mirror of
https://github.com/openjdk/jdk.git
synced 2026-05-23 03:48:13 +00:00
8289743: AArch64: Clean up patching logic
Reviewed-by: adinn, ngasson
This commit is contained in:
Andrew Haley
parent
011958d30b
commit
1c055076e0
@ -155,10 +155,6 @@ void Address::lea(MacroAssembler *as, Register r) const {
|
||||
}
|
||||
}
|
||||
|
||||
void Assembler::adrp(Register reg1, const Address &dest, uint64_t &byte_offset) {
|
||||
ShouldNotReachHere();
|
||||
}
|
||||
|
||||
#undef __
|
||||
|
||||
#define starti Instruction_aarch64 current_insn(this);
|
||||
@ -189,7 +185,7 @@ void Assembler::adrp(Register reg1, const Address &dest, uint64_t &byte_offset)
|
||||
offset >>= 2;
|
||||
starti;
|
||||
f(1, 31), f(offset_lo, 30, 29), f(0b10000, 28, 24), sf(offset, 23, 5);
|
||||
rf(Rd, 0);
|
||||
zrf(Rd, 0);
|
||||
}
|
||||
|
||||
// An "all-purpose" add/subtract immediate, per ARM documentation:
|
||||
|
||||
@ -220,7 +220,7 @@ public:
|
||||
return extend(uval, msb - lsb);
|
||||
}
|
||||
|
||||
static void patch(address a, int msb, int lsb, uint64_t val) {
|
||||
static ALWAYSINLINE void patch(address a, int msb, int lsb, uint64_t val) {
|
||||
int nbits = msb - lsb + 1;
|
||||
guarantee(val < (1ULL << nbits), "Field too big for insn");
|
||||
assert_cond(msb >= lsb);
|
||||
@ -718,7 +718,7 @@ public:
|
||||
wrap_label(Rd, L, &Assembler::_adrp);
|
||||
}
|
||||
|
||||
void adrp(Register Rd, const Address &dest, uint64_t &offset);
|
||||
void adrp(Register Rd, const Address &dest, uint64_t &offset) = delete;
|
||||
|
||||
#undef INSN
|
||||
|
||||
|
||||
@ -2690,7 +2690,7 @@ void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
|
||||
assert_different_registers(val, crc, res);
|
||||
uint64_t offset;
|
||||
__ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
|
||||
if (offset) __ add(res, res, offset);
|
||||
__ add(res, res, offset);
|
||||
|
||||
__ mvnw(crc, crc); // ~crc
|
||||
__ update_byte_crc32(crc, val, res);
|
||||
|
||||
@ -170,7 +170,12 @@ void InterpreterMacroAssembler::get_unsigned_2_byte_index_at_bcp(
|
||||
void InterpreterMacroAssembler::get_dispatch() {
|
||||
uint64_t offset;
|
||||
adrp(rdispatch, ExternalAddress((address)Interpreter::dispatch_table()), offset);
|
||||
lea(rdispatch, Address(rdispatch, offset));
|
||||
// Use add() here after ARDP, rather than lea().
|
||||
// lea() does not generate anything if its offset is zero.
|
||||
// However, relocs expect to find either an ADD or a load/store
|
||||
// insn after an ADRP. add() always generates an ADD insn, even
|
||||
// for add(Rn, Rn, 0).
|
||||
add(rdispatch, rdispatch, offset);
|
||||
}
|
||||
|
||||
void InterpreterMacroAssembler::get_cache_index_at_bcp(Register index,
|
||||
|
||||
@ -73,102 +73,435 @@
|
||||
#define STOP(str) stop(str);
|
||||
#define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
|
||||
|
||||
// Patch any kind of instruction; there may be several instructions.
|
||||
// Return the total length (in bytes) of the instructions.
|
||||
int MacroAssembler::pd_patch_instruction_size(address branch, address target) {
|
||||
int instructions = 1;
|
||||
assert((uint64_t)target < (1ull << 48), "48-bit overflow in address constant");
|
||||
intptr_t offset = (target - branch) >> 2;
|
||||
unsigned insn = *(unsigned*)branch;
|
||||
if ((Instruction_aarch64::extract(insn, 29, 24) & 0b111011) == 0b011000) {
|
||||
// Load register (literal)
|
||||
Instruction_aarch64::spatch(branch, 23, 5, offset);
|
||||
} else if (Instruction_aarch64::extract(insn, 30, 26) == 0b00101) {
|
||||
// Unconditional branch (immediate)
|
||||
Instruction_aarch64::spatch(branch, 25, 0, offset);
|
||||
} else if (Instruction_aarch64::extract(insn, 31, 25) == 0b0101010) {
|
||||
// Conditional branch (immediate)
|
||||
Instruction_aarch64::spatch(branch, 23, 5, offset);
|
||||
} else if (Instruction_aarch64::extract(insn, 30, 25) == 0b011010) {
|
||||
// Compare & branch (immediate)
|
||||
Instruction_aarch64::spatch(branch, 23, 5, offset);
|
||||
} else if (Instruction_aarch64::extract(insn, 30, 25) == 0b011011) {
|
||||
// Test & branch (immediate)
|
||||
Instruction_aarch64::spatch(branch, 18, 5, offset);
|
||||
} else if (Instruction_aarch64::extract(insn, 28, 24) == 0b10000) {
|
||||
// PC-rel. addressing
|
||||
offset = target-branch;
|
||||
int shift = Instruction_aarch64::extract(insn, 31, 31);
|
||||
if (shift) {
|
||||
uint64_t dest = (uint64_t)target;
|
||||
uint64_t pc_page = (uint64_t)branch >> 12;
|
||||
uint64_t adr_page = (uint64_t)target >> 12;
|
||||
unsigned offset_lo = dest & 0xfff;
|
||||
offset = adr_page - pc_page;
|
||||
#ifdef ASSERT
|
||||
extern "C" void disnm(intptr_t p);
|
||||
#endif
|
||||
// Target-dependent relocation processing
|
||||
//
|
||||
// Instruction sequences whose target may need to be retrieved or
|
||||
// patched are distinguished by their leading instruction, sorting
|
||||
// them into three main instruction groups and related subgroups.
|
||||
//
|
||||
// 1) Branch, Exception and System (insn count = 1)
|
||||
// 1a) Unconditional branch (immediate):
|
||||
// b/bl imm19
|
||||
// 1b) Compare & branch (immediate):
|
||||
// cbz/cbnz Rt imm19
|
||||
// 1c) Test & branch (immediate):
|
||||
// tbz/tbnz Rt imm14
|
||||
// 1d) Conditional branch (immediate):
|
||||
// b.cond imm19
|
||||
//
|
||||
// 2) Loads and Stores (insn count = 1)
|
||||
// 2a) Load register literal:
|
||||
// ldr Rt imm19
|
||||
//
|
||||
// 3) Data Processing Immediate (insn count = 2 or 3)
|
||||
// 3a) PC-rel. addressing
|
||||
// adr/adrp Rx imm21; ldr/str Ry Rx #imm12
|
||||
// adr/adrp Rx imm21; add Ry Rx #imm12
|
||||
// adr/adrp Rx imm21; movk Rx #imm16<<32; ldr/str Ry, [Rx, #offset_in_page]
|
||||
// adr/adrp Rx imm21
|
||||
// adr/adrp Rx imm21; movk Rx #imm16<<32
|
||||
// adr/adrp Rx imm21; movk Rx #imm16<<32; add Ry, Rx, #offset_in_page
|
||||
// The latter form can only happen when the target is an
|
||||
// ExternalAddress, and (by definition) ExternalAddresses don't
|
||||
// move. Because of that property, there is never any need to
|
||||
// patch the last of the three instructions. However,
|
||||
// MacroAssembler::target_addr_for_insn takes all three
|
||||
// instructions into account and returns the correct address.
|
||||
// 3b) Move wide (immediate)
|
||||
// movz Rx #imm16; movk Rx #imm16 << 16; movk Rx #imm16 << 32;
|
||||
//
|
||||
// A switch on a subset of the instruction's bits provides an
|
||||
// efficient dispatch to these subcases.
|
||||
//
|
||||
// insn[28:26] -> main group ('x' == don't care)
|
||||
// 00x -> UNALLOCATED
|
||||
// 100 -> Data Processing Immediate
|
||||
// 101 -> Branch, Exception and System
|
||||
// x1x -> Loads and Stores
|
||||
//
|
||||
// insn[30:25] -> subgroup ('_' == group, 'x' == don't care).
|
||||
// n.b. in some cases extra bits need to be checked to verify the
|
||||
// instruction is as expected
|
||||
//
|
||||
// 1) ... xx101x Branch, Exception and System
|
||||
// 1a) 00___x Unconditional branch (immediate)
|
||||
// 1b) 01___0 Compare & branch (immediate)
|
||||
// 1c) 01___1 Test & branch (immediate)
|
||||
// 1d) 10___0 Conditional branch (immediate)
|
||||
// other Should not happen
|
||||
//
|
||||
// 2) ... xxx1x0 Loads and Stores
|
||||
// 2a) xx1__00 Load/Store register (insn[28] == 1 && insn[24] == 0)
|
||||
// 2aa) x01__00 Load register literal (i.e. requires insn[29] == 0)
|
||||
// strictly should be 64 bit non-FP/SIMD i.e.
|
||||
// 0101_000 (i.e. requires insn[31:24] == 01011000)
|
||||
//
|
||||
// 3) ... xx100x Data Processing Immediate
|
||||
// 3a) xx___00 PC-rel. addressing (n.b. requires insn[24] == 0)
|
||||
// 3b) xx___101 Move wide (immediate) (n.b. requires insn[24:23] == 01)
|
||||
// strictly should be 64 bit movz #imm16<<0
|
||||
// 110___10100 (i.e. requires insn[31:21] == 11010010100)
|
||||
//
|
||||
class RelocActions {
|
||||
protected:
|
||||
typedef int (*reloc_insn)(address insn_addr, address &target);
|
||||
|
||||
// We handle 4 types of PC relative addressing
|
||||
// 1 - adrp Rx, target_page
|
||||
// ldr/str Ry, [Rx, #offset_in_page]
|
||||
// 2 - adrp Rx, target_page
|
||||
// add Ry, Rx, #offset_in_page
|
||||
// 3 - adrp Rx, target_page (page aligned reloc, offset == 0)
|
||||
// movk Rx, #imm16<<32
|
||||
// 4 - adrp Rx, target_page (page aligned reloc, offset == 0)
|
||||
// In the first 3 cases we must check that Rx is the same in the adrp and the
|
||||
// subsequent ldr/str, add or movk instruction. Otherwise we could accidentally end
|
||||
// up treating a type 4 relocation as a type 1, 2 or 3 just because it happened
|
||||
// to be followed by a random unrelated ldr/str, add or movk instruction.
|
||||
//
|
||||
unsigned insn2 = ((unsigned*)branch)[1];
|
||||
if (Instruction_aarch64::extract(insn2, 29, 24) == 0b111001 &&
|
||||
Instruction_aarch64::extract(insn, 4, 0) ==
|
||||
Instruction_aarch64::extract(insn2, 9, 5)) {
|
||||
// Load/store register (unsigned immediate)
|
||||
unsigned size = Instruction_aarch64::extract(insn2, 31, 30);
|
||||
Instruction_aarch64::patch(branch + sizeof (unsigned),
|
||||
21, 10, offset_lo >> size);
|
||||
guarantee(((dest >> size) << size) == dest, "misaligned target");
|
||||
instructions = 2;
|
||||
} else if (Instruction_aarch64::extract(insn2, 31, 22) == 0b1001000100 &&
|
||||
Instruction_aarch64::extract(insn, 4, 0) ==
|
||||
Instruction_aarch64::extract(insn2, 4, 0)) {
|
||||
// add (immediate)
|
||||
Instruction_aarch64::patch(branch + sizeof (unsigned),
|
||||
21, 10, offset_lo);
|
||||
instructions = 2;
|
||||
} else if (Instruction_aarch64::extract(insn2, 31, 21) == 0b11110010110 &&
|
||||
Instruction_aarch64::extract(insn, 4, 0) ==
|
||||
virtual reloc_insn adrpMem() = 0;
|
||||
virtual reloc_insn adrpAdd() = 0;
|
||||
virtual reloc_insn adrpMovk() = 0;
|
||||
|
||||
const address _insn_addr;
|
||||
const uint32_t _insn;
|
||||
|
||||
static uint32_t insn_at(address insn_addr, int n) {
|
||||
return ((uint32_t*)insn_addr)[n];
|
||||
}
|
||||
uint32_t insn_at(int n) const {
|
||||
return insn_at(_insn_addr, n);
|
||||
}
|
||||
|
||||
public:
|
||||
|
||||
RelocActions(address insn_addr) : _insn_addr(insn_addr), _insn(insn_at(insn_addr, 0)) {}
|
||||
RelocActions(address insn_addr, uint32_t insn)
|
||||
: _insn_addr(insn_addr), _insn(insn) {}
|
||||
|
||||
virtual int unconditionalBranch(address insn_addr, address &target) = 0;
|
||||
virtual int conditionalBranch(address insn_addr, address &target) = 0;
|
||||
virtual int testAndBranch(address insn_addr, address &target) = 0;
|
||||
virtual int loadStore(address insn_addr, address &target) = 0;
|
||||
virtual int adr(address insn_addr, address &target) = 0;
|
||||
virtual int adrp(address insn_addr, address &target, reloc_insn inner) = 0;
|
||||
virtual int immediate(address insn_addr, address &target) = 0;
|
||||
virtual void verify(address insn_addr, address &target) = 0;
|
||||
|
||||
int ALWAYSINLINE run(address insn_addr, address &target) {
|
||||
int instructions = 1;
|
||||
|
||||
uint32_t dispatch = Instruction_aarch64::extract(_insn, 30, 25);
|
||||
switch(dispatch) {
|
||||
case 0b001010:
|
||||
case 0b001011: {
|
||||
instructions = unconditionalBranch(insn_addr, target);
|
||||
break;
|
||||
}
|
||||
case 0b101010: // Conditional branch (immediate)
|
||||
case 0b011010: { // Compare & branch (immediate)
|
||||
instructions = conditionalBranch(insn_addr, target);
|
||||
break;
|
||||
}
|
||||
case 0b011011: {
|
||||
instructions = testAndBranch(insn_addr, target);
|
||||
break;
|
||||
}
|
||||
case 0b001100:
|
||||
case 0b001110:
|
||||
case 0b011100:
|
||||
case 0b011110:
|
||||
case 0b101100:
|
||||
case 0b101110:
|
||||
case 0b111100:
|
||||
case 0b111110: {
|
||||
// load/store
|
||||
if ((Instruction_aarch64::extract(_insn, 29, 24) & 0b111011) == 0b011000) {
|
||||
// Load register (literal)
|
||||
instructions = loadStore(insn_addr, target);
|
||||
break;
|
||||
} else {
|
||||
// nothing to do
|
||||
assert(target == 0, "did not expect to relocate target for polling page load");
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 0b001000:
|
||||
case 0b011000:
|
||||
case 0b101000:
|
||||
case 0b111000: {
|
||||
// adr/adrp
|
||||
assert(Instruction_aarch64::extract(_insn, 28, 24) == 0b10000, "must be");
|
||||
int shift = Instruction_aarch64::extract(_insn, 31, 31);
|
||||
if (shift) {
|
||||
uint32_t insn2 = insn_at(1);
|
||||
if (Instruction_aarch64::extract(insn2, 29, 24) == 0b111001 &&
|
||||
Instruction_aarch64::extract(_insn, 4, 0) ==
|
||||
Instruction_aarch64::extract(insn2, 9, 5)) {
|
||||
instructions = adrp(insn_addr, target, adrpMem());
|
||||
} else if (Instruction_aarch64::extract(insn2, 31, 22) == 0b1001000100 &&
|
||||
Instruction_aarch64::extract(_insn, 4, 0) ==
|
||||
Instruction_aarch64::extract(insn2, 4, 0)) {
|
||||
// movk #imm16<<32
|
||||
Instruction_aarch64::patch(branch + 4, 20, 5, (uint64_t)target >> 32);
|
||||
uintptr_t dest = ((uintptr_t)target & 0xffffffffULL) | ((uintptr_t)branch & 0xffff00000000ULL);
|
||||
uintptr_t pc_page = (uintptr_t)branch >> 12;
|
||||
uintptr_t adr_page = (uintptr_t)dest >> 12;
|
||||
offset = adr_page - pc_page;
|
||||
instructions = 2;
|
||||
instructions = adrp(insn_addr, target, adrpAdd());
|
||||
} else if (Instruction_aarch64::extract(insn2, 31, 21) == 0b11110010110 &&
|
||||
Instruction_aarch64::extract(_insn, 4, 0) ==
|
||||
Instruction_aarch64::extract(insn2, 4, 0)) {
|
||||
instructions = adrp(insn_addr, target, adrpMovk());
|
||||
} else {
|
||||
ShouldNotReachHere();
|
||||
}
|
||||
} else {
|
||||
instructions = adr(insn_addr, target);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 0b001001:
|
||||
case 0b011001:
|
||||
case 0b101001:
|
||||
case 0b111001: {
|
||||
instructions = immediate(insn_addr, target);
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
ShouldNotReachHere();
|
||||
}
|
||||
}
|
||||
|
||||
verify(insn_addr, target);
|
||||
return instructions * NativeInstruction::instruction_size;
|
||||
}
|
||||
};
|
||||
|
||||
class Patcher : public RelocActions {
|
||||
virtual reloc_insn adrpMem() { return &Patcher::adrpMem_impl; }
|
||||
virtual reloc_insn adrpAdd() { return &Patcher::adrpAdd_impl; }
|
||||
virtual reloc_insn adrpMovk() { return &Patcher::adrpMovk_impl; }
|
||||
|
||||
public:
|
||||
Patcher(address insn_addr) : RelocActions(insn_addr) {}
|
||||
|
||||
virtual int unconditionalBranch(address insn_addr, address &target) {
|
||||
intptr_t offset = (target - insn_addr) >> 2;
|
||||
Instruction_aarch64::spatch(insn_addr, 25, 0, offset);
|
||||
return 1;
|
||||
}
|
||||
virtual int conditionalBranch(address insn_addr, address &target) {
|
||||
intptr_t offset = (target - insn_addr) >> 2;
|
||||
Instruction_aarch64::spatch(insn_addr, 23, 5, offset);
|
||||
return 1;
|
||||
}
|
||||
virtual int testAndBranch(address insn_addr, address &target) {
|
||||
intptr_t offset = (target - insn_addr) >> 2;
|
||||
Instruction_aarch64::spatch(insn_addr, 18, 5, offset);
|
||||
return 1;
|
||||
}
|
||||
virtual int loadStore(address insn_addr, address &target) {
|
||||
intptr_t offset = (target - insn_addr) >> 2;
|
||||
Instruction_aarch64::spatch(insn_addr, 23, 5, offset);
|
||||
return 1;
|
||||
}
|
||||
virtual int adr(address insn_addr, address &target) {
|
||||
#ifdef ASSERT
|
||||
assert(Instruction_aarch64::extract(_insn, 28, 24) == 0b10000, "must be");
|
||||
#endif
|
||||
// PC-rel. addressing
|
||||
ptrdiff_t offset = target - insn_addr;
|
||||
int offset_lo = offset & 3;
|
||||
offset >>= 2;
|
||||
Instruction_aarch64::spatch(branch, 23, 5, offset);
|
||||
Instruction_aarch64::patch(branch, 30, 29, offset_lo);
|
||||
} else if (Instruction_aarch64::extract(insn, 31, 21) == 0b11010010100) {
|
||||
Instruction_aarch64::spatch(insn_addr, 23, 5, offset);
|
||||
Instruction_aarch64::patch(insn_addr, 30, 29, offset_lo);
|
||||
return 1;
|
||||
}
|
||||
virtual int adrp(address insn_addr, address &target, reloc_insn inner) {
|
||||
int instructions = 1;
|
||||
#ifdef ASSERT
|
||||
assert(Instruction_aarch64::extract(_insn, 28, 24) == 0b10000, "must be");
|
||||
#endif
|
||||
ptrdiff_t offset = target - insn_addr;
|
||||
instructions = 2;
|
||||
precond(inner != nullptr);
|
||||
uintptr_t dest = (uintptr_t)target;
|
||||
uintptr_t pc_page = (uintptr_t)insn_addr >> 12;
|
||||
uintptr_t adr_page = (uintptr_t)target >> 12;
|
||||
offset = adr_page - pc_page;
|
||||
instructions = (*inner)(insn_addr, target);
|
||||
// Now we extract the lower 21 bits of the signed offset field for
|
||||
// the ADRP.
|
||||
offset = offset << (64-21) >> (64-21);
|
||||
int offset_lo = offset & 3;
|
||||
offset >>= 2;
|
||||
Instruction_aarch64::spatch(insn_addr, 23, 5, offset);
|
||||
Instruction_aarch64::patch(insn_addr, 30, 29, offset_lo);
|
||||
return instructions;
|
||||
}
|
||||
static int adrpMem_impl(address insn_addr, address &target) {
|
||||
uintptr_t dest = (uintptr_t)target;
|
||||
int offset_lo = dest & 0xfff;
|
||||
uint32_t insn2 = insn_at(insn_addr, 1);
|
||||
uint32_t size = Instruction_aarch64::extract(insn2, 31, 30);
|
||||
Instruction_aarch64::patch(insn_addr + sizeof (uint32_t), 21, 10, offset_lo >> size);
|
||||
guarantee(((dest >> size) << size) == dest, "misaligned target");
|
||||
return 2;
|
||||
}
|
||||
static int adrpAdd_impl(address insn_addr, address &target) {
|
||||
uintptr_t dest = (uintptr_t)target;
|
||||
int offset_lo = dest & 0xfff;
|
||||
Instruction_aarch64::patch(insn_addr + sizeof (uint32_t), 21, 10, offset_lo);
|
||||
return 2;
|
||||
}
|
||||
static int adrpMovk_impl(address insn_addr, address &target) {
|
||||
uintptr_t dest = (uintptr_t)target;
|
||||
Instruction_aarch64::patch(insn_addr + sizeof (uint32_t), 20, 5, (uintptr_t)target >> 32);
|
||||
return 2;
|
||||
}
|
||||
virtual int immediate(address insn_addr, address &target) {
|
||||
assert(Instruction_aarch64::extract(_insn, 31, 21) == 0b11010010100, "must be");
|
||||
uint64_t dest = (uint64_t)target;
|
||||
// Move wide constant
|
||||
assert(nativeInstruction_at(branch+4)->is_movk(), "wrong insns in patch");
|
||||
assert(nativeInstruction_at(branch+8)->is_movk(), "wrong insns in patch");
|
||||
Instruction_aarch64::patch(branch, 20, 5, dest & 0xffff);
|
||||
Instruction_aarch64::patch(branch+4, 20, 5, (dest >>= 16) & 0xffff);
|
||||
Instruction_aarch64::patch(branch+8, 20, 5, (dest >>= 16) & 0xffff);
|
||||
assert(target_addr_for_insn(branch) == target, "should be");
|
||||
instructions = 3;
|
||||
} else if (NativeInstruction::is_ldrw_to_zr(address(&insn))) {
|
||||
// nothing to do
|
||||
assert(target == 0, "did not expect to relocate target for polling page load");
|
||||
} else {
|
||||
ShouldNotReachHere();
|
||||
assert(nativeInstruction_at(insn_addr+4)->is_movk(), "wrong insns in patch");
|
||||
assert(nativeInstruction_at(insn_addr+8)->is_movk(), "wrong insns in patch");
|
||||
Instruction_aarch64::patch(insn_addr, 20, 5, dest & 0xffff);
|
||||
Instruction_aarch64::patch(insn_addr+4, 20, 5, (dest >>= 16) & 0xffff);
|
||||
Instruction_aarch64::patch(insn_addr+8, 20, 5, (dest >>= 16) & 0xffff);
|
||||
return 3;
|
||||
}
|
||||
return instructions * NativeInstruction::instruction_size;
|
||||
virtual void verify(address insn_addr, address &target) {
|
||||
#ifdef ASSERT
|
||||
address address_is = MacroAssembler::target_addr_for_insn(insn_addr);
|
||||
if (!(address_is == target)) {
|
||||
tty->print_cr("%p at %p should be %p", address_is, insn_addr, target);
|
||||
disnm((intptr_t)insn_addr);
|
||||
assert(address_is == target, "should be");
|
||||
}
|
||||
#endif
|
||||
}
|
||||
};
|
||||
|
||||
// If insn1 and insn2 use the same register to form an address, either
|
||||
// by an offsetted LDR or a simple ADD, return the offset. If the
|
||||
// second instruction is an LDR, the offset may be scaled.
|
||||
static bool offset_for(uint32_t insn1, uint32_t insn2, ptrdiff_t &byte_offset) {
|
||||
if (Instruction_aarch64::extract(insn2, 29, 24) == 0b111001 &&
|
||||
Instruction_aarch64::extract(insn1, 4, 0) ==
|
||||
Instruction_aarch64::extract(insn2, 9, 5)) {
|
||||
// Load/store register (unsigned immediate)
|
||||
byte_offset = Instruction_aarch64::extract(insn2, 21, 10);
|
||||
uint32_t size = Instruction_aarch64::extract(insn2, 31, 30);
|
||||
byte_offset <<= size;
|
||||
return true;
|
||||
} else if (Instruction_aarch64::extract(insn2, 31, 22) == 0b1001000100 &&
|
||||
Instruction_aarch64::extract(insn1, 4, 0) ==
|
||||
Instruction_aarch64::extract(insn2, 4, 0)) {
|
||||
// add (immediate)
|
||||
byte_offset = Instruction_aarch64::extract(insn2, 21, 10);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
class Decoder : public RelocActions {
|
||||
virtual reloc_insn adrpMem() { return &Decoder::adrpMem_impl; }
|
||||
virtual reloc_insn adrpAdd() { return &Decoder::adrpAdd_impl; }
|
||||
virtual reloc_insn adrpMovk() { return &Decoder::adrpMovk_impl; }
|
||||
|
||||
public:
|
||||
Decoder(address insn_addr, uint32_t insn) : RelocActions(insn_addr, insn) {}
|
||||
|
||||
virtual int loadStore(address insn_addr, address &target) {
|
||||
intptr_t offset = Instruction_aarch64::sextract(_insn, 23, 5);
|
||||
target = insn_addr + (offset << 2);
|
||||
return 1;
|
||||
}
|
||||
virtual int unconditionalBranch(address insn_addr, address &target) {
|
||||
intptr_t offset = Instruction_aarch64::sextract(_insn, 25, 0);
|
||||
target = insn_addr + (offset << 2);
|
||||
return 1;
|
||||
}
|
||||
virtual int conditionalBranch(address insn_addr, address &target) {
|
||||
intptr_t offset = Instruction_aarch64::sextract(_insn, 23, 5);
|
||||
target = address(((uint64_t)insn_addr + (offset << 2)));
|
||||
return 1;
|
||||
}
|
||||
virtual int testAndBranch(address insn_addr, address &target) {
|
||||
intptr_t offset = Instruction_aarch64::sextract(_insn, 18, 5);
|
||||
target = address(((uint64_t)insn_addr + (offset << 2)));
|
||||
return 1;
|
||||
}
|
||||
virtual int adr(address insn_addr, address &target) {
|
||||
// PC-rel. addressing
|
||||
intptr_t offset = Instruction_aarch64::extract(_insn, 30, 29);
|
||||
offset |= Instruction_aarch64::sextract(_insn, 23, 5) << 2;
|
||||
target = address((uint64_t)insn_addr + offset);
|
||||
return 1;
|
||||
}
|
||||
virtual int adrp(address insn_addr, address &target, reloc_insn inner) {
|
||||
assert(Instruction_aarch64::extract(_insn, 28, 24) == 0b10000, "must be");
|
||||
intptr_t offset = Instruction_aarch64::extract(_insn, 30, 29);
|
||||
offset |= Instruction_aarch64::sextract(_insn, 23, 5) << 2;
|
||||
int shift = 12;
|
||||
offset <<= shift;
|
||||
uint64_t target_page = ((uint64_t)insn_addr) + offset;
|
||||
target_page &= ((uint64_t)-1) << shift;
|
||||
uint32_t insn2 = insn_at(1);
|
||||
target = address(target_page);
|
||||
precond(inner != nullptr);
|
||||
(*inner)(insn_addr, target);
|
||||
return 2;
|
||||
}
|
||||
static int adrpMem_impl(address insn_addr, address &target) {
|
||||
uint32_t insn2 = insn_at(insn_addr, 1);
|
||||
// Load/store register (unsigned immediate)
|
||||
ptrdiff_t byte_offset = Instruction_aarch64::extract(insn2, 21, 10);
|
||||
uint32_t size = Instruction_aarch64::extract(insn2, 31, 30);
|
||||
byte_offset <<= size;
|
||||
target += byte_offset;
|
||||
return 2;
|
||||
}
|
||||
static int adrpAdd_impl(address insn_addr, address &target) {
|
||||
uint32_t insn2 = insn_at(insn_addr, 1);
|
||||
// add (immediate)
|
||||
ptrdiff_t byte_offset = Instruction_aarch64::extract(insn2, 21, 10);
|
||||
target += byte_offset;
|
||||
return 2;
|
||||
}
|
||||
static int adrpMovk_impl(address insn_addr, address &target) {
|
||||
uint32_t insn2 = insn_at(insn_addr, 1);
|
||||
uint64_t dest = uint64_t(target);
|
||||
dest = (dest & 0xffffffff) |
|
||||
((uint64_t)Instruction_aarch64::extract(insn2, 20, 5) << 32);
|
||||
target = address(dest);
|
||||
|
||||
// We know the destination 4k page. Maybe we have a third
|
||||
// instruction.
|
||||
uint32_t insn = insn_at(insn_addr, 0);
|
||||
uint32_t insn3 = insn_at(insn_addr, 2);
|
||||
ptrdiff_t byte_offset;
|
||||
if (offset_for(insn, insn3, byte_offset)) {
|
||||
target += byte_offset;
|
||||
return 3;
|
||||
} else {
|
||||
return 2;
|
||||
}
|
||||
}
|
||||
virtual int immediate(address insn_addr, address &target) {
|
||||
uint32_t *insns = (uint32_t *)insn_addr;
|
||||
assert(Instruction_aarch64::extract(_insn, 31, 21) == 0b11010010100, "must be");
|
||||
// Move wide constant: movz, movk, movk. See movptr().
|
||||
assert(nativeInstruction_at(insns+1)->is_movk(), "wrong insns in patch");
|
||||
assert(nativeInstruction_at(insns+2)->is_movk(), "wrong insns in patch");
|
||||
target = address(uint64_t(Instruction_aarch64::extract(_insn, 20, 5))
|
||||
+ (uint64_t(Instruction_aarch64::extract(insns[1], 20, 5)) << 16)
|
||||
+ (uint64_t(Instruction_aarch64::extract(insns[2], 20, 5)) << 32));
|
||||
assert(nativeInstruction_at(insn_addr+4)->is_movk(), "wrong insns in patch");
|
||||
assert(nativeInstruction_at(insn_addr+8)->is_movk(), "wrong insns in patch");
|
||||
return 3;
|
||||
}
|
||||
virtual void verify(address insn_addr, address &target) {
|
||||
}
|
||||
};
|
||||
|
||||
address MacroAssembler::target_addr_for_insn(address insn_addr, uint32_t insn) {
|
||||
Decoder decoder(insn_addr, insn);
|
||||
address target;
|
||||
decoder.run(insn_addr, target);
|
||||
return target;
|
||||
}
|
||||
|
||||
// Patch any kind of instruction; there may be several instructions.
|
||||
// Return the total length (in bytes) of the instructions.
|
||||
int MacroAssembler::pd_patch_instruction_size(address insn_addr, address target) {
|
||||
Patcher patcher(insn_addr);
|
||||
return patcher.run(insn_addr, target);
|
||||
}
|
||||
|
||||
int MacroAssembler::patch_oop(address insn_addr, address o) {
|
||||
@ -210,90 +543,9 @@ int MacroAssembler::patch_narrow_klass(address insn_addr, narrowKlass n) {
|
||||
return 2 * NativeInstruction::instruction_size;
|
||||
}
|
||||
|
||||
address MacroAssembler::target_addr_for_insn(address insn_addr, unsigned insn) {
|
||||
intptr_t offset = 0;
|
||||
if ((Instruction_aarch64::extract(insn, 29, 24) & 0b011011) == 0b00011000) {
|
||||
// Load register (literal)
|
||||
offset = Instruction_aarch64::sextract(insn, 23, 5);
|
||||
return address(((uint64_t)insn_addr + (offset << 2)));
|
||||
} else if (Instruction_aarch64::extract(insn, 30, 26) == 0b00101) {
|
||||
// Unconditional branch (immediate)
|
||||
offset = Instruction_aarch64::sextract(insn, 25, 0);
|
||||
} else if (Instruction_aarch64::extract(insn, 31, 25) == 0b0101010) {
|
||||
// Conditional branch (immediate)
|
||||
offset = Instruction_aarch64::sextract(insn, 23, 5);
|
||||
} else if (Instruction_aarch64::extract(insn, 30, 25) == 0b011010) {
|
||||
// Compare & branch (immediate)
|
||||
offset = Instruction_aarch64::sextract(insn, 23, 5);
|
||||
} else if (Instruction_aarch64::extract(insn, 30, 25) == 0b011011) {
|
||||
// Test & branch (immediate)
|
||||
offset = Instruction_aarch64::sextract(insn, 18, 5);
|
||||
} else if (Instruction_aarch64::extract(insn, 28, 24) == 0b10000) {
|
||||
// PC-rel. addressing
|
||||
offset = Instruction_aarch64::extract(insn, 30, 29);
|
||||
offset |= Instruction_aarch64::sextract(insn, 23, 5) << 2;
|
||||
int shift = Instruction_aarch64::extract(insn, 31, 31) ? 12 : 0;
|
||||
if (shift) {
|
||||
offset <<= shift;
|
||||
uint64_t target_page = ((uint64_t)insn_addr) + offset;
|
||||
target_page &= ((uint64_t)-1) << shift;
|
||||
// Return the target address for the following sequences
|
||||
// 1 - adrp Rx, target_page
|
||||
// ldr/str Ry, [Rx, #offset_in_page]
|
||||
// 2 - adrp Rx, target_page
|
||||
// add Ry, Rx, #offset_in_page
|
||||
// 3 - adrp Rx, target_page (page aligned reloc, offset == 0)
|
||||
// movk Rx, #imm12<<32
|
||||
// 4 - adrp Rx, target_page (page aligned reloc, offset == 0)
|
||||
//
|
||||
// In the first two cases we check that the register is the same and
|
||||
// return the target_page + the offset within the page.
|
||||
// Otherwise we assume it is a page aligned relocation and return
|
||||
// the target page only.
|
||||
//
|
||||
unsigned insn2 = ((unsigned*)insn_addr)[1];
|
||||
if (Instruction_aarch64::extract(insn2, 29, 24) == 0b111001 &&
|
||||
Instruction_aarch64::extract(insn, 4, 0) ==
|
||||
Instruction_aarch64::extract(insn2, 9, 5)) {
|
||||
// Load/store register (unsigned immediate)
|
||||
unsigned int byte_offset = Instruction_aarch64::extract(insn2, 21, 10);
|
||||
unsigned int size = Instruction_aarch64::extract(insn2, 31, 30);
|
||||
return address(target_page + (byte_offset << size));
|
||||
} else if (Instruction_aarch64::extract(insn2, 31, 22) == 0b1001000100 &&
|
||||
Instruction_aarch64::extract(insn, 4, 0) ==
|
||||
Instruction_aarch64::extract(insn2, 4, 0)) {
|
||||
// add (immediate)
|
||||
unsigned int byte_offset = Instruction_aarch64::extract(insn2, 21, 10);
|
||||
return address(target_page + byte_offset);
|
||||
} else {
|
||||
if (Instruction_aarch64::extract(insn2, 31, 21) == 0b11110010110 &&
|
||||
Instruction_aarch64::extract(insn, 4, 0) ==
|
||||
Instruction_aarch64::extract(insn2, 4, 0)) {
|
||||
target_page = (target_page & 0xffffffff) |
|
||||
((uint64_t)Instruction_aarch64::extract(insn2, 20, 5) << 32);
|
||||
}
|
||||
return (address)target_page;
|
||||
}
|
||||
} else {
|
||||
ShouldNotReachHere();
|
||||
}
|
||||
} else if (Instruction_aarch64::extract(insn, 31, 23) == 0b110100101) {
|
||||
uint32_t *insns = (uint32_t *)insn_addr;
|
||||
// Move wide constant: movz, movk, movk. See movptr().
|
||||
assert(nativeInstruction_at(insns+1)->is_movk(), "wrong insns in patch");
|
||||
assert(nativeInstruction_at(insns+2)->is_movk(), "wrong insns in patch");
|
||||
return address(uint64_t(Instruction_aarch64::extract(insns[0], 20, 5))
|
||||
+ (uint64_t(Instruction_aarch64::extract(insns[1], 20, 5)) << 16)
|
||||
+ (uint64_t(Instruction_aarch64::extract(insns[2], 20, 5)) << 32));
|
||||
} else {
|
||||
ShouldNotReachHere();
|
||||
}
|
||||
return address(((uint64_t)insn_addr + (offset << 2)));
|
||||
}
|
||||
|
||||
address MacroAssembler::target_addr_for_insn_or_null(address insn_addr, unsigned insn) {
|
||||
if (NativeInstruction::is_ldrw_to_zr(address(&insn))) {
|
||||
return 0;
|
||||
return nullptr;
|
||||
}
|
||||
return MacroAssembler::target_addr_for_insn(insn_addr, insn);
|
||||
}
|
||||
@ -3436,7 +3688,6 @@ void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len,
|
||||
Register table0, Register table1, Register table2, Register table3,
|
||||
Register tmp, Register tmp2, Register tmp3) {
|
||||
Label L_by16, L_by16_loop, L_by4, L_by4_loop, L_by1, L_by1_loop, L_exit;
|
||||
uint64_t offset;
|
||||
|
||||
if (UseCRC32) {
|
||||
kernel_crc32_using_crc32(crc, buf, len, table0, table1, table2, table3);
|
||||
@ -3445,8 +3696,11 @@ void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len,
|
||||
|
||||
mvnw(crc, crc);
|
||||
|
||||
adrp(table0, ExternalAddress(StubRoutines::crc_table_addr()), offset);
|
||||
if (offset) add(table0, table0, offset);
|
||||
{
|
||||
uint64_t offset;
|
||||
adrp(table0, ExternalAddress(StubRoutines::crc_table_addr()), offset);
|
||||
add(table0, table0, offset);
|
||||
}
|
||||
add(table1, table0, 1*256*sizeof(juint));
|
||||
add(table2, table0, 2*256*sizeof(juint));
|
||||
add(table3, table0, 3*256*sizeof(juint));
|
||||
|
||||
@ -6949,6 +6949,22 @@ class StubGenerator: public StubCodeGenerator {
|
||||
|
||||
__ leave();
|
||||
|
||||
#ifdef ASSERT
|
||||
{
|
||||
// Stress relocs for adrp() by trying to reach a page beyond
|
||||
// the range of a simple ADRP instruction.
|
||||
ExternalAddress longWayAway(__ pc() - (1ll << 34));
|
||||
if (! __ is_valid_AArch64_address(longWayAway.target())) {
|
||||
longWayAway = ExternalAddress(__ pc() + (1ll << 34));
|
||||
}
|
||||
if (__ is_valid_AArch64_address(longWayAway.target())) {
|
||||
uint64_t offset;
|
||||
__ adrp(rscratch1, longWayAway, offset);
|
||||
__ add(rscratch1, rscratch1, offset);
|
||||
}
|
||||
}
|
||||
#endif // ASSERT
|
||||
|
||||
// check for pending exceptions
|
||||
#ifdef ASSERT
|
||||
Label L;
|
||||
@ -6959,7 +6975,6 @@ class StubGenerator: public StubCodeGenerator {
|
||||
#endif // ASSERT
|
||||
__ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
|
||||
|
||||
|
||||
// codeBlob framesize is in words (not VMRegImpl::slot_size)
|
||||
RuntimeStub* stub =
|
||||
RuntimeStub::new_runtime_stub(name,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user