diff --git a/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.cpp index 90b9b64e699..0dda457b449 100644 --- a/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.cpp @@ -299,3 +299,18 @@ void BarrierSetAssembler::c2i_entry_barrier(MacroAssembler* masm) { __ bind(method_live); } +void BarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) { + // Check if the oop is in the right area of memory + __ mov(tmp2, (intptr_t) Universe::verify_oop_mask()); + __ andr(tmp1, obj, tmp2); + __ mov(tmp2, (intptr_t) Universe::verify_oop_bits()); + + // Compare tmp1 and tmp2. We don't use a compare + // instruction here because the flags register is live. + __ eor(tmp1, tmp1, tmp2); + __ cbnz(tmp1, error); + + // make sure klass is 'reasonable', which is not zero. + __ load_klass(obj, obj); // get klass + __ cbz(obj, error); // if klass is NULL it is broken +} diff --git a/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.hpp index f308603a8cb..b4d5161ed74 100644 --- a/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.hpp @@ -72,6 +72,8 @@ public: virtual void nmethod_entry_barrier(MacroAssembler* masm, Label* slow_path, Label* continuation, Label* guard); virtual void c2i_entry_barrier(MacroAssembler* masm); + virtual void check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error); + virtual bool supports_instruction_patching() { NMethodPatchingType patching_type = nmethod_patching_type(); return patching_type == NMethodPatchingType::conc_instruction_and_data_patch || diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp index 0693a2c063b..b958393bbbb 100644 --- a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp @@ -447,3 +447,17 @@ void ZBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, Z #undef __ #endif // COMPILER2 + +#define __ masm-> + +void ZBarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) { + // Check if mask is good. + // verifies that ZAddressBadMask & r0 == 0 + __ ldr(tmp2, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); + __ andr(tmp1, obj, tmp2); + __ cbnz(tmp1, error); + + BarrierSetAssembler::check_oop(masm, obj, tmp1, tmp2, error); +} + +#undef __ diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp index ac32d9111e1..c852dac3a4d 100644 --- a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp @@ -97,6 +97,8 @@ public: void generate_c2_load_barrier_stub(MacroAssembler* masm, ZLoadBarrierStubC2* stub) const; #endif // COMPILER2 + + void check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error); }; #endif // CPU_AARCH64_GC_Z_ZBARRIERSETASSEMBLER_AARCH64_HPP diff --git a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp index 34fc3eff9ad..2bc60b43ba6 100644 --- a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp @@ -568,29 +568,8 @@ class StubGenerator: public StubCodeGenerator { // make sure object is 'reasonable' __ cbz(r0, exit); // if obj is NULL it is OK -#if INCLUDE_ZGC - if (UseZGC) { - // Check if mask is good. - // verifies that ZAddressBadMask & r0 == 0 - __ ldr(c_rarg3, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); - __ andr(c_rarg2, r0, c_rarg3); - __ cbnz(c_rarg2, error); - } -#endif - - // Check if the oop is in the right area of memory - __ mov(c_rarg3, (intptr_t) Universe::verify_oop_mask()); - __ andr(c_rarg2, r0, c_rarg3); - __ mov(c_rarg3, (intptr_t) Universe::verify_oop_bits()); - - // Compare c_rarg2 and c_rarg3. We don't use a compare - // instruction here because the flags register is live. - __ eor(c_rarg2, c_rarg2, c_rarg3); - __ cbnz(c_rarg2, error); - - // make sure klass is 'reasonable', which is not zero. - __ load_klass(r0, r0); // get klass - __ cbz(r0, error); // if klass is NULL it is broken + BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler(); + bs_asm->check_oop(_masm, r0, c_rarg2, c_rarg3, error); // return if everything seems ok __ bind(exit); diff --git a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp index 93616b0fe88..fbbd16466ef 100644 --- a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp +++ b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp @@ -731,7 +731,10 @@ int LIR_Assembler::store(LIR_Opr from_reg, Register base, int offset, BasicType __ verify_coop(from_reg->as_register(), FILE_AND_LINE); } else { __ std(from_reg->as_register(), offset, base); - __ verify_oop(from_reg->as_register(), FILE_AND_LINE); + if (VerifyOops) { + BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); + bs->check_oop(_masm, from_reg->as_register(), FILE_AND_LINE); // kills R0 + } } break; } @@ -772,7 +775,10 @@ int LIR_Assembler::store(LIR_Opr from_reg, Register base, Register disp, BasicTy __ verify_coop(from_reg->as_register(), FILE_AND_LINE); // kills R0 } else { __ stdx(from_reg->as_register(), base, disp); - __ verify_oop(from_reg->as_register(), FILE_AND_LINE); // kills R0 + if (VerifyOops) { + BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); + bs->check_oop(_masm, from_reg->as_register(), FILE_AND_LINE); // kills R0 + } } break; } @@ -813,7 +819,10 @@ int LIR_Assembler::load(Register base, int offset, LIR_Opr to_reg, BasicType typ } else { __ ld(to_reg->as_register(), offset, base); } - __ verify_oop(to_reg->as_register(), FILE_AND_LINE); + if (VerifyOops) { + BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); + bs->check_oop(_masm, to_reg->as_register(), FILE_AND_LINE); // kills R0 + } break; } case T_FLOAT: __ lfs(to_reg->as_float_reg(), offset, base); break; @@ -844,7 +853,10 @@ int LIR_Assembler::load(Register base, Register disp, LIR_Opr to_reg, BasicType } else { __ ldx(to_reg->as_register(), base, disp); } - __ verify_oop(to_reg->as_register(), FILE_AND_LINE); + if (VerifyOops) { + BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); + bs->check_oop(_masm, to_reg->as_register(), FILE_AND_LINE); // kills R0 + } break; } case T_FLOAT: __ lfsx(to_reg->as_float_reg() , base, disp); break; diff --git a/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.cpp b/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.cpp index 051e1d08aec..f507c290aa1 100644 --- a/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.cpp +++ b/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.cpp @@ -255,3 +255,7 @@ void BarrierSetAssembler::c2i_entry_barrier(MacroAssembler *masm, Register tmp1, __ block_comment("} c2i_entry_barrier (c2i_entry_barrier)"); } + +void BarrierSetAssembler::check_oop(MacroAssembler *masm, Register oop, const char* msg) { + __ verify_oop(oop, msg); +} diff --git a/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.hpp b/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.hpp index 9ad474961cb..0265f97b639 100644 --- a/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.hpp +++ b/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.hpp @@ -69,6 +69,8 @@ public: virtual void nmethod_entry_barrier(MacroAssembler* masm, Register tmp); virtual void c2i_entry_barrier(MacroAssembler* masm, Register tmp1, Register tmp2, Register tmp3); + + virtual void check_oop(MacroAssembler *masm, Register oop, const char* msg); }; #endif // CPU_PPC_GC_SHARED_BARRIERSETASSEMBLER_PPC_HPP diff --git a/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.cpp b/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.cpp index d8d0beeadfa..662de44de29 100644 --- a/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.cpp +++ b/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.cpp @@ -308,3 +308,17 @@ void BarrierSetAssembler::c2i_entry_barrier(MacroAssembler* masm) { __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); __ bind(method_live); } + +void BarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) { + // Check if the oop is in the right area of memory + __ mv(tmp2, (intptr_t) Universe::verify_oop_mask()); + __ andr(tmp1, obj, tmp2); + __ mv(tmp2, (intptr_t) Universe::verify_oop_bits()); + + // Compare tmp1 and tmp2. + __ bne(tmp1, tmp2, error); + + // Make sure klass is 'reasonable', which is not zero. + __ load_klass(obj, obj, tmp1); // get klass + __ beqz(obj, error); // if klass is NULL it is broken +} diff --git a/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.hpp b/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.hpp index 7e24c920926..fe6d5b2da92 100644 --- a/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.hpp +++ b/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.hpp @@ -74,6 +74,8 @@ public: virtual void nmethod_entry_barrier(MacroAssembler* masm, Label* slow_path, Label* continuation, Label* guard); virtual void c2i_entry_barrier(MacroAssembler* masm); + virtual void check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error); + virtual bool supports_instruction_patching() { NMethodPatchingType patching_type = nmethod_patching_type(); return patching_type == NMethodPatchingType::conc_instruction_and_data_patch || diff --git a/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.cpp b/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.cpp index 0240217abff..1ebfb8f9e23 100644 --- a/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.cpp +++ b/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.cpp @@ -444,3 +444,17 @@ void ZBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler* #undef __ #endif // COMPILER1 + +#define __ masm-> + +void ZBarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) { + // Check if mask is good. + // verifies that ZAddressBadMask & obj == 0 + __ ld(tmp2, Address(xthread, ZThreadLocalData::address_bad_mask_offset())); + __ andr(tmp1, obj, tmp2); + __ bnez(tmp1, error); + + BarrierSetAssembler::check_oop(masm, obj, tmp1, tmp2, error); +} + +#undef __ diff --git a/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.hpp b/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.hpp index 25d512b9307..c7f29684a64 100644 --- a/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.hpp +++ b/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.hpp @@ -99,6 +99,8 @@ public: void generate_c2_load_barrier_stub(MacroAssembler* masm, ZLoadBarrierStubC2* stub) const; #endif // COMPILER2 + + void check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error); }; #endif // CPU_RISCV_GC_Z_ZBARRIERSETASSEMBLER_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp index a0b9e862c58..1b092a76bc0 100644 --- a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp +++ b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp @@ -614,27 +614,8 @@ class StubGenerator: public StubCodeGenerator { // make sure object is 'reasonable' __ beqz(x10, exit); // if obj is NULL it is OK -#if INCLUDE_ZGC - if (UseZGC) { - // Check if mask is good. - // verifies that ZAddressBadMask & x10 == 0 - __ ld(c_rarg3, Address(xthread, ZThreadLocalData::address_bad_mask_offset())); - __ andr(c_rarg2, x10, c_rarg3); - __ bnez(c_rarg2, error); - } -#endif - - // Check if the oop is in the right area of memory - __ mv(c_rarg3, (intptr_t) Universe::verify_oop_mask()); - __ andr(c_rarg2, x10, c_rarg3); - __ mv(c_rarg3, (intptr_t) Universe::verify_oop_bits()); - - // Compare c_rarg2 and c_rarg3. - __ bne(c_rarg2, c_rarg3, error); - - // make sure klass is 'reasonable', which is not zero. - __ load_klass(x10, x10); // get klass - __ beqz(x10, error); // if klass is NULL it is broken + BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler(); + bs_asm->check_oop(_masm, x10, c_rarg2, c_rarg3, error); // return if everything seems ok __ bind(exit); diff --git a/src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.cpp index 348fd00c9d4..06007d8be58 100644 --- a/src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.cpp @@ -366,3 +366,18 @@ void BarrierSetAssembler::c2i_entry_barrier(MacroAssembler* masm) { __ pop(tmp1); #endif } + +void BarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) { + // Check if the oop is in the right area of memory + __ movptr(tmp1, obj); + __ movptr(tmp2, (intptr_t) Universe::verify_oop_mask()); + __ andptr(tmp1, tmp2); + __ movptr(tmp2, (intptr_t) Universe::verify_oop_bits()); + __ cmpptr(tmp1, tmp2); + __ jcc(Assembler::notZero, error); + + // make sure klass is 'reasonable', which is not zero. + __ load_klass(obj, obj, tmp1); // get klass + __ testptr(obj, obj); + __ jcc(Assembler::zero, error); // if klass is NULL it is broken +} diff --git a/src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.hpp index d793a299eb5..1ba289e4fa4 100644 --- a/src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.hpp @@ -64,6 +64,8 @@ public: virtual void nmethod_entry_barrier(MacroAssembler* masm, Label* slow_path, Label* continuation); virtual void c2i_entry_barrier(MacroAssembler* masm); + + virtual void check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error); }; #endif // CPU_X86_GC_SHARED_BARRIERSETASSEMBLER_X86_HPP diff --git a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp index c93d1d4de6f..597376ceba6 100644 --- a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp @@ -29,6 +29,7 @@ #include "gc/z/zBarrierSet.hpp" #include "gc/z/zBarrierSetAssembler.hpp" #include "gc/z/zBarrierSetRuntime.hpp" +#include "gc/z/zThreadLocalData.hpp" #include "memory/resourceArea.hpp" #include "runtime/sharedRuntime.hpp" #include "utilities/macros.hpp" @@ -700,3 +701,14 @@ void ZBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, Z #undef __ #endif // COMPILER2 + +#define __ masm-> + +void ZBarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) { + // Check if metadata bits indicate a bad oop + __ testptr(obj, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset())); + __ jcc(Assembler::notZero, error); + BarrierSetAssembler::check_oop(masm, obj, tmp1, tmp2, error); +} + +#undef __ diff --git a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp index 2446bd1e46a..2c5887361eb 100644 --- a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp @@ -96,6 +96,8 @@ public: void generate_c2_load_barrier_stub(MacroAssembler* masm, ZLoadBarrierStubC2* stub) const; #endif // COMPILER2 + + void check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error); }; #endif // CPU_X86_GC_Z_ZBARRIERSETASSEMBLER_X86_HPP diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp index 05b55348658..ea0ae37d153 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp @@ -1093,26 +1093,8 @@ address StubGenerator::generate_verify_oop() { __ testptr(rax, rax); __ jcc(Assembler::zero, exit); // if obj is NULL it is OK -#if INCLUDE_ZGC - if (UseZGC) { - // Check if metadata bits indicate a bad oop - __ testptr(rax, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset())); - __ jcc(Assembler::notZero, error); - } -#endif - - // Check if the oop is in the right area of memory - __ movptr(c_rarg2, rax); - __ movptr(c_rarg3, (intptr_t) Universe::verify_oop_mask()); - __ andptr(c_rarg2, c_rarg3); - __ movptr(c_rarg3, (intptr_t) Universe::verify_oop_bits()); - __ cmpptr(c_rarg2, c_rarg3); - __ jcc(Assembler::notZero, error); - - // make sure klass is 'reasonable', which is not zero. - __ load_klass(rax, rax, rscratch1); // get klass - __ testptr(rax, rax); - __ jcc(Assembler::zero, error); // if klass is NULL it is broken + BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler(); + bs_asm->check_oop(_masm, rax, c_rarg2, c_rarg3, error); // return if everything seems ok __ bind(exit);