8236555: [s390] Fix VerifyOops

Reviewed-by: goetz, lucy

src/hotspot/cpu/s390/c1_CodeStubs_s390.cpp

@@ -294,7 +294,7 @@ void PatchingStub::align_patch_site(MacroAssembler* masm) {
 void PatchingStub::emit_code(LIR_Assembler* ce) {
   // Copy original code here.
   assert(NativeGeneralJump::instruction_size <= _bytes_to_copy && _bytes_to_copy <= 0xFF,
-         "not enough room for call");
+         "not enough room for call, need %d", _bytes_to_copy);
 
   NearLabel call_patch;
 
@@ -331,7 +331,7 @@ void PatchingStub::emit_code(LIR_Assembler* ce) {
     }
 #endif
   } else {
-    // Make a copy the code which is going to be patched.
+    // Make a copy of the code which is going to be patched.
     for (int i = 0; i < _bytes_to_copy; i++) {
       address ptr = (address)(_pc_start + i);
       int a_byte = (*ptr) & 0xFF;

src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp

@@ -897,7 +897,7 @@ void LIR_Assembler::mem2reg(LIR_Opr src_opr, LIR_Opr dest, BasicType type, LIR_P
   bool needs_patching = (patch_code != lir_patch_none);
 
   if (addr->base()->type() == T_OBJECT) {
-    __ verify_oop(src);
+    __ verify_oop(src, FILE_AND_LINE);
   }
 
   PatchingStub* patch = NULL;
@@ -972,7 +972,7 @@ void LIR_Assembler::mem2reg(LIR_Opr src_opr, LIR_Opr dest, BasicType type, LIR_P
       } else {
         __ z_lg(dest->as_register(), disp_value, disp_reg, src);
       }
-      __ verify_oop(dest->as_register());
+      __ verify_oop(dest->as_register(), FILE_AND_LINE);
       break;
     }
     case T_FLOAT:
@@ -1006,7 +1006,7 @@ void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
   if (dest->is_single_cpu()) {
     if (is_reference_type(type)) {
       __ mem2reg_opt(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()), true);
-      __ verify_oop(dest->as_register());
+      __ verify_oop(dest->as_register(), FILE_AND_LINE);
     } else if (type == T_METADATA || type == T_ADDRESS) {
       __ mem2reg_opt(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()), true);
     } else {
@@ -1033,7 +1033,7 @@ void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool po
   if (src->is_single_cpu()) {
     const Address dst = frame_map()->address_for_slot(dest->single_stack_ix());
     if (is_reference_type(type)) {
-      __ verify_oop(src->as_register());
+      __ verify_oop(src->as_register(), FILE_AND_LINE);
       __ reg2mem_opt(src->as_register(), dst, true);
     } else if (type == T_METADATA || type == T_ADDRESS) {
       __ reg2mem_opt(src->as_register(), dst, true);
@@ -1079,7 +1079,7 @@ void LIR_Assembler::reg2reg(LIR_Opr from_reg, LIR_Opr to_reg) {
     ShouldNotReachHere();
   }
   if (is_reference_type(to_reg->type())) {
-    __ verify_oop(to_reg->as_register());
+    __ verify_oop(to_reg->as_register(), FILE_AND_LINE);
   }
 }
 
@@ -1095,7 +1095,7 @@ void LIR_Assembler::reg2mem(LIR_Opr from, LIR_Opr dest_opr, BasicType type,
   bool needs_patching = (patch_code != lir_patch_none);
 
   if (addr->base()->is_oop_register()) {
-    __ verify_oop(dest);
+    __ verify_oop(dest, FILE_AND_LINE);
   }
 
   PatchingStub* patch = NULL;
@@ -1130,7 +1130,7 @@ void LIR_Assembler::reg2mem(LIR_Opr from, LIR_Opr dest_opr, BasicType type,
   assert(disp_reg != Z_R0 || Immediate::is_simm20(disp_value), "should have set this up");
 
   if (is_reference_type(type)) {
-    __ verify_oop(from->as_register());
+    __ verify_oop(from->as_register(), FILE_AND_LINE);
   }
 
   bool short_disp = Immediate::is_uimm12(disp_value);
@@ -2412,7 +2412,7 @@ void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
                      op->klass()->as_register(),
                      *op->stub()->entry());
   __ bind(*op->stub()->continuation());
-  __ verify_oop(op->obj()->as_register());
+  __ verify_oop(op->obj()->as_register(), FILE_AND_LINE);
 }
 
 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
@@ -2548,7 +2548,7 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, L
   }
   assert(obj != k_RInfo, "must be different");
 
-  __ verify_oop(obj);
+  __ verify_oop(obj, FILE_AND_LINE);
 
   // Get object class.
   // Not a safepoint as obj null check happens earlier.
@@ -3009,7 +3009,7 @@ void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
   assert(do_null || do_update, "why are we here?");
   assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
 
-  __ verify_oop(obj);
+  __ verify_oop(obj, FILE_AND_LINE);
 
   if (do_null || tmp1 != obj DEBUG_ONLY(|| true)) {
     __ z_ltgr(tmp1, obj);

src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp

@@ -40,7 +40,7 @@
 
 void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) {
   Label ic_miss, ic_hit;
-  verify_oop(receiver);
+  verify_oop(receiver, FILE_AND_LINE);
   int klass_offset = oopDesc::klass_offset_in_bytes();
 
   if (!ImplicitNullChecks || MacroAssembler::needs_explicit_null_check(klass_offset)) {
@@ -83,7 +83,7 @@ void C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hd
   assert_different_registers(hdr, obj, disp_hdr);
   NearLabel done;
 
-  verify_oop(obj);
+  verify_oop(obj, FILE_AND_LINE);
 
   // Load object header.
   z_lg(hdr, Address(obj, hdr_offset));
@@ -158,7 +158,7 @@ void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_
     // Load object.
     z_lg(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
   }
-  verify_oop(obj);
+  verify_oop(obj, FILE_AND_LINE);
   // Test if object header is pointing to the displaced header, and if so, restore
   // the displaced header in the object. If the object header is not pointing to
   // the displaced header, get the object header instead.
@@ -278,7 +278,7 @@ void C1_MacroAssembler::initialize_object(
   //    call(RuntimeAddress(Runtime1::entry_for (Runtime1::dtrace_object_alloc_id)));
   //  }
 
-  verify_oop(obj);
+  verify_oop(obj, FILE_AND_LINE);
 }
 
 void C1_MacroAssembler::allocate_array(
@@ -336,16 +336,15 @@ void C1_MacroAssembler::allocate_array(
   //   call(RuntimeAddress(Runtime1::entry_for (Runtime1::dtrace_object_alloc_id)));
   // }
 
-  verify_oop(obj);
+  verify_oop(obj, FILE_AND_LINE);
 }
 
 
 #ifndef PRODUCT
 
 void C1_MacroAssembler::verify_stack_oop(int stack_offset) {
-  Unimplemented();
-  // if (!VerifyOops) return;
-  // verify_oop_addr(Address(SP, stack_offset + STACK_BIAS));
+  if (!VerifyOops) return;
+  verify_oop_addr(Address(Z_SP, stack_offset), FILE_AND_LINE);
 }
 
 void C1_MacroAssembler::verify_not_null_oop(Register r) {
@@ -354,7 +353,7 @@ void C1_MacroAssembler::verify_not_null_oop(Register r) {
   compareU64_and_branch(r, (intptr_t)0, bcondNotEqual, not_null);
   stop("non-null oop required");
   bind(not_null);
-  verify_oop(r);
+  verify_oop(r, FILE_AND_LINE);
 }
 
 void C1_MacroAssembler::invalidate_registers(Register preserve1,

src/hotspot/cpu/s390/c1_Runtime1_s390.cpp

@@ -339,7 +339,7 @@ OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
         oop_maps->add_gc_map(call_offset, map);
         restore_live_registers_except_r2(sasm);
 
-        __ verify_oop(obj);
+        __ verify_oop(obj, FILE_AND_LINE);
         __ z_br(Z_R14);
       }
       break;
@@ -405,7 +405,7 @@ OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
         oop_maps->add_gc_map(call_offset, map);
         restore_live_registers_except_r2(sasm);
 
-        __ verify_oop(obj);
+        __ verify_oop(obj, FILE_AND_LINE);
         __ z_br(Z_R14);
       }
       break;
@@ -423,7 +423,7 @@ OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
         restore_live_registers_except_r2(sasm);
 
         // Z_R2,: new multi array
-        __ verify_oop(Z_R2);
+        __ verify_oop(Z_R2, FILE_AND_LINE);
         __ z_br(Z_R14);
       }
       break;

src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.cpp

@@ -400,11 +400,11 @@ void G1BarrierSetAssembler::resolve_jobject(MacroAssembler* masm, Register value
 
   __ z_tmll(tmp1, JNIHandles::weak_tag_mask); // Test for jweak tag.
   __ z_braz(Lnot_weak);
-  __ verify_oop(value);
+  __ verify_oop(value, FILE_AND_LINE);
   DecoratorSet decorators = IN_NATIVE | ON_PHANTOM_OOP_REF;
   g1_write_barrier_pre(masm, decorators, (const Address*)NULL, value, noreg, tmp1, tmp2, true);
   __ bind(Lnot_weak);
-  __ verify_oop(value);
+  __ verify_oop(value, FILE_AND_LINE);
   __ bind(Ldone);
 }
 

src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.cpp

@@ -108,7 +108,7 @@ void BarrierSetAssembler::resolve_jobject(MacroAssembler* masm, Register value,
   __ z_nill(value, ~JNIHandles::weak_tag_mask);
   __ z_lg(value, 0, value); // Resolve (untagged) jobject.
 
-  __ verify_oop(value);
+  __ verify_oop(value, FILE_AND_LINE);
   __ bind(Ldone);
 }
 

src/hotspot/cpu/s390/interp_masm_s390.cpp

@@ -1664,7 +1664,7 @@ void InterpreterMacroAssembler::profile_obj_type(Register obj, Address mdo_addr,
     compareU64_and_branch(obj, (intptr_t)0, Assembler::bcondEqual, null_seen);
   }
 
-  verify_oop(obj);
+  MacroAssembler::verify_oop(obj, FILE_AND_LINE);
   load_klass(klass, obj);
 
   // Klass seen before, nothing to do (regardless of unknown bit).
@@ -2073,7 +2073,7 @@ void InterpreterMacroAssembler::access_local_int(Register index, Register dst) {
 }
 
 void InterpreterMacroAssembler::verify_oop(Register reg, TosState state) {
-  if (state == atos) { MacroAssembler::verify_oop(reg); }
+  if (state == atos) { MacroAssembler::verify_oop(reg, FILE_AND_LINE); }
 }
 
 // Inline assembly for:

src/hotspot/cpu/s390/macroAssembler_s390.cpp

@@ -3587,7 +3587,7 @@ void MacroAssembler::get_vm_result(Register oop_result) {
   z_lg(oop_result, Address(Z_thread, JavaThread::vm_result_offset()));
   clear_mem(Address(Z_thread, JavaThread::vm_result_offset()), sizeof(void*));
 
-  verify_oop(oop_result);
+  verify_oop(oop_result, FILE_AND_LINE);
 }
 
 void MacroAssembler::get_vm_result_2(Register result) {
@@ -6813,26 +6813,94 @@ void MacroAssembler::verify_thread() {
   }
 }
 
+// Save and restore functions: Exclude Z_R0.
+void MacroAssembler::save_volatile_regs(Register dst, int offset, bool include_fp, bool include_flags) {
+  z_stmg(Z_R1, Z_R5, offset, dst); offset += 5 * BytesPerWord;
+  if (include_fp) {
+    z_std(Z_F0, Address(dst, offset)); offset += BytesPerWord;
+    z_std(Z_F1, Address(dst, offset)); offset += BytesPerWord;
+    z_std(Z_F2, Address(dst, offset)); offset += BytesPerWord;
+    z_std(Z_F3, Address(dst, offset)); offset += BytesPerWord;
+    z_std(Z_F4, Address(dst, offset)); offset += BytesPerWord;
+    z_std(Z_F5, Address(dst, offset)); offset += BytesPerWord;
+    z_std(Z_F6, Address(dst, offset)); offset += BytesPerWord;
+    z_std(Z_F7, Address(dst, offset)); offset += BytesPerWord;
+  }
+  if (include_flags) {
+    Label done;
+    z_mvi(Address(dst, offset), 2); // encoding: equal
+    z_bre(done);
+    z_mvi(Address(dst, offset), 4); // encoding: higher
+    z_brh(done);
+    z_mvi(Address(dst, offset), 1); // encoding: lower
+    bind(done);
+  }
+}
+void MacroAssembler::restore_volatile_regs(Register src, int offset, bool include_fp, bool include_flags) {
+  z_lmg(Z_R1, Z_R5, offset, src); offset += 5 * BytesPerWord;
+  if (include_fp) {
+    z_ld(Z_F0, Address(src, offset)); offset += BytesPerWord;
+    z_ld(Z_F1, Address(src, offset)); offset += BytesPerWord;
+    z_ld(Z_F2, Address(src, offset)); offset += BytesPerWord;
+    z_ld(Z_F3, Address(src, offset)); offset += BytesPerWord;
+    z_ld(Z_F4, Address(src, offset)); offset += BytesPerWord;
+    z_ld(Z_F5, Address(src, offset)); offset += BytesPerWord;
+    z_ld(Z_F6, Address(src, offset)); offset += BytesPerWord;
+    z_ld(Z_F7, Address(src, offset)); offset += BytesPerWord;
+  }
+  if (include_flags) {
+    z_cli(Address(src, offset), 2); // see encoding above
+  }
+}
+
 // Plausibility check for oops.
 void MacroAssembler::verify_oop(Register oop, const char* msg) {
   if (!VerifyOops) return;
 
   BLOCK_COMMENT("verify_oop {");
-  Register tmp = Z_R0;
-  unsigned int nbytes_save = 5*BytesPerWord;
-  address entry = StubRoutines::verify_oop_subroutine_entry_address();
+  unsigned int nbytes_save = (5 + 8 + 1) * BytesPerWord;
+  address entry_addr = StubRoutines::verify_oop_subroutine_entry_address();
 
   save_return_pc();
-  push_frame_abi160(nbytes_save);
-  z_stmg(Z_R1, Z_R5, frame::z_abi_160_size, Z_SP);
 
-  z_lgr(Z_ARG2, oop);
-  load_const(Z_ARG1, (address) msg);
-  load_const(Z_R1, entry);
+  // Push frame, but preserve flags
+  z_lgr(Z_R0, Z_SP);
+  z_lay(Z_SP, -((int64_t)nbytes_save + frame::z_abi_160_size), Z_SP);
+  z_stg(Z_R0, _z_abi(callers_sp), Z_SP);
+
+  save_volatile_regs(Z_SP, frame::z_abi_160_size, true, true);
+
+  lgr_if_needed(Z_ARG2, oop);
+  load_const_optimized(Z_ARG1, (address)msg);
+  load_const_optimized(Z_R1, entry_addr);
   z_lg(Z_R1, 0, Z_R1);
   call_c(Z_R1);
 
-  z_lmg(Z_R1, Z_R5, frame::z_abi_160_size, Z_SP);
+  restore_volatile_regs(Z_SP, frame::z_abi_160_size, true, true);
+  pop_frame();
+  restore_return_pc();
+
+  BLOCK_COMMENT("} verify_oop ");
+}
+
+void MacroAssembler::verify_oop_addr(Address addr, const char* msg) {
+  if (!VerifyOops) return;
+
+  BLOCK_COMMENT("verify_oop {");
+  unsigned int nbytes_save = (5 + 8) * BytesPerWord;
+  address entry_addr = StubRoutines::verify_oop_subroutine_entry_address();
+
+  save_return_pc();
+  unsigned int frame_size = push_frame_abi160(nbytes_save); // kills Z_R0
+  save_volatile_regs(Z_SP, frame::z_abi_160_size, true, false);
+
+  z_lg(Z_ARG2, addr.plus_disp(frame_size));
+  load_const_optimized(Z_ARG1, (address)msg);
+  load_const_optimized(Z_R1, entry_addr);
+  z_lg(Z_R1, 0, Z_R1);
+  call_c(Z_R1);
+
+  restore_volatile_regs(Z_SP, frame::z_abi_160_size, true, false);
   pop_frame();
   restore_return_pc();
 

src/hotspot/cpu/s390/macroAssembler_s390.hpp

@@ -973,8 +973,15 @@ class MacroAssembler: public Assembler {
   // Verify Z_thread contents.
   void verify_thread();
 
+  // Save and restore functions: Exclude Z_R0.
+  void save_volatile_regs(   Register dst, int offset, bool include_fp, bool include_flags);
+  void restore_volatile_regs(Register src, int offset, bool include_fp, bool include_flags);
+
   // Only if +VerifyOops.
+  // Kills Z_R0.
   void verify_oop(Register reg, const char* s = "broken oop");
+  // Kills Z_R0, condition code.
+  void verify_oop_addr(Address addr, const char* msg = "contains broken oop");
 
   // TODO: verify_method and klass metadata (compare against vptr?).
   void _verify_method_ptr(Register reg, const char * msg, const char * file, int line) {}

src/hotspot/cpu/s390/methodHandles_s390.cpp

@@ -85,7 +85,7 @@ void MethodHandles::verify_klass(MacroAssembler* _masm,
 
   BLOCK_COMMENT("verify_klass {");
 
-  __ verify_oop(obj_reg);
+  __ verify_oop(obj_reg, FILE_AND_LINE);
   __ compareU64_and_branch(obj_reg, (intptr_t)0L, Assembler::bcondEqual, L_bad);
   __ load_klass(temp_reg, obj_reg);
   // klass_addr is a klass in allstatic SystemDictionaryHandles. Can't get GCed.
@@ -194,22 +194,22 @@ void MethodHandles::jump_to_lambda_form(MacroAssembler* _masm,
   BLOCK_COMMENT("jump_to_lambda_form {");
 
   // Load the invoker, as MH -> MH.form -> LF.vmentry
-  __ verify_oop(recv);
+  __ verify_oop(recv, FILE_AND_LINE);
   __ load_heap_oop(method_temp,
                    Address(recv,
                            NONZERO(java_lang_invoke_MethodHandle::form_offset_in_bytes())),
                    noreg, noreg, IS_NOT_NULL);
-  __ verify_oop(method_temp);
+  __ verify_oop(method_temp, FILE_AND_LINE);
   __ load_heap_oop(method_temp,
                    Address(method_temp,
                            NONZERO(java_lang_invoke_LambdaForm::vmentry_offset_in_bytes())),
                    noreg, noreg, IS_NOT_NULL);
-  __ verify_oop(method_temp);
+  __ verify_oop(method_temp, FILE_AND_LINE);
   __ load_heap_oop(method_temp,
                    Address(method_temp,
                            NONZERO(java_lang_invoke_MemberName::method_offset_in_bytes())),
                    noreg, noreg, IS_NOT_NULL);
-  __ verify_oop(method_temp);
+  __ verify_oop(method_temp, FILE_AND_LINE);
   __ z_lg(method_temp,
           Address(method_temp,
                   NONZERO(java_lang_invoke_ResolvedMethodName::vmtarget_offset_in_bytes())));
@@ -385,7 +385,7 @@ void MethodHandles::generate_method_handle_dispatch(MacroAssembler* _masm,
   Register temp1_recv_klass = temp1;
 
   if (iid != vmIntrinsics::_linkToStatic) {
-    __ verify_oop(receiver_reg);
+    __ verify_oop(receiver_reg, FILE_AND_LINE);
     if (iid == vmIntrinsics::_linkToSpecial) {
       // Don't actually load the klass; just null-check the receiver.
       __ null_check(receiver_reg);

src/hotspot/cpu/s390/sharedRuntime_s390.cpp

@@ -892,9 +892,9 @@ static void verify_oop_args(MacroAssembler *masm,
       if (r->is_stack()) {
         __ z_lg(Z_R0_scratch,
                 Address(Z_SP, r->reg2stack() * VMRegImpl::stack_slot_size + wordSize));
-        __ verify_oop(Z_R0_scratch);
+        __ verify_oop(Z_R0_scratch, FILE_AND_LINE);
       } else {
-        __ verify_oop(r->as_Register());
+        __ verify_oop(r->as_Register(), FILE_AND_LINE);
       }
     }
   }
@@ -2686,7 +2686,7 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
       __ z_ltgr(Z_ARG1, Z_ARG1);
       __ z_bre(ic_miss);
     }
-    __ verify_oop(Z_ARG1);
+    __ verify_oop(Z_ARG1, FILE_AND_LINE);
 
     // Check ic: object class <-> cached class
     // Compress cached class for comparison. That's more efficient.
@@ -2955,7 +2955,7 @@ void SharedRuntime::generate_deopt_blob() {
 #ifdef ASSERT
   // verify that there is really an exception oop in JavaThread
   __ z_lg(Z_ARG1, Address(Z_thread, JavaThread::exception_oop_offset()));
-  __ verify_oop(Z_ARG1);
+  __ MacroAssembler::verify_oop(Z_ARG1, FILE_AND_LINE);
 
   // verify that there is no pending exception
   __ asm_assert_mem8_is_zero(in_bytes(Thread::pending_exception_offset()), Z_thread,

src/hotspot/cpu/s390/stubGenerator_s390.cpp

@@ -667,6 +667,17 @@ class StubGenerator: public StubCodeGenerator {
     return start;
   }
 
+#if !defined(PRODUCT)
+  // Wrapper which calls oopDesc::is_oop_or_null()
+  // Only called by MacroAssembler::verify_oop
+  static void verify_oop_helper(const char* message, oopDesc* o) {
+    if (!oopDesc::is_oop_or_null(o)) {
+      fatal("%s. oop: " PTR_FORMAT, message, p2i(o));
+    }
+    ++ StubRoutines::_verify_oop_count;
+  }
+#endif
+
   // Return address of code to be called from code generated by
   // MacroAssembler::verify_oop.
   //
@@ -679,6 +690,11 @@ class StubGenerator: public StubCodeGenerator {
     // StubCodeMark mark(this, "StubRoutines", "verify_oop_stub");
 
     address start = 0;
+
+#if !defined(PRODUCT)
+    start = CAST_FROM_FN_PTR(address, verify_oop_helper);
+#endif
+
     return start;
   }