From 7539cc092d0a6b5604351d19e555101fcff75f58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20Gr=C3=B6nlund?= Date: Fri, 14 Jul 2023 10:46:49 +0000 Subject: [PATCH] 8303134: JFR: Missing stack trace during chunk rotation stress Reviewed-by: egahlin, thartmann --- .../cpu/aarch64/stubGenerator_aarch64.cpp | 52 ++++++- src/hotspot/cpu/arm/stubGenerator_arm.cpp | 52 ++++++- src/hotspot/cpu/ppc/stubGenerator_ppc.cpp | 47 ++++++- src/hotspot/cpu/riscv/stubGenerator_riscv.cpp | 52 ++++++- src/hotspot/cpu/s390/stubGenerator_s390.cpp | 21 ++- src/hotspot/cpu/x86/stubGenerator_x86_32.cpp | 55 +++++++- src/hotspot/cpu/x86/stubGenerator_x86_64.cpp | 53 ++++++- src/hotspot/cpu/x86/stubGenerator_x86_64.hpp | 5 +- src/hotspot/share/c1/c1_LIRGenerator.hpp | 4 - src/hotspot/share/jfr/jni/jfrJniMethod.cpp | 10 +- src/hotspot/share/jfr/jni/jfrJniMethod.hpp | 1 + .../jfr/jni/jfrJniMethodRegistration.cpp | 3 +- .../checkpoint/jfrCheckpointManager.cpp | 2 + .../share/jfr/recorder/storage/jfrBuffer.cpp | 8 ++ .../share/jfr/recorder/storage/jfrBuffer.hpp | 8 +- .../share/jfr/recorder/storage/jfrStorage.cpp | 28 +++- .../share/jfr/recorder/storage/jfrStorage.hpp | 3 +- .../jfr/recorder/stringpool/jfrStringPool.cpp | 55 ++++++++ .../jfr/recorder/stringpool/jfrStringPool.hpp | 2 + .../share/jfr/support/jfrIntrinsics.cpp | 10 ++ .../share/jfr/support/jfrIntrinsics.hpp | 5 +- .../share/jfr/support/jfrThreadExtension.hpp | 13 ++ .../share/jfr/support/jfrThreadLocal.cpp | 9 ++ .../share/jfr/support/jfrThreadLocal.hpp | 15 ++ .../share/jfr/writers/jfrJavaEventWriter.cpp | 76 +++++----- .../share/jfr/writers/jfrJavaEventWriter.hpp | 6 +- src/hotspot/share/opto/c2compiler.cpp | 1 + src/hotspot/share/opto/library_call.cpp | 131 ++++++++++++++++++ src/hotspot/share/opto/library_call.hpp | 1 + src/hotspot/share/runtime/objectMonitor.cpp | 2 +- src/hotspot/share/runtime/stubRoutines.cpp | 2 + src/hotspot/share/runtime/stubRoutines.hpp | 3 + .../share/classes/jdk/jfr/internal/JVM.java | 12 +- .../classes/jdk/jfr/internal/StringPool.java | 83 ++++++++--- .../jdk/jfr/internal/event/EventWriter.java | 65 +++------ test/jdk/jdk/jfr/jvm/TestChunkIntegrity.java | 3 +- 36 files changed, 760 insertions(+), 138 deletions(-) diff --git a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp index a5113f994df..6f1979e8b5a 100644 --- a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp @@ -7221,7 +7221,6 @@ class StubGenerator: public StubCodeGenerator { // The handle is dereferenced through a load barrier. static void jfr_epilogue(MacroAssembler* _masm) { __ reset_last_Java_frame(true); - __ resolve_global_jobject(r0, rscratch1, rscratch2); } // For c2: c_rarg0 is junk, call to runtime to write a checkpoint. @@ -7250,6 +7249,7 @@ class StubGenerator: public StubCodeGenerator { jfr_prologue(the_pc, _masm, rthread); __ call_VM_leaf(CAST_FROM_FN_PTR(address, JfrIntrinsicSupport::write_checkpoint), 1); jfr_epilogue(_masm); + __ resolve_global_jobject(r0, rscratch1, rscratch2); __ leave(); __ ret(lr); @@ -7263,6 +7263,44 @@ class StubGenerator: public StubCodeGenerator { return stub; } + // For c2: call to return a leased buffer. + static RuntimeStub* generate_jfr_return_lease() { + enum layout { + rbp_off, + rbpH_off, + return_off, + return_off2, + framesize // inclusive of return address + }; + + int insts_size = 1024; + int locs_size = 64; + CodeBuffer code("jfr_return_lease", insts_size, locs_size); + OopMapSet* oop_maps = new OopMapSet(); + MacroAssembler* masm = new MacroAssembler(&code); + MacroAssembler* _masm = masm; + + address start = __ pc(); + __ enter(); + int frame_complete = __ pc() - start; + address the_pc = __ pc(); + jfr_prologue(the_pc, _masm, rthread); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, JfrIntrinsicSupport::return_lease), 1); + jfr_epilogue(_masm); + + __ leave(); + __ ret(lr); + + OopMap* map = new OopMap(framesize, 1); // rfp + oop_maps->add_gc_map(the_pc - start, map); + + RuntimeStub* stub = // codeBlob framesize is in words (not VMRegImpl::slot_size) + RuntimeStub::new_runtime_stub("jfr_return_lease", &code, frame_complete, + (framesize >> (LogBytesPerWord - LogBytesPerInt)), + oop_maps, false); + return stub; + } + #endif // INCLUDE_JFR // Continuation point for throwing of implicit exceptions that are @@ -8261,10 +8299,18 @@ class StubGenerator: public StubCodeGenerator { StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier(); StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception(); - JFR_ONLY(StubRoutines::_jfr_write_checkpoint_stub = generate_jfr_write_checkpoint();) - JFR_ONLY(StubRoutines::_jfr_write_checkpoint = StubRoutines::_jfr_write_checkpoint_stub->entry_point();) + JFR_ONLY(generate_jfr_stubs();) } +#if INCLUDE_JFR + void generate_jfr_stubs() { + StubRoutines::_jfr_write_checkpoint_stub = generate_jfr_write_checkpoint(); + StubRoutines::_jfr_write_checkpoint = StubRoutines::_jfr_write_checkpoint_stub->entry_point(); + StubRoutines::_jfr_return_lease_stub = generate_jfr_return_lease(); + StubRoutines::_jfr_return_lease = StubRoutines::_jfr_return_lease_stub->entry_point(); + } +#endif // INCLUDE_JFR + void generate_final_stubs() { // support for verify_oop (must happen after universe_init) if (VerifyOops) { diff --git a/src/hotspot/cpu/arm/stubGenerator_arm.cpp b/src/hotspot/cpu/arm/stubGenerator_arm.cpp index f6e7b46b030..fe01d5f3fa5 100644 --- a/src/hotspot/cpu/arm/stubGenerator_arm.cpp +++ b/src/hotspot/cpu/arm/stubGenerator_arm.cpp @@ -3072,6 +3072,46 @@ class StubGenerator: public StubCodeGenerator { return stub; } + // For c2: call to return a leased buffer. + static RuntimeStub* generate_jfr_return_lease() { + enum layout { + r1_off, + r2_off, + return_off, + framesize // inclusive of return address + }; + + CodeBuffer code("jfr_return_lease", 512, 64); + MacroAssembler* masm = new MacroAssembler(&code); + + address start = __ pc(); + __ raw_push(R1, R2, LR); + address the_pc = __ pc(); + + int frame_complete = the_pc - start; + + __ set_last_Java_frame(SP, FP, true, Rtemp); + __ mov(c_rarg0, Rthread); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, JfrIntrinsicSupport::return_lease), c_rarg0); + __ reset_last_Java_frame(Rtemp); + + __ raw_pop(R1, R2, LR); + __ ret(); + + OopMapSet* oop_maps = new OopMapSet(); + OopMap* map = new OopMap(framesize, 1); + oop_maps->add_gc_map(frame_complete, map); + + RuntimeStub* stub = + RuntimeStub::new_runtime_stub(code.name(), + &code, + frame_complete, + (framesize >> (LogBytesPerWord - LogBytesPerInt)), + oop_maps, + false); + return stub; + } + #endif // INCLUDE_JFR //--------------------------------------------------------------------------- @@ -3116,10 +3156,18 @@ class StubGenerator: public StubCodeGenerator { StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier(); StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception(); - JFR_ONLY(StubRoutines::_jfr_write_checkpoint_stub = generate_jfr_write_checkpoint();) - JFR_ONLY(StubRoutines::_jfr_write_checkpoint = StubRoutines::_jfr_write_checkpoint_stub->entry_point();) + JFR_ONLY(generate_jfr_stubs();) } +#if INCLUDE_JFR + void generate_jfr_stubs() { + StubRoutines::_jfr_write_checkpoint_stub = generate_jfr_write_checkpoint(); + StubRoutines::_jfr_write_checkpoint = StubRoutines::_jfr_write_checkpoint_stub->entry_point(); + StubRoutines::_jfr_return_lease_stub = generate_jfr_return_lease(); + StubRoutines::_jfr_return_lease = StubRoutines::_jfr_return_lease_stub->entry_point(); + } +#endif // INCLUDE_JFR + void generate_final_stubs() { // Generates all stubs and initializes the entry points diff --git a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp index 2e9e18bd8db..c06f22e0fbb 100644 --- a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp +++ b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp @@ -4680,6 +4680,41 @@ class StubGenerator: public StubCodeGenerator { return stub; } + // For c2: call to return a leased buffer. + RuntimeStub* generate_jfr_return_lease() { + CodeBuffer code("jfr_return_lease", 512, 64); + MacroAssembler* _masm = new MacroAssembler(&code); + + Register tmp1 = R10_ARG8; + Register tmp2 = R9_ARG7; + + int framesize = frame::native_abi_reg_args_size / VMRegImpl::stack_slot_size; + address start = __ pc(); + __ mflr(tmp1); + __ std(tmp1, _abi0(lr), R1_SP); // save return pc + __ push_frame_reg_args(0, tmp1); + int frame_complete = __ pc() - start; + __ set_last_Java_frame(R1_SP, noreg); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, JfrIntrinsicSupport::return_lease), R16_thread); + address calls_return_pc = __ last_calls_return_pc(); + __ reset_last_Java_frame(); + __ pop_frame(); + __ ld(tmp1, _abi0(lr), R1_SP); + __ mtlr(tmp1); + __ blr(); + + OopMapSet* oop_maps = new OopMapSet(); + OopMap* map = new OopMap(framesize, 0); + oop_maps->add_gc_map(calls_return_pc - start, map); + + RuntimeStub* stub = // codeBlob framesize is in words (not VMRegImpl::slot_size) + RuntimeStub::new_runtime_stub(code.name(), + &code, frame_complete, + (framesize >> (LogBytesPerWord - LogBytesPerInt)), + oop_maps, false); + return stub; + } + #endif // INCLUDE_JFR @@ -4728,10 +4763,18 @@ class StubGenerator: public StubCodeGenerator { StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier(); StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception(); - JFR_ONLY(StubRoutines::_jfr_write_checkpoint_stub = generate_jfr_write_checkpoint();) - JFR_ONLY(StubRoutines::_jfr_write_checkpoint = StubRoutines::_jfr_write_checkpoint_stub->entry_point();) + JFR_ONLY(generate_jfr_stubs();) } +#if INCLUDE_JFR + void generate_jfr_stubs() { + StubRoutines::_jfr_write_checkpoint_stub = generate_jfr_write_checkpoint(); + StubRoutines::_jfr_write_checkpoint = StubRoutines::_jfr_write_checkpoint_stub->entry_point(); + StubRoutines::_jfr_return_lease_stub = generate_jfr_return_lease(); + StubRoutines::_jfr_return_lease = StubRoutines::_jfr_return_lease_stub->entry_point(); + } +#endif // INCLUDE_JFR + void generate_final_stubs() { // Generates all stubs and initializes the entry points diff --git a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp index 026649f3a6a..811cc9575ee 100644 --- a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp +++ b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp @@ -3923,7 +3923,6 @@ class StubGenerator: public StubCodeGenerator { static void jfr_epilogue(MacroAssembler* _masm) { __ reset_last_Java_frame(true); - __ resolve_global_jobject(x10, t0, t1); } // For c2: c_rarg0 is junk, call to runtime to write a checkpoint. // It returns a jobject handle to the event writer. @@ -3952,6 +3951,7 @@ class StubGenerator: public StubCodeGenerator { __ call_VM_leaf(CAST_FROM_FN_PTR(address, JfrIntrinsicSupport::write_checkpoint), 1); jfr_epilogue(_masm); + __ resolve_global_jobject(x10, t0, t1); __ leave(); __ ret(); @@ -3965,6 +3965,44 @@ class StubGenerator: public StubCodeGenerator { return stub; } + // For c2: call to return a leased buffer. + static RuntimeStub* generate_jfr_return_lease() { + enum layout { + fp_off, + fp_off2, + return_off, + return_off2, + framesize // inclusive of return address + }; + + int insts_size = 1024; + int locs_size = 64; + CodeBuffer code("jfr_return_lease", insts_size, locs_size); + OopMapSet* oop_maps = new OopMapSet(); + MacroAssembler* masm = new MacroAssembler(&code); + MacroAssembler* _masm = masm; + + address start = __ pc(); + __ enter(); + int frame_complete = __ pc() - start; + address the_pc = __ pc(); + jfr_prologue(the_pc, _masm, xthread); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, JfrIntrinsicSupport::return_lease), 1); + + jfr_epilogue(_masm); + __ leave(); + __ ret(); + + OopMap* map = new OopMap(framesize, 1); + oop_maps->add_gc_map(the_pc - start, map); + + RuntimeStub* stub = // codeBlob framesize is in words (not VMRegImpl::slot_size) + RuntimeStub::new_runtime_stub("jfr_return_lease", &code, frame_complete, + (framesize >> (LogBytesPerWord - LogBytesPerInt)), + oop_maps, false); + return stub; + } + #endif // INCLUDE_JFR #undef __ @@ -4008,10 +4046,18 @@ class StubGenerator: public StubCodeGenerator { StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier(); StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception(); - JFR_ONLY(StubRoutines::_jfr_write_checkpoint_stub = generate_jfr_write_checkpoint();) - JFR_ONLY(StubRoutines::_jfr_write_checkpoint = StubRoutines::_jfr_write_checkpoint_stub->entry_point();) + JFR_ONLY(generate_jfr_stubs();) } +#if INCLUDE_JFR + void generate_jfr_stubs() { + StubRoutines::_jfr_write_checkpoint_stub = generate_jfr_write_checkpoint(); + StubRoutines::_jfr_write_checkpoint = StubRoutines::_jfr_write_checkpoint_stub->entry_point(); + StubRoutines::_jfr_return_lease_stub = generate_jfr_return_lease(); + StubRoutines::_jfr_return_lease = StubRoutines::_jfr_return_lease_stub->entry_point(); + } +#endif // INCLUDE_JFR + void generate_final_stubs() { // support for verify_oop (must happen after universe_init) if (VerifyOops) { diff --git a/src/hotspot/cpu/s390/stubGenerator_s390.cpp b/src/hotspot/cpu/s390/stubGenerator_s390.cpp index d5465343131..a26bcd04de2 100644 --- a/src/hotspot/cpu/s390/stubGenerator_s390.cpp +++ b/src/hotspot/cpu/s390/stubGenerator_s390.cpp @@ -3085,7 +3085,14 @@ class StubGenerator: public StubCodeGenerator { Unimplemented(); return nullptr; } - #endif // INCLUD_JFR + + RuntimeStub* generate_jfr_return_lease() { + if (!Continuations::enabled()) return nullptr; + Unimplemented(); + return nullptr; + } + + #endif // INCLUDE_JFR void generate_initial_stubs() { // Generates all stubs and initializes the entry points. @@ -3133,10 +3140,18 @@ class StubGenerator: public StubCodeGenerator { StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier(); StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception(); - JFR_ONLY(StubRoutines::_jfr_write_checkpoint_stub = generate_jfr_write_checkpoint();) - JFR_ONLY(StubRoutines::_jfr_write_checkpoint = StubRoutines::_jfr_write_checkpoint_stub->entry_point();) + JFR_ONLY(generate_jfr_stubs();) } +#if INCLUDE_JFR + void generate_jfr_stubs() { + StubRoutines::_jfr_write_checkpoint_stub = generate_jfr_write_checkpoint(); + StubRoutines::_jfr_write_checkpoint = StubRoutines::_jfr_write_checkpoint_stub->entry_point(); + StubRoutines::_jfr_return_lease_stub = generate_jfr_return_lease(); + StubRoutines::_jfr_return_lease = StubRoutines::_jfr_return_lease_stub->entry_point(); + } +#endif // INCLUDE_JFR + void generate_final_stubs() { // Generates all stubs and initializes the entry points. diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_32.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_32.cpp index ed3e75ee1d3..41294a4f207 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_32.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_32.cpp @@ -4011,7 +4011,6 @@ class StubGenerator: public StubCodeGenerator { Register java_thread = rdi; __ get_thread(java_thread); __ reset_last_Java_frame(java_thread, true); - __ resolve_global_jobject(rax, java_thread, rdx); } // For c2: c_rarg0 is junk, call to runtime to write a checkpoint. @@ -4044,6 +4043,7 @@ class StubGenerator: public StubCodeGenerator { jfr_prologue(the_pc, _masm); __ call_VM_leaf(CAST_FROM_FN_PTR(address, JfrIntrinsicSupport::write_checkpoint), 1); jfr_epilogue(_masm); + __ resolve_global_jobject(rax, rdi, rdx); __ leave(); __ ret(0); @@ -4057,6 +4057,47 @@ class StubGenerator: public StubCodeGenerator { return stub; } + // For c2: call to return a leased buffer. + static RuntimeStub* generate_jfr_return_lease() { + enum layout { + FPUState_off = 0, + rbp_off = FPUStateSizeInWords, + rdi_off, + rsi_off, + rcx_off, + rbx_off, + saved_argument_off, + saved_argument_off2, // 2nd half of double + framesize + }; + + int insts_size = 1024; + int locs_size = 64; + CodeBuffer code("jfr_return_lease", insts_size, locs_size); + OopMapSet* oop_maps = new OopMapSet(); + MacroAssembler* masm = new MacroAssembler(&code); + MacroAssembler* _masm = masm; + + address start = __ pc(); + __ enter(); + int frame_complete = __ pc() - start; + address the_pc = __ pc(); + jfr_prologue(the_pc, _masm); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, JfrIntrinsicSupport::return_lease), 1); + jfr_epilogue(_masm); + __ leave(); + __ ret(0); + + OopMap* map = new OopMap(framesize, 1); // rbp + oop_maps->add_gc_map(the_pc - start, map); + + RuntimeStub* stub = // codeBlob framesize is in words (not VMRegImpl::slot_size) + RuntimeStub::new_runtime_stub("jfr_return_lease", &code, frame_complete, + (framesize >> (LogBytesPerWord - LogBytesPerInt)), + oop_maps, false); + return stub; + } + #endif // INCLUDE_JFR //--------------------------------------------------------------------------- @@ -4148,10 +4189,18 @@ class StubGenerator: public StubCodeGenerator { StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier(); StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception(); - JFR_ONLY(StubRoutines::_jfr_write_checkpoint_stub = generate_jfr_write_checkpoint();) - JFR_ONLY(StubRoutines::_jfr_write_checkpoint = StubRoutines::_jfr_write_checkpoint_stub->entry_point();) + JFR_ONLY(generate_jfr_stubs();) } +#if INCLUDE_JFR + void generate_jfr_stubs() { + StubRoutines::_jfr_write_checkpoint_stub = generate_jfr_write_checkpoint(); + StubRoutines::_jfr_write_checkpoint = StubRoutines::_jfr_write_checkpoint_stub->entry_point(); + StubRoutines::_jfr_return_lease_stub = generate_jfr_return_lease(); + StubRoutines::_jfr_return_lease = StubRoutines::_jfr_return_lease_stub->entry_point(); + } +#endif // INCLUDE_JFR + void generate_final_stubs() { // Generates all stubs and initializes the entry points diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp index 2c100da4a5d..e70723730cd 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp @@ -3750,6 +3750,47 @@ RuntimeStub* StubGenerator::generate_jfr_write_checkpoint() { return stub; } +// For c2: call to return a leased buffer. +RuntimeStub* StubGenerator::generate_jfr_return_lease() { + enum layout { + rbp_off, + rbpH_off, + return_off, + return_off2, + framesize // inclusive of return address + }; + + CodeBuffer code("jfr_return_lease", 1024, 64); + MacroAssembler* _masm = new MacroAssembler(&code); + address start = __ pc(); + + __ enter(); + address the_pc = __ pc(); + + int frame_complete = the_pc - start; + + __ set_last_Java_frame(rsp, rbp, the_pc, rscratch2); + __ movptr(c_rarg0, r15_thread); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, JfrIntrinsicSupport::return_lease), 1); + __ reset_last_Java_frame(true); + + __ leave(); + __ ret(0); + + OopMapSet* oop_maps = new OopMapSet(); + OopMap* map = new OopMap(framesize, 1); + oop_maps->add_gc_map(frame_complete, map); + + RuntimeStub* stub = + RuntimeStub::new_runtime_stub(code.name(), + &code, + frame_complete, + (framesize >> (LogBytesPerWord - LogBytesPerInt)), + oop_maps, + false); + return stub; +} + #endif // INCLUDE_JFR // Continuation point for throwing of implicit exceptions that are @@ -3949,10 +3990,18 @@ void StubGenerator::generate_continuation_stubs() { StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier(); StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception(); - JFR_ONLY(StubRoutines::_jfr_write_checkpoint_stub = generate_jfr_write_checkpoint();) - JFR_ONLY(StubRoutines::_jfr_write_checkpoint = StubRoutines::_jfr_write_checkpoint_stub->entry_point();) + JFR_ONLY(generate_jfr_stubs();) } +#if INCLUDE_JFR +void StubGenerator::generate_jfr_stubs() { + StubRoutines::_jfr_write_checkpoint_stub = generate_jfr_write_checkpoint(); + StubRoutines::_jfr_write_checkpoint = StubRoutines::_jfr_write_checkpoint_stub->entry_point(); + StubRoutines::_jfr_return_lease_stub = generate_jfr_return_lease(); + StubRoutines::_jfr_return_lease = StubRoutines::_jfr_return_lease_stub->entry_point(); +} +#endif + void StubGenerator::generate_final_stubs() { // Generates the rest of stubs and initializes the entry points diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp index e7cc8e3b40a..be3a5f0ea0f 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp @@ -520,12 +520,13 @@ class StubGenerator: public StubCodeGenerator { address generate_cont_returnBarrier_exception(); #if INCLUDE_JFR - + void generate_jfr_stubs(); // For c2: c_rarg0 is junk, call to runtime to write a checkpoint. // It returns a jobject handle to the event writer. // The handle is dereferenced and the return value is the event writer oop. RuntimeStub* generate_jfr_write_checkpoint(); - + // For c2: call to runtime to return a buffer lease. + RuntimeStub* generate_jfr_return_lease(); #endif // INCLUDE_JFR // Continuation point for throwing of implicit exceptions that are diff --git a/src/hotspot/share/c1/c1_LIRGenerator.hpp b/src/hotspot/share/c1/c1_LIRGenerator.hpp index bcd89beb7d5..20101fd02dd 100644 --- a/src/hotspot/share/c1/c1_LIRGenerator.hpp +++ b/src/hotspot/share/c1/c1_LIRGenerator.hpp @@ -469,10 +469,6 @@ class LIRGenerator: public InstructionVisitor, public BlockClosure { SwitchRangeArray* create_lookup_ranges(LookupSwitch* x); void do_SwitchRanges(SwitchRangeArray* x, LIR_Opr value, BlockBegin* default_sux); -#ifdef JFR_HAVE_INTRINSICS - void do_getEventWriter(Intrinsic* x); -#endif - void do_RuntimeCall(address routine, Intrinsic* x); ciKlass* profile_type(ciMethodData* md, int md_first_offset, int md_offset, intptr_t profiled_k, diff --git a/src/hotspot/share/jfr/jni/jfrJniMethod.cpp b/src/hotspot/share/jfr/jni/jfrJniMethod.cpp index a9dae138796..4951e74dfd4 100644 --- a/src/hotspot/share/jfr/jni/jfrJniMethod.cpp +++ b/src/hotspot/share/jfr/jni/jfrJniMethod.cpp @@ -290,9 +290,13 @@ JVM_ENTRY_NO_ENV(jobject, jfr_new_event_writer(JNIEnv* env, jclass jvm)) return JfrJavaEventWriter::new_event_writer(thread); JVM_END -JVM_ENTRY_NO_ENV(jboolean, jfr_event_writer_flush(JNIEnv* env, jclass jvm, jobject writer, jint used_size, jint requested_size)) - return JfrJavaEventWriter::flush(writer, used_size, requested_size, thread); -JVM_END +NO_TRANSITION(void, jfr_event_writer_flush(JNIEnv* env, jclass jvm, jobject writer, jint used_size, jint requested_size)) + JfrJavaEventWriter::flush(writer, used_size, requested_size, JavaThread::current()); +NO_TRANSITION_END + +NO_TRANSITION(jlong, jfr_commit(JNIEnv* env, jclass jvm, jlong next_position)) + return JfrJavaEventWriter::commit(next_position); +NO_TRANSITION_END JVM_ENTRY_NO_ENV(void, jfr_flush(JNIEnv* env, jclass jvm)) JfrRepository::flush(thread); diff --git a/src/hotspot/share/jfr/jni/jfrJniMethod.hpp b/src/hotspot/share/jfr/jni/jfrJniMethod.hpp index 7d6fc3f20bc..28bafc4b73f 100644 --- a/src/hotspot/share/jfr/jni/jfrJniMethod.hpp +++ b/src/hotspot/share/jfr/jni/jfrJniMethod.hpp @@ -115,6 +115,7 @@ jobject JNICALL jfr_new_event_writer(JNIEnv* env, jclass jvm); jboolean JNICALL jfr_event_writer_flush(JNIEnv* env, jclass jvm, jobject writer, jint used_size, jint requested_size); +jlong JNICALL jfr_commit(JNIEnv* env, jclass cls, jlong next_position); void JNICALL jfr_flush(JNIEnv* env, jclass jvm); void JNICALL jfr_abort(JNIEnv* env, jclass jvm, jstring errorMsg); diff --git a/src/hotspot/share/jfr/jni/jfrJniMethodRegistration.cpp b/src/hotspot/share/jfr/jni/jfrJniMethodRegistration.cpp index 8183f5a9358..7c571abbd69 100644 --- a/src/hotspot/share/jfr/jni/jfrJniMethodRegistration.cpp +++ b/src/hotspot/share/jfr/jni/jfrJniMethodRegistration.cpp @@ -71,7 +71,8 @@ JfrJniMethodRegistration::JfrJniMethodRegistration(JNIEnv* env) { (char*)"getTypeId", (char*)"(Ljava/lang/Class;)J", (void*)jfr_type_id, (char*)"getEventWriter", (char*)"()Ljdk/jfr/internal/event/EventWriter;", (void*)jfr_get_event_writer, (char*)"newEventWriter", (char*)"()Ljdk/jfr/internal/event/EventWriter;", (void*)jfr_new_event_writer, - (char*)"flush", (char*)"(Ljdk/jfr/internal/event/EventWriter;II)Z", (void*)jfr_event_writer_flush, + (char*)"flush", (char*)"(Ljdk/jfr/internal/event/EventWriter;II)V", (void*)jfr_event_writer_flush, + (char*)"commit", (char*)"(J)J", (void*)jfr_commit, (char*)"flush", (char*)"()V", (void*)jfr_flush, (char*)"setRepositoryLocation", (char*)"(Ljava/lang/String;)V", (void*)jfr_set_repository_location, (char*)"setDumpPath", (char*)"(Ljava/lang/String;)V", (void*)jfr_set_dump_path, diff --git a/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp b/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp index 680e6ba621c..8da6600f89e 100644 --- a/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp +++ b/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp @@ -38,6 +38,7 @@ #include "jfr/recorder/storage/jfrEpochStorage.inline.hpp" #include "jfr/recorder/storage/jfrMemorySpace.inline.hpp" #include "jfr/recorder/storage/jfrStorageUtils.inline.hpp" +#include "jfr/recorder/stringpool/jfrStringPool.hpp" #include "jfr/support/jfrKlassUnloading.hpp" #include "jfr/support/jfrThreadLocal.hpp" #include "jfr/utilities/jfrBigEndian.hpp" @@ -493,6 +494,7 @@ void JfrCheckpointManager::end_epoch_shift() { debug_only(const u1 current_epoch = JfrTraceIdEpoch::current();) JfrTraceIdEpoch::end_epoch_shift(); assert(current_epoch != JfrTraceIdEpoch::current(), "invariant"); + JfrStringPool::on_epoch_shift(); } size_t JfrCheckpointManager::write() { diff --git a/src/hotspot/share/jfr/recorder/storage/jfrBuffer.cpp b/src/hotspot/share/jfr/recorder/storage/jfrBuffer.cpp index e89147b97df..edbf5ef6981 100644 --- a/src/hotspot/share/jfr/recorder/storage/jfrBuffer.cpp +++ b/src/hotspot/share/jfr/recorder/storage/jfrBuffer.cpp @@ -259,3 +259,11 @@ void JfrBuffer::set_context(u1 context) { void JfrBuffer::clear_context() { set(&_context, 0); } + +ByteSize JfrBuffer::pos_offset() { + return byte_offset_of(JfrBuffer, _pos); +} + +ByteSize JfrBuffer::flags_offset() { + return byte_offset_of(JfrBuffer, _flags); +} \ No newline at end of file diff --git a/src/hotspot/share/jfr/recorder/storage/jfrBuffer.hpp b/src/hotspot/share/jfr/recorder/storage/jfrBuffer.hpp index ae059e9230c..7a7e7fdf21f 100644 --- a/src/hotspot/share/jfr/recorder/storage/jfrBuffer.hpp +++ b/src/hotspot/share/jfr/recorder/storage/jfrBuffer.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -27,6 +27,7 @@ #include "memory/allocation.hpp" #include "runtime/atomic.hpp" +#include "utilities/sizes.hpp" // // Represents a piece of committed memory. @@ -174,6 +175,11 @@ class JfrBuffer { u1 context() const; void set_context(u1 context); void clear_context(); + + // Code generation + static ByteSize pos_offset(); + static ByteSize flags_offset(); + }; #endif // SHARE_JFR_RECORDER_STORAGE_JFRBUFFER_HPP diff --git a/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp b/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp index 420232743a0..49f7719d00d 100644 --- a/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp +++ b/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp @@ -41,6 +41,7 @@ #include "jfr/utilities/jfrTryLock.hpp" #include "jfr/writers/jfrNativeEventWriter.hpp" #include "logging/log.hpp" +#include "runtime/interfaceSupport.inline.hpp" #include "runtime/javaThread.hpp" #include "runtime/mutexLocker.hpp" #include "runtime/safepoint.hpp" @@ -173,15 +174,18 @@ static BufferPtr acquire_lease(size_t size, JfrStorageMspace* mspace, JfrStorage } } -static BufferPtr acquire_promotion_buffer(size_t size, JfrStorageMspace* mspace, JfrStorage& storage_instance, size_t retry_count, Thread* thread) { +BufferPtr JfrStorage::acquire_promotion_buffer(size_t size, JfrStorageMspace* mspace, JfrStorage& storage_instance, size_t retry_count, Thread* thread) { assert(size <= mspace->min_element_size(), "invariant"); while (true) { - BufferPtr buffer= mspace_acquire_live_with_retry(size, mspace, retry_count, thread); - if (buffer == nullptr && storage_instance.control().should_discard()) { + BufferPtr buffer = mspace_acquire_live_with_retry(size, mspace, retry_count, thread); + if (buffer != nullptr) { + return buffer; + } + if (storage_instance.control().should_discard()) { storage_instance.discard_oldest(thread); continue; } - return buffer; + return storage_instance.control().to_disk() ? JfrStorage::acquire_transient(size, thread) : nullptr; } } @@ -251,6 +255,10 @@ bool JfrStorage::flush_regular_buffer(BufferPtr buffer, Thread* thread) { assert(promotion_buffer->free_size() >= unflushed_size, "invariant"); buffer->move(promotion_buffer, unflushed_size); assert(buffer->empty(), "invariant"); + if (promotion_buffer->transient()) { + promotion_buffer->set_retired(); + register_full(promotion_buffer, thread); + } return true; } @@ -277,9 +285,17 @@ void JfrStorage::release_large(BufferPtr buffer, Thread* thread) { void JfrStorage::register_full(BufferPtr buffer, Thread* thread) { assert(buffer != nullptr, "invariant"); - assert(buffer->acquired_by(thread), "invariant"); assert(buffer->retired(), "invariant"); if (_full_list->add(buffer)) { + if (thread->is_Java_thread()) { + JavaThread* jt = JavaThread::cast(thread); + if (jt->thread_state() == _thread_in_native) { + // Transition java thread to vm so it can issue a notify. + ThreadInVMfromNative transition(jt); + _post_box.post(MSG_FULLBUFFER); + return; + } + } _post_box.post(MSG_FULLBUFFER); } } @@ -328,7 +344,6 @@ void JfrStorage::discard_oldest(Thread* thread) { while (_full_list->is_nonempty()) { BufferPtr oldest = _full_list->remove(); assert(oldest != nullptr, "invariant"); - assert(oldest->identity() != nullptr, "invariant"); discarded_size += oldest->discard(); assert(oldest->unflushed_size() == 0, "invariant"); if (oldest->transient()) { @@ -337,6 +352,7 @@ void JfrStorage::discard_oldest(Thread* thread) { } oldest->reinitialize(); assert(!oldest->retired(), "invariant"); + assert(oldest->identity() != nullptr, "invariant"); oldest->release(); // publish break; } diff --git a/src/hotspot/share/jfr/recorder/storage/jfrStorage.hpp b/src/hotspot/share/jfr/recorder/storage/jfrStorage.hpp index 4d0d5ac7230..39d7e330db2 100644 --- a/src/hotspot/share/jfr/recorder/storage/jfrStorage.hpp +++ b/src/hotspot/share/jfr/recorder/storage/jfrStorage.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -64,6 +64,7 @@ class JfrStorage : public JfrCHeapObj { BufferPtr flush_regular(BufferPtr cur, const u1* cur_pos, size_t used, size_t req, bool native, Thread* thread); BufferPtr flush_large(BufferPtr cur, const u1* cur_pos, size_t used, size_t req, bool native, Thread* thread); BufferPtr provision_large(BufferPtr cur, const u1* cur_pos, size_t used, size_t req, bool native, Thread* thread); + BufferPtr acquire_promotion_buffer(size_t size, JfrStorageMspace* mspace, JfrStorage& storage_instance, size_t retry_count, Thread* thread); void release(BufferPtr buffer, Thread* thread); size_t clear(); diff --git a/src/hotspot/share/jfr/recorder/stringpool/jfrStringPool.cpp b/src/hotspot/share/jfr/recorder/stringpool/jfrStringPool.cpp index cc099cdb92e..47cfca6b892 100644 --- a/src/hotspot/share/jfr/recorder/stringpool/jfrStringPool.cpp +++ b/src/hotspot/share/jfr/recorder/stringpool/jfrStringPool.cpp @@ -24,6 +24,9 @@ #include "precompiled.hpp" #include "classfile/javaClasses.inline.hpp" +#include "classfile/symbolTable.hpp" +#include "classfile/systemDictionary.hpp" +#include "classfile/vmSymbols.hpp" #include "jfr/recorder/checkpoint/types/traceid/jfrTraceIdEpoch.hpp" #include "jfr/recorder/service/jfrOptionSet.hpp" #include "jfr/recorder/storage/jfrMemorySpace.inline.hpp" @@ -39,6 +42,45 @@ #include "runtime/javaThread.hpp" #include "runtime/safepoint.hpp" +static int generation_offset = invalid_offset; +static jobject string_pool = nullptr; + +static unsigned short generation = 0; + +static bool setup_string_pool_offsets(TRAPS) { + const char class_name[] = "jdk/jfr/internal/StringPool"; + Symbol* const k_sym = SymbolTable::new_symbol(class_name); + assert(k_sym != nullptr, "invariant"); + Klass* klass = SystemDictionary::resolve_or_fail(k_sym, true, CHECK_false); + assert(klass != nullptr, "invariant"); + klass->initialize(CHECK_false); + assert(!klass->should_be_initialized(), "invariant"); + assert(string_pool == nullptr, "invariant"); + jobject pool = JfrJavaSupport::global_jni_handle(klass->java_mirror(), THREAD); + if (pool == nullptr) { + return false; + } + const char generation_name[] = "generation"; + Symbol* const generation_sym = SymbolTable::new_symbol(generation_name); + assert(generation_sym != nullptr, "invariant"); + assert(invalid_offset == generation_offset, "invariant"); + if (!JfrJavaSupport::compute_field_offset(generation_offset, klass, generation_sym, vmSymbols::short_signature(), true)) { + JfrJavaSupport::destroy_global_jni_handle(pool); + return false; + } + assert(generation_offset != invalid_offset, "invariant"); + string_pool = pool; + return true; +} + +static bool initialize_java_string_pool() { + static bool initialized = false; + if (!initialized) { + initialized = setup_string_pool_offsets(JavaThread::current()); + } + return initialized; +} + typedef JfrStringPool::BufferPtr BufferPtr; static JfrSignal _new_string; @@ -75,6 +117,10 @@ static const size_t string_pool_cache_count = 2; static const size_t string_pool_buffer_size = 512 * K; bool JfrStringPool::initialize() { + if (!initialize_java_string_pool()) { + return false; + } + assert(_mspace == nullptr, "invariant"); _mspace = create_mspace(string_pool_buffer_size, 0, @@ -230,3 +276,12 @@ void JfrStringPool::register_full(BufferPtr buffer, Thread* thread) { assert(buffer->acquired_by(thread), "invariant"); assert(buffer->retired(), "invariant"); } + +void JfrStringPool::on_epoch_shift() { + assert(SafepointSynchronize::is_at_safepoint(), "invariant"); + assert(!JfrTraceIdEpoch::is_synchronizing(), "invariant"); + assert(string_pool != nullptr, "invariant"); + oop mirror = JfrJavaSupport::resolve_non_null(string_pool); + assert(mirror != nullptr, "invariant"); + mirror->short_field_put(generation_offset, generation++); +} diff --git a/src/hotspot/share/jfr/recorder/stringpool/jfrStringPool.hpp b/src/hotspot/share/jfr/recorder/stringpool/jfrStringPool.hpp index c4ff2af6c5c..a65e6cd4d93 100644 --- a/src/hotspot/share/jfr/recorder/stringpool/jfrStringPool.hpp +++ b/src/hotspot/share/jfr/recorder/stringpool/jfrStringPool.hpp @@ -69,10 +69,12 @@ class JfrStringPool : public JfrCHeapObj { bool initialize(); static void destroy(); static bool is_modified(); + static void on_epoch_shift(); // mspace callback void register_full(BufferPtr buffer, Thread* thread); + friend class JfrCheckpointManager; friend class JfrRecorder; friend class JfrRecorderService; friend class JfrStringPoolFlush; diff --git a/src/hotspot/share/jfr/support/jfrIntrinsics.cpp b/src/hotspot/share/jfr/support/jfrIntrinsics.cpp index aaecd4500f2..59d4b37f028 100644 --- a/src/hotspot/share/jfr/support/jfrIntrinsics.cpp +++ b/src/hotspot/share/jfr/support/jfrIntrinsics.cpp @@ -71,6 +71,16 @@ void* JfrIntrinsicSupport::write_checkpoint(JavaThread* jt) { return JfrJavaEventWriter::event_writer(jt); } +void JfrIntrinsicSupport::return_lease(JavaThread* jt) { + DEBUG_ONLY(assert_precondition(jt);) + ThreadStateTransition::transition_from_java(jt, _thread_in_native); + assert(jt->jfr_thread_local()->has_java_event_writer(), "invariant"); + assert(jt->jfr_thread_local()->shelved_buffer() != nullptr, "invariant"); + JfrJavaEventWriter::flush(jt->jfr_thread_local()->java_event_writer(), 0, 0, jt); + assert(jt->jfr_thread_local()->shelved_buffer() == nullptr, "invariant"); + ThreadStateTransition::transition_from_native(jt, _thread_in_Java); +} + void JfrIntrinsicSupport::load_barrier(const Klass* klass) { assert(klass != nullptr, "sanity"); JfrTraceIdLoadBarrier::load_barrier(klass); diff --git a/src/hotspot/share/jfr/support/jfrIntrinsics.hpp b/src/hotspot/share/jfr/support/jfrIntrinsics.hpp index aa307062cfa..2e59749a61d 100644 --- a/src/hotspot/share/jfr/support/jfrIntrinsics.hpp +++ b/src/hotspot/share/jfr/support/jfrIntrinsics.hpp @@ -1,5 +1,5 @@ /* -* Copyright (c) 2012, 2022, Oracle and/or its affiliates. All rights reserved. +* Copyright (c) 2012, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -37,6 +37,7 @@ class JfrIntrinsicSupport : AllStatic { public: static void* write_checkpoint(JavaThread* jt); + static void return_lease(JavaThread* jt); static void load_barrier(const Klass* klass); static address epoch_address(); static address signal_address(); @@ -61,7 +62,7 @@ class JfrIntrinsicSupport : AllStatic { do_name( getClassId_name, "getClassId") \ do_intrinsic(_getEventWriter, jdk_jfr_internal_JVM, getEventWriter_name, getEventWriter_signature, F_SN) \ do_name( getEventWriter_name, "getEventWriter") \ - + do_intrinsic(_jvm_commit, jdk_jfr_internal_JVM, commit_name, long_long_signature, F_SN) #else // !INCLUDE_JFR #define JFR_TEMPLATES(template) diff --git a/src/hotspot/share/jfr/support/jfrThreadExtension.hpp b/src/hotspot/share/jfr/support/jfrThreadExtension.hpp index a98cda7d415..1396b6d51c1 100644 --- a/src/hotspot/share/jfr/support/jfrThreadExtension.hpp +++ b/src/hotspot/share/jfr/support/jfrThreadExtension.hpp @@ -26,6 +26,7 @@ #define SHARE_JFR_SUPPORT_JFRTHREADEXTENSION_HPP #include "jfr/periodic/sampling/jfrThreadSampler.hpp" +#include "jfr/recorder/storage/jfrBuffer.hpp" #include "jfr/support/jfrThreadId.hpp" #define DEFINE_THREAD_LOCAL_FIELD_JFR mutable JfrThreadLocal _jfr_thread_local @@ -49,6 +50,18 @@ #define VTHREAD_EXCLUDED_OFFSET_JFR JfrThreadLocal::vthread_excluded_offset() +#define JAVA_BUFFER_OFFSET_JFR \ + JfrThreadLocal::java_buffer_offset() + THREAD_LOCAL_OFFSET_JFR + +#define NOTIFY_OFFSET_JFR \ + JfrThreadLocal::notified_offset() + THREAD_LOCAL_OFFSET_JFR + +#define JFR_BUFFER_POS_OFFSET \ + JfrBuffer::pos_offset() + +#define JFR_BUFFER_FLAGS_OFFSET \ + JfrBuffer::flags_offset() + #define THREAD_LOCAL_WRITER_OFFSET_JFR \ JfrThreadLocal::java_event_writer_offset() + THREAD_LOCAL_OFFSET_JFR diff --git a/src/hotspot/share/jfr/support/jfrThreadLocal.cpp b/src/hotspot/share/jfr/support/jfrThreadLocal.cpp index 9b661efe18b..6e4e6644080 100644 --- a/src/hotspot/share/jfr/support/jfrThreadLocal.cpp +++ b/src/hotspot/share/jfr/support/jfrThreadLocal.cpp @@ -76,6 +76,7 @@ JfrThreadLocal::JfrThreadLocal() : _vthread_excluded(false), _jvm_thread_excluded(false), _vthread(false), + _notified(false), _dead(false) { Thread* thread = Thread::current_or_null(); _parent_trace_id = thread != nullptr ? jvm_thread_id(thread) : (traceid)0; @@ -247,6 +248,10 @@ ByteSize JfrThreadLocal::java_event_writer_offset() { return byte_offset_of(JfrThreadLocal, _java_event_writer); } +ByteSize JfrThreadLocal::java_buffer_offset() { + return byte_offset_of(JfrThreadLocal, _java_buffer); +} + ByteSize JfrThreadLocal::vthread_id_offset() { return byte_offset_of(JfrThreadLocal, _vthread_id); } @@ -263,6 +268,10 @@ ByteSize JfrThreadLocal::vthread_excluded_offset() { return byte_offset_of(JfrThreadLocal, _vthread_excluded); } +ByteSize JfrThreadLocal::notified_offset() { + return byte_offset_of(JfrThreadLocal, _notified); +} + void JfrThreadLocal::set(bool* exclusion_field, bool state) { assert(exclusion_field != nullptr, "invariant"); *exclusion_field = state; diff --git a/src/hotspot/share/jfr/support/jfrThreadLocal.hpp b/src/hotspot/share/jfr/support/jfrThreadLocal.hpp index 6d71b94362e..da7d1f5e2ef 100644 --- a/src/hotspot/share/jfr/support/jfrThreadLocal.hpp +++ b/src/hotspot/share/jfr/support/jfrThreadLocal.hpp @@ -70,6 +70,7 @@ class JfrThreadLocal { bool _vthread_excluded; bool _jvm_thread_excluded; bool _vthread; + bool _notified; bool _dead; JfrBuffer* install_native_buffer() const; @@ -250,6 +251,18 @@ class JfrThreadLocal { _wallclock_time = wallclock_time; } + bool is_notified() { + return _notified; + } + + void notify() { + _notified = true; + } + + void clear_notification() { + _notified = false; + } + bool is_dead() const { return _dead; } @@ -273,10 +286,12 @@ class JfrThreadLocal { // Code generation static ByteSize java_event_writer_offset(); + static ByteSize java_buffer_offset(); static ByteSize vthread_id_offset(); static ByteSize vthread_offset(); static ByteSize vthread_epoch_offset(); static ByteSize vthread_excluded_offset(); + static ByteSize notified_offset(); friend class JfrJavaThread; friend class JfrCheckpointManager; diff --git a/src/hotspot/share/jfr/writers/jfrJavaEventWriter.cpp b/src/hotspot/share/jfr/writers/jfrJavaEventWriter.cpp index 2c56fad846a..5ff319782b2 100644 --- a/src/hotspot/share/jfr/writers/jfrJavaEventWriter.cpp +++ b/src/hotspot/share/jfr/writers/jfrJavaEventWriter.cpp @@ -37,6 +37,7 @@ #include "oops/oop.inline.hpp" #include "runtime/fieldDescriptor.inline.hpp" #include "runtime/handles.inline.hpp" +#include "runtime/interfaceSupport.inline.hpp" #include "runtime/jniHandles.inline.hpp" #include "runtime/safepoint.hpp" #include "runtime/threads.hpp" @@ -45,7 +46,6 @@ static int start_pos_offset = invalid_offset; static int start_pos_address_offset = invalid_offset; static int current_pos_offset = invalid_offset; static int max_pos_offset = invalid_offset; -static int notified_offset = invalid_offset; static int excluded_offset = invalid_offset; static int thread_id_offset = invalid_offset; static int valid_offset = invalid_offset; @@ -64,13 +64,6 @@ static bool setup_event_writer_offsets(TRAPS) { JfrJavaSupport::compute_field_offset(start_pos_offset, klass, start_pos_sym, vmSymbols::long_signature()); assert(start_pos_offset != invalid_offset, "invariant"); - const char start_pos_address_name[] = "startPositionAddress"; - Symbol* const start_pos_address_sym = SymbolTable::new_symbol(start_pos_address_name); - assert(start_pos_address_sym != nullptr, "invariant"); - assert(invalid_offset == start_pos_address_offset, "invariant"); - JfrJavaSupport::compute_field_offset(start_pos_address_offset, klass, start_pos_address_sym, vmSymbols::long_signature()); - assert(start_pos_address_offset != invalid_offset, "invariant"); - const char event_pos_name[] = "currentPosition"; Symbol* const event_pos_sym = SymbolTable::new_symbol(event_pos_name); assert(event_pos_sym != nullptr, "invariant"); @@ -85,13 +78,6 @@ static bool setup_event_writer_offsets(TRAPS) { JfrJavaSupport::compute_field_offset(max_pos_offset, klass, max_pos_sym, vmSymbols::long_signature()); assert(max_pos_offset != invalid_offset, "invariant"); - const char notified_name[] = "notified"; - Symbol* const notified_sym = SymbolTable::new_symbol(notified_name); - assert (notified_sym != nullptr, "invariant"); - assert(invalid_offset == notified_offset, "invariant"); - JfrJavaSupport::compute_field_offset(notified_offset, klass, notified_sym, vmSymbols::bool_signature()); - assert(notified_offset != invalid_offset, "invariant"); - const char excluded_name[] = "excluded"; Symbol* const excluded_sym = SymbolTable::new_symbol(excluded_name); assert(excluded_sym != nullptr, "invariant"); @@ -123,11 +109,9 @@ bool JfrJavaEventWriter::initialize() { return initialized; } -jboolean JfrJavaEventWriter::flush(jobject writer, jint used, jint requested, JavaThread* jt) { - DEBUG_ONLY(JfrJavaSupport::check_java_thread_in_vm(jt)); +void JfrJavaEventWriter::flush(jobject writer, jint used, jint requested, JavaThread* jt) { + DEBUG_ONLY(JfrJavaSupport::check_java_thread_in_native(jt)); assert(writer != nullptr, "invariant"); - oop const w = JNIHandles::resolve_non_null(writer); - assert(w != nullptr, "invariant"); JfrBuffer* const current = jt->jfr_thread_local()->java_buffer(); assert(current != nullptr, "invariant"); JfrBuffer* const buffer = JfrStorage::flush(current, used, requested, false, jt); @@ -138,23 +122,48 @@ jboolean JfrJavaEventWriter::flush(jobject writer, jint used, jint requested, Ja const bool is_valid = buffer->free_size() >= (size_t)(used + requested); u1* const new_current_position = is_valid ? buffer->pos() + used : buffer->pos(); assert(start_pos_offset != invalid_offset, "invariant"); + // can safepoint here + ThreadInVMfromNative transition(jt); + oop const w = JNIHandles::resolve_non_null(writer); + assert(w != nullptr, "invariant"); w->long_field_put(start_pos_offset, (jlong)buffer->pos()); w->long_field_put(current_pos_offset, (jlong)new_current_position); // only update java writer if underlying memory changed if (buffer != current) { - w->long_field_put(start_pos_address_offset, (jlong)buffer->pos_address()); w->long_field_put(max_pos_offset, (jlong)buffer->end()); } if (!is_valid) { // mark writer as invalid for this write attempt w->release_bool_field_put(valid_offset, JNI_FALSE); - return JNI_FALSE; } - // An exclusive use of a leased buffer is treated equivalent to - // holding a system resource. As such, it should be released as soon as possible. - // Returning true here signals that the thread will need to call flush again - // on EventWriter.endEvent() and that flush will return the lease. - return buffer->lease() ? JNI_TRUE : JNI_FALSE; +} + +jlong JfrJavaEventWriter::commit(jlong next_position) { + assert(next_position != 0, "invariant"); + JavaThread* const jt = JavaThread::current(); + assert(jt != nullptr, "invariant"); + DEBUG_ONLY(JfrJavaSupport::check_java_thread_in_native(jt)); + JfrThreadLocal* const tl = jt->jfr_thread_local(); + assert(tl != nullptr, "invariant"); + assert(tl->has_java_event_writer(), "invariant"); + assert(tl->has_java_buffer(), "invariant"); + JfrBuffer* const current = tl->java_buffer(); + assert(current != nullptr, "invariant"); + u1* const next = reinterpret_cast(next_position); + assert(next >= current->start(), "invariant"); + assert(next <= current->end(), "invariant"); + if (tl->is_notified()) { + tl->clear_notification(); + return reinterpret_cast(current->pos()); + } + // set_pos() has release semantics + current->set_pos(next); + if (!current->lease()) { + return next_position; + } + assert(current->lease(), "invariant"); + flush(tl->java_event_writer(), 0, 0, jt); + return 0; // signals that the buffer lease was returned. } class JfrJavaEventWriterNotificationClosure : public ThreadClosure { @@ -198,9 +207,14 @@ void JfrJavaEventWriter::notify(JavaThread* jt) { assert(jt != nullptr, "invariant"); assert(SafepointSynchronize::is_at_safepoint(), "invariant"); if (jt->jfr_thread_local()->has_java_event_writer()) { - oop buffer_writer = JNIHandles::resolve_non_null(jt->jfr_thread_local()->java_event_writer()); - assert(buffer_writer != nullptr, "invariant"); - buffer_writer->release_bool_field_put(notified_offset, JNI_TRUE); + JfrThreadLocal* const tl = jt->jfr_thread_local(); + assert(tl != nullptr, "invariant"); + oop event_writer = JNIHandles::resolve_non_null(tl->java_event_writer()); + assert(event_writer != nullptr, "invariant"); + const jlong start_pos = event_writer->long_field(start_pos_offset); + if (event_writer->long_field(current_pos_offset) > start_pos) { + tl->notify(); + } } } @@ -210,14 +224,13 @@ static jobject create_new_event_writer(JfrBuffer* buffer, JfrThreadLocal* tl, TR HandleMark hm(THREAD); static const char klass[] = "jdk/jfr/internal/event/EventWriter"; static const char method[] = ""; - static const char signature[] = "(JJJJZZ)V"; + static const char signature[] = "(JJJZZ)V"; JavaValue result(T_OBJECT); JfrJavaArguments args(&result, klass, method, signature, CHECK_NULL); // parameters args.push_long((jlong)buffer->pos()); args.push_long((jlong)buffer->end()); - args.push_long((jlong)buffer->pos_address()); args.push_long((jlong)JfrThreadLocal::thread_id(THREAD)); args.push_int((jint)JNI_TRUE); // valid args.push_int(tl->is_excluded() ? (jint)JNI_TRUE : (jint)JNI_FALSE); // excluded @@ -228,7 +241,6 @@ static jobject create_new_event_writer(JfrBuffer* buffer, JfrThreadLocal* tl, TR jobject JfrJavaEventWriter::event_writer(JavaThread* jt) { DEBUG_ONLY(JfrJavaSupport::check_java_thread_in_vm(jt)); JfrThreadLocal* const tl = jt->jfr_thread_local(); - assert(tl->shelved_buffer() == nullptr, "invariant"); jobject h_writer = tl->java_event_writer(); if (h_writer != nullptr) { oop writer = JNIHandles::resolve_non_null(h_writer); diff --git a/src/hotspot/share/jfr/writers/jfrJavaEventWriter.hpp b/src/hotspot/share/jfr/writers/jfrJavaEventWriter.hpp index af99690ba47..b93d6ad8cac 100644 --- a/src/hotspot/share/jfr/writers/jfrJavaEventWriter.hpp +++ b/src/hotspot/share/jfr/writers/jfrJavaEventWriter.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -31,6 +31,7 @@ #include "utilities/exceptions.hpp" class JavaThread; +class JfrBuffer; class Thread; class JfrJavaEventWriter : AllStatic { @@ -48,7 +49,8 @@ class JfrJavaEventWriter : AllStatic { static void include(traceid tid, const JavaThread* jt); static jobject event_writer(JavaThread* t); static jobject new_event_writer(TRAPS); - static jboolean flush(jobject writer, jint used, jint requested, JavaThread* jt); + static void flush(jobject writer, jint used, jint requested, JavaThread* jt); + static jlong commit(jlong next_position); }; #endif // SHARE_JFR_WRITERS_JFRJAVAEVENTWRITER_HPP diff --git a/src/hotspot/share/opto/c2compiler.cpp b/src/hotspot/share/opto/c2compiler.cpp index 998a7741f13..fac11e9a8a0 100644 --- a/src/hotspot/share/opto/c2compiler.cpp +++ b/src/hotspot/share/opto/c2compiler.cpp @@ -672,6 +672,7 @@ bool C2Compiler::is_intrinsic_supported(const methodHandle& method) { #ifdef JFR_HAVE_INTRINSICS case vmIntrinsics::_counterTime: case vmIntrinsics::_getEventWriter: + case vmIntrinsics::_jvm_commit: #endif case vmIntrinsics::_currentTimeMillis: case vmIntrinsics::_nanoTime: diff --git a/src/hotspot/share/opto/library_call.cpp b/src/hotspot/share/opto/library_call.cpp index dffd0e3d2d7..833b76ae212 100644 --- a/src/hotspot/share/opto/library_call.cpp +++ b/src/hotspot/share/opto/library_call.cpp @@ -494,6 +494,7 @@ bool LibraryCallKit::try_to_inline(int predicate) { #ifdef JFR_HAVE_INTRINSICS case vmIntrinsics::_counterTime: return inline_native_time_funcs(CAST_FROM_FN_PTR(address, JfrTime::time_function()), "counterTime"); case vmIntrinsics::_getEventWriter: return inline_native_getEventWriter(); + case vmIntrinsics::_jvm_commit: return inline_native_jvm_commit(); #endif case vmIntrinsics::_currentTimeMillis: return inline_native_time_funcs(CAST_FROM_FN_PTR(address, os::javaTimeMillis), "currentTimeMillis"); case vmIntrinsics::_nanoTime: return inline_native_time_funcs(CAST_FROM_FN_PTR(address, os::javaTimeNanos), "nanoTime"); @@ -3003,6 +3004,136 @@ bool LibraryCallKit::inline_native_classID() { return true; } +//------------------------inline_native_jvm_commit------------------ +bool LibraryCallKit::inline_native_jvm_commit() { + enum { _true_path = 1, _false_path = 2, PATH_LIMIT }; + + // Save input memory and i_o state. + Node* input_memory_state = reset_memory(); + set_all_memory(input_memory_state); + Node* input_io_state = i_o(); + + // TLS. + Node* tls_ptr = _gvn.transform(new ThreadLocalNode()); + // Jfr java buffer. + Node* java_buffer_offset = _gvn.transform(new AddPNode(top(), tls_ptr, _gvn.transform(MakeConX(in_bytes(JAVA_BUFFER_OFFSET_JFR))))); + Node* java_buffer = _gvn.transform(new LoadPNode(control(), input_memory_state, java_buffer_offset, TypePtr::BOTTOM, TypeRawPtr::NOTNULL, MemNode::unordered)); + Node* java_buffer_pos_offset = _gvn.transform(new AddPNode(top(), java_buffer, _gvn.transform(MakeConX(in_bytes(JFR_BUFFER_POS_OFFSET))))); + + // Load the current value of the notified field in the JfrThreadLocal. + Node* notified_offset = basic_plus_adr(top(), tls_ptr, in_bytes(NOTIFY_OFFSET_JFR)); + Node* notified = make_load(control(), notified_offset, TypeInt::BOOL, T_BOOLEAN, MemNode::unordered); + + // Test for notification. + Node* notified_cmp = _gvn.transform(new CmpINode(notified, _gvn.intcon(1))); + Node* test_notified = _gvn.transform(new BoolNode(notified_cmp, BoolTest::eq)); + IfNode* iff_notified = create_and_map_if(control(), test_notified, PROB_MIN, COUNT_UNKNOWN); + + // True branch, is notified. + Node* is_notified = _gvn.transform(new IfTrueNode(iff_notified)); + set_control(is_notified); + + // Reset notified state. + Node* notified_reset_memory = store_to_memory(control(), notified_offset, _gvn.intcon(0), T_BOOLEAN, Compile::AliasIdxRaw, MemNode::unordered); + + // Iff notified, the return address of the commit method is the current position of the backing java buffer. This is used to reset the event writer. + Node* current_pos_X = _gvn.transform(new LoadXNode(control(), input_memory_state, java_buffer_pos_offset, TypeRawPtr::NOTNULL, TypeX_X, MemNode::unordered)); + // Convert the machine-word to a long. + Node* current_pos = _gvn.transform(ConvX2L(current_pos_X)); + + // False branch, not notified. + Node* not_notified = _gvn.transform(new IfFalseNode(iff_notified)); + set_control(not_notified); + set_all_memory(input_memory_state); + + // Arg is the next position as a long. + Node* arg = argument(0); + // Convert long to machine-word. + Node* next_pos_X = _gvn.transform(ConvL2X(arg)); + + // Store the next_position to the underlying jfr java buffer. + Node* commit_memory; +#ifdef _LP64 + commit_memory = store_to_memory(control(), java_buffer_pos_offset, next_pos_X, T_LONG, Compile::AliasIdxRaw, MemNode::release); +#else + commit_memory = store_to_memory(control(), java_buffer_pos_offset, next_pos_X, T_INT, Compile::AliasIdxRaw, MemNode::release); +#endif + + // Now load the flags from off the java buffer and decide if the buffer is a lease. If so, it needs to be returned post-commit. + Node* java_buffer_flags_offset = _gvn.transform(new AddPNode(top(), java_buffer, _gvn.transform(MakeConX(in_bytes(JFR_BUFFER_FLAGS_OFFSET))))); + Node* flags = make_load(control(), java_buffer_flags_offset, TypeInt::UBYTE, T_BYTE, MemNode::unordered); + Node* lease_constant = _gvn.transform(_gvn.intcon(4)); + + // And flags with lease constant. + Node* lease = _gvn.transform(new AndINode(flags, lease_constant)); + + // Branch on lease to conditionalize returning the leased java buffer. + Node* lease_cmp = _gvn.transform(new CmpINode(lease, lease_constant)); + Node* test_lease = _gvn.transform(new BoolNode(lease_cmp, BoolTest::eq)); + IfNode* iff_lease = create_and_map_if(control(), test_lease, PROB_MIN, COUNT_UNKNOWN); + + // False branch, not a lease. + Node* not_lease = _gvn.transform(new IfFalseNode(iff_lease)); + + // True branch, is lease. + Node* is_lease = _gvn.transform(new IfTrueNode(iff_lease)); + set_control(is_lease); + + // Make a runtime call, which can safepoint, to return the leased buffer. This updates both the JfrThreadLocal and the Java event writer oop. + Node* call_return_lease = make_runtime_call(RC_NO_LEAF, + OptoRuntime::void_void_Type(), + StubRoutines::jfr_return_lease(), + "return_lease", TypePtr::BOTTOM); + Node* call_return_lease_control = _gvn.transform(new ProjNode(call_return_lease, TypeFunc::Control)); + + RegionNode* lease_compare_rgn = new RegionNode(PATH_LIMIT); + record_for_igvn(lease_compare_rgn); + PhiNode* lease_compare_mem = new PhiNode(lease_compare_rgn, Type::MEMORY, TypePtr::BOTTOM); + record_for_igvn(lease_compare_mem); + PhiNode* lease_compare_io = new PhiNode(lease_compare_rgn, Type::ABIO); + record_for_igvn(lease_compare_io); + PhiNode* lease_result_value = new PhiNode(lease_compare_rgn, TypeLong::LONG); + record_for_igvn(lease_result_value); + + // Update control and phi nodes. + lease_compare_rgn->init_req(_true_path, call_return_lease_control); + lease_compare_rgn->init_req(_false_path, not_lease); + + lease_compare_mem->init_req(_true_path, _gvn.transform(reset_memory())); + lease_compare_mem->init_req(_false_path, commit_memory); + + lease_compare_io->init_req(_true_path, i_o()); + lease_compare_io->init_req(_false_path, input_io_state); + + lease_result_value->init_req(_true_path, null()); // if the lease was returned, return 0. + lease_result_value->init_req(_false_path, arg); // if not lease, return new updated position. + + RegionNode* result_rgn = new RegionNode(PATH_LIMIT); + PhiNode* result_mem = new PhiNode(result_rgn, Type::MEMORY, TypePtr::BOTTOM); + PhiNode* result_io = new PhiNode(result_rgn, Type::ABIO); + PhiNode* result_value = new PhiNode(result_rgn, TypeLong::LONG); + + // Update control and phi nodes. + result_rgn->init_req(_true_path, is_notified); + result_rgn->init_req(_false_path, _gvn.transform(lease_compare_rgn)); + + result_mem->init_req(_true_path, notified_reset_memory); + result_mem->init_req(_false_path, _gvn.transform(lease_compare_mem)); + + result_io->init_req(_true_path, input_io_state); + result_io->init_req(_false_path, _gvn.transform(lease_compare_io)); + + result_value->init_req(_true_path, current_pos); + result_value->init_req(_false_path, _gvn.transform(lease_result_value)); + + // Set output state. + set_control(_gvn.transform(result_rgn)); + set_all_memory(_gvn.transform(result_mem)); + set_i_o(_gvn.transform(result_io)); + set_result(result_rgn, result_value); + return true; +} + /* * The intrinsic is a model of this pseudo-code: * diff --git a/src/hotspot/share/opto/library_call.hpp b/src/hotspot/share/opto/library_call.hpp index bec6f2d8829..d4d6e50b2a5 100644 --- a/src/hotspot/share/opto/library_call.hpp +++ b/src/hotspot/share/opto/library_call.hpp @@ -249,6 +249,7 @@ class LibraryCallKit : public GraphKit { #ifdef JFR_HAVE_INTRINSICS bool inline_native_classID(); bool inline_native_getEventWriter(); + bool inline_native_jvm_commit(); void extend_setCurrentThread(Node* jt, Node* thread); #endif bool inline_native_Class_query(vmIntrinsics::ID id); diff --git a/src/hotspot/share/runtime/objectMonitor.cpp b/src/hotspot/share/runtime/objectMonitor.cpp index 0739de77b8a..12a0e953771 100644 --- a/src/hotspot/share/runtime/objectMonitor.cpp +++ b/src/hotspot/share/runtime/objectMonitor.cpp @@ -386,7 +386,7 @@ bool ObjectMonitor::enter(JavaThread* current) { return false; } - JFR_ONLY(JfrConditionalFlushWithStacktrace flush(current);) + JFR_ONLY(JfrConditionalFlush flush(current);) EventJavaMonitorEnter event; if (event.is_started()) { event.set_monitorClass(object()->klass()); diff --git a/src/hotspot/share/runtime/stubRoutines.cpp b/src/hotspot/share/runtime/stubRoutines.cpp index 73bc3a003fb..1f54b8871a3 100644 --- a/src/hotspot/share/runtime/stubRoutines.cpp +++ b/src/hotspot/share/runtime/stubRoutines.cpp @@ -182,6 +182,8 @@ address StubRoutines::_cont_returnBarrierExc = nullptr; JFR_ONLY(RuntimeStub* StubRoutines::_jfr_write_checkpoint_stub = nullptr;) JFR_ONLY(address StubRoutines::_jfr_write_checkpoint = nullptr;) +JFR_ONLY(RuntimeStub* StubRoutines::_jfr_return_lease_stub = nullptr;) +JFR_ONLY(address StubRoutines::_jfr_return_lease = nullptr;) // Initialization // diff --git a/src/hotspot/share/runtime/stubRoutines.hpp b/src/hotspot/share/runtime/stubRoutines.hpp index ae37b9409e9..4d8688f68cb 100644 --- a/src/hotspot/share/runtime/stubRoutines.hpp +++ b/src/hotspot/share/runtime/stubRoutines.hpp @@ -260,6 +260,8 @@ class StubRoutines: AllStatic { JFR_ONLY(static RuntimeStub* _jfr_write_checkpoint_stub;) JFR_ONLY(static address _jfr_write_checkpoint;) + JFR_ONLY(static RuntimeStub* _jfr_return_lease_stub;) + JFR_ONLY(static address _jfr_return_lease;) // Vector Math Routines static address _vector_f_math[VectorSupport::NUM_VEC_SIZES][VectorSupport::NUM_SVML_OP]; @@ -457,6 +459,7 @@ class StubRoutines: AllStatic { static address cont_returnBarrierExc(){return _cont_returnBarrierExc; } JFR_ONLY(static address jfr_write_checkpoint() { return _jfr_write_checkpoint; }) + JFR_ONLY(static address jfr_return_lease() { return _jfr_return_lease; }) static address select_fill_function(BasicType t, bool aligned, const char* &name); diff --git a/src/jdk.jfr/share/classes/jdk/jfr/internal/JVM.java b/src/jdk.jfr/share/classes/jdk/jfr/internal/JVM.java index f749a458287..986b6f2450b 100644 --- a/src/jdk.jfr/share/classes/jdk/jfr/internal/JVM.java +++ b/src/jdk.jfr/share/classes/jdk/jfr/internal/JVM.java @@ -427,7 +427,17 @@ public final class JVM { /** * Flushes the EventWriter for this thread. */ - public static native boolean flush(EventWriter writer, int uncommittedSize, int requestedSize); + public static native void flush(EventWriter writer, int uncommittedSize, int requestedSize); + + /** + * Commits an event to the underlying buffer by setting the nextPosition. + * + * @param nextPosition + * + * @return the next startPosition + */ + @IntrinsicCandidate + public static native long commit(long nextPosition); /** * Flushes all thread buffers to disk and the constant pool data needed to read diff --git a/src/jdk.jfr/share/classes/jdk/jfr/internal/StringPool.java b/src/jdk.jfr/share/classes/jdk/jfr/internal/StringPool.java index 0ce7832dd99..6513895069e 100644 --- a/src/jdk.jfr/share/classes/jdk/jfr/internal/StringPool.java +++ b/src/jdk.jfr/share/classes/jdk/jfr/internal/StringPool.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -36,6 +36,8 @@ public final class StringPool { private static final int MAX_SIZE = 32 * 1024; /* max size bytes */ private static final long MAX_SIZE_UTF16 = 16 * 1024 * 1024; + /* mask for constructing generation relative string id. */ + private static final long SID_MASK = -65536; /* string id index */ private static final AtomicLong sidIdx = new AtomicLong(1); /* looking at a biased data set 4 is a good value */ @@ -48,18 +50,69 @@ public final class StringPool { private static int preCacheOld = 0; /* max size bytes */ private static long currentSizeUTF16; + /* string pool generation (0-65535) set by the JVM on epoch shift. Not private to avoid being optimized away. */ + static short generation = 0; - public static void reset() { - cache.clear(); - synchronized (StringPool.class) { - currentSizeUTF16 = 0; - } + /* internalSid is a composite id [48-bit externalSid][16-bit generation]. */ + private static boolean isCurrentGeneration(long internalSid) { + return generation == (short)internalSid; } + private static long updateInternalSid(long internalSid) { + return (internalSid & SID_MASK) | generation; + } + + private static long nextInternalSid() { + return sidIdx.getAndIncrement() << 16 | generation; + } + + /* externalSid is the most significant 48-bits of the internalSid. */ + private static long externalSid(long internalSid) { + return internalSid >> 16; + } + + /* synchronized because of writing the string to the JVM. */ + private static synchronized long storeString(String s) { + Long lsid = cache.get(s); + long internalSid; + if (lsid != null) { + internalSid = lsid.longValue(); + if (isCurrentGeneration(internalSid)) { + // Someone already updated the cache. + return externalSid(internalSid); + } + internalSid = updateInternalSid(internalSid); + } else { + // Not yet added or the cache was cleared. + internalSid = nextInternalSid(); + currentSizeUTF16 += s.length(); + } + long extSid = externalSid(internalSid); + // Write the string to the JVM before publishing to the cache. + JVM.addStringConstant(extSid, s); + cache.put(s, internalSid); + return extSid; + } + + /* a string fetched from the string pool must be of the current generation */ + private static long ensureCurrentGeneration(String s, Long lsid) { + long internalSid = lsid.longValue(); + return isCurrentGeneration(internalSid) ? externalSid(internalSid) : storeString(s); + } + + /* + * The string pool uses a generational id scheme to sync the JVM and Java sides. + * The string pool relies on the EventWriter and its implementation, especially + * its ability to restart event write attempts on interleaving epoch shifts. + * Even though a string id is generationally valid during StringPool lookup, + * the JVM can evolve the generation before the event is committed, + * effectively invalidating the fetched string id. The event restart mechanism + * of the EventWriter ensures that committed strings are in the correct generation. + */ public static long addString(String s) { Long lsid = cache.get(s); if (lsid != null) { - return lsid.longValue(); + return ensureCurrentGeneration(s, lsid); } if (!preCache(s)) { /* we should not pool this string */ @@ -72,17 +125,6 @@ public final class StringPool { return storeString(s); } - private static long storeString(String s) { - long sid = sidIdx.getAndIncrement(); - /* we can race but it is ok */ - cache.put(s, sid); - synchronized (StringPool.class) { - JVM.addStringConstant(sid, s); - currentSizeUTF16 += s.length(); - } - return sid; - } - private static boolean preCache(String s) { if (preCache[0].equals(s)) { return true; @@ -100,4 +142,9 @@ public final class StringPool { preCache[preCacheOld] = s; return false; } + + private static synchronized void reset() { + cache.clear(); + currentSizeUTF16 = 0; + } } diff --git a/src/jdk.jfr/share/classes/jdk/jfr/internal/event/EventWriter.java b/src/jdk.jfr/share/classes/jdk/jfr/internal/event/EventWriter.java index b430de41668..43fc6130495 100644 --- a/src/jdk.jfr/share/classes/jdk/jfr/internal/event/EventWriter.java +++ b/src/jdk.jfr/share/classes/jdk/jfr/internal/event/EventWriter.java @@ -62,15 +62,12 @@ public final class EventWriter { // The JVM needs access to these values. Don't remove private final long threadID; private long startPosition; - private long startPositionAddress; private long currentPosition; private long maxPosition; private boolean valid; - boolean notified; // Not private to avoid being optimized away boolean excluded; private PlatformEventType eventType; - private boolean flushOnEnd; private boolean largeSize = false; // User code must not be able to instantiate @@ -212,10 +209,6 @@ public final class EventWriter { public void reset() { currentPosition = startPosition; - if (flushOnEnd) { - flushOnEnd = flush(); - } - valid = true; } private boolean isValidForSize(int requestedSize) { @@ -223,7 +216,7 @@ public final class EventWriter { return false; } if (currentPosition + requestedSize > maxPosition) { - flushOnEnd = flush(usedSize(), requestedSize); + flush(usedSize(), requestedSize); // retry if (!valid) { return false; @@ -232,31 +225,18 @@ public final class EventWriter { return true; } - private boolean isNotified() { - return notified; - } - - private void resetNotified() { - notified = false; - } - - private void resetStringPool() { - StringPool.reset(); - } - private int usedSize() { return (int) (currentPosition - startPosition); } - private boolean flush() { - return flush(usedSize(), 0); + private void flush() { + flush(usedSize(), 0); } - private boolean flush(int usedSize, int requestedSize) { - return JVM.flush(this, usedSize, requestedSize); + private void flush(int usedSize, int requestedSize) { + JVM.flush(this, usedSize, requestedSize); } - public boolean beginEvent(EventConfiguration configuration, long typeId) { // Malicious code could take the EventConfiguration object from one // event class field and assign it to another. This check makes sure @@ -277,6 +257,7 @@ public final class EventWriter { public boolean endEvent() { if (!valid) { reset(); + valid = true; return true; } final int eventSize = usedSize(); @@ -284,7 +265,6 @@ public final class EventWriter { reset(); return true; } - if (largeSize) { Bits.putInt(startPosition, makePaddedInt(eventSize)); } else { @@ -298,32 +278,29 @@ public final class EventWriter { return false; } } - - if (isNotified()) { - resetNotified(); - resetStringPool(); - reset(); - // returning false will trigger restart of the event write attempt - return false; + long nextPosition = JVM.commit(currentPosition); + if (nextPosition == currentPosition) { + // Successful commit. Update the writer start position. + startPosition = nextPosition; + return true; } - startPosition = currentPosition; - unsafe.storeStoreFence(); - unsafe.putAddress(startPositionAddress, currentPosition); - // the event is now committed - if (flushOnEnd) { - flushOnEnd = flush(); + // If nextPosition == 0, the event was committed, the underlying buffer lease + // returned and new writer positions updated. Nothing to do. + if (nextPosition == 0) { + return true; } - return true; + // The commit was aborted because of an interleaving epoch shift. + // The nextPosition returned is the current start position. + // Reset the writer and return false to restart the write attempt. + currentPosition = nextPosition; + return false; } - private EventWriter(long startPos, long maxPos, long startPosAddress, long threadID, boolean valid, boolean excluded) { + private EventWriter(long startPos, long maxPos, long threadID, boolean valid, boolean excluded) { startPosition = currentPosition = startPos; maxPosition = maxPos; - startPositionAddress = startPosAddress; this.threadID = threadID; - flushOnEnd = false; this.valid = valid; - notified = false; this.excluded = excluded; } diff --git a/test/jdk/jdk/jfr/jvm/TestChunkIntegrity.java b/test/jdk/jdk/jfr/jvm/TestChunkIntegrity.java index 0a192431811..0ca887609f4 100644 --- a/test/jdk/jdk/jfr/jvm/TestChunkIntegrity.java +++ b/test/jdk/jdk/jfr/jvm/TestChunkIntegrity.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -56,7 +56,6 @@ import jdk.test.lib.jfr.TestClassLoader; /** * @test - * @ignore * @key jfr * @requires vm.hasJFR * @library /test/lib /test/jdk