diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index af4cec03221..6723bb89021 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -255,8 +255,10 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { return false; } - if (VerifyAfterGC) { - vmop_entry_verify_final_roots(); + // In normal cycle, final-update-refs would verify at the end of the cycle. + // In abbreviated cycle, we need to verify separately. + if (ShenandoahVerify) { + vmop_entry_final_verify(); } } @@ -344,14 +346,14 @@ void ShenandoahConcurrentGC::vmop_entry_final_update_refs() { VMThread::execute(&op); } -void ShenandoahConcurrentGC::vmop_entry_verify_final_roots() { +void ShenandoahConcurrentGC::vmop_entry_final_verify() { ShenandoahHeap* const heap = ShenandoahHeap::heap(); TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters()); - ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_roots_gross); + ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_verify_gross); // This phase does not use workers, no need for setup heap->try_inject_alloc_failure(); - VM_ShenandoahFinalRoots op(this); + VM_ShenandoahFinalVerify op(this); VMThread::execute(&op); } @@ -400,12 +402,12 @@ void ShenandoahConcurrentGC::entry_final_update_refs() { op_final_update_refs(); } -void ShenandoahConcurrentGC::entry_verify_final_roots() { - const char* msg = verify_final_roots_event_message(); - ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_roots); +void ShenandoahConcurrentGC::entry_final_verify() { + const char* msg = verify_final_event_message(); + ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_verify); EventMark em("%s", msg); - op_verify_final_roots(); + op_verify_final(); } void ShenandoahConcurrentGC::entry_reset() { @@ -1263,10 +1265,10 @@ bool ShenandoahConcurrentGC::entry_final_roots() { return true; } -void ShenandoahConcurrentGC::op_verify_final_roots() { - if (VerifyAfterGC) { - Universe::verify(); - } +void ShenandoahConcurrentGC::op_verify_final() { + assert(ShenandoahVerify, "Should have been checked before"); + ShenandoahHeap* const heap = ShenandoahHeap::heap(); + heap->verifier()->verify_after_gc(_generation); } void ShenandoahConcurrentGC::op_cleanup_complete() { @@ -1351,11 +1353,11 @@ const char* ShenandoahConcurrentGC::conc_reset_after_collect_event_message() con } } -const char* ShenandoahConcurrentGC::verify_final_roots_event_message() const { +const char* ShenandoahConcurrentGC::verify_final_event_message() const { if (ShenandoahHeap::heap()->unload_classes()) { - SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Verify Final Roots", " (unload classes)"); + SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Verify Final", " (unload classes)"); } else { - SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Verify Final Roots", ""); + SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Verify Final", ""); } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp index ba228901d72..fde585b4aa9 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp @@ -43,7 +43,7 @@ class ShenandoahConcurrentGC : public ShenandoahGC { friend class VM_ShenandoahFinalMarkStartEvac; friend class VM_ShenandoahInitUpdateRefs; friend class VM_ShenandoahFinalUpdateRefs; - friend class VM_ShenandoahFinalRoots; + friend class VM_ShenandoahFinalVerify; protected: ShenandoahConcurrentMark _mark; @@ -69,7 +69,7 @@ protected: void vmop_entry_final_mark(); void vmop_entry_init_update_refs(); void vmop_entry_final_update_refs(); - void vmop_entry_verify_final_roots(); + void vmop_entry_final_verify(); // Entry methods to normally STW GC operations. These set up logging, monitoring // and workers for next VM operation @@ -77,7 +77,7 @@ protected: void entry_final_mark(); void entry_init_update_refs(); void entry_final_update_refs(); - void entry_verify_final_roots(); + void entry_final_verify(); // Entry methods to normally concurrent GC operations. These set up logging, monitoring // for concurrent operation. @@ -122,7 +122,7 @@ protected: void op_update_thread_roots(); void op_final_update_refs(); - void op_verify_final_roots(); + void op_verify_final(); void op_cleanup_complete(); void op_reset_after_collect(); @@ -143,7 +143,7 @@ private: // passing around the logging/tracing systems const char* init_mark_event_message() const; const char* final_mark_event_message() const; - const char* verify_final_roots_event_message() const; + const char* verify_final_event_message() const; const char* conc_final_roots_event_message() const; const char* conc_mark_event_message() const; const char* conc_reset_event_message() const; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp index a454de68f00..6a316e2265a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp @@ -112,8 +112,8 @@ class outputStream; f(conc_update_card_table, "Concurrent Update Cards") \ f(conc_final_roots, "Concurrent Final Roots") \ f(promote_in_place, " Promote Regions") \ - f(final_roots_gross, "Pause Verify Final Roots (G)") \ - f(final_roots, "Pause Verify Final Roots (N)") \ + f(final_verify_gross, "Pause Final Verify (G)") \ + f(final_verify, "Pause Final Verify (N)") \ \ f(init_update_refs_gross, "Pause Init Update Refs (G)") \ f(init_update_refs, "Pause Init Update Refs (N)") \ diff --git a/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp b/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp index 6ef4cd7c702..0169941c3d9 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp @@ -187,7 +187,7 @@ public: type == VM_Operation::VMOp_ShenandoahFinalMarkStartEvac || type == VM_Operation::VMOp_ShenandoahInitUpdateRefs || type == VM_Operation::VMOp_ShenandoahFinalUpdateRefs || - type == VM_Operation::VMOp_ShenandoahFinalRoots || + type == VM_Operation::VMOp_ShenandoahFinalVerify || type == VM_Operation::VMOp_ShenandoahFullGC || type == VM_Operation::VMOp_ShenandoahDegeneratedGC; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp index 6b45842f781..97dd7e5cda1 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp @@ -135,12 +135,12 @@ void VM_ShenandoahFinalUpdateRefs::doit() { _gc->entry_final_update_refs(); } -VM_ShenandoahFinalRoots::VM_ShenandoahFinalRoots(ShenandoahConcurrentGC* gc) +VM_ShenandoahFinalVerify::VM_ShenandoahFinalVerify(ShenandoahConcurrentGC* gc) : VM_ShenandoahOperation(gc->generation()), _gc(gc) { } -void VM_ShenandoahFinalRoots::doit() { - ShenandoahGCPauseMark mark(_gc_id, "Final Roots", SvcGCMarker::CONCURRENT); +void VM_ShenandoahFinalVerify::doit() { + ShenandoahGCPauseMark mark(_gc_id, "Final Verify", SvcGCMarker::CONCURRENT); set_active_generation(); - _gc->entry_verify_final_roots(); + _gc->entry_final_verify(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp index d565a3df22c..f8b99c71b14 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp @@ -38,7 +38,7 @@ class ShenandoahFullGC; // - VM_ShenandoahFinalMarkStartEvac: finish up concurrent marking, and start evacuation // - VM_ShenandoahInitUpdateRefs: initiate update references // - VM_ShenandoahFinalUpdateRefs: finish up update references -// - VM_ShenandoahFinalRoots: finish up roots on a non-evacuating cycle +// - VM_ShenandoahFinalVerify: final verification at the end of the cycle // - VM_ShenandoahReferenceOperation: // - VM_ShenandoahFullGC: do full GC // - VM_ShenandoahDegeneratedGC: do STW degenerated GC @@ -127,12 +127,12 @@ public: void doit() override; }; -class VM_ShenandoahFinalRoots: public VM_ShenandoahOperation { +class VM_ShenandoahFinalVerify: public VM_ShenandoahOperation { ShenandoahConcurrentGC* const _gc; public: - explicit VM_ShenandoahFinalRoots(ShenandoahConcurrentGC* gc); - VM_Operation::VMOp_Type type() const override { return VMOp_ShenandoahFinalRoots; } - const char* name() const override { return "Shenandoah Final Roots"; } + explicit VM_ShenandoahFinalVerify(ShenandoahConcurrentGC* gc); + VM_Operation::VMOp_Type type() const override { return VMOp_ShenandoahFinalVerify; } + const char* name() const override { return "Shenandoah Final Verify"; } void doit() override; }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp index afef11640c9..8299cbe62c6 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp @@ -1180,7 +1180,7 @@ void ShenandoahVerifier::verify_before_update_refs(ShenandoahGeneration* generat ); } -// We have not yet cleanup (reclaimed) the collection set +// We have not yet cleaned up (reclaimed) the collection set void ShenandoahVerifier::verify_after_update_refs(ShenandoahGeneration* generation) { verify_at_safepoint( generation, @@ -1197,6 +1197,23 @@ void ShenandoahVerifier::verify_after_update_refs(ShenandoahGeneration* generati ); } +// We have not yet cleaned up (reclaimed) the collection set +void ShenandoahVerifier::verify_after_gc(ShenandoahGeneration* generation) { + verify_at_safepoint( + generation, + "After GC", + _verify_remembered_disable, // do not verify remembered set + _verify_forwarded_none, // no forwarded references + _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well + _verify_cset_none, // no cset references, all updated + _verify_liveness_disable, // no reliable liveness data anymore + _verify_regions_nocset, // no cset regions, trash regions have appeared + // expect generation and heap sizes to match exactly, including trash + _verify_size_exact_including_trash, + _verify_gcstate_stable // GC state was turned off + ); +} + void ShenandoahVerifier::verify_after_degenerated(ShenandoahGeneration* generation) { verify_at_safepoint( generation, diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp index 7e683cf7af8..0479d5f67ce 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp @@ -220,6 +220,7 @@ public: void verify_before_evacuation(ShenandoahGeneration* generation); void verify_before_update_refs(ShenandoahGeneration* generation); void verify_after_update_refs(ShenandoahGeneration* generation); + void verify_after_gc(ShenandoahGeneration* generation); void verify_before_fullgc(ShenandoahGeneration* generation); void verify_after_fullgc(ShenandoahGeneration* generation); void verify_after_degenerated(ShenandoahGeneration* generation); diff --git a/src/hotspot/share/runtime/vmOperation.hpp b/src/hotspot/share/runtime/vmOperation.hpp index 5140d0401fb..6078600c16e 100644 --- a/src/hotspot/share/runtime/vmOperation.hpp +++ b/src/hotspot/share/runtime/vmOperation.hpp @@ -90,7 +90,7 @@ template(ShenandoahFinalMarkStartEvac) \ template(ShenandoahInitUpdateRefs) \ template(ShenandoahFinalUpdateRefs) \ - template(ShenandoahFinalRoots) \ + template(ShenandoahFinalVerify) \ template(ShenandoahDegeneratedGC) \ template(Exit) \ template(LinuxDllLoad) \