From 5698f7ad29c939b7e52882ace575dd7113bf41de Mon Sep 17 00:00:00 2001 From: Aleksey Shipilev Date: Mon, 1 Apr 2024 17:27:10 +0000 Subject: [PATCH] 8329134: Reconsider TLAB zapping Reviewed-by: stefank, rkennke --- src/hotspot/share/gc/shared/memAllocator.cpp | 7 ++----- .../gc/shared/threadLocalAllocBuffer.inline.hpp | 12 +++--------- src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp | 7 ++----- .../share/interpreter/zero/bytecodeInterpreter.cpp | 10 ++++------ src/hotspot/share/runtime/globals.hpp | 3 +++ 5 files changed, 14 insertions(+), 25 deletions(-) diff --git a/src/hotspot/share/gc/shared/memAllocator.cpp b/src/hotspot/share/gc/shared/memAllocator.cpp index dc94b83c139..053f49dd6dd 100644 --- a/src/hotspot/share/gc/shared/memAllocator.cpp +++ b/src/hotspot/share/gc/shared/memAllocator.cpp @@ -316,18 +316,15 @@ HeapWord* MemAllocator::mem_allocate_inside_tlab_slow(Allocation& allocation) co PTR_FORMAT " min: " SIZE_FORMAT ", desired: " SIZE_FORMAT, p2i(mem), min_tlab_size, new_tlab_size); + // ...and clear or zap just allocated TLAB, if needed. if (ZeroTLAB) { - // ..and clear it. Copy::zero_to_words(mem, allocation._allocated_tlab_size); - } else { - // ...and zap just allocated object. -#ifdef ASSERT + } else if (ZapTLAB) { // Skip mangling the space corresponding to the object header to // ensure that the returned space is not considered parsable by // any concurrent GC thread. size_t hdr_size = oopDesc::header_size(); Copy::fill_to_words(mem + hdr_size, allocation._allocated_tlab_size - hdr_size, badHeapWordVal); -#endif // ASSERT } tlab.fill(mem, mem + _word_size, allocation._allocated_tlab_size); diff --git a/src/hotspot/share/gc/shared/threadLocalAllocBuffer.inline.hpp b/src/hotspot/share/gc/shared/threadLocalAllocBuffer.inline.hpp index 3988134a8e9..6910d9c1860 100644 --- a/src/hotspot/share/gc/shared/threadLocalAllocBuffer.inline.hpp +++ b/src/hotspot/share/gc/shared/threadLocalAllocBuffer.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -39,14 +39,8 @@ inline HeapWord* ThreadLocalAllocBuffer::allocate(size_t size) { invariants(); HeapWord* obj = top(); if (pointer_delta(end(), obj) >= size) { - // successful thread-local allocation -#ifdef ASSERT - // Skip mangling the space corresponding to the object header to - // ensure that the returned space is not considered parsable by - // any concurrent GC thread. - size_t hdr_size = oopDesc::header_size(); - Copy::fill_to_words(obj + hdr_size, size - hdr_size, badHeapWordVal); -#endif // ASSERT + // Successful thread-local allocation. + // This addition is safe because we know that top is // at least size below end, so the add can't wrap. set_top(obj + size); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 24a94e9d867..8e6081fd0bf 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -878,18 +878,15 @@ HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) assert (size <= actual_size, "allocation should fit"); + // ...and clear or zap just allocated TLAB, if needed. if (ZeroTLAB) { - // ..and clear it. Copy::zero_to_words(gclab_buf, actual_size); - } else { - // ...and zap just allocated object. -#ifdef ASSERT + } else if (ZapTLAB) { // Skip mangling the space corresponding to the object header to // ensure that the returned space is not considered parsable by // any concurrent GC thread. size_t hdr_size = oopDesc::header_size(); Copy::fill_to_words(gclab_buf + hdr_size, actual_size - hdr_size, badHeapWordVal); -#endif // ASSERT } gclab->set_buf(gclab_buf, actual_size); return gclab->allocate(size); diff --git a/src/hotspot/share/interpreter/zero/bytecodeInterpreter.cpp b/src/hotspot/share/interpreter/zero/bytecodeInterpreter.cpp index feb2a2ede03..616ce240869 100644 --- a/src/hotspot/share/interpreter/zero/bytecodeInterpreter.cpp +++ b/src/hotspot/share/interpreter/zero/bytecodeInterpreter.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2002, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1991,11 +1991,9 @@ run: size_t obj_size = ik->size_helper(); HeapWord* result = THREAD->tlab().allocate(obj_size); if (result != nullptr) { - // Initialize object field block: - // - if TLAB is pre-zeroed, we can skip this path - // - in debug mode, ThreadLocalAllocBuffer::allocate mangles - // this area, and we still need to initialize it - if (DEBUG_ONLY(true ||) !ZeroTLAB) { + // Initialize object field block. + if (!ZeroTLAB) { + // The TLAB was not pre-zeroed, we need to clear the memory here. size_t hdr_size = oopDesc::header_size(); Copy::fill_to_words(result + hdr_size, obj_size - hdr_size, 0); } diff --git a/src/hotspot/share/runtime/globals.hpp b/src/hotspot/share/runtime/globals.hpp index f6a7a345dd1..c48d6d8c2e5 100644 --- a/src/hotspot/share/runtime/globals.hpp +++ b/src/hotspot/share/runtime/globals.hpp @@ -483,6 +483,9 @@ const int ObjectAlignmentInBytes = 8; develop(bool, ZapFillerObjects, trueInDebug, \ "Zap filler objects") \ \ + develop(bool, ZapTLAB, trueInDebug, \ + "Zap allocated TLABs") \ + \ product(bool, ExecutingUnitTests, false, \ "Whether the JVM is running unit tests or not") \ \