Go to Debian
Package Search, search for the following packages for your
diff --git a/doc/building.md b/doc/building.md
index b626027f101..d653d36eb55 100644
--- a/doc/building.md
+++ b/doc/building.md
@@ -1178,10 +1178,8 @@ Note that alsa is needed even if you only want to build a headless JDK.
#### X11
-You will need X11 libraries suitable for your *target* system. In most cases,
-using Debian's pre-built libraries work fine.
-
-Note that X11 is needed even if you only want to build a headless JDK.
+When not building a headless JDK, you will need X11 libraries suitable for your
+*target* system. In most cases, using Debian's pre-built libraries work fine.
* Go to [Debian Package Search](https://www.debian.org/distrib/packages),
search for the following packages for your *target* system, and download them
diff --git a/make/autoconf/libraries.m4 b/make/autoconf/libraries.m4
index 8dc3d55ed0c..5daacdc1ced 100644
--- a/make/autoconf/libraries.m4
+++ b/make/autoconf/libraries.m4
@@ -42,12 +42,12 @@ m4_include([lib-tests.m4])
AC_DEFUN_ONCE([LIB_DETERMINE_DEPENDENCIES],
[
# Check if X11 is needed
- if test "x$OPENJDK_TARGET_OS" = xwindows || test "x$OPENJDK_TARGET_OS" = xmacosx; then
- # No X11 support on windows or macosx
+ if test "x$OPENJDK_TARGET_OS" = xwindows ||
+ test "x$OPENJDK_TARGET_OS" = xmacosx ||
+ test "x$ENABLE_HEADLESS_ONLY" = xtrue; then
NEEDS_LIB_X11=false
else
- # All other instances need X11, even if building headless only, libawt still
- # needs X11 headers.
+ # All other instances need X11 for libawt.
NEEDS_LIB_X11=true
fi
diff --git a/make/modules/java.desktop/lib/AwtLibraries.gmk b/make/modules/java.desktop/lib/AwtLibraries.gmk
index 463e09e12dc..8b6b50b9e62 100644
--- a/make/modules/java.desktop/lib/AwtLibraries.gmk
+++ b/make/modules/java.desktop/lib/AwtLibraries.gmk
@@ -88,6 +88,10 @@ LIBAWT_EXTRA_HEADER_DIRS := \
LIBAWT_CFLAGS := -D__MEDIALIB_OLD_NAMES -D__USE_J2D_NAMES -DMLIB_NO_LIBSUNMATH
+ifeq ($(ENABLE_HEADLESS_ONLY), true)
+ LIBAWT_CFLAGS += -DHEADLESS
+endif
+
ifeq ($(call isTargetOs, windows), true)
LIBAWT_CFLAGS += -EHsc -DUNICODE -D_UNICODE -DMLIB_OS64BIT
LIBAWT_RCFLAGS ?= -I$(TOPDIR)/src/java.base/windows/native/launcher/icons
@@ -167,11 +171,18 @@ ifeq ($(call isTargetOs, windows macosx), false)
$(TOPDIR)/src/$(MODULE)/$(OPENJDK_TARGET_OS_TYPE)/native/common/awt \
#
+ LIBAWT_HEADLESS_EXCLUDE_FILES := \
+ GLXGraphicsConfig.c \
+ GLXSurfaceData.c \
+ X11PMBlitLoops.c \
+ X11Renderer.c \
+ X11SurfaceData.c \
+ #
+
LIBAWT_HEADLESS_EXTRA_HEADER_DIRS := \
$(LIBAWT_DEFAULT_HEADER_DIRS) \
common/awt/debug \
common/font \
- common/java2d/opengl \
java.base:libjvm \
#
@@ -191,7 +202,8 @@ ifeq ($(call isTargetOs, windows macosx), false)
$(eval $(call SetupJdkLibrary, BUILD_LIBAWT_HEADLESS, \
NAME := awt_headless, \
EXTRA_SRC := $(LIBAWT_HEADLESS_EXTRA_SRC), \
- EXCLUDES := medialib, \
+ EXCLUDES := medialib opengl, \
+ EXCLUDE_FILES := $(LIBAWT_HEADLESS_EXCLUDE_FILES), \
ONLY_EXPORTED := $(LIBAWT_HEADLESS_ONLY_EXPORTED), \
OPTIMIZATION := LOW, \
CFLAGS := -DHEADLESS=true $(CUPS_CFLAGS) $(FONTCONFIG_CFLAGS) \
diff --git a/make/modules/jdk.incubator.vector/Java.gmk b/make/modules/jdk.incubator.vector/Java.gmk
index e51ffe57cea..eafcbd1e1e2 100644
--- a/make/modules/jdk.incubator.vector/Java.gmk
+++ b/make/modules/jdk.incubator.vector/Java.gmk
@@ -25,7 +25,7 @@
################################################################################
-DISABLED_WARNINGS_java += dangling-doc-comments
+DISABLED_WARNINGS_java += dangling-doc-comments suppression
DOCLINT += -Xdoclint:all/protected
diff --git a/make/modules/jdk.jpackage/Lib.gmk b/make/modules/jdk.jpackage/Lib.gmk
index 704436bbde6..86b11bdafee 100644
--- a/make/modules/jdk.jpackage/Lib.gmk
+++ b/make/modules/jdk.jpackage/Lib.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -54,7 +54,7 @@ $(eval $(call SetupJdkExecutable, BUILD_JPACKAGEAPPLAUNCHER, \
SRC := applauncher, \
EXTRA_SRC := common, \
INCLUDE_FILES := $(JPACKAGEAPPLAUNCHER_INCLUDE_FILES), \
- OPTIMIZATION := LOW, \
+ OPTIMIZATION := SIZE, \
DISABLED_WARNINGS_clang_JvmLauncherLib.c := format-nonliteral, \
DISABLED_WARNINGS_clang_LinuxPackage.c := format-nonliteral, \
DISABLED_WARNINGS_clang_Log.cpp := unused-const-variable, \
@@ -91,7 +91,7 @@ ifeq ($(call isTargetOs, linux), true)
common, \
EXCLUDE_FILES := LinuxLauncher.c LinuxPackage.c, \
LINK_TYPE := C++, \
- OPTIMIZATION := LOW, \
+ OPTIMIZATION := SIZE, \
DISABLED_WARNINGS_gcc_Log.cpp := unused-const-variable, \
DISABLED_WARNINGS_clang_JvmLauncherLib.c := format-nonliteral, \
DISABLED_WARNINGS_clang_tstrings.cpp := format-nonliteral, \
diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad
index b9252cc56ff..a9ca91d9309 100644
--- a/src/hotspot/cpu/aarch64/aarch64.ad
+++ b/src/hotspot/cpu/aarch64/aarch64.ad
@@ -1229,7 +1229,7 @@ public:
// predicate controlling addressing modes
bool size_fits_all_mem_uses(AddPNode* addp, int shift);
- // Convert BootTest condition to Assembler condition.
+ // Convert BoolTest condition to Assembler condition.
// Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
Assembler::Condition to_assembler_cond(BoolTest::mask cond);
%}
@@ -2579,7 +2579,7 @@ bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
return true;
}
-// Convert BootTest condition to Assembler condition.
+// Convert BoolTest condition to Assembler condition.
// Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
Assembler::Condition result;
diff --git a/src/hotspot/cpu/aarch64/aarch64_vector.ad b/src/hotspot/cpu/aarch64/aarch64_vector.ad
index 78ef121bd29..19f03d97a72 100644
--- a/src/hotspot/cpu/aarch64/aarch64_vector.ad
+++ b/src/hotspot/cpu/aarch64/aarch64_vector.ad
@@ -1,5 +1,5 @@
//
-// Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
// Copyright (c) 2020, 2025, Arm Limited. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
@@ -201,6 +201,8 @@ source %{
case Op_XorReductionV:
case Op_MinReductionV:
case Op_MaxReductionV:
+ case Op_UMinReductionV:
+ case Op_UMaxReductionV:
// Reductions with less than 8 bytes vector length are
// not supported.
if (length_in_bytes < 8) {
@@ -383,6 +385,8 @@ source %{
return !VM_Version::use_neon_for_vector(length_in_bytes);
case Op_MinReductionV:
case Op_MaxReductionV:
+ case Op_UMinReductionV:
+ case Op_UMaxReductionV:
// For BYTE/SHORT/INT/FLOAT/DOUBLE types, we prefer using NEON
// instructions rather than SVE predicated instructions for
// better performance.
@@ -4218,6 +4222,224 @@ instruct reduce_minD_masked(vRegD dst, vRegD dsrc, vReg vsrc, pRegGov pg) %{
ins_pipe(pipe_slow);
%}
+// -------------------- Vector reduction unsigned min/max ----------------------
+
+// reduction uminI
+
+instruct reduce_uminI_neon(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc,
+ vReg tmp, rFlagsReg cr) %{
+ predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(2))) &&
+ (Matcher::vector_element_basic_type(n->in(2)) == T_BYTE ||
+ Matcher::vector_element_basic_type(n->in(2)) == T_SHORT ||
+ Matcher::vector_element_basic_type(n->in(2)) == T_INT));
+ match(Set dst (UMinReductionV isrc vsrc));
+ effect(TEMP_DEF dst, TEMP tmp, KILL cr);
+ format %{ "reduce_uminI_neon $dst, $isrc, $vsrc\t# KILL $tmp, cr" %}
+ ins_encode %{
+ BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
+ uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
+ __ neon_reduce_minmax_integral(this->ideal_Opcode(), $dst$$Register, bt,
+ $isrc$$Register, $vsrc$$FloatRegister,
+ length_in_bytes, $tmp$$FloatRegister);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct reduce_uminI_sve(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc,
+ vRegD tmp, rFlagsReg cr) %{
+ predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(2))) &&
+ (Matcher::vector_element_basic_type(n->in(2)) == T_BYTE ||
+ Matcher::vector_element_basic_type(n->in(2)) == T_SHORT ||
+ Matcher::vector_element_basic_type(n->in(2)) == T_INT));
+ match(Set dst (UMinReductionV isrc vsrc));
+ effect(TEMP_DEF dst, TEMP tmp, KILL cr);
+ format %{ "reduce_uminI_sve $dst, $isrc, $vsrc\t# KILL $tmp, cr" %}
+ ins_encode %{
+ assert(UseSVE > 0, "must be sve");
+ BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
+ uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
+ assert(length_in_bytes == MaxVectorSize, "invalid vector length");
+ __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
+ $isrc$$Register, $vsrc$$FloatRegister,
+ ptrue, $tmp$$FloatRegister);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+// reduction uminL
+
+instruct reduce_uminL_neon(iRegLNoSp dst, iRegL isrc, vReg vsrc, rFlagsReg cr) %{
+ predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
+ match(Set dst (UMinReductionV isrc vsrc));
+ effect(TEMP_DEF dst, KILL cr);
+ format %{ "reduce_uminL_neon $dst, $isrc, $vsrc\t# 2L. KILL cr" %}
+ ins_encode %{
+ __ neon_reduce_minmax_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
+ $isrc$$Register, $vsrc$$FloatRegister,
+ /* vector_length_in_bytes */ 16, fnoreg);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct reduce_uminL_sve(iRegLNoSp dst, iRegL isrc, vReg vsrc,
+ vRegD tmp, rFlagsReg cr) %{
+ predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
+ match(Set dst (UMinReductionV isrc vsrc));
+ effect(TEMP_DEF dst, TEMP tmp, KILL cr);
+ format %{ "reduce_uminL_sve $dst, $isrc, $vsrc\t# KILL $tmp, cr" %}
+ ins_encode %{
+ uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
+ assert(length_in_bytes == MaxVectorSize, "invalid vector length");
+ __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
+ $isrc$$Register, $vsrc$$FloatRegister,
+ ptrue, $tmp$$FloatRegister);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+// reduction umin - predicated
+
+instruct reduce_uminI_masked(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc, pRegGov pg,
+ vRegD tmp, rFlagsReg cr) %{
+ predicate(UseSVE > 0 &&
+ (Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_BYTE ||
+ Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_SHORT ||
+ Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_INT));
+ match(Set dst (UMinReductionV (Binary isrc vsrc) pg));
+ effect(TEMP_DEF dst, TEMP tmp, KILL cr);
+ format %{ "reduce_uminI_masked $dst, $isrc, $pg, $vsrc\t# KILL $tmp, cr" %}
+ ins_encode %{
+ BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
+ __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
+ $isrc$$Register, $vsrc$$FloatRegister,
+ $pg$$PRegister, $tmp$$FloatRegister);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct reduce_uminL_masked(iRegLNoSp dst, iRegL isrc, vReg vsrc, pRegGov pg,
+ vRegD tmp, rFlagsReg cr) %{
+ predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_LONG);
+ match(Set dst (UMinReductionV (Binary isrc vsrc) pg));
+ effect(TEMP_DEF dst, TEMP tmp, KILL cr);
+ format %{ "reduce_uminL_masked $dst, $isrc, $pg, $vsrc\t# KILL $tmp, cr" %}
+ ins_encode %{
+ BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
+ __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
+ $isrc$$Register, $vsrc$$FloatRegister,
+ $pg$$PRegister, $tmp$$FloatRegister);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+// reduction umaxI
+
+instruct reduce_umaxI_neon(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc,
+ vReg tmp, rFlagsReg cr) %{
+ predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(2))) &&
+ (Matcher::vector_element_basic_type(n->in(2)) == T_BYTE ||
+ Matcher::vector_element_basic_type(n->in(2)) == T_SHORT ||
+ Matcher::vector_element_basic_type(n->in(2)) == T_INT));
+ match(Set dst (UMaxReductionV isrc vsrc));
+ effect(TEMP_DEF dst, TEMP tmp, KILL cr);
+ format %{ "reduce_umaxI_neon $dst, $isrc, $vsrc\t# KILL $tmp, cr" %}
+ ins_encode %{
+ BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
+ uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
+ __ neon_reduce_minmax_integral(this->ideal_Opcode(), $dst$$Register, bt,
+ $isrc$$Register, $vsrc$$FloatRegister,
+ length_in_bytes, $tmp$$FloatRegister);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct reduce_umaxI_sve(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc,
+ vRegD tmp, rFlagsReg cr) %{
+ predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(2))) &&
+ (Matcher::vector_element_basic_type(n->in(2)) == T_BYTE ||
+ Matcher::vector_element_basic_type(n->in(2)) == T_SHORT ||
+ Matcher::vector_element_basic_type(n->in(2)) == T_INT));
+ match(Set dst (UMaxReductionV isrc vsrc));
+ effect(TEMP_DEF dst, TEMP tmp, KILL cr);
+ format %{ "reduce_umaxI_sve $dst, $isrc, $vsrc\t# KILL $tmp, cr" %}
+ ins_encode %{
+ assert(UseSVE > 0, "must be sve");
+ BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
+ uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
+ assert(length_in_bytes == MaxVectorSize, "invalid vector length");
+ __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
+ $isrc$$Register, $vsrc$$FloatRegister,
+ ptrue, $tmp$$FloatRegister);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+// reduction umaxL
+
+instruct reduce_umaxL_neon(iRegLNoSp dst, iRegL isrc, vReg vsrc, rFlagsReg cr) %{
+ predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
+ match(Set dst (UMaxReductionV isrc vsrc));
+ effect(TEMP_DEF dst, KILL cr);
+ format %{ "reduce_umaxL_neon $dst, $isrc, $vsrc\t# 2L. KILL cr" %}
+ ins_encode %{
+ __ neon_reduce_minmax_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
+ $isrc$$Register, $vsrc$$FloatRegister,
+ /* vector_length_in_bytes */ 16, fnoreg);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct reduce_umaxL_sve(iRegLNoSp dst, iRegL isrc, vReg vsrc,
+ vRegD tmp, rFlagsReg cr) %{
+ predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
+ match(Set dst (UMaxReductionV isrc vsrc));
+ effect(TEMP_DEF dst, TEMP tmp, KILL cr);
+ format %{ "reduce_umaxL_sve $dst, $isrc, $vsrc\t# KILL $tmp, cr" %}
+ ins_encode %{
+ uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
+ assert(length_in_bytes == MaxVectorSize, "invalid vector length");
+ __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
+ $isrc$$Register, $vsrc$$FloatRegister,
+ ptrue, $tmp$$FloatRegister);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+// reduction umax - predicated
+
+instruct reduce_umaxI_masked(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc, pRegGov pg,
+ vRegD tmp, rFlagsReg cr) %{
+ predicate(UseSVE > 0 &&
+ (Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_BYTE ||
+ Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_SHORT ||
+ Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_INT));
+ match(Set dst (UMaxReductionV (Binary isrc vsrc) pg));
+ effect(TEMP_DEF dst, TEMP tmp, KILL cr);
+ format %{ "reduce_umaxI_masked $dst, $isrc, $pg, $vsrc\t# KILL $tmp, cr" %}
+ ins_encode %{
+ BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
+ __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
+ $isrc$$Register, $vsrc$$FloatRegister,
+ $pg$$PRegister, $tmp$$FloatRegister);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct reduce_umaxL_masked(iRegLNoSp dst, iRegL isrc, vReg vsrc, pRegGov pg,
+ vRegD tmp, rFlagsReg cr) %{
+ predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_LONG);
+ match(Set dst (UMaxReductionV (Binary isrc vsrc) pg));
+ effect(TEMP_DEF dst, TEMP tmp, KILL cr);
+ format %{ "reduce_umaxL_masked $dst, $isrc, $pg, $vsrc\t# KILL $tmp, cr" %}
+ ins_encode %{
+ BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
+ __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
+ $isrc$$Register, $vsrc$$FloatRegister,
+ $pg$$PRegister, $tmp$$FloatRegister);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
// ------------------------------ Vector reinterpret ---------------------------
instruct reinterpret_same_size(vReg dst_src) %{
diff --git a/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 b/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4
index 66dc22c3758..48bffb3cf35 100644
--- a/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4
+++ b/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4
@@ -1,5 +1,5 @@
//
-// Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
// Copyright (c) 2020, 2025, Arm Limited. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
@@ -191,6 +191,8 @@ source %{
case Op_XorReductionV:
case Op_MinReductionV:
case Op_MaxReductionV:
+ case Op_UMinReductionV:
+ case Op_UMaxReductionV:
// Reductions with less than 8 bytes vector length are
// not supported.
if (length_in_bytes < 8) {
@@ -373,6 +375,8 @@ source %{
return !VM_Version::use_neon_for_vector(length_in_bytes);
case Op_MinReductionV:
case Op_MaxReductionV:
+ case Op_UMinReductionV:
+ case Op_UMaxReductionV:
// For BYTE/SHORT/INT/FLOAT/DOUBLE types, we prefer using NEON
// instructions rather than SVE predicated instructions for
// better performance.
@@ -2505,6 +2509,32 @@ REDUCE_MAXMIN_INT_PREDICATE(min, L, iRegL, MinReductionV)
REDUCE_MAXMIN_FP_PREDICATE(min, F, fsrc, MinReductionV, sve_fminv, fmins)
REDUCE_MAXMIN_FP_PREDICATE(min, D, dsrc, MinReductionV, sve_fminv, fmind)
+// -------------------- Vector reduction unsigned min/max ----------------------
+
+// reduction uminI
+REDUCE_MAXMIN_I_NEON(umin, UMinReductionV)
+REDUCE_MAXMIN_I_SVE(umin, UMinReductionV)
+
+// reduction uminL
+REDUCE_MAXMIN_L_NEON(umin, UMinReductionV)
+REDUCE_MAXMIN_L_SVE(umin, UMinReductionV)
+
+// reduction umin - predicated
+REDUCE_MAXMIN_INT_PREDICATE(umin, I, iRegIorL2I, UMinReductionV)
+REDUCE_MAXMIN_INT_PREDICATE(umin, L, iRegL, UMinReductionV)
+
+// reduction umaxI
+REDUCE_MAXMIN_I_NEON(umax, UMaxReductionV)
+REDUCE_MAXMIN_I_SVE(umax, UMaxReductionV)
+
+// reduction umaxL
+REDUCE_MAXMIN_L_NEON(umax, UMaxReductionV)
+REDUCE_MAXMIN_L_SVE(umax, UMaxReductionV)
+
+// reduction umax - predicated
+REDUCE_MAXMIN_INT_PREDICATE(umax, I, iRegIorL2I, UMaxReductionV)
+REDUCE_MAXMIN_INT_PREDICATE(umax, L, iRegL, UMaxReductionV)
+
// ------------------------------ Vector reinterpret ---------------------------
instruct reinterpret_same_size(vReg dst_src) %{
diff --git a/src/hotspot/cpu/aarch64/assembler_aarch64.hpp b/src/hotspot/cpu/aarch64/assembler_aarch64.hpp
index 18807c667e3..19b3bb1a65b 100644
--- a/src/hotspot/cpu/aarch64/assembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/assembler_aarch64.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2024, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -2658,6 +2658,8 @@ template
INSN(uminv, 1, 0b011011, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
INSN(smaxp, 0, 0b101001, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
INSN(sminp, 0, 0b101011, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
+ INSN(umaxp, 1, 0b101001, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
+ INSN(uminp, 1, 0b101011, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
INSN(sqdmulh,0, 0b101101, false); // accepted arrangements: T4H, T8H, T2S, T4S
INSN(shsubv, 0, 0b001001, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
@@ -3490,7 +3492,9 @@ public:
INSN(sve_sub, 0b00000100, 0b000001000); // vector sub
INSN(sve_uaddv, 0b00000100, 0b000001001); // unsigned add reduction to scalar
INSN(sve_umax, 0b00000100, 0b001001000); // unsigned maximum vectors
+ INSN(sve_umaxv, 0b00000100, 0b001001001); // unsigned maximum reduction to scalar
INSN(sve_umin, 0b00000100, 0b001011000); // unsigned minimum vectors
+ INSN(sve_uminv, 0b00000100, 0b001011001); // unsigned minimum reduction to scalar
#undef INSN
// SVE floating-point arithmetic - predicate
@@ -4325,6 +4329,7 @@ public:
#undef INSN
Assembler(CodeBuffer* code) : AbstractAssembler(code) {
+ MACOS_AARCH64_ONLY(os::thread_wx_enable_write());
}
// Stack overflow checking
diff --git a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
index 75897a16fe4..958855c7685 100644
--- a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1960,50 +1960,76 @@ void C2_MacroAssembler::neon_reduce_logical(int opc, Register dst, BasicType bt,
BLOCK_COMMENT("} neon_reduce_logical");
}
-// Vector reduction min/max for integral type with ASIMD instructions.
+// Helper function to decode min/max reduction operation properties
+void C2_MacroAssembler::decode_minmax_reduction_opc(int opc, bool* is_min,
+ bool* is_unsigned,
+ Condition* cond) {
+ switch(opc) {
+ case Op_MinReductionV:
+ *is_min = true; *is_unsigned = false; *cond = LT; break;
+ case Op_MaxReductionV:
+ *is_min = false; *is_unsigned = false; *cond = GT; break;
+ case Op_UMinReductionV:
+ *is_min = true; *is_unsigned = true; *cond = LO; break;
+ case Op_UMaxReductionV:
+ *is_min = false; *is_unsigned = true; *cond = HI; break;
+ default:
+ ShouldNotReachHere();
+ }
+}
+
+// Vector reduction min/max/umin/umax for integral type with ASIMD instructions.
// Note: vtmp is not used and expected to be fnoreg for T_LONG case.
// Clobbers: rscratch1, rflags
void C2_MacroAssembler::neon_reduce_minmax_integral(int opc, Register dst, BasicType bt,
Register isrc, FloatRegister vsrc,
unsigned vector_length_in_bytes,
FloatRegister vtmp) {
- assert(opc == Op_MinReductionV || opc == Op_MaxReductionV, "unsupported");
+ assert(opc == Op_MinReductionV || opc == Op_MaxReductionV ||
+ opc == Op_UMinReductionV || opc == Op_UMaxReductionV, "unsupported");
assert(vector_length_in_bytes == 8 || vector_length_in_bytes == 16, "unsupported");
assert(bt == T_BYTE || bt == T_SHORT || bt == T_INT || bt == T_LONG, "unsupported");
assert_different_registers(dst, isrc);
bool isQ = vector_length_in_bytes == 16;
- bool is_min = opc == Op_MinReductionV;
-
+ bool is_min;
+ bool is_unsigned;
+ Condition cond;
+ decode_minmax_reduction_opc(opc, &is_min, &is_unsigned, &cond);
BLOCK_COMMENT("neon_reduce_minmax_integral {");
if (bt == T_LONG) {
assert(vtmp == fnoreg, "should be");
assert(isQ, "should be");
umov(rscratch1, vsrc, D, 0);
cmp(isrc, rscratch1);
- csel(dst, isrc, rscratch1, is_min ? LT : GT);
+ csel(dst, isrc, rscratch1, cond);
umov(rscratch1, vsrc, D, 1);
cmp(dst, rscratch1);
- csel(dst, dst, rscratch1, is_min ? LT : GT);
+ csel(dst, dst, rscratch1, cond);
} else {
SIMD_Arrangement size = esize2arrangement((unsigned)type2aelembytes(bt), isQ);
if (size == T2S) {
- is_min ? sminp(vtmp, size, vsrc, vsrc) : smaxp(vtmp, size, vsrc, vsrc);
+ // For T2S (2x32-bit elements), use pairwise instructions because
+ // uminv/umaxv/sminv/smaxv don't support arrangement 2S.
+ neon_minmaxp(is_unsigned, is_min, vtmp, size, vsrc, vsrc);
} else {
- is_min ? sminv(vtmp, size, vsrc) : smaxv(vtmp, size, vsrc);
+ // For other sizes, use reduction to scalar instructions.
+ neon_minmaxv(is_unsigned, is_min, vtmp, size, vsrc);
}
if (bt == T_INT) {
umov(dst, vtmp, S, 0);
+ } else if (is_unsigned) {
+ umov(dst, vtmp, elemType_to_regVariant(bt), 0);
} else {
smov(dst, vtmp, elemType_to_regVariant(bt), 0);
}
cmpw(dst, isrc);
- cselw(dst, dst, isrc, is_min ? LT : GT);
+ cselw(dst, dst, isrc, cond);
}
BLOCK_COMMENT("} neon_reduce_minmax_integral");
}
// Vector reduction for integral type with SVE instruction.
-// Supported operations are Add, And, Or, Xor, Max, Min.
+// Supported operations are Add, And, Or, Xor, Max, Min, UMax, UMin.
// rflags would be clobbered if opc is Op_MaxReductionV or Op_MinReductionV.
void C2_MacroAssembler::sve_reduce_integral(int opc, Register dst, BasicType bt, Register src1,
FloatRegister src2, PRegister pg, FloatRegister tmp) {
@@ -2075,35 +2101,27 @@ void C2_MacroAssembler::sve_reduce_integral(int opc, Register dst, BasicType bt,
}
break;
}
- case Op_MaxReductionV: {
- sve_smaxv(tmp, size, pg, src2);
- if (bt == T_INT || bt == T_LONG) {
+ case Op_MaxReductionV:
+ case Op_MinReductionV:
+ case Op_UMaxReductionV:
+ case Op_UMinReductionV: {
+ bool is_min;
+ bool is_unsigned;
+ Condition cond;
+ decode_minmax_reduction_opc(opc, &is_min, &is_unsigned, &cond);
+ sve_minmaxv(is_unsigned, is_min, tmp, size, pg, src2);
+ // Move result from vector to general register
+ if (is_unsigned || bt == T_INT || bt == T_LONG) {
umov(dst, tmp, size, 0);
} else {
smov(dst, tmp, size, 0);
}
if (bt == T_LONG) {
cmp(dst, src1);
- csel(dst, dst, src1, Assembler::GT);
+ csel(dst, dst, src1, cond);
} else {
cmpw(dst, src1);
- cselw(dst, dst, src1, Assembler::GT);
- }
- break;
- }
- case Op_MinReductionV: {
- sve_sminv(tmp, size, pg, src2);
- if (bt == T_INT || bt == T_LONG) {
- umov(dst, tmp, size, 0);
- } else {
- smov(dst, tmp, size, 0);
- }
- if (bt == T_LONG) {
- cmp(dst, src1);
- csel(dst, dst, src1, Assembler::LT);
- } else {
- cmpw(dst, src1);
- cselw(dst, dst, src1, Assembler::LT);
+ cselw(dst, dst, src1, cond);
}
break;
}
diff --git a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp
index 412f0f37e9e..4f3a41da402 100644
--- a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -34,6 +34,37 @@
void neon_reduce_logical_helper(int opc, bool sf, Register Rd, Register Rn, Register Rm,
enum shift_kind kind = Assembler::LSL, unsigned shift = 0);
+ // Helper functions for min/max reduction operations
+
+ void decode_minmax_reduction_opc(int opc, bool* is_min, bool* is_unsigned, Condition* cond);
+
+ void neon_minmaxp(bool is_unsigned, bool is_min, FloatRegister dst,
+ SIMD_Arrangement size, FloatRegister src1, FloatRegister src2) {
+ auto m = is_unsigned ? (is_min ? &Assembler::uminp : &Assembler::umaxp)
+ : (is_min ? &Assembler::sminp : &Assembler::smaxp);
+ (this->*m)(dst, size, src1, src2);
+ }
+
+ // Typedefs used to disambiguate overloaded member functions.
+ typedef void (Assembler::*neon_reduction2)
+ (FloatRegister, SIMD_Arrangement, FloatRegister);
+
+ void neon_minmaxv(bool is_unsigned, bool is_min, FloatRegister dst,
+ SIMD_Arrangement size, FloatRegister src) {
+ auto m = is_unsigned ? (is_min ? (neon_reduction2)&Assembler::uminv
+ : (neon_reduction2)&Assembler::umaxv)
+ : (is_min ? &Assembler::sminv
+ : &Assembler::smaxv);
+ (this->*m)(dst, size, src);
+ }
+
+ void sve_minmaxv(bool is_unsigned, bool is_min, FloatRegister dst,
+ SIMD_RegVariant size, PRegister pg, FloatRegister src) {
+ auto m = is_unsigned ? (is_min ? &Assembler::sve_uminv : &Assembler::sve_umaxv)
+ : (is_min ? &Assembler::sve_sminv : &Assembler::sve_smaxv);
+ (this->*m)(dst, size, pg, src);
+ }
+
void select_from_two_vectors_neon(FloatRegister dst, FloatRegister src1,
FloatRegister src2, FloatRegister index,
FloatRegister tmp, unsigned vector_length_in_bytes);
diff --git a/src/hotspot/cpu/aarch64/gc/shared/barrierSetNMethod_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shared/barrierSetNMethod_aarch64.cpp
index 4d5ca01b6b4..3d5261c31d1 100644
--- a/src/hotspot/cpu/aarch64/gc/shared/barrierSetNMethod_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/gc/shared/barrierSetNMethod_aarch64.cpp
@@ -209,6 +209,10 @@ void BarrierSetNMethod::set_guard_value(nmethod* nm, int value, int bit_mask) {
bs_asm->increment_patching_epoch();
}
+ // Enable WXWrite: the function is called directly from nmethod_entry_barrier
+ // stub.
+ MACOS_AARCH64_ONLY(ThreadWXEnable wx(WXWrite, Thread::current()));
+
NativeNMethodBarrier barrier(nm);
barrier.set_value(value, bit_mask);
}
diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp
index 9a035d9f40e..ad7bac4e067 100644
--- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp
@@ -85,26 +85,16 @@ void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, Dec
}
}
-void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
- Register obj,
- Register pre_val,
- Register thread,
- Register tmp,
- bool tosca_live,
- bool expand_call) {
- if (ShenandoahSATBBarrier) {
- satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, rscratch1, tosca_live, expand_call);
- }
-}
+void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler* masm,
+ Register obj,
+ Register pre_val,
+ Register thread,
+ Register tmp1,
+ Register tmp2,
+ bool tosca_live,
+ bool expand_call) {
+ assert(ShenandoahSATBBarrier, "Should be checked by caller");
-void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
- Register obj,
- Register pre_val,
- Register thread,
- Register tmp1,
- Register tmp2,
- bool tosca_live,
- bool expand_call) {
// If expand_call is true then we expand the call_VM_leaf macro
// directly to skip generating the check by
// InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
@@ -358,20 +348,20 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet d
if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
__ enter(/*strip_ret_addr*/true);
__ push_call_clobbered_registers();
- satb_write_barrier_pre(masm /* masm */,
- noreg /* obj */,
- dst /* pre_val */,
- rthread /* thread */,
- tmp1 /* tmp1 */,
- tmp2 /* tmp2 */,
- true /* tosca_live */,
- true /* expand_call */);
+ satb_barrier(masm /* masm */,
+ noreg /* obj */,
+ dst /* pre_val */,
+ rthread /* thread */,
+ tmp1 /* tmp1 */,
+ tmp2 /* tmp2 */,
+ true /* tosca_live */,
+ true /* expand_call */);
__ pop_call_clobbered_registers();
__ leave();
}
}
-void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj) {
+void ShenandoahBarrierSetAssembler::card_barrier(MacroAssembler* masm, Register obj) {
assert(ShenandoahCardBarrier, "Should have been checked by caller");
__ lsr(obj, obj, CardTable::card_shift());
@@ -394,13 +384,13 @@ void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register o
void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
- bool on_oop = is_reference_type(type);
- if (!on_oop) {
+ // 1: non-reference types require no barriers
+ if (!is_reference_type(type)) {
BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
return;
}
- // flatten object address if needed
+ // Flatten object address right away for simplicity: likely needed by barriers
if (dst.index() == noreg && dst.offset() == 0) {
if (dst.base() != tmp3) {
__ mov(tmp3, dst.base());
@@ -409,20 +399,26 @@ void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet
__ lea(tmp3, dst);
}
- shenandoah_write_barrier_pre(masm,
- tmp3 /* obj */,
- tmp2 /* pre_val */,
- rthread /* thread */,
- tmp1 /* tmp */,
- val != noreg /* tosca_live */,
- false /* expand_call */);
+ bool storing_non_null = (val != noreg);
+ // 2: pre-barrier: SATB needs the previous value
+ if (ShenandoahBarrierSet::need_satb_barrier(decorators, type)) {
+ satb_barrier(masm,
+ tmp3 /* obj */,
+ tmp2 /* pre_val */,
+ rthread /* thread */,
+ tmp1 /* tmp */,
+ rscratch1 /* tmp2 */,
+ storing_non_null /* tosca_live */,
+ false /* expand_call */);
+ }
+
+ // Store!
BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
- bool in_heap = (decorators & IN_HEAP) != 0;
- bool needs_post_barrier = (val != noreg) && in_heap && ShenandoahCardBarrier;
- if (needs_post_barrier) {
- store_check(masm, tmp3);
+ // 3: post-barrier: card barrier needs store address
+ if (ShenandoahBarrierSet::need_card_barrier(decorators, type) && storing_non_null) {
+ card_barrier(masm, tmp3);
}
}
diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp
index c0e708e1292..362fcae1ccd 100644
--- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp
@@ -40,23 +40,16 @@ class StubCodeGenerator;
class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
private:
- void satb_write_barrier_pre(MacroAssembler* masm,
- Register obj,
- Register pre_val,
- Register thread,
- Register tmp1,
- Register tmp2,
- bool tosca_live,
- bool expand_call);
- void shenandoah_write_barrier_pre(MacroAssembler* masm,
- Register obj,
- Register pre_val,
- Register thread,
- Register tmp,
- bool tosca_live,
- bool expand_call);
+ void satb_barrier(MacroAssembler* masm,
+ Register obj,
+ Register pre_val,
+ Register thread,
+ Register tmp1,
+ Register tmp2,
+ bool tosca_live,
+ bool expand_call);
- void store_check(MacroAssembler* masm, Register obj);
+ void card_barrier(MacroAssembler* masm, Register obj);
void resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp = noreg);
void resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp = noreg);
diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
index f8b5a6f825c..409343b6b8d 100644
--- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2024, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -473,6 +473,7 @@ address MacroAssembler::target_addr_for_insn(address insn_addr) {
// Patch any kind of instruction; there may be several instructions.
// Return the total length (in bytes) of the instructions.
int MacroAssembler::pd_patch_instruction_size(address insn_addr, address target) {
+ MACOS_AARCH64_ONLY(os::thread_wx_enable_write());
return RelocActions::run(insn_addr, target);
}
@@ -481,6 +482,8 @@ int MacroAssembler::patch_oop(address insn_addr, address o) {
unsigned insn = *(unsigned*)insn_addr;
assert(nativeInstruction_at(insn_addr+4)->is_movk(), "wrong insns in patch");
+ MACOS_AARCH64_ONLY(os::thread_wx_enable_write());
+
// OOPs are either narrow (32 bits) or wide (48 bits). We encode
// narrow OOPs by setting the upper 16 bits in the first
// instruction.
@@ -510,6 +513,8 @@ int MacroAssembler::patch_narrow_klass(address insn_addr, narrowKlass n) {
assert(Instruction_aarch64::extract(insn->encoding(), 31, 21) == 0b11010010101 &&
nativeInstruction_at(insn_addr+4)->is_movk(), "wrong insns in patch");
+ MACOS_AARCH64_ONLY(os::thread_wx_enable_write());
+
Instruction_aarch64::patch(insn_addr, 20, 5, n >> 16);
Instruction_aarch64::patch(insn_addr+4, 20, 5, n & 0xffff);
return 2 * NativeInstruction::instruction_size;
@@ -6419,10 +6424,14 @@ void MacroAssembler::fill_words(Register base, Register cnt, Register value)
// Intrinsic for
//
-// - sun/nio/cs/ISO_8859_1$Encoder.implEncodeISOArray
-// return the number of characters copied.
-// - java/lang/StringUTF16.compress
-// return index of non-latin1 character if copy fails, otherwise 'len'.
+// - sun.nio.cs.ISO_8859_1.Encoder#encodeISOArray0(byte[] sa, int sp, byte[] da, int dp, int len)
+// Encodes char[] to byte[] in ISO-8859-1
+//
+// - java.lang.StringCoding#encodeISOArray0(byte[] sa, int sp, byte[] da, int dp, int len)
+// Encodes byte[] (containing UTF-16) to byte[] in ISO-8859-1
+//
+// - java.lang.StringCoding#encodeAsciiArray0(char[] sa, int sp, byte[] da, int dp, int len)
+// Encodes char[] to byte[] in ASCII
//
// This version always returns the number of characters copied, and does not
// clobber the 'len' register. A successful copy will complete with the post-
diff --git a/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp b/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp
index f2003dd9b55..0cdf36f0bc5 100644
--- a/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp
@@ -133,7 +133,6 @@ void NativeMovConstReg::verify() {
intptr_t NativeMovConstReg::data() const {
- // das(uint64_t(instruction_address()),2);
address addr = MacroAssembler::target_addr_for_insn(instruction_address());
if (maybe_cpool_ref(instruction_address())) {
return *(intptr_t*)addr;
@@ -144,6 +143,7 @@ intptr_t NativeMovConstReg::data() const {
void NativeMovConstReg::set_data(intptr_t x) {
if (maybe_cpool_ref(instruction_address())) {
+ MACOS_AARCH64_ONLY(os::thread_wx_enable_write());
address addr = MacroAssembler::target_addr_for_insn(instruction_address());
*(intptr_t*)addr = x;
} else {
@@ -350,8 +350,6 @@ bool NativeInstruction::is_stop() {
//-------------------------------------------------------------------
-void NativeGeneralJump::verify() { }
-
// MT-safe patching of a long jump instruction.
void NativeGeneralJump::replace_mt_safe(address instr_addr, address code_buffer) {
ShouldNotCallThis();
diff --git a/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp b/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp
index c30cb911d96..15b6c9ff215 100644
--- a/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp
@@ -90,16 +90,18 @@ protected:
s_char sbyte_at(int offset) const { return *(s_char*)addr_at(offset); }
u_char ubyte_at(int offset) const { return *(u_char*)addr_at(offset); }
- jint int_at(int offset) const { return *(jint*)addr_at(offset); }
- juint uint_at(int offset) const { return *(juint*)addr_at(offset); }
- address ptr_at(int offset) const { return *(address*)addr_at(offset); }
- oop oop_at(int offset) const { return *(oop*)addr_at(offset); }
+ jint int_at(int offset) const { return *(jint*)addr_at(offset); }
+ juint uint_at(int offset) const { return *(juint*)addr_at(offset); }
+ address ptr_at(int offset) const { return *(address*)addr_at(offset); }
+ oop oop_at(int offset) const { return *(oop*)addr_at(offset); }
- void set_char_at(int offset, char c) { *addr_at(offset) = (u_char)c; }
- void set_int_at(int offset, jint i) { *(jint*)addr_at(offset) = i; }
- void set_uint_at(int offset, jint i) { *(juint*)addr_at(offset) = i; }
- void set_ptr_at(int offset, address ptr) { *(address*)addr_at(offset) = ptr; }
- void set_oop_at(int offset, oop o) { *(oop*)addr_at(offset) = o; }
+#define MACOS_WX_WRITE MACOS_AARCH64_ONLY(os::thread_wx_enable_write())
+ void set_char_at(int offset, char c) { MACOS_WX_WRITE; *addr_at(offset) = (u_char)c; }
+ void set_int_at(int offset, jint i) { MACOS_WX_WRITE; *(jint*)addr_at(offset) = i; }
+ void set_uint_at(int offset, jint i) { MACOS_WX_WRITE; *(juint*)addr_at(offset) = i; }
+ void set_ptr_at(int offset, address ptr) { MACOS_WX_WRITE; *(address*)addr_at(offset) = ptr; }
+ void set_oop_at(int offset, oop o) { MACOS_WX_WRITE; *(oop*)addr_at(offset) = o; }
+#undef MACOS_WX_WRITE
void wrote(int offset);
@@ -380,7 +382,6 @@ public:
void set_jump_destination(address dest);
static void replace_mt_safe(address instr_addr, address code_buffer);
- static void verify();
};
inline NativeGeneralJump* nativeGeneralJump_at(address address) {
diff --git a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp
index db653bcf236..a459a28b09e 100644
--- a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp
@@ -11742,7 +11742,9 @@ class StubGenerator: public StubCodeGenerator {
}
#endif
- StubRoutines::_unsafe_setmemory = generate_unsafe_setmemory();
+ if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_setMemory)) {
+ StubRoutines::_unsafe_setmemory = generate_unsafe_setmemory();
+ }
StubRoutines::aarch64::set_completed(); // Inidicate that arraycopy and zero_blocks stubs are generated
}
diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
index 659c231464a..0a7bc5a8962 100644
--- a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
@@ -201,16 +201,14 @@ void VM_Version::initialize() {
}
}
- // Cortex A53
- if (_cpu == CPU_ARM && model_is(0xd03)) {
+ if (_cpu == CPU_ARM && model_is(CPU_MODEL_ARM_CORTEX_A53)) {
set_feature(CPU_A53MAC);
if (FLAG_IS_DEFAULT(UseSIMDForArrayEquals)) {
FLAG_SET_DEFAULT(UseSIMDForArrayEquals, false);
}
}
- // Cortex A73
- if (_cpu == CPU_ARM && model_is(0xd09)) {
+ if (_cpu == CPU_ARM && model_is(CPU_MODEL_ARM_CORTEX_A73)) {
if (FLAG_IS_DEFAULT(SoftwarePrefetchHintDistance)) {
FLAG_SET_DEFAULT(SoftwarePrefetchHintDistance, -1);
}
@@ -220,16 +218,11 @@ void VM_Version::initialize() {
}
}
- // Neoverse
- // N1: 0xd0c
- // N2: 0xd49
- // N3: 0xd8e
- // V1: 0xd40
- // V2: 0xd4f
- // V3: 0xd84
- if (_cpu == CPU_ARM && (model_is(0xd0c) || model_is(0xd49) ||
- model_is(0xd40) || model_is(0xd4f) ||
- model_is(0xd8e) || model_is(0xd84))) {
+ if (_cpu == CPU_ARM &&
+ model_is_in({ CPU_MODEL_ARM_NEOVERSE_N1, CPU_MODEL_ARM_NEOVERSE_V1,
+ CPU_MODEL_ARM_NEOVERSE_N2, CPU_MODEL_ARM_NEOVERSE_V2,
+ CPU_MODEL_ARM_NEOVERSE_N3, CPU_MODEL_ARM_NEOVERSE_V3,
+ CPU_MODEL_ARM_NEOVERSE_V3AE })) {
if (FLAG_IS_DEFAULT(UseSIMDForMemoryOps)) {
FLAG_SET_DEFAULT(UseSIMDForMemoryOps, true);
}
@@ -261,12 +254,9 @@ void VM_Version::initialize() {
FLAG_SET_DEFAULT(UseCRC32, false);
}
- // Neoverse
- // V1: 0xd40
- // V2: 0xd4f
- // V3: 0xd84
if (_cpu == CPU_ARM &&
- (model_is(0xd40) || model_is(0xd4f) || model_is(0xd84))) {
+ model_is_in({ CPU_MODEL_ARM_NEOVERSE_V1, CPU_MODEL_ARM_NEOVERSE_V2,
+ CPU_MODEL_ARM_NEOVERSE_V3, CPU_MODEL_ARM_NEOVERSE_V3AE })) {
if (FLAG_IS_DEFAULT(UseCryptoPmullForCRC32)) {
FLAG_SET_DEFAULT(UseCryptoPmullForCRC32, true);
}
@@ -632,6 +622,22 @@ void VM_Version::initialize() {
check_virtualizations();
+#ifdef __APPLE__
+ DefaultWXWriteMode = UseOldWX ? WXWrite : WXArmedForWrite;
+
+ if (TraceWXHealing) {
+ if (pthread_jit_write_protect_supported_np()) {
+ tty->print_cr("### TraceWXHealing is in use");
+ if (StressWXHealing) {
+ tty->print_cr("### StressWXHealing is in use");
+ }
+ } else {
+ tty->print_cr("WX Healing is not in use because MAP_JIT write protection "
+ "does not work on this system.");
+ }
+ }
+#endif
+
// Sync SVE related CPU features with flags
if (UseSVE < 2) {
clear_feature(CPU_SVE2);
diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp
index 17087d243d3..38b112d9936 100644
--- a/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp
@@ -30,6 +30,8 @@
#include "runtime/abstract_vm_version.hpp"
#include "utilities/sizes.hpp"
+#include
+
class stringStream;
#define BIT_MASK(flag) (1ULL<<(flag))
@@ -112,14 +114,26 @@ public:
CPU_APPLE = 'a',
};
-enum Ampere_CPU_Model {
+ enum Ampere_CPU_Model {
CPU_MODEL_EMAG = 0x0, /* CPU implementer is CPU_AMCC */
CPU_MODEL_ALTRA = 0xd0c, /* CPU implementer is CPU_ARM, Neoverse N1 */
CPU_MODEL_ALTRAMAX = 0xd0c, /* CPU implementer is CPU_ARM, Neoverse N1 */
CPU_MODEL_AMPERE_1 = 0xac3, /* CPU implementer is CPU_AMPERE */
CPU_MODEL_AMPERE_1A = 0xac4, /* CPU implementer is CPU_AMPERE */
CPU_MODEL_AMPERE_1B = 0xac5 /* AMPERE_1B core Implements ARMv8.7 with CSSC, MTE, SM3/SM4 extensions */
-};
+ };
+
+ enum ARM_CPU_Model {
+ CPU_MODEL_ARM_CORTEX_A53 = 0xd03,
+ CPU_MODEL_ARM_CORTEX_A73 = 0xd09,
+ CPU_MODEL_ARM_NEOVERSE_N1 = 0xd0c,
+ CPU_MODEL_ARM_NEOVERSE_V1 = 0xd40,
+ CPU_MODEL_ARM_NEOVERSE_N2 = 0xd49,
+ CPU_MODEL_ARM_NEOVERSE_V2 = 0xd4f,
+ CPU_MODEL_ARM_NEOVERSE_V3AE = 0xd83,
+ CPU_MODEL_ARM_NEOVERSE_V3 = 0xd84,
+ CPU_MODEL_ARM_NEOVERSE_N3 = 0xd8e,
+ };
#define CPU_FEATURE_FLAGS(decl) \
decl(FP, fp, 0) \
@@ -181,6 +195,15 @@ enum Ampere_CPU_Model {
return _model == cpu_model || _model2 == cpu_model;
}
+ static bool model_is_in(std::initializer_list cpu_models) {
+ for (const int& cpu_model : cpu_models) {
+ if (_model == cpu_model || _model2 == cpu_model) {
+ return true;
+ }
+ }
+ return false;
+ }
+
static bool is_zva_enabled() { return 0 <= _zva_length; }
static int zva_length() {
assert(is_zva_enabled(), "ZVA not available");
diff --git a/src/hotspot/cpu/ppc/assembler_ppc.hpp b/src/hotspot/cpu/ppc/assembler_ppc.hpp
index 15e38411482..23775a3a52e 100644
--- a/src/hotspot/cpu/ppc/assembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/assembler_ppc.hpp
@@ -568,6 +568,9 @@ class Assembler : public AbstractAssembler {
XSCVDPHP_OPCODE= (60u << OPCODE_SHIFT | 347u << 2 | 17u << 16), // XX2-FORM
XXPERM_OPCODE = (60u << OPCODE_SHIFT | 26u << 3),
XXSEL_OPCODE = (60u << OPCODE_SHIFT | 3u << 4),
+ XSCMPEQDP_OPCODE=(60u << OPCODE_SHIFT | 3u << 3),
+ XSCMPGEDP_OPCODE=(60u << OPCODE_SHIFT | 19u << 3),
+ XSCMPGTDP_OPCODE=(60u << OPCODE_SHIFT | 11u << 3),
XXSPLTIB_OPCODE= (60u << OPCODE_SHIFT | 360u << 1),
XVDIVDP_OPCODE = (60u << OPCODE_SHIFT | 120u << 3),
XVABSSP_OPCODE = (60u << OPCODE_SHIFT | 409u << 2),
@@ -2424,6 +2427,9 @@ class Assembler : public AbstractAssembler {
inline void xscvdphp( VectorSRegister d, VectorSRegister b);
inline void xxland( VectorSRegister d, VectorSRegister a, VectorSRegister b);
inline void xxsel( VectorSRegister d, VectorSRegister a, VectorSRegister b, VectorSRegister c);
+ inline void xscmpeqdp(VectorSRegister t, VectorSRegister a, VectorSRegister b); // Requires Power9
+ inline void xscmpgedp(VectorSRegister t, VectorSRegister a, VectorSRegister b); // Requires Power9
+ inline void xscmpgtdp(VectorSRegister t, VectorSRegister a, VectorSRegister b); // Requires Power9
inline void xxspltib( VectorSRegister d, int ui8);
inline void xvdivsp( VectorSRegister d, VectorSRegister a, VectorSRegister b);
inline void xvdivdp( VectorSRegister d, VectorSRegister a, VectorSRegister b);
diff --git a/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp b/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp
index 7e49ec7455d..4cda782067e 100644
--- a/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp
+++ b/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp
@@ -923,6 +923,10 @@ inline void Assembler::xxmrghw( VectorSRegister d, VectorSRegister a, VectorSReg
inline void Assembler::xxmrglw( VectorSRegister d, VectorSRegister a, VectorSRegister b) { emit_int32( XXMRGHW_OPCODE | vsrt(d) | vsra(a) | vsrb(b)); }
inline void Assembler::xxsel( VectorSRegister d, VectorSRegister a, VectorSRegister b, VectorSRegister c) { emit_int32( XXSEL_OPCODE | vsrt(d) | vsra(a) | vsrb(b) | vsrc(c)); }
+inline void Assembler::xscmpeqdp(VectorSRegister t, VectorSRegister a, VectorSRegister b) { emit_int32( XSCMPEQDP_OPCODE | vsrt(t) | vsra(a) | vsrb(b) );}
+inline void Assembler::xscmpgedp(VectorSRegister t, VectorSRegister a, VectorSRegister b) { emit_int32( XSCMPGEDP_OPCODE | vsrt(t) | vsra(a) | vsrb(b) );}
+inline void Assembler::xscmpgtdp(VectorSRegister t, VectorSRegister a, VectorSRegister b) { emit_int32( XSCMPGTDP_OPCODE | vsrt(t) | vsra(a) | vsrb(b) );}
+
// VSX Extended Mnemonics
inline void Assembler::xxspltd( VectorSRegister d, VectorSRegister a, int x) { xxpermdi(d, a, a, x ? 3 : 0); }
inline void Assembler::xxmrghd( VectorSRegister d, VectorSRegister a, VectorSRegister b) { xxpermdi(d, a, b, 0); }
diff --git a/src/hotspot/cpu/ppc/c2_MacroAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c2_MacroAssembler_ppc.cpp
index edf348fdc50..73b6b132895 100644
--- a/src/hotspot/cpu/ppc/c2_MacroAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/c2_MacroAssembler_ppc.cpp
@@ -664,3 +664,37 @@ void C2_MacroAssembler::reduceI(int opcode, Register dst, Register iSrc, VectorR
fn_scalar_op(opcode, dst, iSrc, R0); // dst <- op(iSrc, R0)
}
+// Works for single and double precision floats.
+// dst = (op1 cmp(cc) op2) ? src1 : src2;
+// Unordered semantics are the same as for CmpF3Node/CmpD3Node which implement the fcmpl/dcmpl bytecodes.
+// Comparing unordered values has the same result as when src1 is less than src2.
+// So dst = src1 for <, <=, != and dst = src2 for >, >=, ==.
+void C2_MacroAssembler::cmovF(int cc, VectorSRegister dst, VectorSRegister op1, VectorSRegister op2,
+ VectorSRegister src1, VectorSRegister src2, VectorSRegister tmp) {
+ // See operand cmpOp() for details.
+ bool invert_cond = (cc & 8) == 0; // invert reflects bcondCRbiIs0
+ auto cmp = (Assembler::Condition)(cc & 3);
+
+ switch(cmp) {
+ case Assembler::Condition::equal:
+ // Use false_result if "unordered".
+ xscmpeqdp(tmp, op1, op2);
+ break;
+ case Assembler::Condition::greater:
+ // Use false_result if "unordered".
+ xscmpgtdp(tmp, op1, op2);
+ break;
+ case Assembler::Condition::less:
+ // Use true_result if "unordered".
+ xscmpgedp(tmp, op1, op2);
+ invert_cond = !invert_cond;
+ break;
+ default:
+ assert(false, "unsupported compare condition: %d", cc);
+ ShouldNotReachHere();
+ }
+
+ VectorSRegister true_result = invert_cond ? src2 : src1;
+ VectorSRegister false_result = invert_cond ? src1 : src2;
+ xxsel(dst, false_result, true_result, tmp);
+}
diff --git a/src/hotspot/cpu/ppc/c2_MacroAssembler_ppc.hpp b/src/hotspot/cpu/ppc/c2_MacroAssembler_ppc.hpp
index 5a114294c1f..e0dffec8396 100644
--- a/src/hotspot/cpu/ppc/c2_MacroAssembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/c2_MacroAssembler_ppc.hpp
@@ -74,5 +74,7 @@
void count_positives(Register src, Register cnt, Register result, Register tmp1, Register tmp2);
void reduceI(int opcode, Register dst, Register iSrc, VectorRegister vSrc, VectorRegister vTmp1, VectorRegister vTmp2);
+ void cmovF(int cc, VectorSRegister dst, VectorSRegister op1, VectorSRegister op2,
+ VectorSRegister src1, VectorSRegister src2, VectorSRegister tmp);
#endif // CPU_PPC_C2_MACROASSEMBLER_PPC_HPP
diff --git a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp
index 9d143c14d27..c3bb1811031 100644
--- a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp
@@ -50,14 +50,14 @@
#define __ masm->
-void ShenandoahBarrierSetAssembler::satb_write_barrier(MacroAssembler *masm,
- Register base, RegisterOrConstant ind_or_offs,
- Register tmp1, Register tmp2, Register tmp3,
- MacroAssembler::PreservationLevel preservation_level) {
+void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler *masm,
+ Register base, RegisterOrConstant ind_or_offs,
+ Register tmp1, Register tmp2, Register tmp3,
+ MacroAssembler::PreservationLevel preservation_level) {
if (ShenandoahSATBBarrier) {
- __ block_comment("satb_write_barrier (shenandoahgc) {");
- satb_write_barrier_impl(masm, 0, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);
- __ block_comment("} satb_write_barrier (shenandoahgc)");
+ __ block_comment("satb_barrier (shenandoahgc) {");
+ satb_barrier_impl(masm, 0, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);
+ __ block_comment("} satb_barrier (shenandoahgc)");
}
}
@@ -198,11 +198,12 @@ void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, Dec
// In "load mode", this register acts as a temporary register and must
// thus not be 'noreg'. In "preloaded mode", its content will be sustained.
// tmp1/tmp2: Temporary registers, one of which must be non-volatile in "preloaded mode".
-void ShenandoahBarrierSetAssembler::satb_write_barrier_impl(MacroAssembler *masm, DecoratorSet decorators,
- Register base, RegisterOrConstant ind_or_offs,
- Register pre_val,
- Register tmp1, Register tmp2,
- MacroAssembler::PreservationLevel preservation_level) {
+void ShenandoahBarrierSetAssembler::satb_barrier_impl(MacroAssembler *masm, DecoratorSet decorators,
+ Register base, RegisterOrConstant ind_or_offs,
+ Register pre_val,
+ Register tmp1, Register tmp2,
+ MacroAssembler::PreservationLevel preservation_level) {
+ assert(ShenandoahSATBBarrier, "Should be checked by caller");
assert_different_registers(tmp1, tmp2, pre_val, noreg);
Label skip_barrier;
@@ -574,13 +575,13 @@ void ShenandoahBarrierSetAssembler::load_at(
if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
if (ShenandoahSATBBarrier) {
__ block_comment("keep_alive_barrier (shenandoahgc) {");
- satb_write_barrier_impl(masm, 0, noreg, noreg, dst, tmp1, tmp2, preservation_level);
+ satb_barrier_impl(masm, 0, noreg, noreg, dst, tmp1, tmp2, preservation_level);
__ block_comment("} keep_alive_barrier (shenandoahgc)");
}
}
}
-void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register base, RegisterOrConstant ind_or_offs, Register tmp) {
+void ShenandoahBarrierSetAssembler::card_barrier(MacroAssembler* masm, Register base, RegisterOrConstant ind_or_offs, Register tmp) {
assert(ShenandoahCardBarrier, "Should have been checked by caller");
assert_different_registers(base, tmp, R0);
@@ -603,21 +604,33 @@ void ShenandoahBarrierSetAssembler::store_at(MacroAssembler *masm, DecoratorSet
Register base, RegisterOrConstant ind_or_offs, Register val,
Register tmp1, Register tmp2, Register tmp3,
MacroAssembler::PreservationLevel preservation_level) {
- if (is_reference_type(type)) {
- if (ShenandoahSATBBarrier) {
- satb_write_barrier(masm, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);
- }
+ // 1: non-reference types require no barriers
+ if (!is_reference_type(type)) {
+ BarrierSetAssembler::store_at(masm, decorators, type,
+ base, ind_or_offs,
+ val,
+ tmp1, tmp2, tmp3,
+ preservation_level);
+ return;
}
+ bool storing_non_null = (val != noreg);
+
+ // 2: pre-barrier: SATB needs the previous value
+ if (ShenandoahBarrierSet::need_satb_barrier(decorators, type)) {
+ satb_barrier(masm, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);
+ }
+
+ // Store!
BarrierSetAssembler::store_at(masm, decorators, type,
base, ind_or_offs,
val,
tmp1, tmp2, tmp3,
preservation_level);
- // No need for post barrier if storing null
- if (ShenandoahCardBarrier && is_reference_type(type) && val != noreg) {
- store_check(masm, base, ind_or_offs, tmp1);
+ // 3: post-barrier: card barrier needs store address
+ if (ShenandoahBarrierSet::need_card_barrier(decorators, type) && storing_non_null) {
+ card_barrier(masm, base, ind_or_offs, tmp1);
}
}
diff --git a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.hpp b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.hpp
index b058dcf1a2e..52615a740af 100644
--- a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.hpp
@@ -45,15 +45,15 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
private:
/* ==== Actual barrier implementations ==== */
- void satb_write_barrier_impl(MacroAssembler* masm, DecoratorSet decorators,
- Register base, RegisterOrConstant ind_or_offs,
- Register pre_val,
- Register tmp1, Register tmp2,
- MacroAssembler::PreservationLevel preservation_level);
+ void satb_barrier_impl(MacroAssembler* masm, DecoratorSet decorators,
+ Register base, RegisterOrConstant ind_or_offs,
+ Register pre_val,
+ Register tmp1, Register tmp2,
+ MacroAssembler::PreservationLevel preservation_level);
- void store_check(MacroAssembler* masm,
- Register base, RegisterOrConstant ind_or_offs,
- Register tmp);
+ void card_barrier(MacroAssembler* masm,
+ Register base, RegisterOrConstant ind_or_offs,
+ Register tmp);
void load_reference_barrier_impl(MacroAssembler* masm, DecoratorSet decorators,
Register base, RegisterOrConstant ind_or_offs,
@@ -85,10 +85,10 @@ public:
#endif
/* ==== Available barriers (facades of the actual implementations) ==== */
- void satb_write_barrier(MacroAssembler* masm,
- Register base, RegisterOrConstant ind_or_offs,
- Register tmp1, Register tmp2, Register tmp3,
- MacroAssembler::PreservationLevel preservation_level);
+ void satb_barrier(MacroAssembler* masm,
+ Register base, RegisterOrConstant ind_or_offs,
+ Register tmp1, Register tmp2, Register tmp3,
+ MacroAssembler::PreservationLevel preservation_level);
void load_reference_barrier(MacroAssembler* masm, DecoratorSet decorators,
Register base, RegisterOrConstant ind_or_offs,
diff --git a/src/hotspot/cpu/ppc/matcher_ppc.hpp b/src/hotspot/cpu/ppc/matcher_ppc.hpp
index aad41fb7b1c..b50de6323de 100644
--- a/src/hotspot/cpu/ppc/matcher_ppc.hpp
+++ b/src/hotspot/cpu/ppc/matcher_ppc.hpp
@@ -64,12 +64,10 @@
return true;
}
- // Use conditional move (CMOVL) on Power7.
static constexpr int long_cmove_cost() { return 0; } // this only makes long cmoves more expensive than int cmoves
- // Suppress CMOVF. Conditional move available (sort of) on PPC64 only from P7 onwards. Not exploited yet.
- // fsel doesn't accept a condition register as input, so this would be slightly different.
- static int float_cmove_cost() { return ConditionalMoveLimit; }
+ // Suppress CMOVF for Power8 because there are no fast nodes.
+ static int float_cmove_cost() { return (PowerArchitecturePPC64 >= 9) ? 0 : ConditionalMoveLimit; }
// This affects two different things:
// - how Decode nodes are matched
diff --git a/src/hotspot/cpu/ppc/ppc.ad b/src/hotspot/cpu/ppc/ppc.ad
index 2a0a9149bb3..d926fabd353 100644
--- a/src/hotspot/cpu/ppc/ppc.ad
+++ b/src/hotspot/cpu/ppc/ppc.ad
@@ -3024,7 +3024,6 @@ encode %{
%}
enc_class postalloc_expand_encode_oop(iRegNdst dst, iRegPdst src, flagsReg crx) %{
- // use isel instruction with Power 7
cmpP_reg_imm16Node *n_compare = new cmpP_reg_imm16Node();
encodeP_subNode *n_sub_base = new encodeP_subNode();
encodeP_shiftNode *n_shift = new encodeP_shiftNode();
@@ -3099,7 +3098,6 @@ encode %{
n_shift->_opnds[1] = op_src;
n_shift->_bottom_type = _bottom_type;
- // use isel instruction with Power 7
decodeN_addNode *n_add_base = new decodeN_addNode();
n_add_base->add_req(n_region, n_shift);
n_add_base->_opnds[0] = op_dst;
@@ -6618,7 +6616,6 @@ instruct cond_sub_base(iRegNdst dst, flagsRegSrc crx, iRegPsrc src1) %{
ins_pipe(pipe_class_default);
%}
-// Power 7 can use isel instruction
instruct cond_set_0_oop(iRegNdst dst, flagsRegSrc crx, iRegPsrc src1) %{
// The match rule is needed to make it a 'MachTypeNode'!
match(Set dst (EncodeP (Binary crx src1)));
@@ -7293,7 +7290,6 @@ instruct cmovF_reg(cmpOp cmp, flagsRegSrc crx, regF dst, regF src) %{
ins_variable_size_depending_on_alignment(true);
format %{ "CMOVEF $cmp, $crx, $dst, $src\n\t" %}
- // Worst case is branch + move + stop, no stop without scheduler.
size(8);
ins_encode %{
Label done;
@@ -7313,7 +7309,6 @@ instruct cmovD_reg(cmpOp cmp, flagsRegSrc crx, regD dst, regD src) %{
ins_variable_size_depending_on_alignment(true);
format %{ "CMOVEF $cmp, $crx, $dst, $src\n\t" %}
- // Worst case is branch + move + stop, no stop without scheduler.
size(8);
ins_encode %{
Label done;
@@ -7326,6 +7321,70 @@ instruct cmovD_reg(cmpOp cmp, flagsRegSrc crx, regD dst, regD src) %{
ins_pipe(pipe_class_default);
%}
+instruct cmovF_cmpF(cmpOp cop, regF op1, regF op2, regF dst, regF false_result, regF true_result, regD tmp) %{
+ match(Set dst (CMoveF (Binary cop (CmpF op1 op2)) (Binary false_result true_result)));
+ predicate(PowerArchitecturePPC64 >= 9);
+ effect(TEMP tmp);
+ ins_cost(2*DEFAULT_COST);
+ format %{ "cmovF_cmpF $dst = ($op1 $cop $op2) ? $true_result : $false_result\n\t" %}
+ size(8);
+ ins_encode %{
+ __ cmovF($cop$$cmpcode, $dst$$FloatRegister->to_vsr(),
+ $op1$$FloatRegister->to_vsr(), $op2$$FloatRegister->to_vsr(),
+ $true_result$$FloatRegister->to_vsr(), $false_result$$FloatRegister->to_vsr(),
+ $tmp$$FloatRegister->to_vsr());
+ %}
+ ins_pipe(pipe_class_default);
+%}
+
+instruct cmovF_cmpD(cmpOp cop, regD op1, regD op2, regF dst, regF false_result, regF true_result, regD tmp) %{
+ match(Set dst (CMoveF (Binary cop (CmpD op1 op2)) (Binary false_result true_result)));
+ predicate(PowerArchitecturePPC64 >= 9);
+ effect(TEMP tmp);
+ ins_cost(2*DEFAULT_COST);
+ format %{ "cmovF_cmpD $dst = ($op1 $cop $op2) ? $true_result : $false_result\n\t" %}
+ size(8);
+ ins_encode %{
+ __ cmovF($cop$$cmpcode, $dst$$FloatRegister->to_vsr(),
+ $op1$$FloatRegister->to_vsr(), $op2$$FloatRegister->to_vsr(),
+ $true_result$$FloatRegister->to_vsr(), $false_result$$FloatRegister->to_vsr(),
+ $tmp$$FloatRegister->to_vsr());
+ %}
+ ins_pipe(pipe_class_default);
+%}
+
+instruct cmovD_cmpD(cmpOp cop, regD op1, regD op2, regD dst, regD false_result, regD true_result, regD tmp) %{
+ match(Set dst (CMoveD (Binary cop (CmpD op1 op2)) (Binary false_result true_result)));
+ predicate(PowerArchitecturePPC64 >= 9);
+ effect(TEMP tmp);
+ ins_cost(2*DEFAULT_COST);
+ format %{ "cmovD_cmpD $dst = ($op1 $cop $op2) ? $true_result : $false_result\n\t" %}
+ size(8);
+ ins_encode %{
+ __ cmovF($cop$$cmpcode, $dst$$FloatRegister->to_vsr(),
+ $op1$$FloatRegister->to_vsr(), $op2$$FloatRegister->to_vsr(),
+ $true_result$$FloatRegister->to_vsr(), $false_result$$FloatRegister->to_vsr(),
+ $tmp$$FloatRegister->to_vsr());
+ %}
+ ins_pipe(pipe_class_default);
+%}
+
+instruct cmovD_cmpF(cmpOp cop, regF op1, regF op2, regD dst, regD false_result, regD true_result, regD tmp) %{
+ match(Set dst (CMoveD (Binary cop (CmpF op1 op2)) (Binary false_result true_result)));
+ predicate(PowerArchitecturePPC64 >= 9);
+ effect(TEMP tmp);
+ ins_cost(2*DEFAULT_COST);
+ format %{ "cmovD_cmpF $dst = ($op1 $cop $op2) ? $true_result : $false_result\n\t" %}
+ size(8);
+ ins_encode %{
+ __ cmovF($cop$$cmpcode, $dst$$FloatRegister->to_vsr(),
+ $op1$$FloatRegister->to_vsr(), $op2$$FloatRegister->to_vsr(),
+ $true_result$$FloatRegister->to_vsr(), $false_result$$FloatRegister->to_vsr(),
+ $tmp$$FloatRegister->to_vsr());
+ %}
+ ins_pipe(pipe_class_default);
+%}
+
//----------Compare-And-Swap---------------------------------------------------
// CompareAndSwap{P,I,L} have more than one output, therefore "CmpI
@@ -8492,7 +8551,6 @@ instruct cmovI_bne_negI_reg(iRegIdst dst, flagsRegSrc crx, iRegIsrc src1) %{
ins_variable_size_depending_on_alignment(true);
format %{ "CMOVE $dst, neg($src1), $crx" %}
- // Worst case is branch + move + stop, no stop without scheduler.
size(8);
ins_encode %{
Label done;
@@ -8551,7 +8609,6 @@ instruct cmovL_bne_negL_reg(iRegLdst dst, flagsRegSrc crx, iRegLsrc src1) %{
ins_variable_size_depending_on_alignment(true);
format %{ "CMOVE $dst, neg($src1), $crx" %}
- // Worst case is branch + move + stop, no stop without scheduler.
size(8);
ins_encode %{
Label done;
@@ -10262,7 +10319,6 @@ instruct cmovI_bso_stackSlotL(iRegIdst dst, flagsRegSrc crx, stackSlotL src) %{
ins_variable_size_depending_on_alignment(true);
format %{ "cmovI $crx, $dst, $src" %}
- // Worst case is branch + move + stop, no stop without scheduler.
size(8);
ins_encode( enc_cmove_bso_stackSlotL(dst, crx, src) );
ins_pipe(pipe_class_default);
@@ -10276,7 +10332,6 @@ instruct cmovI_bso_reg(iRegIdst dst, flagsRegSrc crx, regD src) %{
ins_variable_size_depending_on_alignment(true);
format %{ "cmovI $crx, $dst, $src" %}
- // Worst case is branch + move + stop, no stop without scheduler.
size(8);
ins_encode( enc_cmove_bso_reg(dst, crx, src) );
ins_pipe(pipe_class_default);
@@ -10439,7 +10494,6 @@ instruct cmovL_bso_stackSlotL(iRegLdst dst, flagsRegSrc crx, stackSlotL src) %{
ins_variable_size_depending_on_alignment(true);
format %{ "cmovL $crx, $dst, $src" %}
- // Worst case is branch + move + stop, no stop without scheduler.
size(8);
ins_encode( enc_cmove_bso_stackSlotL(dst, crx, src) );
ins_pipe(pipe_class_default);
@@ -10453,7 +10507,6 @@ instruct cmovL_bso_reg(iRegLdst dst, flagsRegSrc crx, regD src) %{
ins_variable_size_depending_on_alignment(true);
format %{ "cmovL $crx, $dst, $src" %}
- // Worst case is branch + move + stop, no stop without scheduler.
size(8);
ins_encode( enc_cmove_bso_reg(dst, crx, src) );
ins_pipe(pipe_class_default);
@@ -11080,7 +11133,6 @@ instruct cmov_bns_less(flagsReg crx) %{
ins_variable_size_depending_on_alignment(true);
format %{ "cmov $crx" %}
- // Worst case is branch + move + stop, no stop without scheduler.
size(12);
ins_encode %{
Label done;
diff --git a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
index 4eb2028f529..5260ed978ff 100644
--- a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
+++ b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
@@ -1,6 +1,6 @@
/*
* Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -775,7 +775,6 @@ int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
return stk;
}
-#if defined(COMPILER1) || defined(COMPILER2)
// Calling convention for calling C code.
int SharedRuntime::c_calling_convention(const BasicType *sig_bt,
VMRegPair *regs,
@@ -913,7 +912,6 @@ int SharedRuntime::c_calling_convention(const BasicType *sig_bt,
return MAX2(arg, 8) * 2 + additional_frame_header_slots;
#endif
}
-#endif // COMPILER2
int SharedRuntime::vector_calling_convention(VMRegPair *regs,
uint num_bits,
@@ -2874,7 +2872,6 @@ void SharedRuntime::generate_deopt_blob() {
CodeBuffer buffer(name, 2048, 1024);
InterpreterMacroAssembler* masm = new InterpreterMacroAssembler(&buffer);
Label exec_mode_initialized;
- int frame_size_in_words;
OopMap* map = nullptr;
OopMapSet *oop_maps = new OopMapSet();
@@ -2886,6 +2883,9 @@ void SharedRuntime::generate_deopt_blob() {
const Register exec_mode_reg = R21_tmp1;
const address start = __ pc();
+ int exception_offset = 0;
+ int exception_in_tls_offset = 0;
+ int reexecute_offset = 0;
#if defined(COMPILER1) || defined(COMPILER2)
// --------------------------------------------------------------------------
@@ -2925,7 +2925,7 @@ void SharedRuntime::generate_deopt_blob() {
// - R3_ARG1: exception oop
// - R4_ARG2: exception pc
- int exception_offset = __ pc() - start;
+ exception_offset = __ pc() - start;
BLOCK_COMMENT("Prolog for exception case");
@@ -2936,7 +2936,7 @@ void SharedRuntime::generate_deopt_blob() {
__ std(R4_ARG2, _abi0(lr), R1_SP);
// Vanilla deoptimization with an exception pending in exception_oop.
- int exception_in_tls_offset = __ pc() - start;
+ exception_in_tls_offset = __ pc() - start;
// Push the "unpack frame".
// Save everything in sight.
@@ -2949,8 +2949,6 @@ void SharedRuntime::generate_deopt_blob() {
__ li(exec_mode_reg, Deoptimization::Unpack_exception);
// fall through
-
- int reexecute_offset = 0;
#ifdef COMPILER1
__ b(exec_mode_initialized);
@@ -3068,11 +3066,12 @@ void SharedRuntime::generate_deopt_blob() {
// Return to the interpreter entry point.
__ blr();
- __ flush();
-#else // COMPILER2
+#else // !defined(COMPILER1) && !defined(COMPILER2)
__ unimplemented("deopt blob needed only with compiler");
- int exception_offset = __ pc() - start;
-#endif // COMPILER2
+#endif
+
+ // Make sure all code is generated
+ __ flush();
_deopt_blob = DeoptimizationBlob::create(&buffer, oop_maps, 0, exception_offset,
reexecute_offset, first_frame_size_in_bytes / wordSize);
diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp
index e77a2067e89..63e2fd015d7 100644
--- a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp
@@ -1041,31 +1041,10 @@ void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
__ bind(*op->stub()->continuation());
}
-void LIR_Assembler::type_profile_helper(Register mdo, ciMethodData *md, ciProfileData *data,
- Register recv, Label* update_done) {
- for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
- Label next_test;
- // See if the receiver is receiver[n].
- __ ld(t1, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
- __ bne(recv, t1, next_test);
- Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
- __ increment(data_addr, DataLayout::counter_increment);
- __ j(*update_done);
- __ bind(next_test);
- }
-
- // Didn't find receiver; find next empty slot and fill it in
- for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
- Label next_test;
- Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
- __ ld(t1, recv_addr);
- __ bnez(t1, next_test);
- __ sd(recv, recv_addr);
- __ mv(t1, DataLayout::counter_increment);
- __ sd(t1, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
- __ j(*update_done);
- __ bind(next_test);
- }
+void LIR_Assembler::type_profile_helper(Register mdo, ciMethodData *md,
+ ciProfileData *data, Register recv) {
+ int mdp_offset = md->byte_offset_of_slot(data, in_ByteSize(0));
+ __ profile_receiver_type(recv, mdo, mdp_offset);
}
void LIR_Assembler::data_check(LIR_OpTypeCheck *op, ciMethodData **md, ciProfileData **data) {
@@ -1139,14 +1118,9 @@ void LIR_Assembler::profile_object(ciMethodData* md, ciProfileData* data, Regist
__ j(*obj_is_null);
__ bind(not_null);
- Label update_done;
Register recv = k_RInfo;
__ load_klass(recv, obj);
- type_profile_helper(mdo, md, data, recv, &update_done);
- Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
- __ increment(counter_addr, DataLayout::counter_increment);
-
- __ bind(update_done);
+ type_profile_helper(mdo, md, data, recv);
}
void LIR_Assembler::typecheck_loaded(LIR_OpTypeCheck *op, ciKlass* k, Register k_RInfo) {
@@ -1554,11 +1528,8 @@ void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
// We know the type that will be seen at this call site; we can
// statically update the MethodData* rather than needing to do
// dynamic tests on the receiver type
- // NOTE: we should probably put a lock around this search to
- // avoid collisions by concurrent compilations
ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
- uint i;
- for (i = 0; i < VirtualCallData::row_limit(); i++) {
+ for (uint i = 0; i < VirtualCallData::row_limit(); i++) {
ciKlass* receiver = vc_data->receiver(i);
if (known_klass->equals(receiver)) {
Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
@@ -1566,32 +1537,13 @@ void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
return;
}
}
-
- // Receiver type not found in profile data; select an empty slot
- // Note that this is less efficient than it should be because it
- // always does a write to the receiver part of the
- // VirtualCallData rather than just the first time
- for (i = 0; i < VirtualCallData::row_limit(); i++) {
- ciKlass* receiver = vc_data->receiver(i);
- if (receiver == nullptr) {
- Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
- __ mov_metadata(t1, known_klass->constant_encoding());
- __ sd(t1, recv_addr);
- Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
- __ increment(data_addr, DataLayout::counter_increment);
- return;
- }
- }
+ // Receiver type is not found in profile data.
+ // Fall back to runtime helper to handle the rest at runtime.
+ __ mov_metadata(recv, known_klass->constant_encoding());
} else {
__ load_klass(recv, recv);
- Label update_done;
- type_profile_helper(mdo, md, data, recv, &update_done);
- // Receiver did not match any saved receiver and there is no empty row for it.
- // Increment total counter to indicate polymorphic case.
- __ increment(counter_addr, DataLayout::counter_increment);
-
- __ bind(update_done);
}
+ type_profile_helper(mdo, md, data, recv);
} else {
// Static call
__ increment(counter_addr, DataLayout::counter_increment);
diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.hpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.hpp
index 1e466e90d37..90b6b3ee4f4 100644
--- a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.hpp
@@ -54,9 +54,8 @@ private:
Address stack_slot_address(int index, uint shift, int adjust = 0);
// Record the type of the receiver in ReceiverTypeData
- void type_profile_helper(Register mdo,
- ciMethodData *md, ciProfileData *data,
- Register recv, Label* update_done);
+ void type_profile_helper(Register mdo, ciMethodData *md,
+ ciProfileData *data, Register recv);
void casw(Register addr, Register newval, Register cmpval);
void caswu(Register addr, Register newval, Register cmpval);
diff --git a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp
index 824ea872935..b4e0ba69042 100644
--- a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -2813,10 +2813,14 @@ void C2_MacroAssembler::char_array_compress_v(Register src, Register dst, Regist
// Intrinsic for
//
-// - sun/nio/cs/ISO_8859_1$Encoder.implEncodeISOArray
-// return the number of characters copied.
-// - java/lang/StringUTF16.compress
-// return index of non-latin1 character if copy fails, otherwise 'len'.
+// - sun.nio.cs.ISO_8859_1.Encoder#encodeISOArray0(byte[] sa, int sp, byte[] da, int dp, int len)
+// Encodes char[] to byte[] in ISO-8859-1
+//
+// - java.lang.StringCoding#encodeISOArray0(byte[] sa, int sp, byte[] da, int dp, int len)
+// Encodes byte[] (containing UTF-16) to byte[] in ISO-8859-1
+//
+// - java.lang.StringCoding#encodeAsciiArray0(char[] sa, int sp, byte[] da, int dp, int len)
+// Encodes char[] to byte[] in ASCII
//
// This version always returns the number of characters copied. A successful
// copy will complete with the post-condition: 'res' == 'len', while an
diff --git a/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.cpp b/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.cpp
index dd6c8556307..3cbbb783258 100644
--- a/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.cpp
@@ -88,26 +88,16 @@ void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, Dec
}
}
-void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
- Register obj,
- Register pre_val,
- Register thread,
- Register tmp,
- bool tosca_live,
- bool expand_call) {
- if (ShenandoahSATBBarrier) {
- satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, t0, tosca_live, expand_call);
- }
-}
+void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler* masm,
+ Register obj,
+ Register pre_val,
+ Register thread,
+ Register tmp1,
+ Register tmp2,
+ bool tosca_live,
+ bool expand_call) {
+ assert(ShenandoahSATBBarrier, "Should be checked by caller");
-void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
- Register obj,
- Register pre_val,
- Register thread,
- Register tmp1,
- Register tmp2,
- bool tosca_live,
- bool expand_call) {
// If expand_call is true then we expand the call_VM_leaf macro
// directly to skip generating the check by
// InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
@@ -376,21 +366,21 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm,
if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
__ enter();
__ push_call_clobbered_registers();
- satb_write_barrier_pre(masm /* masm */,
- noreg /* obj */,
- dst /* pre_val */,
- xthread /* thread */,
- tmp1 /* tmp1 */,
- tmp2 /* tmp2 */,
- true /* tosca_live */,
- true /* expand_call */);
+ satb_barrier(masm /* masm */,
+ noreg /* obj */,
+ dst /* pre_val */,
+ xthread /* thread */,
+ tmp1 /* tmp1 */,
+ tmp2 /* tmp2 */,
+ true /* tosca_live */,
+ true /* expand_call */);
__ pop_call_clobbered_registers();
__ leave();
}
}
-void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj) {
- assert(ShenandoahCardBarrier, "Did you mean to enable ShenandoahCardBarrier?");
+void ShenandoahBarrierSetAssembler::card_barrier(MacroAssembler* masm, Register obj) {
+ assert(ShenandoahCardBarrier, "Should have been checked by caller");
__ srli(obj, obj, CardTable::card_shift());
@@ -413,13 +403,13 @@ void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register o
void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
- bool on_oop = is_reference_type(type);
- if (!on_oop) {
+ // 1: non-reference types require no barriers
+ if (!is_reference_type(type)) {
BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
return;
}
- // flatten object address if needed
+ // Flatten object address right away for simplicity: likely needed by barriers
if (dst.offset() == 0) {
if (dst.base() != tmp3) {
__ mv(tmp3, dst.base());
@@ -428,20 +418,26 @@ void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet
__ la(tmp3, dst);
}
- shenandoah_write_barrier_pre(masm,
- tmp3 /* obj */,
- tmp2 /* pre_val */,
- xthread /* thread */,
- tmp1 /* tmp */,
- val != noreg /* tosca_live */,
- false /* expand_call */);
+ bool storing_non_null = (val != noreg);
+ // 2: pre-barrier: SATB needs the previous value
+ if (ShenandoahBarrierSet::need_satb_barrier(decorators, type)) {
+ satb_barrier(masm,
+ tmp3 /* obj */,
+ tmp2 /* pre_val */,
+ xthread /* thread */,
+ tmp1 /* tmp */,
+ t0 /* tmp2 */,
+ storing_non_null /* tosca_live */,
+ false /* expand_call */);
+ }
+
+ // Store!
BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
- bool in_heap = (decorators & IN_HEAP) != 0;
- bool needs_post_barrier = (val != noreg) && in_heap && ShenandoahCardBarrier;
- if (needs_post_barrier) {
- store_check(masm, tmp3);
+ // 3: post-barrier: card barrier needs store address
+ if (ShenandoahBarrierSet::need_card_barrier(decorators, type) && storing_non_null) {
+ card_barrier(masm, tmp3);
}
}
diff --git a/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.hpp b/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.hpp
index c8a7c35fb83..5085be26b2e 100644
--- a/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.hpp
@@ -41,23 +41,16 @@ class StubCodeGenerator;
class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
private:
- void satb_write_barrier_pre(MacroAssembler* masm,
- Register obj,
- Register pre_val,
- Register thread,
- Register tmp1,
- Register tmp2,
- bool tosca_live,
- bool expand_call);
- void shenandoah_write_barrier_pre(MacroAssembler* masm,
- Register obj,
- Register pre_val,
- Register thread,
- Register tmp,
- bool tosca_live,
- bool expand_call);
+ void satb_barrier(MacroAssembler* masm,
+ Register obj,
+ Register pre_val,
+ Register thread,
+ Register tmp1,
+ Register tmp2,
+ bool tosca_live,
+ bool expand_call);
- void store_check(MacroAssembler* masm, Register obj);
+ void card_barrier(MacroAssembler* masm, Register obj);
void resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp = noreg);
void resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp = noreg);
diff --git a/src/hotspot/cpu/riscv/icache_riscv.cpp b/src/hotspot/cpu/riscv/icache_riscv.cpp
index 258bc665770..20de2dbb2ad 100644
--- a/src/hotspot/cpu/riscv/icache_riscv.cpp
+++ b/src/hotspot/cpu/riscv/icache_riscv.cpp
@@ -39,7 +39,8 @@ static int icache_flush(address addr, int lines, int magic) {
// We need to make sure stores happens before the I/D cache synchronization.
__asm__ volatile("fence rw, rw" : : : "memory");
- RiscvFlushIcache::flush((uintptr_t)addr, ((uintptr_t)lines) << ICache::log2_line_size);
+ uintptr_t end = (uintptr_t)addr + ((uintptr_t)lines << ICache::log2_line_size);
+ RiscvFlushIcache::flush((uintptr_t)addr, end);
return magic;
}
diff --git a/src/hotspot/cpu/riscv/interp_masm_riscv.cpp b/src/hotspot/cpu/riscv/interp_masm_riscv.cpp
index 189c7c93d07..744590bec2b 100644
--- a/src/hotspot/cpu/riscv/interp_masm_riscv.cpp
+++ b/src/hotspot/cpu/riscv/interp_masm_riscv.cpp
@@ -237,15 +237,14 @@ void InterpreterMacroAssembler::load_resolved_klass_at_offset(
// Rsub_klass: subklass
//
// Kills:
-// x12, x15
+// x12
void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass,
Label& ok_is_subtype) {
assert(Rsub_klass != x10, "x10 holds superklass");
assert(Rsub_klass != x12, "x12 holds 2ndary super array length");
- assert(Rsub_klass != x15, "x15 holds 2ndary super array scan ptr");
// Profile the not-null value's klass.
- profile_typecheck(x12, Rsub_klass, x15); // blows x12, reloads x15
+ profile_typecheck(x12, Rsub_klass); // blows x12
// Do the check.
check_klass_subtype(Rsub_klass, x10, x12, ok_is_subtype); // blows x12
@@ -1042,7 +1041,6 @@ void InterpreterMacroAssembler::profile_final_call(Register mdp) {
void InterpreterMacroAssembler::profile_virtual_call(Register receiver,
Register mdp,
- Register reg2,
bool receiver_can_be_null) {
if (ProfileInterpreter) {
Label profile_continue;
@@ -1060,7 +1058,7 @@ void InterpreterMacroAssembler::profile_virtual_call(Register receiver,
}
// Record the receiver type.
- record_klass_in_profile(receiver, mdp, reg2);
+ profile_receiver_type(receiver, mdp, 0);
bind(skip_receiver_profile);
// The method data pointer needs to be updated to reflect the new target.
@@ -1072,153 +1070,6 @@ void InterpreterMacroAssembler::profile_virtual_call(Register receiver,
}
}
-// This routine creates a state machine for updating the multi-row
-// type profile at a virtual call site (or other type-sensitive bytecode).
-// The machine visits each row (of receiver/count) until the receiver type
-// is found, or until it runs out of rows. At the same time, it remembers
-// the location of the first empty row. (An empty row records null for its
-// receiver, and can be allocated for a newly-observed receiver type.)
-// Because there are two degrees of freedom in the state, a simple linear
-// search will not work; it must be a decision tree. Hence this helper
-// function is recursive, to generate the required tree structured code.
-// It's the interpreter, so we are trading off code space for speed.
-// See below for example code.
-void InterpreterMacroAssembler::record_klass_in_profile_helper(
- Register receiver, Register mdp,
- Register reg2, Label& done) {
- if (TypeProfileWidth == 0) {
- increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
- } else {
- record_item_in_profile_helper(receiver, mdp, reg2, 0, done, TypeProfileWidth,
- &VirtualCallData::receiver_offset, &VirtualCallData::receiver_count_offset);
- }
-}
-
-void InterpreterMacroAssembler::record_item_in_profile_helper(Register item, Register mdp,
- Register reg2, int start_row, Label& done, int total_rows,
- OffsetFunction item_offset_fn, OffsetFunction item_count_offset_fn) {
- int last_row = total_rows - 1;
- assert(start_row <= last_row, "must be work left to do");
- // Test this row for both the item and for null.
- // Take any of three different outcomes:
- // 1. found item => increment count and goto done
- // 2. found null => keep looking for case 1, maybe allocate this cell
- // 3. found something else => keep looking for cases 1 and 2
- // Case 3 is handled by a recursive call.
- for (int row = start_row; row <= last_row; row++) {
- Label next_test;
- bool test_for_null_also = (row == start_row);
-
- // See if the item is item[n].
- int item_offset = in_bytes(item_offset_fn(row));
- test_mdp_data_at(mdp, item_offset, item,
- (test_for_null_also ? reg2 : noreg),
- next_test);
- // (Reg2 now contains the item from the CallData.)
-
- // The item is item[n]. Increment count[n].
- int count_offset = in_bytes(item_count_offset_fn(row));
- increment_mdp_data_at(mdp, count_offset);
- j(done);
- bind(next_test);
-
- if (test_for_null_also) {
- Label found_null;
- // Failed the equality check on item[n]... Test for null.
- if (start_row == last_row) {
- // The only thing left to do is handle the null case.
- beqz(reg2, found_null);
- // Item did not match any saved item and there is no empty row for it.
- // Increment total counter to indicate polymorphic case.
- increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
- j(done);
- bind(found_null);
- break;
- }
- // Since null is rare, make it be the branch-taken case.
- beqz(reg2, found_null);
-
- // Put all the "Case 3" tests here.
- record_item_in_profile_helper(item, mdp, reg2, start_row + 1, done, total_rows,
- item_offset_fn, item_count_offset_fn);
-
- // Found a null. Keep searching for a matching item,
- // but remember that this is an empty (unused) slot.
- bind(found_null);
- }
- }
-
- // In the fall-through case, we found no matching item, but we
- // observed the item[start_row] is null.
- // Fill in the item field and increment the count.
- int item_offset = in_bytes(item_offset_fn(start_row));
- set_mdp_data_at(mdp, item_offset, item);
- int count_offset = in_bytes(item_count_offset_fn(start_row));
- mv(reg2, DataLayout::counter_increment);
- set_mdp_data_at(mdp, count_offset, reg2);
- if (start_row > 0) {
- j(done);
- }
-}
-
-// Example state machine code for three profile rows:
-// # main copy of decision tree, rooted at row[1]
-// if (row[0].rec == rec) then [
-// row[0].incr()
-// goto done
-// ]
-// if (row[0].rec != nullptr) then [
-// # inner copy of decision tree, rooted at row[1]
-// if (row[1].rec == rec) then [
-// row[1].incr()
-// goto done
-// ]
-// if (row[1].rec != nullptr) then [
-// # degenerate decision tree, rooted at row[2]
-// if (row[2].rec == rec) then [
-// row[2].incr()
-// goto done
-// ]
-// if (row[2].rec != nullptr) then [
-// count.incr()
-// goto done
-// ] # overflow
-// row[2].init(rec)
-// goto done
-// ] else [
-// # remember row[1] is empty
-// if (row[2].rec == rec) then [
-// row[2].incr()
-// goto done
-// ]
-// row[1].init(rec)
-// goto done
-// ]
-// else [
-// # remember row[0] is empty
-// if (row[1].rec == rec) then [
-// row[1].incr()
-// goto done
-// ]
-// if (row[2].rec == rec) then [
-// row[2].incr()
-// goto done
-// ]
-// row[0].init(rec)
-// goto done
-// ]
-// done:
-
-void InterpreterMacroAssembler::record_klass_in_profile(Register receiver,
- Register mdp, Register reg2) {
- assert(ProfileInterpreter, "must be profiling");
- Label done;
-
- record_klass_in_profile_helper(receiver, mdp, reg2, done);
-
- bind(done);
-}
-
void InterpreterMacroAssembler::profile_ret(Register return_bci, Register mdp) {
if (ProfileInterpreter) {
Label profile_continue;
@@ -1274,7 +1125,7 @@ void InterpreterMacroAssembler::profile_null_seen(Register mdp) {
}
}
-void InterpreterMacroAssembler::profile_typecheck(Register mdp, Register klass, Register reg2) {
+void InterpreterMacroAssembler::profile_typecheck(Register mdp, Register klass) {
if (ProfileInterpreter) {
Label profile_continue;
@@ -1287,7 +1138,7 @@ void InterpreterMacroAssembler::profile_typecheck(Register mdp, Register klass,
mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size());
// Record the object type.
- record_klass_in_profile(klass, mdp, reg2);
+ profile_receiver_type(klass, mdp, 0);
}
update_mdp_by_constant(mdp, mdp_delta);
diff --git a/src/hotspot/cpu/riscv/interp_masm_riscv.hpp b/src/hotspot/cpu/riscv/interp_masm_riscv.hpp
index a9df09d656a..59cc76b022f 100644
--- a/src/hotspot/cpu/riscv/interp_masm_riscv.hpp
+++ b/src/hotspot/cpu/riscv/interp_masm_riscv.hpp
@@ -262,14 +262,6 @@ class InterpreterMacroAssembler: public MacroAssembler {
Register test_value_out,
Label& not_equal_continue);
- void record_klass_in_profile(Register receiver, Register mdp,
- Register reg2);
- void record_klass_in_profile_helper(Register receiver, Register mdp,
- Register reg2, Label& done);
- void record_item_in_profile_helper(Register item, Register mdp,
- Register reg2, int start_row, Label& done, int total_rows,
- OffsetFunction item_offset_fn, OffsetFunction item_count_offset_fn);
-
void update_mdp_by_offset(Register mdp_in, int offset_of_offset);
void update_mdp_by_offset(Register mdp_in, Register reg, int offset_of_disp);
void update_mdp_by_constant(Register mdp_in, int constant);
@@ -283,11 +275,10 @@ class InterpreterMacroAssembler: public MacroAssembler {
void profile_call(Register mdp);
void profile_final_call(Register mdp);
void profile_virtual_call(Register receiver, Register mdp,
- Register t1,
bool receiver_can_be_null = false);
void profile_ret(Register return_bci, Register mdp);
void profile_null_seen(Register mdp);
- void profile_typecheck(Register mdp, Register klass, Register temp);
+ void profile_typecheck(Register mdp, Register klass);
void profile_typecheck_failed(Register mdp);
void profile_switch_default(Register mdp);
void profile_switch_case(Register index_in_scratch, Register mdp,
diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
index fb30f64e9ed..4f5e7afc166 100644
--- a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
@@ -543,6 +543,160 @@ void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file,
BLOCK_COMMENT("} verify_oop");
}
+// Handle the receiver type profile update given the "recv" klass.
+//
+// Normally updates the ReceiverData (RD) that starts at "mdp" + "mdp_offset".
+// If there are no matching or claimable receiver entries in RD, updates
+// the polymorphic counter.
+//
+// This code expected to run by either the interpreter or JIT-ed code, without
+// extra synchronization. For safety, receiver cells are claimed atomically, which
+// avoids grossly misrepresenting the profiles under concurrent updates. For speed,
+// counter updates are not atomic.
+//
+void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_offset) {
+ assert_different_registers(recv, mdp, t0, t1);
+
+ int base_receiver_offset = in_bytes(ReceiverTypeData::receiver_offset(0));
+ int end_receiver_offset = in_bytes(ReceiverTypeData::receiver_offset(ReceiverTypeData::row_limit()));
+ int poly_count_offset = in_bytes(CounterData::count_offset());
+ int receiver_step = in_bytes(ReceiverTypeData::receiver_offset(1)) - base_receiver_offset;
+ int receiver_to_count_step = in_bytes(ReceiverTypeData::receiver_count_offset(0)) - base_receiver_offset;
+
+ // Adjust for MDP offsets. Slots are pointer-sized, so is the global offset.
+ base_receiver_offset += mdp_offset;
+ end_receiver_offset += mdp_offset;
+ poly_count_offset += mdp_offset;
+
+#ifdef ASSERT
+ // We are about to walk the MDO slots without asking for offsets.
+ // Check that our math hits all the right spots.
+ for (uint c = 0; c < ReceiverTypeData::row_limit(); c++) {
+ int real_recv_offset = mdp_offset + in_bytes(ReceiverTypeData::receiver_offset(c));
+ int real_count_offset = mdp_offset + in_bytes(ReceiverTypeData::receiver_count_offset(c));
+ int offset = base_receiver_offset + receiver_step*c;
+ int count_offset = offset + receiver_to_count_step;
+ assert(offset == real_recv_offset, "receiver slot math");
+ assert(count_offset == real_count_offset, "receiver count math");
+ }
+ int real_poly_count_offset = mdp_offset + in_bytes(CounterData::count_offset());
+ assert(poly_count_offset == real_poly_count_offset, "poly counter math");
+#endif
+
+ // Corner case: no profile table. Increment poly counter and exit.
+ if (ReceiverTypeData::row_limit() == 0) {
+ increment(Address(mdp, poly_count_offset), DataLayout::counter_increment);
+ return;
+ }
+
+ Register offset = t1;
+
+ Label L_loop_search_receiver, L_loop_search_empty;
+ Label L_restart, L_found_recv, L_found_empty, L_polymorphic, L_count_update;
+
+ // The code here recognizes three major cases:
+ // A. Fastest: receiver found in the table
+ // B. Fast: no receiver in the table, and the table is full
+ // C. Slow: no receiver in the table, free slots in the table
+ //
+ // The case A performance is most important, as perfectly-behaved code would end up
+ // there, especially with larger TypeProfileWidth. The case B performance is
+ // important as well, this is where bulk of code would land for normally megamorphic
+ // cases. The case C performance is not essential, its job is to deal with installation
+ // races, we optimize for code density instead. Case C needs to make sure that receiver
+ // rows are only claimed once. This makes sure we never overwrite a row for another
+ // receiver and never duplicate the receivers in the list, making profile type-accurate.
+ //
+ // It is very tempting to handle these cases in a single loop, and claim the first slot
+ // without checking the rest of the table. But, profiling code should tolerate free slots
+ // in the table, as class unloading can clear them. After such cleanup, the receiver
+ // we need might be _after_ the free slot. Therefore, we need to let at least full scan
+ // to complete, before trying to install new slots. Splitting the code in several tight
+ // loops also helpfully optimizes for cases A and B.
+ //
+ // This code is effectively:
+ //
+ // restart:
+ // // Fastest: receiver is already installed
+ // for (i = 0; i < receiver_count(); i++) {
+ // if (receiver(i) == recv) goto found_recv(i);
+ // }
+ //
+ // // Fast: no receiver, but profile is full
+ // for (i = 0; i < receiver_count(); i++) {
+ // if (receiver(i) == null) goto found_null(i);
+ // }
+ // goto polymorphic
+ //
+ // // Slow: try to install receiver
+ // found_null(i):
+ // CAS(&receiver(i), null, recv);
+ // goto restart
+ //
+ // polymorphic:
+ // count++;
+ // return
+ //
+ // found_recv(i):
+ // *receiver_count(i)++
+ //
+
+ bind(L_restart);
+
+ // Fastest: receiver is already installed
+ mv(offset, base_receiver_offset);
+ bind(L_loop_search_receiver);
+ add(t0, mdp, offset);
+ ld(t0, Address(t0));
+ beq(recv, t0, L_found_recv);
+ add(offset, offset, receiver_step);
+ sub(t0, offset, end_receiver_offset);
+ bnez(t0, L_loop_search_receiver);
+
+ // Fast: no receiver, but profile is full
+ mv(offset, base_receiver_offset);
+ bind(L_loop_search_empty);
+ add(t0, mdp, offset);
+ ld(t0, Address(t0));
+ beqz(t0, L_found_empty);
+ add(offset, offset, receiver_step);
+ sub(t0, offset, end_receiver_offset);
+ bnez(t0, L_loop_search_empty);
+ j(L_polymorphic);
+
+ // Slow: try to install receiver
+ bind(L_found_empty);
+
+ // Atomically swing receiver slot: null -> recv.
+ //
+ // The update uses CAS, which clobbers t0. Therefore, t1
+ // is used to hold the destination address. This is safe because the
+ // offset is no longer needed after the address is computed.
+ add(t1, mdp, offset);
+ weak_cmpxchg(/*addr*/ t1, /*expected*/ zr, /*new*/ recv, Assembler::int64,
+ /*acquire*/ Assembler::relaxed, /*release*/ Assembler::relaxed, /*result*/ t0);
+
+ // CAS success means the slot now has the receiver we want. CAS failure means
+ // something had claimed the slot concurrently: it can be the same receiver we want,
+ // or something else. Since this is a slow path, we can optimize for code density,
+ // and just restart the search from the beginning.
+ j(L_restart);
+
+ // Counter updates:
+ // Increment polymorphic counter instead of receiver slot.
+ bind(L_polymorphic);
+ mv(offset, poly_count_offset);
+ j(L_count_update);
+
+ // Found a receiver, convert its slot offset to corresponding count offset.
+ bind(L_found_recv);
+ add(offset, offset, receiver_to_count_step);
+
+ bind(L_count_update);
+ add(t1, mdp, offset);
+ increment(Address(t1), DataLayout::counter_increment);
+}
+
void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) {
if (!VerifyOops) {
return;
diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
index 3b021388fa5..f5e985c28a2 100644
--- a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
@@ -390,6 +390,8 @@ class MacroAssembler: public Assembler {
Address argument_address(RegisterOrConstant arg_slot, int extra_slot_offset = 0);
+ void profile_receiver_type(Register recv, Register mdp, int mdp_offset);
+
// only if +VerifyOops
void _verify_oop(Register reg, const char* s, const char* file, int line);
void _verify_oop_addr(Address addr, const char* s, const char* file, int line);
diff --git a/src/hotspot/cpu/riscv/templateTable_riscv.cpp b/src/hotspot/cpu/riscv/templateTable_riscv.cpp
index 5a3644f70bb..5cc725e3af4 100644
--- a/src/hotspot/cpu/riscv/templateTable_riscv.cpp
+++ b/src/hotspot/cpu/riscv/templateTable_riscv.cpp
@@ -708,7 +708,6 @@ void TemplateTable::index_check(Register array, Register index) {
__ mv(x11, index);
}
Label ok;
- __ sext(index, index, 32);
__ bltu(index, length, ok);
__ mv(x13, array);
__ mv(t1, Interpreter::_throw_ArrayIndexOutOfBoundsException_entry);
@@ -1052,7 +1051,7 @@ void TemplateTable::aastore() {
transition(vtos, vtos);
// stack: ..., array, index, value
__ ld(x10, at_tos()); // value
- __ ld(x12, at_tos_p1()); // index
+ __ lw(x12, at_tos_p1()); // index
__ ld(x13, at_tos_p2()); // array
index_check(x13, x12); // kills x11
@@ -1462,9 +1461,9 @@ void TemplateTable::iinc() {
transition(vtos, vtos);
__ load_signed_byte(x11, at_bcp(2)); // get constant
locals_index(x12);
- __ ld(x10, iaddress(x12, x10, _masm));
+ __ lw(x10, iaddress(x12, x10, _masm));
__ addw(x10, x10, x11);
- __ sd(x10, iaddress(x12, t0, _masm));
+ __ sw(x10, iaddress(x12, t0, _masm));
}
void TemplateTable::wide_iinc() {
@@ -1477,9 +1476,9 @@ void TemplateTable::wide_iinc() {
__ orr(x11, x11, t1);
locals_index_wide(x12);
- __ ld(x10, iaddress(x12, t0, _masm));
+ __ lw(x10, iaddress(x12, t0, _masm));
__ addw(x10, x10, x11);
- __ sd(x10, iaddress(x12, t0, _masm));
+ __ sw(x10, iaddress(x12, t0, _masm));
}
void TemplateTable::convert() {
@@ -3280,7 +3279,7 @@ void TemplateTable::invokevirtual_helper(Register index,
__ load_klass(x10, recv);
// profile this call
- __ profile_virtual_call(x10, xlocals, x13);
+ __ profile_virtual_call(x10, xlocals);
// get target Method & entry point
__ lookup_virtual_method(x10, index, method);
@@ -3407,7 +3406,7 @@ void TemplateTable::invokeinterface(int byte_no) {
/*return_method=*/false);
// profile this call
- __ profile_virtual_call(x13, x30, x9);
+ __ profile_virtual_call(x13, x30);
// Get declaring interface class from method, and itable index
__ load_method_holder(x10, xmethod);
diff --git a/src/hotspot/cpu/x86/assembler_x86.cpp b/src/hotspot/cpu/x86/assembler_x86.cpp
index cbc5c6988d4..3c8defe62d9 100644
--- a/src/hotspot/cpu/x86/assembler_x86.cpp
+++ b/src/hotspot/cpu/x86/assembler_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -7320,6 +7320,25 @@ void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
emit_int16(0x2E, (0xC0 | encode));
}
+void Assembler::vucomxsd(XMMRegister dst, Address src) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
+ attributes.set_is_evex_instruction();
+ vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
+ emit_int8(0x2E);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::vucomxsd(XMMRegister dst, XMMRegister src) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ attributes.set_is_evex_instruction();
+ int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
+ emit_int16(0x2E, (0xC0 | encode));
+}
+
void Assembler::ucomiss(XMMRegister dst, Address src) {
InstructionMark im(this);
InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
@@ -7335,6 +7354,25 @@ void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
emit_int16(0x2E, (0xC0 | encode));
}
+void Assembler::vucomxss(XMMRegister dst, Address src) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
+ attributes.set_is_evex_instruction();
+ vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
+ emit_int8(0x2E);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::vucomxss(XMMRegister dst, XMMRegister src) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ attributes.set_is_evex_instruction();
+ int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
+ emit_int16(0x2E, (0xC0 | encode));
+}
+
void Assembler::xabort(int8_t imm8) {
emit_int24((unsigned char)0xC6, (unsigned char)0xF8, (imm8 & 0xFF));
}
diff --git a/src/hotspot/cpu/x86/assembler_x86.hpp b/src/hotspot/cpu/x86/assembler_x86.hpp
index 26c57fc2d80..97854f712cf 100644
--- a/src/hotspot/cpu/x86/assembler_x86.hpp
+++ b/src/hotspot/cpu/x86/assembler_x86.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -2331,10 +2331,14 @@ private:
// Unordered Compare Scalar Double-Precision Floating-Point Values and set EFLAGS
void ucomisd(XMMRegister dst, Address src);
void ucomisd(XMMRegister dst, XMMRegister src);
+ void vucomxsd(XMMRegister dst, Address src);
+ void vucomxsd(XMMRegister dst, XMMRegister src);
// Unordered Compare Scalar Single-Precision Floating-Point Values and set EFLAGS
void ucomiss(XMMRegister dst, Address src);
void ucomiss(XMMRegister dst, XMMRegister src);
+ void vucomxss(XMMRegister dst, Address src);
+ void vucomxss(XMMRegister dst, XMMRegister src);
void xabort(int8_t imm8);
diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
index 8fc3d18abb1..c65b439604b 100644
--- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1046,17 +1046,28 @@ void C2_MacroAssembler::signum_fp(int opcode, XMMRegister dst, XMMRegister zero,
Label DONE_LABEL;
+ // Handle special cases +0.0/-0.0 and NaN, if argument is +0.0/-0.0 or NaN, return argument
+ // If AVX10.2 (or newer) floating point comparison instructions used, SF=1 for equal and unordered cases
+ // If other floating point comparison instructions used, ZF=1 for equal and unordered cases
if (opcode == Op_SignumF) {
- ucomiss(dst, zero);
- jcc(Assembler::equal, DONE_LABEL); // handle special case +0.0/-0.0, if argument is +0.0/-0.0, return argument
- jcc(Assembler::parity, DONE_LABEL); // handle special case NaN, if argument NaN, return NaN
+ if (VM_Version::supports_avx10_2()) {
+ vucomxss(dst, zero);
+ jcc(Assembler::negative, DONE_LABEL);
+ } else {
+ ucomiss(dst, zero);
+ jcc(Assembler::equal, DONE_LABEL);
+ }
movflt(dst, one);
jcc(Assembler::above, DONE_LABEL);
xorps(dst, ExternalAddress(StubRoutines::x86::vector_float_sign_flip()), noreg);
} else if (opcode == Op_SignumD) {
- ucomisd(dst, zero);
- jcc(Assembler::equal, DONE_LABEL); // handle special case +0.0/-0.0, if argument is +0.0/-0.0, return argument
- jcc(Assembler::parity, DONE_LABEL); // handle special case NaN, if argument NaN, return NaN
+ if (VM_Version::supports_avx10_2()) {
+ vucomxsd(dst, zero);
+ jcc(Assembler::negative, DONE_LABEL);
+ } else {
+ ucomisd(dst, zero);
+ jcc(Assembler::equal, DONE_LABEL);
+ }
movdbl(dst, one);
jcc(Assembler::above, DONE_LABEL);
xorpd(dst, ExternalAddress(StubRoutines::x86::vector_double_sign_flip()), noreg);
diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp
index 9e321391f6c..97829a10a3b 100644
--- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp
@@ -174,24 +174,14 @@ void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, Dec
}
}
-void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
- Register obj,
- Register pre_val,
- Register tmp,
- bool tosca_live,
- bool expand_call) {
+void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler* masm,
+ Register obj,
+ Register pre_val,
+ Register tmp,
+ bool tosca_live,
+ bool expand_call) {
+ assert(ShenandoahSATBBarrier, "Should be checked by caller");
- if (ShenandoahSATBBarrier) {
- satb_write_barrier_pre(masm, obj, pre_val, tmp, tosca_live, expand_call);
- }
-}
-
-void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
- Register obj,
- Register pre_val,
- Register tmp,
- bool tosca_live,
- bool expand_call) {
// If expand_call is true then we expand the call_VM_leaf macro
// directly to skip generating the check by
// InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
@@ -533,18 +523,18 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet d
assert_different_registers(dst, tmp1, r15_thread);
// Generate the SATB pre-barrier code to log the value of
// the referent field in an SATB buffer.
- shenandoah_write_barrier_pre(masm /* masm */,
- noreg /* obj */,
- dst /* pre_val */,
- tmp1 /* tmp */,
- true /* tosca_live */,
- true /* expand_call */);
+ satb_barrier(masm /* masm */,
+ noreg /* obj */,
+ dst /* pre_val */,
+ tmp1 /* tmp */,
+ true /* tosca_live */,
+ true /* expand_call */);
restore_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ true);
}
}
-void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj) {
+void ShenandoahBarrierSetAssembler::card_barrier(MacroAssembler* masm, Register obj) {
assert(ShenandoahCardBarrier, "Should have been checked by caller");
// Does a store check for the oop in register obj. The content of
@@ -575,41 +565,40 @@ void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register o
void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
- bool on_oop = is_reference_type(type);
- bool in_heap = (decorators & IN_HEAP) != 0;
- bool as_normal = (decorators & AS_NORMAL) != 0;
- if (on_oop && in_heap) {
- bool needs_pre_barrier = as_normal;
+ // 1: non-reference types require no barriers
+ if (!is_reference_type(type)) {
+ BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
+ return;
+ }
- // flatten object address if needed
- // We do it regardless of precise because we need the registers
- if (dst.index() == noreg && dst.disp() == 0) {
- if (dst.base() != tmp1) {
- __ movptr(tmp1, dst.base());
- }
- } else {
- __ lea(tmp1, dst);
- }
-
- assert_different_registers(val, tmp1, tmp2, tmp3, r15_thread);
-
- if (needs_pre_barrier) {
- shenandoah_write_barrier_pre(masm /*masm*/,
- tmp1 /* obj */,
- tmp2 /* pre_val */,
- tmp3 /* tmp */,
- val != noreg /* tosca_live */,
- false /* expand_call */);
- }
-
- BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
- if (val != noreg) {
- if (ShenandoahCardBarrier) {
- store_check(masm, tmp1);
- }
+ // Flatten object address right away for simplicity: likely needed by barriers
+ assert_different_registers(val, tmp1, tmp2, tmp3, r15_thread);
+ if (dst.index() == noreg && dst.disp() == 0) {
+ if (dst.base() != tmp1) {
+ __ movptr(tmp1, dst.base());
}
} else {
- BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
+ __ lea(tmp1, dst);
+ }
+
+ bool storing_non_null = (val != noreg);
+
+ // 2: pre-barrier: SATB needs the previous value
+ if (ShenandoahBarrierSet::need_satb_barrier(decorators, type)) {
+ satb_barrier(masm,
+ tmp1 /* obj */,
+ tmp2 /* pre_val */,
+ tmp3 /* tmp */,
+ storing_non_null /* tosca_live */,
+ false /* expand_call */);
+ }
+
+ // Store!
+ BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
+
+ // 3: post-barrier: card barrier needs store address
+ if (ShenandoahBarrierSet::need_card_barrier(decorators, type) && storing_non_null) {
+ card_barrier(masm, tmp1);
}
}
diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp
index b0185f2dbff..b5cc5c8d834 100644
--- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp
@@ -41,21 +41,14 @@ class StubCodeGenerator;
class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
private:
- void satb_write_barrier_pre(MacroAssembler* masm,
- Register obj,
- Register pre_val,
- Register tmp,
- bool tosca_live,
- bool expand_call);
+ void satb_barrier(MacroAssembler* masm,
+ Register obj,
+ Register pre_val,
+ Register tmp,
+ bool tosca_live,
+ bool expand_call);
- void shenandoah_write_barrier_pre(MacroAssembler* masm,
- Register obj,
- Register pre_val,
- Register tmp,
- bool tosca_live,
- bool expand_call);
-
- void store_check(MacroAssembler* masm, Register obj);
+ void card_barrier(MacroAssembler* masm, Register obj);
void gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
Register addr, Register count,
diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp
index 7f7bb2c4c7f..d4d3ec85bcf 100644
--- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp
@@ -2656,6 +2656,17 @@ void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src, Register rscra
}
}
+void MacroAssembler::vucomxsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
+ assert(rscratch != noreg || always_reachable(src), "missing");
+
+ if (reachable(src)) {
+ Assembler::vucomxsd(dst, as_Address(src));
+ } else {
+ lea(rscratch, src);
+ Assembler::vucomxsd(dst, Address(rscratch, 0));
+ }
+}
+
void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src, Register rscratch) {
assert(rscratch != noreg || always_reachable(src), "missing");
@@ -2667,6 +2678,17 @@ void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src, Register rscra
}
}
+void MacroAssembler::vucomxss(XMMRegister dst, AddressLiteral src, Register rscratch) {
+ assert(rscratch != noreg || always_reachable(src), "missing");
+
+ if (reachable(src)) {
+ Assembler::vucomxss(dst, as_Address(src));
+ } else {
+ lea(rscratch, src);
+ Assembler::vucomxss(dst, Address(rscratch, 0));
+ }
+}
+
void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
assert(rscratch != noreg || always_reachable(src), "missing");
@@ -6251,32 +6273,46 @@ void MacroAssembler::evpbroadcast(BasicType type, XMMRegister dst, Register src,
}
}
-// encode char[] to byte[] in ISO_8859_1 or ASCII
- //@IntrinsicCandidate
- //private static int implEncodeISOArray(byte[] sa, int sp,
- //byte[] da, int dp, int len) {
- // int i = 0;
- // for (; i < len; i++) {
- // char c = StringUTF16.getChar(sa, sp++);
- // if (c > '\u00FF')
- // break;
- // da[dp++] = (byte)c;
- // }
- // return i;
- //}
- //
- //@IntrinsicCandidate
- //private static int implEncodeAsciiArray(char[] sa, int sp,
- // byte[] da, int dp, int len) {
- // int i = 0;
- // for (; i < len; i++) {
- // char c = sa[sp++];
- // if (c >= '\u0080')
- // break;
- // da[dp++] = (byte)c;
- // }
- // return i;
- //}
+// Encode given char[]/byte[] to byte[] in ISO_8859_1 or ASCII
+//
+// @IntrinsicCandidate
+// int sun.nio.cs.ISO_8859_1.Encoder#encodeISOArray0(
+// char[] sa, int sp, byte[] da, int dp, int len) {
+// int i = 0;
+// for (; i < len; i++) {
+// char c = sa[sp++];
+// if (c > '\u00FF')
+// break;
+// da[dp++] = (byte) c;
+// }
+// return i;
+// }
+//
+// @IntrinsicCandidate
+// int java.lang.StringCoding.encodeISOArray0(
+// byte[] sa, int sp, byte[] da, int dp, int len) {
+// int i = 0;
+// for (; i < len; i++) {
+// char c = StringUTF16.getChar(sa, sp++);
+// if (c > '\u00FF')
+// break;
+// da[dp++] = (byte) c;
+// }
+// return i;
+// }
+//
+// @IntrinsicCandidate
+// int java.lang.StringCoding.encodeAsciiArray0(
+// char[] sa, int sp, byte[] da, int dp, int len) {
+// int i = 0;
+// for (; i < len; i++) {
+// char c = sa[sp++];
+// if (c >= '\u0080')
+// break;
+// da[dp++] = (byte) c;
+// }
+// return i;
+// }
void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,
XMMRegister tmp1Reg, XMMRegister tmp2Reg,
XMMRegister tmp3Reg, XMMRegister tmp4Reg,
diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.hpp b/src/hotspot/cpu/x86/macroAssembler_x86.hpp
index 93e3529ac1e..eb23199ca63 100644
--- a/src/hotspot/cpu/x86/macroAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/macroAssembler_x86.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1313,10 +1313,18 @@ public:
void ucomiss(XMMRegister dst, Address src) { Assembler::ucomiss(dst, src); }
void ucomiss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
+ void vucomxss(XMMRegister dst, XMMRegister src) { Assembler::vucomxss(dst, src); }
+ void vucomxss(XMMRegister dst, Address src) { Assembler::vucomxss(dst, src); }
+ void vucomxss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
+
void ucomisd(XMMRegister dst, XMMRegister src) { Assembler::ucomisd(dst, src); }
void ucomisd(XMMRegister dst, Address src) { Assembler::ucomisd(dst, src); }
void ucomisd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
+ void vucomxsd(XMMRegister dst, XMMRegister src) { Assembler::vucomxsd(dst, src); }
+ void vucomxsd(XMMRegister dst, Address src) { Assembler::vucomxsd(dst, src); }
+ void vucomxsd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
+
// Bitwise Logical XOR of Packed Double-Precision Floating-Point Values
void xorpd(XMMRegister dst, XMMRegister src);
void xorpd(XMMRegister dst, Address src) { Assembler::xorpd(dst, src); }
diff --git a/src/hotspot/cpu/x86/vm_version_x86.cpp b/src/hotspot/cpu/x86/vm_version_x86.cpp
index 74df41f8682..c65c1c7d219 100644
--- a/src/hotspot/cpu/x86/vm_version_x86.cpp
+++ b/src/hotspot/cpu/x86/vm_version_x86.cpp
@@ -1152,6 +1152,10 @@ void VM_Version::get_processor_features() {
warning("AES intrinsics require UseAES flag to be enabled. Intrinsics will be disabled.");
}
FLAG_SET_DEFAULT(UseAESIntrinsics, false);
+ if (UseAESCTRIntrinsics && !FLAG_IS_DEFAULT(UseAESCTRIntrinsics)) {
+ warning("AES_CTR intrinsics require UseAES flag to be enabled. AES_CTR intrinsics will be disabled.");
+ }
+ FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
} else {
if (UseSSE > 2) {
if (FLAG_IS_DEFAULT(UseAESIntrinsics)) {
@@ -1170,8 +1174,8 @@ void VM_Version::get_processor_features() {
if (!UseAESIntrinsics) {
if (UseAESCTRIntrinsics && !FLAG_IS_DEFAULT(UseAESCTRIntrinsics)) {
warning("AES-CTR intrinsics require UseAESIntrinsics flag to be enabled. Intrinsics will be disabled.");
- FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
}
+ FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
} else {
if (supports_sse4_1()) {
if (FLAG_IS_DEFAULT(UseAESCTRIntrinsics)) {
@@ -1191,16 +1195,16 @@ void VM_Version::get_processor_features() {
} else if (UseAES || UseAESIntrinsics || UseAESCTRIntrinsics) {
if (UseAES && !FLAG_IS_DEFAULT(UseAES)) {
warning("AES instructions are not available on this CPU");
- FLAG_SET_DEFAULT(UseAES, false);
}
+ FLAG_SET_DEFAULT(UseAES, false);
if (UseAESIntrinsics && !FLAG_IS_DEFAULT(UseAESIntrinsics)) {
warning("AES intrinsics are not available on this CPU");
- FLAG_SET_DEFAULT(UseAESIntrinsics, false);
}
+ FLAG_SET_DEFAULT(UseAESIntrinsics, false);
if (UseAESCTRIntrinsics && !FLAG_IS_DEFAULT(UseAESCTRIntrinsics)) {
warning("AES-CTR intrinsics are not available on this CPU");
- FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
}
+ FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
}
// Use CLMUL instructions if available.
@@ -1355,16 +1359,16 @@ void VM_Version::get_processor_features() {
FLAG_SET_DEFAULT(UseSHA512Intrinsics, false);
}
- if (supports_evex() && supports_avx512bw()) {
- if (FLAG_IS_DEFAULT(UseSHA3Intrinsics)) {
- UseSHA3Intrinsics = true;
- }
+ if (UseSHA && supports_evex() && supports_avx512bw()) {
+ if (FLAG_IS_DEFAULT(UseSHA3Intrinsics)) {
+ FLAG_SET_DEFAULT(UseSHA3Intrinsics, true);
+ }
} else if (UseSHA3Intrinsics) {
- warning("Intrinsics for SHA3-224, SHA3-256, SHA3-384 and SHA3-512 crypto hash functions not available on this CPU.");
- FLAG_SET_DEFAULT(UseSHA3Intrinsics, false);
+ warning("Intrinsics for SHA3-224, SHA3-256, SHA3-384 and SHA3-512 crypto hash functions not available on this CPU.");
+ FLAG_SET_DEFAULT(UseSHA3Intrinsics, false);
}
- if (!(UseSHA1Intrinsics || UseSHA256Intrinsics || UseSHA512Intrinsics)) {
+ if (!(UseSHA1Intrinsics || UseSHA256Intrinsics || UseSHA512Intrinsics || UseSHA3Intrinsics)) {
FLAG_SET_DEFAULT(UseSHA, false);
}
diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad
index 93b306c37d6..aed54fe93d4 100644
--- a/src/hotspot/cpu/x86/x86.ad
+++ b/src/hotspot/cpu/x86/x86.ad
@@ -1699,9 +1699,10 @@ static void emit_cmpfp_fixup(MacroAssembler* masm) {
}
static void emit_cmpfp3(MacroAssembler* masm, Register dst) {
+ // If any floating point comparison instruction is used, unordered case always triggers jump
+ // for below condition, CF=1 is true when at least one input is NaN
Label done;
__ movl(dst, -1);
- __ jcc(Assembler::parity, done);
__ jcc(Assembler::below, done);
__ setcc(Assembler::notEqual, dst);
__ bind(done);
@@ -5529,12 +5530,21 @@ operand rFlagsRegU()
operand rFlagsRegUCF() %{
constraint(ALLOC_IN_RC(int_flags));
match(RegFlags);
- predicate(false);
+ predicate(!UseAPX || !VM_Version::supports_avx10_2());
format %{ "RFLAGS_U_CF" %}
interface(REG_INTER);
%}
+operand rFlagsRegUCFE() %{
+ constraint(ALLOC_IN_RC(int_flags));
+ match(RegFlags);
+ predicate(UseAPX && VM_Version::supports_avx10_2());
+
+ format %{ "RFLAGS_U_CFE" %}
+ interface(REG_INTER);
+%}
+
// Float register operands
operand regF() %{
constraint(ALLOC_IN_RC(float_reg));
@@ -6027,10 +6037,10 @@ operand cmpOp()
interface(COND_INTER) %{
equal(0x4, "e");
not_equal(0x5, "ne");
- less(0xC, "l");
- greater_equal(0xD, "ge");
- less_equal(0xE, "le");
- greater(0xF, "g");
+ less(0xc, "l");
+ greater_equal(0xd, "ge");
+ less_equal(0xe, "le");
+ greater(0xf, "g");
overflow(0x0, "o");
no_overflow(0x1, "no");
%}
@@ -6062,11 +6072,12 @@ operand cmpOpU()
// don't need to use cmpOpUCF2 for eq/ne
operand cmpOpUCF() %{
match(Bool);
- predicate(n->as_Bool()->_test._test == BoolTest::lt ||
- n->as_Bool()->_test._test == BoolTest::ge ||
- n->as_Bool()->_test._test == BoolTest::le ||
- n->as_Bool()->_test._test == BoolTest::gt ||
- n->in(1)->in(1) == n->in(1)->in(2));
+ predicate((!UseAPX || !VM_Version::supports_avx10_2()) &&
+ (n->as_Bool()->_test._test == BoolTest::lt ||
+ n->as_Bool()->_test._test == BoolTest::ge ||
+ n->as_Bool()->_test._test == BoolTest::le ||
+ n->as_Bool()->_test._test == BoolTest::gt ||
+ n->in(1)->in(1) == n->in(1)->in(2)));
format %{ "" %}
interface(COND_INTER) %{
equal(0xb, "np");
@@ -6084,7 +6095,8 @@ operand cmpOpUCF() %{
// Floating comparisons that can be fixed up with extra conditional jumps
operand cmpOpUCF2() %{
match(Bool);
- predicate((n->as_Bool()->_test._test == BoolTest::ne ||
+ predicate((!UseAPX || !VM_Version::supports_avx10_2()) &&
+ (n->as_Bool()->_test._test == BoolTest::ne ||
n->as_Bool()->_test._test == BoolTest::eq) &&
n->in(1)->in(1) != n->in(1)->in(2));
format %{ "" %}
@@ -6100,6 +6112,37 @@ operand cmpOpUCF2() %{
%}
%}
+
+// Floating point comparisons that set condition flags to test more directly,
+// Unsigned tests are used for G (>) and GE (>=) conditions while signed tests
+// are used for L (<) and LE (<=) conditions. It's important to convert these
+// latter conditions to ones that use unsigned tests before passing into an
+// instruction because the preceding comparison might be based on a three way
+// comparison (CmpF3 or CmpD3) that also assigns unordered outcomes to -1.
+operand cmpOpUCFE()
+%{
+ match(Bool);
+ predicate((UseAPX && VM_Version::supports_avx10_2()) &&
+ (n->as_Bool()->_test._test == BoolTest::ne ||
+ n->as_Bool()->_test._test == BoolTest::eq ||
+ n->as_Bool()->_test._test == BoolTest::lt ||
+ n->as_Bool()->_test._test == BoolTest::ge ||
+ n->as_Bool()->_test._test == BoolTest::le ||
+ n->as_Bool()->_test._test == BoolTest::gt));
+
+ format %{ "" %}
+ interface(COND_INTER) %{
+ equal(0x4, "e");
+ not_equal(0x5, "ne");
+ less(0x2, "b");
+ greater_equal(0x3, "ae");
+ less_equal(0x6, "be");
+ greater(0x7, "a");
+ overflow(0x0, "o");
+ no_overflow(0x1, "no");
+ %}
+%}
+
// Operands for bound floating pointer register arguments
operand rxmm0() %{
constraint(ALLOC_IN_RC(xmm0_reg));
@@ -9116,20 +9159,34 @@ instruct cmovI_imm_01UCF(rRegI dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
ins_pipe(ialu_reg);
%}
+instruct cmovI_imm_01UCFE(rRegI dst, immI_1 src, rFlagsRegUCFE cr, cmpOpUCFE cop)
+%{
+ predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
+ match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
+
+ ins_cost(100); // XXX
+ format %{ "setbn$cop $dst\t# signed, unsigned, int" %}
+ ins_encode %{
+ Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
+ __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
- predicate(!UseAPX);
match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
+
ins_cost(200);
expand %{
cmovI_regU(cop, cr, dst, src);
%}
%}
-instruct cmovI_regUCF_ndd(rRegI dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegI src1, rRegI src2) %{
- predicate(UseAPX);
+instruct cmovI_regUCFE_ndd(rRegI dst, cmpOpUCFE cop, rFlagsRegUCFE cr, rRegI src1, rRegI src2) %{
match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
+
ins_cost(200);
- format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
+ format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, unsigned, int ndd" %}
ins_encode %{
__ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
%}
@@ -9137,7 +9194,7 @@ instruct cmovI_regUCF_ndd(rRegI dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegI src1,
%}
instruct cmovI_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
- predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
+ predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
ins_cost(200); // XXX
@@ -9150,25 +9207,10 @@ instruct cmovI_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src)
ins_pipe(pipe_cmov_reg);
%}
-instruct cmovI_regUCF2_ne_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src1, rRegI src2) %{
- predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
- match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
- effect(TEMP dst);
-
- ins_cost(200);
- format %{ "ecmovpl $dst, $src1, $src2\n\t"
- "cmovnel $dst, $src2" %}
- ins_encode %{
- __ ecmovl(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
- __ cmovl(Assembler::notEqual, $dst$$Register, $src2$$Register);
- %}
- ins_pipe(pipe_cmov_reg);
-%}
-
// Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
// inputs of the CMove
instruct cmovI_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
- predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
+ predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
effect(TEMP dst);
@@ -9182,23 +9224,6 @@ instruct cmovI_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src)
ins_pipe(pipe_cmov_reg);
%}
-// We need this special handling for only eq / neq comparison since NaN == NaN is false,
-// and parity flag bit is set if any of the operand is a NaN.
-instruct cmovI_regUCF2_eq_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src1, rRegI src2) %{
- predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
- match(Set dst (CMoveI (Binary cop cr) (Binary src2 src1)));
- effect(TEMP dst);
-
- ins_cost(200);
- format %{ "ecmovpl $dst, $src1, $src2\n\t"
- "cmovnel $dst, $src2" %}
- ins_encode %{
- __ ecmovl(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
- __ cmovl(Assembler::notEqual, $dst$$Register, $src2$$Register);
- %}
- ins_pipe(pipe_cmov_reg);
-%}
-
// Conditional move
instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
predicate(!UseAPX);
@@ -9241,8 +9266,8 @@ instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
%}
instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
- predicate(!UseAPX);
match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
+
ins_cost(250);
expand %{
cmovI_memU(cop, cr, dst, src);
@@ -9262,12 +9287,12 @@ instruct cmovI_rReg_rReg_memU_ndd(rRegI dst, cmpOpU cop, rFlagsRegU cr, rRegI sr
ins_pipe(pipe_cmov_mem);
%}
-instruct cmovI_rReg_rReg_memUCF_ndd(rRegI dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegI src1, memory src2)
+instruct cmovI_rReg_rReg_memUCFE_ndd(rRegI dst, cmpOpUCFE cop, rFlagsRegUCFE cr, rRegI src1, memory src2)
%{
- predicate(UseAPX);
match(Set dst (CMoveI (Binary cop cr) (Binary src1 (LoadI src2))));
+
ins_cost(250);
- format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
+ format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, unsigned, int ndd" %}
ins_encode %{
__ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
%}
@@ -9317,8 +9342,8 @@ instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
%}
instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
- predicate(!UseAPX);
match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
+
ins_cost(200);
expand %{
cmovN_regU(cop, cr, dst, src);
@@ -9339,11 +9364,11 @@ instruct cmovN_regU_ndd(rRegN dst, cmpOpU cop, rFlagsRegU cr, rRegN src1, rRegN
ins_pipe(pipe_cmov_reg);
%}
-instruct cmovN_regUCF_ndd(rRegN dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegN src1, rRegN src2) %{
- predicate(UseAPX);
+instruct cmovN_regUCFE_ndd(rRegN dst, cmpOpUCFE cop, rFlagsRegUCFE cr, rRegN src1, rRegN src2) %{
match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
+
ins_cost(200);
- format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, compressed ptr ndd" %}
+ format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, unsigned, compressed ptr ndd" %}
ins_encode %{
__ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
%}
@@ -9437,19 +9462,19 @@ instruct cmovP_regU_ndd(rRegP dst, cmpOpU cop, rFlagsRegU cr, rRegP src1, rRegP
%}
instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
- predicate(!UseAPX);
match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
+
ins_cost(200);
expand %{
cmovP_regU(cop, cr, dst, src);
%}
%}
-instruct cmovP_regUCF_ndd(rRegP dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegP src1, rRegP src2) %{
- predicate(UseAPX);
+instruct cmovP_regUCFE_ndd(rRegP dst, cmpOpUCFE cop, rFlagsRegUCFE cr, rRegP src1, rRegP src2) %{
match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
+
ins_cost(200);
- format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, ptr ndd" %}
+ format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, unsigned, ptr ndd" %}
ins_encode %{
__ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
%}
@@ -9457,7 +9482,7 @@ instruct cmovP_regUCF_ndd(rRegP dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegP src1,
%}
instruct cmovP_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
- predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
+ predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
ins_cost(200); // XXX
@@ -9470,25 +9495,10 @@ instruct cmovP_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src)
ins_pipe(pipe_cmov_reg);
%}
-instruct cmovP_regUCF2_ne_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src1, rRegP src2) %{
- predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
- match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
- effect(TEMP dst);
-
- ins_cost(200);
- format %{ "ecmovpq $dst, $src1, $src2\n\t"
- "cmovneq $dst, $src2" %}
- ins_encode %{
- __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
- __ cmovq(Assembler::notEqual, $dst$$Register, $src2$$Register);
- %}
- ins_pipe(pipe_cmov_reg);
-%}
-
// Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
// inputs of the CMove
instruct cmovP_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
- predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
+ predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
match(Set dst (CMoveP (Binary cop cr) (Binary src dst)));
ins_cost(200); // XXX
@@ -9501,21 +9511,6 @@ instruct cmovP_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src)
ins_pipe(pipe_cmov_reg);
%}
-instruct cmovP_regUCF2_eq_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src1, rRegP src2) %{
- predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
- match(Set dst (CMoveP (Binary cop cr) (Binary src2 src1)));
- effect(TEMP dst);
-
- ins_cost(200);
- format %{ "ecmovpq $dst, $src1, $src2\n\t"
- "cmovneq $dst, $src2" %}
- ins_encode %{
- __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
- __ cmovq(Assembler::notEqual, $dst$$Register, $src2$$Register);
- %}
- ins_pipe(pipe_cmov_reg);
-%}
-
instruct cmovL_imm_01(rRegL dst, immL1 src, rFlagsReg cr, cmpOp cop)
%{
predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
@@ -9636,21 +9631,35 @@ instruct cmovL_imm_01UCF(rRegL dst, immL1 src, rFlagsRegUCF cr, cmpOpUCF cop)
ins_pipe(ialu_reg);
%}
+instruct cmovL_imm_01UCFE(rRegL dst, immL1 src, rFlagsRegUCFE cr, cmpOpUCFE cop)
+%{
+ predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
+ match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
+
+ ins_cost(100); // XXX
+ format %{ "setbn$cop $dst\t# signed, unsigned, long" %}
+ ins_encode %{
+ Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
+ __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
- predicate(!UseAPX);
match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
+
ins_cost(200);
expand %{
cmovL_regU(cop, cr, dst, src);
%}
%}
-instruct cmovL_regUCF_ndd(rRegL dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegL src1, rRegL src2)
+instruct cmovL_regUCFE_ndd(rRegL dst, cmpOpUCFE cop, rFlagsRegUCFE cr, rRegL src1, rRegL src2)
%{
- predicate(UseAPX);
match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
+
ins_cost(200);
- format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
+ format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, unsigned, long ndd" %}
ins_encode %{
__ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
%}
@@ -9658,7 +9667,7 @@ instruct cmovL_regUCF_ndd(rRegL dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegL src1,
%}
instruct cmovL_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
- predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
+ predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
ins_cost(200); // XXX
@@ -9671,25 +9680,10 @@ instruct cmovL_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src)
ins_pipe(pipe_cmov_reg);
%}
-instruct cmovL_regUCF2_ne_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src1, rRegL src2) %{
- predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
- match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
- effect(TEMP dst);
-
- ins_cost(200);
- format %{ "ecmovpq $dst, $src1, $src2\n\t"
- "cmovneq $dst, $src2" %}
- ins_encode %{
- __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
- __ cmovq(Assembler::notEqual, $dst$$Register, $src2$$Register);
- %}
- ins_pipe(pipe_cmov_reg);
-%}
-
// Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
// inputs of the CMove
instruct cmovL_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
- predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
+ predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
ins_cost(200); // XXX
@@ -9702,21 +9696,6 @@ instruct cmovL_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src)
ins_pipe(pipe_cmov_reg);
%}
-instruct cmovL_regUCF2_eq_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src1, rRegL src2) %{
- predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
- match(Set dst (CMoveL (Binary cop cr) (Binary src2 src1)));
- effect(TEMP dst);
-
- ins_cost(200);
- format %{ "ecmovpq $dst, $src1, $src2\n\t"
- "cmovneq $dst, $src2" %}
- ins_encode %{
- __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
- __ cmovq(Assembler::notEqual, $dst$$Register, $src2$$Register);
- %}
- ins_pipe(pipe_cmov_reg);
-%}
-
instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
%{
predicate(!UseAPX);
@@ -9731,8 +9710,8 @@ instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
%}
instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
- predicate(!UseAPX);
match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
+
ins_cost(200);
expand %{
cmovL_memU(cop, cr, dst, src);
@@ -9752,12 +9731,12 @@ instruct cmovL_rReg_rReg_memU_ndd(rRegL dst, cmpOpU cop, rFlagsRegU cr, rRegL sr
ins_pipe(pipe_cmov_mem);
%}
-instruct cmovL_rReg_rReg_memUCF_ndd(rRegL dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegL src1, memory src2)
+instruct cmovL_rReg_rReg_memUCFE_ndd(rRegL dst, cmpOpUCFE cop, rFlagsRegUCFE cr, rRegL src1, memory src2)
%{
- predicate(UseAPX);
match(Set dst (CMoveL (Binary cop cr) (Binary src1 (LoadL src2))));
+
ins_cost(200);
- format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
+ format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, unsigned, long ndd" %}
ins_encode %{
__ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
%}
@@ -9802,12 +9781,31 @@ instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
+
ins_cost(200);
expand %{
cmovF_regU(cop, cr, dst, src);
%}
%}
+instruct cmovF_regUCFE(cmpOpUCFE cop, rFlagsRegUCFE cr, regF dst, regF src)
+%{
+ match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
+
+ ins_cost(200); // XXX
+ format %{ "jn$cop skip\t# signed, unsigned cmove float\n\t"
+ "movss $dst, $src\n"
+ "skip:" %}
+ ins_encode %{
+ Label Lskip;
+ // Invert sense of branch from sense of CMOV
+ __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
+ __ movflt($dst$$XMMRegister, $src$$XMMRegister);
+ __ bind(Lskip);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
%{
match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
@@ -9846,12 +9844,31 @@ instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
+
ins_cost(200);
expand %{
cmovD_regU(cop, cr, dst, src);
%}
%}
+instruct cmovD_regUCFE(cmpOpUCFE cop, rFlagsRegUCFE cr, regD dst, regD src)
+%{
+ match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
+
+ ins_cost(200); // XXX
+ format %{ "jn$cop skip\t# signed, unsigned cmove double\n\t"
+ "movsd $dst, $src\n"
+ "skip:" %}
+ ins_encode %{
+ Label Lskip;
+ // Invert sense of branch from sense of CMOV
+ __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
+ __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
+ __ bind(Lskip);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
//----------Arithmetic Instructions--------------------------------------------
//----------Addition Instructions----------------------------------------------
@@ -14319,7 +14336,7 @@ instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
ins_pipe(pipe_slow);
%}
-instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
+instruct cmpF_cc_regCF(rFlagsRegUCF cr, regF src1, regF src2) %{
match(Set cr (CmpF src1 src2));
ins_cost(100);
@@ -14330,6 +14347,17 @@ instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
ins_pipe(pipe_slow);
%}
+instruct cmpF_cc_regCFE(rFlagsRegUCFE cr, regF src1, regF src2) %{
+ match(Set cr (CmpF src1 src2));
+
+ ins_cost(100);
+ format %{ "vucomxss $src1, $src2" %}
+ ins_encode %{
+ __ vucomxss($src1$$XMMRegister, $src2$$XMMRegister);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
match(Set cr (CmpF src1 (LoadF src2)));
@@ -14341,8 +14369,20 @@ instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
ins_pipe(pipe_slow);
%}
+instruct cmpF_cc_memCFE(rFlagsRegUCFE cr, regF src1, memory src2) %{
+ match(Set cr (CmpF src1 (LoadF src2)));
+
+ ins_cost(100);
+ format %{ "vucomxss $src1, $src2" %}
+ ins_encode %{
+ __ vucomxss($src1$$XMMRegister, $src2$$Address);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
match(Set cr (CmpF src con));
+
ins_cost(100);
format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
ins_encode %{
@@ -14351,6 +14391,17 @@ instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
ins_pipe(pipe_slow);
%}
+instruct cmpF_cc_immCFE(rFlagsRegUCFE cr, regF src, immF con) %{
+ match(Set cr (CmpF src con));
+
+ ins_cost(100);
+ format %{ "vucomxss $src, [$constantaddress]\t# load from constant table: float=$con" %}
+ ins_encode %{
+ __ vucomxss($src$$XMMRegister, $constantaddress($con));
+ %}
+ ins_pipe(pipe_slow);
+%}
+
// Really expensive, avoid
instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
%{
@@ -14370,7 +14421,7 @@ instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
ins_pipe(pipe_slow);
%}
-instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
+instruct cmpD_cc_regCF(rFlagsRegUCF cr, regD src1, regD src2) %{
match(Set cr (CmpD src1 src2));
ins_cost(100);
@@ -14381,6 +14432,17 @@ instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
ins_pipe(pipe_slow);
%}
+instruct cmpD_cc_regCFE(rFlagsRegUCFE cr, regD src1, regD src2) %{
+ match(Set cr (CmpD src1 src2));
+
+ ins_cost(100);
+ format %{ "vucomxsd $src1, $src2 test" %}
+ ins_encode %{
+ __ vucomxsd($src1$$XMMRegister, $src2$$XMMRegister);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
match(Set cr (CmpD src1 (LoadD src2)));
@@ -14392,6 +14454,17 @@ instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
ins_pipe(pipe_slow);
%}
+instruct cmpD_cc_memCFE(rFlagsRegUCFE cr, regD src1, memory src2) %{
+ match(Set cr (CmpD src1 (LoadD src2)));
+
+ ins_cost(100);
+ format %{ "vucomxsd $src1, $src2" %}
+ ins_encode %{
+ __ vucomxsd($src1$$XMMRegister, $src2$$Address);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
match(Set cr (CmpD src con));
ins_cost(100);
@@ -14402,6 +14475,17 @@ instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
ins_pipe(pipe_slow);
%}
+instruct cmpD_cc_immCFE(rFlagsRegUCFE cr, regD src, immD con) %{
+ match(Set cr (CmpD src con));
+
+ ins_cost(100);
+ format %{ "vucomxsd $src, [$constantaddress]\t# load from constant table: double=$con" %}
+ ins_encode %{
+ __ vucomxsd($src$$XMMRegister, $constantaddress($con));
+ %}
+ ins_pipe(pipe_slow);
+%}
+
// Compare into -1,0,1
instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
%{
@@ -16808,6 +16892,21 @@ instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
ins_pipe(pipe_jcc);
%}
+// Jump Direct Conditional - using signed and unsigned comparison
+instruct jmpConUCFE(cmpOpUCFE cop, rFlagsRegUCFE cmp, label labl) %{
+ match(If cop cmp);
+ effect(USE labl);
+
+ ins_cost(200);
+ format %{ "j$cop,su $labl" %}
+ size(6);
+ ins_encode %{
+ Label* L = $labl$$label;
+ __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
+ %}
+ ins_pipe(pipe_jcc);
+%}
+
// ============================================================================
// The 2nd slow-half of a subtype check. Scan the subklass's 2ndary
// superklass array for an instance of the superklass. Set a hidden
@@ -17026,6 +17125,22 @@ instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
ins_short_branch(1);
%}
+// Jump Direct Conditional - using signed and unsigned comparison
+instruct jmpConUCFE_short(cmpOpUCFE cop, rFlagsRegUCFE cmp, label labl) %{
+ match(If cop cmp);
+ effect(USE labl);
+
+ ins_cost(300);
+ format %{ "j$cop,sus $labl" %}
+ size(2);
+ ins_encode %{
+ Label* L = $labl$$label;
+ __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
+ %}
+ ins_pipe(pipe_jcc);
+ ins_short_branch(1);
+%}
+
// ============================================================================
// inlined locking and unlocking
diff --git a/src/hotspot/os/aix/decoder_aix.hpp b/src/hotspot/os/aix/decoder_aix.hpp
index 2ba3e1c5a3a..632355ccf4e 100644
--- a/src/hotspot/os/aix/decoder_aix.hpp
+++ b/src/hotspot/os/aix/decoder_aix.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2013 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -38,7 +38,7 @@ class AIXDecoder: public AbstractDecoder {
virtual bool demangle(const char* symbol, char* buf, int buflen) { return false; } // use AixSymbols::get_function_name to demangle
virtual bool decode(address addr, char* buf, int buflen, int* offset, const char* modulepath, bool demangle) {
- return AixSymbols::get_function_name(addr, buf, buflen, offset, 0, demangle);
+ return AixSymbols::get_function_name(addr, buf, buflen, offset, nullptr, demangle);
}
virtual bool decode(address addr, char *buf, int buflen, int* offset, const void *base) {
ShouldNotReachHere();
diff --git a/src/hotspot/os/aix/os_aix.cpp b/src/hotspot/os/aix/os_aix.cpp
index d7c1911a914..0a8efbece8d 100644
--- a/src/hotspot/os/aix/os_aix.cpp
+++ b/src/hotspot/os/aix/os_aix.cpp
@@ -703,7 +703,7 @@ static void *thread_native_entry(Thread *thread) {
log_info(os, thread)("Thread finished (tid: %zu, kernel thread id: %zu).",
os::current_thread_id(), (uintx) kernel_thread_id);
- return 0;
+ return nullptr;
}
bool os::create_thread(Thread* thread, ThreadType thr_type,
diff --git a/src/hotspot/os/aix/porting_aix.cpp b/src/hotspot/os/aix/porting_aix.cpp
index 7311afc197b..b3f878fbfdd 100644
--- a/src/hotspot/os/aix/porting_aix.cpp
+++ b/src/hotspot/os/aix/porting_aix.cpp
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2012, 2024 SAP SE. All rights reserved.
- * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -78,7 +78,7 @@ class fixed_strings {
public:
- fixed_strings() : first(0) {}
+ fixed_strings() : first(nullptr) {}
~fixed_strings() {
node* n = first;
while (n) {
@@ -113,7 +113,7 @@ bool AixSymbols::get_function_name (
// information (null if not available)
bool demangle // [in] whether to demangle the name
) {
- struct tbtable* tb = 0;
+ struct tbtable* tb = nullptr;
unsigned int searchcount = 0;
// initialize output parameters
@@ -653,10 +653,10 @@ void AixNativeCallstack::print_callstack_for_context(outputStream* st, const uco
// To print the first frame, use the current value of iar:
// current entry indicated by iar (the current pc)
- codeptr_t cur_iar = 0;
- stackptr_t cur_sp = 0;
- codeptr_t cur_rtoc = 0;
- codeptr_t cur_lr = 0;
+ codeptr_t cur_iar = nullptr;
+ stackptr_t cur_sp = nullptr;
+ codeptr_t cur_rtoc = nullptr;
+ codeptr_t cur_lr = nullptr;
const ucontext_t* uc = (const ucontext_t*) context;
@@ -926,7 +926,7 @@ static struct handletableentry* p_handletable = nullptr;
static const char* rtv_linkedin_libpath() {
constexpr int bufsize = 4096;
static char buffer[bufsize];
- static const char* libpath = 0;
+ static const char* libpath = nullptr;
// we only try to retrieve the libpath once. After that try we
// let libpath point to buffer, which then contains a valid libpath
diff --git a/src/hotspot/os/bsd/globals_bsd.hpp b/src/hotspot/os/bsd/globals_bsd.hpp
index 850d491a11f..22f587ed789 100644
--- a/src/hotspot/os/bsd/globals_bsd.hpp
+++ b/src/hotspot/os/bsd/globals_bsd.hpp
@@ -28,6 +28,7 @@
//
// Declare Bsd specific flags. They are not available on other platforms.
//
+#ifdef AARCH64
#define RUNTIME_OS_FLAGS(develop, \
develop_pd, \
product, \
@@ -35,9 +36,21 @@
range, \
constraint) \
\
- AARCH64_ONLY(develop(bool, AssertWXAtThreadSync, true, \
- "Conservatively check W^X thread state at possible safepoint" \
- "or handshake"))
+ develop(bool, TraceWXHealing, false, \
+ "track occurrences of W^X mode healing") \
+ develop(bool, UseOldWX, false, \
+ "Choose old W^X implementation.") \
+ product(bool, StressWXHealing, false, DIAGNOSTIC, \
+ "Stress W xor X healing on MacOS")
+
+#else
+#define RUNTIME_OS_FLAGS(develop, \
+ develop_pd, \
+ product, \
+ product_pd, \
+ range, \
+ constraint)
+#endif
// end of RUNTIME_OS_FLAGS
diff --git a/src/hotspot/os/bsd/os_bsd.cpp b/src/hotspot/os/bsd/os_bsd.cpp
index 0e21c2d1785..81320b4f1aa 100644
--- a/src/hotspot/os/bsd/os_bsd.cpp
+++ b/src/hotspot/os/bsd/os_bsd.cpp
@@ -841,6 +841,7 @@ jlong os::javaTimeNanos() {
// We might also condition (c) on the magnitude of the delta between obsv and now.
// Avoiding excessive CAS operations to hot RW locations is critical.
// See https://blogs.oracle.com/dave/entry/cas_and_cache_trivia_invalidate
+ // https://web.archive.org/web/20131214182431/https://blogs.oracle.com/dave/entry/cas_and_cache_trivia_invalidate
return (prev == obsv) ? now : obsv;
}
diff --git a/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp b/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp
index 62dba218b2f..36599594842 100644
--- a/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp
+++ b/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp
@@ -54,8 +54,11 @@
#include "signals_posix.hpp"
#include "utilities/align.hpp"
#include "utilities/debug.hpp"
+#include "utilities/decoder.hpp"
#include "utilities/events.hpp"
+#include "utilities/nativeStackPrinter.hpp"
#include "utilities/vmError.hpp"
+#include "compiler/disassembler.hpp"
// put OS-includes here
# include
@@ -85,6 +88,8 @@
#define SPELL_REG_SP "sp"
#ifdef __APPLE__
+WXMode DefaultWXWriteMode;
+
// see darwin-xnu/osfmk/mach/arm/_structs.h
// 10.5 UNIX03 member name prefixes
@@ -233,19 +238,56 @@ NOINLINE frame os::current_frame() {
bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info,
ucontext_t* uc, JavaThread* thread) {
- // Enable WXWrite: this function is called by the signal handler at arbitrary
- // point of execution.
- ThreadWXEnable wx(WXWrite, thread);
-
// decide if this trap can be handled by a stub
address stub = nullptr;
-
- address pc = nullptr;
+ address pc = nullptr;
//%note os_trap_1
if (info != nullptr && uc != nullptr && thread != nullptr) {
pc = (address) os::Posix::ucontext_get_pc(uc);
+#ifdef MACOS_AARCH64
+ // If we got a SIGBUS because we tried to write into the code
+ // cache, try enabling WXWrite mode.
+ if (sig == SIGBUS
+ && pc != info->si_addr
+ && CodeCache::contains(info->si_addr)
+ && os::address_is_in_vm(pc)) {
+ WXMode *entry_mode = thread->_cur_wx_mode;
+ if (entry_mode != nullptr && *entry_mode == WXArmedForWrite) {
+ if (TraceWXHealing) {
+ static const char *mode_names[3] = {"WXWrite", "WXExec", "WXArmedForWrite"};
+ tty->print("Healing WXMode %s at %p to WXWrite",
+ mode_names[*entry_mode], entry_mode);
+ char name[128];
+ int offset = 0;
+ if (os::dll_address_to_function_name(pc, name, sizeof name, &offset)) {
+ tty->print_cr(" (%s+0x%x)", name, offset);
+ } else {
+ tty->cr();
+ }
+ if (Verbose) {
+ char buf[O_BUFLEN];
+ NativeStackPrinter nsp(thread);
+ nsp.print_stack(tty, buf, sizeof(buf), pc,
+ true /* print_source_info */, -1 /* max stack */);
+ }
+ }
+#ifndef PRODUCT
+ guarantee(StressWXHealing,
+ "We should not reach here unless StressWXHealing");
+#endif
+ *(thread->_cur_wx_mode) = WXWrite;
+ return thread->wx_enable_write();
+ }
+ }
+
+ // There may be cases where code after this point that we call
+ // from the signal handler changes WX state, so we protect against
+ // that by saving and restoring the state.
+ ThreadWXEnable wx(thread->get_wx_state(), thread);
+#endif
+
// Handle ALL stack overflow variations here
if (sig == SIGSEGV || sig == SIGBUS) {
address addr = (address) info->si_addr;
@@ -515,11 +557,42 @@ int os::extra_bang_size_in_bytes() {
return 0;
}
-#ifdef __APPLE__
+#ifdef MACOS_AARCH64
+THREAD_LOCAL bool os::_jit_exec_enabled;
+
+// This is a wrapper around the standard library function
+// pthread_jit_write_protect_np(3). We keep track of the state of
+// per-thread write protection on the MAP_JIT region in the
+// thread-local variable os::_jit_exec_enabled
void os::current_thread_enable_wx(WXMode mode) {
- pthread_jit_write_protect_np(mode == WXExec);
+ bool exec_enabled = mode != WXWrite;
+ if (exec_enabled != _jit_exec_enabled NOT_PRODUCT( || DefaultWXWriteMode == WXWrite)) {
+ permit_forbidden_function::pthread_jit_write_protect_np(exec_enabled);
+ _jit_exec_enabled = exec_enabled;
+ }
}
-#endif
+
+// If the current thread is in the WX state WXArmedForWrite, change
+// the state to WXWrite.
+bool Thread::wx_enable_write() {
+ if (_wx_state == WXArmedForWrite) {
+ _wx_state = WXWrite;
+ os::current_thread_enable_wx(WXWrite);
+ return true;
+ } else {
+ return false;
+ }
+}
+
+// A wrapper around wx_enable_write() for when the current thread is
+// not known.
+void os::thread_wx_enable_write_impl() {
+ if (!StressWXHealing) {
+ Thread::current()->wx_enable_write();
+ }
+}
+
+#endif // MACOS_AARCH64
static inline void atomic_copy64(const volatile void *src, volatile void *dst) {
*(jlong *) dst = *(const jlong *) src;
diff --git a/src/hotspot/os_cpu/linux_arm/javaThread_linux_arm.cpp b/src/hotspot/os_cpu/linux_arm/javaThread_linux_arm.cpp
index d82b9d90417..2b96e978980 100644
--- a/src/hotspot/os_cpu/linux_arm/javaThread_linux_arm.cpp
+++ b/src/hotspot/os_cpu/linux_arm/javaThread_linux_arm.cpp
@@ -42,6 +42,16 @@ frame JavaThread::pd_last_frame() {
void JavaThread::cache_global_variables() {
BarrierSet* bs = BarrierSet::barrier_set();
+#if INCLUDE_G1GC
+ if (bs->is_a(BarrierSet::G1BarrierSet)) {
+ _card_table_base = nullptr;
+ } else
+#endif
+#if INCLUDE_SHENANDOAHGC
+ if (bs->is_a(BarrierSet::ShenandoahBarrierSet)) {
+ _card_table_base = nullptr;
+ } else
+#endif
if (bs->is_a(BarrierSet::CardTableBarrierSet)) {
CardTableBarrierSet* ctbs = CardTableBarrierSet::barrier_set();
_card_table_base = (address)ctbs->card_table_base_const();
diff --git a/src/hotspot/share/asm/codeBuffer.cpp b/src/hotspot/share/asm/codeBuffer.cpp
index 7871134e923..d94f52c18f6 100644
--- a/src/hotspot/share/asm/codeBuffer.cpp
+++ b/src/hotspot/share/asm/codeBuffer.cpp
@@ -98,6 +98,8 @@ CodeBuffer::CodeBuffer(const CodeBlob* blob) DEBUG_ONLY(: Scrubber(this, sizeof(
}
void CodeBuffer::initialize(csize_t code_size, csize_t locs_size) {
+ MACOS_AARCH64_ONLY(os::thread_wx_enable_write());
+
// Always allow for empty slop around each section.
int slop = (int) CodeSection::end_slop();
diff --git a/src/hotspot/share/c1/c1_Runtime1.cpp b/src/hotspot/share/c1/c1_Runtime1.cpp
index a4c956ff5be..63764dd113a 100644
--- a/src/hotspot/share/c1/c1_Runtime1.cpp
+++ b/src/hotspot/share/c1/c1_Runtime1.cpp
@@ -541,6 +541,7 @@ extern void vm_exit(int code);
// unpack_with_exception entry instead. This makes life for the exception blob easier
// because making that same check and diverting is painful from assembly language.
JRT_ENTRY_NO_ASYNC(static address, exception_handler_for_pc_helper(JavaThread* current, oopDesc* ex, address pc, nmethod*& nm))
+ MACOS_AARCH64_ONLY(current->wx_enable_write());
Handle exception(current, ex);
// This function is called when we are about to throw an exception. Therefore,
diff --git a/src/hotspot/share/classfile/classLoader.cpp b/src/hotspot/share/classfile/classLoader.cpp
index f631bfaa102..eced83577cb 100644
--- a/src/hotspot/share/classfile/classLoader.cpp
+++ b/src/hotspot/share/classfile/classLoader.cpp
@@ -127,6 +127,7 @@ PerfCounter* ClassLoader::_perf_ik_link_methods_count = nullptr;
PerfCounter* ClassLoader::_perf_method_adapters_count = nullptr;
PerfCounter* ClassLoader::_unsafe_defineClassCallCounter = nullptr;
PerfCounter* ClassLoader::_perf_secondary_hash_time = nullptr;
+PerfCounter* ClassLoader::_perf_change_wx_time = nullptr;
PerfCounter* ClassLoader::_perf_resolve_indy_time = nullptr;
PerfCounter* ClassLoader::_perf_resolve_invokehandle_time = nullptr;
@@ -1370,6 +1371,7 @@ void ClassLoader::initialize(TRAPS) {
NEWPERFBYTECOUNTER(_perf_sys_classfile_bytes_read, SUN_CLS, "sysClassBytes");
NEWPERFEVENTCOUNTER(_unsafe_defineClassCallCounter, SUN_CLS, "unsafeDefineClassCalls");
NEWPERFTICKCOUNTER(_perf_secondary_hash_time, SUN_CLS, "secondarySuperHashTime");
+ NEWPERFTICKCOUNTER(_perf_change_wx_time, SUN_CLS, "changeWXTime");
if (log_is_enabled(Info, perf, class, link)) {
NEWPERFTICKCOUNTER(_perf_ik_link_methods_time, SUN_CLS, "linkMethodsTime");
diff --git a/src/hotspot/share/classfile/classLoader.hpp b/src/hotspot/share/classfile/classLoader.hpp
index afb0a581dcc..a935d3027ac 100644
--- a/src/hotspot/share/classfile/classLoader.hpp
+++ b/src/hotspot/share/classfile/classLoader.hpp
@@ -184,6 +184,7 @@ class ClassLoader: AllStatic {
// Count the time taken to hash the scondary superclass arrays.
static PerfCounter* _perf_secondary_hash_time;
+ static PerfCounter* _perf_change_wx_time;
// The boot class path consists of 3 ordered pieces:
// 1. the module/path pairs specified to --patch-module
@@ -268,6 +269,9 @@ class ClassLoader: AllStatic {
static PerfCounter* perf_secondary_hash_time() {
return _perf_secondary_hash_time;
}
+ static PerfCounter* perf_change_wx_time() {
+ return _perf_change_wx_time;
+ }
static PerfCounter* perf_sys_classload_time() { return _perf_sys_classload_time; }
static PerfCounter* perf_app_classload_time() { return _perf_app_classload_time; }
static PerfCounter* perf_app_classload_selftime() { return _perf_app_classload_selftime; }
diff --git a/src/hotspot/share/classfile/javaClasses.cpp b/src/hotspot/share/classfile/javaClasses.cpp
index b650bf8cfb8..c6b0fcb90e0 100644
--- a/src/hotspot/share/classfile/javaClasses.cpp
+++ b/src/hotspot/share/classfile/javaClasses.cpp
@@ -1908,16 +1908,16 @@ oop java_lang_Thread::park_blocker(oop java_thread) {
return java_thread->obj_field_access(_park_blocker_offset);
}
-// Obtain stack trace for platform or mounted virtual thread.
-// If jthread is a virtual thread and it has been unmounted (or remounted to different carrier) the method returns null.
-// The caller (java.lang.VirtualThread) handles returned nulls via retry.
+// Obtain stack trace for a platform or virtual thread.
oop java_lang_Thread::async_get_stack_trace(jobject jthread, TRAPS) {
ThreadsListHandle tlh(THREAD);
JavaThread* java_thread = nullptr;
- oop thread_oop;
+ oop thread_oop = nullptr;
bool has_java_thread = tlh.cv_internal_thread_to_JavaThread(jthread, &java_thread, &thread_oop);
- if (!has_java_thread) {
+ assert(thread_oop != nullptr, "Missing Thread oop");
+ bool is_virtual = java_lang_VirtualThread::is_instance(thread_oop);
+ if (!has_java_thread && !is_virtual) {
return nullptr;
}
@@ -1925,12 +1925,11 @@ oop java_lang_Thread::async_get_stack_trace(jobject jthread, TRAPS) {
public:
const Handle _thread_h;
int _depth;
- bool _retry_handshake;
GrowableArray* _methods;
GrowableArray* _bcis;
GetStackTraceHandshakeClosure(Handle thread_h) :
- HandshakeClosure("GetStackTraceHandshakeClosure"), _thread_h(thread_h), _depth(0), _retry_handshake(false),
+ HandshakeClosure("GetStackTraceHandshakeClosure"), _thread_h(thread_h), _depth(0),
_methods(nullptr), _bcis(nullptr) {
}
~GetStackTraceHandshakeClosure() {
@@ -1938,37 +1937,15 @@ oop java_lang_Thread::async_get_stack_trace(jobject jthread, TRAPS) {
delete _bcis;
}
- bool read_reset_retry() {
- bool ret = _retry_handshake;
- // If we re-execute the handshake this method need to return false
- // when the handshake cannot be performed. (E.g. thread terminating)
- _retry_handshake = false;
- return ret;
- }
-
void do_thread(Thread* th) {
- if (!Thread::current()->is_Java_thread()) {
- _retry_handshake = true;
+ JavaThread* java_thread = th != nullptr ? JavaThread::cast(th) : nullptr;
+ if (java_thread != nullptr && !java_thread->has_last_Java_frame()) {
+ // stack trace is empty
return;
}
- JavaThread* java_thread = JavaThread::cast(th);
-
- if (!java_thread->has_last_Java_frame()) {
- return;
- }
-
- bool carrier = false;
- if (java_lang_VirtualThread::is_instance(_thread_h())) {
- // Ensure _thread_h is still mounted to java_thread.
- const ContinuationEntry* ce = java_thread->vthread_continuation();
- if (ce == nullptr || ce->cont_oop(java_thread) != java_lang_VirtualThread::continuation(_thread_h())) {
- // Target thread has been unmounted.
- return;
- }
- } else {
- carrier = (java_thread->vthread_continuation() != nullptr);
- }
+ bool is_virtual = java_lang_VirtualThread::is_instance(_thread_h());
+ bool vthread_carrier = !is_virtual && (java_thread->vthread_continuation() != nullptr);
const int max_depth = MaxJavaStackTraceDepth;
const bool skip_hidden = !ShowHiddenFrames;
@@ -1979,7 +1956,10 @@ oop java_lang_Thread::async_get_stack_trace(jobject jthread, TRAPS) {
_bcis = new (mtInternal) GrowableArray(init_length, mtInternal);
int total_count = 0;
- for (vframeStream vfst(java_thread, false, false, carrier); // we don't process frames as we don't care about oops
+ vframeStream vfst(java_thread != nullptr
+ ? vframeStream(java_thread, false, false, vthread_carrier) // we don't process frames as we don't care about oops
+ : vframeStream(java_lang_VirtualThread::continuation(_thread_h())));
+ for (;
!vfst.at_end() && (max_depth == 0 || max_depth != total_count);
vfst.next()) {
@@ -2001,9 +1981,11 @@ oop java_lang_Thread::async_get_stack_trace(jobject jthread, TRAPS) {
ResourceMark rm(THREAD);
HandleMark hm(THREAD);
GetStackTraceHandshakeClosure gsthc(Handle(THREAD, thread_oop));
- do {
- Handshake::execute(&gsthc, &tlh, java_thread);
- } while (gsthc.read_reset_retry());
+ if (is_virtual) {
+ Handshake::execute(&gsthc, thread_oop);
+ } else {
+ Handshake::execute(&gsthc, &tlh, java_thread);
+ }
// Stop if no stack trace is found.
if (gsthc._depth == 0) {
@@ -2200,7 +2182,7 @@ void java_lang_VirtualThread::set_timeout(oop vthread, jlong value) {
JavaThreadStatus java_lang_VirtualThread::map_state_to_thread_status(int state) {
JavaThreadStatus status = JavaThreadStatus::NEW;
- switch (state & ~SUSPENDED) {
+ switch (state) {
case NEW:
status = JavaThreadStatus::NEW;
break;
diff --git a/src/hotspot/share/classfile/javaClasses.hpp b/src/hotspot/share/classfile/javaClasses.hpp
index a8562a345c8..3276d398faf 100644
--- a/src/hotspot/share/classfile/javaClasses.hpp
+++ b/src/hotspot/share/classfile/javaClasses.hpp
@@ -592,9 +592,6 @@ class java_lang_VirtualThread : AllStatic {
TIMED_WAITING = 17,
TIMED_WAIT = 18, // waiting in timed-Object.wait
TERMINATED = 99,
-
- // additional state bits
- SUSPENDED = 1 << 8, // suspended when unmounted
};
static void compute_offsets();
diff --git a/src/hotspot/share/classfile/verifier.cpp b/src/hotspot/share/classfile/verifier.cpp
index 38dba1d3d5f..30f147b9ae7 100644
--- a/src/hotspot/share/classfile/verifier.cpp
+++ b/src/hotspot/share/classfile/verifier.cpp
@@ -190,9 +190,8 @@ bool Verifier::verify(InstanceKlass* klass, bool should_verify_class, TRAPS) {
// effect (sic!) for external_name(), but instead of doing that, we opt to
// explicitly push the hashcode in here. This is signify the following block
// is IMPORTANT:
- if (klass->java_mirror() != nullptr) {
- klass->java_mirror()->identity_hash();
- }
+ assert(klass->java_mirror() != nullptr, "must be");
+ klass->java_mirror()->identity_hash();
if (!is_eligible_for_verification(klass, should_verify_class)) {
return true;
diff --git a/src/hotspot/share/classfile/vmIntrinsics.hpp b/src/hotspot/share/classfile/vmIntrinsics.hpp
index 07fa294e8e1..75592fd61c8 100644
--- a/src/hotspot/share/classfile/vmIntrinsics.hpp
+++ b/src/hotspot/share/classfile/vmIntrinsics.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -415,18 +415,18 @@ class methodHandle;
\
do_class(java_lang_StringCoding, "java/lang/StringCoding") \
do_intrinsic(_countPositives, java_lang_StringCoding, countPositives_name, countPositives_signature, F_S) \
- do_name( countPositives_name, "countPositives") \
+ do_name( countPositives_name, "countPositives0") \
do_signature(countPositives_signature, "([BII)I") \
\
do_class(sun_nio_cs_iso8859_1_Encoder, "sun/nio/cs/ISO_8859_1$Encoder") \
do_intrinsic(_encodeISOArray, sun_nio_cs_iso8859_1_Encoder, encodeISOArray_name, encodeISOArray_signature, F_S) \
- do_name( encodeISOArray_name, "implEncodeISOArray") \
+ do_name( encodeISOArray_name, "encodeISOArray0") \
do_signature(encodeISOArray_signature, "([CI[BII)I") \
\
do_intrinsic(_encodeByteISOArray, java_lang_StringCoding, encodeISOArray_name, indexOfI_signature, F_S) \
\
do_intrinsic(_encodeAsciiArray, java_lang_StringCoding, encodeAsciiArray_name, encodeISOArray_signature, F_S) \
- do_name( encodeAsciiArray_name, "implEncodeAsciiArray") \
+ do_name( encodeAsciiArray_name, "encodeAsciiArray0") \
\
do_class(java_math_BigInteger, "java/math/BigInteger") \
do_intrinsic(_multiplyToLen, java_math_BigInteger, multiplyToLen_name, multiplyToLen_signature, F_S) \
@@ -1003,7 +1003,7 @@ class methodHandle;
do_signature(vector_unary_op_sig, "(I" \
"Ljava/lang/Class;" \
"Ljava/lang/Class;" \
- "Ljava/lang/Class;" \
+ "I" \
"I" \
"Ljdk/internal/vm/vector/VectorSupport$Vector;" \
"Ljdk/internal/vm/vector/VectorSupport$VectorMask;" \
@@ -1015,7 +1015,7 @@ class methodHandle;
do_signature(vector_binary_op_sig, "(I" \
"Ljava/lang/Class;" \
"Ljava/lang/Class;" \
- "Ljava/lang/Class;" \
+ "I" \
"I" \
"Ljdk/internal/vm/vector/VectorSupport$VectorPayload;" \
"Ljdk/internal/vm/vector/VectorSupport$VectorPayload;" \
@@ -1051,7 +1051,7 @@ class methodHandle;
do_signature(vector_ternary_op_sig, "(I" \
"Ljava/lang/Class;" \
"Ljava/lang/Class;" \
- "Ljava/lang/Class;" \
+ "I" \
"I" \
"Ljdk/internal/vm/vector/VectorSupport$Vector;" \
"Ljdk/internal/vm/vector/VectorSupport$Vector;" \
@@ -1063,7 +1063,7 @@ class methodHandle;
\
do_intrinsic(_VectorSelectFromTwoVectorOp, jdk_internal_vm_vector_VectorSupport, vector_select_from_op_name, vector_select_from_op_sig, F_S) \
do_signature(vector_select_from_op_sig, "(Ljava/lang/Class;" \
- "Ljava/lang/Class;" \
+ "I" \
"I" \
"Ljdk/internal/vm/vector/VectorSupport$Vector;" \
"Ljdk/internal/vm/vector/VectorSupport$Vector;" \
@@ -1074,7 +1074,7 @@ class methodHandle;
\
do_intrinsic(_VectorFromBitsCoerced, jdk_internal_vm_vector_VectorSupport, vector_frombits_coerced_name, vector_frombits_coerced_sig, F_S) \
do_signature(vector_frombits_coerced_sig, "(Ljava/lang/Class;" \
- "Ljava/lang/Class;" \
+ "I" \
"I" \
"J" \
"I" \
@@ -1085,7 +1085,7 @@ class methodHandle;
\
do_intrinsic(_VectorLoadOp, jdk_internal_vm_vector_VectorSupport, vector_load_op_name, vector_load_op_sig, F_S) \
do_signature(vector_load_op_sig, "(Ljava/lang/Class;" \
- "Ljava/lang/Class;" \
+ "I" \
"I" \
"Ljava/lang/Object;" \
"J" \
@@ -1100,7 +1100,7 @@ class methodHandle;
do_intrinsic(_VectorLoadMaskedOp, jdk_internal_vm_vector_VectorSupport, vector_load_masked_op_name, vector_load_masked_op_sig, F_S) \
do_signature(vector_load_masked_op_sig, "(Ljava/lang/Class;" \
"Ljava/lang/Class;" \
- "Ljava/lang/Class;" \
+ "I" \
"I" \
"Ljava/lang/Object;" \
"J" \
@@ -1116,7 +1116,7 @@ class methodHandle;
\
do_intrinsic(_VectorStoreOp, jdk_internal_vm_vector_VectorSupport, vector_store_op_name, vector_store_op_sig, F_S) \
do_signature(vector_store_op_sig, "(Ljava/lang/Class;" \
- "Ljava/lang/Class;" \
+ "I" \
"I" \
"Ljava/lang/Object;" \
"J" \
@@ -1131,7 +1131,7 @@ class methodHandle;
do_intrinsic(_VectorStoreMaskedOp, jdk_internal_vm_vector_VectorSupport, vector_store_masked_op_name, vector_store_masked_op_sig, F_S) \
do_signature(vector_store_masked_op_sig, "(Ljava/lang/Class;" \
"Ljava/lang/Class;" \
- "Ljava/lang/Class;" \
+ "I" \
"I" \
"Ljava/lang/Object;" \
"J" \
@@ -1148,7 +1148,7 @@ class methodHandle;
do_signature(vector_reduction_coerced_sig, "(I" \
"Ljava/lang/Class;" \
"Ljava/lang/Class;" \
- "Ljava/lang/Class;" \
+ "I" \
"I" \
"Ljdk/internal/vm/vector/VectorSupport$Vector;" \
"Ljdk/internal/vm/vector/VectorSupport$VectorMask;" \
@@ -1159,7 +1159,7 @@ class methodHandle;
do_intrinsic(_VectorTest, jdk_internal_vm_vector_VectorSupport, vector_test_name, vector_test_sig, F_S) \
do_signature(vector_test_sig, "(I" \
"Ljava/lang/Class;" \
- "Ljava/lang/Class;" \
+ "I" \
"I" \
"Ljdk/internal/vm/vector/VectorSupport$VectorMask;" \
"Ljdk/internal/vm/vector/VectorSupport$VectorMask;" \
@@ -1170,7 +1170,7 @@ class methodHandle;
do_intrinsic(_VectorBlend, jdk_internal_vm_vector_VectorSupport, vector_blend_name, vector_blend_sig, F_S) \
do_signature(vector_blend_sig, "(Ljava/lang/Class;" \
"Ljava/lang/Class;" \
- "Ljava/lang/Class;" \
+ "I" \
"I" \
"Ljdk/internal/vm/vector/VectorSupport$Vector;" \
"Ljdk/internal/vm/vector/VectorSupport$Vector;" \
@@ -1182,7 +1182,8 @@ class methodHandle;
do_intrinsic(_VectorCompare, jdk_internal_vm_vector_VectorSupport, vector_compare_name, vector_compare_sig, F_S) \
do_signature(vector_compare_sig, "(I" \
"Ljava/lang/Class;" \
- "Ljava/lang/Class;Ljava/lang/Class;" \
+ "Ljava/lang/Class;" \
+ "I" \
"I" \
"Ljdk/internal/vm/vector/VectorSupport$Vector;" \
"Ljdk/internal/vm/vector/VectorSupport$Vector;" \
@@ -1195,7 +1196,7 @@ class methodHandle;
do_signature(vector_rearrange_sig, "(Ljava/lang/Class;" \
"Ljava/lang/Class;" \
"Ljava/lang/Class;" \
- "Ljava/lang/Class;" \
+ "I" \
"I" \
"Ljdk/internal/vm/vector/VectorSupport$Vector;" \
"Ljdk/internal/vm/vector/VectorSupport$VectorShuffle;" \
@@ -1207,18 +1208,18 @@ class methodHandle;
do_intrinsic(_VectorSelectFrom, jdk_internal_vm_vector_VectorSupport, vector_select_from_name, vector_select_from_sig, F_S) \
do_signature(vector_select_from_sig, "(Ljava/lang/Class;" \
"Ljava/lang/Class;" \
- "Ljava/lang/Class;" \
+ "I" \
"I" \
"Ljdk/internal/vm/vector/VectorSupport$Vector;" \
"Ljdk/internal/vm/vector/VectorSupport$Vector;" \
"Ljdk/internal/vm/vector/VectorSupport$VectorMask;" \
"Ljdk/internal/vm/vector/VectorSupport$VectorSelectFromOp;)" \
"Ljdk/internal/vm/vector/VectorSupport$Vector;") \
- do_name(vector_select_from_name, "selectFromOp") \
+ do_name(vector_select_from_name, "selectFromOp") \
\
do_intrinsic(_VectorExtract, jdk_internal_vm_vector_VectorSupport, vector_extract_name, vector_extract_sig, F_S) \
do_signature(vector_extract_sig, "(Ljava/lang/Class;" \
- "Ljava/lang/Class;" \
+ "I" \
"I" \
"Ljdk/internal/vm/vector/VectorSupport$VectorPayload;" \
"I" \
@@ -1228,7 +1229,7 @@ class methodHandle;
\
do_intrinsic(_VectorInsert, jdk_internal_vm_vector_VectorSupport, vector_insert_name, vector_insert_sig, F_S) \
do_signature(vector_insert_sig, "(Ljava/lang/Class;" \
- "Ljava/lang/Class;" \
+ "I" \
"I" \
"Ljdk/internal/vm/vector/VectorSupport$Vector;" \
"IJ" \
@@ -1240,7 +1241,7 @@ class methodHandle;
do_signature(vector_broadcast_int_sig, "(I" \
"Ljava/lang/Class;" \
"Ljava/lang/Class;" \
- "Ljava/lang/Class;" \
+ "I" \
"I" \
"Ljdk/internal/vm/vector/VectorSupport$Vector;" \
"I" \
@@ -1252,10 +1253,10 @@ class methodHandle;
do_intrinsic(_VectorConvert, jdk_internal_vm_vector_VectorSupport, vector_convert_name, vector_convert_sig, F_S) \
do_signature(vector_convert_sig, "(I" \
"Ljava/lang/Class;" \
- "Ljava/lang/Class;" \
+ "I" \
"I" \
"Ljava/lang/Class;" \
- "Ljava/lang/Class;" \
+ "I" \
"I" \
"Ljdk/internal/vm/vector/VectorSupport$VectorPayload;" \
"Ljdk/internal/vm/vector/VectorSupport$VectorSpecies;" \
@@ -1266,7 +1267,7 @@ class methodHandle;
do_intrinsic(_VectorGatherOp, jdk_internal_vm_vector_VectorSupport, vector_gather_name, vector_gather_sig, F_S) \
do_signature(vector_gather_sig, "(Ljava/lang/Class;" \
"Ljava/lang/Class;" \
- "Ljava/lang/Class;" \
+ "I" \
"I" \
"Ljava/lang/Class;" \
"I" \
@@ -1287,7 +1288,7 @@ class methodHandle;
do_intrinsic(_VectorScatterOp, jdk_internal_vm_vector_VectorSupport, vector_scatter_name, vector_scatter_sig, F_S) \
do_signature(vector_scatter_sig, "(Ljava/lang/Class;" \
"Ljava/lang/Class;" \
- "Ljava/lang/Class;" \
+ "I" \
"I" \
"Ljava/lang/Class;" \
"I" \
@@ -1309,7 +1310,7 @@ class methodHandle;
do_intrinsic(_VectorMaskOp, jdk_internal_vm_vector_VectorSupport, vector_mask_oper_name, vector_mask_oper_sig, F_S) \
do_signature(vector_mask_oper_sig, "(I" \
"Ljava/lang/Class;" \
- "Ljava/lang/Class;" \
+ "I" \
"I" \
"Ljdk/internal/vm/vector/VectorSupport$VectorMask;" \
"Ljdk/internal/vm/vector/VectorSupport$VectorMaskOp;)" \
@@ -1320,7 +1321,7 @@ class methodHandle;
do_signature(vector_compress_expand_op_sig, "(I" \
"Ljava/lang/Class;" \
"Ljava/lang/Class;" \
- "Ljava/lang/Class;" \
+ "I" \
"I" \
"Ljdk/internal/vm/vector/VectorSupport$Vector;" \
"Ljdk/internal/vm/vector/VectorSupport$VectorMask;" \
@@ -1330,7 +1331,7 @@ class methodHandle;
\
do_intrinsic(_IndexVector, jdk_internal_vm_vector_VectorSupport, index_vector_op_name, index_vector_op_sig, F_S) \
do_signature(index_vector_op_sig, "(Ljava/lang/Class;" \
- "Ljava/lang/Class;" \
+ "I" \
"I" \
"Ljdk/internal/vm/vector/VectorSupport$Vector;" \
"I" \
@@ -1341,7 +1342,7 @@ class methodHandle;
\
do_intrinsic(_IndexPartiallyInUpperRange, jdk_internal_vm_vector_VectorSupport, index_partially_in_upper_range_name, index_partially_in_upper_range_sig, F_S)\
do_signature(index_partially_in_upper_range_sig, "(Ljava/lang/Class;" \
- "Ljava/lang/Class;" \
+ "I" \
"I" \
"J" \
"J" \
diff --git a/src/hotspot/share/classfile/vmSymbols.hpp b/src/hotspot/share/classfile/vmSymbols.hpp
index 79646f24d0e..0054b7ba3f2 100644
--- a/src/hotspot/share/classfile/vmSymbols.hpp
+++ b/src/hotspot/share/classfile/vmSymbols.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -748,8 +748,6 @@ class SerializeClosure;
template(jdk_internal_vm_ThreadLock, "jdk/internal/vm/ThreadSnapshot$ThreadLock") \
template(jdk_internal_vm_ThreadLock_signature, "Ljdk/internal/vm/ThreadSnapshot$ThreadLock;") \
template(jdk_internal_vm_ThreadLock_array, "[Ljdk/internal/vm/ThreadSnapshot$ThreadLock;") \
- template(java_lang_StackTraceElement_of_name, "of") \
- template(java_lang_StackTraceElement_of_signature, "([Ljava/lang/StackTraceElement;)[Ljava/lang/StackTraceElement;") \
\
/* jcmd Thread.vthread_scheduler and Thread.vthread_pollers */ \
template(jdk_internal_vm_JcmdVThreadCommands, "jdk/internal/vm/JcmdVThreadCommands") \
diff --git a/src/hotspot/share/code/codeBlob.cpp b/src/hotspot/share/code/codeBlob.cpp
index a0a34ec23fa..094b4f82cf0 100644
--- a/src/hotspot/share/code/codeBlob.cpp
+++ b/src/hotspot/share/code/codeBlob.cpp
@@ -520,6 +520,8 @@ VtableBlob* VtableBlob::create(const char* name, int buffer_size) {
// eventually.
return nullptr;
}
+
+ MACOS_AARCH64_ONLY(os::thread_wx_enable_write());
blob = new (size) VtableBlob(name, size);
CodeCache_lock->unlock();
}
diff --git a/src/hotspot/share/code/codeCache.cpp b/src/hotspot/share/code/codeCache.cpp
index 95a2fb908de..481eb51bd5c 100644
--- a/src/hotspot/share/code/codeCache.cpp
+++ b/src/hotspot/share/code/codeCache.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -248,13 +248,61 @@ void CodeCache::initialize_heaps() {
set_size_of_unset_code_heap(&non_nmethod, cache_size, profiled.size + non_profiled.size, non_nmethod_min_size);
}
- size_t total = non_nmethod.size + profiled.size + non_profiled.size;
- if (total != cache_size && !cache_size_set) {
- log_info(codecache)("ReservedCodeCache size %zuK changed to total segments size NonNMethod "
- "%zuK NonProfiled %zuK Profiled %zuK = %zuK",
- cache_size/K, non_nmethod.size/K, non_profiled.size/K, profiled.size/K, total/K);
- // Adjust ReservedCodeCacheSize as necessary because it was not set explicitly
- cache_size = total;
+ // Note: if large page support is enabled, min_size is at least the large
+ // page size. This ensures that the code cache is covered by large pages.
+ non_nmethod.size = align_up(non_nmethod.size, min_size);
+ profiled.size = align_up(profiled.size, min_size);
+ non_profiled.size = align_up(non_profiled.size, min_size);
+
+ size_t aligned_total = non_nmethod.size + profiled.size + non_profiled.size;
+ if (!cache_size_set) {
+ // If ReservedCodeCacheSize is explicitly set and exceeds CODE_CACHE_SIZE_LIMIT,
+ // it is rejected by flag validation elsewhere. Here we only handle the case
+ // where ReservedCodeCacheSize is not set explicitly, but the computed segmented
+ // sizes (after alignment) exceed the platform limit.
+ if (aligned_total > CODE_CACHE_SIZE_LIMIT) {
+ err_msg message("ReservedCodeCacheSize (%zuK), Max (%zuK)."
+ "Segments: NonNMethod (%zuK), NonProfiled (%zuK), Profiled (%zuK).",
+ aligned_total/K, CODE_CACHE_SIZE_LIMIT/K,
+ non_nmethod.size/K, non_profiled.size/K, profiled.size/K);
+ vm_exit_during_initialization("Code cache size exceeds platform limit", message);
+ }
+ if (aligned_total != cache_size) {
+ log_info(codecache)("ReservedCodeCache size %zuK changed to total segments size NonNMethod "
+ "%zuK NonProfiled %zuK Profiled %zuK = %zuK",
+ cache_size/K, non_nmethod.size/K, non_profiled.size/K, profiled.size/K, aligned_total/K);
+ // Adjust ReservedCodeCacheSize as necessary because it was not set explicitly
+ cache_size = aligned_total;
+ }
+ } else {
+ check_min_size("reserved code cache", cache_size, min_cache_size);
+ // ReservedCodeCacheSize was set explicitly, so treat it as a hard cap.
+ // If alignment causes the total to exceed the cap, shrink unset heaps
+ // in min_size steps, never below their minimum sizes.
+ //
+ // A total smaller than cache_size typically happens when all segment sizes
+ // are explicitly set. In that case there is nothing to adjust, so we
+ // only validate the sizes.
+ if (aligned_total > cache_size) {
+ size_t delta = (aligned_total - cache_size) / min_size;
+ while (delta > 0) {
+ size_t start_delta = delta;
+ // Do not shrink the non-nmethod heap here: running out of non-nmethod space
+ // is more critical and may lead to unrecoverable VM errors.
+ if (non_profiled.enabled && !non_profiled.set && non_profiled.size > min_size) {
+ non_profiled.size -= min_size;
+ if (--delta == 0) break;
+ }
+ if (profiled.enabled && !profiled.set && profiled.size > min_size) {
+ profiled.size -= min_size;
+ delta--;
+ }
+ if (delta == start_delta) {
+ break;
+ }
+ }
+ aligned_total = non_nmethod.size + profiled.size + non_profiled.size;
+ }
}
log_debug(codecache)("Initializing code heaps ReservedCodeCache %zuK NonNMethod %zuK"
@@ -270,12 +318,9 @@ void CodeCache::initialize_heaps() {
if (non_profiled.enabled) { // non_profiled.enabled is always ON for segmented code heap, leave it checked for clarity
check_min_size("non-profiled code heap", non_profiled.size, min_size);
}
- if (cache_size_set) {
- check_min_size("reserved code cache", cache_size, min_cache_size);
- }
// ReservedCodeCacheSize was set explicitly, so report an error and abort if it doesn't match the segment sizes
- if (total != cache_size && cache_size_set) {
+ if (aligned_total != cache_size && cache_size_set) {
err_msg message("NonNMethodCodeHeapSize (%zuK)", non_nmethod.size/K);
if (profiled.enabled) {
message.append(" + ProfiledCodeHeapSize (%zuK)", profiled.size/K);
@@ -283,8 +328,8 @@ void CodeCache::initialize_heaps() {
if (non_profiled.enabled) {
message.append(" + NonProfiledCodeHeapSize (%zuK)", non_profiled.size/K);
}
- message.append(" = %zuK", total/K);
- message.append((total > cache_size) ? " is greater than " : " is less than ");
+ message.append(" = %zuK", aligned_total/K);
+ message.append((aligned_total > cache_size) ? " is greater than " : " is less than ");
message.append("ReservedCodeCacheSize (%zuK).", cache_size/K);
vm_exit_during_initialization("Invalid code heap sizes", message);
@@ -300,13 +345,6 @@ void CodeCache::initialize_heaps() {
}
}
- // Note: if large page support is enabled, min_size is at least the large
- // page size. This ensures that the code cache is covered by large pages.
- non_nmethod.size = align_up(non_nmethod.size, min_size);
- profiled.size = align_up(profiled.size, min_size);
- non_profiled.size = align_up(non_profiled.size, min_size);
- cache_size = non_nmethod.size + profiled.size + non_profiled.size;
-
FLAG_SET_ERGO(NonNMethodCodeHeapSize, non_nmethod.size);
FLAG_SET_ERGO(ProfiledCodeHeapSize, profiled.size);
FLAG_SET_ERGO(NonProfiledCodeHeapSize, non_profiled.size);
diff --git a/src/hotspot/share/code/nmethod.cpp b/src/hotspot/share/code/nmethod.cpp
index edfca5c98ee..13eb1ff1604 100644
--- a/src/hotspot/share/code/nmethod.cpp
+++ b/src/hotspot/share/code/nmethod.cpp
@@ -2137,6 +2137,9 @@ void nmethod::make_deoptimized() {
ResourceMark rm;
RelocIterator iter(this, oops_reloc_begin());
+ // Assume there will be some calls to make deoptimized.
+ MACOS_AARCH64_ONLY(os::thread_wx_enable_write());
+
while (iter.next()) {
switch (iter.type()) {
@@ -2213,6 +2216,7 @@ void nmethod::verify_clean_inline_caches() {
}
void nmethod::mark_as_maybe_on_stack() {
+ MACOS_AARCH64_ONLY(os::thread_wx_enable_write());
AtomicAccess::store(&_gc_epoch, CodeCache::gc_epoch());
}
@@ -2305,6 +2309,8 @@ bool nmethod::make_not_entrant(InvalidationReason invalidation_reason) {
return false;
}
+ MACOS_AARCH64_ONLY(os::thread_wx_enable_write());
+
{
// Enter critical section. Does not block for safepoint.
ConditionalMutexLocker ml(NMethodState_lock, !NMethodState_lock->owned_by_self(), Mutex::_no_safepoint_check_flag);
@@ -2740,6 +2746,8 @@ bool nmethod::is_unloading() {
state_is_unloading = IsUnloadingBehaviour::is_unloading(this);
uint8_t new_state = IsUnloadingState::create(state_is_unloading, state_unloading_cycle);
+ MACOS_AARCH64_ONLY(os::thread_wx_enable_write());
+
// Note that if an nmethod has dead oops, everyone will agree that the
// nmethod is_unloading. However, the is_cold heuristics can yield
// different outcomes, so we guard the computed result with a CAS
diff --git a/src/hotspot/share/code/vtableStubs.cpp b/src/hotspot/share/code/vtableStubs.cpp
index b926888595d..35b226a8798 100644
--- a/src/hotspot/share/code/vtableStubs.cpp
+++ b/src/hotspot/share/code/vtableStubs.cpp
@@ -51,6 +51,9 @@ VMReg VtableStub::_receiver_location = VMRegImpl::Bad();
void* VtableStub::operator new(size_t size, int code_size) throw() {
assert_lock_strong(VtableStubs_lock);
assert(size == sizeof(VtableStub), "mismatched size");
+
+ MACOS_AARCH64_ONLY(os::thread_wx_enable_write());
+
// compute real VtableStub size (rounded to nearest word)
const int real_size = align_up(code_size + (int)sizeof(VtableStub), wordSize);
// malloc them in chunks to minimize header overhead
diff --git a/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp b/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp
index 61402301eb1..34d31702e80 100644
--- a/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp
+++ b/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -351,7 +351,6 @@ Node* G1BarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) co
Node* G1BarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
Node* new_val, const Type* value_type) const {
- GraphKit* kit = access.kit();
if (!access.is_oop()) {
return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
}
@@ -361,7 +360,6 @@ Node* G1BarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access
Node* G1BarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
Node* new_val, const Type* value_type) const {
- GraphKit* kit = access.kit();
if (!access.is_oop()) {
return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
}
@@ -370,7 +368,6 @@ Node* G1BarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& acces
}
Node* G1BarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
- GraphKit* kit = access.kit();
if (!access.is_oop()) {
return BarrierSetC2::atomic_xchg_at_resolved(access, new_val, value_type);
}
diff --git a/src/hotspot/share/gc/g1/g1BarrierSet.cpp b/src/hotspot/share/gc/g1/g1BarrierSet.cpp
index 622651ce0d8..dee50500e07 100644
--- a/src/hotspot/share/gc/g1/g1BarrierSet.cpp
+++ b/src/hotspot/share/gc/g1/g1BarrierSet.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -64,13 +64,13 @@ G1BarrierSet::G1BarrierSet(G1CardTable* card_table,
{}
G1BarrierSet::~G1BarrierSet() {
- delete _refinement_table;
+ delete refinement_table();
}
void G1BarrierSet::swap_global_card_table() {
- G1CardTable* temp = static_cast(_card_table);
- _card_table = _refinement_table;
- _refinement_table = temp;
+ G1CardTable* temp = static_cast(card_table());
+ _card_table.store_relaxed(refinement_table());
+ _refinement_table.store_relaxed(temp);
}
void G1BarrierSet::update_card_table_base(Thread* thread) {
@@ -80,7 +80,7 @@ void G1BarrierSet::update_card_table_base(Thread* thread) {
assert(thread->is_Java_thread(), "may only update card table base of JavaThreads, not %s", thread->name());
}
#endif
- G1ThreadLocalData::set_byte_map_base(thread, _card_table->byte_map_base());
+ G1ThreadLocalData::set_byte_map_base(thread, card_table()->byte_map_base());
}
template void
@@ -135,10 +135,10 @@ void G1BarrierSet::write_region(MemRegion mr) {
// marks next time.
// If we write to the old card table (after the switching, then the refinement
// table) the oncoming handshake will do the memory synchronization.
- CardTable* card_table = AtomicAccess::load(&_card_table);
+ CardTable* local_card_table = card_table();
- volatile CardValue* byte = card_table->byte_for(mr.start());
- CardValue* last_byte = card_table->byte_for(mr.last());
+ volatile CardValue* byte = local_card_table->byte_for(mr.start());
+ CardValue* last_byte = local_card_table->byte_for(mr.last());
// Dirty cards only if necessary.
for (; byte <= last_byte; byte++) {
@@ -190,6 +190,6 @@ void G1BarrierSet::on_thread_detach(Thread* thread) {
}
void G1BarrierSet::print_on(outputStream* st) const {
- _card_table->print_on(st, "Card");
- _refinement_table->print_on(st, "Refinement");
+ card_table()->print_on(st, "Card");
+ refinement_table()->print_on(st, "Refinement");
}
diff --git a/src/hotspot/share/gc/g1/g1BarrierSet.hpp b/src/hotspot/share/gc/g1/g1BarrierSet.hpp
index bf595973a32..406096acf10 100644
--- a/src/hotspot/share/gc/g1/g1BarrierSet.hpp
+++ b/src/hotspot/share/gc/g1/g1BarrierSet.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -29,6 +29,7 @@
#include "gc/shared/bufferNode.hpp"
#include "gc/shared/cardTable.hpp"
#include "gc/shared/cardTableBarrierSet.hpp"
+#include "runtime/atomic.hpp"
class G1CardTable;
class Thread;
@@ -66,7 +67,7 @@ class G1BarrierSet: public CardTableBarrierSet {
BufferNode::Allocator _satb_mark_queue_buffer_allocator;
G1SATBMarkQueueSet _satb_mark_queue_set;
- G1CardTable* _refinement_table;
+ Atomic _refinement_table;
public:
G1BarrierSet(G1CardTable* card_table, G1CardTable* refinement_table);
@@ -76,7 +77,7 @@ class G1BarrierSet: public CardTableBarrierSet {
return barrier_set_cast(BarrierSet::barrier_set());
}
- G1CardTable* refinement_table() const { return _refinement_table; }
+ G1CardTable* refinement_table() const { return _refinement_table.load_relaxed(); }
// Swap the global card table references, without synchronization.
void swap_global_card_table();
diff --git a/src/hotspot/share/gc/g1/g1BarrierSet.inline.hpp b/src/hotspot/share/gc/g1/g1BarrierSet.inline.hpp
index 794e5db0634..54892c9191d 100644
--- a/src/hotspot/share/gc/g1/g1BarrierSet.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1BarrierSet.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -73,8 +73,8 @@ inline void G1BarrierSet::write_ref_field_post(T* field) {
// Make sure that the card table reference is read only once. Otherwise the compiler
// might reload that value in the two accesses below, that could cause writes to
// the wrong card table.
- CardTable* card_table = AtomicAccess::load(&_card_table);
- CardValue* byte = card_table->byte_for(field);
+ CardTable* local_card_table = card_table();
+ CardValue* byte = local_card_table->byte_for(field);
if (*byte == G1CardTable::clean_card_val()) {
*byte = G1CardTable::dirty_card_val();
}
diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp
index 8f83a653885..8fc2c7c9941 100644
--- a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp
+++ b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp
@@ -138,6 +138,26 @@ void G1RegionMappingChangedListener::on_commit(uint start_idx, size_t num_region
reset_from_card_cache(start_idx, num_regions);
}
+// Collects commonly used scoped objects that are related to initial setup.
+class G1GCMark : StackObj {
+ ResourceMark _rm;
+ IsSTWGCActiveMark _active_gc_mark;
+ GCIdMark _gc_id_mark;
+ SvcGCMarker _sgcm;
+ GCTraceCPUTime _tcpu;
+
+public:
+ G1GCMark(GCTracer* tracer, bool is_full_gc) :
+ _rm(),
+ _active_gc_mark(),
+ _gc_id_mark(),
+ _sgcm(is_full_gc ? SvcGCMarker::FULL : SvcGCMarker::MINOR),
+ _tcpu(tracer) {
+
+ assert_at_safepoint_on_vm_thread();
+ }
+};
+
void G1CollectedHeap::run_batch_task(G1BatchedTask* cl) {
uint num_workers = MAX2(1u, MIN2(cl->num_workers_estimate(), workers()->active_workers()));
cl->set_max_workers(num_workers);
@@ -914,12 +934,11 @@ void G1CollectedHeap::verify_after_full_collection() {
void G1CollectedHeap::do_full_collection(size_t allocation_word_size,
bool clear_all_soft_refs,
bool do_maximal_compaction) {
- assert_at_safepoint_on_vm_thread();
-
- G1FullGCMark gc_mark;
+ G1FullGCTracer tracer;
+ G1GCMark gc_mark(&tracer, true /* is_full_gc */);
GCTraceTime(Info, gc) tm("Pause Full", nullptr, gc_cause(), true);
- G1FullCollector collector(this, clear_all_soft_refs, do_maximal_compaction, gc_mark.tracer());
+ G1FullCollector collector(this, clear_all_soft_refs, do_maximal_compaction, &tracer);
collector.prepare_collection();
collector.collect();
collector.complete_collection(allocation_word_size);
@@ -1320,7 +1339,6 @@ G1CollectedHeap::G1CollectedHeap() :
_card_set_freelist_pool(G1CardSetConfiguration::num_mem_object_types()),
_young_regions_cset_group(card_set_config(), &_card_set_freelist_pool, G1CSetCandidateGroup::YoungRegionId),
_cm(nullptr),
- _cm_thread(nullptr),
_cr(nullptr),
_task_queues(nullptr),
_partial_array_state_manager(nullptr),
@@ -1564,7 +1582,6 @@ jint G1CollectedHeap::initialize() {
// Create the G1ConcurrentMark data structure and thread.
// (Must do this late, so that "max_[reserved_]regions" is defined.)
_cm = new G1ConcurrentMark(this, bitmap_storage);
- _cm_thread = _cm->cm_thread();
// Now expand into the initial heap size.
if (!expand(init_byte_size, _workers)) {
@@ -1636,7 +1653,9 @@ jint G1CollectedHeap::initialize() {
}
bool G1CollectedHeap::concurrent_mark_is_terminating() const {
- return _cm_thread->should_terminate();
+ assert(_cm != nullptr, "_cm must have been created");
+ assert(_cm->is_fully_initialized(), "thread must exist in order to check if mark is terminating");
+ return _cm->cm_thread()->should_terminate();
}
void G1CollectedHeap::stop() {
@@ -1645,7 +1664,9 @@ void G1CollectedHeap::stop() {
// that are destroyed during shutdown.
_cr->stop();
_service_thread->stop();
- _cm_thread->stop();
+ if (_cm->is_fully_initialized()) {
+ _cm->cm_thread()->stop();
+ }
}
void G1CollectedHeap::safepoint_synchronize_begin() {
@@ -1842,7 +1863,7 @@ void G1CollectedHeap::increment_old_marking_cycles_completed(bool concurrent,
// is set) so that if a waiter requests another System.gc() it doesn't
// incorrectly see that a marking cycle is still in progress.
if (concurrent) {
- _cm_thread->set_idle();
+ _cm->cm_thread()->set_idle();
}
// Notify threads waiting in System.gc() (with ExplicitGCInvokesConcurrent)
@@ -2421,7 +2442,6 @@ void G1CollectedHeap::print_gc_on(outputStream* st) const {
void G1CollectedHeap::gc_threads_do(ThreadClosure* tc) const {
workers()->threads_do(tc);
- tc->do_thread(_cm_thread);
_cm->threads_do(tc);
_cr->threads_do(tc);
tc->do_thread(_service_thread);
@@ -2542,15 +2562,15 @@ HeapWord* G1CollectedHeap::do_collection_pause(size_t word_size,
}
void G1CollectedHeap::start_concurrent_cycle(bool concurrent_operation_is_full_mark) {
- assert(!_cm_thread->in_progress(), "Can not start concurrent operation while in progress");
-
+ assert(_cm->is_fully_initialized(), "sanity");
+ assert(!_cm->in_progress(), "Can not start concurrent operation while in progress");
MutexLocker x(G1CGC_lock, Mutex::_no_safepoint_check_flag);
if (concurrent_operation_is_full_mark) {
_cm->post_concurrent_mark_start();
- _cm_thread->start_full_mark();
+ _cm->cm_thread()->start_full_mark();
} else {
_cm->post_concurrent_undo_start();
- _cm_thread->start_undo_mark();
+ _cm->cm_thread()->start_undo_mark();
}
G1CGC_lock->notify();
}
@@ -2713,19 +2733,12 @@ void G1CollectedHeap::flush_region_pin_cache() {
}
void G1CollectedHeap::do_collection_pause_at_safepoint(size_t allocation_word_size) {
- assert_at_safepoint_on_vm_thread();
- assert(!is_stw_gc_active(), "collection is not reentrant");
-
- ResourceMark rm;
-
- IsSTWGCActiveMark active_gc_mark;
- GCIdMark gc_id_mark;
- SvcGCMarker sgcm(SvcGCMarker::MINOR);
-
- GCTraceCPUTime tcpu(_gc_tracer_stw);
+ G1GCMark gcm(_gc_tracer_stw, false /* is_full_gc */);
_bytes_used_during_gc = 0;
+ _cm->fully_initialize();
+
policy()->decide_on_concurrent_start_pause();
// Record whether this pause may need to trigger a concurrent operation. Later,
// when we signal the G1ConcurrentMarkThread, the collector state has already
diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.hpp b/src/hotspot/share/gc/g1/g1CollectedHeap.hpp
index 1f900c76851..8ff9d481000 100644
--- a/src/hotspot/share/gc/g1/g1CollectedHeap.hpp
+++ b/src/hotspot/share/gc/g1/g1CollectedHeap.hpp
@@ -823,7 +823,6 @@ public:
// The concurrent marker (and the thread it runs in.)
G1ConcurrentMark* _cm;
- G1ConcurrentMarkThread* _cm_thread;
// The concurrent refiner.
G1ConcurrentRefine* _cr;
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp b/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp
index 2bbfb5032b3..8f3cafe1f5b 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp
@@ -382,12 +382,12 @@ G1CMRootMemRegions::~G1CMRootMemRegions() {
}
void G1CMRootMemRegions::reset() {
- _num_root_regions = 0;
+ _num_root_regions.store_relaxed(0);
}
void G1CMRootMemRegions::add(HeapWord* start, HeapWord* end) {
assert_at_safepoint();
- size_t idx = AtomicAccess::fetch_then_add(&_num_root_regions, 1u);
+ size_t idx = _num_root_regions.fetch_then_add(1u);
assert(idx < _max_regions, "Trying to add more root MemRegions than there is space %zu", _max_regions);
assert(start != nullptr && end != nullptr && start <= end, "Start (" PTR_FORMAT ") should be less or equal to "
"end (" PTR_FORMAT ")", p2i(start), p2i(end));
@@ -398,36 +398,38 @@ void G1CMRootMemRegions::add(HeapWord* start, HeapWord* end) {
void G1CMRootMemRegions::prepare_for_scan() {
assert(!scan_in_progress(), "pre-condition");
- _scan_in_progress = _num_root_regions > 0;
+ _scan_in_progress.store_relaxed(num_root_regions() > 0);
- _claimed_root_regions = 0;
- _should_abort = false;
+ _claimed_root_regions.store_relaxed(0);
+ _should_abort.store_relaxed(false);
}
const MemRegion* G1CMRootMemRegions::claim_next() {
- if (_should_abort) {
+ if (_should_abort.load_relaxed()) {
// If someone has set the should_abort flag, we return null to
// force the caller to bail out of their loop.
return nullptr;
}
- if (_claimed_root_regions >= _num_root_regions) {
+ uint local_num_root_regions = num_root_regions();
+ if (_claimed_root_regions.load_relaxed() >= local_num_root_regions) {
return nullptr;
}
- size_t claimed_index = AtomicAccess::fetch_then_add(&_claimed_root_regions, 1u);
- if (claimed_index < _num_root_regions) {
+ size_t claimed_index = _claimed_root_regions.fetch_then_add(1u);
+ if (claimed_index < local_num_root_regions) {
return &_root_regions[claimed_index];
}
return nullptr;
}
uint G1CMRootMemRegions::num_root_regions() const {
- return (uint)_num_root_regions;
+ return (uint)_num_root_regions.load_relaxed();
}
bool G1CMRootMemRegions::contains(const MemRegion mr) const {
- for (uint i = 0; i < _num_root_regions; i++) {
+ uint local_num_root_regions = num_root_regions();
+ for (uint i = 0; i < local_num_root_regions; i++) {
if (_root_regions[i].equals(mr)) {
return true;
}
@@ -437,7 +439,7 @@ bool G1CMRootMemRegions::contains(const MemRegion mr) const {
void G1CMRootMemRegions::notify_scan_done() {
MutexLocker x(G1RootRegionScan_lock, Mutex::_no_safepoint_check_flag);
- _scan_in_progress = false;
+ _scan_in_progress.store_relaxed(false);
G1RootRegionScan_lock->notify_all();
}
@@ -448,10 +450,10 @@ void G1CMRootMemRegions::cancel_scan() {
void G1CMRootMemRegions::scan_finished() {
assert(scan_in_progress(), "pre-condition");
- if (!_should_abort) {
- assert(_claimed_root_regions >= num_root_regions(),
+ if (!_should_abort.load_relaxed()) {
+ assert(_claimed_root_regions.load_relaxed() >= num_root_regions(),
"we should have claimed all root regions, claimed %zu, length = %u",
- _claimed_root_regions, num_root_regions());
+ _claimed_root_regions.load_relaxed(), num_root_regions());
}
notify_scan_done();
@@ -473,7 +475,7 @@ bool G1CMRootMemRegions::wait_until_scan_finished() {
G1ConcurrentMark::G1ConcurrentMark(G1CollectedHeap* g1h,
G1RegionToSpaceMapper* bitmap_storage) :
- // _cm_thread set inside the constructor
+ _cm_thread(nullptr),
_g1h(g1h),
_mark_bitmap(),
@@ -484,13 +486,12 @@ G1ConcurrentMark::G1ConcurrentMark(G1CollectedHeap* g1h,
_global_mark_stack(),
- // _finger set in set_non_marking_state
+ _finger(nullptr), // _finger set in set_non_marking_state
_worker_id_offset(G1ConcRefinementThreads), // The refinement control thread does not refine cards, so it's just the worker threads.
_max_num_tasks(MAX2(ConcGCThreads, ParallelGCThreads)),
- // _num_active_tasks set in set_non_marking_state()
- // _tasks set inside the constructor
-
+ _num_active_tasks(0), // _num_active_tasks set in set_non_marking_state()
+ _tasks(nullptr), // _tasks set inside late_init()
_task_queues(new G1CMTaskQueueSet(_max_num_tasks)),
_terminator(_max_num_tasks, _task_queues),
_partial_array_state_manager(new PartialArrayStateManager(_max_num_tasks)),
@@ -525,6 +526,12 @@ G1ConcurrentMark::G1ConcurrentMark(G1CollectedHeap* g1h,
assert(G1CGC_lock != nullptr, "CGC_lock must be initialized");
_mark_bitmap.initialize(g1h->reserved(), bitmap_storage);
+}
+
+void G1ConcurrentMark::fully_initialize() {
+ if (is_fully_initialized()) {
+ return;
+ }
// Create & start ConcurrentMark thread.
_cm_thread = new G1ConcurrentMarkThread(this);
@@ -560,6 +567,10 @@ G1ConcurrentMark::G1ConcurrentMark(G1CollectedHeap* g1h,
reset_at_marking_complete();
}
+bool G1ConcurrentMark::in_progress() const {
+ return is_fully_initialized() ? _cm_thread->in_progress() : false;
+}
+
PartialArrayStateManager* G1ConcurrentMark::partial_array_state_manager() const {
return _partial_array_state_manager;
}
@@ -628,8 +639,7 @@ void G1ConcurrentMark::reset_marking_for_restart() {
_finger = _heap.start();
for (uint i = 0; i < _max_num_tasks; ++i) {
- G1CMTaskQueue* queue = _task_queues->queue(i);
- queue->set_empty();
+ _tasks[i]->reset_for_restart();
}
}
@@ -765,7 +775,7 @@ private:
// as asserts here to minimize their overhead on the product. However, we
// will have them as guarantees at the beginning / end of the bitmap
// clearing to get some checking in the product.
- assert(!suspendible() || _cm->cm_thread()->in_progress(), "invariant");
+ assert(!suspendible() || _cm->in_progress(), "invariant");
assert(!suspendible() || !G1CollectedHeap::heap()->collector_state()->mark_or_rebuild_in_progress(), "invariant");
// Abort iteration if necessary.
@@ -821,7 +831,8 @@ void G1ConcurrentMark::clear_bitmap(WorkerThreads* workers, bool may_yield) {
void G1ConcurrentMark::cleanup_for_next_mark() {
// Make sure that the concurrent mark thread looks to still be in
// the current cycle.
- guarantee(cm_thread()->in_progress(), "invariant");
+ guarantee(is_fully_initialized(), "should be initializd");
+ guarantee(in_progress(), "invariant");
// We are finishing up the current cycle by clearing the next
// marking bitmap and getting it ready for the next cycle. During
@@ -834,7 +845,8 @@ void G1ConcurrentMark::cleanup_for_next_mark() {
reset_partial_array_state_manager();
// Repeat the asserts from above.
- guarantee(cm_thread()->in_progress(), "invariant");
+ guarantee(is_fully_initialized(), "should be initializd");
+ guarantee(in_progress(), "invariant");
guarantee(!_g1h->collector_state()->mark_or_rebuild_in_progress(), "invariant");
}
@@ -1925,15 +1937,12 @@ bool G1ConcurrentMark::concurrent_cycle_abort() {
// nothing, but this situation should be extremely rare (a full gc after shutdown
// has been signalled is already rare), and this work should be negligible compared
// to actual full gc work.
- if (!cm_thread()->in_progress() && !_g1h->concurrent_mark_is_terminating()) {
+
+ if (!is_fully_initialized() || (!cm_thread()->in_progress() && !_g1h->concurrent_mark_is_terminating())) {
return false;
}
- // Empty mark stack
reset_marking_for_restart();
- for (uint i = 0; i < _max_num_tasks; ++i) {
- _tasks[i]->clear_region_fields();
- }
abort_marking_threads();
@@ -1987,6 +1996,10 @@ void G1ConcurrentMark::print_summary_info() {
}
log.trace(" Concurrent marking:");
+ if (!is_fully_initialized()) {
+ log.trace(" has not been initialized yet");
+ return;
+ }
print_ms_time_info(" ", "remarks", _remark_times);
{
print_ms_time_info(" ", "final marks", _remark_mark_times);
@@ -2003,7 +2016,10 @@ void G1ConcurrentMark::print_summary_info() {
}
void G1ConcurrentMark::threads_do(ThreadClosure* tc) const {
- _concurrent_workers->threads_do(tc);
+ if (is_fully_initialized()) { // they are initialized late
+ tc->do_thread(_cm_thread);
+ _concurrent_workers->threads_do(tc);
+ }
}
void G1ConcurrentMark::print_on(outputStream* st) const {
@@ -2097,6 +2113,13 @@ void G1CMTask::reset(G1CMBitMap* mark_bitmap) {
_mark_stats_cache.reset();
}
+void G1CMTask::reset_for_restart() {
+ clear_region_fields();
+ _task_queue->set_empty();
+ TASKQUEUE_STATS_ONLY(_partial_array_splitter.stats()->reset());
+ TASKQUEUE_STATS_ONLY(_task_queue->stats.reset());
+}
+
void G1CMTask::register_partial_array_splitter() {
::new (&_partial_array_splitter) PartialArraySplitter(_cm->partial_array_state_manager(),
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMark.hpp b/src/hotspot/share/gc/g1/g1ConcurrentMark.hpp
index 836d7793f81..0271e6a4208 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentMark.hpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentMark.hpp
@@ -290,12 +290,12 @@ class G1CMRootMemRegions {
MemRegion* _root_regions;
size_t const _max_regions;
- volatile size_t _num_root_regions; // Actual number of root regions.
+ Atomic _num_root_regions; // Actual number of root regions.
- volatile size_t _claimed_root_regions; // Number of root regions currently claimed.
+ Atomic _claimed_root_regions; // Number of root regions currently claimed.
- volatile bool _scan_in_progress;
- volatile bool _should_abort;
+ Atomic _scan_in_progress;
+ Atomic _should_abort;
void notify_scan_done();
@@ -312,11 +312,11 @@ public:
void prepare_for_scan();
// Forces get_next() to return null so that the iteration aborts early.
- void abort() { _should_abort = true; }
+ void abort() { _should_abort.store_relaxed(true); }
// Return true if the CM thread are actively scanning root regions,
// false otherwise.
- bool scan_in_progress() { return _scan_in_progress; }
+ bool scan_in_progress() { return _scan_in_progress.load_relaxed(); }
// Claim the next root MemRegion to scan atomically, or return null if
// all have been claimed.
@@ -555,6 +555,9 @@ public:
uint worker_id_offset() const { return _worker_id_offset; }
+ void fully_initialize();
+ bool is_fully_initialized() const { return _cm_thread != nullptr; }
+ bool in_progress() const;
uint max_num_tasks() const {return _max_num_tasks; }
// Clear statistics gathered during the concurrent cycle for the given region after
@@ -841,8 +844,10 @@ private:
// Apply the closure to the given range of elements in the objArray.
inline void process_array_chunk(objArrayOop obj, size_t start, size_t end);
public:
- // Resets the task; should be called right at the beginning of a marking phase.
+ // Resets the task completely for a new marking; should be called right at the beginning of a marking phase.
void reset(G1CMBitMap* mark_bitmap);
+ // Minimal reset of the task, making it ready for continuing to mark.
+ void reset_for_restart();
// Register/unregister Partial Array Splitter Allocator with the PartialArrayStateManager.
// This allows us to discard memory arenas used for partial object array states at the end
// of a concurrent mark cycle.
diff --git a/src/hotspot/share/gc/g1/g1FullCollector.cpp b/src/hotspot/share/gc/g1/g1FullCollector.cpp
index 6c8cc7028cc..b6388c2f722 100644
--- a/src/hotspot/share/gc/g1/g1FullCollector.cpp
+++ b/src/hotspot/share/gc/g1/g1FullCollector.cpp
@@ -110,7 +110,7 @@ uint G1FullCollector::calc_active_workers() {
G1FullCollector::G1FullCollector(G1CollectedHeap* heap,
bool clear_soft_refs,
bool do_maximal_compaction,
- G1FullGCTracer* tracer) :
+ GCTracer* tracer) :
_heap(heap),
_scope(heap->monitoring_support(), clear_soft_refs, do_maximal_compaction, tracer),
_num_workers(calc_active_workers()),
@@ -276,6 +276,21 @@ void G1FullCollector::before_marking_update_attribute_table(G1HeapRegion* hr) {
class G1FullGCRefProcProxyTask : public RefProcProxyTask {
G1FullCollector& _collector;
+ // G1 Full GC specific closure for handling discovered fields. Do NOT need any
+ // barriers as Full GC discards all this information anyway.
+ class G1FullGCDiscoveredFieldClosure : public EnqueueDiscoveredFieldClosure {
+ G1CollectedHeap* _g1h;
+
+ public:
+ G1FullGCDiscoveredFieldClosure() : _g1h(G1CollectedHeap::heap()) { }
+
+ void enqueue(HeapWord* discovered_field_addr, oop value) override {
+ assert(_g1h->is_in(discovered_field_addr), PTR_FORMAT " is not in heap ", p2i(discovered_field_addr));
+ // Store the value and done.
+ RawAccess<>::oop_store(discovered_field_addr, value);
+ }
+ };
+
public:
G1FullGCRefProcProxyTask(G1FullCollector &collector, uint max_workers)
: RefProcProxyTask("G1FullGCRefProcProxyTask", max_workers),
@@ -286,7 +301,7 @@ public:
G1IsAliveClosure is_alive(&_collector);
uint index = (_tm == RefProcThreadModel::Single) ? 0 : worker_id;
G1FullKeepAliveClosure keep_alive(_collector.marker(index));
- BarrierEnqueueDiscoveredFieldClosure enqueue;
+ G1FullGCDiscoveredFieldClosure enqueue;
G1MarkStackClosure* complete_marking = _collector.marker(index)->stack_closure();
_rp_task->rp_work(worker_id, &is_alive, &keep_alive, &enqueue, complete_marking);
}
diff --git a/src/hotspot/share/gc/g1/g1FullCollector.hpp b/src/hotspot/share/gc/g1/g1FullCollector.hpp
index 7e455b07013..605556a2ba6 100644
--- a/src/hotspot/share/gc/g1/g1FullCollector.hpp
+++ b/src/hotspot/share/gc/g1/g1FullCollector.hpp
@@ -58,18 +58,6 @@ public:
}
};
-// Full GC Mark that holds GC id and CPU time trace. Needs to be separate
-// from the G1FullCollector and G1FullGCScope to allow the Full GC logging
-// to have the same structure as the Young GC logging.
-class G1FullGCMark : StackObj {
- GCIdMark _gc_id;
- G1FullGCTracer _tracer;
- GCTraceCPUTime _cpu_time;
-public:
- G1FullGCMark() : _gc_id(), _tracer(), _cpu_time(&_tracer) { }
- G1FullGCTracer* tracer() { return &_tracer; }
-};
-
// The G1FullCollector holds data associated with the current Full GC.
class G1FullCollector : StackObj {
G1CollectedHeap* _heap;
@@ -102,7 +90,7 @@ public:
G1FullCollector(G1CollectedHeap* heap,
bool clear_soft_refs,
bool do_maximal_compaction,
- G1FullGCTracer* tracer);
+ GCTracer* tracer);
~G1FullCollector();
void prepare_collection();
diff --git a/src/hotspot/share/gc/g1/g1FullGCScope.cpp b/src/hotspot/share/gc/g1/g1FullGCScope.cpp
index 8b92d51a8a3..cb4ebe423ff 100644
--- a/src/hotspot/share/gc/g1/g1FullGCScope.cpp
+++ b/src/hotspot/share/gc/g1/g1FullGCScope.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -38,18 +38,14 @@ G1FullGCJFRTracerMark::~G1FullGCJFRTracerMark() {
G1FullGCScope::G1FullGCScope(G1MonitoringSupport* monitoring_support,
bool clear_soft,
bool do_maximal_compaction,
- G1FullGCTracer* tracer) :
- _rm(),
+ GCTracer* tracer) :
_should_clear_soft_refs(clear_soft),
_do_maximal_compaction(do_maximal_compaction),
- _g1h(G1CollectedHeap::heap()),
- _svc_marker(SvcGCMarker::FULL),
_timer(),
_tracer(tracer),
- _active(),
_tracer_mark(&_timer, _tracer),
_monitoring_scope(monitoring_support),
- _heap_printer(_g1h),
+ _heap_printer(G1CollectedHeap::heap()),
_region_compaction_threshold(do_maximal_compaction ?
G1HeapRegion::GrainWords :
(1 - MarkSweepDeadRatio / 100.0) * G1HeapRegion::GrainWords) { }
@@ -58,7 +54,7 @@ STWGCTimer* G1FullGCScope::timer() {
return &_timer;
}
-G1FullGCTracer* G1FullGCScope::tracer() {
+GCTracer* G1FullGCScope::tracer() {
return _tracer;
}
diff --git a/src/hotspot/share/gc/g1/g1FullGCScope.hpp b/src/hotspot/share/gc/g1/g1FullGCScope.hpp
index 7a0d0a5395e..fc9d5a71f92 100644
--- a/src/hotspot/share/gc/g1/g1FullGCScope.hpp
+++ b/src/hotspot/share/gc/g1/g1FullGCScope.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -46,14 +46,10 @@ public:
// Class used to group scoped objects used in the Full GC together.
class G1FullGCScope : public StackObj {
- ResourceMark _rm;
bool _should_clear_soft_refs;
bool _do_maximal_compaction;
- G1CollectedHeap* _g1h;
- SvcGCMarker _svc_marker;
STWGCTimer _timer;
- G1FullGCTracer* _tracer;
- IsSTWGCActiveMark _active;
+ GCTracer* _tracer;
G1FullGCJFRTracerMark _tracer_mark;
G1FullGCMonitoringScope _monitoring_scope;
G1HeapPrinterMark _heap_printer;
@@ -63,13 +59,13 @@ public:
G1FullGCScope(G1MonitoringSupport* monitoring_support,
bool clear_soft,
bool do_maximal_compaction,
- G1FullGCTracer* tracer);
+ GCTracer* tracer);
bool should_clear_soft_refs() const { return _should_clear_soft_refs; }
bool do_maximal_compaction() { return _do_maximal_compaction; }
STWGCTimer* timer();
- G1FullGCTracer* tracer();
+ GCTracer* tracer();
size_t region_compaction_threshold() const;
};
diff --git a/src/hotspot/share/gc/g1/g1HeapRegion.cpp b/src/hotspot/share/gc/g1/g1HeapRegion.cpp
index 361e19d4be5..2052a3ce156 100644
--- a/src/hotspot/share/gc/g1/g1HeapRegion.cpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegion.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -44,7 +44,7 @@
#include "oops/access.inline.hpp"
#include "oops/compressedOops.inline.hpp"
#include "oops/oop.inline.hpp"
-#include "runtime/atomicAccess.hpp"
+#include "runtime/atomic.hpp"
#include "runtime/globals_extension.hpp"
#include "utilities/powerOfTwo.hpp"
@@ -131,8 +131,8 @@ void G1HeapRegion::hr_clear(bool clear_space) {
G1CollectedHeap::heap()->concurrent_mark()->reset_top_at_mark_start(this);
- _parsable_bottom = bottom();
- _garbage_bytes = 0;
+ _parsable_bottom.store_relaxed(bottom());
+ _garbage_bytes.store_relaxed(0);
_incoming_refs = 0;
if (clear_space) clear(SpaceDecorator::Mangle);
@@ -294,12 +294,12 @@ void G1HeapRegion::report_region_type_change(G1HeapRegionTraceType::Type to) {
// young gen regions never have their PB set to anything other than bottom.
assert(parsable_bottom_acquire() == bottom(), "must be");
- _garbage_bytes = 0;
+ _garbage_bytes.store_relaxed(0);
_incoming_refs = 0;
}
void G1HeapRegion::note_self_forward_chunk_done(size_t garbage_bytes) {
- AtomicAccess::add(&_garbage_bytes, garbage_bytes, memory_order_relaxed);
+ _garbage_bytes.add_then_fetch(garbage_bytes, memory_order_relaxed);
}
// Code roots support
@@ -448,7 +448,7 @@ void G1HeapRegion::print_on(outputStream* st) const {
st->print("|-");
}
}
- st->print("|%3zu", AtomicAccess::load(&_pinned_object_count));
+ st->print("|%3zu", _pinned_object_count.load_relaxed());
st->print_cr("");
}
diff --git a/src/hotspot/share/gc/g1/g1HeapRegion.hpp b/src/hotspot/share/gc/g1/g1HeapRegion.hpp
index fe915b0dafe..2b4b640d52b 100644
--- a/src/hotspot/share/gc/g1/g1HeapRegion.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegion.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -33,6 +33,7 @@
#include "gc/shared/ageTable.hpp"
#include "gc/shared/spaceDecorator.hpp"
#include "gc/shared/verifyOption.hpp"
+#include "runtime/atomic.hpp"
#include "runtime/mutex.hpp"
#include "utilities/macros.hpp"
@@ -73,7 +74,7 @@ class G1HeapRegion : public CHeapObj {
HeapWord* const _bottom;
HeapWord* const _end;
- HeapWord* volatile _top;
+ Atomic _top;
G1BlockOffsetTable* _bot;
@@ -89,8 +90,8 @@ public:
HeapWord* bottom() const { return _bottom; }
HeapWord* end() const { return _end; }
- void set_top(HeapWord* value) { _top = value; }
- HeapWord* top() const { return _top; }
+ void set_top(HeapWord* value) { _top.store_relaxed(value); }
+ HeapWord* top() const { return _top.load_relaxed(); }
// See the comment above in the declaration of _pre_dummy_top for an
// explanation of what it is.
@@ -231,10 +232,10 @@ private:
//
// Below this limit the marking bitmap must be used to determine size and
// liveness.
- HeapWord* volatile _parsable_bottom;
+ Atomic _parsable_bottom;
// Amount of dead data in the region.
- size_t _garbage_bytes;
+ Atomic _garbage_bytes;
// Approximate number of references to this regions at the end of concurrent
// marking. We we do not mark through all objects, so this is an estimate.
@@ -249,7 +250,7 @@ private:
uint _node_index;
// Number of objects in this region that are currently pinned.
- volatile size_t _pinned_object_count;
+ Atomic _pinned_object_count;
void report_region_type_change(G1HeapRegionTraceType::Type to);
@@ -331,7 +332,7 @@ public:
}
// A lower bound on the amount of garbage bytes in the region.
- size_t garbage_bytes() const { return _garbage_bytes; }
+ size_t garbage_bytes() const { return _garbage_bytes.load_relaxed(); }
// Return the amount of bytes we'll reclaim if we collect this
// region. This includes not only the known garbage bytes in the
@@ -393,8 +394,8 @@ public:
bool is_old_or_humongous() const { return _type.is_old_or_humongous(); }
- size_t pinned_count() const { return AtomicAccess::load(&_pinned_object_count); }
- bool has_pinned_objects() const { return pinned_count() > 0; }
+ inline size_t pinned_count() const;
+ inline bool has_pinned_objects() const;
void set_free();
diff --git a/src/hotspot/share/gc/g1/g1HeapRegion.inline.hpp b/src/hotspot/share/gc/g1/g1HeapRegion.inline.hpp
index f25bf62c9be..4f242b7a537 100644
--- a/src/hotspot/share/gc/g1/g1HeapRegion.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegion.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -35,7 +35,6 @@
#include "gc/g1/g1Policy.hpp"
#include "gc/g1/g1Predictions.hpp"
#include "oops/oop.inline.hpp"
-#include "runtime/atomicAccess.hpp"
#include "runtime/init.hpp"
#include "runtime/prefetch.inline.hpp"
#include "runtime/safepoint.hpp"
@@ -131,7 +130,7 @@ inline void G1HeapRegion::prepare_for_full_gc() {
// After marking and class unloading the heap temporarily contains dead objects
// with unloaded klasses. Moving parsable_bottom makes some (debug) code correctly
// skip dead objects.
- _parsable_bottom = top();
+ _parsable_bottom.store_relaxed(top());
}
inline void G1HeapRegion::reset_compacted_after_full_gc(HeapWord* new_top) {
@@ -154,7 +153,7 @@ inline void G1HeapRegion::reset_after_full_gc_common() {
// Everything above bottom() is parsable and live.
reset_parsable_bottom();
- _garbage_bytes = 0;
+ _garbage_bytes.store_relaxed(0);
_incoming_refs = 0;
@@ -188,20 +187,22 @@ inline void G1HeapRegion::apply_to_marked_objects(G1CMBitMap* bitmap, ApplyToMar
inline HeapWord* G1HeapRegion::par_allocate(size_t min_word_size,
size_t desired_word_size,
size_t* actual_word_size) {
+ HeapWord* obj = top();
do {
- HeapWord* obj = top();
size_t available = pointer_delta(end(), obj);
size_t want_to_allocate = MIN2(available, desired_word_size);
if (want_to_allocate >= min_word_size) {
HeapWord* new_top = obj + want_to_allocate;
- HeapWord* result = AtomicAccess::cmpxchg(&_top, obj, new_top);
- // result can be one of two:
- // the old top value: the exchange succeeded
+ HeapWord* result = _top.compare_exchange(obj, new_top);
+ // Result can be one of two:
+ // the old top value: the exchange succeeded, return.
// otherwise: the new value of the top is returned.
if (result == obj) {
assert(is_object_aligned(obj) && is_object_aligned(new_top), "checking alignment");
*actual_word_size = want_to_allocate;
return obj;
+ } else {
+ obj = result;
}
} else {
return nullptr;
@@ -254,27 +255,27 @@ inline void G1HeapRegion::update_bot_for_block(HeapWord* start, HeapWord* end) {
inline HeapWord* G1HeapRegion::parsable_bottom() const {
assert(!is_init_completed() || SafepointSynchronize::is_at_safepoint(), "only during initialization or safepoint");
- return _parsable_bottom;
+ return _parsable_bottom.load_relaxed();
}
inline HeapWord* G1HeapRegion::parsable_bottom_acquire() const {
- return AtomicAccess::load_acquire(&_parsable_bottom);
+ return _parsable_bottom.load_acquire();
}
inline void G1HeapRegion::reset_parsable_bottom() {
- AtomicAccess::release_store(&_parsable_bottom, bottom());
+ _parsable_bottom.release_store(bottom());
}
inline void G1HeapRegion::note_end_of_marking(HeapWord* top_at_mark_start, size_t marked_bytes, size_t incoming_refs) {
assert_at_safepoint();
if (top_at_mark_start != bottom()) {
- _garbage_bytes = byte_size(bottom(), top_at_mark_start) - marked_bytes;
+ _garbage_bytes.store_relaxed(byte_size(bottom(), top_at_mark_start) - marked_bytes);
_incoming_refs = incoming_refs;
}
if (needs_scrubbing()) {
- _parsable_bottom = top_at_mark_start;
+ _parsable_bottom.store_relaxed(top_at_mark_start);
}
}
@@ -286,6 +287,14 @@ inline bool G1HeapRegion::needs_scrubbing() const {
return is_old();
}
+inline size_t G1HeapRegion::pinned_count() const {
+ return _pinned_object_count.load_relaxed();
+}
+
+inline bool G1HeapRegion::has_pinned_objects() const {
+ return pinned_count() > 0;
+}
+
inline bool G1HeapRegion::in_collection_set() const {
return G1CollectedHeap::heap()->is_in_cset(this);
}
@@ -511,7 +520,7 @@ inline void G1HeapRegion::record_surv_words_in_group(size_t words_survived) {
inline void G1HeapRegion::add_pinned_object_count(size_t value) {
assert(value != 0, "wasted effort");
assert(!is_free(), "trying to pin free region %u, adding %zu", hrm_index(), value);
- AtomicAccess::add(&_pinned_object_count, value, memory_order_relaxed);
+ _pinned_object_count.add_then_fetch(value, memory_order_relaxed);
}
inline void G1HeapRegion::install_cset_group(G1CSetCandidateGroup* cset_group) {
diff --git a/src/hotspot/share/gc/g1/g1HeapRegionManager.cpp b/src/hotspot/share/gc/g1/g1HeapRegionManager.cpp
index 795b6543bae..44897c8a277 100644
--- a/src/hotspot/share/gc/g1/g1HeapRegionManager.cpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionManager.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -713,8 +713,10 @@ void G1HeapRegionManager::verify_optional() {
G1HeapRegionClaimer::G1HeapRegionClaimer(uint n_workers) :
_n_workers(n_workers), _n_regions(G1CollectedHeap::heap()->_hrm._next_highest_used_hrm_index), _claims(nullptr) {
- uint* new_claims = NEW_C_HEAP_ARRAY(uint, _n_regions, mtGC);
- memset(new_claims, Unclaimed, sizeof(*_claims) * _n_regions);
+ Atomic* new_claims = NEW_C_HEAP_ARRAY(Atomic, _n_regions, mtGC);
+ for (uint i = 0; i < _n_regions; i++) {
+ new_claims[i].store_relaxed(Unclaimed);
+ }
_claims = new_claims;
}
@@ -730,13 +732,12 @@ uint G1HeapRegionClaimer::offset_for_worker(uint worker_id) const {
bool G1HeapRegionClaimer::is_region_claimed(uint region_index) const {
assert(region_index < _n_regions, "Invalid index.");
- return _claims[region_index] == Claimed;
+ return _claims[region_index].load_relaxed() == Claimed;
}
bool G1HeapRegionClaimer::claim_region(uint region_index) {
assert(region_index < _n_regions, "Invalid index.");
- uint old_val = AtomicAccess::cmpxchg(&_claims[region_index], Unclaimed, Claimed);
- return old_val == Unclaimed;
+ return _claims[region_index].compare_set(Unclaimed, Claimed);
}
class G1RebuildFreeListTask : public WorkerTask {
diff --git a/src/hotspot/share/gc/g1/g1HeapRegionManager.hpp b/src/hotspot/share/gc/g1/g1HeapRegionManager.hpp
index b4ce3b0a8be..eb593ff408e 100644
--- a/src/hotspot/share/gc/g1/g1HeapRegionManager.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionManager.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -30,6 +30,7 @@
#include "gc/g1/g1HeapRegionSet.hpp"
#include "gc/g1/g1RegionToSpaceMapper.hpp"
#include "memory/allocation.hpp"
+#include "runtime/atomic.hpp"
#include "services/memoryUsage.hpp"
class G1HeapRegion;
@@ -294,7 +295,7 @@ public:
class G1HeapRegionClaimer : public StackObj {
uint _n_workers;
uint _n_regions;
- volatile uint* _claims;
+ Atomic* _claims;
static const uint Unclaimed = 0;
static const uint Claimed = 1;
diff --git a/src/hotspot/share/gc/g1/g1PeriodicGCTask.cpp b/src/hotspot/share/gc/g1/g1PeriodicGCTask.cpp
index 50002ac2bfe..f280d76f3c7 100644
--- a/src/hotspot/share/gc/g1/g1PeriodicGCTask.cpp
+++ b/src/hotspot/share/gc/g1/g1PeriodicGCTask.cpp
@@ -39,7 +39,7 @@ bool G1PeriodicGCTask::should_start_periodic_gc(G1CollectedHeap* g1h,
SuspendibleThreadSetJoiner sts;
// If we are currently in a concurrent mark we are going to uncommit memory soon.
- if (g1h->concurrent_mark()->cm_thread()->in_progress()) {
+ if (g1h->concurrent_mark()->in_progress()) {
log_debug(gc, periodic)("Concurrent cycle in progress. Skipping.");
return false;
}
diff --git a/src/hotspot/share/gc/g1/g1Policy.cpp b/src/hotspot/share/gc/g1/g1Policy.cpp
index 1d0b29c303e..98e6acc1d77 100644
--- a/src/hotspot/share/gc/g1/g1Policy.cpp
+++ b/src/hotspot/share/gc/g1/g1Policy.cpp
@@ -739,7 +739,7 @@ double G1Policy::constant_other_time_ms(double pause_time_ms) const {
}
bool G1Policy::about_to_start_mixed_phase() const {
- return _g1h->concurrent_mark()->cm_thread()->in_progress() || collector_state()->in_young_gc_before_mixed();
+ return _g1h->concurrent_mark()->in_progress() || collector_state()->in_young_gc_before_mixed();
}
bool G1Policy::need_to_start_conc_mark(const char* source, size_t allocation_word_size) {
@@ -1235,7 +1235,7 @@ bool G1Policy::force_concurrent_start_if_outside_cycle(GCCause::Cause gc_cause)
// We actually check whether we are marking here and not if we are in a
// reclamation phase. This means that we will schedule a concurrent mark
// even while we are still in the process of reclaiming memory.
- bool during_cycle = _g1h->concurrent_mark()->cm_thread()->in_progress();
+ bool during_cycle = _g1h->concurrent_mark()->in_progress();
if (!during_cycle) {
log_debug(gc, ergo)("Request concurrent cycle initiation (requested by GC cause). "
"GC cause: %s",
diff --git a/src/hotspot/share/gc/g1/g1RegionMarkStatsCache.cpp b/src/hotspot/share/gc/g1/g1RegionMarkStatsCache.cpp
index d9b7ec294bd..c5f55e1d20c 100644
--- a/src/hotspot/share/gc/g1/g1RegionMarkStatsCache.cpp
+++ b/src/hotspot/share/gc/g1/g1RegionMarkStatsCache.cpp
@@ -29,12 +29,12 @@
G1RegionMarkStatsCache::G1RegionMarkStatsCache(G1RegionMarkStats* target, uint num_cache_entries) :
_target(target),
+ _cache(NEW_C_HEAP_ARRAY(G1RegionMarkStatsCacheEntry, num_cache_entries, mtGC)),
_num_cache_entries(num_cache_entries),
_num_cache_entries_mask(_num_cache_entries - 1) {
guarantee(is_power_of_2(num_cache_entries),
"Number of cache entries must be power of two, but is %u", num_cache_entries);
- _cache = NEW_C_HEAP_ARRAY(G1RegionMarkStatsCacheEntry, _num_cache_entries, mtGC);
}
G1RegionMarkStatsCache::~G1RegionMarkStatsCache() {
diff --git a/src/hotspot/share/gc/g1/g1VMOperations.cpp b/src/hotspot/share/gc/g1/g1VMOperations.cpp
index 1c024f2943b..56ab3a4b0fe 100644
--- a/src/hotspot/share/gc/g1/g1VMOperations.cpp
+++ b/src/hotspot/share/gc/g1/g1VMOperations.cpp
@@ -85,7 +85,7 @@ void VM_G1TryInitiateConcMark::doit() {
GCCauseSetter x(g1h, _gc_cause);
_mark_in_progress = g1h->collector_state()->mark_in_progress();
- _cycle_already_in_progress = g1h->concurrent_mark()->cm_thread()->in_progress();
+ _cycle_already_in_progress = g1h->concurrent_mark()->in_progress();
if (!g1h->policy()->force_concurrent_start_if_outside_cycle(_gc_cause)) {
// Failure to force the next GC pause to be a concurrent start indicates
diff --git a/src/hotspot/share/gc/g1/g1YoungCollector.cpp b/src/hotspot/share/gc/g1/g1YoungCollector.cpp
index 36cc44a8b7c..a9db9a7c269 100644
--- a/src/hotspot/share/gc/g1/g1YoungCollector.cpp
+++ b/src/hotspot/share/gc/g1/g1YoungCollector.cpp
@@ -36,6 +36,7 @@
#include "gc/g1/g1EvacFailureRegions.inline.hpp"
#include "gc/g1/g1EvacInfo.hpp"
#include "gc/g1/g1GCPhaseTimes.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
#include "gc/g1/g1HeapRegionPrinter.hpp"
#include "gc/g1/g1MonitoringSupport.hpp"
#include "gc/g1/g1ParScanThreadState.inline.hpp"
diff --git a/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp b/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp
index 46d12df575c..3f47d386015 100644
--- a/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp
+++ b/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp
@@ -854,7 +854,7 @@ public:
void do_thread(Thread* thread) {
if (UseTLAB && ResizeTLAB) {
- static_cast(thread)->tlab().resize();
+ thread->tlab().resize();
}
G1BarrierSet::g1_barrier_set()->update_card_table_base(thread);
diff --git a/src/hotspot/share/gc/g1/vmStructs_g1.hpp b/src/hotspot/share/gc/g1/vmStructs_g1.hpp
index 21c86d47a6b..af236ec8581 100644
--- a/src/hotspot/share/gc/g1/vmStructs_g1.hpp
+++ b/src/hotspot/share/gc/g1/vmStructs_g1.hpp
@@ -28,6 +28,7 @@
#include "gc/g1/g1CollectedHeap.hpp"
#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1HeapRegionManager.hpp"
+#include "runtime/atomic.hpp"
#include "utilities/macros.hpp"
#define VM_STRUCTS_G1GC(nonstatic_field, \
@@ -39,9 +40,9 @@
\
nonstatic_field(G1HeapRegion, _type, G1HeapRegionType) \
nonstatic_field(G1HeapRegion, _bottom, HeapWord* const) \
- nonstatic_field(G1HeapRegion, _top, HeapWord* volatile) \
+ nonstatic_field(G1HeapRegion, _top, Atomic) \
nonstatic_field(G1HeapRegion, _end, HeapWord* const) \
- volatile_nonstatic_field(G1HeapRegion, _pinned_object_count, size_t) \
+ volatile_nonstatic_field(G1HeapRegion, _pinned_object_count, Atomic)\
\
nonstatic_field(G1HeapRegionType, _tag, G1HeapRegionType::Tag volatile) \
\
diff --git a/src/hotspot/share/gc/parallel/mutableNUMASpace.cpp b/src/hotspot/share/gc/parallel/mutableNUMASpace.cpp
index e0b1edf2efc..c5d112ffbc1 100644
--- a/src/hotspot/share/gc/parallel/mutableNUMASpace.cpp
+++ b/src/hotspot/share/gc/parallel/mutableNUMASpace.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2006, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2006, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -31,7 +31,7 @@
#include "memory/allocation.inline.hpp"
#include "oops/oop.inline.hpp"
#include "oops/typeArrayOop.hpp"
-#include "runtime/atomicAccess.hpp"
+#include "runtime/atomic.hpp"
#include "runtime/java.hpp"
#include "runtime/javaThread.hpp"
#include "runtime/os.inline.hpp"
@@ -489,7 +489,7 @@ HeapWord* MutableNUMASpace::cas_allocate(size_t size) {
if (p != nullptr) {
HeapWord* cur_top, *cur_chunk_top = p + size;
while ((cur_top = top()) < cur_chunk_top) { // Keep _top updated.
- if (AtomicAccess::cmpxchg(top_addr(), cur_top, cur_chunk_top) == cur_top) {
+ if (top_addr()->compare_set(cur_top, cur_chunk_top)) {
break;
}
}
diff --git a/src/hotspot/share/gc/parallel/mutableSpace.cpp b/src/hotspot/share/gc/parallel/mutableSpace.cpp
index fc42fc1eab2..d99db493989 100644
--- a/src/hotspot/share/gc/parallel/mutableSpace.cpp
+++ b/src/hotspot/share/gc/parallel/mutableSpace.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -28,7 +28,6 @@
#include "memory/iterator.inline.hpp"
#include "memory/universe.hpp"
#include "oops/oop.inline.hpp"
-#include "runtime/atomicAccess.hpp"
#include "runtime/javaThread.hpp"
#include "runtime/safepoint.hpp"
#include "utilities/align.hpp"
@@ -123,7 +122,7 @@ void MutableSpace::initialize(MemRegion mr,
// makes the new space available for allocation by other threads. So this
// assignment must follow all other configuration and initialization that
// might be done for expansion.
- AtomicAccess::release_store(end_addr(), mr.end());
+ _end.release_store(mr.end());
if (clear_space) {
clear(mangle_space);
@@ -140,7 +139,7 @@ void MutableSpace::clear(bool mangle_space) {
#ifndef PRODUCT
void MutableSpace::mangle_unused_area() {
- mangle_region(MemRegion(_top, _end));
+ mangle_region(MemRegion(top(), end()));
}
void MutableSpace::mangle_region(MemRegion mr) {
@@ -155,14 +154,10 @@ HeapWord* MutableSpace::cas_allocate(size_t size) {
// If end is read first, other threads may advance end and top such that
// current top > old end and current top + size > current end. Then
// pointer_delta underflows, allowing installation of top > current end.
- HeapWord* obj = AtomicAccess::load_acquire(top_addr());
+ HeapWord* obj = _top.load_acquire();
if (pointer_delta(end(), obj) >= size) {
HeapWord* new_top = obj + size;
- HeapWord* result = AtomicAccess::cmpxchg(top_addr(), obj, new_top);
- // result can be one of two:
- // the old top value: the exchange succeeded
- // otherwise: the new value of the top is returned.
- if (result != obj) {
+ if (!_top.compare_set(obj, new_top)) {
continue; // another thread beat us to the allocation, try again
}
assert(is_object_aligned(obj) && is_object_aligned(new_top),
@@ -177,7 +172,7 @@ HeapWord* MutableSpace::cas_allocate(size_t size) {
// Try to deallocate previous allocation. Returns true upon success.
bool MutableSpace::cas_deallocate(HeapWord *obj, size_t size) {
HeapWord* expected_top = obj + size;
- return AtomicAccess::cmpxchg(top_addr(), expected_top, obj) == expected_top;
+ return _top.compare_set(expected_top, obj);
}
void MutableSpace::oop_iterate(OopIterateClosure* cl) {
diff --git a/src/hotspot/share/gc/parallel/mutableSpace.hpp b/src/hotspot/share/gc/parallel/mutableSpace.hpp
index 9d3894e2489..28df19a7c4b 100644
--- a/src/hotspot/share/gc/parallel/mutableSpace.hpp
+++ b/src/hotspot/share/gc/parallel/mutableSpace.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -28,6 +28,7 @@
#include "memory/allocation.hpp"
#include "memory/iterator.hpp"
#include "memory/memRegion.hpp"
+#include "runtime/atomic.hpp"
#include "utilities/copy.hpp"
#include "utilities/globalDefinitions.hpp"
#include "utilities/macros.hpp"
@@ -53,8 +54,8 @@ class MutableSpace: public CHeapObj {
MemRegion _last_setup_region;
size_t _page_size;
HeapWord* _bottom;
- HeapWord* volatile _top;
- HeapWord* _end;
+ Atomic _top;
+ Atomic _end;
void numa_setup_pages(MemRegion mr, bool clear_space);
@@ -64,21 +65,20 @@ class MutableSpace: public CHeapObj {
protected:
size_t page_size() const { return _page_size; }
+ Atomic* top_addr() { return &_top; }
+
public:
virtual ~MutableSpace() = default;
MutableSpace(size_t page_size);
// Accessors
HeapWord* bottom() const { return _bottom; }
- HeapWord* top() const { return _top; }
- HeapWord* end() const { return _end; }
+ HeapWord* top() const { return _top.load_relaxed(); }
+ HeapWord* end() const { return _end.load_relaxed(); }
void set_bottom(HeapWord* value) { _bottom = value; }
- virtual void set_top(HeapWord* value) { _top = value; }
- void set_end(HeapWord* value) { _end = value; }
-
- HeapWord* volatile* top_addr() { return &_top; }
- HeapWord** end_addr() { return &_end; }
+ virtual void set_top(HeapWord* value) { _top.store_relaxed(value); }
+ void set_end(HeapWord* value) { _end.store_relaxed(value); }
MemRegion region() const { return MemRegion(bottom(), end()); }
@@ -110,7 +110,7 @@ public:
// Boolean queries.
bool is_empty() const { return used_in_words() == 0; }
bool not_empty() const { return used_in_words() > 0; }
- bool contains(const void* p) const { return _bottom <= p && p < _end; }
+ bool contains(const void* p) const { return _bottom <= p && p < end(); }
// Size computations. Sizes are in bytes.
size_t used_in_bytes() const { return used_in_words() * HeapWordSize; }
diff --git a/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp b/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp
index 0d8a3166f79..f49419595e1 100644
--- a/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp
+++ b/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -48,6 +48,7 @@
#include "memory/reservedSpace.hpp"
#include "memory/universe.hpp"
#include "oops/oop.inline.hpp"
+#include "runtime/atomic.hpp"
#include "runtime/cpuTimeCounters.hpp"
#include "runtime/globals_extension.hpp"
#include "runtime/handles.inline.hpp"
@@ -438,7 +439,7 @@ bool ParallelScavengeHeap::check_gc_overhead_limit() {
log_debug(gc)("GC Overhead Limit: GC Time %f Free Space Young %f Old %f Counter %zu",
(100 - _size_policy->mutator_time_percent()),
percent_of(_young_gen->free_in_bytes(), _young_gen->capacity_in_bytes()),
- percent_of(_old_gen->free_in_bytes(), _young_gen->capacity_in_bytes()),
+ percent_of(_old_gen->free_in_bytes(), _old_gen->capacity_in_bytes()),
_gc_overhead_counter);
if (little_mutator_time && little_free_space) {
@@ -594,7 +595,7 @@ void ParallelScavengeHeap::object_iterate(ObjectClosure* cl) {
// these spaces.
// The old space is divided into fixed-size blocks.
class HeapBlockClaimer : public StackObj {
- size_t _claimed_index;
+ Atomic _claimed_index;
public:
static const size_t InvalidIndex = SIZE_MAX;
@@ -606,7 +607,7 @@ public:
// Claim the block and get the block index.
size_t claim_and_get_block() {
size_t block_index;
- block_index = AtomicAccess::fetch_then_add(&_claimed_index, 1u);
+ block_index = _claimed_index.fetch_then_add(1u);
PSOldGen* old_gen = ParallelScavengeHeap::heap()->old_gen();
size_t num_claims = old_gen->num_iterable_blocks() + NumNonOldGenClaims;
diff --git a/src/hotspot/share/gc/parallel/psCardTable.cpp b/src/hotspot/share/gc/parallel/psCardTable.cpp
index fcd0dd67a45..6429766309a 100644
--- a/src/hotspot/share/gc/parallel/psCardTable.cpp
+++ b/src/hotspot/share/gc/parallel/psCardTable.cpp
@@ -108,7 +108,7 @@ void PSCardTable::scan_obj_with_limit(PSPromotionManager* pm,
}
void PSCardTable::pre_scavenge(uint active_workers) {
- _preprocessing_active_workers = active_workers;
+ _preprocessing_active_workers.store_relaxed(active_workers);
}
// The "shadow" table is a copy of the card table entries of the current stripe.
@@ -382,9 +382,9 @@ void PSCardTable::scavenge_contents_parallel(ObjectStartArray* start_array,
preprocess_card_table_parallel(object_start, old_gen_bottom, old_gen_top, stripe_index, n_stripes);
// Sync with other workers.
- AtomicAccess::dec(&_preprocessing_active_workers);
+ _preprocessing_active_workers.sub_then_fetch(1);
SpinYield spin_yield;
- while (AtomicAccess::load_acquire(&_preprocessing_active_workers) > 0) {
+ while (_preprocessing_active_workers.load_acquire() > 0) {
spin_yield.wait();
}
diff --git a/src/hotspot/share/gc/parallel/psCardTable.hpp b/src/hotspot/share/gc/parallel/psCardTable.hpp
index 70c32d23b7f..033933bcbf1 100644
--- a/src/hotspot/share/gc/parallel/psCardTable.hpp
+++ b/src/hotspot/share/gc/parallel/psCardTable.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -27,6 +27,7 @@
#include "gc/shared/cardTable.hpp"
#include "oops/oop.hpp"
+#include "runtime/atomic.hpp"
class MutableSpace;
class ObjectStartArray;
@@ -37,7 +38,7 @@ class PSCardTable: public CardTable {
static constexpr size_t num_cards_in_stripe = 128;
static_assert(num_cards_in_stripe >= 1, "progress");
- volatile int _preprocessing_active_workers;
+ Atomic _preprocessing_active_workers;
bool is_dirty(CardValue* card) {
return !is_clean(card);
diff --git a/src/hotspot/share/gc/parallel/psPromotionManager.cpp b/src/hotspot/share/gc/parallel/psPromotionManager.cpp
index a41a9403082..d6208755374 100644
--- a/src/hotspot/share/gc/parallel/psPromotionManager.cpp
+++ b/src/hotspot/share/gc/parallel/psPromotionManager.cpp
@@ -43,6 +43,7 @@
#include "memory/resourceArea.hpp"
#include "oops/access.inline.hpp"
#include "oops/compressedOops.inline.hpp"
+#include "oops/oopsHierarchy.hpp"
#include "utilities/checkedCast.hpp"
PaddedEnd* PSPromotionManager::_manager_array = nullptr;
@@ -248,30 +249,19 @@ void PSPromotionManager::flush_labs() {
}
}
-template
-void PSPromotionManager::process_array_chunk_work(oop obj, int start, int end) {
- assert(start <= end, "invariant");
- T* const base = (T*)objArrayOop(obj)->base();
- T* p = base + start;
- T* const chunk_end = base + end;
- while (p < chunk_end) {
- claim_or_forward_depth(p);
- ++p;
- }
+void PSPromotionManager::process_array_chunk(objArrayOop obj, size_t start, size_t end) {
+ PSPushContentsClosure pcc(this);
+ obj->oop_iterate_elements_range(&pcc,
+ checked_cast(start),
+ checked_cast(end));
}
void PSPromotionManager::process_array_chunk(PartialArrayState* state, bool stolen) {
// Access before release by claim().
- oop new_obj = state->destination();
+ objArrayOop to_array = objArrayOop(state->destination());
PartialArraySplitter::Claim claim =
_partial_array_splitter.claim(state, &_claimed_stack_depth, stolen);
- int start = checked_cast(claim._start);
- int end = checked_cast(claim._end);
- if (UseCompressedOops) {
- process_array_chunk_work(new_obj, start, end);
- } else {
- process_array_chunk_work(new_obj, start, end);
- }
+ process_array_chunk(to_array, claim._start, claim._end);
}
void PSPromotionManager::push_objArray(oop old_obj, oop new_obj) {
@@ -284,12 +274,8 @@ void PSPromotionManager::push_objArray(oop old_obj, oop new_obj) {
size_t initial_chunk_size =
// The source array is unused when processing states.
_partial_array_splitter.start(&_claimed_stack_depth, nullptr, to_array, array_length);
- int end = checked_cast(initial_chunk_size);
- if (UseCompressedOops) {
- process_array_chunk_work(to_array, 0, end);
- } else {
- process_array_chunk_work(to_array, 0, end);
- }
+
+ process_array_chunk(to_array, 0, initial_chunk_size);
}
oop PSPromotionManager::oop_promotion_failed(oop obj, markWord obj_mark) {
diff --git a/src/hotspot/share/gc/parallel/psPromotionManager.hpp b/src/hotspot/share/gc/parallel/psPromotionManager.hpp
index 44df708eea4..2b0fc56c0bf 100644
--- a/src/hotspot/share/gc/parallel/psPromotionManager.hpp
+++ b/src/hotspot/share/gc/parallel/psPromotionManager.hpp
@@ -97,9 +97,8 @@ class PSPromotionManager {
inline static PSPromotionManager* manager_array(uint index);
- template void process_array_chunk_work(oop obj,
- int start, int end);
void process_array_chunk(PartialArrayState* state, bool stolen);
+ void process_array_chunk(objArrayOop obj, size_t start, size_t end);
void push_objArray(oop old_obj, oop new_obj);
inline void promotion_trace_event(oop new_obj, Klass* klass, size_t obj_size,
diff --git a/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp b/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp
index f1fd49c7dfe..9e904e44b22 100644
--- a/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp
+++ b/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp
@@ -51,7 +51,7 @@ inline PSPromotionManager* PSPromotionManager::manager_array(uint index) {
}
template
-inline void PSPromotionManager::claim_or_forward_depth(T* p) {
+ALWAYSINLINE void PSPromotionManager::claim_or_forward_depth(T* p) {
assert(ParallelScavengeHeap::heap()->is_in(p), "pointer outside heap");
T heap_oop = RawAccess<>::oop_load(p);
if (PSScavenge::is_obj_in_young(heap_oop)) {
diff --git a/src/hotspot/share/gc/parallel/vmStructs_parallelgc.hpp b/src/hotspot/share/gc/parallel/vmStructs_parallelgc.hpp
index f69219a1f40..f8dabc4539e 100644
--- a/src/hotspot/share/gc/parallel/vmStructs_parallelgc.hpp
+++ b/src/hotspot/share/gc/parallel/vmStructs_parallelgc.hpp
@@ -47,8 +47,8 @@
nonstatic_field(PSVirtualSpace, _committed_high_addr, char*) \
\
nonstatic_field(MutableSpace, _bottom, HeapWord*) \
- nonstatic_field(MutableSpace, _end, HeapWord*) \
- volatile_nonstatic_field(MutableSpace, _top, HeapWord*) \
+ nonstatic_field(MutableSpace, _end, Atomic) \
+ volatile_nonstatic_field(MutableSpace, _top, Atomic) \
\
nonstatic_field(PSYoungGen, _reserved, MemRegion) \
nonstatic_field(PSYoungGen, _virtual_space, PSVirtualSpace*) \
diff --git a/src/hotspot/share/gc/shared/barrierSetNMethod.cpp b/src/hotspot/share/gc/shared/barrierSetNMethod.cpp
index ab94bae079a..a1f03a4bf50 100644
--- a/src/hotspot/share/gc/shared/barrierSetNMethod.cpp
+++ b/src/hotspot/share/gc/shared/barrierSetNMethod.cpp
@@ -111,6 +111,8 @@ bool BarrierSetNMethod::nmethod_entry_barrier(nmethod* nm) {
return true;
}
+ // Enable WXWrite: the function is called directly from nmethod_entry_barrier
+ // stub.
MACOS_AARCH64_ONLY(ThreadWXEnable wx(WXWrite, Thread::current()));
// If the nmethod is the only thing pointing to the oops, and we are using a
diff --git a/src/hotspot/share/gc/shared/cardTableBarrierSet.cpp b/src/hotspot/share/gc/shared/cardTableBarrierSet.cpp
index 539e40820a8..d6541198858 100644
--- a/src/hotspot/share/gc/shared/cardTableBarrierSet.cpp
+++ b/src/hotspot/share/gc/shared/cardTableBarrierSet.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -73,15 +73,15 @@ CardTableBarrierSet::CardTableBarrierSet(CardTable* card_table) :
{}
CardTableBarrierSet::~CardTableBarrierSet() {
- delete _card_table;
+ delete card_table();
}
void CardTableBarrierSet::write_region(MemRegion mr) {
- _card_table->dirty_MemRegion(mr);
+ card_table()->dirty_MemRegion(mr);
}
void CardTableBarrierSet::print_on(outputStream* st) const {
- _card_table->print_on(st);
+ card_table()->print_on(st);
}
// Helper for ReduceInitialCardMarks. For performance,
@@ -116,7 +116,7 @@ void CardTableBarrierSet::on_slowpath_allocation_exit(JavaThread* thread, oop ne
if (!ReduceInitialCardMarks) {
return;
}
- if (new_obj->is_typeArray() || _card_table->is_in_young(new_obj)) {
+ if (new_obj->is_typeArray() || card_table()->is_in_young(new_obj)) {
// Arrays of non-references don't need a post-barrier.
} else {
MemRegion mr(cast_from_oop(new_obj), new_obj->size());
diff --git a/src/hotspot/share/gc/shared/cardTableBarrierSet.hpp b/src/hotspot/share/gc/shared/cardTableBarrierSet.hpp
index c298bfcd0c2..3a9b46d9df8 100644
--- a/src/hotspot/share/gc/shared/cardTableBarrierSet.hpp
+++ b/src/hotspot/share/gc/shared/cardTableBarrierSet.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -29,6 +29,7 @@
#include "gc/shared/cardTable.hpp"
#include "gc/shared/gc_globals.hpp"
#include "memory/memRegion.hpp"
+#include "runtime/atomic.hpp"
#include "utilities/align.hpp"
// This kind of "BarrierSet" allows a "CollectedHeap" to detect and
@@ -49,7 +50,7 @@ class CardTableBarrierSet: public BarrierSet {
protected:
typedef CardTable::CardValue CardValue;
- CardTable* _card_table;
+ Atomic _card_table;
CardTableBarrierSet(BarrierSetAssembler* barrier_set_assembler,
BarrierSetC1* barrier_set_c1,
@@ -90,10 +91,12 @@ public:
// at the address "start", which may not necessarily be HeapWord-aligned
inline void write_ref_array(HeapWord* start, size_t count);
- CardTable* card_table() const { return _card_table; }
+ CardTable* card_table() { return _card_table.load_relaxed(); }
+ CardTable* card_table() const { return _card_table.load_relaxed(); }
+
CardValue* card_table_base_const() const {
assert(UseSerialGC || UseParallelGC, "Only these GCs have constant card table base");
- return _card_table->byte_map_base();
+ return card_table()->byte_map_base();
}
virtual void on_slowpath_allocation_exit(JavaThread* thread, oop new_obj);
diff --git a/src/hotspot/share/gc/shared/cardTableBarrierSet.inline.hpp b/src/hotspot/share/gc/shared/cardTableBarrierSet.inline.hpp
index ea539a70be5..f60a7f47a19 100644
--- a/src/hotspot/share/gc/shared/cardTableBarrierSet.inline.hpp
+++ b/src/hotspot/share/gc/shared/cardTableBarrierSet.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -35,7 +35,7 @@
template
inline void CardTableBarrierSet::write_ref_field_post(T* field) {
- volatile CardValue* byte = _card_table->byte_for(field);
+ volatile CardValue* byte = card_table()->byte_for(field);
*byte = CardTable::dirty_card_val();
}
diff --git a/src/hotspot/share/gc/shared/parallelCleaning.cpp b/src/hotspot/share/gc/shared/parallelCleaning.cpp
index e302085d0cc..1a0d536f3b3 100644
--- a/src/hotspot/share/gc/shared/parallelCleaning.cpp
+++ b/src/hotspot/share/gc/shared/parallelCleaning.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -39,7 +39,7 @@ CodeCacheUnloadingTask::CodeCacheUnloadingTask(bool unloading_occurred) :
if(iter.next()) {
_first_nmethod = iter.method();
}
- _claimed_nmethod = _first_nmethod;
+ _claimed_nmethod.store_relaxed(_first_nmethod);
}
CodeCacheUnloadingTask::~CodeCacheUnloadingTask() {
@@ -53,7 +53,7 @@ void CodeCacheUnloadingTask::claim_nmethods(nmethod** claimed_nmethods, int *num
do {
*num_claimed_nmethods = 0;
- first = _claimed_nmethod;
+ first = _claimed_nmethod.load_relaxed();
last = NMethodIterator(NMethodIterator::all, first);
if (first != nullptr) {
@@ -67,7 +67,7 @@ void CodeCacheUnloadingTask::claim_nmethods(nmethod** claimed_nmethods, int *num
}
}
- } while (AtomicAccess::cmpxchg(&_claimed_nmethod, first, last.method()) != first);
+ } while (!_claimed_nmethod.compare_set(first, last.method()));
}
void CodeCacheUnloadingTask::work(uint worker_id) {
diff --git a/src/hotspot/share/gc/shared/parallelCleaning.hpp b/src/hotspot/share/gc/shared/parallelCleaning.hpp
index ed76c4c9df9..0f5cb78bf55 100644
--- a/src/hotspot/share/gc/shared/parallelCleaning.hpp
+++ b/src/hotspot/share/gc/shared/parallelCleaning.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -29,6 +29,7 @@
#include "code/codeCache.hpp"
#include "gc/shared/oopStorageParState.hpp"
#include "gc/shared/workerThread.hpp"
+#include "runtime/atomic.hpp"
class CodeCacheUnloadingTask {
@@ -36,7 +37,7 @@ class CodeCacheUnloadingTask {
// Variables used to claim nmethods.
nmethod* _first_nmethod;
- nmethod* volatile _claimed_nmethod;
+ Atomic _claimed_nmethod;
public:
CodeCacheUnloadingTask(bool unloading_occurred);
diff --git a/src/hotspot/share/gc/shared/space.cpp b/src/hotspot/share/gc/shared/space.cpp
index 011a0f5cfd8..84ba21527fd 100644
--- a/src/hotspot/share/gc/shared/space.cpp
+++ b/src/hotspot/share/gc/shared/space.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -30,7 +30,6 @@
#include "memory/iterator.inline.hpp"
#include "memory/universe.hpp"
#include "oops/oop.inline.hpp"
-#include "runtime/atomicAccess.hpp"
#include "runtime/java.hpp"
#include "runtime/safepoint.hpp"
#include "utilities/align.hpp"
@@ -69,7 +68,7 @@ void ContiguousSpace::clear(bool mangle_space) {
#ifndef PRODUCT
void ContiguousSpace::mangle_unused_area() {
- mangle_unused_area(MemRegion(_top, _end));
+ mangle_unused_area(MemRegion(top(), _end));
}
void ContiguousSpace::mangle_unused_area(MemRegion mr) {
@@ -128,11 +127,8 @@ inline HeapWord* ContiguousSpace::par_allocate_impl(size_t size) {
HeapWord* obj = top();
if (pointer_delta(end(), obj) >= size) {
HeapWord* new_top = obj + size;
- HeapWord* result = AtomicAccess::cmpxchg(top_addr(), obj, new_top);
- // result can be one of two:
- // the old top value: the exchange succeeded
- // otherwise: the new value of the top is returned.
- if (result == obj) {
+ // Retry if we did not successfully updated the top pointers.
+ if (_top.compare_set(obj, new_top)) {
assert(is_object_aligned(obj) && is_object_aligned(new_top), "checking alignment");
return obj;
}
diff --git a/src/hotspot/share/gc/shared/space.hpp b/src/hotspot/share/gc/shared/space.hpp
index 7f2887275b3..05b22f680bf 100644
--- a/src/hotspot/share/gc/shared/space.hpp
+++ b/src/hotspot/share/gc/shared/space.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -32,6 +32,7 @@
#include "memory/iterator.hpp"
#include "memory/memRegion.hpp"
#include "oops/markWord.hpp"
+#include "runtime/atomic.hpp"
#include "runtime/mutexLocker.hpp"
#include "utilities/align.hpp"
#include "utilities/macros.hpp"
@@ -53,7 +54,7 @@ class ContiguousSpace: public CHeapObj {
private:
HeapWord* _bottom;
HeapWord* _end;
- HeapWord* _top;
+ Atomic _top;
// Allocation helpers (return null if full).
inline HeapWord* allocate_impl(size_t word_size);
@@ -64,12 +65,12 @@ public:
// Accessors
HeapWord* bottom() const { return _bottom; }
- HeapWord* end() const { return _end; }
- HeapWord* top() const { return _top; }
+ HeapWord* end() const { return _end; }
+ HeapWord* top() const { return _top.load_relaxed(); }
void set_bottom(HeapWord* value) { _bottom = value; }
void set_end(HeapWord* value) { _end = value; }
- void set_top(HeapWord* value) { _top = value; }
+ void set_top(HeapWord* value) { _top.store_relaxed(value); }
// Testers
bool is_empty() const { return used() == 0; }
@@ -121,9 +122,6 @@ public:
// Iteration
void object_iterate(ObjectClosure* blk);
- // Addresses for inlined allocation
- HeapWord** top_addr() { return &_top; }
-
// Debugging
void verify() const;
};
diff --git a/src/hotspot/share/gc/shared/threadLocalAllocBuffer.cpp b/src/hotspot/share/gc/shared/threadLocalAllocBuffer.cpp
index 9635ed4d0cb..e86881d3523 100644
--- a/src/hotspot/share/gc/shared/threadLocalAllocBuffer.cpp
+++ b/src/hotspot/share/gc/shared/threadLocalAllocBuffer.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -458,10 +458,19 @@ size_t ThreadLocalAllocBuffer::end_reserve() {
return MAX2(reserve_size, (size_t)_reserve_for_allocation_prefetch);
}
-const HeapWord* ThreadLocalAllocBuffer::start_relaxed() const {
- return AtomicAccess::load(&_start);
-}
-
-const HeapWord* ThreadLocalAllocBuffer::top_relaxed() const {
- return AtomicAccess::load(&_top);
+size_t ThreadLocalAllocBuffer::estimated_used_bytes() const {
+ HeapWord* start = AtomicAccess::load(&_start);
+ HeapWord* top = AtomicAccess::load(&_top);
+ // There has been a race when retrieving _top and _start. Return 0.
+ if (_top < _start) {
+ return 0;
+ }
+ size_t used_bytes = pointer_delta(_top, _start, 1);
+ // Comparing diff with the maximum allowed size will ensure that we don't add
+ // the used bytes from a semi-initialized TLAB ending up with implausible values.
+ // In this case also just return 0.
+ if (used_bytes > ThreadLocalAllocBuffer::max_size_in_bytes()) {
+ return 0;
+ }
+ return used_bytes;
}
diff --git a/src/hotspot/share/gc/shared/threadLocalAllocBuffer.hpp b/src/hotspot/share/gc/shared/threadLocalAllocBuffer.hpp
index 8267a103539..a50e7c9533c 100644
--- a/src/hotspot/share/gc/shared/threadLocalAllocBuffer.hpp
+++ b/src/hotspot/share/gc/shared/threadLocalAllocBuffer.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -33,16 +33,13 @@
class ThreadLocalAllocStats;
// ThreadLocalAllocBuffer: a descriptor for thread-local storage used by
-// the threads for allocation.
-// It is thread-private at any time, but maybe multiplexed over
-// time across multiple threads. The park()/unpark() pair is
-// used to make it available for such multiplexing.
+// the threads for allocation. It is thread-private at any time.
//
-// Heap sampling is performed via the end and allocation_end
-// fields.
-// allocation_end contains the real end of the tlab allocation,
-// whereas end can be set to an arbitrary spot in the tlab to
-// trip the return and sample the allocation.
+// Heap sampling is performed via the end and allocation_end
+// fields.
+// allocation_end contains the real end of the tlab allocation,
+// whereas end can be set to an arbitrary spot in the tlab to
+// trip the return and sample the allocation.
class ThreadLocalAllocBuffer: public CHeapObj {
friend class VMStructs;
friend class JVMCIVMStructs;
@@ -116,17 +113,18 @@ public:
HeapWord* end() const { return _end; }
HeapWord* top() const { return _top; }
HeapWord* hard_end();
- HeapWord* pf_top() const { return _pf_top; }
size_t desired_size() const { return _desired_size; }
- size_t used() const { return pointer_delta(top(), start()); }
size_t used_bytes() const { return pointer_delta(top(), start(), 1); }
size_t free() const { return pointer_delta(end(), top()); }
// Don't discard tlab if remaining space is larger than this.
size_t refill_waste_limit() const { return _refill_waste_limit; }
- // For external inspection.
- const HeapWord* start_relaxed() const;
- const HeapWord* top_relaxed() const;
+ // Returns an estimate of the number of bytes currently used in the TLAB.
+ // Due to races with concurrent allocations and/or resetting the TLAB the return
+ // value may be inconsistent with any other metrics (e.g. total allocated
+ // bytes), and may just incorrectly return 0.
+ // Intented fo external inspection only where accuracy is not 100% required.
+ size_t estimated_used_bytes() const;
// Allocate size HeapWords. The memory is NOT initialized to zero.
inline HeapWord* allocate(size_t size);
@@ -171,14 +169,6 @@ public:
static size_t refill_waste_limit_increment();
- template void addresses_do(T f) {
- f(&_start);
- f(&_top);
- f(&_pf_top);
- f(&_end);
- f(&_allocation_end);
- }
-
// Code generation support
static ByteSize start_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _start); }
static ByteSize end_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _end); }
diff --git a/src/hotspot/share/gc/shared/vmStructs_gc.hpp b/src/hotspot/share/gc/shared/vmStructs_gc.hpp
index db968e28f67..9348fd980f4 100644
--- a/src/hotspot/share/gc/shared/vmStructs_gc.hpp
+++ b/src/hotspot/share/gc/shared/vmStructs_gc.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -48,6 +48,7 @@
#if INCLUDE_ZGC
#include "gc/z/vmStructs_z.hpp"
#endif
+#include "runtime/atomic.hpp"
#define VM_STRUCTS_GC(nonstatic_field, \
volatile_static_field, \
@@ -88,7 +89,7 @@
nonstatic_field(CardTable, _byte_map_size, const size_t) \
nonstatic_field(CardTable, _byte_map, CardTable::CardValue*) \
nonstatic_field(CardTable, _byte_map_base, CardTable::CardValue*) \
- nonstatic_field(CardTableBarrierSet, _card_table, CardTable*) \
+ nonstatic_field(CardTableBarrierSet, _card_table, Atomic) \
\
static_field(CollectedHeap, _lab_alignment_reserve, size_t) \
nonstatic_field(CollectedHeap, _reserved, MemRegion) \
@@ -97,7 +98,7 @@
\
nonstatic_field(ContiguousSpace, _bottom, HeapWord*) \
nonstatic_field(ContiguousSpace, _end, HeapWord*) \
- nonstatic_field(ContiguousSpace, _top, HeapWord*) \
+ nonstatic_field(ContiguousSpace, _top, Atomic) \
\
nonstatic_field(MemRegion, _start, HeapWord*) \
nonstatic_field(MemRegion, _word_size, size_t)
@@ -149,6 +150,7 @@
\
declare_toplevel_type(BarrierSet*) \
declare_toplevel_type(CardTable*) \
+ declare_toplevel_type(Atomic) \
declare_toplevel_type(CardTable*const) \
declare_toplevel_type(CardTableBarrierSet*) \
declare_toplevel_type(CardTableBarrierSet**) \
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp
index 46d9f19d35f..7a8bd55c795 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp
@@ -68,9 +68,9 @@ ShenandoahAdaptiveHeuristics::ShenandoahAdaptiveHeuristics(ShenandoahSpaceInfo*
ShenandoahAdaptiveHeuristics::~ShenandoahAdaptiveHeuristics() {}
-size_t ShenandoahAdaptiveHeuristics::choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset,
- RegionData* data, size_t size,
- size_t actual_free) {
+void ShenandoahAdaptiveHeuristics::choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset,
+ RegionData* data, size_t size,
+ size_t actual_free) {
size_t garbage_threshold = ShenandoahHeapRegion::region_size_bytes() * ShenandoahGarbageThreshold / 100;
// The logic for cset selection in adaptive is as follows:
@@ -124,7 +124,6 @@ size_t ShenandoahAdaptiveHeuristics::choose_collection_set_from_regiondata(Shena
cur_garbage = new_garbage;
}
}
- return 0;
}
void ShenandoahAdaptiveHeuristics::record_cycle_start() {
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp
index c4fdf819391..9b7824a50d7 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp
@@ -33,7 +33,7 @@
#include "utilities/numberSeq.hpp"
/**
- * ShenanoahAllocationRate maintains a truncated history of recently sampled allocation rates for the purpose of providing
+ * ShenandoahAllocationRate maintains a truncated history of recently sampled allocation rates for the purpose of providing
* informed estimates of current and future allocation rates based on weighted averages and standard deviations of the
* truncated history. More recently sampled allocations are weighted more heavily than older samples when computing
* averages and standard deviations.
@@ -108,20 +108,20 @@ public:
virtual ~ShenandoahAdaptiveHeuristics();
- virtual size_t choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset,
+ void choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset,
RegionData* data, size_t size,
size_t actual_free) override;
- virtual void record_cycle_start() override;
- virtual void record_success_concurrent() override;
- virtual void record_degenerated() override;
- virtual void record_success_full() override;
+ void record_cycle_start() override;
+ void record_success_concurrent() override;
+ void record_degenerated() override;
+ void record_success_full() override;
- virtual bool should_start_gc() override;
+ bool should_start_gc() override;
- virtual const char* name() override { return "Adaptive"; }
- virtual bool is_diagnostic() override { return false; }
- virtual bool is_experimental() override { return false; }
+ const char* name() override { return "Adaptive"; }
+ bool is_diagnostic() override { return false; }
+ bool is_experimental() override { return false; }
private:
// These are used to adjust the margin of error and the spike threshold
@@ -185,7 +185,7 @@ protected:
// in the generational case. Controlled by global flag ShenandoahMinFreeThreshold.
size_t min_free_threshold();
- inline void accept_trigger_with_type(Trigger trigger_type) {
+ void accept_trigger_with_type(Trigger trigger_type) {
_last_trigger = trigger_type;
ShenandoahHeuristics::accept_trigger();
}
@@ -193,7 +193,7 @@ protected:
public:
// Sample the allocation rate at GC trigger time if possible. Return the number of allocated bytes that were
// not accounted for in the sample. This must be called before resetting bytes allocated since gc start.
- virtual size_t force_alloc_rate_sample(size_t bytes_allocated) override {
+ size_t force_alloc_rate_sample(size_t bytes_allocated) override {
size_t unaccounted_bytes;
_allocation_rate.force_sample(bytes_allocated, unaccounted_bytes);
return unaccounted_bytes;
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp
index 990b59ec853..a833e39631c 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp
@@ -39,16 +39,15 @@ ShenandoahAggressiveHeuristics::ShenandoahAggressiveHeuristics(ShenandoahSpaceIn
SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahEvacReserveOverflow);
}
-size_t ShenandoahAggressiveHeuristics::choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset,
- RegionData* data, size_t size,
- size_t free) {
+void ShenandoahAggressiveHeuristics::choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset,
+ RegionData* data, size_t size,
+ size_t free) {
for (size_t idx = 0; idx < size; idx++) {
ShenandoahHeapRegion* r = data[idx].get_region();
if (r->garbage() > 0) {
cset->add_region(r);
}
}
- return 0;
}
bool ShenandoahAggressiveHeuristics::should_start_gc() {
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.hpp
index 25c8635489f..9dc88a61bf5 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.hpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.hpp
@@ -35,17 +35,17 @@ class ShenandoahAggressiveHeuristics : public ShenandoahHeuristics {
public:
ShenandoahAggressiveHeuristics(ShenandoahSpaceInfo* space_info);
- virtual size_t choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset,
- RegionData* data, size_t size,
- size_t free);
+ void choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset,
+ RegionData* data, size_t size,
+ size_t free) override;
- virtual bool should_start_gc();
+ bool should_start_gc() override;
- virtual bool should_unload_classes();
+ bool should_unload_classes() override;
- virtual const char* name() { return "Aggressive"; }
- virtual bool is_diagnostic() { return true; }
- virtual bool is_experimental() { return false; }
+ const char* name() override { return "Aggressive"; }
+ bool is_diagnostic() override { return true; }
+ bool is_experimental() override { return false; }
};
#endif // SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHAGGRESSIVEHEURISTICS_HPP
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp
index 09a8394a4b1..28673b28612 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp
@@ -76,9 +76,9 @@ bool ShenandoahCompactHeuristics::should_start_gc() {
return ShenandoahHeuristics::should_start_gc();
}
-size_t ShenandoahCompactHeuristics::choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset,
- RegionData* data, size_t size,
- size_t actual_free) {
+void ShenandoahCompactHeuristics::choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset,
+ RegionData* data, size_t size,
+ size_t actual_free) {
// Do not select too large CSet that would overflow the available free space
size_t max_cset = actual_free * 3 / 4;
@@ -97,5 +97,4 @@ size_t ShenandoahCompactHeuristics::choose_collection_set_from_regiondata(Shenan
cset->add_region(r);
}
}
- return 0;
}
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp
index 4988d5d495d..a32c9c88478 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp
@@ -33,17 +33,17 @@
*/
class ShenandoahCompactHeuristics : public ShenandoahHeuristics {
public:
- ShenandoahCompactHeuristics(ShenandoahSpaceInfo* space_info);
+ explicit ShenandoahCompactHeuristics(ShenandoahSpaceInfo* space_info);
- virtual bool should_start_gc();
+ bool should_start_gc() override;
- virtual size_t choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset,
- RegionData* data, size_t size,
- size_t actual_free);
+ void choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset,
+ RegionData* data, size_t size,
+ size_t actual_free) override;
- virtual const char* name() { return "Compact"; }
- virtual bool is_diagnostic() { return false; }
- virtual bool is_experimental() { return false; }
+ const char* name() override { return "Compact"; }
+ bool is_diagnostic() override { return false; }
+ bool is_experimental() override { return false; }
};
#endif // SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHCOMPACTHEURISTICS_HPP
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.cpp
index ee315ce5c7e..80e6decf57d 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.cpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.cpp
@@ -25,19 +25,205 @@
#include "gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.hpp"
#include "gc/shenandoah/shenandoahCollectionSet.hpp"
+#include "gc/shenandoah/shenandoahCollectionSetPreselector.hpp"
#include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
#include "gc/shenandoah/shenandoahGeneration.hpp"
#include "gc/shenandoah/shenandoahGenerationalHeap.inline.hpp"
#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
#include "gc/shenandoah/shenandoahOldGeneration.hpp"
#include "gc/shenandoah/shenandoahTrace.hpp"
+#include "gc/shenandoah/shenandoahYoungGeneration.hpp"
#include "logging/log.hpp"
+#include "utilities/quickSort.hpp"
-ShenandoahGenerationalHeuristics::ShenandoahGenerationalHeuristics(ShenandoahGeneration* generation)
- : ShenandoahAdaptiveHeuristics(generation), _generation(generation) {
+using idx_t = ShenandoahSimpleBitMap::idx_t;
+
+typedef struct {
+ ShenandoahHeapRegion* _region;
+ size_t _live_data;
+} AgedRegionData;
+
+static int compare_by_aged_live(AgedRegionData a, AgedRegionData b) {
+ if (a._live_data < b._live_data)
+ return -1;
+ if (a._live_data > b._live_data)
+ return 1;
+ return 0;
}
-size_t ShenandoahGenerationalHeuristics::choose_collection_set(ShenandoahCollectionSet* collection_set) {
+inline void assert_no_in_place_promotions() {
+#ifdef ASSERT
+ class ShenandoahNoInPlacePromotions : public ShenandoahHeapRegionClosure {
+ public:
+ void heap_region_do(ShenandoahHeapRegion *r) override {
+ assert(r->get_top_before_promote() == nullptr,
+ "Region %zu should not be ready for in-place promotion", r->index());
+ }
+ } cl;
+ ShenandoahHeap::heap()->heap_region_iterate(&cl);
+#endif
+}
+
+ShenandoahGenerationalHeuristics::ShenandoahGenerationalHeuristics(ShenandoahGeneration* generation)
+ : ShenandoahAdaptiveHeuristics(generation), _generation(generation), _add_regions_to_old(0) {
+}
+
+void ShenandoahGenerationalHeuristics::choose_collection_set(ShenandoahCollectionSet* collection_set) {
+ ShenandoahHeap* heap = ShenandoahHeap::heap();
+
+ _add_regions_to_old = 0;
+
+ // Seed the collection set with resource area-allocated
+ // preselected regions, which are removed when we exit this scope.
+ ShenandoahCollectionSetPreselector preselector(collection_set, heap->num_regions());
+
+ // Find the amount that will be promoted, regions that will be promoted in
+ // place, and preselected older regions that will be promoted by evacuation.
+ compute_evacuation_budgets(heap);
+
+ // Choose the collection set, including the regions preselected above for promotion into the old generation.
+ filter_regions(collection_set);
+
+ // Even if collection_set->is_empty(), we want to adjust budgets, making reserves available to mutator.
+ adjust_evacuation_budgets(heap, collection_set);
+
+ if (_generation->is_global()) {
+ // We have just chosen a collection set for a global cycle. The mark bitmap covering old regions is complete, so
+ // the remembered set scan can use that to avoid walking into garbage. When the next old mark begins, we will
+ // use the mark bitmap to make the old regions parsable by coalescing and filling any unmarked objects. Thus,
+ // we prepare for old collections by remembering which regions are old at this time. Note that any objects
+ // promoted into old regions will be above TAMS, and so will be considered marked. However, free regions that
+ // become old after this point will not be covered correctly by the mark bitmap, so we must be careful not to
+ // coalesce those regions. Only the old regions which are not part of the collection set at this point are
+ // eligible for coalescing. As implemented now, this has the side effect of possibly initiating mixed-evacuations
+ // after a global cycle for old regions that were not included in this collection set.
+ heap->old_generation()->prepare_for_mixed_collections_after_global_gc();
+ }
+}
+
+void ShenandoahGenerationalHeuristics::compute_evacuation_budgets(ShenandoahHeap* const heap) {
+ shenandoah_assert_generational();
+
+ ShenandoahOldGeneration* const old_generation = heap->old_generation();
+ ShenandoahYoungGeneration* const young_generation = heap->young_generation();
+ const size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes();
+
+ // During initialization and phase changes, it is more likely that fewer objects die young and old-gen
+ // memory is not yet full (or is in the process of being replaced). During these times especially, it
+ // is beneficial to loan memory from old-gen to young-gen during the evacuation and update-refs phases
+ // of execution.
+
+ // Calculate EvacuationReserve before PromotionReserve. Evacuation is more critical than promotion.
+ // If we cannot evacuate old-gen, we will not be able to reclaim old-gen memory. Promotions are less
+ // critical. If we cannot promote, there may be degradation of young-gen memory because old objects
+ // accumulate there until they can be promoted. This increases the young-gen marking and evacuation work.
+
+ // First priority is to reclaim the easy garbage out of young-gen.
+
+ // maximum_young_evacuation_reserve is upper bound on memory to be evacuated into young Collector Reserve. This is
+ // bounded at the end of previous GC cycle, based on available memory and balancing of evacuation to old and young.
+ size_t maximum_young_evacuation_reserve = young_generation->get_evacuation_reserve();
+
+ // maximum_old_evacuation_reserve is an upper bound on memory evacuated from old and evacuated to old (promoted),
+ // clamped by the old generation space available.
+ //
+ // Here's the algebra.
+ // Let SOEP = ShenandoahOldEvacPercent,
+ // OE = old evac,
+ // YE = young evac, and
+ // TE = total evac = OE + YE
+ // By definition:
+ // SOEP/100 = OE/TE
+ // = OE/(OE+YE)
+ // => SOEP/(100-SOEP) = OE/((OE+YE)-OE) // componendo-dividendo: If a/b = c/d, then a/(b-a) = c/(d-c)
+ // = OE/YE
+ // => OE = YE*SOEP/(100-SOEP)
+
+ // We have to be careful in the event that SOEP is set to 100 by the user.
+ assert(ShenandoahOldEvacPercent <= 100, "Error");
+ const size_t old_available = old_generation->available();
+ const size_t maximum_old_evacuation_reserve = (ShenandoahOldEvacPercent == 100) ?
+ old_available : MIN2((maximum_young_evacuation_reserve * ShenandoahOldEvacPercent) / (100 - ShenandoahOldEvacPercent),
+ old_available);
+
+ // In some cases, maximum_old_reserve < old_available (when limited by ShenandoahOldEvacPercent)
+ // This limit affects mixed evacuations, but does not affect promotions.
+
+ // Second priority is to reclaim garbage out of old-gen if there are old-gen collection candidates. Third priority
+ // is to promote as much as we have room to promote. However, if old-gen memory is in short supply, this means young
+ // GC is operating under "duress" and was unable to transfer the memory that we would normally expect. In this case,
+ // old-gen will refrain from compacting itself in order to allow a quicker young-gen cycle (by avoiding the update-refs
+ // through ALL of old-gen). If there is some memory available in old-gen, we will use this for promotions as promotions
+ // do not add to the update-refs burden of GC.
+
+ size_t old_evacuation_reserve, old_promo_reserve;
+ if (_generation->is_global()) {
+ // Global GC is typically triggered by user invocation of System.gc(), and typically indicates that there is lots
+ // of garbage to be reclaimed because we are starting a new phase of execution. Marking for global GC may take
+ // significantly longer than typical young marking because we must mark through all old objects. To expedite
+ // evacuation and update-refs, we give emphasis to reclaiming garbage first, wherever that garbage is found.
+ // Global GC will adjust generation sizes to accommodate the collection set it chooses.
+
+ // Use remnant of old_available to hold promotions.
+ old_promo_reserve = old_available - maximum_old_evacuation_reserve;
+
+ // Dedicate all available old memory to old_evacuation reserve. This may be small, because old-gen is only
+ // expanded based on an existing mixed evacuation workload at the end of the previous GC cycle. We'll expand
+ // the budget for evacuation of old during GLOBAL cset selection.
+ old_evacuation_reserve = maximum_old_evacuation_reserve;
+ } else if (old_generation->has_unprocessed_collection_candidates()) {
+ // We reserved all old-gen memory at end of previous GC to hold anticipated evacuations to old-gen. If this is
+ // mixed evacuation, reserve all of this memory for compaction of old-gen and do not promote. Prioritize compaction
+ // over promotion in order to defragment OLD so that it will be better prepared to efficiently receive promoted memory.
+ old_evacuation_reserve = maximum_old_evacuation_reserve;
+ old_promo_reserve = old_available - maximum_old_evacuation_reserve;
+ } else {
+ // Make all old-evacuation memory for promotion, but if we can't use it all for promotion, we'll allow some evacuation.
+ old_evacuation_reserve = old_available - maximum_old_evacuation_reserve;
+ old_promo_reserve = maximum_old_evacuation_reserve;
+ }
+ assert(old_evacuation_reserve <= old_available, "Error");
+
+
+ // We see too many old-evacuation failures if we force ourselves to evacuate into regions that are not initially empty.
+ // So we limit the old-evacuation reserve to unfragmented memory. Even so, old-evacuation is free to fill in nooks and
+ // crannies within existing partially used regions and it generally tries to do so.
+ const size_t old_free_unfragmented = old_generation->free_unaffiliated_regions() * region_size_bytes;
+ if (old_evacuation_reserve > old_free_unfragmented) {
+ const size_t delta = old_evacuation_reserve - old_free_unfragmented;
+ old_evacuation_reserve -= delta;
+ // Let promo consume fragments of old-gen memory
+ old_promo_reserve += delta;
+ }
+
+ // If is_global(), we let garbage-first heuristic determine cset membership. Otherwise, we give priority
+ // to tenurable regions by preselecting regions for promotion by evacuation (obtaining the live data to seed promoted_reserve).
+ // This also identifies regions that will be promoted in place. These use the tenuring threshold.
+ const size_t consumed_by_advance_promotion = select_aged_regions(_generation->is_global()? 0: old_promo_reserve);
+ assert(consumed_by_advance_promotion <= old_promo_reserve, "Do not promote more than budgeted");
+
+ // The young evacuation reserve can be no larger than young_unaffiliated. Planning to evacuate into partially consumed
+ // young regions is doomed to failure if any of those partially consumed regions is selected for the collection set.
+ size_t young_unaffiliated = young_generation->free_unaffiliated_regions() * region_size_bytes;
+
+ // If any regions have been selected for promotion in place, this has the effect of decreasing available within mutator
+ // and collector partitions, due to padding of remnant memory within each promoted in place region. This will affect
+ // young_evacuation_reserve but not old_evacuation_reserve or consumed_by_advance_promotion. So recompute.
+ size_t young_evacuation_reserve = MIN2(maximum_young_evacuation_reserve, young_unaffiliated);
+
+ // Note that unused old_promo_reserve might not be entirely consumed_by_advance_promotion. Do not transfer this
+ // to old_evacuation_reserve because this memory is likely very fragmented, and we do not want to increase the likelihood
+ // of old evacuation failure. Leave this memory in the promoted reserve as it may be targeted by opportunistic
+ // promotions (found during evacuation of young regions).
+ young_generation->set_evacuation_reserve(young_evacuation_reserve);
+ old_generation->set_evacuation_reserve(old_evacuation_reserve);
+ old_generation->set_promoted_reserve(old_promo_reserve);
+
+ // There is no need to expand OLD because all memory used here was set aside at end of previous GC, except in the
+ // case of a GLOBAL gc. During choose_collection_set() of GLOBAL, old will be expanded on demand.
+}
+
+void ShenandoahGenerationalHeuristics::filter_regions(ShenandoahCollectionSet* collection_set) {
assert(collection_set->is_empty(), "Must be empty");
auto heap = ShenandoahGenerationalHeap::heap();
@@ -170,10 +356,9 @@ size_t ShenandoahGenerationalHeuristics::choose_collection_set(ShenandoahCollect
size_t immediate_percent = (total_garbage == 0) ? 0 : (immediate_garbage * 100 / total_garbage);
bool doing_promote_in_place = (humongous_regions_promoted + regular_regions_promoted_in_place > 0);
- size_t add_regions_to_old = 0;
if (doing_promote_in_place || (preselected_candidates > 0) || (immediate_percent <= ShenandoahImmediateThreshold)) {
// Call the subclasses to add young-gen regions into the collection set.
- add_regions_to_old = choose_collection_set_from_regiondata(collection_set, candidates, cand_idx, immediate_garbage + free);
+ choose_collection_set_from_regiondata(collection_set, candidates, cand_idx, immediate_garbage + free);
}
if (collection_set->has_old_regions()) {
@@ -190,9 +375,359 @@ size_t ShenandoahGenerationalHeuristics::choose_collection_set(ShenandoahCollect
regular_regions_promoted_free,
immediate_regions,
immediate_garbage);
- return add_regions_to_old;
}
+// Preselect for inclusion into the collection set all regions whose age is at or above tenure age and for which the
+// garbage percentage exceeds a dynamically adjusted threshold (known as the old-garbage threshold percentage). We
+// identify these regions by setting the appropriate entry of the collection set's preselected regions array to true.
+// All entries are initialized to false before calling this function.
+//
+// During the subsequent selection of the collection set, we give priority to these promotion set candidates.
+// Without this prioritization, we found that the aged regions tend to be ignored because they typically have
+// much less garbage and much more live data than the recently allocated "eden" regions. When aged regions are
+// repeatedly excluded from the collection set, the amount of live memory within the young generation tends to
+// accumulate and this has the undesirable side effect of causing young-generation collections to require much more
+// CPU and wall-clock time.
+//
+// A second benefit of treating aged regions differently than other regions during collection set selection is
+// that this allows us to more accurately budget memory to hold the results of evacuation. Memory for evacuation
+// of aged regions must be reserved in the old generation. Memory for evacuation of all other regions must be
+// reserved in the young generation.
+size_t ShenandoahGenerationalHeuristics::select_aged_regions(const size_t old_promotion_reserve) {
+
+ // There should be no regions configured for subsequent in-place-promotions carried over from the previous cycle.
+ assert_no_in_place_promotions();
+
+ auto const heap = ShenandoahGenerationalHeap::heap();
+ ShenandoahFreeSet* free_set = heap->free_set();
+ bool* const candidate_regions_for_promotion_by_copy = heap->collection_set()->preselected_regions();
+ ShenandoahMarkingContext* const ctx = heap->marking_context();
+
+ const size_t old_garbage_threshold =
+ (ShenandoahHeapRegion::region_size_bytes() * heap->old_generation()->heuristics()->get_old_garbage_threshold()) / 100;
+
+ const size_t pip_used_threshold = (ShenandoahHeapRegion::region_size_bytes() * ShenandoahGenerationalMinPIPUsage) / 100;
+
+ size_t promo_potential = 0;
+ size_t candidates = 0;
+
+ // Tracks the padding of space above top in regions eligible for promotion in place
+ size_t promote_in_place_pad = 0;
+
+ // Sort the promotion-eligible regions in order of increasing live-data-bytes so that we can first reclaim regions that require
+ // less evacuation effort. This prioritizes garbage first, expanding the allocation pool early before we reclaim regions that
+ // have more live data.
+ const idx_t num_regions = heap->num_regions();
+
+ ResourceMark rm;
+ AgedRegionData* sorted_regions = NEW_RESOURCE_ARRAY(AgedRegionData, num_regions);
+
+ ShenandoahFreeSet* freeset = heap->free_set();
+
+ // Any region that is to be promoted in place needs to be retired from its Collector or Mutator partition.
+ idx_t pip_low_collector_idx = freeset->max_regions();
+ idx_t pip_high_collector_idx = -1;
+ idx_t pip_low_mutator_idx = freeset->max_regions();
+ idx_t pip_high_mutator_idx = -1;
+ size_t collector_regions_to_pip = 0;
+ size_t mutator_regions_to_pip = 0;
+
+ size_t pip_mutator_regions = 0;
+ size_t pip_collector_regions = 0;
+ size_t pip_mutator_bytes = 0;
+ size_t pip_collector_bytes = 0;
+
+ for (idx_t i = 0; i < num_regions; i++) {
+ ShenandoahHeapRegion* const r = heap->get_region(i);
+ if (r->is_empty() || !r->has_live() || !r->is_young() || !r->is_regular()) {
+ // skip over regions that aren't regular young with some live data
+ continue;
+ }
+ if (heap->is_tenurable(r)) {
+ if ((r->garbage() < old_garbage_threshold) && (r->used() > pip_used_threshold)) {
+ // We prefer to promote this region in place because it has a small amount of garbage and a large usage.
+ HeapWord* tams = ctx->top_at_mark_start(r);
+ HeapWord* original_top = r->top();
+ if (!heap->is_concurrent_old_mark_in_progress() && tams == original_top) {
+ // No allocations from this region have been made during concurrent mark. It meets all the criteria
+ // for in-place-promotion. Though we only need the value of top when we fill the end of the region,
+ // we use this field to indicate that this region should be promoted in place during the evacuation
+ // phase.
+ r->save_top_before_promote();
+ size_t remnant_bytes = r->free();
+ size_t remnant_words = remnant_bytes / HeapWordSize;
+ assert(ShenandoahHeap::min_fill_size() <= PLAB::min_size(), "Implementation makes invalid assumptions");
+ if (remnant_words >= ShenandoahHeap::min_fill_size()) {
+ ShenandoahHeap::fill_with_object(original_top, remnant_words);
+ // Fill the remnant memory within this region to assure no allocations prior to promote in place. Otherwise,
+ // newly allocated objects will not be parsable when promote in place tries to register them. Furthermore, any
+ // new allocations would not necessarily be eligible for promotion. This addresses both issues.
+ r->set_top(r->end());
+ // The region r is either in the Mutator or Collector partition if remnant_words > heap()->plab_min_size.
+ // Otherwise, the region is in the NotFree partition.
+ ShenandoahFreeSetPartitionId p = free_set->membership(i);
+ if (p == ShenandoahFreeSetPartitionId::Mutator) {
+ mutator_regions_to_pip++;
+ if (i < pip_low_mutator_idx) {
+ pip_low_mutator_idx = i;
+ }
+ if (i > pip_high_mutator_idx) {
+ pip_high_mutator_idx = i;
+ }
+ pip_mutator_regions++;
+ pip_mutator_bytes += remnant_bytes;
+ } else if (p == ShenandoahFreeSetPartitionId::Collector) {
+ collector_regions_to_pip++;
+ if (i < pip_low_collector_idx) {
+ pip_low_collector_idx = i;
+ }
+ if (i > pip_high_collector_idx) {
+ pip_high_collector_idx = i;
+ }
+ pip_collector_regions++;
+ pip_collector_bytes += remnant_bytes;
+ } else {
+ assert((p == ShenandoahFreeSetPartitionId::NotFree) && (remnant_words < heap->plab_min_size()),
+ "Should be NotFree if not in Collector or Mutator partitions");
+ // In this case, the memory is already counted as used and the region has already been retired. There is
+ // no need for further adjustments to used. Further, the remnant memory for this region will not be
+ // unallocated or made available to OldCollector after pip.
+ remnant_bytes = 0;
+ }
+ promote_in_place_pad += remnant_bytes;
+ free_set->prepare_to_promote_in_place(i, remnant_bytes);
+ } else {
+ // Since the remnant is so small that this region has already been retired, we don't have to worry about any
+ // accidental allocations occurring within this region before the region is promoted in place.
+
+ // This region was already not in the Collector or Mutator set, so no need to remove it.
+ assert(free_set->membership(i) == ShenandoahFreeSetPartitionId::NotFree, "sanity");
+ }
+ }
+ // Else, we do not promote this region (either in place or by copy) because it has received new allocations.
+
+ // During evacuation, we exclude from promotion regions for which age > tenure threshold, garbage < garbage-threshold,
+ // used > pip_used_threshold, and get_top_before_promote() != tams
+ } else {
+ // Record this promotion-eligible candidate region. After sorting and selecting the best candidates below,
+ // we may still decide to exclude this promotion-eligible region from the current collection set. If this
+ // happens, we will consider this region as part of the anticipated promotion potential for the next GC
+ // pass; see further below.
+ sorted_regions[candidates]._region = r;
+ sorted_regions[candidates++]._live_data = r->get_live_data_bytes();
+ }
+ } else {
+ // We only evacuate & promote objects from regular regions whose garbage() is above old-garbage-threshold.
+ // Objects in tenure-worthy regions with less garbage are promoted in place. These take a different path to
+ // old-gen. Regions excluded from promotion because their garbage content is too low (causing us to anticipate that
+ // the region would be promoted in place) may be eligible for evacuation promotion by the time promotion takes
+ // place during a subsequent GC pass because more garbage is found within the region between now and then. This
+ // should not happen if we are properly adapting the tenure age. The theory behind adaptive tenuring threshold
+ // is to choose the youngest age that demonstrates no "significant" further loss of population since the previous
+ // age. If not this, we expect the tenure age to demonstrate linear population decay for at least two population
+ // samples, whereas we expect to observe exponential population decay for ages younger than the tenure age.
+ //
+ // In the case that certain regions which were anticipated to be promoted in place need to be promoted by
+ // evacuation, it may be the case that there is not sufficient reserve within old-gen to hold evacuation of
+ // these regions. The likely outcome is that these regions will not be selected for evacuation or promotion
+ // in the current cycle and we will anticipate that they will be promoted in the next cycle. This will cause
+ // us to reserve more old-gen memory so that these objects can be promoted in the subsequent cycle.
+ if (heap->is_aging_cycle() && heap->age_census()->is_tenurable(r->age() + 1)) {
+ if (r->garbage() >= old_garbage_threshold) {
+ promo_potential += r->get_live_data_bytes();
+ }
+ }
+ }
+ // Note that we keep going even if one region is excluded from selection.
+ // Subsequent regions may be selected if they have smaller live data.
+ }
+
+ if (pip_mutator_regions + pip_collector_regions > 0) {
+ freeset->account_for_pip_regions(pip_mutator_regions, pip_mutator_bytes, pip_collector_regions, pip_collector_bytes);
+ }
+
+ // Retire any regions that have been selected for promote in place
+ if (collector_regions_to_pip > 0) {
+ freeset->shrink_interval_if_range_modifies_either_boundary(ShenandoahFreeSetPartitionId::Collector,
+ pip_low_collector_idx, pip_high_collector_idx,
+ collector_regions_to_pip);
+ }
+ if (mutator_regions_to_pip > 0) {
+ freeset->shrink_interval_if_range_modifies_either_boundary(ShenandoahFreeSetPartitionId::Mutator,
+ pip_low_mutator_idx, pip_high_mutator_idx,
+ mutator_regions_to_pip);
+ }
+
+ // Sort in increasing order according to live data bytes. Note that candidates represents the number of regions
+ // that qualify to be promoted by evacuation.
+ size_t old_consumed = 0;
+ if (candidates > 0) {
+ size_t selected_regions = 0;
+ size_t selected_live = 0;
+ QuickSort::sort(sorted_regions, candidates, compare_by_aged_live);
+ for (size_t i = 0; i < candidates; i++) {
+ ShenandoahHeapRegion* const region = sorted_regions[i]._region;
+ const size_t region_live_data = sorted_regions[i]._live_data;
+ const size_t promotion_need = (size_t) (region_live_data * ShenandoahPromoEvacWaste);
+ if (old_consumed + promotion_need <= old_promotion_reserve) {
+ old_consumed += promotion_need;
+ candidate_regions_for_promotion_by_copy[region->index()] = true;
+ selected_regions++;
+ selected_live += region_live_data;
+ } else {
+ // We rejected this promotable region from the collection set because we had no room to hold its copy.
+ // Add this region to promo potential for next GC.
+ promo_potential += region_live_data;
+ assert(!candidate_regions_for_promotion_by_copy[region->index()], "Shouldn't be selected");
+ }
+ // We keep going even if one region is excluded from selection because we need to accumulate all eligible
+ // regions that are not preselected into promo_potential
+ }
+ log_debug(gc, ergo)("Preselected %zu regions containing " PROPERFMT " live data,"
+ " consuming: " PROPERFMT " of budgeted: " PROPERFMT,
+ selected_regions, PROPERFMTARGS(selected_live), PROPERFMTARGS(old_consumed), PROPERFMTARGS(old_promotion_reserve));
+ }
+
+ log_info(gc, ergo)("Promotion potential of aged regions with sufficient garbage: " PROPERFMT, PROPERFMTARGS(promo_potential));
+
+ heap->old_generation()->set_pad_for_promote_in_place(promote_in_place_pad);
+ heap->old_generation()->set_promotion_potential(promo_potential);
+ return old_consumed;
+}
+
+// Having chosen the collection set, adjust the budgets for generational mode based on its composition. Note
+// that young_generation->available() now knows about recently discovered immediate garbage.
+void ShenandoahGenerationalHeuristics::adjust_evacuation_budgets(ShenandoahHeap* const heap,
+ ShenandoahCollectionSet* const collection_set) {
+ shenandoah_assert_generational();
+ // We may find that old_evacuation_reserve and/or loaned_for_young_evacuation are not fully consumed, in which case we may
+ // be able to increase regions_available_to_loan
+
+ // The role of adjust_evacuation_budgets() is to compute the correct value of regions_available_to_loan and to make
+ // effective use of this memory, including the remnant memory within these regions that may result from rounding loan to
+ // integral number of regions. Excess memory that is available to be loaned is applied to an allocation supplement,
+ // which allows mutators to allocate memory beyond the current capacity of young-gen on the promise that the loan
+ // will be repaid as soon as we finish updating references for the recently evacuated collection set.
+
+ // We cannot recalculate regions_available_to_loan by simply dividing old_generation->available() by region_size_bytes
+ // because the available memory may be distributed between many partially occupied regions that are already holding old-gen
+ // objects. Memory in partially occupied regions is not "available" to be loaned. Note that an increase in old-gen
+ // available that results from a decrease in memory consumed by old evacuation is not necessarily available to be loaned
+ // to young-gen.
+
+ const size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes();
+ ShenandoahOldGeneration* const old_generation = heap->old_generation();
+ ShenandoahYoungGeneration* const young_generation = heap->young_generation();
+
+ const size_t old_evacuated = collection_set->get_live_bytes_in_old_regions();
+ size_t old_evacuated_committed = (size_t) (ShenandoahOldEvacWaste * double(old_evacuated));
+ size_t old_evacuation_reserve = old_generation->get_evacuation_reserve();
+
+ if (old_evacuated_committed > old_evacuation_reserve) {
+ // This should only happen due to round-off errors when enforcing ShenandoahOldEvacWaste
+ assert(old_evacuated_committed <= (33 * old_evacuation_reserve) / 32,
+ "Round-off errors should be less than 3.125%%, committed: %zu, reserved: %zu",
+ old_evacuated_committed, old_evacuation_reserve);
+ old_evacuated_committed = old_evacuation_reserve;
+ // Leave old_evac_reserve as previously configured
+ } else if (old_evacuated_committed < old_evacuation_reserve) {
+ // This happens if the old-gen collection consumes less than full budget.
+ log_debug(gc, cset)("Shrinking old evac reserve to match old_evac_commited: " PROPERFMT,
+ PROPERFMTARGS(old_evacuated_committed));
+ old_evacuation_reserve = old_evacuated_committed;
+ old_generation->set_evacuation_reserve(old_evacuation_reserve);
+ }
+
+ size_t young_advance_promoted = collection_set->get_live_bytes_in_tenurable_regions();
+ size_t young_advance_promoted_reserve_used = (size_t) (ShenandoahPromoEvacWaste * double(young_advance_promoted));
+
+ size_t young_evacuated = collection_set->get_live_bytes_in_untenurable_regions();
+ size_t young_evacuated_reserve_used = (size_t) (ShenandoahEvacWaste * double(young_evacuated));
+
+ size_t total_young_available = young_generation->available_with_reserve() - _add_regions_to_old * region_size_bytes;;
+ assert(young_evacuated_reserve_used <= total_young_available, "Cannot evacuate (%zu) more than is available in young (%zu)",
+ young_evacuated_reserve_used, total_young_available);
+ young_generation->set_evacuation_reserve(young_evacuated_reserve_used);
+
+ // We have not yet rebuilt the free set. Some of the memory that is thought to be avaiable within old may no
+ // longer be available if that memory had been free within regions that were selected for the collection set.
+ // Make the necessary adjustments to old_available.
+ size_t old_available =
+ old_generation->available() + _add_regions_to_old * region_size_bytes - collection_set->get_old_available_bytes_collected();
+
+ // Now that we've established the collection set, we know how much memory is really required by old-gen for evacuation
+ // and promotion reserves. Try shrinking OLD now in case that gives us a bit more runway for mutator allocations during
+ // evac and update phases.
+ size_t old_consumed = old_evacuated_committed + young_advance_promoted_reserve_used;
+
+ if (old_available < old_consumed) {
+ // This can happen due to round-off errors when adding the results of truncated integer arithmetic.
+ // We've already truncated old_evacuated_committed. Truncate young_advance_promoted_reserve_used here.
+
+ assert(young_advance_promoted_reserve_used <= (33 * (old_available - old_evacuated_committed)) / 32,
+ "Round-off errors should be less than 3.125%%, committed: %zu, reserved: %zu",
+ young_advance_promoted_reserve_used, old_available - old_evacuated_committed);
+ if (old_available > old_evacuated_committed) {
+ young_advance_promoted_reserve_used = old_available - old_evacuated_committed;
+ } else {
+ young_advance_promoted_reserve_used = 0;
+ old_evacuated_committed = old_available;
+ }
+ // TODO: reserve for full promotion reserve, not just for advance (preselected) promotion
+ old_consumed = old_evacuated_committed + young_advance_promoted_reserve_used;
+ }
+
+ assert(old_available >= old_consumed, "Cannot consume (%zu) more than is available (%zu)",
+ old_consumed, old_available);
+ size_t excess_old = old_available - old_consumed;
+ size_t unaffiliated_old_regions = old_generation->free_unaffiliated_regions() + _add_regions_to_old;
+ size_t unaffiliated_old = unaffiliated_old_regions * region_size_bytes;
+ assert(unaffiliated_old >= old_evacuated_committed, "Do not evacuate (%zu) more than unaffiliated old (%zu)",
+ old_evacuated_committed, unaffiliated_old);
+
+ // Make sure old_evac_committed is unaffiliated
+ if (old_evacuated_committed > 0) {
+ if (unaffiliated_old > old_evacuated_committed) {
+ size_t giveaway = unaffiliated_old - old_evacuated_committed;
+ size_t giveaway_regions = giveaway / region_size_bytes; // round down
+ if (giveaway_regions > 0) {
+ excess_old = MIN2(excess_old, giveaway_regions * region_size_bytes);
+ } else {
+ excess_old = 0;
+ }
+ } else {
+ excess_old = 0;
+ }
+ }
+
+ // If we find that OLD has excess regions, give them back to YOUNG now to reduce likelihood we run out of allocation
+ // runway during evacuation and update-refs. We may make further adjustments to balance.
+ ssize_t add_regions_to_young = 0;
+ if (excess_old > unaffiliated_old) {
+ // we can give back unaffiliated_old (all of unaffiliated is excess)
+ if (unaffiliated_old_regions > 0) {
+ add_regions_to_young = unaffiliated_old_regions;
+ }
+ } else if (unaffiliated_old_regions > 0) {
+ // excess_old < unaffiliated old: we can give back MIN(excess_old/region_size_bytes, unaffiliated_old_regions)
+ size_t excess_regions = excess_old / region_size_bytes;
+ add_regions_to_young = MIN2(excess_regions, unaffiliated_old_regions);
+ }
+
+ if (add_regions_to_young > 0) {
+ assert(excess_old >= add_regions_to_young * region_size_bytes, "Cannot xfer more than excess old");
+ excess_old -= add_regions_to_young * region_size_bytes;
+ log_debug(gc, ergo)("Before start of evacuation, total_promotion reserve is young_advance_promoted_reserve: %zu "
+ "plus excess: old: %zu", young_advance_promoted_reserve_used, excess_old);
+ }
+
+ // Add in the excess_old memory to hold unanticipated promotions, if any. If there are more unanticipated
+ // promotions than fit in reserved memory, they will be deferred until a future GC pass.
+ size_t total_promotion_reserve = young_advance_promoted_reserve_used + excess_old;
+
+ old_generation->set_promoted_reserve(total_promotion_reserve);
+ old_generation->reset_promoted_expended();
+}
size_t ShenandoahGenerationalHeuristics::add_preselected_regions_to_collection_set(ShenandoahCollectionSet* cset,
const RegionData* data,
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.hpp
index 9b4c93af9b4..74d657feab7 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.hpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.hpp
@@ -29,6 +29,9 @@
#include "gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp"
class ShenandoahGeneration;
+class ShenandoahHeap;
+class ShenandoahCollectionSet;
+class RegionData;
/*
* This class serves as the base class for heuristics used to trigger and
@@ -44,10 +47,42 @@ class ShenandoahGenerationalHeuristics : public ShenandoahAdaptiveHeuristics {
public:
explicit ShenandoahGenerationalHeuristics(ShenandoahGeneration* generation);
- size_t choose_collection_set(ShenandoahCollectionSet* collection_set) override;
+ void choose_collection_set(ShenandoahCollectionSet* collection_set) override;
+
+private:
+ // Compute evacuation budgets prior to choosing collection set.
+ void compute_evacuation_budgets(ShenandoahHeap* const heap);
+
+ // Preselect for possible inclusion into the collection set exactly the most
+ // garbage-dense regions, including those that satisfy criteria 1 & 2 below,
+ // and whose live bytes will fit within old_available budget:
+ // Criterion 1. region age >= tenuring threshold
+ // Criterion 2. region garbage percentage > old garbage threshold
+ //
+ // Identifies regions eligible for promotion in place,
+ // being those of at least tenuring_threshold age that have lower garbage
+ // density.
+ //
+ // Updates promotion_potential and pad_for_promote_in_place fields
+ // of the heap. Returns bytes of live object memory in the preselected
+ // regions, which are marked in the preselected_regions() indicator
+ // array of the heap's collection set, which should be initialized
+ // to false.
+ size_t select_aged_regions(const size_t old_promotion_reserve);
+
+ // Filter and sort remaining regions before adding to collection set.
+ void filter_regions(ShenandoahCollectionSet* collection_set);
+
+ // Adjust evacuation budgets after choosing collection set. The argument regions_to_xfer
+ // represents regions to be transferred to old based on decisions made in top_off_collection_set()
+ void adjust_evacuation_budgets(ShenandoahHeap* const heap,
+ ShenandoahCollectionSet* const collection_set);
+
protected:
ShenandoahGeneration* _generation;
+ size_t _add_regions_to_old;
+
size_t add_preselected_regions_to_collection_set(ShenandoahCollectionSet* cset,
const RegionData* data,
size_t size) const;
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGlobalHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGlobalHeuristics.cpp
index f47371c14d5..dd2ad28aa4b 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGlobalHeuristics.cpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGlobalHeuristics.cpp
@@ -36,14 +36,13 @@ ShenandoahGlobalHeuristics::ShenandoahGlobalHeuristics(ShenandoahGlobalGeneratio
}
-size_t ShenandoahGlobalHeuristics::choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset,
- RegionData* data, size_t size,
- size_t actual_free) {
+void ShenandoahGlobalHeuristics::choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset,
+ RegionData* data, size_t size,
+ size_t actual_free) {
// Better select garbage-first regions
- QuickSort::sort(data, (int) size, compare_by_garbage);
+ QuickSort::sort(data, size, compare_by_garbage);
choose_global_collection_set(cset, data, size, actual_free, 0 /* cur_young_garbage */);
- return 0;
}
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGlobalHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGlobalHeuristics.hpp
index e0513f60da9..1f95f75c521 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGlobalHeuristics.hpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGlobalHeuristics.hpp
@@ -39,9 +39,9 @@ class ShenandoahGlobalHeuristics : public ShenandoahGenerationalHeuristics {
public:
ShenandoahGlobalHeuristics(ShenandoahGlobalGeneration* generation);
- size_t choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset,
- RegionData* data, size_t size,
- size_t actual_free) override;
+ void choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset,
+ RegionData* data, size_t size,
+ size_t actual_free) override;
private:
void choose_global_collection_set(ShenandoahCollectionSet* cset,
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp
index aeb64b6f1df..8fc744112bf 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp
@@ -72,7 +72,7 @@ ShenandoahHeuristics::~ShenandoahHeuristics() {
FREE_C_HEAP_ARRAY(RegionGarbage, _region_data);
}
-size_t ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collection_set) {
+void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collection_set) {
ShenandoahHeap* heap = ShenandoahHeap::heap();
assert(collection_set->is_empty(), "Must be empty");
@@ -154,7 +154,6 @@ size_t ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* coll
choose_collection_set_from_regiondata(collection_set, candidates, cand_idx, immediate_garbage + free);
}
collection_set->summarize(total_garbage, immediate_garbage, immediate_regions);
- return 0;
}
void ShenandoahHeuristics::record_cycle_start() {
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp
index ae34a9743a9..633c4e87126 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp
@@ -183,12 +183,10 @@ protected:
static int compare_by_garbage(RegionData a, RegionData b);
- // This is a helper function to choose_collection_set(), returning the number of regions that need to be transferred to
- // the old reserve from the young reserve in order to effectively evacuate the chosen collection set. In non-generational
- // mode, the return value is 0.
- virtual size_t choose_collection_set_from_regiondata(ShenandoahCollectionSet* set,
- RegionData* data, size_t data_size,
- size_t free) = 0;
+ // This is a helper function to choose_collection_set()
+ virtual void choose_collection_set_from_regiondata(ShenandoahCollectionSet* set,
+ RegionData* data, size_t data_size,
+ size_t free) = 0;
void adjust_penalty(intx step);
@@ -238,7 +236,7 @@ public:
// Choose the collection set, returning the number of regions that need to be transferred to the old reserve from the young
// reserve in order to effectively evacuate the chosen collection set. In non-generational mode, the return value is 0.
- virtual size_t choose_collection_set(ShenandoahCollectionSet* collection_set);
+ virtual void choose_collection_set(ShenandoahCollectionSet* collection_set);
virtual bool can_unload_classes();
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp
index f47d0cbe819..e0cab781674 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp
@@ -884,9 +884,8 @@ bool ShenandoahOldHeuristics::is_experimental() {
return true;
}
-size_t ShenandoahOldHeuristics::choose_collection_set_from_regiondata(ShenandoahCollectionSet* set,
- ShenandoahHeuristics::RegionData* data,
- size_t data_size, size_t free) {
+void ShenandoahOldHeuristics::choose_collection_set_from_regiondata(ShenandoahCollectionSet* set,
+ ShenandoahHeuristics::RegionData* data,
+ size_t data_size, size_t free) {
ShouldNotReachHere();
- return 0;
}
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp
index 97a5b1ebf24..fc7a35aa6c8 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp
@@ -155,8 +155,8 @@ private:
void set_trigger_if_old_is_overgrown();
protected:
- size_t
- choose_collection_set_from_regiondata(ShenandoahCollectionSet* set, RegionData* data, size_t data_size, size_t free) override;
+ void choose_collection_set_from_regiondata(ShenandoahCollectionSet* set,
+ RegionData* data, size_t data_size, size_t free) override;
// This internal helper routine adds as many mixed evacuation candidate regions as fit within the old-gen evacuation budget
// to the collection set. This may be called twice to prepare for any given mixed evacuation cycle, the first time with
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp
index d4a38278161..b5e9cc433ea 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp
@@ -50,9 +50,9 @@ bool ShenandoahPassiveHeuristics::should_degenerate_cycle() {
return ShenandoahDegeneratedGC;
}
-size_t ShenandoahPassiveHeuristics::choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset,
- RegionData* data, size_t size,
- size_t actual_free) {
+void ShenandoahPassiveHeuristics::choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset,
+ RegionData* data, size_t size,
+ size_t actual_free) {
assert(ShenandoahDegeneratedGC, "This path is only taken for Degenerated GC");
// Do not select too large CSet that would overflow the available free space.
@@ -76,5 +76,4 @@ size_t ShenandoahPassiveHeuristics::choose_collection_set_from_regiondata(Shenan
cset->add_region(r);
}
}
- return 0;
}
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.hpp
index 7a64fad7cc9..3cb85f5d05f 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.hpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.hpp
@@ -40,19 +40,19 @@ class ShenandoahPassiveHeuristics : public ShenandoahHeuristics {
public:
ShenandoahPassiveHeuristics(ShenandoahSpaceInfo* space_info);
- virtual bool should_start_gc();
+ bool should_start_gc() override;
- virtual bool should_unload_classes();
+ bool should_unload_classes() override;
- virtual bool should_degenerate_cycle();
+ bool should_degenerate_cycle() override;
- virtual size_t choose_collection_set_from_regiondata(ShenandoahCollectionSet* set,
- RegionData* data, size_t data_size,
- size_t free);
+ void choose_collection_set_from_regiondata(ShenandoahCollectionSet* set,
+ RegionData* data, size_t data_size,
+ size_t free) override;
- virtual const char* name() { return "Passive"; }
- virtual bool is_diagnostic() { return true; }
- virtual bool is_experimental() { return false; }
+ const char* name() override { return "Passive"; }
+ bool is_diagnostic() override { return true; }
+ bool is_experimental() override { return false; }
};
#endif // SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHPASSIVEHEURISTICS_HPP
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp
index 3843e434781..5f384f3dc73 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp
@@ -37,8 +37,6 @@ ShenandoahStaticHeuristics::ShenandoahStaticHeuristics(ShenandoahSpaceInfo* spac
SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahImplicitGCInvokesConcurrent);
}
-ShenandoahStaticHeuristics::~ShenandoahStaticHeuristics() {}
-
bool ShenandoahStaticHeuristics::should_start_gc() {
size_t capacity = ShenandoahHeap::heap()->soft_max_capacity();
size_t available = _space_info->soft_mutator_available();
@@ -59,9 +57,9 @@ bool ShenandoahStaticHeuristics::should_start_gc() {
return ShenandoahHeuristics::should_start_gc();
}
-size_t ShenandoahStaticHeuristics::choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset,
- RegionData* data, size_t size,
- size_t free) {
+void ShenandoahStaticHeuristics::choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset,
+ RegionData* data, size_t size,
+ size_t free) {
size_t threshold = ShenandoahHeapRegion::region_size_bytes() * ShenandoahGarbageThreshold / 100;
for (size_t idx = 0; idx < size; idx++) {
@@ -70,5 +68,4 @@ size_t ShenandoahStaticHeuristics::choose_collection_set_from_regiondata(Shenand
cset->add_region(r);
}
}
- return 0;
}
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp
index 27dc3c8e0ae..b1514b55e5a 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp
@@ -34,19 +34,17 @@
*/
class ShenandoahStaticHeuristics : public ShenandoahHeuristics {
public:
- ShenandoahStaticHeuristics(ShenandoahSpaceInfo* space_info);
+ explicit ShenandoahStaticHeuristics(ShenandoahSpaceInfo* space_info);
- virtual ~ShenandoahStaticHeuristics();
+ bool should_start_gc() override;
- virtual bool should_start_gc();
+ void choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset,
+ RegionData* data, size_t size,
+ size_t free) override;
- virtual size_t choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset,
- RegionData* data, size_t size,
- size_t free);
-
- virtual const char* name() { return "Static"; }
- virtual bool is_diagnostic() { return false; }
- virtual bool is_experimental() { return false; }
+ const char* name() override { return "Static"; }
+ bool is_diagnostic() override { return false; }
+ bool is_experimental() override { return false; }
};
#endif // SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHSTATICHEURISTICS_HPP
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahYoungHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahYoungHeuristics.cpp
index 01c3873df72..beff2200d90 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahYoungHeuristics.cpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahYoungHeuristics.cpp
@@ -37,7 +37,7 @@ ShenandoahYoungHeuristics::ShenandoahYoungHeuristics(ShenandoahYoungGeneration*
}
-size_t ShenandoahYoungHeuristics::choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset,
+void ShenandoahYoungHeuristics::choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset,
RegionData* data, size_t size,
size_t actual_free) {
// See comments in ShenandoahAdaptiveHeuristics::choose_collection_set_from_regiondata():
@@ -52,7 +52,7 @@ size_t ShenandoahYoungHeuristics::choose_collection_set_from_regiondata(Shenando
bool need_to_finalize_mixed = heap->old_generation()->heuristics()->prime_collection_set(cset);
// Better select garbage-first regions
- QuickSort::sort(data, (int) size, compare_by_garbage);
+ QuickSort::sort(data, size, compare_by_garbage);
size_t cur_young_garbage = add_preselected_regions_to_collection_set(cset, data, size);
@@ -62,12 +62,10 @@ size_t ShenandoahYoungHeuristics::choose_collection_set_from_regiondata(Shenando
// enough consolidated garbage to make effective use of young-gen evacuation reserve. If there is still
// young-gen reserve available following selection of the young-gen collection set, see if we can use
// this memory to expand the old-gen evacuation collection set.
- size_t add_regions_to_old;
- need_to_finalize_mixed |= heap->old_generation()->heuristics()->top_off_collection_set(add_regions_to_old);
+ need_to_finalize_mixed |= heap->old_generation()->heuristics()->top_off_collection_set(_add_regions_to_old);
if (need_to_finalize_mixed) {
heap->old_generation()->heuristics()->finalize_mixed_evacs();
}
- return add_regions_to_old;
}
void ShenandoahYoungHeuristics::choose_young_collection_set(ShenandoahCollectionSet* cset,
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahYoungHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahYoungHeuristics.hpp
index 85587887663..b9d64059680 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahYoungHeuristics.hpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahYoungHeuristics.hpp
@@ -38,9 +38,9 @@ public:
explicit ShenandoahYoungHeuristics(ShenandoahYoungGeneration* generation);
- size_t choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset,
- RegionData* data, size_t size,
- size_t actual_free) override;
+ void choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset,
+ RegionData* data, size_t size,
+ size_t actual_free) override;
bool should_start_gc() override;
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp
index 2aa37d7c575..004558a9fa8 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp
@@ -72,21 +72,33 @@ void ShenandoahBarrierSet::print_on(outputStream* st) const {
bool ShenandoahBarrierSet::need_load_reference_barrier(DecoratorSet decorators, BasicType type) {
if (!ShenandoahLoadRefBarrier) return false;
- // Only needed for references
return is_reference_type(type);
}
bool ShenandoahBarrierSet::need_keep_alive_barrier(DecoratorSet decorators, BasicType type) {
if (!ShenandoahSATBBarrier) return false;
- // Only needed for references
if (!is_reference_type(type)) return false;
-
bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0;
bool unknown = (decorators & ON_UNKNOWN_OOP_REF) != 0;
bool on_weak_ref = (decorators & (ON_WEAK_OOP_REF | ON_PHANTOM_OOP_REF)) != 0;
return (on_weak_ref || unknown) && keep_alive;
}
+bool ShenandoahBarrierSet::need_satb_barrier(DecoratorSet decorators, BasicType type) {
+ if (!ShenandoahSATBBarrier) return false;
+ if (!is_reference_type(type)) return false;
+ bool as_normal = (decorators & AS_NORMAL) != 0;
+ bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
+ return as_normal && !dest_uninitialized;
+}
+
+bool ShenandoahBarrierSet::need_card_barrier(DecoratorSet decorators, BasicType type) {
+ if (!ShenandoahCardBarrier) return false;
+ if (!is_reference_type(type)) return false;
+ bool in_heap = (decorators & IN_HEAP) != 0;
+ return in_heap;
+}
+
void ShenandoahBarrierSet::on_slowpath_allocation_exit(JavaThread* thread, oop new_obj) {
#if COMPILER2_OR_JVMCI
if (ReduceInitialCardMarks && ShenandoahCardBarrier && !ShenandoahHeap::heap()->is_in_young(new_obj)) {
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp
index 28765605267..e7a0ed57740 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp
@@ -60,6 +60,8 @@ public:
static bool need_load_reference_barrier(DecoratorSet decorators, BasicType type);
static bool need_keep_alive_barrier(DecoratorSet decorators, BasicType type);
+ static bool need_satb_barrier(DecoratorSet decorators, BasicType type);
+ static bool need_card_barrier(DecoratorSet decorators, BasicType type);
static bool is_strong_access(DecoratorSet decorators) {
return (decorators & (ON_WEAK_OOP_REF | ON_PHANTOM_OOP_REF)) == 0;
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp
index 5fe90f64f98..1166333ae3a 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp
@@ -63,17 +63,10 @@ private:
public:
// The most common scenario for lack of good progress following a degenerated GC is an accumulation of floating
- // garbage during the most recently aborted concurrent GC effort. With generational GC, it is far more effective to
+ // garbage during the most recently aborted concurrent GC effort. Usually, it is far more effective to
// reclaim this floating garbage with another degenerated cycle (which focuses on young generation and might require
- // a pause of 200 ms) rather than a full GC cycle (which may require over 2 seconds with a 10 GB old generation).
- //
- // In generational mode, we'll only upgrade to full GC if we've done two degen cycles in a row and both indicated
- // bad progress. In non-generational mode, we'll preserve the original behavior, which is to upgrade to full
- // immediately following a degenerated cycle with bad progress. This preserves original behavior of non-generational
- // Shenandoah to avoid introducing "surprising new behavior." It also makes less sense with non-generational
- // Shenandoah to replace a full GC with a degenerated GC, because both have similar pause times in non-generational
- // mode.
- static constexpr size_t GENERATIONAL_CONSECUTIVE_BAD_DEGEN_PROGRESS_THRESHOLD = 2;
+ // a pause of 200 ms) rather than a full GC cycle (which may require multiple seconds with a 10 GB old generation).
+ static constexpr size_t CONSECUTIVE_BAD_DEGEN_PROGRESS_THRESHOLD = 2;
ShenandoahCollectorPolicy();
@@ -117,9 +110,9 @@ public:
return _consecutive_degenerated_gcs;
}
- // Genshen will only upgrade to a full gc after the configured number of futile degenerated cycles.
- bool generational_should_upgrade_degenerated_gc() const {
- return _consecutive_degenerated_gcs_without_progress >= GENERATIONAL_CONSECUTIVE_BAD_DEGEN_PROGRESS_THRESHOLD;
+ // Only upgrade to a full gc after the configured number of futile degenerated cycles.
+ bool should_upgrade_degenerated_gc() const {
+ return _consecutive_degenerated_gcs_without_progress >= CONSECUTIVE_BAD_DEGEN_PROGRESS_THRESHOLD;
}
static bool is_allocation_failure(GCCause::Cause cause);
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp
index 333bdbc6e72..99776e38bfe 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp
@@ -314,7 +314,7 @@ void ShenandoahDegenGC::op_degenerated() {
if (progress) {
heap->notify_gc_progress();
_generation->heuristics()->record_degenerated();
- } else if (!heap->mode()->is_generational() || policy->generational_should_upgrade_degenerated_gc()) {
+ } else if (policy->should_upgrade_degenerated_gc()) {
// Upgrade to full GC, register full-GC impact on heuristics.
op_degenerated_futile();
} else {
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp
index cdc7e1a328a..ddb50ee0020 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp
@@ -24,7 +24,6 @@
*/
#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
-#include "gc/shenandoah/shenandoahCollectionSetPreselector.hpp"
#include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
#include "gc/shenandoah/shenandoahFreeSet.hpp"
#include "gc/shenandoah/shenandoahGeneration.hpp"
@@ -245,506 +244,6 @@ void ShenandoahGeneration::parallel_heap_region_iterate_free(ShenandoahHeapRegio
ShenandoahHeap::heap()->parallel_heap_region_iterate(cl);
}
-void ShenandoahGeneration::compute_evacuation_budgets(ShenandoahHeap* const heap) {
- shenandoah_assert_generational();
-
- ShenandoahOldGeneration* const old_generation = heap->old_generation();
- ShenandoahYoungGeneration* const young_generation = heap->young_generation();
- const size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes();
-
- // During initialization and phase changes, it is more likely that fewer objects die young and old-gen
- // memory is not yet full (or is in the process of being replaced). During these times especially, it
- // is beneficial to loan memory from old-gen to young-gen during the evacuation and update-refs phases
- // of execution.
-
- // Calculate EvacuationReserve before PromotionReserve. Evacuation is more critical than promotion.
- // If we cannot evacuate old-gen, we will not be able to reclaim old-gen memory. Promotions are less
- // critical. If we cannot promote, there may be degradation of young-gen memory because old objects
- // accumulate there until they can be promoted. This increases the young-gen marking and evacuation work.
-
- // First priority is to reclaim the easy garbage out of young-gen.
-
- // maximum_young_evacuation_reserve is upper bound on memory to be evacuated into young Collector Reserve. This is
- // bounded at the end of previous GC cycle, based on available memory and balancing of evacuation to old and young.
- size_t maximum_young_evacuation_reserve = young_generation->get_evacuation_reserve();
-
- // maximum_old_evacuation_reserve is an upper bound on memory evacuated from old and evacuated to old (promoted),
- // clamped by the old generation space available.
- //
- // Here's the algebra.
- // Let SOEP = ShenandoahOldEvacPercent,
- // OE = old evac,
- // YE = young evac, and
- // TE = total evac = OE + YE
- // By definition:
- // SOEP/100 = OE/TE
- // = OE/(OE+YE)
- // => SOEP/(100-SOEP) = OE/((OE+YE)-OE) // componendo-dividendo: If a/b = c/d, then a/(b-a) = c/(d-c)
- // = OE/YE
- // => OE = YE*SOEP/(100-SOEP)
-
- // We have to be careful in the event that SOEP is set to 100 by the user.
- assert(ShenandoahOldEvacPercent <= 100, "Error");
- const size_t old_available = old_generation->available();
- const size_t maximum_old_evacuation_reserve = (ShenandoahOldEvacPercent == 100) ?
- old_available : MIN2((maximum_young_evacuation_reserve * ShenandoahOldEvacPercent) / (100 - ShenandoahOldEvacPercent),
- old_available);
-
- // In some cases, maximum_old_reserve < old_available (when limited by ShenandoahOldEvacPercent)
- // This limit affects mixed evacuations, but does not affect promotions.
-
- // Second priority is to reclaim garbage out of old-gen if there are old-gen collection candidates. Third priority
- // is to promote as much as we have room to promote. However, if old-gen memory is in short supply, this means young
- // GC is operating under "duress" and was unable to transfer the memory that we would normally expect. In this case,
- // old-gen will refrain from compacting itself in order to allow a quicker young-gen cycle (by avoiding the update-refs
- // through ALL of old-gen). If there is some memory available in old-gen, we will use this for promotions as promotions
- // do not add to the update-refs burden of GC.
-
- size_t old_evacuation_reserve, old_promo_reserve;
- if (is_global()) {
- // Global GC is typically triggered by user invocation of System.gc(), and typically indicates that there is lots
- // of garbage to be reclaimed because we are starting a new phase of execution. Marking for global GC may take
- // significantly longer than typical young marking because we must mark through all old objects. To expedite
- // evacuation and update-refs, we give emphasis to reclaiming garbage first, wherever that garbage is found.
- // Global GC will adjust generation sizes to accommodate the collection set it chooses.
-
- // Use remnant of old_available to hold promotions.
- old_promo_reserve = old_available - maximum_old_evacuation_reserve;
-
- // Dedicate all available old memory to old_evacuation reserve. This may be small, because old-gen is only
- // expanded based on an existing mixed evacuation workload at the end of the previous GC cycle. We'll expand
- // the budget for evacuation of old during GLOBAL cset selection.
- old_evacuation_reserve = maximum_old_evacuation_reserve;
- } else if (old_generation->has_unprocessed_collection_candidates()) {
- // We reserved all old-gen memory at end of previous GC to hold anticipated evacuations to old-gen. If this is
- // mixed evacuation, reserve all of this memory for compaction of old-gen and do not promote. Prioritize compaction
- // over promotion in order to defragment OLD so that it will be better prepared to efficiently receive promoted memory.
- old_evacuation_reserve = maximum_old_evacuation_reserve;
- old_promo_reserve = old_available - maximum_old_evacuation_reserve;
- } else {
- // Make all old-evacuation memory for promotion, but if we can't use it all for promotion, we'll allow some evacuation.
- old_evacuation_reserve = old_available - maximum_old_evacuation_reserve;
- old_promo_reserve = maximum_old_evacuation_reserve;
- }
- assert(old_evacuation_reserve <= old_available, "Error");
-
-
- // We see too many old-evacuation failures if we force ourselves to evacuate into regions that are not initially empty.
- // So we limit the old-evacuation reserve to unfragmented memory. Even so, old-evacuation is free to fill in nooks and
- // crannies within existing partially used regions and it generally tries to do so.
- const size_t old_free_unfragmented = old_generation->free_unaffiliated_regions() * region_size_bytes;
- if (old_evacuation_reserve > old_free_unfragmented) {
- const size_t delta = old_evacuation_reserve - old_free_unfragmented;
- old_evacuation_reserve -= delta;
- // Let promo consume fragments of old-gen memory
- old_promo_reserve += delta;
- }
-
- // If is_global(), we let garbage-first heuristic determine cset membership. Otherwise, we give priority
- // to tenurable regions by preselecting regions for promotion by evacuation (obtaining the live data to seed promoted_reserve).
- // This also identifies regions that will be promoted in place. These use the tenuring threshold.
- const size_t consumed_by_advance_promotion = select_aged_regions(is_global()? 0: old_promo_reserve);
- assert(consumed_by_advance_promotion <= old_promo_reserve, "Do not promote more than budgeted");
-
- // The young evacuation reserve can be no larger than young_unaffiliated. Planning to evacuate into partially consumed
- // young regions is doomed to failure if any of those partially consumed regions is selected for the collection set.
- size_t young_unaffiliated = young_generation->free_unaffiliated_regions() * region_size_bytes;
-
- // If any regions have been selected for promotion in place, this has the effect of decreasing available within mutator
- // and collector partitions, due to padding of remnant memory within each promoted in place region. This will affect
- // young_evacuation_reserve but not old_evacuation_reserve or consumed_by_advance_promotion. So recompute.
- size_t young_evacuation_reserve = MIN2(maximum_young_evacuation_reserve, young_unaffiliated);
-
- // Note that unused old_promo_reserve might not be entirely consumed_by_advance_promotion. Do not transfer this
- // to old_evacuation_reserve because this memory is likely very fragmented, and we do not want to increase the likelihood
- // of old evacuation failure. Leave this memory in the promoted reserve as it may be targeted by opportunistic
- // promotions (found during evacuation of young regions).
- young_generation->set_evacuation_reserve(young_evacuation_reserve);
- old_generation->set_evacuation_reserve(old_evacuation_reserve);
- old_generation->set_promoted_reserve(old_promo_reserve);
-
- // There is no need to expand OLD because all memory used here was set aside at end of previous GC, except in the
- // case of a GLOBAL gc. During choose_collection_set() of GLOBAL, old will be expanded on demand.
-}
-
-// Having chosen the collection set, adjust the budgets for generational mode based on its composition. Note
-// that young_generation->available() now knows about recently discovered immediate garbage.
-void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* const heap,
- ShenandoahCollectionSet* const collection_set, size_t add_regions_to_old) {
- shenandoah_assert_generational();
- // We may find that old_evacuation_reserve and/or loaned_for_young_evacuation are not fully consumed, in which case we may
- // be able to increase regions_available_to_loan
-
- // The role of adjust_evacuation_budgets() is to compute the correct value of regions_available_to_loan and to make
- // effective use of this memory, including the remnant memory within these regions that may result from rounding loan to
- // integral number of regions. Excess memory that is available to be loaned is applied to an allocation supplement,
- // which allows mutators to allocate memory beyond the current capacity of young-gen on the promise that the loan
- // will be repaid as soon as we finish updating references for the recently evacuated collection set.
-
- // We cannot recalculate regions_available_to_loan by simply dividing old_generation->available() by region_size_bytes
- // because the available memory may be distributed between many partially occupied regions that are already holding old-gen
- // objects. Memory in partially occupied regions is not "available" to be loaned. Note that an increase in old-gen
- // available that results from a decrease in memory consumed by old evacuation is not necessarily available to be loaned
- // to young-gen.
-
- const size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes();
- ShenandoahOldGeneration* const old_generation = heap->old_generation();
- ShenandoahYoungGeneration* const young_generation = heap->young_generation();
-
- const size_t old_evacuated = collection_set->get_live_bytes_in_old_regions();
- size_t old_evacuated_committed = (size_t) (ShenandoahOldEvacWaste * double(old_evacuated));
- size_t old_evacuation_reserve = old_generation->get_evacuation_reserve();
-
- if (old_evacuated_committed > old_evacuation_reserve) {
- // This should only happen due to round-off errors when enforcing ShenandoahOldEvacWaste
- assert(old_evacuated_committed <= (33 * old_evacuation_reserve) / 32,
- "Round-off errors should be less than 3.125%%, committed: %zu, reserved: %zu",
- old_evacuated_committed, old_evacuation_reserve);
- old_evacuated_committed = old_evacuation_reserve;
- // Leave old_evac_reserve as previously configured
- } else if (old_evacuated_committed < old_evacuation_reserve) {
- // This happens if the old-gen collection consumes less than full budget.
- log_debug(gc, cset)("Shrinking old evac reserve to match old_evac_commited: " PROPERFMT,
- PROPERFMTARGS(old_evacuated_committed));
- old_evacuation_reserve = old_evacuated_committed;
- old_generation->set_evacuation_reserve(old_evacuation_reserve);
- }
-
- size_t young_advance_promoted = collection_set->get_live_bytes_in_tenurable_regions();
- size_t young_advance_promoted_reserve_used = (size_t) (ShenandoahPromoEvacWaste * double(young_advance_promoted));
-
- size_t young_evacuated = collection_set->get_live_bytes_in_untenurable_regions();
- size_t young_evacuated_reserve_used = (size_t) (ShenandoahEvacWaste * double(young_evacuated));
-
- size_t total_young_available = young_generation->available_with_reserve() - add_regions_to_old * region_size_bytes;;
- assert(young_evacuated_reserve_used <= total_young_available, "Cannot evacuate (%zu) more than is available in young (%zu)",
- young_evacuated_reserve_used, total_young_available);
- young_generation->set_evacuation_reserve(young_evacuated_reserve_used);
-
- // We have not yet rebuilt the free set. Some of the memory that is thought to be avaiable within old may no
- // longer be available if that memory had been free within regions that were selected for the collection set.
- // Make the necessary adjustments to old_available.
- size_t old_available =
- old_generation->available() + add_regions_to_old * region_size_bytes - collection_set->get_old_available_bytes_collected();
-
- // Now that we've established the collection set, we know how much memory is really required by old-gen for evacuation
- // and promotion reserves. Try shrinking OLD now in case that gives us a bit more runway for mutator allocations during
- // evac and update phases.
- size_t old_consumed = old_evacuated_committed + young_advance_promoted_reserve_used;
-
- if (old_available < old_consumed) {
- // This can happen due to round-off errors when adding the results of truncated integer arithmetic.
- // We've already truncated old_evacuated_committed. Truncate young_advance_promoted_reserve_used here.
-
- assert(young_advance_promoted_reserve_used <= (33 * (old_available - old_evacuated_committed)) / 32,
- "Round-off errors should be less than 3.125%%, committed: %zu, reserved: %zu",
- young_advance_promoted_reserve_used, old_available - old_evacuated_committed);
- if (old_available > old_evacuated_committed) {
- young_advance_promoted_reserve_used = old_available - old_evacuated_committed;
- } else {
- young_advance_promoted_reserve_used = 0;
- old_evacuated_committed = old_available;
- }
- // TODO: reserve for full promotion reserve, not just for advance (preselected) promotion
- old_consumed = old_evacuated_committed + young_advance_promoted_reserve_used;
- }
-
- assert(old_available >= old_consumed, "Cannot consume (%zu) more than is available (%zu)",
- old_consumed, old_available);
- size_t excess_old = old_available - old_consumed;
- size_t unaffiliated_old_regions = old_generation->free_unaffiliated_regions() + add_regions_to_old;
- size_t unaffiliated_old = unaffiliated_old_regions * region_size_bytes;
- assert(unaffiliated_old >= old_evacuated_committed, "Do not evacuate (%zu) more than unaffiliated old (%zu)",
- old_evacuated_committed, unaffiliated_old);
-
- // Make sure old_evac_committed is unaffiliated
- if (old_evacuated_committed > 0) {
- if (unaffiliated_old > old_evacuated_committed) {
- size_t giveaway = unaffiliated_old - old_evacuated_committed;
- size_t giveaway_regions = giveaway / region_size_bytes; // round down
- if (giveaway_regions > 0) {
- excess_old = MIN2(excess_old, giveaway_regions * region_size_bytes);
- } else {
- excess_old = 0;
- }
- } else {
- excess_old = 0;
- }
- }
-
- // If we find that OLD has excess regions, give them back to YOUNG now to reduce likelihood we run out of allocation
- // runway during evacuation and update-refs. We may make further adjustments to balance.
- ssize_t add_regions_to_young = 0;
- if (excess_old > unaffiliated_old) {
- // we can give back unaffiliated_old (all of unaffiliated is excess)
- if (unaffiliated_old_regions > 0) {
- add_regions_to_young = unaffiliated_old_regions;
- }
- } else if (unaffiliated_old_regions > 0) {
- // excess_old < unaffiliated old: we can give back MIN(excess_old/region_size_bytes, unaffiliated_old_regions)
- size_t excess_regions = excess_old / region_size_bytes;
- add_regions_to_young = MIN2(excess_regions, unaffiliated_old_regions);
- }
-
- if (add_regions_to_young > 0) {
- assert(excess_old >= add_regions_to_young * region_size_bytes, "Cannot xfer more than excess old");
- excess_old -= add_regions_to_young * region_size_bytes;
- log_debug(gc, ergo)("Before start of evacuation, total_promotion reserve is young_advance_promoted_reserve: %zu "
- "plus excess: old: %zu", young_advance_promoted_reserve_used, excess_old);
- }
-
- // Add in the excess_old memory to hold unanticipated promotions, if any. If there are more unanticipated
- // promotions than fit in reserved memory, they will be deferred until a future GC pass.
- size_t total_promotion_reserve = young_advance_promoted_reserve_used + excess_old;
-
- old_generation->set_promoted_reserve(total_promotion_reserve);
- old_generation->reset_promoted_expended();
-}
-
-typedef struct {
- ShenandoahHeapRegion* _region;
- size_t _live_data;
-} AgedRegionData;
-
-static int compare_by_aged_live(AgedRegionData a, AgedRegionData b) {
- if (a._live_data < b._live_data)
- return -1;
- else if (a._live_data > b._live_data)
- return 1;
- else return 0;
-}
-
-inline void assert_no_in_place_promotions() {
-#ifdef ASSERT
- class ShenandoahNoInPlacePromotions : public ShenandoahHeapRegionClosure {
- public:
- void heap_region_do(ShenandoahHeapRegion *r) override {
- assert(r->get_top_before_promote() == nullptr,
- "Region %zu should not be ready for in-place promotion", r->index());
- }
- } cl;
- ShenandoahHeap::heap()->heap_region_iterate(&cl);
-#endif
-}
-
-// Preselect for inclusion into the collection set all regions whose age is at or above tenure age and for which the
-// garbage percentage exceeds a dynamically adjusted threshold (known as the old-garbage threshold percentage). We
-// identify these regions by setting the appropriate entry of the collection set's preselected regions array to true.
-// All entries are initialized to false before calling this function.
-//
-// During the subsequent selection of the collection set, we give priority to these promotion set candidates.
-// Without this prioritization, we found that the aged regions tend to be ignored because they typically have
-// much less garbage and much more live data than the recently allocated "eden" regions. When aged regions are
-// repeatedly excluded from the collection set, the amount of live memory within the young generation tends to
-// accumulate and this has the undesirable side effect of causing young-generation collections to require much more
-// CPU and wall-clock time.
-//
-// A second benefit of treating aged regions differently than other regions during collection set selection is
-// that this allows us to more accurately budget memory to hold the results of evacuation. Memory for evacuation
-// of aged regions must be reserved in the old generation. Memory for evacuation of all other regions must be
-// reserved in the young generation.
-size_t ShenandoahGeneration::select_aged_regions(const size_t old_promotion_reserve) {
-
- // There should be no regions configured for subsequent in-place-promotions carried over from the previous cycle.
- assert_no_in_place_promotions();
-
- auto const heap = ShenandoahGenerationalHeap::heap();
- ShenandoahFreeSet* free_set = heap->free_set();
- bool* const candidate_regions_for_promotion_by_copy = heap->collection_set()->preselected_regions();
- ShenandoahMarkingContext* const ctx = heap->marking_context();
-
- const size_t old_garbage_threshold =
- (ShenandoahHeapRegion::region_size_bytes() * heap->old_generation()->heuristics()->get_old_garbage_threshold()) / 100;
-
- const size_t pip_used_threshold = (ShenandoahHeapRegion::region_size_bytes() * ShenandoahGenerationalMinPIPUsage) / 100;
-
- size_t promo_potential = 0;
- size_t candidates = 0;
-
- // Tracks the padding of space above top in regions eligible for promotion in place
- size_t promote_in_place_pad = 0;
-
- // Sort the promotion-eligible regions in order of increasing live-data-bytes so that we can first reclaim regions that require
- // less evacuation effort. This prioritizes garbage first, expanding the allocation pool early before we reclaim regions that
- // have more live data.
- const idx_t num_regions = heap->num_regions();
-
- ResourceMark rm;
- AgedRegionData* sorted_regions = NEW_RESOURCE_ARRAY(AgedRegionData, num_regions);
-
- ShenandoahFreeSet* freeset = heap->free_set();
-
- // Any region that is to be promoted in place needs to be retired from its Collector or Mutator partition.
- idx_t pip_low_collector_idx = freeset->max_regions();
- idx_t pip_high_collector_idx = -1;
- idx_t pip_low_mutator_idx = freeset->max_regions();
- idx_t pip_high_mutator_idx = -1;
- size_t collector_regions_to_pip = 0;
- size_t mutator_regions_to_pip = 0;
-
- size_t pip_mutator_regions = 0;
- size_t pip_collector_regions = 0;
- size_t pip_mutator_bytes = 0;
- size_t pip_collector_bytes = 0;
-
- for (idx_t i = 0; i < num_regions; i++) {
- ShenandoahHeapRegion* const r = heap->get_region(i);
- if (r->is_empty() || !r->has_live() || !r->is_young() || !r->is_regular()) {
- // skip over regions that aren't regular young with some live data
- continue;
- }
- if (heap->is_tenurable(r)) {
- if ((r->garbage() < old_garbage_threshold) && (r->used() > pip_used_threshold)) {
- // We prefer to promote this region in place because it has a small amount of garbage and a large usage.
- HeapWord* tams = ctx->top_at_mark_start(r);
- HeapWord* original_top = r->top();
- if (!heap->is_concurrent_old_mark_in_progress() && tams == original_top) {
- // No allocations from this region have been made during concurrent mark. It meets all the criteria
- // for in-place-promotion. Though we only need the value of top when we fill the end of the region,
- // we use this field to indicate that this region should be promoted in place during the evacuation
- // phase.
- r->save_top_before_promote();
- size_t remnant_bytes = r->free();
- size_t remnant_words = remnant_bytes / HeapWordSize;
- assert(ShenandoahHeap::min_fill_size() <= PLAB::min_size(), "Implementation makes invalid assumptions");
- if (remnant_words >= ShenandoahHeap::min_fill_size()) {
- ShenandoahHeap::fill_with_object(original_top, remnant_words);
- // Fill the remnant memory within this region to assure no allocations prior to promote in place. Otherwise,
- // newly allocated objects will not be parsable when promote in place tries to register them. Furthermore, any
- // new allocations would not necessarily be eligible for promotion. This addresses both issues.
- r->set_top(r->end());
- // The region r is either in the Mutator or Collector partition if remnant_words > heap()->plab_min_size.
- // Otherwise, the region is in the NotFree partition.
- ShenandoahFreeSetPartitionId p = free_set->membership(i);
- if (p == ShenandoahFreeSetPartitionId::Mutator) {
- mutator_regions_to_pip++;
- if (i < pip_low_mutator_idx) {
- pip_low_mutator_idx = i;
- }
- if (i > pip_high_mutator_idx) {
- pip_high_mutator_idx = i;
- }
- pip_mutator_regions++;
- pip_mutator_bytes += remnant_bytes;
- } else if (p == ShenandoahFreeSetPartitionId::Collector) {
- collector_regions_to_pip++;
- if (i < pip_low_collector_idx) {
- pip_low_collector_idx = i;
- }
- if (i > pip_high_collector_idx) {
- pip_high_collector_idx = i;
- }
- pip_collector_regions++;
- pip_collector_bytes += remnant_bytes;
- } else {
- assert((p == ShenandoahFreeSetPartitionId::NotFree) && (remnant_words < heap->plab_min_size()),
- "Should be NotFree if not in Collector or Mutator partitions");
- // In this case, the memory is already counted as used and the region has already been retired. There is
- // no need for further adjustments to used. Further, the remnant memory for this region will not be
- // unallocated or made available to OldCollector after pip.
- remnant_bytes = 0;
- }
- promote_in_place_pad += remnant_bytes;
- free_set->prepare_to_promote_in_place(i, remnant_bytes);
- } else {
- // Since the remnant is so small that this region has already been retired, we don't have to worry about any
- // accidental allocations occurring within this region before the region is promoted in place.
-
- // This region was already not in the Collector or Mutator set, so no need to remove it.
- assert(free_set->membership(i) == ShenandoahFreeSetPartitionId::NotFree, "sanity");
- }
- }
- // Else, we do not promote this region (either in place or by copy) because it has received new allocations.
-
- // During evacuation, we exclude from promotion regions for which age > tenure threshold, garbage < garbage-threshold,
- // used > pip_used_threshold, and get_top_before_promote() != tams
- } else {
- // Record this promotion-eligible candidate region. After sorting and selecting the best candidates below,
- // we may still decide to exclude this promotion-eligible region from the current collection set. If this
- // happens, we will consider this region as part of the anticipated promotion potential for the next GC
- // pass; see further below.
- sorted_regions[candidates]._region = r;
- sorted_regions[candidates++]._live_data = r->get_live_data_bytes();
- }
- } else {
- // We only evacuate & promote objects from regular regions whose garbage() is above old-garbage-threshold.
- // Objects in tenure-worthy regions with less garbage are promoted in place. These take a different path to
- // old-gen. Regions excluded from promotion because their garbage content is too low (causing us to anticipate that
- // the region would be promoted in place) may be eligible for evacuation promotion by the time promotion takes
- // place during a subsequent GC pass because more garbage is found within the region between now and then. This
- // should not happen if we are properly adapting the tenure age. The theory behind adaptive tenuring threshold
- // is to choose the youngest age that demonstrates no "significant" further loss of population since the previous
- // age. If not this, we expect the tenure age to demonstrate linear population decay for at least two population
- // samples, whereas we expect to observe exponential population decay for ages younger than the tenure age.
- //
- // In the case that certain regions which were anticipated to be promoted in place need to be promoted by
- // evacuation, it may be the case that there is not sufficient reserve within old-gen to hold evacuation of
- // these regions. The likely outcome is that these regions will not be selected for evacuation or promotion
- // in the current cycle and we will anticipate that they will be promoted in the next cycle. This will cause
- // us to reserve more old-gen memory so that these objects can be promoted in the subsequent cycle.
- if (heap->is_aging_cycle() && heap->age_census()->is_tenurable(r->age() + 1)) {
- if (r->garbage() >= old_garbage_threshold) {
- promo_potential += r->get_live_data_bytes();
- }
- }
- }
- // Note that we keep going even if one region is excluded from selection.
- // Subsequent regions may be selected if they have smaller live data.
- }
-
- if (pip_mutator_regions + pip_collector_regions > 0) {
- freeset->account_for_pip_regions(pip_mutator_regions, pip_mutator_bytes, pip_collector_regions, pip_collector_bytes);
- }
-
- // Retire any regions that have been selected for promote in place
- if (collector_regions_to_pip > 0) {
- freeset->shrink_interval_if_range_modifies_either_boundary(ShenandoahFreeSetPartitionId::Collector,
- pip_low_collector_idx, pip_high_collector_idx,
- collector_regions_to_pip);
- }
- if (mutator_regions_to_pip > 0) {
- freeset->shrink_interval_if_range_modifies_either_boundary(ShenandoahFreeSetPartitionId::Mutator,
- pip_low_mutator_idx, pip_high_mutator_idx,
- mutator_regions_to_pip);
- }
-
- // Sort in increasing order according to live data bytes. Note that candidates represents the number of regions
- // that qualify to be promoted by evacuation.
- size_t old_consumed = 0;
- if (candidates > 0) {
- size_t selected_regions = 0;
- size_t selected_live = 0;
- QuickSort::sort(sorted_regions, candidates, compare_by_aged_live);
- for (size_t i = 0; i < candidates; i++) {
- ShenandoahHeapRegion* const region = sorted_regions[i]._region;
- const size_t region_live_data = sorted_regions[i]._live_data;
- const size_t promotion_need = (size_t) (region_live_data * ShenandoahPromoEvacWaste);
- if (old_consumed + promotion_need <= old_promotion_reserve) {
- old_consumed += promotion_need;
- candidate_regions_for_promotion_by_copy[region->index()] = true;
- selected_regions++;
- selected_live += region_live_data;
- } else {
- // We rejected this promotable region from the collection set because we had no room to hold its copy.
- // Add this region to promo potential for next GC.
- promo_potential += region_live_data;
- assert(!candidate_regions_for_promotion_by_copy[region->index()], "Shouldn't be selected");
- }
- // We keep going even if one region is excluded from selection because we need to accumulate all eligible
- // regions that are not preselected into promo_potential
- }
- log_debug(gc, ergo)("Preselected %zu regions containing " PROPERFMT " live data,"
- " consuming: " PROPERFMT " of budgeted: " PROPERFMT,
- selected_regions, PROPERFMTARGS(selected_live), PROPERFMTARGS(old_consumed), PROPERFMTARGS(old_promotion_reserve));
- }
-
- log_info(gc, ergo)("Promotion potential of aged regions with sufficient garbage: " PROPERFMT, PROPERFMTARGS(promo_potential));
-
- heap->old_generation()->set_pad_for_promote_in_place(promote_in_place_pad);
- heap->old_generation()->set_promotion_potential(promo_potential);
- return old_consumed;
-}
-
void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) {
ShenandoahHeap* heap = ShenandoahHeap::heap();
ShenandoahCollectionSet* collection_set = heap->collection_set();
@@ -798,34 +297,7 @@ void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) {
collection_set->clear();
ShenandoahHeapLocker locker(heap->lock());
- if (is_generational) {
- // Seed the collection set with resource area-allocated
- // preselected regions, which are removed when we exit this scope.
- ShenandoahCollectionSetPreselector preselector(collection_set, heap->num_regions());
-
- // Find the amount that will be promoted, regions that will be promoted in
- // place, and preselected older regions that will be promoted by evacuation.
- compute_evacuation_budgets(heap);
-
- // Choose the collection set, including the regions preselected above for promotion into the old generation.
- size_t add_regions_to_old = _heuristics->choose_collection_set(collection_set);
- // Even if collection_set->is_empty(), we want to adjust budgets, making reserves available to mutator.
- adjust_evacuation_budgets(heap, collection_set, add_regions_to_old);
- if (is_global()) {
- // We have just chosen a collection set for a global cycle. The mark bitmap covering old regions is complete, so
- // the remembered set scan can use that to avoid walking into garbage. When the next old mark begins, we will
- // use the mark bitmap to make the old regions parsable by coalescing and filling any unmarked objects. Thus,
- // we prepare for old collections by remembering which regions are old at this time. Note that any objects
- // promoted into old regions will be above TAMS, and so will be considered marked. However, free regions that
- // become old after this point will not be covered correctly by the mark bitmap, so we must be careful not to
- // coalesce those regions. Only the old regions which are not part of the collection set at this point are
- // eligible for coalescing. As implemented now, this has the side effect of possibly initiating mixed-evacuations
- // after a global cycle for old regions that were not included in this collection set.
- heap->old_generation()->prepare_for_mixed_collections_after_global_gc();
- }
- } else {
- _heuristics->choose_collection_set(collection_set);
- }
+ _heuristics->choose_collection_set(collection_set);
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp
index d49e3bed5f8..946f2b91520 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp
@@ -60,31 +60,6 @@ protected:
ShenandoahHeuristics* _heuristics;
private:
- // Compute evacuation budgets prior to choosing collection set.
- void compute_evacuation_budgets(ShenandoahHeap* heap);
-
- // Adjust evacuation budgets after choosing collection set. The argument regions_to_xfer represents regions to be
- // transfered to old based on decisions made in top_off_collection_set()
- void adjust_evacuation_budgets(ShenandoahHeap* heap,
- ShenandoahCollectionSet* collection_set, size_t regions_to_xfer);
-
- // Preselect for possible inclusion into the collection set exactly the most
- // garbage-dense regions, including those that satisfy criteria 1 & 2 below,
- // and whose live bytes will fit within old_available budget:
- // Criterion 1. region age >= tenuring threshold
- // Criterion 2. region garbage percentage > old garbage threshold
- //
- // Identifies regions eligible for promotion in place,
- // being those of at least tenuring_threshold age that have lower garbage
- // density.
- //
- // Updates promotion_potential and pad_for_promote_in_place fields
- // of the heap. Returns bytes of live object memory in the preselected
- // regions, which are marked in the preselected_regions() indicator
- // array of the heap's collection set, which should be initialized
- // to false.
- size_t select_aged_regions(size_t old_promotion_reserve);
-
// Return available assuming that we can allocate no more than capacity bytes within this generation.
size_t available(size_t capacity) const;
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
index ef99bd98c93..ccfc1c036c2 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
@@ -2719,18 +2719,6 @@ bool ShenandoahRegionIterator::has_next() const {
return _index < _heap->num_regions();
}
-char ShenandoahHeap::gc_state() const {
- return _gc_state.raw_value();
-}
-
-bool ShenandoahHeap::is_gc_state(GCState state) const {
- // If the global gc state has been changed, but hasn't yet been propagated to all threads, then
- // the global gc state is the correct value. Once the gc state has been synchronized with all threads,
- // _gc_state_changed will be toggled to false and we need to use the thread local state.
- return _gc_state_changed ? _gc_state.is_set(state) : ShenandoahThreadLocalData::is_gc_state(state);
-}
-
-
ShenandoahLiveData* ShenandoahHeap::get_liveness_cache(uint worker_id) {
#ifdef ASSERT
assert(_liveness_cache != nullptr, "sanity");
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp
index 174001170f4..9240091070b 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp
@@ -353,7 +353,7 @@ private:
public:
// This returns the raw value of the singular, global gc state.
- char gc_state() const;
+ inline char gc_state() const;
// Compares the given state against either the global gc state, or the thread local state.
// The global gc state may change on a safepoint and is the correct value to use until
@@ -361,7 +361,7 @@ public:
// compare against the thread local state). The thread local gc state may also be changed
// by a handshake operation, in which case, this function continues using the updated thread
// local value.
- bool is_gc_state(GCState state) const;
+ inline bool is_gc_state(GCState state) const;
// This copies the global gc state into a thread local variable for all threads.
// The thread local gc state is primarily intended to support quick access at barriers.
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp
index 34c279a1495..e35f116b843 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp
@@ -452,6 +452,17 @@ inline bool ShenandoahHeap::in_collection_set_loc(void* p) const {
return collection_set()->is_in_loc(p);
}
+inline char ShenandoahHeap::gc_state() const {
+ return _gc_state.raw_value();
+}
+
+inline bool ShenandoahHeap::is_gc_state(GCState state) const {
+ // If the global gc state has been changed, but hasn't yet been propagated to all threads, then
+ // the global gc state is the correct value. Once the gc state has been synchronized with all threads,
+ // _gc_state_changed will be toggled to false and we need to use the thread local state.
+ return _gc_state_changed ? _gc_state.is_set(state) : ShenandoahThreadLocalData::is_gc_state(state);
+}
+
inline bool ShenandoahHeap::is_idle() const {
return _gc_state_changed ? _gc_state.is_clear() : ShenandoahThreadLocalData::gc_state(Thread::current()) == 0;
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp
index 6bb8382de0a..b0c13df6c4f 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp
@@ -43,7 +43,6 @@
#include "memory/resourceArea.hpp"
#include "memory/universe.hpp"
#include "oops/oop.inline.hpp"
-#include "runtime/atomicAccess.hpp"
#include "runtime/globals_extension.hpp"
#include "runtime/java.hpp"
#include "runtime/mutexLocker.hpp"
@@ -384,7 +383,7 @@ size_t ShenandoahHeapRegion::get_plab_allocs() const {
void ShenandoahHeapRegion::set_live_data(size_t s) {
assert(Thread::current()->is_VM_thread(), "by VM thread");
- _live_data = (s >> LogHeapWordSize);
+ _live_data.store_relaxed(s >> LogHeapWordSize);
}
void ShenandoahHeapRegion::print_on(outputStream* st) const {
@@ -435,7 +434,7 @@ void ShenandoahHeapRegion::print_on(outputStream* st) const {
st->print("|TAMS " SHR_PTR_FORMAT,
p2i(ShenandoahHeap::heap()->marking_context()->top_at_mark_start(const_cast(this))));
st->print("|UWM " SHR_PTR_FORMAT,
- p2i(_update_watermark));
+ p2i(_update_watermark.load_relaxed()));
st->print("|U %5zu%1s", byte_size_in_proper_unit(used()), proper_unit_for_byte_size(used()));
st->print("|T %5zu%1s", byte_size_in_proper_unit(get_tlab_allocs()), proper_unit_for_byte_size(get_tlab_allocs()));
st->print("|G %5zu%1s", byte_size_in_proper_unit(get_gclab_allocs()), proper_unit_for_byte_size(get_gclab_allocs()));
@@ -839,20 +838,20 @@ void ShenandoahHeapRegion::set_state(RegionState to) {
evt.set_to(to);
evt.commit();
}
- AtomicAccess::store(&_state, to);
+ _state.store_relaxed(to);
}
void ShenandoahHeapRegion::record_pin() {
- AtomicAccess::add(&_critical_pins, (size_t)1);
+ _critical_pins.add_then_fetch((size_t)1);
}
void ShenandoahHeapRegion::record_unpin() {
assert(pin_count() > 0, "Region %zu should have non-zero pins", index());
- AtomicAccess::sub(&_critical_pins, (size_t)1);
+ _critical_pins.sub_then_fetch((size_t)1);
}
size_t ShenandoahHeapRegion::pin_count() const {
- return AtomicAccess::load(&_critical_pins);
+ return _critical_pins.load_relaxed();
}
void ShenandoahHeapRegion::set_affiliation(ShenandoahAffiliation new_affiliation) {
@@ -864,7 +863,7 @@ void ShenandoahHeapRegion::set_affiliation(ShenandoahAffiliation new_affiliation
log_debug(gc)("Setting affiliation of Region %zu from %s to %s, top: " PTR_FORMAT ", TAMS: " PTR_FORMAT
", watermark: " PTR_FORMAT ", top_bitmap: " PTR_FORMAT,
index(), shenandoah_affiliation_name(region_affiliation), shenandoah_affiliation_name(new_affiliation),
- p2i(top()), p2i(ctx->top_at_mark_start(this)), p2i(_update_watermark), p2i(ctx->top_bitmap(this)));
+ p2i(top()), p2i(ctx->top_at_mark_start(this)), p2i(_update_watermark.load_relaxed()), p2i(ctx->top_bitmap(this)));
}
#ifdef ASSERT
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp
index 9da2816e2c9..3a0ac042f57 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp
@@ -34,6 +34,7 @@
#include "gc/shenandoah/shenandoahAsserts.hpp"
#include "gc/shenandoah/shenandoahHeap.hpp"
#include "gc/shenandoah/shenandoahPadding.hpp"
+#include "runtime/atomic.hpp"
#include "utilities/sizes.hpp"
class VMStructs;
@@ -217,7 +218,7 @@ public:
bool is_alloc_allowed() const { auto cur_state = state(); return is_empty_state(cur_state) || cur_state == _regular || cur_state == _pinned; }
bool is_stw_move_allowed() const { auto cur_state = state(); return cur_state == _regular || cur_state == _cset || (ShenandoahHumongousMoves && cur_state == _humongous_start); }
- RegionState state() const { return AtomicAccess::load(&_state); }
+ RegionState state() const { return _state.load_relaxed(); }
int state_ordinal() const { return region_state_to_ordinal(state()); }
void record_pin();
@@ -247,7 +248,7 @@ private:
HeapWord* _top_before_promoted;
// Seldom updated fields
- volatile RegionState _state;
+ Atomic _state;
HeapWord* _coalesce_and_fill_boundary; // for old regions not selected as collection set candidates.
// Frequently updated fields
@@ -257,12 +258,12 @@ private:
size_t _gclab_allocs;
size_t _plab_allocs;
- volatile size_t _live_data;
- volatile size_t _critical_pins;
+ Atomic _live_data;
+ Atomic _critical_pins;
size_t _mixed_candidate_garbage_words;
- HeapWord* volatile _update_watermark;
+ Atomic _update_watermark;
uint _age;
bool _promoted_in_place;
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp
index be982433885..39b7c732703 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp
@@ -32,7 +32,6 @@
#include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahOldGeneration.hpp"
-#include "runtime/atomicAccess.hpp"
HeapWord* ShenandoahHeapRegion::allocate_aligned(size_t size, ShenandoahAllocRequest &req, size_t alignment_in_bytes) {
shenandoah_assert_heaplocked_or_safepoint();
@@ -147,16 +146,16 @@ inline void ShenandoahHeapRegion::increase_live_data_gc_words(size_t s) {
}
inline void ShenandoahHeapRegion::internal_increase_live_data(size_t s) {
- AtomicAccess::add(&_live_data, s, memory_order_relaxed);
+ _live_data.add_then_fetch(s, memory_order_relaxed);
}
inline void ShenandoahHeapRegion::clear_live_data() {
- AtomicAccess::store(&_live_data, (size_t)0);
+ _live_data.store_relaxed((size_t)0);
_promoted_in_place = false;
}
inline size_t ShenandoahHeapRegion::get_live_data_words() const {
- return AtomicAccess::load(&_live_data);
+ return _live_data.load_relaxed();
}
inline size_t ShenandoahHeapRegion::get_live_data_bytes() const {
@@ -205,21 +204,21 @@ inline size_t ShenandoahHeapRegion::garbage_before_padded_for_promote() const {
}
inline HeapWord* ShenandoahHeapRegion::get_update_watermark() const {
- HeapWord* watermark = AtomicAccess::load_acquire(&_update_watermark);
+ HeapWord* watermark = _update_watermark.load_acquire();
assert(bottom() <= watermark && watermark <= top(), "within bounds");
return watermark;
}
inline void ShenandoahHeapRegion::set_update_watermark(HeapWord* w) {
assert(bottom() <= w && w <= top(), "within bounds");
- AtomicAccess::release_store(&_update_watermark, w);
+ _update_watermark.release_store(w);
}
// Fast version that avoids synchronization, only to be used at safepoints.
inline void ShenandoahHeapRegion::set_update_watermark_at_safepoint(HeapWord* w) {
assert(bottom() <= w && w <= top(), "within bounds");
assert(SafepointSynchronize::is_at_safepoint(), "Should be at Shenandoah safepoint");
- _update_watermark = w;
+ _update_watermark.store_relaxed(w);
}
inline ShenandoahAffiliation ShenandoahHeapRegion::affiliation() const {
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp
index 0cc6d4c6ed4..225339a3219 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp
@@ -42,7 +42,7 @@
#include "memory/iterator.inline.hpp"
#include "memory/resourceArea.hpp"
#include "oops/compressedOops.inline.hpp"
-#include "runtime/atomicAccess.hpp"
+#include "runtime/atomic.hpp"
#include "runtime/orderAccess.hpp"
#include "runtime/threads.hpp"
#include "utilities/align.hpp"
@@ -110,15 +110,15 @@ private:
void do_oop_work(T* p) {
T o = RawAccess<>::oop_load(p);
if (!CompressedOops::is_null(o)) {
- oop obj = CompressedOops::decode_not_null(o);
+ // Basic verification should happen before we touch anything else.
+ // For performance reasons, only fully verify non-marked field values.
+ // We are here when the host object for *p is already marked.
+ oop obj = CompressedOops::decode_raw_not_null(o);
+ verify_oop_at_basic(p, obj);
+
if (is_instance_ref_klass(ShenandoahForwarding::klass(obj))) {
obj = ShenandoahForwarding::get_forwardee(obj);
}
- // Single threaded verification can use faster non-atomic stack and bitmap
- // methods.
- //
- // For performance reasons, only fully verify non-marked field values.
- // We are here when the host object for *p is already marked.
if (in_generation(obj) && _map->par_mark(obj)) {
verify_oop_at(p, obj);
_stack->push(ShenandoahVerifierTask(obj));
@@ -131,7 +131,7 @@ private:
return _generation->contains(region);
}
- void verify_oop(oop obj) {
+ void verify_oop(oop obj, bool basic = false) {
// Perform consistency checks with gradually decreasing safety level. This guarantees
// that failure report would not try to touch something that was not yet verified to be
// safe to process.
@@ -174,17 +174,21 @@ private:
}
}
+ check(ShenandoahAsserts::_safe_unknown, obj, obj_reg->is_active(),
+ "Object should be in active region");
+
// ------------ obj is safe at this point --------------
- check(ShenandoahAsserts::_safe_oop, obj, obj_reg->is_active(),
- "Object should be in active region");
+ if (basic) {
+ return;
+ }
switch (_options._verify_liveness) {
case ShenandoahVerifier::_verify_liveness_disable:
// skip
break;
case ShenandoahVerifier::_verify_liveness_complete:
- AtomicAccess::add(&_ld[obj_reg->index()], (uint) ShenandoahForwarding::size(obj), memory_order_relaxed);
+ _ld[obj_reg->index()].add_then_fetch((uint) ShenandoahForwarding::size(obj), memory_order_relaxed);
// fallthrough for fast failure for un-live regions:
case ShenandoahVerifier::_verify_liveness_conservative:
check(ShenandoahAsserts::_safe_oop, obj, obj_reg->has_live() ||
@@ -331,6 +335,18 @@ public:
_interior_loc = nullptr;
}
+ /**
+ * Verify object with known interior reference, with only basic verification.
+ * @param p interior reference where the object is referenced from; can be off-heap
+ * @param obj verified object
+ */
+ template
+ void verify_oop_at_basic(T* p, oop obj) {
+ _interior_loc = p;
+ verify_oop(obj, /* basic = */ true);
+ _interior_loc = nullptr;
+ }
+
/**
* Verify object without known interior reference.
* Useful when picking up the object at known offset in heap,
@@ -593,7 +609,7 @@ private:
ShenandoahHeap* _heap;
ShenandoahLivenessData* _ld;
MarkBitMap* _bitmap;
- volatile size_t _processed;
+ Atomic _processed;
ShenandoahGeneration* _generation;
public:
@@ -612,7 +628,7 @@ public:
_generation(generation) {};
size_t processed() const {
- return _processed;
+ return _processed.load_relaxed();
}
void work(uint worker_id) override {
@@ -648,7 +664,7 @@ public:
}
}
- AtomicAccess::add(&_processed, processed, memory_order_relaxed);
+ _processed.add_then_fetch(processed, memory_order_relaxed);
}
};
@@ -669,8 +685,8 @@ class ShenandoahVerifierMarkedRegionTask : public WorkerTask {
ShenandoahHeap *_heap;
MarkBitMap* _bitmap;
ShenandoahLivenessData* _ld;
- volatile size_t _claimed;
- volatile size_t _processed;
+ Atomic _claimed;
+ Atomic _processed;
ShenandoahGeneration* _generation;
public:
@@ -690,7 +706,7 @@ public:
_generation(generation) {}
size_t processed() {
- return AtomicAccess::load(&_processed);
+ return _processed.load_relaxed();
}
void work(uint worker_id) override {
@@ -705,7 +721,7 @@ public:
_options);
while (true) {
- size_t v = AtomicAccess::fetch_then_add(&_claimed, 1u, memory_order_relaxed);
+ size_t v = _claimed.fetch_then_add(1u, memory_order_relaxed);
if (v < _heap->num_regions()) {
ShenandoahHeapRegion* r = _heap->get_region(v);
if (!in_generation(r)) {
@@ -733,7 +749,7 @@ public:
if (_generation->complete_marking_context()->is_marked(cast_to_oop(obj))) {
verify_and_follow(obj, stack, cl, &processed);
}
- AtomicAccess::add(&_processed, processed, memory_order_relaxed);
+ _processed.add_then_fetch(processed, memory_order_relaxed);
}
virtual void work_regular(ShenandoahHeapRegion *r, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl) {
@@ -766,7 +782,7 @@ public:
}
}
- AtomicAccess::add(&_processed, processed, memory_order_relaxed);
+ _processed.add_then_fetch(processed, memory_order_relaxed);
}
void verify_and_follow(HeapWord *addr, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl, size_t *processed) {
@@ -1035,12 +1051,12 @@ void ShenandoahVerifier::verify_at_safepoint(ShenandoahGeneration* generation,
if (r->is_humongous()) {
// For humongous objects, test if start region is marked live, and if so,
// all humongous regions in that chain have live data equal to their "used".
- juint start_live = AtomicAccess::load(&ld[r->humongous_start_region()->index()]);
+ juint start_live = ld[r->humongous_start_region()->index()].load_relaxed();
if (start_live > 0) {
verf_live = (juint)(r->used() / HeapWordSize);
}
} else {
- verf_live = AtomicAccess::load(&ld[r->index()]);
+ verf_live = ld[r->index()].load_relaxed();
}
size_t reg_live = r->get_live_data_words();
@@ -1232,7 +1248,9 @@ private:
void do_oop_work(T* p) {
T o = RawAccess<>::oop_load(p);
if (!CompressedOops::is_null(o)) {
- oop obj = CompressedOops::decode_not_null(o);
+ oop obj = CompressedOops::decode_raw_not_null(o);
+ ShenandoahAsserts::assert_correct(p, obj, __FILE__, __LINE__);
+
oop fwd = ShenandoahForwarding::get_forwardee_raw_unchecked(obj);
if (obj != fwd) {
ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, nullptr,
@@ -1252,7 +1270,9 @@ private:
void do_oop_work(T* p) {
T o = RawAccess<>::oop_load(p);
if (!CompressedOops::is_null(o)) {
- oop obj = CompressedOops::decode_not_null(o);
+ oop obj = CompressedOops::decode_raw_not_null(o);
+ ShenandoahAsserts::assert_correct(p, obj, __FILE__, __LINE__);
+
ShenandoahHeap* heap = ShenandoahHeap::heap();
if (!heap->marking_context()->is_marked_or_old(obj)) {
@@ -1306,7 +1326,9 @@ public:
inline void work(T* p) {
T o = RawAccess<>::oop_load(p);
if (!CompressedOops::is_null(o)) {
- oop obj = CompressedOops::decode_not_null(o);
+ oop obj = CompressedOops::decode_raw_not_null(o);
+ ShenandoahAsserts::assert_correct(p, obj, __FILE__, __LINE__);
+
if (_heap->is_in_young(obj) && !_scanner->is_card_dirty((HeapWord*) p)) {
ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, nullptr,
_message, "clean card, it should be dirty.", __FILE__, __LINE__);
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp
index f66d7bbec77..7e683cf7af8 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp
@@ -52,7 +52,7 @@ private:
};
typedef Stack ShenandoahVerifierStack;
-typedef volatile juint ShenandoahLivenessData;
+typedef Atomic ShenandoahLivenessData;
class ShenandoahVerifier : public CHeapObj {
private:
diff --git a/src/hotspot/share/gc/shenandoah/vmStructs_shenandoah.hpp b/src/hotspot/share/gc/shenandoah/vmStructs_shenandoah.hpp
index 3968575d089..e5e2b14a3a1 100644
--- a/src/hotspot/share/gc/shenandoah/vmStructs_shenandoah.hpp
+++ b/src/hotspot/share/gc/shenandoah/vmStructs_shenandoah.hpp
@@ -29,21 +29,22 @@
#include "gc/shenandoah/shenandoahHeap.hpp"
#include "gc/shenandoah/shenandoahHeapRegion.hpp"
#include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
+#include "runtime/atomic.hpp"
-#define VM_STRUCTS_SHENANDOAH(nonstatic_field, volatile_nonstatic_field, static_field) \
- nonstatic_field(ShenandoahHeap, _num_regions, size_t) \
- nonstatic_field(ShenandoahHeap, _regions, ShenandoahHeapRegion**) \
- nonstatic_field(ShenandoahHeap, _log_min_obj_alignment_in_bytes, int) \
- nonstatic_field(ShenandoahHeap, _free_set, ShenandoahFreeSet*) \
- volatile_nonstatic_field(ShenandoahHeap, _committed, size_t) \
- static_field(ShenandoahHeapRegion, RegionSizeBytes, size_t) \
- static_field(ShenandoahHeapRegion, RegionSizeBytesShift, size_t) \
- volatile_nonstatic_field(ShenandoahHeapRegion, _state, ShenandoahHeapRegion::RegionState) \
- nonstatic_field(ShenandoahHeapRegion, _index, size_t const) \
- nonstatic_field(ShenandoahHeapRegion, _bottom, HeapWord* const) \
- nonstatic_field(ShenandoahHeapRegion, _top, HeapWord*) \
- nonstatic_field(ShenandoahHeapRegion, _end, HeapWord* const) \
- nonstatic_field(ShenandoahFreeSet, _total_global_used, size_t) \
+#define VM_STRUCTS_SHENANDOAH(nonstatic_field, volatile_nonstatic_field, static_field) \
+ nonstatic_field(ShenandoahHeap, _num_regions, size_t) \
+ nonstatic_field(ShenandoahHeap, _regions, ShenandoahHeapRegion**) \
+ nonstatic_field(ShenandoahHeap, _log_min_obj_alignment_in_bytes, int) \
+ nonstatic_field(ShenandoahHeap, _free_set, ShenandoahFreeSet*) \
+ volatile_nonstatic_field(ShenandoahHeap, _committed, size_t) \
+ static_field(ShenandoahHeapRegion, RegionSizeBytes, size_t) \
+ static_field(ShenandoahHeapRegion, RegionSizeBytesShift, size_t) \
+ nonstatic_field(ShenandoahHeapRegion, _state, Atomic) \
+ nonstatic_field(ShenandoahHeapRegion, _index, size_t const) \
+ nonstatic_field(ShenandoahHeapRegion, _bottom, HeapWord* const) \
+ nonstatic_field(ShenandoahHeapRegion, _top, HeapWord*) \
+ nonstatic_field(ShenandoahHeapRegion, _end, HeapWord* const) \
+ nonstatic_field(ShenandoahFreeSet, _total_global_used, size_t) \
#define VM_INT_CONSTANTS_SHENANDOAH(declare_constant, declare_constant_with_value) \
declare_constant(ShenandoahHeapRegion::_empty_uncommitted) \
@@ -65,7 +66,7 @@
declare_toplevel_type(ShenandoahHeapRegion) \
declare_toplevel_type(ShenandoahHeap*) \
declare_toplevel_type(ShenandoahHeapRegion*) \
- declare_toplevel_type(ShenandoahHeapRegion::RegionState) \
+ declare_toplevel_type(Atomic) \
declare_toplevel_type(ShenandoahFreeSet) \
declare_toplevel_type(ShenandoahFreeSet*) \
diff --git a/src/hotspot/share/gc/z/zUtils.inline.hpp b/src/hotspot/share/gc/z/zUtils.inline.hpp
index b83f42d18e6..a221a925498 100644
--- a/src/hotspot/share/gc/z/zUtils.inline.hpp
+++ b/src/hotspot/share/gc/z/zUtils.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -37,6 +37,9 @@
inline uintptr_t ZUtils::alloc_aligned_unfreeable(size_t alignment, size_t size) {
const size_t padded_size = size + (alignment - 1);
void* const addr = os::malloc(padded_size, mtGC);
+ if (addr == nullptr) {
+ vm_exit_out_of_memory(padded_size, OOM_MALLOC_ERROR, "ZGC alloc_aligned_unfreeable malloc failed");
+ }
void* const aligned_addr = align_up(addr, alignment);
memset(aligned_addr, 0, size);
diff --git a/src/hotspot/share/include/jvm.h b/src/hotspot/share/include/jvm.h
index 7236679d8f3..f99e8f162c6 100644
--- a/src/hotspot/share/include/jvm.h
+++ b/src/hotspot/share/include/jvm.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -301,7 +301,7 @@ JVM_Interrupt(JNIEnv *env, jobject thread);
JNIEXPORT jboolean JNICALL
JVM_HoldsLock(JNIEnv *env, jclass threadClass, jobject obj);
-JNIEXPORT jobject JNICALL
+JNIEXPORT jobjectArray JNICALL
JVM_GetStackTrace(JNIEnv *env, jobject thread);
JNIEXPORT jobject JNICALL
diff --git a/src/hotspot/share/jfr/recorder/jfrRecorder.cpp b/src/hotspot/share/jfr/recorder/jfrRecorder.cpp
index 6a6da0f9b04..061e3feac6f 100644
--- a/src/hotspot/share/jfr/recorder/jfrRecorder.cpp
+++ b/src/hotspot/share/jfr/recorder/jfrRecorder.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2012, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -240,7 +240,7 @@ bool JfrRecorder::on_create_vm_2() {
}
bool JfrRecorder::on_create_vm_3() {
- JVMTI_ONLY( assert(JvmtiEnvBase::get_phase() == JVMTI_PHASE_LIVE, "invalid init sequence"); )
+ JVMTI_ONLY( assert(JvmtiEnvBase::get_phase() == JVMTI_PHASE_LIVE, "invalid init sequence, phase is %d", (int)JvmtiEnvBase::get_phase()); )
return CDSConfig::is_dumping_archive() || launch_command_line_recordings(JavaThread::current());
}
diff --git a/src/hotspot/share/memory/arena.hpp b/src/hotspot/share/memory/arena.hpp
index a8450b5543a..7d88c79ca52 100644
--- a/src/hotspot/share/memory/arena.hpp
+++ b/src/hotspot/share/memory/arena.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -101,6 +101,7 @@ public:
FN(ra, Resource areas) \
FN(node, C2 Node arena) \
FN(comp, C2 Compile arena) \
+ FN(idealloop, C2 Ideal Loop arena) \
FN(type, C2 Type arena) \
FN(states, C2 Matcher States Arena) \
FN(reglive, C2 Register Allocation Live Arenas) \
diff --git a/src/hotspot/share/memory/heap.cpp b/src/hotspot/share/memory/heap.cpp
index a1333ed13e9..3ade5169eef 100644
--- a/src/hotspot/share/memory/heap.cpp
+++ b/src/hotspot/share/memory/heap.cpp
@@ -163,6 +163,7 @@ void CodeHeap::mark_segmap_as_used(size_t beg, size_t end, bool is_FreeBlock_joi
void CodeHeap::invalidate(size_t beg, size_t end, size_t hdr_size) {
#ifndef PRODUCT
+ MACOS_AARCH64_ONLY(os::thread_wx_enable_write());
// Fill the given range with some bad value.
// length is expected to be in segment_size units.
// This prevents inadvertent execution of code leftover from previous use.
@@ -172,11 +173,13 @@ void CodeHeap::invalidate(size_t beg, size_t end, size_t hdr_size) {
}
void CodeHeap::clear(size_t beg, size_t end) {
+ MACOS_AARCH64_ONLY(os::thread_wx_enable_write());
mark_segmap_as_free(beg, end);
invalidate(beg, end, 0);
}
void CodeHeap::clear() {
+ MACOS_AARCH64_ONLY(os::thread_wx_enable_write());
_next_segment = 0;
clear(_next_segment, _number_of_committed_segments);
}
@@ -190,6 +193,7 @@ static size_t align_to_page_size(size_t size) {
bool CodeHeap::reserve(ReservedSpace rs, size_t committed_size, size_t segment_size) {
+ MACOS_AARCH64_ONLY(os::thread_wx_enable_write());
assert(rs.size() >= committed_size, "reserved < committed");
assert(is_aligned(committed_size, rs.page_size()), "must be page aligned");
assert(segment_size >= sizeof(FreeBlock), "segment size is too small");
@@ -230,6 +234,7 @@ bool CodeHeap::reserve(ReservedSpace rs, size_t committed_size, size_t segment_s
bool CodeHeap::expand_by(size_t size) {
+ MACOS_AARCH64_ONLY(os::thread_wx_enable_write());
assert_locked_or_safepoint(CodeCache_lock);
// expand _memory space
@@ -259,6 +264,7 @@ bool CodeHeap::expand_by(size_t size) {
void* CodeHeap::allocate(size_t instance_size) {
+ MACOS_AARCH64_ONLY(os::thread_wx_enable_write());
size_t number_of_segments = size_to_segments(instance_size + header_size());
assert(segments_to_size(number_of_segments) >= sizeof(FreeBlock), "not enough room for FreeList");
assert_locked_or_safepoint(CodeCache_lock);
diff --git a/src/hotspot/share/memory/metaspace/blockTree.cpp b/src/hotspot/share/memory/metaspace/blockTree.cpp
index 7ad24353c96..bdae317a0b9 100644
--- a/src/hotspot/share/memory/metaspace/blockTree.cpp
+++ b/src/hotspot/share/memory/metaspace/blockTree.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, 2022 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -39,18 +39,14 @@ const size_t BlockTree::MinWordSize;
#define NODE_FORMAT \
"@" PTR_FORMAT \
": canary " INTPTR_FORMAT \
- ", parent " PTR_FORMAT \
- ", left " PTR_FORMAT \
- ", right " PTR_FORMAT \
+ ", tree " PTR_FORMAT \
", next " PTR_FORMAT \
", size %zu"
#define NODE_FORMAT_ARGS(n) \
p2i(n), \
(n)->_canary, \
- p2i((n)->_parent), \
- p2i((n)->_left), \
- p2i((n)->_right), \
+ p2i(&(n)->_tree_node), \
p2i((n)->_next), \
(n)->_word_size
@@ -74,15 +70,6 @@ const size_t BlockTree::MinWordSize;
#define tree_assert_invalid_node(cond, failure_node) \
tree_assert(cond, "Invalid node: " NODE_FORMAT, NODE_FORMAT_ARGS(failure_node))
-// walkinfo keeps a node plus the size corridor it and its children
-// are supposed to be in.
-struct BlockTree::walkinfo {
- BlockTree::Node* n;
- int depth;
- size_t lim1; // (
- size_t lim2; // )
-};
-
// Helper for verify()
void BlockTree::verify_node_pointer(const Node* n) const {
tree_assert(os::is_readable_pointer(n),
@@ -98,80 +85,32 @@ void BlockTree::verify_node_pointer(const Node* n) const {
void BlockTree::verify() const {
// Traverse the tree and test that all nodes are in the correct order.
-
MemRangeCounter counter;
- if (_root != nullptr) {
- ResourceMark rm;
- GrowableArray stack;
+ // Verifies node ordering (n1 < n2 => word_size1 < word_size2),
+ // node validity, and that the tree is balanced and not ill-formed.
+ _tree.verify_self([&](const TreeNode* tree_node) {
+ const Node* n = Node::cast_to_node(tree_node);
- walkinfo info;
- info.n = _root;
- info.lim1 = 0;
- info.lim2 = SIZE_MAX;
- info.depth = 0;
+ verify_node_pointer(n);
- stack.push(info);
+ counter.add(n->_word_size);
- while (stack.length() > 0) {
- info = stack.pop();
- const Node* n = info.n;
+ tree_assert_invalid_node(n->_word_size >= MinWordSize, n);
+ tree_assert_invalid_node(n->_word_size <= chunklevel::MAX_CHUNK_WORD_SIZE, n);
- verify_node_pointer(n);
-
- // Assume a (ridiculously large) edge limit to catch cases
- // of badly degenerated or circular trees.
- tree_assert(info.depth < 10000, "too deep (%d)", info.depth);
- counter.add(n->_word_size);
-
- if (n == _root) {
- tree_assert_invalid_node(n->_parent == nullptr, n);
- } else {
- tree_assert_invalid_node(n->_parent != nullptr, n);
- }
-
- // check size and ordering
- tree_assert_invalid_node(n->_word_size >= MinWordSize, n);
- tree_assert_invalid_node(n->_word_size <= chunklevel::MAX_CHUNK_WORD_SIZE, n);
- tree_assert_invalid_node(n->_word_size > info.lim1, n);
- tree_assert_invalid_node(n->_word_size < info.lim2, n);
-
- // Check children
- if (n->_left != nullptr) {
- tree_assert_invalid_node(n->_left != n, n);
- tree_assert_invalid_node(n->_left->_parent == n, n);
-
- walkinfo info2;
- info2.n = n->_left;
- info2.lim1 = info.lim1;
- info2.lim2 = n->_word_size;
- info2.depth = info.depth + 1;
- stack.push(info2);
- }
-
- if (n->_right != nullptr) {
- tree_assert_invalid_node(n->_right != n, n);
- tree_assert_invalid_node(n->_right->_parent == n, n);
-
- walkinfo info2;
- info2.n = n->_right;
- info2.lim1 = n->_word_size;
- info2.lim2 = info.lim2;
- info2.depth = info.depth + 1;
- stack.push(info2);
- }
-
- // If node has same-sized siblings check those too.
- const Node* n2 = n->_next;
- while (n2 != nullptr) {
- verify_node_pointer(n2);
- tree_assert_invalid_node(n2 != n, n2); // catch simple circles
- tree_assert_invalid_node(n2->_word_size == n->_word_size, n2);
- counter.add(n2->_word_size);
- n2 = n2->_next;
- }
+ // If node has same-sized siblings check those too.
+ const Node* n2 = n->_next;
+ while (n2 != nullptr) {
+ verify_node_pointer(n2);
+ tree_assert_invalid_node(n2 != n, n2); // catch simple circles
+ tree_assert_invalid_node(n2->_word_size == n->_word_size, n2);
+ counter.add(n2->_word_size);
+ n2 = n2->_next;
}
- }
+
+ return true;
+ });
// At the end, check that counters match
// (which also verifies that we visited every node, or at least
@@ -189,64 +128,34 @@ void BlockTree::print_tree(outputStream* st) const {
// as a quasi list is much clearer to the eye.
// We print the tree depth-first, with stacked nodes below normal ones
// (normal "real" nodes are marked with a leading '+')
- if (_root != nullptr) {
+ if (is_empty()) {
+ st->print_cr("");
+ return;
+ }
- ResourceMark rm;
- GrowableArray stack;
+ _tree.print_on(st, [&](outputStream *st, const TreeNode *tree_node, int depth) {
+ const Node* n = Node::cast_to_node(tree_node);
- walkinfo info;
- info.n = _root;
- info.depth = 0;
-
- stack.push(info);
- while (stack.length() > 0) {
- info = stack.pop();
- const Node* n = info.n;
-
- // Print node.
- st->print("%4d + ", info.depth);
- if (os::is_readable_pointer(n)) {
- st->print_cr(NODE_FORMAT, NODE_FORMAT_ARGS(n));
- } else {
- st->print_cr("@" PTR_FORMAT ": unreadable (skipping subtree)", p2i(n));
- continue; // don't print this subtree
- }
-
- // Print same-sized-nodes stacked under this node
- for (Node* n2 = n->_next; n2 != nullptr; n2 = n2->_next) {
- st->print_raw(" ");
- if (os::is_readable_pointer(n2)) {
- st->print_cr(NODE_FORMAT, NODE_FORMAT_ARGS(n2));
- } else {
- st->print_cr("@" PTR_FORMAT ": unreadable (skipping rest of chain).", p2i(n2));
- break; // stop printing this chain.
- }
- }
-
- // Handle simple circularities
- if (n == n->_right || n == n->_left || n == n->_next) {
- st->print_cr("@" PTR_FORMAT ": circularity detected.", p2i(n));
- return; // stop printing
- }
-
- // Handle children.
- if (n->_right != nullptr) {
- walkinfo info2;
- info2.n = n->_right;
- info2.depth = info.depth + 1;
- stack.push(info2);
- }
- if (n->_left != nullptr) {
- walkinfo info2;
- info2.n = n->_left;
- info2.depth = info.depth + 1;
- stack.push(info2);
- }
+ // Print node.
+ st->print("%4d + ", depth);
+ if (os::is_readable_pointer(n)) {
+ st->print_cr(NODE_FORMAT, NODE_FORMAT_ARGS(n));
+ } else {
+ st->print_cr("@" PTR_FORMAT ": unreadable", p2i(n));
+ return;
}
- } else {
- st->print_cr("");
- }
+ // Print same-sized-nodes stacked under this node
+ for (Node* n2 = n->_next; n2 != nullptr; n2 = n2->_next) {
+ st->print_raw(" ");
+ if (os::is_readable_pointer(n2)) {
+ st->print_cr(NODE_FORMAT, NODE_FORMAT_ARGS(n2));
+ } else {
+ st->print_cr("@" PTR_FORMAT ": unreadable (skipping rest of chain).", p2i(n2));
+ break; // stop printing this chain.
+ }
+ }
+ });
}
#endif // ASSERT
diff --git a/src/hotspot/share/memory/metaspace/blockTree.hpp b/src/hotspot/share/memory/metaspace/blockTree.hpp
index a01f60b166f..e7c1edf9c4f 100644
--- a/src/hotspot/share/memory/metaspace/blockTree.hpp
+++ b/src/hotspot/share/memory/metaspace/blockTree.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -32,17 +32,18 @@
#include "memory/metaspace/metablock.hpp"
#include "utilities/debug.hpp"
#include "utilities/globalDefinitions.hpp"
+#include "utilities/rbTree.inline.hpp"
namespace metaspace {
-// BlockTree is a rather simple binary search tree. It is used to
-// manage medium to large free memory blocks.
+// BlockTree is tree built on an intrusive red-black tree.
+// It is used to manage medium to large free memory blocks.
//
// There is no separation between payload (managed blocks) and nodes: the
// memory blocks themselves are the nodes, with the block size being the key.
//
// We store node pointer information in these blocks when storing them. That
-// imposes a minimum size to the managed memory blocks (1 word)
+// imposes a minimum size to the managed memory blocks (1 MinWordSize)
//
// We want to manage many memory blocks of the same size, but we want
// to prevent the tree from blowing up and degenerating into a list. Therefore
@@ -53,9 +54,9 @@ namespace metaspace {
// | 100 |
// +-----+
// / \
-// +-----+
-// | 80 |
-// +-----+
+// +-----+ +-----+
+// | 80 | | 120 |
+// +-----+ +-----+
// / | \
// / +-----+ \
// +-----+ | 80 | +-----+
@@ -65,16 +66,11 @@ namespace metaspace {
// | 80 |
// +-----+
//
-//
-// Todo: This tree is unbalanced. It would be a good fit for a red-black tree.
-// In order to make this a red-black tree, we need an algorithm which can deal
-// with nodes which are their own payload (most red-black tree implementations
-// swap payloads of their nodes at some point, see e.g. j.u.TreeSet).
-// A good example is the Linux kernel rbtree, which is a clean, easy-to-read
-// implementation.
class BlockTree: public CHeapObj {
+ using TreeNode = IntrusiveRBNode;
+
struct Node {
static const intptr_t _canary_value =
@@ -86,29 +82,27 @@ class BlockTree: public CHeapObj {
// in debug.
const intptr_t _canary;
- // Normal tree node stuff...
- // (Note: all null if this is a stacked node)
- Node* _parent;
- Node* _left;
- Node* _right;
+ // Tree node for linking blocks in the intrusive tree.
+ TreeNode _tree_node;
// Blocks with the same size are put in a list with this node as head.
Node* _next;
// Word size of node. Note that size cannot be larger than max metaspace size,
- // so this could be very well a 32bit value (in case we ever make this a balancing
- // tree and need additional space for weighting information).
+ // so this could very well be a 32bit value.
const size_t _word_size;
Node(size_t word_size) :
_canary(_canary_value),
- _parent(nullptr),
- _left(nullptr),
- _right(nullptr),
+ _tree_node{},
_next(nullptr),
_word_size(word_size)
{}
+ static Node* cast_to_node(const TreeNode* tree_node) {
+ return (Node*)((uintptr_t)tree_node - offset_of(Node, _tree_node));
+ }
+
#ifdef ASSERT
bool valid() const {
return _canary == _canary_value &&
@@ -118,8 +112,23 @@ class BlockTree: public CHeapObj {
#endif
};
- // Needed for verify() and print_tree()
- struct walkinfo;
+ struct TreeComparator {
+ static RBTreeOrdering cmp(const size_t a, const TreeNode* b) {
+ const size_t node_word_size = Node::cast_to_node(b)->_word_size;
+
+ if (a < node_word_size) { return RBTreeOrdering::LT; }
+ if (a > node_word_size) { return RBTreeOrdering::GT; }
+ return RBTreeOrdering::EQ;
+ }
+
+ static bool less_than(const TreeNode* a, const TreeNode* b) {
+ const size_t a_word_size = Node::cast_to_node(a)->_word_size;
+ const size_t b_word_size = Node::cast_to_node(b)->_word_size;
+
+ if (a_word_size < b_word_size) { return true; }
+ return false;
+ }
+ };
#ifdef ASSERT
// Run a quick check on a node; upon suspicion dive into a full tree check.
@@ -134,7 +143,7 @@ public:
private:
- Node* _root;
+ IntrusiveRBTree _tree;
MemRangeCounter _counter;
@@ -143,7 +152,7 @@ private:
assert(head->_word_size == n->_word_size, "sanity");
n->_next = head->_next;
head->_next = n;
- DEBUG_ONLY(n->_left = n->_right = n->_parent = nullptr;)
+ DEBUG_ONLY(n->_tree_node = TreeNode());
}
// Given a node list starting at head, remove one of the follow up nodes from
@@ -157,183 +166,6 @@ private:
return n;
}
- // Given a node c and a node p, wire up c as left child of p.
- static void set_left_child(Node* p, Node* c) {
- p->_left = c;
- if (c != nullptr) {
- assert(c->_word_size < p->_word_size, "sanity");
- c->_parent = p;
- }
- }
-
- // Given a node c and a node p, wire up c as right child of p.
- static void set_right_child(Node* p, Node* c) {
- p->_right = c;
- if (c != nullptr) {
- assert(c->_word_size > p->_word_size, "sanity");
- c->_parent = p;
- }
- }
-
- // Given a node n, return its successor in the tree
- // (node with the next-larger size).
- static Node* successor(Node* n) {
- Node* succ = nullptr;
- if (n->_right != nullptr) {
- // If there is a right child, search the left-most
- // child of that child.
- succ = n->_right;
- while (succ->_left != nullptr) {
- succ = succ->_left;
- }
- } else {
- succ = n->_parent;
- Node* n2 = n;
- // As long as I am the right child of my parent, search upward
- while (succ != nullptr && n2 == succ->_right) {
- n2 = succ;
- succ = succ->_parent;
- }
- }
- return succ;
- }
-
- // Given a node, replace it with a replacement node as a child for its parent.
- // If the node is root and has no parent, sets it as root.
- void replace_node_in_parent(Node* child, Node* replace) {
- Node* parent = child->_parent;
- if (parent != nullptr) {
- if (parent->_left == child) { // Child is left child
- set_left_child(parent, replace);
- } else {
- set_right_child(parent, replace);
- }
- } else {
- assert(child == _root, "must be root");
- _root = replace;
- if (replace != nullptr) {
- replace->_parent = nullptr;
- }
- }
- return;
- }
-
- // Given a node n and an insertion point, insert n under insertion point.
- void insert(Node* insertion_point, Node* n) {
- assert(n->_parent == nullptr, "Sanity");
- for (;;) {
- DEBUG_ONLY(check_node(insertion_point);)
- if (n->_word_size == insertion_point->_word_size) {
- add_to_list(n, insertion_point); // parent stays null in this case.
- break;
- } else if (n->_word_size > insertion_point->_word_size) {
- if (insertion_point->_right == nullptr) {
- set_right_child(insertion_point, n);
- break;
- } else {
- insertion_point = insertion_point->_right;
- }
- } else {
- if (insertion_point->_left == nullptr) {
- set_left_child(insertion_point, n);
- break;
- } else {
- insertion_point = insertion_point->_left;
- }
- }
- }
- }
-
- // Given a node and a wish size, search this node and all children for
- // the node closest (equal or larger sized) to the size s.
- Node* find_closest_fit(Node* n, size_t s) {
- Node* best_match = nullptr;
- while (n != nullptr) {
- DEBUG_ONLY(check_node(n);)
- if (n->_word_size >= s) {
- best_match = n;
- if (n->_word_size == s) {
- break; // perfect match or max depth reached
- }
- n = n->_left;
- } else {
- n = n->_right;
- }
- }
- return best_match;
- }
-
- // Given a wish size, search the whole tree for a
- // node closest (equal or larger sized) to the size s.
- Node* find_closest_fit(size_t s) {
- if (_root != nullptr) {
- return find_closest_fit(_root, s);
- }
- return nullptr;
- }
-
- // Given a node n, remove it from the tree and repair tree.
- void remove_node_from_tree(Node* n) {
- assert(n->_next == nullptr, "do not delete a node which has a non-empty list");
-
- if (n->_left == nullptr && n->_right == nullptr) {
- replace_node_in_parent(n, nullptr);
-
- } else if (n->_left == nullptr && n->_right != nullptr) {
- replace_node_in_parent(n, n->_right);
-
- } else if (n->_left != nullptr && n->_right == nullptr) {
- replace_node_in_parent(n, n->_left);
-
- } else {
- // Node has two children.
-
- // 1) Find direct successor (the next larger node).
- Node* succ = successor(n);
-
- // There has to be a successor since n->right was != null...
- assert(succ != nullptr, "must be");
-
- // ... and it should not have a left child since successor
- // is supposed to be the next larger node, so it must be the mostleft node
- // in the sub tree rooted at n->right
- assert(succ->_left == nullptr, "must be");
- assert(succ->_word_size > n->_word_size, "sanity");
-
- Node* successor_parent = succ->_parent;
- Node* successor_right_child = succ->_right;
-
- // Remove successor from its parent.
- if (successor_parent == n) {
-
- // special case: successor is a direct child of n. Has to be the right child then.
- assert(n->_right == succ, "sanity");
-
- // Just replace n with this successor.
- replace_node_in_parent(n, succ);
-
- // Take over n's old left child, too.
- // We keep the successor's right child.
- set_left_child(succ, n->_left);
- } else {
- // If the successors parent is not n, we are deeper in the tree,
- // the successor has to be the left child of its parent.
- assert(successor_parent->_left == succ, "sanity");
-
- // The right child of the successor (if there was one) replaces
- // the successor at its parent's left child.
- set_left_child(successor_parent, succ->_right);
-
- // and the successor replaces n at its parent
- replace_node_in_parent(n, succ);
-
- // and takes over n's old children
- set_left_child(succ, n->_left);
- set_right_child(succ, n->_right);
- }
- }
- }
-
#ifdef ASSERT
void zap_block(MetaBlock block);
// Helper for verify()
@@ -342,7 +174,7 @@ private:
public:
- BlockTree() : _root(nullptr) {}
+ BlockTree() {}
// Add a memory block to the tree. Its content will be overwritten.
void add_block(MetaBlock block) {
@@ -350,10 +182,12 @@ public:
const size_t word_size = block.word_size();
assert(word_size >= MinWordSize, "invalid block size %zu", word_size);
Node* n = new(block.base()) Node(word_size);
- if (_root == nullptr) {
- _root = n;
- } else {
- insert(_root, n);
+ IntrusiveRBTree::Cursor cursor = _tree.cursor(word_size);
+ if (cursor.found()) {
+ add_to_list(n, Node::cast_to_node(cursor.node()));
+ }
+ else {
+ _tree.insert_at_cursor(&n->_tree_node, cursor);
}
_counter.add(word_size);
}
@@ -364,9 +198,10 @@ public:
assert(word_size >= MinWordSize, "invalid block size %zu", word_size);
MetaBlock result;
- Node* n = find_closest_fit(word_size);
+ TreeNode* tree_node = _tree.closest_ge(word_size);
- if (n != nullptr) {
+ if (tree_node != nullptr) {
+ Node* n = Node::cast_to_node(tree_node);
DEBUG_ONLY(check_node(n);)
assert(n->_word_size >= word_size, "sanity");
@@ -377,7 +212,7 @@ public:
// node into its place in the tree).
n = remove_from_list(n);
} else {
- remove_node_from_tree(n);
+ _tree.remove(tree_node);
}
result = MetaBlock((MetaWord*)n, n->_word_size);
@@ -395,7 +230,7 @@ public:
// Returns total size, in words, of all elements.
size_t total_size() const { return _counter.total_size(); }
- bool is_empty() const { return _root == nullptr; }
+ bool is_empty() const { return _tree.size() == 0; }
DEBUG_ONLY(void print_tree(outputStream* st) const;)
DEBUG_ONLY(void verify() const;)
diff --git a/src/hotspot/share/nmt/mallocHeader.cpp b/src/hotspot/share/nmt/mallocHeader.cpp
index d88b5c790fb..e74e74fde48 100644
--- a/src/hotspot/share/nmt/mallocHeader.cpp
+++ b/src/hotspot/share/nmt/mallocHeader.cpp
@@ -75,4 +75,4 @@ void MallocHeader::print_block_on_error(outputStream* st, address bad_address, a
// print one hex dump
os::print_hex_dump(st, from1, to2, 1);
}
-}
+}
\ No newline at end of file
diff --git a/src/hotspot/share/nmt/mallocHeader.hpp b/src/hotspot/share/nmt/mallocHeader.hpp
index 45568243ef3..33e288dd8e4 100644
--- a/src/hotspot/share/nmt/mallocHeader.hpp
+++ b/src/hotspot/share/nmt/mallocHeader.hpp
@@ -27,10 +27,18 @@
#define SHARE_NMT_MALLOCHEADER_HPP
#include "nmt/memTag.hpp"
+#include "sanitizers/address.hpp"
#include "utilities/globalDefinitions.hpp"
#include "utilities/macros.hpp"
#include "utilities/nativeCallStack.hpp"
+// With ASAN, we omit NMT block integrity checks since ASAN does a better and faster
+// job of alerting us to memory corruptions
+#if INCLUDE_ASAN
+#undef NMT_BLOCK_INTEGRITY_CHECKS
+#else
+#define NMT_BLOCK_INTEGRITY_CHECKS
+#endif
class outputStream;
/*
@@ -118,6 +126,7 @@ class MallocHeader {
inline static OutTypeParam resolve_checked_impl(InTypeParam memblock);
public:
+ static constexpr size_t footer_size = sizeof(uint16_t);
// Contains all of the necessary data to to deaccount block with NMT.
struct FreeInfo {
const size_t size;
@@ -126,8 +135,9 @@ public:
};
inline MallocHeader(size_t size, MemTag mem_tag, uint32_t mst_marker);
-
- inline static size_t malloc_overhead() { return sizeof(MallocHeader) + sizeof(uint16_t); }
+ inline static size_t malloc_overhead() { return sizeof(MallocHeader) + footer_size; }
+ inline static MallocHeader* kill_block(void* memblock);
+ inline static void revive_block(void* memblock);
inline size_t size() const { return _size; }
inline MemTag mem_tag() const { return _mem_tag; }
inline uint32_t mst_marker() const { return _mst_marker; }
@@ -164,5 +174,4 @@ public:
// This needs to be true on both 64-bit and 32-bit platforms
STATIC_ASSERT(sizeof(MallocHeader) == (sizeof(uint64_t) * 2));
-
-#endif // SHARE_NMT_MALLOCHEADER_HPP
+#endif // SHARE_NMT_MALLOCHEADER_HPP
\ No newline at end of file
diff --git a/src/hotspot/share/nmt/mallocHeader.inline.hpp b/src/hotspot/share/nmt/mallocHeader.inline.hpp
index 0ad0b3ddc81..09ab872c780 100644
--- a/src/hotspot/share/nmt/mallocHeader.inline.hpp
+++ b/src/hotspot/share/nmt/mallocHeader.inline.hpp
@@ -46,9 +46,6 @@ inline MallocHeader::MallocHeader(size_t size, MemTag mem_tag, uint32_t mst_mark
}
inline void MallocHeader::revive() {
- assert(_canary == _header_canary_dead_mark, "must be dead");
- assert(get_footer() == _footer_canary_dead_mark, "must be dead");
- NOT_LP64(assert(_alt_canary == _header_alt_canary_dead_mark, "must be dead"));
_canary = _header_canary_live_mark;
NOT_LP64(_alt_canary = _header_alt_canary_live_mark);
set_footer(_footer_canary_live_mark);
@@ -96,16 +93,18 @@ inline bool MallocHeader::is_valid_malloced_pointer(const void* payload, char* m
template
inline OutTypeParam MallocHeader::resolve_checked_impl(InTypeParam memblock) {
- char msg[256];
- address corruption = nullptr;
- if (!is_valid_malloced_pointer(memblock, msg, sizeof(msg))) {
- fatal("Not a valid malloc pointer: " PTR_FORMAT ": %s", p2i(memblock), msg);
- }
OutTypeParam header_pointer = (OutTypeParam)memblock - 1;
- if (!header_pointer->check_block_integrity(msg, sizeof(msg), &corruption)) {
- header_pointer->print_block_on_error(tty, corruption != nullptr ? corruption : (address)header_pointer, (address)header_pointer);
- fatal("NMT has detected a memory corruption bug. Block at " PTR_FORMAT ": %s", p2i(memblock), msg);
- }
+ #ifdef NMT_BLOCK_INTEGRITY_CHECKS
+ char msg[256];
+ address corruption = nullptr;
+ if (!is_valid_malloced_pointer(memblock, msg, sizeof(msg))) {
+ fatal("Not a valid malloc pointer: " PTR_FORMAT ": %s", p2i(memblock), msg);
+ }
+ if (!header_pointer->check_block_integrity(msg, sizeof(msg), &corruption)) {
+ header_pointer->print_block_on_error(tty, corruption != nullptr ? corruption : (address)header_pointer, (address)header_pointer);
+ fatal("NMT has detected a memory corruption bug. Block at " PTR_FORMAT ": %s", p2i(memblock), msg);
+ }
+ #endif
return header_pointer;
}
@@ -163,4 +162,20 @@ inline bool MallocHeader::check_block_integrity(char* msg, size_t msglen, addres
return true;
}
+MallocHeader* MallocHeader::kill_block(void* memblock) {
+ MallocHeader* header = (MallocHeader*)memblock - 1;
+ ASAN_UNPOISON_MEMORY_REGION(header, sizeof(MallocHeader));
+ ASAN_UNPOISON_MEMORY_REGION(header->footer_address(), footer_size);
+ resolve_checked(memblock);
+ header->mark_block_as_dead();
+ return header;
+}
+
+void MallocHeader::revive_block(void* memblock) {
+ MallocHeader* header = (MallocHeader*)memblock - 1;
+ header->revive();
+ ASAN_POISON_MEMORY_REGION(header->footer_address(), footer_size);
+ ASAN_POISON_MEMORY_REGION(header, sizeof(MallocHeader));
+}
+
#endif // SHARE_NMT_MALLOCHEADER_INLINE_HPP
diff --git a/src/hotspot/share/nmt/mallocTracker.cpp b/src/hotspot/share/nmt/mallocTracker.cpp
index 2cf5034c0bf..532b6f41bd0 100644
--- a/src/hotspot/share/nmt/mallocTracker.cpp
+++ b/src/hotspot/share/nmt/mallocTracker.cpp
@@ -199,7 +199,7 @@ void* MallocTracker::record_malloc(void* malloc_base, size_t size, MemTag mem_ta
assert(header2->mem_tag() == mem_tag, "Wrong memory tag");
}
#endif
-
+ MallocHeader::revive_block(memblock);
return memblock;
}
@@ -207,7 +207,7 @@ void* MallocTracker::record_free_block(void* memblock) {
assert(MemTracker::enabled(), "Sanity");
assert(memblock != nullptr, "precondition");
- MallocHeader* header = MallocHeader::resolve_checked(memblock);
+ MallocHeader* header = MallocHeader::kill_block(memblock);
deaccount(header->free_info());
@@ -218,7 +218,6 @@ void* MallocTracker::record_free_block(void* memblock) {
}
header->mark_block_as_dead();
-
return (void*)header;
}
diff --git a/src/hotspot/share/nmt/mallocTracker.hpp b/src/hotspot/share/nmt/mallocTracker.hpp
index fc03faf7212..e96396ebbcb 100644
--- a/src/hotspot/share/nmt/mallocTracker.hpp
+++ b/src/hotspot/share/nmt/mallocTracker.hpp
@@ -311,15 +311,6 @@ class MallocTracker : AllStatic {
// totally failproof. Only use this during debugging or when you can afford
// signals popping up, e.g. when writing an hs_err file.
static bool print_pointer_information(const void* p, outputStream* st);
-
- static inline MallocHeader* malloc_header(void *memblock) {
- assert(memblock != nullptr, "null pointer");
- return (MallocHeader*)memblock -1;
- }
- static inline const MallocHeader* malloc_header(const void *memblock) {
- assert(memblock != nullptr, "null pointer");
- return (const MallocHeader*)memblock -1;
- }
};
#endif // SHARE_NMT_MALLOCTRACKER_HPP
diff --git a/src/hotspot/share/opto/classes.hpp b/src/hotspot/share/opto/classes.hpp
index d56abaa17dd..abd93fdd876 100644
--- a/src/hotspot/share/opto/classes.hpp
+++ b/src/hotspot/share/opto/classes.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -281,7 +281,7 @@ macro(OpaqueLoopInit)
macro(OpaqueLoopStride)
macro(OpaqueMultiversioning)
macro(OpaqueZeroTripGuard)
-macro(OpaqueNotNull)
+macro(OpaqueConstantBool)
macro(OpaqueInitializedAssertionPredicate)
macro(OpaqueTemplateAssertionPredicate)
macro(ProfileBoolean)
@@ -462,6 +462,8 @@ macro(UMinV)
macro(UMaxV)
macro(MinReductionV)
macro(MaxReductionV)
+macro(UMinReductionV)
+macro(UMaxReductionV)
macro(CompressV)
macro(CompressM)
macro(ExpandV)
diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp
index c4c9445b61a..80bb1dcf408 100644
--- a/src/hotspot/share/opto/compile.cpp
+++ b/src/hotspot/share/opto/compile.cpp
@@ -2171,6 +2171,21 @@ void Compile::inline_incrementally_cleanup(PhaseIterGVN& igvn) {
print_method(PHASE_INCREMENTAL_INLINE_CLEANUP, 3);
}
+template
+static void shuffle_array(Compile& C, GrowableArray& array) {
+ if (array.length() < 2) {
+ return;
+ }
+ for (uint i = array.length() - 1; i >= 1; i--) {
+ uint j = C.random() % (i + 1);
+ swap(array.at(i), array.at(j));
+ }
+}
+
+void Compile::shuffle_late_inlines() {
+ shuffle_array(*C, _late_inlines);
+}
+
// Perform incremental inlining until bound on number of live nodes is reached
void Compile::inline_incrementally(PhaseIterGVN& igvn) {
TracePhase tp(_t_incrInline);
@@ -2178,6 +2193,10 @@ void Compile::inline_incrementally(PhaseIterGVN& igvn) {
set_inlining_incrementally(true);
uint low_live_nodes = 0;
+ if (StressIncrementalInlining) {
+ shuffle_late_inlines();
+ }
+
while (_late_inlines.length() > 0) {
if (live_nodes() > (uint)LiveNodeCountInliningCutoff) {
if (low_live_nodes < (uint)LiveNodeCountInliningCutoff * 8 / 10) {
@@ -2250,6 +2269,10 @@ void Compile::process_late_inline_calls_no_inline(PhaseIterGVN& igvn) {
assert(_modified_nodes == nullptr, "not allowed");
assert(_late_inlines.length() > 0, "sanity");
+ if (StressIncrementalInlining) {
+ shuffle_late_inlines();
+ }
+
while (_late_inlines.length() > 0) {
igvn_worklist()->ensure_empty(); // should be done with igvn
@@ -3789,6 +3812,8 @@ void Compile::final_graph_reshaping_main_switch(Node* n, Final_Reshape_Counts& f
case Op_MulReductionVD:
case Op_MinReductionV:
case Op_MaxReductionV:
+ case Op_UMinReductionV:
+ case Op_UMaxReductionV:
case Op_AndReductionV:
case Op_OrReductionV:
case Op_XorReductionV:
@@ -5141,13 +5166,7 @@ void CloneMap::dump(node_idx_t key, outputStream* st) const {
}
void Compile::shuffle_macro_nodes() {
- if (_macro_nodes.length() < 2) {
- return;
- }
- for (uint i = _macro_nodes.length() - 1; i >= 1; i--) {
- uint j = C->random() % (i + 1);
- swap(_macro_nodes.at(i), _macro_nodes.at(j));
- }
+ shuffle_array(*C, _macro_nodes);
}
// Move Allocate nodes to the start of the list
diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp
index d9a8ea15724..eb6be669f24 100644
--- a/src/hotspot/share/opto/compile.hpp
+++ b/src/hotspot/share/opto/compile.hpp
@@ -795,6 +795,7 @@ public:
void remove_from_merge_stores_igvn(Node* n);
void process_for_merge_stores_igvn(PhaseIterGVN& igvn);
+ void shuffle_late_inlines();
void shuffle_macro_nodes();
void sort_macro_nodes();
@@ -1059,6 +1060,13 @@ public:
// Record this CallGenerator for inlining at the end of parsing.
void add_late_inline(CallGenerator* cg) {
_late_inlines.insert_before(_late_inlines_pos, cg);
+ if (StressIncrementalInlining) {
+ assert(_late_inlines_pos < _late_inlines.length(), "unthinkable!");
+ if (_late_inlines.length() - _late_inlines_pos >= 2) {
+ int j = (C->random() % (_late_inlines.length() - _late_inlines_pos)) + _late_inlines_pos;
+ swap(_late_inlines.at(_late_inlines_pos), _late_inlines.at(j));
+ }
+ }
_late_inlines_pos++;
}
diff --git a/src/hotspot/share/opto/escape.cpp b/src/hotspot/share/opto/escape.cpp
index b46a12fcf89..5befdd924ff 100644
--- a/src/hotspot/share/opto/escape.cpp
+++ b/src/hotspot/share/opto/escape.cpp
@@ -588,7 +588,7 @@ bool ConnectionGraph::can_reduce_check_users(Node* n, uint nesting) const {
// CmpP/N used by the If controlling the cast.
if (use->in(0)->is_IfTrue() || use->in(0)->is_IfFalse()) {
Node* iff = use->in(0)->in(0);
- // We may have an OpaqueNotNull node between If and Bool nodes. But we could also have a sub class of IfNode,
+ // We may have an OpaqueConstantBool node between If and Bool nodes. But we could also have a sub class of IfNode,
// for example, an OuterStripMinedLoopEnd or a Parse Predicate. Bail out in all these cases.
bool can_reduce = (iff->Opcode() == Op_If) && iff->in(1)->is_Bool() && iff->in(1)->in(1)->is_Cmp();
if (can_reduce) {
diff --git a/src/hotspot/share/opto/graphKit.cpp b/src/hotspot/share/opto/graphKit.cpp
index 3d127322439..084b137f313 100644
--- a/src/hotspot/share/opto/graphKit.cpp
+++ b/src/hotspot/share/opto/graphKit.cpp
@@ -1472,7 +1472,7 @@ Node* GraphKit::cast_not_null(Node* obj, bool do_replace_in_map) {
// In that case that data path will die and we need the control path
// to become dead as well to keep the graph consistent. So we have to
// add a check for null for which one branch can't be taken. It uses
-// an OpaqueNotNull node that will cause the check to be removed after loop
+// an OpaqueConstantBool node that will cause the check to be removed after loop
// opts so the test goes away and the compiled code doesn't execute a
// useless check.
Node* GraphKit::must_be_not_null(Node* value, bool do_replace_in_map) {
@@ -1481,7 +1481,7 @@ Node* GraphKit::must_be_not_null(Node* value, bool do_replace_in_map) {
}
Node* chk = _gvn.transform(new CmpPNode(value, null()));
Node* tst = _gvn.transform(new BoolNode(chk, BoolTest::ne));
- Node* opaq = _gvn.transform(new OpaqueNotNullNode(C, tst));
+ Node* opaq = _gvn.transform(new OpaqueConstantBoolNode(C, tst, true));
IfNode* iff = new IfNode(control(), opaq, PROB_MAX, COUNT_UNKNOWN);
_gvn.set_type(iff, iff->Value(&_gvn));
if (!tst->is_Con()) {
diff --git a/src/hotspot/share/opto/library_call.cpp b/src/hotspot/share/opto/library_call.cpp
index e481833c816..b4e18b596e4 100644
--- a/src/hotspot/share/opto/library_call.cpp
+++ b/src/hotspot/share/opto/library_call.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -891,13 +891,16 @@ inline Node* LibraryCallKit::generate_fair_guard(Node* test, RegionNode* region)
}
inline Node* LibraryCallKit::generate_negative_guard(Node* index, RegionNode* region,
- Node* *pos_index) {
+ Node** pos_index, bool with_opaque) {
if (stopped())
return nullptr; // already stopped
if (_gvn.type(index)->higher_equal(TypeInt::POS)) // [0,maxint]
return nullptr; // index is already adequately typed
Node* cmp_lt = _gvn.transform(new CmpINode(index, intcon(0)));
Node* bol_lt = _gvn.transform(new BoolNode(cmp_lt, BoolTest::lt));
+ if (with_opaque) {
+ bol_lt = _gvn.transform(new OpaqueConstantBoolNode(C, bol_lt, false));
+ }
Node* is_neg = generate_guard(bol_lt, region, PROB_MIN);
if (is_neg != nullptr && pos_index != nullptr) {
// Emulate effect of Parse::adjust_map_after_if.
@@ -924,7 +927,8 @@ inline Node* LibraryCallKit::generate_negative_guard(Node* index, RegionNode* re
inline Node* LibraryCallKit::generate_limit_guard(Node* offset,
Node* subseq_length,
Node* array_length,
- RegionNode* region) {
+ RegionNode* region,
+ bool with_opaque) {
if (stopped())
return nullptr; // already stopped
bool zero_offset = _gvn.type(offset) == TypeInt::ZERO;
@@ -935,12 +939,19 @@ inline Node* LibraryCallKit::generate_limit_guard(Node* offset,
last = _gvn.transform(new AddINode(last, offset));
Node* cmp_lt = _gvn.transform(new CmpUNode(array_length, last));
Node* bol_lt = _gvn.transform(new BoolNode(cmp_lt, BoolTest::lt));
+ if (with_opaque) {
+ bol_lt = _gvn.transform(new OpaqueConstantBoolNode(C, bol_lt, false));
+ }
Node* is_over = generate_guard(bol_lt, region, PROB_MIN);
return is_over;
}
// Emit range checks for the given String.value byte array
-void LibraryCallKit::generate_string_range_check(Node* array, Node* offset, Node* count, bool char_count) {
+void LibraryCallKit::generate_string_range_check(Node* array,
+ Node* offset,
+ Node* count,
+ bool char_count,
+ bool halt_on_oob) {
if (stopped()) {
return; // already stopped
}
@@ -952,16 +963,23 @@ void LibraryCallKit::generate_string_range_check(Node* array, Node* offset, Node
}
// Offset and count must not be negative
- generate_negative_guard(offset, bailout);
- generate_negative_guard(count, bailout);
+ generate_negative_guard(offset, bailout, nullptr, halt_on_oob);
+ generate_negative_guard(count, bailout, nullptr, halt_on_oob);
// Offset + count must not exceed length of array
- generate_limit_guard(offset, count, load_array_length(array), bailout);
+ generate_limit_guard(offset, count, load_array_length(array), bailout, halt_on_oob);
if (bailout->req() > 1) {
- PreserveJVMState pjvms(this);
- set_control(_gvn.transform(bailout));
- uncommon_trap(Deoptimization::Reason_intrinsic,
- Deoptimization::Action_maybe_recompile);
+ if (halt_on_oob) {
+ bailout = _gvn.transform(bailout)->as_Region();
+ Node* frame = _gvn.transform(new ParmNode(C->start(), TypeFunc::FramePtr));
+ Node* halt = _gvn.transform(new HaltNode(bailout, frame, "unexpected guard failure in intrinsic"));
+ C->root()->add_req(halt);
+ } else {
+ PreserveJVMState pjvms(this);
+ set_control(_gvn.transform(bailout));
+ uncommon_trap(Deoptimization::Reason_intrinsic,
+ Deoptimization::Action_maybe_recompile);
+ }
}
}
@@ -1119,6 +1137,7 @@ bool LibraryCallKit::inline_array_equals(StrIntrinsicNode::ArgEnc ae) {
//------------------------------inline_countPositives------------------------------
+// int java.lang.StringCoding#countPositives0(byte[] ba, int off, int len)
bool LibraryCallKit::inline_countPositives() {
if (too_many_traps(Deoptimization::Reason_intrinsic)) {
return false;
@@ -1131,12 +1150,11 @@ bool LibraryCallKit::inline_countPositives() {
Node* len = argument(2);
ba = must_be_not_null(ba, true);
-
- // Range checks
- generate_string_range_check(ba, offset, len, false);
+ generate_string_range_check(ba, offset, len, false, true);
if (stopped()) {
return true;
}
+
Node* ba_start = array_element_address(ba, offset, T_BYTE);
Node* result = new CountPositivesNode(control(), memory(TypeAryPtr::BYTES), ba_start, len);
set_result(_gvn.transform(result));
@@ -6171,6 +6189,9 @@ CallStaticJavaNode* LibraryCallKit::get_uncommon_trap_from_success_proj(Node* no
}
//-------------inline_encodeISOArray-----------------------------------
+// int sun.nio.cs.ISO_8859_1.Encoder#encodeISOArray0(byte[] sa, int sp, byte[] da, int dp, int len)
+// int java.lang.StringCoding#encodeISOArray0(byte[] sa, int sp, byte[] da, int dp, int len)
+// int java.lang.StringCoding#encodeAsciiArray0(char[] sa, int sp, byte[] da, int dp, int len)
// encode char[] to byte[] in ISO_8859_1 or ASCII
bool LibraryCallKit::inline_encodeISOArray(bool ascii) {
assert(callee()->signature()->size() == 5, "encodeISOArray has 5 parameters");
@@ -6181,8 +6202,12 @@ bool LibraryCallKit::inline_encodeISOArray(bool ascii) {
Node *dst_offset = argument(3);
Node *length = argument(4);
+ // Cast source & target arrays to not-null
src = must_be_not_null(src, true);
dst = must_be_not_null(dst, true);
+ if (stopped()) {
+ return true;
+ }
const TypeAryPtr* src_type = src->Value(&_gvn)->isa_aryptr();
const TypeAryPtr* dst_type = dst->Value(&_gvn)->isa_aryptr();
@@ -6199,6 +6224,13 @@ bool LibraryCallKit::inline_encodeISOArray(bool ascii) {
return false;
}
+ // Check source & target bounds
+ generate_string_range_check(src, src_offset, length, src_elem == T_BYTE, true);
+ generate_string_range_check(dst, dst_offset, length, false, true);
+ if (stopped()) {
+ return true;
+ }
+
Node* src_start = array_element_address(src, src_offset, T_CHAR);
Node* dst_start = array_element_address(dst, dst_offset, dst_elem);
// 'src_start' points to src array + scaled offset
diff --git a/src/hotspot/share/opto/library_call.hpp b/src/hotspot/share/opto/library_call.hpp
index 00ba4c795f1..56141be2362 100644
--- a/src/hotspot/share/opto/library_call.hpp
+++ b/src/hotspot/share/opto/library_call.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -158,12 +158,15 @@ class LibraryCallKit : public GraphKit {
Node* generate_fair_guard(Node* test, RegionNode* region);
Node* generate_negative_guard(Node* index, RegionNode* region,
// resulting CastII of index:
- Node* *pos_index = nullptr);
+ Node** pos_index = nullptr,
+ bool with_opaque = false);
Node* generate_limit_guard(Node* offset, Node* subseq_length,
Node* array_length,
- RegionNode* region);
+ RegionNode* region,
+ bool with_opaque = false);
void generate_string_range_check(Node* array, Node* offset,
- Node* length, bool char_count);
+ Node* length, bool char_count,
+ bool halt_on_oob = false);
Node* current_thread_helper(Node* &tls_output, ByteSize handle_offset,
bool is_immutable);
Node* generate_current_thread(Node* &tls_output);
diff --git a/src/hotspot/share/opto/loopTransform.cpp b/src/hotspot/share/opto/loopTransform.cpp
index fdb3ab89b82..4e221a9a0ef 100644
--- a/src/hotspot/share/opto/loopTransform.cpp
+++ b/src/hotspot/share/opto/loopTransform.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1234,7 +1234,7 @@ bool IdealLoopTree::policy_range_check(PhaseIdealLoop* phase, bool provisional,
continue;
}
if (!bol->is_Bool()) {
- assert(bol->is_OpaqueNotNull() ||
+ assert(bol->is_OpaqueConstantBool() ||
bol->is_OpaqueTemplateAssertionPredicate() ||
bol->is_OpaqueInitializedAssertionPredicate() ||
bol->is_OpaqueMultiversioning(),
diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp
index fab354e3e3d..d68505836d4 100644
--- a/src/hotspot/share/opto/loopnode.cpp
+++ b/src/hotspot/share/opto/loopnode.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -3752,6 +3752,20 @@ void CountedLoopEndNode::dump_spec(outputStream *st) const {
}
#endif
+IdealLoopTree::IdealLoopTree(PhaseIdealLoop* phase, Node* head, Node* tail): _parent(nullptr), _next(nullptr), _child(nullptr),
+ _head(head), _tail(tail),
+ _phase(phase),
+ _local_loop_unroll_limit(0), _local_loop_unroll_factor(0),
+ _body(phase->arena()),
+ _nest(0), _irreducible(0), _has_call(0), _has_sfpt(0), _rce_candidate(0),
+ _has_range_checks(0), _has_range_checks_computed(0),
+ _safepts(nullptr),
+ _required_safept(nullptr),
+ _allow_optimizations(true) {
+ precond(_head != nullptr);
+ precond(_tail != nullptr);
+}
+
//=============================================================================
//------------------------------is_member--------------------------------------
// Is 'l' a member of 'this'?
@@ -5089,8 +5103,8 @@ void PhaseIdealLoop::build_and_optimize() {
// Since nodes do not have a slot for immediate dominator, make
// a persistent side array for that info indexed on node->_idx.
_idom_size = C->unique();
- _idom = NEW_RESOURCE_ARRAY( Node*, _idom_size );
- _dom_depth = NEW_RESOURCE_ARRAY( uint, _idom_size );
+ _idom = NEW_ARENA_ARRAY(&_arena, Node*, _idom_size);
+ _dom_depth = NEW_ARENA_ARRAY(&_arena, uint, _idom_size);
_dom_stk = nullptr; // Allocated on demand in recompute_dom_depth
memset( _dom_depth, 0, _idom_size * sizeof(uint) );
@@ -5691,8 +5705,8 @@ void PhaseIdealLoop::set_idom(Node* d, Node* n, uint dom_depth) {
uint idx = d->_idx;
if (idx >= _idom_size) {
uint newsize = next_power_of_2(idx);
- _idom = REALLOC_RESOURCE_ARRAY( Node*, _idom,_idom_size,newsize);
- _dom_depth = REALLOC_RESOURCE_ARRAY( uint, _dom_depth,_idom_size,newsize);
+ _idom = REALLOC_ARENA_ARRAY(&_arena, Node*, _idom,_idom_size,newsize);
+ _dom_depth = REALLOC_ARENA_ARRAY(&_arena, uint, _dom_depth,_idom_size,newsize);
memset( _dom_depth + _idom_size, 0, (newsize - _idom_size) * sizeof(uint) );
_idom_size = newsize;
}
diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp
index 24976d76a51..5b06f0555ab 100644
--- a/src/hotspot/share/opto/loopnode.hpp
+++ b/src/hotspot/share/opto/loopnode.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -669,21 +669,7 @@ public:
Node_List* _required_safept; // A inner loop cannot delete these safepts;
bool _allow_optimizations; // Allow loop optimizations
- IdealLoopTree( PhaseIdealLoop* phase, Node *head, Node *tail )
- : _parent(nullptr), _next(nullptr), _child(nullptr),
- _head(head), _tail(tail),
- _phase(phase),
- _local_loop_unroll_limit(0), _local_loop_unroll_factor(0),
- _body(Compile::current()->comp_arena()),
- _nest(0), _irreducible(0), _has_call(0), _has_sfpt(0), _rce_candidate(0),
- _has_range_checks(0), _has_range_checks_computed(0),
- _safepts(nullptr),
- _required_safept(nullptr),
- _allow_optimizations(true)
- {
- precond(_head != nullptr);
- precond(_tail != nullptr);
- }
+ IdealLoopTree(PhaseIdealLoop* phase, Node* head, Node* tail);
// Is 'l' a member of 'this'?
bool is_member(const IdealLoopTree *l) const; // Test for nested membership
@@ -889,6 +875,8 @@ class PhaseIdealLoop : public PhaseTransform {
friend class ShenandoahBarrierC2Support;
friend class AutoNodeBudget;
+ Arena _arena; // For data whose lifetime is a single pass of loop optimizations
+
// Map loop membership for CFG nodes, and ctrl for non-CFG nodes.
//
// Exception: dead CFG nodes may instead have a ctrl/idom forwarding
@@ -1049,6 +1037,8 @@ public:
PhaseIterGVN &igvn() const { return _igvn; }
+ Arena* arena() { return &_arena; };
+
bool has_node(const Node* n) const {
guarantee(n != nullptr, "No Node.");
return _loop_or_ctrl[n->_idx] != nullptr;
@@ -1223,7 +1213,8 @@ private:
// Compute the Ideal Node to Loop mapping
PhaseIdealLoop(PhaseIterGVN& igvn, LoopOptsMode mode) :
PhaseTransform(Ideal_Loop),
- _loop_or_ctrl(igvn.C->comp_arena()),
+ _arena(mtCompiler, Arena::Tag::tag_idealloop),
+ _loop_or_ctrl(&_arena),
_igvn(igvn),
_verify_me(nullptr),
_verify_only(false),
@@ -1238,7 +1229,8 @@ private:
// or only verify that the graph is valid if verify_me is null.
PhaseIdealLoop(PhaseIterGVN& igvn, const PhaseIdealLoop* verify_me = nullptr) :
PhaseTransform(Ideal_Loop),
- _loop_or_ctrl(igvn.C->comp_arena()),
+ _arena(mtCompiler, Arena::Tag::tag_idealloop),
+ _loop_or_ctrl(&_arena),
_igvn(igvn),
_verify_me(verify_me),
_verify_only(verify_me == nullptr),
diff --git a/src/hotspot/share/opto/loopopts.cpp b/src/hotspot/share/opto/loopopts.cpp
index 1855263539b..2b48780f9b0 100644
--- a/src/hotspot/share/opto/loopopts.cpp
+++ b/src/hotspot/share/opto/loopopts.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1704,7 +1704,7 @@ void PhaseIdealLoop::try_sink_out_of_loop(Node* n) {
!n->is_Proj() &&
!n->is_MergeMem() &&
!n->is_CMove() &&
- !n->is_OpaqueNotNull() &&
+ !n->is_OpaqueConstantBool() &&
!n->is_OpaqueInitializedAssertionPredicate() &&
!n->is_OpaqueTemplateAssertionPredicate() &&
!is_raw_to_oop_cast && // don't extend live ranges of raw oops
@@ -2045,14 +2045,14 @@ Node* PhaseIdealLoop::clone_iff(PhiNode* phi) {
if (b->is_Phi()) {
_igvn.replace_input_of(phi, i, clone_iff(b->as_Phi()));
} else {
- assert(b->is_Bool() || b->is_OpaqueNotNull() || b->is_OpaqueInitializedAssertionPredicate(),
- "bool, non-null check with OpaqueNotNull or Initialized Assertion Predicate with its Opaque node");
+ assert(b->is_Bool() || b->is_OpaqueConstantBool() || b->is_OpaqueInitializedAssertionPredicate(),
+ "bool, non-null check with OpaqueConstantBool or Initialized Assertion Predicate with its Opaque node");
}
}
Node* n = phi->in(1);
Node* sample_opaque = nullptr;
Node *sample_bool = nullptr;
- if (n->is_OpaqueNotNull() || n->is_OpaqueInitializedAssertionPredicate()) {
+ if (n->is_OpaqueConstantBool() || n->is_OpaqueInitializedAssertionPredicate()) {
sample_opaque = n;
sample_bool = n->in(1);
assert(sample_bool->is_Bool(), "wrong type");
@@ -2228,7 +2228,7 @@ void PhaseIdealLoop::clone_loop_handle_data_uses(Node* old, Node_List &old_new,
// split if to break.
assert(!use->is_OpaqueTemplateAssertionPredicate(),
"should not clone a Template Assertion Predicate which should be removed once it's useless");
- if (use->is_If() || use->is_CMove() || use->is_OpaqueNotNull() || use->is_OpaqueInitializedAssertionPredicate() ||
+ if (use->is_If() || use->is_CMove() || use->is_OpaqueConstantBool() || use->is_OpaqueInitializedAssertionPredicate() ||
(use->Opcode() == Op_AllocateArray && use->in(AllocateNode::ValidLengthTest) == old)) {
// Since this code is highly unlikely, we lazily build the worklist
// of such Nodes to go split.
diff --git a/src/hotspot/share/opto/macro.cpp b/src/hotspot/share/opto/macro.cpp
index 9470001b2d2..ef38a511a88 100644
--- a/src/hotspot/share/opto/macro.cpp
+++ b/src/hotspot/share/opto/macro.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -2499,7 +2499,7 @@ void PhaseMacroExpand::eliminate_macro_nodes() {
assert(n->Opcode() == Op_LoopLimit ||
n->Opcode() == Op_ModD ||
n->Opcode() == Op_ModF ||
- n->is_OpaqueNotNull() ||
+ n->is_OpaqueConstantBool() ||
n->is_OpaqueInitializedAssertionPredicate() ||
n->Opcode() == Op_MaxL ||
n->Opcode() == Op_MinL ||
@@ -2547,14 +2547,14 @@ void PhaseMacroExpand::eliminate_opaque_looplimit_macro_nodes() {
} else if (n->is_Opaque1()) {
_igvn.replace_node(n, n->in(1));
success = true;
- } else if (n->is_OpaqueNotNull()) {
- // Tests with OpaqueNotNull nodes are implicitly known to be true. Replace the node with true. In debug builds,
+ } else if (n->is_OpaqueConstantBool()) {
+ // Tests with OpaqueConstantBool nodes are implicitly known. Replace the node with true/false. In debug builds,
// we leave the test in the graph to have an additional sanity check at runtime. If the test fails (i.e. a bug),
// we will execute a Halt node.
#ifdef ASSERT
_igvn.replace_node(n, n->in(1));
#else
- _igvn.replace_node(n, _igvn.intcon(1));
+ _igvn.replace_node(n, _igvn.intcon(n->as_OpaqueConstantBool()->constant()));
#endif
success = true;
} else if (n->is_OpaqueInitializedAssertionPredicate()) {
diff --git a/src/hotspot/share/opto/node.hpp b/src/hotspot/share/opto/node.hpp
index 5ddc4236b3e..e65578924d1 100644
--- a/src/hotspot/share/opto/node.hpp
+++ b/src/hotspot/share/opto/node.hpp
@@ -143,7 +143,7 @@ class Opaque1Node;
class OpaqueLoopInitNode;
class OpaqueLoopStrideNode;
class OpaqueMultiversioningNode;
-class OpaqueNotNullNode;
+class OpaqueConstantBoolNode;
class OpaqueInitializedAssertionPredicateNode;
class OpaqueTemplateAssertionPredicateNode;
class OuterStripMinedLoopNode;
@@ -818,7 +818,7 @@ public:
DEFINE_CLASS_ID(OpaqueLoopInit, Opaque1, 0)
DEFINE_CLASS_ID(OpaqueLoopStride, Opaque1, 1)
DEFINE_CLASS_ID(OpaqueMultiversioning, Opaque1, 2)
- DEFINE_CLASS_ID(OpaqueNotNull, Node, 17)
+ DEFINE_CLASS_ID(OpaqueConstantBool, Node, 17)
DEFINE_CLASS_ID(OpaqueInitializedAssertionPredicate, Node, 18)
DEFINE_CLASS_ID(OpaqueTemplateAssertionPredicate, Node, 19)
DEFINE_CLASS_ID(Move, Node, 20)
@@ -1000,7 +1000,7 @@ public:
DEFINE_CLASS_QUERY(NegV)
DEFINE_CLASS_QUERY(NeverBranch)
DEFINE_CLASS_QUERY(Opaque1)
- DEFINE_CLASS_QUERY(OpaqueNotNull)
+ DEFINE_CLASS_QUERY(OpaqueConstantBool)
DEFINE_CLASS_QUERY(OpaqueInitializedAssertionPredicate)
DEFINE_CLASS_QUERY(OpaqueTemplateAssertionPredicate)
DEFINE_CLASS_QUERY(OpaqueLoopInit)
diff --git a/src/hotspot/share/opto/opaquenode.cpp b/src/hotspot/share/opto/opaquenode.cpp
index 29d625db190..428379e84ae 100644
--- a/src/hotspot/share/opto/opaquenode.cpp
+++ b/src/hotspot/share/opto/opaquenode.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2014, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -108,10 +108,16 @@ void OpaqueMultiversioningNode::dump_spec(outputStream *st) const {
}
#endif
-const Type* OpaqueNotNullNode::Value(PhaseGVN* phase) const {
+const Type* OpaqueConstantBoolNode::Value(PhaseGVN* phase) const {
return phase->type(in(1));
}
+#ifndef PRODUCT
+void OpaqueConstantBoolNode::dump_spec(outputStream *st) const {
+ st->print(_constant ? " #true" : " #false");
+}
+#endif
+
OpaqueTemplateAssertionPredicateNode::OpaqueTemplateAssertionPredicateNode(BoolNode* bol, CountedLoopNode* loop_node)
: Node(nullptr, bol),
_loop_node(loop_node),
diff --git a/src/hotspot/share/opto/opaquenode.hpp b/src/hotspot/share/opto/opaquenode.hpp
index 9edfe2a7258..bb3da2aa65f 100644
--- a/src/hotspot/share/opto/opaquenode.hpp
+++ b/src/hotspot/share/opto/opaquenode.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2014, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -130,15 +130,26 @@ public:
// This node is used in the context of intrinsics. We sometimes implicitly know that an object is non-null even though
// the compiler cannot prove it. We therefore add a corresponding cast to propagate this implicit knowledge. However,
// this cast could become top during optimizations (input to cast becomes null) and the data path is folded. To ensure
-// that the control path is also properly folded, we insert an If node with a OpaqueNotNullNode as condition. During
-// macro expansion, we replace the OpaqueNotNullNodes with true in product builds such that the actually unneeded checks
-// are folded and do not end up in the emitted code. In debug builds, we keep the actual checks as additional
-// verification code (i.e. removing OpaqueNotNullNodes and use the BoolNode inputs instead). For more details, also see
-// GraphKit::must_be_not_null().
-class OpaqueNotNullNode : public Node {
+// that the control path is also properly folded, we insert an If node with a OpaqueConstantBoolNode as condition.
+// During macro expansion, we replace the OpaqueConstantBoolNodes with true in product builds such that the actually
+// unneeded checks are folded and do not end up in the emitted code. In debug builds, we keep the actual checks as
+// additional verification code (i.e. removing OpaqueConstantBoolNodes and use the BoolNode inputs instead). For more
+// details, also see GraphKit::must_be_not_null().
+// Similarly, sometimes we know that a size or limit guard is checked (e.g. there is already a guard in the caller) but
+// the compiler cannot prove it. We could in principle avoid adding a guard in the intrinsic but in some cases (e.g.
+// when the input is a constant that breaks the guard and the caller guard is not inlined) the input of the intrinsic
+// can become top and the data path is folded. To ensure that the control path is also properly folded, we insert an
+// OpaqueConstantBoolNode before the If node in the guard. During macro expansion, we replace the OpaqueConstantBoolNode
+// with false in product builds such that the actually unneeded guards are folded and do not end up in the emitted code.
+// In debug builds, we keep the actual checks as additional verification code (i.e. removing OpaqueConstantBoolNodes and
+// use the BoolNode inputs instead).
+class OpaqueConstantBoolNode : public Node {
+ private:
+ const bool _constant;
public:
- OpaqueNotNullNode(Compile* C, Node* tst) : Node(nullptr, tst) {
- init_class_id(Class_OpaqueNotNull);
+ OpaqueConstantBoolNode(Compile* C, Node* tst, bool constant) : Node(nullptr, tst), _constant(constant) {
+ assert(tst->is_Bool() || tst->is_Con(), "Test node must be a BoolNode or a constant");
+ init_class_id(Class_OpaqueConstantBool);
init_flags(Flag_is_macro);
C->add_macro_node(this);
}
@@ -146,6 +157,9 @@ class OpaqueNotNullNode : public Node {
virtual int Opcode() const;
virtual const Type* Value(PhaseGVN* phase) const;
virtual const Type* bottom_type() const { return TypeInt::BOOL; }
+ int constant() const { return _constant ? 1 : 0; }
+ virtual uint size_of() const { return sizeof(OpaqueConstantBoolNode); }
+ NOT_PRODUCT(void dump_spec(outputStream* st) const);
};
// This node is used for Template Assertion Predicate BoolNodes. A Template Assertion Predicate is always removed
diff --git a/src/hotspot/share/opto/phaseX.cpp b/src/hotspot/share/opto/phaseX.cpp
index 52badca8050..ce24c46590d 100644
--- a/src/hotspot/share/opto/phaseX.cpp
+++ b/src/hotspot/share/opto/phaseX.cpp
@@ -532,6 +532,10 @@ void PhaseValues::init_con_caches() {
memset(_zcons,0,sizeof(_zcons));
}
+PhaseIterGVN* PhaseValues::is_IterGVN() {
+ return (_phase == PhaseValuesType::iter_gvn || _phase == PhaseValuesType::ccp) ? static_cast(this) : nullptr;
+}
+
//--------------------------------find_int_type--------------------------------
const TypeInt* PhaseValues::find_int_type(Node* n) {
if (n == nullptr) return nullptr;
@@ -812,7 +816,7 @@ void PhaseGVN::dump_infinite_loop_info(Node* n, const char* where) {
PhaseIterGVN::PhaseIterGVN(PhaseIterGVN* igvn) : _delay_transform(igvn->_delay_transform),
_worklist(*C->igvn_worklist())
{
- _iterGVN = true;
+ _phase = PhaseValuesType::iter_gvn;
assert(&_worklist == &igvn->_worklist, "sanity");
}
@@ -821,7 +825,7 @@ PhaseIterGVN::PhaseIterGVN(PhaseIterGVN* igvn) : _delay_transform(igvn->_delay_t
PhaseIterGVN::PhaseIterGVN() : _delay_transform(false),
_worklist(*C->igvn_worklist())
{
- _iterGVN = true;
+ _phase = PhaseValuesType::iter_gvn;
uint max;
// Dead nodes in the hash table inherited from GVN were not treated as
@@ -1090,16 +1094,28 @@ void PhaseIterGVN::verify_optimize() {
is_verify_invariants()) {
ResourceMark rm;
Unique_Node_List worklist;
- bool failure = false;
// BFS all nodes, starting at root
worklist.push(C->root());
for (uint j = 0; j < worklist.size(); ++j) {
Node* n = worklist.at(j);
- if (is_verify_Value()) { failure |= verify_Value_for(n); }
- if (is_verify_Ideal()) { failure |= verify_Ideal_for(n, false); }
- if (is_verify_Ideal()) { failure |= verify_Ideal_for(n, true); }
- if (is_verify_Identity()) { failure |= verify_Identity_for(n); }
- if (is_verify_invariants()) { failure |= verify_node_invariants_for(n); }
+ // If we get an assert here, check why the reported node was not processed again in IGVN.
+ // We should either make sure that this node is properly added back to the IGVN worklist
+ // in PhaseIterGVN::add_users_to_worklist to update it again or add an exception
+ // in the verification methods below if that is not possible for some reason (like Load nodes).
+ if (is_verify_Value()) {
+ verify_Value_for(n);
+ }
+ if (is_verify_Ideal()) {
+ verify_Ideal_for(n, false);
+ verify_Ideal_for(n, true);
+ }
+ if (is_verify_Identity()) {
+ verify_Identity_for(n);
+ }
+ if (is_verify_invariants()) {
+ verify_node_invariants_for(n);
+ }
+
// traverse all inputs and outputs
for (uint i = 0; i < n->req(); i++) {
if (n->in(i) != nullptr) {
@@ -1110,11 +1126,6 @@ void PhaseIterGVN::verify_optimize() {
worklist.push(n->fast_out(i));
}
}
- // If we get this assert, check why the reported nodes were not processed again in IGVN.
- // We should either make sure that these nodes are properly added back to the IGVN worklist
- // in PhaseIterGVN::add_users_to_worklist to update them again or add an exception
- // in the verification code above if that is not possible for some reason (like Load nodes).
- assert(!failure, "Missed optimization opportunity/broken graph in PhaseIterGVN");
}
verify_empty_worklist(nullptr);
@@ -1139,18 +1150,18 @@ void PhaseIterGVN::verify_empty_worklist(Node* node) {
assert(false, "igvn worklist must still be empty after verify");
}
-// Check that type(n) == n->Value(), return true if we have a failure.
+// Check that type(n) == n->Value(), asserts if we have a failure.
// We have a list of exceptions, see detailed comments in code.
// (1) Integer "widen" changes, but the range is the same.
// (2) LoadNode performs deep traversals. Load is not notified for changes far away.
// (3) CmpPNode performs deep traversals if it compares oopptr. CmpP is not notified for changes far away.
-bool PhaseIterGVN::verify_Value_for(Node* n, bool strict) {
+void PhaseIterGVN::verify_Value_for(const Node* n, bool strict) {
// If we assert inside type(n), because the type is still a null, then maybe
// the node never went through gvn.transform, which would be a bug.
const Type* told = type(n);
const Type* tnew = n->Value(this);
if (told == tnew) {
- return false;
+ return;
}
// Exception (1)
// Integer "widen" changes, but range is the same.
@@ -1159,7 +1170,7 @@ bool PhaseIterGVN::verify_Value_for(Node* n, bool strict) {
const TypeInteger* t1 = tnew->is_integer(tnew->basic_type());
if (t0->lo_as_long() == t1->lo_as_long() &&
t0->hi_as_long() == t1->hi_as_long()) {
- return false; // ignore integer widen
+ return; // ignore integer widen
}
}
// Exception (2)
@@ -1168,7 +1179,7 @@ bool PhaseIterGVN::verify_Value_for(Node* n, bool strict) {
// MemNode::can_see_stored_value looks up through many memory nodes,
// which means we would need to notify modifications from far up in
// the inputs all the way down to the LoadNode. We don't do that.
- return false;
+ return;
}
// Exception (3)
// CmpPNode performs deep traversals if it compares oopptr. CmpP is not notified for changes far away.
@@ -1184,10 +1195,10 @@ bool PhaseIterGVN::verify_Value_for(Node* n, bool strict) {
// control sub of the allocation. The problems is that sometimes dominates answers
// false conservatively, and later it can determine that it is indeed true. Loops with
// Region heads can lead to giving up, whereas LoopNodes can be skipped easier, and
- // so the traversal becomes more powerful. This is difficult to remidy, we would have
+ // so the traversal becomes more powerful. This is difficult to remedy, we would have
// to notify the CmpP of CFG updates. Luckily, we recompute CmpP::Value during CCP
// after loop-opts, so that should take care of many of these cases.
- return false;
+ return;
}
stringStream ss; // Print as a block without tty lock.
@@ -1201,13 +1212,24 @@ bool PhaseIterGVN::verify_Value_for(Node* n, bool strict) {
tnew->dump_on(&ss);
ss.cr();
tty->print_cr("%s", ss.as_string());
- return true;
+
+ switch (_phase) {
+ case PhaseValuesType::iter_gvn:
+ assert(false, "Missed Value optimization opportunity in PhaseIterGVN for %s",n->Name());
+ break;
+ case PhaseValuesType::ccp:
+ assert(false, "PhaseCCP not at fixpoint: analysis result may be unsound for %s", n->Name());
+ break;
+ default:
+ assert(false, "Unexpected phase");
+ break;
+ }
}
// Check that all Ideal optimizations that could be done were done.
-// Returns true if it found missed optimization opportunities and
-// false otherwise (no missed optimization, or skipped verification).
-bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
+// Asserts if it found missed optimization opportunities or encountered unexpected changes, and
+// returns normally otherwise (no missed optimization, or skipped verification).
+void PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
// First, we check a list of exceptions, where we skip verification,
// because there are known cases where Ideal can optimize after IGVN.
// Some may be expected and cannot be fixed, and others should be fixed.
@@ -1221,7 +1243,7 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
// Found with:
// java -XX:VerifyIterativeGVN=0100 -Xbatch --version
case Op_RangeCheck:
- return false;
+ return;
// IfNode::Ideal does:
// Node* prev_dom = search_identical(dist, igvn);
@@ -1232,7 +1254,7 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
// Found with:
// java -XX:VerifyIterativeGVN=0100 -Xcomp --version
case Op_If:
- return false;
+ return;
// IfNode::simple_subsuming
// Looks for dominating test that subsumes the current test.
@@ -1242,7 +1264,7 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
// runtime/exceptionMsgs/ArrayIndexOutOfBoundsException/ArrayIndexOutOfBoundsExceptionTest.java#id1
// -XX:VerifyIterativeGVN=1110
case Op_CountedLoopEnd:
- return false;
+ return;
// LongCountedLoopEndNode::Ideal
// Probably same issue as above.
@@ -1251,7 +1273,7 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
// compiler/predicates/assertion/TestAssertionPredicates.java#NoLoopPredicationXbatch
// -XX:StressLongCountedLoop=2000000 -XX:+IgnoreUnrecognizedVMOptions -XX:VerifyIterativeGVN=1110
case Op_LongCountedLoopEnd:
- return false;
+ return;
// RegionNode::Ideal does "Skip around the useless IF diamond".
// 245 IfTrue === 244
@@ -1270,7 +1292,7 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
// Found with:
// java -XX:VerifyIterativeGVN=0100 -Xcomp --version
case Op_Region:
- return false;
+ return;
// In AddNode::Ideal, we call "commute", which swaps the inputs so
// that smaller idx are first. Tracking it back, it led me to
@@ -1349,7 +1371,7 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
case Op_MulHF:
case Op_MaxHF:
case Op_MinHF:
- return false;
+ return;
// In MulNode::Ideal the edges can be swapped to help value numbering:
//
@@ -1373,7 +1395,7 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
// compiler/intrinsics/bigInteger/MontgomeryMultiplyTest.java
// -XX:VerifyIterativeGVN=1110
case Op_AndL:
- return false;
+ return;
// SubLNode::Ideal does transform like:
// Convert "c1 - (y+c0)" into "(c1-c0) - y"
@@ -1405,7 +1427,7 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
// Found with:
// java -XX:VerifyIterativeGVN=0100 -Xcomp --version
case Op_SubL:
- return false;
+ return;
// SubINode::Ideal does
// Convert "x - (y+c0)" into "(x-y) - c0" AND
@@ -1417,7 +1439,7 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
// test/hotspot/jtreg/compiler/c2/IVTest.java
// -XX:VerifyIterativeGVN=1110
case Op_SubI:
- return false;
+ return;
// AddNode::IdealIL does transform like:
// Convert x + (con - y) into "(x - y) + con"
@@ -1446,7 +1468,7 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
// Found with:
// java -XX:VerifyIterativeGVN=0100 -Xcomp --version
case Op_AddL:
- return false;
+ return;
// SubTypeCheckNode::Ideal calls SubTypeCheckNode::verify_helper, which does
// Node* cmp = phase->transform(new CmpPNode(subklass, in(SuperKlass)));
@@ -1471,7 +1493,7 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
// Found with:
// java -XX:VerifyIterativeGVN=0100 -Xbatch --version
case Op_SubTypeCheck:
- return false;
+ return;
// LoopLimitNode::Ideal when stride is constant power-of-2, we can do a lowering
// to other nodes: Conv, Add, Sub, Mul, And ...
@@ -1491,7 +1513,7 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
// Fond with:
// java -XX:VerifyIterativeGVN=0100 -Xcomp --version
case Op_LoopLimit:
- return false;
+ return;
// PhiNode::Ideal calls split_flow_path, which tries to do this:
// "This optimization tries to find two or more inputs of phi with the same constant
@@ -1514,7 +1536,7 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
// Found with:
// java -XX:VerifyIterativeGVN=0100 -Xcomp --version
case Op_Phi:
- return false;
+ return;
// MemBarNode::Ideal does "Eliminate volatile MemBars for scalar replaced objects".
// For examle "The allocated object does not escape".
@@ -1527,7 +1549,7 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
// Found with:
// java -XX:VerifyIterativeGVN=0100 -Xcomp --version
case Op_MemBarStoreStore:
- return false;
+ return;
// ConvI2LNode::Ideal converts
// 648 AddI === _ 583 645 [[ 661 ]]
@@ -1543,7 +1565,7 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
// Found with:
// java -XX:VerifyIterativeGVN=0100 -Xcomp --version
case Op_ConvI2L:
- return false;
+ return;
// AddNode::IdealIL can do this transform (and similar other ones):
// Convert "a*b+a*c into a*(b+c)
@@ -1557,7 +1579,7 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
// test/hotspot/jtreg/compiler/loopopts/superword/ReductionPerf.java
// -XX:VerifyIterativeGVN=1110
case Op_AddI:
- return false;
+ return;
// ArrayCopyNode::Ideal
// calls ArrayCopyNode::prepare_array_copy
@@ -1583,7 +1605,7 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
// compiler/arraycopy/TestArrayCopyAsLoadsStores.java
// -XX:VerifyIterativeGVN=1110
case Op_ArrayCopy:
- return false;
+ return;
// CastLLNode::Ideal
// calls ConstraintCastNode::optimize_integer_cast -> pushes CastLL through SubL
@@ -1595,7 +1617,7 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
// compiler/c2/TestMergeStoresMemorySegment.java#byte-array
// -XX:VerifyIterativeGVN=1110
case Op_CastLL:
- return false;
+ return;
// Similar case happens to CastII
//
@@ -1603,7 +1625,7 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
// compiler/c2/TestScalarReplacementMaxLiveNodes.java
// -XX:VerifyIterativeGVN=1110
case Op_CastII:
- return false;
+ return;
// MaxLNode::Ideal
// calls AddNode::Ideal
@@ -1618,7 +1640,7 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
// -XX:VerifyIterativeGVN=1110
case Op_MaxL:
case Op_MinL:
- return false;
+ return;
// OrINode::Ideal
// calls AddNode::Ideal
@@ -1632,7 +1654,7 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
// -XX:VerifyIterativeGVN=1110
case Op_OrI:
case Op_OrL:
- return false;
+ return;
// Bool -> constant folded to 1.
// Issue with notification?
@@ -1641,7 +1663,7 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
// compiler/c2/irTests/TestVectorizationMismatchedAccess.java
// -XX:VerifyIterativeGVN=1110
case Op_Bool:
- return false;
+ return;
// LShiftLNode::Ideal
// Looks at pattern: "(x + x) << c0", converts it to "x << (c0 + 1)"
@@ -1651,7 +1673,7 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
// compiler/conversions/TestMoveConvI2LOrCastIIThruAddIs.java
// -ea -esa -XX:CompileThreshold=100 -XX:+UnlockExperimentalVMOptions -server -XX:-TieredCompilation -XX:+IgnoreUnrecognizedVMOptions -XX:VerifyIterativeGVN=1110
case Op_LShiftL:
- return false;
+ return;
// LShiftINode::Ideal
// pattern: ((x + con1) << con2) -> x << con2 + con1 << con2
@@ -1664,18 +1686,18 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
// compiler/escapeAnalysis/Test6689060.java
// -XX:+IgnoreUnrecognizedVMOptions -XX:VerifyIterativeGVN=1110 -ea -esa -XX:CompileThreshold=100 -XX:+UnlockExperimentalVMOptions -server -XX:-TieredCompilation -XX:+IgnoreUnrecognizedVMOptions -XX:VerifyIterativeGVN=1110
case Op_LShiftI:
- return false;
+ return;
// AddPNode::Ideal seems to do set_req without removing lock first.
// Found with various vector tests tier1-tier3.
case Op_AddP:
- return false;
+ return;
// StrIndexOfNode::Ideal
// Found in tier1-3.
case Op_StrIndexOf:
case Op_StrIndexOfChar:
- return false;
+ return;
// StrEqualsNode::Identity
//
@@ -1684,7 +1706,7 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
// -XX:+UnlockExperimentalVMOptions -XX:LockingMode=1 -XX:+IgnoreUnrecognizedVMOptions -XX:VerifyIterativeGVN=1110
// Note: The -XX:LockingMode option is not available anymore.
case Op_StrEquals:
- return false;
+ return;
// AryEqNode::Ideal
// Not investigated. Reshapes itself and adds lots of nodes to the worklist.
@@ -1693,22 +1715,22 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
// vmTestbase/vm/mlvm/meth/stress/compiler/i2c_c2i/Test.java
// -XX:+UnlockDiagnosticVMOptions -XX:-TieredCompilation -XX:+StressUnstableIfTraps -XX:+IgnoreUnrecognizedVMOptions -XX:VerifyIterativeGVN=1110
case Op_AryEq:
- return false;
+ return;
// MergeMemNode::Ideal
// Found in tier1-3. Did not investigate further yet.
case Op_MergeMem:
- return false;
+ return;
// URShiftINode::Ideal
// Found in tier1-3. Did not investigate further yet.
case Op_URShiftI:
- return false;
+ return;
// CMoveINode::Ideal
// Found in tier1-3. Did not investigate further yet.
case Op_CMoveI:
- return false;
+ return;
// CmpPNode::Ideal calls isa_const_java_mirror
// and generates new constant nodes, even if no progress is made.
@@ -1719,14 +1741,14 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
// Found with:
// java -XX:VerifyIterativeGVN=1110 -Xcomp --version
case Op_CmpP:
- return false;
+ return;
// MinINode::Ideal
// Did not investigate, but there are some patterns that might
// need more notification.
case Op_MinI:
case Op_MaxI: // preemptively removed it as well.
- return false;
+ return;
}
if (n->is_Load()) {
@@ -1742,7 +1764,7 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
// Found with:
// test/hotspot/jtreg/compiler/arraycopy/TestCloneAccess.java
// -XX:VerifyIterativeGVN=1110
- return false;
+ return;
}
if (n->is_Store()) {
@@ -1756,7 +1778,7 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
//
// Found with:
// java -XX:VerifyIterativeGVN=0100 -Xcomp --version
- return false;
+ return;
}
if (n->is_Vector()) {
@@ -1775,7 +1797,7 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
// Found with:
// compiler/vectorapi/TestMaskedMacroLogicVector.java
// -XX:+IgnoreUnrecognizedVMOptions -XX:VerifyIterativeGVN=1110 -XX:+UseParallelGC -XX:+UseNUMA
- return false;
+ return;
}
if (n->is_Region()) {
@@ -1794,7 +1816,7 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
// Found with:
// compiler/eliminateAutobox/TestShortBoxing.java
// -ea -esa -XX:CompileThreshold=100 -XX:+UnlockExperimentalVMOptions -server -XX:-TieredCompilation -XX:+IgnoreUnrecognizedVMOptions -XX:VerifyIterativeGVN=1110
- return false;
+ return;
}
if (n->is_CallJava()) {
@@ -1826,13 +1848,19 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
// Found with:
// compiler/loopopts/superword/TestDependencyOffsets.java#vanilla-U
// -XX:+IgnoreUnrecognizedVMOptions -XX:VerifyIterativeGVN=1110
- return false;
+ return;
}
// The number of nodes shoud not increase.
uint old_unique = C->unique();
// The hash of a node should not change, this would indicate different inputs
uint old_hash = n->hash();
+ // Remove 'n' from hash table in case it gets modified. We want to avoid
+ // hitting the "Need to remove from hash before changing edges" assert if
+ // a change occurs. Instead, we would like to proceed with the optimization,
+ // return and finally hit the assert in PhaseIterGVN::verify_optimize to get
+ // a more meaningful message
+ _table.hash_delete(n);
Node* i = n->Ideal(this, can_reshape);
// If there was no new Idealization, we are probably happy.
if (i == nullptr) {
@@ -1843,7 +1871,7 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
ss.print_cr(" old_unique = %d, unique = %d", old_unique, C->unique());
n->dump_bfs(1, nullptr, "", &ss);
tty->print_cr("%s", ss.as_string());
- return true;
+ assert(false, "Unexpected new unused nodes from applying Ideal optimization on %s", n->Name());
}
if (old_hash != n->hash()) {
@@ -1853,13 +1881,14 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
ss.print_cr(" old_hash = %d, hash = %d", old_hash, n->hash());
n->dump_bfs(1, nullptr, "", &ss);
tty->print_cr("%s", ss.as_string());
- return true;
+ assert(false, "Unexpected hash change from applying Ideal optimization on %s", n->Name());
}
verify_empty_worklist(n);
// Everything is good.
- return false;
+ hash_find_insert(n);
+ return;
}
// We just saw a new Idealization which was not done during IGVN.
@@ -1876,13 +1905,14 @@ bool PhaseIterGVN::verify_Ideal_for(Node* n, bool can_reshape) {
ss.print_cr("The result after Ideal:");
i->dump_bfs(1, nullptr, "", &ss);
tty->print_cr("%s", ss.as_string());
- return true;
+
+ assert(false, "Missed Ideal optimization opportunity in PhaseIterGVN for %s", n->Name());
}
// Check that all Identity optimizations that could be done were done.
-// Returns true if it found missed optimization opportunities and
-// false otherwise (no missed optimization, or skipped verification).
-bool PhaseIterGVN::verify_Identity_for(Node* n) {
+// Asserts if it found missed optimization opportunities, and
+// returns normally otherwise (no missed optimization, or skipped verification).
+void PhaseIterGVN::verify_Identity_for(Node* n) {
// First, we check a list of exceptions, where we skip verification,
// because there are known cases where Ideal can optimize after IGVN.
// Some may be expected and cannot be fixed, and others should be fixed.
@@ -1901,7 +1931,7 @@ bool PhaseIterGVN::verify_Identity_for(Node* n) {
// Found with:
// java -XX:VerifyIterativeGVN=1000 -Xcomp --version
case Op_SafePoint:
- return false;
+ return;
// MergeMemNode::Identity replaces the MergeMem with its base_memory if it
// does not record any other memory splits.
@@ -1912,7 +1942,7 @@ bool PhaseIterGVN::verify_Identity_for(Node* n) {
// Found with:
// java -XX:VerifyIterativeGVN=1000 -Xcomp --version
case Op_MergeMem:
- return false;
+ return;
// ConstraintCastNode::Identity finds casts that are the same, except that
// the control is "higher up", i.e. dominates. The call goes via
@@ -1926,7 +1956,7 @@ bool PhaseIterGVN::verify_Identity_for(Node* n) {
case Op_CastPP:
case Op_CastII:
case Op_CastLL:
- return false;
+ return;
// Same issue for CheckCastPP, uses ConstraintCastNode::Identity and
// checks dominator, which may be changed, but too far up for notification
@@ -1936,7 +1966,7 @@ bool PhaseIterGVN::verify_Identity_for(Node* n) {
// compiler/c2/irTests/TestSkeletonPredicates.java
// -XX:VerifyIterativeGVN=1110
case Op_CheckCastPP:
- return false;
+ return;
// In SubNode::Identity, we do:
// Convert "(X+Y) - Y" into X and "(X+Y) - X" into Y
@@ -1954,7 +1984,7 @@ bool PhaseIterGVN::verify_Identity_for(Node* n) {
// java -XX:VerifyIterativeGVN=1000 -Xcomp --version
case Op_SubI:
case Op_SubL:
- return false;
+ return;
// PhiNode::Identity checks for patterns like:
// r = (x != con) ? x : con;
@@ -1968,7 +1998,7 @@ bool PhaseIterGVN::verify_Identity_for(Node* n) {
// test/hotspot/jtreg/gc/stress/gcbasher/TestGCBasherWithG1.java
// -XX:VerifyIterativeGVN=1110
case Op_Phi:
- return false;
+ return;
// ConvI2LNode::Identity does
// convert I2L(L2I(x)) => x
@@ -1979,7 +2009,7 @@ bool PhaseIterGVN::verify_Identity_for(Node* n) {
// compiler/loopopts/superword/TestDependencyOffsets.java#vanilla-A
// -XX:VerifyIterativeGVN=1110
case Op_ConvI2L:
- return false;
+ return;
// MaxNode::find_identity_operation
// Finds patterns like Max(A, Max(A, B)) -> Max(A, B)
@@ -1999,7 +2029,7 @@ bool PhaseIterGVN::verify_Identity_for(Node* n) {
case Op_MinHF:
case Op_MaxD:
case Op_MinD:
- return false;
+ return;
// AddINode::Identity
@@ -2013,12 +2043,12 @@ bool PhaseIterGVN::verify_Identity_for(Node* n) {
// -ea -esa -XX:CompileThreshold=100 -XX:+UnlockExperimentalVMOptions -server -XX:-TieredCompilation -XX:+IgnoreUnrecognizedVMOptions -XX:VerifyIterativeGVN=1110
case Op_AddI:
case Op_AddL:
- return false;
+ return;
// AbsINode::Identity
// Not investigated yet.
case Op_AbsI:
- return false;
+ return;
}
if (n->is_Load()) {
@@ -2032,7 +2062,7 @@ bool PhaseIterGVN::verify_Identity_for(Node* n) {
//
// Found with:
// java -XX:VerifyIterativeGVN=1000 -Xcomp --version
- return false;
+ return;
}
if (n->is_Store()) {
@@ -2043,20 +2073,20 @@ bool PhaseIterGVN::verify_Identity_for(Node* n) {
// Found with:
// applications/ctw/modules/java_base_2.java
// -ea -esa -XX:CompileThreshold=100 -XX:+UnlockExperimentalVMOptions -server -XX:-TieredCompilation -Djava.awt.headless=true -XX:+IgnoreUnrecognizedVMOptions -XX:VerifyIterativeGVN=1110
- return false;
+ return;
}
if (n->is_Vector()) {
// Found with tier1-3. Not investigated yet.
// The observed issue was with AndVNode::Identity
- return false;
+ return;
}
Node* i = n->Identity(this);
// If we cannot find any other Identity, we are happy.
if (i == n) {
verify_empty_worklist(n);
- return false;
+ return;
}
// The verification just found a new Identity that was not found during IGVN.
@@ -2068,11 +2098,12 @@ bool PhaseIterGVN::verify_Identity_for(Node* n) {
ss.print_cr("New node:");
i->dump_bfs(1, nullptr, "", &ss);
tty->print_cr("%s", ss.as_string());
- return true;
+
+ assert(false, "Missed Identity optimization opportunity in PhaseIterGVN for %s", n->Name());
}
// Some other verifications that are not specific to a particular transformation.
-bool PhaseIterGVN::verify_node_invariants_for(const Node* n) {
+void PhaseIterGVN::verify_node_invariants_for(const Node* n) {
if (n->is_AddP()) {
if (!n->as_AddP()->address_input_has_same_base()) {
stringStream ss; // Print as a block without tty lock.
@@ -2080,10 +2111,10 @@ bool PhaseIterGVN::verify_node_invariants_for(const Node* n) {
ss.print_cr("Base pointers must match for AddP chain:");
n->dump_bfs(2, nullptr, "", &ss);
tty->print_cr("%s", ss.as_string());
- return true;
+
+ assert(false, "Broken node invariant for %s", n->Name());
}
}
- return false;
}
#endif
@@ -2794,6 +2825,7 @@ uint PhaseCCP::_total_constants = 0;
PhaseCCP::PhaseCCP( PhaseIterGVN *igvn ) : PhaseIterGVN(igvn) {
NOT_PRODUCT( clear_constants(); )
assert( _worklist.size() == 0, "" );
+ _phase = PhaseValuesType::ccp;
analyze();
}
@@ -2914,17 +2946,18 @@ bool PhaseCCP::needs_revisit(Node* n) const {
// Note for CCP the non-convergence can lead to unsound analysis and mis-compilation.
// Therefore, we are verifying Value convergence strictly.
void PhaseCCP::verify_analyze(Unique_Node_List& worklist_verify) {
- bool failure = false;
while (worklist_verify.size()) {
Node* n = worklist_verify.pop();
- failure |= verify_Value_for(n, /* strict = */ true);
+
+ // An assert in verify_Value_for means that PhaseCCP is not at fixpoint
+ // and that the analysis result may be unsound.
+ // If this happens, check why the reported nodes were not processed again in CCP.
+ // We should either make sure that these nodes are properly added back to the CCP worklist
+ // in PhaseCCP::push_child_nodes_to_worklist() to update their type in the same round,
+ // or that they are added in PhaseCCP::needs_revisit() so that analysis revisits
+ // them at the end of the round.
+ verify_Value_for(n, true);
}
- // If we get this assert, check why the reported nodes were not processed again in CCP.
- // We should either make sure that these nodes are properly added back to the CCP worklist
- // in PhaseCCP::push_child_nodes_to_worklist() to update their type in the same round,
- // or that they are added in PhaseCCP::needs_revisit() so that analysis revisits
- // them at the end of the round.
- assert(!failure, "PhaseCCP not at fixpoint: analysis result may be unsound.");
}
#endif
diff --git a/src/hotspot/share/opto/phaseX.hpp b/src/hotspot/share/opto/phaseX.hpp
index 3f75aab8980..ce02f456c00 100644
--- a/src/hotspot/share/opto/phaseX.hpp
+++ b/src/hotspot/share/opto/phaseX.hpp
@@ -224,7 +224,13 @@ public:
// 3) NodeHash table, to find identical nodes (and remove/update the hash of a node on modification).
class PhaseValues : public PhaseTransform {
protected:
- bool _iterGVN;
+ enum class PhaseValuesType {
+ gvn,
+ iter_gvn,
+ ccp
+ };
+
+ PhaseValuesType _phase;
// Hash table for value-numbering. Reference to "C->node_hash()",
NodeHash &_table;
@@ -247,7 +253,7 @@ protected:
void init_con_caches();
public:
- PhaseValues() : PhaseTransform(GVN), _iterGVN(false),
+ PhaseValues() : PhaseTransform(GVN), _phase(PhaseValuesType::gvn),
_table(*C->node_hash()), _types(*C->types())
{
NOT_PRODUCT( clear_new_values(); )
@@ -256,7 +262,7 @@ public:
init_con_caches();
}
NOT_PRODUCT(~PhaseValues();)
- PhaseIterGVN* is_IterGVN() { return (_iterGVN) ? (PhaseIterGVN*)this : nullptr; }
+ PhaseIterGVN* is_IterGVN();
// Some Ideal and other transforms delete --> modify --> insert values
bool hash_delete(Node* n) { return _table.hash_delete(n); }
@@ -490,10 +496,10 @@ public:
void optimize();
#ifdef ASSERT
void verify_optimize();
- bool verify_Value_for(Node* n, bool strict = false);
- bool verify_Ideal_for(Node* n, bool can_reshape);
- bool verify_Identity_for(Node* n);
- bool verify_node_invariants_for(const Node* n);
+ void verify_Value_for(const Node* n, bool strict = false);
+ void verify_Ideal_for(Node* n, bool can_reshape);
+ void verify_Identity_for(Node* n);
+ void verify_node_invariants_for(const Node* n);
void verify_empty_worklist(Node* n);
#endif
diff --git a/src/hotspot/share/opto/runtime.cpp b/src/hotspot/share/opto/runtime.cpp
index 6dbbfb0a130..0d2dbb813bd 100644
--- a/src/hotspot/share/opto/runtime.cpp
+++ b/src/hotspot/share/opto/runtime.cpp
@@ -1865,6 +1865,8 @@ JRT_ENTRY_NO_ASYNC(address, OptoRuntime::handle_exception_C_helper(JavaThread* c
// has updated oops.
StackWatermarkSet::after_unwind(current);
+ MACOS_AARCH64_ONLY(os::thread_wx_enable_write());
+
// Do not confuse exception_oop with pending_exception. The exception_oop
// is only used to pass arguments into the method. Not for general
// exception handling. DO NOT CHANGE IT to use pending_exception, since
diff --git a/src/hotspot/share/opto/split_if.cpp b/src/hotspot/share/opto/split_if.cpp
index 425c0c9e99d..de9ddb60b0c 100644
--- a/src/hotspot/share/opto/split_if.cpp
+++ b/src/hotspot/share/opto/split_if.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -307,7 +307,7 @@ bool PhaseIdealLoop::clone_cmp_down(Node* n, const Node* blk1, const Node* blk2)
assert( bol->is_Bool(), "" );
if (bol->outcnt() == 1) {
Node* use = bol->unique_out();
- if (use->is_OpaqueNotNull() || use->is_OpaqueTemplateAssertionPredicate() ||
+ if (use->is_OpaqueConstantBool() || use->is_OpaqueTemplateAssertionPredicate() ||
use->is_OpaqueInitializedAssertionPredicate()) {
if (use->outcnt() == 1) {
Node* iff = use->unique_out();
@@ -331,8 +331,8 @@ bool PhaseIdealLoop::clone_cmp_down(Node* n, const Node* blk1, const Node* blk2)
// Recursively sink any BoolNode
for (DUIterator j = bol->outs(); bol->has_out(j); j++) {
Node* u = bol->out(j);
- // Uses are either IfNodes, CMoves, OpaqueNotNull, or Opaque*AssertionPredicate
- if (u->is_OpaqueNotNull() || u->is_OpaqueTemplateAssertionPredicate() ||
+ // Uses are either IfNodes, CMoves, OpaqueConstantBool or Opaque*AssertionPredicate
+ if (u->is_OpaqueConstantBool() || u->is_OpaqueTemplateAssertionPredicate() ||
u->is_OpaqueInitializedAssertionPredicate()) {
assert(u->in(1) == bol, "bad input");
for (DUIterator_Last kmin, k = u->last_outs(kmin); k >= kmin; --k) {
diff --git a/src/hotspot/share/opto/vectorIntrinsics.cpp b/src/hotspot/share/opto/vectorIntrinsics.cpp
index 883a0526053..9df10fefdd1 100644
--- a/src/hotspot/share/opto/vectorIntrinsics.cpp
+++ b/src/hotspot/share/opto/vectorIntrinsics.cpp
@@ -297,40 +297,63 @@ static bool is_klass_initialized(const TypeInstPtr* vec_klass) {
return klass->is_initialized();
}
-// public static
-// ,
-// M extends VectorMask,
-// E>
-// V unaryOp(int oprId, Class extends V> vmClass, Class extends M> maskClass, Class elementType,
-// int length, V v, M m,
-// UnaryOperation defaultImpl)
+static bool is_primitive_lane_type(VectorSupport::LaneType laneType) {
+ return laneType >= VectorSupport::LT_FLOAT && laneType <= VectorSupport::LT_LONG;
+}
+
+static BasicType get_vector_primitive_lane_type(VectorSupport::LaneType lane_type) {
+ assert(is_primitive_lane_type(lane_type), "");
+ switch (lane_type) {
+ case VectorSupport::LaneType::LT_FLOAT: return T_FLOAT;
+ case VectorSupport::LaneType::LT_DOUBLE: return T_DOUBLE;
+ case VectorSupport::LaneType::LT_LONG: return T_LONG;
+ case VectorSupport::LaneType::LT_INT: return T_INT;
+ case VectorSupport::LaneType::LT_SHORT: return T_SHORT;
+ case VectorSupport::LaneType::LT_BYTE: return T_BYTE;
+ }
+ return T_ILLEGAL;
+}
+
//
-// public static
-// ,
-// E>
-// V binaryOp(int oprId, Class extends V> vmClass, Class extends M> maskClass, Class elementType,
-// int length, V v1, V v2, M m,
-// BinaryOperation defaultImpl)
+// ,
+// M extends VectorMask,
+// E>
+// V unaryOp(int oprId,
+// Class extends V> vClass, Class extends M> mClass, int laneType,
+// int length,
+// V v, M m,
+// UnaryOperation defaultImpl) {
+//
+// ,
+// E>
+// VM binaryOp(int oprId,
+// Class extends VM> vmClass, Class extends M> mClass, int laneType,
+// int length,
+// VM v1, VM v2, M m,
+// BinaryOperation defaultImpl) {
+//
+//
+// ,
+// M extends VectorMask,
+// E>
+// V ternaryOp(int oprId,
+// Class extends V> vClass, Class extends M> mClass, int laneType,
+// int length,
+// V v1, V v2, V v3, M m,
+// TernaryOperation defaultImpl) {
//
-// public static
-// ,
-// M extends VectorMask,
-// E>
-// V ternaryOp(int oprId, Class extends V> vmClass, Class extends M> maskClass, Class elementType,
-// int length, V v1, V v2, V v3, M m,
-// TernaryOperation defaultImpl)
//
bool LibraryCallKit::inline_vector_nary_operation(int n) {
const TypeInt* opr = gvn().type(argument(0))->isa_int();
const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
- const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
+ const TypeInt* laneType = gvn().type(argument(3))->isa_int();
const TypeInt* vlen = gvn().type(argument(4))->isa_int();
if (opr == nullptr || !opr->is_con() ||
+ laneType == nullptr || !laneType->is_con() ||
vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
- elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
vlen == nullptr || !vlen->is_con()) {
log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
NodeClassNames[argument(0)->Opcode()],
@@ -340,11 +363,12 @@ bool LibraryCallKit::inline_vector_nary_operation(int n) {
return false; // not enough info for intrinsification
}
- ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
- if (!elem_type->is_primitive_type()) {
- log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
- return false; // should be primitive type
+ VectorSupport::LaneType vltype = static_cast(laneType->get_con());
+ if (!is_primitive_lane_type(vltype)) {
+ log_if_needed(" ** not a primitive lt=%s", VectorSupport::lanetype2name(vltype));
+ return false;
}
+
if (!is_klass_initialized(vector_klass)) {
log_if_needed(" ** klass argument not initialized");
return false;
@@ -371,12 +395,13 @@ bool LibraryCallKit::inline_vector_nary_operation(int n) {
}
}
- BasicType elem_bt = elem_type->basic_type();
bool has_scalar_op = VectorSupport::has_scalar_op(opr->get_con());
bool is_unsigned = VectorSupport::is_unsigned_op(opr->get_con());
int num_elem = vlen->get_con();
- int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
+ BasicType elem_bt = get_vector_primitive_lane_type(vltype);
+ int opc = VectorSupport::vop2ideal(opr->get_con(), vltype);
+
int sopc = has_scalar_op ? VectorNode::opcode(opc, elem_bt) : opc;
if (sopc == 0 || num_elem == 1) {
log_if_needed(" ** operation not supported: arity=%d opc=%s[%d] vlen=%d etype=%s",
@@ -487,29 +512,29 @@ bool LibraryCallKit::inline_vector_nary_operation(int n) {
return true;
}
-// public static
-// , E>
-// V libraryUnaryOp(long address, Class extends V> vClass, Class elementType, int length, String debugName,
-// V v,
-// UnaryOperation defaultImpl)
//
-// public static
-//
-// V libraryBinaryOp(long address, Class extends V> vClass, Class elementType, int length, String debugName,
-// V v1, V v2,
-// BinaryOperation defaultImpl)
+// , E>
+// V libraryUnaryOp(long addr, Class extends V> vClass, int laneType, int length, String debugName,
+// V v,
+// UnaryOperation defaultImpl)
+//
+//
+// V libraryBinaryOp(long addr, Class extends V> vClass, int laneType, int length, String debugName,
+// V v1, V v2,
+// BinaryOperation defaultImpl)
+//
bool LibraryCallKit::inline_vector_call(int arity) {
assert(Matcher::supports_vector_calling_convention(), "required");
const TypeLong* entry = gvn().type(argument(0))->isa_long();
const TypeInstPtr* vector_klass = gvn().type(argument(2))->isa_instptr();
- const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
+ const TypeInt* laneType = gvn().type(argument(3))->isa_int();
const TypeInt* vlen = gvn().type(argument(4))->isa_int();
const TypeInstPtr* debug_name_oop = gvn().type(argument(5))->isa_instptr();
if (entry == nullptr || !entry->is_con() ||
vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
- elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
+ laneType == nullptr || !laneType->is_con() ||
vlen == nullptr || !vlen->is_con() ||
debug_name_oop == nullptr || debug_name_oop->const_oop() == nullptr) {
log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s debug_name=%s",
@@ -526,18 +551,20 @@ bool LibraryCallKit::inline_vector_call(int arity) {
return false;
}
- ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
- if (!elem_type->is_primitive_type()) {
- log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
- return false; // should be primitive type
+ VectorSupport::LaneType vltype = static_cast(laneType->get_con());
+ if (!is_primitive_lane_type(vltype)) {
+ log_if_needed(" ** not a primitive lt=%s", VectorSupport::lanetype2name(vltype));
+ return false;
}
+
+
if (!is_klass_initialized(vector_klass)) {
log_if_needed(" ** klass argument not initialized");
return false;
}
- BasicType elem_bt = elem_type->basic_type();
int num_elem = vlen->get_con();
+ BasicType elem_bt = get_vector_primitive_lane_type(vltype);
if (!Matcher::vector_size_supported(elem_bt, num_elem)) {
log_if_needed(" ** vector size (vlen=%d, etype=%s) is not supported",
num_elem, type2name(elem_bt));
@@ -589,18 +616,24 @@ bool LibraryCallKit::inline_vector_call(int arity) {
return true;
}
-//
-// long maskReductionCoerced(int oper, Class extends M> maskClass, Class> elemClass,
-// int length, M m, VectorMaskOp defaultImpl)
+//
+// ,
+// E>
+// long maskReductionCoerced(int oper,
+// Class extends M> mClass, int laneType,
+// int length,
+// M m,
+// VectorMaskOp defaultImpl)
+//
bool LibraryCallKit::inline_vector_mask_operation() {
const TypeInt* oper = gvn().type(argument(0))->isa_int();
const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
- const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
+ const TypeInt* laneType = gvn().type(argument(2))->isa_int();
const TypeInt* vlen = gvn().type(argument(3))->isa_int();
Node* mask = argument(4);
if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
- elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
+ laneType == nullptr || !laneType->is_con() ||
vlen == nullptr || !vlen->is_con() ||
oper == nullptr || !oper->is_con() ||
mask->is_top()) {
@@ -612,11 +645,15 @@ bool LibraryCallKit::inline_vector_mask_operation() {
return false;
}
- int num_elem = vlen->get_con();
- ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
- BasicType elem_bt = elem_type->basic_type();
+ VectorSupport::LaneType vltype = static_cast(laneType->get_con());
+ if (!is_primitive_lane_type(vltype)) {
+ log_if_needed(" ** not a primitive lt=%s", VectorSupport::lanetype2name(vltype));
+ return false;
+ }
- int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
+ int num_elem = vlen->get_con();
+ BasicType elem_bt = get_vector_primitive_lane_type(vltype);
+ int mopc = VectorSupport::vop2ideal(oper->get_con(), vltype);
if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
mopc, num_elem, type2name(elem_bt));
@@ -646,16 +683,18 @@ bool LibraryCallKit::inline_vector_mask_operation() {
return true;
}
-// public static
-// ,
-// E>
-// M fromBitsCoerced(Class extends M> vmClass, Class elementType, int length,
-// long bits, int mode, S s,
-// BroadcastOperation defaultImpl)
+//
+// ,
+// E>
+// VM fromBitsCoerced(Class extends VM> vmClass, int laneType,
+// int length,
+// long bits, int mode, S s,
+// FromBitsCoercedOperation defaultImpl)
+//
bool LibraryCallKit::inline_vector_frombits_coerced() {
const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
- const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
+ const TypeInt* laneType = gvn().type(argument(1))->isa_int();
const TypeInt* vlen = gvn().type(argument(2))->isa_int();
const TypeLong* bits_type = gvn().type(argument(3))->isa_long();
// Mode argument determines the mode of operation it can take following values:-
@@ -664,7 +703,7 @@ bool LibraryCallKit::inline_vector_frombits_coerced() {
const TypeInt* mode = gvn().type(argument(5))->isa_int();
if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
- elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
+ laneType == nullptr || !laneType->is_con() ||
vlen == nullptr || !vlen->is_con() ||
bits_type == nullptr ||
mode == nullptr || !mode->is_con()) {
@@ -676,17 +715,19 @@ bool LibraryCallKit::inline_vector_frombits_coerced() {
return false; // not enough info for intrinsification
}
+ VectorSupport::LaneType vltype = static_cast(laneType->get_con());
+ if (!is_primitive_lane_type(vltype)) {
+ log_if_needed(" ** not a primitive lt=%s", VectorSupport::lanetype2name(vltype));
+ return false;
+ }
+
if (!is_klass_initialized(vector_klass)) {
log_if_needed(" ** klass argument not initialized");
return false;
}
- ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
- if (!elem_type->is_primitive_type()) {
- log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
- return false; // should be primitive type
- }
- BasicType elem_bt = elem_type->basic_type();
+
int num_elem = vlen->get_con();
+ BasicType elem_bt = get_vector_primitive_lane_type(vltype);
ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
@@ -776,34 +817,34 @@ static bool elem_consistent_with_arr(BasicType elem_bt, const TypeAryPtr* arr_ty
}
}
-// public static
+//
// >
-// VM load(Class extends VM> vmClass, Class eClass,
+// VM load(Class extends VM> vmClass, int laneType,
// int length,
-// Object base, long offset, // Unsafe addressing
-// boolean fromSegment,
-// C container, long index, S s, // Arguments for default implementation
-// LoadOperation defaultImpl) {
-// public static
+// Object base, long offset, boolean fromSegment,
+// C container, long index, S s,
+// LoadOperation defaultImpl)
+//
+//
//
-// void store(Class> vClass, Class> eClass,
+// void store(Class> vClass, int laneType,
// int length,
-// Object base, long offset, // Unsafe addressing
-// boolean fromSegment,
-// V v, C container, long index, // Arguments for default implementation
-// StoreVectorOperation defaultImpl) {
+// Object base, long offset, boolean fromSegment,
+// V v, C container, long index,
+// StoreVectorOperation defaultImpl)
+//
bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
- const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
+ const TypeInt* laneType = gvn().type(argument(1))->isa_int();
const TypeInt* vlen = gvn().type(argument(2))->isa_int();
const TypeInt* from_ms = gvn().type(argument(6))->isa_int();
if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
- elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
+ laneType == nullptr || !laneType->is_con() ||
vlen == nullptr || !vlen->is_con() ||
from_ms == nullptr || !from_ms->is_con()) {
log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
@@ -813,17 +854,19 @@ bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
NodeClassNames[argument(6)->Opcode()]);
return false; // not enough info for intrinsification
}
+
+ VectorSupport::LaneType vltype = static_cast(laneType->get_con());
+ if (!is_primitive_lane_type(vltype)) {
+ log_if_needed(" ** not a primitive lt=%s", VectorSupport::lanetype2name(vltype));
+ return false;
+ }
+
if (!is_klass_initialized(vector_klass)) {
log_if_needed(" ** klass argument not initialized");
return false;
}
- ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
- if (!elem_type->is_primitive_type()) {
- log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
- return false; // should be primitive type
- }
- BasicType elem_bt = elem_type->basic_type();
+ BasicType elem_bt = get_vector_primitive_lane_type(vltype);
int num_elem = vlen->get_con();
// TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
@@ -968,40 +1011,39 @@ bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
return true;
}
-// public static
+//
// ,
// E,
// S extends VectorSpecies,
// M extends VectorMask>
-// V loadMasked(Class extends V> vClass, Class mClass, Class eClass,
-// int length, Object base, long offset, // Unsafe addressing
-// boolean fromSegment,
+// V loadMasked(Class extends V> vClass, Class mClass, int laneType,
+// int length, Object base, long offset, boolean fromSegment,
// M m, int offsetInRange,
-// C container, long index, S s, // Arguments for default implementation
-// LoadVectorMaskedOperation defaultImpl) {
-// public static
+// C container, long index, S s,
+// LoadVectorMaskedOperation defaultImpl)
+//
// ,
// M extends VectorMask,
// E>
-// void storeMasked(Class extends V> vClass, Class mClass, Class eClass,
+// void storeMasked(Class extends V> vClass, Class mClass, int laneType,
// int length,
-// Object base, long offset, // Unsafe addressing
-// boolean fromSegment,
-// V v, M m, C container, long index, // Arguments for default implementation
-// StoreVectorMaskedOperation defaultImpl) {
+// Object base, long offset, boolean fromSegment,
+// V v, M m, C container, long index,
+// StoreVectorMaskedOperation defaultImpl)
+//
bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
- const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
+ const TypeInt* laneType = gvn().type(argument(2))->isa_int();
const TypeInt* vlen = gvn().type(argument(3))->isa_int();
const TypeInt* from_ms = gvn().type(argument(7))->isa_int();
if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
- elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
+ laneType == nullptr || !laneType->is_con() ||
vlen == nullptr || !vlen->is_con() ||
from_ms == nullptr || !from_ms->is_con()) {
log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
@@ -1012,6 +1054,13 @@ bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
NodeClassNames[argument(7)->Opcode()]);
return false; // not enough info for intrinsification
}
+
+ VectorSupport::LaneType vltype = static_cast(laneType->get_con());
+ if (!is_primitive_lane_type(vltype)) {
+ log_if_needed(" ** not a primitive lt=%s", VectorSupport::lanetype2name(vltype));
+ return false;
+ }
+
if (!is_klass_initialized(vector_klass)) {
log_if_needed(" ** klass argument not initialized");
return false;
@@ -1022,13 +1071,7 @@ bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
return false;
}
- ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
- if (!elem_type->is_primitive_type()) {
- log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
- return false; // should be primitive type
- }
-
- BasicType elem_bt = elem_type->basic_type();
+ BasicType elem_bt = get_vector_primitive_lane_type(vltype);
int num_elem = vlen->get_con();
Node* base = argument(4);
@@ -1199,35 +1242,37 @@ bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
// S extends VectorSpecies,
// M extends VectorMask,
// E>
-// V loadWithMap(Class extends V> vClass, Class mClass, Class eClass, int length,
-// Class extends Vector> vectorIndexClass, int indexLength,
-// Object base, long offset,
-// W indexVector1, W indexVector2, W indexVector3, W indexVector4,
-// M m, C container, int index, int[] indexMap, int indexM, S s,
-// LoadVectorOperationWithMap defaultImpl)
-//
+// V loadWithMap(Class extends V> vClass, Class mClass, int laneType,
+// int length,
+// Class extends Vector> vectorIndexClass,
+// int indexLength, Object base, long offset,
+// W indexVector1, W indexVector2, W indexVector3, W indexVector4,
+// M m, C container, int index, int[] indexMap, int indexM, S s,
+// LoadVectorOperationWithMap defaultImpl)
// ,
// W extends Vector,
// M extends VectorMask,
// E>
-// void storeWithMap(Class extends V> vClass, Class mClass, Class eClass, int length,
-// Class extends Vector> vectorIndexClass, int indexLength,
-// Object base, long offset, // Unsafe addressing
-// W indexVector, V v, M m,
-// C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
-// StoreVectorOperationWithMap defaultImpl)
+// void storeWithMap(Class extends V> vClass, Class mClass, int laneType,
+// int length,
+// Class extends Vector> vectorIndexClass,
+// int indexLength, Object base, long offset,
+// W indexVector,
+// V v, M m, C container, int index, int[] indexMap, int indexM,
+// StoreVectorOperationWithMap defaultImpl)
+//
//
bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
- const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
+ const TypeInt* laneType = gvn().type(argument(2))->isa_int();
const TypeInt* vlen = gvn().type(argument(3))->isa_int();
const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
const TypeInt* idx_vlen = gvn().type(argument(5))->isa_int();
if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
- elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
+ laneType == nullptr || !laneType->is_con() ||
vlen == nullptr || !vlen->is_con() ||
vector_idx_klass == nullptr || vector_idx_klass->const_oop() == nullptr ||
idx_vlen == nullptr || !idx_vlen->is_con()) {
@@ -1245,13 +1290,13 @@ bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
return false;
}
- ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
- if (!elem_type->is_primitive_type()) {
- log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
- return false; // should be primitive type
+ VectorSupport::LaneType vltype = static_cast(laneType->get_con());
+ if (!is_primitive_lane_type(vltype)) {
+ log_if_needed(" ** not a primitive lt=%s", VectorSupport::lanetype2name(vltype));
+ return false;
}
- BasicType elem_bt = elem_type->basic_type();
+ BasicType elem_bt = get_vector_primitive_lane_type(vltype);
int num_elem = vlen->get_con();
int idx_num_elem = idx_vlen->get_con();
@@ -1395,23 +1440,26 @@ bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
return true;
}
-// public static
-// ,
-// M extends VectorMask,
-// E>
-// long reductionCoerced(int oprId, Class extends V> vectorClass, Class extends M> maskClass,
-// Class elementType, int length, V v, M m,
-// ReductionOperation defaultImpl)
+//
+// ,
+// M extends VectorMask,
+// E>
+// long reductionCoerced(int oprId,
+// Class extends V> vClass, Class extends M> mClass, int laneType,
+// int length,
+// V v, M m,
+// ReductionOperation defaultImpl)
+//
bool LibraryCallKit::inline_vector_reduction() {
const TypeInt* opr = gvn().type(argument(0))->isa_int();
const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
- const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
+ const TypeInt* laneType = gvn().type(argument(3))->isa_int();
const TypeInt* vlen = gvn().type(argument(4))->isa_int();
if (opr == nullptr || !opr->is_con() ||
vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
- elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
+ laneType == nullptr || !laneType->is_con() ||
vlen == nullptr || !vlen->is_con()) {
log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
NodeClassNames[argument(0)->Opcode()],
@@ -1424,12 +1472,14 @@ bool LibraryCallKit::inline_vector_reduction() {
log_if_needed(" ** klass argument not initialized");
return false;
}
- ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
- if (!elem_type->is_primitive_type()) {
- log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
- return false; // should be primitive type
+
+ VectorSupport::LaneType vltype = static_cast(laneType->get_con());
+ if (!is_primitive_lane_type(vltype)) {
+ log_if_needed(" ** not a primitive lt=%s", VectorSupport::lanetype2name(vltype));
+ return false;
}
+ BasicType elem_bt = get_vector_primitive_lane_type(vltype);
const Type* vmask_type = gvn().type(argument(6));
bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
if (is_masked_op) {
@@ -1449,9 +1499,8 @@ bool LibraryCallKit::inline_vector_reduction() {
}
}
- BasicType elem_bt = elem_type->basic_type();
int num_elem = vlen->get_con();
- int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
+ int opc = VectorSupport::vop2ideal(opr->get_con(), vltype);
int sopc = ReductionNode::opcode(opc, elem_bt);
// Ensure reduction operation for lanewise operation
@@ -1539,19 +1588,27 @@ bool LibraryCallKit::inline_vector_reduction() {
return true;
}
-// public static boolean test(int cond, Class> vectorClass, Class> elementType, int vlen,
-// V v1, V v2,
-// BiFunction defaultImpl)
+
+
+//
+// ,
+// E>
+// boolean test(int cond,
+// Class> mClass, int laneType,
+// int length,
+// M m1, M m2,
+// BiFunction defaultImpl)
+//
//
bool LibraryCallKit::inline_vector_test() {
const TypeInt* cond = gvn().type(argument(0))->isa_int();
const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
- const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
+ const TypeInt* laneType = gvn().type(argument(2))->isa_int();
const TypeInt* vlen = gvn().type(argument(3))->isa_int();
if (cond == nullptr || !cond->is_con() ||
vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
- elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
+ laneType == nullptr || !laneType->is_con() ||
vlen == nullptr || !vlen->is_con()) {
log_if_needed(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
NodeClassNames[argument(0)->Opcode()],
@@ -1560,17 +1617,20 @@ bool LibraryCallKit::inline_vector_test() {
NodeClassNames[argument(3)->Opcode()]);
return false; // not enough info for intrinsification
}
+
+ VectorSupport::LaneType vltype = static_cast(laneType->get_con());
+ if (!is_primitive_lane_type(vltype)) {
+ log_if_needed(" ** not a primitive lt=%s", VectorSupport::lanetype2name(vltype));
+ return false;
+ }
+
if (!is_klass_initialized(vector_klass)) {
log_if_needed(" ** klass argument not initialized");
return false;
}
- ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
- if (!elem_type->is_primitive_type()) {
- log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
- return false; // should be primitive type
- }
- BasicType elem_bt = elem_type->basic_type();
+
int num_elem = vlen->get_con();
+ BasicType elem_bt = get_vector_primitive_lane_type(vltype);
BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
@@ -1605,24 +1665,26 @@ bool LibraryCallKit::inline_vector_test() {
return true;
}
-// public static
-// ,
-// M extends VectorMask,
-// E>
-// V blend(Class extends V> vectorClass, Class maskClass, Class elementType, int vlen,
-// V v1, V v2, M m,
-// VectorBlendOp defaultImpl)
+//
+// ,
+// M extends VectorMask,
+// E>
+// V blend(Class extends V> vClass, Class mClass, int laneType,
+// int length,
+// V v1, V v2, M m,
+// VectorBlendOp defaultImpl)
+//
bool LibraryCallKit::inline_vector_blend() {
const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
- const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
+ const TypeInt* laneType = gvn().type(argument(2))->isa_int();
const TypeInt* vlen = gvn().type(argument(3))->isa_int();
- if (mask_klass == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
+ if (mask_klass == nullptr || vector_klass == nullptr || vlen == nullptr || laneType == nullptr) {
return false; // dead code
}
if (mask_klass->const_oop() == nullptr || vector_klass->const_oop() == nullptr ||
- elem_klass->const_oop() == nullptr || !vlen->is_con()) {
+ !vlen->is_con() || !laneType->is_con()) {
log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
NodeClassNames[argument(0)->Opcode()],
NodeClassNames[argument(1)->Opcode()],
@@ -1630,16 +1692,19 @@ bool LibraryCallKit::inline_vector_blend() {
NodeClassNames[argument(3)->Opcode()]);
return false; // not enough info for intrinsification
}
+
+ VectorSupport::LaneType vltype = static_cast(laneType->get_con());
+ if (!is_primitive_lane_type(vltype)) {
+ log_if_needed(" ** not a primitive lt=%s", VectorSupport::lanetype2name(vltype));
+ return false; // should be primitive type
+ }
+
if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
log_if_needed(" ** klass argument not initialized");
return false;
}
- ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
- if (!elem_type->is_primitive_type()) {
- log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
- return false; // should be primitive type
- }
- BasicType elem_bt = elem_type->basic_type();
+
+ BasicType elem_bt = get_vector_primitive_lane_type(vltype);
BasicType mask_bt = elem_bt;
int num_elem = vlen->get_con();
@@ -1670,25 +1735,29 @@ bool LibraryCallKit::inline_vector_blend() {
return true;
}
-// public static
+
+//
// ,
// M extends VectorMask,
// E>
-// M compare(int cond, Class extends V> vectorClass, Class maskClass, Class elementType, int vlen,
+// M compare(int cond,
+// Class extends V> vectorClass, Class mClass, int laneType,
+// int length,
// V v1, V v2, M m,
-// VectorCompareOp defaultImpl)
+// VectorCompareOp defaultImpl)
+//
bool LibraryCallKit::inline_vector_compare() {
const TypeInt* cond = gvn().type(argument(0))->isa_int();
const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
- const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
+ const TypeInt* laneType = gvn().type(argument(3))->isa_int();
const TypeInt* vlen = gvn().type(argument(4))->isa_int();
- if (cond == nullptr || vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
+ if (cond == nullptr || vector_klass == nullptr || mask_klass == nullptr || laneType == nullptr || vlen == nullptr) {
return false; // dead code
}
if (!cond->is_con() || vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
- elem_klass->const_oop() == nullptr || !vlen->is_con()) {
+ !laneType->is_con() || !vlen->is_con()) {
log_if_needed(" ** missing constant: cond=%s vclass=%s mclass=%s etype=%s vlen=%s",
NodeClassNames[argument(0)->Opcode()],
NodeClassNames[argument(1)->Opcode()],
@@ -1697,18 +1766,20 @@ bool LibraryCallKit::inline_vector_compare() {
NodeClassNames[argument(4)->Opcode()]);
return false; // not enough info for intrinsification
}
+
+ VectorSupport::LaneType vltype = static_cast(laneType->get_con());
+ if (!is_primitive_lane_type(vltype)) {
+ log_if_needed(" ** not a primitive lt=%s", VectorSupport::lanetype2name(vltype));
+ return false;
+ }
+
if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
log_if_needed(" ** klass argument not initialized");
return false;
}
- ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
- if (!elem_type->is_primitive_type()) {
- log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
- return false; // should be primitive type
- }
int num_elem = vlen->get_con();
- BasicType elem_bt = elem_type->basic_type();
+ BasicType elem_bt = get_vector_primitive_lane_type(vltype);
BasicType mask_bt = elem_bt;
if ((cond->get_con() & BoolTest::unsigned_compare) != 0) {
@@ -1777,27 +1848,30 @@ bool LibraryCallKit::inline_vector_compare() {
return true;
}
-// public static
-// ,
-// Sh extends VectorShuffle,
-// M extends VectorMask,
-// E>
-// V rearrangeOp(Class extends V> vectorClass, Class shuffleClass, Class maskClass, Class elementType, int vlen,
-// V v1, Sh sh, M m,
-// VectorRearrangeOp defaultImpl)
+//
+// ,
+// SH extends VectorShuffle,
+// M extends VectorMask,
+// E>
+// V rearrangeOp(Class extends V> vClass, Class shClass, Class mClass, int laneType,
+// int length,
+// V v, SH sh, M m,
+// VectorRearrangeOp defaultImpl)
+//
bool LibraryCallKit::inline_vector_rearrange() {
const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
- const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
+ const TypeInt* laneType = gvn().type(argument(3))->isa_int();
const TypeInt* vlen = gvn().type(argument(4))->isa_int();
- if (vector_klass == nullptr || shuffle_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
+ if (vector_klass == nullptr || shuffle_klass == nullptr || laneType == nullptr || vlen == nullptr) {
return false; // dead code
}
+
if (shuffle_klass->const_oop() == nullptr ||
vector_klass->const_oop() == nullptr ||
- elem_klass->const_oop() == nullptr ||
+ !laneType->is_con() ||
!vlen->is_con()) {
log_if_needed(" ** missing constant: vclass=%s sclass=%s etype=%s vlen=%s",
NodeClassNames[argument(0)->Opcode()],
@@ -1806,18 +1880,20 @@ bool LibraryCallKit::inline_vector_rearrange() {
NodeClassNames[argument(4)->Opcode()]);
return false; // not enough info for intrinsification
}
+
if (!is_klass_initialized(vector_klass) ||
!is_klass_initialized(shuffle_klass)) {
log_if_needed(" ** klass argument not initialized");
return false;
}
- ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
- if (!elem_type->is_primitive_type()) {
- log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
- return false; // should be primitive type
+
+ VectorSupport::LaneType vltype = static_cast(laneType->get_con());
+ if (!is_primitive_lane_type(vltype)) {
+ log_if_needed(" ** not a primitive lt=%s", VectorSupport::lanetype2name(vltype));
+ return false;
}
- BasicType elem_bt = elem_type->basic_type();
+ BasicType elem_bt = get_vector_primitive_lane_type(vltype);
BasicType shuffle_bt = elem_bt;
if (shuffle_bt == T_FLOAT) {
shuffle_bt = T_INT;
@@ -1917,22 +1993,23 @@ bool LibraryCallKit::inline_vector_rearrange() {
return true;
}
-// public static
-// ,
-// M extends VectorMask,
-// E>
-// V selectFromOp(Class extends V> vClass, Class mClass, Class eClass,
-// int length, V v1, V v2, M m,
-// VectorSelectFromOp defaultImpl)
+//
+// ,
+// M extends VectorMask,
+// E>
+// V selectFromOp(Class extends V> vClass, Class mClass, int laneType,
+// int length, V v1, V v2, M m,
+// VectorSelectFromOp defaultImpl)
+//
bool LibraryCallKit::inline_vector_select_from() {
const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
- const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
+ const TypeInt* laneType = gvn().type(argument(2))->isa_int();
const TypeInt* vlen = gvn().type(argument(3))->isa_int();
- if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
+ if (vector_klass == nullptr || laneType == nullptr || vlen == nullptr ||
vector_klass->const_oop() == nullptr ||
- elem_klass->const_oop() == nullptr ||
+ !laneType->is_con() ||
!vlen->is_con()) {
log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
NodeClassNames[argument(0)->Opcode()],
@@ -1940,17 +2017,20 @@ bool LibraryCallKit::inline_vector_select_from() {
NodeClassNames[argument(3)->Opcode()]);
return false; // not enough info for intrinsification
}
+
if (!is_klass_initialized(vector_klass)) {
log_if_needed(" ** klass argument not initialized");
return false;
}
- ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
- if (!elem_type->is_primitive_type()) {
- log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
- return false; // should be primitive type
+
+ VectorSupport::LaneType vltype = static_cast(laneType->get_con());
+ if (!is_primitive_lane_type(vltype)) {
+ log_if_needed(" ** not a primitive lt=%s", VectorSupport::lanetype2name(vltype));
+ return false;
}
- BasicType elem_bt = elem_type->basic_type();
+
int num_elem = vlen->get_con();
+ BasicType elem_bt = get_vector_primitive_lane_type(vltype);
if (!is_power_of_2(num_elem)) {
log_if_needed(" ** vlen not power of two=%d", num_elem);
return false;
@@ -2074,25 +2154,27 @@ bool LibraryCallKit::inline_vector_select_from() {
return true;
}
-// public static
-// ,
-// M extends VectorMask,
-// E>
-// V broadcastInt(int opr, Class extends V> vectorClass, Class extends M> maskClass,
-// Class elementType, int length,
-// V v, int n, M m,
-// VectorBroadcastIntOp defaultImpl)
+//
+// ,
+// M extends VectorMask,
+// E>
+// V broadcastInt(int opr,
+// Class extends V> vClass, Class extends M> mClass, int laneType,
+// int length,
+// V v, int n, M m,
+// VectorBroadcastIntOp defaultImpl) {
+//
bool LibraryCallKit::inline_vector_broadcast_int() {
const TypeInt* opr = gvn().type(argument(0))->isa_int();
const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
- const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
+ const TypeInt* laneType = gvn().type(argument(3))->isa_int();
const TypeInt* vlen = gvn().type(argument(4))->isa_int();
- if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
+ if (opr == nullptr || vector_klass == nullptr || laneType == nullptr || vlen == nullptr) {
return false; // dead code
}
- if (!opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
+ if (!opr->is_con() || vector_klass->const_oop() == nullptr || !laneType->is_con() || !vlen->is_con()) {
log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
NodeClassNames[argument(0)->Opcode()],
NodeClassNames[argument(1)->Opcode()],
@@ -2105,6 +2187,12 @@ bool LibraryCallKit::inline_vector_broadcast_int() {
return false;
}
+ VectorSupport::LaneType vltype = static_cast(laneType->get_con());
+ if (!is_primitive_lane_type(vltype)) {
+ log_if_needed(" ** not a primitive lt=%s", VectorSupport::lanetype2name(vltype));
+ return false;
+ }
+
const Type* vmask_type = gvn().type(argument(7));
bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
if (is_masked_op) {
@@ -2124,15 +2212,9 @@ bool LibraryCallKit::inline_vector_broadcast_int() {
}
}
- ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
- if (!elem_type->is_primitive_type()) {
- log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
- return false; // should be primitive type
- }
-
int num_elem = vlen->get_con();
- BasicType elem_bt = elem_type->basic_type();
- int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
+ BasicType elem_bt = get_vector_primitive_lane_type(vltype);
+ int opc = VectorSupport::vop2ideal(opr->get_con(), vltype);
bool is_shift = VectorNode::is_shift_opcode(opc);
bool is_rotate = VectorNode::is_rotate_opcode(opc);
@@ -2222,12 +2304,12 @@ bool LibraryCallKit::inline_vector_broadcast_int() {
return true;
}
-// public static
+//
// VOUT convert(int oprId,
-// Class> fromVectorClass, Class> fromElementType, int fromVLen,
-// Class> toVectorClass, Class> toElementType, int toVLen,
+// Class> fromVectorClass, int fromLaneType, int fromVLen,
+// Class> toVectorClass, int toLaneType, int toVLen,
// VIN v, S s,
// VectorConvertOp defaultImpl)
//
@@ -2235,22 +2317,22 @@ bool LibraryCallKit::inline_vector_convert() {
const TypeInt* opr = gvn().type(argument(0))->isa_int();
const TypeInstPtr* vector_klass_from = gvn().type(argument(1))->isa_instptr();
- const TypeInstPtr* elem_klass_from = gvn().type(argument(2))->isa_instptr();
+ const TypeInt* laneType_from = gvn().type(argument(2))->isa_int();
const TypeInt* vlen_from = gvn().type(argument(3))->isa_int();
const TypeInstPtr* vector_klass_to = gvn().type(argument(4))->isa_instptr();
- const TypeInstPtr* elem_klass_to = gvn().type(argument(5))->isa_instptr();
+ const TypeInt* laneType_to = gvn().type(argument(5))->isa_int();
const TypeInt* vlen_to = gvn().type(argument(6))->isa_int();
if (opr == nullptr ||
- vector_klass_from == nullptr || elem_klass_from == nullptr || vlen_from == nullptr ||
- vector_klass_to == nullptr || elem_klass_to == nullptr || vlen_to == nullptr) {
+ vector_klass_from == nullptr || laneType_from == nullptr || vlen_from == nullptr ||
+ vector_klass_to == nullptr || laneType_to == nullptr || vlen_to == nullptr) {
return false; // dead code
}
if (!opr->is_con() ||
- vector_klass_from->const_oop() == nullptr || elem_klass_from->const_oop() == nullptr || !vlen_from->is_con() ||
- vector_klass_to->const_oop() == nullptr || elem_klass_to->const_oop() == nullptr || !vlen_to->is_con()) {
- log_if_needed(" ** missing constant: opr=%s vclass_from=%s etype_from=%s vlen_from=%s vclass_to=%s etype_to=%s vlen_to=%s",
+ vector_klass_from->const_oop() == nullptr || !laneType_from->is_con() || !vlen_from->is_con() ||
+ vector_klass_to->const_oop() == nullptr || !laneType_to->is_con() || !vlen_to->is_con()) {
+ log_if_needed(" ** missing constant: opr=%s vclass_from=%s laneType_from=%s vlen_from=%s vclass_to=%s laneType_to=%s vlen_to=%s",
NodeClassNames[argument(0)->Opcode()],
NodeClassNames[argument(1)->Opcode()],
NodeClassNames[argument(2)->Opcode()],
@@ -2276,16 +2358,19 @@ bool LibraryCallKit::inline_vector_convert() {
bool is_mask = is_vector_mask(vbox_klass_from);
- ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type();
- if (!elem_type_from->is_primitive_type()) {
+ VectorSupport::LaneType vltype_from = static_cast(laneType_from->get_con());
+ if (!is_primitive_lane_type(vltype_from)) {
+ log_if_needed(" ** not a primitive from lt=%s", VectorSupport::lanetype2name(vltype_from));
return false; // should be primitive type
}
- BasicType elem_bt_from = elem_type_from->basic_type();
- ciType* elem_type_to = elem_klass_to->const_oop()->as_instance()->java_mirror_type();
- if (!elem_type_to->is_primitive_type()) {
+
+ VectorSupport::LaneType vltype_to = static_cast(laneType_to->get_con());
+ if (!is_primitive_lane_type(vltype_to)) {
+ log_if_needed(" ** not a primitive to lt=%s", VectorSupport::lanetype2name(vltype_to));
return false; // should be primitive type
}
- BasicType elem_bt_to = elem_type_to->basic_type();
+ BasicType elem_bt_from = get_vector_primitive_lane_type(vltype_from);
+ BasicType elem_bt_to = get_vector_primitive_lane_type(vltype_to);
int num_elem_from = vlen_from->get_con();
int num_elem_to = vlen_to->get_con();
@@ -2432,22 +2517,25 @@ bool LibraryCallKit::inline_vector_convert() {
return true;
}
-// public static
+//
// ,
// E>
-// V insert(Class extends V> vectorClass, Class elementType, int vlen,
-// V vec, int ix, long val,
+// V insert(Class extends V> vClass, int laneType,
+// int length,
+// V v, int i, long val,
// VecInsertOp defaultImpl)
+//
bool LibraryCallKit::inline_vector_insert() {
const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
- const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
+ const TypeInt* laneType = gvn().type(argument(1))->isa_int();
const TypeInt* vlen = gvn().type(argument(2))->isa_int();
const TypeInt* idx = gvn().type(argument(4))->isa_int();
- if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
+ if (vector_klass == nullptr || laneType == nullptr || vlen == nullptr || idx == nullptr) {
return false; // dead code
}
- if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
+ if (vector_klass->const_oop() == nullptr || !laneType->is_con() ||
+ !vlen->is_con() || !idx->is_con()) {
log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
NodeClassNames[argument(0)->Opcode()],
NodeClassNames[argument(1)->Opcode()],
@@ -2455,17 +2543,20 @@ bool LibraryCallKit::inline_vector_insert() {
NodeClassNames[argument(4)->Opcode()]);
return false; // not enough info for intrinsification
}
+
+ VectorSupport::LaneType vltype = static_cast