diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml
index 3bb50a137ec..49071de7771 100644
--- a/.github/workflows/build-windows.yml
+++ b/.github/workflows/build-windows.yml
@@ -31,6 +31,9 @@ on:
platform:
required: true
type: string
+ runs-on:
+ required: true
+ type: string
extra-conf-options:
required: false
type: string
@@ -67,7 +70,7 @@ env:
jobs:
build-windows:
name: build
- runs-on: windows-2025
+ runs-on: ${{ inputs.runs-on }}
defaults:
run:
shell: bash
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 20be196b128..94f011ff76c 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -353,6 +353,7 @@ jobs:
uses: ./.github/workflows/build-windows.yml
with:
platform: windows-x64
+ runs-on: windows-2022
msvc-toolset-version: '14.44'
msvc-toolset-architecture: 'x86.x64'
configure-arguments: ${{ github.event.inputs.configure-arguments }}
@@ -366,6 +367,7 @@ jobs:
uses: ./.github/workflows/build-windows.yml
with:
platform: windows-aarch64
+ runs-on: windows-2022
msvc-toolset-version: '14.44'
msvc-toolset-architecture: 'arm64'
make-target: 'hotspot'
@@ -446,6 +448,6 @@ jobs:
with:
platform: windows-x64
bootjdk-platform: windows-x64
- runs-on: windows-2025
+ runs-on: windows-2022
dry-run: ${{ needs.prepare.outputs.dry-run == 'true' }}
debug-suffix: -debug
diff --git a/.gitignore b/.gitignore
index b6b4a1a559a..a45e2113756 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,8 +24,6 @@ NashornProfile.txt
/.gdbinit
/.lldbinit
**/core.[0-9]*
-*.rej
-*.orig
test/benchmarks/**/target
/src/hotspot/CMakeLists.txt
/src/hotspot/compile_commands.json
diff --git a/make/RunTestsPrebuiltSpec.gmk b/make/RunTestsPrebuiltSpec.gmk
index 5fe559eafad..568f69da5a5 100644
--- a/make/RunTestsPrebuiltSpec.gmk
+++ b/make/RunTestsPrebuiltSpec.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2017, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2017, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -64,7 +64,7 @@ TEST_JOBS ?= 0
# Use hard-coded values for java flags (one size, fits all!)
JAVA_FLAGS := -Duser.language=en -Duser.country=US
JAVA_FLAGS_BIG := -Xms64M -Xmx2048M
-JAVA_FLAGS_SMALL := -XX:+UseSerialGC -Xms32M -Xmx512M -XX:TieredStopAtLevel=1
+JAVA_FLAGS_SMALL := -Xms32M -Xmx512M -XX:TieredStopAtLevel=1
BUILDJDK_JAVA_FLAGS_SMALL := -Xms32M -Xmx512M -XX:TieredStopAtLevel=1
BUILD_JAVA_FLAGS := $(JAVA_FLAGS_BIG)
diff --git a/make/autoconf/boot-jdk.m4 b/make/autoconf/boot-jdk.m4
index b3dbc292919..4468a9acf27 100644
--- a/make/autoconf/boot-jdk.m4
+++ b/make/autoconf/boot-jdk.m4
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -481,8 +481,6 @@ AC_DEFUN_ONCE([BOOTJDK_SETUP_BOOT_JDK_ARGUMENTS],
AC_MSG_CHECKING([flags for boot jdk java command for small workloads])
- # Use serial gc for small short lived tools if possible
- UTIL_ADD_JVM_ARG_IF_OK([-XX:+UseSerialGC],boot_jdk_jvmargs_small,[$JAVA])
UTIL_ADD_JVM_ARG_IF_OK([-Xms32M],boot_jdk_jvmargs_small,[$JAVA])
UTIL_ADD_JVM_ARG_IF_OK([-Xmx512M],boot_jdk_jvmargs_small,[$JAVA])
UTIL_ADD_JVM_ARG_IF_OK([-XX:TieredStopAtLevel=1],boot_jdk_jvmargs_small,[$JAVA])
@@ -492,8 +490,6 @@ AC_DEFUN_ONCE([BOOTJDK_SETUP_BOOT_JDK_ARGUMENTS],
JAVA_FLAGS_SMALL=$boot_jdk_jvmargs_small
AC_SUBST(JAVA_FLAGS_SMALL)
- # Don't presuppose SerialGC is present in the buildjdk. Also, we cannot test
- # the buildjdk, but on the other hand we know what it will support.
BUILD_JAVA_FLAGS_SMALL="-Xms32M -Xmx512M -XX:TieredStopAtLevel=1"
AC_SUBST(BUILD_JAVA_FLAGS_SMALL)
diff --git a/make/autoconf/buildjdk-spec.gmk.template b/make/autoconf/buildjdk-spec.gmk.template
index bb020842d59..40758436517 100644
--- a/make/autoconf/buildjdk-spec.gmk.template
+++ b/make/autoconf/buildjdk-spec.gmk.template
@@ -108,3 +108,4 @@ override EXTRA_LDFLAGS :=
# hsdis is not needed
HSDIS_BACKEND := none
ENABLE_HSDIS_BUNDLING := false
+DEFAULT_PRINT_ASSEMBLY_OPTIONS :=
diff --git a/make/autoconf/jdk-options.m4 b/make/autoconf/jdk-options.m4
index 89fcbc88521..5b1e4c3d7f7 100644
--- a/make/autoconf/jdk-options.m4
+++ b/make/autoconf/jdk-options.m4
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -323,6 +323,14 @@ AC_DEFUN_ONCE([JDKOPT_SETUP_DEBUG_SYMBOLS],
AC_SUBST(COPY_DEBUG_SYMBOLS)
AC_SUBST(ZIP_EXTERNAL_DEBUG_SYMBOLS)
+ # Should we enable objcopy debuginfo compression ?
+ UTIL_ARG_ENABLE(NAME: objcopy-debuginfo-compression, DEFAULT: false,
+ RESULT: ENABLE_OBJCOPY_DEBUGINFO_COMPRESSION,
+ DESC: [Set to enable compression in the debuginfo files (Linux only)],
+ CHECKING_MSG: [if debuginfo compression with objcopy is done],
+ IF_ENABLED: [ OBJCOPY_COMPRESS_FLAGS="--compress-debug-sections=zlib-gnu" ])
+ AC_SUBST(OBJCOPY_COMPRESS_FLAGS)
+
# Should we add external native debug symbols to the shipped bundles?
AC_MSG_CHECKING([if we should add external native debug symbols to the shipped bundles])
AC_ARG_WITH([external-symbols-in-bundles],
diff --git a/make/autoconf/lib-hsdis.m4 b/make/autoconf/lib-hsdis.m4
index 0530ef90be8..928aa021f28 100644
--- a/make/autoconf/lib-hsdis.m4
+++ b/make/autoconf/lib-hsdis.m4
@@ -417,4 +417,9 @@ AC_DEFUN_ONCE([LIB_SETUP_HSDIS],
AC_MSG_RESULT([no])
fi
AC_SUBST(ENABLE_HSDIS_BUNDLING)
+
+ UTIL_ARG_WITH(NAME: print-assembly-options, TYPE: string,
+ DEFAULT: [], RESULT: DEFAULT_PRINT_ASSEMBLY_OPTIONS,
+ DESC: [default value for the PrintAssemblyOptions diagnostic flag, passed verbatim to the disassembler])
+ AC_SUBST(DEFAULT_PRINT_ASSEMBLY_OPTIONS)
])
diff --git a/make/autoconf/spec.gmk.template b/make/autoconf/spec.gmk.template
index c4e5a23d31a..7de5056cb83 100644
--- a/make/autoconf/spec.gmk.template
+++ b/make/autoconf/spec.gmk.template
@@ -381,6 +381,7 @@ HSDIS_CFLAGS := @HSDIS_CFLAGS@
HSDIS_LDFLAGS := @HSDIS_LDFLAGS@
HSDIS_LIBS := @HSDIS_LIBS@
CAPSTONE_ARCH_AARCH64_NAME := @CAPSTONE_ARCH_AARCH64_NAME@
+DEFAULT_PRINT_ASSEMBLY_OPTIONS := @DEFAULT_PRINT_ASSEMBLY_OPTIONS@
# The boot jdk to use. This is overridden in bootcycle-spec.gmk. Make sure to keep
# it in sync.
@@ -634,6 +635,7 @@ METALLIB := @METALLIB@
COMPILE_WITH_DEBUG_SYMBOLS := @COMPILE_WITH_DEBUG_SYMBOLS@
COPY_DEBUG_SYMBOLS := @COPY_DEBUG_SYMBOLS@
ZIP_EXTERNAL_DEBUG_SYMBOLS := @ZIP_EXTERNAL_DEBUG_SYMBOLS@
+OBJCOPY_COMPRESS_FLAGS := @OBJCOPY_COMPRESS_FLAGS@
CFLAGS_DEBUG_SYMBOLS := @CFLAGS_DEBUG_SYMBOLS@
ASFLAGS_DEBUG_SYMBOLS := @ASFLAGS_DEBUG_SYMBOLS@
diff --git a/make/common/native/Link.gmk b/make/common/native/Link.gmk
index 855e50bddfb..2307b315877 100644
--- a/make/common/native/Link.gmk
+++ b/make/common/native/Link.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -189,7 +189,7 @@ define CreateDynamicLibraryOrExecutable
# This cannot be run separately since it updates the original target
# file.
$$(call ExecuteWithLog, $$($1_OBJECT_DIR)/$$($1_SAFE_NAME)_create_debuginfo, \
- $$($1_OBJCOPY) --only-keep-debug $$($1_TARGET) $$($1_DEBUGINFO_FILES))
+ $$($1_OBJCOPY) $(OBJCOPY_COMPRESS_FLAGS) --only-keep-debug $$($1_TARGET) $$($1_DEBUGINFO_FILES))
$$(CHMOD) -x $$($1_DEBUGINFO_FILES)
else ifeq ($(call isTargetOs, aix), true)
# AIX does not provide the equivalent of objcopy to extract debug
diff --git a/make/data/cldr/common/dtd/ldml.dtd b/make/data/cldr/common/dtd/ldml.dtd
index aebedd33a43..b4247f2d924 100644
--- a/make/data/cldr/common/dtd/ldml.dtd
+++ b/make/data/cldr/common/dtd/ldml.dtd
@@ -1,5 +1,5 @@
-
+
@@ -493,6 +493,16 @@ CLDR data files are interpreted according to the LDML specification (http://unic
+
+
+
+
+
+
+
+
+
+
diff --git a/make/data/cldr/common/main/aa.xml b/make/data/cldr/common/main/aa.xml
index 3ff6fb6dd06..791c3009658 100644
--- a/make/data/cldr/common/main/aa.xml
+++ b/make/data/cldr/common/main/aa.xml
@@ -1,6 +1,6 @@
-
@@ -1027,6 +1027,7 @@ For terms of use, see http://www.unicode.org/copyright.html
+
diff --git a/make/data/cldr/common/supplemental/likelySubtags.xml b/make/data/cldr/common/supplemental/likelySubtags.xml
index 76e215255fd..a73b8a8c95b 100644
--- a/make/data/cldr/common/supplemental/likelySubtags.xml
+++ b/make/data/cldr/common/supplemental/likelySubtags.xml
@@ -1,7 +1,7 @@
-
+
@@ -1343,7 +1343,7 @@ not be patched by hand, as any changes made in that fashion may be lost.
-
+
diff --git a/make/data/cldr/common/supplemental/metaZones.xml b/make/data/cldr/common/supplemental/metaZones.xml
index 710934fef81..610921a8f6d 100644
--- a/make/data/cldr/common/supplemental/metaZones.xml
+++ b/make/data/cldr/common/supplemental/metaZones.xml
@@ -735,7 +735,7 @@ For terms of use, see http://www.unicode.org/copyright.html
-
+
diff --git a/make/data/cldr/common/supplemental/supplementalData.xml b/make/data/cldr/common/supplemental/supplementalData.xml
index 25684d36c6e..cbfe2c5e875 100644
--- a/make/data/cldr/common/supplemental/supplementalData.xml
+++ b/make/data/cldr/common/supplemental/supplementalData.xml
@@ -1,7 +1,7 @@
@@ -57,7 +57,7 @@ For terms of use, see https://www.unicode.org/copyright.html
-
+
@@ -3147,7 +3147,7 @@ XXX Code for transations where no currency is involved
-
+
diff --git a/make/hotspot/lib/CompileGtest.gmk b/make/hotspot/lib/CompileGtest.gmk
index 4b21d481049..3a5fac34597 100644
--- a/make/hotspot/lib/CompileGtest.gmk
+++ b/make/hotspot/lib/CompileGtest.gmk
@@ -45,6 +45,15 @@ else
GTEST_COPY_DEBUG_SYMBOLS := false
endif
+GTEST_LIBJVM_CFLAGS := $(JVM_CFLAGS)
+# Decoder does not work with debuginfo of the gtest libjvm when sections are used,
+# so we get wrong file names. That's why we filter out the section flags.
+ifeq ($(ENABLE_LINKTIME_GC), true)
+ ifeq ($(TOOLCHAIN_TYPE), gcc)
+ GTEST_LIBJVM_CFLAGS := $(filter-out -ffunction-sections -fdata-sections, $(JVM_CFLAGS))
+ endif
+endif
+
################################################################################
## Build libgtest
################################################################################
@@ -99,7 +108,7 @@ $(eval $(call SetupJdkLibrary, BUILD_GTEST_LIBJVM, \
EXCLUDE_PATTERNS := $(JVM_EXCLUDE_PATTERNS), \
EXTRA_OBJECT_FILES := $(BUILD_LIBJVM_ALL_OBJS), \
DEFAULT_CFLAGS := false, \
- CFLAGS := $(JVM_CFLAGS) \
+ CFLAGS := $(GTEST_LIBJVM_CFLAGS) \
-DHOTSPOT_GTEST \
-I$(GTEST_FRAMEWORK_SRC)/googletest/include \
-I$(GTEST_FRAMEWORK_SRC)/googlemock/include \
diff --git a/make/hotspot/lib/CompileJvm.gmk b/make/hotspot/lib/CompileJvm.gmk
index f41693e05fb..e8db4888d3a 100644
--- a/make/hotspot/lib/CompileJvm.gmk
+++ b/make/hotspot/lib/CompileJvm.gmk
@@ -209,6 +209,8 @@ $(eval $(call SetupJdkLibrary, BUILD_LIBJVM, \
DISABLED_WARNINGS_gcc_jvmtiTagMap.cpp := stringop-overflow, \
DISABLED_WARNINGS_gcc_macroAssembler_ppc_sha.cpp := unused-const-variable, \
DISABLED_WARNINGS_gcc_postaloc.cpp := address, \
+ DISABLED_WARNINGS_gcc_safepointMechanism.cpp := stringop-overflow, \
+ DISABLED_WARNINGS_gcc_shenandoahGenerationalHeap.cpp := stringop-overflow, \
DISABLED_WARNINGS_gcc_shenandoahLock.cpp := stringop-overflow, \
DISABLED_WARNINGS_gcc_stubGenerator_s390.cpp := unused-const-variable, \
DISABLED_WARNINGS_gcc_synchronizer.cpp := stringop-overflow, \
diff --git a/make/hotspot/lib/JvmFlags.gmk b/make/hotspot/lib/JvmFlags.gmk
index 27a96cc4865..121f71bfa27 100644
--- a/make/hotspot/lib/JvmFlags.gmk
+++ b/make/hotspot/lib/JvmFlags.gmk
@@ -102,6 +102,10 @@ ifneq ($(HOTSPOT_OVERRIDE_LIBPATH), )
JVM_CFLAGS += -DOVERRIDE_LIBPATH='"$(HOTSPOT_OVERRIDE_LIBPATH)"'
endif
+ifneq ($(DEFAULT_PRINT_ASSEMBLY_OPTIONS), )
+ JVM_CFLAGS += -DDEFAULT_PRINT_ASSEMBLY_OPTIONS='"$(DEFAULT_PRINT_ASSEMBLY_OPTIONS)"'
+endif
+
ifeq ($(ENABLE_COMPATIBLE_CDS_ALIGNMENT), true)
JVM_CFLAGS += -DCOMPATIBLE_CDS_ALIGNMENT
endif
diff --git a/make/jdk/src/classes/build/tools/cldrconverter/BundleGenerator.java b/make/jdk/src/classes/build/tools/cldrconverter/BundleGenerator.java
index 20e259a0ba7..be5e49f8f7d 100644
--- a/make/jdk/src/classes/build/tools/cldrconverter/BundleGenerator.java
+++ b/make/jdk/src/classes/build/tools/cldrconverter/BundleGenerator.java
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2012, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -50,7 +50,7 @@ public interface BundleGenerator {
};
public void generateBundle(String packageName, String baseName, String localeID,
- boolean useJava, Map map, BundleType type) throws IOException;
+ Map map, BundleType type) throws IOException;
public void generateMetaInfo(Map> metaInfo) throws IOException;
}
diff --git a/make/jdk/src/classes/build/tools/cldrconverter/CLDRConverter.java b/make/jdk/src/classes/build/tools/cldrconverter/CLDRConverter.java
index 9f42326ef09..7b198f97679 100644
--- a/make/jdk/src/classes/build/tools/cldrconverter/CLDRConverter.java
+++ b/make/jdk/src/classes/build/tools/cldrconverter/CLDRConverter.java
@@ -183,7 +183,6 @@ public class CLDRConverter {
}
}
- static boolean USE_UTF8 = false;
private static boolean verbose;
private CLDRConverter() {
@@ -232,10 +231,6 @@ public class CLDRConverter {
DESTINATION_DIR = args[++i];
break;
- case "-utf8":
- USE_UTF8 = true;
- break;
-
case "-verbose":
verbose = true;
break;
@@ -336,7 +331,6 @@ public class CLDRConverter {
+ "\t-year year copyright year in output%n"
+ "\t-zntempfile template file for java.time.format.ZoneName.java%n"
+ "\t-tzdatadir tzdata directory for java.time.format.ZoneName.java%n"
- + "\t-utf8 use UTF-8 rather than \\uxxxx (for debug)%n"
+ "\t-jdk-header-template %n"
+ "\t\t override default GPL header with contents of file%n");
}
@@ -612,31 +606,31 @@ public class CLDRConverter {
if (bundleTypes.contains(Bundle.Type.LOCALENAMES)) {
Map localeNamesMap = extractLocaleNames(targetMap, id);
if (!localeNamesMap.isEmpty() || bundle.isRoot()) {
- bundleGenerator.generateBundle("util", "LocaleNames", id, true, localeNamesMap, BundleType.OPEN);
+ bundleGenerator.generateBundle("util", "LocaleNames", id, localeNamesMap, BundleType.OPEN);
}
}
if (bundleTypes.contains(Bundle.Type.CURRENCYNAMES)) {
Map currencyNamesMap = extractCurrencyNames(targetMap, id, bundle.getCurrencies());
if (!currencyNamesMap.isEmpty() || bundle.isRoot()) {
- bundleGenerator.generateBundle("util", "CurrencyNames", id, true, currencyNamesMap, BundleType.OPEN);
+ bundleGenerator.generateBundle("util", "CurrencyNames", id, currencyNamesMap, BundleType.OPEN);
}
}
if (bundleTypes.contains(Bundle.Type.TIMEZONENAMES)) {
Map zoneNamesMap = extractZoneNames(targetMap, id);
if (!zoneNamesMap.isEmpty() || bundle.isRoot()) {
- bundleGenerator.generateBundle("util", "TimeZoneNames", id, true, zoneNamesMap, BundleType.TIMEZONE);
+ bundleGenerator.generateBundle("util", "TimeZoneNames", id, zoneNamesMap, BundleType.TIMEZONE);
}
}
if (bundleTypes.contains(Bundle.Type.CALENDARDATA)) {
Map calendarDataMap = extractCalendarData(targetMap, id);
if (!calendarDataMap.isEmpty() || bundle.isRoot()) {
- bundleGenerator.generateBundle("util", "CalendarData", id, true, calendarDataMap, BundleType.PLAIN);
+ bundleGenerator.generateBundle("util", "CalendarData", id, calendarDataMap, BundleType.PLAIN);
}
}
if (bundleTypes.contains(Bundle.Type.FORMATDATA)) {
Map formatDataMap = extractFormatData(targetMap, id);
if (!formatDataMap.isEmpty() || bundle.isRoot()) {
- bundleGenerator.generateBundle("text", "FormatData", id, true, formatDataMap, BundleType.PLAIN);
+ bundleGenerator.generateBundle("text", "FormatData", id, formatDataMap, BundleType.PLAIN);
}
}
@@ -821,6 +815,13 @@ public class CLDRConverter {
data = map.get(TIMEZONE_ID_PREFIX + tzLink);
}
+ String meta = handlerMetaZones.get(tzKey);
+ if (meta == null && tzLink != null) {
+ // Check for tzLink
+ meta = handlerMetaZones.get(tzLink);
+ }
+ String metaKey = meta != null ? METAZONE_ID_PREFIX + meta : null;
+
if (data instanceof String[] tznames) {
// Hack for UTC. UTC is an alias to Etc/UTC in CLDR
if (tzid.equals("Etc/UTC") && !map.containsKey(TIMEZONE_ID_PREFIX + "UTC")) {
@@ -832,24 +833,14 @@ public class CLDRConverter {
tznames = Arrays.copyOf(tznames, tznames.length);
fillTZDBShortNames(tzKey, tznames);
names.put(tzid, tznames);
+ if (meta != null && map.get(metaKey) instanceof String[] metaNames) {
+ recordMetazone(names, meta, tzKey, metaNames);
+ }
}
} else {
- String meta = handlerMetaZones.get(tzKey);
- if (meta == null && tzLink != null) {
- // Check for tzLink
- meta = handlerMetaZones.get(tzLink);
- }
if (meta != null) {
- String metaKey = METAZONE_ID_PREFIX + meta;
- data = map.get(metaKey);
- if (data instanceof String[] tznames) {
- if (isDefaultZone(meta, tzKey)) {
- // Record the metazone names only from the default
- // (001) zone, with short names filled from TZDB
- tznames = Arrays.copyOf(tznames, tznames.length);
- fillTZDBShortNames(tzKey, tznames);
- names.put(metaKey, tznames);
- }
+ if (map.get(metaKey) instanceof String[] metaNames) {
+ recordMetazone(names, meta, tzKey, metaNames);
names.put(tzid, meta);
if (tzLink != null && availableIds.contains(tzLink)) {
names.put(tzLink, meta);
@@ -1053,28 +1044,15 @@ public class CLDRConverter {
}
}
- // --- code below here is adapted from java.util.Properties ---
- private static final String specialSaveCharsJava = "\"";
- private static final String specialSaveCharsProperties = "=: \t\r\n\f#!";
-
/*
- * Converts unicodes to encoded \uxxxx
- * and writes out any of the characters in specialSaveChars
- * with a preceding slash
+ * Escapes control codes to ASCII escapes or encoded \uxxxx
+ * and writes out ASCII quotation marks with a preceding slash
*/
- static String saveConvert(String theString, boolean useJava) {
+ static String escape(String theString) {
if (theString == null) {
return "";
}
- String specialSaveChars;
- if (useJava) {
- specialSaveChars = specialSaveCharsJava;
- } else {
- specialSaveChars = specialSaveCharsProperties;
- }
- boolean escapeSpace = false;
-
int len = theString.length();
StringBuilder outBuffer = new StringBuilder(len * 2);
Formatter formatter = new Formatter(outBuffer, Locale.ROOT);
@@ -1083,14 +1061,14 @@ public class CLDRConverter {
char aChar = theString.charAt(x);
switch (aChar) {
case ' ':
- if (x == 0 || escapeSpace) {
+ if (x == 0) {
outBuffer.append('\\');
}
outBuffer.append(' ');
break;
- case '\\':
- outBuffer.append('\\');
+ case '\\', '"':
outBuffer.append('\\');
+ outBuffer.append(aChar);
break;
case '\t':
outBuffer.append('\\');
@@ -1109,12 +1087,9 @@ public class CLDRConverter {
outBuffer.append('f');
break;
default:
- if (aChar < 0x0020 || (!USE_UTF8 && aChar > 0x007e)) {
+ if (aChar < 0x0020) {
formatter.format("\\u%04x", (int)aChar);
} else {
- if (specialSaveChars.indexOf(aChar) != -1) {
- outBuffer.append('\\');
- }
outBuffer.append(aChar);
}
}
@@ -1530,11 +1505,18 @@ public class CLDRConverter {
}
}
- private static boolean isDefaultZone(String meta, String tzid) {
+ private static void recordMetazone(Map names, String meta, String tzid, String[] tznames) {
String zone001 = handlerMetaZones.zidMap().get(meta);
var tzLink = getTZDBLink(tzid);
- return canonicalTZMap.getOrDefault(tzid, tzid).equals(zone001) ||
- tzLink != null && canonicalTZMap.getOrDefault(tzLink, tzLink).equals(zone001);
+
+ // Record the metazone names only from the default
+ // (001) zone, with short names filled from TZDB
+ if (canonicalTZMap.getOrDefault(tzid, tzid).equals(zone001) ||
+ tzLink != null && canonicalTZMap.getOrDefault(tzLink, tzLink).equals(zone001)) {
+ tznames = Arrays.copyOf(tznames, tznames.length);
+ fillTZDBShortNames(tzid, tznames);
+ names.put(METAZONE_ID_PREFIX + meta, tznames);
+ }
}
private static String getTZDBLink(String tzid) {
diff --git a/make/jdk/src/classes/build/tools/cldrconverter/ResourceBundleGenerator.java b/make/jdk/src/classes/build/tools/cldrconverter/ResourceBundleGenerator.java
index 8278bf6bcfa..0bc5a2bdb0d 100644
--- a/make/jdk/src/classes/build/tools/cldrconverter/ResourceBundleGenerator.java
+++ b/make/jdk/src/classes/build/tools/cldrconverter/ResourceBundleGenerator.java
@@ -70,9 +70,8 @@ class ResourceBundleGenerator implements BundleGenerator {
private static final String META_VALUE_PREFIX = "metaValue_";
@Override
- public void generateBundle(String packageName, String baseName, String localeID, boolean useJava,
+ public void generateBundle(String packageName, String baseName, String localeID,
Map map, BundleType type) throws IOException {
- String suffix = useJava ? ".java" : ".properties";
String dirName = CLDRConverter.DESTINATION_DIR + File.separator + "sun" + File.separator
+ packageName + File.separator + "resources" + File.separator + "cldr";
packageName = packageName + ".resources.cldr";
@@ -91,23 +90,12 @@ class ResourceBundleGenerator implements BundleGenerator {
if (!dir.exists()) {
dir.mkdirs();
}
- File file = new File(dir, baseName + ("root".equals(localeID) ? "" : "_" + localeID) + suffix);
+ File file = new File(dir, baseName + ("root".equals(localeID) ? "" : "_" + localeID) + ".java");
if (!file.exists()) {
file.createNewFile();
}
CLDRConverter.info("\tWriting file " + file);
- String encoding;
- if (useJava) {
- if (CLDRConverter.USE_UTF8) {
- encoding = "utf-8";
- } else {
- encoding = "us-ascii";
- }
- } else {
- encoding = "iso-8859-1";
- }
-
Formatter fmt = null;
if (type == BundleType.TIMEZONE) {
fmt = new Formatter();
@@ -119,7 +107,7 @@ class ResourceBundleGenerator implements BundleGenerator {
value = (String[]) map.get(key);
fmt.format(" final String[] %s = new String[] {\n", meta);
for (String s : value) {
- fmt.format(" \"%s\",\n", CLDRConverter.saveConvert(s, useJava));
+ fmt.format(" \"%s\",\n", CLDRConverter.escape(s));
}
fmt.format(" };\n");
metaKeys.add(key);
@@ -159,11 +147,11 @@ class ResourceBundleGenerator implements BundleGenerator {
if (val instanceof String[] values) {
fmt.format(" final String[] %s = new String[] {\n", metaVal);
for (String s : values) {
- fmt.format(" \"%s\",\n", CLDRConverter.saveConvert(s, useJava));
+ fmt.format(" \"%s\",\n", CLDRConverter.escape(s));
}
fmt.format(" };\n");
} else {
- fmt.format(" final String %s = \"%s\";\n", metaVal, CLDRConverter.saveConvert((String)val, useJava));
+ fmt.format(" final String %s = \"%s\";\n", metaVal, CLDRConverter.escape((String)val));
}
newMap.put(oldEntry.key, oldEntry.metaKey());
}
@@ -173,55 +161,47 @@ class ResourceBundleGenerator implements BundleGenerator {
map = newMap;
}
- try (PrintWriter out = new PrintWriter(file, encoding)) {
+ try (PrintWriter out = new PrintWriter(file, "utf-8")) {
// Output copyright headers
out.println(getOpenJDKCopyright());
out.println(CopyrightHeaders.getUnicodeCopyright());
- if (useJava) {
- out.println("package sun." + packageName + ";\n");
- out.printf("import %s;\n\n", type.getPathName());
- out.printf("public class %s%s extends %s {\n", baseName, "root".equals(localeID) ? "" : "_" + localeID, type.getClassName());
+ out.println("package sun." + packageName + ";\n");
+ out.printf("import %s;\n\n", type.getPathName());
+ out.printf("public class %s%s extends %s {\n", baseName, "root".equals(localeID) ? "" : "_" + localeID, type.getClassName());
- out.println(" @Override\n" +
- " protected final Object[][] getContents() {");
- if (fmt != null) {
- out.print(fmt.toString());
- }
- out.println(" final Object[][] data = new Object[][] {");
+ out.println(" @Override\n" +
+ " protected final Object[][] getContents() {");
+ if (fmt != null) {
+ out.print(fmt.toString());
}
+ out.println(" final Object[][] data = new Object[][] {");
for (String key : map.keySet()) {
- if (useJava) {
- Object value = map.get(key);
- if (value == null) {
- CLDRConverter.warning("null value for " + key);
- } else if (value instanceof String) {
- String valStr = (String)value;
- if (type == BundleType.TIMEZONE &&
- !(key.startsWith(CLDRConverter.EXEMPLAR_CITY_PREFIX) ||
- key.startsWith(CLDRConverter.METAZONE_DSTOFFSET_PREFIX)) ||
- valStr.startsWith(META_VALUE_PREFIX)) {
- out.printf(" { \"%s\", %s },\n", key, CLDRConverter.saveConvert(valStr, useJava));
- } else {
- out.printf(" { \"%s\", \"%s\" },\n", key, CLDRConverter.saveConvert(valStr, useJava));
- }
- } else if (value instanceof String[]) {
- String[] values = (String[]) value;
- out.println(" { \"" + key + "\",\n new String[] {");
- for (String s : values) {
- out.println(" \"" + CLDRConverter.saveConvert(s, useJava) + "\",");
- }
- out.println(" }\n },");
+ Object value = map.get(key);
+ if (value == null) {
+ CLDRConverter.warning("null value for " + key);
+ } else if (value instanceof String) {
+ String valStr = (String)value;
+ if (type == BundleType.TIMEZONE &&
+ !(key.startsWith(CLDRConverter.EXEMPLAR_CITY_PREFIX) ||
+ key.startsWith(CLDRConverter.METAZONE_DSTOFFSET_PREFIX)) ||
+ valStr.startsWith(META_VALUE_PREFIX)) {
+ out.printf(" { \"%s\", %s },\n", key, CLDRConverter.escape(valStr));
} else {
- throw new RuntimeException("unknown value type: " + value.getClass().getName());
+ out.printf(" { \"%s\", \"%s\" },\n", key, CLDRConverter.escape(valStr));
}
+ } else if (value instanceof String[]) {
+ String[] values = (String[]) value;
+ out.println(" { \"" + key + "\",\n new String[] {");
+ for (String s : values) {
+ out.println(" \"" + CLDRConverter.escape(s) + "\",");
+ }
+ out.println(" }\n },");
} else {
- out.println(key + "=" + CLDRConverter.saveConvert((String) map.get(key), useJava));
+ throw new RuntimeException("unknown value type: " + value.getClass().getName());
}
}
- if (useJava) {
- out.println(" };\n return data;\n }\n}");
- }
+ out.println(" };\n return data;\n }\n}");
}
}
diff --git a/make/modules/java.base/Gensrc.gmk b/make/modules/java.base/Gensrc.gmk
index e8236f0b0e4..675038c8fd5 100644
--- a/make/modules/java.base/Gensrc.gmk
+++ b/make/modules/java.base/Gensrc.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -47,8 +47,6 @@ CLDR_GEN_DONE := $(GENSRC_DIR)/_cldr-gensrc.marker
TZ_DATA_DIR := $(MODULE_SRC)/share/data/tzdata
ZONENAME_TEMPLATE := $(MODULE_SRC)/share/classes/java/time/format/ZoneName.java.template
-# The `-utf8` option is used even for US English, as some names
-# may contain non-ASCII characters, such as “Türkiye”.
$(CLDR_GEN_DONE): $(wildcard $(CLDR_DATA_DIR)/dtd/*.dtd) \
$(wildcard $(CLDR_DATA_DIR)/main/en*.xml) \
$(wildcard $(CLDR_DATA_DIR)/supplemental/*.xml) \
@@ -64,8 +62,7 @@ $(CLDR_GEN_DONE): $(wildcard $(CLDR_DATA_DIR)/dtd/*.dtd) \
-basemodule \
-year $(COPYRIGHT_YEAR) \
-zntempfile $(ZONENAME_TEMPLATE) \
- -tzdatadir $(TZ_DATA_DIR) \
- -utf8)
+ -tzdatadir $(TZ_DATA_DIR))
$(TOUCH) $@
TARGETS += $(CLDR_GEN_DONE)
diff --git a/make/modules/java.desktop/lib/ClientLibraries.gmk b/make/modules/java.desktop/lib/ClientLibraries.gmk
index 3e37fe79643..2326505d11c 100644
--- a/make/modules/java.desktop/lib/ClientLibraries.gmk
+++ b/make/modules/java.desktop/lib/ClientLibraries.gmk
@@ -395,6 +395,8 @@ $(eval $(call SetupJdkLibrary, BUILD_LIBFONTMANAGER, \
AccelGlyphCache.c, \
CFLAGS := $(LIBFONTMANAGER_CFLAGS), \
CXXFLAGS := $(LIBFONTMANAGER_CFLAGS), \
+ CXXFLAGS_gcc := -fno-rtti -fno-exceptions, \
+ CXXFLAGS_clang := -fno-rtti -fno-exceptions, \
OPTIMIZATION := HIGHEST, \
CFLAGS_windows = -DCC_NOEX, \
EXTRA_HEADER_DIRS := $(LIBFONTMANAGER_EXTRA_HEADER_DIRS), \
diff --git a/make/modules/jdk.localedata/Gensrc.gmk b/make/modules/jdk.localedata/Gensrc.gmk
index 93b863df66f..2ff972c7536 100644
--- a/make/modules/jdk.localedata/Gensrc.gmk
+++ b/make/modules/jdk.localedata/Gensrc.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2014, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2014, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -45,8 +45,7 @@ $(CLDR_GEN_DONE): $(wildcard $(CLDR_DATA_DIR)/dtd/*.dtd) \
-baselocales "en-US" \
-year $(COPYRIGHT_YEAR) \
-o $(GENSRC_DIR) \
- -tzdatadir $(TZ_DATA_DIR) \
- -utf8)
+ -tzdatadir $(TZ_DATA_DIR))
$(TOUCH) $@
TARGETS += $(CLDR_GEN_DONE)
diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad
index 53fa4e3066c..7487de2d577 100644
--- a/src/hotspot/cpu/aarch64/aarch64.ad
+++ b/src/hotspot/cpu/aarch64/aarch64.ad
@@ -1986,7 +1986,7 @@ uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *r
assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
- if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
+ if (src_hi != OptoReg::Bad && !bottom_type()->isa_pvectmask()) {
assert((src_lo&1)==0 && src_lo+1==src_hi &&
(dst_lo&1)==0 && dst_lo+1==dst_hi,
"expected aligned-adjacent pairs");
@@ -2001,7 +2001,7 @@ uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *r
int src_offset = ra_->reg2offset(src_lo);
int dst_offset = ra_->reg2offset(dst_lo);
- if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
+ if (bottom_type()->isa_vect() && !bottom_type()->isa_pvectmask()) {
uint ireg = ideal_reg();
DEBUG_ONLY(int algm = MIN2(RegMask::num_registers(ireg), (int)Matcher::stack_alignment_in_slots()) * VMRegImpl::stack_slot_size);
assert((src_lo_rc != rc_stack) || is_aligned(src_offset, algm), "unaligned vector spill sp offset %d (src)", src_offset);
@@ -2149,7 +2149,7 @@ uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *r
} else {
st->print("%s", Matcher::regName[dst_lo]);
}
- if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
+ if (bottom_type()->isa_vect() && !bottom_type()->isa_pvectmask()) {
int vsize = 0;
switch (ideal_reg()) {
case Op_VecD:
@@ -3389,12 +3389,13 @@ encode %{
assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
// load fake address constants using a normal move
if (! __ is_valid_AArch64_address(con) ||
- con < (address)(uintptr_t)os::vm_page_size()) {
+ con < (address)(uintptr_t)os::vm_page_size() ||
+ rtype == relocInfo::none) {
__ mov(dst_reg, con);
} else {
- // no reloc so just use adrp and add
+ // use shorter adrp/add sequence for external_word relocation
uint64_t offset;
- __ adrp(dst_reg, con, offset);
+ __ adrp(dst_reg, Address(con, rtype), offset);
__ add(dst_reg, dst_reg, offset);
}
}
@@ -7708,10 +7709,11 @@ instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
match(Set dst (ReverseBytesUS src));
ins_cost(INSN_COST);
- format %{ "rev16w $dst, $src" %}
+ format %{ "rev16w $dst, $src\t# $dst -> unsigned short" %}
ins_encode %{
__ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
+ __ narrow_subword_type(as_Register($dst$$reg), T_CHAR);
%}
ins_pipe(ialu_reg);
diff --git a/src/hotspot/cpu/aarch64/aarch64_vector.ad b/src/hotspot/cpu/aarch64/aarch64_vector.ad
index 4c854913e63..2ff93c9e288 100644
--- a/src/hotspot/cpu/aarch64/aarch64_vector.ad
+++ b/src/hotspot/cpu/aarch64/aarch64_vector.ad
@@ -311,6 +311,12 @@ source %{
return false;
}
break;
+ case Op_RotateLeftV:
+ case Op_RotateRightV:
+ if (length_in_bytes > 16) {
+ return false; // NEON only, since SLI/USHR are not available in SVE
+ }
+ break;
default:
break;
}
@@ -370,6 +376,11 @@ source %{
case Op_SqrtVHF:
case Op_FmaVHF:
return false;
+ // There's no SLI instruction in SVE, so we can't have an optimal vector
+ // rotate with masking when emitting code for SVE.
+ case Op_RotateLeftV:
+ case Op_RotateRightV:
+ return false;
default:
break;
}
@@ -441,11 +452,11 @@ source %{
// On architectures that do not support predicate, masks are stored in
// general vector registers (TypeVect) with sizes ranging from TypeVectA
// to TypeVectX based on the vector size in bytes.
- assert(vt->isa_vectmask() == nullptr, "mask type is not matched");
+ assert(vt->isa_pvectmask() == nullptr, "mask type is not matched");
return false;
}
- assert(vt->isa_vectmask() != nullptr, "expected TypeVectMask on SVE");
+ assert(vt->isa_pvectmask() != nullptr, "expected TypePVectMask on SVE");
switch (opcode) {
case Op_VectorMaskToLong:
case Op_VectorLongToMask:
@@ -3229,6 +3240,25 @@ instruct vlsra_imm(vReg dst, vReg src, immI_positive shift) %{
ins_pipe(pipe_slow);
%}
+// vector rotate with constant shift count (NEON only)
+// Uses USHR+SLI 2-instruction sequence instead of SHL+USHR+ORR 3-instruction decomposition.
+
+instruct vrotateconstant(vReg dst, vReg src, immI shift) %{
+ predicate(Matcher::vector_length_in_bytes(n) <= 16);
+ match(Set dst (RotateLeftV src shift));
+ match(Set dst (RotateRightV src shift));
+ effect(TEMP_DEF dst);
+ format %{ "vrotateconstant $dst, $src, $shift" %}
+ ins_encode %{
+ int opc = this->ideal_Opcode();
+ int raw_shift = checked_cast(opc == Op_RotateLeftV ?
+ $shift$$constant : -$shift$$constant);
+ __ neon_vector_rotate($dst$$FloatRegister, get_arrangement(this),
+ $src$$FloatRegister, raw_shift);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
// vector shift - predicated
instruct vlsl_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
diff --git a/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 b/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4
index 58ed234194a..c5df949dfb6 100644
--- a/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4
+++ b/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4
@@ -301,6 +301,12 @@ source %{
return false;
}
break;
+ case Op_RotateLeftV:
+ case Op_RotateRightV:
+ if (length_in_bytes > 16) {
+ return false; // NEON only, since SLI/USHR are not available in SVE
+ }
+ break;
default:
break;
}
@@ -360,6 +366,11 @@ source %{
case Op_SqrtVHF:
case Op_FmaVHF:
return false;
+ // There's no SLI instruction in SVE, so we can't have an optimal vector
+ // rotate with masking when emitting code for SVE.
+ case Op_RotateLeftV:
+ case Op_RotateRightV:
+ return false;
default:
break;
}
@@ -431,11 +442,11 @@ source %{
// On architectures that do not support predicate, masks are stored in
// general vector registers (TypeVect) with sizes ranging from TypeVectA
// to TypeVectX based on the vector size in bytes.
- assert(vt->isa_vectmask() == nullptr, "mask type is not matched");
+ assert(vt->isa_pvectmask() == nullptr, "mask type is not matched");
return false;
}
- assert(vt->isa_vectmask() != nullptr, "expected TypeVectMask on SVE");
+ assert(vt->isa_pvectmask() != nullptr, "expected TypePVectMask on SVE");
switch (opcode) {
case Op_VectorMaskToLong:
case Op_VectorLongToMask:
@@ -567,13 +578,9 @@ instruct vloadcon(vReg dst, immI0 src) %{
BasicType bt = Matcher::vector_element_basic_type(this);
if (UseSVE == 0) {
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
+ int entry_idx = __ vector_iota_entry_index(bt);
assert(length_in_bytes <= 16, "must be");
- // The iota indices are ordered by type B/S/I/L/F/D, and the offset between two types is 16.
- int offset = exact_log2(type2aelembytes(bt)) << 4;
- if (is_floating_point_type(bt)) {
- offset += 32;
- }
- __ lea(rscratch1, ExternalAddress(StubRoutines::aarch64::vector_iota_indices() + offset));
+ __ lea(rscratch1, ExternalAddress(StubRoutines::aarch64::vector_iota_indices(entry_idx)));
if (length_in_bytes == 16) {
__ ldrq($dst$$FloatRegister, rscratch1);
} else {
@@ -1969,6 +1976,25 @@ instruct vlsra_imm(vReg dst, vReg src, immI_positive shift) %{
ins_pipe(pipe_slow);
%}
+// vector rotate with constant shift count (NEON only)
+// Uses USHR+SLI 2-instruction sequence instead of SHL+USHR+ORR 3-instruction decomposition.
+
+instruct vrotateconstant(vReg dst, vReg src, immI shift) %{
+ predicate(Matcher::vector_length_in_bytes(n) <= 16);
+ match(Set dst (RotateLeftV src shift));
+ match(Set dst (RotateRightV src shift));
+ effect(TEMP_DEF dst);
+ format %{ "vrotateconstant $dst, $src, $shift" %}
+ ins_encode %{
+ int opc = this->ideal_Opcode();
+ int raw_shift = checked_cast(opc == Op_RotateLeftV ?
+ $shift$$constant : -$shift$$constant);
+ __ neon_vector_rotate($dst$$FloatRegister, get_arrangement(this),
+ $src$$FloatRegister, raw_shift);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
dnl
dnl VSHIFT_PREDICATE($1, $2, $3 )
dnl VSHIFT_PREDICATE(type, op_name, insn)
diff --git a/src/hotspot/cpu/aarch64/assembler_aarch64.hpp b/src/hotspot/cpu/aarch64/assembler_aarch64.hpp
index 4c1c8d9bbc8..c8d5ee2eaeb 100644
--- a/src/hotspot/cpu/aarch64/assembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/assembler_aarch64.hpp
@@ -1,6 +1,7 @@
/*
* Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2024, Red Hat Inc. All rights reserved.
+ * Copyright 2026 Arm Limited and/or its affiliates.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1254,6 +1255,13 @@ public:
sz, 0b000, ordered);
}
+ void load_store_volatile(Register data, BasicType type, Register addr,
+ bool is_load) {
+ load_store_exclusive(dummy_reg, data, dummy_reg, addr,
+ (Assembler::operand_size)exact_log2(type2aelembytes(type)),
+ is_load ? 0b110 : 0b100, /* ordered = */ true);
+ }
+
#define INSN4(NAME, sz, op, o0) /* Four registers */ \
void NAME(Register Rs, Register Rt1, Register Rt2, Register Rn) { \
guarantee(Rs != Rn && Rs != Rt1 && Rs != Rt2, "unpredictable instruction"); \
diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
index 4de6237304d..4eb4e3d5ac7 100644
--- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
@@ -1,6 +1,7 @@
/*
* Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
+ * Copyright 2026 Arm Limited and/or its affiliates.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -911,8 +912,15 @@ void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
reg2stack(temp, dest, dest->type());
}
+void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type,
+ LIR_PatchCode patch_code, CodeEmitInfo* info,
+ bool wide) {
+ mem2reg(src, dest, type, patch_code, info, wide, false);
+}
-void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
+void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type,
+ LIR_PatchCode patch_code, CodeEmitInfo* info,
+ bool wide, bool is_volatile) {
LIR_Address* addr = src->as_address_ptr();
LIR_Address* from_addr = src->as_address_ptr();
@@ -925,10 +933,27 @@ void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_Patch
return;
}
+ if (is_volatile) {
+ load_volatile(from_addr, dest, type, info);
+ } else {
+ load_unordered(from_addr, dest, type, wide, info);
+ }
+
+ if (is_reference_type(type)) {
+ if (UseCompressedOops && !wide) {
+ __ decode_heap_oop(dest->as_register());
+ }
+
+ __ verify_oop(dest->as_register());
+ }
+}
+
+void LIR_Assembler::load_unordered(LIR_Address *from_addr, LIR_Opr dest,
+ BasicType type, bool wide, CodeEmitInfo* info) {
if (info != nullptr) {
add_debug_info_for_null_check_here(info);
}
- int null_check_here = code_offset();
+
switch (type) {
case T_FLOAT: {
__ ldrs(dest->as_float_reg(), as_Address(from_addr));
@@ -986,16 +1011,44 @@ void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_Patch
default:
ShouldNotReachHere();
}
-
- if (is_reference_type(type)) {
- if (UseCompressedOops && !wide) {
- __ decode_heap_oop(dest->as_register());
- }
-
- __ verify_oop(dest->as_register());
- }
}
+void LIR_Assembler::load_volatile(LIR_Address *from_addr, LIR_Opr dest,
+ BasicType type, CodeEmitInfo* info) {
+ __ lea(rscratch1, as_Address(from_addr));
+
+ Register dest_reg = rscratch2;
+ if (!is_floating_point_type(type)) {
+ dest_reg = (dest->is_single_cpu()
+ ? dest->as_register() : dest->as_register_lo());
+ }
+
+ if (info != nullptr) {
+ add_debug_info_for_null_check_here(info);
+ }
+
+ // Uses LDAR to ensure memory ordering.
+ __ load_store_volatile(dest_reg, type, rscratch1, /*is_load*/true);
+
+ switch (type) {
+ // LDAR is unsigned so need to sign-extend for byte and short
+ case T_BYTE:
+ __ sxtb(dest_reg, dest_reg);
+ break;
+ case T_SHORT:
+ __ sxth(dest_reg, dest_reg);
+ break;
+ // need to move from GPR to FPR after LDAR with FMOV for floating types
+ case T_FLOAT:
+ __ fmovs(dest->as_float_reg(), dest_reg);
+ break;
+ case T_DOUBLE:
+ __ fmovd(dest->as_double_reg(), dest_reg);
+ break;
+ default:
+ break;
+ }
+}
int LIR_Assembler::array_element_size(BasicType type) const {
int elem_size = type2aelembytes(type);
@@ -2764,7 +2817,9 @@ void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* arg
}
void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
- if (dest->is_address() || src->is_address()) {
+ if (src->is_address()) {
+ mem2reg(src, dest, type, lir_patch_none, info, /*wide*/false, /*is_volatile*/true);
+ } else if (dest->is_address()) {
move_op(src, dest, type, lir_patch_none, info, /*wide*/false);
} else {
ShouldNotReachHere();
diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.hpp
index 5af06fc6a1c..367256d2f69 100644
--- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.hpp
@@ -57,6 +57,12 @@ friend class ArrayCopyStub;
void casw(Register addr, Register newval, Register cmpval);
void casl(Register addr, Register newval, Register cmpval);
+ void mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type,
+ LIR_PatchCode patch_code,
+ CodeEmitInfo* info, bool wide, bool is_volatile);
+ void load_unordered(LIR_Address *from_addr, LIR_Opr dest, BasicType type, bool wide, CodeEmitInfo* info);
+ void load_volatile(LIR_Address *from_addr, LIR_Opr dest, BasicType type, CodeEmitInfo* info);
+
static const int max_tableswitches = 20;
struct tableswitch switches[max_tableswitches];
int tableswitch_count;
diff --git a/src/hotspot/cpu/aarch64/c1_LIRGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRGenerator_aarch64.cpp
index f10c5197d91..7e82f410a95 100644
--- a/src/hotspot/cpu/aarch64/c1_LIRGenerator_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c1_LIRGenerator_aarch64.cpp
@@ -1398,14 +1398,5 @@ void LIRGenerator::volatile_field_store(LIR_Opr value, LIR_Address* address,
void LIRGenerator::volatile_field_load(LIR_Address* address, LIR_Opr result,
CodeEmitInfo* info) {
- // 8179954: We need to make sure that the code generated for
- // volatile accesses forms a sequentially-consistent set of
- // operations when combined with STLR and LDAR. Without a leading
- // membar it's possible for a simple Dekker test to fail if loads
- // use LD;DMB but stores use STLR. This can happen if C2 compiles
- // the stores in one method and C1 compiles the loads in another.
- if (!CompilerConfig::is_c1_only_no_jvmci()) {
- __ membar();
- }
__ volatile_load_mem_reg(address, result, info);
}
diff --git a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
index 3c179f21c14..67dc4966d64 100644
--- a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
@@ -265,7 +265,7 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box, Register t1,
// Check if object matches.
ldr(t3, Address(t1_monitor, ObjectMonitor::object_offset()));
BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
- bs_asm->try_resolve_weak_handle_in_c2(this, t3, t2, slow_path);
+ bs_asm->try_peek_weak_handle_in_nmethod(this, t3, t3, t2, slow_path);
cmp(t3, obj);
br(Assembler::NE, slow_path);
diff --git a/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.cpp
index 2a78d688097..38efcf80650 100644
--- a/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.cpp
@@ -393,6 +393,11 @@ void BarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register
__ cbz(obj, error); // if klass is null it is broken
}
+void BarrierSetAssembler::try_peek_weak_handle_in_nmethod(MacroAssembler* masm, Register weak_handle, Register obj, Register tmp, Label& slow_path) {
+ // Load the oop from the weak handle without barriers.
+ __ ldr(obj, Address(weak_handle));
+}
+
#ifdef COMPILER2
OptoReg::Name BarrierSetAssembler::encode_float_vector_register_size(const Node* node, OptoReg::Name opto_reg) {
@@ -440,12 +445,6 @@ OptoReg::Name BarrierSetAssembler::refine_register(const Node* node, OptoReg::Na
return opto_reg;
}
-
-void BarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path) {
- // Load the oop from the weak handle.
- __ ldr(obj, Address(obj));
-}
-
#undef __
#define __ _masm->
diff --git a/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.hpp
index c2581b2f962..c8ed7941983 100644
--- a/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.hpp
@@ -130,12 +130,15 @@ public:
static void clear_patching_epoch();
static void increment_patching_epoch();
+ // See AS_NO_KEEPALIVE for peek semantics
+ // weak_handle and obj may alias
+ virtual void try_peek_weak_handle_in_nmethod(MacroAssembler* masm, Register weak_handle, Register obj, Register tmp, Label& slow_path);
+
#ifdef COMPILER2
OptoReg::Name encode_float_vector_register_size(const Node* node,
OptoReg::Name opto_reg);
OptoReg::Name refine_register(const Node* node,
OptoReg::Name opto_reg);
- virtual void try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path);
#endif // COMPILER2
};
diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/c1/shenandoahBarrierSetC1_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shenandoah/c1/shenandoahBarrierSetC1_aarch64.cpp
deleted file mode 100644
index e4db8a9ab1f..00000000000
--- a/src/hotspot/cpu/aarch64/gc/shenandoah/c1/shenandoahBarrierSetC1_aarch64.cpp
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved.
- * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#include "c1/c1_LIRAssembler.hpp"
-#include "c1/c1_MacroAssembler.hpp"
-#include "compiler/compilerDefinitions.inline.hpp"
-#include "gc/shared/gc_globals.hpp"
-#include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
-#include "gc/shenandoah/shenandoahBarrierSet.hpp"
-#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
-
-#define __ masm->masm()->
-
-void LIR_OpShenandoahCompareAndSwap::emit_code(LIR_Assembler* masm) {
- Register addr = _addr->as_register_lo();
- Register newval = _new_value->as_register();
- Register cmpval = _cmp_value->as_register();
- Register tmp1 = _tmp1->as_register();
- Register tmp2 = _tmp2->as_register();
- Register result = result_opr()->as_register();
-
- if (UseCompressedOops) {
- __ encode_heap_oop(tmp1, cmpval);
- cmpval = tmp1;
- __ encode_heap_oop(tmp2, newval);
- newval = tmp2;
- }
-
- ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm->masm(), addr, cmpval, newval, /*acquire*/ true, /*release*/ true, /*is_cae*/ false, result);
-
- if (CompilerConfig::is_c1_only_no_jvmci()) {
- // The membar here is necessary to prevent reordering between the
- // release store in the CAS above and a subsequent volatile load.
- // However for tiered compilation C1 inserts a full barrier before
- // volatile loads which means we don't need an additional barrier
- // here (see LIRGenerator::volatile_field_load()).
- __ membar(__ AnyAny);
- }
-}
-
-#undef __
-
-#ifdef ASSERT
-#define __ gen->lir(__FILE__, __LINE__)->
-#else
-#define __ gen->lir()->
-#endif
-
-LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value) {
- BasicType bt = access.type();
- if (access.is_oop()) {
- LIRGenerator *gen = access.gen();
- if (ShenandoahSATBBarrier) {
- pre_barrier(gen, access.access_emit_info(), access.decorators(), access.resolved_addr(),
- LIR_OprFact::illegalOpr /* pre_val */);
- }
- if (ShenandoahCASBarrier) {
- cmp_value.load_item();
- new_value.load_item();
-
- LIR_Opr t1 = gen->new_register(T_OBJECT);
- LIR_Opr t2 = gen->new_register(T_OBJECT);
- LIR_Opr addr = access.resolved_addr()->as_address_ptr()->base();
- LIR_Opr result = gen->new_register(T_INT);
-
- __ append(new LIR_OpShenandoahCompareAndSwap(addr, cmp_value.result(), new_value.result(), t1, t2, result));
-
- if (ShenandoahCardBarrier) {
- post_barrier(access, access.resolved_addr(), new_value.result());
- }
- return result;
- }
- }
- return BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value);
-}
-
-LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value) {
- LIRGenerator* gen = access.gen();
- BasicType type = access.type();
-
- LIR_Opr result = gen->new_register(type);
- value.load_item();
- LIR_Opr value_opr = value.result();
-
- assert(type == T_INT || is_reference_type(type) LP64_ONLY( || type == T_LONG ), "unexpected type");
- LIR_Opr tmp = gen->new_register(T_INT);
- __ xchg(access.resolved_addr(), value_opr, result, tmp);
-
- if (access.is_oop()) {
- result = load_reference_barrier(access.gen(), result, LIR_OprFact::addressConst(0), access.decorators());
- LIR_Opr tmp = gen->new_register(type);
- __ move(result, tmp);
- result = tmp;
- if (ShenandoahSATBBarrier) {
- pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr,
- result /* pre_val */);
- }
- if (ShenandoahCardBarrier) {
- post_barrier(access, access.resolved_addr(), result);
- }
- }
-
- return result;
-}
diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp
index 2f7707227b4..56835e799f0 100644
--- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp
@@ -57,6 +57,8 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec
__ cbz(count, done);
// Is GC active?
+ assert(!saved_regs.contains(rscratch1), "Sanity: about to clobber rscratch1");
+ assert(!saved_regs.contains(rscratch2), "Sanity: about to clobber rscratch2");
Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
__ ldrb(rscratch1, gc_state);
if (ShenandoahSATBBarrier && dest_uninitialized) {
@@ -67,13 +69,26 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec
__ br(Assembler::EQ, done);
}
- __ push(saved_regs, sp);
- if (UseCompressedOops) {
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop), src, dst, count);
- } else {
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop), src, dst, count);
+ __ push_call_clobbered_registers();
+ // If arguments are not in proper places, shuffle them.
+ // Doing this via the stack is the most straight-forward way to avoid
+ // accidentally smashing any register.
+ if (c_rarg0 != src || c_rarg1 != dst || c_rarg2 != count) {
+ __ push(RegSet::of(src), sp);
+ __ push(RegSet::of(dst), sp);
+ __ push(RegSet::of(count), sp);
+ __ pop(RegSet::of(c_rarg2), sp);
+ __ pop(RegSet::of(c_rarg1), sp);
+ __ pop(RegSet::of(c_rarg0), sp);
}
- __ pop(saved_regs, sp);
+ address target = nullptr;
+ if (UseCompressedOops) {
+ target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop);
+ } else {
+ target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop);
+ }
+ __ call_VM_leaf(target, 3);
+ __ pop_call_clobbered_registers();
__ bind(done);
}
}
@@ -91,15 +106,8 @@ void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler* masm,
Register pre_val,
Register thread,
Register tmp1,
- Register tmp2,
- bool tosca_live,
- bool expand_call) {
+ Register tmp2) {
assert(ShenandoahSATBBarrier, "Should be checked by caller");
-
- // If expand_call is true then we expand the call_VM_leaf macro
- // directly to skip generating the check by
- // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
-
assert(thread == rthread, "must be");
Label done;
@@ -118,7 +126,12 @@ void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler* masm,
// Do we need to load the previous value?
if (obj != noreg) {
- __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
+ if (UseCompressedOops) {
+ __ ldrw(pre_val, Address(obj, 0));
+ __ decode_heap_oop(pre_val);
+ } else {
+ __ ldr(pre_val, Address(obj, 0));
+ }
}
// Is the previous value null?
@@ -142,33 +155,17 @@ void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler* masm,
__ b(done);
__ bind(runtime);
- // save the live input values
- RegSet saved = RegSet::of(pre_val);
- if (tosca_live) saved += RegSet::of(r0);
- if (obj != noreg) saved += RegSet::of(obj);
- __ push(saved, sp);
-
- // Calling the runtime using the regular call_VM_leaf mechanism generates
- // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
- // that checks that the *(rfp+frame::interpreter_frame_last_sp) == nullptr.
- //
- // If we care generating the pre-barrier without a frame (e.g. in the
- // intrinsified Reference.get() routine) then rfp might be pointing to
- // the caller frame and so this check will most likely fail at runtime.
- //
- // Expanding the call directly bypasses the generation of the check.
- // So when we do not have have a full interpreter frame on the stack
- // expand_call should be passed true.
-
- if (expand_call) {
- assert(pre_val != c_rarg1, "smashed arg");
- __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), pre_val);
- } else {
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), pre_val);
+ // Slow-path call
+ __ enter(/* strip_ret_addr = */ true);
+ __ push_call_clobbered_registers();
+ if (c_rarg0 != pre_val) {
+ __ mov(c_rarg0, pre_val);
}
-
- __ pop(saved, sp);
+ // Calling with super_call_VM_leaf with c_rarg0 bypasses interpreter checks and avoids any moves.
+ __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), c_rarg0);
+ __ pop_call_clobbered_registers();
+ __ leave();
__ bind(done);
}
@@ -232,7 +229,6 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
bool is_narrow = UseCompressedOops && !is_native;
Label heap_stable, not_cset;
- __ enter(/*strip_ret_addr*/true);
Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
__ ldrb(rscratch2, gc_state);
@@ -261,35 +257,47 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
// Test for in-cset
if (is_strong) {
- __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
- __ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint());
+ if (AOTCodeCache::is_on_for_dump()) {
+ __ lea(rscratch2, ExternalAddress(AOTRuntimeConstants::cset_base_address()));
+ __ ldr(rscratch2, Address(rscratch2));
+ __ lea(rscratch1, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
+ __ ldrw(rscratch1, Address(rscratch1));
+ __ lsrv(rscratch1, r0, rscratch1);
+ } else {
+ __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
+ __ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint());
+ }
__ ldrb(rscratch2, Address(rscratch2, rscratch1));
__ tbz(rscratch2, 0, not_cset);
}
+ // Slow-path call
+ __ enter(/* strip_ret_addr = */ true);
__ push_call_clobbered_registers();
+ address target = nullptr;
if (is_strong) {
if (is_narrow) {
- __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow));
+ target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
} else {
- __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong));
+ target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
}
} else if (is_weak) {
if (is_narrow) {
- __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow));
+ target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
} else {
- __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
+ target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
}
} else {
assert(is_phantom, "only remaining strength");
assert(!is_narrow, "phantom access cannot be narrow");
- // AOT saved adapters need relocation for this call.
- __ lea(lr, RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom)));
+ target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
}
- __ blr(lr);
+ // Calling with super_call_VM_leaf with c_rarg0/1 bypasses interpreter checks and avoids any moves.
+ __ super_call_VM_leaf(target, c_rarg0, c_rarg1);
__ mov(rscratch1, r0);
__ pop_call_clobbered_registers();
__ mov(r0, rscratch1);
+ __ leave();
__ bind(not_cset);
@@ -297,7 +305,6 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
__ pop(to_save, sp);
__ bind(heap_stable);
- __ leave();
}
//
@@ -347,18 +354,12 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet d
// 3: apply keep-alive barrier if needed
if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
- __ enter(/*strip_ret_addr*/true);
- __ push_call_clobbered_registers();
satb_barrier(masm /* masm */,
noreg /* obj */,
dst /* pre_val */,
rthread /* thread */,
tmp1 /* tmp1 */,
- tmp2 /* tmp2 */,
- true /* tosca_live */,
- true /* expand_call */);
- __ pop_call_clobbered_registers();
- __ leave();
+ tmp2 /* tmp2 */);
}
}
@@ -400,8 +401,6 @@ void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet
__ lea(tmp3, dst);
}
- bool storing_non_null = (val != noreg);
-
// 2: pre-barrier: SATB needs the previous value
if (ShenandoahBarrierSet::need_satb_barrier(decorators, type)) {
satb_barrier(masm,
@@ -409,15 +408,14 @@ void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet
tmp2 /* pre_val */,
rthread /* thread */,
tmp1 /* tmp */,
- rscratch1 /* tmp2 */,
- storing_non_null /* tosca_live */,
- false /* expand_call */);
+ rscratch1 /* tmp2 */);
}
// Store!
BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
// 3: post-barrier: card barrier needs store address
+ bool storing_non_null = (val != noreg);
if (ShenandoahBarrierSet::need_card_barrier(decorators, type) && storing_non_null) {
card_barrier(masm, tmp3);
}
@@ -443,15 +441,15 @@ void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler
__ bind(done);
}
-#ifdef COMPILER2
-void ShenandoahBarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj,
- Register tmp, Label& slow_path) {
- assert_different_registers(obj, tmp);
+void ShenandoahBarrierSetAssembler::try_peek_weak_handle_in_nmethod(MacroAssembler* masm, Register weak_handle, Register obj,
+ Register tmp, Label& slow_path) {
+ assert_different_registers(weak_handle, tmp, noreg);
+ assert_different_registers(obj, tmp, noreg);
Label done;
- // Resolve weak handle using the standard implementation.
- BarrierSetAssembler::try_resolve_weak_handle_in_c2(masm, obj, tmp, slow_path);
+ // Peek weak handle using the standard implementation.
+ BarrierSetAssembler::try_peek_weak_handle_in_nmethod(masm, weak_handle, obj, tmp, slow_path);
// Check if the reference is null, and if it is, take the fast path.
__ cbz(obj, done);
@@ -465,7 +463,6 @@ void ShenandoahBarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler
__ tbnz(tmp, ShenandoahHeap::WEAK_ROOTS_BITPOS, slow_path);
__ bind(done);
}
-#endif
// Special Shenandoah CAS implementation that handles false negatives due
// to concurrent evacuation. The service is more complex than a
@@ -709,8 +706,16 @@ void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assemble
if (is_strong) {
// Check for object in cset.
- __ mov(tmp2, ShenandoahHeap::in_cset_fast_test_addr());
- __ lsr(tmp1, res, ShenandoahHeapRegion::region_size_bytes_shift_jint());
+ if (AOTCodeCache::is_on_for_dump()) {
+ __ lea(tmp2, ExternalAddress(AOTRuntimeConstants::cset_base_address()));
+ __ ldr(tmp2, Address(tmp2));
+ __ lea(tmp1, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
+ __ ldrw(tmp1, Address(tmp1));
+ __ lsrv(tmp1, res, tmp1);
+ } else {
+ __ mov(tmp2, ShenandoahHeap::in_cset_fast_test_addr());
+ __ lsr(tmp1, res, ShenandoahHeapRegion::region_size_bytes_shift_jint());
+ }
__ ldrb(tmp2, Address(tmp2, tmp1));
__ cbz(tmp2, *stub->continuation());
}
@@ -795,25 +800,25 @@ void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_s
bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
if (is_strong) {
if (is_native) {
- __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong));
+ __ lea(lr, RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong)));
} else {
if (UseCompressedOops) {
- __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow));
+ __ lea(lr, RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow)));
} else {
- __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong));
+ __ lea(lr, RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong)));
}
}
} else if (is_weak) {
assert(!is_native, "weak must not be called off-heap");
if (UseCompressedOops) {
- __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow));
+ __ lea(lr, RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow)));
} else {
- __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
+ __ lea(lr, RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak)));
}
} else {
assert(is_phantom, "only remaining strength");
assert(is_native, "phantom must only be called off-heap");
- __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom));
+ __ lea(lr, RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom)));
}
__ blr(lr);
__ mov(rscratch1, r0);
diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp
index d5d5ce8950e..3f96177d009 100644
--- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp
@@ -46,9 +46,7 @@ private:
Register pre_val,
Register thread,
Register tmp1,
- Register tmp2,
- bool tosca_live,
- bool expand_call);
+ Register tmp2);
void card_barrier(MacroAssembler* masm, Register obj);
@@ -80,9 +78,8 @@ public:
Address dst, Register val, Register tmp1, Register tmp2, Register tmp3);
virtual void try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
Register obj, Register tmp, Label& slowpath);
-#ifdef COMPILER2
- virtual void try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path);
-#endif
+ virtual void try_peek_weak_handle_in_nmethod(MacroAssembler* masm, Register weak_handle, Register obj,
+ Register tmp, Label& slow_path);
void cmpxchg_oop(MacroAssembler* masm, Register addr, Register expected, Register new_val,
bool acquire, bool release, bool is_cae, Register result);
};
diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp
index f0885fee93d..1eb96cdb6e7 100644
--- a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp
@@ -1328,12 +1328,20 @@ void ZStoreBarrierStubC2Aarch64::emit_code(MacroAssembler& masm) {
register_stub(this);
}
+#undef __
+
+#endif // COMPILER2
+
#undef __
#define __ masm->
-void ZBarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path) {
- // Resolve weak handle using the standard implementation.
- BarrierSetAssembler::try_resolve_weak_handle_in_c2(masm, obj, tmp, slow_path);
+void ZBarrierSetAssembler::try_peek_weak_handle_in_nmethod(MacroAssembler* masm, Register weak_handle, Register obj,
+ Register tmp, Label& slow_path) {
+ assert_different_registers(weak_handle, tmp, noreg);
+ assert_different_registers(obj, tmp, noreg);
+
+ // Peek weak handle using the standard implementation.
+ BarrierSetAssembler::try_peek_weak_handle_in_nmethod(masm, weak_handle, obj, tmp, slow_path);
// Check if the oop is bad, in which case we need to take the slow path.
__ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatMarkBadBeforeMov);
@@ -1345,13 +1353,6 @@ void ZBarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler* masm, R
__ lsr(obj, obj, ZPointerLoadShift);
}
-#undef __
-
-#endif // COMPILER2
-
-#undef __
-#define __ masm->
-
void ZBarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) {
// C1 calls verfy_oop in the middle of barriers, before they have been uncolored
// and after being colored. Therefore, we must deal with colored oops as well.
diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp
index fbbc5c1b517..a8fcb7696b4 100644
--- a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp
@@ -191,9 +191,10 @@ public:
ZLoadBarrierStubC2* stub) const;
void generate_c2_store_barrier_stub(MacroAssembler* masm,
ZStoreBarrierStubC2* stub) const;
- void try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path);
#endif // COMPILER2
+ void try_peek_weak_handle_in_nmethod(MacroAssembler* masm, Register weak_handle, Register obj, Register tmp, Label& slow_path);
+
void check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error);
};
diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
index 7bec0a3c0ca..8f5eb702714 100644
--- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
@@ -2815,6 +2815,17 @@ void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in
}
}
+void MacroAssembler::narrow_subword_type(Register reg, BasicType bt) {
+ assert(is_subword_type(bt), "required");
+ switch (bt) {
+ case T_BOOLEAN: andw(reg, reg, 1); break;
+ case T_BYTE: sxtbw(reg, reg); break;
+ case T_CHAR: uxthw(reg, reg); break;
+ case T_SHORT: sxthw(reg, reg); break;
+ default: ShouldNotReachHere();
+ }
+}
+
void MacroAssembler::decrementw(Register reg, int value)
{
if (value < 0) { incrementw(reg, -value); return; }
@@ -6724,13 +6735,14 @@ void MacroAssembler::java_round_float(Register dst, FloatRegister src,
// by the call to JavaThread::aarch64_get_thread_helper() or, indeed,
// the call setup code.
//
-// On Linux, aarch64_get_thread_helper() clobbers only r0, r1, and flags.
+// On Linux and Windows, aarch64_get_thread_helper() is implemented in
+// assembly and clobbers only r0, r1, and flags.
// On other systems, the helper is a usual C function.
//
void MacroAssembler::get_thread(Register dst) {
RegSet saved_regs =
- LINUX_ONLY(RegSet::range(r0, r1) + lr - dst)
- NOT_LINUX (RegSet::range(r0, r17) + lr - dst);
+ BSD_ONLY(RegSet::range(r0, r17) + lr - dst)
+ NOT_BSD (RegSet::range(r0, r1) + lr - dst);
protect_return_address();
push(saved_regs, sp);
@@ -7263,3 +7275,20 @@ void MacroAssembler::fast_unlock(Register obj, Register t1, Register t2, Registe
bind(unlocked);
}
+
+// Rotate using USHR and SLI instructions (or copy, if rotate count is zero)
+void MacroAssembler::neon_vector_rotate(FloatRegister dst, SIMD_Arrangement T,
+ FloatRegister src, int shift_amount) {
+ assert(src != dst, "did not expect src and dst to be the same register");
+
+ int esize = BitsPerByte << (T / 2);
+ int lshift = shift_amount & (esize - 1);
+
+ if (lshift == 0) {
+ // T & 1 == 0 => 64-bit arrangements, else 128-bit arrangements
+ orr(dst, (T & 1) == 0 ? T8B : T16B, src, src);
+ } else {
+ ushr(dst, T, src, esize - lshift);
+ sli(dst, T, src, lshift);
+ }
+}
diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
index a6cc862d05c..c02df666a87 100644
--- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
@@ -33,6 +33,7 @@
#include "oops/compressedOops.hpp"
#include "oops/compressedKlass.hpp"
#include "runtime/vm_version.hpp"
+#include "utilities/globalDefinitions.hpp"
#include "utilities/powerOfTwo.hpp"
class OopMap;
@@ -719,6 +720,9 @@ public:
// Support for sign-extension (hi:lo = extend_sign(lo))
void extend_sign(Register hi, Register lo);
+ // Clean up a subword typed value to the representation in compliance with JVMS §2.3
+ void narrow_subword_type(Register reg, BasicType bt);
+
// Load and store values by size and signed-ness
void load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed);
void store_sized_value(Address dst, Register src, size_t size_in_bytes);
@@ -1623,6 +1627,10 @@ public:
const FloatRegister (&stateVectors)[16], int idx1, int idx2,
int idx3, int idx4);
+ // Rotate using ORR (for identity) or USHR + SLI.
+ void neon_vector_rotate(FloatRegister dst, SIMD_Arrangement T,
+ FloatRegister src, int shift_amount);
+
// Place an ISB after code may have been modified due to a safepoint.
void safepoint_isb();
diff --git a/src/hotspot/cpu/aarch64/matcher_aarch64.hpp b/src/hotspot/cpu/aarch64/matcher_aarch64.hpp
index 0fbc2ef141e..aa5a1df5cb0 100644
--- a/src/hotspot/cpu/aarch64/matcher_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/matcher_aarch64.hpp
@@ -141,8 +141,10 @@
}
// Does the CPU supports vector constant rotate instructions?
+ // NEON supports constant rotates via USHR+SLI (2-instruction sequence).
+ // The shift value will be masked to the element width in the .ad rule.
static constexpr bool supports_vector_constant_rotates(int shift) {
- return false;
+ return true;
}
// Does the CPU supports vector unsigned comparison instructions?
diff --git a/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp b/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp
index 69769fb8441..4c64b265d92 100644
--- a/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp
@@ -3479,7 +3479,6 @@ void TemplateTable::invokeinterface(int byte_no) {
__ bind(notVFinal);
// Get receiver klass into r3
- __ restore_locals();
__ load_klass(r3, r2);
Label no_such_method;
diff --git a/src/hotspot/cpu/aarch64/vmStructs_aarch64.hpp b/src/hotspot/cpu/aarch64/vmStructs_aarch64.hpp
index 2ec901f6a2e..a9f2d2ff2aa 100644
--- a/src/hotspot/cpu/aarch64/vmStructs_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/vmStructs_aarch64.hpp
@@ -41,7 +41,7 @@
#define VM_LONG_CONSTANTS_CPU(declare_constant, declare_preprocessor_constant)
-#define DECLARE_INT_CPU_FEATURE_CONSTANT(id, name, bit) GENERATE_VM_INT_CONSTANT_ENTRY(VM_Version::CPU_##id)
+#define DECLARE_INT_CPU_FEATURE_CONSTANT(id, name) GENERATE_VM_INT_CONSTANT_ENTRY(VM_Version::CPU_##id)
#define VM_INT_CPU_FEATURE_CONSTANTS CPU_FEATURE_FLAGS(DECLARE_INT_CPU_FEATURE_CONSTANT)
#endif // CPU_AARCH64_VMSTRUCTS_AARCH64_HPP
diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
index 441bd4859fe..d1cf8b6feed 100644
--- a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
@@ -57,7 +57,9 @@ bool VM_Version::_cache_dic_enabled;
bool VM_Version::_cache_idc_enabled;
bool VM_Version::_ic_ivau_trapped;
-const char* VM_Version::_features_names[MAX_CPU_FEATURES] = { nullptr };
+#define DECLARE_CPU_FEATURE_NAME(id, name) XSTR(name),
+const char* VM_Version::_features_names[] = { CPU_FEATURE_FLAGS(DECLARE_CPU_FEATURE_NAME)};
+#undef DECLARE_CPU_FEATURE_NAME
static SpinWait get_spin_wait_desc() {
SpinWait spin_wait(OnSpinWaitInst, OnSpinWaitInstCount, OnSpinWaitDelay);
@@ -104,11 +106,6 @@ static bool has_neoverse_n1_errata_1542419() {
}
void VM_Version::initialize() {
-#define SET_CPU_FEATURE_NAME(id, name, bit) \
- _features_names[bit] = XSTR(name);
- CPU_FEATURE_FLAGS(SET_CPU_FEATURE_NAME)
-#undef SET_CPU_FEATURE_NAME
-
_supports_atomic_getset4 = true;
_supports_atomic_getadd4 = true;
_supports_atomic_getset8 = true;
@@ -305,9 +302,9 @@ void VM_Version::initialize() {
FLAG_SET_DEFAULT(UseSHA, false);
}
- CHECK_CPU_FEATURE(supports_crc32, CRC32);
- CHECK_CPU_FEATURE(supports_lse, LSE);
- CHECK_CPU_FEATURE(supports_aes, AES);
+ CHECK_CPU_FEATURE(UseCRC32, CRC32, supports_crc32(), "CRC32" MULTI_INST_WARNING_MSG);
+ CHECK_CPU_FEATURE(UseLSE, LSE, supports_lse(), "LSE" MULTI_INST_WARNING_MSG);
+ CHECK_CPU_FEATURE(UseAES, AES, supports_aes(), "AES" MULTI_INST_WARNING_MSG);
if (_cpu == CPU_ARM &&
model_is_in({ CPU_MODEL_ARM_NEOVERSE_V1, CPU_MODEL_ARM_NEOVERSE_V2,
@@ -789,9 +786,9 @@ void VM_Version::store_cpu_features(void* buf) {
*(uint64_t*)buf = _features;
}
-bool VM_Version::supports_features(void* features_buffer) {
+bool VM_Version::verify_aot_code_cache_features(void* features_buffer) {
uint64_t features_to_test = *(uint64_t*)features_buffer;
- return (_features & features_to_test) == features_to_test;
+ return (_features == features_to_test);
}
#if defined(LINUX)
diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp
index 30f1a5d86ca..c67455e6b79 100644
--- a/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp
@@ -144,33 +144,32 @@ public:
CPU_MODEL_ARM_NEOVERSE_N3 = 0xd8e,
};
-#define CPU_FEATURE_FLAGS(decl) \
- decl(FP, fp, 0) \
- decl(ASIMD, asimd, 1) \
- decl(EVTSTRM, evtstrm, 2) \
- decl(AES, aes, 3) \
- decl(PMULL, pmull, 4) \
- decl(SHA1, sha1, 5) \
- decl(SHA2, sha256, 6) \
- decl(CRC32, crc32, 7) \
- decl(LSE, lse, 8) \
- decl(FPHP, fphp, 9) \
- decl(ASIMDHP, asimdhp, 10) \
- decl(DCPOP, dcpop, 16) \
- decl(SHA3, sha3, 17) \
- decl(SHA512, sha512, 21) \
- decl(SVE, sve, 22) \
- decl(SB, sb, 29) \
- decl(PACA, paca, 30) \
- /* flags above must follow Linux HWCAP */ \
- decl(SVEBITPERM, svebitperm, 27) \
- decl(SVE2, sve2, 28) \
- decl(A53MAC, a53mac, 31) \
- decl(ECV, ecv, 32) \
- decl(WFXT, wfxt, 33)
+#define CPU_FEATURE_FLAGS(decl) \
+ decl(FP, fp ) \
+ decl(ASIMD, asimd ) \
+ decl(EVTSTRM, evtstrm ) \
+ decl(AES, aes ) \
+ decl(PMULL, pmull ) \
+ decl(SHA1, sha1 ) \
+ decl(SHA2, sha256 ) \
+ decl(CRC32, crc32 ) \
+ decl(LSE, lse ) \
+ decl(FPHP, fphp ) \
+ decl(ASIMDHP, asimdhp ) \
+ decl(DCPOP, dcpop ) \
+ decl(SHA3, sha3 ) \
+ decl(SHA512, sha512 ) \
+ decl(SVE, sve ) \
+ decl(SB, sb ) \
+ decl(PACA, paca ) \
+ decl(SVEBITPERM, svebitperm ) \
+ decl(SVE2, sve2 ) \
+ decl(A53MAC, a53mac ) \
+ decl(ECV, ecv ) \
+ decl(WFXT, wfxt )
enum Feature_Flag {
-#define DECLARE_CPU_FEATURE_FLAG(id, name, bit) CPU_##id = bit,
+#define DECLARE_CPU_FEATURE_FLAG(id, name) CPU_##id,
CPU_FEATURE_FLAGS(DECLARE_CPU_FEATURE_FLAG)
#undef DECLARE_CPU_FEATURE_FLAG
MAX_CPU_FEATURES
@@ -178,10 +177,10 @@ public:
STATIC_ASSERT(sizeof(_features) * BitsPerByte >= MAX_CPU_FEATURES);
- static const char* _features_names[MAX_CPU_FEATURES];
+ static const char* _features_names[];
// Feature identification
-#define CPU_FEATURE_DETECTION(id, name, bit) \
+#define CPU_FEATURE_DETECTION(id, name) \
static bool supports_##name() { return supports_feature(CPU_##id); }
CPU_FEATURE_FLAGS(CPU_FEATURE_DETECTION)
#undef CPU_FEATURE_DETECTION
@@ -279,7 +278,7 @@ public:
// Size of the buffer must be same as returned by cpu_features_size()
static void store_cpu_features(void* buf);
- static bool supports_features(void* features_to_test);
+ static bool verify_aot_code_cache_features(void* features_buffer);
};
#endif // CPU_AARCH64_VM_VERSION_AARCH64_HPP
diff --git a/src/hotspot/cpu/arm/arm.ad b/src/hotspot/cpu/arm/arm.ad
index 60a0ef307b5..45ae283e05a 100644
--- a/src/hotspot/cpu/arm/arm.ad
+++ b/src/hotspot/cpu/arm/arm.ad
@@ -9214,10 +9214,12 @@ instruct bytes_reverse_long(iRegL dst, iRegL src) %{
instruct bytes_reverse_unsigned_short(iRegI dst, iRegI src) %{
match(Set dst (ReverseBytesUS src));
- size(4);
- format %{ "REV16 $dst,$src" %}
+ size(8);
+ format %{ "REV32 $dst,$src\n\t"
+ "LSR $dst,$dst,#16" %}
ins_encode %{
- __ rev16($dst$$Register, $src$$Register);
+ __ rev($dst$$Register, $src$$Register);
+ __ mov($dst$$Register, AsmOperand($dst$$Register, lsr, 16));
%}
ins_pipe( iload_mem ); // FIXME
%}
diff --git a/src/hotspot/cpu/arm/arm_32.ad b/src/hotspot/cpu/arm/arm_32.ad
index 9438e8da8b5..2af7e253a1a 100644
--- a/src/hotspot/cpu/arm/arm_32.ad
+++ b/src/hotspot/cpu/arm/arm_32.ad
@@ -1,5 +1,5 @@
//
-// Copyright (c) 2008, 2025, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2008, 2026, Oracle and/or its affiliates. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
@@ -501,7 +501,7 @@ operand immIRotn() %{
%}
operand immPRot() %{
- predicate(n->get_ptr() == 0 || (AsmOperand::is_rotated_imm(n->get_ptr()) && ((ConPNode*)n)->type()->reloc() == relocInfo::none));
+ predicate(n->get_ptr() == 0 || (AsmOperand::is_rotated_imm(n->get_ptr()) && ((ConPNode*)n)->type()->is_ptr()->reloc() == relocInfo::none));
match(ConP);
diff --git a/src/hotspot/cpu/arm/c1_LIRGenerator_arm.cpp b/src/hotspot/cpu/arm/c1_LIRGenerator_arm.cpp
index 4c339968f85..46ec87290ae 100644
--- a/src/hotspot/cpu/arm/c1_LIRGenerator_arm.cpp
+++ b/src/hotspot/cpu/arm/c1_LIRGenerator_arm.cpp
@@ -1332,7 +1332,8 @@ void LIRGenerator::volatile_field_load(LIR_Address* address, LIR_Opr result,
load_addr = address;
}
__ volatile_load_mem_reg(load_addr, result, info);
- return;
+ } else {
+ __ load(address, result, info, lir_patch_none);
}
- __ load(address, result, info, lir_patch_none);
+ __ membar_acquire();
}
diff --git a/src/hotspot/cpu/ppc/assembler_ppc.hpp b/src/hotspot/cpu/ppc/assembler_ppc.hpp
index 378e01fc1cc..f62c93e466c 100644
--- a/src/hotspot/cpu/ppc/assembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/assembler_ppc.hpp
@@ -461,10 +461,6 @@ class Assembler : public AbstractAssembler {
FRIN_OPCODE = (63u << OPCODE_SHIFT | 392u << 1),
FRIP_OPCODE = (63u << OPCODE_SHIFT | 456u << 1),
FRIM_OPCODE = (63u << OPCODE_SHIFT | 488u << 1),
- // These are special Power6 opcodes, reused for "lfdepx" and "stfdepx"
- // on Power7. Do not use.
- // MFFGPR_OPCODE = (31u << OPCODE_SHIFT | 607u << 1),
- // MFTGPR_OPCODE = (31u << OPCODE_SHIFT | 735u << 1),
CMPB_OPCODE = (31u << OPCODE_SHIFT | 508 << 1),
POPCNTB_OPCODE = (31u << OPCODE_SHIFT | 122 << 1),
POPCNTW_OPCODE = (31u << OPCODE_SHIFT | 378 << 1),
@@ -518,7 +514,6 @@ class Assembler : public AbstractAssembler {
FSQRT_OPCODE = (63u << OPCODE_SHIFT | 22u << 1), // A-FORM
FSQRTS_OPCODE = (59u << OPCODE_SHIFT | 22u << 1), // A-FORM
- // Vector instruction support for >= Power6
// Vector Storage Access
LVEBX_OPCODE = (31u << OPCODE_SHIFT | 7u << 1),
LVEHX_OPCODE = (31u << OPCODE_SHIFT | 39u << 1),
@@ -1236,7 +1231,7 @@ class Assembler : public AbstractAssembler {
static int u( int x) { return opp_u_field(x, 19, 16); }
static int ui( int x) { return opp_u_field(x, 31, 16); }
- // Support vector instructions for >= Power6.
+ // Support vector instructions.
static int vra( int x) { return opp_u_field(x, 15, 11); }
static int vrb( int x) { return opp_u_field(x, 20, 16); }
static int vrc( int x) { return opp_u_field(x, 25, 21); }
@@ -2036,7 +2031,7 @@ class Assembler : public AbstractAssembler {
inline void stqcx_( Register s, Register a, Register b);
// Instructions for adjusting thread priority for simultaneous
- // multithreading (SMT) on Power5.
+ // multithreading (SMT).
private:
inline void smt_prio_very_low();
inline void smt_prio_medium_high();
@@ -2204,7 +2199,7 @@ class Assembler : public AbstractAssembler {
inline void fsqrt( FloatRegister d, FloatRegister b);
inline void fsqrts(FloatRegister d, FloatRegister b);
- // Vector instructions for >= Power6.
+ // Vector instructions.
inline void lvebx( VectorRegister d, Register s1, Register s2);
inline void lvehx( VectorRegister d, Register s1, Register s2);
inline void lvewx( VectorRegister d, Register s1, Register s2);
diff --git a/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp b/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp
index d349bbc6f87..22b9e268dcd 100644
--- a/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp
+++ b/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp
@@ -642,7 +642,6 @@ inline void Assembler::crorc( ConditionRegister crdst, Condition cdst, Condition
crorc(dst_bit, src_bit, dst_bit);
}
-// Conditional move (>= Power7)
inline void Assembler::isel(Register d, ConditionRegister cr, Condition cc, bool inv, Register a, Register b) {
if (b == noreg) {
b = d; // Can be omitted if old value should be kept in "else" case.
@@ -689,7 +688,7 @@ inline void Assembler::elemental_membar(int e) { assert(0 < e && e < 16, "invali
// Wait instructions for polling.
inline void Assembler::wait() { emit_int32( WAIT_OPCODE); }
-inline void Assembler::waitrsv() { emit_int32( WAIT_OPCODE | 1<<(31-10)); } // WC=0b01 >=Power7
+inline void Assembler::waitrsv() { emit_int32( WAIT_OPCODE | 1<<(31-10)); } // WC=0b01
// atomics
// Use ra0mem to disallow R0 as base.
@@ -709,19 +708,16 @@ inline void Assembler::stwcx_(Register s, Register a, Register b)
inline void Assembler::stdcx_(Register s, Register a, Register b) { emit_int32( STDCX_OPCODE | rs(s) | ra0mem(a) | rb(b) | rc(1)); }
inline void Assembler::stqcx_(Register s, Register a, Register b) { emit_int32( STQCX_OPCODE | rs(s) | ra0mem(a) | rb(b) | rc(1)); }
-// Instructions for adjusting thread priority
-// for simultaneous multithreading (SMT) on >= POWER5.
+// Instructions for adjusting thread priority for simultaneous multithreading (SMT).
inline void Assembler::smt_prio_very_low() { Assembler::or_unchecked(R31, R31, R31); }
inline void Assembler::smt_prio_low() { Assembler::or_unchecked(R1, R1, R1); }
inline void Assembler::smt_prio_medium_low() { Assembler::or_unchecked(R6, R6, R6); }
inline void Assembler::smt_prio_medium() { Assembler::or_unchecked(R2, R2, R2); }
inline void Assembler::smt_prio_medium_high() { Assembler::or_unchecked(R5, R5, R5); }
-inline void Assembler::smt_prio_high() { Assembler::or_unchecked(R3, R3, R3); }
-// >= Power7
+inline void Assembler::smt_prio_high() { Assembler::or_unchecked(R3, R3, R3); } // Restricted to supervisor state since Power9.
inline void Assembler::smt_yield() { Assembler::or_unchecked(R27, R27, R27); } // never actually implemented
-inline void Assembler::smt_mdoio() { Assembler::or_unchecked(R29, R29, R29); } // never actually implemetned
+inline void Assembler::smt_mdoio() { Assembler::or_unchecked(R29, R29, R29); } // never actually implemented
inline void Assembler::smt_mdoom() { Assembler::or_unchecked(R30, R30, R30); } // never actually implemented
-// Power8
inline void Assembler::smt_miso() { Assembler::or_unchecked(R26, R26, R26); } // never actually implemented
inline void Assembler::twi_0(Register a) { twi_unchecked(0, a, 0);}
@@ -766,10 +762,6 @@ inline void Assembler::frin( FloatRegister d, FloatRegister b) { emit_int32( FRI
inline void Assembler::frip( FloatRegister d, FloatRegister b) { emit_int32( FRIP_OPCODE | frt(d) | frb(b) | rc(0)); }
inline void Assembler::frim( FloatRegister d, FloatRegister b) { emit_int32( FRIM_OPCODE | frt(d) | frb(b) | rc(0)); }
-// These are special Power6 opcodes, reused for "lfdepx" and "stfdepx"
-// on Power7. Do not use.
-//inline void Assembler::mffgpr( FloatRegister d, Register b) { emit_int32( MFFGPR_OPCODE | frt(d) | rb(b) | rc(0)); }
-//inline void Assembler::mftgpr( Register d, FloatRegister b) { emit_int32( MFTGPR_OPCODE | rt(d) | frb(b) | rc(0)); }
// add cmpb and popcntb to detect ppc power version.
inline void Assembler::cmpb( Register a, Register s, Register b) { emit_int32( CMPB_OPCODE | rta(a) | rs(s) | rb(b) | rc(0)); }
inline void Assembler::popcntb(Register a, Register s) { emit_int32( POPCNTB_OPCODE | rta(a) | rs(s)); };
@@ -837,7 +829,7 @@ inline void Assembler::fcmpu( ConditionRegister crx, FloatRegister a, FloatRegis
inline void Assembler::fsqrt( FloatRegister d, FloatRegister b) { emit_int32( FSQRT_OPCODE | frt(d) | frb(b) | rc(0)); }
inline void Assembler::fsqrts(FloatRegister d, FloatRegister b) { emit_int32( FSQRTS_OPCODE | frt(d) | frb(b) | rc(0)); }
-// Vector instructions for >= Power6.
+// Vector instructions.
inline void Assembler::lvebx( VectorRegister d, Register s1, Register s2) { emit_int32( LVEBX_OPCODE | vrt(d) | ra0mem(s1) | rb(s2)); }
inline void Assembler::lvehx( VectorRegister d, Register s1, Register s2) { emit_int32( LVEHX_OPCODE | vrt(d) | ra0mem(s1) | rb(s2)); }
inline void Assembler::lvewx( VectorRegister d, Register s1, Register s2) { emit_int32( LVEWX_OPCODE | vrt(d) | ra0mem(s1) | rb(s2)); }
diff --git a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp
index 0b48653ae64..777b41577be 100644
--- a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp
@@ -210,7 +210,7 @@ int LIR_Assembler::emit_unwind_handler() {
_masm->block_comment("Unwind handler");
int offset = code_offset();
- bool preserve_exception = method()->is_synchronized() || compilation()->env()->dtrace_method_probes();
+ bool preserve_exception = method()->is_synchronized();
const Register Rexception = R3 /*LIRGenerator::exceptionOopOpr()*/, Rexception_save = R31;
// Fetch the exception from TLS and clear out exception related thread state.
@@ -232,10 +232,6 @@ int LIR_Assembler::emit_unwind_handler() {
__ bind(*stub->continuation());
}
- if (compilation()->env()->dtrace_method_probes()) {
- Unimplemented();
- }
-
// Dispatch to the unwind logic.
address unwind_stub = Runtime1::entry_for(StubId::c1_unwind_exception_id);
//__ load_const_optimized(R0, unwind_stub);
diff --git a/src/hotspot/cpu/ppc/c1_LIRGenerator_ppc.cpp b/src/hotspot/cpu/ppc/c1_LIRGenerator_ppc.cpp
index 5f030676bcb..a652a155f62 100644
--- a/src/hotspot/cpu/ppc/c1_LIRGenerator_ppc.cpp
+++ b/src/hotspot/cpu/ppc/c1_LIRGenerator_ppc.cpp
@@ -1143,6 +1143,7 @@ void LIRGenerator::volatile_field_load(LIR_Address* address, LIR_Opr result,
Unimplemented();
// __ volatile_load_mem_reg(address, result, info);
#endif
+ __ membar_acquire();
}
diff --git a/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp
index 4d7af0e4a71..359c7cf22ad 100644
--- a/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp
@@ -232,13 +232,6 @@ void C1_MacroAssembler::initialize_object(
initialize_body(obj, t1, t2, con_size_in_bytes, hdr_size_in_bytes);
}
- if (CURRENT_ENV->dtrace_alloc_probes()) {
- Unimplemented();
-// assert(obj == O0, "must be");
-// call(CAST_FROM_FN_PTR(address, Runtime1::entry_for(StubId::c1_dtrace_object_alloc_id)),
-// relocInfo::runtime_call_type);
- }
-
verify_oop(obj, FILE_AND_LINE);
}
@@ -308,13 +301,6 @@ void C1_MacroAssembler::allocate_array(
initialize_body(base, index);
}
- if (CURRENT_ENV->dtrace_alloc_probes()) {
- Unimplemented();
- //assert(obj == O0, "must be");
- //call(CAST_FROM_FN_PTR(address, Runtime1::entry_for(StubId::c1_dtrace_object_alloc_id)),
- // relocInfo::runtime_call_type);
- }
-
verify_oop(obj, FILE_AND_LINE);
}
diff --git a/src/hotspot/cpu/ppc/continuationFreezeThaw_ppc.inline.hpp b/src/hotspot/cpu/ppc/continuationFreezeThaw_ppc.inline.hpp
index f7704ea5b14..82167949065 100644
--- a/src/hotspot/cpu/ppc/continuationFreezeThaw_ppc.inline.hpp
+++ b/src/hotspot/cpu/ppc/continuationFreezeThaw_ppc.inline.hpp
@@ -540,6 +540,8 @@ template frame ThawBase::new_stack_frame(const frame& hf, frame&
intptr_t* frame_sp = caller.sp() - fsize;
if ((bottom && argsize > 0) || caller.is_interpreted_frame()) {
+ assert(!_should_patch_caller_pc, "");
+ _should_patch_caller_pc = caller.is_interpreted_frame();
frame_sp -= argsize + frame::metadata_words_at_top;
frame_sp = align_down(frame_sp, frame::alignment_in_bytes);
caller.set_sp(frame_sp + fsize);
diff --git a/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.cpp b/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.cpp
index 3692b247989..7a9e29191b9 100644
--- a/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.cpp
@@ -179,9 +179,10 @@ void BarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Re
__ ld(dst, 0, dst); // Resolve (untagged) jobject.
}
-void BarrierSetAssembler::try_resolve_weak_handle(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path) {
- // Load the oop from the weak handle.
- __ ld(obj, 0, obj);
+void BarrierSetAssembler::try_peek_weak_handle_in_nmethod(MacroAssembler* masm, Register weak_handle, Register obj,
+ Register tmp, Label& slow_path) {
+ // Load the oop from the weak handle without barriers.
+ __ ld(obj, 0, weak_handle);
}
void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm, Register tmp) {
@@ -354,19 +355,9 @@ int SaveLiveRegisters::iterate_over_register_mask(IterationAction action, int of
Register spill_addr = R0;
int spill_offset = offset - reg_save_index * BytesPerWord;
if (action == ACTION_SAVE) {
- if (PowerArchitecturePPC64 >= 9) {
- _masm->stxv(vs_reg, spill_offset, R1_SP);
- } else {
- _masm->addi(spill_addr, R1_SP, spill_offset);
- _masm->stxvd2x(vs_reg, spill_addr);
- }
+ _masm->stxv(vs_reg, spill_offset, R1_SP);
} else if (action == ACTION_RESTORE) {
- if (PowerArchitecturePPC64 >= 9) {
- _masm->lxv(vs_reg, spill_offset, R1_SP);
- } else {
- _masm->addi(spill_addr, R1_SP, spill_offset);
- _masm->lxvd2x(vs_reg, spill_addr);
- }
+ _masm->lxv(vs_reg, spill_offset, R1_SP);
} else {
assert(action == ACTION_COUNT_ONLY, "Sanity");
}
diff --git a/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.hpp b/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.hpp
index 8112542d761..8e8c4d41c3b 100644
--- a/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.hpp
@@ -70,11 +70,10 @@ public:
virtual void try_resolve_jobject_in_native(MacroAssembler* masm, Register dst, Register jni_env,
Register obj, Register tmp, Label& slowpath);
- // Can be used in nmethods including native wrappers.
- // Attention: obj will only be valid until next safepoint (no SATB barrier).
- // TODO: maybe rename to try_peek_weak_handle on all platforms (try: operation may fail, peek: obj is not kept alive)
- // (other platforms currently use it for C2 only: try_resolve_weak_handle_in_c2)
- virtual void try_resolve_weak_handle(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path);
+ // See AS_NO_KEEPALIVE for peek semantics
+ // weak_handle and obj may alias
+ virtual void try_peek_weak_handle_in_nmethod(MacroAssembler* masm, Register weak_handle, Register obj,
+ Register tmp, Label& slow_path);
virtual void barrier_stubs_init() {}
diff --git a/src/hotspot/cpu/ppc/gc/shenandoah/c1/shenandoahBarrierSetC1_ppc.cpp b/src/hotspot/cpu/ppc/gc/shenandoah/c1/shenandoahBarrierSetC1_ppc.cpp
deleted file mode 100644
index 5b24259103f..00000000000
--- a/src/hotspot/cpu/ppc/gc/shenandoah/c1/shenandoahBarrierSetC1_ppc.cpp
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Copyright (c) 2018, 2023, Red Hat, Inc. All rights reserved.
- * Copyright (c) 2012, 2023 SAP SE. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#include "asm/macroAssembler.inline.hpp"
-#include "c1/c1_LIRAssembler.hpp"
-#include "c1/c1_MacroAssembler.hpp"
-#include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
-#include "gc/shenandoah/shenandoahBarrierSet.hpp"
-#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
-
-#define __ masm->masm()->
-
-void LIR_OpShenandoahCompareAndSwap::emit_code(LIR_Assembler *masm) {
- __ block_comment("LIR_OpShenandoahCompareAndSwap (shenandaohgc) {");
-
- Register addr = _addr->as_register_lo();
- Register new_val = _new_value->as_register();
- Register cmp_val = _cmp_value->as_register();
- Register tmp1 = _tmp1->as_register();
- Register tmp2 = _tmp2->as_register();
- Register result = result_opr()->as_register();
-
- if (UseCompressedOops) {
- __ encode_heap_oop(cmp_val, cmp_val);
- __ encode_heap_oop(new_val, new_val);
- }
-
- // There might be a volatile load before this Unsafe CAS.
- if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
- __ sync();
- } else {
- __ lwsync();
- }
-
- ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm->masm(), addr, cmp_val, new_val, tmp1, tmp2,
- false, result);
-
- if (UseCompressedOops) {
- __ decode_heap_oop(cmp_val);
- __ decode_heap_oop(new_val);
- }
-
- if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
- __ isync();
- } else {
- __ sync();
- }
-
- __ block_comment("} LIR_OpShenandoahCompareAndSwap (shenandaohgc)");
-}
-
-#undef __
-
-#ifdef ASSERT
-#define __ gen->lir(__FILE__, __LINE__)->
-#else
-#define __ gen->lir()->
-#endif
-
-LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess &access, LIRItem &cmp_value, LIRItem &new_value) {
- BasicType bt = access.type();
-
- if (access.is_oop()) {
- LIRGenerator* gen = access.gen();
-
- if (ShenandoahSATBBarrier) {
- pre_barrier(gen, access.access_emit_info(), access.decorators(), access.resolved_addr(),
- LIR_OprFact::illegalOpr);
- }
-
- if (ShenandoahCASBarrier) {
- cmp_value.load_item();
- new_value.load_item();
-
- LIR_Opr t1 = gen->new_register(T_OBJECT);
- LIR_Opr t2 = gen->new_register(T_OBJECT);
- LIR_Opr addr = access.resolved_addr()->as_address_ptr()->base();
- LIR_Opr result = gen->new_register(T_INT);
-
- __ append(new LIR_OpShenandoahCompareAndSwap(addr, cmp_value.result(), new_value.result(), t1, t2, result));
-
- if (ShenandoahCardBarrier) {
- post_barrier(access, access.resolved_addr(), new_value.result());
- }
-
- return result;
- }
- }
-
- return BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value);
-}
-
-LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess &access, LIRItem &value) {
- LIRGenerator* gen = access.gen();
- BasicType type = access.type();
-
- LIR_Opr result = gen->new_register(type);
- value.load_item();
- LIR_Opr value_opr = value.result();
-
- assert(type == T_INT || is_reference_type(type) LP64_ONLY( || type == T_LONG ), "unexpected type");
- LIR_Opr tmp_xchg = gen->new_register(T_INT);
- __ xchg(access.resolved_addr(), value_opr, result, tmp_xchg);
-
- if (access.is_oop()) {
- result = load_reference_barrier_impl(access.gen(), result, LIR_OprFact::addressConst(0),
- access.decorators());
-
- LIR_Opr tmp_barrier = gen->new_register(type);
- __ move(result, tmp_barrier);
- result = tmp_barrier;
-
- if (ShenandoahSATBBarrier) {
- pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr, result);
- }
-
- if (ShenandoahCardBarrier) {
- post_barrier(access, access.resolved_addr(), result);
- }
- }
-
- return result;
-}
diff --git a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp
index 8e99d23cc99..82f3bb38012 100644
--- a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp
@@ -663,16 +663,18 @@ void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler
__ block_comment("} try_resolve_jobject_in_native (shenandoahgc)");
}
-void ShenandoahBarrierSetAssembler::try_resolve_weak_handle(MacroAssembler *masm, Register obj,
- Register tmp, Label &slow_path) {
- __ block_comment("try_resolve_weak_handle (shenandoahgc) {");
+void ShenandoahBarrierSetAssembler::try_peek_weak_handle_in_nmethod(MacroAssembler *masm, Register weak_handle,
+ Register obj, Register tmp, Label &slow_path) {
+ __ block_comment("try_peek_weak_handle_in_nmethod (shenandoahgc) {");
+
+ assert_different_registers(weak_handle, tmp, noreg);
+ assert_different_registers(obj, tmp, noreg);
- assert_different_registers(obj, tmp);
Label done;
- // Resolve weak handle using the standard implementation.
- BarrierSetAssembler::try_resolve_weak_handle(masm, obj, tmp, slow_path);
+ // Peek weak handle using the standard implementation.
+ BarrierSetAssembler::try_peek_weak_handle_in_nmethod(masm, weak_handle, obj, tmp, slow_path);
// Check if the reference is null, and if it is, take the fast path.
__ cmpdi(CR0, obj, 0);
@@ -685,7 +687,7 @@ void ShenandoahBarrierSetAssembler::try_resolve_weak_handle(MacroAssembler *masm
__ bne(CR0, slow_path);
__ bind(done);
- __ block_comment("} try_resolve_weak_handle (shenandoahgc)");
+ __ block_comment("} try_peek_weak_handle_in_nmethod (shenandoahgc)");
}
// Special shenandoah CAS implementation that handles false negatives due
@@ -891,13 +893,11 @@ void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assemble
Register tmp2 = stub->tmp2()->as_register();
assert_different_registers(addr, res, tmp1, tmp2);
-#ifdef ASSERT
- // Ensure that 'res' is 'R3_ARG1' and contains the same value as 'obj' to reduce the number of required
- // copy instructions.
assert(R3_RET == res, "res must be r3");
- __ cmpd(CR0, res, obj);
- __ asm_assert_eq("result register must contain the reference stored in obj");
-#endif
+
+ if (res != obj) {
+ __ mr(res, obj);
+ }
DecoratorSet decorators = stub->decorators();
@@ -1032,7 +1032,7 @@ void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_s
__ save_volatile_gprs(R1_SP, -nbytes_save, true, false);
// Load arguments from stack.
- // No load required, as assured by assertions in 'ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub'.
+ // No load required, as caller has already loaded obj into R3.
Register R3_obj = R3_ARG1;
Register R4_load_addr = R4_ARG2;
__ ld(R4_load_addr, -8, R1_SP);
diff --git a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.hpp b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.hpp
index 58180c49642..6033fbc54c4 100644
--- a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.hpp
@@ -123,7 +123,8 @@ public:
virtual void try_resolve_jobject_in_native(MacroAssembler* masm, Register dst, Register jni_env,
Register obj, Register tmp, Label& slowpath);
- virtual void try_resolve_weak_handle(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path);
+ virtual void try_peek_weak_handle_in_nmethod(MacroAssembler* masm, Register weak_handle, Register obj,
+ Register tmp, Label& slow_path);
};
#endif // CPU_PPC_GC_SHENANDOAH_SHENANDOAHBARRIERSETASSEMBLER_PPC_HPP
diff --git a/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.cpp b/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.cpp
index 3e74dfb88cb..5264b284458 100644
--- a/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.cpp
@@ -627,9 +627,13 @@ void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, R
__ block_comment("} try_resolve_jobject_in_native (zgc)");
}
-void ZBarrierSetAssembler::try_resolve_weak_handle(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path) {
- // Resolve weak handle using the standard implementation.
- BarrierSetAssembler::try_resolve_weak_handle(masm, obj, tmp, slow_path);
+void ZBarrierSetAssembler::try_peek_weak_handle_in_nmethod(MacroAssembler* masm, Register weak_handle, Register obj,
+ Register tmp, Label& slow_path) {
+ assert_different_registers(weak_handle, tmp, noreg);
+ assert_different_registers(obj, tmp, noreg);
+
+ // Peek weak handle using the standard implementation.
+ BarrierSetAssembler::try_peek_weak_handle_in_nmethod(masm, weak_handle, obj, tmp, slow_path);
// Check if the oop is bad, in which case we need to take the slow path.
__ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatMarkBadMask);
diff --git a/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.hpp b/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.hpp
index 655184cf6a3..8e4bd209222 100644
--- a/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.hpp
@@ -72,7 +72,8 @@ public:
virtual void try_resolve_jobject_in_native(MacroAssembler* masm, Register dst, Register jni_env,
Register obj, Register tmp, Label& slowpath);
- virtual void try_resolve_weak_handle(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path);
+ virtual void try_peek_weak_handle_in_nmethod(MacroAssembler* masm, Register weak_handle, Register obj,
+ Register tmp, Label& slow_path);
virtual void check_oop(MacroAssembler *masm, Register obj, const char* msg);
diff --git a/src/hotspot/cpu/ppc/globals_ppc.hpp b/src/hotspot/cpu/ppc/globals_ppc.hpp
index 927a8cc2be3..d46bb733ea7 100644
--- a/src/hotspot/cpu/ppc/globals_ppc.hpp
+++ b/src/hotspot/cpu/ppc/globals_ppc.hpp
@@ -116,7 +116,8 @@ define_pd_global(intx, InitArrayShortSize, 9*BytesPerLong);
\
/* special instructions */ \
product(bool, SuperwordUseVSX, false, \
- "Use VSX instructions for superword optimization.") \
+ "Use VSX instructions for superword optimization " \
+ "(default for Power9 and later).") \
\
product(bool, UseByteReverseInstructions, false, DIAGNOSTIC, \
"Use byte reverse instructions.") \
diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp
index 5fbcce94029..24c314b8be3 100644
--- a/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp
@@ -799,13 +799,7 @@ void MacroAssembler::save_nonvolatile_registers(Register dst, int offset, bool i
}
} else {
for (int i = 20; i < 32; i++) {
- if (PowerArchitecturePPC64 >= 9) {
- stxv(as_VectorRegister(i)->to_vsr(), offset, dst);
- } else {
- Register spill_addr = R0;
- addi(spill_addr, dst, offset);
- stxvd2x(as_VectorRegister(i)->to_vsr(), spill_addr);
- }
+ stxv(as_VectorRegister(i)->to_vsr(), offset, dst);
offset += 16;
}
}
@@ -838,13 +832,7 @@ void MacroAssembler::restore_nonvolatile_registers(Register src, int offset, boo
}
} else {
for (int i = 20; i < 32; i++) {
- if (PowerArchitecturePPC64 >= 9) {
- lxv(as_VectorRegister(i)->to_vsr(), offset, src);
- } else {
- Register spill_addr = R0;
- addi(spill_addr, src, offset);
- lxvd2x(as_VectorRegister(i)->to_vsr(), spill_addr);
- }
+ lxv(as_VectorRegister(i)->to_vsr(), offset, src);
offset += 16;
}
}
@@ -2800,7 +2788,7 @@ void MacroAssembler::compiler_fast_lock_object(ConditionRegister flag, Register
// Check if object matches.
ld(tmp3, in_bytes(ObjectMonitor::object_offset()), monitor);
BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
- bs_asm->try_resolve_weak_handle(this, tmp3, tmp2, slow_path);
+ bs_asm->try_peek_weak_handle_in_nmethod(this, tmp3, tmp3, tmp2, slow_path);
cmpd(CR0, tmp3, obj);
bne(CR0, slow_path);
@@ -3214,7 +3202,7 @@ void MacroAssembler::store_klass_gap(Register dst_oop, Register val) {
stw(val, oopDesc::klass_gap_offset_in_bytes(), dst_oop);
}
-int MacroAssembler::instr_size_for_decode_klass_not_null() {
+int MacroAssembler::instr_size_for_load_klass() {
static int computed_size = -1;
// Not yet computed?
@@ -3222,10 +3210,10 @@ int MacroAssembler::instr_size_for_decode_klass_not_null() {
// Determine by scratch emit.
ResourceMark rm;
- int code_size = 8 * BytesPerInstWord;
- CodeBuffer cb("decode_klass_not_null scratch buffer", code_size, 0);
+ int code_size = 16 * BytesPerInstWord;
+ CodeBuffer cb("load_klass scratch buffer", code_size, 0);
MacroAssembler* a = new MacroAssembler(&cb);
- a->decode_klass_not_null(R11_scratch1);
+ a->load_klass(R11_scratch1, R11_scratch1);
computed_size = a->offset();
}
diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp
index 4be62098bdf..b2f5e8f0b60 100644
--- a/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp
@@ -802,7 +802,7 @@ class MacroAssembler: public Assembler {
MacroAssembler::PreservationLevel preservation_level);
void load_method_holder(Register holder, Register method);
- static int instr_size_for_decode_klass_not_null();
+ static int instr_size_for_load_klass();
void decode_klass_not_null(Register dst, Register src = noreg);
Register encode_klass_not_null(Register dst, Register src = noreg);
diff --git a/src/hotspot/cpu/ppc/matcher_ppc.hpp b/src/hotspot/cpu/ppc/matcher_ppc.hpp
index cbe882648b8..a3ab382564c 100644
--- a/src/hotspot/cpu/ppc/matcher_ppc.hpp
+++ b/src/hotspot/cpu/ppc/matcher_ppc.hpp
@@ -38,10 +38,10 @@
return false;
}
- // The PPC implementation uses VSX lxvd2x/stxvd2x instructions (if
+ // The PPC implementation uses VSX lxv/stxv instructions (if
// SuperwordUseVSX). They do not have alignment requirements.
// Some VSX storage access instructions cannot encode arbitrary displacements
- // (e.g. lxv). None of them is currently used.
+ // (e.g. lxv). We use memoryAlg16 for them.
static constexpr bool misaligned_vectors_ok() {
return true;
}
@@ -54,7 +54,7 @@
// PowerPC requires masked shift counts.
static const bool need_masked_shift_count = true;
- // Power6 requires postalloc expand (see block.cpp for description of postalloc expand).
+ // PPC64 requires postalloc expand (see block.cpp for description of postalloc expand).
static const bool require_postalloc_expand = true;
// No support for generic vector operands.
@@ -157,7 +157,7 @@
// true means we have fast l2f conversion
static constexpr bool convL2FSupported(void) {
- // fcfids can do the conversion (>= Power7).
+ // fcfids can do the conversion.
// fcfid + frsp showed rounding problem when result should be 0x3f800001.
return true;
}
diff --git a/src/hotspot/cpu/ppc/ppc.ad b/src/hotspot/cpu/ppc/ppc.ad
index f3d33b4305d..00549ac8508 100644
--- a/src/hotspot/cpu/ppc/ppc.ad
+++ b/src/hotspot/cpu/ppc/ppc.ad
@@ -1187,7 +1187,7 @@ int MachCallDynamicJavaNode::ret_addr_offset() {
assert(vtable_index == Method::invalid_vtable_index, "correct sentinel value");
return 12;
} else {
- return 24 + MacroAssembler::instr_size_for_decode_klass_not_null();
+ return 20 + MacroAssembler::instr_size_for_load_klass();
}
}
@@ -1818,52 +1818,26 @@ uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *r
// VectorRegister->Memory Spill.
else if (src_lo_rc == rc_vec && dst_lo_rc == rc_stack) {
VectorSRegister Rsrc = as_VectorRegister(Matcher::_regEncode[src_lo]).to_vsr();
- if (PowerArchitecturePPC64 >= 9) {
- if (masm) {
- __ stxv(Rsrc, dst_offset, R1_SP); // matches storeV16_Power9
- }
- size += 4;
- } else {
- if (masm) {
- __ addi(R0, R1_SP, dst_offset);
- __ stxvd2x(Rsrc, R0); // matches storeV16_Power8
- }
- size += 8;
+ if (masm) {
+ __ stxv(Rsrc, dst_offset, R1_SP); // matches storeV16
}
+ size += 4;
#ifndef PRODUCT
if (st != nullptr) {
- if (PowerArchitecturePPC64 >= 9) {
- st->print("%-7s %s, [R1_SP + #%d] \t// vector spill copy", "STXV", Matcher::regName[src_lo], dst_offset);
- } else {
- st->print("%-7s R0, R1_SP, %d \t// vector spill copy\n\t"
- "%-7s %s, [R0] \t// vector spill copy", "ADDI", dst_offset, "STXVD2X", Matcher::regName[src_lo]);
- }
+ st->print("%-7s %s, [R1_SP + #%d] \t// vector spill copy", "STXV", Matcher::regName[src_lo], dst_offset);
}
#endif // !PRODUCT
}
// Memory->VectorRegister Spill.
else if (src_lo_rc == rc_stack && dst_lo_rc == rc_vec) {
VectorSRegister Rdst = as_VectorRegister(Matcher::_regEncode[dst_lo]).to_vsr();
- if (PowerArchitecturePPC64 >= 9) {
- if (masm) {
- __ lxv(Rdst, src_offset, R1_SP);
- }
- size += 4;
- } else {
- if (masm) {
- __ addi(R0, R1_SP, src_offset);
- __ lxvd2x(Rdst, R0);
- }
- size += 8;
+ if (masm) {
+ __ lxv(Rdst, src_offset, R1_SP);
}
+ size += 4;
#ifndef PRODUCT
if (st != nullptr) {
- if (PowerArchitecturePPC64 >= 9) {
- st->print("%-7s %s, [R1_SP + #%d] \t// vector spill copy", "LXV", Matcher::regName[dst_lo], src_offset);
- } else {
- st->print("%-7s R0, R1_SP, %d \t// vector spill copy\n\t"
- "%-7s %s, [R0] \t// vector spill copy", "ADDI", src_offset, "LXVD2X", Matcher::regName[dst_lo]);
- }
+ st->print("%-7s %s, [R1_SP + #%d] \t// vector spill copy", "LXV", Matcher::regName[dst_lo], src_offset);
}
#endif // !PRODUCT
}
@@ -2284,7 +2258,7 @@ bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
case Op_UMaxV:
return bt == T_INT || bt == T_LONG;
case Op_NegVI:
- return PowerArchitecturePPC64 >= 9 && bt == T_INT;
+ return bt == T_INT;
}
return true; // Per default match rules are supported.
}
@@ -2322,10 +2296,12 @@ OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
// Vector width in bytes.
int Matcher::vector_width_in_bytes(BasicType bt) {
if (SuperwordUseVSX) {
- assert(MaxVectorSize == 16, "");
+ assert(MaxVectorSize == 16,
+ "SuperwordUseVSX requires MaxVectorSize 16, got " INT64_FORMAT, (int64_t)MaxVectorSize);
return 16;
} else {
- assert(MaxVectorSize == 8, "");
+ assert(MaxVectorSize == 8,
+ "expected MaxVectorSize 8, got " INT64_FORMAT, (int64_t)MaxVectorSize);
return 8;
}
}
@@ -2333,10 +2309,14 @@ int Matcher::vector_width_in_bytes(BasicType bt) {
// Vector ideal reg.
uint Matcher::vector_ideal_reg(int size) {
if (SuperwordUseVSX) {
- assert(MaxVectorSize == 16 && size == 16, "");
+ assert(MaxVectorSize == 16 && size == 16,
+ "SuperwordUseVSX requires MaxVectorSize 16 and size 16, got MaxVectorSize=" INT64_FORMAT ", size=%d",
+ (int64_t)MaxVectorSize, size);
return Op_VecX;
} else {
- assert(MaxVectorSize == 8 && size == 8, "");
+ assert(MaxVectorSize == 8 && size == 8,
+ "expected MaxVectorSize 8 and size 8, got MaxVectorSize=" INT64_FORMAT ", size=%d",
+ (int64_t)MaxVectorSize, size);
return Op_RegL;
}
}
@@ -5413,23 +5393,9 @@ instruct loadV8(iRegLdst dst, memoryAlg4 mem) %{
ins_pipe(pipe_class_memory);
%}
-// Load Aligned Packed Byte
-// Note: The Power8 instruction loads the contents in a special order in Little Endian mode.
-instruct loadV16_Power8(vecX dst, indirect mem) %{
- predicate(n->as_LoadVector()->memory_size() == 16 && PowerArchitecturePPC64 == 8);
- match(Set dst (LoadVector mem));
- ins_cost(MEMORY_REF_COST);
- format %{ "LXVD2X $dst, $mem \t// load 16-byte Vector" %}
- size(4);
- ins_encode %{
- __ lxvd2x($dst$$VectorRegister.to_vsr(), $mem$$Register);
- %}
- ins_pipe(pipe_class_default);
-%}
-
-instruct loadV16_Power9(vecX dst, memoryAlg16 mem) %{
- predicate(n->as_LoadVector()->memory_size() == 16 && PowerArchitecturePPC64 >= 9);
+instruct loadV16(vecX dst, memoryAlg16 mem) %{
+ predicate(n->as_LoadVector()->memory_size() == 16);
match(Set dst (LoadVector mem));
ins_cost(MEMORY_REF_COST);
@@ -6424,23 +6390,9 @@ instruct storeA8B(memoryAlg4 mem, iRegLsrc src) %{
ins_pipe(pipe_class_memory);
%}
-// Store Packed Byte long register to memory
-// Note: The Power8 instruction stores the contents in a special order in Little Endian mode.
-instruct storeV16_Power8(indirect mem, vecX src) %{
- predicate(n->as_StoreVector()->memory_size() == 16 && PowerArchitecturePPC64 == 8);
- match(Set mem (StoreVector mem src));
- ins_cost(MEMORY_REF_COST);
- format %{ "STXVD2X $mem, $src \t// store 16-byte Vector" %}
- size(4);
- ins_encode %{
- __ stxvd2x($src$$VectorRegister.to_vsr(), $mem$$Register);
- %}
- ins_pipe(pipe_class_default);
-%}
-
-instruct storeV16_Power9(memoryAlg16 mem, vecX src) %{
- predicate(n->as_StoreVector()->memory_size() == 16 && PowerArchitecturePPC64 >= 9);
+instruct storeV16(memoryAlg16 mem, vecX src) %{
+ predicate(n->as_StoreVector()->memory_size() == 16);
match(Set mem (StoreVector mem src));
ins_cost(MEMORY_REF_COST);
@@ -12480,6 +12432,19 @@ instruct countTrailingZerosL_cnttzd(iRegIdst dst, iRegLsrc src) %{
ins_pipe(pipe_class_default);
%}
+// Expand nodes for byte_reverse_int/ushort/short.
+instruct rlwinm(iRegIdst dst, iRegIsrc src, immI16 shift, immI16 mb, immI16 me) %{
+ effect(DEF dst, USE src, USE shift, USE mb, USE me);
+ predicate(false);
+
+ format %{ "RLWINM $dst, $src, $shift, $mb, $me" %}
+ size(4);
+ ins_encode %{
+ __ rlwinm($dst$$Register, $src$$Register, $shift$$constant, $mb$$constant, $me$$constant);
+ %}
+ ins_pipe(pipe_class_default);
+%}
+
// Expand nodes for byte_reverse_int.
instruct insrwi_a(iRegIdst dst, iRegIsrc src, immI16 n, immI16 b) %{
effect(DEF dst, USE src, USE n, USE b);
@@ -12636,34 +12601,22 @@ instruct bytes_reverse_long(iRegLdst dst, iRegLsrc src) %{
ins_pipe(pipe_class_default);
%}
+// Need zero extend. Must not use brh only.
instruct bytes_reverse_ushort_Ex(iRegIdst dst, iRegIsrc src) %{
match(Set dst (ReverseBytesUS src));
- predicate(!UseByteReverseInstructions);
ins_cost(2*DEFAULT_COST);
expand %{
+ immI16 imm31 %{ (int) 31 %}
+ immI16 imm24 %{ (int) 24 %}
immI16 imm16 %{ (int) 16 %}
immI16 imm8 %{ (int) 8 %}
- urShiftI_reg_imm(dst, src, imm8);
+ rlwinm(dst, src, imm24, imm24, imm31);
insrwi(dst, src, imm8, imm16);
%}
%}
-instruct bytes_reverse_ushort(iRegIdst dst, iRegIsrc src) %{
- match(Set dst (ReverseBytesUS src));
- predicate(UseByteReverseInstructions);
- ins_cost(DEFAULT_COST);
- size(4);
-
- format %{ "BRH $dst, $src" %}
-
- ins_encode %{
- __ brh($dst$$Register, $src$$Register);
- %}
- ins_pipe(pipe_class_default);
-%}
-
instruct bytes_reverse_short_Ex(iRegIdst dst, iRegIsrc src) %{
match(Set dst (ReverseBytesS src));
predicate(!UseByteReverseInstructions);
@@ -13656,7 +13609,7 @@ instruct vneg2D_reg(vecX dst, vecX src) %{
instruct vneg4I_reg(vecX dst, vecX src) %{
match(Set dst (NegVI src));
- predicate(PowerArchitecturePPC64 >= 9 && Matcher::vector_element_basic_type(n) == T_INT);
+ predicate(Matcher::vector_element_basic_type(n) == T_INT);
format %{ "VNEGW $dst,$src\t// negate int vector" %}
size(4);
ins_encode %{
diff --git a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
index 53644210415..54336e9f62b 100644
--- a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
+++ b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
@@ -360,7 +360,7 @@ OopMap* RegisterSaver::push_frame_reg_args_and_save_live_registers(MacroAssemble
assert(RegisterSaver_LiveVecRegs[i + 1].reg_num == reg_num + 1, "or use other instructions!");
__ stxvp(as_VectorRegister(reg_num).to_vsr(), offset, R1_SP);
- // Note: The contents were read in the same order (see loadV16_Power9 node in ppc.ad).
+ // Note: The contents were read in the same order (see loadV16 node in ppc.ad).
// RegisterMap::pd_location only uses the first VMReg for each VectorRegister.
if (generate_oop_map) {
map->set_callee_saved(VMRegImpl::stack2reg(offset >> 2),
@@ -374,13 +374,8 @@ OopMap* RegisterSaver::push_frame_reg_args_and_save_live_registers(MacroAssemble
for (int i = 0; i < vecregstosave_num; i++) {
int reg_num = RegisterSaver_LiveVecRegs[i].reg_num;
- if (PowerArchitecturePPC64 >= 9) {
- __ stxv(as_VectorRegister(reg_num)->to_vsr(), offset, R1_SP);
- } else {
- __ li(R31, offset);
- __ stxvd2x(as_VectorRegister(reg_num)->to_vsr(), R31, R1_SP);
- }
- // Note: The contents were read in the same order (see loadV16_Power8 / loadV16_Power9 node in ppc.ad).
+ __ stxv(as_VectorRegister(reg_num)->to_vsr(), offset, R1_SP);
+ // Note: The contents were read in the same order (see loadV16 node in ppc.ad).
// RegisterMap::pd_location only uses the first VMReg for each VectorRegister.
if (generate_oop_map) {
VMReg vsr = RegisterSaver_LiveVecRegs[i].vmreg;
@@ -464,12 +459,7 @@ void RegisterSaver::restore_live_registers_and_pop_frame(MacroAssembler* masm,
for (int i = 0; i < vecregstosave_num; i++) {
int reg_num = RegisterSaver_LiveVecRegs[i].reg_num;
- if (PowerArchitecturePPC64 >= 9) {
- __ lxv(as_VectorRegister(reg_num).to_vsr(), offset, R1_SP);
- } else {
- __ li(R31, offset);
- __ lxvd2x(as_VectorRegister(reg_num).to_vsr(), R31, R1_SP);
- }
+ __ lxv(as_VectorRegister(reg_num).to_vsr(), offset, R1_SP);
offset += vec_reg_size;
}
diff --git a/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp b/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp
index 37f780535b4..252425fb104 100644
--- a/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp
+++ b/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp
@@ -3852,13 +3852,6 @@ void TemplateTable::_new() {
__ store_klass(RallocatedObject, RinstanceKlass, Rscratch);
}
- // Check and trigger dtrace event.
- if (DTraceAllocProbes) {
- __ push(atos);
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, static_cast(SharedRuntime::dtrace_object_alloc)));
- __ pop(atos);
- }
-
__ b(Ldone);
}
diff --git a/src/hotspot/cpu/ppc/vm_version_ppc.cpp b/src/hotspot/cpu/ppc/vm_version_ppc.cpp
index 3e3b1103c86..be05ec1dfb3 100644
--- a/src/hotspot/cpu/ppc/vm_version_ppc.cpp
+++ b/src/hotspot/cpu/ppc/vm_version_ppc.cpp
@@ -109,6 +109,9 @@ void VM_Version::initialize() {
if (FLAG_IS_DEFAULT(SuperwordUseVSX) && CompilerConfig::is_c2_enabled()) {
FLAG_SET_ERGO(SuperwordUseVSX, true);
}
+ } else if (SuperwordUseVSX) {
+ warning("SuperwordUseVSX specified, but needs at least Power9.");
+ FLAG_SET_DEFAULT(SuperwordUseVSX, false);
}
MaxVectorSize = SuperwordUseVSX ? 16 : 8;
diff --git a/src/hotspot/cpu/riscv/c1_LIRGenerator_riscv.cpp b/src/hotspot/cpu/riscv/c1_LIRGenerator_riscv.cpp
index f290708a231..5e0deb84a14 100644
--- a/src/hotspot/cpu/riscv/c1_LIRGenerator_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c1_LIRGenerator_riscv.cpp
@@ -1169,4 +1169,5 @@ void LIRGenerator::volatile_field_store(LIR_Opr value, LIR_Address* address,
void LIRGenerator::volatile_field_load(LIR_Address* address, LIR_Opr result,
CodeEmitInfo* info) {
__ volatile_load_mem_reg(address, result, info);
+ __ membar_acquire();
}
diff --git a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp
index 0d06fd469de..8d383f07c9a 100644
--- a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp
@@ -167,7 +167,7 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box,
// Check if object matches.
ld(tmp3, Address(tmp1_monitor, ObjectMonitor::object_offset()));
BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
- bs_asm->try_resolve_weak_handle_in_c2(this, tmp3, tmp2, slow_path);
+ bs_asm->try_peek_weak_handle_in_nmethod(this, tmp3, tmp3, tmp2, slow_path);
bne(tmp3, obj, slow_path);
bind(monitor_found);
@@ -3069,12 +3069,12 @@ void C2_MacroAssembler::reduce_mul_integral_v(Register dst, Register src1, Vecto
// If the operation is MUL, then the identity value is one.
vmv_v_i(vtmp1, 1);
vmerge_vvm(vtmp2, vtmp1, src2); // vm == v0
- vslidedown_vi(vtmp1, vtmp2, vector_length);
+ slidedown_v(vtmp1, vtmp2, vector_length);
vsetvli_helper(bt, vector_length);
vmul_vv(vtmp1, vtmp1, vtmp2);
} else {
- vslidedown_vi(vtmp1, src2, vector_length);
+ slidedown_v(vtmp1, src2, vector_length);
vsetvli_helper(bt, vector_length);
vmul_vv(vtmp1, vtmp1, src2);
@@ -3082,7 +3082,7 @@ void C2_MacroAssembler::reduce_mul_integral_v(Register dst, Register src1, Vecto
while (vector_length > 1) {
vector_length /= 2;
- vslidedown_vi(vtmp2, vtmp1, vector_length);
+ slidedown_v(vtmp2, vtmp1, vector_length);
vsetvli_helper(bt, vector_length);
vmul_vv(vtmp1, vtmp1, vtmp2);
}
@@ -3281,40 +3281,44 @@ VFCVT_SAFE(vfcvt_rtz_x_f_v);
// Extract a scalar element from an vector at position 'idx'.
// The input elements in src are expected to be of integral type.
-void C2_MacroAssembler::extract_v(Register dst, VectorRegister src, BasicType bt,
- int idx, VectorRegister tmp) {
+void C2_MacroAssembler::extract_v(Register dst, VectorRegister src,
+ BasicType bt, int idx, VectorRegister vtmp) {
assert(is_integral_type(bt), "unsupported element type");
assert(idx >= 0, "idx cannot be negative");
// Only need the first element after vector slidedown
vsetvli_helper(bt, 1);
if (idx == 0) {
vmv_x_s(dst, src);
- } else if (idx <= 31) {
- vslidedown_vi(tmp, src, idx);
- vmv_x_s(dst, tmp);
} else {
- mv(t0, idx);
- vslidedown_vx(tmp, src, t0);
- vmv_x_s(dst, tmp);
+ slidedown_v(vtmp, src, idx);
+ vmv_x_s(dst, vtmp);
}
}
// Extract a scalar element from an vector at position 'idx'.
// The input elements in src are expected to be of floating point type.
-void C2_MacroAssembler::extract_fp_v(FloatRegister dst, VectorRegister src, BasicType bt,
- int idx, VectorRegister tmp) {
+void C2_MacroAssembler::extract_fp_v(FloatRegister dst, VectorRegister src,
+ BasicType bt, int idx, VectorRegister vtmp) {
assert(is_floating_point_type(bt), "unsupported element type");
assert(idx >= 0, "idx cannot be negative");
// Only need the first element after vector slidedown
vsetvli_helper(bt, 1);
if (idx == 0) {
vfmv_f_s(dst, src);
- } else if (idx <= 31) {
- vslidedown_vi(tmp, src, idx);
- vfmv_f_s(dst, tmp);
} else {
- mv(t0, idx);
- vslidedown_vx(tmp, src, t0);
- vfmv_f_s(dst, tmp);
+ slidedown_v(vtmp, src, idx);
+ vfmv_f_s(dst, vtmp);
+ }
+}
+
+// Move elements down a vector register group.
+// Offset is the start index (offset) for the source.
+void C2_MacroAssembler::slidedown_v(VectorRegister dst, VectorRegister src,
+ uint32_t offset, Register tmp) {
+ if (is_uimm5(offset)) {
+ vslidedown_vi(dst, src, offset);
+ } else {
+ mv(tmp, offset);
+ vslidedown_vx(dst, src, tmp);
}
}
diff --git a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp
index fa87ceba295..468d53b1a54 100644
--- a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -296,7 +296,13 @@
void vfcvt_rtz_x_f_v_safe(VectorRegister dst, VectorRegister src);
- void extract_v(Register dst, VectorRegister src, BasicType bt, int idx, VectorRegister tmp);
- void extract_fp_v(FloatRegister dst, VectorRegister src, BasicType bt, int idx, VectorRegister tmp);
+ void extract_v(Register dst, VectorRegister src,
+ BasicType bt, int idx, VectorRegister vtmp);
+
+ void extract_fp_v(FloatRegister dst, VectorRegister src,
+ BasicType bt, int idx, VectorRegister vtmp);
+
+ void slidedown_v(VectorRegister dst, VectorRegister src,
+ uint32_t offset, Register tmp = t0);
#endif // CPU_RISCV_C2_MACROASSEMBLER_RISCV_HPP
diff --git a/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.cpp b/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.cpp
index aeb9df06de6..fd78b429ee4 100644
--- a/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.cpp
@@ -228,7 +228,7 @@ void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm, Label* slo
BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
Assembler::IncompressibleScope scope(masm); // Fixed length: see entry_barrier_offset()
- Label local_guard;
+ Label local_guard, skip_barrier;
NMethodPatchingType patching_type = nmethod_patching_type();
if (slow_path == nullptr) {
@@ -290,24 +290,26 @@ void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm, Label* slo
ShouldNotReachHere();
}
+ Label& barrier_target = slow_path == nullptr ? skip_barrier : *slow_path;
if (slow_path == nullptr) {
- Label skip_barrier;
- __ beq(t0, t1, skip_barrier);
+ __ beq(t0, t1, barrier_target, true /* is_far */);
+ } else {
+ __ bne(t0, t1, barrier_target, true /* is_far */);
+ }
+ if (slow_path == nullptr) {
__ rt_call(StubRoutines::method_entry_barrier());
-
__ j(skip_barrier);
__ bind(local_guard);
MacroAssembler::assert_alignment(__ pc());
__ emit_int32(0); // nmethod guard value. Skipped over in common case.
- __ bind(skip_barrier);
} else {
- __ beq(t0, t1, *continuation);
- __ j(*slow_path);
__ bind(*continuation);
}
+
+ __ bind(skip_barrier);
}
void BarrierSetAssembler::c2i_entry_barrier(MacroAssembler* masm) {
@@ -354,6 +356,12 @@ void BarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register
__ beqz(obj, error); // if klass is null it is broken
}
+void BarrierSetAssembler::try_peek_weak_handle_in_nmethod(MacroAssembler* masm, Register weak_handle, Register obj,
+ Register tmp, Label& slow_path) {
+ // Load the oop from the weak handle without barriers.
+ __ ld(obj, Address(weak_handle));
+}
+
#ifdef COMPILER2
OptoReg::Name BarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) {
@@ -368,12 +376,6 @@ OptoReg::Name BarrierSetAssembler::refine_register(const Node* node, OptoReg::Na
return opto_reg;
}
-
-void BarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path) {
- // Load the oop from the weak handle.
- __ ld(obj, Address(obj));
-}
-
#undef __
#define __ _masm->
diff --git a/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.hpp b/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.hpp
index bbb2a5af824..4a14fa65e78 100644
--- a/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.hpp
@@ -107,11 +107,14 @@ public:
static void clear_patching_epoch();
static void increment_patching_epoch();
+ // See AS_NO_KEEPALIVE for peek semantics
+ // weak_handle and obj may alias
+ virtual void try_peek_weak_handle_in_nmethod(MacroAssembler* masm, Register weak_handle, Register obj,
+ Register tmp, Label& slow_path);
+
#ifdef COMPILER2
OptoReg::Name refine_register(const Node* node,
OptoReg::Name opto_reg);
- virtual void try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj,
- Register tmp, Label& slow_path);
#endif // COMPILER2
};
diff --git a/src/hotspot/cpu/riscv/gc/shared/barrierSetNMethod_riscv.cpp b/src/hotspot/cpu/riscv/gc/shared/barrierSetNMethod_riscv.cpp
index 5003b9584a3..9b318dbe579 100644
--- a/src/hotspot/cpu/riscv/gc/shared/barrierSetNMethod_riscv.cpp
+++ b/src/hotspot/cpu/riscv/gc/shared/barrierSetNMethod_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -41,17 +41,16 @@
static int slow_path_size(nmethod* nm) {
// The slow path code is out of line with C2.
- // Leave a jal to the stub in the fast path.
- return nm->is_compiled_by_c2() ? 1 : 8;
+ return nm->is_compiled_by_c2() ? 0 : 4;
}
static int entry_barrier_offset(nmethod* nm) {
BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
switch (bs_asm->nmethod_patching_type()) {
case NMethodPatchingType::stw_instruction_and_data_patch:
- return -4 * (4 + slow_path_size(nm));
+ return -4 * (5 + slow_path_size(nm));
case NMethodPatchingType::conc_instruction_and_data_patch:
- return -4 * (15 + slow_path_size(nm));
+ return -4 * ((UseZtso ? 14 : 16) + slow_path_size(nm));
}
ShouldNotReachHere();
return 0;
@@ -103,6 +102,10 @@ public:
}
_guard_addr = reinterpret_cast(instruction_address() + local_guard_offset(nm));
}
+
+ // Perform the checking as verification.
+ err_msg msg("%s", "");
+ assert(check_barrier(msg), "%s", msg.buffer());
}
int get_value() {
@@ -128,10 +131,6 @@ public:
}
bool check_barrier(err_msg& msg) const;
- void verify() const {
- err_msg msg("%s", "");
- assert(check_barrier(msg), "%s", msg.buffer());
- }
};
// Store the instruction bitmask, bits and name for checking the barrier.
@@ -142,8 +141,8 @@ struct CheckInsn {
};
static const struct CheckInsn barrierInsn[] = {
- { 0x00000fff, 0x00000297, "auipc t0, 0 "},
- { 0x000fffff, 0x0002e283, "lwu t0, guard_offset(t0) "},
+ { 0x00000fff, 0x00000297, "auipc t0, 0 " },
+ { 0x000fffff, 0x0002e283, "lwu t0, guard_offset(t0)" },
/* ...... */
/* ...... */
/* guard: */
@@ -155,10 +154,11 @@ static const struct CheckInsn barrierInsn[] = {
// register numbers and immediate values in the encoding.
bool NativeNMethodBarrier::check_barrier(err_msg& msg) const {
address addr = instruction_address();
- for(unsigned int i = 0; i < sizeof(barrierInsn)/sizeof(struct CheckInsn); i++ ) {
+ for (unsigned int i = 0; i < sizeof(barrierInsn) / sizeof(struct CheckInsn); i++) {
uint32_t inst = Assembler::ld_instr(addr);
if ((inst & barrierInsn[i].mask) != barrierInsn[i].bits) {
- msg.print("Addr: " INTPTR_FORMAT " Code: 0x%x not an %s instruction", p2i(addr), inst, barrierInsn[i].name);
+ msg.print("Nmethod entry barrier did not start with auipc & lwu as expected. "
+ "Addr: " INTPTR_FORMAT " Code: 0x%x not an %s instruction.", p2i(addr), inst, barrierInsn[i].name);
return false;
}
addr += 4;
diff --git a/src/hotspot/cpu/riscv/gc/shenandoah/c1/shenandoahBarrierSetC1_riscv.cpp b/src/hotspot/cpu/riscv/gc/shenandoah/c1/shenandoahBarrierSetC1_riscv.cpp
deleted file mode 100644
index 11c4e5dc81b..00000000000
--- a/src/hotspot/cpu/riscv/gc/shenandoah/c1/shenandoahBarrierSetC1_riscv.cpp
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved.
- * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#include "c1/c1_LIRAssembler.hpp"
-#include "c1/c1_MacroAssembler.hpp"
-#include "gc/shared/gc_globals.hpp"
-#include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
-#include "gc/shenandoah/shenandoahBarrierSet.hpp"
-#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
-
-#define __ masm->masm()->
-
-void LIR_OpShenandoahCompareAndSwap::emit_code(LIR_Assembler* masm) {
- Register addr = _addr->as_register_lo();
- Register newval = _new_value->as_register();
- Register cmpval = _cmp_value->as_register();
- Register tmp1 = _tmp1->as_register();
- Register tmp2 = _tmp2->as_register();
- Register result = result_opr()->as_register();
-
- if (UseCompressedOops) {
- __ encode_heap_oop(tmp1, cmpval);
- cmpval = tmp1;
- __ encode_heap_oop(tmp2, newval);
- newval = tmp2;
- }
-
- ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm->masm(), addr, cmpval, newval, /* acquire */ Assembler::aq,
- /* release */ Assembler::rl, /* is_cae */ false, result);
-}
-
-#undef __
-
-#ifdef ASSERT
-#define __ gen->lir(__FILE__, __LINE__)->
-#else
-#define __ gen->lir()->
-#endif
-
-LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value) {
- BasicType bt = access.type();
- if (access.is_oop()) {
- LIRGenerator *gen = access.gen();
- if (ShenandoahSATBBarrier) {
- pre_barrier(gen, access.access_emit_info(), access.decorators(), access.resolved_addr(),
- LIR_OprFact::illegalOpr /* pre_val */);
- }
- if (ShenandoahCASBarrier) {
- cmp_value.load_item();
- new_value.load_item();
-
- LIR_Opr tmp1 = gen->new_register(T_OBJECT);
- LIR_Opr tmp2 = gen->new_register(T_OBJECT);
- LIR_Opr addr = access.resolved_addr()->as_address_ptr()->base();
- LIR_Opr result = gen->new_register(T_INT);
-
- __ append(new LIR_OpShenandoahCompareAndSwap(addr, cmp_value.result(), new_value.result(), tmp1, tmp2, result));
-
- if (ShenandoahCardBarrier) {
- post_barrier(access, access.resolved_addr(), new_value.result());
- }
- return result;
- }
- }
-
- return BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value);
-}
-
-LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value) {
- LIRGenerator* gen = access.gen();
- BasicType type = access.type();
-
- LIR_Opr result = gen->new_register(type);
- value.load_item();
- LIR_Opr value_opr = value.result();
-
- assert(type == T_INT || is_reference_type(type) LP64_ONLY( || type == T_LONG ), "unexpected type");
- LIR_Opr tmp = gen->new_register(T_INT);
- __ xchg(access.resolved_addr(), value_opr, result, tmp);
-
- if (access.is_oop()) {
- result = load_reference_barrier(access.gen(), result, LIR_OprFact::addressConst(0), access.decorators());
- LIR_Opr tmp_opr = gen->new_register(type);
- __ move(result, tmp_opr);
- result = tmp_opr;
- if (ShenandoahSATBBarrier) {
- pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr,
- result /* pre_val */);
- }
- if (ShenandoahCardBarrier) {
- post_barrier(access, access.resolved_addr(), result);
- }
- }
-
- return result;
-}
diff --git a/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.cpp b/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.cpp
index 8d530d15ee5..156c6b9e5a8 100644
--- a/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.cpp
@@ -60,6 +60,8 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec
Address gc_state(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
assert_different_registers(src, dst, count, t0);
+ assert(!saved_regs.contains(t0), "Sanity: about to clobber t0");
+
__ lbu(t0, gc_state);
if (ShenandoahSATBBarrier && dest_uninitialized) {
__ test_bit(t0, t0, ShenandoahHeap::HAS_FORWARDED_BITPOS);
@@ -69,14 +71,26 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec
__ beqz(t0, done);
}
- __ push_reg(saved_regs, sp);
- if (UseCompressedOops) {
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop),
- src, dst, count);
- } else {
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop), src, dst, count);
+ __ push_call_clobbered_registers();
+ // If arguments are not in proper places, shuffle them.
+ // Doing this via the stack is the most straight-forward way to avoid
+ // accidentally smashing any register.
+ if (c_rarg0 != src || c_rarg1 != dst || c_rarg2 != count) {
+ __ push_reg(RegSet::of(src), sp);
+ __ push_reg(RegSet::of(dst), sp);
+ __ push_reg(RegSet::of(count), sp);
+ __ pop_reg(RegSet::of(c_rarg2), sp);
+ __ pop_reg(RegSet::of(c_rarg1), sp);
+ __ pop_reg(RegSet::of(c_rarg0), sp);
}
- __ pop_reg(saved_regs, sp);
+ address target = nullptr;
+ if (UseCompressedOops) {
+ target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop);
+ } else {
+ target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop);
+ }
+ __ call_VM_leaf(target, 3);
+ __ pop_call_clobbered_registers();
__ bind(done);
}
}
@@ -94,14 +108,8 @@ void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler* masm,
Register pre_val,
Register thread,
Register tmp1,
- Register tmp2,
- bool tosca_live,
- bool expand_call) {
+ Register tmp2) {
assert(ShenandoahSATBBarrier, "Should be checked by caller");
-
- // If expand_call is true then we expand the call_VM_leaf macro
- // directly to skip generating the check by
- // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
assert(thread == xthread, "must be");
Label done;
@@ -121,7 +129,12 @@ void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler* masm,
// Do we need to load the previous value?
if (obj != noreg) {
- __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
+ if (UseCompressedOops) {
+ __ lwu(pre_val, Address(obj, 0));
+ __ decode_heap_oop(pre_val);
+ } else {
+ __ ld(pre_val, Address(obj, 0));
+ }
}
// Is the previous value null?
@@ -142,33 +155,17 @@ void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler* masm,
__ sd(pre_val, Address(tmp1, 0));
__ j(done);
+ // Slow-path call.
__ bind(runtime);
- // save the live input values
- RegSet saved = RegSet::of(pre_val);
- if (tosca_live) saved += RegSet::of(x10);
- if (obj != noreg) saved += RegSet::of(obj);
-
- __ push_reg(saved, sp);
-
- // Calling the runtime using the regular call_VM_leaf mechanism generates
- // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
- // that checks that the *(rfp+frame::interpreter_frame_last_sp) is null.
- //
- // If we care generating the pre-barrier without a frame (e.g. in the
- // intrinsified Reference.get() routine) then ebp might be pointing to
- // the caller frame and so this check will most likely fail at runtime.
- //
- // Expanding the call directly bypasses the generation of the check.
- // So when we do not have have a full interpreter frame on the stack
- // expand_call should be passed true.
- if (expand_call) {
- assert(pre_val != c_rarg1, "smashed arg");
- __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), pre_val);
- } else {
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), pre_val);
+ __ enter();
+ __ push_call_clobbered_registers();
+ if (c_rarg0 != pre_val) {
+ __ mv(c_rarg0, pre_val);
}
-
- __ pop_reg(saved, sp);
+ // Calling with super_call_VM_leaf with c_rarg0 bypasses interpreter checks and avoids any moves.
+ __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), c_rarg0);
+ __ pop_call_clobbered_registers();
+ __ leave();
__ bind(done);
}
@@ -235,7 +232,6 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
bool is_narrow = UseCompressedOops && !is_native;
Label heap_stable, not_cset;
- __ enter();
Address gc_state(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
__ lbu(t1, gc_state);
@@ -275,6 +271,8 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
__ beqz(t0, not_cset);
}
+ // Slow-path call
+ __ enter();
__ push_call_clobbered_registers();
address target = nullptr;
if (is_strong) {
@@ -292,18 +290,20 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
} else {
assert(is_phantom, "only remaining strength");
assert(!is_narrow, "phantom access cannot be narrow");
- target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
+ target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
}
- __ rt_call(target);
+ // Calling with super_call_VM_leaf with c_rarg0/1 bypasses interpreter checks and avoids any moves.
+ __ super_call_VM_leaf(target, c_rarg0, c_rarg1);
__ mv(t0, x10);
__ pop_call_clobbered_registers();
__ mv(x10, t0);
+ __ leave();
+
__ bind(not_cset);
__ mv(result_dst, x10);
__ pop_reg(saved_regs, sp);
__ bind(heap_stable);
- __ leave();
}
//
@@ -365,18 +365,12 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm,
// 3: apply keep-alive barrier if needed
if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
- __ enter();
- __ push_call_clobbered_registers();
satb_barrier(masm /* masm */,
noreg /* obj */,
dst /* pre_val */,
xthread /* thread */,
tmp1 /* tmp1 */,
- tmp2 /* tmp2 */,
- true /* tosca_live */,
- true /* expand_call */);
- __ pop_call_clobbered_registers();
- __ leave();
+ tmp2 /* tmp2 */);
}
}
@@ -419,8 +413,6 @@ void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet
__ la(tmp3, dst);
}
- bool storing_non_null = (val != noreg);
-
// 2: pre-barrier: SATB needs the previous value
if (ShenandoahBarrierSet::need_satb_barrier(decorators, type)) {
satb_barrier(masm,
@@ -428,15 +420,14 @@ void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet
tmp2 /* pre_val */,
xthread /* thread */,
tmp1 /* tmp */,
- t0 /* tmp2 */,
- storing_non_null /* tosca_live */,
- false /* expand_call */);
+ t0 /* tmp2 */);
}
// Store!
BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
// 3: post-barrier: card barrier needs store address
+ bool storing_non_null = (val != noreg);
if (ShenandoahBarrierSet::need_card_barrier(decorators, type) && storing_non_null) {
card_barrier(masm, tmp3);
}
@@ -462,15 +453,16 @@ void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler
__ bind(done);
}
-#ifdef COMPILER2
-void ShenandoahBarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler *masm, Register obj,
- Register tmp, Label& slow_path) {
- assert_different_registers(obj, tmp);
+void ShenandoahBarrierSetAssembler::try_peek_weak_handle_in_nmethod(MacroAssembler *masm, Register weak_handle,
+ Register obj, Register tmp, Label& slow_path) {
+ assert_different_registers(weak_handle, tmp, noreg);
+ assert_different_registers(obj, tmp, noreg);
+
Label done;
- // Resolve weak handle using the standard implementation.
- BarrierSetAssembler::try_resolve_weak_handle_in_c2(masm, obj, tmp, slow_path);
+ // Peek weak handle using the standard implementation.
+ BarrierSetAssembler::try_peek_weak_handle_in_nmethod(masm, weak_handle, obj, tmp, slow_path);
// Check if the reference is null, and if it is, take the fast path.
__ beqz(obj, done);
@@ -484,7 +476,6 @@ void ShenandoahBarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler
__ bnez(tmp, slow_path);
__ bind(done);
}
-#endif
// Special Shenandoah CAS implementation that handles false negatives due
// to concurrent evacuation. The service is more complex than a
diff --git a/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.hpp b/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.hpp
index e35e09c93da..eed20e00a65 100644
--- a/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.hpp
@@ -47,9 +47,7 @@ private:
Register pre_val,
Register thread,
Register tmp1,
- Register tmp2,
- bool tosca_live,
- bool expand_call);
+ Register tmp2);
void card_barrier(MacroAssembler* masm, Register obj);
@@ -85,9 +83,8 @@ public:
virtual void try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
Register obj, Register tmp, Label& slowpath);
-#ifdef COMPILER2
- virtual void try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path);
-#endif
+ virtual void try_peek_weak_handle_in_nmethod(MacroAssembler* masm, Register weak_handle, Register obj,
+ Register tmp, Label& slow_path);
void cmpxchg_oop(MacroAssembler* masm, Register addr, Register expected, Register new_val,
Assembler::Aqrl acquire, Assembler::Aqrl release, bool is_cae, Register result);
};
diff --git a/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.cpp b/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.cpp
index 163271a2f11..bf37ccf64e2 100644
--- a/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.cpp
@@ -602,12 +602,16 @@ void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm,
BLOCK_COMMENT("} ZBarrierSetAssembler::try_resolve_jobject_in_native");
}
-#ifdef COMPILER2
-void ZBarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path) {
- BLOCK_COMMENT("ZBarrierSetAssembler::try_resolve_weak_handle_in_c2 {");
+void ZBarrierSetAssembler::try_peek_weak_handle_in_nmethod(MacroAssembler* masm, Register weak_handle, Register obj,
+ Register tmp, Label& slow_path) {
+ BLOCK_COMMENT("ZBarrierSetAssembler::try_peek_weak_handle_in_nmethod {");
- // Resolve weak handle using the standard implementation.
- BarrierSetAssembler::try_resolve_weak_handle_in_c2(masm, obj, tmp, slow_path);
+ assert_different_registers(weak_handle, tmp, noreg);
+ assert_different_registers(obj, tmp, noreg);
+
+
+ // Peek weak handle using the standard implementation.
+ BarrierSetAssembler::try_peek_weak_handle_in_nmethod(masm, weak_handle, obj, tmp, slow_path);
// Check if the oop is bad, in which case we need to take the slow path.
__ relocate(barrier_Relocation::spec(), [&] {
@@ -619,9 +623,8 @@ void ZBarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler* masm, R
// Oop is okay, so we uncolor it.
__ srli(obj, obj, ZPointerLoadShift);
- BLOCK_COMMENT("} ZBarrierSetAssembler::try_resolve_weak_handle_in_c2");
+ BLOCK_COMMENT("} ZBarrierSetAssembler::try_peek_weak_handle_in_nmethod");
}
-#endif
static uint16_t patch_barrier_relocation_value(int format) {
switch (format) {
@@ -734,6 +737,7 @@ public:
#define __ masm->
void ZBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, ZLoadBarrierStubC2* stub) const {
+ Assembler::InlineSkippedInstructionsCounter skipped_counter(masm);
BLOCK_COMMENT("ZLoadBarrierStubC2");
// Stub entry
@@ -753,6 +757,7 @@ void ZBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, Z
}
void ZBarrierSetAssembler::generate_c2_store_barrier_stub(MacroAssembler* masm, ZStoreBarrierStubC2* stub) const {
+ Assembler::InlineSkippedInstructionsCounter skipped_counter(masm);
BLOCK_COMMENT("ZStoreBarrierStubC2");
// Stub entry
diff --git a/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.hpp b/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.hpp
index 648cb3bf63d..f99df299a77 100644
--- a/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.hpp
@@ -170,12 +170,14 @@ public:
ZLoadBarrierStubC2* stub) const;
void generate_c2_store_barrier_stub(MacroAssembler* masm,
ZStoreBarrierStubC2* stub) const;
- void try_resolve_weak_handle_in_c2(MacroAssembler* masm,
- Register obj,
- Register tmp,
- Label& slow_path);
#endif // COMPILER2
+ void try_peek_weak_handle_in_nmethod(MacroAssembler* masm,
+ Register weak_handle,
+ Register obj,
+ Register tmp,
+ Label& slow_path);
+
void check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error);
};
diff --git a/src/hotspot/cpu/riscv/gc/z/z_riscv.ad b/src/hotspot/cpu/riscv/gc/z/z_riscv.ad
index a408cf309d5..0078deb76e8 100644
--- a/src/hotspot/cpu/riscv/gc/z/z_riscv.ad
+++ b/src/hotspot/cpu/riscv/gc/z/z_riscv.ad
@@ -1,5 +1,5 @@
//
-// Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2019, 2026, Oracle and/or its affiliates. All rights reserved.
// Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
@@ -57,6 +57,7 @@ static void check_color(MacroAssembler* masm, Register ref, bool on_non_strong,
}
static void z_load_barrier(MacroAssembler* masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) {
+ Assembler::InlineSkippedInstructionsCounter skipped_counter(masm);
const bool on_non_strong =
((node->barrier_data() & ZBarrierWeak) != 0) ||
((node->barrier_data() & ZBarrierPhantom) != 0);
@@ -78,6 +79,7 @@ static void z_load_barrier(MacroAssembler* masm, const MachNode* node, Address r
}
static void z_store_barrier(MacroAssembler* masm, const MachNode* node, Address ref_addr, Register rnew_zaddress, Register rnew_zpointer, Register tmp, bool is_atomic) {
+ Assembler::InlineSkippedInstructionsCounter skipped_counter(masm);
if (node->barrier_data() == ZBarrierElided) {
z_color(masm, node, rnew_zpointer, rnew_zaddress, tmp);
} else {
diff --git a/src/hotspot/cpu/riscv/riscv.ad b/src/hotspot/cpu/riscv/riscv.ad
index e236d03e6d2..e615b946b44 100644
--- a/src/hotspot/cpu/riscv/riscv.ad
+++ b/src/hotspot/cpu/riscv/riscv.ad
@@ -1580,7 +1580,7 @@ uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *r
assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
- if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
+ if (src_hi != OptoReg::Bad && !bottom_type()->isa_pvectmask()) {
assert((src_lo & 1) == 0 && src_lo + 1 == src_hi &&
(dst_lo & 1) == 0 && dst_lo + 1 == dst_hi,
"expected aligned-adjacent pairs");
@@ -1616,7 +1616,7 @@ uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *r
} else {
ShouldNotReachHere();
}
- } else if (bottom_type()->isa_vectmask() && masm) {
+ } else if (bottom_type()->isa_pvectmask() && masm) {
int vmask_size_in_bytes = Matcher::scalable_predicate_reg_slots() * 32 / 8;
if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
// stack to stack
@@ -1718,7 +1718,7 @@ uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *r
} else {
st->print("%s", Matcher::regName[dst_lo]);
}
- if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
+ if (bottom_type()->isa_vect() && !bottom_type()->isa_pvectmask()) {
int vsize = 0;
if (ideal_reg() == Op_VecA) {
vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
diff --git a/src/hotspot/cpu/riscv/riscv_v.ad b/src/hotspot/cpu/riscv/riscv_v.ad
index aeb8398f06f..a0af43364cb 100644
--- a/src/hotspot/cpu/riscv/riscv_v.ad
+++ b/src/hotspot/cpu/riscv/riscv_v.ad
@@ -181,8 +181,8 @@ source %{
}
bool Matcher::mask_op_prefers_predicate(int opcode, const TypeVect* vt) {
- // Prefer predicate if the mask type is "TypeVectMask".
- return vt->isa_vectmask() != nullptr;
+ // Prefer predicate if the mask type is "TypePVectMask".
+ return vt->isa_pvectmask() != nullptr;
}
%}
diff --git a/src/hotspot/cpu/riscv/templateTable_riscv.cpp b/src/hotspot/cpu/riscv/templateTable_riscv.cpp
index 5cc725e3af4..bae5bb7b57b 100644
--- a/src/hotspot/cpu/riscv/templateTable_riscv.cpp
+++ b/src/hotspot/cpu/riscv/templateTable_riscv.cpp
@@ -3389,7 +3389,6 @@ void TemplateTable::invokeinterface(int byte_no) {
__ bind(notVFinal);
// Get receiver klass into x13
- __ restore_locals();
__ load_klass(x13, x12);
Label no_such_method;
diff --git a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp
index e1d8d062c23..38698370faa 100644
--- a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp
@@ -2884,10 +2884,23 @@ void LIR_Assembler::on_spin_wait() {
}
void LIR_Assembler::leal(LIR_Opr addr_opr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
- assert(patch_code == lir_patch_none, "Patch code not supported");
+ assert(addr_opr->is_address(), "must be an address");
+ assert(dest->is_register(), "must be a register");
+
LIR_Address* addr = addr_opr->as_address_ptr();
+ Register reg = dest->as_pointer_register();
assert(addr->scale() == LIR_Address::times_1, "scaling unsupported");
- __ load_address(dest->as_pointer_register(), as_Address(addr));
+
+ if (addr->index()->is_illegal() && patch_code != lir_patch_none) {
+ PatchingStub* patch = new PatchingStub(_masm, PatchingStub::access_field_id);
+
+ // TODO: Use load_const_32to64 here by extending NativeMovRegMem to support both instruction patterns.
+ __ load_const(Z_R0_scratch, (intptr_t)0);
+ __ z_agrk(reg, addr->base()->as_pointer_register(), Z_R0_scratch);
+ patching_epilog(patch, patch_code, addr->base()->as_register(), info);
+ } else {
+ __ load_address(reg, as_Address(addr));
+ }
}
void LIR_Assembler::get_thread(LIR_Opr result_reg) {
diff --git a/src/hotspot/cpu/s390/c1_LIRGenerator_s390.cpp b/src/hotspot/cpu/s390/c1_LIRGenerator_s390.cpp
index 5a0fd5f9561..1ffd172df8f 100644
--- a/src/hotspot/cpu/s390/c1_LIRGenerator_s390.cpp
+++ b/src/hotspot/cpu/s390/c1_LIRGenerator_s390.cpp
@@ -1046,6 +1046,7 @@ void LIRGenerator::volatile_field_store(LIR_Opr value, LIR_Address* address,
void LIRGenerator::volatile_field_load(LIR_Address* address, LIR_Opr result,
CodeEmitInfo* info) {
__ load(address, result, info);
+ __ membar_acquire();
}
void LIRGenerator::do_update_CRC32(Intrinsic* x) {
diff --git a/src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.cpp b/src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.cpp
index 617bc7cd00c..881fb613114 100644
--- a/src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.cpp
@@ -281,6 +281,7 @@ void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorator
if (on_oop && on_reference && L_handle_null == nullptr) { L_handle_null = &done; }
CardTableBarrierSetAssembler::load_at(masm, decorators, type, src, dst, tmp1, tmp2, L_handle_null);
if (on_oop && on_reference) {
+ assert(tmp1 != noreg && tmp2 != noreg, "need temp registers for G1 pre-barrier");
// Generate the G1 pre-barrier code to log the value of
// the referent field in an SATB buffer.
g1_write_barrier_pre(masm, decorators | IS_NOT_NULL,
diff --git a/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.cpp b/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.cpp
index 9fac231df47..9a401766200 100644
--- a/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.cpp
@@ -169,9 +169,10 @@ void BarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Re
__ z_lg(obj, 0, obj); // Resolve (untagged) jobject.
}
-void BarrierSetAssembler::try_resolve_weak_handle(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path) {
- // Load the oop from the weak handle.
- __ z_lg(obj, Address(obj));
+void BarrierSetAssembler::try_peek_weak_handle_in_nmethod(MacroAssembler* masm, Register weak_handle, Register obj,
+ Register tmp, Label& slow_path) {
+ // Load the oop from the weak handle without barriers.
+ __ z_lg(obj, Address(weak_handle));
}
void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm) {
diff --git a/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.hpp b/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.hpp
index 8e76ec2f4b4..6c729528a67 100644
--- a/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.hpp
+++ b/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.hpp
@@ -58,10 +58,10 @@ public:
virtual void try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
Register obj, Register tmp, Label& slowpath);
- // Can be used in nmethods including native wrappers.
- // Attention: obj will only be valid until next safepoint (no SATB barrier).
- // (other platforms currently use it for C2 only: try_resolve_weak_handle_in_c2)
- virtual void try_resolve_weak_handle(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path);
+ // See AS_NO_KEEPALIVE for peek semantics
+ // weak_handle and obj may alias
+ virtual void try_peek_weak_handle_in_nmethod(MacroAssembler* masm, Register weak_handle, Register obj,
+ Register tmp, Label& slow_path);
virtual void nmethod_entry_barrier(MacroAssembler* masm);
diff --git a/src/hotspot/cpu/s390/interp_masm_s390.cpp b/src/hotspot/cpu/s390/interp_masm_s390.cpp
index d5239898dd7..7327e2a13f2 100644
--- a/src/hotspot/cpu/s390/interp_masm_s390.cpp
+++ b/src/hotspot/cpu/s390/interp_masm_s390.cpp
@@ -411,7 +411,7 @@ void InterpreterMacroAssembler::load_resolved_reference_at_index(Register result
// Load pointer for resolved_references[] objArray.
z_lg(result, in_bytes(ConstantPool::cache_offset()), result);
z_lg(result, in_bytes(ConstantPoolCache::resolved_references_offset()), result);
- resolve_oop_handle(result); // Load resolved references array itself.
+ resolve_oop_handle(result, Z_R0_scratch, Z_R1_scratch); // Load resolved references array itself.
#ifdef ASSERT
NearLabel index_ok;
z_lgf(Z_R0, Address(result, arrayOopDesc::length_offset_in_bytes()));
diff --git a/src/hotspot/cpu/s390/macroAssembler_s390.cpp b/src/hotspot/cpu/s390/macroAssembler_s390.cpp
index de3608e74ba..4d2bbe796fb 100644
--- a/src/hotspot/cpu/s390/macroAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/macroAssembler_s390.cpp
@@ -4705,16 +4705,8 @@ void MacroAssembler::oop_decoder(Register Rdst, Register Rsrc, bool maybenull, R
}
// ((OopHandle)result).resolve();
-void MacroAssembler::resolve_oop_handle(Register result) {
- // OopHandle::resolve is an indirection.
- z_lg(result, 0, result);
-}
-
-void MacroAssembler::load_mirror_from_const_method(Register mirror, Register const_method) {
- mem2reg_opt(mirror, Address(const_method, ConstMethod::constants_offset()));
- mem2reg_opt(mirror, Address(mirror, ConstantPool::pool_holder_offset()));
- mem2reg_opt(mirror, Address(mirror, Klass::java_mirror_offset()));
- resolve_oop_handle(mirror);
+void MacroAssembler::resolve_oop_handle(Register result, Register tmp1, Register tmp2) {
+ access_load_at(T_OBJECT, IN_NATIVE, Address(result, 0), result, tmp1, tmp2);
}
void MacroAssembler::load_method_holder(Register holder, Register method) {
@@ -5886,6 +5878,28 @@ void MacroAssembler::asm_assert_frame_size(Register expected_size, Register tmp,
#endif // ASSERT
}
+#ifdef ASSERT
+bool is_excluded(Register excluded_register[], Register reg, int n) {
+ for (int i = 0; i < n; i++) {
+ if (excluded_register[i] == reg) {
+ return true;
+ }
+ }
+ return false;
+}
+
+void MacroAssembler::clobber_volatile_registers(Register excluded_register[], int n) {
+ const int magic_number = 0x82;
+
+ for (int i = 0; i < 6 /* R0 to R5 */; i++) {
+ Register reg = as_Register(i);
+ if (!is_excluded(excluded_register, reg, n)) {
+ load_const_optimized(reg, magic_number);
+ }
+ }
+}
+#endif // ASSERT
+
// Save and restore functions: Exclude Z_R0.
void MacroAssembler::save_volatile_regs(Register dst, int offset, bool include_fp, bool include_flags) {
z_stmg(Z_R1, Z_R5, offset, dst); offset += 5 * BytesPerWord;
@@ -6381,7 +6395,7 @@ void MacroAssembler::compiler_fast_lock_object(Register obj, Register box, Regis
// Check if object matches.
z_lg(tmp2, Address(tmp1_monitor, ObjectMonitor::object_offset()));
BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
- bs_asm->try_resolve_weak_handle(this, tmp2, Z_R0_scratch, slow_path);
+ bs_asm->try_peek_weak_handle_in_nmethod(this, tmp2, tmp2, Z_R0_scratch, slow_path);
z_cgr(obj, tmp2);
z_brne(slow_path);
diff --git a/src/hotspot/cpu/s390/macroAssembler_s390.hpp b/src/hotspot/cpu/s390/macroAssembler_s390.hpp
index 32e484d4790..34389917cef 100644
--- a/src/hotspot/cpu/s390/macroAssembler_s390.hpp
+++ b/src/hotspot/cpu/s390/macroAssembler_s390.hpp
@@ -484,6 +484,10 @@ class MacroAssembler: public Assembler {
// Pop current C frame and restore return PC register (Z_R14).
void pop_frame_restore_retPC(int frame_size_in_bytes);
+#ifdef ASSERT
+ void clobber_volatile_registers(Register excluded_register[], int n);
+#endif // ASSERT
+
//
// Calls
//
@@ -885,8 +889,7 @@ class MacroAssembler: public Assembler {
void oop_decoder(Register Rdst, Register Rsrc, bool maybenull,
Register Rbase = Z_R1, int pow2_offset = -1);
- void resolve_oop_handle(Register result);
- void load_mirror_from_const_method(Register mirror, Register const_method);
+ void resolve_oop_handle(Register result, Register tmp1, Register tmp2);
void load_method_holder(Register holder, Register method);
//--------------------------
diff --git a/src/hotspot/cpu/s390/registerMap_s390.cpp b/src/hotspot/cpu/s390/registerMap_s390.cpp
new file mode 100644
index 00000000000..85a49ff1d60
--- /dev/null
+++ b/src/hotspot/cpu/s390/registerMap_s390.cpp
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2026 IBM Corp. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "runtime/registerMap.hpp"
+
+address RegisterMap::pd_location(VMReg base_reg, int slot_idx) const {
+ if (base_reg->is_VectorRegister()) {
+ // Not all physical slots belonging to a VectorRegister have corresponding
+ // valid VMReg locations in the RegisterMap.
+ // (See RegisterSaver::save_live_registers.)
+ // However, the slots are always saved to the stack in a contiguous region
+ // of memory so we can calculate the address of the upper slots by
+ // offsetting from the base address.
+ assert(base_reg->is_concrete(), "must pass base reg");
+ address base_location = location(base_reg, nullptr);
+ if (base_location != nullptr) {
+ intptr_t offset_in_bytes = slot_idx * VMRegImpl::stack_slot_size;
+ return base_location + offset_in_bytes;
+ } else {
+ return nullptr;
+ }
+ } else {
+ return location(base_reg->next(slot_idx), nullptr);
+ }
+}
diff --git a/src/hotspot/cpu/s390/registerMap_s390.hpp b/src/hotspot/cpu/s390/registerMap_s390.hpp
index 827e3b44e04..9069fb1e31d 100644
--- a/src/hotspot/cpu/s390/registerMap_s390.hpp
+++ b/src/hotspot/cpu/s390/registerMap_s390.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -36,9 +36,7 @@
// Since there is none, we just return null.
address pd_location(VMReg reg) const {return nullptr;}
- address pd_location(VMReg base_reg, int slot_idx) const {
- return location(base_reg->next(slot_idx), nullptr);
- }
+ address pd_location(VMReg base_reg, int slot_idx) const;
// No PD state to clear or copy.
void pd_clear() {}
diff --git a/src/hotspot/cpu/s390/registerSaver_s390.hpp b/src/hotspot/cpu/s390/registerSaver_s390.hpp
index a049f8b581b..2d3c35250ba 100644
--- a/src/hotspot/cpu/s390/registerSaver_s390.hpp
+++ b/src/hotspot/cpu/s390/registerSaver_s390.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -58,7 +58,7 @@ class RegisterSaver {
// During deoptimization only the result register need to be restored
// all the other values have already been extracted.
- static void restore_result_registers(MacroAssembler* masm);
+ static void restore_result_registers(MacroAssembler* masm, bool save_vectors);
// Constants and data structures:
diff --git a/src/hotspot/cpu/s390/sharedRuntime_s390.cpp b/src/hotspot/cpu/s390/sharedRuntime_s390.cpp
index 00a830a80cd..e5a27e66968 100644
--- a/src/hotspot/cpu/s390/sharedRuntime_s390.cpp
+++ b/src/hotspot/cpu/s390/sharedRuntime_s390.cpp
@@ -402,9 +402,7 @@ OopMap* RegisterSaver::save_live_registers(MacroAssembler* masm, RegisterSet reg
break;
}
- // Second set_callee_saved is really a waste but we'll keep things as they were for now
map->set_callee_saved(VMRegImpl::stack2reg(offset >> 2), live_regs[i].vmreg);
- map->set_callee_saved(VMRegImpl::stack2reg((offset + half_reg_size) >> 2), live_regs[i].vmreg->next());
}
assert(first != noreg, "Should spill at least one int reg.");
__ z_stmg(first, last, first_offset, Z_SP);
@@ -416,12 +414,6 @@ OopMap* RegisterSaver::save_live_registers(MacroAssembler* masm, RegisterSet reg
map->set_callee_saved(VMRegImpl::stack2reg(offset>>2),
RegisterSaver_LiveVRegs[i].vmreg);
- map->set_callee_saved(VMRegImpl::stack2reg((offset + half_reg_size ) >> 2),
- RegisterSaver_LiveVRegs[i].vmreg->next());
- map->set_callee_saved(VMRegImpl::stack2reg((offset + (half_reg_size * 2)) >> 2),
- RegisterSaver_LiveVRegs[i].vmreg->next(2));
- map->set_callee_saved(VMRegImpl::stack2reg((offset + (half_reg_size * 3)) >> 2),
- RegisterSaver_LiveVRegs[i].vmreg->next(3));
}
assert(offset == frame_size_in_bytes, "consistency check");
@@ -473,7 +465,6 @@ OopMap* RegisterSaver::generate_oop_map(MacroAssembler* masm, RegisterSet reg_se
for (int i = 0; i < regstosave_num; i++) {
if (live_regs[i].reg_type < RegisterSaver::excluded_reg) {
map->set_callee_saved(VMRegImpl::stack2reg(offset>>2), live_regs[i].vmreg);
- map->set_callee_saved(VMRegImpl::stack2reg((offset + half_reg_size)>>2), live_regs[i].vmreg->next());
}
offset += reg_size;
}
@@ -580,10 +571,12 @@ void RegisterSaver::restore_live_registers(MacroAssembler* masm, RegisterSet reg
// Pop the current frame and restore the registers that might be holding a result.
-void RegisterSaver::restore_result_registers(MacroAssembler* masm) {
+void RegisterSaver::restore_result_registers(MacroAssembler* masm, bool save_vectors) {
const int regstosave_num = sizeof(RegisterSaver_LiveRegs) /
sizeof(RegisterSaver::LiveRegType);
- const int register_save_offset = live_reg_frame_size(all_registers) - live_reg_save_size(all_registers);
+ const int vecregstosave_num = save_vectors ? calculate_vregstosave_num() : 0;
+ const int vreg_save_size = vecregstosave_num * v_reg_size;
+ const int register_save_offset = live_reg_frame_size(all_registers, save_vectors) - (live_reg_save_size(all_registers) + vreg_save_size);
// Restore all result registers (ints and floats).
int offset = register_save_offset;
@@ -609,7 +602,7 @@ void RegisterSaver::restore_result_registers(MacroAssembler* masm) {
ShouldNotReachHere();
}
}
- assert(offset == live_reg_frame_size(all_registers), "consistency check");
+ assert(offset == live_reg_frame_size(all_registers, save_vectors) - (save_vectors ? vreg_save_size : 0) , "consistency check");
}
// ---------------------------------------------------------------------------
@@ -2557,7 +2550,7 @@ void SharedRuntime::generate_deopt_blob() {
// nmethod that was valid just before the nmethod was deoptimized.
// save R14 into the deoptee frame. the `fetch_unroll_info'
// procedure called below will read it from there.
- map = RegisterSaver::save_live_registers(masm, RegisterSaver::all_registers);
+ map = RegisterSaver::save_live_registers(masm, RegisterSaver::all_registers, Z_R14, /* save_vectors= */ SuperwordUseVX);
// note the entry point.
__ load_const_optimized(exec_mode_reg, Deoptimization::Unpack_deopt);
@@ -2573,7 +2566,7 @@ void SharedRuntime::generate_deopt_blob() {
int reexecute_offset = __ offset() - start_off;
// No need to update map as each call to save_live_registers will produce identical oopmap
- (void) RegisterSaver::save_live_registers(masm, RegisterSaver::all_registers);
+ (void) RegisterSaver::save_live_registers(masm, RegisterSaver::all_registers, Z_R14, /* save_vectors= */ SuperwordUseVX);
__ load_const_optimized(exec_mode_reg, Deoptimization::Unpack_reexecute);
__ z_bru(exec_mode_initialized);
@@ -2611,7 +2604,7 @@ void SharedRuntime::generate_deopt_blob() {
__ z_lg(Z_R1_scratch, Address(Z_thread, JavaThread::exception_pc_offset()));
// Save everything in sight.
- (void) RegisterSaver::save_live_registers(masm, RegisterSaver::all_registers, Z_R1_scratch);
+ (void) RegisterSaver::save_live_registers(masm, RegisterSaver::all_registers, Z_R1_scratch, /* save_vectors= */ SuperwordUseVX);
// Now it is safe to overwrite any register
@@ -2661,7 +2654,7 @@ void SharedRuntime::generate_deopt_blob() {
__ z_lgr(unroll_block_reg, Z_RET);
// restore the return registers that have been saved
// (among other registers) by save_live_registers(...).
- RegisterSaver::restore_result_registers(masm);
+ RegisterSaver::restore_result_registers(masm, /* save_vectors= */ SuperwordUseVX);
// reload the exec mode from the UnrollBlock (it might have changed)
__ z_llgf(exec_mode_reg, Address(unroll_block_reg, Deoptimization::UnrollBlock::unpack_kind_offset()));
@@ -2737,7 +2730,7 @@ void SharedRuntime::generate_deopt_blob() {
// Make sure all code is generated
masm->flush();
- _deopt_blob = DeoptimizationBlob::create(&buffer, oop_maps, 0, exception_offset, reexecute_offset, RegisterSaver::live_reg_frame_size(RegisterSaver::all_registers)/wordSize);
+ _deopt_blob = DeoptimizationBlob::create(&buffer, oop_maps, 0, exception_offset, reexecute_offset, RegisterSaver::live_reg_frame_size(RegisterSaver::all_registers, SuperwordUseVX)/wordSize);
_deopt_blob->set_unpack_with_exception_in_tls_offset(exception_in_tls_offset);
}
diff --git a/src/hotspot/cpu/s390/templateInterpreterGenerator_s390.cpp b/src/hotspot/cpu/s390/templateInterpreterGenerator_s390.cpp
index 2da21f08bbc..dba04fc0e85 100644
--- a/src/hotspot/cpu/s390/templateInterpreterGenerator_s390.cpp
+++ b/src/hotspot/cpu/s390/templateInterpreterGenerator_s390.cpp
@@ -1113,6 +1113,7 @@ void TemplateInterpreterGenerator::generate_fixed_frame(bool native_call) {
{
// locals
const Register local_addr = Z_ARG4;
+ const Register constants_addr = Z_ARG2;
BLOCK_COMMENT("generate_fixed_frame: initialize interpreter state {");
@@ -1128,8 +1129,8 @@ void TemplateInterpreterGenerator::generate_fixed_frame(bool native_call) {
__ z_stg(Z_R10, _z_ijava_state_neg(sender_sp), fp);
// Load cp cache and save it at the end of this block.
- __ z_lg(Z_R1_scratch, Address(const_method, ConstMethod::constants_offset()));
- __ z_lg(Z_R1_scratch, Address(Z_R1_scratch, ConstantPool::cache_offset()));
+ __ z_lg(constants_addr, Address(const_method, ConstMethod::constants_offset()));
+ __ z_lg(Z_R1_scratch, Address(constants_addr, ConstantPool::cache_offset()));
// z_ijava_state->method = method;
__ z_stg(Z_method, _z_ijava_state_neg(method), fp);
@@ -1192,7 +1193,9 @@ void TemplateInterpreterGenerator::generate_fixed_frame(bool native_call) {
__ z_stg(Z_R1_scratch, _z_ijava_state_neg(cpoolCache), fp);
// Get mirror and store it in the frame as GC root for this Method*.
- __ load_mirror_from_const_method(Z_R1_scratch, const_method);
+ __ mem2reg_opt(Z_R1_scratch, Address(constants_addr, ConstantPool::pool_holder_offset()));
+ __ mem2reg_opt(Z_R1_scratch, Address(Z_R1_scratch, Klass::java_mirror_offset()));
+ __ resolve_oop_handle(Z_R1_scratch, Z_R0_scratch, Z_R1_scratch);
__ z_stg(Z_R1_scratch, _z_ijava_state_neg(mirror), fp);
BLOCK_COMMENT("} generate_fixed_frame: initialize interpreter state");
@@ -2028,7 +2031,7 @@ address TemplateInterpreterGenerator::generate_currentThread() {
uint64_t entry_off = __ offset();
__ z_lg(Z_RET, Address(Z_thread, JavaThread::threadObj_offset()));
- __ resolve_oop_handle(Z_RET);
+ __ resolve_oop_handle(Z_RET, Z_R0_scratch, Z_R1_scratch);
// Restore caller sp for c2i case.
__ resize_frame_absolute(Z_R10, Z_R0, true); // Cut the stack back to where the caller started.
diff --git a/src/hotspot/cpu/s390/templateTable_s390.cpp b/src/hotspot/cpu/s390/templateTable_s390.cpp
index 647915ef4fa..3b0929608a3 100644
--- a/src/hotspot/cpu/s390/templateTable_s390.cpp
+++ b/src/hotspot/cpu/s390/templateTable_s390.cpp
@@ -480,8 +480,9 @@ void TemplateTable::fast_aldc(LdcType type) {
// Convert null sentinel to null.
__ load_const_optimized(Z_R1_scratch, (intptr_t)Universe::the_null_sentinel_addr());
- __ resolve_oop_handle(Z_R1_scratch);
- __ z_cg(Z_tos, Address(Z_R1_scratch));
+ __ z_lg(Z_R1_scratch, Address(Z_R1_scratch));
+ __ resolve_oop_handle(Z_R1_scratch, Z_R0_scratch, Z_R1_scratch);
+ __ z_cgr(Z_tos, Z_R1_scratch);
__ z_brne(L_resolved);
__ clear_reg(Z_tos);
__ z_bru(L_resolved);
@@ -2478,7 +2479,7 @@ void TemplateTable::load_resolved_field_entry(Register obj,
if (is_static) {
__ load_sized_value(obj, Address(cache, ResolvedFieldEntry::field_holder_offset()), sizeof(void*), false);
__ load_sized_value(obj, Address(obj, in_bytes(Klass::java_mirror_offset())), sizeof(void*), false);
- __ resolve_oop_handle(obj);
+ __ resolve_oop_handle(obj, Z_R0_scratch, Z_R1_scratch);
}
}
diff --git a/src/hotspot/cpu/s390/vm_version_s390.cpp b/src/hotspot/cpu/s390/vm_version_s390.cpp
index 7e9000991ca..c3f981f159a 100644
--- a/src/hotspot/cpu/s390/vm_version_s390.cpp
+++ b/src/hotspot/cpu/s390/vm_version_s390.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -24,8 +24,9 @@
*/
#include "asm/assembler.inline.hpp"
-#include "compiler/disassembler.hpp"
#include "code/compiledIC.hpp"
+#include "compiler/compilerDefinitions.inline.hpp"
+#include "compiler/disassembler.hpp"
#include "jvm.h"
#include "memory/resourceArea.hpp"
#include "runtime/java.hpp"
@@ -105,7 +106,7 @@ void VM_Version::initialize() {
int model_ix = get_model_index();
if ( model_ix >= 7 ) {
- if (FLAG_IS_DEFAULT(SuperwordUseVX)) {
+ if (FLAG_IS_DEFAULT(SuperwordUseVX) && CompilerConfig::is_c2_enabled()) {
FLAG_SET_ERGO(SuperwordUseVX, true);
}
if (model_ix > 7 && FLAG_IS_DEFAULT(UseSFPV) && SuperwordUseVX) {
diff --git a/src/hotspot/cpu/s390/vmreg_s390.hpp b/src/hotspot/cpu/s390/vmreg_s390.hpp
index 517fb8e2130..5fb5b7b40b1 100644
--- a/src/hotspot/cpu/s390/vmreg_s390.hpp
+++ b/src/hotspot/cpu/s390/vmreg_s390.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -59,7 +59,12 @@ inline VectorRegister as_VectorRegister() {
inline bool is_concrete() {
assert(is_reg(), "must be");
- return is_even(value());
+ if (is_Register() || is_FloatRegister()) return is_even(value());
+ if (is_VectorRegister()) {
+ int base = value() - ConcreteRegisterImpl::max_fpr;
+ return (base & 3) == 0;
+ }
+ return true;
}
#endif // CPU_S390_VMREG_S390_HPP
diff --git a/src/hotspot/cpu/x86/assembler_x86.cpp b/src/hotspot/cpu/x86/assembler_x86.cpp
index a4f2968f0d1..0c8dd85b15d 100644
--- a/src/hotspot/cpu/x86/assembler_x86.cpp
+++ b/src/hotspot/cpu/x86/assembler_x86.cpp
@@ -1664,14 +1664,14 @@ void Assembler::eandl(Register dst, Register src1, Register src2, bool no_flags)
}
void Assembler::andnl(Register dst, Register src1, Register src2) {
- assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
+ assert(VM_Version::supports_bmi1() && VM_Version::supports_avx(), "bit manipulation instructions not supported");
InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes, true);
emit_int16((unsigned char)0xF2, (0xC0 | encode));
}
void Assembler::andnl(Register dst, Register src1, Address src2) {
- assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
+ assert(VM_Version::supports_bmi1() && VM_Version::supports_avx(), "bit manipulation instructions not supported");
InstructionMark im(this);
InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
@@ -1696,14 +1696,14 @@ void Assembler::bswapl(Register reg) { // bswap
}
void Assembler::blsil(Register dst, Register src) {
- assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
+ assert(VM_Version::supports_bmi1() && VM_Version::supports_avx(), "bit manipulation instructions not supported");
InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes, true);
emit_int16((unsigned char)0xF3, (0xC0 | encode));
}
void Assembler::blsil(Register dst, Address src) {
- assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
+ assert(VM_Version::supports_bmi1() && VM_Version::supports_avx(), "bit manipulation instructions not supported");
InstructionMark im(this);
InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
@@ -1713,7 +1713,7 @@ void Assembler::blsil(Register dst, Address src) {
}
void Assembler::blsmskl(Register dst, Register src) {
- assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
+ assert(VM_Version::supports_bmi1() && VM_Version::supports_avx(), "bit manipulation instructions not supported");
InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes, true);
emit_int16((unsigned char)0xF3,
@@ -1721,7 +1721,7 @@ void Assembler::blsmskl(Register dst, Register src) {
}
void Assembler::blsmskl(Register dst, Address src) {
- assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
+ assert(VM_Version::supports_bmi1() && VM_Version::supports_avx(), "bit manipulation instructions not supported");
InstructionMark im(this);
InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
@@ -1731,14 +1731,14 @@ void Assembler::blsmskl(Register dst, Address src) {
}
void Assembler::blsrl(Register dst, Register src) {
- assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
+ assert(VM_Version::supports_bmi1() && VM_Version::supports_avx(), "bit manipulation instructions not supported");
InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes, true);
emit_int16((unsigned char)0xF3, (0xC0 | encode));
}
void Assembler::blsrl(Register dst, Address src) {
- assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
+ assert(VM_Version::supports_bmi1() && VM_Version::supports_avx(), "bit manipulation instructions not supported");
InstructionMark im(this);
InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
@@ -7275,21 +7275,21 @@ void Assembler::testl(Register dst, Address src) {
}
void Assembler::tzcntl(Register dst, Register src) {
- assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
+ assert(UseCountTrailingZerosInstruction, "tzcnt instruction not supported");
emit_int8((unsigned char)0xF3);
int encode = prefix_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */);
emit_opcode_prefix_and_encoding((unsigned char)0xBC, 0xC0, encode);
}
void Assembler::etzcntl(Register dst, Register src, bool no_flags) {
- assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
+ assert(UseCountTrailingZerosInstruction, "tzcnt instruction not supported");
InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
int encode = eevex_prefix_and_encode_nf(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
emit_int16((unsigned char)0xF4, (0xC0 | encode));
}
void Assembler::tzcntl(Register dst, Address src) {
- assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
+ assert(UseCountTrailingZerosInstruction, "tzcnt instruction not supported");
InstructionMark im(this);
emit_int8((unsigned char)0xF3);
prefix(src, dst, false, true /* is_map1 */);
@@ -7298,7 +7298,7 @@ void Assembler::tzcntl(Register dst, Address src) {
}
void Assembler::etzcntl(Register dst, Address src, bool no_flags) {
- assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
+ assert(UseCountTrailingZerosInstruction, "tzcnt instruction not supported");
InstructionMark im(this);
InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
@@ -7308,21 +7308,21 @@ void Assembler::etzcntl(Register dst, Address src, bool no_flags) {
}
void Assembler::tzcntq(Register dst, Register src) {
- assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
+ assert(UseCountTrailingZerosInstruction, "tzcnt instruction not supported");
emit_int8((unsigned char)0xF3);
int encode = prefixq_and_encode(dst->encoding(), src->encoding(), true /* is_map1 */);
emit_opcode_prefix_and_encoding((unsigned char)0xBC, 0xC0, encode);
}
void Assembler::etzcntq(Register dst, Register src, bool no_flags) {
- assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
+ assert(UseCountTrailingZerosInstruction, "tzcnt instruction not supported");
InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
int encode = eevex_prefix_and_encode_nf(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
emit_int16((unsigned char)0xF4, (0xC0 | encode));
}
void Assembler::tzcntq(Register dst, Address src) {
- assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
+ assert(UseCountTrailingZerosInstruction, "tzcnt instruction not supported");
InstructionMark im(this);
emit_int8((unsigned char)0xF3);
prefixq(src, dst, true /* is_map1 */);
@@ -7331,7 +7331,7 @@ void Assembler::tzcntq(Register dst, Address src) {
}
void Assembler::etzcntq(Register dst, Address src, bool no_flags) {
- assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
+ assert(UseCountTrailingZerosInstruction, "tzcnt instruction not supported");
InstructionMark im(this);
InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
@@ -15000,14 +15000,14 @@ void Assembler::eandq(Register dst, Address src1, Register src2, bool no_flags)
}
void Assembler::andnq(Register dst, Register src1, Register src2) {
- assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
+ assert(VM_Version::supports_bmi1() && VM_Version::supports_avx(), "bit manipulation instructions not supported");
InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes, true);
emit_int16((unsigned char)0xF2, (0xC0 | encode));
}
void Assembler::andnq(Register dst, Register src1, Address src2) {
- assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
+ assert(VM_Version::supports_bmi1() && VM_Version::supports_avx(), "bit manipulation instructions not supported");
InstructionMark im(this);
InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
@@ -15032,14 +15032,14 @@ void Assembler::bswapq(Register reg) {
}
void Assembler::blsiq(Register dst, Register src) {
- assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
+ assert(VM_Version::supports_bmi1() && VM_Version::supports_avx(), "bit manipulation instructions not supported");
InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes, true);
emit_int16((unsigned char)0xF3, (0xC0 | encode));
}
void Assembler::blsiq(Register dst, Address src) {
- assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
+ assert(VM_Version::supports_bmi1() && VM_Version::supports_avx(), "bit manipulation instructions not supported");
InstructionMark im(this);
InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
@@ -15049,14 +15049,14 @@ void Assembler::blsiq(Register dst, Address src) {
}
void Assembler::blsmskq(Register dst, Register src) {
- assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
+ assert(VM_Version::supports_bmi1() && VM_Version::supports_avx(), "bit manipulation instructions not supported");
InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes, true);
emit_int16((unsigned char)0xF3, (0xC0 | encode));
}
void Assembler::blsmskq(Register dst, Address src) {
- assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
+ assert(VM_Version::supports_bmi1() && VM_Version::supports_avx(), "bit manipulation instructions not supported");
InstructionMark im(this);
InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
@@ -15066,14 +15066,14 @@ void Assembler::blsmskq(Register dst, Address src) {
}
void Assembler::blsrq(Register dst, Register src) {
- assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
+ assert(VM_Version::supports_bmi1() && VM_Version::supports_avx(), "bit manipulation instructions not supported");
InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes, true);
emit_int16((unsigned char)0xF3, (0xC0 | encode));
}
void Assembler::blsrq(Register dst, Address src) {
- assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
+ assert(VM_Version::supports_bmi1() && VM_Version::supports_avx(), "bit manipulation instructions not supported");
InstructionMark im(this);
InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
diff --git a/src/hotspot/cpu/x86/c1_CodeStubs_x86.cpp b/src/hotspot/cpu/x86/c1_CodeStubs_x86.cpp
index 95ce48f34db..9a4044a4f0c 100644
--- a/src/hotspot/cpu/x86/c1_CodeStubs_x86.cpp
+++ b/src/hotspot/cpu/x86/c1_CodeStubs_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -293,7 +293,7 @@ void PatchingStub::emit_code(LIR_Assembler* ce) {
address ptr = (address)(_pc_start + i);
int a_byte = (*ptr) & 0xFF;
__ emit_int8(a_byte);
- *ptr = 0x90; // make the site look like a nop
+ *ptr = NativeInstruction::nop_instruction_code; // make the site look like a nop
}
}
@@ -342,6 +342,38 @@ void PatchingStub::emit_code(LIR_Assembler* ce) {
assert(patch_info_pc - end_of_patch == bytes_to_skip, "incorrect patch info");
address entry = __ pc();
+ // NativeGeneralJump::insert_unconditional will be writing a jmp rel32 at _pc_start over the existing instructions.
+ // There are 2 cases:
+ // - the existing instruction is a mov r64 imm64 from LIR_Assembler::klass2reg_with_patching
+ // - or there are nops there (from higher in this function).
+ // In the first case, since a jmp rel32 is 5-byte long, but a mov r64 imm64 is 10-byte long
+ // (resp. 11 if using a REX2 prefix), so we are left with the last 5 (resp. 6) bytes of the
+ // immediate operand (which are all 0x00). When debugging, this confuses the disassembler
+ // because it tries to recognize an instruction starting immediately after the jmp rel32,
+ // leading to wrong instructions, and possibly failure to disassemble further the whole function.
+ //
+ // To be disassembler-friendly, let's replace the leftover 0x00 with nops.
+ // There are 2 shapes:
+ // - without REX2 prefix: REX prefix | MOV r64
+ // - with REX2 prefix: REX2 prefix | REX prefix | MOV r64
+ // then, we know the 8 bytes after are the immediate operand.
+ if (NativeInstruction* ni = nativeInstruction_at(_pc_start); ni->is_mov_literal64()) {
+ int length_before_immediate = ni->has_rex2_prefix() ? 3 : 2;
+ assert(*(long long int*)(_pc_start + length_before_immediate) == 0, "imm64 must be 0 in mov r64, imm64");
+ // We don't need to replace the NativeGeneralJump::instruction_size first bytes, since insert_unconditional
+ // will overwrite.
+ for (int i = NativeGeneralJump::instruction_size; i < length_before_immediate + BytesPerLong; ++i) {
+ _pc_start[i] = NativeInstruction::nop_instruction_code;
+ }
+ }
+#ifdef ASSERT
+ else { // and we make sure otherwise, we indeed have just nops.
+ for (int i = 0; i < NativeGeneralJump::instruction_size; ++i) {
+ assert(_pc_start[i] == NativeInstruction::nop_instruction_code, "patching over an unexpected instruction");
+ }
+ }
+#endif
+
NativeGeneralJump::insert_unconditional((address)_pc_start, entry);
address target = nullptr;
relocInfo::relocType reloc_type = relocInfo::none;
diff --git a/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp b/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp
index f448e4ee17f..cc068cda7a9 100644
--- a/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp
+++ b/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp
@@ -1432,4 +1432,5 @@ void LIRGenerator::volatile_field_load(LIR_Address* address, LIR_Opr result,
} else {
__ load(address, result, info);
}
+ __ membar_acquire();
}
diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
index b4d8aa10de2..69308bb2a7e 100644
--- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
@@ -336,7 +336,7 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box, Register rax_reg,
// Check if object matches.
movptr(rax_reg, Address(monitor, ObjectMonitor::object_offset()));
BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
- bs_asm->try_resolve_weak_handle_in_c2(this, rax_reg, slow_path);
+ bs_asm->try_peek_weak_handle_in_nmethod(this, rax_reg, rax_reg, slow_path);
cmpptr(rax_reg, obj);
jcc(Assembler::notEqual, slow_path);
@@ -483,7 +483,7 @@ void C2_MacroAssembler::fast_unlock(Register obj, Register reg_rax, Register t,
// Try to unlock. Transition lock bits 0b00 => 0b01
movptr(reg_rax, mark);
- andptr(reg_rax, ~(int32_t)markWord::lock_mask);
+ andptr(reg_rax, ~(int32_t)markWord::lock_mask_in_place);
orptr(mark, markWord::unlocked_value);
lock(); cmpxchgptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
jcc(Assembler::notEqual, push_and_slow_path);
@@ -5558,7 +5558,7 @@ void C2_MacroAssembler::vector_mask_operation_helper(int opc, Register dst, Regi
}
break;
case Op_VectorMaskFirstTrue:
- if (VM_Version::supports_bmi1()) {
+ if (UseCountTrailingZerosInstruction) {
if (masklen < 32) {
orl(tmp, 1 << masklen);
tzcntl(dst, tmp);
@@ -6350,7 +6350,7 @@ void C2_MacroAssembler::udivI(Register rax, Register divisor, Register rdx) {
// See Hacker's Delight (2nd ed), section 9.3 which is implemented in java.lang.Long.divideUnsigned()
movl(rdx, rax);
subl(rdx, divisor);
- if (VM_Version::supports_bmi1()) {
+ if (VM_Version::supports_bmi1() && VM_Version::supports_avx()) {
andnl(rax, rdx, rax);
} else {
notl(rdx);
@@ -6374,7 +6374,7 @@ void C2_MacroAssembler::umodI(Register rax, Register divisor, Register rdx) {
// See Hacker's Delight (2nd ed), section 9.3 which is implemented in java.lang.Long.remainderUnsigned()
movl(rdx, rax);
subl(rax, divisor);
- if (VM_Version::supports_bmi1()) {
+ if (VM_Version::supports_bmi1() && VM_Version::supports_avx()) {
andnl(rax, rax, rdx);
} else {
notl(rax);
@@ -6403,7 +6403,7 @@ void C2_MacroAssembler::udivmodI(Register rax, Register divisor, Register rdx, R
// java.lang.Long.divideUnsigned() and java.lang.Long.remainderUnsigned()
movl(rdx, rax);
subl(rax, divisor);
- if (VM_Version::supports_bmi1()) {
+ if (VM_Version::supports_bmi1() && VM_Version::supports_avx()) {
andnl(rax, rax, rdx);
} else {
notl(rax);
@@ -6515,7 +6515,7 @@ void C2_MacroAssembler::udivL(Register rax, Register divisor, Register rdx) {
// See Hacker's Delight (2nd ed), section 9.3 which is implemented in java.lang.Long.divideUnsigned()
movq(rdx, rax);
subq(rdx, divisor);
- if (VM_Version::supports_bmi1()) {
+ if (VM_Version::supports_bmi1() && VM_Version::supports_avx()) {
andnq(rax, rdx, rax);
} else {
notq(rdx);
@@ -6539,7 +6539,7 @@ void C2_MacroAssembler::umodL(Register rax, Register divisor, Register rdx) {
// See Hacker's Delight (2nd ed), section 9.3 which is implemented in java.lang.Long.remainderUnsigned()
movq(rdx, rax);
subq(rax, divisor);
- if (VM_Version::supports_bmi1()) {
+ if (VM_Version::supports_bmi1() && VM_Version::supports_avx()) {
andnq(rax, rax, rdx);
} else {
notq(rax);
@@ -6567,7 +6567,7 @@ void C2_MacroAssembler::udivmodL(Register rax, Register divisor, Register rdx, R
// java.lang.Long.divideUnsigned() and java.lang.Long.remainderUnsigned()
movq(rdx, rax);
subq(rax, divisor);
- if (VM_Version::supports_bmi1()) {
+ if (VM_Version::supports_bmi1() && VM_Version::supports_avx()) {
andnq(rax, rax, rdx);
} else {
notq(rax);
diff --git a/src/hotspot/cpu/x86/c2_stubGenerator_x86_64_string.cpp b/src/hotspot/cpu/x86/c2_stubGenerator_x86_64_string.cpp
index 77a149addb5..b71fb46f075 100644
--- a/src/hotspot/cpu/x86/c2_stubGenerator_x86_64_string.cpp
+++ b/src/hotspot/cpu/x86/c2_stubGenerator_x86_64_string.cpp
@@ -28,6 +28,7 @@
#include "oops/arrayOop.hpp"
#include "opto/c2_MacroAssembler.hpp"
#include "opto/intrinsicnode.hpp"
+#include "runtime/stubRoutines.hpp"
/******************************************************************************/
// String handling intrinsics
@@ -98,7 +99,8 @@
__ blsrl(mask, mask); \
}
-#define NUMBER_OF_CASES 10
+#define NUMBER_OF_CASES StubRoutines::x86::STRING_INDEXOF_NUMBER_OF_CASES
+#define TABLE_COUNT StubRoutines::x86::STRING_INDEXOF_TABLE_COUNT
#undef STACK_SPACE
#undef MAX_NEEDLE_LEN_TO_EXPAND
@@ -186,9 +188,9 @@ static void generate_string_indexof_stubs(StubGenerator *stubgen, address *fnptr
////////////////////////////////////////////////////////////////////////////////////////
void StubGenerator::generate_string_indexof(address *fnptrs) {
- assert((int) StrIntrinsicNode::LL < 4, "Enum out of range");
- assert((int) StrIntrinsicNode::UL < 4, "Enum out of range");
- assert((int) StrIntrinsicNode::UU < 4, "Enum out of range");
+ assert((int) StrIntrinsicNode::LL < TABLE_COUNT, "Enum out of range");
+ assert((int) StrIntrinsicNode::UL < TABLE_COUNT, "Enum out of range");
+ assert((int) StrIntrinsicNode::UU < TABLE_COUNT, "Enum out of range");
generate_string_indexof_stubs(this, fnptrs, StrIntrinsicNode::LL, _masm);
generate_string_indexof_stubs(this, fnptrs, StrIntrinsicNode::UU, _masm);
generate_string_indexof_stubs(this, fnptrs, StrIntrinsicNode::UL, _masm);
@@ -206,6 +208,33 @@ static void generate_string_indexof_stubs(StubGenerator *stubgen, address *fnptr
assert(isLL || isUL || isUU, "Encoding not recognized");
StubId stub_id = (isLL ? StubId::stubgen_string_indexof_linear_ll_id : (isUL ? StubId::stubgen_string_indexof_linear_ul_id : StubId::stubgen_string_indexof_linear_uu_id));
+
+ // Addresses of the two jump tables used for small needle processing
+ address *big_jump_table = StubRoutines::x86::big_jump_table_base(ae);
+ address *small_jump_table = StubRoutines::x86::big_jump_table_base(ae);
+
+ // If the stub has been cached we can retrieve the stub entry. The
+ // target addresses we need to install into the jump tables will
+ // also have been cached, as extra addresses rather than secondary
+ // entries because they are only used internally to this stub.
+
+ assert(StubInfo::entry_count(stub_id) == 1, "sanity check");
+ GrowableArray extras;
+ const int expected_extra_count = 2 * NUMBER_OF_CASES;
+ address start = stubgen->load_archive_data(stub_id, nullptr, &extras);
+ if (start != nullptr) {
+ assert(extras.length() == expected_extra_count,
+ "expecting %d extra addresses but got %d!",
+ expected_extra_count,
+ extras.length());
+ fnptrs[ae] = start;
+ for (int i = 0; i < NUMBER_OF_CASES; i++) {
+ big_jump_table[i]= extras.at(i);
+ small_jump_table[i]= extras.at(NUMBER_OF_CASES + i);
+ }
+ return;
+ }
+
StubCodeMark mark(stubgen, stub_id);
// Keep track of isUL since we need to generate UU code in the main body
// for the case where we expand the needle from bytes to words on the stack.
@@ -253,10 +282,6 @@ static void generate_string_indexof_stubs(StubGenerator *stubgen, address *fnptr
const Register needle_p = c_rarg2;
const Register needle_len_p = c_rarg3;
- // Addresses of the two jump tables used for small needle processing
- address big_jump_table;
- address small_jump_table;
-
Label L_begin;
Label L_returnError, L_bigCaseFixupAndReturn;
@@ -265,7 +290,7 @@ static void generate_string_indexof_stubs(StubGenerator *stubgen, address *fnptr
Label L_wcharBegin, L_continue, L_wideNoExpand, L_returnR11;
__ align(CodeEntryAlignment);
- fnptrs[ae] = __ pc();
+ start = __ pc();
__ enter(); // required for proper stackwalking of RuntimeStub frame
// Check for trivial cases
@@ -311,8 +336,8 @@ static void generate_string_indexof_stubs(StubGenerator *stubgen, address *fnptr
}
// Set up jump tables. Used when needle size <= NUMBER_OF_CASES
- setup_jump_tables(ae, L_returnError, L_returnR11, L_bigCaseFixupAndReturn, &big_jump_table,
- &small_jump_table, _masm);
+ setup_jump_tables(ae, L_returnError, L_returnR11, L_bigCaseFixupAndReturn, big_jump_table,
+ small_jump_table, _masm);
////////////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////////////
@@ -422,7 +447,7 @@ static void generate_string_indexof_stubs(StubGenerator *stubgen, address *fnptr
__ leaq(r13, Address(save_ndl_len, -1));
__ cmpq(r13, NUMBER_OF_CASES - 1);
__ ja(L_smallCaseDefault);
- __ lea(r15, InternalAddress(small_jump_table));
+ __ lea(r15, ExternalAddress((address)small_jump_table));
__ jmp(Address(r15, r13, Address::times_8));
// Dispatch to handlers for small needle and large haystack
@@ -431,7 +456,7 @@ static void generate_string_indexof_stubs(StubGenerator *stubgen, address *fnptr
__ leaq(rax, Address(save_ndl_len, -1));
__ cmpq(rax, NUMBER_OF_CASES - 1);
__ ja(L_bigCaseDefault);
- __ lea(r15, InternalAddress(big_jump_table));
+ __ lea(r15, ExternalAddress((address)big_jump_table));
__ jmp(Address(r15, rax, Address::times_8));
////////////////////////////////////////////////////////////////////////////////////////
@@ -942,6 +967,23 @@ static void generate_string_indexof_stubs(StubGenerator *stubgen, address *fnptr
}
}
+ // Record and possibly cache the entry address. In the latter case
+ // the target addresses installed into the jump tables also need to
+ // be cached, as extra addresses rather than secondary entries
+ // because they are only used internally to this stub.
+
+ for (int i = 0; i < NUMBER_OF_CASES; i++) {
+ extras.append(big_jump_table[i]);
+ }
+ for (int i = 0; i < NUMBER_OF_CASES; i++) {
+ extras.append(small_jump_table[i]);
+ }
+ stubgen->store_archive_data(stub_id, start, __ pc(), nullptr, &extras);
+
+ // x86 consumes the stub via a shared (i.e. non-cpu specific) static
+ // array
+ fnptrs[ae] = start;
+
return;
}
@@ -1722,8 +1764,8 @@ static void copy_to_stack(Register haystack, Register haystack_len, bool isU,
// L_error - Label to branch to if no match found
// L_checkRange - label to jump to when match found. Checks validity of returned index
// L_fixup - Jump to here for big cases. Return value is pointer to matching haystack byte
-// *big_jump_table - Address of pointer to the first element of big jump table
-// *small_jump_table - Address of pointer to the first element of small jump table
+// big_jump_table - jump table to be populated with jump addresses
+// small_jump_table - jump table to be populated with jump addresses
// _masm - Current MacroAssembler instance pointer
static void setup_jump_tables(StrIntrinsicNode::ArgEncoding ae, Label &L_error, Label &L_checkRange,
@@ -1734,8 +1776,6 @@ static void setup_jump_tables(StrIntrinsicNode::ArgEncoding ae, Label &L_error,
bool isU = isUL || isUU; // At least one is UTF-16
const XMMRegister byte_1 = XMM_BYTE_1;
- address big_hs_jmp_table[NUMBER_OF_CASES]; // Jump table for large haystacks
- address small_hs_jmp_table[NUMBER_OF_CASES]; // Jump table for small haystacks
int jmp_ndx = 0;
////////////////////////////////////////////////
@@ -1786,7 +1826,7 @@ static void setup_jump_tables(StrIntrinsicNode::ArgEncoding ae, Label &L_error,
const Register rTmp = rax;
for (int i = 6; i < NUMBER_OF_CASES; i++) {
- small_hs_jmp_table[i] = __ pc();
+ small_jump_table[i] = __ pc();
if (isU && ((i + 1) & 1)) {
continue;
} else {
@@ -1830,7 +1870,7 @@ static void setup_jump_tables(StrIntrinsicNode::ArgEncoding ae, Label &L_error,
const Register rTmp4 = r13;
for (int i = 0; i < NUMBER_OF_CASES; i++) {
- big_hs_jmp_table[i] = __ pc();
+ big_jump_table[i] = __ pc();
if (isU && ((i + 1) & 1)) {
continue;
} else {
@@ -1843,24 +1883,6 @@ static void setup_jump_tables(StrIntrinsicNode::ArgEncoding ae, Label &L_error,
}
}
}
- ////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////
- ////////////////////////////////////////////////////////////////////////////////////////
- // JUMP TABLES
- __ align(8);
-
- *big_jump_table = __ pc();
-
- for (jmp_ndx = 0; jmp_ndx < NUMBER_OF_CASES; jmp_ndx++) {
- __ emit_address(big_hs_jmp_table[jmp_ndx]);
- }
-
- *small_jump_table = __ pc();
-
- for (jmp_ndx = 0; jmp_ndx < NUMBER_OF_CASES; jmp_ndx++) {
- __ emit_address(small_hs_jmp_table[jmp_ndx]);
- }
}
#undef STACK_SPACE
diff --git a/src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.cpp
index 215dc30f7fd..731eef09c37 100644
--- a/src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.cpp
@@ -362,6 +362,11 @@ void BarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register
__ jcc(Assembler::zero, error); // if klass is null it is broken
}
+void BarrierSetAssembler::try_peek_weak_handle_in_nmethod(MacroAssembler* masm, Register weak_handle, Register obj, Label& slowpath) {
+ // Load the oop from the weak handle without barriers.
+ __ movptr(obj, Address(weak_handle));
+}
+
#ifdef COMPILER2
OptoReg::Name BarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) {
@@ -395,11 +400,6 @@ OptoReg::Name BarrierSetAssembler::refine_register(const Node* node, OptoReg::Na
extern void vec_spill_helper(C2_MacroAssembler *masm, bool is_load,
int stack_offset, int reg, uint ireg, outputStream* st);
-void BarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Label& slowpath) {
- // Load the oop from the weak handle.
- __ movptr(obj, Address(obj));
-}
-
#undef __
#define __ _masm->
diff --git a/src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.hpp
index 6aff29850e3..caf341dcdf3 100644
--- a/src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.hpp
@@ -106,11 +106,13 @@ public:
virtual void check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error);
+ // See AS_NO_KEEPALIVE for peek semantics
+ // weak_handle and obj may alias
+ virtual void try_peek_weak_handle_in_nmethod(MacroAssembler* masm, Register weak_handle, Register obj, Label& slowpath);
+
#ifdef COMPILER2
OptoReg::Name refine_register(const Node* node,
OptoReg::Name opto_reg);
-
- virtual void try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Label& slowpath);
#endif // COMPILER2
};
diff --git a/src/hotspot/cpu/x86/gc/shenandoah/c1/shenandoahBarrierSetC1_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/c1/shenandoahBarrierSetC1_x86.cpp
deleted file mode 100644
index 66fb4cbb8c7..00000000000
--- a/src/hotspot/cpu/x86/gc/shenandoah/c1/shenandoahBarrierSetC1_x86.cpp
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved.
- * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#include "c1/c1_LIRAssembler.hpp"
-#include "c1/c1_MacroAssembler.hpp"
-#include "gc/shared/gc_globals.hpp"
-#include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
-#include "gc/shenandoah/shenandoahBarrierSet.hpp"
-#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
-
-#define __ masm->masm()->
-
-void LIR_OpShenandoahCompareAndSwap::emit_code(LIR_Assembler* masm) {
- Register addr = _addr->is_single_cpu() ? _addr->as_register() : _addr->as_register_lo();
- Register newval = _new_value->as_register();
- Register cmpval = _cmp_value->as_register();
- Register tmp1 = _tmp1->as_register();
- Register tmp2 = _tmp2->as_register();
- Register result = result_opr()->as_register();
- assert(cmpval == rax, "wrong register");
- assert(newval != noreg, "new val must be register");
- assert(cmpval != newval, "cmp and new values must be in different registers");
- assert(cmpval != addr, "cmp and addr must be in different registers");
- assert(newval != addr, "new value and addr must be in different registers");
-
- if (UseCompressedOops) {
- __ encode_heap_oop(cmpval);
- __ mov(rscratch1, newval);
- __ encode_heap_oop(rscratch1);
- newval = rscratch1;
- }
-
- ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm->masm(), result, Address(addr, 0), cmpval, newval, false, tmp1, tmp2);
-}
-
-#undef __
-
-#ifdef ASSERT
-#define __ gen->lir(__FILE__, __LINE__)->
-#else
-#define __ gen->lir()->
-#endif
-
-LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value) {
-
- if (access.is_oop()) {
- LIRGenerator* gen = access.gen();
- if (ShenandoahSATBBarrier) {
- pre_barrier(gen, access.access_emit_info(), access.decorators(), access.resolved_addr(),
- LIR_OprFact::illegalOpr /* pre_val */);
- }
- if (ShenandoahCASBarrier) {
- cmp_value.load_item_force(FrameMap::rax_oop_opr);
- new_value.load_item();
-
- LIR_Opr t1 = gen->new_register(T_OBJECT);
- LIR_Opr t2 = gen->new_register(T_OBJECT);
- LIR_Opr addr = access.resolved_addr()->as_address_ptr()->base();
- LIR_Opr result = gen->new_register(T_INT);
-
- __ append(new LIR_OpShenandoahCompareAndSwap(addr, cmp_value.result(), new_value.result(), t1, t2, result));
-
- if (ShenandoahCardBarrier) {
- post_barrier(access, access.resolved_addr(), new_value.result());
- }
- return result;
- }
- }
- return BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value);
-}
-
-LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value) {
- LIRGenerator* gen = access.gen();
- BasicType type = access.type();
-
- LIR_Opr result = gen->new_register(type);
- value.load_item();
- LIR_Opr value_opr = value.result();
-
- // Because we want a 2-arg form of xchg and xadd
- __ move(value_opr, result);
-
- assert(type == T_INT || is_reference_type(type) || type == T_LONG, "unexpected type");
- __ xchg(access.resolved_addr(), result, result, LIR_OprFact::illegalOpr);
-
- if (access.is_oop()) {
- result = load_reference_barrier(access.gen(), result, LIR_OprFact::addressConst(0), access.decorators());
- LIR_Opr tmp = gen->new_register(type);
- __ move(result, tmp);
- result = tmp;
- if (ShenandoahSATBBarrier) {
- pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr,
- result /* pre_val */);
- }
- if (ShenandoahCardBarrier) {
- post_barrier(access, access.resolved_addr(), result);
- }
- }
-
- return result;
-}
diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp
index 67510fac58f..1df8de2352d 100644
--- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp
@@ -45,47 +45,6 @@
#define __ masm->
-static void save_machine_state(MacroAssembler* masm, bool handle_gpr, bool handle_fp) {
- if (handle_gpr) {
- __ push_IU_state();
- }
-
- if (handle_fp) {
- // Some paths can be reached from the c2i adapter with live fp arguments in registers.
- assert(Argument::n_float_register_parameters_j == 8, "8 fp registers to save at java call");
-
- const int xmm_size = wordSize * 2;
- __ subptr(rsp, xmm_size * 8);
- __ movdbl(Address(rsp, xmm_size * 0), xmm0);
- __ movdbl(Address(rsp, xmm_size * 1), xmm1);
- __ movdbl(Address(rsp, xmm_size * 2), xmm2);
- __ movdbl(Address(rsp, xmm_size * 3), xmm3);
- __ movdbl(Address(rsp, xmm_size * 4), xmm4);
- __ movdbl(Address(rsp, xmm_size * 5), xmm5);
- __ movdbl(Address(rsp, xmm_size * 6), xmm6);
- __ movdbl(Address(rsp, xmm_size * 7), xmm7);
- }
-}
-
-static void restore_machine_state(MacroAssembler* masm, bool handle_gpr, bool handle_fp) {
- if (handle_fp) {
- const int xmm_size = wordSize * 2;
- __ movdbl(xmm0, Address(rsp, xmm_size * 0));
- __ movdbl(xmm1, Address(rsp, xmm_size * 1));
- __ movdbl(xmm2, Address(rsp, xmm_size * 2));
- __ movdbl(xmm3, Address(rsp, xmm_size * 3));
- __ movdbl(xmm4, Address(rsp, xmm_size * 4));
- __ movdbl(xmm5, Address(rsp, xmm_size * 5));
- __ movdbl(xmm6, Address(rsp, xmm_size * 6));
- __ movdbl(xmm7, Address(rsp, xmm_size * 7));
- __ addptr(rsp, xmm_size * 8);
- }
-
- if (handle_gpr) {
- __ pop_IU_state();
- }
-}
-
void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Register src, Register dst, Register count) {
@@ -130,20 +89,27 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec
__ testb(gc_state, flags);
__ jcc(Assembler::zero, L_done);
- save_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ false);
-
- assert(src == rdi, "expected");
- assert(dst == rsi, "expected");
- assert(count == rdx, "expected");
- if (UseCompressedOops) {
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop),
- src, dst, count);
- } else {
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop),
- src, dst, count);
+ __ push_call_clobbered_registers(/* save_fpu = */ false);
+ // If arguments are not in proper places, shuffle them.
+ // Doing this via the stack is the most straight-forward way to avoid
+ // accidentally smashing any register.
+ if (c_rarg0 != src || c_rarg1 != dst || c_rarg2 != count) {
+ __ push(src);
+ __ push(dst);
+ __ push(count);
+ __ pop(c_rarg2);
+ __ pop(c_rarg1);
+ __ pop(c_rarg0);
}
+ address target = nullptr;
+ if (UseCompressedOops) {
+ target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop);
+ } else {
+ target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop);
+ }
+ __ call_VM_leaf(target, 3);
- restore_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ false);
+ __ pop_call_clobbered_registers(/* restore_fpu = */ false);
__ bind(L_done);
}
@@ -178,15 +144,8 @@ void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, Dec
void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler* masm,
Register obj,
Register pre_val,
- Register tmp,
- bool tosca_live,
- bool expand_call) {
+ Register tmp) {
assert(ShenandoahSATBBarrier, "Should be checked by caller");
-
- // If expand_call is true then we expand the call_VM_leaf macro
- // directly to skip generating the check by
- // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
-
const Register thread = r15_thread;
Label done;
@@ -208,7 +167,12 @@ void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler* masm,
// Do we need to load the previous value?
if (obj != noreg) {
- __ load_heap_oop(pre_val, Address(obj, 0), noreg, AS_RAW);
+ if (UseCompressedOops) {
+ __ movl(pre_val, Address(obj, 0));
+ __ decode_heap_oop(pre_val);
+ } else {
+ __ movq(pre_val, Address(obj, 0));
+ }
}
// Is the previous value null?
@@ -232,54 +196,22 @@ void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler* masm,
__ jmp(done);
__ bind(runtime);
- // save the live input values
- if(tosca_live) __ push(rax);
- if (obj != noreg && obj != rax)
- __ push(obj);
+ // Slow-path call.
+ // Some paths can be reached from the c2i adapter with live fp arguments in registers.
+ __ enter();
+ __ push_call_clobbered_registers(/* save_fpu = */ true);
- if (pre_val != rax)
- __ push(pre_val);
-
- // Calling the runtime using the regular call_VM_leaf mechanism generates
- // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
- // that checks that the *(ebp+frame::interpreter_frame_last_sp) == nullptr.
- //
- // If we care generating the pre-barrier without a frame (e.g. in the
- // intrinsified Reference.get() routine) then ebp might be pointing to
- // the caller frame and so this check will most likely fail at runtime.
- //
- // Expanding the call directly bypasses the generation of the check.
- // So when we do not have have a full interpreter frame on the stack
- // expand_call should be passed true.
-
- // We move pre_val into c_rarg0 early, in order to avoid smashing it, should
- // pre_val be c_rarg1 (where the call prologue would copy thread argument).
- // Note: this should not accidentally smash thread, because thread is always r15.
assert(thread != c_rarg0, "smashed arg");
if (c_rarg0 != pre_val) {
__ mov(c_rarg0, pre_val);
}
- if (expand_call) {
- assert(pre_val != c_rarg1, "smashed arg");
- if (c_rarg1 != thread) {
- __ mov(c_rarg1, thread);
- }
- // Already moved pre_val into c_rarg0 above
- __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), 1);
- } else {
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), c_rarg0);
- }
+ // Calling with super_call_VM_leaf with c_rarg0 bypasses interpreter checks and avoids any moves.
+ __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), c_rarg0);
- // save the live input values
- if (pre_val != rax)
- __ pop(pre_val);
-
- if (obj != noreg && obj != rax)
- __ pop(obj);
-
- if(tosca_live) __ pop(rax);
+ __ pop_call_clobbered_registers(/* restore_fpu = */ true);
+ __ leave();
__ bind(done);
}
@@ -312,9 +244,9 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
if (is_strong) {
// Test for object in cset
// Allocate temporary registers
- for (int i = 0; i < 8; i++) {
+ for (int i = 0; i < Register::available_gp_registers(); i++) {
Register r = as_Register(i);
- if (r != rsp && r != rbp && r != dst && r != src.base() && r != src.index()) {
+ if (r != rsp && r != rbp && r != rcx && r != dst && r != src.base() && r != src.index() ) {
if (tmp1 == noreg) {
tmp1 = r;
} else {
@@ -333,118 +265,69 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
// Optimized cset-test
__ movptr(tmp1, dst);
- __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
- __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
+ if (AOTCodeCache::is_on_for_dump()) {
+ assert_different_registers(tmp1, tmp2, rcx);
+ __ lea(tmp2, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
+ __ push(rcx);
+ __ movb(rcx, Address(tmp2));
+ __ shrptr(tmp1);
+ __ pop(rcx);
+ __ lea(tmp2, ExternalAddress(AOTRuntimeConstants::cset_base_address()));
+ __ movptr(tmp2, Address(tmp2));
+ } else {
+ __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
+ __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
+ }
__ movbool(tmp1, Address(tmp1, tmp2, Address::times_1));
__ testbool(tmp1);
__ jcc(Assembler::zero, not_cset);
}
- save_machine_state(masm, /* handle_gpr = */ false, /* handle_fp = */ true);
-
- // The rest is saved with the optimized path
-
- uint num_saved_regs = 4 + (dst != rax ? 1 : 0) + 4 + (UseAPX ? 16 : 0);
- __ subptr(rsp, num_saved_regs * wordSize);
- uint slot = num_saved_regs;
+ // Slow-path call.
+ // Save registers that can be clobbered by call.
+ // Some paths can be reached from the c2i adapter with live fp arguments in registers.
+ __ enter();
if (dst != rax) {
- __ movptr(Address(rsp, (--slot) * wordSize), rax);
+ __ push(rax);
}
- __ movptr(Address(rsp, (--slot) * wordSize), rcx);
- __ movptr(Address(rsp, (--slot) * wordSize), rdx);
- __ movptr(Address(rsp, (--slot) * wordSize), rdi);
- __ movptr(Address(rsp, (--slot) * wordSize), rsi);
- __ movptr(Address(rsp, (--slot) * wordSize), r8);
- __ movptr(Address(rsp, (--slot) * wordSize), r9);
- __ movptr(Address(rsp, (--slot) * wordSize), r10);
- __ movptr(Address(rsp, (--slot) * wordSize), r11);
- // Save APX extended registers r16–r31 if enabled
- if (UseAPX) {
- __ movptr(Address(rsp, (--slot) * wordSize), r16);
- __ movptr(Address(rsp, (--slot) * wordSize), r17);
- __ movptr(Address(rsp, (--slot) * wordSize), r18);
- __ movptr(Address(rsp, (--slot) * wordSize), r19);
- __ movptr(Address(rsp, (--slot) * wordSize), r20);
- __ movptr(Address(rsp, (--slot) * wordSize), r21);
- __ movptr(Address(rsp, (--slot) * wordSize), r22);
- __ movptr(Address(rsp, (--slot) * wordSize), r23);
- __ movptr(Address(rsp, (--slot) * wordSize), r24);
- __ movptr(Address(rsp, (--slot) * wordSize), r25);
- __ movptr(Address(rsp, (--slot) * wordSize), r26);
- __ movptr(Address(rsp, (--slot) * wordSize), r27);
- __ movptr(Address(rsp, (--slot) * wordSize), r28);
- __ movptr(Address(rsp, (--slot) * wordSize), r29);
- __ movptr(Address(rsp, (--slot) * wordSize), r30);
- __ movptr(Address(rsp, (--slot) * wordSize), r31);
- }
- // r12-r15 are callee saved in all calling conventions
- assert(slot == 0, "must use all slots");
+ __ push_call_clobbered_registers_except(rax, /* save_fpu = */ true);
// Shuffle registers such that dst is in c_rarg0 and addr in c_rarg1.
- Register arg0 = c_rarg0, arg1 = c_rarg1;
- if (dst == arg1) {
- __ lea(arg0, src);
- __ xchgptr(arg1, arg0);
+ if (dst == c_rarg1) {
+ __ lea(c_rarg0, src);
+ __ xchgptr(c_rarg1, c_rarg0);
} else {
- __ lea(arg1, src);
- __ movptr(arg0, dst);
+ __ lea(c_rarg1, src);
+ __ movptr(c_rarg0, dst);
}
+ address target = nullptr;
if (is_strong) {
if (is_narrow) {
- __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow), arg0, arg1);
+ target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
} else {
- __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong), arg0, arg1);
+ target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
}
} else if (is_weak) {
if (is_narrow) {
- __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow), arg0, arg1);
+ target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
} else {
- __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), arg0, arg1);
+ target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
}
} else {
assert(is_phantom, "only remaining strength");
assert(!is_narrow, "phantom access cannot be narrow");
- __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom), arg0, arg1);
+ target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
}
- // Restore APX extended registers r31–r16 if previously saved
- if (UseAPX) {
- __ movptr(r31, Address(rsp, (slot++) * wordSize));
- __ movptr(r30, Address(rsp, (slot++) * wordSize));
- __ movptr(r29, Address(rsp, (slot++) * wordSize));
- __ movptr(r28, Address(rsp, (slot++) * wordSize));
- __ movptr(r27, Address(rsp, (slot++) * wordSize));
- __ movptr(r26, Address(rsp, (slot++) * wordSize));
- __ movptr(r25, Address(rsp, (slot++) * wordSize));
- __ movptr(r24, Address(rsp, (slot++) * wordSize));
- __ movptr(r23, Address(rsp, (slot++) * wordSize));
- __ movptr(r22, Address(rsp, (slot++) * wordSize));
- __ movptr(r21, Address(rsp, (slot++) * wordSize));
- __ movptr(r20, Address(rsp, (slot++) * wordSize));
- __ movptr(r19, Address(rsp, (slot++) * wordSize));
- __ movptr(r18, Address(rsp, (slot++) * wordSize));
- __ movptr(r17, Address(rsp, (slot++) * wordSize));
- __ movptr(r16, Address(rsp, (slot++) * wordSize));
- }
- __ movptr(r11, Address(rsp, (slot++) * wordSize));
- __ movptr(r10, Address(rsp, (slot++) * wordSize));
- __ movptr(r9, Address(rsp, (slot++) * wordSize));
- __ movptr(r8, Address(rsp, (slot++) * wordSize));
- __ movptr(rsi, Address(rsp, (slot++) * wordSize));
- __ movptr(rdi, Address(rsp, (slot++) * wordSize));
- __ movptr(rdx, Address(rsp, (slot++) * wordSize));
- __ movptr(rcx, Address(rsp, (slot++) * wordSize));
-
+ // Calling with super_call_VM_leaf with c_rarg0/1 bypasses interpreter checks and avoids any moves.
+ __ super_call_VM_leaf(target, c_rarg0, c_rarg1);
+ __ pop_call_clobbered_registers_except(rax, /* restore_fpu = */ true);
if (dst != rax) {
__ movptr(dst, rax);
- __ movptr(rax, Address(rsp, (slot++) * wordSize));
+ __ pop(rax);
}
-
- assert(slot == num_saved_regs, "must use all slots");
- __ addptr(rsp, num_saved_regs * wordSize);
-
- restore_machine_state(masm, /* handle_gpr = */ false, /* handle_fp = */ true);
+ __ leave();
__ bind(not_cset);
@@ -519,19 +402,10 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet d
// 3: apply keep-alive barrier if needed
if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
- save_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ true);
-
- assert_different_registers(dst, tmp1, r15_thread);
- // Generate the SATB pre-barrier code to log the value of
- // the referent field in an SATB buffer.
satb_barrier(masm /* masm */,
noreg /* obj */,
dst /* pre_val */,
- tmp1 /* tmp */,
- true /* tosca_live */,
- true /* expand_call */);
-
- restore_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ true);
+ tmp1 /* tmp */);
}
}
@@ -582,22 +456,19 @@ void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet
__ lea(tmp1, dst);
}
- bool storing_non_null = (val != noreg);
-
// 2: pre-barrier: SATB needs the previous value
if (ShenandoahBarrierSet::need_satb_barrier(decorators, type)) {
satb_barrier(masm,
tmp1 /* obj */,
tmp2 /* pre_val */,
- tmp3 /* tmp */,
- storing_non_null /* tosca_live */,
- false /* expand_call */);
+ tmp3 /* tmp */);
}
// Store!
BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
// 3: post-barrier: card barrier needs store address
+ bool storing_non_null = (val != noreg);
if (ShenandoahBarrierSet::need_card_barrier(decorators, type) && storing_non_null) {
card_barrier(masm, tmp1);
}
@@ -619,12 +490,11 @@ void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler
__ bind(done);
}
-#ifdef COMPILER2
-void ShenandoahBarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Label& slowpath) {
+void ShenandoahBarrierSetAssembler::try_peek_weak_handle_in_nmethod(MacroAssembler* masm, Register weak_handle, Register obj, Label& slowpath) {
Label done;
- // Resolve weak handle using the standard implementation.
- BarrierSetAssembler::try_resolve_weak_handle_in_c2(masm, obj, slowpath);
+ // Peek weak handle using the standard implementation.
+ BarrierSetAssembler::try_peek_weak_handle_in_nmethod(masm, weak_handle, obj, slowpath);
// Check if the reference is null, and if it is, take the fast path.
__ testptr(obj, obj);
@@ -638,7 +508,6 @@ void ShenandoahBarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler
__ jcc(Assembler::notZero, slowpath);
__ bind(done);
}
-#endif // COMPILER2
// Special Shenandoah CAS implementation that handles false negatives
// due to concurrent evacuation.
@@ -886,8 +755,27 @@ void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assemble
if (is_strong) {
// Check for object being in the collection set.
__ mov(tmp1, res);
- __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
- __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
+ if (AOTCodeCache::is_on_for_dump()) {
+ __ push(rcx);
+ __ lea(rcx, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
+ __ movl(rcx, Address(rcx));
+ if (tmp1 != rcx) {
+ __ mov(tmp1, res);
+ __ shrptr(tmp1);
+ __ pop(rcx);
+ } else {
+ assert_different_registers(tmp2, rcx);
+ __ mov(tmp2, res);
+ __ shrptr(tmp2);
+ __ pop(rcx);
+ __ movptr(tmp1, tmp2);
+ }
+ __ lea(tmp2, ExternalAddress(AOTRuntimeConstants::cset_base_address()));
+ __ movptr(tmp2, Address(tmp2));
+ } else {
+ __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
+ __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
+ }
__ movbool(tmp2, Address(tmp2, tmp1, Address::times_1));
__ testbool(tmp2);
__ jcc(Assembler::zero, *stub->continuation());
diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp
index 79540aa19e1..85fb98fa15c 100644
--- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp
@@ -45,9 +45,7 @@ private:
void satb_barrier(MacroAssembler* masm,
Register obj,
Register pre_val,
- Register tmp,
- bool tosca_live,
- bool expand_call);
+ Register tmp);
void card_barrier(MacroAssembler* masm, Register obj);
@@ -78,9 +76,7 @@ public:
Address dst, Register val, Register tmp1, Register tmp2, Register tmp3);
virtual void try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
Register obj, Register tmp, Label& slowpath);
-#ifdef COMPILER2
- virtual void try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Label& slowpath);
-#endif // COMPILER2
+ virtual void try_peek_weak_handle_in_nmethod(MacroAssembler* masm, Register weak_handle, Register obj, Label& slowpath);
};
#endif // CPU_X86_GC_SHENANDOAH_SHENANDOAHBARRIERSETASSEMBLER_X86_HPP
diff --git a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp
index c20551b5084..3301d6ace49 100644
--- a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp
@@ -1329,9 +1329,14 @@ void ZBarrierSetAssembler::generate_c2_store_barrier_stub(MacroAssembler* masm,
__ jmp(slow_continuation);
}
-void ZBarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Label& slow_path) {
- // Resolve weak handle using the standard implementation.
- BarrierSetAssembler::try_resolve_weak_handle_in_c2(masm, obj, slow_path);
+#endif // COMPILER2
+
+#undef __
+#define __ masm->
+
+void ZBarrierSetAssembler::try_peek_weak_handle_in_nmethod(MacroAssembler* masm, Register weak_handle, Register obj, Label& slow_path) {
+ // Peek weak handle using the standard implementation.
+ BarrierSetAssembler::try_peek_weak_handle_in_nmethod(masm, weak_handle, obj, slow_path);
// Check if the oop is bad, in which case we need to take the slow path.
__ testptr(obj, Address(r15_thread, ZThreadLocalData::mark_bad_mask_offset()));
@@ -1343,7 +1348,6 @@ void ZBarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler* masm, R
}
#undef __
-#endif // COMPILER2
static int patch_barrier_relocation_offset(int format) {
switch (format) {
diff --git a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp
index ce0c4769716..476a316b2e1 100644
--- a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp
@@ -167,10 +167,10 @@ public:
ZLoadBarrierStubC2* stub) const;
void generate_c2_store_barrier_stub(MacroAssembler* masm,
ZStoreBarrierStubC2* stub) const;
-
- virtual void try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Label& slow_path);
#endif // COMPILER2
+ virtual void try_peek_weak_handle_in_nmethod(MacroAssembler* masm, Register weak_handle, Register obj, Label& slow_path);
+
void store_barrier_fast(MacroAssembler* masm,
Address ref_addr,
Register rnew_persistent,
diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp
index 5ab3ca339aa..f64c4d3f086 100644
--- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp
@@ -55,6 +55,7 @@
#include "runtime/sharedRuntime.hpp"
#include "runtime/stubRoutines.hpp"
#include "utilities/checkedCast.hpp"
+#include "utilities/globalDefinitions.hpp"
#include "utilities/macros.hpp"
#ifdef PRODUCT
@@ -2540,6 +2541,17 @@ void MacroAssembler::sign_extend_short(Register reg) {
movswl(reg, reg); // movsxw
}
+void MacroAssembler::narrow_subword_type(Register reg, BasicType bt) {
+ assert(is_subword_type(bt), "required");
+ switch (bt) {
+ case T_BOOLEAN: andl(reg, 1); break;
+ case T_BYTE: movsbl(reg, reg); break;
+ case T_CHAR: movzwl(reg, reg); break;
+ case T_SHORT: movswl(reg, reg); break;
+ default: ShouldNotReachHere();
+ }
+}
+
void MacroAssembler::testl(Address dst, int32_t imm32) {
if (imm32 >= 0 && is8bit(imm32)) {
testb(dst, imm32);
@@ -6955,7 +6967,7 @@ void MacroAssembler::vectorized_mismatch(Register obja, Register objb, Register
xorq(result, result);
if ((AVX3Threshold == 0) && (UseAVX > 2) &&
- VM_Version::supports_avx512vlbw()) {
+ VM_Version::supports_avx512vlbw() && UseCountTrailingZerosInstruction) {
Label VECTOR64_LOOP, VECTOR64_NOT_EQUAL, VECTOR32_TAIL;
cmpq(length, 64);
diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.hpp b/src/hotspot/cpu/x86/macroAssembler_x86.hpp
index 021d2943ee8..b73339c217f 100644
--- a/src/hotspot/cpu/x86/macroAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/macroAssembler_x86.hpp
@@ -444,6 +444,9 @@ class MacroAssembler: public Assembler {
void sign_extend_short(Register reg);
void sign_extend_byte(Register reg);
+ // Clean up a subword typed value to the representation in compliance with JVMS §2.3
+ void narrow_subword_type(Register reg, BasicType bt);
+
// Division by power of 2, rounding towards 0
void division_with_shift(Register reg, int shift_value);
diff --git a/src/hotspot/cpu/x86/methodHandles_x86.hpp b/src/hotspot/cpu/x86/methodHandles_x86.hpp
index 83b59834261..c4dde903d29 100644
--- a/src/hotspot/cpu/x86/methodHandles_x86.hpp
+++ b/src/hotspot/cpu/x86/methodHandles_x86.hpp
@@ -27,7 +27,7 @@
// Adapters
enum /* platform_dependent_constants */ {
- adapter_code_size = 4000 DEBUG_ONLY(+ 6000)
+ adapter_code_size = 6000 DEBUG_ONLY(+ 6000)
};
// Additional helper methods for MethodHandles code generation:
diff --git a/src/hotspot/cpu/x86/nativeInst_x86.hpp b/src/hotspot/cpu/x86/nativeInst_x86.hpp
index aba4400515f..cbb72508cc1 100644
--- a/src/hotspot/cpu/x86/nativeInst_x86.hpp
+++ b/src/hotspot/cpu/x86/nativeInst_x86.hpp
@@ -97,11 +97,7 @@ class NativeInstruction {
};
inline NativeInstruction* nativeInstruction_at(address address) {
- NativeInstruction* inst = (NativeInstruction*)address;
-#ifdef ASSERT
- //inst->verify();
-#endif
- return inst;
+ return (NativeInstruction*)address;
}
class NativeCall;
diff --git a/src/hotspot/cpu/x86/peephole_x86_64.cpp b/src/hotspot/cpu/x86/peephole_x86_64.cpp
index 5fc39e163e1..9dccea8515e 100644
--- a/src/hotspot/cpu/x86/peephole_x86_64.cpp
+++ b/src/hotspot/cpu/x86/peephole_x86_64.cpp
@@ -25,6 +25,7 @@
#ifdef COMPILER2
#include "opto/addnode.hpp"
+#include "opto/compile.hpp"
#include "peephole_x86_64.hpp"
#include "adfiles/ad_x86.hpp"
@@ -343,10 +344,15 @@ bool Peephole::lea_remove_redundant(Block* block, int block_index, PhaseCFG* cfg
cfg_->map_node_to_block(proj, nullptr);
// Rewire the base of all leas currently depending on the decode we are removing.
+ // Use top() rather than a real value: leaP* emission does not consult Base, and the
+ // unused MachProj of decode confirms the derived oops produced by these leaP*s are
+ // not consumed by any OopMap. Avoiding a real input here also avoids creating a
+ // register conflict that would later be flagged by verify_good_schedule.
+ Node* top = Compile::current()->top();
for (DUIterator_Fast imax, i = decode->fast_outs(imax); i < imax; i++) {
Node* dependant_lea = decode->fast_out(i);
if (dependant_lea->is_Mach() && dependant_lea->as_Mach()->ideal_Opcode() == Op_AddP) {
- dependant_lea->set_req(AddPNode::Base, lea_derived_oop->in(AddPNode::Address));
+ dependant_lea->set_req(AddPNode::Base, top);
// This deleted something in the out array, hence adjust i, imax.
--i;
--imax;
diff --git a/src/hotspot/cpu/x86/stubDeclarations_x86.hpp b/src/hotspot/cpu/x86/stubDeclarations_x86.hpp
index 24886deb3c5..031e95170cc 100644
--- a/src/hotspot/cpu/x86/stubDeclarations_x86.hpp
+++ b/src/hotspot/cpu/x86/stubDeclarations_x86.hpp
@@ -178,8 +178,17 @@
/* other arches use arch-specific entries */ \
/* this really needs rationalising */ \
do_stub(compiler, string_indexof_linear_ll) \
+ do_arch_entry(x86, compiler, string_indexof_linear_ll, \
+ string_indexof_linear_ll, \
+ string_indexof_linear_ll) \
do_stub(compiler, string_indexof_linear_uu) \
+ do_arch_entry(x86, compiler, string_indexof_linear_uu, \
+ string_indexof_linear_uu, \
+ string_indexof_linear_uu) \
do_stub(compiler, string_indexof_linear_ul) \
+ do_arch_entry(x86, compiler, string_indexof_linear_ul, \
+ string_indexof_linear_ul, \
+ string_indexof_linear_ul) \
do_stub(compiler, pshuffle_byte_flip_mask_sha512) \
do_arch_entry(x86, compiler, pshuffle_byte_flip_mask_sha512, \
pshuffle_byte_flip_mask_addr_sha512, \
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp
index 993d1964034..b0612d21437 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -4891,7 +4891,7 @@ void StubGenerator::generate_compiler_stubs() {
StubRoutines::_data_cache_writeback_sync = generate_data_cache_writeback_sync();
#ifdef COMPILER2
- if ((UseAVX == 2) && EnableX86ECoreOpts) {
+ if ((UseAVX == 2) && EnableX86ECoreOpts && UseCountTrailingZerosInstruction) {
generate_string_indexof(StubRoutines::_string_indexof_array);
}
#endif
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_kyber.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_kyber.cpp
index 347a9b936a8..13b1c942213 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_kyber.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_kyber.cpp
@@ -957,6 +957,9 @@ address generate_kyber12To16_avx512(StubGenerator *stubgen,
__ mov64(rax, 0); // return 0
__ ret(0);
+ // record the stub entry and end
+ stubgen->store_archive_data(stub_id, start, __ pc());
+
return start;
}
diff --git a/src/hotspot/cpu/x86/stubRoutines_x86.cpp b/src/hotspot/cpu/x86/stubRoutines_x86.cpp
index ce11925dde2..63851ab69ec 100644
--- a/src/hotspot/cpu/x86/stubRoutines_x86.cpp
+++ b/src/hotspot/cpu/x86/stubRoutines_x86.cpp
@@ -420,6 +420,12 @@ ATTRIBUTE_ALIGNED(64) const julong StubRoutines::x86::_k512_W[] =
0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL,
};
+ATTRIBUTE_ALIGNED(64)
+address StubRoutines::x86::small_jump_table[STRING_INDEXOF_TABLE_COUNT * STRING_INDEXOF_NUMBER_OF_CASES] = { nullptr, };
+
+ATTRIBUTE_ALIGNED(64)
+address StubRoutines::x86::big_jump_table[STRING_INDEXOF_TABLE_COUNT * STRING_INDEXOF_NUMBER_OF_CASES] = { nullptr, };
+
#if INCLUDE_CDS
void StubRoutines::init_AOTAddressTable() {
@@ -459,6 +465,12 @@ void StubRoutines::x86::init_AOTAddressTable(GrowableArray& external_ad
ADD(_k256);
ADD(_k256_W);
ADD(_k512_W);
+ for (int i = 0; i < STRING_INDEXOF_TABLE_COUNT; i++) {
+ address a = (address)StubRoutines::x86::small_jump_table_base(i);
+ ADD(a);
+ a = (address)StubRoutines::x86::big_jump_table_base(i);
+ ADD(a);
+ }
#undef ADD
}
#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubRoutines_x86.hpp b/src/hotspot/cpu/x86/stubRoutines_x86.hpp
index 7283798888b..0dce0f44cb9 100644
--- a/src/hotspot/cpu/x86/stubRoutines_x86.hpp
+++ b/src/hotspot/cpu/x86/stubRoutines_x86.hpp
@@ -118,6 +118,26 @@ public:
static address k256_W_addr() { return _k256_W_adr; }
static address k512_W_addr() { return _k512_W_addr; }
+ // declare storage for jump tables used by string index stubs
+
+ // we assert this equals the number of tags in
+ // StrIntrinsicNode::ArgEncoding in code that uses the value to
+ // avoid having to inlcude opto headers here
+ static constexpr int STRING_INDEXOF_TABLE_COUNT = 4;
+ static constexpr int STRING_INDEXOF_NUMBER_OF_CASES = 10;
+ static address small_jump_table[STRING_INDEXOF_TABLE_COUNT * STRING_INDEXOF_NUMBER_OF_CASES];
+ static address big_jump_table[STRING_INDEXOF_TABLE_COUNT * STRING_INDEXOF_NUMBER_OF_CASES];
+
+ static address *small_jump_table_base(int idx) {
+ assert(idx >= 0 && idx < STRING_INDEXOF_TABLE_COUNT, "invalid jump table index %d", idx);
+ return small_jump_table + (idx * STRING_INDEXOF_NUMBER_OF_CASES);
+ }
+
+ static address *big_jump_table_base(int idx) {
+ assert(idx >= 0 && idx < STRING_INDEXOF_TABLE_COUNT, "invalid jump table index %d", idx);
+ return big_jump_table + (idx * STRING_INDEXOF_NUMBER_OF_CASES);
+ }
+
static address arrays_hashcode_powers_of_31() { return (address)_arrays_hashcode_powers_of_31; }
static void generate_CRC32C_table(bool is_pclmulqdq_supported);
diff --git a/src/hotspot/cpu/x86/vmStructs_x86.hpp b/src/hotspot/cpu/x86/vmStructs_x86.hpp
index b8089a6413e..e0fcc7d375a 100644
--- a/src/hotspot/cpu/x86/vmStructs_x86.hpp
+++ b/src/hotspot/cpu/x86/vmStructs_x86.hpp
@@ -46,7 +46,7 @@
#define VM_LONG_CONSTANTS_CPU(declare_constant, declare_preprocessor_constant)
-#define DECLARE_LONG_CPU_FEATURE_CONSTANT(id, name, bit) GENERATE_VM_LONG_CONSTANT_ENTRY(VM_Version::CPU_##id)
+#define DECLARE_LONG_CPU_FEATURE_CONSTANT(id, name) GENERATE_VM_LONG_CONSTANT_ENTRY(VM_Version::CPU_##id)
#define VM_LONG_CPU_FEATURE_CONSTANTS CPU_FEATURE_FLAGS(DECLARE_LONG_CPU_FEATURE_CONSTANT)
#endif // CPU_X86_VMSTRUCTS_X86_HPP
diff --git a/src/hotspot/cpu/x86/vm_version_x86.cpp b/src/hotspot/cpu/x86/vm_version_x86.cpp
index cf9de40a237..80d88f2ecb8 100644
--- a/src/hotspot/cpu/x86/vm_version_x86.cpp
+++ b/src/hotspot/cpu/x86/vm_version_x86.cpp
@@ -48,7 +48,7 @@ int VM_Version::_stepping;
bool VM_Version::_has_intel_jcc_erratum;
VM_Version::CpuidInfo VM_Version::_cpuid_info = { 0, };
-#define DECLARE_CPU_FEATURE_NAME(id, name, bit) XSTR(name),
+#define DECLARE_CPU_FEATURE_NAME(id, name) XSTR(name),
const char* VM_Version::_features_names[] = { CPU_FEATURE_FLAGS(DECLARE_CPU_FEATURE_NAME)};
#undef DECLARE_CPU_FEATURE_NAME
@@ -942,21 +942,21 @@ void VM_Version::get_processor_features() {
}
if (UseSSE < 4) {
- _features.clear_feature(CPU_SSE4_1);
- _features.clear_feature(CPU_SSE4_2);
+ clear_feature(CPU_SSE4_1);
+ clear_feature(CPU_SSE4_2);
}
if (UseSSE < 3) {
- _features.clear_feature(CPU_SSE3);
- _features.clear_feature(CPU_SSSE3);
- _features.clear_feature(CPU_SSE4A);
+ clear_feature(CPU_SSE3);
+ clear_feature(CPU_SSSE3);
+ clear_feature(CPU_SSE4A);
}
if (UseSSE < 2)
- _features.clear_feature(CPU_SSE2);
+ clear_feature(CPU_SSE2);
if (UseSSE < 1)
- _features.clear_feature(CPU_SSE);
+ clear_feature(CPU_SSE);
// ZX cpus specific settings
if (is_zx() && FLAG_IS_DEFAULT(UseAVX)) {
@@ -1030,102 +1030,119 @@ void VM_Version::get_processor_features() {
}
if (UseAVX < 3) {
- _features.clear_feature(CPU_AVX512F);
- _features.clear_feature(CPU_AVX512DQ);
- _features.clear_feature(CPU_AVX512CD);
- _features.clear_feature(CPU_AVX512BW);
- _features.clear_feature(CPU_AVX512ER);
- _features.clear_feature(CPU_AVX512PF);
- _features.clear_feature(CPU_AVX512VL);
- _features.clear_feature(CPU_AVX512_VPOPCNTDQ);
- _features.clear_feature(CPU_AVX512_VPCLMULQDQ);
- _features.clear_feature(CPU_AVX512_VAES);
- _features.clear_feature(CPU_AVX512_VNNI);
- _features.clear_feature(CPU_AVX512_VBMI);
- _features.clear_feature(CPU_AVX512_VBMI2);
- _features.clear_feature(CPU_AVX512_BITALG);
- _features.clear_feature(CPU_AVX512_IFMA);
- _features.clear_feature(CPU_APX_F);
- _features.clear_feature(CPU_AVX512_FP16);
- _features.clear_feature(CPU_AVX10_1);
- _features.clear_feature(CPU_AVX10_2);
+ clear_feature(CPU_AVX512F);
+ clear_feature(CPU_AVX512DQ);
+ clear_feature(CPU_AVX512CD);
+ clear_feature(CPU_AVX512BW);
+ clear_feature(CPU_AVX512ER);
+ clear_feature(CPU_AVX512PF);
+ clear_feature(CPU_AVX512VL);
+ clear_feature(CPU_AVX512_VPOPCNTDQ);
+ clear_feature(CPU_AVX512_VPCLMULQDQ);
+ clear_feature(CPU_AVX512_VAES);
+ clear_feature(CPU_AVX512_VNNI);
+ clear_feature(CPU_AVX512_VBMI);
+ clear_feature(CPU_AVX512_VBMI2);
+ clear_feature(CPU_AVX512_BITALG);
+ clear_feature(CPU_AVX512_IFMA);
+ clear_feature(CPU_APX_F);
+ clear_feature(CPU_AVX512_FP16);
+ clear_feature(CPU_AVX10_1);
+ clear_feature(CPU_AVX10_2);
}
if (UseAVX < 2) {
- _features.clear_feature(CPU_AVX2);
- _features.clear_feature(CPU_AVX_IFMA);
+ clear_feature(CPU_AVX2);
+ clear_feature(CPU_AVX_IFMA);
}
if (UseAVX < 1) {
- _features.clear_feature(CPU_AVX);
- _features.clear_feature(CPU_VZEROUPPER);
- _features.clear_feature(CPU_F16C);
- _features.clear_feature(CPU_SHA512);
+ clear_feature(CPU_AVX);
+ clear_feature(CPU_VZEROUPPER);
+ clear_feature(CPU_F16C);
+ clear_feature(CPU_SHA512);
}
if (logical_processors_per_package() == 1) {
// HT processor could be installed on a system which doesn't support HT.
- _features.clear_feature(CPU_HT);
+ clear_feature(CPU_HT);
}
if (is_intel()) { // Intel cpus specific settings
if (is_knights_family()) {
- _features.clear_feature(CPU_VZEROUPPER);
- _features.clear_feature(CPU_AVX512BW);
- _features.clear_feature(CPU_AVX512VL);
- _features.clear_feature(CPU_APX_F);
- _features.clear_feature(CPU_AVX512DQ);
- _features.clear_feature(CPU_AVX512_VNNI);
- _features.clear_feature(CPU_AVX512_VAES);
- _features.clear_feature(CPU_AVX512_VPOPCNTDQ);
- _features.clear_feature(CPU_AVX512_VPCLMULQDQ);
- _features.clear_feature(CPU_AVX512_VBMI);
- _features.clear_feature(CPU_AVX512_VBMI2);
- _features.clear_feature(CPU_CLWB);
- _features.clear_feature(CPU_FLUSHOPT);
- _features.clear_feature(CPU_GFNI);
- _features.clear_feature(CPU_AVX512_BITALG);
- _features.clear_feature(CPU_AVX512_IFMA);
- _features.clear_feature(CPU_AVX_IFMA);
- _features.clear_feature(CPU_AVX512_FP16);
- _features.clear_feature(CPU_AVX10_1);
- _features.clear_feature(CPU_AVX10_2);
+ clear_feature(CPU_VZEROUPPER);
+ clear_feature(CPU_AVX512BW);
+ clear_feature(CPU_AVX512VL);
+ clear_feature(CPU_APX_F);
+ clear_feature(CPU_AVX512DQ);
+ clear_feature(CPU_AVX512_VNNI);
+ clear_feature(CPU_AVX512_VAES);
+ clear_feature(CPU_AVX512_VPOPCNTDQ);
+ clear_feature(CPU_AVX512_VPCLMULQDQ);
+ clear_feature(CPU_AVX512_VBMI);
+ clear_feature(CPU_AVX512_VBMI2);
+ clear_feature(CPU_CLWB);
+ clear_feature(CPU_FLUSHOPT);
+ clear_feature(CPU_GFNI);
+ clear_feature(CPU_AVX512_BITALG);
+ clear_feature(CPU_AVX512_IFMA);
+ clear_feature(CPU_AVX_IFMA);
+ clear_feature(CPU_AVX512_FP16);
+ clear_feature(CPU_AVX10_1);
+ clear_feature(CPU_AVX10_2);
}
}
// Currently APX support is only enabled for targets supporting AVX512VL feature.
if (supports_apx_f() && os_supports_apx_egprs() && supports_avx512vl()) {
if (FLAG_IS_DEFAULT(UseAPX)) {
- UseAPX = false; // by default UseAPX is false
- _features.clear_feature(CPU_APX_F);
+ FLAG_SET_DEFAULT(UseAPX, false); // by default UseAPX is false
+ clear_feature(CPU_APX_F);
} else if (!UseAPX) {
- _features.clear_feature(CPU_APX_F);
+ clear_feature(CPU_APX_F);
}
- } else if (UseAPX) {
- if (!FLAG_IS_DEFAULT(UseAPX)) {
- warning("APX is not supported on this CPU, setting it to false)");
+ } else {
+ if (!os_supports_apx_egprs() || !supports_avx512vl()) {
+ clear_feature(CPU_APX_F);
+ }
+ if (UseAPX) {
+ if (!FLAG_IS_DEFAULT(UseAPX)) {
+ warning("APX instructions are not available on this CPU");
+ }
+ FLAG_SET_DEFAULT(UseAPX, false);
}
- FLAG_SET_DEFAULT(UseAPX, false);
}
- CHECK_CPU_FEATURE(supports_clmul, CLMUL);
- CHECK_CPU_FEATURE(supports_aes, AES);
- CHECK_CPU_FEATURE(supports_fma, FMA);
+ CHECK_CPU_FEATURE(UseCLMUL, CLMUL, supports_clmul(), "CLMUL" MULTI_INST_WARNING_MSG);
+ CHECK_CPU_FEATURE(UseAES, AES, supports_aes(), "AES" MULTI_INST_WARNING_MSG);
+ CHECK_CPU_FEATURE(UseFMA, FMA, supports_fma(), "FMA" MULTI_INST_WARNING_MSG);
+ CHECK_CPU_FEATURE(UseCountLeadingZerosInstruction, LZCNT, supports_lzcnt(), "lzcnt" SINGLE_INST_WARNING_MSG);
+ // BMI instructions (except tzcnt) use an encoding with VEX prefix.
+ // VEX prefix is generated only when AVX > 0.
+ CHECK_CPU_FEATURE(UseBMI1Instructions, BMI1, supports_bmi1(), "BMI1" MULTI_INST_WARNING_MSG);
- if (supports_sha() || (supports_avx2() && supports_bmi2())) {
- if (FLAG_IS_DEFAULT(UseSHA)) {
- UseSHA = true;
- } else if (!UseSHA) {
- _features.clear_feature(CPU_SHA);
+ if (supports_bmi2() && supports_avx()) {
+ if (FLAG_IS_DEFAULT(UseBMI2Instructions)) {
+ FLAG_SET_DEFAULT(UseBMI2Instructions, true);
+ } else if (!UseBMI2Instructions) {
+ clear_feature(CPU_BMI2);
}
- } else if (UseSHA) {
- if (!FLAG_IS_DEFAULT(UseSHA)) {
- warning("SHA instructions are not available on this CPU");
+ } else {
+ if (!supports_avx()) {
+ clear_feature(CPU_BMI2);
+ }
+ if (UseBMI2Instructions) {
+ if (!FLAG_IS_DEFAULT(UseBMI2Instructions)) {
+ warning("BMI2 instructions are not available on this CPU (AVX is also required)");
+ }
+ FLAG_SET_DEFAULT(UseBMI2Instructions, false);
}
- FLAG_SET_DEFAULT(UseSHA, false);
}
+ CHECK_CPU_FEATURE(UsePopCountInstruction, POPCNT, supports_popcnt(), "popcnt" SINGLE_INST_WARNING_MSG);
+ CHECK_CPU_FEATURE(UseSHA, SHA, supports_sha() || (supports_avx2() && supports_bmi2()), "SHA" MULTI_INST_WARNING_MSG);
+
if (FLAG_IS_DEFAULT(IntelJccErratumMitigation)) {
_has_intel_jcc_erratum = compute_has_intel_jcc_erratum();
FLAG_SET_ERGO(IntelJccErratumMitigation, _has_intel_jcc_erratum);
@@ -1716,28 +1733,11 @@ void VM_Version::get_processor_features() {
FLAG_SET_DEFAULT(UseVectorizedHashCodeIntrinsic, false);
}
- // Use count leading zeros count instruction if available.
- if (supports_lzcnt()) {
- if (FLAG_IS_DEFAULT(UseCountLeadingZerosInstruction)) {
- UseCountLeadingZerosInstruction = true;
- }
- } else if (UseCountLeadingZerosInstruction) {
- if (!FLAG_IS_DEFAULT(UseCountLeadingZerosInstruction)) {
- warning("lzcnt instruction is not available on this CPU");
- }
- FLAG_SET_DEFAULT(UseCountLeadingZerosInstruction, false);
- }
-
// Use count trailing zeros instruction if available
if (supports_bmi1()) {
// tzcnt does not require VEX prefix
if (FLAG_IS_DEFAULT(UseCountTrailingZerosInstruction)) {
- if (!UseBMI1Instructions && !FLAG_IS_DEFAULT(UseBMI1Instructions)) {
- // Don't use tzcnt if BMI1 is switched off on command line.
- UseCountTrailingZerosInstruction = false;
- } else {
- UseCountTrailingZerosInstruction = true;
- }
+ UseCountTrailingZerosInstruction = true;
}
} else if (UseCountTrailingZerosInstruction) {
if (!FLAG_IS_DEFAULT(UseCountTrailingZerosInstruction)) {
@@ -1746,42 +1746,6 @@ void VM_Version::get_processor_features() {
FLAG_SET_DEFAULT(UseCountTrailingZerosInstruction, false);
}
- // BMI instructions (except tzcnt) use an encoding with VEX prefix.
- // VEX prefix is generated only when AVX > 0.
- if (supports_bmi1() && supports_avx()) {
- if (FLAG_IS_DEFAULT(UseBMI1Instructions)) {
- UseBMI1Instructions = true;
- }
- } else if (UseBMI1Instructions) {
- if (!FLAG_IS_DEFAULT(UseBMI1Instructions)) {
- warning("BMI1 instructions are not available on this CPU (AVX is also required)");
- }
- FLAG_SET_DEFAULT(UseBMI1Instructions, false);
- }
-
- if (supports_bmi2() && supports_avx()) {
- if (FLAG_IS_DEFAULT(UseBMI2Instructions)) {
- UseBMI2Instructions = true;
- }
- } else if (UseBMI2Instructions) {
- if (!FLAG_IS_DEFAULT(UseBMI2Instructions)) {
- warning("BMI2 instructions are not available on this CPU (AVX is also required)");
- }
- FLAG_SET_DEFAULT(UseBMI2Instructions, false);
- }
-
- // Use population count instruction if available.
- if (supports_popcnt()) {
- if (FLAG_IS_DEFAULT(UsePopCountInstruction)) {
- UsePopCountInstruction = true;
- }
- } else if (UsePopCountInstruction) {
- if (!FLAG_IS_DEFAULT(UsePopCountInstruction)) {
- warning("POPCNT instruction is not available on this CPU");
- }
- FLAG_SET_DEFAULT(UsePopCountInstruction, false);
- }
-
// Use fast-string operations if available.
if (supports_erms()) {
if (FLAG_IS_DEFAULT(UseFastStosb)) {
@@ -2527,7 +2491,7 @@ const char* VM_Version::cpu_brand_string(void) {
}
int ret_val = cpu_extended_brand_string(_cpu_brand_string, CPU_EBS_MAX_LENGTH);
if (ret_val != OS_OK) {
- FREE_C_HEAP_ARRAY(char, _cpu_brand_string);
+ FREE_C_HEAP_ARRAY(_cpu_brand_string);
_cpu_brand_string = nullptr;
}
}
@@ -3366,12 +3330,12 @@ int VM_Version::cpu_features_size() {
}
void VM_Version::store_cpu_features(void* buf) {
- VM_Features copy = _features;
- copy.clear_feature(CPU_HT); // HT does not result in incompatibility of aot code cache
+ VM_Features copy = _features.aot_code_cache_features();
memcpy(buf, ©, sizeof(VM_Features));
}
-bool VM_Version::supports_features(void* features_buffer) {
+bool VM_Version::verify_aot_code_cache_features(void* features_buffer) {
VM_Features* features_to_test = (VM_Features*)features_buffer;
- return _features.supports_features(features_to_test);
+ VM_Features rt_features = _features.aot_code_cache_features();
+ return rt_features.verify_aot_code_cache_features(features_to_test);
}
diff --git a/src/hotspot/cpu/x86/vm_version_x86.hpp b/src/hotspot/cpu/x86/vm_version_x86.hpp
index f721635a02e..fe6d424f50c 100644
--- a/src/hotspot/cpu/x86/vm_version_x86.hpp
+++ b/src/hotspot/cpu/x86/vm_version_x86.hpp
@@ -377,86 +377,86 @@ protected:
*/
enum Feature_Flag {
#define CPU_FEATURE_FLAGS(decl) \
- decl(CX8, cx8, 0) /* next bits are from cpuid 1 (EDX) */ \
- decl(CMOV, cmov, 1) \
- decl(FXSR, fxsr, 2) \
- decl(HT, ht, 3) \
- \
- decl(MMX, mmx, 4) \
- decl(3DNOW_PREFETCH, 3dnowpref, 5) /* Processor supports 3dnow prefetch and prefetchw instructions */ \
- /* may not necessarily support other 3dnow instructions */ \
- decl(SSE, sse, 6) \
- decl(SSE2, sse2, 7) \
- \
- decl(SSE3, sse3, 8 ) /* SSE3 comes from cpuid 1 (ECX) */ \
- decl(SSSE3, ssse3, 9 ) \
- decl(SSE4A, sse4a, 10) \
- decl(SSE4_1, sse4.1, 11) \
- \
- decl(SSE4_2, sse4.2, 12) \
- decl(POPCNT, popcnt, 13) \
- decl(LZCNT, lzcnt, 14) \
- decl(TSC, tsc, 15) \
- \
- decl(TSCINV_BIT, tscinvbit, 16) \
- decl(TSCINV, tscinv, 17) \
- decl(AVX, avx, 18) \
- decl(AVX2, avx2, 19) \
- \
- decl(AES, aes, 20) \
- decl(ERMS, erms, 21) /* enhanced 'rep movsb/stosb' instructions */ \
- decl(CLMUL, clmul, 22) /* carryless multiply for CRC */ \
- decl(BMI1, bmi1, 23) \
- \
- decl(BMI2, bmi2, 24) \
- decl(RTM, rtm, 25) /* Restricted Transactional Memory instructions */ \
- decl(ADX, adx, 26) \
- decl(AVX512F, avx512f, 27) /* AVX 512bit foundation instructions */ \
- \
- decl(AVX512DQ, avx512dq, 28) \
- decl(AVX512PF, avx512pf, 29) \
- decl(AVX512ER, avx512er, 30) \
- decl(AVX512CD, avx512cd, 31) \
- \
- decl(AVX512BW, avx512bw, 32) /* Byte and word vector instructions */ \
- decl(AVX512VL, avx512vl, 33) /* EVEX instructions with smaller vector length */ \
- decl(SHA, sha, 34) /* SHA instructions */ \
- decl(FMA, fma, 35) /* FMA instructions */ \
- \
- decl(VZEROUPPER, vzeroupper, 36) /* Vzeroupper instruction */ \
- decl(AVX512_VPOPCNTDQ, avx512_vpopcntdq, 37) /* Vector popcount */ \
- decl(AVX512_VPCLMULQDQ, avx512_vpclmulqdq, 38) /* Vector carryless multiplication */ \
- decl(AVX512_VAES, avx512_vaes, 39) /* Vector AES instruction */ \
- \
- decl(AVX512_VNNI, avx512_vnni, 40) /* Vector Neural Network Instructions */ \
- decl(FLUSH, clflush, 41) /* flush instruction */ \
- decl(FLUSHOPT, clflushopt, 42) /* flusopth instruction */ \
- decl(CLWB, clwb, 43) /* clwb instruction */ \
- \
- decl(AVX512_VBMI2, avx512_vbmi2, 44) /* VBMI2 shift left double instructions */ \
- decl(AVX512_VBMI, avx512_vbmi, 45) /* Vector BMI instructions */ \
- decl(HV, hv, 46) /* Hypervisor instructions */ \
- decl(SERIALIZE, serialize, 47) /* CPU SERIALIZE */ \
- decl(RDTSCP, rdtscp, 48) /* RDTSCP instruction */ \
- decl(RDPID, rdpid, 49) /* RDPID instruction */ \
- decl(FSRM, fsrm, 50) /* Fast Short REP MOV */ \
- decl(GFNI, gfni, 51) /* Vector GFNI instructions */ \
- decl(AVX512_BITALG, avx512_bitalg, 52) /* Vector sub-word popcount and bit gather instructions */\
- decl(F16C, f16c, 53) /* Half-precision and single precision FP conversion instructions*/ \
- decl(PKU, pku, 54) /* Protection keys for user-mode pages */ \
- decl(OSPKE, ospke, 55) /* OS enables protection keys */ \
- decl(CET_IBT, cet_ibt, 56) /* Control Flow Enforcement - Indirect Branch Tracking */ \
- decl(CET_SS, cet_ss, 57) /* Control Flow Enforcement - Shadow Stack */ \
- decl(AVX512_IFMA, avx512_ifma, 58) /* Integer Vector FMA instructions*/ \
- decl(AVX_IFMA, avx_ifma, 59) /* 256-bit VEX-coded variant of AVX512-IFMA*/ \
- decl(APX_F, apx_f, 60) /* Intel Advanced Performance Extensions*/ \
- decl(SHA512, sha512, 61) /* SHA512 instructions*/ \
- decl(AVX512_FP16, avx512_fp16, 62) /* AVX512 FP16 ISA support*/ \
- decl(AVX10_1, avx10_1, 63) /* AVX10 512 bit vector ISA Version 1 support*/ \
- decl(AVX10_2, avx10_2, 64) /* AVX10 512 bit vector ISA Version 2 support*/ \
- decl(HYBRID, hybrid, 65) /* Hybrid architecture */
+ decl(CX8, cx8 ) /* next bits are from cpuid 1 (EDX) */ \
+ decl(CMOV, cmov ) \
+ decl(FXSR, fxsr ) \
+ decl(HT, ht ) \
+ \
+ decl(MMX, mmx ) \
+ decl(3DNOW_PREFETCH, 3dnowpref ) /* Processor supports 3dnow prefetch and prefetchw instructions */ \
+ /* may not necessarily support other 3dnow instructions */ \
+ decl(SSE, sse ) \
+ decl(SSE2, sse2 ) \
+ \
+ decl(SSE3, sse3 ) /* SSE3 comes from cpuid 1 (ECX) */ \
+ decl(SSSE3, ssse3 ) \
+ decl(SSE4A, sse4a ) \
+ decl(SSE4_1, sse4.1 ) \
+ \
+ decl(SSE4_2, sse4.2 ) \
+ decl(POPCNT, popcnt ) \
+ decl(LZCNT, lzcnt ) \
+ decl(TSC, tsc ) \
+ \
+ decl(TSCINV_BIT, tscinvbit ) \
+ decl(TSCINV, tscinv ) \
+ decl(AVX, avx ) \
+ decl(AVX2, avx2 ) \
+ \
+ decl(AES, aes ) \
+ decl(ERMS, erms ) /* enhanced 'rep movsb/stosb' instructions */ \
+ decl(CLMUL, clmul ) /* carryless multiply for CRC */ \
+ decl(BMI1, bmi1 ) \
+ \
+ decl(BMI2, bmi2 ) \
+ decl(RTM, rtm ) /* Restricted Transactional Memory instructions */ \
+ decl(ADX, adx ) \
+ decl(AVX512F, avx512f ) /* AVX 512bit foundation instructions */ \
+ \
+ decl(AVX512DQ, avx512dq ) \
+ decl(AVX512PF, avx512pf ) \
+ decl(AVX512ER, avx512er ) \
+ decl(AVX512CD, avx512cd ) \
+ \
+ decl(AVX512BW, avx512bw ) /* Byte and word vector instructions */ \
+ decl(AVX512VL, avx512vl ) /* EVEX instructions with smaller vector length */ \
+ decl(SHA, sha ) /* SHA instructions */ \
+ decl(FMA, fma ) /* FMA instructions */ \
+ \
+ decl(VZEROUPPER, vzeroupper ) /* Vzeroupper instruction */ \
+ decl(AVX512_VPOPCNTDQ, avx512_vpopcntdq ) /* Vector popcount */ \
+ decl(AVX512_VPCLMULQDQ, avx512_vpclmulqdq ) /* Vector carryless multiplication */ \
+ decl(AVX512_VAES, avx512_vaes ) /* Vector AES instruction */ \
+ \
+ decl(AVX512_VNNI, avx512_vnni ) /* Vector Neural Network Instructions */ \
+ decl(FLUSH, clflush ) /* flush instruction */ \
+ decl(FLUSHOPT, clflushopt ) /* flusopth instruction */ \
+ decl(CLWB, clwb ) /* clwb instruction */ \
+ \
+ decl(AVX512_VBMI2, avx512_vbmi2 ) /* VBMI2 shift left double instructions */ \
+ decl(AVX512_VBMI, avx512_vbmi ) /* Vector BMI instructions */ \
+ decl(HV, hv ) /* Hypervisor instructions */ \
+ decl(SERIALIZE, serialize ) /* CPU SERIALIZE */ \
+ decl(RDTSCP, rdtscp ) /* RDTSCP instruction */ \
+ decl(RDPID, rdpid ) /* RDPID instruction */ \
+ decl(FSRM, fsrm ) /* Fast Short REP MOV */ \
+ decl(GFNI, gfni ) /* Vector GFNI instructions */ \
+ decl(AVX512_BITALG, avx512_bitalg ) /* Vector sub-word popcount and bit gather instructions */\
+ decl(F16C, f16c ) /* Half-precision and single precision FP conversion instructions*/ \
+ decl(PKU, pku ) /* Protection keys for user-mode pages */ \
+ decl(OSPKE, ospke ) /* OS enables protection keys */ \
+ decl(CET_IBT, cet_ibt ) /* Control Flow Enforcement - Indirect Branch Tracking */ \
+ decl(CET_SS, cet_ss ) /* Control Flow Enforcement - Shadow Stack */ \
+ decl(AVX512_IFMA, avx512_ifma ) /* Integer Vector FMA instructions*/ \
+ decl(AVX_IFMA, avx_ifma ) /* 256-bit VEX-coded variant of AVX512-IFMA*/ \
+ decl(APX_F, apx_f ) /* Intel Advanced Performance Extensions*/ \
+ decl(SHA512, sha512 ) /* SHA512 instructions*/ \
+ decl(AVX512_FP16, avx512_fp16 ) /* AVX512 FP16 ISA support*/ \
+ decl(AVX10_1, avx10_1 ) /* AVX10 512 bit vector ISA Version 1 support*/ \
+ decl(AVX10_2, avx10_2 ) /* AVX10 512 bit vector ISA Version 2 support*/ \
+ decl(HYBRID, hybrid ) /* Hybrid architecture */
-#define DECLARE_CPU_FEATURE_FLAG(id, name, bit) CPU_##id = (bit),
+#define DECLARE_CPU_FEATURE_FLAG(id, name) CPU_##id,
CPU_FEATURE_FLAGS(DECLARE_CPU_FEATURE_FLAG)
#undef DECLARE_CPU_FEATURE_FLAG
MAX_CPU_FEATURES
@@ -517,14 +517,21 @@ protected:
return (_features_bitmap[idx] & bit_mask(feature)) != 0;
}
- bool supports_features(VM_Features* features_to_test) {
+ bool verify_aot_code_cache_features(VM_Features* features_to_test) {
for (int i = 0; i < features_bitmap_element_count(); i++) {
- if ((_features_bitmap[i] & features_to_test->_features_bitmap[i]) != features_to_test->_features_bitmap[i]) {
+ if (_features_bitmap[i] != features_to_test->_features_bitmap[i]) {
return false;
- }
+ }
}
return true;
}
+
+ VM_Features aot_code_cache_features() {
+ VM_Features copy = *this;
+ // HT does not result in incompatibility of aot code cache
+ copy.clear_feature(CPU_HT);
+ return copy;
+ }
};
// CPU feature flags vector, can be affected by VM settings.
@@ -1134,7 +1141,7 @@ public:
// Size of the buffer must be same as returned by cpu_features_size()
static void store_cpu_features(void* buf);
- static bool supports_features(void* features_to_test);
+ static bool verify_aot_code_cache_features(void* features_buffer);
};
#endif // CPU_X86_VM_VERSION_X86_HPP
diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad
index eaa88d900c7..db87f81d6c4 100644
--- a/src/hotspot/cpu/x86/x86.ad
+++ b/src/hotspot/cpu/x86/x86.ad
@@ -2159,7 +2159,7 @@ uint MachSpillCopyNode::implementation(C2_MacroAssembler* masm,
// Self copy, no move
return 0;
}
- if (bottom_type()->isa_vect() != nullptr && bottom_type()->isa_vectmask() == nullptr) {
+ if (bottom_type()->isa_vect() != nullptr && bottom_type()->isa_pvectmask() == nullptr) {
uint ireg = ideal_reg();
assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
@@ -3789,8 +3789,8 @@ bool Matcher::vector_rearrange_requires_load_shuffle(BasicType elem_bt, int vlen
}
bool Matcher::mask_op_prefers_predicate(int opcode, const TypeVect* vt) {
- // Prefer predicate if the mask type is "TypeVectMask".
- return vt->isa_vectmask() != nullptr;
+ // Prefer predicate if the mask type is "TypePVectMask".
+ return vt->isa_pvectmask() != nullptr;
}
MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
@@ -4048,7 +4048,7 @@ class FusedPatternMatcher {
};
static bool is_bmi_pattern(Node* n, Node* m) {
- assert(UseBMI1Instructions, "sanity");
+ assert(VM_Version::supports_bmi1() && VM_Version::supports_avx(), "sanity");
if (n != nullptr && m != nullptr) {
if (m->Opcode() == Op_LoadI) {
FusedPatternMatcher bmii(n, m, Op_ConI);
@@ -4068,7 +4068,7 @@ static bool is_bmi_pattern(Node* n, Node* m) {
// Should the matcher clone input 'm' of node 'n'?
bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
// If 'n' and 'm' are part of a graph for BMI instruction, clone the input 'm'.
- if (UseBMI1Instructions && is_bmi_pattern(n, m)) {
+ if (VM_Version::supports_bmi1() && VM_Version::supports_avx() && is_bmi_pattern(n, m)) {
mstack.push(m, Visit);
return true;
}
@@ -4568,7 +4568,7 @@ encode %{
} else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
// The NOP here is purely to ensure that eliding a call to
// JVM_EnsureMaterializedForStackWalk doesn't change the code size.
- __ addr_nop_5();
+ __ nop(5);
__ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
} else {
int method_index = resolved_method_index(masm);
@@ -4950,7 +4950,7 @@ operand immN0() %{
operand immP31()
%{
- predicate(n->as_Type()->type()->reloc() == relocInfo::none
+ predicate(n->as_Type()->type()->is_ptr()->reloc() == relocInfo::none
&& (n->get_ptr() >> 31) == 0);
match(ConP);
@@ -10899,10 +10899,11 @@ instruct xaddB(memory mem, rRegI newval, rFlagsReg cr) %{
predicate(!n->as_LoadStore()->result_not_used());
match(Set newval (GetAndAddB mem newval));
effect(KILL cr);
- format %{ "xaddb_lock $mem, $newval" %}
+ format %{ "xaddb_lock $mem, $newval\t# $newval -> byte" %}
ins_encode %{
__ lock();
__ xaddb($mem$$Address, $newval$$Register);
+ __ narrow_subword_type($newval$$Register, T_BYTE);
%}
ins_pipe(pipe_cmpxchg);
%}
@@ -10935,10 +10936,11 @@ instruct xaddS(memory mem, rRegI newval, rFlagsReg cr) %{
predicate(!n->as_LoadStore()->result_not_used());
match(Set newval (GetAndAddS mem newval));
effect(KILL cr);
- format %{ "xaddw_lock $mem, $newval" %}
+ format %{ "xaddw_lock $mem, $newval\t# $newval -> short" %}
ins_encode %{
__ lock();
__ xaddw($mem$$Address, $newval$$Register);
+ __ narrow_subword_type($newval$$Register, T_SHORT);
%}
ins_pipe(pipe_cmpxchg);
%}
@@ -11017,18 +11019,20 @@ instruct xaddL(memory mem, rRegL newval, rFlagsReg cr) %{
instruct xchgB( memory mem, rRegI newval) %{
match(Set newval (GetAndSetB mem newval));
- format %{ "XCHGB $newval,[$mem]" %}
+ format %{ "XCHGB $newval,[$mem]\t# $newval -> byte" %}
ins_encode %{
__ xchgb($newval$$Register, $mem$$Address);
+ __ narrow_subword_type($newval$$Register, T_BYTE);
%}
ins_pipe( pipe_cmpxchg );
%}
instruct xchgS( memory mem, rRegI newval) %{
match(Set newval (GetAndSetS mem newval));
- format %{ "XCHGW $newval,[$mem]" %}
+ format %{ "XCHGW $newval,[$mem]\t# $newval -> short" %}
ins_encode %{
__ xchgw($newval$$Register, $mem$$Address);
+ __ narrow_subword_type($newval$$Register, T_SHORT);
%}
ins_pipe( pipe_cmpxchg );
%}
@@ -13161,7 +13165,7 @@ instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
// BMI1 instructions
instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
- predicate(UseBMI1Instructions);
+ predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
@@ -13176,7 +13180,7 @@ instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1
instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
match(Set dst (AndI (XorI src1 minus_1) src2));
- predicate(UseBMI1Instructions);
+ predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
@@ -13190,7 +13194,7 @@ instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1
instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
match(Set dst (AndI (SubI imm_zero src) src));
- predicate(UseBMI1Instructions);
+ predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
@@ -13204,7 +13208,7 @@ instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
- predicate(UseBMI1Instructions);
+ predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
@@ -13220,7 +13224,7 @@ instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
%{
match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
- predicate(UseBMI1Instructions);
+ predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
@@ -13236,7 +13240,7 @@ instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
%{
match(Set dst (XorI (AddI src minus_1) src));
- predicate(UseBMI1Instructions);
+ predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
@@ -13252,7 +13256,7 @@ instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
%{
match(Set dst (AndI (AddI src minus_1) src) );
- predicate(UseBMI1Instructions);
+ predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
@@ -13268,7 +13272,7 @@ instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
%{
match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
- predicate(UseBMI1Instructions);
+ predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
@@ -13809,7 +13813,7 @@ instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
// BMI1 instructions
instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
- predicate(UseBMI1Instructions);
+ predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
@@ -13824,7 +13828,7 @@ instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1
instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
match(Set dst (AndL (XorL src1 minus_1) src2));
- predicate(UseBMI1Instructions);
+ predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
@@ -13838,7 +13842,7 @@ instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1
instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
match(Set dst (AndL (SubL imm_zero src) src));
- predicate(UseBMI1Instructions);
+ predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
@@ -13852,7 +13856,7 @@ instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
- predicate(UseBMI1Instructions);
+ predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
@@ -13868,7 +13872,7 @@ instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
%{
match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
- predicate(UseBMI1Instructions);
+ predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
@@ -13884,7 +13888,7 @@ instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
%{
match(Set dst (XorL (AddL src minus_1) src));
- predicate(UseBMI1Instructions);
+ predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
@@ -13900,7 +13904,7 @@ instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
%{
match(Set dst (AndL (AddL src minus_1) src) );
- predicate(UseBMI1Instructions);
+ predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
@@ -13916,7 +13920,7 @@ instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
%{
match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
- predicate(UseBMI1Instructions);
+ predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
@@ -16333,7 +16337,7 @@ instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
// and raw pointers have no anti-dependencies.
instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
%{
- predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none &&
+ predicate(n->in(2)->in(2)->bottom_type()->isa_rawptr() != nullptr &&
n->in(2)->as_Load()->barrier_data() == 0);
match(Set cr (CmpP op1 (LoadP op2)));
@@ -18211,7 +18215,7 @@ instruct vconvHF2F(vec dst, vec src) %{
// ---------------------------------------- VectorReinterpret ------------------------------------
instruct reinterpret_mask(kReg dst) %{
- predicate(n->bottom_type()->isa_vectmask() &&
+ predicate(n->bottom_type()->isa_pvectmask() &&
Matcher::vector_length(n) == Matcher::vector_length(n->in(1))); // dst == src
match(Set dst (VectorReinterpret dst));
ins_cost(125);
@@ -18224,10 +18228,10 @@ instruct reinterpret_mask(kReg dst) %{
instruct reinterpret_mask_W2B(kReg dst, kReg src, vec xtmp) %{
predicate(UseAVX > 2 && Matcher::vector_length(n) != Matcher::vector_length(n->in(1)) &&
- n->bottom_type()->isa_vectmask() &&
- n->in(1)->bottom_type()->isa_vectmask() &&
- n->in(1)->bottom_type()->is_vectmask()->element_basic_type() == T_SHORT &&
- n->bottom_type()->is_vectmask()->element_basic_type() == T_BYTE); // dst == src
+ n->bottom_type()->isa_pvectmask() &&
+ n->in(1)->bottom_type()->isa_pvectmask() &&
+ n->in(1)->bottom_type()->is_pvectmask()->element_basic_type() == T_SHORT &&
+ n->bottom_type()->is_pvectmask()->element_basic_type() == T_BYTE); // dst == src
match(Set dst (VectorReinterpret src));
effect(TEMP xtmp);
format %{ "vector_mask_reinterpret_W2B $dst $src\t!" %}
@@ -18244,11 +18248,11 @@ instruct reinterpret_mask_W2B(kReg dst, kReg src, vec xtmp) %{
instruct reinterpret_mask_D2B(kReg dst, kReg src, vec xtmp) %{
predicate(UseAVX > 2 && Matcher::vector_length(n) != Matcher::vector_length(n->in(1)) &&
- n->bottom_type()->isa_vectmask() &&
- n->in(1)->bottom_type()->isa_vectmask() &&
- (n->in(1)->bottom_type()->is_vectmask()->element_basic_type() == T_INT ||
- n->in(1)->bottom_type()->is_vectmask()->element_basic_type() == T_FLOAT) &&
- n->bottom_type()->is_vectmask()->element_basic_type() == T_BYTE); // dst == src
+ n->bottom_type()->isa_pvectmask() &&
+ n->in(1)->bottom_type()->isa_pvectmask() &&
+ (n->in(1)->bottom_type()->is_pvectmask()->element_basic_type() == T_INT ||
+ n->in(1)->bottom_type()->is_pvectmask()->element_basic_type() == T_FLOAT) &&
+ n->bottom_type()->is_pvectmask()->element_basic_type() == T_BYTE); // dst == src
match(Set dst (VectorReinterpret src));
effect(TEMP xtmp);
format %{ "vector_mask_reinterpret_D2B $dst $src\t!" %}
@@ -18265,11 +18269,11 @@ instruct reinterpret_mask_D2B(kReg dst, kReg src, vec xtmp) %{
instruct reinterpret_mask_Q2B(kReg dst, kReg src, vec xtmp) %{
predicate(UseAVX > 2 && Matcher::vector_length(n) != Matcher::vector_length(n->in(1)) &&
- n->bottom_type()->isa_vectmask() &&
- n->in(1)->bottom_type()->isa_vectmask() &&
- (n->in(1)->bottom_type()->is_vectmask()->element_basic_type() == T_LONG ||
- n->in(1)->bottom_type()->is_vectmask()->element_basic_type() == T_DOUBLE) &&
- n->bottom_type()->is_vectmask()->element_basic_type() == T_BYTE); // dst == src
+ n->bottom_type()->isa_pvectmask() &&
+ n->in(1)->bottom_type()->isa_pvectmask() &&
+ (n->in(1)->bottom_type()->is_pvectmask()->element_basic_type() == T_LONG ||
+ n->in(1)->bottom_type()->is_pvectmask()->element_basic_type() == T_DOUBLE) &&
+ n->bottom_type()->is_pvectmask()->element_basic_type() == T_BYTE); // dst == src
match(Set dst (VectorReinterpret src));
effect(TEMP xtmp);
format %{ "vector_mask_reinterpret_Q2B $dst $src\t!" %}
@@ -18285,7 +18289,7 @@ instruct reinterpret_mask_Q2B(kReg dst, kReg src, vec xtmp) %{
%}
instruct reinterpret(vec dst) %{
- predicate(!n->bottom_type()->isa_vectmask() &&
+ predicate(!n->bottom_type()->isa_pvectmask() &&
Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1))); // dst == src
match(Set dst (VectorReinterpret dst));
ins_cost(125);
@@ -18321,7 +18325,7 @@ instruct reinterpret_expand(vec dst, vec src) %{
instruct vreinterpret_expand4(legVec dst, vec src) %{
predicate(UseAVX > 0 &&
- !n->bottom_type()->isa_vectmask() &&
+ !n->bottom_type()->isa_pvectmask() &&
(Matcher::vector_length_in_bytes(n->in(1)) == 4) && // src
(Matcher::vector_length_in_bytes(n->in(1)) < Matcher::vector_length_in_bytes(n))); // src < dst
match(Set dst (VectorReinterpret src));
@@ -18336,7 +18340,7 @@ instruct vreinterpret_expand4(legVec dst, vec src) %{
instruct vreinterpret_expand(legVec dst, vec src) %{
predicate(UseAVX > 0 &&
- !n->bottom_type()->isa_vectmask() &&
+ !n->bottom_type()->isa_pvectmask() &&
(Matcher::vector_length_in_bytes(n->in(1)) > 4) && // src
(Matcher::vector_length_in_bytes(n->in(1)) < Matcher::vector_length_in_bytes(n))); // src < dst
match(Set dst (VectorReinterpret src));
@@ -18354,7 +18358,7 @@ instruct vreinterpret_expand(legVec dst, vec src) %{
%}
instruct reinterpret_shrink(vec dst, legVec src) %{
- predicate(!n->bottom_type()->isa_vectmask() &&
+ predicate(!n->bottom_type()->isa_pvectmask() &&
Matcher::vector_length_in_bytes(n->in(1)) > Matcher::vector_length_in_bytes(n)); // src > dst
match(Set dst (VectorReinterpret src));
ins_cost(125);
@@ -22462,7 +22466,7 @@ instruct vround_reg_evex(vec dst, vec src, rRegP tmp, vec xtmp1, vec xtmp2, kReg
// --------------------------------- VectorMaskCmp --------------------------------------
instruct vcmpFD(legVec dst, legVec src1, legVec src2, immI8 cond) %{
- predicate(n->bottom_type()->isa_vectmask() == nullptr &&
+ predicate(n->bottom_type()->isa_pvectmask() == nullptr &&
Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 8 && // src1
Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
is_floating_point_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
@@ -22482,7 +22486,7 @@ instruct vcmpFD(legVec dst, legVec src1, legVec src2, immI8 cond) %{
instruct evcmpFD64(vec dst, vec src1, vec src2, immI8 cond, kReg ktmp) %{
predicate(Matcher::vector_length_in_bytes(n->in(1)->in(1)) == 64 && // src1
- n->bottom_type()->isa_vectmask() == nullptr &&
+ n->bottom_type()->isa_pvectmask() == nullptr &&
is_floating_point_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
effect(TEMP ktmp);
@@ -22503,12 +22507,12 @@ instruct evcmpFD64(vec dst, vec src1, vec src2, immI8 cond, kReg ktmp) %{
%}
instruct evcmpFD(kReg dst, vec src1, vec src2, immI8 cond) %{
- predicate(n->bottom_type()->isa_vectmask() &&
+ predicate(n->bottom_type()->isa_pvectmask() &&
is_floating_point_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
format %{ "vector_compare_evex $dst,$src1,$src2,$cond\t!" %}
ins_encode %{
- assert(bottom_type()->isa_vectmask(), "TypeVectMask expected");
+ assert(bottom_type()->isa_pvectmask(), "TypePVectMask expected");
int vlen_enc = vector_length_encoding(this, $src1);
Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
KRegister mask = k0; // The comparison itself is not being masked.
@@ -22522,7 +22526,7 @@ instruct evcmpFD(kReg dst, vec src1, vec src2, immI8 cond) %{
%}
instruct vcmp_direct(legVec dst, legVec src1, legVec src2, immI8 cond) %{
- predicate(n->bottom_type()->isa_vectmask() == nullptr &&
+ predicate(n->bottom_type()->isa_pvectmask() == nullptr &&
!Matcher::is_unsigned_booltest_pred(n->in(2)->get_int()) &&
Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 4 && // src1
Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
@@ -22542,7 +22546,7 @@ instruct vcmp_direct(legVec dst, legVec src1, legVec src2, immI8 cond) %{
%}
instruct vcmp_negate(legVec dst, legVec src1, legVec src2, immI8 cond, legVec xtmp) %{
- predicate(n->bottom_type()->isa_vectmask() == nullptr &&
+ predicate(n->bottom_type()->isa_pvectmask() == nullptr &&
!Matcher::is_unsigned_booltest_pred(n->in(2)->get_int()) &&
Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 4 && // src1
Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
@@ -22563,7 +22567,7 @@ instruct vcmp_negate(legVec dst, legVec src1, legVec src2, immI8 cond, legVec xt
%}
instruct vcmpu(legVec dst, legVec src1, legVec src2, immI8 cond, legVec xtmp) %{
- predicate(n->bottom_type()->isa_vectmask() == nullptr &&
+ predicate(n->bottom_type()->isa_pvectmask() == nullptr &&
Matcher::is_unsigned_booltest_pred(n->in(2)->get_int()) &&
Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 4 && // src1
Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
@@ -22590,7 +22594,7 @@ instruct vcmpu(legVec dst, legVec src1, legVec src2, immI8 cond, legVec xtmp) %{
%}
instruct vcmp64(vec dst, vec src1, vec src2, immI8 cond, kReg ktmp) %{
- predicate((n->bottom_type()->isa_vectmask() == nullptr &&
+ predicate((n->bottom_type()->isa_pvectmask() == nullptr &&
Matcher::vector_length_in_bytes(n->in(1)->in(1)) == 64) && // src1
is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1
match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
@@ -22625,13 +22629,13 @@ instruct vcmp64(vec dst, vec src1, vec src2, immI8 cond, kReg ktmp) %{
instruct evcmp(kReg dst, vec src1, vec src2, immI8 cond) %{
- predicate(n->bottom_type()->isa_vectmask() &&
+ predicate(n->bottom_type()->isa_pvectmask() &&
is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1
match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
format %{ "vector_compared_evex $dst,$src1,$src2,$cond\t!" %}
ins_encode %{
assert(UseAVX > 2, "required");
- assert(bottom_type()->isa_vectmask(), "TypeVectMask expected");
+ assert(bottom_type()->isa_pvectmask(), "TypePVectMask expected");
int vlen_enc = vector_length_encoding(this, $src1);
Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
@@ -22800,7 +22804,7 @@ instruct blendvp(vec dst, vec src, vec mask, rxmm0 tmp) %{
instruct vblendvpI(legVec dst, legVec src1, legVec src2, legVec mask) %{
predicate(UseAVX > 0 && !EnableX86ECoreOpts &&
- n->in(2)->bottom_type()->isa_vectmask() == nullptr &&
+ n->in(2)->bottom_type()->isa_pvectmask() == nullptr &&
Matcher::vector_length_in_bytes(n) <= 32 &&
is_integral_type(Matcher::vector_element_basic_type(n)));
match(Set dst (VectorBlend (Binary src1 src2) mask));
@@ -22814,7 +22818,7 @@ instruct vblendvpI(legVec dst, legVec src1, legVec src2, legVec mask) %{
instruct vblendvpFD(legVec dst, legVec src1, legVec src2, legVec mask) %{
predicate(UseAVX > 0 && !EnableX86ECoreOpts &&
- n->in(2)->bottom_type()->isa_vectmask() == nullptr &&
+ n->in(2)->bottom_type()->isa_pvectmask() == nullptr &&
Matcher::vector_length_in_bytes(n) <= 32 &&
!is_integral_type(Matcher::vector_element_basic_type(n)));
match(Set dst (VectorBlend (Binary src1 src2) mask));
@@ -22828,7 +22832,7 @@ instruct vblendvpFD(legVec dst, legVec src1, legVec src2, legVec mask) %{
instruct vblendvp(legVec dst, legVec src1, legVec src2, legVec mask, legVec vtmp) %{
predicate(UseAVX > 0 && EnableX86ECoreOpts &&
- n->in(2)->bottom_type()->isa_vectmask() == nullptr &&
+ n->in(2)->bottom_type()->isa_pvectmask() == nullptr &&
Matcher::vector_length_in_bytes(n) <= 32);
match(Set dst (VectorBlend (Binary src1 src2) mask));
format %{ "vector_blend $dst,$src1,$src2,$mask\t! using $vtmp as TEMP" %}
@@ -22844,7 +22848,7 @@ instruct vblendvp(legVec dst, legVec src1, legVec src2, legVec mask, legVec vtmp
instruct evblendvp64(vec dst, vec src1, vec src2, vec mask, kReg ktmp) %{
predicate(Matcher::vector_length_in_bytes(n) == 64 &&
- n->in(2)->bottom_type()->isa_vectmask() == nullptr);
+ n->in(2)->bottom_type()->isa_pvectmask() == nullptr);
match(Set dst (VectorBlend (Binary src1 src2) mask));
format %{ "vector_blend $dst,$src1,$src2,$mask\t! using k2 as TEMP" %}
effect(TEMP ktmp);
@@ -22859,7 +22863,7 @@ instruct evblendvp64(vec dst, vec src1, vec src2, vec mask, kReg ktmp) %{
instruct evblendvp64_masked(vec dst, vec src1, vec src2, kReg mask) %{
- predicate(n->in(2)->bottom_type()->isa_vectmask() &&
+ predicate(n->in(2)->bottom_type()->isa_pvectmask() &&
(!is_subword_type(Matcher::vector_element_basic_type(n)) ||
VM_Version::supports_avx512bw()));
match(Set dst (VectorBlend (Binary src1 src2) mask));
@@ -23058,7 +23062,7 @@ instruct ktest_ge8(rFlagsRegU cr, kReg src1, kReg src2) %{
//------------------------------------- LoadMask --------------------------------------------
instruct loadMask(legVec dst, legVec src) %{
- predicate(n->bottom_type()->isa_vectmask() == nullptr && !VM_Version::supports_avx512vlbw());
+ predicate(n->bottom_type()->isa_pvectmask() == nullptr && !VM_Version::supports_avx512vlbw());
match(Set dst (VectorLoadMask src));
effect(TEMP dst);
format %{ "vector_loadmask_byte $dst, $src\n\t" %}
@@ -23071,7 +23075,7 @@ instruct loadMask(legVec dst, legVec src) %{
%}
instruct loadMask64(kReg dst, vec src, vec xtmp) %{
- predicate(n->bottom_type()->isa_vectmask() && !VM_Version::supports_avx512vlbw());
+ predicate(n->bottom_type()->isa_pvectmask() && !VM_Version::supports_avx512vlbw());
match(Set dst (VectorLoadMask src));
effect(TEMP xtmp);
format %{ "vector_loadmask_64byte $dst, $src\t! using $xtmp as TEMP" %}
@@ -23083,7 +23087,7 @@ instruct loadMask64(kReg dst, vec src, vec xtmp) %{
%}
instruct loadMask_evex(kReg dst, vec src, vec xtmp) %{
- predicate(n->bottom_type()->isa_vectmask() && VM_Version::supports_avx512vlbw());
+ predicate(n->bottom_type()->isa_pvectmask() && VM_Version::supports_avx512vlbw());
match(Set dst (VectorLoadMask src));
effect(TEMP xtmp);
format %{ "vector_loadmask_byte $dst, $src\t! using $xtmp as TEMP" %}
@@ -23098,7 +23102,7 @@ instruct loadMask_evex(kReg dst, vec src, vec xtmp) %{
//------------------------------------- StoreMask --------------------------------------------
instruct vstoreMask1B(vec dst, vec src, immI_1 size) %{
- predicate(Matcher::vector_length(n) < 64 && n->in(1)->bottom_type()->isa_vectmask() == nullptr);
+ predicate(Matcher::vector_length(n) < 64 && n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
match(Set dst (VectorStoreMask src size));
format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
ins_encode %{
@@ -23116,7 +23120,7 @@ instruct vstoreMask1B(vec dst, vec src, immI_1 size) %{
%}
instruct vstoreMask2B(vec dst, vec src, vec xtmp, immI_2 size) %{
- predicate(Matcher::vector_length(n) <= 16 && n->in(1)->bottom_type()->isa_vectmask() == nullptr);
+ predicate(Matcher::vector_length(n) <= 16 && n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
match(Set dst (VectorStoreMask src size));
effect(TEMP_DEF dst, TEMP xtmp);
format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
@@ -23139,7 +23143,7 @@ instruct vstoreMask2B(vec dst, vec src, vec xtmp, immI_2 size) %{
%}
instruct vstoreMask4B(vec dst, vec src, vec xtmp, immI_4 size) %{
- predicate(UseAVX <= 2 && Matcher::vector_length(n) <= 8 && n->in(1)->bottom_type()->isa_vectmask() == nullptr);
+ predicate(UseAVX <= 2 && Matcher::vector_length(n) <= 8 && n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
match(Set dst (VectorStoreMask src size));
format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
effect(TEMP_DEF dst, TEMP xtmp);
@@ -23199,7 +23203,7 @@ instruct storeMask8B_avx(vec dst, vec src, immI_8 size, vec vtmp) %{
%}
instruct vstoreMask4B_evex_novectmask(vec dst, vec src, immI_4 size) %{
- predicate(UseAVX > 2 && n->in(1)->bottom_type()->isa_vectmask() == nullptr);
+ predicate(UseAVX > 2 && n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
match(Set dst (VectorStoreMask src size));
format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
ins_encode %{
@@ -23215,7 +23219,7 @@ instruct vstoreMask4B_evex_novectmask(vec dst, vec src, immI_4 size) %{
%}
instruct vstoreMask8B_evex_novectmask(vec dst, vec src, immI_8 size) %{
- predicate(UseAVX > 2 && n->in(1)->bottom_type()->isa_vectmask() == nullptr);
+ predicate(UseAVX > 2 && n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
match(Set dst (VectorStoreMask src size));
format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
ins_encode %{
@@ -23231,7 +23235,7 @@ instruct vstoreMask8B_evex_novectmask(vec dst, vec src, immI_8 size) %{
%}
instruct vstoreMask_evex_vectmask(vec dst, kReg mask, immI size) %{
- predicate(n->in(1)->bottom_type()->isa_vectmask() && !VM_Version::supports_avx512vlbw());
+ predicate(n->in(1)->bottom_type()->isa_pvectmask() && !VM_Version::supports_avx512vlbw());
match(Set dst (VectorStoreMask mask size));
effect(TEMP_DEF dst);
format %{ "vector_store_mask $dst, $mask \t! elem size is $size byte[s]" %}
@@ -23245,7 +23249,7 @@ instruct vstoreMask_evex_vectmask(vec dst, kReg mask, immI size) %{
%}
instruct vstoreMask_evex(vec dst, kReg mask, immI size) %{
- predicate(n->in(1)->bottom_type()->isa_vectmask() && VM_Version::supports_avx512vlbw());
+ predicate(n->in(1)->bottom_type()->isa_pvectmask() && VM_Version::supports_avx512vlbw());
match(Set dst (VectorStoreMask mask size));
effect(TEMP_DEF dst);
format %{ "vector_store_mask $dst, $mask \t! elem size is $size byte[s]" %}
@@ -23866,7 +23870,7 @@ instruct vprorate(vec dst, vec src, vec shift) %{
// ---------------------------------- Masked Operations ------------------------------------
instruct vmasked_load_avx_non_subword(vec dst, memory mem, vec mask) %{
- predicate(!n->in(3)->bottom_type()->isa_vectmask());
+ predicate(!n->in(3)->bottom_type()->isa_pvectmask());
match(Set dst (LoadVectorMasked mem mask));
format %{ "vector_masked_load $dst, $mem, $mask \t! vector masked copy" %}
ins_encode %{
@@ -23879,7 +23883,7 @@ instruct vmasked_load_avx_non_subword(vec dst, memory mem, vec mask) %{
instruct vmasked_load_evex(vec dst, memory mem, kReg mask) %{
- predicate(n->in(3)->bottom_type()->isa_vectmask());
+ predicate(n->in(3)->bottom_type()->isa_pvectmask());
match(Set dst (LoadVectorMasked mem mask));
format %{ "vector_masked_load $dst, $mem, $mask \t! vector masked copy" %}
ins_encode %{
@@ -23891,7 +23895,7 @@ instruct vmasked_load_evex(vec dst, memory mem, kReg mask) %{
%}
instruct vmasked_store_avx_non_subword(memory mem, vec src, vec mask) %{
- predicate(!n->in(3)->in(2)->bottom_type()->isa_vectmask());
+ predicate(!n->in(3)->in(2)->bottom_type()->isa_pvectmask());
match(Set mem (StoreVectorMasked mem (Binary src mask)));
format %{ "vector_masked_store $mem, $src, $mask \t! vector masked store" %}
ins_encode %{
@@ -23904,7 +23908,7 @@ instruct vmasked_store_avx_non_subword(memory mem, vec src, vec mask) %{
%}
instruct vmasked_store_evex(memory mem, vec src, kReg mask) %{
- predicate(n->in(3)->in(2)->bottom_type()->isa_vectmask());
+ predicate(n->in(3)->in(2)->bottom_type()->isa_pvectmask());
match(Set mem (StoreVectorMasked mem (Binary src mask)));
format %{ "vector_masked_store $mem, $src, $mask \t! vector masked store" %}
ins_encode %{
@@ -23983,7 +23987,7 @@ instruct vmask_gen_imm(kReg dst, immL len, rRegL temp) %{
%}
instruct vmask_tolong_evex(rRegL dst, kReg mask, rFlagsReg cr) %{
- predicate(n->in(1)->bottom_type()->isa_vectmask());
+ predicate(n->in(1)->bottom_type()->isa_pvectmask());
match(Set dst (VectorMaskToLong mask));
effect(TEMP dst, KILL cr);
format %{ "vector_tolong_evex $dst, $mask \t! vector mask tolong" %}
@@ -24000,7 +24004,7 @@ instruct vmask_tolong_evex(rRegL dst, kReg mask, rFlagsReg cr) %{
%}
instruct vmask_tolong_bool(rRegL dst, vec mask, vec xtmp, rFlagsReg cr) %{
- predicate(n->in(1)->bottom_type()->isa_vectmask() == nullptr);
+ predicate(n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
match(Set dst (VectorMaskToLong mask));
format %{ "vector_tolong_bool $dst, $mask \t! using $xtmp as TEMP" %}
effect(TEMP_DEF dst, TEMP xtmp, KILL cr);
@@ -24016,7 +24020,7 @@ instruct vmask_tolong_bool(rRegL dst, vec mask, vec xtmp, rFlagsReg cr) %{
%}
instruct vmask_tolong_avx(rRegL dst, vec mask, immI size, vec xtmp, rFlagsReg cr) %{
- predicate(n->in(1)->in(1)->bottom_type()->isa_vectmask() == nullptr);
+ predicate(n->in(1)->in(1)->bottom_type()->isa_pvectmask() == nullptr);
match(Set dst (VectorMaskToLong (VectorStoreMask mask size)));
format %{ "vector_tolong_avx $dst, $mask \t! using $xtmp as TEMP" %}
effect(TEMP_DEF dst, TEMP xtmp, KILL cr);
@@ -24032,7 +24036,7 @@ instruct vmask_tolong_avx(rRegL dst, vec mask, immI size, vec xtmp, rFlagsReg cr
%}
instruct vmask_truecount_evex(rRegI dst, kReg mask, rRegL tmp, rFlagsReg cr) %{
- predicate(n->in(1)->bottom_type()->isa_vectmask());
+ predicate(n->in(1)->bottom_type()->isa_pvectmask());
match(Set dst (VectorMaskTrueCount mask));
effect(TEMP_DEF dst, TEMP tmp, KILL cr);
format %{ "vector_truecount_evex $dst, $mask \t! using $tmp as TEMP" %}
@@ -24049,7 +24053,7 @@ instruct vmask_truecount_evex(rRegI dst, kReg mask, rRegL tmp, rFlagsReg cr) %{
%}
instruct vmask_truecount_bool(rRegI dst, vec mask, rRegL tmp, vec xtmp, rFlagsReg cr) %{
- predicate(n->in(1)->bottom_type()->isa_vectmask() == nullptr);
+ predicate(n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
match(Set dst (VectorMaskTrueCount mask));
effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
format %{ "vector_truecount_bool $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
@@ -24065,7 +24069,7 @@ instruct vmask_truecount_bool(rRegI dst, vec mask, rRegL tmp, vec xtmp, rFlagsRe
%}
instruct vmask_truecount_avx(rRegI dst, vec mask, immI size, rRegL tmp, vec xtmp, rFlagsReg cr) %{
- predicate(n->in(1)->in(1)->bottom_type()->isa_vectmask() == nullptr);
+ predicate(n->in(1)->in(1)->bottom_type()->isa_pvectmask() == nullptr);
match(Set dst (VectorMaskTrueCount (VectorStoreMask mask size)));
effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
format %{ "vector_truecount_avx $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
@@ -24081,7 +24085,7 @@ instruct vmask_truecount_avx(rRegI dst, vec mask, immI size, rRegL tmp, vec xtmp
%}
instruct vmask_first_or_last_true_evex(rRegI dst, kReg mask, rRegL tmp, rFlagsReg cr) %{
- predicate(n->in(1)->bottom_type()->isa_vectmask());
+ predicate(n->in(1)->bottom_type()->isa_pvectmask());
match(Set dst (VectorMaskFirstTrue mask));
match(Set dst (VectorMaskLastTrue mask));
effect(TEMP_DEF dst, TEMP tmp, KILL cr);
@@ -24099,7 +24103,7 @@ instruct vmask_first_or_last_true_evex(rRegI dst, kReg mask, rRegL tmp, rFlagsRe
%}
instruct vmask_first_or_last_true_bool(rRegI dst, vec mask, rRegL tmp, vec xtmp, rFlagsReg cr) %{
- predicate(n->in(1)->bottom_type()->isa_vectmask() == nullptr);
+ predicate(n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
match(Set dst (VectorMaskFirstTrue mask));
match(Set dst (VectorMaskLastTrue mask));
effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
@@ -24116,7 +24120,7 @@ instruct vmask_first_or_last_true_bool(rRegI dst, vec mask, rRegL tmp, vec xtmp,
%}
instruct vmask_first_or_last_true_avx(rRegI dst, vec mask, immI size, rRegL tmp, vec xtmp, rFlagsReg cr) %{
- predicate(n->in(1)->in(1)->bottom_type()->isa_vectmask() == nullptr);
+ predicate(n->in(1)->in(1)->bottom_type()->isa_pvectmask() == nullptr);
match(Set dst (VectorMaskFirstTrue (VectorStoreMask mask size)));
match(Set dst (VectorMaskLastTrue (VectorStoreMask mask size)));
effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
@@ -24168,7 +24172,7 @@ instruct vcompress_mask_reg_evex(kReg dst, kReg mask, rRegL rtmp1, rRegL rtmp2,
effect(TEMP rtmp1, TEMP rtmp2, KILL cr);
format %{ "mask_compress_evex $dst, $mask\t! using $rtmp1 and $rtmp2 as TEMP" %}
ins_encode %{
- assert(this->in(1)->bottom_type()->isa_vectmask(), "");
+ assert(this->in(1)->bottom_type()->isa_pvectmask(), "");
int mask_len = Matcher::vector_length(this);
__ vector_mask_compress($dst$$KRegister, $mask$$KRegister, $rtmp1$$Register, $rtmp2$$Register, mask_len);
%}
@@ -24838,7 +24842,7 @@ instruct evcmp_masked(kReg dst, vec src1, vec src2, immI8 cond, kReg mask) %{
match(Set dst (VectorMaskCmp (Binary src1 src2) (Binary cond mask)));
format %{ "vcmp_masked $dst, $src1, $src2, $cond, $mask" %}
ins_encode %{
- assert(bottom_type()->isa_vectmask(), "TypeVectMask expected");
+ assert(bottom_type()->isa_pvectmask(), "TypePVectMask expected");
int vlen_enc = vector_length_encoding(this, $src1);
BasicType src1_elem_bt = Matcher::vector_element_basic_type(this, $src1);
@@ -24920,11 +24924,11 @@ instruct mask_not_imm(kReg dst, kReg src, immI_M1 cnt) %{
ins_pipe( pipe_slow );
%}
-instruct long_to_maskLE8_avx(vec dst, rRegL src, rRegL rtmp1, rRegL rtmp2, vec xtmp) %{
- predicate(n->bottom_type()->isa_vectmask() == nullptr && Matcher::vector_length(n) <= 8);
+instruct long_to_maskLE8_avx(vec dst, rRegL src, rRegL rtmp1, rRegL rtmp2) %{
+ predicate(n->bottom_type()->isa_pvectmask() == nullptr && Matcher::vector_length(n) <= 8);
match(Set dst (VectorLongToMask src));
- effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, TEMP xtmp);
- format %{ "long_to_mask_avx $dst, $src\t! using $rtmp1, $rtmp2, $xtmp as TEMP" %}
+ effect(TEMP dst, TEMP rtmp1, TEMP rtmp2);
+ format %{ "long_to_mask_avx $dst, $src\t! using $rtmp1, $rtmp2" %}
ins_encode %{
int mask_len = Matcher::vector_length(this);
int vec_enc = vector_length_encoding(mask_len);
@@ -24936,7 +24940,7 @@ instruct long_to_maskLE8_avx(vec dst, rRegL src, rRegL rtmp1, rRegL rtmp2, vec x
instruct long_to_maskGT8_avx(vec dst, rRegL src, rRegL rtmp1, rRegL rtmp2, vec xtmp1, rFlagsReg cr) %{
- predicate(n->bottom_type()->isa_vectmask() == nullptr && Matcher::vector_length(n) > 8);
+ predicate(n->bottom_type()->isa_pvectmask() == nullptr && Matcher::vector_length(n) > 8);
match(Set dst (VectorLongToMask src));
effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, TEMP xtmp1, KILL cr);
format %{ "long_to_mask_avx $dst, $src\t! using $rtmp1, $rtmp2, $xtmp1, as TEMP" %}
@@ -24951,7 +24955,7 @@ instruct long_to_maskGT8_avx(vec dst, rRegL src, rRegL rtmp1, rRegL rtmp2, vec x
%}
instruct long_to_mask_evex(kReg dst, rRegL src) %{
- predicate(n->bottom_type()->isa_vectmask());
+ predicate(n->bottom_type()->isa_pvectmask());
match(Set dst (VectorLongToMask src));
format %{ "long_to_mask_evex $dst, $src\t!" %}
ins_encode %{
@@ -25317,6 +25321,7 @@ instruct reinterpretHF2S(rRegI dst, regF src)
format %{ "evmovw $dst, $src" %}
ins_encode %{
__ evmovw($dst$$Register, $src$$XMMRegister);
+ __ narrow_subword_type($dst$$Register, T_SHORT);
%}
ins_pipe(pipe_slow);
%}
diff --git a/src/hotspot/os/aix/os_aix.cpp b/src/hotspot/os/aix/os_aix.cpp
index 3cad24d388c..32d845b2b6d 100644
--- a/src/hotspot/os/aix/os_aix.cpp
+++ b/src/hotspot/os/aix/os_aix.cpp
@@ -578,13 +578,13 @@ void os::init_system_properties_values() {
char *ld_library_path = NEW_C_HEAP_ARRAY(char, pathsize, mtInternal);
os::snprintf_checked(ld_library_path, pathsize, "%s%s" DEFAULT_LIBPATH, v, v_colon);
Arguments::set_library_path(ld_library_path);
- FREE_C_HEAP_ARRAY(char, ld_library_path);
+ FREE_C_HEAP_ARRAY(ld_library_path);
// Extensions directories.
os::snprintf_checked(buf, bufsize, "%s" EXTENSIONS_DIR, Arguments::get_java_home());
Arguments::set_ext_dirs(buf);
- FREE_C_HEAP_ARRAY(char, buf);
+ FREE_C_HEAP_ARRAY(buf);
#undef DEFAULT_LIBPATH
#undef EXTENSIONS_DIR
diff --git a/src/hotspot/os/aix/os_perf_aix.cpp b/src/hotspot/os/aix/os_perf_aix.cpp
index cbf78083483..3668ac6ba3f 100644
--- a/src/hotspot/os/aix/os_perf_aix.cpp
+++ b/src/hotspot/os/aix/os_perf_aix.cpp
@@ -258,10 +258,10 @@ bool CPUPerformanceInterface::CPUPerformance::initialize() {
CPUPerformanceInterface::CPUPerformance::~CPUPerformance() {
if (_lcpu_names) {
- FREE_C_HEAP_ARRAY(perfstat_id_t, _lcpu_names);
+ FREE_C_HEAP_ARRAY(_lcpu_names);
}
if (_prev_ticks) {
- FREE_C_HEAP_ARRAY(cpu_tick_store_t, _prev_ticks);
+ FREE_C_HEAP_ARRAY(_prev_ticks);
}
}
@@ -511,12 +511,12 @@ CPUInformationInterface::~CPUInformationInterface() {
if (_cpu_info != nullptr) {
if (_cpu_info->cpu_name() != nullptr) {
const char* cpu_name = _cpu_info->cpu_name();
- FREE_C_HEAP_ARRAY(char, cpu_name);
+ FREE_C_HEAP_ARRAY(cpu_name);
_cpu_info->set_cpu_name(nullptr);
}
if (_cpu_info->cpu_description() != nullptr) {
const char* cpu_desc = _cpu_info->cpu_description();
- FREE_C_HEAP_ARRAY(char, cpu_desc);
+ FREE_C_HEAP_ARRAY(cpu_desc);
_cpu_info->set_cpu_description(nullptr);
}
delete _cpu_info;
@@ -576,7 +576,7 @@ int NetworkPerformanceInterface::NetworkPerformance::network_utilization(Network
// check for error
if (n_records < 0) {
- FREE_C_HEAP_ARRAY(perfstat_netinterface_t, net_stats);
+ FREE_C_HEAP_ARRAY(net_stats);
return OS_ERR;
}
@@ -593,7 +593,7 @@ int NetworkPerformanceInterface::NetworkPerformance::network_utilization(Network
*network_interfaces = new_interface;
}
- FREE_C_HEAP_ARRAY(perfstat_netinterface_t, net_stats);
+ FREE_C_HEAP_ARRAY(net_stats);
return OS_OK;
}
diff --git a/src/hotspot/os/bsd/os_bsd.cpp b/src/hotspot/os/bsd/os_bsd.cpp
index a4d9a2197a5..fc5b9952f78 100644
--- a/src/hotspot/os/bsd/os_bsd.cpp
+++ b/src/hotspot/os/bsd/os_bsd.cpp
@@ -106,6 +106,14 @@
#include
#include
#include
+
+ // needed by current_stack_base_and_size() workaround for Mavericks
+ #define DEFAULT_MAIN_THREAD_STACK_PAGES 2048
+ #define OS_X_10_9_0_KERNEL_MAJOR_VERSION 13
+#endif
+
+#if !defined(__APPLE__) && !defined(__NetBSD__)
+ #include
#endif
#ifndef MAP_ANONYMOUS
@@ -444,14 +452,14 @@ void os::init_system_properties_values() {
char *ld_library_path = NEW_C_HEAP_ARRAY(char, ld_library_path_size, mtInternal);
os::snprintf_checked(ld_library_path, ld_library_path_size, "%s%s" SYS_EXT_DIR "/lib/%s:" DEFAULT_LIBPATH, v, v_colon, cpu_arch);
Arguments::set_library_path(ld_library_path);
- FREE_C_HEAP_ARRAY(char, ld_library_path);
+ FREE_C_HEAP_ARRAY(ld_library_path);
}
// Extensions directories.
os::snprintf_checked(buf, bufsize, "%s" EXTENSIONS_DIR ":" SYS_EXT_DIR EXTENSIONS_DIR, Arguments::get_java_home());
Arguments::set_ext_dirs(buf);
- FREE_C_HEAP_ARRAY(char, buf);
+ FREE_C_HEAP_ARRAY(buf);
#else // __APPLE__
@@ -538,7 +546,7 @@ void os::init_system_properties_values() {
os::snprintf_checked(ld_library_path, ld_library_path_size, "%s%s%s%s%s" SYS_EXTENSIONS_DIR ":" SYS_EXTENSIONS_DIRS ":.",
v, v_colon, l, l_colon, user_home_dir);
Arguments::set_library_path(ld_library_path);
- FREE_C_HEAP_ARRAY(char, ld_library_path);
+ FREE_C_HEAP_ARRAY(ld_library_path);
}
// Extensions directories.
@@ -550,7 +558,7 @@ void os::init_system_properties_values() {
user_home_dir, Arguments::get_java_home());
Arguments::set_ext_dirs(buf);
- FREE_C_HEAP_ARRAY(char, buf);
+ FREE_C_HEAP_ARRAY(buf);
#undef SYS_EXTENSIONS_DIR
#undef SYS_EXTENSIONS_DIRS
@@ -1115,7 +1123,7 @@ bool os::dll_address_to_library_name(address addr, char* buf,
// in case of error it checks if .dll/.so was built for the
// same architecture as Hotspot is running on
-void *os::Bsd::dlopen_helper(const char *filename, int mode, char *ebuf, int ebuflen) {
+static void *dlopen_helper(const char *filename, char *ebuf, int ebuflen) {
bool ieee_handling = IEEE_subnormal_handling_OK();
if (!ieee_handling) {
Events::log_dll_message(nullptr, "IEEE subnormal handling check failed before loading %s", filename);
@@ -1199,7 +1207,7 @@ void * os::dll_load(const char *filename, char *ebuf, int ebuflen) {
log_info(os)("attempting shared library load of %s", filename);
- return os::Bsd::dlopen_helper(filename, RTLD_LAZY, ebuf, ebuflen);
+ return dlopen_helper(filename, ebuf, ebuflen);
}
#else
void * os::dll_load(const char *filename, char *ebuf, int ebuflen) {
@@ -1210,7 +1218,7 @@ void * os::dll_load(const char *filename, char *ebuf, int ebuflen) {
log_info(os)("attempting shared library load of %s", filename);
void* result;
- result = os::Bsd::dlopen_helper(filename, RTLD_LAZY, ebuf, ebuflen);
+ result = dlopen_helper(filename, ebuf, ebuflen);
if (result != nullptr) {
return result;
}
@@ -2545,6 +2553,106 @@ bool os::start_debugging(char *buf, int buflen) {
return yes;
}
+// Java thread:
+//
+// Low memory addresses
+// +------------------------+
+// | |\ Java thread created by VM does not have glibc
+// | glibc guard page | - guard, attached Java thread usually has
+// | |/ 1 glibc guard page.
+// P1 +------------------------+ Thread::stack_base() - Thread::stack_size()
+// | |\
+// | HotSpot Guard Pages | - red, yellow and reserved pages
+// | |/
+// +------------------------+ StackOverflow::stack_reserved_zone_base()
+// | |\
+// | Normal Stack | -
+// | |/
+// P2 +------------------------+ Thread::stack_base()
+//
+// Non-Java thread:
+//
+// Low memory addresses
+// +------------------------+
+// | |\
+// | glibc guard page | - usually 1 page
+// | |/
+// P1 +------------------------+ Thread::stack_base() - Thread::stack_size()
+// | |\
+// | Normal Stack | -
+// | |/
+// P2 +------------------------+ Thread::stack_base()
+//
+// ** P1 (aka bottom) and size are the address and stack size
+// returned from pthread_attr_getstack().
+// ** P2 (aka stack top or base) = P1 + size
+
+void os::current_stack_base_and_size(address* base, size_t* size) {
+ address bottom;
+#ifdef __APPLE__
+ pthread_t self = pthread_self();
+ *base = (address) pthread_get_stackaddr_np(self);
+ *size = pthread_get_stacksize_np(self);
+# ifdef __x86_64__
+ // workaround for OS X 10.9.0 (Mavericks)
+ // pthread_get_stacksize_np returns 128 pages even though the actual size is 2048 pages
+ if (pthread_main_np() == 1) {
+ // At least on Mac OS 10.12 we have observed stack sizes not aligned
+ // to pages boundaries. This can be provoked by e.g. setrlimit() (ulimit -s xxxx in the
+ // shell). Apparently Mac OS actually rounds upwards to next multiple of page size,
+ // however, we round downwards here to be on the safe side.
+ *size = align_down(*size, getpagesize());
+
+ if ((*size) < (DEFAULT_MAIN_THREAD_STACK_PAGES * (size_t)getpagesize())) {
+ char kern_osrelease[256];
+ size_t kern_osrelease_size = sizeof(kern_osrelease);
+ int ret = sysctlbyname("kern.osrelease", kern_osrelease, &kern_osrelease_size, nullptr, 0);
+ if (ret == 0) {
+ // get the major number, atoi will ignore the minor amd micro portions of the version string
+ if (atoi(kern_osrelease) >= OS_X_10_9_0_KERNEL_MAJOR_VERSION) {
+ *size = (DEFAULT_MAIN_THREAD_STACK_PAGES*getpagesize());
+ }
+ }
+ }
+ }
+# endif
+ bottom = *base - *size;
+#elif defined(__OpenBSD__)
+ stack_t ss;
+ int rslt = pthread_stackseg_np(pthread_self(), &ss);
+
+ if (rslt != 0)
+ fatal("pthread_stackseg_np failed with error = %d", rslt);
+
+ *base = (address) ss.ss_sp;
+ *size = ss.ss_size;
+ bottom = *base - *size;
+#else
+ pthread_attr_t attr;
+
+ int rslt = pthread_attr_init(&attr);
+
+ // JVM needs to know exact stack location, abort if it fails
+ if (rslt != 0)
+ fatal("pthread_attr_init failed with error = %d", rslt);
+
+ rslt = pthread_attr_get_np(pthread_self(), &attr);
+
+ if (rslt != 0)
+ fatal("pthread_attr_get_np failed with error = %d", rslt);
+
+ if (pthread_attr_getstackaddr(&attr, (void **)&bottom) != 0 ||
+ pthread_attr_getstacksize(&attr, size) != 0) {
+ fatal("Can not locate current stack attributes!");
+ }
+
+ *base = bottom + *size;
+
+ pthread_attr_destroy(&attr);
+#endif
+ assert(os::current_stack_pointer() >= bottom &&
+ os::current_stack_pointer() < *base, "just checking");
+}
void os::print_memory_mappings(char* addr, size_t bytes, outputStream* st) {}
#if INCLUDE_JFR
diff --git a/src/hotspot/os/bsd/os_bsd.hpp b/src/hotspot/os/bsd/os_bsd.hpp
index e87a680b2d2..91fcb090f50 100644
--- a/src/hotspot/os/bsd/os_bsd.hpp
+++ b/src/hotspot/os/bsd/os_bsd.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -76,8 +76,6 @@ class os::Bsd {
// Real-time clock functions
static void clock_init(void);
- static void *dlopen_helper(const char *path, int mode, char *ebuf, int ebuflen);
-
// Stack repair handling
// none present
@@ -105,6 +103,7 @@ class os::Bsd {
static void set_numa_tonode_memory(numa_tonode_memory_func_t func) { _numa_tonode_memory = func; }
static void set_numa_interleave_memory(numa_interleave_memory_func_t func) { _numa_interleave_memory = func; }
static void set_numa_all_nodes(unsigned long* ptr) { _numa_all_nodes = ptr; }
+
public:
static int sched_getcpu() { return _sched_getcpu != nullptr ? _sched_getcpu() : -1; }
static int numa_node_to_cpus(int node, unsigned long *buffer, int bufferlen) {
diff --git a/src/hotspot/os/bsd/os_perf_bsd.cpp b/src/hotspot/os/bsd/os_perf_bsd.cpp
index 78d9519c3a7..47fe3a0d7e9 100644
--- a/src/hotspot/os/bsd/os_perf_bsd.cpp
+++ b/src/hotspot/os/bsd/os_perf_bsd.cpp
@@ -301,7 +301,7 @@ int SystemProcessInterface::SystemProcesses::system_processes(SystemProcess** sy
pids_bytes = proc_listpids(PROC_ALL_PIDS, 0, pids, pids_bytes);
if (pids_bytes <= 0) {
// couldn't fit buffer, retry.
- FREE_RESOURCE_ARRAY(pid_t, pids, pid_count);
+ FREE_RESOURCE_ARRAY(pids, pid_count);
pids = nullptr;
try_count++;
if (try_count > 3) {
@@ -381,12 +381,12 @@ CPUInformationInterface::~CPUInformationInterface() {
if (_cpu_info != nullptr) {
if (_cpu_info->cpu_name() != nullptr) {
const char* cpu_name = _cpu_info->cpu_name();
- FREE_C_HEAP_ARRAY(char, cpu_name);
+ FREE_C_HEAP_ARRAY(cpu_name);
_cpu_info->set_cpu_name(nullptr);
}
if (_cpu_info->cpu_description() != nullptr) {
const char* cpu_desc = _cpu_info->cpu_description();
- FREE_C_HEAP_ARRAY(char, cpu_desc);
+ FREE_C_HEAP_ARRAY(cpu_desc);
_cpu_info->set_cpu_description(nullptr);
}
delete _cpu_info;
diff --git a/src/hotspot/os/linux/cgroupSubsystem_linux.cpp b/src/hotspot/os/linux/cgroupSubsystem_linux.cpp
index 4a2d75ecdf3..1c183a9bbab 100644
--- a/src/hotspot/os/linux/cgroupSubsystem_linux.cpp
+++ b/src/hotspot/os/linux/cgroupSubsystem_linux.cpp
@@ -649,7 +649,7 @@ bool CgroupSubsystem::active_processor_count(int (*cpu_bound_func)(), double& va
return true;
}
- int cpu_count = cpu_bound_func();
+ double cpu_count = static_cast(cpu_bound_func());
double result = -1;
if (!CgroupUtil::processor_count(contrl->controller(), cpu_count, result)) {
return false;
diff --git a/src/hotspot/os/linux/cgroupUtil_linux.cpp b/src/hotspot/os/linux/cgroupUtil_linux.cpp
index f166f6cd5e4..1f7775acfc3 100644
--- a/src/hotspot/os/linux/cgroupUtil_linux.cpp
+++ b/src/hotspot/os/linux/cgroupUtil_linux.cpp
@@ -25,7 +25,7 @@
#include "cgroupUtil_linux.hpp"
-bool CgroupUtil::processor_count(CgroupCpuController* cpu_ctrl, int upper_bound, double& value) {
+bool CgroupUtil::processor_count(CgroupCpuController* cpu_ctrl, double upper_bound, double& value) {
assert(upper_bound > 0, "upper bound of cpus must be positive");
int quota = -1;
int period = -1;
@@ -68,8 +68,8 @@ physical_memory_size_type CgroupUtil::get_updated_mem_limit(CgroupMemoryControll
// Get an updated cpu limit. The return value is strictly less than or equal to the
// passed in 'lowest' value.
double CgroupUtil::get_updated_cpu_limit(CgroupCpuController* cpu,
- int lowest,
- int upper_bound) {
+ double lowest,
+ double upper_bound) {
assert(lowest > 0 && lowest <= upper_bound, "invariant");
double cpu_limit_val = -1;
if (CgroupUtil::processor_count(cpu, upper_bound, cpu_limit_val) && cpu_limit_val != upper_bound) {
@@ -145,7 +145,7 @@ void CgroupUtil::adjust_controller(CgroupMemoryController* mem, physical_memory_
os::free(limit_cg_path);
}
-void CgroupUtil::adjust_controller(CgroupCpuController* cpu, int upper_bound) {
+void CgroupUtil::adjust_controller(CgroupCpuController* cpu, double upper_bound) {
assert(cpu->cgroup_path() != nullptr, "invariant");
if (strstr(cpu->cgroup_path(), "../") != nullptr) {
log_warning(os, container)("Cgroup cpu controller path at '%s' seems to have moved "
@@ -163,9 +163,9 @@ void CgroupUtil::adjust_controller(CgroupCpuController* cpu, int upper_bound) {
char* cg_path = os::strdup(orig);
char* last_slash;
assert(cg_path[0] == '/', "cgroup path must start with '/'");
- int lowest_limit = upper_bound;
+ double lowest_limit = upper_bound;
double cpus = get_updated_cpu_limit(cpu, lowest_limit, upper_bound);
- int orig_limit = lowest_limit != upper_bound ? lowest_limit : upper_bound;
+ double orig_limit = lowest_limit != upper_bound ? lowest_limit : upper_bound;
char* limit_cg_path = nullptr;
while ((last_slash = strrchr(cg_path, '/')) != cg_path) {
*last_slash = '\0'; // strip path
@@ -193,10 +193,10 @@ void CgroupUtil::adjust_controller(CgroupCpuController* cpu, int upper_bound) {
assert(limit_cg_path != nullptr, "limit path must be set");
cpu->set_subsystem_path(limit_cg_path);
log_trace(os, container)("Adjusted controller path for cpu to: %s. "
- "Lowest limit was: %d",
+ "Lowest limit was: %.2f",
cpu->subsystem_path(), lowest_limit);
} else {
- log_trace(os, container)("Lowest limit was: %d", lowest_limit);
+ log_trace(os, container)("Lowest limit was: %.2f", lowest_limit);
log_trace(os, container)("No lower limit found for cpu in hierarchy %s, "
"adjusting to original path %s",
cpu->mount_point(), orig);
diff --git a/src/hotspot/os/linux/cgroupUtil_linux.hpp b/src/hotspot/os/linux/cgroupUtil_linux.hpp
index 68585c22c2d..c5df7e8efa9 100644
--- a/src/hotspot/os/linux/cgroupUtil_linux.hpp
+++ b/src/hotspot/os/linux/cgroupUtil_linux.hpp
@@ -32,18 +32,18 @@
class CgroupUtil: AllStatic {
public:
- static bool processor_count(CgroupCpuController* cpu, int upper_bound, double& value);
+ static bool processor_count(CgroupCpuController* cpu, double upper_bound, double& value);
// Given a memory controller, adjust its path to a point in the hierarchy
// that represents the closest memory limit.
static void adjust_controller(CgroupMemoryController* m, physical_memory_size_type upper_bound);
// Given a cpu controller, adjust its path to a point in the hierarchy
// that represents the closest cpu limit.
- static void adjust_controller(CgroupCpuController* c, int upper_bound);
+ static void adjust_controller(CgroupCpuController* c, double upper_bound);
private:
static physical_memory_size_type get_updated_mem_limit(CgroupMemoryController* m,
physical_memory_size_type lowest,
physical_memory_size_type upper_bound);
- static double get_updated_cpu_limit(CgroupCpuController* c, int lowest, int upper_bound);
+ static double get_updated_cpu_limit(CgroupCpuController* c, double lowest, double upper_bound);
};
#endif // CGROUP_UTIL_LINUX_HPP
diff --git a/src/hotspot/os/linux/osContainer_linux.cpp b/src/hotspot/os/linux/osContainer_linux.cpp
index da2cbf381e6..511c7bebff7 100644
--- a/src/hotspot/os/linux/osContainer_linux.cpp
+++ b/src/hotspot/os/linux/osContainer_linux.cpp
@@ -79,9 +79,12 @@ void OSContainer::init() {
* that limits enforced by other means (e.g. systemd slice) are properly
* detected.
*/
- const char *reason;
- bool any_mem_cpu_limit_present = false;
+ const char* reason;
bool controllers_read_only = cgroup_subsystem->is_containerized();
+
+ bool any_mem_limit_present = false;
+ bool cpu_limit_present = false;
+
if (controllers_read_only) {
// in-container case
reason = " because all controllers are mounted read-only (container case)";
@@ -89,19 +92,30 @@ void OSContainer::init() {
// We can be in one of two cases:
// 1.) On a physical Linux system without any limit
// 2.) On a physical Linux system with a limit enforced by other means (like systemd slice)
+
physical_memory_size_type mem_limit_val = value_unlimited;
- (void)memory_limit_in_bytes(mem_limit_val); // discard error and use default
- double host_cpus = os::Linux::active_processor_count();
+ any_mem_limit_present = any_mem_limit_present || (memory_limit_in_bytes(mem_limit_val) &&
+ mem_limit_val != value_unlimited);
+
+ physical_memory_size_type throttle_limit_val = value_unlimited;
+ any_mem_limit_present = any_mem_limit_present || (memory_throttle_limit_in_bytes(throttle_limit_val) &&
+ throttle_limit_val != value_unlimited);
+
+ physical_memory_size_type soft_limit_val = value_unlimited;
+ any_mem_limit_present = any_mem_limit_present || (memory_soft_limit_in_bytes(soft_limit_val) &&
+ (soft_limit_val != value_unlimited && soft_limit_val != 0));
+
+ const double host_cpus = os::Linux::active_processor_count();
double cpus = host_cpus;
- (void)active_processor_count(cpus); // discard error and use default
- any_mem_cpu_limit_present = mem_limit_val != value_unlimited || host_cpus != cpus;
- if (any_mem_cpu_limit_present) {
+ cpu_limit_present = active_processor_count(cpus) && host_cpus != cpus;
+
+ if (any_mem_limit_present || cpu_limit_present) {
reason = " because either a cpu or a memory limit is present";
} else {
reason = " because no cpu or memory limit is present";
}
}
- _is_containerized = controllers_read_only || any_mem_cpu_limit_present;
+ _is_containerized = controllers_read_only || any_mem_limit_present || cpu_limit_present;
log_debug(os, container)("OSContainer::init: is_containerized() = %s%s",
_is_containerized ? "true" : "false",
reason);
diff --git a/src/hotspot/os/linux/os_linux.cpp b/src/hotspot/os/linux/os_linux.cpp
index a87c0ab33fa..6927f5108ac 100644
--- a/src/hotspot/os/linux/os_linux.cpp
+++ b/src/hotspot/os/linux/os_linux.cpp
@@ -710,14 +710,14 @@ void os::init_system_properties_values() {
char *ld_library_path = NEW_C_HEAP_ARRAY(char, pathsize, mtInternal);
os::snprintf_checked(ld_library_path, pathsize, "%s%s" SYS_EXT_DIR "/lib:" DEFAULT_LIBPATH, v, v_colon);
Arguments::set_library_path(ld_library_path);
- FREE_C_HEAP_ARRAY(char, ld_library_path);
+ FREE_C_HEAP_ARRAY(ld_library_path);
}
// Extensions directories.
os::snprintf_checked(buf, bufsize, "%s" EXTENSIONS_DIR ":" SYS_EXT_DIR EXTENSIONS_DIR, Arguments::get_java_home());
Arguments::set_ext_dirs(buf);
- FREE_C_HEAP_ARRAY(char, buf);
+ FREE_C_HEAP_ARRAY(buf);
#undef DEFAULT_LIBPATH
#undef SYS_EXT_DIR
@@ -3435,7 +3435,7 @@ void os::Linux::rebuild_cpu_to_node_map() {
}
}
}
- FREE_C_HEAP_ARRAY(unsigned long, cpu_map);
+ FREE_C_HEAP_ARRAY(cpu_map);
}
int os::Linux::numa_node_to_cpus(int node, unsigned long *buffer, int bufferlen) {
diff --git a/src/hotspot/os/linux/os_perf_linux.cpp b/src/hotspot/os/linux/os_perf_linux.cpp
index 9f91f3b4c0d..c0e863ed2a2 100644
--- a/src/hotspot/os/linux/os_perf_linux.cpp
+++ b/src/hotspot/os/linux/os_perf_linux.cpp
@@ -545,7 +545,7 @@ bool CPUPerformanceInterface::CPUPerformance::initialize() {
CPUPerformanceInterface::CPUPerformance::~CPUPerformance() {
if (_counters.cpus != nullptr) {
- FREE_C_HEAP_ARRAY(char, _counters.cpus);
+ FREE_C_HEAP_ARRAY(_counters.cpus);
}
}
@@ -811,7 +811,7 @@ int SystemProcessInterface::SystemProcesses::ProcessIterator::current(SystemProc
cmdline = get_cmdline();
if (cmdline != nullptr) {
process_info->set_command_line(allocate_string(cmdline));
- FREE_C_HEAP_ARRAY(char, cmdline);
+ FREE_C_HEAP_ARRAY(cmdline);
}
return OS_OK;
@@ -937,12 +937,12 @@ CPUInformationInterface::~CPUInformationInterface() {
if (_cpu_info != nullptr) {
if (_cpu_info->cpu_name() != nullptr) {
const char* cpu_name = _cpu_info->cpu_name();
- FREE_C_HEAP_ARRAY(char, cpu_name);
+ FREE_C_HEAP_ARRAY(cpu_name);
_cpu_info->set_cpu_name(nullptr);
}
if (_cpu_info->cpu_description() != nullptr) {
const char* cpu_desc = _cpu_info->cpu_description();
- FREE_C_HEAP_ARRAY(char, cpu_desc);
+ FREE_C_HEAP_ARRAY(cpu_desc);
_cpu_info->set_cpu_description(nullptr);
}
delete _cpu_info;
diff --git a/src/hotspot/os/linux/procMapsParser.cpp b/src/hotspot/os/linux/procMapsParser.cpp
index 0663cae61f3..00675683e34 100644
--- a/src/hotspot/os/linux/procMapsParser.cpp
+++ b/src/hotspot/os/linux/procMapsParser.cpp
@@ -45,7 +45,7 @@ ProcSmapsParser::ProcSmapsParser(FILE* f) :
}
ProcSmapsParser::~ProcSmapsParser() {
- FREE_C_HEAP_ARRAY(char, _line);
+ FREE_C_HEAP_ARRAY(_line);
}
bool ProcSmapsParser::read_line() {
diff --git a/src/hotspot/os/posix/perfMemory_posix.cpp b/src/hotspot/os/posix/perfMemory_posix.cpp
index c5046797e02..300c86ffc47 100644
--- a/src/hotspot/os/posix/perfMemory_posix.cpp
+++ b/src/hotspot/os/posix/perfMemory_posix.cpp
@@ -118,7 +118,7 @@ static void save_memory_to_file(char* addr, size_t size) {
}
}
}
- FREE_C_HEAP_ARRAY(char, destfile);
+ FREE_C_HEAP_ARRAY(destfile);
}
@@ -483,14 +483,14 @@ static char* get_user_name(uid_t uid) {
p->pw_name == nullptr ? "pw_name = null" : "pw_name zero length");
}
}
- FREE_C_HEAP_ARRAY(char, pwbuf);
+ FREE_C_HEAP_ARRAY(pwbuf);
return nullptr;
}
char* user_name = NEW_C_HEAP_ARRAY(char, strlen(p->pw_name) + 1, mtInternal);
strcpy(user_name, p->pw_name);
- FREE_C_HEAP_ARRAY(char, pwbuf);
+ FREE_C_HEAP_ARRAY(pwbuf);
return user_name;
}
@@ -572,7 +572,7 @@ static char* get_user_name_slow(int vmid, int nspid, TRAPS) {
DIR* subdirp = open_directory_secure(usrdir_name);
if (subdirp == nullptr) {
- FREE_C_HEAP_ARRAY(char, usrdir_name);
+ FREE_C_HEAP_ARRAY(usrdir_name);
continue;
}
@@ -583,7 +583,7 @@ static char* get_user_name_slow(int vmid, int nspid, TRAPS) {
// symlink can be exploited.
//
if (!is_directory_secure(usrdir_name)) {
- FREE_C_HEAP_ARRAY(char, usrdir_name);
+ FREE_C_HEAP_ARRAY(usrdir_name);
os::closedir(subdirp);
continue;
}
@@ -607,13 +607,13 @@ static char* get_user_name_slow(int vmid, int nspid, TRAPS) {
// don't follow symbolic links for the file
RESTARTABLE(::lstat(filename, &statbuf), result);
if (result == OS_ERR) {
- FREE_C_HEAP_ARRAY(char, filename);
+ FREE_C_HEAP_ARRAY(filename);
continue;
}
// skip over files that are not regular files.
if (!S_ISREG(statbuf.st_mode)) {
- FREE_C_HEAP_ARRAY(char, filename);
+ FREE_C_HEAP_ARRAY(filename);
continue;
}
@@ -623,7 +623,7 @@ static char* get_user_name_slow(int vmid, int nspid, TRAPS) {
if (statbuf.st_ctime > oldest_ctime) {
char* user = strchr(dentry->d_name, '_') + 1;
- FREE_C_HEAP_ARRAY(char, oldest_user);
+ FREE_C_HEAP_ARRAY(oldest_user);
oldest_user = NEW_C_HEAP_ARRAY(char, strlen(user)+1, mtInternal);
strcpy(oldest_user, user);
@@ -631,11 +631,11 @@ static char* get_user_name_slow(int vmid, int nspid, TRAPS) {
}
}
- FREE_C_HEAP_ARRAY(char, filename);
+ FREE_C_HEAP_ARRAY(filename);
}
}
os::closedir(subdirp);
- FREE_C_HEAP_ARRAY(char, usrdir_name);
+ FREE_C_HEAP_ARRAY(usrdir_name);
}
os::closedir(tmpdirp);
@@ -1105,11 +1105,11 @@ static char* mmap_create_shared(size_t size) {
log_info(perf, memops)("Trying to open %s/%s", dirname, short_filename);
fd = create_sharedmem_file(dirname, short_filename, size);
- FREE_C_HEAP_ARRAY(char, user_name);
- FREE_C_HEAP_ARRAY(char, dirname);
+ FREE_C_HEAP_ARRAY(user_name);
+ FREE_C_HEAP_ARRAY(dirname);
if (fd == -1) {
- FREE_C_HEAP_ARRAY(char, filename);
+ FREE_C_HEAP_ARRAY(filename);
return nullptr;
}
@@ -1121,7 +1121,7 @@ static char* mmap_create_shared(size_t size) {
if (mapAddress == MAP_FAILED) {
log_debug(perf)("mmap failed - %s", os::strerror(errno));
remove_file(filename);
- FREE_C_HEAP_ARRAY(char, filename);
+ FREE_C_HEAP_ARRAY(filename);
return nullptr;
}
@@ -1171,7 +1171,7 @@ static void delete_shared_memory(char* addr, size_t size) {
remove_file(backing_store_file_name);
// Don't.. Free heap memory could deadlock os::abort() if it is called
// from signal handler. OS will reclaim the heap memory.
- // FREE_C_HEAP_ARRAY(char, backing_store_file_name);
+ // FREE_C_HEAP_ARRAY(backing_store_file_name);
backing_store_file_name = nullptr;
}
}
@@ -1223,8 +1223,8 @@ static void mmap_attach_shared(int vmid, char** addr, size_t* sizep, TRAPS) {
// store file, we don't follow them when attaching either.
//
if (!is_directory_secure(dirname)) {
- FREE_C_HEAP_ARRAY(char, dirname);
- FREE_C_HEAP_ARRAY(char, luser);
+ FREE_C_HEAP_ARRAY(dirname);
+ FREE_C_HEAP_ARRAY(luser);
THROW_MSG(vmSymbols::java_lang_IllegalArgumentException(),
"Process not found");
}
@@ -1236,9 +1236,9 @@ static void mmap_attach_shared(int vmid, char** addr, size_t* sizep, TRAPS) {
int fd = open_sharedmem_file(filename, file_flags, THREAD);
// free the c heap resources that are no longer needed
- FREE_C_HEAP_ARRAY(char, luser);
- FREE_C_HEAP_ARRAY(char, dirname);
- FREE_C_HEAP_ARRAY(char, filename);
+ FREE_C_HEAP_ARRAY(luser);
+ FREE_C_HEAP_ARRAY(dirname);
+ FREE_C_HEAP_ARRAY(filename);
if (HAS_PENDING_EXCEPTION) {
assert(fd == OS_ERR, "open_sharedmem_file always return OS_ERR on exceptions");
diff --git a/src/hotspot/os/windows/os_perf_windows.cpp b/src/hotspot/os/windows/os_perf_windows.cpp
index 9d04ae65954..d083c72c2e0 100644
--- a/src/hotspot/os/windows/os_perf_windows.cpp
+++ b/src/hotspot/os/windows/os_perf_windows.cpp
@@ -178,9 +178,9 @@ static void destroy(MultiCounterQueryP query) {
for (int i = 0; i < query->noOfCounters; ++i) {
close_query(nullptr, &query->counters[i]);
}
- FREE_C_HEAP_ARRAY(char, query->counters);
+ FREE_C_HEAP_ARRAY(query->counters);
close_query(&query->query.pdh_query_handle, nullptr);
- FREE_C_HEAP_ARRAY(MultiCounterQueryS, query);
+ FREE_C_HEAP_ARRAY(query);
}
}
@@ -189,15 +189,15 @@ static void destroy_query_set(MultiCounterQuerySetP query_set) {
for (int j = 0; j < query_set->queries[i].noOfCounters; ++j) {
close_query(nullptr, &query_set->queries[i].counters[j]);
}
- FREE_C_HEAP_ARRAY(char, query_set->queries[i].counters);
+ FREE_C_HEAP_ARRAY(query_set->queries[i].counters);
close_query(&query_set->queries[i].query.pdh_query_handle, nullptr);
}
- FREE_C_HEAP_ARRAY(MultiCounterQueryS, query_set->queries);
+ FREE_C_HEAP_ARRAY(query_set->queries);
}
static void destroy(MultiCounterQuerySetP query) {
destroy_query_set(query);
- FREE_C_HEAP_ARRAY(MultiCounterQuerySetS, query);
+ FREE_C_HEAP_ARRAY(query);
}
static void destroy(ProcessQueryP query) {
@@ -229,7 +229,7 @@ static void allocate_counters(ProcessQueryP query, size_t nofCounters) {
}
static void deallocate_counters(MultiCounterQueryP query) {
- FREE_C_HEAP_ARRAY(char, query->counters);
+ FREE_C_HEAP_ARRAY(query->counters);
query->counters = nullptr;
query->noOfCounters = 0;
}
@@ -710,11 +710,11 @@ static const char* pdh_process_image_name() {
}
static void deallocate_pdh_constants() {
- FREE_C_HEAP_ARRAY(char, process_image_name);
+ FREE_C_HEAP_ARRAY(process_image_name);
process_image_name = nullptr;
- FREE_C_HEAP_ARRAY(char, pdh_process_instance_IDProcess_counter_fmt);
+ FREE_C_HEAP_ARRAY(pdh_process_instance_IDProcess_counter_fmt);
pdh_process_instance_IDProcess_counter_fmt = nullptr;
- FREE_C_HEAP_ARRAY(char, pdh_process_instance_wildcard_IDProcess_counter);
+ FREE_C_HEAP_ARRAY(pdh_process_instance_wildcard_IDProcess_counter);
pdh_process_instance_wildcard_IDProcess_counter = nullptr;
}
@@ -779,6 +779,114 @@ static OSReturn allocate_pdh_constants() {
return OS_OK;
}
+// Look up the PDH index by reading the English (locale 009) counter name
+// registry. See KB Q287159: Using PDH APIs Correctly in a Localized Language
+// for details.
+static OSReturn lookup_perf_index_by_english_name(const char* english_name,
+ DWORD* result) {
+ ResourceMark rm;
+
+ DWORD type = 0;
+ DWORD size = 0;
+
+ // Determine the required buffer size
+ if (RegQueryValueEx(HKEY_PERFORMANCE_DATA, "Counter 009",
+ nullptr, &type, nullptr, &size) != ERROR_SUCCESS) {
+ return OS_ERR;
+ }
+
+ // Since registry entries in `HKEY_PERFORMANCE_DATA` are generated on the fly,
+ // they could change between calls, so we can't rely just on the size returned
+ // by the first call. Instead, Microsoft's documentation suggests running
+ // these calls in a loop until the return code is no longer `ERROR_MORE_DATA`.
+
+ char* buffer;
+ do {
+ if (size == 0) {
+ return OS_ERR;
+ }
+
+ // When `RegQueryValueEx()` returns `ERROR_MORE_DATA`, the value in the
+ // callback argument is undefined, so we need to create a new variable whose
+ // address is passed as the callback size argument.
+ buffer = NEW_RESOURCE_ARRAY(char, size);
+
+ DWORD cb_size = size;
+ LSTATUS status = RegQueryValueEx(HKEY_PERFORMANCE_DATA, "Counter 009",
+ nullptr, &type, (LPBYTE)buffer,
+ &cb_size);
+ if (status == ERROR_MORE_DATA) {
+ // We need to increase the buffer size. Since we don't know _how much_ to
+ // increase it by, we use an estimate (4096) for the increment.
+ DWORD increment = 4096;
+ if (size > MAXDWORD - increment) {
+ return OS_ERR;
+ }
+ size += increment;
+ } else if (status == ERROR_SUCCESS) {
+ break;
+ } else {
+ // If there was some other problem fetching this registry entry, tell the
+ // caller that we couldn't lookup the index.
+ return OS_ERR;
+ }
+ } while (true);
+
+ if (type != REG_MULTI_SZ) {
+ return OS_ERR;
+ }
+
+ // The buffer contains indices and names in the form (\0\0)*, so
+ // iterate character by character to parse the name and if it matches the
+ // English name, then we return the integer value of the index.
+ for (const char* p = buffer; *p != '\0'; ) {
+ const char* idx_str = p;
+ p += strlen(p) + 1;
+ if (*p == '\0') {
+ break;
+ }
+
+ const char* name = p;
+ p += strlen(p) + 1;
+ if (strcmp(name, english_name) == 0) {
+ errno = 0;
+ char* end = nullptr;
+ unsigned long value = strtoul(idx_str, &end, 10);
+ if (errno == 0 && end != idx_str && value <= MAXDWORD) {
+ *result = (DWORD)value;
+ return OS_OK;
+ }
+ }
+ }
+
+ return OS_ERR;
+}
+
+// Return the counter index of the 'Processor Information' counter, if
+// available, or else the 'Processor' counter. The former is aware of the
+// possibility of multiple processor groups and thus provides a more accurate
+// processor count whereas the latter serves as fallback.
+static DWORD get_proc_counter() {
+ static DWORD pdh_idx = 0;
+ if (pdh_idx != 0) {
+ return pdh_idx;
+ }
+
+ // Some APIs accept English counter names whereas others accept counter names
+ // in the specific user's locale. We determine the locale-specific name using
+ // the counter index, but to find the counter index, we use the English name
+ // of the counter and look for it in a specific registry key.
+ DWORD info_idx;
+ if (lookup_perf_index_by_english_name("Processor Information",
+ &info_idx) != OS_OK) {
+ info_idx = PDH_PROCESSOR_IDX;
+ }
+
+ // Assign to the static variable so that the value persists across calls.
+ pdh_idx = info_idx;
+ return pdh_idx;
+}
+
/*
* Enuerate the Processor PDH object and returns a buffer containing the enumerated instances.
* Caller needs ResourceMark;
@@ -786,8 +894,11 @@ static OSReturn allocate_pdh_constants() {
* @return buffer if successful, null on failure.
*/
static const char* enumerate_cpu_instances() {
- char* processor; //'Processor' == PDH_PROCESSOR_IDX
- if (lookup_name_by_index(PDH_PROCESSOR_IDX, &processor) != OS_OK) {
+ // The `PdhEnumObjectItems()` function accepts a localized name of the perf
+ // counter. To obtain the name that is specific to the user's locale, we
+ // perform a reverse lookup from counter index to counter name.
+ char* processor;
+ if (lookup_name_by_index(get_proc_counter(), &processor) != OS_OK) {
return nullptr;
}
DWORD c_size = 0;
@@ -821,13 +932,17 @@ static const char* enumerate_cpu_instances() {
static int count_logical_cpus(const char* instances) {
assert(instances != nullptr, "invariant");
- // count logical instances.
- DWORD count;
- char* tmp;
- for (count = 0, tmp = const_cast(instances); *tmp != '\0'; tmp = &tmp[strlen(tmp) + 1], count++);
- // PDH reports an instance for each logical processor plus an instance for the total (_Total)
- assert(count == os::processor_count() + 1, "invalid enumeration!");
- return count - 1;
+ DWORD count = 0;
+ for (const char* tmp = instances; *tmp != '\0'; tmp += strlen(tmp) + 1) {
+ // In both the 'Processor' counter and the 'Processor Information' counter,
+ // the output contains totals for the processor group(s). We filter those
+ // out by looking for the `_Total` substring.
+ if (strstr(tmp, "_Total") == nullptr) {
+ count++;
+ }
+ }
+ assert(count >= 1, "invalid enumeration!");
+ return count;
}
static int number_of_logical_cpus() {
@@ -847,7 +962,16 @@ static double cpu_factor() {
static double cpuFactor = .0;
if (numCpus == 0) {
numCpus = number_of_logical_cpus();
- assert(os::processor_count() <= (int)numCpus, "invariant");
+
+ // If we are using the legacy 'Processor' counter, which counts processors
+ // only in the first processor group, then `numCpus` can undercount, in
+ // which case, `numCpus` will be likely smaller than `os_processor_count`.
+ // However, when we use the 'Processor Information' counter, we expect both
+ // `numCpus` and `os::processorCount` to be identical. In both cases, we
+ // expect to see at least one CPU.
+ assert(numCpus >= 1 && numCpus <= (DWORD)os::processor_count(),
+ "unexpected cpu count");
+
cpuFactor = numCpus * 100;
}
return cpuFactor;
@@ -861,8 +985,8 @@ static void log_error_message_on_no_PDH_artifact(const char* counter_path) {
static int initialize_cpu_query_counters(MultiCounterQueryP query, DWORD pdh_counter_idx) {
assert(query != nullptr, "invariant");
assert(query->counters != nullptr, "invariant");
- char* processor; //'Processor' == PDH_PROCESSOR_IDX
- if (lookup_name_by_index(PDH_PROCESSOR_IDX, &processor) != OS_OK) {
+ char* processor;
+ if (lookup_name_by_index(get_proc_counter(), &processor) != OS_OK) {
return OS_ERR;
}
char* counter_name = nullptr;
@@ -880,7 +1004,11 @@ static int initialize_cpu_query_counters(MultiCounterQueryP query, DWORD pdh_cou
counter_len += OBJECT_WITH_INSTANCES_COUNTER_FMT_LEN; // "\\%s(%s)\\%s"
const char* instances = enumerate_cpu_instances();
DWORD index = 0;
- for (char* tmp = const_cast(instances); *tmp != '\0'; tmp = &tmp[strlen(tmp) + 1], index++) {
+ for (char* tmp = const_cast(instances); *tmp != '\0'; tmp = &tmp[strlen(tmp) + 1]) {
+ // Skip totals for each processor group.
+ if (strstr(tmp, ",_Total") != nullptr) {
+ continue;
+ }
const size_t tmp_len = strlen(tmp);
char* counter_path = NEW_RESOURCE_ARRAY(char, counter_len + tmp_len + 1);
const size_t jio_snprintf_result = jio_snprintf(counter_path,
@@ -896,6 +1024,7 @@ static int initialize_cpu_query_counters(MultiCounterQueryP query, DWORD pdh_cou
// return OS_OK to have the system continue to run without the missing counter
return OS_OK;
}
+ index++;
}
// Query once to initialize the counters which require at least two samples
// (like the % CPU usage) to calculate correctly.
@@ -1445,9 +1574,9 @@ bool CPUInformationInterface::initialize() {
CPUInformationInterface::~CPUInformationInterface() {
if (_cpu_info != nullptr) {
- FREE_C_HEAP_ARRAY(char, _cpu_info->cpu_name());
+ FREE_C_HEAP_ARRAY(_cpu_info->cpu_name());
_cpu_info->set_cpu_name(nullptr);
- FREE_C_HEAP_ARRAY(char, _cpu_info->cpu_description());
+ FREE_C_HEAP_ARRAY(_cpu_info->cpu_description());
_cpu_info->set_cpu_description(nullptr);
delete _cpu_info;
}
diff --git a/src/hotspot/os/windows/os_windows.cpp b/src/hotspot/os/windows/os_windows.cpp
index 9d8fb45f0d1..9a987bf3762 100644
--- a/src/hotspot/os/windows/os_windows.cpp
+++ b/src/hotspot/os/windows/os_windows.cpp
@@ -334,14 +334,14 @@ void os::init_system_properties_values() {
home_path = NEW_C_HEAP_ARRAY(char, strlen(home_dir) + 1, mtInternal);
strcpy(home_path, home_dir);
Arguments::set_java_home(home_path);
- FREE_C_HEAP_ARRAY(char, home_path);
+ FREE_C_HEAP_ARRAY(home_path);
dll_path = NEW_C_HEAP_ARRAY(char, strlen(home_dir) + strlen(bin) + 1,
mtInternal);
strcpy(dll_path, home_dir);
strcat(dll_path, bin);
Arguments::set_dll_dir(dll_path);
- FREE_C_HEAP_ARRAY(char, dll_path);
+ FREE_C_HEAP_ARRAY(dll_path);
if (!set_boot_path('\\', ';')) {
vm_exit_during_initialization("Failed setting boot class path.", nullptr);
@@ -396,7 +396,7 @@ void os::init_system_properties_values() {
strcat(library_path, ";.");
Arguments::set_library_path(library_path);
- FREE_C_HEAP_ARRAY(char, library_path);
+ FREE_C_HEAP_ARRAY(library_path);
}
// Default extensions directory
@@ -1079,7 +1079,7 @@ void os::set_native_thread_name(const char *name) {
HRESULT hr = _SetThreadDescription(current, unicode_name);
if (FAILED(hr)) {
log_debug(os, thread)("set_native_thread_name: SetThreadDescription failed - falling back to debugger method");
- FREE_C_HEAP_ARRAY(WCHAR, unicode_name);
+ FREE_C_HEAP_ARRAY(unicode_name);
} else {
log_trace(os, thread)("set_native_thread_name: SetThreadDescription succeeded - new name: %s", name);
@@ -1102,7 +1102,7 @@ void os::set_native_thread_name(const char *name) {
LocalFree(thread_name);
}
#endif
- FREE_C_HEAP_ARRAY(WCHAR, unicode_name);
+ FREE_C_HEAP_ARRAY(unicode_name);
return;
}
} else {
@@ -2897,7 +2897,7 @@ class NUMANodeListHolder {
int _numa_used_node_count;
void free_node_list() {
- FREE_C_HEAP_ARRAY(int, _numa_used_node_list);
+ FREE_C_HEAP_ARRAY(_numa_used_node_list);
}
public:
@@ -4744,7 +4744,7 @@ static wchar_t* wide_abs_unc_path(char const* path, errno_t & err, int additiona
LPWSTR unicode_path = nullptr;
err = convert_to_unicode(buf, &unicode_path);
- FREE_C_HEAP_ARRAY(char, buf);
+ FREE_C_HEAP_ARRAY(buf);
if (err != ERROR_SUCCESS) {
return nullptr;
}
@@ -4772,9 +4772,9 @@ static wchar_t* wide_abs_unc_path(char const* path, errno_t & err, int additiona
}
if (converted_path != unicode_path) {
- FREE_C_HEAP_ARRAY(WCHAR, converted_path);
+ FREE_C_HEAP_ARRAY(converted_path);
}
- FREE_C_HEAP_ARRAY(WCHAR, unicode_path);
+ FREE_C_HEAP_ARRAY(unicode_path);
return static_cast(result); // LPWSTR and wchat_t* are the same type on Windows.
}
@@ -5827,7 +5827,7 @@ int os::fork_and_exec(const char* cmd) {
exit_code = -1;
}
- FREE_C_HEAP_ARRAY(char, cmd_string);
+ FREE_C_HEAP_ARRAY(cmd_string);
return (int)exit_code;
}
diff --git a/src/hotspot/os/windows/perfMemory_windows.cpp b/src/hotspot/os/windows/perfMemory_windows.cpp
index dad2804f18a..8e698c53d28 100644
--- a/src/hotspot/os/windows/perfMemory_windows.cpp
+++ b/src/hotspot/os/windows/perfMemory_windows.cpp
@@ -113,7 +113,7 @@ static void save_memory_to_file(char* addr, size_t size) {
}
}
- FREE_C_HEAP_ARRAY(char, destfile);
+ FREE_C_HEAP_ARRAY(destfile);
}
// Shared Memory Implementation Details
@@ -319,7 +319,7 @@ static char* get_user_name_slow(int vmid) {
DIR* subdirp = os::opendir(usrdir_name);
if (subdirp == nullptr) {
- FREE_C_HEAP_ARRAY(char, usrdir_name);
+ FREE_C_HEAP_ARRAY(usrdir_name);
continue;
}
@@ -330,7 +330,7 @@ static char* get_user_name_slow(int vmid) {
// symlink can be exploited.
//
if (!is_directory_secure(usrdir_name)) {
- FREE_C_HEAP_ARRAY(char, usrdir_name);
+ FREE_C_HEAP_ARRAY(usrdir_name);
os::closedir(subdirp);
continue;
}
@@ -350,13 +350,13 @@ static char* get_user_name_slow(int vmid) {
strcat(filename, udentry->d_name);
if (::stat(filename, &statbuf) == OS_ERR) {
- FREE_C_HEAP_ARRAY(char, filename);
+ FREE_C_HEAP_ARRAY(filename);
continue;
}
// skip over files that are not regular files.
if ((statbuf.st_mode & S_IFMT) != S_IFREG) {
- FREE_C_HEAP_ARRAY(char, filename);
+ FREE_C_HEAP_ARRAY(filename);
continue;
}
@@ -378,18 +378,18 @@ static char* get_user_name_slow(int vmid) {
if (statbuf.st_ctime > latest_ctime) {
char* user = strchr(dentry->d_name, '_') + 1;
- FREE_C_HEAP_ARRAY(char, latest_user);
+ FREE_C_HEAP_ARRAY(latest_user);
latest_user = NEW_C_HEAP_ARRAY(char, strlen(user)+1, mtInternal);
strcpy(latest_user, user);
latest_ctime = statbuf.st_ctime;
}
- FREE_C_HEAP_ARRAY(char, filename);
+ FREE_C_HEAP_ARRAY(filename);
}
}
os::closedir(subdirp);
- FREE_C_HEAP_ARRAY(char, usrdir_name);
+ FREE_C_HEAP_ARRAY(usrdir_name);
}
os::closedir(tmpdirp);
@@ -481,7 +481,7 @@ static void remove_file(const char* dirname, const char* filename) {
}
}
- FREE_C_HEAP_ARRAY(char, path);
+ FREE_C_HEAP_ARRAY(path);
}
// returns true if the process represented by pid is alive, otherwise
@@ -708,11 +708,11 @@ static void free_security_desc(PSECURITY_DESCRIPTOR pSD) {
// be an ACL we enlisted. free the resources.
//
if (success && exists && pACL != nullptr && !isdefault) {
- FREE_C_HEAP_ARRAY(char, pACL);
+ FREE_C_HEAP_ARRAY(pACL);
}
// free the security descriptor
- FREE_C_HEAP_ARRAY(char, pSD);
+ FREE_C_HEAP_ARRAY(pSD);
}
}
@@ -768,7 +768,7 @@ static PSID get_user_sid(HANDLE hProcess) {
if (!GetTokenInformation(hAccessToken, TokenUser, token_buf, rsize, &rsize)) {
log_debug(perf)("GetTokenInformation failure: lasterror = %d, rsize = %d",
GetLastError(), rsize);
- FREE_C_HEAP_ARRAY(char, token_buf);
+ FREE_C_HEAP_ARRAY(token_buf);
CloseHandle(hAccessToken);
return nullptr;
}
@@ -779,15 +779,15 @@ static PSID get_user_sid(HANDLE hProcess) {
if (!CopySid(nbytes, pSID, token_buf->User.Sid)) {
log_debug(perf)("GetTokenInformation failure: lasterror = %d, rsize = %d",
GetLastError(), rsize);
- FREE_C_HEAP_ARRAY(char, token_buf);
- FREE_C_HEAP_ARRAY(char, pSID);
+ FREE_C_HEAP_ARRAY(token_buf);
+ FREE_C_HEAP_ARRAY(pSID);
CloseHandle(hAccessToken);
return nullptr;
}
// close the access token.
CloseHandle(hAccessToken);
- FREE_C_HEAP_ARRAY(char, token_buf);
+ FREE_C_HEAP_ARRAY(token_buf);
return pSID;
}
@@ -865,7 +865,7 @@ static bool add_allow_aces(PSECURITY_DESCRIPTOR pSD,
if (!InitializeAcl(newACL, newACLsize, ACL_REVISION)) {
log_debug(perf)("InitializeAcl failure: lasterror = %d", GetLastError());
- FREE_C_HEAP_ARRAY(char, newACL);
+ FREE_C_HEAP_ARRAY(newACL);
return false;
}
@@ -876,7 +876,7 @@ static bool add_allow_aces(PSECURITY_DESCRIPTOR pSD,
LPVOID ace;
if (!GetAce(oldACL, ace_index, &ace)) {
log_debug(perf)("InitializeAcl failure: lasterror = %d", GetLastError());
- FREE_C_HEAP_ARRAY(char, newACL);
+ FREE_C_HEAP_ARRAY(newACL);
return false;
}
if (((ACCESS_ALLOWED_ACE *)ace)->Header.AceFlags && INHERITED_ACE) {
@@ -901,7 +901,7 @@ static bool add_allow_aces(PSECURITY_DESCRIPTOR pSD,
if (!AddAce(newACL, ACL_REVISION, MAXDWORD, ace,
((PACE_HEADER)ace)->AceSize)) {
log_debug(perf)("AddAce failure: lasterror = %d", GetLastError());
- FREE_C_HEAP_ARRAY(char, newACL);
+ FREE_C_HEAP_ARRAY(newACL);
return false;
}
}
@@ -915,7 +915,7 @@ static bool add_allow_aces(PSECURITY_DESCRIPTOR pSD,
aces[i].mask, aces[i].pSid)) {
log_debug(perf)("AddAccessAllowedAce failure: lasterror = %d",
GetLastError());
- FREE_C_HEAP_ARRAY(char, newACL);
+ FREE_C_HEAP_ARRAY(newACL);
return false;
}
}
@@ -928,13 +928,13 @@ static bool add_allow_aces(PSECURITY_DESCRIPTOR pSD,
LPVOID ace;
if (!GetAce(oldACL, ace_index, &ace)) {
log_debug(perf)("InitializeAcl failure: lasterror = %d", GetLastError());
- FREE_C_HEAP_ARRAY(char, newACL);
+ FREE_C_HEAP_ARRAY(newACL);
return false;
}
if (!AddAce(newACL, ACL_REVISION, MAXDWORD, ace,
((PACE_HEADER)ace)->AceSize)) {
log_debug(perf)("AddAce failure: lasterror = %d", GetLastError());
- FREE_C_HEAP_ARRAY(char, newACL);
+ FREE_C_HEAP_ARRAY(newACL);
return false;
}
ace_index++;
@@ -944,7 +944,7 @@ static bool add_allow_aces(PSECURITY_DESCRIPTOR pSD,
// add the new ACL to the security descriptor.
if (!SetSecurityDescriptorDacl(pSD, TRUE, newACL, FALSE)) {
log_debug(perf)("SetSecurityDescriptorDacl failure: lasterror = %d", GetLastError());
- FREE_C_HEAP_ARRAY(char, newACL);
+ FREE_C_HEAP_ARRAY(newACL);
return false;
}
@@ -952,7 +952,7 @@ static bool add_allow_aces(PSECURITY_DESCRIPTOR pSD,
// protected prevents that.
if (!SetSecurityDescriptorControl(pSD, SE_DACL_PROTECTED, SE_DACL_PROTECTED)) {
log_debug(perf)("SetSecurityDescriptorControl failure: lasterror = %d", GetLastError());
- FREE_C_HEAP_ARRAY(char, newACL);
+ FREE_C_HEAP_ARRAY(newACL);
return false;
}
@@ -1057,7 +1057,7 @@ static LPSECURITY_ATTRIBUTES make_user_everybody_admin_security_attr(
// create a security attributes structure with access control
// entries as initialized above.
LPSECURITY_ATTRIBUTES lpSA = make_security_attr(aces, 3);
- FREE_C_HEAP_ARRAY(char, aces[0].pSid);
+ FREE_C_HEAP_ARRAY(aces[0].pSid);
FreeSid(everybodySid);
FreeSid(administratorsSid);
return(lpSA);
@@ -1341,8 +1341,8 @@ static char* mapping_create_shared(size_t size) {
// check that the file system is secure - i.e. it supports ACLs.
if (!is_filesystem_secure(dirname)) {
- FREE_C_HEAP_ARRAY(char, dirname);
- FREE_C_HEAP_ARRAY(char, user);
+ FREE_C_HEAP_ARRAY(dirname);
+ FREE_C_HEAP_ARRAY(user);
return nullptr;
}
@@ -1358,15 +1358,15 @@ static char* mapping_create_shared(size_t size) {
assert(((size != 0) && (size % os::vm_page_size() == 0)),
"unexpected PerfMemry region size");
- FREE_C_HEAP_ARRAY(char, user);
+ FREE_C_HEAP_ARRAY(user);
// create the shared memory resources
sharedmem_fileMapHandle =
create_sharedmem_resources(dirname, filename, objectname, size);
- FREE_C_HEAP_ARRAY(char, filename);
- FREE_C_HEAP_ARRAY(char, objectname);
- FREE_C_HEAP_ARRAY(char, dirname);
+ FREE_C_HEAP_ARRAY(filename);
+ FREE_C_HEAP_ARRAY(objectname);
+ FREE_C_HEAP_ARRAY(dirname);
if (sharedmem_fileMapHandle == nullptr) {
return nullptr;
@@ -1480,8 +1480,8 @@ static void open_file_mapping(int vmid, char** addrp, size_t* sizep, TRAPS) {
// store file, we also don't following them when attaching
//
if (!is_directory_secure(dirname)) {
- FREE_C_HEAP_ARRAY(char, dirname);
- FREE_C_HEAP_ARRAY(char, luser);
+ FREE_C_HEAP_ARRAY(dirname);
+ FREE_C_HEAP_ARRAY(luser);
THROW_MSG(vmSymbols::java_lang_IllegalArgumentException(),
"Process not found");
}
@@ -1498,10 +1498,10 @@ static void open_file_mapping(int vmid, char** addrp, size_t* sizep, TRAPS) {
char* robjectname = ResourceArea::strdup(THREAD, objectname);
// free the c heap resources that are no longer needed
- FREE_C_HEAP_ARRAY(char, luser);
- FREE_C_HEAP_ARRAY(char, dirname);
- FREE_C_HEAP_ARRAY(char, filename);
- FREE_C_HEAP_ARRAY(char, objectname);
+ FREE_C_HEAP_ARRAY(luser);
+ FREE_C_HEAP_ARRAY(dirname);
+ FREE_C_HEAP_ARRAY(filename);
+ FREE_C_HEAP_ARRAY(objectname);
size_t size;
if (*sizep == 0) {
diff --git a/src/hotspot/os/windows/safefetch_static_windows.cpp b/src/hotspot/os/windows/safefetch_static_windows.cpp
index 3ea8b96b32d..f037459f1fb 100644
--- a/src/hotspot/os/windows/safefetch_static_windows.cpp
+++ b/src/hotspot/os/windows/safefetch_static_windows.cpp
@@ -46,7 +46,8 @@ extern "C" char _SafeFetchN_fault[];
bool handle_safefetch(int exception_code, address pc, void* context) {
CONTEXT* ctx = (CONTEXT*)context;
- if (exception_code == EXCEPTION_ACCESS_VIOLATION && ctx != nullptr) {
+ if ((exception_code == EXCEPTION_ACCESS_VIOLATION ||
+ exception_code == EXCEPTION_GUARD_PAGE) && ctx != nullptr) {
if (pc == (address)_SafeFetch32_fault) {
os::win32::context_set_pc(ctx, (address)_SafeFetch32_continuation);
return true;
diff --git a/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp b/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp
index 49d879731ff..6f31bc284e3 100644
--- a/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp
+++ b/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp
@@ -81,10 +81,6 @@
# include
#endif
-#if !defined(__APPLE__) && !defined(__NetBSD__)
-# include
-#endif
-
#define SPELL_REG_SP "sp"
#ifdef __APPLE__
@@ -415,49 +411,6 @@ size_t os::Posix::default_stack_size(os::ThreadType thr_type) {
size_t s = (thr_type == os::compiler_thread ? 4 * M : 1 * M);
return s;
}
-void os::current_stack_base_and_size(address* base, size_t* size) {
- address bottom;
-#ifdef __APPLE__
- pthread_t self = pthread_self();
- *base = (address) pthread_get_stackaddr_np(self);
- *size = pthread_get_stacksize_np(self);
- bottom = *base - *size;
-#elif defined(__OpenBSD__)
- stack_t ss;
- int rslt = pthread_stackseg_np(pthread_self(), &ss);
-
- if (rslt != 0)
- fatal("pthread_stackseg_np failed with error = %d", rslt);
-
- *base = (address) ss.ss_sp;
- *size = ss.ss_size;
- bottom = *base - *size;
-#else
- pthread_attr_t attr;
-
- int rslt = pthread_attr_init(&attr);
-
- // JVM needs to know exact stack location, abort if it fails
- if (rslt != 0)
- fatal("pthread_attr_init failed with error = %d", rslt);
-
- rslt = pthread_attr_get_np(pthread_self(), &attr);
-
- if (rslt != 0)
- fatal("pthread_attr_get_np failed with error = %d", rslt);
-
- if (pthread_attr_getstackaddr(&attr, (void **)&bottom) != 0 ||
- pthread_attr_getstacksize(&attr, size) != 0) {
- fatal("Can not locate current stack attributes!");
- }
-
- *base = bottom + *size;
-
- pthread_attr_destroy(&attr);
-#endif
- assert(os::current_stack_pointer() >= bottom &&
- os::current_stack_pointer() < *base, "just checking");
-}
/////////////////////////////////////////////////////////////////////////////
// helper functions for fatal error handler
diff --git a/src/hotspot/os_cpu/bsd_x86/os_bsd_x86.cpp b/src/hotspot/os_cpu/bsd_x86/os_bsd_x86.cpp
index f1c80594eaf..8668f20e371 100644
--- a/src/hotspot/os_cpu/bsd_x86/os_bsd_x86.cpp
+++ b/src/hotspot/os_cpu/bsd_x86/os_bsd_x86.cpp
@@ -55,6 +55,7 @@
// put OS-includes here
# include
# include
+# include
# include
# include
# include
@@ -73,19 +74,6 @@
# include
#endif
-#if !defined(__APPLE__) && !defined(__NetBSD__)
-# include
-#endif
-
-// needed by current_stack_base_and_size() workaround for Mavericks
-#if defined(__APPLE__)
-# include
-# include
-# include
-# define DEFAULT_MAIN_THREAD_STACK_PAGES 2048
-# define OS_X_10_9_0_KERNEL_MAJOR_VERSION 13
-#endif
-
#define SPELL_REG_SP "rsp"
#define SPELL_REG_FP "rbp"
#define REG_BCP context_r13
@@ -499,104 +487,6 @@ size_t os::Posix::default_stack_size(os::ThreadType thr_type) {
}
-// Java thread:
-//
-// Low memory addresses
-// +------------------------+
-// | |\ Java thread created by VM does not have glibc
-// | glibc guard page | - guard, attached Java thread usually has
-// | |/ 1 glibc guard page.
-// P1 +------------------------+ Thread::stack_base() - Thread::stack_size()
-// | |\
-// | HotSpot Guard Pages | - red, yellow and reserved pages
-// | |/
-// +------------------------+ StackOverflow::stack_reserved_zone_base()
-// | |\
-// | Normal Stack | -
-// | |/
-// P2 +------------------------+ Thread::stack_base()
-//
-// Non-Java thread:
-//
-// Low memory addresses
-// +------------------------+
-// | |\
-// | glibc guard page | - usually 1 page
-// | |/
-// P1 +------------------------+ Thread::stack_base() - Thread::stack_size()
-// | |\
-// | Normal Stack | -
-// | |/
-// P2 +------------------------+ Thread::stack_base()
-//
-// ** P1 (aka bottom) and size are the address and stack size
-// returned from pthread_attr_getstack().
-// ** P2 (aka stack top or base) = P1 + size
-
-void os::current_stack_base_and_size(address* base, size_t* size) {
- address bottom;
-#ifdef __APPLE__
- pthread_t self = pthread_self();
- *base = (address) pthread_get_stackaddr_np(self);
- *size = pthread_get_stacksize_np(self);
- // workaround for OS X 10.9.0 (Mavericks)
- // pthread_get_stacksize_np returns 128 pages even though the actual size is 2048 pages
- if (pthread_main_np() == 1) {
- // At least on Mac OS 10.12 we have observed stack sizes not aligned
- // to pages boundaries. This can be provoked by e.g. setrlimit() (ulimit -s xxxx in the
- // shell). Apparently Mac OS actually rounds upwards to next multiple of page size,
- // however, we round downwards here to be on the safe side.
- *size = align_down(*size, getpagesize());
-
- if ((*size) < (DEFAULT_MAIN_THREAD_STACK_PAGES * (size_t)getpagesize())) {
- char kern_osrelease[256];
- size_t kern_osrelease_size = sizeof(kern_osrelease);
- int ret = sysctlbyname("kern.osrelease", kern_osrelease, &kern_osrelease_size, nullptr, 0);
- if (ret == 0) {
- // get the major number, atoi will ignore the minor amd micro portions of the version string
- if (atoi(kern_osrelease) >= OS_X_10_9_0_KERNEL_MAJOR_VERSION) {
- *size = (DEFAULT_MAIN_THREAD_STACK_PAGES*getpagesize());
- }
- }
- }
- }
- bottom = *base - *size;
-#elif defined(__OpenBSD__)
- stack_t ss;
- int rslt = pthread_stackseg_np(pthread_self(), &ss);
-
- if (rslt != 0)
- fatal("pthread_stackseg_np failed with error = %d", rslt);
-
- *base = (address) ss.ss_sp;
- *size = ss.ss_size;
- bottom = *base - *size;
-#else
- pthread_attr_t attr;
-
- int rslt = pthread_attr_init(&attr);
-
- // JVM needs to know exact stack location, abort if it fails
- if (rslt != 0)
- fatal("pthread_attr_init failed with error = %d", rslt);
-
- rslt = pthread_attr_get_np(pthread_self(), &attr);
-
- if (rslt != 0)
- fatal("pthread_attr_get_np failed with error = %d", rslt);
-
- if (pthread_attr_getstackaddr(&attr, (void **)&bottom) != 0 ||
- pthread_attr_getstacksize(&attr, size) != 0) {
- fatal("Can not locate current stack attributes!");
- }
-
- *base = bottom + *size;
-
- pthread_attr_destroy(&attr);
-#endif
- assert(os::current_stack_pointer() >= bottom &&
- os::current_stack_pointer() < *base, "just checking");
-}
/////////////////////////////////////////////////////////////////////////////
// helper functions for fatal error handler
diff --git a/src/hotspot/os_cpu/bsd_zero/os_bsd_zero.cpp b/src/hotspot/os_cpu/bsd_zero/os_bsd_zero.cpp
index facad184426..a089d5981ca 100644
--- a/src/hotspot/os_cpu/bsd_zero/os_bsd_zero.cpp
+++ b/src/hotspot/os_cpu/bsd_zero/os_bsd_zero.cpp
@@ -52,7 +52,6 @@
#if !defined(__APPLE__) && !defined(__NetBSD__)
#include
-# include /* For pthread_attr_get_np */
#endif
address os::current_stack_pointer() {
@@ -179,52 +178,6 @@ size_t os::Posix::default_stack_size(os::ThreadType thr_type) {
return s;
}
-void os::current_stack_base_and_size(address* base, size_t* size) {
- address bottom;
-
-#ifdef __APPLE__
- pthread_t self = pthread_self();
- *base = (address) pthread_get_stackaddr_np(self);
- *size = pthread_get_stacksize_np(self);
- bottom = *base - *size;
-#elif defined(__OpenBSD__)
- stack_t ss;
- int rslt = pthread_stackseg_np(pthread_self(), &ss);
-
- if (rslt != 0)
- fatal("pthread_stackseg_np failed with error = %d", rslt);
-
- *base = (address) ss.ss_sp;
- *size = ss.ss_size;
- bottom = *base - *size;
-#else
- pthread_attr_t attr;
-
- int rslt = pthread_attr_init(&attr);
-
- // JVM needs to know exact stack location, abort if it fails
- if (rslt != 0)
- fatal("pthread_attr_init failed with error = %d", rslt);
-
- rslt = pthread_attr_get_np(pthread_self(), &attr);
-
- if (rslt != 0)
- fatal("pthread_attr_get_np failed with error = %d", rslt);
-
- if (pthread_attr_getstackaddr(&attr, (void **) &bottom) != 0 ||
- pthread_attr_getstacksize(&attr, size) != 0) {
- fatal("Can not locate current stack attributes!");
- }
-
- *base = bottom + *size;
-
- pthread_attr_destroy(&attr);
-
-#endif
- assert(os::current_stack_pointer() >= bottom &&
- os::current_stack_pointer() < *base, "just checking");
-}
-
/////////////////////////////////////////////////////////////////////////////
// helper functions for fatal error handler
diff --git a/src/hotspot/os_cpu/linux_aarch64/vm_version_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/vm_version_linux_aarch64.cpp
index ee2d3013c4c..e5762fa3ba5 100644
--- a/src/hotspot/os_cpu/linux_aarch64/vm_version_linux_aarch64.cpp
+++ b/src/hotspot/os_cpu/linux_aarch64/vm_version_linux_aarch64.cpp
@@ -123,58 +123,40 @@ int VM_Version::set_and_get_current_sve_vector_length(int length) {
return new_length;
}
+static uint64_t check_feature(uint64_t hwcap, uint64_t feature_bit_mask, uint64_t hwcap_bitmask) {
+ if (hwcap & hwcap_bitmask) {
+ return feature_bit_mask;
+ } else {
+ return 0;
+ }
+}
+
void VM_Version::get_os_cpu_info() {
uint64_t auxv = getauxval(AT_HWCAP);
uint64_t auxv2 = getauxval(AT_HWCAP2);
- static_assert(BIT_MASK(CPU_FP) == HWCAP_FP, "Flag CPU_FP must follow Linux HWCAP");
- static_assert(BIT_MASK(CPU_ASIMD) == HWCAP_ASIMD, "Flag CPU_ASIMD must follow Linux HWCAP");
- static_assert(BIT_MASK(CPU_EVTSTRM) == HWCAP_EVTSTRM, "Flag CPU_EVTSTRM must follow Linux HWCAP");
- static_assert(BIT_MASK(CPU_AES) == HWCAP_AES, "Flag CPU_AES must follow Linux HWCAP");
- static_assert(BIT_MASK(CPU_PMULL) == HWCAP_PMULL, "Flag CPU_PMULL must follow Linux HWCAP");
- static_assert(BIT_MASK(CPU_SHA1) == HWCAP_SHA1, "Flag CPU_SHA1 must follow Linux HWCAP");
- static_assert(BIT_MASK(CPU_SHA2) == HWCAP_SHA2, "Flag CPU_SHA2 must follow Linux HWCAP");
- static_assert(BIT_MASK(CPU_CRC32) == HWCAP_CRC32, "Flag CPU_CRC32 must follow Linux HWCAP");
- static_assert(BIT_MASK(CPU_LSE) == HWCAP_ATOMICS, "Flag CPU_LSE must follow Linux HWCAP");
- static_assert(BIT_MASK(CPU_DCPOP) == HWCAP_DCPOP, "Flag CPU_DCPOP must follow Linux HWCAP");
- static_assert(BIT_MASK(CPU_SHA3) == HWCAP_SHA3, "Flag CPU_SHA3 must follow Linux HWCAP");
- static_assert(BIT_MASK(CPU_SHA512) == HWCAP_SHA512, "Flag CPU_SHA512 must follow Linux HWCAP");
- static_assert(BIT_MASK(CPU_SVE) == HWCAP_SVE, "Flag CPU_SVE must follow Linux HWCAP");
- static_assert(BIT_MASK(CPU_PACA) == HWCAP_PACA, "Flag CPU_PACA must follow Linux HWCAP");
- static_assert(BIT_MASK(CPU_FPHP) == HWCAP_FPHP, "Flag CPU_FPHP must follow Linux HWCAP");
- static_assert(BIT_MASK(CPU_ASIMDHP) == HWCAP_ASIMDHP, "Flag CPU_ASIMDHP must follow Linux HWCAP");
- _features = auxv & (
- HWCAP_FP |
- HWCAP_ASIMD |
- HWCAP_EVTSTRM |
- HWCAP_AES |
- HWCAP_PMULL |
- HWCAP_SHA1 |
- HWCAP_SHA2 |
- HWCAP_CRC32 |
- HWCAP_ATOMICS |
- HWCAP_DCPOP |
- HWCAP_SHA3 |
- HWCAP_SHA512 |
- HWCAP_SVE |
- HWCAP_SB |
- HWCAP_PACA |
- HWCAP_FPHP |
- HWCAP_ASIMDHP);
-
- if (auxv2 & HWCAP2_SVE2) {
- set_feature(CPU_SVE2);
- }
- if (auxv2 & HWCAP2_SVEBITPERM) {
- set_feature(CPU_SVEBITPERM);
- }
- if (auxv2 & HWCAP2_ECV) {
- set_feature(CPU_ECV);
- }
- if (auxv2 & HWCAP2_WFXT) {
- set_feature(CPU_WFXT);
- }
+ _features =
+ check_feature(auxv, BIT_MASK(CPU_FP), HWCAP_FP) |
+ check_feature(auxv, BIT_MASK(CPU_ASIMD), HWCAP_ASIMD) |
+ check_feature(auxv, BIT_MASK(CPU_EVTSTRM), HWCAP_EVTSTRM) |
+ check_feature(auxv, BIT_MASK(CPU_AES), HWCAP_AES) |
+ check_feature(auxv, BIT_MASK(CPU_PMULL), HWCAP_PMULL) |
+ check_feature(auxv, BIT_MASK(CPU_SHA1), HWCAP_SHA1) |
+ check_feature(auxv, BIT_MASK(CPU_SHA2), HWCAP_SHA2) |
+ check_feature(auxv, BIT_MASK(CPU_CRC32), HWCAP_CRC32) |
+ check_feature(auxv, BIT_MASK(CPU_LSE), HWCAP_ATOMICS) |
+ check_feature(auxv, BIT_MASK(CPU_DCPOP), HWCAP_DCPOP) |
+ check_feature(auxv, BIT_MASK(CPU_SHA3), HWCAP_SHA3) |
+ check_feature(auxv, BIT_MASK(CPU_SHA512), HWCAP_SHA512) |
+ check_feature(auxv, BIT_MASK(CPU_SVE), HWCAP_SVE) |
+ check_feature(auxv, BIT_MASK(CPU_PACA), HWCAP_PACA) |
+ check_feature(auxv, BIT_MASK(CPU_FPHP), HWCAP_FPHP) |
+ check_feature(auxv, BIT_MASK(CPU_ASIMDHP), HWCAP_ASIMDHP) |
+ check_feature(auxv2, BIT_MASK(CPU_SVE2), HWCAP2_SVE2) |
+ check_feature(auxv2, BIT_MASK(CPU_SVEBITPERM), HWCAP2_SVEBITPERM) |
+ check_feature(auxv2, BIT_MASK(CPU_ECV), HWCAP2_ECV) |
+ check_feature(auxv2, BIT_MASK(CPU_WFXT), HWCAP2_WFXT);
uint64_t ctr_el0;
uint64_t dczid_el0;
diff --git a/src/hotspot/os_cpu/windows_aarch64/javaThread_windows_aarch64.cpp b/src/hotspot/os_cpu/windows_aarch64/javaThread_windows_aarch64.cpp
index 8f6f1ccd38a..3f77a27f051 100644
--- a/src/hotspot/os_cpu/windows_aarch64/javaThread_windows_aarch64.cpp
+++ b/src/hotspot/os_cpu/windows_aarch64/javaThread_windows_aarch64.cpp
@@ -26,6 +26,12 @@
#include "runtime/frame.inline.hpp"
#include "runtime/javaThread.hpp"
+// CRT-provided TLS slot for this module (jvm.dll), set by the OS loader.
+extern "C" unsigned long _tls_index;
+
+// TLS offset read by the assembly code in `aarch64_get_thread_helper()`.
+extern "C" ptrdiff_t _jvm_thr_current_tls_offset = JavaThread::get_thr_tls_offset();
+
frame JavaThread::pd_last_frame() {
assert(has_last_Java_frame(), "must have last_Java_sp() when suspended");
vmassert(_anchor.last_Java_pc() != nullptr, "not walkable");
@@ -87,3 +93,25 @@ bool JavaThread::pd_get_top_frame(frame* fr_addr, void* ucontext, bool isInJava)
}
void JavaThread::cache_global_variables() { }
+
+ptrdiff_t JavaThread::get_thr_tls_offset() {
+ char* tebPointer = (char*)NtCurrentTeb();
+
+ // 0x58 is the offset of ThreadLocalStoragePointer within the TEB. This is
+ // a stable Windows ABI constant but is not exposed in the SDK's minimal
+ // _TEB struct.
+ void** tls_array = *(void***)(tebPointer + 0x58);
+ char* curr_ptr = (char*)&Thread::_thr_current;
+ char* tls_block = (char*)tls_array[_tls_index];
+
+ // Compute the offset of Thread::_thr_current within this module's TLS
+ // block. Unlike ELF, which provides `tlsdesc` relocations that lets
+ // assembly code resolve TLS variables symbolically at link/load time,
+ // Windows PE/COFF has no equivalent mechanism for armasm64. So we compute
+ // the offset here in C++ (where the compiler knows how to access
+ // __declspec(thread) variables) and store it in a plain global that the
+ // assembly can load directly. In subsequent calls to
+ // `aarch64_get_thread_helper()`, the assembly will read the TEB to find the
+ // TLS block and then add this offset to find `Thread::_thr_current`.
+ return curr_ptr - tls_block;
+}
diff --git a/src/hotspot/os_cpu/windows_aarch64/javaThread_windows_aarch64.hpp b/src/hotspot/os_cpu/windows_aarch64/javaThread_windows_aarch64.hpp
index 7d6ed16e629..34984607814 100644
--- a/src/hotspot/os_cpu/windows_aarch64/javaThread_windows_aarch64.hpp
+++ b/src/hotspot/os_cpu/windows_aarch64/javaThread_windows_aarch64.hpp
@@ -46,8 +46,11 @@ private:
bool pd_get_top_frame(frame* fr_addr, void* ucontext, bool isInJava);
public:
- static Thread *aarch64_get_thread_helper() {
- return Thread::current();
- }
+ static Thread *aarch64_get_thread_helper();
+
+ // Compute the offset of `Thread::_thr_current` in the thread-local storage
+ // This offset is then used by the assembly code implementation of
+ // `aarch64_get_thread_helper()`.
+ static ptrdiff_t get_thr_tls_offset();
#endif // OS_CPU_WINDOWS_AARCH64_JAVATHREAD_WINDOWS_AARCH64_HPP
diff --git a/src/hotspot/os_cpu/windows_aarch64/sve_windows_aarch64.S b/src/hotspot/os_cpu/windows_aarch64/sve_windows_aarch64.S
index e0c85830bd4..137cd0f7753 100644
--- a/src/hotspot/os_cpu/windows_aarch64/sve_windows_aarch64.S
+++ b/src/hotspot/os_cpu/windows_aarch64/sve_windows_aarch64.S
@@ -24,19 +24,26 @@
; Support for int get_sve_vector_length();
;
; Returns the current SVE vector length in bytes.
- ; This function uses the INCB instruction which increments a register
- ; by the number of bytes in an SVE vector register.
+ ; This function uses the RDVL instruction which reads a multiple of the
+ ; vector register size into a scalar register.
;
- ; Note: This function will fault if SVE is not available or enabled.
- ; The caller must ensure SVE support is detected before calling.
+ ; Note: This function will fault if SVE is not available or enabled. The
+ ; caller must ensure SVE support is detected before calling.
ALIGN 4
EXPORT get_sve_vector_length
AREA sve_text, CODE
get_sve_vector_length
- mov x0, #0
- incb x0
+ ; Older versions of Visual Studio aren't aware of SVE mnemonics, so we use
+ ; the raw instruction encoding to satisfy the compiler. This function call
+ ; is gated by `VM_Version::supports_sve()`, so this instruction will never
+ ; run on non-SVE hardware.
+ ;
+ ; See https://www.scs.stanford.edu/~zyedidia/arm64/rdvl_r_i.html for a quick
+ ; reference to the instruction encoding.
+
+ DCD 0x04BF5020 ; rdvl x0, #1 (i.e. x0 = 1 * vector_length_in_bytes)
ret
END
diff --git a/src/hotspot/os_cpu/windows_aarch64/threadLS_windows_aarch64.S b/src/hotspot/os_cpu/windows_aarch64/threadLS_windows_aarch64.S
new file mode 100644
index 00000000000..81749b9a372
--- /dev/null
+++ b/src/hotspot/os_cpu/windows_aarch64/threadLS_windows_aarch64.S
@@ -0,0 +1,64 @@
+;
+; Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
+; DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+;
+; This code is free software; you can redistribute it and/or modify it
+; under the terms of the GNU General Public License version 2 only, as
+; published by the Free Software Foundation.
+;
+; This code is distributed in the hope that it will be useful, but WITHOUT
+; ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+; FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+; version 2 for more details (a copy is included in the LICENSE file that
+; accompanied this code).
+;
+; You should have received a copy of the GNU General Public License version
+; 2 along with this work; if not, write to the Free Software Foundation,
+; Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+;
+; Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+; or visit www.oracle.com if you need additional information or have any
+; questions.
+;
+
+ ; JavaThread::aarch64_get_thread_helper()
+ ;
+ ; Optimized TLS access to `Thread::_thr_current` on Windows AArch64.
+ ; Returns the current thread pointer in x0, clobbers x1, while all other
+ ; registers are preserved.
+
+ IMPORT _tls_index
+ IMPORT _jvm_thr_current_tls_offset
+
+ AREA threadls_text, CODE, READONLY
+ ALIGN 4
+
+ ; MSVC-decorated name for: static Thread* JavaThread::aarch64_get_thread_helper()
+ EXPORT |?aarch64_get_thread_helper@JavaThread@@SAPEAVThread@@XZ|
+
+|?aarch64_get_thread_helper@JavaThread@@SAPEAVThread@@XZ| PROC
+
+ ; x18 holds the TEB, 0x58 is a well-defined offset into the TEB on 64-bit
+ ; systems, so the following line loads the thread-local storage pointer
+ ; inside the TEB
+ ldr x1, [x18, #0x58]
+
+ ; Load `_tls_index` and zero-extend it to 64 bits to occupy x0
+ adrp x0, _tls_index
+ ldr w0, [x0, _tls_index]
+
+ ; `x0` holds the index, `x1` holds the array base address (each entry is 64
+ ; bits long), so in the following line, x1 = array_base[_tls_index]
+ ldr x1, [x1, x0, lsl #3]
+
+ ; Load cached TLS offset of `Thread::_thr_current`
+ adrp x0, _jvm_thr_current_tls_offset
+ ldr x0, [x0, _jvm_thr_current_tls_offset]
+
+ ; Load `Thread::_thr_current` value
+ ldr x0, [x1, x0]
+
+ ret
+
+ ENDP
+ END
diff --git a/src/hotspot/os_cpu/windows_aarch64/vm_version_windows_aarch64.cpp b/src/hotspot/os_cpu/windows_aarch64/vm_version_windows_aarch64.cpp
index e78a37b4178..e485ae6b8c2 100644
--- a/src/hotspot/os_cpu/windows_aarch64/vm_version_windows_aarch64.cpp
+++ b/src/hotspot/os_cpu/windows_aarch64/vm_version_windows_aarch64.cpp
@@ -26,19 +26,52 @@
#include "runtime/os.hpp"
#include "runtime/vm_version.hpp"
-// Assembly function to get SVE vector length using INCB instruction
+// Since PF_ARM_SVE_INSTRUCTIONS_AVAILABLE and related constants were added in
+// Windows 11 (version 24H2) and in Windows Server 2025, we define them here for
+// compatibility with older SDK versions.
+#ifndef PF_ARM_SVE_INSTRUCTIONS_AVAILABLE
+#define PF_ARM_SVE_INSTRUCTIONS_AVAILABLE 46
+#endif
+
+#ifndef PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE
+#define PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE 47
+#endif
+
+#ifndef PF_ARM_SVE_BITPERM_INSTRUCTIONS_AVAILABLE
+#define PF_ARM_SVE_BITPERM_INSTRUCTIONS_AVAILABLE 51
+#endif
+
+#ifndef PF_ARM_SHA3_INSTRUCTIONS_AVAILABLE
+#define PF_ARM_SHA3_INSTRUCTIONS_AVAILABLE 64
+#endif
+
+#ifndef PF_ARM_SHA512_INSTRUCTIONS_AVAILABLE
+#define PF_ARM_SHA512_INSTRUCTIONS_AVAILABLE 65
+#endif
+
+#ifndef PF_ARM_V82_FP16_INSTRUCTIONS_AVAILABLE
+#define PF_ARM_V82_FP16_INSTRUCTIONS_AVAILABLE 67
+#endif
+
+// Assembly function to get SVE vector length using RDVL instruction
extern "C" int get_sve_vector_length();
int VM_Version::get_current_sve_vector_length() {
assert(VM_Version::supports_sve(), "should not call this");
- // Use assembly instruction to get the actual SVE vector length
- return VM_Version::supports_sve() ? get_sve_vector_length() : 0; // This value is in bytes
+ return VM_Version::supports_sve() ? get_sve_vector_length() : 0;
}
int VM_Version::set_and_get_current_sve_vector_length(int length) {
assert(VM_Version::supports_sve(), "should not call this");
- // Use assembly instruction to get the SVE vector length
- return VM_Version::supports_sve() ? get_sve_vector_length() : 0; // This value is in bytes
+
+ // Unlike Linux, Windows does not present a way to modify the VL (the
+ // rationale is that the OS expects the application to use the maximum vector
+ // length supported by the hardware), so we simply return the current VL. If
+ // the user sets `MaxVectorSize` that is not the same as the maximum possible
+ // vector length, then the caller (`VM_Version::initialize()`) will print a
+ // warning, set `MaxVectorSize` to the value returned by this function, and
+ // move on.
+ return VM_Version::supports_sve() ? get_sve_vector_length() : 0;
}
void VM_Version::get_os_cpu_info() {
@@ -67,6 +100,10 @@ void VM_Version::get_os_cpu_info() {
if (IsProcessorFeaturePresent(PF_ARM_SVE_BITPERM_INSTRUCTIONS_AVAILABLE)) {
set_feature(CPU_SVEBITPERM);
}
+ if (IsProcessorFeaturePresent(PF_ARM_V82_FP16_INSTRUCTIONS_AVAILABLE)) {
+ set_feature(CPU_FPHP);
+ set_feature(CPU_ASIMDHP);
+ }
if (IsProcessorFeaturePresent(PF_ARM_SHA3_INSTRUCTIONS_AVAILABLE)) {
set_feature(CPU_SHA3);
}
diff --git a/src/hotspot/share/asm/codeBuffer.cpp b/src/hotspot/share/asm/codeBuffer.cpp
index 854cf73049b..c6475050592 100644
--- a/src/hotspot/share/asm/codeBuffer.cpp
+++ b/src/hotspot/share/asm/codeBuffer.cpp
@@ -417,7 +417,7 @@ void CodeSection::expand_locs(int new_capacity) {
new_capacity = old_capacity * 2;
relocInfo* locs_start;
if (_locs_own) {
- locs_start = REALLOC_RESOURCE_ARRAY(relocInfo, _locs_start, old_capacity, new_capacity);
+ locs_start = REALLOC_RESOURCE_ARRAY(_locs_start, old_capacity, new_capacity);
} else {
locs_start = NEW_RESOURCE_ARRAY(relocInfo, new_capacity);
Copy::conjoint_jbytes(_locs_start, locs_start, old_capacity * sizeof(relocInfo));
diff --git a/src/hotspot/share/c1/c1_GraphBuilder.cpp b/src/hotspot/share/c1/c1_GraphBuilder.cpp
index f910ecadc16..db55b8c5fa8 100644
--- a/src/hotspot/share/c1/c1_GraphBuilder.cpp
+++ b/src/hotspot/share/c1/c1_GraphBuilder.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -3558,7 +3558,7 @@ const char* GraphBuilder::check_can_parse(ciMethod* callee) const {
// negative filter: should callee NOT be inlined? returns null, ok to inline, or rejection msg
const char* GraphBuilder::should_not_inline(ciMethod* callee) const {
- if ( compilation()->directive()->should_not_inline(callee)) return "disallowed by CompileCommand";
+ if ( compilation()->directive()->should_not_inline(callee, compilation()->env()->comp_level())) return "disallowed by CompileCommand";
if ( callee->dont_inline()) return "don't inline by annotation";
return nullptr;
}
diff --git a/src/hotspot/share/c1/c1_LIRGenerator.hpp b/src/hotspot/share/c1/c1_LIRGenerator.hpp
index ec0ea5dc047..8e30d05af6d 100644
--- a/src/hotspot/share/c1/c1_LIRGenerator.hpp
+++ b/src/hotspot/share/c1/c1_LIRGenerator.hpp
@@ -330,8 +330,9 @@ class LIRGenerator: public InstructionVisitor, public BlockClosure {
// volatile field operations are never patchable because a klass
// must be loaded to know it's volatile which means that the offset
- // it always known as well.
+ // is always known as well.
void volatile_field_store(LIR_Opr value, LIR_Address* address, CodeEmitInfo* info);
+ // volatile_field_load provides trailing membar semantics
void volatile_field_load(LIR_Address* address, LIR_Opr result, CodeEmitInfo* info);
void put_Object_unsafe(LIR_Opr src, LIR_Opr offset, LIR_Opr data, BasicType type, bool is_volatile);
diff --git a/src/hotspot/share/c1/c1_Runtime1.cpp b/src/hotspot/share/c1/c1_Runtime1.cpp
index 38f563935e0..41504c74dd2 100644
--- a/src/hotspot/share/c1/c1_Runtime1.cpp
+++ b/src/hotspot/share/c1/c1_Runtime1.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -982,6 +982,7 @@ JRT_ENTRY(void, Runtime1::patch_code(JavaThread* current, StubId stub_id ))
constantPoolHandle constants(current, caller_method->constants());
LinkResolver::resolve_field_access(result, constants, field_access.index(), caller_method, Bytecodes::java_code(code), CHECK);
patch_field_offset = result.offset();
+ patch_field_type = result.field_type();
// If we're patching a field which is volatile then at compile it
// must not have been know to be volatile, so the generated code
@@ -993,22 +994,6 @@ JRT_ENTRY(void, Runtime1::patch_code(JavaThread* current, StubId stub_id ))
// handling in the volatile case.
deoptimize_for_volatile = result.access_flags().is_volatile();
-
- // If we are patching a field which should be atomic, then
- // the generated code is not correct either, force deoptimizing.
- // We need to only cover T_LONG and T_DOUBLE fields, as we can
- // break access atomicity only for them.
-
- // Strictly speaking, the deoptimization on 64-bit platforms
- // is unnecessary, and T_LONG stores on 32-bit platforms need
- // to be handled by special patching code when AlwaysAtomicAccesses
- // becomes product feature. At this point, we are still going
- // for the deoptimization for consistency against volatile
- // accesses.
-
- patch_field_type = result.field_type();
- deoptimize_for_atomic = (AlwaysAtomicAccesses && (patch_field_type == T_DOUBLE || patch_field_type == T_LONG));
-
} else if (load_klass_or_mirror_patch_id) {
Klass* k = nullptr;
switch (code) {
@@ -1081,17 +1066,12 @@ JRT_ENTRY(void, Runtime1::patch_code(JavaThread* current, StubId stub_id ))
ShouldNotReachHere();
}
- if (deoptimize_for_volatile || deoptimize_for_atomic) {
- // At compile time we assumed the field wasn't volatile/atomic but after
- // loading it turns out it was volatile/atomic so we have to throw the
+ if (deoptimize_for_volatile) {
+ // At compile time we assumed the field wasn't volatile but after
+ // loading it turns out it was volatile so we have to throw the
// compiled code out and let it be regenerated.
if (TracePatching) {
- if (deoptimize_for_volatile) {
- tty->print_cr("Deoptimizing for patching volatile field reference");
- }
- if (deoptimize_for_atomic) {
- tty->print_cr("Deoptimizing for patching atomic field reference");
- }
+ tty->print_cr("Deoptimizing for patching volatile field reference");
}
// It's possible the nmethod was invalidated in the last
diff --git a/src/hotspot/share/cds/aotMetaspace.hpp b/src/hotspot/share/cds/aotMetaspace.hpp
index 4607a936abe..2236bae91f3 100644
--- a/src/hotspot/share/cds/aotMetaspace.hpp
+++ b/src/hotspot/share/cds/aotMetaspace.hpp
@@ -105,7 +105,9 @@ public:
// Return true if given address is in the shared metaspace regions (i.e., excluding the
// mapped heap region.)
static bool in_aot_cache(const void* p) {
- return MetaspaceObj::in_aot_cache((const MetaspaceObj*)p);
+ // This function is called only after the AOT metaspace is initialized, so
+ // we can skip init checks.
+ return MetaspaceObj::is_pointer_in_aot_cache_no_init_check(p);
}
static void set_aot_metaspace_range(void* base, void *static_top, void* top) NOT_CDS_RETURN;
diff --git a/src/hotspot/share/cds/aotStreamedHeapLoader.cpp b/src/hotspot/share/cds/aotStreamedHeapLoader.cpp
index 39f735543cd..7f9f8cf0628 100644
--- a/src/hotspot/share/cds/aotStreamedHeapLoader.cpp
+++ b/src/hotspot/share/cds/aotStreamedHeapLoader.cpp
@@ -797,7 +797,7 @@ void AOTStreamedHeapLoader::cleanup() {
Universe::vm_global()->release(&handles[num_null_handles], num_handles - num_null_handles);
}
- FREE_C_HEAP_ARRAY(void*, _object_index_to_heap_object_table);
+ FREE_C_HEAP_ARRAY(_object_index_to_heap_object_table);
// Unmap regions
FileMapInfo::current_info()->unmap_region(AOTMetaspace::hp);
diff --git a/src/hotspot/share/cds/archiveBuilder.cpp b/src/hotspot/share/cds/archiveBuilder.cpp
index 21eef3d7b0b..cf51897c2f1 100644
--- a/src/hotspot/share/cds/archiveBuilder.cpp
+++ b/src/hotspot/share/cds/archiveBuilder.cpp
@@ -1171,7 +1171,7 @@ void ArchiveBuilder::write_archive(FileMapInfo* mapinfo, AOTMappedHeapInfo* mapp
AOTMapLogger::dumptime_log(this, mapinfo, mapped_heap_info, streamed_heap_info, bitmap, bitmap_size_in_bytes);
}
CDS_JAVA_HEAP_ONLY(HeapShared::destroy_archived_object_cache());
- FREE_C_HEAP_ARRAY(char, bitmap);
+ FREE_C_HEAP_ARRAY(bitmap);
}
void ArchiveBuilder::write_region(FileMapInfo* mapinfo, int region_idx, DumpRegion* dump_region, bool read_only, bool allow_exec) {
diff --git a/src/hotspot/share/cds/archiveUtils.cpp b/src/hotspot/share/cds/archiveUtils.cpp
index 6e0608e196b..7985c62d67b 100644
--- a/src/hotspot/share/cds/archiveUtils.cpp
+++ b/src/hotspot/share/cds/archiveUtils.cpp
@@ -416,13 +416,22 @@ void DumpRegion::report_gaps(DumpAllocStats* stats) {
stats->record_gap(checked_cast(node->key().gap_bytes()));
return true;
});
+
+ double unfilled_percent = 0.0;
if (_gap_tree.size() > 0) {
- log_warning(aot)("Unexpected %zu gaps (%zu bytes) for Klass alignment",
- _gap_tree.size(), _total_gap_bytes);
+ unfilled_percent = percent_of(_total_gap_bytes, _total_gap_allocs);
+ if (unfilled_percent > 5.0) {
+ // We have a limited number of small objects, so some small gaps may remain
+ // unfilled. If more than 5% of the gaps are unfilled, this likely indicates
+ // a systematic error that should be investigated. Otherwise, do not warn to
+ // avoid noise.
+ log_warning(aot)("Unexpected %zu gaps (%zu bytes) for Klass alignment",
+ _gap_tree.size(), _total_gap_bytes);
+ }
}
if (_total_gap_allocs > 0) {
- log_info(aot)("Allocated %zu objects of %zu bytes in gaps (remain = %zu bytes)",
- _total_gap_allocs, _total_gap_bytes_used, _total_gap_bytes);
+ log_info(aot)("Allocated %zu objects of %zu bytes in gaps (remain = %zu bytes, %.2f%%)",
+ _total_gap_allocs, _total_gap_bytes_used, _total_gap_bytes, unfilled_percent);
}
}
diff --git a/src/hotspot/share/cds/cdsConfig.cpp b/src/hotspot/share/cds/cdsConfig.cpp
index ecf3c6d2231..21066f76932 100644
--- a/src/hotspot/share/cds/cdsConfig.cpp
+++ b/src/hotspot/share/cds/cdsConfig.cpp
@@ -520,7 +520,7 @@ static void substitute_aot_filename(JVMFlagsEnum flag_enum) {
JVMFlag::Error err = JVMFlagAccess::set_ccstr(flag, &new_filename, JVMFlagOrigin::ERGONOMIC);
assert(err == JVMFlag::SUCCESS, "must never fail");
}
- FREE_C_HEAP_ARRAY(char, new_filename);
+ FREE_C_HEAP_ARRAY(new_filename);
}
void CDSConfig::check_aotmode_record() {
diff --git a/src/hotspot/share/cds/cds_globals.hpp b/src/hotspot/share/cds/cds_globals.hpp
index 447914b3101..7df498ca5b9 100644
--- a/src/hotspot/share/cds/cds_globals.hpp
+++ b/src/hotspot/share/cds/cds_globals.hpp
@@ -163,7 +163,7 @@
\
product(uint, AOTCodeMaxSize, 10*M, DIAGNOSTIC, \
"Buffer size in bytes for AOT code caching") \
- range(1*M, max_jint) \
+ range(1*M, CODE_CACHE_SIZE_LIMIT) \
\
product(bool, AbortVMOnAOTCodeFailure, false, DIAGNOSTIC, \
"Abort VM on the first occurrence of AOT code load or store " \
diff --git a/src/hotspot/share/cds/classListWriter.cpp b/src/hotspot/share/cds/classListWriter.cpp
index 8e1f298e8e3..c90e233df73 100644
--- a/src/hotspot/share/cds/classListWriter.cpp
+++ b/src/hotspot/share/cds/classListWriter.cpp
@@ -49,7 +49,7 @@ void ClassListWriter::init() {
_classlist_file->print_cr("# This file is generated via the -XX:DumpLoadedClassList= option");
_classlist_file->print_cr("# and is used at CDS archive dump time (see -Xshare:dump).");
_classlist_file->print_cr("#");
- FREE_C_HEAP_ARRAY(char, list_name);
+ FREE_C_HEAP_ARRAY(list_name);
}
}
diff --git a/src/hotspot/share/cds/filemap.cpp b/src/hotspot/share/cds/filemap.cpp
index 38502b2b2d8..1ed9979ff85 100644
--- a/src/hotspot/share/cds/filemap.cpp
+++ b/src/hotspot/share/cds/filemap.cpp
@@ -402,7 +402,7 @@ public:
~FileHeaderHelper() {
if (_header != nullptr) {
- FREE_C_HEAP_ARRAY(char, _header);
+ FREE_C_HEAP_ARRAY(_header);
}
if (_fd != -1) {
::close(_fd);
@@ -1362,6 +1362,13 @@ bool FileMapInfo::map_aot_code_region(ReservedSpace rs) {
return false;
} else {
assert(mapped_base == requested_base, "must be");
+
+ if (VerifySharedSpaces && !r->check_region_crc(mapped_base)) {
+ aot_log_error(aot)("region %d CRC error", AOTMetaspace::ac);
+ os::unmap_memory(mapped_base, r->used_aligned());
+ return false;
+ }
+
r->set_mapped_from_file(true);
r->set_mapped_base(mapped_base);
aot_log_info(aot)("Mapped static region #%d at base " INTPTR_FORMAT " top " INTPTR_FORMAT " (%s)",
@@ -1464,14 +1471,14 @@ size_t FileMapInfo::read_bytes(void* buffer, size_t count) {
return count;
}
-// Get the total size in bytes of a read only region
+// Get the total size in bytes of all mapped read only region
size_t FileMapInfo::readonly_total() {
size_t total = 0;
- if (current_info() != nullptr) {
+ if (current_info() != nullptr && current_info()->is_mapped()) {
FileMapRegion* r = FileMapInfo::current_info()->region_at(AOTMetaspace::ro);
if (r->read_only()) total += r->used();
}
- if (dynamic_info() != nullptr) {
+ if (dynamic_info() != nullptr && current_info()->is_mapped()) {
FileMapRegion* r = FileMapInfo::dynamic_info()->region_at(AOTMetaspace::ro);
if (r->read_only()) total += r->used();
}
diff --git a/src/hotspot/share/ci/ciEnv.hpp b/src/hotspot/share/ci/ciEnv.hpp
index b384ff47a89..8167697e84b 100644
--- a/src/hotspot/share/ci/ciEnv.hpp
+++ b/src/hotspot/share/ci/ciEnv.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -236,6 +236,9 @@ private:
ciInstanceKlass* declared_holder = get_instance_klass_for_declared_method_holder(holder);
return _factory->get_unloaded_method(declared_holder, name, signature, accessor);
}
+ InstanceKlass::ClassState get_cached_init_state(uint id) {
+ return (InstanceKlass::ClassState)_factory->cached_init_state(id);
+ }
// Get a ciKlass representing an unloaded klass.
// Ensures uniqueness of the result.
diff --git a/src/hotspot/share/ci/ciInstanceKlass.cpp b/src/hotspot/share/ci/ciInstanceKlass.cpp
index 6243258acd9..293063d0b68 100644
--- a/src/hotspot/share/ci/ciInstanceKlass.cpp
+++ b/src/hotspot/share/ci/ciInstanceKlass.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -136,12 +136,14 @@ ciInstanceKlass::ciInstanceKlass(ciSymbol* name,
// ------------------------------------------------------------------
-// ciInstanceKlass::compute_shared_is_initialized
-void ciInstanceKlass::compute_shared_init_state() {
- GUARDED_VM_ENTRY(
- InstanceKlass* ik = get_instanceKlass();
- _init_state = ik->init_state();
- )
+InstanceKlass::ClassState ciInstanceKlass::compute_init_state() {
+ if (_is_shared && is_loaded()) {
+ // Return cached init state of shared klass
+ ciEnv* env = CURRENT_ENV;
+ assert(env->task() != nullptr, "only calls from compilation are expected here");
+ return env->get_cached_init_state(ident());
+ }
+ return _init_state;
}
// ------------------------------------------------------------------
@@ -319,11 +321,11 @@ void ciInstanceKlass::print_impl(outputStream* st) {
bool_to_str(has_subklass()),
layout_helper());
- _flags.print_klass_flags();
+ _flags.print_klass_flags(st);
if (_super) {
st->print(" super=");
- _super->print_name();
+ _super->print_name_on(st);
}
if (_java_mirror) {
st->print(" mirror=PRESENT");
diff --git a/src/hotspot/share/ci/ciInstanceKlass.hpp b/src/hotspot/share/ci/ciInstanceKlass.hpp
index a84c63981c9..221e6806b5a 100644
--- a/src/hotspot/share/ci/ciInstanceKlass.hpp
+++ b/src/hotspot/share/ci/ciInstanceKlass.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -106,43 +106,36 @@ protected:
bool is_shared() { return _is_shared; }
- void compute_shared_init_state();
+ InstanceKlass::ClassState compute_init_state();
bool compute_shared_has_subklass();
int compute_nonstatic_fields();
GrowableArray* compute_nonstatic_fields_impl(GrowableArray* super_fields);
bool compute_has_trusted_loader();
- // Update the init_state for shared klasses
- void update_if_shared(InstanceKlass::ClassState expected) {
- if (_is_shared && _init_state != expected) {
- if (is_loaded()) compute_shared_init_state();
- }
- }
-
public:
// Has this klass been initialized?
bool is_initialized() {
- update_if_shared(InstanceKlass::fully_initialized);
- return _init_state == InstanceKlass::fully_initialized;
+ InstanceKlass::ClassState state = compute_init_state();
+ return state == InstanceKlass::fully_initialized;
}
bool is_not_initialized() {
- update_if_shared(InstanceKlass::fully_initialized);
- return _init_state < InstanceKlass::being_initialized;
+ InstanceKlass::ClassState state = compute_init_state();
+ return state < InstanceKlass::being_initialized;
}
// Is this klass being initialized?
bool is_being_initialized() {
- update_if_shared(InstanceKlass::being_initialized);
- return _init_state == InstanceKlass::being_initialized;
+ InstanceKlass::ClassState state = compute_init_state();
+ return state == InstanceKlass::being_initialized;
}
// Has this klass been linked?
bool is_linked() {
- update_if_shared(InstanceKlass::linked);
- return _init_state >= InstanceKlass::linked;
+ InstanceKlass::ClassState state = compute_init_state();
+ return state >= InstanceKlass::linked;
}
// Is this klass in error state?
bool is_in_error_state() {
- update_if_shared(InstanceKlass::initialization_error);
- return _init_state == InstanceKlass::initialization_error;
+ InstanceKlass::ClassState state = compute_init_state();
+ return state == InstanceKlass::initialization_error;
}
// General klass information.
diff --git a/src/hotspot/share/ci/ciObjectFactory.cpp b/src/hotspot/share/ci/ciObjectFactory.cpp
index 2af5d812922..d3bef01f852 100644
--- a/src/hotspot/share/ci/ciObjectFactory.cpp
+++ b/src/hotspot/share/ci/ciObjectFactory.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -48,6 +48,7 @@
#include "gc/shared/collectedHeap.inline.hpp"
#include "memory/allocation.inline.hpp"
#include "memory/universe.hpp"
+#include "oops/instanceKlass.hpp"
#include "oops/oop.inline.hpp"
#include "oops/trainingData.hpp"
#include "runtime/handles.inline.hpp"
@@ -83,6 +84,7 @@ ciObjectFactory::ciObjectFactory(Arena* arena,
int expected_size)
: _arena(arena),
_ci_metadata(arena, expected_size, 0, nullptr),
+ _cached_init_state(arena, _shared_ident_limit, 0, (u1)0),
_unloaded_methods(arena, 4, 0, nullptr),
_unloaded_klasses(arena, 8, 0, nullptr),
_unloaded_instances(arena, 4, 0, nullptr),
@@ -97,6 +99,28 @@ ciObjectFactory::ciObjectFactory(Arena* arena,
// If the shared ci objects exist append them to this factory's objects
if (_shared_ci_metadata != nullptr) {
_ci_metadata.appendAll(_shared_ci_metadata);
+ // ciInstanceKlass for well-known class is shared by all
+ // compiler threads and can be updated concurrently by
+ // other compiler threads during compilation.
+ // Make local copy of class state to avoid state change
+ // during compilation.
+ int len = _ci_metadata.length();
+ for (int i = 0; i < len; i++) {
+ ciMetadata* obj = _ci_metadata.at(i);
+ if (obj->is_loaded() && obj->is_instance_klass()) {
+ ciInstanceKlass* cik = obj->as_instance_klass();
+ precond(cik->is_shared());
+ InstanceKlass::ClassState current_state = cik->_init_state;
+ InstanceKlass::ClassState state = InstanceKlass::fully_initialized;
+ if (current_state != state) {
+ GUARDED_VM_ENTRY( state = cik->get_instanceKlass()->init_state(); )
+ // Update state of shared ciInstanceKlass
+ cik->_init_state = state;
+ }
+ // Cache state for current compilation
+ _cached_init_state.at_put_grow(cik->ident(), (u1)state, 0);
+ }
+ }
}
}
diff --git a/src/hotspot/share/ci/ciObjectFactory.hpp b/src/hotspot/share/ci/ciObjectFactory.hpp
index fd7ca6bb801..c578aecb564 100644
--- a/src/hotspot/share/ci/ciObjectFactory.hpp
+++ b/src/hotspot/share/ci/ciObjectFactory.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -47,6 +47,8 @@ private:
Arena* _arena;
GrowableArray _ci_metadata;
+ // Local copy of shared ciInstanceKlass init state for current compilation
+ GrowableArray _cached_init_state;
GrowableArray _unloaded_methods;
GrowableArray _unloaded_klasses;
GrowableArray _unloaded_instances;
@@ -103,6 +105,11 @@ public:
ciMetadata* cached_metadata(Metadata* key);
ciSymbol* get_symbol(Symbol* key);
+ // Get cached init state of shared ciInstanceKlass
+ u1 cached_init_state(uint id) {
+ return _cached_init_state.at(id);
+ }
+
// Get the ciSymbol corresponding to one of the vmSymbols.
static ciSymbol* vm_symbol_at(vmSymbolID index);
diff --git a/src/hotspot/share/ci/ciReplay.cpp b/src/hotspot/share/ci/ciReplay.cpp
index 6266c024260..35522e75877 100644
--- a/src/hotspot/share/ci/ciReplay.cpp
+++ b/src/hotspot/share/ci/ciReplay.cpp
@@ -600,7 +600,7 @@ class CompileReplay : public StackObj {
_nesting.check(); // Check if a reallocation in the resource arena is safe
int new_length = _buffer_length * 2;
// Next call will throw error in case of OOM.
- _buffer = REALLOC_RESOURCE_ARRAY(char, _buffer, _buffer_length, new_length);
+ _buffer = REALLOC_RESOURCE_ARRAY(_buffer, _buffer_length, new_length);
_buffer_length = new_length;
}
if (c == '\n') {
diff --git a/src/hotspot/share/classfile/classFileParser.cpp b/src/hotspot/share/classfile/classFileParser.cpp
index a9ea6fbea11..d5ee16fec32 100644
--- a/src/hotspot/share/classfile/classFileParser.cpp
+++ b/src/hotspot/share/classfile/classFileParser.cpp
@@ -2363,8 +2363,8 @@ Method* ClassFileParser::parse_method(const ClassFileStream* const cfs,
}
if (lvt_cnt == max_lvt_cnt) {
max_lvt_cnt <<= 1;
- localvariable_table_length = REALLOC_RESOURCE_ARRAY(u2, localvariable_table_length, lvt_cnt, max_lvt_cnt);
- localvariable_table_start = REALLOC_RESOURCE_ARRAY(const unsafe_u2*, localvariable_table_start, lvt_cnt, max_lvt_cnt);
+ localvariable_table_length = REALLOC_RESOURCE_ARRAY(localvariable_table_length, lvt_cnt, max_lvt_cnt);
+ localvariable_table_start = REALLOC_RESOURCE_ARRAY(localvariable_table_start, lvt_cnt, max_lvt_cnt);
}
localvariable_table_start[lvt_cnt] =
parse_localvariable_table(cfs,
@@ -2393,8 +2393,8 @@ Method* ClassFileParser::parse_method(const ClassFileStream* const cfs,
// Parse local variable type table
if (lvtt_cnt == max_lvtt_cnt) {
max_lvtt_cnt <<= 1;
- localvariable_type_table_length = REALLOC_RESOURCE_ARRAY(u2, localvariable_type_table_length, lvtt_cnt, max_lvtt_cnt);
- localvariable_type_table_start = REALLOC_RESOURCE_ARRAY(const unsafe_u2*, localvariable_type_table_start, lvtt_cnt, max_lvtt_cnt);
+ localvariable_type_table_length = REALLOC_RESOURCE_ARRAY(localvariable_type_table_length, lvtt_cnt, max_lvtt_cnt);
+ localvariable_type_table_start = REALLOC_RESOURCE_ARRAY(localvariable_type_table_start, lvtt_cnt, max_lvtt_cnt);
}
localvariable_type_table_start[lvtt_cnt] =
parse_localvariable_table(cfs,
diff --git a/src/hotspot/share/classfile/classLoader.cpp b/src/hotspot/share/classfile/classLoader.cpp
index eced83577cb..bf00185ffa9 100644
--- a/src/hotspot/share/classfile/classLoader.cpp
+++ b/src/hotspot/share/classfile/classLoader.cpp
@@ -96,9 +96,18 @@ static JImageClose_t JImageClose = nullptr;
static JImageFindResource_t JImageFindResource = nullptr;
static JImageGetResource_t JImageGetResource = nullptr;
-// JimageFile pointer, or null if exploded JDK build.
+// JImageFile pointer, or null if exploded JDK build.
static JImageFile* JImage_file = nullptr;
+// PreviewMode status to control preview behaviour. JImage_file is unusable
+// for normal lookup until (Preview_mode != PREVIEW_MODE_UNINITIALIZED).
+enum PreviewMode {
+ PREVIEW_MODE_UNINITIALIZED = 0,
+ PREVIEW_MODE_DEFAULT = 1,
+ PREVIEW_MODE_ENABLE_PREVIEW = 2
+};
+static PreviewMode Preview_mode = PREVIEW_MODE_UNINITIALIZED;
+
// Globals
PerfCounter* ClassLoader::_perf_accumulated_time = nullptr;
@@ -154,7 +163,7 @@ void ClassLoader::print_counters(outputStream *st) {
GrowableArray* ClassLoader::_patch_mod_entries = nullptr;
GrowableArray* ClassLoader::_exploded_entries = nullptr;
-ClassPathEntry* ClassLoader::_jrt_entry = nullptr;
+ClassPathImageEntry* ClassLoader::_jrt_entry = nullptr;
ClassPathEntry* volatile ClassLoader::_first_append_entry_list = nullptr;
ClassPathEntry* volatile ClassLoader::_last_append_entry = nullptr;
@@ -171,15 +180,6 @@ static bool string_starts_with(const char* str, const char* str_to_find) {
}
#endif
-static const char* get_jimage_version_string() {
- static char version_string[10] = "";
- if (version_string[0] == '\0') {
- jio_snprintf(version_string, sizeof(version_string), "%d.%d",
- VM_Version::vm_major_version(), VM_Version::vm_minor_version());
- }
- return (const char*)version_string;
-}
-
bool ClassLoader::string_ends_with(const char* str, const char* str_to_find) {
size_t str_len = strlen(str);
size_t str_to_find_len = strlen(str_to_find);
@@ -234,6 +234,69 @@ Symbol* ClassLoader::package_from_class_name(const Symbol* name, bool* bad_class
return SymbolTable::new_symbol(name, pointer_delta_as_int(start, base), pointer_delta_as_int(end, base));
}
+// --------------------------------
+// The following jimage_xxx static functions encapsulate all JImage_file and Preview_mode access.
+// This is done to make it easy to reason about the JImage file state (exists vs initialized etc.).
+
+// Opens the named JImage file and sets the JImage file reference.
+// Returns true if opening the JImage file was successful (see also jimage_is_open()).
+static bool jimage_open(const char* modules_path) {
+ // Currently 'error' is not set to anything useful, so ignore it here.
+ jint error;
+ JImage_file = (*JImageOpen)(modules_path, &error);
+ if (Arguments::has_jimage() && JImage_file == nullptr) {
+ // The modules file exists but is unreadable or corrupt
+ vm_exit_during_initialization(err_msg("Unable to load %s", modules_path));
+ }
+ return JImage_file != nullptr;
+}
+
+// Closes and clears the JImage file reference (this will only be called during shutdown).
+static void jimage_close() {
+ if (JImage_file != nullptr) {
+ (*JImageClose)(JImage_file);
+ JImage_file = nullptr;
+ }
+}
+
+// Returns whether a JImage file was opened (but NOT whether it was initialized yet).
+static bool jimage_is_open() {
+ return JImage_file != nullptr;
+}
+
+// Returns the JImage file reference (which may or may not be initialized).
+static JImageFile* jimage_non_null() {
+ assert(jimage_is_open(), "should have been opened by ClassLoader::lookup_vm_options "
+ "and remains open throughout normal JVM lifetime");
+ return JImage_file;
+}
+
+// Returns true if jimage_init() has been called. Once the JImage file is initialized,
+// jimage_is_preview_enabled() can be called to correctly determine the access mode.
+static bool jimage_is_initialized() {
+ return jimage_is_open() && Preview_mode != PREVIEW_MODE_UNINITIALIZED;
+}
+
+// Returns the access mode for an initialized JImage file (reflects --enable-preview).
+static bool is_preview_enabled() {
+ return Preview_mode == PREVIEW_MODE_ENABLE_PREVIEW;
+}
+
+// Looks up the location of a named JImage resource. This "raw" lookup function allows
+// the preview mode to be manually specified, so must not be accessible outside this
+// class. ClassPathImageEntry manages all calls for resources after startup is complete.
+static JImageLocationRef jimage_find_resource(const char* module_name,
+ const char* file_name,
+ bool is_preview,
+ jlong* size) {
+ return ((*JImageFindResource)(jimage_non_null(),
+ module_name,
+ file_name,
+ is_preview,
+ size));
+}
+// --------------------------------
+
// Given a fully qualified package name, find its defining package in the class loader's
// package entry table.
PackageEntry* ClassLoader::get_package_entry(Symbol* pkg_name, ClassLoaderData* loader_data) {
@@ -251,7 +314,7 @@ const char* ClassPathEntry::copy_path(const char* path) {
}
ClassPathDirEntry::~ClassPathDirEntry() {
- FREE_C_HEAP_ARRAY(char, _dir);
+ FREE_C_HEAP_ARRAY(_dir);
}
ClassFileStream* ClassPathDirEntry::open_stream(JavaThread* current, const char* name) {
@@ -280,7 +343,7 @@ ClassFileStream* ClassPathDirEntry::open_stream(JavaThread* current, const char*
#ifdef ASSERT
// Freeing path is a no-op here as buffer prevents it from being reclaimed. But we keep it for
// debug builds so that we guard against use-after-free bugs.
- FREE_RESOURCE_ARRAY_IN_THREAD(current, char, path, path_len);
+ FREE_RESOURCE_ARRAY_IN_THREAD(current, path, path_len);
#endif
// We don't verify the length of the classfile stream fits in an int, but this is the
// bootloader so we have control of this.
@@ -291,7 +354,7 @@ ClassFileStream* ClassPathDirEntry::open_stream(JavaThread* current, const char*
}
}
}
- FREE_RESOURCE_ARRAY_IN_THREAD(current, char, path, path_len);
+ FREE_RESOURCE_ARRAY_IN_THREAD(current, path, path_len);
return nullptr;
}
@@ -302,7 +365,7 @@ ClassPathZipEntry::ClassPathZipEntry(jzfile* zip, const char* zip_name) : ClassP
ClassPathZipEntry::~ClassPathZipEntry() {
ZipLibrary::close(_zip);
- FREE_C_HEAP_ARRAY(char, _zip_name);
+ FREE_C_HEAP_ARRAY(_zip_name);
}
bool ClassPathZipEntry::has_entry(JavaThread* current, const char* name) {
@@ -372,28 +435,15 @@ ClassFileStream* ClassPathZipEntry::open_stream(JavaThread* current, const char*
DEBUG_ONLY(ClassPathImageEntry* ClassPathImageEntry::_singleton = nullptr;)
-JImageFile* ClassPathImageEntry::jimage() const {
- return JImage_file;
-}
-
-JImageFile* ClassPathImageEntry::jimage_non_null() const {
- assert(ClassLoader::has_jrt_entry(), "must be");
- assert(jimage() != nullptr, "should have been opened by ClassLoader::lookup_vm_options "
- "and remained throughout normal JVM lifetime");
- return jimage();
-}
-
void ClassPathImageEntry::close_jimage() {
- if (jimage() != nullptr) {
- (*JImageClose)(jimage());
- JImage_file = nullptr;
- }
+ jimage_close();
}
-ClassPathImageEntry::ClassPathImageEntry(JImageFile* jimage, const char* name) :
+ClassPathImageEntry::ClassPathImageEntry(const char* name) :
ClassPathEntry() {
- guarantee(jimage != nullptr, "jimage file is null");
+ guarantee(jimage_is_initialized(), "jimage is not initialized");
guarantee(name != nullptr, "jimage file name is null");
+
assert(_singleton == nullptr, "VM supports only one jimage");
DEBUG_ONLY(_singleton = this);
size_t len = strlen(name) + 1;
@@ -412,6 +462,8 @@ ClassFileStream* ClassPathImageEntry::open_stream(JavaThread* current, const cha
// 2. A package is in at most one module in the jimage file.
//
ClassFileStream* ClassPathImageEntry::open_stream_for_loader(JavaThread* current, const char* name, ClassLoaderData* loader_data) {
+ const bool is_preview = is_preview_enabled();
+
jlong size;
JImageLocationRef location = 0;
@@ -420,7 +472,7 @@ ClassFileStream* ClassPathImageEntry::open_stream_for_loader(JavaThread* current
if (pkg_name != nullptr) {
if (!Universe::is_module_initialized()) {
- location = (*JImageFindResource)(jimage_non_null(), JAVA_BASE_NAME, get_jimage_version_string(), name, &size);
+ location = jimage_find_resource(JAVA_BASE_NAME, name, is_preview, &size);
} else {
PackageEntry* package_entry = ClassLoader::get_package_entry(pkg_name, loader_data);
if (package_entry != nullptr) {
@@ -431,7 +483,7 @@ ClassFileStream* ClassPathImageEntry::open_stream_for_loader(JavaThread* current
assert(module->is_named(), "Boot classLoader package is in unnamed module");
const char* module_name = module->name()->as_C_string();
if (module_name != nullptr) {
- location = (*JImageFindResource)(jimage_non_null(), module_name, get_jimage_version_string(), name, &size);
+ location = jimage_find_resource(module_name, name, is_preview, &size);
}
}
}
@@ -444,7 +496,7 @@ ClassFileStream* ClassPathImageEntry::open_stream_for_loader(JavaThread* current
char* data = NEW_RESOURCE_ARRAY(char, size);
(*JImageGetResource)(jimage_non_null(), location, data, size);
// Resource allocated
- assert(this == (ClassPathImageEntry*)ClassLoader::get_jrt_entry(), "must be");
+ assert(this == ClassLoader::get_jrt_entry(), "must be");
return new ClassFileStream((u1*)data,
checked_cast(size),
_name,
@@ -454,16 +506,9 @@ ClassFileStream* ClassPathImageEntry::open_stream_for_loader(JavaThread* current
return nullptr;
}
-JImageLocationRef ClassLoader::jimage_find_resource(JImageFile* jf,
- const char* module_name,
- const char* file_name,
- jlong &size) {
- return ((*JImageFindResource)(jf, module_name, get_jimage_version_string(), file_name, &size));
-}
-
bool ClassPathImageEntry::is_modules_image() const {
assert(this == _singleton, "VM supports a single jimage");
- assert(this == (ClassPathImageEntry*)ClassLoader::get_jrt_entry(), "must be used for jrt entry");
+ assert(this == ClassLoader::get_jrt_entry(), "must be used for jrt entry");
return true;
}
@@ -618,14 +663,15 @@ void ClassLoader::setup_bootstrap_search_path_impl(JavaThread* current, const ch
struct stat st;
if (os::stat(path, &st) == 0) {
// Directory found
- if (JImage_file != nullptr) {
+ if (jimage_is_open()) {
assert(Arguments::has_jimage(), "sanity check");
const char* canonical_path = get_canonical_path(path, current);
assert(canonical_path != nullptr, "canonical_path issue");
- _jrt_entry = new ClassPathImageEntry(JImage_file, canonical_path);
+ // Hand over lifecycle control of the JImage file to the _jrt_entry singleton
+ // (see ClassPathImageEntry::close_jimage). The image must be initialized by now.
+ _jrt_entry = new ClassPathImageEntry(canonical_path);
assert(_jrt_entry != nullptr && _jrt_entry->is_modules_image(), "No java runtime image present");
- assert(_jrt_entry->jimage() != nullptr, "No java runtime image");
} // else it's an exploded build.
} else {
// If path does not exist, exit
@@ -645,7 +691,7 @@ void ClassLoader::setup_bootstrap_search_path_impl(JavaThread* current, const ch
static const char* get_exploded_module_path(const char* module_name, bool c_heap) {
const char *home = Arguments::get_java_home();
const char file_sep = os::file_separator()[0];
- // 10 represents the length of "modules" + 2 file separators + \0
+ // 10 represents the length of "modules" (7) + 2 file separators + \0
size_t len = strlen(home) + strlen(module_name) + 10;
char *path = c_heap ? NEW_C_HEAP_ARRAY(char, len, mtModule) : NEW_RESOURCE_ARRAY(char, len);
jio_snprintf(path, len, "%s%cmodules%c%s", home, file_sep, file_sep, module_name);
@@ -707,6 +753,7 @@ ClassPathEntry* ClassLoader::create_class_path_entry(JavaThread* current,
if (zip != nullptr && error_msg == nullptr) {
new_entry = new ClassPathZipEntry(zip, path);
} else {
+ log_info(class, path)("failed: %s, err: %s", path, error_msg);
return nullptr;
}
log_info(class, path)("opened: %s", path);
@@ -1397,20 +1444,8 @@ void ClassLoader::initialize(TRAPS) {
setup_bootstrap_search_path(THREAD);
}
-static char* lookup_vm_resource(JImageFile *jimage, const char *jimage_version, const char *path) {
- jlong size;
- JImageLocationRef location = (*JImageFindResource)(jimage, "java.base", jimage_version, path, &size);
- if (location == 0)
- return nullptr;
- char *val = NEW_C_HEAP_ARRAY(char, size+1, mtClass);
- (*JImageGetResource)(jimage, location, val, size);
- val[size] = '\0';
- return val;
-}
-
// Lookup VM options embedded in the modules jimage file
char* ClassLoader::lookup_vm_options() {
- jint error;
char modules_path[JVM_MAXPATHLEN];
const char* fileSep = os::file_separator();
@@ -1418,32 +1453,41 @@ char* ClassLoader::lookup_vm_options() {
load_jimage_library();
jio_snprintf(modules_path, JVM_MAXPATHLEN, "%s%slib%smodules", Arguments::get_java_home(), fileSep, fileSep);
- JImage_file =(*JImageOpen)(modules_path, &error);
- if (JImage_file == nullptr) {
- if (Arguments::has_jimage()) {
- // The modules file exists but is unreadable or corrupt
- vm_exit_during_initialization(err_msg("Unable to load %s", modules_path));
+ if (jimage_open(modules_path)) {
+ // Special case where we lookup the options string *before* set_preview_mode() is called.
+ // Since VM arguments have not been parsed, and the ClassPathImageEntry singleton
+ // has not been created yet, we access the JImage file directly in non-preview mode.
+ jlong size;
+ JImageLocationRef location =
+ jimage_find_resource(JAVA_BASE_NAME, "jdk/internal/vm/options", /* is_preview */ false, &size);
+ if (location != 0) {
+ char* options = NEW_C_HEAP_ARRAY(char, size+1, mtClass);
+ (*JImageGetResource)(jimage_non_null(), location, options, size);
+ options[size] = '\0';
+ return options;
}
- return nullptr;
}
+ return nullptr;
+}
- const char *jimage_version = get_jimage_version_string();
- char *options = lookup_vm_resource(JImage_file, jimage_version, "jdk/internal/vm/options");
- return options;
+// Finishes initializing the JImageFile (if present) by setting the access mode.
+void ClassLoader::set_preview_mode(bool enable_preview) {
+ assert(Preview_mode == PREVIEW_MODE_UNINITIALIZED, "set_preview_mode must not be called twice");
+ Preview_mode = enable_preview ? PREVIEW_MODE_ENABLE_PREVIEW : PREVIEW_MODE_DEFAULT;
}
bool ClassLoader::is_module_observable(const char* module_name) {
assert(JImageOpen != nullptr, "jimage library should have been opened");
- if (JImage_file == nullptr) {
+ if (!jimage_is_open()) {
struct stat st;
const char *path = get_exploded_module_path(module_name, true);
bool res = os::stat(path, &st) == 0;
- FREE_C_HEAP_ARRAY(char, path);
+ FREE_C_HEAP_ARRAY(path);
return res;
}
+ // We don't expect preview mode (i.e. --enable-preview) to affect module visibility.
jlong size;
- const char *jimage_version = get_jimage_version_string();
- return (*JImageFindResource)(JImage_file, module_name, jimage_version, "module-info.class", &size) != 0;
+ return jimage_find_resource(module_name, "module-info.class", /* is_preview */ false, &size) != 0;
}
jlong ClassLoader::classloader_time_ms() {
diff --git a/src/hotspot/share/classfile/classLoader.hpp b/src/hotspot/share/classfile/classLoader.hpp
index a935d3027ac..ff7e8999688 100644
--- a/src/hotspot/share/classfile/classLoader.hpp
+++ b/src/hotspot/share/classfile/classLoader.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -99,7 +99,8 @@ class ClassPathZipEntry: public ClassPathEntry {
};
-// For java image files
+// A singleton path entry which takes ownership of the initialized JImageFile
+// reference. Not used for exploded builds.
class ClassPathImageEntry: public ClassPathEntry {
private:
const char* _name;
@@ -107,11 +108,12 @@ private:
public:
bool is_modules_image() const;
const char* name() const { return _name == nullptr ? "" : _name; }
- JImageFile* jimage() const;
- JImageFile* jimage_non_null() const;
+ // Called to close the JImage during os::abort (normally not called).
void close_jimage();
- ClassPathImageEntry(JImageFile* jimage, const char* name);
+ // Takes effective ownership of the static JImageFile pointer.
+ ClassPathImageEntry(const char* name);
virtual ~ClassPathImageEntry() { ShouldNotReachHere(); }
+
ClassFileStream* open_stream(JavaThread* current, const char* name);
ClassFileStream* open_stream_for_loader(JavaThread* current, const char* name, ClassLoaderData* loader_data);
};
@@ -201,10 +203,10 @@ class ClassLoader: AllStatic {
static GrowableArray* _patch_mod_entries;
// 2. the base piece
- // Contains the ClassPathEntry of the modular java runtime image.
+ // Contains the ClassPathImageEntry of the modular java runtime image.
// If no java runtime image is present, this indicates a
// build with exploded modules is being used instead.
- static ClassPathEntry* _jrt_entry;
+ static ClassPathImageEntry* _jrt_entry;
static GrowableArray* _exploded_entries;
enum { EXPLODED_ENTRY_SIZE = 80 }; // Initial number of exploded modules
@@ -354,15 +356,20 @@ class ClassLoader: AllStatic {
static void append_boot_classpath(ClassPathEntry* new_entry);
#endif
+ // Retrieves additional VM options prior to flags processing. Options held
+ // in the JImage file are retrieved without fully initializing it. (this is
+ // the only JImage lookup which can succeed before init_jimage() is called).
static char* lookup_vm_options();
+ // Called once, after all flags are processed, to finish initializing the
+ // JImage file. Until this is called, jimage_find_resource(), and any other
+ // JImage resource lookups or access will fail.
+ static void set_preview_mode(bool enable_preview);
+
// Determines if the named module is present in the
// modules jimage file or in the exploded modules directory.
static bool is_module_observable(const char* module_name);
- static JImageLocationRef jimage_find_resource(JImageFile* jf, const char* module_name,
- const char* file_name, jlong &size);
-
static void trace_class_path(const char* msg, const char* name = nullptr);
// VM monitoring and management support
diff --git a/src/hotspot/share/classfile/compactHashtable.cpp b/src/hotspot/share/classfile/compactHashtable.cpp
index de67971c403..f06f1986d8b 100644
--- a/src/hotspot/share/classfile/compactHashtable.cpp
+++ b/src/hotspot/share/classfile/compactHashtable.cpp
@@ -69,7 +69,7 @@ CompactHashtableWriter::~CompactHashtableWriter() {
delete bucket;
}
- FREE_C_HEAP_ARRAY(GrowableArray*, _buckets);
+ FREE_C_HEAP_ARRAY(_buckets);
}
// Add an entry to the temporary hash table
diff --git a/src/hotspot/share/classfile/resolutionErrors.cpp b/src/hotspot/share/classfile/resolutionErrors.cpp
index c41d5d2f052..958d4812cbb 100644
--- a/src/hotspot/share/classfile/resolutionErrors.cpp
+++ b/src/hotspot/share/classfile/resolutionErrors.cpp
@@ -114,15 +114,15 @@ ResolutionErrorEntry::~ResolutionErrorEntry() {
Symbol::maybe_decrement_refcount(_cause);
if (_message != nullptr) {
- FREE_C_HEAP_ARRAY(char, _message);
+ FREE_C_HEAP_ARRAY(_message);
}
if (_cause_msg != nullptr) {
- FREE_C_HEAP_ARRAY(char, _cause_msg);
+ FREE_C_HEAP_ARRAY(_cause_msg);
}
if (nest_host_error() != nullptr) {
- FREE_C_HEAP_ARRAY(char, nest_host_error());
+ FREE_C_HEAP_ARRAY(nest_host_error());
}
}
diff --git a/src/hotspot/share/classfile/systemDictionaryShared.cpp b/src/hotspot/share/classfile/systemDictionaryShared.cpp
index fd30fc6766f..330b8e81d7f 100644
--- a/src/hotspot/share/classfile/systemDictionaryShared.cpp
+++ b/src/hotspot/share/classfile/systemDictionaryShared.cpp
@@ -1277,7 +1277,7 @@ unsigned int SystemDictionaryShared::hash_for_shared_dictionary(address ptr) {
uintx offset = ArchiveBuilder::current()->any_to_offset(ptr);
unsigned int hash = primitive_hash(offset);
DEBUG_ONLY({
- if (MetaspaceObj::in_aot_cache((const MetaspaceObj*)ptr)) {
+ if (AOTMetaspace::in_aot_cache(ptr)) {
assert(hash == SystemDictionaryShared::hash_for_shared_dictionary_quick(ptr), "must be");
}
});
diff --git a/src/hotspot/share/classfile/systemDictionaryShared.hpp b/src/hotspot/share/classfile/systemDictionaryShared.hpp
index c837a386344..740c7370d28 100644
--- a/src/hotspot/share/classfile/systemDictionaryShared.hpp
+++ b/src/hotspot/share/classfile/systemDictionaryShared.hpp
@@ -25,6 +25,7 @@
#ifndef SHARE_CLASSFILE_SYSTEMDICTIONARYSHARED_HPP
#define SHARE_CLASSFILE_SYSTEMDICTIONARYSHARED_HPP
+#include "cds/aotMetaspace.hpp"
#include "cds/cds_globals.hpp"
#include "cds/dumpTimeClassInfo.hpp"
#include "cds/filemap.hpp"
@@ -312,7 +313,7 @@ public:
template
static unsigned int hash_for_shared_dictionary_quick(T* ptr) {
- assert(MetaspaceObj::in_aot_cache((const MetaspaceObj*)ptr), "must be");
+ assert(AOTMetaspace::in_aot_cache(ptr), "must be");
assert(ptr > (T*)SharedBaseAddress, "must be");
uintx offset = uintx(ptr) - uintx(SharedBaseAddress);
return primitive_hash(offset);
diff --git a/src/hotspot/share/code/aotCodeCache.cpp b/src/hotspot/share/code/aotCodeCache.cpp
index d4f12936e96..b330ed35d0b 100644
--- a/src/hotspot/share/code/aotCodeCache.cpp
+++ b/src/hotspot/share/code/aotCodeCache.cpp
@@ -63,6 +63,7 @@
#include "gc/g1/g1HeapRegion.hpp"
#endif
#if INCLUDE_SHENANDOAHGC
+#include "gc/shenandoah/shenandoahHeapRegion.hpp"
#include "gc/shenandoah/shenandoahRuntime.hpp"
#endif
#if INCLUDE_ZGC
@@ -83,15 +84,24 @@ const char* aot_code_entry_kind_name[] = {
static LogStream& load_failure_log() {
static LogStream err_stream(LogLevel::Error, LogTagSetMapping::tagset());
static LogStream dbg_stream(LogLevel::Debug, LogTagSetMapping::tagset());
- if (RequireSharedSpaces) {
+ if (RequireSharedSpaces || AbortVMOnAOTCodeFailure) {
return err_stream;
} else {
return dbg_stream;
}
}
+// Report AOT code cache failure and exit VM
+// if (AOTMode is `on` and AbortVMOnAOTCodeFailure is default)
+// or AbortVMOnAOTCodeFailure is `true`.
+//
+// Note, specifying -XX:-AbortVMOnAOTCodeFailure on command line
+// will prevent aborting VM when AOTMode is `on`. It is used for testing.
+
static void report_load_failure() {
- if (AbortVMOnAOTCodeFailure) {
+ bool abort_vm = AbortVMOnAOTCodeFailure ||
+ (FLAG_IS_DEFAULT(AbortVMOnAOTCodeFailure) && RequireSharedSpaces);
+ if (abort_vm) {
vm_exit_during_initialization("Unable to use AOT Code Cache.", nullptr);
}
load_failure_log().print_cr("Unable to use AOT Code Cache.");
@@ -482,25 +492,25 @@ bool AOTCodeCache::Config::verify_cpu_features(AOTCodeCache* cache) const {
log.print_cr("CPU features recorded in AOTCodeCache: %s", ss.as_string());
}
- if (VM_Version::supports_features(cached_cpu_features_buffer)) {
- if (log.is_enabled()) {
- ResourceMark rm; // required for stringStream::as_string()
- stringStream ss;
- char* runtime_cpu_features = NEW_RESOURCE_ARRAY(char, VM_Version::cpu_features_size());
- VM_Version::store_cpu_features(runtime_cpu_features);
- VM_Version::get_missing_features_name(runtime_cpu_features, cached_cpu_features_buffer, ss);
- if (!ss.is_empty()) {
- log.print_cr("Additional runtime CPU features: %s", ss.as_string());
- }
- }
- } else {
+ if (!VM_Version::verify_aot_code_cache_features(cached_cpu_features_buffer)) {
if (load_failure_log().is_enabled()) {
ResourceMark rm; // required for stringStream::as_string()
- stringStream ss;
+ load_failure_log().print_cr("AOT Code Cache disabled: cpu features are incompatible");
char* runtime_cpu_features = NEW_RESOURCE_ARRAY(char, VM_Version::cpu_features_size());
VM_Version::store_cpu_features(runtime_cpu_features);
- VM_Version::get_missing_features_name(cached_cpu_features_buffer, runtime_cpu_features, ss);
- load_failure_log().print_cr("AOT Code Cache disabled: required cpu features are missing: %s", ss.as_string());
+
+ stringStream missing_features;
+ VM_Version::get_missing_features_name(cached_cpu_features_buffer, runtime_cpu_features, missing_features);
+ if (!missing_features.is_empty()) {
+ load_failure_log().print_cr("cpu features that are required: \"%s\"", missing_features.as_string());
+ }
+
+ stringStream additional_features;
+ VM_Version::get_missing_features_name(runtime_cpu_features, cached_cpu_features_buffer, additional_features);
+ if (!additional_features.is_empty()) {
+ load_failure_log().print("cpu features that are additional: \"%s\"", additional_features.as_string());
+ }
+ load_failure_log().print_cr("");
}
return false;
}
@@ -652,6 +662,10 @@ void AOTCodeReader::set_read_position(uint pos) {
_read_position = pos;
}
+uint AOTCodeReader::align_read_int() {
+ return align_up(_read_position, sizeof(int));
+}
+
bool AOTCodeCache::set_write_position(uint pos) {
if (pos == _write_position) {
return true;
@@ -666,21 +680,29 @@ bool AOTCodeCache::set_write_position(uint pos) {
static char align_buffer[256] = { 0 };
-bool AOTCodeCache::align_write() {
- // We are not executing code from cache - we copy it by bytes first.
- // No need for big alignment (or at all).
- uint padding = DATA_ALIGNMENT - (_write_position & (DATA_ALIGNMENT - 1));
- if (padding == DATA_ALIGNMENT) {
+bool AOTCodeCache::align_write_bytes(uint alignment) {
+ uint padding = alignment - (_write_position & (alignment - 1));
+ if (padding == alignment) {
return true;
}
uint n = write_bytes((const void*)&align_buffer, padding);
if (n != padding) {
return false;
}
- log_trace(aot, codecache)("Adjust write alignment in AOT Code Cache");
+ log_trace(aot, codecache)("Adjust write alignment to %d bytes in AOT Code Cache", alignment);
return true;
}
+bool AOTCodeCache::align_write() {
+ // We are not executing code from cache - we copy it by bytes first.
+ // No need for big alignment (or at all).
+ return align_write_bytes(DATA_ALIGNMENT);
+}
+
+bool AOTCodeCache::align_write_int() {
+ return align_write_bytes(sizeof(int));
+}
+
// Check to see if AOT code cache has required space to store "nbytes" of data
address AOTCodeCache::reserve_bytes(uint nbytes) {
assert(for_dump(), "Code Cache file is not created");
@@ -876,7 +898,7 @@ bool AOTCodeCache::finish_write() {
current += size;
uint n = write_bytes(&(entries_address[i]), sizeof(AOTCodeEntry));
if (n != sizeof(AOTCodeEntry)) {
- FREE_C_HEAP_ARRAY(uint, search);
+ FREE_C_HEAP_ARRAY(search);
return false;
}
search[entries_count*2 + 0] = entries_address[i].id();
@@ -897,7 +919,7 @@ bool AOTCodeCache::finish_write() {
}
if (entries_count == 0) {
log_info(aot, codecache, exit)("AOT Code Cache was not created: no entires");
- FREE_C_HEAP_ARRAY(uint, search);
+ FREE_C_HEAP_ARRAY(search);
return true; // Nothing to write
}
assert(entries_count <= store_count, "%d > %d", entries_count, store_count);
@@ -913,7 +935,7 @@ bool AOTCodeCache::finish_write() {
qsort(search, entries_count, 2*sizeof(uint), uint_cmp);
search_size = 2 * entries_count * sizeof(uint);
copy_bytes((const char*)search, (address)current, search_size);
- FREE_C_HEAP_ARRAY(uint, search);
+ FREE_C_HEAP_ARRAY(search);
current += search_size;
// Write entries
@@ -1012,19 +1034,8 @@ bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind
}
uint entry_position = cache->_write_position;
- // Write name
- uint name_offset = cache->_write_position - entry_position;
- uint name_size = (uint)strlen(name) + 1; // Includes '/0'
- uint n = cache->write_bytes(name, name_size);
- if (n != name_size) {
- return false;
- }
-
- // Write CodeBlob
- if (!cache->align_write()) {
- return false;
- }
uint blob_offset = cache->_write_position - entry_position;
+ // Code blob's size is aligned to oopSize
address archive_buffer = cache->reserve_bytes(blob.size());
if (archive_buffer == nullptr) {
return false;
@@ -1056,7 +1067,7 @@ bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind
reloc_count = blob.relocation_size() / sizeof(relocInfo);
reloc_data = (address)blob.relocation_begin();
}
- n = cache->write_bytes(&reloc_count, sizeof(int));
+ uint n = cache->write_bytes(&reloc_count, sizeof(int));
if (n != sizeof(int)) {
return false;
}
@@ -1123,6 +1134,14 @@ bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind
}
#endif /* PRODUCT */
+ // Write name after code comments
+ uint name_offset = cache->_write_position - entry_position;
+ uint name_size = (uint)strlen(name) + 1; // Includes '/0'
+ n = cache->write_bytes(name, name_size);
+ if (n != name_size) {
+ return false;
+ }
+
uint entry_size = cache->_write_position - entry_position;
AOTCodeEntry* entry = new(cache) AOTCodeEntry(entry_kind, encode_id(entry_kind, id),
@@ -1151,6 +1170,9 @@ bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind
}
bool AOTCodeCache::write_stub_data(CodeBlob &blob, AOTStubData *stub_data) {
+ if (!align_write_int()) {
+ return false;
+ }
BlobId blob_id = stub_data->blob_id();
StubId stub_id = StubInfo::stub_base(blob_id);
address blob_base = blob.code_begin();
@@ -1320,7 +1342,8 @@ CodeBlob* AOTCodeReader::compile_code_blob(const char* name, AOTCodeEntry::Kind
CodeBlob* archived_blob = (CodeBlob*)addr(offset);
offset += archived_blob->size();
- _reloc_count = *(int*)addr(offset); offset += sizeof(int);
+ _reloc_count = *(int*)addr(offset);
+ offset += sizeof(int);
if (AOTCodeEntry::is_multi_stub_blob(entry_kind)) {
// position of relocs will have been aligned to heap word size so
// we can install them into a code buffer
@@ -1443,7 +1466,7 @@ void AOTCodeReader::read_stub_data(CodeBlob* code_blob, AOTStubData* stub_data)
address blob_base = code_blob->code_begin();
uint blob_size = (uint)(code_blob->code_end() - blob_base);
- int offset = read_position();
+ uint offset = align_read_int();
LogStreamHandle(Trace, aot, codecache, stubs) log;
if (log.is_enabled()) {
log.print_cr("======== Stub data starts at offset %d", offset);
@@ -1571,6 +1594,9 @@ void AOTCodeCache::publish_stub_addresses(CodeBlob &code_blob, BlobId blob_id, A
#define BAD_ADDRESS_ID -2
bool AOTCodeCache::write_relocations(CodeBlob& code_blob, RelocIterator& iter) {
+ if (!align_write_int()) {
+ return false;
+ }
GrowableArray reloc_data;
LogStreamHandle(Trace, aot, codecache, reloc) log;
while (iter.next()) {
@@ -1653,7 +1679,7 @@ bool AOTCodeCache::write_relocations(CodeBlob& code_blob, RelocIterator& iter) {
}
void AOTCodeReader::fix_relocations(CodeBlob *code_blob, RelocIterator& iter) {
- uint offset = read_position();
+ uint offset = align_read_int();
int reloc_count = *(int*)addr(offset);
offset += sizeof(int);
uint* reloc_data = (uint*)addr(offset);
@@ -1726,6 +1752,9 @@ void AOTCodeReader::fix_relocations(CodeBlob *code_blob, RelocIterator& iter) {
}
bool AOTCodeCache::write_oop_map_set(CodeBlob& cb) {
+ if (!align_write_int()) {
+ return false;
+ }
ImmutableOopMapSet* oopmaps = cb.oop_maps();
int oopmaps_size = oopmaps->nr_of_bytes();
if (!write_bytes(&oopmaps_size, sizeof(int))) {
@@ -1739,7 +1768,7 @@ bool AOTCodeCache::write_oop_map_set(CodeBlob& cb) {
}
ImmutableOopMapSet* AOTCodeReader::read_oop_map_set() {
- uint offset = read_position();
+ uint offset = align_read_int();
int size = *(int *)addr(offset);
offset += sizeof(int);
ImmutableOopMapSet* oopmaps = (ImmutableOopMapSet *)addr(offset);
@@ -1750,6 +1779,9 @@ ImmutableOopMapSet* AOTCodeReader::read_oop_map_set() {
#ifndef PRODUCT
bool AOTCodeCache::write_asm_remarks(CodeBlob& cb) {
+ if (!align_write_int()) {
+ return false;
+ }
// Write asm remarks
uint* count_ptr = (uint *)reserve_bytes(sizeof(uint));
if (count_ptr == nullptr) {
@@ -1778,7 +1810,7 @@ bool AOTCodeCache::write_asm_remarks(CodeBlob& cb) {
void AOTCodeReader::read_asm_remarks(AsmRemarks& asm_remarks) {
// Read asm remarks
- uint offset = read_position();
+ uint offset = align_read_int();
uint count = *(uint *)addr(offset);
offset += sizeof(uint);
for (uint i = 0; i < count; i++) {
@@ -1793,6 +1825,9 @@ void AOTCodeReader::read_asm_remarks(AsmRemarks& asm_remarks) {
}
bool AOTCodeCache::write_dbg_strings(CodeBlob& cb) {
+ if (!align_write_int()) {
+ return false;
+ }
// Write dbg strings
uint* count_ptr = (uint *)reserve_bytes(sizeof(uint));
if (count_ptr == nullptr) {
@@ -1817,7 +1852,7 @@ bool AOTCodeCache::write_dbg_strings(CodeBlob& cb) {
void AOTCodeReader::read_dbg_strings(DbgStrings& dbg_strings) {
// Read dbg strings
- uint offset = read_position();
+ uint offset = align_read_int();
uint count = *(uint *)addr(offset);
offset += sizeof(uint);
for (uint i = 0; i < count; i++) {
@@ -2365,10 +2400,6 @@ int AOTCodeAddressTable::id_for_address(address addr, RelocIterator reloc, CodeB
if (addr == (address)-1) { // Static call stub has jump to itself
return id;
}
- // Check card_table_base address first since it can point to any address
- BarrierSet* bs = BarrierSet::barrier_set();
- bool is_const_card_table_base = !UseG1GC && !UseShenandoahGC && bs->is_a(BarrierSet::CardTableBarrierSet);
- guarantee(!is_const_card_table_base || addr != ci_card_table_address_const(), "sanity");
// fast path for stubs and external addresses
if (_hash_table != nullptr) {
int *result = _hash_table->get(addr);
@@ -2442,6 +2473,7 @@ void AOTRuntimeConstants::initialize_from_runtime() {
BarrierSet* bs = BarrierSet::barrier_set();
address card_table_base = nullptr;
uint grain_shift = 0;
+ address cset_base = nullptr;
#if INCLUDE_G1GC
if (bs->is_a(BarrierSet::G1BarrierSet)) {
grain_shift = G1HeapRegion::LogOfHRGrainBytes;
@@ -2449,7 +2481,8 @@ void AOTRuntimeConstants::initialize_from_runtime() {
#endif
#if INCLUDE_SHENANDOAHGC
if (bs->is_a(BarrierSet::ShenandoahBarrierSet)) {
- grain_shift = 0;
+ grain_shift = ShenandoahHeapRegion::region_size_bytes_shift_jint();
+ cset_base = ShenandoahHeap::in_cset_fast_test_addr();
} else
#endif
if (bs->is_a(BarrierSet::CardTableBarrierSet)) {
@@ -2461,11 +2494,13 @@ void AOTRuntimeConstants::initialize_from_runtime() {
}
_aot_runtime_constants._card_table_base = card_table_base;
_aot_runtime_constants._grain_shift = grain_shift;
+ _aot_runtime_constants._cset_base = cset_base;
}
address AOTRuntimeConstants::_field_addresses_list[] = {
((address)&_aot_runtime_constants._card_table_base),
((address)&_aot_runtime_constants._grain_shift),
+ ((address)&_aot_runtime_constants._cset_base),
nullptr
};
@@ -2576,10 +2611,6 @@ address AOTStubData::load_archive_data(StubId stub_id, address& end, GrowableArr
StubAddrRange &range = _ranges[idx];
int base = range.start_index();
if (base < 0) {
-#ifdef DEBUG
- // reset index so we can idenitfy which ones we failed to find
- range.init_entry(-2, 0);
-#endif
return nullptr;
}
int count = range.count();
@@ -2656,3 +2687,23 @@ void AOTStubData::store_archive_data(StubId stub_id, address start, address end,
}
range.init_entry(base, _address_array.length() - base);
}
+
+void AOTStubData::stub_epilog(StubId stub_id) {
+ DEBUG_ONLY(check_stored(stub_id));
+}
+
+#ifdef ASSERT
+void AOTStubData::check_stored(StubId stub_id) {
+ // Only need to check if we are dumping
+ //
+ // This excludes cases where the cache got closed because of error
+ // plus the pre-universe stubs we can never store because they are
+ // generated prior to cache opening.
+ if (is_dumping()) {
+ int idx = StubInfo::stubgen_offset_in_blob(_blob_id, stub_id);
+ assert(idx >= 0 && idx < _stub_cnt, "invalid index %d for stub count %d", idx, _stub_cnt);
+ StubAddrRange& range = _ranges[idx];
+ assert(range.start_index() != -1, "missing store_archive_data for generated stub %s", StubInfo::name(stub_id));
+ }
+}
+#endif
diff --git a/src/hotspot/share/code/aotCodeCache.hpp b/src/hotspot/share/code/aotCodeCache.hpp
index 5b773a986f1..039735dc954 100644
--- a/src/hotspot/share/code/aotCodeCache.hpp
+++ b/src/hotspot/share/code/aotCodeCache.hpp
@@ -252,7 +252,7 @@ private:
public:
AOTStubData(BlobId blob_id) NOT_CDS({});
- ~AOTStubData() CDS_ONLY({FREE_C_HEAP_ARRAY(StubAddrRange, _ranges);}) NOT_CDS({})
+ ~AOTStubData() CDS_ONLY({FREE_C_HEAP_ARRAY(_ranges);}) NOT_CDS({})
bool is_open() CDS_ONLY({ return (_flags & OPEN) != 0; }) NOT_CDS_RETURN_(false);
bool is_using() CDS_ONLY({ return (_flags & USING) != 0; }) NOT_CDS_RETURN_(false);
@@ -266,6 +266,10 @@ public:
address load_archive_data(StubId stub_id, address &end, GrowableArray* entries = nullptr, GrowableArray* extras = nullptr) NOT_CDS_RETURN_(nullptr);
void store_archive_data(StubId stub_id, address start, address end, GrowableArray* entries = nullptr, GrowableArray* extras = nullptr) NOT_CDS_RETURN;
+ void stub_epilog(StubId stub_id);
+#ifdef ASSERT
+ void check_stored(StubId stub_id);
+#endif
const AOTStubData* as_const() { return (const AOTStubData*)this; }
};
@@ -331,8 +335,6 @@ public:
do_var(intx, PrefetchCopyIntervalInBytes) /* array copy stubs */ \
do_var(int, SoftwarePrefetchHintDistance) /* array fill stubs */ \
do_var(bool, UseBlockZeroing) \
- do_var(bool, UseLSE) /* stubs and nmethods */ \
- do_var(uint, UseSVE) /* stubs and nmethods */ \
do_var(bool, UseSecondarySupersCache) \
do_var(bool, UseSIMDForArrayEquals) /* array copy stubs and nmethods */ \
do_var(bool, UseSIMDForBigIntegerShiftIntrinsics) \
@@ -348,8 +350,6 @@ public:
#define AOTCODECACHE_CONFIGS_X86_DO(do_var, do_fun) \
do_var(int, AVX3Threshold) /* array copy stubs and nmethods */ \
do_var(bool, EnableX86ECoreOpts) /* nmethods */ \
- do_var(int, UseAVX) /* array copy stubs and nmethods */ \
- do_var(bool, UseAPX) /* nmethods and stubs */ \
do_var(bool, UseLibmIntrinsic) \
do_var(bool, UseIntPolyIntrinsics) \
// END
@@ -478,6 +478,8 @@ private:
bool set_write_position(uint pos);
bool align_write();
+ bool align_write_int();
+ bool align_write_bytes(uint alignment);
address reserve_bytes(uint nbytes);
uint write_bytes(const void* buffer, uint nbytes);
const char* addr(uint offset) const { return _load_buffer + offset; }
@@ -643,6 +645,7 @@ private:
uint _read_position; // Position in _load_buffer
uint read_position() const { return _read_position; }
void set_read_position(uint pos);
+ uint align_read_int();
const char* addr(uint offset) const { return _load_buffer + offset; }
bool _lookup_failed; // Failed to lookup for info (skip only this code load)
@@ -685,6 +688,7 @@ class AOTRuntimeConstants {
private:
address _card_table_base;
uint _grain_shift;
+ address _cset_base;
static address _field_addresses_list[];
static AOTRuntimeConstants _aot_runtime_constants;
// private constructor for unique singleton
@@ -700,6 +704,7 @@ class AOTRuntimeConstants {
}
static address card_table_base_address();
static address grain_shift_address() { return (address)&_aot_runtime_constants._grain_shift; }
+ static address cset_base_address() { return (address)&_aot_runtime_constants._cset_base; }
static address* field_addresses_list() {
return _field_addresses_list;
}
@@ -707,6 +712,7 @@ class AOTRuntimeConstants {
static bool contains(address adr) { return false; }
static address card_table_base_address() { return nullptr; }
static address grain_shift_address() { return nullptr; }
+ static address cset_base_address() { return nullptr; }
static address* field_addresses_list() { return nullptr; }
#endif
};
diff --git a/src/hotspot/share/code/codeBlob.cpp b/src/hotspot/share/code/codeBlob.cpp
index e0c286937d0..d69ae40be19 100644
--- a/src/hotspot/share/code/codeBlob.cpp
+++ b/src/hotspot/share/code/codeBlob.cpp
@@ -357,9 +357,12 @@ void RuntimeBlob::trace_new_stub(RuntimeBlob* stub, const char* name1, const cha
if (stub != nullptr && (PrintStubCode ||
Forte::is_enabled() ||
JvmtiExport::should_post_dynamic_code_generated())) {
- char stub_id[256];
- assert(strlen(name1) + strlen(name2) < sizeof(stub_id), "");
- jio_snprintf(stub_id, sizeof(stub_id), "%s%s", name1, name2);
+ ResourceMark rm;
+ const size_t name1_len = strlen(name1);
+ const size_t name2_len = strlen(name2);
+ const size_t stub_id_size = name1_len + name2_len + 1;
+ char* stub_id = NEW_RESOURCE_ARRAY(char, stub_id_size);
+ jio_snprintf(stub_id, stub_id_size, "%s%s", name1, name2);
if (PrintStubCode) {
ttyLocker ttyl;
tty->print_cr("- - - [BEGIN] - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -");
diff --git a/src/hotspot/share/code/codeCache.cpp b/src/hotspot/share/code/codeCache.cpp
index c0b4918102e..ffa88a88b29 100644
--- a/src/hotspot/share/code/codeCache.cpp
+++ b/src/hotspot/share/code/codeCache.cpp
@@ -65,6 +65,7 @@
#include "sanitizers/leak.hpp"
#include "services/memoryService.hpp"
#include "utilities/align.hpp"
+#include "utilities/integerCast.hpp"
#include "utilities/vmError.hpp"
#include "utilities/xmlstream.hpp"
#ifdef COMPILER1
@@ -228,9 +229,14 @@ void CodeCache::initialize_heaps() {
assert(heap_available(CodeBlobType::MethodNonProfiled), "MethodNonProfiled heap is always available for segmented code heap");
- size_t compiler_buffer_size = 0;
- COMPILER1_PRESENT(compiler_buffer_size += CompilationPolicy::c1_count() * Compiler::code_buffer_size());
- COMPILER2_PRESENT(compiler_buffer_size += CompilationPolicy::c2_count() * C2Compiler::initial_code_buffer_size());
+ uint64_t compiler_buffer_size_uint64 = 0;
+ COMPILER1_PRESENT(compiler_buffer_size_uint64 += (uint64_t)CompilationPolicy::c1_count() * Compiler::code_buffer_size());
+ COMPILER2_PRESENT(compiler_buffer_size_uint64 += (uint64_t)CompilationPolicy::c2_count() * C2Compiler::initial_code_buffer_size());
+ if (compiler_buffer_size_uint64 > (uint64_t)CODE_CACHE_SIZE_LIMIT) {
+ err_msg msg("CICompilerCount is too large (%" PRIdPTR "): compiler buffer size exceeds the CodeCache size limit", CICompilerCount);
+ vm_exit_during_initialization(msg);
+ }
+ size_t compiler_buffer_size = integer_cast_permit_tautology(compiler_buffer_size_uint64);
if (!non_nmethod.set) {
non_nmethod.size += compiler_buffer_size;
@@ -1715,7 +1721,7 @@ void CodeCache::print_internals() {
}
}
- FREE_C_HEAP_ARRAY(int, buckets);
+ FREE_C_HEAP_ARRAY(buckets);
print_memory_overhead();
}
@@ -1838,11 +1844,15 @@ void CodeCache::print() {
}
void CodeCache::print_summary(outputStream* st, bool detailed) {
+ int total_blob_count = 0;
+ int total_nmethod_count = 0;
+ int total_adapter_count = 0;
int full_count = 0;
julong total_used = 0;
julong total_max_used = 0;
julong total_free = 0;
julong total_size = 0;
+
FOR_ALL_HEAPS(heap_iterator) {
CodeHeap* heap = (*heap_iterator);
size_t total = (heap->high_boundary() - heap->low_boundary());
@@ -1868,8 +1878,13 @@ void CodeCache::print_summary(outputStream* st, bool detailed) {
p2i(heap->low_boundary()),
p2i(heap->high()),
p2i(heap->high_boundary()));
-
- full_count += get_codemem_full_count(heap->code_blob_type());
+ st->print_cr(" blobs=" UINT32_FORMAT ", nmethods=" UINT32_FORMAT
+ ", adapters=" UINT32_FORMAT ", full_count=" UINT32_FORMAT,
+ heap->blob_count(), heap->nmethod_count(), heap->adapter_count(), heap->full_count());
+ total_blob_count += heap->blob_count();
+ total_nmethod_count += heap->nmethod_count();
+ total_adapter_count += heap->adapter_count();
+ full_count += heap->full_count();
}
}
@@ -1879,10 +1894,10 @@ void CodeCache::print_summary(outputStream* st, bool detailed) {
st->print_cr(" size=" JULONG_FORMAT "Kb, used=" JULONG_FORMAT
"Kb, max_used=" JULONG_FORMAT "Kb, free=" JULONG_FORMAT "Kb",
total_size, total_used, total_max_used, total_free);
+ st->print_cr(" total blobs=" UINT32_FORMAT ", nmethods=" UINT32_FORMAT
+ ", adapters=" UINT32_FORMAT ", full_count=" UINT32_FORMAT,
+ total_blob_count, total_nmethod_count, total_adapter_count, full_count);
}
- st->print_cr(" total_blobs=" UINT32_FORMAT ", nmethods=" UINT32_FORMAT
- ", adapters=" UINT32_FORMAT ", full_count=" UINT32_FORMAT,
- blob_count(), nmethod_count(), adapter_count(), full_count);
st->print_cr("Compilation: %s, stopped_count=%d, restarted_count=%d",
CompileBroker::should_compile_new_jobs() ?
"enabled" : Arguments::mode() == Arguments::_int ?
diff --git a/src/hotspot/share/code/exceptionHandlerTable.cpp b/src/hotspot/share/code/exceptionHandlerTable.cpp
index a295d1271aa..f0ad9921cd0 100644
--- a/src/hotspot/share/code/exceptionHandlerTable.cpp
+++ b/src/hotspot/share/code/exceptionHandlerTable.cpp
@@ -32,7 +32,7 @@ void ExceptionHandlerTable::add_entry(HandlerTableEntry entry) {
// not enough space => grow the table (amortized growth, double its size)
guarantee(_size > 0, "no space allocated => cannot grow the table since it is part of nmethod");
int new_size = _size * 2;
- _table = REALLOC_RESOURCE_ARRAY(HandlerTableEntry, _table, _size, new_size);
+ _table = REALLOC_RESOURCE_ARRAY(_table, _size, new_size);
_size = new_size;
}
assert(_length < _size, "sanity check");
@@ -178,7 +178,7 @@ void ImplicitExceptionTable::append( uint exec_off, uint cont_off ) {
if (_size == 0) _size = 4;
_size *= 2;
uint new_size_in_elements = _size*2;
- _data = REALLOC_RESOURCE_ARRAY(uint, _data, old_size_in_elements, new_size_in_elements);
+ _data = REALLOC_RESOURCE_ARRAY(_data, old_size_in_elements, new_size_in_elements);
}
*(adr(l) ) = exec_off;
*(adr(l)+1) = cont_off;
diff --git a/src/hotspot/share/compiler/cHeapStringHolder.cpp b/src/hotspot/share/compiler/cHeapStringHolder.cpp
index 261658e04eb..5a9c124688e 100644
--- a/src/hotspot/share/compiler/cHeapStringHolder.cpp
+++ b/src/hotspot/share/compiler/cHeapStringHolder.cpp
@@ -36,7 +36,7 @@ void CHeapStringHolder::set(const char* string) {
void CHeapStringHolder::clear() {
if (_string != nullptr) {
- FREE_C_HEAP_ARRAY(char, _string);
+ FREE_C_HEAP_ARRAY(_string);
_string = nullptr;
}
}
diff --git a/src/hotspot/share/compiler/compilationMemoryStatistic.cpp b/src/hotspot/share/compiler/compilationMemoryStatistic.cpp
index 1951fd066fc..82bd4c160d2 100644
--- a/src/hotspot/share/compiler/compilationMemoryStatistic.cpp
+++ b/src/hotspot/share/compiler/compilationMemoryStatistic.cpp
@@ -487,7 +487,7 @@ public:
void clean_details() {
if (_detail_stats != nullptr) {
- FREE_C_HEAP_ARRAY(Details, _detail_stats);
+ FREE_C_HEAP_ARRAY(_detail_stats);
_detail_stats = nullptr;
}
}
diff --git a/src/hotspot/share/compiler/compilationPolicy.cpp b/src/hotspot/share/compiler/compilationPolicy.cpp
index 1cc44602186..e69480560ae 100644
--- a/src/hotspot/share/compiler/compilationPolicy.cpp
+++ b/src/hotspot/share/compiler/compilationPolicy.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2010, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2010, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -814,23 +814,32 @@ CompileTask* CompilationPolicy::select_task(CompileQueue* compile_queue, JavaThr
max_method = max_task->method();
}
- methodHandle max_method_h(THREAD, max_method);
+ if (max_task != nullptr && max_method != nullptr) {
+ methodHandle max_method_h(THREAD, max_method);
- if (max_task != nullptr && max_task->comp_level() == CompLevel_full_profile && TieredStopAtLevel > CompLevel_full_profile &&
- max_method != nullptr && is_method_profiled(max_method_h) && !Arguments::is_compiler_only()) {
- max_task->set_comp_level(CompLevel_limited_profile);
+ if (max_task->comp_level() == CompLevel_full_profile && TieredStopAtLevel > CompLevel_full_profile &&
+ is_method_profiled(max_method_h) && !Arguments::is_compiler_only()) {
- if (CompileBroker::compilation_is_complete(max_method_h, max_task->osr_bci(), CompLevel_limited_profile)) {
- if (PrintTieredEvents) {
- print_event(REMOVE_FROM_QUEUE, max_method, max_method, max_task->osr_bci(), (CompLevel)max_task->comp_level());
+ CompilerDirectiveMatcher directive_matcher(max_method_h, CompLevel_limited_profile);
+ bool exclude_limited_profile = directive_matcher.directive_set()->ExcludeOption;
+
+ if (!exclude_limited_profile) {
+ max_task->set_comp_level(CompLevel_limited_profile);
+ max_task->transfer_directive(directive_matcher);
+
+ if (CompileBroker::compilation_is_complete(max_method_h, max_task->osr_bci(), CompLevel_limited_profile)) {
+ if (PrintTieredEvents) {
+ print_event(REMOVE_FROM_QUEUE, max_method, max_method, max_task->osr_bci(), (CompLevel)max_task->comp_level());
+ }
+ compile_queue->remove_and_mark_stale(max_task);
+ max_method->clear_queued_for_compilation();
+ return nullptr;
+ }
+
+ if (PrintTieredEvents) {
+ print_event(UPDATE_IN_QUEUE, max_method, max_method, max_task->osr_bci(), (CompLevel)max_task->comp_level());
+ }
}
- compile_queue->remove_and_mark_stale(max_task);
- max_method->clear_queued_for_compilation();
- return nullptr;
- }
-
- if (PrintTieredEvents) {
- print_event(UPDATE_IN_QUEUE, max_method, max_method, max_task->osr_bci(), (CompLevel)max_task->comp_level());
}
}
return max_task;
diff --git a/src/hotspot/share/compiler/compileBroker.cpp b/src/hotspot/share/compiler/compileBroker.cpp
index 7b236ed3589..ddd3f8ae5f8 100644
--- a/src/hotspot/share/compiler/compileBroker.cpp
+++ b/src/hotspot/share/compiler/compileBroker.cpp
@@ -345,6 +345,8 @@ void CompileQueue::add(CompileTask* task) {
// Mark the method as being in the compile queue.
task->method()->set_queued_for_compilation();
+ task->mark_queued(os::elapsed_counter());
+
if (CIPrintCompileQueue) {
print_tty();
}
@@ -1380,10 +1382,9 @@ nmethod* CompileBroker::compile_method(const methodHandle& method, int osr_bci,
}
#endif
- DirectiveSet* directive = DirectivesStack::getMatchingDirective(method, comp);
+ CompilerDirectiveMatcher matcher(method, comp_level);
// CompileBroker::compile_method can trap and can have pending async exception.
- nmethod* nm = CompileBroker::compile_method(method, osr_bci, comp_level, hot_count, compile_reason, directive, THREAD);
- DirectivesStack::release(directive);
+ nmethod* nm = CompileBroker::compile_method(method, osr_bci, comp_level, hot_count, compile_reason, matcher.directive_set(), THREAD);
return nm;
}
@@ -2363,11 +2364,10 @@ void CompileBroker::invoke_compiler_on_method(CompileTask* task) {
if (!ci_env.failing() && !task->is_success()) {
- assert(ci_env.failure_reason() != nullptr, "expect failure reason");
- assert(false, "compiler should always document failure: %s", ci_env.failure_reason());
- // The compiler elected, without comment, not to register a result.
+ const char* reason = task->failure_reason();
+ assert(reason != nullptr, "compiler should always document failure");
// Do not attempt further compilations of this method.
- ci_env.record_method_not_compilable("compile failed");
+ ci_env.record_method_not_compilable(reason != nullptr ? reason : "compile failed: reason unknown");
}
// Copy this bit to the enclosing block:
@@ -2402,7 +2402,7 @@ void CompileBroker::invoke_compiler_on_method(CompileTask* task) {
}
}
- DirectivesStack::release(directive);
+ task->mark_finished(os::elapsed_counter());
methodHandle method(thread, task->method());
@@ -2410,14 +2410,9 @@ void CompileBroker::invoke_compiler_on_method(CompileTask* task) {
collect_statistics(thread, time, task);
- if (PrintCompilation && PrintCompilation2) {
- tty->print("%7d ", (int) tty->time_stamp().milliseconds()); // print timestamp
- tty->print("%4d ", compile_id); // print compilation number
- tty->print("%s ", (is_osr ? "%" : " "));
- if (task->is_success()) {
- tty->print("size: %d(%d) ", task->nm_total_size(), task->nm_insts_size());
- }
- tty->print_cr("time: %d inlined: %d bytes", (int)time.milliseconds(), task->num_inlined_bytecodes());
+ if (PrintCompilation2 || directive->PrintCompilation2Option) {
+ ResourceMark rm;
+ task->print_post(tty);
}
Log(compilation, codecache) log;
@@ -2614,7 +2609,7 @@ void CompileBroker::collect_statistics(CompilerThread* thread, elapsedTimer time
}
// Collect statistic per compiler
- AbstractCompiler* comp = compiler(comp_level);
+ AbstractCompiler* comp = task->compiler();
if (comp) {
CompilerStatistics* stats = comp->stats();
if (is_osr) {
diff --git a/src/hotspot/share/compiler/compileLog.cpp b/src/hotspot/share/compiler/compileLog.cpp
index d0ea80d6019..d0833fbe7fa 100644
--- a/src/hotspot/share/compiler/compileLog.cpp
+++ b/src/hotspot/share/compiler/compileLog.cpp
@@ -64,8 +64,8 @@ CompileLog::~CompileLog() {
_out = nullptr;
// Remove partial file after merging in CompileLog::finish_log_on_error
unlink(_file);
- FREE_C_HEAP_ARRAY(char, _identities);
- FREE_C_HEAP_ARRAY(char, _file);
+ FREE_C_HEAP_ARRAY(_identities);
+ FREE_C_HEAP_ARRAY(_file);
}
@@ -96,7 +96,7 @@ int CompileLog::identify(ciBaseObject* obj) {
if (id >= _identities_capacity) {
int new_cap = _identities_capacity * 2;
if (new_cap <= id) new_cap = id + 100;
- _identities = REALLOC_C_HEAP_ARRAY(char, _identities, new_cap, mtCompiler);
+ _identities = REALLOC_C_HEAP_ARRAY(_identities, new_cap, mtCompiler);
_identities_capacity = new_cap;
}
while (id >= _identities_limit) {
diff --git a/src/hotspot/share/compiler/compileTask.cpp b/src/hotspot/share/compiler/compileTask.cpp
index 536d81045d7..b22aa4466a4 100644
--- a/src/hotspot/share/compiler/compileTask.cpp
+++ b/src/hotspot/share/compiler/compileTask.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -44,38 +44,36 @@ CompileTask::CompileTask(int compile_id,
int comp_level,
int hot_count,
CompileReason compile_reason,
- bool is_blocking) {
- Thread* thread = Thread::current();
- _compile_id = compile_id;
- _method = method();
- _method_holder = JNIHandles::make_weak_global(Handle(thread, method->method_holder()->klass_holder()));
- _osr_bci = osr_bci;
- _is_blocking = is_blocking;
- JVMCI_ONLY(_has_waiter = CompileBroker::compiler(comp_level)->is_jvmci();)
- JVMCI_ONLY(_blocking_jvmci_compile_state = nullptr;)
- _comp_level = comp_level;
- _num_inlined_bytecodes = 0;
-
- _is_complete = false;
- _is_success = false;
-
- _hot_count = hot_count;
- _time_queued = os::elapsed_counter();
- _time_started = 0;
- _compile_reason = compile_reason;
- _nm_content_size = 0;
- AbstractCompiler* comp = compiler();
- _directive = DirectivesStack::getMatchingDirective(method, comp);
- _nm_insts_size = 0;
- _nm_total_size = 0;
- _failure_reason = nullptr;
- _failure_reason_on_C_heap = false;
- _training_data = nullptr;
- _arena_bytes = 0;
-
- _next = nullptr;
- _prev = nullptr;
-
+ bool is_blocking) :
+ _compile_id(compile_id),
+ _method(method()),
+ _method_holder(JNIHandles::make_weak_global(Handle(Thread::current(), method->method_holder()->klass_holder()))),
+ _osr_bci(osr_bci),
+ _is_complete(false),
+ _is_success(false),
+ _is_blocking(is_blocking),
+ _nm_content_size(0),
+ _nm_total_size(0),
+ _nm_insts_size(0),
+ _comp_level(comp_level),
+ _compiler(CompileBroker::compiler(comp_level)),
+ _comp_directive_matcher(method, static_cast(comp_level)),
+ JVMCI_ONLY(_has_waiter(_compiler->is_jvmci()) COMMA)
+ JVMCI_ONLY(_blocking_jvmci_compile_state(nullptr) COMMA)
+ _num_inlined_bytecodes(0),
+ _next(nullptr),
+ _prev(nullptr),
+ _time_created(os::elapsed_counter()),
+ _time_queued(0),
+ _time_started(0),
+ _time_finished(0),
+ _hot_count(hot_count),
+ _compile_reason(compile_reason),
+ _failure_reason(nullptr),
+ _failure_reason_on_C_heap(false),
+ _training_data(nullptr),
+ _arena_bytes(0)
+{
AtomicAccess::add(&_active_tasks, 1, memory_order_relaxed);
}
@@ -108,7 +106,8 @@ void CompileTask::wait_for_no_active_tasks() {
* Returns the compiler for this task.
*/
AbstractCompiler* CompileTask::compiler() const {
- return CompileBroker::compiler(_comp_level);
+ assert(_compiler != nullptr, "should be set");
+ return _compiler;
}
// Replace weak handles by strong handles to avoid unloading during compilation.
@@ -157,7 +156,7 @@ void CompileTask::metadata_do(MetadataClosure* f) {
//
void CompileTask::print_line_on_error(outputStream* st, char* buf, int buflen) {
// print compiler name
- st->print("%s:", CompileBroker::compiler_name(comp_level()));
+ st->print("%s:", compiler()->name());
print(st);
}
@@ -168,29 +167,71 @@ void CompileTask::print_tty() {
print(tty);
}
+void CompileTask::print_post(outputStream* st) {
+ bool is_osr_method = osr_bci() != InvocationEntryBci;
+ print_impl(st, is_unloaded() ? nullptr : method(), compile_id(), comp_level(),
+ is_osr_method, osr_bci(), is_blocking(),
+ compiler()->name(), nullptr, false /* short_form */, true /* cr */,
+ true /* after_compile_details */,
+ _num_inlined_bytecodes, _nm_total_size, _nm_insts_size,
+ _time_created, _time_queued, _time_started, _time_finished);
+}
+
// ------------------------------------------------------------------
// CompileTask::print_impl
void CompileTask::print_impl(outputStream* st, Method* method, int compile_id, int comp_level,
bool is_osr_method, int osr_bci, bool is_blocking,
- const char* msg, bool short_form, bool cr,
- jlong time_queued, jlong time_started) {
- if (!short_form) {
+ const char* compiler_name,
+ const char* msg, bool short_form, bool cr, bool after_compile_details,
+ int inlined_bytecodes, int nm_total_size, int nm_insts_size,
+ jlong time_created, jlong time_queued, jlong time_started, jlong time_finished) {
+ // Use stringStream to avoid breaking the line
+ stringStream sst;
+ if (after_compile_details) {
+ { // Print current time
+ stringStream ss;
+ ss.print(UINT64_FORMAT, (uint64_t) tty->time_stamp().milliseconds());
+ sst.print("%7s ", ss.freeze());
+ }
+ { // Time waiting to be put on queue
+ stringStream ss;
+ if (time_created != 0 && time_queued != 0) {
+ ss.print("W%.1f", TimeHelper::counter_to_millis(time_queued - time_created));
+ }
+ sst.print("%7s ", ss.freeze());
+ }
+ { // Time in queue
+ stringStream ss;
+ if (time_queued != 0 && time_started != 0) {
+ ss.print("Q%.1f", TimeHelper::counter_to_millis(time_started - time_queued));
+ }
+ sst.print("%7s ", ss.freeze());
+ }
+ { // Time in compilation
+ stringStream ss;
+ if (time_started != 0 && time_finished != 0) {
+ ss.print("C%.1f", TimeHelper::counter_to_millis(time_finished - time_started));
+ }
+ sst.print("%7s ", ss.freeze());
+ }
+ } else if (!short_form) {
// Print current time
- st->print(UINT64_FORMAT " ", (uint64_t) tty->time_stamp().milliseconds());
+ sst.print(UINT64_FORMAT " ", (uint64_t) tty->time_stamp().milliseconds());
if (Verbose && time_queued != 0) {
// Print time in queue and time being processed by compiler thread
jlong now = os::elapsed_counter();
- st->print("%.0f ", TimeHelper::counter_to_millis(now-time_queued));
+ sst.print("%.0f ", TimeHelper::counter_to_millis(now-time_queued));
if (time_started != 0) {
- st->print("%.0f ", TimeHelper::counter_to_millis(now-time_started));
+ sst.print("%.0f ", TimeHelper::counter_to_millis(now-time_started));
}
}
}
+
// print compiler name if requested
if (CIPrintCompilerName) {
- st->print("%s:", CompileBroker::compiler_name(comp_level));
+ sst.print("%s:", compiler_name);
}
- st->print("%4d ", compile_id); // print compilation number
+ sst.print("%4d ", compile_id); // print compilation number
bool is_synchronized = false;
bool has_exception_handler = false;
@@ -208,40 +249,52 @@ void CompileTask::print_impl(outputStream* st, Method* method, int compile_id, i
const char native_char = is_native ? 'n' : ' ';
// print method attributes
- st->print("%c%c%c%c%c ", compile_type, sync_char, exception_char, blocking_char, native_char);
+ sst.print("%c%c%c%c%c ", compile_type, sync_char, exception_char, blocking_char, native_char);
if (TieredCompilation) {
- if (comp_level != -1) st->print("%d ", comp_level);
- else st->print("- ");
+ if (comp_level != -1) sst.print("%d ", comp_level);
+ else sst.print("- ");
}
- st->print(" "); // more indent
+ sst.print(" "); // more indent
if (method == nullptr) {
- st->print("(method)");
+ sst.print("(method)");
} else {
- method->print_short_name(st);
- if (is_osr_method) {
- st->print(" @ %d", osr_bci);
+ if (after_compile_details) {
+ sst.print("%s", method->name_and_sig_as_C_string(true /* use_double_colon */));
+ } else {
+ method->print_short_name(&sst);
}
- if (method->is_native())
- st->print(" (native)");
- else
- st->print(" (%d bytes)", method->code_size());
+ if (is_osr_method) {
+ sst.print(" @ %d", osr_bci);
+ }
+ if (method->is_native()) {
+ sst.print(" (native)");
+ } else {
+ sst.print(" (%d bytes)", method->code_size());
+ }
+ }
+ if (after_compile_details) {
+ sst.print(" (inlined %d)", inlined_bytecodes);
+ sst.print(" (size %d/%d)", nm_total_size, nm_insts_size);
}
if (msg != nullptr) {
- st->print(" %s", msg);
+ sst.print(" %s", msg);
}
if (cr) {
- st->cr();
+ sst.cr();
}
+ st->print("%s",sst.freeze());
}
// ------------------------------------------------------------------
// CompileTask::print_compilation
void CompileTask::print(outputStream* st, const char* msg, bool short_form, bool cr) {
bool is_osr_method = osr_bci() != InvocationEntryBci;
- print_impl(st, is_unloaded() ? nullptr : method(), compile_id(), comp_level(), is_osr_method, osr_bci(), is_blocking(), msg, short_form, cr, _time_queued, _time_started);
+ print_impl(st, is_unloaded() ? nullptr : method(), compile_id(), comp_level(),
+ is_osr_method, osr_bci(), is_blocking(),
+ compiler()->name(), msg, short_form, cr);
}
// ------------------------------------------------------------------
@@ -435,6 +488,7 @@ void CompileTask::print_ul(const nmethod* nm, const char* msg) {
nm->comp_level(), nm->is_osr_method(),
nm->is_osr_method() ? nm->osr_entry_bci() : -1,
/*is_blocking*/ false,
+ nm->compiler_name(),
msg, /* short form */ true, /* cr */ true);
}
}
diff --git a/src/hotspot/share/compiler/compileTask.hpp b/src/hotspot/share/compiler/compileTask.hpp
index 2cc5e9afe3c..b6174af72eb 100644
--- a/src/hotspot/share/compiler/compileTask.hpp
+++ b/src/hotspot/share/compiler/compileTask.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -28,12 +28,12 @@
#include "ci/ciMethod.hpp"
#include "code/nmethod.hpp"
#include "compiler/compileLog.hpp"
+#include "compiler/compilerDirectives.hpp"
#include "memory/allocation.hpp"
#include "runtime/mutexLocker.hpp"
#include "utilities/xmlstream.hpp"
class CompileTrainingData;
-class DirectiveSet;
JVMCI_ONLY(class JVMCICompileState;)
@@ -93,19 +93,22 @@ class CompileTask : public CHeapObj {
CodeSection::csize_t _nm_content_size;
CodeSection::csize_t _nm_total_size;
CodeSection::csize_t _nm_insts_size;
- DirectiveSet* _directive;
+ int _comp_level;
+ AbstractCompiler* _compiler;
+ CompilerDirectiveMatcher _comp_directive_matcher;
#if INCLUDE_JVMCI
bool _has_waiter;
// Compilation state for a blocking JVMCI compilation
JVMCICompileState* _blocking_jvmci_compile_state;
#endif
- int _comp_level;
int _num_inlined_bytecodes;
CompileTask* _next;
CompileTask* _prev;
// Fields used for logging why the compilation was initiated:
+ jlong _time_created; // time when task was created
jlong _time_queued; // time when task was enqueued
jlong _time_started; // time when compilation started
+ jlong _time_finished; // time when compilation finished
int _hot_count; // information about its invocation counter
CompileReason _compile_reason; // more info about the task
const char* _failure_reason;
@@ -118,6 +121,7 @@ class CompileTask : public CHeapObj {
CompileTask(int compile_id, const methodHandle& method, int osr_bci, int comp_level,
int hot_count, CompileReason compile_reason, bool is_blocking);
~CompileTask();
+
static void wait_for_no_active_tasks();
int compile_id() const { return _compile_id; }
@@ -126,7 +130,9 @@ class CompileTask : public CHeapObj {
bool is_complete() const { return _is_complete; }
bool is_blocking() const { return _is_blocking; }
bool is_success() const { return _is_success; }
- DirectiveSet* directive() const { return _directive; }
+ DirectiveSet* directive() const { return _comp_directive_matcher.directive_set(); }
+ void transfer_directive(CompilerDirectiveMatcher& matcher) { _comp_directive_matcher.transfer_from(matcher); }
+ CompileReason compile_reason() const { return _compile_reason; }
CodeSection::csize_t nm_content_size() { return _nm_content_size; }
void set_nm_content_size(CodeSection::csize_t size) { _nm_content_size = size; }
CodeSection::csize_t nm_insts_size() { return _nm_insts_size; }
@@ -166,8 +172,9 @@ class CompileTask : public CHeapObj {
void mark_complete() { _is_complete = true; }
void mark_success() { _is_success = true; }
+ void mark_queued(jlong time) { _time_queued = time; }
void mark_started(jlong time) { _time_started = time; }
-
+ void mark_finished(jlong time) { _time_finished = time; }
int comp_level() { return _comp_level;}
void set_comp_level(int comp_level) { _comp_level = comp_level;}
@@ -198,16 +205,20 @@ class CompileTask : public CHeapObj {
private:
static void print_impl(outputStream* st, Method* method, int compile_id, int comp_level,
bool is_osr_method = false, int osr_bci = -1, bool is_blocking = false,
+ const char* compiler_name = nullptr,
const char* msg = nullptr, bool short_form = false, bool cr = true,
- jlong time_queued = 0, jlong time_started = 0);
+ bool after_compile_details = false,
+ int inlined_bytecodes = 0, int nm_total_size = 0, int nm_insts_size = 0,
+ jlong time_created = 0, jlong time_queued = 0,
+ jlong time_started = 0, jlong time_finished = 0);
public:
void print(outputStream* st = tty, const char* msg = nullptr, bool short_form = false, bool cr = true);
void print_ul(const char* msg = nullptr);
static void print(outputStream* st, const nmethod* nm, const char* msg = nullptr, bool short_form = false, bool cr = true) {
print_impl(st, nm->method(), nm->compile_id(), nm->comp_level(),
- nm->is_osr_method(), nm->is_osr_method() ? nm->osr_entry_bci() : -1, /*is_blocking*/ false,
- msg, short_form, cr);
+ nm->is_osr_method(), nm->is_osr_method() ? nm->osr_entry_bci() : -1, /*is_blocking*/ false,
+ nm->compiler_name(), msg, short_form, cr);
}
static void print_ul(const nmethod* nm, const char* msg = nullptr);
@@ -217,6 +228,7 @@ public:
static void print_inline_indent(int inline_level, outputStream* st = tty);
void print_tty();
+ void print_post(outputStream* st);
void print_line_on_error(outputStream* st, char* buf, int buflen);
void log_task(xmlStream* log);
@@ -224,6 +236,7 @@ public:
void log_task_start(CompileLog* log);
void log_task_done(CompileLog* log);
+ const char* failure_reason() const { return _failure_reason; }
void set_failure_reason(const char* reason, bool on_C_heap = false) {
_failure_reason = reason;
_failure_reason_on_C_heap = on_C_heap;
diff --git a/src/hotspot/share/compiler/compilerDirectives.cpp b/src/hotspot/share/compiler/compilerDirectives.cpp
index 1cd8bd1b510..f61aa111e65 100644
--- a/src/hotspot/share/compiler/compilerDirectives.cpp
+++ b/src/hotspot/share/compiler/compilerDirectives.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -25,6 +25,7 @@
#include "ci/ciMethod.hpp"
#include "ci/ciUtilities.inline.hpp"
#include "compiler/abstractCompiler.hpp"
+#include "compiler/compileBroker.hpp"
#include "compiler/compilerDefinitions.inline.hpp"
#include "compiler/compilerDirectives.hpp"
#include "compiler/compilerOracle.hpp"
@@ -256,7 +257,7 @@ ControlIntrinsicIter::ControlIntrinsicIter(ccstrlist option_value, bool disable_
}
ControlIntrinsicIter::~ControlIntrinsicIter() {
- FREE_C_HEAP_ARRAY(char, _list);
+ FREE_C_HEAP_ARRAY(_list);
}
// pre-increment
@@ -378,7 +379,7 @@ class DirectiveSetPtr {
// - if some option is changed we need to copy directiveset since it no longer can be shared
// - Need to free copy after use
// - Requires a modified bit so we don't overwrite options that is set by directives
-DirectiveSet* DirectiveSet::compilecommand_compatibility_init(const methodHandle& method) {
+DirectiveSet* DirectiveSet::compilecommand_compatibility_init(const methodHandle& method, int comp_level) {
// Early bail out - checking all options is expensive - we rely on them not being used
// Only set a flag if it has not been modified and value changes.
// Only copy set if a flag needs to be set
@@ -397,7 +398,7 @@ DirectiveSet* DirectiveSet::compilecommand_compatibility_init(const methodHandle
// All CompileCommands are not equal so this gets a bit verbose
// When CompileCommands have been refactored less clutter will remain.
- if (CompilerOracle::should_break_at(method)) {
+ if (CompilerOracle::should_break_at(method, static_cast(comp_level))) {
// If the directives didn't have 'BreakAtCompile' or 'BreakAtExecute',
// the sub-command 'Break' of the 'CompileCommand' would become effective.
if (!_modified[BreakAtCompileIndex]) {
@@ -414,13 +415,13 @@ DirectiveSet* DirectiveSet::compilecommand_compatibility_init(const methodHandle
}
}
- if (CompilerOracle::should_print(method)) {
+ if (CompilerOracle::should_print(method, static_cast(comp_level))) {
if (!_modified[PrintAssemblyIndex]) {
set.cloned()->PrintAssemblyOption = true;
}
}
// Exclude as in should not compile == Enabled
- if (CompilerOracle::should_exclude(method)) {
+ if (CompilerOracle::should_exclude(method, static_cast(comp_level))) {
if (!_modified[ExcludeIndex]) {
set.cloned()->ExcludeOption = true;
}
@@ -547,7 +548,7 @@ bool DirectiveSet::should_inline(ciMethod* inlinee) {
return false;
}
-bool DirectiveSet::should_not_inline(ciMethod* inlinee) {
+bool DirectiveSet::should_not_inline(ciMethod* inlinee, int comp_level) {
inlinee->check_is_loaded();
VM_ENTRY_MARK;
methodHandle mh(THREAD, inlinee->get_Method());
@@ -556,7 +557,7 @@ bool DirectiveSet::should_not_inline(ciMethod* inlinee) {
return matches_inline(mh, InlineMatcher::dont_inline);
}
if (!CompilerDirectivesIgnoreCompileCommandsOption) {
- return CompilerOracle::should_not_inline(mh);
+ return CompilerOracle::should_not_inline(mh, static_cast(comp_level));
}
return false;
}
@@ -755,7 +756,7 @@ void DirectivesStack::release(DirectiveSet* set) {
assert(set != nullptr, "Never nullptr");
MutexLocker locker(DirectivesStack_lock, Mutex::_no_safepoint_check_flag);
if (set->is_exclusive_copy()) {
- // Old CompilecCmmands forced us to create an exclusive copy
+ // Old CompileCommands forced us to create an exclusive copy
delete set;
} else {
assert(set->directive() != nullptr, "Never nullptr");
@@ -772,8 +773,9 @@ void DirectivesStack::release(CompilerDirectives* dir) {
}
}
-DirectiveSet* DirectivesStack::getMatchingDirective(const methodHandle& method, AbstractCompiler *comp) {
+DirectiveSet* DirectivesStack::getMatchingDirective(const methodHandle& method, int comp_level) {
assert(_depth > 0, "Must never be empty");
+ AbstractCompiler* comp = CompileBroker::compiler(comp_level);
DirectiveSet* match = nullptr;
{
@@ -798,5 +800,5 @@ DirectiveSet* DirectivesStack::getMatchingDirective(const methodHandle& method,
guarantee(match != nullptr, "There should always be a default directive that matches");
// Check for legacy compile commands update, without DirectivesStack_lock
- return match->compilecommand_compatibility_init(method);
+ return match->compilecommand_compatibility_init(method, comp_level);
}
diff --git a/src/hotspot/share/compiler/compilerDirectives.hpp b/src/hotspot/share/compiler/compilerDirectives.hpp
index e4826b3056c..ae814cdc491 100644
--- a/src/hotspot/share/compiler/compilerDirectives.hpp
+++ b/src/hotspot/share/compiler/compilerDirectives.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -44,6 +44,7 @@
cflags(MemStat, uintx, 0, MemStat) \
cflags(PrintAssembly, bool, PrintAssembly, PrintAssembly) \
cflags(PrintCompilation, bool, PrintCompilation, PrintCompilation) \
+ cflags(PrintCompilation2, bool, PrintCompilation2, PrintCompilation2) \
cflags(PrintInlining, bool, PrintInlining, PrintInlining) \
cflags(PrintNMethods, bool, PrintNMethods, PrintNMethods) \
cflags(BackgroundCompilation, bool, BackgroundCompilation, BackgroundCompilation) \
@@ -105,23 +106,26 @@ class CompilerDirectives;
class DirectiveSet;
class DirectivesStack : AllStatic {
+ // To allow access to private methods
+ friend class CompilerDirectiveMatcher;
+ friend class DirectiveSetPtr;
private:
static CompilerDirectives* _top;
static CompilerDirectives* _bottom;
static int _depth;
static void pop_inner(); // no lock version of pop
+ static DirectiveSet* getMatchingDirective(const methodHandle& mh, int comp_level);
+ static DirectiveSet* getDefaultDirective(AbstractCompiler* comp);
+ static void release(DirectiveSet* set);
+ static void release(CompilerDirectives* dir);
public:
static void init();
- static DirectiveSet* getMatchingDirective(const methodHandle& mh, AbstractCompiler* comp);
- static DirectiveSet* getDefaultDirective(AbstractCompiler* comp);
static void push(CompilerDirectives* directive);
static void pop(int count);
static bool check_capacity(int request_size, outputStream* st);
static void clear();
static void print(outputStream* st);
- static void release(DirectiveSet* set);
- static void release(CompilerDirectives* dir);
};
class DirectiveSet : public CHeapObj {
@@ -141,10 +145,10 @@ public:
bool parse_and_add_inline(char* str, const char*& error_msg);
void append_inline(InlineMatcher* m);
bool should_inline(ciMethod* inlinee);
- bool should_not_inline(ciMethod* inlinee);
+ bool should_not_inline(ciMethod* inlinee, int comp_level);
bool should_delay_inline(ciMethod* inlinee);
void print_inline(outputStream* st);
- DirectiveSet* compilecommand_compatibility_init(const methodHandle& method);
+ DirectiveSet* compilecommand_compatibility_init(const methodHandle& method, int comp_level);
bool is_exclusive_copy() { return _directive == nullptr; }
bool matches_inline(const methodHandle& method, int inline_action);
static DirectiveSet* clone(DirectiveSet const* src);
@@ -283,7 +287,7 @@ class ControlIntrinsicValidator {
~ControlIntrinsicValidator() {
if (_bad != nullptr) {
- FREE_C_HEAP_ARRAY(char, _bad);
+ FREE_C_HEAP_ARRAY(_bad);
}
}
@@ -326,4 +330,40 @@ public:
DirectiveSet* _c2_store;
};
+// Helper class to get a matching CompilerDirective using RAII pattern.
+// CompileDirective ref count is decremented in the destructor.
+class CompilerDirectiveMatcher {
+private:
+ DirectiveSet* _match;
+
+ void release_match() {
+ if (_match != nullptr) {
+ DirectivesStack::release(_match);
+ _match = nullptr;
+ }
+ }
+
+public:
+ // Use this constructor to get default directive
+ CompilerDirectiveMatcher(AbstractCompiler* comp) {
+ _match = DirectivesStack::getDefaultDirective(comp);
+ }
+
+ CompilerDirectiveMatcher(const methodHandle& mh, int comp_level) {
+ _match = DirectivesStack::getMatchingDirective(mh, comp_level);
+ }
+
+ ~CompilerDirectiveMatcher() {
+ release_match();
+ }
+
+ DirectiveSet* directive_set() const { return _match; }
+
+ void transfer_from(CompilerDirectiveMatcher& src) {
+ release_match();
+ _match = src._match;
+ src._match = nullptr;
+ }
+};
+
#endif // SHARE_COMPILER_COMPILERDIRECTIVES_HPP
diff --git a/src/hotspot/share/compiler/compilerOracle.cpp b/src/hotspot/share/compiler/compilerOracle.cpp
index 5bcd01a4d09..241d27eb6be 100644
--- a/src/hotspot/share/compiler/compilerOracle.cpp
+++ b/src/hotspot/share/compiler/compilerOracle.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -60,6 +60,37 @@ static const char* const default_compile_commands[] = {
#endif
nullptr };
+// CompLevel | -XX:CompileCommand bitmask
+// ----------------------------------------------------
+// 0 (interpreter) | N/A
+// 1 (C1) | 1
+// 2 (C1 + counters) | 10
+// 3 (C1 + counters + mdo) | 100
+// 4 (C2/JVMCI) | 1000
+// All C1 levels | 111
+// All levels | 1111
+
+static const int comp_level_bitmask[CompLevel_count] = {0, 1, 10, 100, 1000};
+static const int comp_level_bitmask_all_levels = 1111;
+static const intx default_comp_level_argument = comp_level_bitmask_all_levels;
+
+inline bool bitmask_applies_to_comp_level(int bitmask, int comp_level) {
+ assert(comp_level > CompLevel_none && comp_level < CompLevel_count, "CompLevel out of bounds");
+ return (bitmask / comp_level_bitmask[comp_level]) % 10 == 1;
+}
+
+static bool is_valid_comp_level_bitmask(intx bitmask) {
+ if (bitmask < 0 || bitmask > comp_level_bitmask_all_levels) {
+ return false;
+ }
+ for (; bitmask != 0; bitmask /= 10) {
+ if (bitmask % 10 > 1) {
+ return false;
+ }
+ }
+ return true;
+}
+
static const char* optiontype_names[] = {
#define enum_of_types(type, name) name,
OPTION_TYPES(enum_of_types)
@@ -456,36 +487,56 @@ template bool CompilerOracle::option_matches_type(CompileCommandEnum optio
template bool CompilerOracle::option_matches_type(CompileCommandEnum option, ccstr& value);
template bool CompilerOracle::option_matches_type(CompileCommandEnum option, double& value);
+bool CompilerOracle::applies_to_comp_level(const methodHandle& method, CompileCommandEnum command, CompLevel current_level) {
+ if (current_level == CompLevel_none) {
+ return false;
+ }
+
+ intx bitmask = 0;
+ if (!has_option_value(method, command, bitmask)) {
+ return false;
+ }
+
+ // Since we don't have bitmask for interpreter level (0), but still need to call CompilerOracle::should_print()
+ // from collect_profiled_methods() in java.cpp, a special value of CompLevel_any produces a match with any bitmask, even 0
+ return current_level == CompLevel_any
+ || bitmask_applies_to_comp_level(bitmask, current_level);
+}
+
bool CompilerOracle::has_option(const methodHandle& method, CompileCommandEnum option) {
bool value = false;
has_option_value(method, option, value);
return value;
}
-bool CompilerOracle::should_exclude(const methodHandle& method) {
- if (check_predicate(CompileCommandEnum::Exclude, method)) {
+bool CompilerOracle::should_exclude(const methodHandle& method, const CompLevel level) {
+ if (has_exclude(method, level)) {
return true;
}
if (has_command(CompileCommandEnum::CompileOnly)) {
- return !check_predicate(CompileCommandEnum::CompileOnly, method);
+ return !applies_to_comp_level(method, CompileCommandEnum::CompileOnly, level);
}
return false;
}
+bool CompilerOracle::has_exclude(const methodHandle& method, const CompLevel level) {
+ return applies_to_comp_level(method, CompileCommandEnum::Exclude, level);
+}
+
bool CompilerOracle::should_inline(const methodHandle& method) {
return (check_predicate(CompileCommandEnum::Inline, method));
}
-bool CompilerOracle::should_not_inline(const methodHandle& method) {
- return check_predicate(CompileCommandEnum::DontInline, method) || check_predicate(CompileCommandEnum::Exclude, method);
+bool CompilerOracle::should_not_inline(const methodHandle& method, const CompLevel level) {
+ return check_predicate(CompileCommandEnum::DontInline, method) || has_exclude(method, level);
}
bool CompilerOracle::should_delay_inline(const methodHandle& method) {
return (check_predicate(CompileCommandEnum::DelayInline, method));
}
-bool CompilerOracle::should_print(const methodHandle& method) {
- return check_predicate(CompileCommandEnum::Print, method);
+bool CompilerOracle::should_print(const methodHandle& method, const CompLevel level) {
+ return applies_to_comp_level(method, CompileCommandEnum::Print, level);
}
bool CompilerOracle::should_print_methods() {
@@ -505,8 +556,8 @@ bool CompilerOracle::should_log(const methodHandle& method) {
return (check_predicate(CompileCommandEnum::Log, method));
}
-bool CompilerOracle::should_break_at(const methodHandle& method) {
- return check_predicate(CompileCommandEnum::Break, method);
+bool CompilerOracle::should_break_at(const methodHandle& method, const CompLevel level) {
+ return applies_to_comp_level(method, CompileCommandEnum::Break, level);
}
void CompilerOracle::tag_blackhole_if_possible(const methodHandle& method) {
@@ -678,6 +729,19 @@ static void usage() {
tty->print_cr("from inlining, whereas the 'compileonly' command only excludes methods from");
tty->print_cr("top-level compilations (i.e. they can still be inlined into other compilation units).");
tty->cr();
+ tty->print_cr("Compilation levels can be specified in the 'compileonly', 'exclude', 'print',");
+ tty->print_cr("and 'break' commands using a binary bitmask as an optional value:");
+ tty->print_cr(" -XX:CompileCommand=exclude,java/*.*,1011 -XX:CompileCommand=print,java/*.*,100");
+ tty->cr();
+ tty->print_cr("The bitmask is calculated by summing the desired compilation level values:");
+ tty->print_cr(" C1 without profiling = 1");
+ tty->print_cr(" C1 with limited profiling = 10");
+ tty->print_cr(" C1 with full profiling = 100");
+ tty->print_cr(" C2 = 1000");
+ tty->cr();
+ tty->print_cr("Note: Excluding specific compilation levels may disrupt normal state transitions");
+ tty->print_cr("between the levels, as the VM will not automatically work around the excluded ones.");
+ tty->cr();
};
static int skip_whitespace(char* &line) {
@@ -712,7 +776,7 @@ static bool parseMemLimit(const char* line, intx& value, int& bytes_read, char*
size_t s = 0;
char* end;
if (!parse_integer(line, &end, &s)) {
- jio_snprintf(errorbuf, buf_size, "MemLimit: invalid value");
+ jio_snprintf(errorbuf, buf_size, ": invalid integer: '%.20s'", line);
return false;
}
bytes_read = (int)(end - line);
@@ -726,7 +790,7 @@ static bool parseMemLimit(const char* line, intx& value, int& bytes_read, char*
// ok, this is the default
bytes_read += 5;
} else {
- jio_snprintf(errorbuf, buf_size, "MemLimit: invalid option");
+ jio_snprintf(errorbuf, buf_size, ": invalid suffix: '%.6s'", end);
return false;
}
}
@@ -751,7 +815,7 @@ static bool parseMemStat(const char* line, uintx& value, int& bytes_read, char*
});
#undef IF_ENUM_STRING
- jio_snprintf(errorbuf, buf_size, "MemStat: invalid option");
+ jio_snprintf(errorbuf, buf_size, ": invalid option: '%.8s'", line);
return false;
}
@@ -763,21 +827,42 @@ static bool scan_value(enum OptionType type, char* line, int& total_bytes_read,
const char* type_str = optiontype2name(type);
int skipped = skip_whitespace(line);
total_bytes_read += skipped;
+ char parse_error_buf[80] = {};
+
if (type == OptionType::Intx) {
intx value;
bool success = false;
- if (option == CompileCommandEnum::MemLimit) {
- // Special parsing for MemLimit
- success = parseMemLimit(line, value, bytes_read, errorbuf, buf_size);
- } else {
- // Is it a raw number?
- success = sscanf(line, "%zd%n", &value, &bytes_read) == 1;
+ switch (option) {
+ case CompileCommandEnum::MemLimit:
+ // Special parsing for MemLimit
+ success = parseMemLimit(line, value, bytes_read, parse_error_buf, sizeof(parse_error_buf));
+ break;
+ case CompileCommandEnum::Break:
+ case CompileCommandEnum::CompileOnly:
+ case CompileCommandEnum::Exclude:
+ case CompileCommandEnum::Print:
+ // In the commands above the parameter used to be a boolean. Now it is an int (a compilation level mask).
+ // For compatibility with previous versions we keep it optional. If user did not specify the mask, assume default value
+ if (*line == '\0') {
+ value = default_comp_level_argument;
+ success = true;
+ } else {
+ success = sscanf(line, "%zd%n", &value, &bytes_read) == 1;
+ if (success && !is_valid_comp_level_bitmask(value)) {
+ jio_snprintf(parse_error_buf, sizeof(parse_error_buf), ": invalid compilation level bitmask '%.*s'", bytes_read, line);
+ success = false;
+ }
+ }
+ break;
+ default:
+ // Is it a raw number?
+ success = sscanf(line, "%zd%n", &value, &bytes_read) == 1;
}
if (success) {
total_bytes_read += bytes_read;
return register_command(matcher, option, errorbuf, buf_size, value);
} else {
- jio_snprintf(errorbuf, buf_size, "Value cannot be read for option '%s' of type '%s'", ccname, type_str);
+ jio_snprintf(errorbuf, buf_size, "Value cannot be read for option '%s' of type '%s'%s", ccname, type_str, parse_error_buf);
return false;
}
} else if (type == OptionType::Uintx) {
@@ -785,7 +870,7 @@ static bool scan_value(enum OptionType type, char* line, int& total_bytes_read,
bool success = false;
if (option == CompileCommandEnum::MemStat) {
// Special parsing for MemStat
- success = parseMemStat(line, value, bytes_read, errorbuf, buf_size);
+ success = parseMemStat(line, value, bytes_read, parse_error_buf, sizeof(parse_error_buf));
} else {
// parse as raw number
success = sscanf(line, "%zu%n", &value, &bytes_read) == 1;
@@ -794,7 +879,7 @@ static bool scan_value(enum OptionType type, char* line, int& total_bytes_read,
total_bytes_read += bytes_read;
return register_command(matcher, option, errorbuf, buf_size, value);
} else {
- jio_snprintf(errorbuf, buf_size, "Value cannot be read for option '%s' of type '%s'", ccname, type_str);
+ jio_snprintf(errorbuf, buf_size, "Value cannot be read for option '%s' of type '%s'%s", ccname, type_str, parse_error_buf);
return false;
}
} else if (type == OptionType::Ccstr) {
@@ -1089,17 +1174,25 @@ bool CompilerOracle::parse_from_line(char* line) {
return false;
}
return true;
- } else if (option == CompileCommandEnum::MemStat) {
- // MemStat default action is to collect data but to not print
- if (!register_command(matcher, option, error_buf, sizeof(error_buf), (uintx)MemStatAction::collect)) {
+ }
+
+ switch (option) {
+ case CompileCommandEnum::Break:
+ case CompileCommandEnum::CompileOnly:
+ case CompileCommandEnum::Exclude:
+ case CompileCommandEnum::Print:
+ break;
+ case CompileCommandEnum::MemStat:
+ // MemStat default action is to collect data but to not print
+ if (!register_command(matcher, option, error_buf, sizeof(error_buf), (uintx)MemStatAction::collect)) {
+ print_parse_error(error_buf, original.get());
+ return false;
+ }
+ return true;
+ default:
+ jio_snprintf(error_buf, sizeof(error_buf), " Option '%s' is not followed by a value", option2name(option));
print_parse_error(error_buf, original.get());
return false;
- }
- return true;
- } else {
- jio_snprintf(error_buf, sizeof(error_buf), " Option '%s' is not followed by a value", option2name(option));
- print_parse_error(error_buf, original.get());
- return false;
}
}
if (!scan_value(type, line, bytes_read, matcher, option, error_buf, sizeof(error_buf))) {
@@ -1209,7 +1302,7 @@ bool CompilerOracle::parse_compile_only(char* line) {
if (method_pattern != nullptr) {
TypedMethodOptionMatcher* matcher = TypedMethodOptionMatcher::parse_method_pattern(method_pattern, error_buf, sizeof(error_buf));
if (matcher != nullptr) {
- if (register_command(matcher, CompileCommandEnum::CompileOnly, error_buf, sizeof(error_buf), true)) {
+ if (register_command(matcher, CompileCommandEnum::CompileOnly, error_buf, sizeof(error_buf), default_comp_level_argument)) {
continue;
}
}
diff --git a/src/hotspot/share/compiler/compilerOracle.hpp b/src/hotspot/share/compiler/compilerOracle.hpp
index 665f3b2fbfd..bfed52f12e7 100644
--- a/src/hotspot/share/compiler/compilerOracle.hpp
+++ b/src/hotspot/share/compiler/compilerOracle.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -25,6 +25,7 @@
#ifndef SHARE_COMPILER_COMPILERORACLE_HPP
#define SHARE_COMPILER_COMPILERORACLE_HPP
+#include "compiler/compilerDirectives.hpp"
#include "memory/allStatic.hpp"
#include "oops/oopsHierarchy.hpp"
#include "utilities/istream.hpp"
@@ -49,20 +50,21 @@ class methodHandle;
option(Help, "help", Unknown) \
option(Quiet, "quiet", Unknown) \
option(Log, "log", Bool) \
- option(Print, "print", Bool) \
+ option(Print, "print", Intx) \
option(Inline, "inline", Bool) \
option(DelayInline, "delayinline", Bool) \
option(DontInline, "dontinline", Bool) \
option(Blackhole, "blackhole", Bool) \
- option(CompileOnly, "compileonly", Bool)\
- option(Exclude, "exclude", Bool) \
- option(Break, "break", Bool) \
+ option(CompileOnly, "compileonly", Intx) \
+ option(Exclude, "exclude", Intx) \
+ option(Break, "break", Intx) \
option(BreakAtExecute, "BreakAtExecute", Bool) \
option(BreakAtCompile, "BreakAtCompile", Bool) \
option(MemLimit, "MemLimit", Intx) \
option(MemStat, "MemStat", Uintx) \
option(PrintAssembly, "PrintAssembly", Bool) \
option(PrintCompilation, "PrintCompilation", Bool) \
+ option(PrintCompilation2, "PrintCompilation2", Bool) \
option(PrintInlining, "PrintInlining", Bool) \
option(PrintIntrinsics, "PrintIntrinsics", Bool) \
option(PrintNMethods, "PrintNMethods", Bool) \
@@ -134,6 +136,9 @@ class CompilerOracle : AllStatic {
static bool parse_from_input(inputStream::Input* input,
parse_from_line_fn_t* parse_from_line);
+ static bool has_exclude(const methodHandle& method, CompLevel level);
+ static bool applies_to_comp_level(const methodHandle& method, CompileCommandEnum command, CompLevel current_level);
+
public:
// True if the command file has been specified or is implicit
static bool has_command_file();
@@ -142,14 +147,15 @@ class CompilerOracle : AllStatic {
static bool parse_from_file();
// Tells whether we to exclude compilation of method
- static bool should_exclude(const methodHandle& method);
+ static bool should_exclude(const methodHandle & method, CompLevel level);
+
static bool be_quiet() { return _quiet; }
// Tells whether we want to inline this method
static bool should_inline(const methodHandle& method);
// Tells whether we want to disallow inlining of this method
- static bool should_not_inline(const methodHandle& method);
+ static bool should_not_inline(const methodHandle& method, CompLevel level);
// Tells whether we want to delay inlining of this method
static bool should_delay_inline(const methodHandle& method);
@@ -158,13 +164,14 @@ class CompilerOracle : AllStatic {
static bool changes_current_thread(const methodHandle& method);
// Tells whether we should print the assembly for this method
- static bool should_print(const methodHandle& method);
+ // If level == CompLevel_none or CompLevel_any, returns true if there is a print command with any mask
+ static bool should_print(const methodHandle& method, CompLevel level);
// Tells whether we should log the compilation data for this method
static bool should_log(const methodHandle& method);
// Tells whether to break when compiling method
- static bool should_break_at(const methodHandle& method);
+ static bool should_break_at(const methodHandle& method, CompLevel level);
// Tells whether there are any methods to print for print_method_statistics()
static bool should_print_methods();
diff --git a/src/hotspot/share/compiler/directivesParser.cpp b/src/hotspot/share/compiler/directivesParser.cpp
index a2da7c7e0e4..53a8325702e 100644
--- a/src/hotspot/share/compiler/directivesParser.cpp
+++ b/src/hotspot/share/compiler/directivesParser.cpp
@@ -198,7 +198,7 @@ bool DirectivesParser::push_key(const char* str, size_t len) {
strncpy(s, str, len);
s[len] = '\0';
error(KEY_ERROR, "No such key: '%s'.", s);
- FREE_C_HEAP_ARRAY(char, s);
+ FREE_C_HEAP_ARRAY(s);
return false;
}
@@ -370,7 +370,7 @@ bool DirectivesParser::set_option_flag(JSON_TYPE t, JSON_VAL* v, const key* opti
#endif
if (!valid) {
- FREE_C_HEAP_ARRAY(char, s);
+ FREE_C_HEAP_ARRAY(s);
return false;
}
(set->*test)((void *)&s); // Takes ownership.
@@ -440,7 +440,7 @@ bool DirectivesParser::set_option(JSON_TYPE t, JSON_VAL* v) {
assert (error_msg != nullptr, "Must have valid error message");
error(VALUE_ERROR, "Method pattern error: %s", error_msg);
}
- FREE_C_HEAP_ARRAY(char, s);
+ FREE_C_HEAP_ARRAY(s);
}
break;
@@ -472,7 +472,7 @@ bool DirectivesParser::set_option(JSON_TYPE t, JSON_VAL* v) {
error(VALUE_ERROR, "Method pattern error: %s", error_msg);
}
}
- FREE_C_HEAP_ARRAY(char, s);
+ FREE_C_HEAP_ARRAY(s);
}
break;
@@ -622,4 +622,3 @@ bool DirectivesParser::callback(JSON_TYPE t, JSON_VAL* v, uint rlimit) {
}
}
}
-
diff --git a/src/hotspot/share/compiler/oopMap.cpp b/src/hotspot/share/compiler/oopMap.cpp
index 87467d06400..c8d0c5d22ba 100644
--- a/src/hotspot/share/compiler/oopMap.cpp
+++ b/src/hotspot/share/compiler/oopMap.cpp
@@ -869,7 +869,7 @@ ImmutableOopMapSet* ImmutableOopMapSet::clone() const {
}
void ImmutableOopMapSet::operator delete(void* p) {
- FREE_C_HEAP_ARRAY(unsigned char, p);
+ FREE_C_HEAP_ARRAY(p);
}
//------------------------------DerivedPointerTable---------------------------
diff --git a/src/hotspot/share/gc/epsilon/epsilonMonitoringSupport.cpp b/src/hotspot/share/gc/epsilon/epsilonMonitoringSupport.cpp
index 213fc18b8ff..7ec4a0016db 100644
--- a/src/hotspot/share/gc/epsilon/epsilonMonitoringSupport.cpp
+++ b/src/hotspot/share/gc/epsilon/epsilonMonitoringSupport.cpp
@@ -70,7 +70,7 @@ public:
}
~EpsilonSpaceCounters() {
- FREE_C_HEAP_ARRAY(char, _name_space);
+ FREE_C_HEAP_ARRAY(_name_space);
}
inline void update_all(size_t capacity, size_t used) {
diff --git a/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp b/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp
index 34d31702e80..3bf26bf46c9 100644
--- a/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp
+++ b/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp
@@ -68,7 +68,7 @@
* has happened since the allocation.
*/
bool G1BarrierSetC2::g1_can_remove_pre_barrier(GraphKit* kit,
- PhaseValues* phase,
+ PhaseGVN* phase,
Node* adr,
BasicType bt,
uint adr_idx) const {
diff --git a/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.hpp b/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.hpp
index 601d0f1138e..e8a0e797dfa 100644
--- a/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.hpp
+++ b/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.hpp
@@ -74,7 +74,7 @@ private:
protected:
bool g1_can_remove_pre_barrier(GraphKit* kit,
- PhaseValues* phase,
+ PhaseGVN* phase,
Node* adr,
BasicType bt,
uint adr_idx) const;
diff --git a/src/hotspot/share/gc/g1/g1Allocator.cpp b/src/hotspot/share/gc/g1/g1Allocator.cpp
index 78710084ee3..e9d1c13af7a 100644
--- a/src/hotspot/share/gc/g1/g1Allocator.cpp
+++ b/src/hotspot/share/gc/g1/g1Allocator.cpp
@@ -63,8 +63,8 @@ G1Allocator::~G1Allocator() {
_mutator_alloc_regions[i].~MutatorAllocRegion();
_survivor_gc_alloc_regions[i].~SurvivorGCAllocRegion();
}
- FREE_C_HEAP_ARRAY(MutatorAllocRegion, _mutator_alloc_regions);
- FREE_C_HEAP_ARRAY(SurvivorGCAllocRegion, _survivor_gc_alloc_regions);
+ FREE_C_HEAP_ARRAY(_mutator_alloc_regions);
+ FREE_C_HEAP_ARRAY(_survivor_gc_alloc_regions);
}
#ifdef ASSERT
@@ -315,7 +315,7 @@ G1PLABAllocator::PLABData::~PLABData() {
for (uint node_index = 0; node_index < _num_alloc_buffers; node_index++) {
delete _alloc_buffer[node_index];
}
- FREE_C_HEAP_ARRAY(PLAB*, _alloc_buffer);
+ FREE_C_HEAP_ARRAY(_alloc_buffer);
}
void G1PLABAllocator::PLABData::initialize(uint num_alloc_buffers, size_t desired_plab_size, size_t tolerated_refills) {
diff --git a/src/hotspot/share/gc/g1/g1Arguments.cpp b/src/hotspot/share/gc/g1/g1Arguments.cpp
index c3bbd5a3b52..a0acd903b0f 100644
--- a/src/hotspot/share/gc/g1/g1Arguments.cpp
+++ b/src/hotspot/share/gc/g1/g1Arguments.cpp
@@ -98,7 +98,7 @@ void G1Arguments::initialize_verification_types() {
parse_verification_type(token);
token = strtok_r(nullptr, delimiter, &save_ptr);
}
- FREE_C_HEAP_ARRAY(char, type_list);
+ FREE_C_HEAP_ARRAY(type_list);
}
}
diff --git a/src/hotspot/share/gc/g1/g1CardSet.cpp b/src/hotspot/share/gc/g1/g1CardSet.cpp
index 60ad63e812c..f0db638a2fe 100644
--- a/src/hotspot/share/gc/g1/g1CardSet.cpp
+++ b/src/hotspot/share/gc/g1/g1CardSet.cpp
@@ -145,7 +145,7 @@ G1CardSetConfiguration::G1CardSetConfiguration(uint inline_ptr_bits_per_card,
}
G1CardSetConfiguration::~G1CardSetConfiguration() {
- FREE_C_HEAP_ARRAY(size_t, _card_set_alloc_options);
+ FREE_C_HEAP_ARRAY(_card_set_alloc_options);
}
void G1CardSetConfiguration::init_card_set_alloc_options() {
diff --git a/src/hotspot/share/gc/g1/g1CardSetMemory.cpp b/src/hotspot/share/gc/g1/g1CardSetMemory.cpp
index 0da2f90da3f..95a32bae766 100644
--- a/src/hotspot/share/gc/g1/g1CardSetMemory.cpp
+++ b/src/hotspot/share/gc/g1/g1CardSetMemory.cpp
@@ -90,7 +90,7 @@ G1CardSetMemoryManager::~G1CardSetMemoryManager() {
for (uint i = 0; i < num_mem_object_types(); i++) {
_allocators[i].~G1CardSetAllocator();
}
- FREE_C_HEAP_ARRAY(G1CardSetAllocator, _allocators);
+ FREE_C_HEAP_ARRAY(_allocators);
}
void G1CardSetMemoryManager::free(uint type, void* value) {
diff --git a/src/hotspot/share/gc/g1/g1CardTableClaimTable.cpp b/src/hotspot/share/gc/g1/g1CardTableClaimTable.cpp
index d8cabaa00a4..27b41ef165f 100644
--- a/src/hotspot/share/gc/g1/g1CardTableClaimTable.cpp
+++ b/src/hotspot/share/gc/g1/g1CardTableClaimTable.cpp
@@ -39,7 +39,7 @@ G1CardTableClaimTable::G1CardTableClaimTable(uint chunks_per_region) :
}
G1CardTableClaimTable::~G1CardTableClaimTable() {
- FREE_C_HEAP_ARRAY(uint, _card_claims);
+ FREE_C_HEAP_ARRAY(_card_claims);
}
void G1CardTableClaimTable::initialize(uint max_reserved_regions) {
diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp
index 2709e6b3008..0fb2e65f1b9 100644
--- a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp
+++ b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp
@@ -896,7 +896,7 @@ void G1CollectedHeap::abort_refinement() {
// Record any available refinement statistics.
policy()->record_refinement_stats(sweep_state.stats());
- sweep_state.complete_work(false /* concurrent */, false /* print_log */);
+ sweep_state.cancel_refinement();
}
sweep_state.reset_stats();
}
@@ -944,10 +944,6 @@ void G1CollectedHeap::do_full_collection(size_t allocation_word_size,
}
void G1CollectedHeap::do_full_collection(bool clear_all_soft_refs) {
- // Currently, there is no facility in the do_full_collection(bool) API to notify
- // the caller that the collection did not succeed (e.g., because it was locked
- // out by the GC locker). So, right now, we'll ignore the return value.
-
do_full_collection(size_t(0) /* allocation_word_size */,
clear_all_soft_refs,
false /* do_maximal_compaction */);
@@ -2960,7 +2956,7 @@ void G1CollectedHeap::abandon_collection_set() {
collection_set()->abandon();
}
-size_t G1CollectedHeap::non_young_occupancy_after_allocation(size_t allocation_word_size) {
+size_t G1CollectedHeap::non_young_occupancy_after_allocation(size_t allocation_word_size) const {
const size_t cur_occupancy = (old_regions_count() + humongous_regions_count()) * G1HeapRegion::GrainBytes -
_allocator->free_bytes_in_retained_old_region();
// Humongous allocations will always be assigned to non-young heap, so consider
diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.hpp b/src/hotspot/share/gc/g1/g1CollectedHeap.hpp
index 3a47453819e..a68d1030636 100644
--- a/src/hotspot/share/gc/g1/g1CollectedHeap.hpp
+++ b/src/hotspot/share/gc/g1/g1CollectedHeap.hpp
@@ -1032,7 +1032,7 @@ public:
// Returns how much memory there is assigned to non-young heap that can not be
// allocated into any more without garbage collection after a hypothetical
// allocation of allocation_word_size.
- size_t non_young_occupancy_after_allocation(size_t allocation_word_size);
+ size_t non_young_occupancy_after_allocation(size_t allocation_word_size) const;
// Determine whether the given region is one that we are using as an
// old GC alloc region.
diff --git a/src/hotspot/share/gc/g1/g1CollectionSet.cpp b/src/hotspot/share/gc/g1/g1CollectionSet.cpp
index b3bcf6094ab..7329e679519 100644
--- a/src/hotspot/share/gc/g1/g1CollectionSet.cpp
+++ b/src/hotspot/share/gc/g1/g1CollectionSet.cpp
@@ -72,7 +72,7 @@ G1CollectionSet::G1CollectionSet(G1CollectedHeap* g1h, G1Policy* policy) :
}
G1CollectionSet::~G1CollectionSet() {
- FREE_C_HEAP_ARRAY(uint, _regions);
+ FREE_C_HEAP_ARRAY(_regions);
abandon_all_candidates();
}
@@ -373,7 +373,7 @@ double G1CollectionSet::finalize_young_part(double target_pause_time_ms, G1Survi
// made to regular old regions without remembered sets after a few attempts to save computation costs
// of keeping them candidates for very long living pinned regions.
void G1CollectionSet::finalize_old_part(double time_remaining_ms) {
- double non_young_start_time_sec = os::elapsedTime();
+ Ticks start_time = Ticks::now();
if (!candidates()->is_empty()) {
candidates()->verify();
@@ -392,8 +392,7 @@ void G1CollectionSet::finalize_old_part(double time_remaining_ms) {
log_debug(gc, ergo, cset)("No candidates to reclaim.");
}
- double non_young_end_time_sec = os::elapsedTime();
- phase_times()->record_non_young_cset_choice_time_ms((non_young_end_time_sec - non_young_start_time_sec) * 1000.0);
+ phase_times()->record_non_young_cset_choice_time_ms((Ticks::now() - start_time).seconds() * MILLIUNITS);
}
static void print_finish_message(const char* reason, bool from_marking) {
@@ -766,7 +765,7 @@ public:
}
}
~G1VerifyYoungCSetIndicesClosure() {
- FREE_C_HEAP_ARRAY(int, _heap_region_indices);
+ FREE_C_HEAP_ARRAY(_heap_region_indices);
}
virtual bool do_heap_region(G1HeapRegion* r) {
diff --git a/src/hotspot/share/gc/g1/g1CollectionSetCandidates.cpp b/src/hotspot/share/gc/g1/g1CollectionSetCandidates.cpp
index 2113db1163b..3637d477229 100644
--- a/src/hotspot/share/gc/g1/g1CollectionSetCandidates.cpp
+++ b/src/hotspot/share/gc/g1/g1CollectionSetCandidates.cpp
@@ -214,7 +214,7 @@ G1CollectionSetCandidates::G1CollectionSetCandidates() :
{ }
G1CollectionSetCandidates::~G1CollectionSetCandidates() {
- FREE_C_HEAP_ARRAY(CandidateOrigin, _contains_map);
+ FREE_C_HEAP_ARRAY(_contains_map);
_from_marking_groups.clear();
_retained_groups.clear();
}
@@ -413,7 +413,7 @@ void G1CollectionSetCandidates::verify() {
static_cast::type>(verify_map[i]));
}
- FREE_C_HEAP_ARRAY(CandidateOrigin, verify_map);
+ FREE_C_HEAP_ARRAY(verify_map);
}
#endif
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp b/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp
index dbb5ba509a2..83dda2a043b 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp
@@ -249,7 +249,7 @@ G1CMMarkStack::ChunkAllocator::~ChunkAllocator() {
}
}
- FREE_C_HEAP_ARRAY(TaskQueueEntryChunk*, _buckets);
+ FREE_C_HEAP_ARRAY(_buckets);
}
bool G1CMMarkStack::ChunkAllocator::reserve(size_t new_capacity) {
@@ -524,7 +524,10 @@ void G1ConcurrentMark::fully_initialize() {
uint max_num_regions = _g1h->max_num_regions();
::new (_region_mark_stats) G1RegionMarkStats[max_num_regions]{};
- ::new (_top_at_mark_starts) Atomic[max_num_regions]{};
+ for (uint i = 0; i < max_num_regions; i++) {
+ ::new (&_top_at_mark_starts[i]) Atomic(_g1h->bottom_addr_for_region(i));
+ }
+ // Contrary to TAMS, the default value of _top_at_rebuild_starts needs to be null.
::new (_top_at_rebuild_starts) Atomic[max_num_regions]{};
reset_at_marking_complete();
@@ -676,9 +679,9 @@ void G1ConcurrentMark::reset_at_marking_complete() {
}
G1ConcurrentMark::~G1ConcurrentMark() {
- FREE_C_HEAP_ARRAY(Atomic, _top_at_mark_starts);
- FREE_C_HEAP_ARRAY(Atomic, _top_at_rebuild_starts);
- FREE_C_HEAP_ARRAY(G1RegionMarkStats, _region_mark_stats);
+ FREE_C_HEAP_ARRAY(_top_at_mark_starts);
+ FREE_C_HEAP_ARRAY(_top_at_rebuild_starts);
+ FREE_C_HEAP_ARRAY(_region_mark_stats);
// The G1ConcurrentMark instance is never freed.
ShouldNotReachHere();
}
@@ -1146,7 +1149,6 @@ bool G1ConcurrentMark::scan_root_regions(WorkerThreads* workers, bool concurrent
// completing this work during GC.
const uint num_workers = MIN2(num_remaining,
_max_concurrent_workers);
- assert(num_workers > 0, "no more remaining root regions to process");
G1CMRootRegionScanTask task(this, concurrent);
log_debug(gc, ergo)("Running %s using %u workers for %u work units.",
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMark.hpp b/src/hotspot/share/gc/g1/g1ConcurrentMark.hpp
index f9287f673d2..1ab4654a490 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentMark.hpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentMark.hpp
@@ -280,9 +280,9 @@ private:
// Typically they contain the areas from TAMS to top of the regions.
// We could scan and mark through these objects during the concurrent start pause,
// but for pause time reasons we move this work to the concurrent phase.
-// We need to complete this procedure before we can evacuate a particular region
-// because evacuation might determine that some of these "root objects" are dead,
-// potentially dropping some required references.
+// Garbage collections that evacuate must either complete or abort this procedure
+// before they can move objects because evacuation might determine that some of these
+// "root objects" are dead, potentially dropping some references.
// Root MemRegions comprise of the contents of survivor regions at the end
// of the GC, and any objects copied into the old gen during GC.
class G1CMRootMemRegions {
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMarkThread.cpp b/src/hotspot/share/gc/g1/g1ConcurrentMarkThread.cpp
index b8c97acd1b0..8fc4d7a2e20 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentMarkThread.cpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentMarkThread.cpp
@@ -263,23 +263,6 @@ void G1ConcurrentMarkThread::concurrent_mark_cycle_do() {
HandleMark hm(Thread::current());
ResourceMark rm;
- // We have to ensure that we finish scanning the root regions
- // before the next GC takes place. To ensure this we have to
- // make sure that we do not join the STS until the root regions
- // have been scanned. If we did then it's possible that a
- // subsequent GC could block us from joining the STS and proceed
- // without the root regions have been scanned which would be a
- // correctness issue.
- //
- // So do not return before the scan root regions phase as a GC waits for a
- // notification from it.
- //
- // For the same reason ConcurrentGCBreakpoints (in the phase methods) before
- // here risk deadlock, because a young GC must wait for root region scanning.
- //
- // We can not easily abort before root region scan either because of the
- // reasons mentioned in G1CollectedHeap::abort_concurrent_cycle().
-
// Phase 1: Scan root regions.
if (phase_scan_root_regions()) return;
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMarkThread.inline.hpp b/src/hotspot/share/gc/g1/g1ConcurrentMarkThread.inline.hpp
index 8cb7881e000..64441ccac65 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentMarkThread.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentMarkThread.inline.hpp
@@ -32,7 +32,7 @@
// Total virtual time so far.
inline double G1ConcurrentMarkThread::total_mark_cpu_time_s() {
- return static_cast(os::thread_cpu_time(this)) + worker_threads_cpu_time_s();
+ return static_cast(os::thread_cpu_time(this)) / NANOSECS_PER_SEC + worker_threads_cpu_time_s();
}
// Marking virtual time so far
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentRefine.cpp b/src/hotspot/share/gc/g1/g1ConcurrentRefine.cpp
index e12a8c284de..d58d980b651 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentRefine.cpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentRefine.cpp
@@ -101,8 +101,6 @@ void G1ConcurrentRefineThreadControl::activate() {
}
void G1ConcurrentRefineThreadControl::run_task(WorkerTask* task, uint num_workers) {
- assert(num_workers >= 1, "must be");
-
WithActiveWorkers w(_workers, num_workers);
_workers->run_task(task);
}
@@ -137,12 +135,26 @@ G1ConcurrentRefineSweepState::~G1ConcurrentRefineSweepState() {
delete _sweep_table;
}
-void G1ConcurrentRefineSweepState::set_state_start_time() {
- _state_start[static_cast(_state)] = Ticks::now();
+void G1ConcurrentRefineSweepState::enter_state(State state, Ticks timestamp) {
+ assert(state > State::Idle, "precondition");
+ assert(state != State::Last, "preconditon");
+ assert(_state == State(static_cast(state) - 1),
+ "must come from previous state but is %s", state_name(_state));
+
+ _state_start[static_cast(state)] = timestamp;
+ _state = state;
}
-Tickspan G1ConcurrentRefineSweepState::get_duration(State start, State end) {
- return _state_start[static_cast(end)] - _state_start[static_cast(start)];
+Tickspan G1ConcurrentRefineSweepState::time_since_start(Ticks completion_time) const {
+ assert(_state >= State::SwapGlobalCT, "precondition");
+ return completion_time - _state_start[static_cast(State::SwapGlobalCT)];
+}
+
+Tickspan G1ConcurrentRefineSweepState::time_until_state(State state) const {
+ assert(_state >= State::SwapGlobalCT, "precondition");
+ assert(state >= State::SwapGlobalCT, "precondition");
+ assert(state <= _state, "precondition");
+ return _state_start[static_cast(state)] - _state_start[static_cast(State::SwapGlobalCT)];
}
void G1ConcurrentRefineSweepState::reset_stats() {
@@ -153,35 +165,11 @@ void G1ConcurrentRefineSweepState::add_yield_during_sweep_duration(jlong duratio
stats()->inc_yield_during_sweep_duration(duration);
}
-bool G1ConcurrentRefineSweepState::advance_state(State next_state) {
- bool result = is_in_progress();
- if (result) {
- _state = next_state;
- } else {
- _state = State::Idle;
- }
- return result;
-}
-
-void G1ConcurrentRefineSweepState::assert_state(State expected) {
- assert(_state == expected, "must be %s but is %s", state_name(expected), state_name(_state));
-}
-
-void G1ConcurrentRefineSweepState::start_work() {
- assert_state(State::Idle);
-
- set_state_start_time();
-
+bool G1ConcurrentRefineSweepState::swap_global_card_table() {
+ enter_state(State::SwapGlobalCT, Ticks::now());
_stats.reset();
- _state = State::SwapGlobalCT;
-}
-
-bool G1ConcurrentRefineSweepState::swap_global_card_table() {
- assert_state(State::SwapGlobalCT);
-
GCTraceTime(Info, gc, refine) tm("Concurrent Refine Global Card Table Swap");
- set_state_start_time();
{
// We can't have any new threads being in the process of created while we
@@ -192,22 +180,20 @@ bool G1ConcurrentRefineSweepState::swap_global_card_table() {
MutexLocker mu(Threads_lock);
// A GC that advanced the epoch might have happened, which already switched
- // The global card table. Do nothing.
+ // the global card table. Do nothing.
if (is_in_progress()) {
G1BarrierSet::g1_barrier_set()->swap_global_card_table();
}
}
- return advance_state(State::SwapJavaThreadsCT);
+ return is_in_progress();
}
bool G1ConcurrentRefineSweepState::swap_java_threads_ct() {
- assert_state(State::SwapJavaThreadsCT);
+ enter_state(State::SwapJavaThreadsCT, Ticks::now());
GCTraceTime(Info, gc, refine) tm("Concurrent Refine Java Thread CT swap");
- set_state_start_time();
-
{
// Need to leave the STS to avoid potential deadlock in the handshake.
SuspendibleThreadSetLeaver sts;
@@ -224,16 +210,14 @@ bool G1ConcurrentRefineSweepState::swap_java_threads_ct() {
Handshake::execute(&cl);
}
- return advance_state(State::SynchronizeGCThreads);
- }
+ return is_in_progress();
+}
bool G1ConcurrentRefineSweepState::swap_gc_threads_ct() {
- assert_state(State::SynchronizeGCThreads);
+ enter_state(State::SynchronizeGCThreads, Ticks::now());
GCTraceTime(Info, gc, refine) tm("Concurrent Refine GC Thread CT swap");
- set_state_start_time();
-
{
class RendezvousGCThreads: public VM_Operation {
public:
@@ -267,93 +251,148 @@ bool G1ConcurrentRefineSweepState::swap_gc_threads_ct() {
VMThread::execute(&op);
}
- return advance_state(State::SnapshotHeap);
+ return is_in_progress();
}
-void G1ConcurrentRefineSweepState::snapshot_heap(bool concurrent) {
- if (concurrent) {
- GCTraceTime(Info, gc, refine) tm("Concurrent Refine Snapshot Heap");
+void G1ConcurrentRefineSweepState::snapshot_heap() {
+ enter_state(State::SnapshotHeap, Ticks::now());
- assert_state(State::SnapshotHeap);
+ GCTraceTime(Info, gc, refine) tm("Concurrent Refine Snapshot Heap");
- set_state_start_time();
+ snapshot_heap_inner();
+}
+bool G1ConcurrentRefineSweepState::sweep_refinement_table(jlong& total_yield_duration) {
+ enter_state(State::SweepRT, Ticks::now());
+
+ while (true) {
+ {
+ GCTraceTime(Info, gc, refine) tm("Concurrent Refine Table Step");
+
+ G1ConcurrentRefine* cr = G1CollectedHeap::heap()->concurrent_refine();
+
+ G1ConcurrentRefineSweepTask task(_sweep_table, &_stats, cr->num_threads_wanted());
+ cr->run_with_refinement_workers(&task);
+
+ assert(is_in_progress(), "inv");
+ if (task.sweep_completed()) {
+ return true;
+ }
+ }
+
+ assert(SuspendibleThreadSet::should_yield(), "must be");
+ // Interrupted by safepoint request.
+ {
+ jlong yield_start = os::elapsed_counter();
+ SuspendibleThreadSet::yield();
+
+ if (!is_in_progress()) {
+ return false;
+ } else {
+ jlong yield_during_sweep_duration = os::elapsed_counter() - yield_start;
+ log_trace(gc, refine)("Yielded from card table sweeping for %.2fms, no GC inbetween, continue",
+ TimeHelper::counter_to_millis(yield_during_sweep_duration));
+ total_yield_duration += yield_during_sweep_duration;
+ }
+ }
+ }
+}
+
+static void print_refinement_stats(const Tickspan& total_duration,
+ const Tickspan& pre_sweep_duration,
+ const G1ConcurrentRefineStats* stats) {
+ assert(total_duration >= Tickspan(), "must be non-negative");
+ assert(pre_sweep_duration >= Tickspan(), "must be non-negative");
+ assert(pre_sweep_duration <= total_duration, "must be bounded by total duration");
+
+ log_debug(gc, refine)("Refinement took %.2fms (pre-sweep %.2fms card refine %.2fms) "
+ "(scanned %zu clean %zu (%.2f%%) not_clean %zu (%.2f%%) not_parsable %zu "
+ "refers_to_cset %zu (%.2f%%) still_refers_to_cset %zu (%.2f%%) no_cross_region %zu pending %zu)",
+ total_duration.seconds() * 1000.0,
+ pre_sweep_duration.seconds() * 1000.0,
+ TimeHelper::counter_to_millis(stats->refine_duration()),
+ stats->cards_scanned(),
+ stats->cards_clean(),
+ percent_of(stats->cards_clean(), stats->cards_scanned()),
+ stats->cards_not_clean(),
+ percent_of(stats->cards_not_clean(), stats->cards_scanned()),
+ stats->cards_not_parsable(),
+ stats->cards_refer_to_cset(),
+ percent_of(stats->cards_refer_to_cset(), stats->cards_not_clean()),
+ stats->cards_already_refer_to_cset(),
+ percent_of(stats->cards_already_refer_to_cset(), stats->cards_not_clean()),
+ stats->cards_no_cross_region(),
+ stats->cards_pending()
+ );
+}
+
+void G1ConcurrentRefineSweepState::handle_ongoing_refinement_at_safepoint() {
+ assert_at_safepoint();
+ if (!is_in_progress()) {
+ return;
+ }
+
+ const Ticks completion_time = Ticks::now();
+
+ const Tickspan total_duration = time_since_start(completion_time);
+ const Tickspan pre_sweep_duration = time_until_state(MIN2(_state, State::SweepRT));
+
+ print_refinement_stats(total_duration, pre_sweep_duration, &_stats);
+
+ const bool is_in_sweep_rt = _state == State::SweepRT;
+
+ if (!is_in_sweep_rt) {
+ // Refinement has been interrupted without having a snapshot. There may
+ // be a mix of already swapped and not-swapped card tables assigned to threads,
+ // so they might have already dirtied the swapped card tables.
+ // Conservatively scan all (non-free, non-committed) region's card tables,
+ // creating the snapshot right now.
+ log_debug(gc, refine)("Create work from scratch");
snapshot_heap_inner();
-
- advance_state(State::SweepRT);
} else {
- assert_state(State::Idle);
- assert_at_safepoint();
-
- snapshot_heap_inner();
+ log_debug(gc, refine)("Continue existing work");
}
+
+ _state = State::Idle;
}
-void G1ConcurrentRefineSweepState::sweep_refinement_table_start() {
- assert_state(State::SweepRT);
-
- set_state_start_time();
+void G1ConcurrentRefineSweepState::cancel_refinement() {
+ _state = State::Idle;
}
-bool G1ConcurrentRefineSweepState::sweep_refinement_table_step() {
- assert_state(State::SweepRT);
+void G1ConcurrentRefineSweepState::complete_refinement(jlong total_yield_during_sweep_duration,
+ jlong epoch_yield_duration,
+ jlong next_epoch_start) {
+ enter_state(State::CompleteRefineWork, Ticks::now());
- GCTraceTime(Info, gc, refine) tm("Concurrent Refine Table Step");
+ GCTraceTime(Info, gc, refine) tm("Concurrent Refine Complete Work");
- G1ConcurrentRefine* cr = G1CollectedHeap::heap()->concurrent_refine();
+ add_yield_during_sweep_duration(total_yield_during_sweep_duration);
- G1ConcurrentRefineSweepTask task(_sweep_table, &_stats, cr->num_threads_wanted());
- cr->run_with_refinement_workers(&task);
+ const Ticks completion_time = Ticks::now();
- if (task.sweep_completed()) {
- advance_state(State::CompleteRefineWork);
- return true;
- } else {
- return false;
+ const Tickspan total_duration = time_since_start(completion_time);
+ const Tickspan pre_sweep_duration = time_until_state(State::SweepRT);
+
+ print_refinement_stats(total_duration, pre_sweep_duration, &_stats);
+
+ G1CollectedHeap* g1h = G1CollectedHeap::heap();
+ G1Policy* policy = g1h->policy();
+ policy->record_refinement_stats(stats());
+
+ {
+ MutexLocker x(G1ReviseYoungLength_lock, Mutex::_no_safepoint_check_flag);
+ policy->record_dirtying_stats(TimeHelper::counter_to_millis(g1h->last_refinement_epoch_start()),
+ TimeHelper::counter_to_millis(next_epoch_start),
+ _stats.cards_pending(),
+ TimeHelper::counter_to_millis(epoch_yield_duration),
+ 0 /* pending_cards_from_gc */,
+ _stats.cards_to_cset());
+ g1h->set_last_refinement_epoch_start(next_epoch_start, epoch_yield_duration);
}
-}
+ _stats.reset();
-bool G1ConcurrentRefineSweepState::complete_work(bool concurrent, bool print_log) {
- if (concurrent) {
- assert_state(State::CompleteRefineWork);
- } else {
- // May have been forced to complete at any other time.
- assert(is_in_progress() && _state != State::CompleteRefineWork, "must be but is %s", state_name(_state));
- }
-
- set_state_start_time();
-
- if (print_log) {
- G1ConcurrentRefineStats* s = &_stats;
-
- State state_bounded_by_sweeprt = (_state == State::SweepRT || _state == State::CompleteRefineWork)
- ? State::SweepRT : _state;
-
- log_debug(gc, refine)("Refinement took %.2fms (pre-sweep %.2fms card refine %.2fms) "
- "(scanned %zu clean %zu (%.2f%%) not_clean %zu (%.2f%%) not_parsable %zu "
- "refers_to_cset %zu (%.2f%%) still_refers_to_cset %zu (%.2f%%) no_cross_region %zu pending %zu)",
- get_duration(State::Idle, _state).seconds() * 1000.0,
- get_duration(State::Idle, state_bounded_by_sweeprt).seconds() * 1000.0,
- TimeHelper::counter_to_millis(s->refine_duration()),
- s->cards_scanned(),
- s->cards_clean(),
- percent_of(s->cards_clean(), s->cards_scanned()),
- s->cards_not_clean(),
- percent_of(s->cards_not_clean(), s->cards_scanned()),
- s->cards_not_parsable(),
- s->cards_refer_to_cset(),
- percent_of(s->cards_refer_to_cset(), s->cards_not_clean()),
- s->cards_already_refer_to_cset(),
- percent_of(s->cards_already_refer_to_cset(), s->cards_not_clean()),
- s->cards_no_cross_region(),
- s->cards_pending()
- );
- }
-
- bool has_sweep_rt_work = _state == State::SweepRT;
-
- advance_state(State::Idle);
- return has_sweep_rt_work;
+ _state = State::Idle;
}
void G1ConcurrentRefineSweepState::snapshot_heap_inner() {
@@ -385,10 +424,6 @@ void G1ConcurrentRefineSweepState::snapshot_heap_inner() {
G1CollectedHeap::heap()->heap_region_iterate(&cl);
}
-bool G1ConcurrentRefineSweepState::is_in_progress() const {
- return _state != State::Idle;
-}
-
bool G1ConcurrentRefineSweepState::are_java_threads_synched() const {
return _state > State::SwapJavaThreadsCT || !is_in_progress();
}
@@ -424,19 +459,8 @@ jint G1ConcurrentRefine::initialize() {
}
G1ConcurrentRefineSweepState& G1ConcurrentRefine::sweep_state_for_merge() {
- bool has_sweep_claims = sweep_state().complete_work(false /* concurrent */);
- if (has_sweep_claims) {
- log_debug(gc, refine)("Continue existing work");
- } else {
- // Refinement has been interrupted without having a snapshot. There may
- // be a mix of already swapped and not-swapped card tables assigned to threads,
- // so they might have already dirtied the swapped card tables.
- // Conservatively scan all (non-free, non-committed) region's card tables,
- // creating the snapshot right now.
- log_debug(gc, refine)("Create work from scratch");
-
- sweep_state().snapshot_heap(false /* concurrent */);
- }
+ sweep_state().handle_ongoing_refinement_at_safepoint();
+ assert(!sweep_state().is_in_progress(), "postcondition");
return sweep_state();
}
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentRefine.hpp b/src/hotspot/share/gc/g1/g1ConcurrentRefine.hpp
index 5e96ed738fd..50fb412f3af 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentRefine.hpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentRefine.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -32,6 +32,7 @@
#include "utilities/globalDefinitions.hpp"
#include "utilities/growableArray.hpp"
#include "utilities/macros.hpp"
+#include "utilities/ticks.hpp"
// Forward decl
class G1CardTableClaimTable;
@@ -104,8 +105,10 @@ public:
// Calculates statistics about the process to be used in various parts of
// the garbage collection.
//
-// All but step 4 are interruptible by safepoints. In case of a garbage collection,
-// the garbage collection will interrupt this process, and go to Idle state.
+// Steps 1, 2, 3, and 5 can be interrupted by safepoints. In case of a
+// garbage collection, the garbage collection will interrupt this process,
+// and go to Idle state.
+// Step 4 and 6 can not be interrupted.
//
class G1ConcurrentRefineSweepState {
@@ -137,36 +140,39 @@ class G1ConcurrentRefineSweepState {
// Current heap snapshot.
G1CardTableClaimTable* _sweep_table;
- // Start times for all states.
+ // Entry timestamps for states in the current refinement cycle.
+ // The timestamp of a state is only valid if that state has been reached in
+ // this cycle. State transitions must update _state and _state_start together.
Ticks _state_start[static_cast(State::Last)];
- void set_state_start_time();
- Tickspan get_duration(State start, State end);
+ void enter_state(State state, Ticks timestamp);
+ Tickspan time_since_start(Ticks completion_time) const;
+ Tickspan time_until_state(State state) const;
G1ConcurrentRefineStats _stats;
- // Advances the state to next_state if not interrupted by a changed epoch. Returns
- // to Idle otherwise.
- bool advance_state(State next_state);
-
- void assert_state(State expected);
-
void snapshot_heap_inner();
public:
G1ConcurrentRefineSweepState(uint max_reserved_regions);
~G1ConcurrentRefineSweepState();
- void start_work();
-
bool swap_global_card_table();
bool swap_java_threads_ct();
bool swap_gc_threads_ct();
- void snapshot_heap(bool concurrent = true);
- void sweep_refinement_table_start();
- bool sweep_refinement_table_step();
+ void snapshot_heap();
+ bool sweep_refinement_table(jlong& total_yield_duration);
- bool complete_work(bool concurrent, bool print_log = true);
+ // Complete refinement for the current epoch. Finalizes the sweep state,
+ // records stats, and updates policy.
+ void complete_refinement(jlong total_yield_during_sweep_duration,
+ jlong epoch_yield_duration,
+ jlong next_epoch_start);
+
+ void cancel_refinement();
+ // Called at safepoint when refinement was interrupted and we need to merge state.
+ // Logs any accumulated stats and creates a snapshot if SweepRT was not reached.
+ void handle_ongoing_refinement_at_safepoint();
G1CardTableClaimTable* sweep_table() { return _sweep_table; }
G1ConcurrentRefineStats* stats() { return &_stats; }
@@ -174,7 +180,7 @@ public:
void add_yield_during_sweep_duration(jlong duration);
- bool is_in_progress() const;
+ bool is_in_progress() const { return _state != State::Idle; }
bool are_java_threads_synched() const;
};
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentRefineStats.cpp b/src/hotspot/share/gc/g1/g1ConcurrentRefineStats.cpp
index 7da0066e2f1..4fa32b388bd 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentRefineStats.cpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentRefineStats.cpp
@@ -25,18 +25,15 @@
#include "gc/g1/g1ConcurrentRefineStats.inline.hpp"
#include "runtime/timer.hpp"
-void G1ConcurrentRefineStats::add_atomic(G1ConcurrentRefineStats* other) {
- _sweep_duration.add_then_fetch(other->_sweep_duration.load_relaxed(), memory_order_relaxed);
- _yield_during_sweep_duration.add_then_fetch(other->yield_during_sweep_duration(), memory_order_relaxed);
+void G1ConcurrentRefineStats::add_atomic(const G1LocalRefineStats* other) {
+ _cards_scanned.add_then_fetch(other->_cards_scanned, memory_order_relaxed);
+ _cards_clean.add_then_fetch(other->_cards_clean, memory_order_relaxed);
+ _cards_not_parsable.add_then_fetch(other->_cards_not_parsable, memory_order_relaxed);
+ _cards_already_refer_to_cset.add_then_fetch(other->_cards_already_refer_to_cset, memory_order_relaxed);
+ _cards_refer_to_cset.add_then_fetch(other->_cards_refer_to_cset, memory_order_relaxed);
+ _cards_no_cross_region.add_then_fetch(other->_cards_no_cross_region, memory_order_relaxed);
- _cards_scanned.add_then_fetch(other->cards_scanned(), memory_order_relaxed);
- _cards_clean.add_then_fetch(other->cards_clean(), memory_order_relaxed);
- _cards_not_parsable.add_then_fetch(other->cards_not_parsable(), memory_order_relaxed);
- _cards_already_refer_to_cset.add_then_fetch(other->cards_already_refer_to_cset(), memory_order_relaxed);
- _cards_refer_to_cset.add_then_fetch(other->cards_refer_to_cset(), memory_order_relaxed);
- _cards_no_cross_region.add_then_fetch(other->cards_no_cross_region(), memory_order_relaxed);
-
- _refine_duration.add_then_fetch(other->refine_duration(), memory_order_relaxed);
+ _refine_duration.add_then_fetch(other->_refine_duration, memory_order_relaxed);
}
void G1ConcurrentRefineStats::reset() {
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentRefineStats.hpp b/src/hotspot/share/gc/g1/g1ConcurrentRefineStats.hpp
index a91ad0eb2e4..6f4af71081b 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentRefineStats.hpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentRefineStats.hpp
@@ -29,9 +29,27 @@
#include "runtime/atomic.hpp"
#include "utilities/globalDefinitions.hpp"
-// Collection of statistics for concurrent refinement processing.
-// Used for collecting per-thread statistics and for summaries over a
-// collection of threads.
+// Thread-local refinement statistics.
+struct G1LocalRefineStats {
+ size_t _cards_scanned;
+ size_t _cards_clean;
+ size_t _cards_not_parsable;
+ size_t _cards_already_refer_to_cset;
+ size_t _cards_refer_to_cset;
+ size_t _cards_no_cross_region;
+ jlong _refine_duration;
+
+ G1LocalRefineStats() :
+ _cards_scanned(0),
+ _cards_clean(0),
+ _cards_not_parsable(0),
+ _cards_already_refer_to_cset(0),
+ _cards_refer_to_cset(0),
+ _cards_no_cross_region(0),
+ _refine_duration(0) {}
+};
+
+// Global statistics for concurrent refinement processing.
class G1ConcurrentRefineStats : public CHeapObj {
Atomic _sweep_duration; // Time spent sweeping the table finding non-clean cards
// and refining them.
@@ -69,18 +87,10 @@ public:
inline size_t cards_to_cset() const;
- inline void inc_sweep_time(jlong t);
+ void add_atomic(const G1LocalRefineStats* other);
+
+ inline void inc_sweep_duration(jlong t);
inline void inc_yield_during_sweep_duration(jlong t);
- inline void inc_refine_duration(jlong t);
-
- inline void inc_cards_scanned(size_t increment);
- inline void inc_cards_clean(size_t increment);
- inline void inc_cards_not_parsable();
- inline void inc_cards_already_refer_to_cset();
- inline void inc_cards_refer_to_cset();
- inline void inc_cards_no_cross_region();
-
- void add_atomic(G1ConcurrentRefineStats* other);
void reset();
};
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentRefineStats.inline.hpp b/src/hotspot/share/gc/g1/g1ConcurrentRefineStats.inline.hpp
index e1a296c6494..2ef35caab08 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentRefineStats.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentRefineStats.inline.hpp
@@ -79,40 +79,12 @@ inline size_t G1ConcurrentRefineStats::cards_to_cset() const {
return cards_already_refer_to_cset() + cards_refer_to_cset();
}
-inline void G1ConcurrentRefineStats::inc_sweep_time(jlong t) {
- _sweep_duration.store_relaxed(_sweep_duration.load_relaxed() + t);
+inline void G1ConcurrentRefineStats::inc_sweep_duration(jlong t) {
+ _sweep_duration.fetch_then_add(t, memory_order_relaxed);
}
inline void G1ConcurrentRefineStats::inc_yield_during_sweep_duration(jlong t) {
- _yield_during_sweep_duration.store_relaxed(yield_during_sweep_duration() + t);
-}
-
-inline void G1ConcurrentRefineStats::inc_refine_duration(jlong t) {
- _refine_duration.store_relaxed(refine_duration() + t);
-}
-
-inline void G1ConcurrentRefineStats::inc_cards_scanned(size_t increment) {
- _cards_scanned.store_relaxed(cards_scanned() + increment);
-}
-
-inline void G1ConcurrentRefineStats::inc_cards_clean(size_t increment) {
- _cards_clean.store_relaxed(cards_clean() + increment);
-}
-
-inline void G1ConcurrentRefineStats::inc_cards_not_parsable() {
- _cards_not_parsable.store_relaxed(cards_not_parsable() + 1);
-}
-
-inline void G1ConcurrentRefineStats::inc_cards_already_refer_to_cset() {
- _cards_already_refer_to_cset.store_relaxed(cards_already_refer_to_cset() + 1);
-}
-
-inline void G1ConcurrentRefineStats::inc_cards_refer_to_cset() {
- _cards_refer_to_cset.store_relaxed(cards_refer_to_cset() + 1);
-}
-
-inline void G1ConcurrentRefineStats::inc_cards_no_cross_region() {
- _cards_no_cross_region.store_relaxed(cards_no_cross_region() + 1);
+ _yield_during_sweep_duration.fetch_then_add(t, memory_order_relaxed);
}
#endif // SHARE_GC_G1_G1CONCURRENTREFINESTATS_INLINE_HPP
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentRefineSweepTask.cpp b/src/hotspot/share/gc/g1/g1ConcurrentRefineSweepTask.cpp
index ce944f2254d..2f99611bb99 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentRefineSweepTask.cpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentRefineSweepTask.cpp
@@ -35,8 +35,6 @@ class G1RefineRegionClosure : public G1HeapRegionClosure {
uint _worker_id;
- size_t _num_collections_at_start;
-
bool has_work(G1HeapRegion* r) {
return _scan_state->has_unclaimed_cards(r->hrm_index());
}
@@ -62,22 +60,22 @@ class G1RefineRegionClosure : public G1HeapRegionClosure {
switch (res) {
case G1RemSet::HasRefToCSet: {
*dest_card = G1CardTable::g1_to_cset_card;
- _refine_stats.inc_cards_refer_to_cset();
+ _per_worker_refine_data._cards_refer_to_cset++;
break;
}
case G1RemSet::AlreadyToCSet: {
*dest_card = G1CardTable::g1_to_cset_card;
- _refine_stats.inc_cards_already_refer_to_cset();
+ _per_worker_refine_data._cards_already_refer_to_cset++;
break;
}
case G1RemSet::NoCrossRegion: {
- _refine_stats.inc_cards_no_cross_region();
+ _per_worker_refine_data._cards_no_cross_region++;
break;
}
case G1RemSet::CouldNotParse: {
// Could not refine - redirty with the original value.
*dest_card = *source_card;
- _refine_stats.inc_cards_not_parsable();
+ _per_worker_refine_data._cards_not_parsable++;
break;
}
case G1RemSet::HasRefToOld : break; // Nothing special to do.
@@ -94,7 +92,7 @@ class G1RefineRegionClosure : public G1HeapRegionClosure {
public:
bool _completed;
- G1ConcurrentRefineStats _refine_stats;
+ G1LocalRefineStats _per_worker_refine_data;
G1RefineRegionClosure(uint worker_id, G1CardTableClaimTable* scan_state) :
G1HeapRegionClosure(),
@@ -102,7 +100,7 @@ public:
_scan_state(scan_state),
_worker_id(worker_id),
_completed(true),
- _refine_stats() { }
+ _per_worker_refine_data() { }
bool do_heap_region(G1HeapRegion* r) override {
@@ -143,7 +141,7 @@ public:
do_claimed_block(dirty_l, dirty_r, dest_card + pointer_delta(dirty_l, start_card, sizeof(CardValue)));
num_dirty_cards += pointer_delta(dirty_r, dirty_l, sizeof(CardValue));
- _refine_stats.inc_refine_duration(os::elapsed_counter() - refine_start);
+ _per_worker_refine_data._refine_duration += os::elapsed_counter() - refine_start;
});
if (VerifyDuringGC) {
@@ -152,8 +150,8 @@ public:
}
}
- _refine_stats.inc_cards_scanned(claim.size());
- _refine_stats.inc_cards_clean(claim.size() - num_dirty_cards);
+ _per_worker_refine_data._cards_scanned += claim.size();
+ _per_worker_refine_data._cards_clean += claim.size() - num_dirty_cards;
if (SuspendibleThreadSet::should_yield()) {
_completed = false;
@@ -185,8 +183,8 @@ void G1ConcurrentRefineSweepTask::work(uint worker_id) {
_sweep_completed = false;
}
- sweep_cl._refine_stats.inc_sweep_time(os::elapsed_counter() - start);
- _stats->add_atomic(&sweep_cl._refine_stats);
+ _stats->inc_sweep_duration(os::elapsed_counter() - start);
+ _stats->add_atomic(&sweep_cl._per_worker_refine_data);
}
-bool G1ConcurrentRefineSweepTask::sweep_completed() const { return _sweep_completed; }
\ No newline at end of file
+bool G1ConcurrentRefineSweepTask::sweep_completed() const { return _sweep_completed; }
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentRefineThread.cpp b/src/hotspot/share/gc/g1/g1ConcurrentRefineThread.cpp
index 6b51e5eef62..2c2f8a1977f 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentRefineThread.cpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentRefineThread.cpp
@@ -141,8 +141,6 @@ bool G1ConcurrentRefineThread::wait_for_work() {
void G1ConcurrentRefineThread::do_refinement() {
G1ConcurrentRefineSweepState& state = _cr->sweep_state();
- state.start_work();
-
// Swap card tables.
// 1. Global card table
@@ -163,72 +161,23 @@ void G1ConcurrentRefineThread::do_refinement() {
return;
}
- G1CollectedHeap* g1h = G1CollectedHeap::heap();
- jlong epoch_yield_duration = g1h->yield_duration_in_refinement_epoch();
+ jlong epoch_yield_duration = G1CollectedHeap::heap()->yield_duration_in_refinement_epoch();
jlong next_epoch_start = os::elapsed_counter();
- jlong total_yield_during_sweep_duration = 0;
-
// 4. Snapshot heap.
state.snapshot_heap();
- // 5. Sweep refinement table until done
- bool interrupted_by_gc = false;
+ // 5. Sweep refinement table.
+ log_info(gc, task)("Concurrent Refine Sweep Using %u of %u Workers", cr()->num_threads_wanted(), cr()->max_num_threads());
- log_info(gc, task)("Concurrent Refine Sweep Using %u of %u Workers", _cr->num_threads_wanted(), _cr->max_num_threads());
-
- state.sweep_refinement_table_start();
- while (true) {
- bool completed = state.sweep_refinement_table_step();
-
- if (completed) {
- break;
- }
-
- if (SuspendibleThreadSet::should_yield()) {
- jlong yield_during_sweep_start = os::elapsed_counter();
- SuspendibleThreadSet::yield();
-
- // The yielding may have completed the task, check.
- if (!state.is_in_progress()) {
- log_debug(gc, refine)("GC completed sweeping, aborting concurrent operation");
- interrupted_by_gc = true;
- break;
- } else {
- jlong yield_during_sweep_duration = os::elapsed_counter() - yield_during_sweep_start;
- log_debug(gc, refine)("Yielded from card table sweeping for %.2fms, no GC inbetween, continue",
- TimeHelper::counter_to_millis(yield_during_sweep_duration));
- total_yield_during_sweep_duration += yield_during_sweep_duration;
- }
- }
+ jlong total_yield_during_sweep_duration = 0;
+ if (!state.sweep_refinement_table(total_yield_during_sweep_duration)) {
+ log_debug(gc, refine)("GC completed sweeping, aborting concurrent operation");
+ return;
}
- if (!interrupted_by_gc) {
- GCTraceTime(Info, gc, refine) tm("Concurrent Refine Complete Work");
-
- state.add_yield_during_sweep_duration(total_yield_during_sweep_duration);
-
- state.complete_work(true);
-
- G1CollectedHeap* g1h = G1CollectedHeap::heap();
- G1Policy* policy = g1h->policy();
- G1ConcurrentRefineStats* stats = state.stats();
- policy->record_refinement_stats(stats);
-
- {
- // The young gen revising mechanism reads the predictor and the values set
- // here. Avoid inconsistencies by locking.
- MutexLocker x(G1ReviseYoungLength_lock, Mutex::_no_safepoint_check_flag);
- policy->record_dirtying_stats(TimeHelper::counter_to_millis(G1CollectedHeap::heap()->last_refinement_epoch_start()),
- TimeHelper::counter_to_millis(next_epoch_start),
- stats->cards_pending(),
- TimeHelper::counter_to_millis(epoch_yield_duration),
- 0 /* pending_cards_from_gc */,
- stats->cards_to_cset());
- G1CollectedHeap::heap()->set_last_refinement_epoch_start(next_epoch_start, epoch_yield_duration);
- }
- stats->reset();
- }
+ // 6. Complete refinement.
+ state.complete_refinement(total_yield_during_sweep_duration, epoch_yield_duration, next_epoch_start);
}
void G1ConcurrentRefineThread::update_perf_counter_cpu_time() {
diff --git a/src/hotspot/share/gc/g1/g1EvacFailureRegions.cpp b/src/hotspot/share/gc/g1/g1EvacFailureRegions.cpp
index 37553e2aa56..03c20346eb3 100644
--- a/src/hotspot/share/gc/g1/g1EvacFailureRegions.cpp
+++ b/src/hotspot/share/gc/g1/g1EvacFailureRegions.cpp
@@ -55,7 +55,7 @@ void G1EvacFailureRegions::post_collection() {
_regions_pinned.resize(0);
_regions_alloc_failed.resize(0);
- FREE_C_HEAP_ARRAY(uint, _evac_failed_regions);
+ FREE_C_HEAP_ARRAY(_evac_failed_regions);
_evac_failed_regions = nullptr;
}
diff --git a/src/hotspot/share/gc/g1/g1FullCollector.cpp b/src/hotspot/share/gc/g1/g1FullCollector.cpp
index c835dd159a6..8b38509d1d8 100644
--- a/src/hotspot/share/gc/g1/g1FullCollector.cpp
+++ b/src/hotspot/share/gc/g1/g1FullCollector.cpp
@@ -167,10 +167,10 @@ G1FullCollector::~G1FullCollector() {
delete _partial_array_state_manager;
- FREE_C_HEAP_ARRAY(G1FullGCMarker*, _markers);
- FREE_C_HEAP_ARRAY(G1FullGCCompactionPoint*, _compaction_points);
- FREE_C_HEAP_ARRAY(Atomic, _compaction_tops);
- FREE_C_HEAP_ARRAY(G1RegionMarkStats, _live_stats);
+ FREE_C_HEAP_ARRAY(_markers);
+ FREE_C_HEAP_ARRAY(_compaction_points);
+ FREE_C_HEAP_ARRAY(_compaction_tops);
+ FREE_C_HEAP_ARRAY(_live_stats);
}
class PrepareRegionsClosure : public G1HeapRegionClosure {
diff --git a/src/hotspot/share/gc/g1/g1GCPhaseTimes.cpp b/src/hotspot/share/gc/g1/g1GCPhaseTimes.cpp
index 023790a2422..e13b9d91bc5 100644
--- a/src/hotspot/share/gc/g1/g1GCPhaseTimes.cpp
+++ b/src/hotspot/share/gc/g1/g1GCPhaseTimes.cpp
@@ -181,7 +181,6 @@ void G1GCPhaseTimes::reset() {
_cur_resize_heap_time_ms = 0.0;
_cur_ref_proc_time_ms = 0.0;
_root_region_scan_time_ms = 0.0;
- _external_accounted_time_ms = 0.0;
_recorded_prepare_heap_roots_time_ms = 0.0;
_recorded_young_cset_choice_time_ms = 0.0;
_recorded_non_young_cset_choice_time_ms = 0.0;
@@ -416,8 +415,6 @@ double G1GCPhaseTimes::print_pre_evacuate_collection_set() const {
info_time("Pre Evacuate Collection Set", sum_ms);
- // Concurrent tasks of ResetMarkingState and NoteStartOfMark are triggered during
- // young collection. However, their execution time are not included in _gc_pause_time_ms.
if (_cur_prepare_concurrent_task_time_ms > 0.0) {
debug_time("Prepare Concurrent Start", _cur_prepare_concurrent_task_time_ms);
debug_phase(_gc_par_phases[ResetMarkingState], 1);
@@ -543,10 +540,9 @@ void G1GCPhaseTimes::print_other(double accounted_ms) const {
info_time("Other", _gc_pause_time_ms - accounted_ms);
}
-// Root-region-scan-wait, verify-before and verify-after are part of young GC,
+// Root region scan, verify before and verify after are part of young GC,
// but these are not measured by G1Policy. i.e. these are not included in
// G1Policy::record_young_collection_start() and record_young_collection_end().
-// In addition, these are not included in G1GCPhaseTimes::_gc_pause_time_ms.
// See G1YoungCollector::collect().
void G1GCPhaseTimes::print(bool evacuation_failed) {
if (_root_region_scan_time_ms > 0.0) {
diff --git a/src/hotspot/share/gc/g1/g1GCPhaseTimes.hpp b/src/hotspot/share/gc/g1/g1GCPhaseTimes.hpp
index b57bf0d617e..eb51b340da3 100644
--- a/src/hotspot/share/gc/g1/g1GCPhaseTimes.hpp
+++ b/src/hotspot/share/gc/g1/g1GCPhaseTimes.hpp
@@ -175,7 +175,6 @@ class G1GCPhaseTimes : public CHeapObj {
double _cur_collection_nmethod_list_cleanup_time_ms;
double _cur_merge_heap_roots_time_ms;
- // Merge refinement table time. Note that this time is included in _cur_merge_heap_roots_time_ms.
double _cur_merge_refinement_table_time_ms;
double _cur_optional_merge_heap_roots_time_ms;
@@ -190,11 +189,8 @@ class G1GCPhaseTimes : public CHeapObj {
double _cur_resize_heap_time_ms;
double _cur_ref_proc_time_ms;
- // Not included in _gc_pause_time_ms
double _root_region_scan_time_ms;
- double _external_accounted_time_ms;
-
double _recorded_prepare_heap_roots_time_ms;
double _recorded_young_cset_choice_time_ms;
@@ -210,7 +206,6 @@ class G1GCPhaseTimes : public CHeapObj {
double _cur_region_register_time;
- // Not included in _gc_pause_time_ms
double _cur_verify_before_time_ms;
double _cur_verify_after_time_ms;
@@ -298,7 +293,7 @@ class G1GCPhaseTimes : public CHeapObj {
}
void record_merge_heap_roots_time(double ms) {
- _cur_merge_heap_roots_time_ms += ms;
+ _cur_merge_heap_roots_time_ms = ms;
}
void record_merge_refinement_table_time(double ms) {
@@ -373,10 +368,6 @@ class G1GCPhaseTimes : public CHeapObj {
_cur_verify_after_time_ms = time_ms;
}
- void inc_external_accounted_time_ms(double time_ms) {
- _external_accounted_time_ms += time_ms;
- }
-
void record_prepare_heap_roots_time_ms(double recorded_prepare_heap_roots_time_ms) {
_recorded_prepare_heap_roots_time_ms = recorded_prepare_heap_roots_time_ms;
}
diff --git a/src/hotspot/share/gc/g1/g1HeapRegion.inline.hpp b/src/hotspot/share/gc/g1/g1HeapRegion.inline.hpp
index f92e37fee3c..619aef35a9a 100644
--- a/src/hotspot/share/gc/g1/g1HeapRegion.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegion.inline.hpp
@@ -187,7 +187,16 @@ inline void G1HeapRegion::apply_to_marked_objects(G1CMBitMap* bitmap, ApplyToMar
}
}
- assert(next_addr == limit, "Should stop the scan at the limit.");
+#ifdef ASSERT
+ if (is_starts_humongous() && bitmap->is_marked(bottom())) {
+ HeapWord* humongous_end = bottom() + cast_to_oop(bottom())->size();
+ assert(next_addr == MAX2(limit, humongous_end),
+ "Should stop the scan at limit or end of humongous object. r %u (%s)",
+ hrm_index(), get_short_type_str());
+ } else {
+ assert(next_addr == limit, "Should stop the scan at the limit. r %u (%s)", hrm_index(), get_short_type_str());
+ }
+#endif
}
inline HeapWord* G1HeapRegion::par_allocate(size_t min_word_size,
diff --git a/src/hotspot/share/gc/g1/g1HeapRegionManager.cpp b/src/hotspot/share/gc/g1/g1HeapRegionManager.cpp
index 3c0318827ef..214f1a2d2b6 100644
--- a/src/hotspot/share/gc/g1/g1HeapRegionManager.cpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionManager.cpp
@@ -718,7 +718,7 @@ G1HeapRegionClaimer::G1HeapRegionClaimer(uint n_workers) :
}
G1HeapRegionClaimer::~G1HeapRegionClaimer() {
- FREE_C_HEAP_ARRAY(uint, _claims);
+ FREE_C_HEAP_ARRAY(_claims);
}
uint G1HeapRegionClaimer::offset_for_worker(uint worker_id) const {
@@ -759,7 +759,7 @@ public:
for (uint worker = 0; worker < _num_workers; worker++) {
_worker_freelists[worker].~G1FreeRegionList();
}
- FREE_C_HEAP_ARRAY(G1FreeRegionList, _worker_freelists);
+ FREE_C_HEAP_ARRAY(_worker_freelists);
}
G1FreeRegionList* worker_freelist(uint worker) {
diff --git a/src/hotspot/share/gc/g1/g1HeapRegionSet.cpp b/src/hotspot/share/gc/g1/g1HeapRegionSet.cpp
index 70186adcdfc..930a4bd953f 100644
--- a/src/hotspot/share/gc/g1/g1HeapRegionSet.cpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionSet.cpp
@@ -384,7 +384,7 @@ G1FreeRegionList::NodeInfo::NodeInfo() : _numa(G1NUMA::numa()), _length_of_node(
}
G1FreeRegionList::NodeInfo::~NodeInfo() {
- FREE_C_HEAP_ARRAY(uint, _length_of_node);
+ FREE_C_HEAP_ARRAY(_length_of_node);
}
void G1FreeRegionList::NodeInfo::clear() {
diff --git a/src/hotspot/share/gc/g1/g1HeapTransition.cpp b/src/hotspot/share/gc/g1/g1HeapTransition.cpp
index 30ad4c72bf6..690bda4e7e6 100644
--- a/src/hotspot/share/gc/g1/g1HeapTransition.cpp
+++ b/src/hotspot/share/gc/g1/g1HeapTransition.cpp
@@ -55,8 +55,8 @@ G1HeapTransition::Data::Data(G1CollectedHeap* g1_heap) :
}
G1HeapTransition::Data::~Data() {
- FREE_C_HEAP_ARRAY(uint, _eden_length_per_node);
- FREE_C_HEAP_ARRAY(uint, _survivor_length_per_node);
+ FREE_C_HEAP_ARRAY(_eden_length_per_node);
+ FREE_C_HEAP_ARRAY(_survivor_length_per_node);
}
G1HeapTransition::G1HeapTransition(G1CollectedHeap* g1_heap) : _g1_heap(g1_heap), _before(g1_heap) { }
diff --git a/src/hotspot/share/gc/g1/g1HeapVerifier.cpp b/src/hotspot/share/gc/g1/g1HeapVerifier.cpp
index 714a2473a08..304722c13a1 100644
--- a/src/hotspot/share/gc/g1/g1HeapVerifier.cpp
+++ b/src/hotspot/share/gc/g1/g1HeapVerifier.cpp
@@ -461,35 +461,34 @@ public:
G1ConcurrentMark* cm = G1CollectedHeap::heap()->concurrent_mark();
- bool part_of_marking = r->is_old_or_humongous() && !r->is_collection_set_candidate();
HeapWord* top_at_mark_start = cm->top_at_mark_start(r);
- if (part_of_marking) {
- guarantee(r->bottom() != top_at_mark_start,
- "region %u (%s) does not have TAMS set",
- r->hrm_index(), r->get_short_type_str());
- size_t marked_bytes = cm->live_bytes(r->hrm_index());
-
+ if (r->is_old_or_humongous()) {
+ if (!cm->is_root_region(r)) {
+ guarantee(r->bottom() != top_at_mark_start,
+ "region %u (%s) does not have TAMS set although it's going to be marked through",
+ r->hrm_index(), r->get_short_type_str());
+ }
MarkedBytesClosure cl;
r->apply_to_marked_objects(cm->mark_bitmap(), &cl);
+ size_t marked_bytes = cm->live_bytes(r->hrm_index());
guarantee(cl.marked_bytes() == marked_bytes,
"region %u (%s) live bytes actual %zu and cache %zu differ",
r->hrm_index(), r->get_short_type_str(), cl.marked_bytes(), marked_bytes);
- } else {
+ } else if (r->is_young()) {
guarantee(r->bottom() == top_at_mark_start,
"region %u (%s) has TAMS set " PTR_FORMAT " " PTR_FORMAT,
r->hrm_index(), r->get_short_type_str(), p2i(r->bottom()), p2i(top_at_mark_start));
guarantee(cm->live_bytes(r->hrm_index()) == 0,
"region %u (%s) has %zu live bytes recorded",
r->hrm_index(), r->get_short_type_str(), cm->live_bytes(r->hrm_index()));
- guarantee(cm->mark_bitmap()->get_next_marked_addr(r->bottom(), r->end()) == r->end(),
- "region %u (%s) has mark",
- r->hrm_index(), r->get_short_type_str());
- guarantee(cm->is_root_region(r),
- "region %u (%s) should be root region",
- r->hrm_index(), r->get_short_type_str());
+ guarantee(cm->is_root_region(r), "must be for %u (%s)", r->hrm_index(), r->get_short_type_str());
}
+
+ guarantee(cm->mark_bitmap()->get_next_marked_addr(top_at_mark_start, r->end()) == r->end(),
+ "region %u (%s) has mark from TAMS to top",
+ r->hrm_index(), r->get_short_type_str());
return false;
}
};
diff --git a/src/hotspot/share/gc/g1/g1IHOPControl.cpp b/src/hotspot/share/gc/g1/g1IHOPControl.cpp
index 1e1c52477f9..164486123f7 100644
--- a/src/hotspot/share/gc/g1/g1IHOPControl.cpp
+++ b/src/hotspot/share/gc/g1/g1IHOPControl.cpp
@@ -114,7 +114,7 @@ void G1IHOPControl::add_marking_start_to_mixed_length(double length_s) {
// Determine the old generation occupancy threshold at which to start
// concurrent marking such that reclamation (first Mixed GC) begins
// before the heap reaches a critical occupancy level.
-size_t G1IHOPControl::old_gen_threshold_for_conc_mark_start() {
+size_t G1IHOPControl::old_gen_threshold_for_conc_mark_start() const {
guarantee(_target_occupancy > 0, "Target occupancy must be initialized");
if (!_is_adaptive || !have_enough_data_for_prediction()) {
diff --git a/src/hotspot/share/gc/g1/g1IHOPControl.hpp b/src/hotspot/share/gc/g1/g1IHOPControl.hpp
index ff209012f02..2836408978b 100644
--- a/src/hotspot/share/gc/g1/g1IHOPControl.hpp
+++ b/src/hotspot/share/gc/g1/g1IHOPControl.hpp
@@ -115,7 +115,7 @@ class G1IHOPControl : public CHeapObj {
void add_marking_start_to_mixed_length(double length_s);
// Get the current non-young occupancy at which concurrent marking should start.
- size_t old_gen_threshold_for_conc_mark_start();
+ size_t old_gen_threshold_for_conc_mark_start() const;
void report_statistics(G1NewTracer* tracer, size_t non_young_occupancy);
};
diff --git a/src/hotspot/share/gc/g1/g1MonotonicArena.cpp b/src/hotspot/share/gc/g1/g1MonotonicArena.cpp
index 3f97870a67f..aea6f4335e8 100644
--- a/src/hotspot/share/gc/g1/g1MonotonicArena.cpp
+++ b/src/hotspot/share/gc/g1/g1MonotonicArena.cpp
@@ -52,7 +52,7 @@ void G1MonotonicArena::Segment::delete_segment(Segment* segment) {
GlobalCounter::write_synchronize();
}
segment->~Segment();
- FREE_C_HEAP_ARRAY(_mem_tag, segment);
+ FREE_C_HEAP_ARRAY(segment);
}
void G1MonotonicArena::SegmentFreeList::bulk_add(Segment& first,
diff --git a/src/hotspot/share/gc/g1/g1MonotonicArenaFreePool.cpp b/src/hotspot/share/gc/g1/g1MonotonicArenaFreePool.cpp
index 922c68bfba4..c12321b851a 100644
--- a/src/hotspot/share/gc/g1/g1MonotonicArenaFreePool.cpp
+++ b/src/hotspot/share/gc/g1/g1MonotonicArenaFreePool.cpp
@@ -159,7 +159,7 @@ G1MonotonicArenaFreePool::~G1MonotonicArenaFreePool() {
for (uint i = 0; i < _num_free_lists; i++) {
_free_lists[i].~SegmentFreeList();
}
- FREE_C_HEAP_ARRAY(mtGC, _free_lists);
+ FREE_C_HEAP_ARRAY(_free_lists);
}
G1MonotonicArenaMemoryStats G1MonotonicArenaFreePool::memory_sizes() const {
diff --git a/src/hotspot/share/gc/g1/g1NUMA.cpp b/src/hotspot/share/gc/g1/g1NUMA.cpp
index 778ed31d7b5..db42b8c10fc 100644
--- a/src/hotspot/share/gc/g1/g1NUMA.cpp
+++ b/src/hotspot/share/gc/g1/g1NUMA.cpp
@@ -123,8 +123,8 @@ void G1NUMA::initialize(bool use_numa) {
G1NUMA::~G1NUMA() {
delete _stats;
- FREE_C_HEAP_ARRAY(uint, _node_id_to_index_map);
- FREE_C_HEAP_ARRAY(uint, _node_ids);
+ FREE_C_HEAP_ARRAY(_node_id_to_index_map);
+ FREE_C_HEAP_ARRAY(_node_ids);
}
void G1NUMA::set_region_info(size_t region_size, size_t page_size) {
@@ -280,9 +280,9 @@ G1NodeIndexCheckClosure::~G1NodeIndexCheckClosure() {
_ls->print("%u: %u/%u/%u ", numa_ids[i], _matched[i], _mismatched[i], _total[i]);
}
- FREE_C_HEAP_ARRAY(uint, _matched);
- FREE_C_HEAP_ARRAY(uint, _mismatched);
- FREE_C_HEAP_ARRAY(uint, _total);
+ FREE_C_HEAP_ARRAY(_matched);
+ FREE_C_HEAP_ARRAY(_mismatched);
+ FREE_C_HEAP_ARRAY(_total);
}
bool G1NodeIndexCheckClosure::do_heap_region(G1HeapRegion* hr) {
diff --git a/src/hotspot/share/gc/g1/g1NUMAStats.cpp b/src/hotspot/share/gc/g1/g1NUMAStats.cpp
index aaebfa1be8f..ce62d34f847 100644
--- a/src/hotspot/share/gc/g1/g1NUMAStats.cpp
+++ b/src/hotspot/share/gc/g1/g1NUMAStats.cpp
@@ -45,9 +45,9 @@ G1NUMAStats::NodeDataArray::NodeDataArray(uint num_nodes) {
G1NUMAStats::NodeDataArray::~NodeDataArray() {
for (uint row = 0; row < _num_row; row++) {
- FREE_C_HEAP_ARRAY(size_t, _data[row]);
+ FREE_C_HEAP_ARRAY(_data[row]);
}
- FREE_C_HEAP_ARRAY(size_t*, _data);
+ FREE_C_HEAP_ARRAY(_data);
}
void G1NUMAStats::NodeDataArray::create_hit_rate(Stat* result) const {
diff --git a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
index 52c8d4d4389..45e1b25cb95 100644
--- a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
+++ b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
@@ -131,9 +131,9 @@ size_t G1ParScanThreadState::flush_stats(size_t* surviving_young_words, uint num
G1ParScanThreadState::~G1ParScanThreadState() {
delete _plab_allocator;
delete _closures;
- FREE_C_HEAP_ARRAY(size_t, _surviving_young_words_base);
+ FREE_C_HEAP_ARRAY(_surviving_young_words_base);
delete[] _oops_into_optional_regions;
- FREE_C_HEAP_ARRAY(size_t, _obj_alloc_stat);
+ FREE_C_HEAP_ARRAY(_obj_alloc_stat);
}
size_t G1ParScanThreadState::lab_waste_words() const {
@@ -444,7 +444,7 @@ void G1ParScanThreadState::do_iterate_object(oop const obj,
return;
}
- ContinuationGCSupport::transform_stack_chunk(obj);
+ ContinuationGCSupport::transform_stack_chunk(obj, klass);
// Check for deduplicating young Strings.
if (G1StringDedup::is_candidate_from_evacuation(klass,
@@ -730,8 +730,8 @@ G1ParScanThreadStateSet::G1ParScanThreadStateSet(G1CollectedHeap* g1h,
G1ParScanThreadStateSet::~G1ParScanThreadStateSet() {
assert(_flushed, "thread local state from the per thread states should have been flushed");
- FREE_C_HEAP_ARRAY(G1ParScanThreadState*, _states);
- FREE_C_HEAP_ARRAY(size_t, _surviving_young_words_total);
+ FREE_C_HEAP_ARRAY(_states);
+ FREE_C_HEAP_ARRAY(_surviving_young_words_total);
}
#if TASKQUEUE_STATS
diff --git a/src/hotspot/share/gc/g1/g1Policy.cpp b/src/hotspot/share/gc/g1/g1Policy.cpp
index 78a533d62c0..01afb6a5c77 100644
--- a/src/hotspot/share/gc/g1/g1Policy.cpp
+++ b/src/hotspot/share/gc/g1/g1Policy.cpp
@@ -728,7 +728,7 @@ bool G1Policy::about_to_start_mixed_phase() const {
return collector_state()->is_in_concurrent_cycle() || collector_state()->is_in_prepare_mixed_gc();
}
-bool G1Policy::need_to_start_conc_mark(const char* source, size_t allocation_word_size) {
+bool G1Policy::need_to_start_conc_mark(const char* source, size_t allocation_word_size) const {
if (about_to_start_mixed_phase()) {
return false;
}
@@ -1006,7 +1006,7 @@ void G1Policy::record_young_collection_end(bool concurrent_operation_is_full_mar
G1IHOPControl* G1Policy::create_ihop_control(const G1OldGenAllocationTracker* old_gen_alloc_tracker,
const G1Predictions* predictor) {
- return new G1IHOPControl(InitiatingHeapOccupancyPercent,
+ return new G1IHOPControl(G1IHOP,
old_gen_alloc_tracker,
G1UseAdaptiveIHOP,
predictor,
diff --git a/src/hotspot/share/gc/g1/g1Policy.hpp b/src/hotspot/share/gc/g1/g1Policy.hpp
index 5c5c2bc3572..0aa15be9cae 100644
--- a/src/hotspot/share/gc/g1/g1Policy.hpp
+++ b/src/hotspot/share/gc/g1/g1Policy.hpp
@@ -296,7 +296,7 @@ public:
void record_young_gc_pause_start();
void record_young_gc_pause_end(bool evacuation_failed);
- bool need_to_start_conc_mark(const char* source, size_t allocation_word_size);
+ bool need_to_start_conc_mark(const char* source, size_t allocation_word_size) const;
bool concurrent_operation_is_full_mark(const char* msg, size_t allocation_word_size);
diff --git a/src/hotspot/share/gc/g1/g1RegionMarkStatsCache.cpp b/src/hotspot/share/gc/g1/g1RegionMarkStatsCache.cpp
index c5f55e1d20c..a9f4115df94 100644
--- a/src/hotspot/share/gc/g1/g1RegionMarkStatsCache.cpp
+++ b/src/hotspot/share/gc/g1/g1RegionMarkStatsCache.cpp
@@ -38,7 +38,7 @@ G1RegionMarkStatsCache::G1RegionMarkStatsCache(G1RegionMarkStats* target, uint n
}
G1RegionMarkStatsCache::~G1RegionMarkStatsCache() {
- FREE_C_HEAP_ARRAY(G1RegionMarkStatsCacheEntry, _cache);
+ FREE_C_HEAP_ARRAY(_cache);
}
void G1RegionMarkStatsCache::add_live_words(oop obj) {
diff --git a/src/hotspot/share/gc/g1/g1RegionsOnNodes.cpp b/src/hotspot/share/gc/g1/g1RegionsOnNodes.cpp
index c1c0d471796..9550e57698e 100644
--- a/src/hotspot/share/gc/g1/g1RegionsOnNodes.cpp
+++ b/src/hotspot/share/gc/g1/g1RegionsOnNodes.cpp
@@ -32,7 +32,7 @@ G1RegionsOnNodes::G1RegionsOnNodes() : _count_per_node(nullptr), _numa(G1NUMA::n
}
G1RegionsOnNodes::~G1RegionsOnNodes() {
- FREE_C_HEAP_ARRAY(uint, _count_per_node);
+ FREE_C_HEAP_ARRAY(_count_per_node);
}
void G1RegionsOnNodes::add(G1HeapRegion* hr) {
diff --git a/src/hotspot/share/gc/g1/g1RemSet.cpp b/src/hotspot/share/gc/g1/g1RemSet.cpp
index 9f9f0ecdf3a..be18a3065e9 100644
--- a/src/hotspot/share/gc/g1/g1RemSet.cpp
+++ b/src/hotspot/share/gc/g1/g1RemSet.cpp
@@ -124,8 +124,8 @@ class G1RemSetScanState : public CHeapObj {
}
~G1DirtyRegions() {
- FREE_C_HEAP_ARRAY(uint, _buffer);
- FREE_C_HEAP_ARRAY(Atomic, _contains);
+ FREE_C_HEAP_ARRAY(_buffer);
+ FREE_C_HEAP_ARRAY(_contains);
}
void reset() {
@@ -245,7 +245,7 @@ public:
_scan_top(nullptr) { }
~G1RemSetScanState() {
- FREE_C_HEAP_ARRAY(HeapWord*, _scan_top);
+ FREE_C_HEAP_ARRAY(_scan_top);
}
void initialize(uint max_reserved_regions) {
diff --git a/src/hotspot/share/gc/g1/g1RemSetSummary.cpp b/src/hotspot/share/gc/g1/g1RemSetSummary.cpp
index 3e9cf938097..1c0e15757cc 100644
--- a/src/hotspot/share/gc/g1/g1RemSetSummary.cpp
+++ b/src/hotspot/share/gc/g1/g1RemSetSummary.cpp
@@ -98,7 +98,7 @@ G1RemSetSummary::G1RemSetSummary(bool should_update) :
}
G1RemSetSummary::~G1RemSetSummary() {
- FREE_C_HEAP_ARRAY(jlong, _worker_threads_cpu_times);
+ FREE_C_HEAP_ARRAY(_worker_threads_cpu_times);
}
void G1RemSetSummary::set(G1RemSetSummary* other) {
diff --git a/src/hotspot/share/gc/g1/g1SurvRateGroup.cpp b/src/hotspot/share/gc/g1/g1SurvRateGroup.cpp
index f858b93b13d..15c6d3d1f15 100644
--- a/src/hotspot/share/gc/g1/g1SurvRateGroup.cpp
+++ b/src/hotspot/share/gc/g1/g1SurvRateGroup.cpp
@@ -65,8 +65,8 @@ void G1SurvRateGroup::start_adding_regions() {
void G1SurvRateGroup::stop_adding_regions() {
if (_num_added_regions > _stats_arrays_length) {
- _accum_surv_rate_pred = REALLOC_C_HEAP_ARRAY(double, _accum_surv_rate_pred, _num_added_regions, mtGC);
- _surv_rate_predictors = REALLOC_C_HEAP_ARRAY(TruncatedSeq*, _surv_rate_predictors, _num_added_regions, mtGC);
+ _accum_surv_rate_pred = REALLOC_C_HEAP_ARRAY(_accum_surv_rate_pred, _num_added_regions, mtGC);
+ _surv_rate_predictors = REALLOC_C_HEAP_ARRAY(_surv_rate_predictors, _num_added_regions, mtGC);
for (uint i = _stats_arrays_length; i < _num_added_regions; ++i) {
// Initialize predictors and accumulated survivor rate predictions.
diff --git a/src/hotspot/share/gc/g1/g1YoungCollector.cpp b/src/hotspot/share/gc/g1/g1YoungCollector.cpp
index 9c12127c864..d26bcc23c08 100644
--- a/src/hotspot/share/gc/g1/g1YoungCollector.cpp
+++ b/src/hotspot/share/gc/g1/g1YoungCollector.cpp
@@ -246,8 +246,6 @@ G1YoungGCAllocationFailureInjector* G1YoungCollector::allocation_failure_injecto
void G1YoungCollector::complete_root_region_scan() {
Ticks start = Ticks::now();
- // We have to complete root region scan as it's the only way to ensure that all the
- // objects on them have been correctly scanned before we start moving them during the GC.
if (concurrent_mark()->complete_root_regions_scan_in_safepoint()) {
phase_times()->record_root_region_scan_time((Ticks::now() - start).seconds() * MILLIUNITS);
}
@@ -1138,9 +1136,7 @@ void G1YoungCollector::collect() {
// Individual parallel phases may override this.
set_young_collection_default_active_worker_threads();
- // Wait for root region scan here to make sure that it is done before any
- // use of the STW workers to maximize cpu use (i.e. all cores are available
- // just to do that).
+ // Complete root region scan before moving any objects to preserve the SATB invariant.
complete_root_region_scan();
G1YoungGCVerifierMark vm(this);
diff --git a/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp b/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp
index 14282383e29..11da3cb8263 100644
--- a/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp
+++ b/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp
@@ -802,7 +802,7 @@ public:
for (uint worker = 0; worker < _active_workers; worker++) {
_worker_stats[worker].~FreeCSetStats();
}
- FREE_C_HEAP_ARRAY(FreeCSetStats, _worker_stats);
+ FREE_C_HEAP_ARRAY(_worker_stats);
_g1h->clear_collection_set();
diff --git a/src/hotspot/share/gc/g1/g1YoungGCPreEvacuateTasks.cpp b/src/hotspot/share/gc/g1/g1YoungGCPreEvacuateTasks.cpp
index 936457659b6..7889fd4fb31 100644
--- a/src/hotspot/share/gc/g1/g1YoungGCPreEvacuateTasks.cpp
+++ b/src/hotspot/share/gc/g1/g1YoungGCPreEvacuateTasks.cpp
@@ -73,7 +73,7 @@ public:
~JavaThreadRetireTLABs() {
static_assert(std::is_trivially_destructible::value, "must be");
- FREE_C_HEAP_ARRAY(ThreadLocalAllocStats, _local_tlab_stats);
+ FREE_C_HEAP_ARRAY(_local_tlab_stats);
}
void do_work(uint worker_id) override {
diff --git a/src/hotspot/share/gc/g1/g1_globals.hpp b/src/hotspot/share/gc/g1/g1_globals.hpp
index b338c11d5be..14daac4800b 100644
--- a/src/hotspot/share/gc/g1/g1_globals.hpp
+++ b/src/hotspot/share/gc/g1/g1_globals.hpp
@@ -100,9 +100,8 @@
\
product(bool, G1UseAdaptiveIHOP, true, \
"Adaptively adjust the initiating heap occupancy from the " \
- "initial value of InitiatingHeapOccupancyPercent. The policy " \
- "attempts to start marking in time based on application " \
- "behavior.") \
+ "initial value of G1IHOP. The policy attempts to start marking " \
+ "in time based on application behavior.") \
\
product(size_t, G1AdaptiveIHOPNumInitialSamples, 3, EXPERIMENTAL, \
"How many completed time periods from concurrent start to first " \
@@ -110,6 +109,19 @@
"of the optimal occupancy to start marking.") \
range(1, max_intx) \
\
+ product(uint, G1IHOP, 45, \
+ "The Initiating Heap Occupancy Percentage (IHOP) for the " \
+ "concurrent cycle. G1IHOP sets the percentage of the current " \
+ "Java heap capacity occupied by the old generation at which G1 " \
+ "starts this process. If G1UseAdaptiveIHOP is enabled, this " \
+ "value is used as the initial threshold and may be adjusted " \
+ "ergonomically by G1. " \
+ "A value of 0 will result in as frequent as possible concurrent " \
+ "cycles. A value of 100 disables concurrent cycles. " \
+ "Fragmentation waste in the old generation is not considered " \
+ "free space in this calculation.") \
+ range(0, 100) \
+ \
product(uint, G1ConfidencePercent, 50, \
"Confidence level for MMU/pause predictions. A higher value " \
"means that G1 will use less safety margin for its predictions.") \
@@ -316,7 +328,7 @@
product(bool, G1VerifyHeapRegionCodeRoots, false, DIAGNOSTIC, \
"Verify the code root lists attached to each heap region.") \
\
- develop(bool, G1VerifyBitmaps, false, \
+ product(bool, G1VerifyBitmaps, false, DIAGNOSTIC, \
"Verifies the consistency of the marking bitmaps") \
\
product(uintx, G1PeriodicGCInterval, 0, MANAGEABLE, \
diff --git a/src/hotspot/share/gc/parallel/mutableNUMASpace.cpp b/src/hotspot/share/gc/parallel/mutableNUMASpace.cpp
index c5d112ffbc1..8b514fe7199 100644
--- a/src/hotspot/share/gc/parallel/mutableNUMASpace.cpp
+++ b/src/hotspot/share/gc/parallel/mutableNUMASpace.cpp
@@ -55,7 +55,7 @@ MutableNUMASpace::MutableNUMASpace(size_t page_size) : MutableSpace(page_size) {
lgrp_spaces()->append(new LGRPSpace(lgrp_ids[i], page_size));
}
- FREE_C_HEAP_ARRAY(uint, lgrp_ids);
+ FREE_C_HEAP_ARRAY(lgrp_ids);
}
MutableNUMASpace::~MutableNUMASpace() {
diff --git a/src/hotspot/share/gc/parallel/psPromotionManager.cpp b/src/hotspot/share/gc/parallel/psPromotionManager.cpp
index ac22430aa4c..6a0905e82f3 100644
--- a/src/hotspot/share/gc/parallel/psPromotionManager.cpp
+++ b/src/hotspot/share/gc/parallel/psPromotionManager.cpp
@@ -294,7 +294,7 @@ oop PSPromotionManager::oop_promotion_failed(oop obj, markWord obj_mark) {
ContinuationGCSupport::transform_stack_chunk(obj);
- push_contents(obj);
+ push_contents(obj, obj->klass());
// Save the markWord of promotion-failed objs in _preserved_marks for later
// restoration. This way we don't have to walk the young-gen to locate
diff --git a/src/hotspot/share/gc/parallel/psPromotionManager.hpp b/src/hotspot/share/gc/parallel/psPromotionManager.hpp
index 2b0fc56c0bf..cd59fa578d1 100644
--- a/src/hotspot/share/gc/parallel/psPromotionManager.hpp
+++ b/src/hotspot/share/gc/parallel/psPromotionManager.hpp
@@ -165,7 +165,7 @@ class PSPromotionManager {
template inline void claim_or_forward_depth(T* p);
- void push_contents(oop obj);
+ void push_contents(oop obj, Klass* klass);
void push_contents_bounded(oop obj, HeapWord* left, HeapWord* right);
};
diff --git a/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp b/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp
index 68370a33a54..4b9edeb7072 100644
--- a/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp
+++ b/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp
@@ -121,10 +121,10 @@ inline void InstanceRefKlass::oop_oop_iterate_reverse(obj, closure);
}
-inline void PSPromotionManager::push_contents(oop obj) {
- if (!obj->klass()->is_typeArray_klass()) {
+inline void PSPromotionManager::push_contents(oop obj, Klass* klass) {
+ if (!klass->is_typeArray_klass()) {
PSPushContentsClosure pcc(this);
- obj->oop_iterate_backwards(&pcc);
+ obj->oop_iterate_backwards(&pcc, klass);
}
}
@@ -292,7 +292,7 @@ inline oop PSPromotionManager::copy_unmarked_to_survivor_space(oop o,
assert(young_space()->contains(new_obj), "Attempt to push non-promoted obj");
}
- ContinuationGCSupport::transform_stack_chunk(new_obj);
+ ContinuationGCSupport::transform_stack_chunk(new_obj, klass);
// Do the size comparison first with new_obj_size, which we
// already have. Hopefully, only a few objects are larger than
@@ -303,7 +303,7 @@ inline oop PSPromotionManager::copy_unmarked_to_survivor_space(oop o,
push_objArray(o, new_obj);
} else {
// we'll just push its contents
- push_contents(new_obj);
+ push_contents(new_obj, klass);
if (StringDedup::is_enabled_string(klass) &&
psStringDedup::is_candidate_from_evacuation(new_obj, new_obj_is_tenured)) {
diff --git a/src/hotspot/share/gc/serial/defNewGeneration.cpp b/src/hotspot/share/gc/serial/defNewGeneration.cpp
index ec3726d1dce..0c44e7a8c14 100644
--- a/src/hotspot/share/gc/serial/defNewGeneration.cpp
+++ b/src/hotspot/share/gc/serial/defNewGeneration.cpp
@@ -698,7 +698,8 @@ void DefNewGeneration::handle_promotion_failure(oop old) {
oop DefNewGeneration::copy_to_survivor_space(oop old) {
assert(is_in_reserved(old) && !old->is_forwarded(),
"shouldn't be scavenging this oop");
- size_t s = old->size();
+ Klass* klass = old->klass();
+ size_t s = old->size_given_klass(klass);
oop obj = nullptr;
// Try allocating obj in to-space (unless too old)
@@ -725,7 +726,7 @@ oop DefNewGeneration::copy_to_survivor_space(oop old) {
// Copy obj
Copy::aligned_disjoint_words(cast_from_oop(old), cast_from_oop(obj), s);
- ContinuationGCSupport::transform_stack_chunk(obj);
+ ContinuationGCSupport::transform_stack_chunk(obj, klass);
if (!new_obj_is_tenured) {
// Increment age if obj still in new generation
@@ -736,7 +737,7 @@ oop DefNewGeneration::copy_to_survivor_space(oop old) {
// Done, insert forward pointer to obj in this header
old->forward_to(obj);
- if (SerialStringDedup::is_candidate_from_evacuation(obj, new_obj_is_tenured)) {
+ if (SerialStringDedup::is_candidate_from_evacuation(obj, klass, new_obj_is_tenured)) {
// Record old; request adds a new weak reference, which reference
// processing expects to refer to a from-space object.
_string_dedup_requests.add(old);
diff --git a/src/hotspot/share/gc/serial/serialStringDedup.hpp b/src/hotspot/share/gc/serial/serialStringDedup.hpp
index 9d4548acf36..76af8a6f5ab 100644
--- a/src/hotspot/share/gc/serial/serialStringDedup.hpp
+++ b/src/hotspot/share/gc/serial/serialStringDedup.hpp
@@ -27,6 +27,8 @@
#include "memory/allStatic.hpp"
#include "oops/oopsHierarchy.hpp"
+class Klass;
+
class SerialStringDedup : AllStatic {
public:
@@ -37,7 +39,7 @@ public:
static bool is_candidate_from_mark(oop java_string);
// Candidate selection policy for young during evacuation.
- static inline bool is_candidate_from_evacuation(oop obj, bool obj_is_tenured);
+ static inline bool is_candidate_from_evacuation(oop obj, const Klass* klass, bool obj_is_tenured);
};
diff --git a/src/hotspot/share/gc/serial/serialStringDedup.inline.hpp b/src/hotspot/share/gc/serial/serialStringDedup.inline.hpp
index ebf29113a0b..bc54f3b34b9 100644
--- a/src/hotspot/share/gc/serial/serialStringDedup.inline.hpp
+++ b/src/hotspot/share/gc/serial/serialStringDedup.inline.hpp
@@ -30,9 +30,9 @@
#include "oops/oop.inline.hpp"
bool SerialStringDedup::is_candidate_from_evacuation(oop obj,
+ const Klass* klass,
bool obj_is_tenured) {
- return StringDedup::is_enabled() &&
- java_lang_String::is_instance(obj) &&
+ return StringDedup::is_enabled_string(klass) &&
(obj_is_tenured ?
StringDedup::is_below_threshold_age(obj->age()) :
StringDedup::is_threshold_age(obj->age()));
diff --git a/src/hotspot/share/gc/shared/bufferNode.cpp b/src/hotspot/share/gc/shared/bufferNode.cpp
index 90e50f52e84..855f872afab 100644
--- a/src/hotspot/share/gc/shared/bufferNode.cpp
+++ b/src/hotspot/share/gc/shared/bufferNode.cpp
@@ -41,7 +41,7 @@ void* BufferNode::AllocatorConfig::allocate() {
void BufferNode::AllocatorConfig::deallocate(void* node) {
assert(node != nullptr, "precondition");
- FREE_C_HEAP_ARRAY(char, node);
+ FREE_C_HEAP_ARRAY(node);
}
BufferNode::Allocator::Allocator(const char* name, size_t buffer_capacity) :
diff --git a/src/hotspot/share/gc/shared/c1/barrierSetC1.cpp b/src/hotspot/share/gc/shared/c1/barrierSetC1.cpp
index a31078f7e67..97c24611de4 100644
--- a/src/hotspot/share/gc/shared/c1/barrierSetC1.cpp
+++ b/src/hotspot/share/gc/shared/c1/barrierSetC1.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -38,16 +38,6 @@
#define __ gen->lir()->
#endif
-// Return true iff an access to bt is single-copy atomic.
-
-// The JMM requires atomicity for all accesses to fields of primitive
-// types other than double and long. In practice, HotSpot assumes that
-// on all processors, accesses to memory operands of wordSize and
-// smaller are atomic.
-static bool access_is_atomic(BasicType bt) {
- return type2aelembytes(bt) <= wordSize;
-}
-
LIR_Opr BarrierSetC1::resolve_address(LIRAccess& access, bool resolve_in_register) {
DecoratorSet decorators = access.decorators();
bool is_array = (decorators & IS_ARRAY) != 0;
@@ -150,7 +140,6 @@ LIR_Opr BarrierSetC1::atomic_add_at(LIRAccess& access, LIRItem& value) {
void BarrierSetC1::store_at_resolved(LIRAccess& access, LIR_Opr value) {
DecoratorSet decorators = access.decorators();
bool is_volatile = (decorators & MO_SEQ_CST) != 0;
- bool needs_atomic = AlwaysAtomicAccesses && !access_is_atomic(value->type());
bool needs_patching = (decorators & C1_NEEDS_PATCHING) != 0;
bool mask_boolean = (decorators & C1_MASK_BOOLEAN) != 0;
LIRGenerator* gen = access.gen();
@@ -164,7 +153,7 @@ void BarrierSetC1::store_at_resolved(LIRAccess& access, LIR_Opr value) {
}
LIR_PatchCode patch_code = needs_patching ? lir_patch_normal : lir_patch_none;
- if ((is_volatile || needs_atomic) && !needs_patching) {
+ if (is_volatile && !needs_patching) {
gen->volatile_field_store(value, access.resolved_addr()->as_address_ptr(), access.access_emit_info());
} else {
__ store(value, access.resolved_addr()->as_address_ptr(), access.access_emit_info(), patch_code);
@@ -179,10 +168,10 @@ void BarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) {
LIRGenerator *gen = access.gen();
DecoratorSet decorators = access.decorators();
bool is_volatile = (decorators & MO_SEQ_CST) != 0;
- bool needs_atomic = AlwaysAtomicAccesses && !access_is_atomic(result->type());
bool needs_patching = (decorators & C1_NEEDS_PATCHING) != 0;
bool mask_boolean = (decorators & C1_MASK_BOOLEAN) != 0;
bool in_native = (decorators & IN_NATIVE) != 0;
+ bool needs_trailing_membar = is_volatile;
if (support_IRIW_for_not_multiple_copy_atomic_cpu && is_volatile) {
__ membar();
@@ -191,13 +180,16 @@ void BarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) {
LIR_PatchCode patch_code = needs_patching ? lir_patch_normal : lir_patch_none;
if (in_native) {
__ move_wide(access.resolved_addr()->as_address_ptr(), result);
- } else if ((is_volatile || needs_atomic) && !needs_patching) {
+ } else if (is_volatile && !needs_patching) {
+ // volatile_field_load provides trailing membar semantics.
+ // Hence separate trailing membar is not needed.
+ needs_trailing_membar = false;
gen->volatile_field_load(access.resolved_addr()->as_address_ptr(), result, access.access_emit_info());
} else {
__ load(access.resolved_addr()->as_address_ptr(), result, access.access_emit_info(), patch_code);
}
- if (is_volatile) {
+ if (needs_trailing_membar) {
__ membar_acquire();
}
diff --git a/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp b/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp
index afe7d2acfa7..239cce16aa3 100644
--- a/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp
+++ b/src/hotspot/share/gc/shared/c2/barrierSetC2.cpp
@@ -395,17 +395,11 @@ MemNode::MemOrd C2Access::mem_node_mo() const {
void C2Access::fixup_decorators() {
bool default_mo = (_decorators & MO_DECORATOR_MASK) == 0;
- bool is_unordered = (_decorators & MO_UNORDERED) != 0 || default_mo;
bool anonymous = (_decorators & C2_UNSAFE_ACCESS) != 0;
bool is_read = (_decorators & C2_READ_ACCESS) != 0;
bool is_write = (_decorators & C2_WRITE_ACCESS) != 0;
- if (AlwaysAtomicAccesses && is_unordered) {
- _decorators &= ~MO_DECORATOR_MASK; // clear the MO bits
- _decorators |= MO_RELAXED; // Force the MO_RELAXED decorator with AlwaysAtomicAccess
- }
-
_decorators = AccessInternal::decorator_fixup(_decorators, _type);
if (is_read && !is_write && anonymous) {
diff --git a/src/hotspot/share/gc/shared/c2/cardTableBarrierSetC2.cpp b/src/hotspot/share/gc/shared/c2/cardTableBarrierSetC2.cpp
index f7445ff254f..381a9f65295 100644
--- a/src/hotspot/share/gc/shared/c2/cardTableBarrierSetC2.cpp
+++ b/src/hotspot/share/gc/shared/c2/cardTableBarrierSetC2.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -126,7 +126,7 @@ Node* CardTableBarrierSetC2::byte_map_base_node(IdealKit* kit) const {
#endif
CardTable::CardValue* card_table_base = ci_card_table_address_const();
if (card_table_base != nullptr) {
- return kit->makecon(TypeRawPtr::make((address)card_table_base));
+ return kit->makecon(TypeRawPtr::make((address)card_table_base, relocInfo::none));
} else {
return kit->makecon(Type::get_zero_type(T_ADDRESS));
}
diff --git a/src/hotspot/share/gc/shared/classUnloadingContext.cpp b/src/hotspot/share/gc/shared/classUnloadingContext.cpp
index 4eac2561e67..d2b4a2fc636 100644
--- a/src/hotspot/share/gc/shared/classUnloadingContext.cpp
+++ b/src/hotspot/share/gc/shared/classUnloadingContext.cpp
@@ -54,7 +54,7 @@ ClassUnloadingContext::~ClassUnloadingContext() {
for (uint i = 0; i < _num_nmethod_unlink_workers; ++i) {
delete _unlinked_nmethods[i];
}
- FREE_C_HEAP_ARRAY(NMethodSet*, _unlinked_nmethods);
+ FREE_C_HEAP_ARRAY(_unlinked_nmethods);
assert(_context == this, "context not set correctly");
_context = nullptr;
diff --git a/src/hotspot/share/gc/shared/collectorCounters.cpp b/src/hotspot/share/gc/shared/collectorCounters.cpp
index f01997f9854..1624c693470 100644
--- a/src/hotspot/share/gc/shared/collectorCounters.cpp
+++ b/src/hotspot/share/gc/shared/collectorCounters.cpp
@@ -62,7 +62,7 @@ CollectorCounters::CollectorCounters(const char* name, int ordinal) {
}
CollectorCounters::~CollectorCounters() {
- FREE_C_HEAP_ARRAY(char, _name_space);
+ FREE_C_HEAP_ARRAY(_name_space);
}
TraceCollectorStats::TraceCollectorStats(CollectorCounters* c) :
diff --git a/src/hotspot/share/gc/shared/continuationGCSupport.hpp b/src/hotspot/share/gc/shared/continuationGCSupport.hpp
index dda983e0bd8..8aa0245201d 100644
--- a/src/hotspot/share/gc/shared/continuationGCSupport.hpp
+++ b/src/hotspot/share/gc/shared/continuationGCSupport.hpp
@@ -28,6 +28,8 @@
#include "memory/allStatic.hpp"
#include "oops/oopsHierarchy.hpp"
+class Klass;
+
class ContinuationGCSupport : public AllStatic {
public:
// Relativize the given oop if it is a stack chunk.
@@ -35,6 +37,7 @@ public:
// Relativize and transform to use a bitmap for future oop iteration for the
// given oop if it is a stack chunk.
static void transform_stack_chunk(oop obj);
+ static void transform_stack_chunk(oop obj, Klass* klass);
};
#endif // SHARE_GC_SHARED_CONTINUATIONGCSUPPORT_HPP
diff --git a/src/hotspot/share/gc/shared/continuationGCSupport.inline.hpp b/src/hotspot/share/gc/shared/continuationGCSupport.inline.hpp
index 72a828b7396..321ef51f9ae 100644
--- a/src/hotspot/share/gc/shared/continuationGCSupport.inline.hpp
+++ b/src/hotspot/share/gc/shared/continuationGCSupport.inline.hpp
@@ -43,7 +43,11 @@ inline bool ContinuationGCSupport::relativize_stack_chunk(oop obj) {
}
inline void ContinuationGCSupport::transform_stack_chunk(oop obj) {
- if (!obj->is_stackChunk()) {
+ transform_stack_chunk(obj, obj->klass());
+}
+
+inline void ContinuationGCSupport::transform_stack_chunk(oop obj, Klass* klass) {
+ if (!klass->is_stack_chunk_instance_klass()) {
return;
}
diff --git a/src/hotspot/share/gc/shared/gcArguments.cpp b/src/hotspot/share/gc/shared/gcArguments.cpp
index 424427c12b6..ec3d758c00a 100644
--- a/src/hotspot/share/gc/shared/gcArguments.cpp
+++ b/src/hotspot/share/gc/shared/gcArguments.cpp
@@ -25,10 +25,12 @@
#include "gc/shared/cardTable.hpp"
#include "gc/shared/gcArguments.hpp"
+#include "gc/shared/genArguments.hpp"
#include "logging/log.hpp"
#include "runtime/arguments.hpp"
#include "runtime/globals.hpp"
#include "runtime/globals_extension.hpp"
+#include "runtime/os.hpp"
#include "utilities/formatBuffer.hpp"
#include "utilities/macros.hpp"
@@ -56,6 +58,141 @@ void GCArguments::initialize() {
}
}
+size_t GCArguments::limit_heap_by_allocatable_memory(size_t limit) {
+ // Limits the given heap size by the maximum amount of virtual
+ // memory this process is currently allowed to use. It also takes
+ // the virtual-to-physical ratio of the current GC into account.
+ size_t fraction = MaxVirtMemFraction * heap_virtual_to_physical_ratio();
+ size_t max_allocatable = os::commit_memory_limit();
+
+ return MIN2(limit, max_allocatable / fraction);
+}
+
+// Use static initialization to get the default before parsing
+static const size_t DefaultHeapBaseMinAddress = HeapBaseMinAddress;
+
+static size_t clamp_by_size_t_max(uint64_t value) {
+ return (size_t)MIN2(value, (uint64_t)std::numeric_limits::max());
+}
+
+void GCArguments::set_heap_size() {
+ // Check if the user has configured any limit on the amount of RAM we may use.
+ bool has_ram_limit = !FLAG_IS_DEFAULT(MaxRAMPercentage) ||
+ !FLAG_IS_DEFAULT(MinRAMPercentage) ||
+ !FLAG_IS_DEFAULT(InitialRAMPercentage);
+
+ const physical_memory_size_type avail_mem = os::physical_memory();
+
+ // If the maximum heap size has not been set with -Xmx, then set it as
+ // fraction of the size of physical memory, respecting the maximum and
+ // minimum sizes of the heap.
+ if (FLAG_IS_DEFAULT(MaxHeapSize)) {
+ uint64_t min_memory = (uint64_t)(((double)avail_mem * MinRAMPercentage) / 100);
+ uint64_t max_memory = (uint64_t)(((double)avail_mem * MaxRAMPercentage) / 100);
+
+ const size_t reasonable_min = clamp_by_size_t_max(min_memory);
+ size_t reasonable_max = clamp_by_size_t_max(max_memory);
+
+ if (reasonable_min < MaxHeapSize) {
+ // Small physical memory, so use a minimum fraction of it for the heap
+ reasonable_max = reasonable_min;
+ } else {
+ // Not-small physical memory, so require a heap at least
+ // as large as MaxHeapSize
+ reasonable_max = MAX2(reasonable_max, MaxHeapSize);
+ }
+
+ if (!FLAG_IS_DEFAULT(ErgoHeapSizeLimit) && ErgoHeapSizeLimit != 0) {
+ // Limit the heap size to ErgoHeapSizeLimit
+ reasonable_max = MIN2(reasonable_max, ErgoHeapSizeLimit);
+ }
+
+ reasonable_max = limit_heap_by_allocatable_memory(reasonable_max);
+
+ if (!FLAG_IS_DEFAULT(InitialHeapSize)) {
+ // An initial heap size was specified on the command line,
+ // so be sure that the maximum size is consistent. Done
+ // after call to limit_heap_by_allocatable_memory because that
+ // method might reduce the allocation size.
+ reasonable_max = MAX2(reasonable_max, InitialHeapSize);
+ } else if (!FLAG_IS_DEFAULT(MinHeapSize)) {
+ reasonable_max = MAX2(reasonable_max, MinHeapSize);
+ }
+
+#ifdef _LP64
+ if (UseCompressedOops) {
+ // HeapBaseMinAddress can be greater than default but not less than.
+ if (!FLAG_IS_DEFAULT(HeapBaseMinAddress)) {
+ if (HeapBaseMinAddress < DefaultHeapBaseMinAddress) {
+ // matches compressed oops printing flags
+ log_debug(gc, heap, coops)("HeapBaseMinAddress must be at least %zu "
+ "(%zuG) which is greater than value given %zu",
+ DefaultHeapBaseMinAddress,
+ DefaultHeapBaseMinAddress/G,
+ HeapBaseMinAddress);
+ FLAG_SET_ERGO(HeapBaseMinAddress, DefaultHeapBaseMinAddress);
+ }
+ }
+
+ uintptr_t heap_end = HeapBaseMinAddress + MaxHeapSize;
+ uintptr_t max_coop_heap = Arguments::max_heap_for_compressed_oops();
+
+ // Limit the heap size to the maximum possible when using compressed oops
+ if (heap_end < max_coop_heap) {
+ // Heap should be above HeapBaseMinAddress to get zero based compressed
+ // oops but it should be not less than default MaxHeapSize.
+ max_coop_heap -= HeapBaseMinAddress;
+ }
+
+ // If the user has configured any limit on the amount of RAM we may use,
+ // then disable compressed oops if the calculated max exceeds max_coop_heap
+ // and UseCompressedOops was not specified.
+ if (reasonable_max > max_coop_heap) {
+ if (FLAG_IS_ERGO(UseCompressedOops) && has_ram_limit) {
+ log_debug(gc, heap, coops)("UseCompressedOops disabled due to "
+ "max heap %zu > compressed oop heap %zu. "
+ "Please check the setting of MaxRAMPercentage %5.2f.",
+ reasonable_max, (size_t)max_coop_heap, MaxRAMPercentage);
+ FLAG_SET_ERGO(UseCompressedOops, false);
+ } else {
+ reasonable_max = max_coop_heap;
+ }
+ }
+ }
+#endif // _LP64
+
+ log_trace(gc, heap)(" Maximum heap size %zu", reasonable_max);
+ FLAG_SET_ERGO(MaxHeapSize, reasonable_max);
+ }
+
+ // If the minimum or initial heap_size have not been set or requested to be set
+ // ergonomically, set them accordingly.
+ if (InitialHeapSize == 0 || MinHeapSize == 0) {
+ size_t reasonable_minimum = clamp_by_size_t_max((uint64_t)OldSize + (uint64_t)NewSize);
+ reasonable_minimum = MIN2(reasonable_minimum, MaxHeapSize);
+ reasonable_minimum = limit_heap_by_allocatable_memory(reasonable_minimum);
+
+ if (InitialHeapSize == 0) {
+ uint64_t initial_memory = (uint64_t)(((double)avail_mem * InitialRAMPercentage) / 100);
+ size_t reasonable_initial = clamp_by_size_t_max(initial_memory);
+ reasonable_initial = limit_heap_by_allocatable_memory(reasonable_initial);
+
+ reasonable_initial = MAX3(reasonable_initial, reasonable_minimum, MinHeapSize);
+ reasonable_initial = MIN2(reasonable_initial, MaxHeapSize);
+
+ FLAG_SET_ERGO(InitialHeapSize, (size_t)reasonable_initial);
+ log_trace(gc, heap)(" Initial heap size %zu", InitialHeapSize);
+ }
+
+ // If the minimum heap size has not been set (via -Xms or -XX:MinHeapSize),
+ // synchronize with InitialHeapSize to avoid errors with the default value.
+ if (MinHeapSize == 0) {
+ FLAG_SET_ERGO(MinHeapSize, MIN2(reasonable_minimum, InitialHeapSize));
+ log_trace(gc, heap)(" Minimum heap size %zu", MinHeapSize);
+ }
+ }
+}
+
void GCArguments::initialize_heap_sizes() {
initialize_alignments();
initialize_heap_flags_and_sizes();
diff --git a/src/hotspot/share/gc/shared/gcArguments.hpp b/src/hotspot/share/gc/shared/gcArguments.hpp
index 7e9fffedaba..6f44fc420aa 100644
--- a/src/hotspot/share/gc/shared/gcArguments.hpp
+++ b/src/hotspot/share/gc/shared/gcArguments.hpp
@@ -40,10 +40,13 @@ protected:
virtual void initialize_heap_flags_and_sizes();
virtual void initialize_size_info();
+ size_t limit_heap_by_allocatable_memory(size_t size);
+
DEBUG_ONLY(void assert_flags();)
DEBUG_ONLY(void assert_size_info();)
public:
+ virtual void set_heap_size();
virtual void initialize();
// Return the (conservative) maximum heap alignment
diff --git a/src/hotspot/share/gc/shared/gc_globals.hpp b/src/hotspot/share/gc/shared/gc_globals.hpp
index c9102944197..1ff6fd493a7 100644
--- a/src/hotspot/share/gc/shared/gc_globals.hpp
+++ b/src/hotspot/share/gc/shared/gc_globals.hpp
@@ -262,10 +262,6 @@
"and ObjArrayMarkingStride.") \
constraint(ArrayMarkingMinStrideConstraintFunc,AfterErgo) \
\
- product(bool, AggressiveHeap, false, \
- "(Deprecated) Optimize heap options for long-running memory " \
- "intensive apps") \
- \
product(size_t, ErgoHeapSizeLimit, 0, \
"Maximum ergonomically set heap size (in bytes); zero means use " \
"(System RAM) * MaxRAMPercentage / 100") \
diff --git a/src/hotspot/share/gc/shared/generationCounters.cpp b/src/hotspot/share/gc/shared/generationCounters.cpp
index f6bbc1c2618..85da4c99cbb 100644
--- a/src/hotspot/share/gc/shared/generationCounters.cpp
+++ b/src/hotspot/share/gc/shared/generationCounters.cpp
@@ -64,7 +64,7 @@ GenerationCounters::GenerationCounters(const char* name,
}
GenerationCounters::~GenerationCounters() {
- FREE_C_HEAP_ARRAY(char, _name_space);
+ FREE_C_HEAP_ARRAY(_name_space);
}
void GenerationCounters::update_capacity(size_t curr_capacity) {
diff --git a/src/hotspot/share/gc/shared/hSpaceCounters.cpp b/src/hotspot/share/gc/shared/hSpaceCounters.cpp
index a873bc2f45c..5dd9d5bfaa8 100644
--- a/src/hotspot/share/gc/shared/hSpaceCounters.cpp
+++ b/src/hotspot/share/gc/shared/hSpaceCounters.cpp
@@ -66,7 +66,7 @@ HSpaceCounters::HSpaceCounters(const char* name_space,
}
HSpaceCounters::~HSpaceCounters() {
- FREE_C_HEAP_ARRAY(char, _name_space);
+ FREE_C_HEAP_ARRAY(_name_space);
}
void HSpaceCounters::update_capacity(size_t v) {
diff --git a/src/hotspot/share/gc/shared/jvmFlagConstraintsGC.cpp b/src/hotspot/share/gc/shared/jvmFlagConstraintsGC.cpp
index 4d7ffce3a5d..096bee9e4a0 100644
--- a/src/hotspot/share/gc/shared/jvmFlagConstraintsGC.cpp
+++ b/src/hotspot/share/gc/shared/jvmFlagConstraintsGC.cpp
@@ -283,7 +283,7 @@ JVMFlag::Error SoftMaxHeapSizeConstraintFunc(size_t value, bool verbose) {
}
JVMFlag::Error HeapBaseMinAddressConstraintFunc(size_t value, bool verbose) {
- // If an overflow happened in Arguments::set_heap_size(), MaxHeapSize will have too large a value.
+ // If an overflow happened in GCArguments::set_heap_size(), MaxHeapSize will have too large a value.
// Check for this by ensuring that MaxHeapSize plus the requested min base address still fit within max_uintx.
if (value > (max_uintx - MaxHeapSize)) {
JVMFlag::printError(verbose,
diff --git a/src/hotspot/share/gc/shared/oopStorage.cpp b/src/hotspot/share/gc/shared/oopStorage.cpp
index 21e63f6fc32..8aea565cdf7 100644
--- a/src/hotspot/share/gc/shared/oopStorage.cpp
+++ b/src/hotspot/share/gc/shared/oopStorage.cpp
@@ -136,7 +136,7 @@ OopStorage::ActiveArray* OopStorage::ActiveArray::create(size_t size,
void OopStorage::ActiveArray::destroy(ActiveArray* ba) {
ba->~ActiveArray();
- FREE_C_HEAP_ARRAY(char, ba);
+ FREE_C_HEAP_ARRAY(ba);
}
size_t OopStorage::ActiveArray::size() const {
@@ -362,7 +362,7 @@ OopStorage::Block* OopStorage::Block::new_block(const OopStorage* owner) {
void OopStorage::Block::delete_block(const Block& block) {
void* memory = block._memory;
block.Block::~Block();
- FREE_C_HEAP_ARRAY(char, memory);
+ FREE_C_HEAP_ARRAY(memory);
}
// This can return a false positive if ptr is not contained by some
diff --git a/src/hotspot/share/gc/shared/partialArrayState.cpp b/src/hotspot/share/gc/shared/partialArrayState.cpp
index d3b21c2fdaa..4679ac2f51c 100644
--- a/src/hotspot/share/gc/shared/partialArrayState.cpp
+++ b/src/hotspot/share/gc/shared/partialArrayState.cpp
@@ -114,7 +114,7 @@ PartialArrayStateManager::PartialArrayStateManager(uint max_allocators)
PartialArrayStateManager::~PartialArrayStateManager() {
reset();
- FREE_C_HEAP_ARRAY(Arena, _arenas);
+ FREE_C_HEAP_ARRAY(_arenas);
}
Arena* PartialArrayStateManager::register_allocator() {
diff --git a/src/hotspot/share/gc/shared/preservedMarks.cpp b/src/hotspot/share/gc/shared/preservedMarks.cpp
index 605b7afe072..6a69fe10f95 100644
--- a/src/hotspot/share/gc/shared/preservedMarks.cpp
+++ b/src/hotspot/share/gc/shared/preservedMarks.cpp
@@ -148,7 +148,7 @@ void PreservedMarksSet::reclaim() {
}
if (_in_c_heap) {
- FREE_C_HEAP_ARRAY(Padded, _stacks);
+ FREE_C_HEAP_ARRAY(_stacks);
} else {
// the array was resource-allocated, so nothing to do
}
diff --git a/src/hotspot/share/gc/shared/referencePolicy.cpp b/src/hotspot/share/gc/shared/referencePolicy.cpp
index d1867291479..6c5f459ebb4 100644
--- a/src/hotspot/share/gc/shared/referencePolicy.cpp
+++ b/src/hotspot/share/gc/shared/referencePolicy.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -28,21 +28,17 @@
#include "gc/shared/referencePolicy.hpp"
#include "memory/universe.hpp"
#include "runtime/globals.hpp"
+#include "utilities/integerCast.hpp"
-LRUCurrentHeapPolicy::LRUCurrentHeapPolicy() {
- setup();
-}
-
-// Capture state (of-the-VM) information needed to evaluate the policy
-void LRUCurrentHeapPolicy::setup() {
- _max_interval = (Universe::heap()->free_at_last_gc() / M) * SoftRefLRUPolicyMSPerMB;
- assert(_max_interval >= 0,"Sanity check");
+void AbstractLRUReferencePolicy::set_max_interval(jlong max_interval) {
+ assert(max_interval >= 0, "Sanity check");
+ _max_interval = max_interval;
}
// The oop passed in is the SoftReference object, and not
// the object the SoftReference points to.
-bool LRUCurrentHeapPolicy::should_clear_reference(oop p,
- jlong timestamp_clock) {
+bool AbstractLRUReferencePolicy::should_clear_reference(oop p, jlong timestamp_clock) {
+ assert(_max_interval >= 0, "Forgot to call setup");
jlong interval = timestamp_clock - java_lang_ref_SoftReference::timestamp(p);
assert(interval >= 0, "Sanity check");
@@ -54,10 +50,9 @@ bool LRUCurrentHeapPolicy::should_clear_reference(oop p,
return true;
}
-/////////////////////// MaxHeap //////////////////////
-
-LRUMaxHeapPolicy::LRUMaxHeapPolicy() {
- setup();
+// Capture state (of-the-VM) information needed to evaluate the policy
+void LRUCurrentHeapPolicy::setup() {
+ set_max_interval(integer_cast(Universe::heap()->free_at_last_gc() / M) * SoftRefLRUPolicyMSPerMB);
}
// Capture state (of-the-VM) information needed to evaluate the policy
@@ -66,21 +61,5 @@ void LRUMaxHeapPolicy::setup() {
max_heap -= Universe::heap()->used_at_last_gc();
max_heap /= M;
- _max_interval = max_heap * SoftRefLRUPolicyMSPerMB;
- assert(_max_interval >= 0,"Sanity check");
-}
-
-// The oop passed in is the SoftReference object, and not
-// the object the SoftReference points to.
-bool LRUMaxHeapPolicy::should_clear_reference(oop p,
- jlong timestamp_clock) {
- jlong interval = timestamp_clock - java_lang_ref_SoftReference::timestamp(p);
- assert(interval >= 0, "Sanity check");
-
- // The interval will be zero if the ref was accessed since the last scavenge/gc.
- if(interval <= _max_interval) {
- return false;
- }
-
- return true;
+ set_max_interval(integer_cast(max_heap) * SoftRefLRUPolicyMSPerMB);
}
diff --git a/src/hotspot/share/gc/shared/referencePolicy.hpp b/src/hotspot/share/gc/shared/referencePolicy.hpp
index cf0382b036b..0fd918fa723 100644
--- a/src/hotspot/share/gc/shared/referencePolicy.hpp
+++ b/src/hotspot/share/gc/shared/referencePolicy.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -56,28 +56,28 @@ class AlwaysClearPolicy : public ReferencePolicy {
}
};
-class LRUCurrentHeapPolicy : public ReferencePolicy {
+class AbstractLRUReferencePolicy : public ReferencePolicy {
private:
- jlong _max_interval;
+ jlong _max_interval = -1;
+
+ protected:
+ void set_max_interval(jlong max_interval);
public:
- LRUCurrentHeapPolicy();
-
- // Capture state (of-the-VM) information needed to evaluate the policy
- void setup();
- virtual bool should_clear_reference(oop p, jlong timestamp_clock);
+ bool should_clear_reference(oop p, jlong timestamp_clock) final;
+ void setup() override = 0;
};
-class LRUMaxHeapPolicy : public ReferencePolicy {
- private:
- jlong _max_interval;
-
+class LRUCurrentHeapPolicy : public AbstractLRUReferencePolicy {
public:
- LRUMaxHeapPolicy();
-
// Capture state (of-the-VM) information needed to evaluate the policy
- void setup();
- virtual bool should_clear_reference(oop p, jlong timestamp_clock);
+ void setup() final;
+};
+
+class LRUMaxHeapPolicy : public AbstractLRUReferencePolicy {
+ public:
+ // Capture state (of-the-VM) information needed to evaluate the policy
+ void setup() final;
};
#endif // SHARE_GC_SHARED_REFERENCEPOLICY_HPP
diff --git a/src/hotspot/share/gc/shared/stringdedup/stringDedup.cpp b/src/hotspot/share/gc/shared/stringdedup/stringDedup.cpp
index b3f96da1cce..ab1b15b0447 100644
--- a/src/hotspot/share/gc/shared/stringdedup/stringDedup.cpp
+++ b/src/hotspot/share/gc/shared/stringdedup/stringDedup.cpp
@@ -182,7 +182,7 @@ void StringDedup::Requests::flush() {
assert(_storage_for_requests != nullptr, "invariant");
_storage_for_requests->storage()->release(_buffer, _index);
}
- FREE_C_HEAP_ARRAY(oop*, _buffer);
+ FREE_C_HEAP_ARRAY(_buffer);
_buffer = nullptr;
}
if (_storage_for_requests != nullptr) {
diff --git a/src/hotspot/share/gc/shared/stringdedup/stringDedupTable.cpp b/src/hotspot/share/gc/shared/stringdedup/stringDedupTable.cpp
index a376f3b96de..546efa4774b 100644
--- a/src/hotspot/share/gc/shared/stringdedup/stringDedupTable.cpp
+++ b/src/hotspot/share/gc/shared/stringdedup/stringDedupTable.cpp
@@ -455,7 +455,7 @@ void StringDedup::Table::free_buckets(Bucket* buckets, size_t number_of_buckets)
while (number_of_buckets > 0) {
buckets[--number_of_buckets].~Bucket();
}
- FREE_C_HEAP_ARRAY(Bucket, buckets);
+ FREE_C_HEAP_ARRAY(buckets);
}
// Compute the hash code for obj using halfsiphash_32. As this is a high
diff --git a/src/hotspot/share/gc/shared/taskqueue.inline.hpp b/src/hotspot/share/gc/shared/taskqueue.inline.hpp
index b142aadc580..c942b6cc3e9 100644
--- a/src/hotspot/share/gc/shared/taskqueue.inline.hpp
+++ b/src/hotspot/share/gc/shared/taskqueue.inline.hpp
@@ -48,7 +48,7 @@ inline GenericTaskQueueSet::GenericTaskQueueSet(uint n) : _n(n) {
template
inline GenericTaskQueueSet::~GenericTaskQueueSet() {
- FREE_C_HEAP_ARRAY(T*, _queues);
+ FREE_C_HEAP_ARRAY(_queues);
}
#if TASKQUEUE_STATS
diff --git a/src/hotspot/share/gc/shared/threadLocalAllocBuffer.cpp b/src/hotspot/share/gc/shared/threadLocalAllocBuffer.cpp
index f9b8694eb04..59d7befb32d 100644
--- a/src/hotspot/share/gc/shared/threadLocalAllocBuffer.cpp
+++ b/src/hotspot/share/gc/shared/threadLocalAllocBuffer.cpp
@@ -58,7 +58,11 @@ ThreadLocalAllocBuffer::ThreadLocalAllocBuffer() :
// do nothing. TLABs must be inited by initialize() calls
}
-size_t ThreadLocalAllocBuffer::initial_refill_waste_limit() { return desired_size() / TLABRefillWasteFraction; }
+size_t ThreadLocalAllocBuffer::initial_refill_waste_limit() {
+ assert(TLABRefillWasteFraction != 0, "inv");
+ return desired_size() / TLABRefillWasteFraction;
+}
+
size_t ThreadLocalAllocBuffer::min_size() { return align_object_size(MinTLABSize / HeapWordSize) + alignment_reserve(); }
size_t ThreadLocalAllocBuffer::refill_waste_limit_increment() { return TLABWasteIncrement; }
diff --git a/src/hotspot/share/gc/shared/workerDataArray.inline.hpp b/src/hotspot/share/gc/shared/workerDataArray.inline.hpp
index 34549bc079e..3b7658ec4c8 100644
--- a/src/hotspot/share/gc/shared/workerDataArray.inline.hpp
+++ b/src/hotspot/share/gc/shared/workerDataArray.inline.hpp
@@ -72,7 +72,7 @@ WorkerDataArray::~WorkerDataArray() {
for (uint i = 0; i < MaxThreadWorkItems; i++) {
delete _thread_work_items[i];
}
- FREE_C_HEAP_ARRAY(T, _data);
+ FREE_C_HEAP_ARRAY(_data);
}
template
diff --git a/src/hotspot/share/gc/shared/workerThread.cpp b/src/hotspot/share/gc/shared/workerThread.cpp
index 2738c98e5c3..35749452c85 100644
--- a/src/hotspot/share/gc/shared/workerThread.cpp
+++ b/src/hotspot/share/gc/shared/workerThread.cpp
@@ -39,6 +39,8 @@ WorkerTaskDispatcher::WorkerTaskDispatcher() :
_end_semaphore() {}
void WorkerTaskDispatcher::coordinator_distribute_task(WorkerTask* task, uint num_workers) {
+ guarantee(num_workers > 0, "must use at least one worker, deadlocks otherwise");
+
// No workers are allowed to read the state variables until they have been signaled.
_task = task;
_not_finished.store_relaxed(num_workers);
@@ -129,9 +131,9 @@ WorkerThread* WorkerThreads::create_worker(uint name_suffix) {
}
uint WorkerThreads::set_active_workers(uint num_workers) {
- assert(num_workers > 0 && num_workers <= _max_workers,
- "Invalid number of active workers %u (should be 1-%u)",
- num_workers, _max_workers);
+ guarantee(num_workers > 0 && num_workers <= _max_workers,
+ "Invalid number of active workers %u (should be 1-%u)",
+ num_workers, _max_workers);
while (_created_workers < num_workers) {
WorkerThread* const worker = create_worker(_created_workers);
diff --git a/src/hotspot/share/gc/shared/workerUtils.cpp b/src/hotspot/share/gc/shared/workerUtils.cpp
index 1826b9d7df8..736a9b007dd 100644
--- a/src/hotspot/share/gc/shared/workerUtils.cpp
+++ b/src/hotspot/share/gc/shared/workerUtils.cpp
@@ -122,7 +122,7 @@ bool SubTasksDone::try_claim_task(uint t) {
SubTasksDone::~SubTasksDone() {
assert(_verification_done.load_relaxed(), "all_tasks_claimed must have been called.");
- FREE_C_HEAP_ARRAY(Atomic, _tasks);
+ FREE_C_HEAP_ARRAY(_tasks);
}
// *** SequentialSubTasksDone
diff --git a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp
index ad1dca47503..637ed6e6407 100644
--- a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp
+++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp
@@ -133,7 +133,6 @@ LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier_impl(LIRGenerator* gen, L
addr = ensure_in_register(gen, addr, T_ADDRESS);
assert(addr->is_register(), "must be a register at this point");
LIR_Opr result = gen->result_register_for(obj->value_type());
- __ move(obj, result);
LIR_Opr tmp1 = gen->new_register(T_ADDRESS);
LIR_Opr tmp2 = gen->new_register(T_ADDRESS);
@@ -164,6 +163,11 @@ LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier_impl(LIRGenerator* gen, L
CodeStub* slow = new ShenandoahLoadReferenceBarrierStub(obj, addr, result, tmp1, tmp2, decorators);
__ branch(lir_cond_notEqual, slow);
+
+ // No barrier is needed, move obj to result now.
+ __ move(obj, result);
+
+ // Slow-path re-enters here with result set.
__ branch_destination(slow->continuation());
return result;
@@ -199,7 +203,7 @@ void ShenandoahBarrierSetC1::store_at_resolved(LIRAccess& access, LIR_Opr value)
bool precise = is_array || on_anonymous;
LIR_Opr post_addr = precise ? access.resolved_addr() : access.base().opr();
- post_barrier(access, post_addr, value);
+ post_barrier(access, post_addr);
}
}
@@ -314,7 +318,7 @@ bool ShenandoahBarrierSetC1::generate_c1_runtime_stubs(BufferBlob* buffer_blob)
return true;
}
-void ShenandoahBarrierSetC1::post_barrier(LIRAccess& access, LIR_Opr addr, LIR_Opr new_val) {
+void ShenandoahBarrierSetC1::post_barrier(LIRAccess& access, LIR_Opr addr) {
assert(ShenandoahCardBarrier, "Should have been checked by caller");
DecoratorSet decorators = access.decorators();
@@ -368,3 +372,71 @@ void ShenandoahBarrierSetC1::post_barrier(LIRAccess& access, LIR_Opr addr, LIR_O
__ move(dirty, card_addr);
}
}
+
+LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value) {
+ if (!access.is_oop()) {
+ return BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value);
+ }
+
+ LIRGenerator* gen = access.gen();
+
+ LIR_Opr tmp = gen->new_register(T_OBJECT);
+ LIR_Opr addr = access.resolved_addr();
+
+ // Handle the previous value through SATB, as we are about to perform the store.
+ __ load(addr->as_address_ptr(), tmp);
+ if (ShenandoahSATBBarrier) {
+ pre_barrier(gen, access.access_emit_info(), access.decorators(),
+ /* addr_opr (unused) = */ LIR_OprFact::illegalOpr,
+ /* pre_val = */ tmp);
+ }
+
+ // Perform LRB on location to fix it up for this and all following accesses.
+ // This guarantees there are no false negatives due to concurrent evacuation,
+ // and the value loaded later by CAS is sanitized by some LRB, or is null.
+ if (ShenandoahLoadRefBarrier) {
+ load_reference_barrier(gen, /* obj = */ tmp, /* addr = */ addr, access.decorators());
+ }
+
+ LIR_Opr result = BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value);
+
+ if (ShenandoahCardBarrier) {
+ post_barrier(access, /* addr = */ addr);
+ }
+
+ return result;
+}
+
+LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value) {
+ if (!access.is_oop()) {
+ return BarrierSetC1::atomic_xchg_at_resolved(access, value);
+ }
+
+ LIRGenerator* gen = access.gen();
+
+ LIR_Opr tmp = gen->new_register(T_OBJECT);
+ LIR_Opr addr = access.resolved_addr();
+
+ // Handle the previous value through SATB, as we are about to perform the store.
+ __ load(addr->as_address_ptr(), tmp);
+ if (ShenandoahSATBBarrier) {
+ pre_barrier(gen, access.access_emit_info(), access.decorators(),
+ /* addr_opr (unused) = */ LIR_OprFact::illegalOpr,
+ /* pre_val = */ tmp);
+ }
+
+ // Perform LRB on location to fix it up for this and all following accesses.
+ // This is purely opportunistic: we would not have any false negatives here.
+ // This guarantees the value loaded later by XCHG is sanitized by some LRB, or is null.
+ if (ShenandoahLoadRefBarrier) {
+ load_reference_barrier(gen, /* obj = */ tmp, /* addr = */ addr, access.decorators());
+ }
+
+ LIR_Opr result = BarrierSetC1::atomic_xchg_at_resolved(access, value);
+
+ if (ShenandoahCardBarrier) {
+ post_barrier(access, /* addr = */ addr);
+ }
+
+ return result;
+}
diff --git a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp
index 1b4f2c79bd2..413777a61ee 100644
--- a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp
+++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp
@@ -127,6 +127,7 @@ public:
visitor->do_input(_addr);
visitor->do_temp(_addr);
visitor->do_temp(_result);
+ visitor->do_output(_result);
visitor->do_temp(_tmp1);
visitor->do_temp(_tmp2);
}
@@ -135,61 +136,6 @@ public:
#endif // PRODUCT
};
-class LIR_OpShenandoahCompareAndSwap : public LIR_Op {
- friend class LIR_OpVisitState;
-
-private:
- LIR_Opr _addr;
- LIR_Opr _cmp_value;
- LIR_Opr _new_value;
- LIR_Opr _tmp1;
- LIR_Opr _tmp2;
-
-public:
- LIR_OpShenandoahCompareAndSwap(LIR_Opr addr, LIR_Opr cmp_value, LIR_Opr new_value,
- LIR_Opr t1, LIR_Opr t2, LIR_Opr result)
- : LIR_Op(lir_none, result, nullptr) // no info
- , _addr(addr)
- , _cmp_value(cmp_value)
- , _new_value(new_value)
- , _tmp1(t1)
- , _tmp2(t2) { }
-
- LIR_Opr addr() const { return _addr; }
- LIR_Opr cmp_value() const { return _cmp_value; }
- LIR_Opr new_value() const { return _new_value; }
- LIR_Opr tmp1() const { return _tmp1; }
- LIR_Opr tmp2() const { return _tmp2; }
-
- virtual void visit(LIR_OpVisitState* state) {
- if (_info) state->do_info(_info);
- assert(_addr->is_valid(), "used"); state->do_input(_addr);
- state->do_temp(_addr);
- assert(_cmp_value->is_valid(), "used"); state->do_input(_cmp_value);
- state->do_temp(_cmp_value);
- assert(_new_value->is_valid(), "used"); state->do_input(_new_value);
- state->do_temp(_new_value);
- if (_tmp1->is_valid()) state->do_temp(_tmp1);
- if (_tmp2->is_valid()) state->do_temp(_tmp2);
- if (_result->is_valid()) state->do_output(_result);
- }
-
- virtual void emit_code(LIR_Assembler* masm);
-
- virtual void print_instr(outputStream* out) const {
- addr()->print(out); out->print(" ");
- cmp_value()->print(out); out->print(" ");
- new_value()->print(out); out->print(" ");
- tmp1()->print(out); out->print(" ");
- tmp2()->print(out); out->print(" ");
- }
-#ifndef PRODUCT
- virtual const char* name() const {
- return "shenandoah_cas_obj";
- }
-#endif // PRODUCT
-};
-
class ShenandoahBarrierSetC1 : public BarrierSetC1 {
private:
CodeBlob* _pre_barrier_c1_runtime_code_blob;
@@ -244,7 +190,7 @@ protected:
virtual LIR_Opr atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value);
- void post_barrier(LIRAccess& access, LIR_Opr addr, LIR_Opr new_val);
+ void post_barrier(LIRAccess& access, LIR_Opr addr);
public:
diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp
index f721c3cd001..deab648a108 100644
--- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp
@@ -73,7 +73,7 @@ void ShenandoahBarrierSetC2State::remove_load_reference_barrier(ShenandoahLoadRe
#define __ kit->
-bool ShenandoahBarrierSetC2::satb_can_remove_pre_barrier(GraphKit* kit, PhaseValues* phase, Node* adr,
+bool ShenandoahBarrierSetC2::satb_can_remove_pre_barrier(GraphKit* kit, PhaseGVN* phase, Node* adr,
BasicType bt, uint adr_idx) const {
intptr_t offset = 0;
Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
@@ -990,19 +990,23 @@ void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node*
shenandoah_eliminate_wb_pre(node, ¯o->igvn());
}
if (ShenandoahCardBarrier && node->Opcode() == Op_CastP2X) {
- Node* shift = node->unique_out();
- Node* addp = shift->unique_out();
- for (DUIterator_Last jmin, j = addp->last_outs(jmin); j >= jmin; --j) {
- Node* mem = addp->last_out(j);
- if (UseCondCardMark && mem->is_Load()) {
- assert(mem->Opcode() == Op_LoadB, "unexpected code shape");
- // The load is checking if the card has been written so
- // replace it with zero to fold the test.
- macro->replace_node(mem, macro->intcon(0));
- continue;
+ for (DUIterator_Last imin, i = node->last_outs(imin); i >= imin; --i) {
+ Node* shift = node->last_out(i);
+ for (DUIterator_Last kmin, k = shift->last_outs(kmin); k >= kmin; --k) {
+ Node* addp = shift->last_out(k);
+ for (DUIterator_Last jmin, j = addp->last_outs(jmin); j >= jmin; --j) {
+ Node* mem = addp->last_out(j);
+ if (UseCondCardMark && mem->is_Load()) {
+ assert(mem->Opcode() == Op_LoadB, "unexpected code shape");
+ // The load is checking if the card has been written so
+ // replace it with zero to fold the test.
+ macro->replace_node(mem, macro->intcon(0));
+ continue;
+ }
+ assert(mem->is_Store(), "store required");
+ macro->replace_node(mem, mem->in(MemNode::Memory));
+ }
}
- assert(mem->is_Store(), "store required");
- macro->replace_node(mem, mem->in(MemNode::Memory));
}
}
}
diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp
index 108eaa0998b..c77a9da63fc 100644
--- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp
@@ -46,7 +46,7 @@ class ShenandoahBarrierSetC2 : public BarrierSetC2 {
private:
void shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const;
- bool satb_can_remove_pre_barrier(GraphKit* kit, PhaseValues* phase, Node* adr,
+ bool satb_can_remove_pre_barrier(GraphKit* kit, PhaseGVN* phase, Node* adr,
BasicType bt, uint adr_idx) const;
void satb_write_barrier_pre(GraphKit* kit, bool do_load,
Node* obj,
diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp
index 015276feb5c..86de9eef459 100644
--- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp
@@ -25,6 +25,7 @@
#include "classfile/javaClasses.hpp"
+#include "code/aotCodeCache.hpp"
#include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
#include "gc/shenandoah/c2/shenandoahSupport.hpp"
#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
@@ -928,12 +929,31 @@ void ShenandoahBarrierC2Support::test_in_cset(Node*& ctrl, Node*& not_cset_ctrl,
PhaseIterGVN& igvn = phase->igvn();
Node* raw_val = new CastP2XNode(old_ctrl, val);
- Node* cset_idx = new URShiftXNode(raw_val, igvn.intcon(ShenandoahHeapRegion::region_size_bytes_shift_jint()));
+ Node* region_size_shift = nullptr;
+ if (AOTCodeCache::is_on_for_dump()) {
+ Node* aot_addr = igvn.makecon(TypeRawPtr::make(AOTRuntimeConstants::grain_shift_address()));
+ region_size_shift = new LoadINode(old_ctrl, raw_mem, aot_addr,
+ DEBUG_ONLY(phase->C->get_adr_type(Compile::AliasIdxRaw)) NOT_DEBUG(nullptr),
+ TypeInt::INT, MemNode::unordered);
+ phase->register_new_node(region_size_shift, old_ctrl);
+ } else {
+ region_size_shift = igvn.intcon(ShenandoahHeapRegion::region_size_bytes_shift_jint());
+ }
+ Node* cset_idx = new URShiftXNode(raw_val, region_size_shift);
// Figure out the target cset address with raw pointer math.
// This avoids matching AddP+LoadB that would emit inefficient code.
// See JDK-8245465.
- Node* cset_addr_ptr = igvn.makecon(TypeRawPtr::make(ShenandoahHeap::in_cset_fast_test_addr()));
+ Node* cset_addr_ptr = nullptr;
+ if (AOTCodeCache::is_on_for_dump()) {
+ Node* aot_addr = igvn.makecon(TypeRawPtr::make(AOTRuntimeConstants::cset_base_address()));
+ cset_addr_ptr = new LoadPNode(old_ctrl, raw_mem, aot_addr,
+ DEBUG_ONLY(phase->C->get_adr_type(Compile::AliasIdxRaw)) NOT_DEBUG(nullptr),
+ TypeRawPtr::NOTNULL, MemNode::unordered);
+ phase->register_new_node(cset_addr_ptr, old_ctrl);
+ } else {
+ cset_addr_ptr = igvn.makecon(TypeRawPtr::make(ShenandoahHeap::in_cset_fast_test_addr()));
+ }
Node* cset_addr = new CastP2XNode(old_ctrl, cset_addr_ptr);
Node* cset_load_addr = new AddXNode(cset_addr, cset_idx);
Node* cset_load_ptr = new CastX2PNode(cset_load_addr);
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp
index c595d1fd9cd..ce74e8cf199 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp
@@ -111,12 +111,12 @@ ShenandoahAdaptiveHeuristics::ShenandoahAdaptiveHeuristics(ShenandoahSpaceInfo*
}
ShenandoahAdaptiveHeuristics::~ShenandoahAdaptiveHeuristics() {
- FREE_C_HEAP_ARRAY(double, _spike_acceleration_rate_samples);
- FREE_C_HEAP_ARRAY(double, _spike_acceleration_rate_timestamps);
- FREE_C_HEAP_ARRAY(double, _gc_time_timestamps);
- FREE_C_HEAP_ARRAY(double, _gc_time_samples);
- FREE_C_HEAP_ARRAY(double, _gc_time_xy);
- FREE_C_HEAP_ARRAY(double, _gc_time_xx);
+ FREE_C_HEAP_ARRAY(_spike_acceleration_rate_samples);
+ FREE_C_HEAP_ARRAY(_spike_acceleration_rate_timestamps);
+ FREE_C_HEAP_ARRAY(_gc_time_timestamps);
+ FREE_C_HEAP_ARRAY(_gc_time_samples);
+ FREE_C_HEAP_ARRAY(_gc_time_xy);
+ FREE_C_HEAP_ARRAY(_gc_time_xx);
}
void ShenandoahAdaptiveHeuristics::initialize() {
@@ -428,9 +428,7 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() {
double predicted_future_gc_time = 0;
double future_planned_gc_time = 0;
bool future_planned_gc_time_is_average = false;
- double avg_time_to_deplete_available = 0.0;
bool is_spiking = false;
- double spike_time_to_deplete_available = 0.0;
log_debug(gc, ergo)("should_start_gc calculation: available: " PROPERFMT ", soft_max_capacity: " PROPERFMT ", "
"allocated_since_gc_start: " PROPERFMT,
@@ -650,9 +648,8 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() {
_space_info->name(), avg_cycle_time * 1000, predicted_future_gc_time * 1000,
byte_size_in_proper_unit(avg_alloc_rate), proper_unit_for_byte_size(avg_alloc_rate));
size_t allocatable_bytes = allocatable_words * HeapWordSize;
- avg_time_to_deplete_available = allocatable_bytes / avg_alloc_rate;
- if (future_planned_gc_time > avg_time_to_deplete_available) {
+ if (future_planned_gc_time * avg_alloc_rate > allocatable_bytes) {
log_trigger("%s GC time (%.2f ms) is above the time for average allocation rate (%.0f %sB/s)"
" to deplete free headroom (%zu%s) (margin of error = %.2f)",
future_planned_gc_time_is_average? "Average": "Linear prediction of", future_planned_gc_time * 1000,
@@ -675,8 +672,7 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() {
}
is_spiking = _allocation_rate.is_spiking(rate, _spike_threshold_sd);
- spike_time_to_deplete_available = (rate == 0)? 0: allocatable_bytes / rate;
- if (is_spiking && (rate != 0) && (future_planned_gc_time > spike_time_to_deplete_available)) {
+ if (is_spiking && (future_planned_gc_time * rate > allocatable_bytes)) {
log_trigger("%s GC time (%.2f ms) is above the time for instantaneous allocation rate (%.0f %sB/s)"
" to deplete free headroom (%zu%s) (spike threshold = %.2f)",
future_planned_gc_time_is_average? "Average": "Linear prediction of", future_planned_gc_time * 1000,
@@ -837,19 +833,29 @@ double ShenandoahAllocationRate::force_sample(size_t allocated, size_t &unaccoun
const double MinSampleTime = 0.002; // Do not sample if time since last update is less than 2 ms
double now = os::elapsedTime();
double time_since_last_update = now - _last_sample_time;
+ double rate = 0.0;
if (time_since_last_update < MinSampleTime) {
+ // If we choose not to sample right now, the unaccounted_bytes_allocated will be added
+ // into the next sample taken. These unaccounted_bytes_allocated will be added to
+ // any additional bytes that are allocated during this GC cycle at the time the rate is
+ // next sampled. We do not overwrite _last_sample_time on this path, because the
+ // unaccounted_bytes_allocated were allocated following _last_sample_time.
unaccounted_bytes_allocated = allocated - _last_sample_value;
- _last_sample_value = 0;
- return 0.0;
} else {
- double rate = instantaneous_rate(now, allocated);
+ rate = instantaneous_rate(now, allocated);
_rate.add(rate);
_rate_avg.add(_rate.avg());
_last_sample_time = now;
- _last_sample_value = allocated;
unaccounted_bytes_allocated = 0;
- return rate;
}
+ // force_sample() is called when resetting bytes allocated since gc start. All subsequent
+ // requests to sample allocated bytes during this GC cycle are measured as a delta from
+ // _last_sample_value. In the case that we choose not to sample now, we will count the
+ // unaccounted_bytes_allocated as if they were allocated following the start of this GC
+ // cycle (but the time span over which these bytes were allocated begins at
+ // _last_sample_time, which we do not overwrite).
+ _last_sample_value = 0;
+ return rate;
}
double ShenandoahAllocationRate::sample(size_t allocated) {
@@ -898,9 +904,7 @@ bool ShenandoahAllocationRate::is_spiking(double rate, double threshold) const {
}
double ShenandoahAllocationRate::instantaneous_rate(double time, size_t allocated) const {
- size_t last_value = _last_sample_value;
- double last_time = _last_sample_time;
- size_t allocation_delta = (allocated > last_value) ? (allocated - last_value) : 0;
- double time_delta_sec = time - last_time;
- return (time_delta_sec > 0) ? (allocation_delta / time_delta_sec) : 0;
+ assert(allocated >= _last_sample_value, "Must be");
+ assert(time > _last_sample_time, "Must be");
+ return (allocated - _last_sample_value) / (time - _last_sample_time);
}
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.cpp
index 594367e2972..840459288c3 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.cpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.cpp
@@ -91,19 +91,6 @@ void ShenandoahGenerationalHeuristics::choose_collection_set_from_regiondata(She
heap->shenandoah_policy()->record_mixed_cycle();
}
- if (_generation->is_global()) {
- // We have just chosen a collection set for a global cycle. The mark bitmap covering old regions is complete, so
- // the remembered set scan can use that to avoid walking into garbage. When the next old mark begins, we will
- // use the mark bitmap to make the old regions parsable by coalescing and filling any unmarked objects. Thus,
- // we prepare for old collections by remembering which regions are old at this time. Note that any objects
- // promoted into old regions will be above TAMS, and so will be considered marked. However, free regions that
- // become old after this point will not be covered correctly by the mark bitmap, so we must be careful not to
- // coalesce those regions. Only the old regions which are not part of the collection set at this point are
- // eligible for coalescing. As implemented now, this has the side effect of possibly initiating mixed-evacuations
- // after a global cycle for old regions that were not included in this collection set.
- heap->old_generation()->transition_old_generation_after_global_gc();
- }
-
ShenandoahTracer::report_promotion_info(collection_set,
in_place_promotions.humongous_region_stats().count,
in_place_promotions.humongous_region_stats().garbage,
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp
index 3091b19b600..d2010d921b1 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp
@@ -73,7 +73,7 @@ ShenandoahHeuristics::ShenandoahHeuristics(ShenandoahSpaceInfo* space_info) :
}
ShenandoahHeuristics::~ShenandoahHeuristics() {
- FREE_C_HEAP_ARRAY(RegionGarbage, _region_data);
+ FREE_C_HEAP_ARRAY(_region_data);
}
void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collection_set) {
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp
index 0789fd5cb1c..4f820f26075 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp
@@ -428,7 +428,7 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() {
}
// else, regions that were promoted in place had 0 old live data at mark start
- if (region->is_regular() || region->is_regular_pinned()) {
+ if (region->is_regular_or_regular_pinned()) {
// Only place regular or pinned regions with live data into the candidate set.
// Pinned regions cannot be evacuated, but we are not actually choosing candidates
// for the collection set here. That happens later during the next young GC cycle,
@@ -539,7 +539,7 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() {
while ((defrag_count < bound_on_additional_regions) &&
(total_uncollected_old_regions < 7 * span_of_uncollected_regions / 8)) {
ShenandoahHeapRegion* r = candidates[_last_old_collection_candidate].get_region();
- assert(r->is_regular() || r->is_regular_pinned(), "Region %zu has wrong state for collection: %s",
+ assert(r->is_regular_or_regular_pinned(), "Region %zu has wrong state for collection: %s",
r->index(), ShenandoahHeapRegion::region_state_to_string(r->state()));
const size_t region_garbage = r->garbage();
const size_t region_free = r->free();
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.cpp b/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.cpp
index a81efa99d70..4989c929b32 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.cpp
@@ -70,15 +70,15 @@ ShenandoahAgeCensus::~ShenandoahAgeCensus() {
for (uint i = 0; i < MAX_SNAPSHOTS; i++) {
delete _global_age_tables[i];
}
- FREE_C_HEAP_ARRAY(AgeTable*, _global_age_tables);
- FREE_C_HEAP_ARRAY(uint, _tenuring_threshold);
- CENSUS_NOISE(FREE_C_HEAP_ARRAY(ShenandoahNoiseStats, _global_noise));
+ FREE_C_HEAP_ARRAY(_global_age_tables);
+ FREE_C_HEAP_ARRAY(_tenuring_threshold);
+ CENSUS_NOISE(FREE_C_HEAP_ARRAY(_global_noise));
if (_local_age_tables) {
for (uint i = 0; i < _max_workers; i++) {
delete _local_age_tables[i];
}
- FREE_C_HEAP_ARRAY(AgeTable*, _local_age_tables);
- CENSUS_NOISE(FREE_C_HEAP_ARRAY(ShenandoahNoiseStats, _local_noise));
+ FREE_C_HEAP_ARRAY(_local_age_tables);
+ CENSUS_NOISE(FREE_C_HEAP_ARRAY(_local_noise));
}
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp b/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp
index e9d6a686694..5ce131b3c80 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp
@@ -44,6 +44,16 @@ void ShenandoahArguments::initialize() {
vm_exit_during_initialization("Shenandoah GC is not supported on this platform.");
#endif
+ // Shenandoah relies on the object header bits (including the self-forwarded bit
+ // at markWord::self_fwd_mask_in_place) being preserved across monitor inflation,
+ // which only holds with UseObjectMonitorTable.
+ if (!UseObjectMonitorTable) {
+ if (FLAG_IS_CMDLINE(UseObjectMonitorTable)) {
+ vm_exit_during_initialization("Shenandoah requires UseObjectMonitorTable");
+ }
+ FLAG_SET_DEFAULT(UseObjectMonitorTable, true);
+ }
+
#if 0 // leave this block as stepping stone for future platforms
log_warning(gc)("Shenandoah GC is not fully supported on this platform:");
log_warning(gc)(" concurrent modes are not supported, only STW cycles are enabled;");
@@ -207,7 +217,8 @@ void ShenandoahArguments::initialize() {
}
size_t ShenandoahArguments::conservative_max_heap_alignment() {
- size_t align = next_power_of_2(ShenandoahMaxRegionSize);
+ static_assert(is_power_of_2(ShenandoahHeapRegion::MAX_REGION_SIZE), "Max region size must be a power of 2.");
+ size_t align = ShenandoahHeapRegion::MAX_REGION_SIZE;
if (UseLargePages) {
align = MAX2(align, os::large_page_size());
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp
index e7a0ed57740..06e16af24c6 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp
@@ -96,7 +96,6 @@ public:
void on_thread_detach(Thread* thread) override;
static inline oop resolve_forwarded_not_null(oop p);
- static inline oop resolve_forwarded_not_null_mutator(oop p);
static inline oop resolve_forwarded(oop p);
template
@@ -109,7 +108,7 @@ public:
inline oop load_reference_barrier(oop obj);
- template
+ template
inline oop load_reference_barrier_mutator(oop obj, T* load_addr);
template
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp
index f4b35e29b09..f4a6d2b9038 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp
@@ -34,7 +34,6 @@
#include "gc/shenandoah/shenandoahAsserts.hpp"
#include "gc/shenandoah/shenandoahCardTable.hpp"
#include "gc/shenandoah/shenandoahCollectionSet.inline.hpp"
-#include "gc/shenandoah/shenandoahEvacOOMHandler.inline.hpp"
#include "gc/shenandoah/shenandoahForwarding.inline.hpp"
#include "gc/shenandoah/shenandoahGeneration.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
@@ -56,20 +55,52 @@ inline oop ShenandoahBarrierSet::resolve_forwarded(oop p) {
}
}
-inline oop ShenandoahBarrierSet::resolve_forwarded_not_null_mutator(oop p) {
- return ShenandoahForwarding::get_forwardee_mutator(p);
-}
-
-template
+template
inline oop ShenandoahBarrierSet::load_reference_barrier_mutator(oop obj, T* load_addr) {
- assert(ShenandoahLoadRefBarrier, "should be enabled");
- shenandoah_assert_in_cset(load_addr, obj);
+ assert(ShenandoahLoadRefBarrier, "Should be enabled");
- oop fwd = resolve_forwarded_not_null_mutator(obj);
+ constexpr bool on_weak = HasDecorator::value;
+ constexpr bool on_phantom = HasDecorator::value;
+
+ // Handle nulls. Strong loads filtered nulls with cset checks.
+ // Weak/phantom loads need to check for nulls here.
+ if (on_weak || on_phantom) {
+ if (obj == nullptr) {
+ return nullptr;
+ }
+ } else {
+ assert(obj != nullptr, "Should have been filtered before");
+ }
+
+ // Prevent resurrection of unreachable phantom (i.e. weak-native) references.
+ if (on_phantom &&
+ _heap->is_concurrent_weak_root_in_progress() &&
+ _heap->is_in_active_generation(obj) &&
+ !_heap->marking_context()->is_marked(obj)) {
+ return nullptr;
+ }
+
+ // Prevent resurrection of unreachable weak references.
+ if (on_weak &&
+ _heap->is_concurrent_weak_root_in_progress() &&
+ _heap->is_in_active_generation(obj) &&
+ !_heap->marking_context()->is_marked_strong(obj)) {
+ return nullptr;
+ }
+
+ // Weak/phantom loads need additional cset check.
+ if (on_phantom || on_weak) {
+ if (!_heap->has_forwarded_objects() || !_heap->in_collection_set(obj)) {
+ return obj;
+ }
+ } else {
+ shenandoah_assert_in_cset(load_addr, obj);
+ }
+
+ oop fwd = ShenandoahForwarding::get_forwardee_mutator(obj);
if (obj == fwd) {
assert(_heap->is_evacuation_in_progress(), "evac should be in progress");
Thread* const t = Thread::current();
- ShenandoahEvacOOMScope scope(t);
fwd = _heap->evacuate_object(obj, t);
}
@@ -91,7 +122,6 @@ inline oop ShenandoahBarrierSet::load_reference_barrier(oop obj) {
oop fwd = resolve_forwarded_not_null(obj);
if (obj == fwd && _heap->is_evacuation_in_progress()) {
Thread* t = Thread::current();
- ShenandoahEvacOOMScope oom_evac_scope(t);
return _heap->evacuate_object(obj, t);
}
return fwd;
@@ -219,29 +249,35 @@ inline oop ShenandoahBarrierSet::oop_load(DecoratorSet decorators, T* addr) {
template
inline oop ShenandoahBarrierSet::oop_cmpxchg(DecoratorSet decorators, T* addr, oop compare_value, oop new_value) {
- oop res;
- oop expected = compare_value;
- do {
- compare_value = expected;
- res = RawAccess<>::oop_atomic_cmpxchg(addr, compare_value, new_value);
- expected = res;
- } while ((compare_value != expected) && (resolve_forwarded(compare_value) == resolve_forwarded(expected)));
+ shenandoah_assert_not_in_cset_except(nullptr, compare_value, (compare_value == nullptr || ShenandoahHeap::heap()->cancelled_gc()));
+ shenandoah_assert_not_in_cset_except(nullptr, new_value, (new_value == nullptr || ShenandoahHeap::heap()->cancelled_gc()));
- // Note: We don't need a keep-alive-barrier here. We already enqueue any loaded reference for SATB anyway,
- // because it must be the previous value.
- res = load_reference_barrier(decorators, res, static_cast(nullptr));
- satb_enqueue(res);
- return res;
+ // Handle the previous value through SATB, as we are about to perform the store.
+ oop prev = RawAccess<>::oop_load(addr);
+ satb_enqueue(prev);
+
+ // Perform LRB on location to fix it up for this and all following accesses.
+ // This guarantees there are no false negatives due to concurrent evacuation,
+ // and the value loaded later by CAS is sanitized by some LRB, or is null.
+ load_reference_barrier(decorators, prev, addr);
+
+ return RawAccess<>::oop_atomic_cmpxchg(addr, compare_value, new_value);
}
template
inline oop ShenandoahBarrierSet::oop_xchg(DecoratorSet decorators, T* addr, oop new_value) {
- oop previous = RawAccess<>::oop_atomic_xchg(addr, new_value);
- // Note: We don't need a keep-alive-barrier here. We already enqueue any loaded reference for SATB anyway,
- // because it must be the previous value.
- previous = load_reference_barrier(decorators, previous, static_cast(nullptr));
- satb_enqueue(previous);
- return previous;
+ shenandoah_assert_not_in_cset_except(nullptr, new_value, (new_value == nullptr || ShenandoahHeap::heap()->cancelled_gc()));
+
+ // Handle the previous value through SATB, as we are about to perform the store.
+ oop prev = RawAccess<>::oop_load(addr);
+ satb_enqueue(prev);
+
+ // Perform LRB on location to fix it up for this and all following accesses.
+ // This is purely opportunistic: we would not have any false negatives here.
+ // This guarantees the value loaded later by XCHG is sanitized by some LRB, or is null.
+ load_reference_barrier(decorators, prev, addr);
+
+ return RawAccess<>::oop_atomic_xchg(addr, new_value);
}
template
@@ -308,7 +344,8 @@ inline void ShenandoahBarrierSet::AccessBarrier::oop_st
template
template
inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_cmpxchg_not_in_heap(T* addr, oop compare_value, oop new_value) {
- assert((decorators & (AS_NO_KEEPALIVE | ON_UNKNOWN_OOP_REF)) == 0, "must be absent");
+ assert((decorators & AS_NO_KEEPALIVE) == 0, "CAS only with keep-alive");
+ assert((decorators & ON_STRONG_OOP_REF) != 0, "CAS only for strong refs");
ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
return bs->oop_cmpxchg(decorators, addr, compare_value, new_value);
}
@@ -316,7 +353,8 @@ inline oop ShenandoahBarrierSet::AccessBarrier::oop_ato
template
template
inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_cmpxchg_in_heap(T* addr, oop compare_value, oop new_value) {
- assert((decorators & (AS_NO_KEEPALIVE | ON_UNKNOWN_OOP_REF)) == 0, "must be absent");
+ assert((decorators & AS_NO_KEEPALIVE) == 0, "CAS only with keep-alive");
+ assert((decorators & ON_STRONG_OOP_REF) != 0, "CAS only for strong refs");
ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
oop result = bs->oop_cmpxchg(decorators, addr, compare_value, new_value);
if (ShenandoahCardBarrier) {
@@ -327,9 +365,20 @@ inline oop ShenandoahBarrierSet::AccessBarrier::oop_ato
template
inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_cmpxchg_in_heap_at(oop base, ptrdiff_t offset, oop compare_value, oop new_value) {
- assert((decorators & AS_NO_KEEPALIVE) == 0, "must be absent");
+ assert((decorators & AS_NO_KEEPALIVE) == 0, "CAS only with keep-alive");
+ assert((decorators & (ON_STRONG_OOP_REF | ON_UNKNOWN_OOP_REF)) != 0, "CAS only for strong refs OR unknown refs (Unsafe)");
ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
+
+ // Unsafe.compareAndExchange/Set come here with ON_UNKNOWN_OOP_REF set.
+ // These are normally strong refs, but one can use Unsafe on Reference.referent.
+ // We cannot deal with that case. If application does Unsafe operations on
+ // Reference.referent field, this likely breaks weak reference semantics already.
+ // We upgrade the access to strong in (sometimes futile) attempt to maintain heap
+ // integrity, and assert in debug builds for better diagnostics.
DecoratorSet resolved_decorators = AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength(base, offset);
+ assert((resolved_decorators & ON_STRONG_OOP_REF) != 0, "Application error: CAS on weak location");
+ resolved_decorators = (resolved_decorators & ~ON_DECORATOR_MASK) | ON_STRONG_OOP_REF;
+
auto addr = AccessInternal::oop_field_addr(base, offset);
oop result = bs->oop_cmpxchg(resolved_decorators, addr, compare_value, new_value);
if (ShenandoahCardBarrier) {
@@ -341,7 +390,8 @@ inline oop ShenandoahBarrierSet::AccessBarrier::oop_ato
template
template
inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_xchg_not_in_heap(T* addr, oop new_value) {
- assert((decorators & (AS_NO_KEEPALIVE | ON_UNKNOWN_OOP_REF)) == 0, "must be absent");
+ assert((decorators & AS_NO_KEEPALIVE) == 0, "XCHG only with keep-alive");
+ assert((decorators & ON_STRONG_OOP_REF) != 0, "XCHG only for strong refs");
ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
return bs->oop_xchg(decorators, addr, new_value);
}
@@ -349,7 +399,8 @@ inline oop ShenandoahBarrierSet::AccessBarrier::oop_ato
template
template
inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_xchg_in_heap(T* addr, oop new_value) {
- assert((decorators & (AS_NO_KEEPALIVE | ON_UNKNOWN_OOP_REF)) == 0, "must be absent");
+ assert((decorators & AS_NO_KEEPALIVE) == 0, "XCHG only with keep-alive");
+ assert((decorators & ON_STRONG_OOP_REF) != 0, "XCHG only for strong refs");
ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
oop result = bs->oop_xchg(decorators, addr, new_value);
if (ShenandoahCardBarrier) {
@@ -360,9 +411,20 @@ inline oop ShenandoahBarrierSet::AccessBarrier::oop_ato
template
inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_xchg_in_heap_at(oop base, ptrdiff_t offset, oop new_value) {
- assert((decorators & AS_NO_KEEPALIVE) == 0, "must be absent");
+ assert((decorators & AS_NO_KEEPALIVE) == 0, "XCHG only with keep-alive");
+ assert((decorators & (ON_STRONG_OOP_REF | ON_UNKNOWN_OOP_REF)) != 0, "XCHG only for strong refs OR unknown refs (Unsafe)");
ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
+
+ // Unsafe.getAndSet comes here with ON_UNKNOWN_OOP_REF set.
+ // These are normally strong refs, but one can use Unsafe on Reference.referent.
+ // We cannot deal with that case. If application does Unsafe operations on
+ // Reference.referent field, this likely breaks weak reference semantics already.
+ // We upgrade the access to strong in (sometimes futile) attempt to maintain heap
+ // integrity, and assert in debug builds for better diagnostics.
DecoratorSet resolved_decorators = AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength(base, offset);
+ assert((resolved_decorators & ON_STRONG_OOP_REF) != 0, "Application error: XCHG on weak location");
+ resolved_decorators = (resolved_decorators & ~ON_DECORATOR_MASK) | ON_STRONG_OOP_REF;
+
auto addr = AccessInternal::oop_field_addr(base, offset);
oop result = bs->oop_xchg(resolved_decorators, addr, new_value);
if (ShenandoahCardBarrier) {
@@ -478,7 +540,6 @@ template
void ShenandoahBarrierSet::arraycopy_evacuation(T* src, size_t count) {
assert(_heap->is_evacuation_in_progress(), "only during evacuation");
if (need_bulk_update(reinterpret_cast(src))) {
- ShenandoahEvacOOMScope oom_evac;
arraycopy_work(src, count);
}
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp
index 8b83cc6b32c..487fbd9ef62 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp
@@ -29,7 +29,6 @@
#include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
#include "gc/shenandoah/shenandoahCollectionSet.inline.hpp"
-#include "gc/shenandoah/shenandoahEvacOOMHandler.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "memory/iterator.inline.hpp"
#include "oops/access.hpp"
@@ -77,7 +76,6 @@ public:
void ShenandoahBarrierSet::clone_evacuation(oop obj) {
assert(_heap->is_evacuation_in_progress(), "only during evacuation");
if (need_bulk_update(cast_from_oop(obj))) {
- ShenandoahEvacOOMScope oom_evac_scope;
ShenandoahUpdateRefsForOopClosure cl;
obj->oop_iterate(&cl);
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCardStats.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCardStats.hpp
index 0ef0eaadb6b..c9bc94c31a8 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahCardStats.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahCardStats.hpp
@@ -88,10 +88,10 @@ public:
_local_card_stats[MAX_CLEAN_RUN].add(percent_of(_max_clean_run, _cards_in_cluster));
// Update global stats for dirty obj scan counts
- _local_card_stats[DIRTY_SCAN_OBJS].add(_dirty_scan_obj_cnt);
+ _local_card_stats[DIRTY_SCAN_OBJS].add(static_cast(_dirty_scan_obj_cnt));
// Update global stats for alternation counts
- _local_card_stats[ALTERNATIONS].add(_alternation_cnt);
+ _local_card_stats[ALTERNATIONS].add(static_cast(_alternation_cnt));
}
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp
index 96ecbad1145..7580b8d1015 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp
@@ -30,7 +30,6 @@
#include "gc/shared/barrierSetNMethod.hpp"
#include "gc/shenandoah/shenandoahAsserts.hpp"
#include "gc/shenandoah/shenandoahBarrierSet.hpp"
-#include "gc/shenandoah/shenandoahEvacOOMHandler.inline.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahMark.inline.hpp"
#include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
@@ -132,22 +131,12 @@ void ShenandoahKeepAliveClosure::do_oop_work(T* p) {
template
void ShenandoahEvacuateUpdateRootClosureBase::do_oop(oop* p) {
- if (CONCURRENT) {
- ShenandoahEvacOOMScope scope;
- do_oop_work(p);
- } else {
- do_oop_work(p);
- }
+ do_oop_work(p);
}
template
void ShenandoahEvacuateUpdateRootClosureBase::do_oop(narrowOop* p) {
- if (CONCURRENT) {
- ShenandoahEvacOOMScope scope;
- do_oop_work(p);
- } else {
- do_oop_work(p);
- }
+ do_oop_work(p);
}
template
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp
index 7cf60cdf65c..3116ec30665 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp
@@ -27,7 +27,6 @@
#include "code/nmethod.hpp"
#include "gc/shared/classUnloadingContext.hpp"
#include "gc/shenandoah/shenandoahClosures.inline.hpp"
-#include "gc/shenandoah/shenandoahEvacOOMHandler.inline.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahNMethod.inline.hpp"
#include "gc/shenandoah/shenandoahUtils.hpp"
@@ -117,7 +116,6 @@ public:
// Heal oops
if (_bs->is_armed(nm)) {
- ShenandoahEvacOOMScope oom_evac_scope;
ShenandoahNMethod::heal_nmethod_metadata(nm_data);
// Must remain armed to complete remaining work in nmethod entry barrier
assert(_bs->is_armed(nm), "Should remain armed");
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp
index 6723bb89021..929f3b30afe 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp
@@ -854,8 +854,6 @@ public:
}
void work(uint worker_id) override {
- // ShenandoahEvacOOMScope has to be setup by ShenandoahContextEvacuateUpdateRootsClosure.
- // Otherwise, may deadlock with watermark lock
ShenandoahContextEvacuateUpdateRootsClosure oops_cl;
ShenandoahConcurrentEvacThreadClosure thr_cl(&oops_cl);
_java_threads.threads_do(&thr_cl, worker_id);
@@ -969,9 +967,8 @@ public:
void work(uint worker_id) override {
ShenandoahConcurrentWorkerSession worker_session(worker_id);
- ShenandoahSuspendibleThreadSetJoiner sts_join;
+ SuspendibleThreadSetJoiner sts_join;
{
- ShenandoahEvacOOMScope oom;
// jni_roots and weak_roots are OopStorage backed roots, concurrent iteration
// may race against OopStorage::release() calls.
ShenandoahEvacUpdateCleanupOopStorageRootsClosure cl(_generation);
@@ -1044,9 +1041,6 @@ public:
void do_nmethod(nmethod* n) {
ShenandoahNMethod* data = ShenandoahNMethod::gc_data(n);
ShenandoahNMethodLocker locker(data->lock());
- // Setup EvacOOM scope below reentrant lock to avoid deadlock with
- // nmethod_entry_barrier
- ShenandoahEvacOOMScope oom;
data->oops_do(&_cl, /* fix_relocations = */ true);
ShenandoahNMethod::disarm_nmethod(n);
}
@@ -1071,7 +1065,6 @@ public:
void work(uint worker_id) {
ShenandoahConcurrentWorkerSession worker_session(worker_id);
{
- ShenandoahEvacOOMScope oom;
{
// vm_roots and weak_roots are OopStorage backed roots, concurrent iteration
// may race against OopStorage::release() calls.
@@ -1086,7 +1079,6 @@ public:
}
}
- // Cannot setup ShenandoahEvacOOMScope here, due to potential deadlock with nmethod_entry_barrier.
if (!ShenandoahHeap::heap()->unload_classes()) {
ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCacheRoots, worker_id);
ShenandoahEvacUpdateCodeCacheClosure cl;
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp
index 367a15abfa4..be0da3e54ba 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp
@@ -58,7 +58,7 @@ public:
void work(uint worker_id) {
ShenandoahConcurrentWorkerSession worker_session(worker_id);
ShenandoahWorkerTimingsTracker timer(ShenandoahPhaseTimings::conc_mark, ShenandoahPhaseTimings::ParallelMark, worker_id, true);
- ShenandoahSuspendibleThreadSetJoiner stsj;
+ SuspendibleThreadSetJoiner stsj;
StringDedup::Requests requests;
_cm->mark_loop(worker_id, _terminator, GENERATION, true /*cancellable*/,
ShenandoahStringDedup::is_enabled() ? ENQUEUE_DEDUP : NO_DEDUP,
@@ -300,7 +300,11 @@ void ShenandoahConcurrentMark::finish_mark_work() {
default:
ShouldNotReachHere();
}
-
+ if (!generation()->is_old() && heap->is_concurrent_young_mark_in_progress()) {
+ // Lastly, ensure all the invisible roots are marked.
+ ShenandoahInvisibleRootsMarkClosure cl;
+ Threads::java_threads_do(&cl);
+ }
assert(task_queues()->is_empty(), "Should be empty");
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp
index 84b22f13d47..7be3141a4fa 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp
@@ -56,9 +56,10 @@ bool ShenandoahDegenGC::collect(GCCause::Cause cause) {
ShenandoahHeap* heap = ShenandoahHeap::heap();
if (heap->mode()->is_generational()) {
bool is_bootstrap_gc = heap->young_generation()->is_bootstrap_cycle();
- heap->mmu_tracker()->record_degenerated(GCId::current(), is_bootstrap_gc);
- const char* msg = is_bootstrap_gc? "At end of Degenerated Bootstrap Old GC": "At end of Degenerated Young GC";
- heap->log_heap_status(msg);
+ FormatBuffer<32> buf("Degenerated %s GC", _generation->name());
+ const char* msg = is_bootstrap_gc ? "Degenerated Bootstrap Old GC" : buf.buffer();
+ heap->mmu_tracker()->record_degenerated(GCId::current(), msg);
+ heap->log_heap_status(FormatBuffer<64>("At end of %s", msg));
}
return true;
}
@@ -95,6 +96,16 @@ void ShenandoahDegenGC::op_degenerated() {
// some phase, we have to upgrade the Degenerate GC to Full GC.
heap->clear_cancelled_gc();
+ // If we degenerated from evacuation or update-refs, some objects in cset may
+ // have been self-forwarded by the failing thread. Clear those marks now so
+ // the remainder of this cycle (re-evac, update-refs, verification) sees a
+ // clean forwarding state.
+ if (_degen_point == ShenandoahDegenPoint::_degenerated_evac ||
+ _degen_point == ShenandoahDegenPoint::_degenerated_update_refs) {
+ ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc_un_self_forward);
+ heap->un_self_forward_cset_regions();
+ }
+
// If it's passive mode with ShenandoahCardBarrier turned on: clean the write table
// without swapping the tables since no scan happens in passive mode anyway
if (ShenandoahCardBarrier && !heap->mode()->is_generational()) {
@@ -304,6 +315,8 @@ void ShenandoahDegenGC::op_degenerated() {
ShouldNotReachHere();
}
+ DEBUG_ONLY(heap->assert_no_self_forwards());
+
if (ShenandoahVerify) {
heap->verifier()->verify_after_degenerated(_generation);
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.cpp b/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.cpp
deleted file mode 100644
index 5b24140ac1c..00000000000
--- a/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.cpp
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- * Copyright (c) 2018, 2020, Red Hat, Inc. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-
-#include "gc/shenandoah/shenandoahEvacOOMHandler.inline.hpp"
-#include "gc/shenandoah/shenandoahUtils.hpp"
-#include "runtime/javaThread.hpp"
-#include "runtime/os.hpp"
-
-const jint ShenandoahEvacOOMCounter::OOM_MARKER_MASK = 0x80000000;
-
-ShenandoahEvacOOMCounter::ShenandoahEvacOOMCounter() :
- _bits(0) {
-}
-
-void ShenandoahEvacOOMCounter::decrement() {
- assert(unmasked_count() > 0, "sanity");
- // NOTE: It's ok to simply decrement, even with mask set, because unmasked value is positive.
- _bits.fetch_then_sub(1);
-}
-
-void ShenandoahEvacOOMCounter::clear() {
- assert(unmasked_count() == 0, "sanity");
- _bits.release_store_fence((jint)0);
-}
-
-void ShenandoahEvacOOMCounter::set_oom_bit(bool decrement) {
- jint threads_in_evac = _bits.load_acquire();
- while (true) {
- jint newval = decrement
- ? (threads_in_evac - 1) | OOM_MARKER_MASK
- : threads_in_evac | OOM_MARKER_MASK;
-
- jint other = _bits.compare_exchange(threads_in_evac, newval);
- if (other == threads_in_evac) {
- // Success: wait for other threads to get out of the protocol and return.
- break;
- } else {
- // Failure: try again with updated new value.
- threads_in_evac = other;
- }
- }
-}
-
-bool ShenandoahEvacOOMCounter::try_increment()
-{
- jint threads_in_evac = _bits.load_acquire();
-
- while (true) {
- // Cannot enter evacuation if OOM_MARKER_MASK is set.
- if ((threads_in_evac & OOM_MARKER_MASK) != 0) {
- return false;
- }
-
- jint other = _bits.compare_exchange(threads_in_evac, threads_in_evac + 1);
- if (other == threads_in_evac) {
- // Success: caller may safely enter evacuation
- return true;
- } else {
- threads_in_evac = other;
- }
- }
-}
-
-ShenandoahEvacOOMHandler::ShenandoahEvacOOMHandler() :
- _num_counters(calc_num_counters()) {
-
- assert(_num_counters > 0, "sanity");
- assert(is_power_of_2(_num_counters), "must be");
-
- _threads_in_evac = NEW_C_HEAP_ARRAY(ShenandoahEvacOOMCounter, _num_counters, mtGC);
- for (int i = 0; i < _num_counters; i++) {
- new (&_threads_in_evac[i]) ShenandoahEvacOOMCounter();
- }
-}
-
-int ShenandoahEvacOOMHandler::calc_num_counters() {
- // Scale the number of counter buckets with the number of CPUs to
- // minimise contention. Also make sure the number is a power of two
- // so we can map hash values to buckets with a simple mask.
- const int nproc = os::active_processor_count();
- const int clamped = MAX2(1, MIN2(nproc, 128));
- return round_up_power_of_2(clamped);
-}
-
-uint64_t ShenandoahEvacOOMHandler::hash_pointer(const void* p) {
- // Bit mixing function from MurmurHash3
- uint64_t key = (uintptr_t)p;
- key ^= (key >> 33);
- key *= UINT64_C(0xff51afd7ed558ccd);
- key ^= (key >> 33);
- key *= UINT64_C(0xc4ceb9fe1a85ec53);
- key ^= (key >> 33);
- return key;
-}
-
-ShenandoahEvacOOMCounter* ShenandoahEvacOOMHandler::counter_for_thread(Thread* t) {
- const uint64_t key = hash_pointer(t);
- return &_threads_in_evac[key & (_num_counters - 1)];
-}
-
-void ShenandoahEvacOOMHandler::wait_for_one_counter(ShenandoahEvacOOMCounter* ptr) {
- // We might be racing against handle_out_of_memory_during_evacuation()
- // setting the OOM_MARKER_MASK bit so we must make sure it is set here
- // *and* the counter is zero.
- while (ptr->load_acquire() != ShenandoahEvacOOMCounter::OOM_MARKER_MASK) {
- os::naked_short_sleep(1);
- }
-}
-
-void ShenandoahEvacOOMHandler::wait_for_no_evac_threads() {
- // Once the OOM_MARKER_MASK bit is set the counter can only decrease
- // so it's safe to check each bucket in turn.
- for (int i = 0; i < _num_counters; i++) {
- wait_for_one_counter(&_threads_in_evac[i]);
- }
- // At this point we are sure that no threads can evacuate anything. Raise
- // the thread-local oom_during_evac flag to indicate that any attempt
- // to evacuate should simply return the forwarding pointer instead (which is safe now).
- ShenandoahThreadLocalData::set_oom_during_evac(Thread::current(), true);
-}
-
-void ShenandoahEvacOOMHandler::register_thread(Thread* thr) {
- assert(!ShenandoahThreadLocalData::is_oom_during_evac(Thread::current()), "TL oom-during-evac must not be set");
-
- ShenandoahEvacOOMCounter* counter = counter_for_thread(thr);
- if (!counter->try_increment()) {
- // Counter has OOM_MARKER_MASK set, loop until no more threads in evac
- wait_for_no_evac_threads();
- }
-}
-
-void ShenandoahEvacOOMHandler::unregister_thread(Thread* thr) {
- if (!ShenandoahThreadLocalData::is_oom_during_evac(thr)) {
- counter_for_thread(thr)->decrement();
- } else {
- // If we get here, the current thread has already gone through the
- // OOM-during-evac protocol and has thus either never entered or successfully left
- // the evacuation region. Simply flip its TL oom-during-evac flag back off.
- ShenandoahThreadLocalData::set_oom_during_evac(thr, false);
- }
- assert(!ShenandoahThreadLocalData::is_oom_during_evac(thr), "TL oom-during-evac must be turned off");
-}
-
-void ShenandoahEvacOOMHandler::handle_out_of_memory_during_evacuation() {
- assert(ShenandoahThreadLocalData::is_evac_allowed(Thread::current()), "sanity");
- assert(!ShenandoahThreadLocalData::is_oom_during_evac(Thread::current()), "TL oom-during-evac must not be set");
-
- ShenandoahEvacOOMCounter* self = counter_for_thread(Thread::current());
- assert(self->unmasked_count() > 0, "sanity");
-
- for (int i = 0; i < _num_counters; i++) {
- ShenandoahEvacOOMCounter* counter = &_threads_in_evac[i];
- counter->set_oom_bit(counter == self);
- }
-
- wait_for_no_evac_threads();
-}
-
-void ShenandoahEvacOOMHandler::clear() {
- assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at a safepoint");
- for (int i = 0; i < _num_counters; i++) {
- _threads_in_evac[i].clear();
- }
-}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.hpp b/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.hpp
deleted file mode 100644
index 3e28d9ac88e..00000000000
--- a/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.hpp
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Copyright (c) 2018, 2020, Red Hat, Inc. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHEVACOOMHANDLER_HPP
-#define SHARE_GC_SHENANDOAH_SHENANDOAHEVACOOMHANDLER_HPP
-
-#include "gc/shenandoah/shenandoahPadding.hpp"
-#include "memory/allocation.hpp"
-#include "runtime/atomic.hpp"
-#include "runtime/javaThread.hpp"
-#include "utilities/globalDefinitions.hpp"
-
-/**
- * Striped counter used to implement the OOM protocol described below.
- */
-class ShenandoahEvacOOMCounter {
-private:
- // Combination of a 31-bit counter and 1-bit OOM marker.
- Atomic _bits;
-
- // This class must be at least a cache line in size to prevent false sharing.
- shenandoah_padding_minus_size(0, sizeof(jint));
-
-public:
- static const jint OOM_MARKER_MASK;
-
- ShenandoahEvacOOMCounter();
-
- void decrement();
- bool try_increment();
- void clear();
- void set_oom_bit(bool decrement);
-
- inline jint unmasked_count();
- inline jint load_acquire();
-};
-
-/**
- * Provides safe handling of out-of-memory situations during evacuation.
- *
- * When a Java thread encounters out-of-memory while evacuating an object in a
- * load-reference-barrier (i.e. it cannot copy the object to to-space), it does not
- * necessarily follow we can return immediately from the LRB (and store to from-space).
- *
- * In very basic case, on such failure we may wait until the evacuation is over,
- * and then resolve the forwarded copy, and to the store there. This is possible
- * because other threads might still have space in their GCLABs, and successfully
- * evacuate the object.
- *
- * But, there is a race due to non-atomic evac_in_progress transition. Consider
- * thread A is stuck waiting for the evacuation to be over -- it cannot leave with
- * from-space copy yet. Control thread drops evacuation_in_progress preparing for
- * next STW phase that has to recover from OOME. Thread B misses that update, and
- * successfully evacuates the object, does the write to to-copy. But, before
- * Thread B is able to install the fwdptr, thread A discovers evac_in_progress is
- * down, exits from here, reads the fwdptr, discovers old from-copy, and stores there.
- * Thread B then wakes up and installs to-copy. This breaks to-space invariant, and
- * silently corrupts the heap: we accepted two writes to separate copies of the object.
- *
- * The way it is solved here is to maintain a counter of threads inside the
- * 'evacuation path'. The 'evacuation path' is the part of evacuation that does the actual
- * allocation, copying and CASing of the copy object, and is protected by this
- * OOM-during-evac-handler. The handler allows multiple threads to enter and exit
- * evacuation path, but on OOME it requires all threads that experienced OOME to wait
- * for current threads to leave, and blocks other threads from entering. The counter state
- * is striped across multiple cache lines to reduce contention when many threads attempt
- * to enter or leave the protocol at the same time.
- *
- * Detailed state change:
- *
- * Upon entry of the evac-path, entering thread will attempt to increase the counter,
- * using a CAS. Depending on the result of the CAS:
- * - success: carry on with evac
- * - failure:
- * - if offending value is a valid counter, then try again
- * - if offending value is OOM-during-evac special value: loop until
- * counter drops to 0, then exit with resolving the ptr
- *
- * Upon exit, exiting thread will decrease the counter using atomic dec.
- *
- * Upon OOM-during-evac, any thread will attempt to CAS OOM-during-evac
- * special value into the counter. Depending on result:
- * - success: busy-loop until counter drops to zero, then exit with resolve
- * - failure:
- * - offender is valid counter update: try again
- * - offender is OOM-during-evac: busy loop until counter drops to
- * zero, then exit with resolve
- */
-class ShenandoahEvacOOMHandler {
-private:
- const int _num_counters;
-
- shenandoah_padding(0);
- ShenandoahEvacOOMCounter* _threads_in_evac;
-
- ShenandoahEvacOOMCounter* counter_for_thread(Thread* t);
-
- void wait_for_no_evac_threads();
- void wait_for_one_counter(ShenandoahEvacOOMCounter* ptr);
-
- static uint64_t hash_pointer(const void* p);
- static int calc_num_counters();
-public:
- ShenandoahEvacOOMHandler();
-
- /**
- * Attempt to enter the protected evacuation path.
- *
- * When this returns true, it is safe to continue with normal evacuation.
- * When this method returns false, evacuation must not be entered, and caller
- * may safely continue with a simple resolve (if Java thread).
- */
- inline void enter_evacuation(Thread* t);
-
- /**
- * Leave evacuation path.
- */
- inline void leave_evacuation(Thread* t);
-
- /**
- * Signal out-of-memory during evacuation. It will prevent any other threads
- * from entering the evacuation path, then wait until all threads have left the
- * evacuation path, and then return. It is then safe to continue with a simple resolve.
- */
- void handle_out_of_memory_during_evacuation();
-
- void clear();
-
-private:
- // Register/Unregister thread to evacuation OOM protocol
- void register_thread(Thread* t);
- void unregister_thread(Thread* t);
-};
-
-class ShenandoahEvacOOMScope : public StackObj {
-private:
- Thread* const _thread;
-
-public:
- inline ShenandoahEvacOOMScope();
- inline ShenandoahEvacOOMScope(Thread* t);
- inline ~ShenandoahEvacOOMScope();
-};
-
-#endif // SHARE_GC_SHENANDOAH_SHENANDOAHEVACOOMHANDLER_HPP
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.inline.hpp
deleted file mode 100644
index 4bd4381068a..00000000000
--- a/src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.inline.hpp
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Copyright (c) 2020, Red Hat, Inc. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHEVACOOMHANDLER_INLINE_HPP
-#define SHARE_GC_SHENANDOAH_SHENANDOAHEVACOOMHANDLER_INLINE_HPP
-
-#include "gc/shenandoah/shenandoahEvacOOMHandler.hpp"
-
-#include "gc/shenandoah/shenandoahHeap.inline.hpp"
-#include "gc/shenandoah/shenandoahThreadLocalData.hpp"
-
-jint ShenandoahEvacOOMCounter::load_acquire() {
- return _bits.load_acquire();
-}
-
-jint ShenandoahEvacOOMCounter::unmasked_count() {
- return _bits.load_acquire() & ~OOM_MARKER_MASK;
-}
-
-void ShenandoahEvacOOMHandler::enter_evacuation(Thread* thr) {
- uint8_t level = ShenandoahThreadLocalData::push_evac_oom_scope(thr);
- if (level == 0) {
- // Entering top level scope, register this thread.
- register_thread(thr);
- } else if (!ShenandoahThreadLocalData::is_oom_during_evac(thr)) {
- ShenandoahEvacOOMCounter* counter = counter_for_thread(thr);
- jint threads_in_evac = counter->load_acquire();
- // If OOM is in progress, handle it.
- if ((threads_in_evac & ShenandoahEvacOOMCounter::OOM_MARKER_MASK) != 0) {
- counter->decrement();
- wait_for_no_evac_threads();
- }
- }
-}
-
-void ShenandoahEvacOOMHandler::leave_evacuation(Thread* thr) {
- uint8_t level = ShenandoahThreadLocalData::pop_evac_oom_scope(thr);
- // Not top level, just return
- if (level > 1) {
- return;
- }
-
- // Leaving top level scope, unregister this thread.
- unregister_thread(thr);
-}
-
-ShenandoahEvacOOMScope::ShenandoahEvacOOMScope() :
- _thread(Thread::current()) {
- ShenandoahHeap::heap()->enter_evacuation(_thread);
-}
-
-ShenandoahEvacOOMScope::ShenandoahEvacOOMScope(Thread* t) :
- _thread(t) {
- ShenandoahHeap::heap()->enter_evacuation(_thread);
-}
-
-ShenandoahEvacOOMScope::~ShenandoahEvacOOMScope() {
- ShenandoahHeap::heap()->leave_evacuation(_thread);
-}
-
-#endif // SHARE_GC_SHENANDOAH_SHENANDOAHEVACOOMHANDLER_INLINE_HPP
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahForwarding.hpp b/src/hotspot/share/gc/shenandoah/shenandoahForwarding.hpp
index fb57c55e09a..6f2f124f6b1 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahForwarding.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahForwarding.hpp
@@ -25,43 +25,68 @@
#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHFORWARDING_HPP
#define SHARE_GC_SHENANDOAH_SHENANDOAHFORWARDING_HPP
+#include "oops/markWord.hpp"
#include "oops/oop.hpp"
#include "utilities/globalDefinitions.hpp"
class ShenandoahForwarding {
public:
- /* Gets forwardee from the given object.
+ /* Gets forwardee from the given object. For a self-forwarded object
+ * (evacuation failure), returns the object itself.
*/
static inline oop get_forwardee(oop obj);
/* Gets forwardee from the given object. Only from mutator thread.
+ * For a self-forwarded object, returns the object itself.
*/
static inline oop get_forwardee_mutator(oop obj);
- /* Returns the raw value from forwardee slot.
+ /* Returns the raw value from forwardee slot. For a self-forwarded
+ * object, returns the object itself.
*/
static inline oop get_forwardee_raw(oop obj);
/* Returns the raw value from forwardee slot without any checks.
- * Used for quick verification.
+ * Used for quick verification. For a self-forwarded object,
+ * returns the object itself.
*/
static inline oop get_forwardee_raw_unchecked(oop obj);
/**
- * Returns true if the object is forwarded, false otherwise.
+ * Returns true if the object is forwarded (including self-forwarded),
+ * false otherwise.
*/
static inline bool is_forwarded(oop obj);
+ /**
+ * Returns true iff obj has been self-forwarded (i.e. evacuation has
+ * failed for this object in the current cycle).
+ */
+ static inline bool is_self_forwarded(oop obj);
+
/* Tries to atomically update forwardee in $holder object to $update.
* Assumes $holder points at itself.
* Asserts $holder is in from-space.
* Asserts $update is in to-space.
*
* Returns the new object 'update' upon success, or
- * the new forwardee that a competing thread installed.
+ * the new forwardee that a competing thread installed. If another
+ * thread self-forwarded the object, returns the object itself.
*/
static inline oop try_update_forwardee(oop obj, oop update);
+ /* Tries to atomically self-forward obj. Used by the evacuation path
+ * when the copy allocation fails: the failing thread installs the
+ * self-forwarded bit so other threads see the object as "already
+ * handled" and return it unchanged.
+ *
+ * Returns nullptr on success (we installed the self-forward), or
+ * the winning forwardee when another thread raced ahead (either a
+ * real forwardee pointing at a copy, or obj itself if the winner
+ * also self-forwarded).
+ */
+ static inline oop try_forward_to_self(oop obj, markWord old_mark);
+
static inline size_t size(oop obj);
static inline Klass* klass(oop obj);
};
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahForwarding.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahForwarding.inline.hpp
index ccdbb81f33b..6bb58920eb9 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahForwarding.inline.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahForwarding.inline.hpp
@@ -48,6 +48,8 @@ inline oop ShenandoahForwarding::get_forwardee_raw_unchecked(oop obj) {
return cast_to_oop(fwdptr);
}
}
+ // Self-forwarded (evacuation failure): the object stays put; the
+ // self-fwd bit is set alongside normal lock bits.
return obj;
}
@@ -61,9 +63,9 @@ inline oop ShenandoahForwarding::get_forwardee_mutator(oop obj) {
HeapWord* fwdptr = (HeapWord*) mark.clear_lock_bits().to_pointer();
assert(fwdptr != nullptr, "Forwarding pointer is never null here");
return cast_to_oop(fwdptr);
- } else {
- return obj;
}
+ // Self-forwarded or not forwarded: return the object itself.
+ return obj;
}
inline oop ShenandoahForwarding::get_forwardee(oop obj) {
@@ -72,7 +74,11 @@ inline oop ShenandoahForwarding::get_forwardee(oop obj) {
}
inline bool ShenandoahForwarding::is_forwarded(oop obj) {
- return obj->mark().is_marked();
+ return obj->mark().is_forwarded();
+}
+
+inline bool ShenandoahForwarding::is_self_forwarded(oop obj) {
+ return obj->mark().is_self_forwarded();
}
inline oop ShenandoahForwarding::try_update_forwardee(oop obj, oop update) {
@@ -80,14 +86,50 @@ inline oop ShenandoahForwarding::try_update_forwardee(oop obj, oop update) {
if (old_mark.is_marked()) {
return cast_to_oop(old_mark.clear_lock_bits().to_pointer());
}
+ if (old_mark.is_self_forwarded()) {
+ // Another thread lost the evacuation race; the object stays put.
+ return obj;
+ }
markWord new_mark = markWord::encode_pointer_as_mark(update);
markWord prev_mark = obj->cas_set_mark(new_mark, old_mark, memory_order_conservative);
if (prev_mark == old_mark) {
return update;
- } else {
+ }
+ // Concurrent writers on a cset object's mark can only be other evacuation
+ // threads installing forwarding (real or self). Mutators cannot reach the
+ // mark of a not-yet-forwarded cset object: LRB + stack watermark barriers
+ // redirect all reference uses before a Java-level operation can touch it.
+ // So the only possible failure modes are a regular forwardee (marked) or
+ // a self-forward (possibly with mutator lock/hash mods layered on top
+ // after the self-forward became visible).
+ if (prev_mark.is_marked()) {
return cast_to_oop(prev_mark.clear_lock_bits().to_pointer());
}
+ assert(prev_mark.is_self_forwarded(),
+ "concurrent writers on cset objects must install forwarding: prev=" INTPTR_FORMAT,
+ prev_mark.value());
+ return obj;
+}
+
+inline oop ShenandoahForwarding::try_forward_to_self(oop obj, markWord old_mark) {
+ assert(!old_mark.is_forwarded(),
+ "caller must pass a non-forwarded mark: old=" INTPTR_FORMAT, old_mark.value());
+ markWord new_mark = old_mark.set_self_forwarded();
+ markWord prev_mark = obj->cas_set_mark(new_mark, old_mark, memory_order_conservative);
+ if (prev_mark == old_mark) {
+ // We installed the self-forward.
+ return nullptr;
+ }
+ // Same invariant as in try_update_forwardee: the only races on a
+ // cset object's mark come from other evac threads installing forwarding.
+ if (prev_mark.is_marked()) {
+ return cast_to_oop(prev_mark.clear_lock_bits().to_pointer());
+ }
+ assert(prev_mark.is_self_forwarded(),
+ "concurrent writers on cset objects must install forwarding: prev=" INTPTR_FORMAT,
+ prev_mark.value());
+ return obj;
}
inline Klass* ShenandoahForwarding::klass(oop obj) {
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp
index 1807383123b..fdce385e0f6 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp
@@ -223,7 +223,9 @@ ShenandoahFreeSetPartitionId ShenandoahFreeSet::prepare_to_promote_in_place(size
}
inline bool ShenandoahFreeSet::can_allocate_from(ShenandoahHeapRegion *r) const {
- return r->is_empty() || (r->is_trash() && !_heap->is_concurrent_weak_root_in_progress());
+ const auto state = r->state();
+ return ShenandoahHeapRegion::is_empty_state(state)
+ || (ShenandoahHeapRegion::is_trash(state) && !_heap->is_concurrent_weak_root_in_progress());
}
inline bool ShenandoahFreeSet::can_allocate_from(size_t idx) const {
@@ -666,7 +668,7 @@ void ShenandoahRegionPartitions::retire_range_from_partition(
#ifdef ASSERT
ShenandoahHeapRegion* r = ShenandoahHeap::heap()->get_region(idx);
assert (in_free_set(partition, idx), "Must be in partition to remove from partition");
- assert(r->is_empty() || r->is_trash(), "Region must be empty or trash");
+ assert(r->is_empty_or_trash(), "Region must be empty or trash");
#endif
_membership[int(partition)].clear_bit(idx);
}
@@ -2822,7 +2824,7 @@ size_t ShenandoahFreeSet::reserve_regions(size_t to_reserve, size_t to_reserve_o
// be collected in the near future.
if (r->is_trash() || !r->is_affiliated()) {
// OLD regions that have available memory are already in the old_collector free set.
- assert(r->is_empty() || r->is_trash(), "Not affiliated implies region %zu is empty", r->index());
+ assert(r->is_empty_or_trash(), "Not affiliated implies region %zu is empty", r->index());
if (idx < old_collector_low_idx) {
old_collector_low_idx = idx;
}
@@ -3166,7 +3168,7 @@ void ShenandoahFreeSet::log_status() {
size_t free = alloc_capacity(r);
max = MAX2(max, free);
size_t used_in_region = r->used();
- if (r->is_empty() || r->is_trash()) {
+ if (r->is_empty_or_trash()) {
used_in_region = 0;
total_free_ext += free;
if (last_idx + 1 == idx) {
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp
index eeff0fde87c..f7ba1f05f47 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp
@@ -680,32 +680,18 @@ public:
}
inline size_t get_bytes_allocated_since_previous_sample() {
- size_t total_bytes = get_total_bytes_allocated();
- size_t result;
- if (total_bytes < _mutator_bytes_at_last_sample) {
- // This rare condition may occur if bytes allocated overflows (wraps around) size_t tally of allocations.
- // This may also occur in the very rare situation that get_total_bytes_allocated() is queried in the middle of
- // reset_bytes_allocated_since_gc_start(). Note that there is no lock to assure that the two global variables
- // it modifies are modified atomically (_total_bytes_previously_allocated and _mutator_byts_allocated_since_gc_start)
- // This has been observed to occur when an out-of-cycle degenerated cycle is starting (and thus calls
- // reset_bytes_allocated_since_gc_start()) at the same time that the control (non-generational mode) or
- // regulator (generational-mode) thread calls should_start_gc() (which invokes get_bytes_allocated_since_previous_sample()).
- //
- // Handle this rare situation by responding with the "innocent" value 0 and resetting internal state so that the
- // the next query can recalibrate.
- result = 0;
- } else {
- // Note: there's always the possibility that the tally of total allocations exceeds the 64-bit capacity of our size_t
- // counter. We assume that the difference between relevant samples does not exceed this count. Example:
- // Suppose _mutator_words_at_last_sample is 0xffff_ffff_ffff_fff0 (18,446,744,073,709,551,600 Decimal)
- // and _total_words is 0x0000_0000_0000_0800 ( 32,768 Decimal)
- // Then, total_words - _mutator_words_at_last_sample can be done adding 1's complement of subtrahend:
- // 1's complement of _mutator_words_at_last_sample is: 0x0000_0000_0000_0010 ( 16 Decimal))
- // plus total_words: 0x0000_0000_0000_0800 (32,768 Decimal)
- // sum: 0x0000_0000_0000_0810 (32,784 Decimal)
- result = total_bytes - _mutator_bytes_at_last_sample;
- }
- _mutator_bytes_at_last_sample = total_bytes;
+ const size_t total_bytes_allocated = get_total_bytes_allocated();
+ // total_bytes_allocated could overflow (wraps around) size_t in rare condition, we are relying on
+ // wrap-around arithmetic of size_t type to produce meaningful result when total_bytes_allocated overflows
+ // its 64-bit counter. The expression below is equivalent to code:
+ // if (total_bytes < _mutator_bytes_at_last_sample) {
+ // // overflow
+ // return total_bytes + (SIZE_T_MAX - _mutator_bytes_at_last_sample) + 1;
+ // } else {
+ // return total_bytes - _mutator_bytes_at_last_sample;
+ // }
+ const size_t result = total_bytes_allocated - _mutator_bytes_at_last_sample;
+ _mutator_bytes_at_last_sample = total_bytes_allocated;
return result;
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp
index 21b1fd9e0a8..365226a054c 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp
@@ -137,6 +137,15 @@ void ShenandoahFullGC::op_full(GCCause::Cause cause) {
void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) {
ShenandoahHeap* heap = ShenandoahHeap::heap();
+ // A full GC may be entered directly, or as an upgrade from a failed
+ // degenerated GC. In the latter case, self-forwarded objects may be
+ // present from the failed evacuation. Drain those marks before any phase
+ // (verify, update_roots, phase1_mark_heap) walks headers.
+ {
+ ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_un_self_forward);
+ heap->un_self_forward_cset_regions();
+ }
+
if (heap->mode()->is_generational()) {
ShenandoahGenerationalFullGC::prepare();
}
@@ -261,11 +270,13 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) {
for (uint i = 0; i < heap->max_workers(); i++) {
delete worker_slices[i];
}
- FREE_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, worker_slices);
+ FREE_C_HEAP_ARRAY(worker_slices);
heap->set_full_gc_move_in_progress(false);
heap->set_full_gc_in_progress(false);
+ DEBUG_ONLY(heap->assert_no_self_forwards());
+
if (ShenandoahVerify) {
heap->verifier()->verify_after_fullgc(_generation);
}
@@ -688,7 +699,7 @@ void ShenandoahFullGC::distribute_slices(ShenandoahHeapRegionSet** worker_slices
}
}
- FREE_C_HEAP_ARRAY(size_t, live);
+ FREE_C_HEAP_ARRAY(live);
#ifdef ASSERT
ResourceBitMap map(n_regions);
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp
index 5b26ee67653..493736f4194 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp
@@ -284,15 +284,6 @@ void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) {
// along with the census done during marking, and compute the tenuring threshold.
ShenandoahAgeCensus* census = ShenandoahGenerationalHeap::heap()->age_census();
census->update_census(age0_pop);
-#ifndef PRODUCT
- size_t total_pop = age0_cl.get_total_population();
- size_t total_census = census->get_total();
- // Usually total_pop > total_census, but not by too much.
- // We use integer division so anything up to just less than 2 is considered
- // reasonable, and the "+1" is to avoid divide-by-zero.
- assert((total_pop+1)/(total_census+1) == 1, "Extreme divergence: "
- "%zu/%zu", total_pop, total_census);
-#endif
}
{
@@ -303,8 +294,20 @@ void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) {
ShenandoahHeapLocker locker(heap->lock());
heap->assert_pinned_region_status(this);
_heuristics->choose_collection_set(collection_set);
- }
+ if (is_generational && is_global()) {
+ // We have finished marking the entire heap. The mark bitmap covering old regions is complete, so
+ // the remembered set scan can use that to avoid walking into garbage. When the next old mark begins, we will
+ // use the mark bitmap to make the old regions parsable by coalescing and filling any unmarked objects. Thus,
+ // we prepare for old collections by remembering which regions are old at this time. Note that any objects
+ // promoted into old regions will be above TAMS, and so will be considered marked. However, free regions that
+ // become old after this point will not be covered correctly by the mark bitmap, so we must be careful not to
+ // coalesce those regions. Only the old regions which are not part of the collection set at this point are
+ // eligible for coalescing. As implemented now, this has the side effect of possibly initiating mixed-evacuations
+ // after a global cycle for old regions that were not included in this collection set.
+ heap->old_generation()->transition_old_generation_after_global_gc();
+ }
+ }
{
ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_rebuild_freeset :
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalControlThread.cpp
index bbad82de1dc..bc2028d077d 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalControlThread.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalControlThread.cpp
@@ -580,7 +580,7 @@ void ShenandoahGenerationalControlThread::service_concurrent_cycle(ShenandoahGen
assert(generation->is_global(), "If not young, must be GLOBAL");
assert(!do_old_gc_bootstrap, "Do not bootstrap with GLOBAL GC");
if (_heap->cancelled_gc()) {
- msg = "At end of Interrupted Concurrent GLOBAL GC";
+ msg = "At end of Interrupted Concurrent Global GC";
} else {
// We only record GC results if GC was successful
msg = "At end of Concurrent Global GC";
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.cpp
index ca15c6db443..750022b274e 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.cpp
@@ -63,7 +63,7 @@ ShenandoahGenerationalEvacuationTask::ShenandoahGenerationalEvacuationTask(Shena
void ShenandoahGenerationalEvacuationTask::work(uint worker_id) {
if (_concurrent) {
ShenandoahConcurrentWorkerSession worker_session(worker_id);
- ShenandoahSuspendibleThreadSetJoiner stsj;
+ SuspendibleThreadSetJoiner stsj;
do_work();
} else {
ShenandoahParallelWorkerSession worker_session(worker_id);
@@ -73,12 +73,10 @@ void ShenandoahGenerationalEvacuationTask::work(uint worker_id) {
void ShenandoahGenerationalEvacuationTask::do_work() {
if (_only_promote_regions) {
- // No allocations will be made, do not enter oom-during-evac protocol.
assert(_heap->collection_set()->is_empty(), "Should not have a collection set here");
promote_regions();
} else {
assert(!_heap->collection_set()->is_empty(), "Should have a collection set here");
- ShenandoahEvacOOMScope oom_evac_scope;
evacuate_and_promote_regions();
}
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp
index 1694121b955..f203a8d1238 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp
@@ -199,13 +199,6 @@ void ShenandoahGenerationalHeap::promote_regions_in_place(ShenandoahGeneration*
oop ShenandoahGenerationalHeap::evacuate_object(oop p, Thread* thread) {
assert(thread == Thread::current(), "Expected thread parameter to be current thread.");
- if (ShenandoahThreadLocalData::is_oom_during_evac(thread)) {
- // This thread went through the OOM during evac protocol and it is safe to return
- // the forward pointer. It must not attempt to evacuate anymore.
- return ShenandoahBarrierSet::resolve_forwarded(p);
- }
-
- assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope");
ShenandoahHeapRegion* from_region = heap_region_containing(p);
assert(!from_region->is_humongous(), "never evacuate humongous objects");
@@ -329,8 +322,23 @@ oop ShenandoahGenerationalHeap::try_evacuate_object(oop p, Thread* thread, uint
}
control_thread()->handle_alloc_failure_evac(size);
- oom_evac_handler()->handle_out_of_memory_during_evacuation();
- return ShenandoahBarrierSet::resolve_forwarded(p);
+
+ // Install the self-forwarded bit so other evacuators/LRBs see the
+ // object as "already handled, do not try to evacuate". The CAS may
+ // fail if another thread concurrently installed a real forwardee or
+ // self-forwarded first.
+ markWord old_mark = p->mark();
+ if (old_mark.is_forwarded()) {
+ return ShenandoahForwarding::get_forwardee(p);
+ }
+ oop winner = ShenandoahForwarding::try_forward_to_self(p, old_mark);
+ if (winner == nullptr) {
+ // We own the self-forwarding. Flag the from-region so the degen/full
+ // GC entry drain knows to scan it for self_fwd bits to clear.
+ heap_region_containing(p)->set_has_self_forwards();
+ return p;
+ }
+ return winner;
}
if (ShenandoahEvacTracking) {
@@ -717,7 +725,7 @@ public:
void work(uint worker_id) override {
if (CONCURRENT) {
ShenandoahConcurrentWorkerSession worker_session(worker_id);
- ShenandoahSuspendibleThreadSetJoiner stsj;
+ SuspendibleThreadSetJoiner stsj;
do_work(worker_id);
} else {
ShenandoahParallelWorkerSession worker_session(worker_id);
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
index 4b01ea1cd52..2bedc53e24b 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
@@ -62,6 +62,7 @@
#include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
#include "gc/shenandoah/shenandoahMemoryPool.hpp"
#include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
+#include "gc/shenandoah/shenandoahObjArrayAllocator.hpp"
#include "gc/shenandoah/shenandoahOldGeneration.hpp"
#include "gc/shenandoah/shenandoahPadding.hpp"
#include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
@@ -1073,6 +1074,11 @@ HeapWord* ShenandoahHeap::mem_allocate(size_t size) {
return allocate_memory(req);
}
+oop ShenandoahHeap::array_allocate(Klass* klass, size_t size, int length, bool do_zero, TRAPS) {
+ ShenandoahObjArrayAllocator allocator(klass, size, length, do_zero, THREAD);
+ return allocator.allocate();
+}
+
MetaWord* ShenandoahHeap::satisfy_failed_metadata_allocation(ClassLoaderData* loader_data,
size_t size,
Metaspace::MetadataType mdtype) {
@@ -1143,12 +1149,10 @@ public:
void work(uint worker_id) {
if (_concurrent) {
ShenandoahConcurrentWorkerSession worker_session(worker_id);
- ShenandoahSuspendibleThreadSetJoiner stsj;
- ShenandoahEvacOOMScope oom_evac_scope;
+ SuspendibleThreadSetJoiner stsj;
do_work();
} else {
ShenandoahParallelWorkerSession worker_session(worker_id);
- ShenandoahEvacOOMScope oom_evac_scope;
do_work();
}
}
@@ -1296,13 +1300,6 @@ void ShenandoahHeap::concurrent_final_roots(HandshakeClosure* handshake_closure)
oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
assert(thread == Thread::current(), "Expected thread parameter to be current thread.");
- if (ShenandoahThreadLocalData::is_oom_during_evac(thread)) {
- // This thread went through the OOM during evac protocol. It is safe to return
- // the forward pointer. It must not attempt to evacuate any other objects.
- return ShenandoahBarrierSet::resolve_forwarded(p);
- }
-
- assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope");
ShenandoahHeapRegion* r = heap_region_containing(p);
assert(!r->is_humongous(), "never evacuate humongous objects");
@@ -1341,9 +1338,22 @@ oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapReg
if (copy == nullptr) {
control_thread()->handle_alloc_failure_evac(size);
- _oom_evac_handler.handle_out_of_memory_during_evacuation();
-
- return ShenandoahBarrierSet::resolve_forwarded(p);
+ // Install the self-forwarded bit on p so other evacuators/LRBs see
+ // the object as "already handled, do not try to evacuate". The CAS
+ // may fail if another thread concurrently installed a real forwardee
+ // (they succeeded where we failed) or self-forwarded first.
+ markWord old_mark = p->mark();
+ if (old_mark.is_forwarded()) {
+ return ShenandoahForwarding::get_forwardee(p);
+ }
+ oop winner = ShenandoahForwarding::try_forward_to_self(p, old_mark);
+ if (winner == nullptr) {
+ // We own the self-forwarding. Flag the region so the degen/full GC
+ // entry drain knows to scan it for self_fwd bits to clear.
+ from_region->set_has_self_forwards();
+ return p;
+ }
+ return winner;
}
if (ShenandoahEvacTracking) {
@@ -1398,6 +1408,71 @@ oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapReg
}
}
+// Clear the self_fwd bit on a live cset object, if set. Runs at a safepoint,
+// so a plain store is sufficient — no concurrent writers to the mark word.
+class ShenandoahUnSelfForwardObjectClosure : public ObjectClosure {
+public:
+ void do_object(oop obj) override {
+ markWord m = obj->mark();
+ if (m.is_self_forwarded()) {
+ obj->set_mark(m.unset_self_forwarded());
+ }
+ }
+};
+
+// Parallel task over flagged cset regions. Iterates the live objects via the
+// mark bitmap (skipping evacuated and never-marked memory), clears self_fwd
+// bits, and resets the region flag once done.
+class ShenandoahUnSelfForwardTask : public WorkerTask {
+private:
+ ShenandoahHeap* const _heap;
+ ShenandoahCollectionSet* const _cs;
+
+public:
+ ShenandoahUnSelfForwardTask(ShenandoahHeap* heap, ShenandoahCollectionSet* cs) :
+ WorkerTask("Shenandoah Un-Self-Forward"),
+ _heap(heap),
+ _cs(cs) {}
+
+ void work(uint worker_id) override {
+ ShenandoahParallelWorkerSession worker_session(worker_id);
+ ShenandoahUnSelfForwardObjectClosure cl;
+ ShenandoahHeapRegion* r;
+ while ((r = _cs->claim_next()) != nullptr) {
+ if (r->has_self_forwards()) {
+ _heap->marked_object_iterate(r, &cl);
+ r->clear_has_self_forwards();
+ }
+ }
+ }
+};
+
+void ShenandoahHeap::un_self_forward_cset_regions() {
+ assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
+ ShenandoahCollectionSet* cs = collection_set();
+ if (cs == nullptr || cs->is_empty()) {
+ return;
+ }
+ cs->clear_current_index();
+ ShenandoahUnSelfForwardTask task(this, cs);
+ workers()->run_task(&task);
+ DEBUG_ONLY(assert_no_self_forwards());
+}
+
+#ifdef ASSERT
+void ShenandoahHeap::assert_no_self_forwards() const {
+ assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
+ ShenandoahCollectionSet* cs = collection_set();
+ if (cs == nullptr) return;
+ cs->clear_current_index();
+ ShenandoahHeapRegion* r;
+ while ((r = cs->next()) != nullptr) {
+ assert(!r->has_self_forwards(), "region still flagged after drain");
+ }
+ cs->clear_current_index();
+}
+#endif
+
void ShenandoahHeap::trash_cset_regions() {
ShenandoahHeapLocker locker(lock());
@@ -2279,6 +2354,11 @@ void ShenandoahHeap::stw_unload_classes(bool full_gc) {
}
// Resize and verify metaspace
MetaspaceGC::compute_new_size();
+
+ if (mode()->is_generational()) {
+ old_generation()->set_parsable(false);
+ }
+
DEBUG_ONLY(MetaspaceUtils::verify();)
}
@@ -2345,24 +2425,23 @@ address ShenandoahHeap::in_cset_fast_test_addr() {
void ShenandoahHeap::reset_bytes_allocated_since_gc_start() {
// It is important to force_alloc_rate_sample() before the associated generation's bytes_allocated has been reset.
- // Note that there is no lock to prevent additional alloations between sampling bytes_allocated_since_gc_start() and
- // reset_bytes_allocated_since_gc_start(). If additional allocations happen, they will be ignored in the average
- // allocation rate computations. This effect is considered to be be negligible.
-
- // unaccounted_bytes is the bytes not accounted for by our forced sample. If the sample interval is too short,
- // the "forced sample" will not happen, and any recently allocated bytes are "unaccounted for". We pretend these
- // bytes are allocated after the start of subsequent gc.
- size_t unaccounted_bytes;
- ShenandoahFreeSet* _free_set = free_set();
- size_t bytes_allocated = _free_set->get_bytes_allocated_since_gc_start();
- if (mode()->is_generational()) {
- unaccounted_bytes = young_generation()->heuristics()->force_alloc_rate_sample(bytes_allocated);
- } else {
- // Single-gen Shenandoah uses global heuristics.
- unaccounted_bytes = heuristics()->force_alloc_rate_sample(bytes_allocated);
+ // Note that we obtain heap lock to prevent additional allocations between sampling bytes_allocated_since_gc_start()
+ // and reset_bytes_allocated_since_gc_start()
+ {
+ ShenandoahHeapLocker locker(lock());
+ // unaccounted_bytes is the bytes not accounted for by our forced sample. If the sample interval is too short,
+ // the "forced sample" will not happen, and any recently allocated bytes are "unaccounted for". We pretend these
+ // bytes are allocated after the start of subsequent gc.
+ size_t unaccounted_bytes;
+ size_t bytes_allocated = _free_set->get_bytes_allocated_since_gc_start();
+ if (mode()->is_generational()) {
+ unaccounted_bytes = young_generation()->heuristics()->force_alloc_rate_sample(bytes_allocated);
+ } else {
+ // Single-gen Shenandoah uses global heuristics.
+ unaccounted_bytes = heuristics()->force_alloc_rate_sample(bytes_allocated);
+ }
+ _free_set->reset_bytes_allocated_since_gc_start(unaccounted_bytes);
}
- ShenandoahHeapLocker locker(lock());
- _free_set->reset_bytes_allocated_since_gc_start(unaccounted_bytes);
}
void ShenandoahHeap::set_degenerated_gc_in_progress(bool in_progress) {
@@ -2497,7 +2576,7 @@ public:
void work(uint worker_id) {
if (CONCURRENT) {
ShenandoahConcurrentWorkerSession worker_session(worker_id);
- ShenandoahSuspendibleThreadSetJoiner stsj;
+ SuspendibleThreadSetJoiner stsj;
do_work(worker_id);
} else {
ShenandoahParallelWorkerSession worker_session(worker_id);
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp
index bed26a093d0..fa435eaa1be 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp
@@ -33,7 +33,6 @@
#include "gc/shenandoah/shenandoahAllocRequest.hpp"
#include "gc/shenandoah/shenandoahAsserts.hpp"
#include "gc/shenandoah/shenandoahController.hpp"
-#include "gc/shenandoah/shenandoahEvacOOMHandler.hpp"
#include "gc/shenandoah/shenandoahEvacTracker.hpp"
#include "gc/shenandoah/shenandoahGenerationType.hpp"
#include "gc/shenandoah/shenandoahLock.hpp"
@@ -558,8 +557,6 @@ public:
ShenandoahPhaseTimings* phase_timings() const { return _phase_timings; }
- ShenandoahEvacOOMHandler* oom_evac_handler() { return &_oom_evac_handler; }
-
ShenandoahEvacuationTracker* evac_tracker() const {
return _evac_tracker;
}
@@ -709,6 +706,7 @@ private:
public:
HeapWord* allocate_memory(ShenandoahAllocRequest& request);
HeapWord* mem_allocate(size_t size) override;
+ oop array_allocate(Klass* klass, size_t size, int length, bool do_zero, TRAPS) override;
MetaWord* satisfy_failed_metadata_allocation(ClassLoaderData* loader_data,
size_t size,
Metaspace::MetadataType mdtype) override;
@@ -792,7 +790,6 @@ public:
//
private:
ShenandoahCollectionSet* _collection_set;
- ShenandoahEvacOOMHandler _oom_evac_handler;
oop try_evacuate_object(oop src, Thread* thread, ShenandoahHeapRegion* from_region, ShenandoahAffiliation target_gen);
@@ -812,12 +809,16 @@ public:
inline bool in_collection_set_loc(void* loc) const;
// Evacuates or promotes object src. Returns the evacuated object, either evacuated
- // by this thread, or by some other thread.
+ // by this thread, or by some other thread. On allocation failure, installs the
+ // self-forwarded bit on src, flags src's region, and returns src.
virtual oop evacuate_object(oop src, Thread* thread);
- // Call before/after evacuation.
- inline void enter_evacuation(Thread* t);
- inline void leave_evacuation(Thread* t);
+ // Parallel scan of flagged cset regions to clear self-forwarded bits on live
+ // objects. Must be called at a safepoint; intended for the degenerated and
+ // full GC entry paths.
+ void un_self_forward_cset_regions();
+
+ DEBUG_ONLY(void assert_no_self_forwards() const;)
// ---------- Helper functions
//
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp
index 6d77cccaa6a..b8db14e5ee7 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp
@@ -100,14 +100,6 @@ inline ShenandoahHeapRegion* ShenandoahHeap::heap_region_containing(const void*
return result;
}
-inline void ShenandoahHeap::enter_evacuation(Thread* t) {
- _oom_evac_handler.enter_evacuation(t);
-}
-
-inline void ShenandoahHeap::leave_evacuation(Thread* t) {
- _oom_evac_handler.leave_evacuation(t);
-}
-
template
inline void ShenandoahHeap::non_conc_update_with_forwarded(T* p) {
T o = RawAccess<>::oop_load(p);
@@ -272,7 +264,6 @@ inline GCCause::Cause ShenandoahHeap::cancelled_cause() const {
inline void ShenandoahHeap::clear_cancelled_gc() {
_cancelled_gc.set(GCCause::_no_gc);
reset_cancellation_time();
- _oom_evac_handler.clear();
}
inline GCCause::Cause ShenandoahHeap::clear_cancellation(const GCCause::Cause expected) {
@@ -455,7 +446,7 @@ inline bool ShenandoahHeap::in_collection_set_loc(void* p) const {
}
inline char ShenandoahHeap::gc_state() const {
- return _gc_state.raw_value();
+ return integer_cast(_gc_state.raw_value());
}
inline bool ShenandoahHeap::is_gc_state(GCState state) const {
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp
index c031569b7c6..c5b9c929eed 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp
@@ -90,6 +90,7 @@ ShenandoahHeapRegion::ShenandoahHeapRegion(HeapWord* start, size_t index, bool c
SpaceMangler::mangle_region(MemRegion(_bottom, _end));
}
_recycling.unset();
+ _has_self_forwards.unset();
}
void ShenandoahHeapRegion::report_illegal_transition(const char *method) {
@@ -572,12 +573,13 @@ void ShenandoahHeapRegion::recycle_internal() {
reset_alloc_metadata();
heap->marking_context()->reset_top_at_mark_start(this);
set_update_watermark(bottom());
+ clear_has_self_forwards();
if (is_old()) {
heap->old_generation()->clear_cards_for(this);
}
if (ZapUnusedHeapArea) {
- SpaceMangler::mangle_region(MemRegion(bottom(), end()));
+ SpaceMangler::mangle_region(MemRegion(bottom(), top()));
}
set_top(bottom());
set_affiliation(FREE);
@@ -668,13 +670,6 @@ size_t ShenandoahHeapRegion::block_size(const HeapWord* p) const {
}
size_t ShenandoahHeapRegion::setup_sizes(size_t max_heap_size) {
- // Absolute minimums we should not ever break.
- static const size_t MIN_REGION_SIZE = 256*K;
-
- if (FLAG_IS_DEFAULT(ShenandoahMinRegionSize)) {
- FLAG_SET_DEFAULT(ShenandoahMinRegionSize, MIN_REGION_SIZE);
- }
-
// Generational Shenandoah needs this alignment for card tables.
if (strcmp(ShenandoahGCMode, "generational") == 0) {
max_heap_size = align_up(max_heap_size , CardTable::ct_max_alignment_constraint());
@@ -682,47 +677,13 @@ size_t ShenandoahHeapRegion::setup_sizes(size_t max_heap_size) {
size_t region_size;
if (FLAG_IS_DEFAULT(ShenandoahRegionSize)) {
- if (ShenandoahMinRegionSize > max_heap_size / MIN_NUM_REGIONS) {
- err_msg message("Max heap size (%zu%s) is too low to afford the minimum number "
- "of regions (%zu) of minimum region size (%zu%s).",
- byte_size_in_proper_unit(max_heap_size), proper_unit_for_byte_size(max_heap_size),
- MIN_NUM_REGIONS,
- byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize));
- vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
- }
- if (ShenandoahMinRegionSize < MIN_REGION_SIZE) {
- err_msg message("%zu%s should not be lower than minimum region size (%zu%s).",
- byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize),
- byte_size_in_proper_unit(MIN_REGION_SIZE), proper_unit_for_byte_size(MIN_REGION_SIZE));
- vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
- }
- if (ShenandoahMinRegionSize < MinTLABSize) {
- err_msg message("%zu%s should not be lower than TLAB size size (%zu%s).",
- byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize),
- byte_size_in_proper_unit(MinTLABSize), proper_unit_for_byte_size(MinTLABSize));
- vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);
- }
- if (ShenandoahMaxRegionSize < MIN_REGION_SIZE) {
- err_msg message("%zu%s should not be lower than min region size (%zu%s).",
- byte_size_in_proper_unit(ShenandoahMaxRegionSize), proper_unit_for_byte_size(ShenandoahMaxRegionSize),
- byte_size_in_proper_unit(MIN_REGION_SIZE), proper_unit_for_byte_size(MIN_REGION_SIZE));
- vm_exit_during_initialization("Invalid -XX:ShenandoahMaxRegionSize option", message);
- }
- if (ShenandoahMinRegionSize > ShenandoahMaxRegionSize) {
- err_msg message("Minimum (%zu%s) should be larger than maximum (%zu%s).",
- byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize),
- byte_size_in_proper_unit(ShenandoahMaxRegionSize), proper_unit_for_byte_size(ShenandoahMaxRegionSize));
- vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize or -XX:ShenandoahMaxRegionSize", message);
- }
-
// We rapidly expand to max_heap_size in most scenarios, so that is the measure
// for usual heap sizes. Do not depend on initial_heap_size here.
region_size = max_heap_size / ShenandoahTargetNumRegions;
// Now make sure that we don't go over or under our limits.
- region_size = MAX2(ShenandoahMinRegionSize, region_size);
- region_size = MIN2(ShenandoahMaxRegionSize, region_size);
-
+ region_size = MAX2(MIN_REGION_SIZE, region_size);
+ region_size = MIN2(MAX_REGION_SIZE, region_size);
} else {
if (ShenandoahRegionSize > max_heap_size / MIN_NUM_REGIONS) {
err_msg message("Max heap size (%zu%s) is too low to afford the minimum number "
@@ -732,16 +693,16 @@ size_t ShenandoahHeapRegion::setup_sizes(size_t max_heap_size) {
byte_size_in_proper_unit(ShenandoahRegionSize), proper_unit_for_byte_size(ShenandoahRegionSize));
vm_exit_during_initialization("Invalid -XX:ShenandoahRegionSize option", message);
}
- if (ShenandoahRegionSize < ShenandoahMinRegionSize) {
+ if (ShenandoahRegionSize < MIN_REGION_SIZE) {
err_msg message("Heap region size (%zu%s) should be larger than min region size (%zu%s).",
byte_size_in_proper_unit(ShenandoahRegionSize), proper_unit_for_byte_size(ShenandoahRegionSize),
- byte_size_in_proper_unit(ShenandoahMinRegionSize), proper_unit_for_byte_size(ShenandoahMinRegionSize));
+ byte_size_in_proper_unit(MIN_REGION_SIZE), proper_unit_for_byte_size(MIN_REGION_SIZE));
vm_exit_during_initialization("Invalid -XX:ShenandoahRegionSize option", message);
}
- if (ShenandoahRegionSize > ShenandoahMaxRegionSize) {
+ if (ShenandoahRegionSize > MAX_REGION_SIZE) {
err_msg message("Heap region size (%zu%s) should be lower than max region size (%zu%s).",
byte_size_in_proper_unit(ShenandoahRegionSize), proper_unit_for_byte_size(ShenandoahRegionSize),
- byte_size_in_proper_unit(ShenandoahMaxRegionSize), proper_unit_for_byte_size(ShenandoahMaxRegionSize));
+ byte_size_in_proper_unit(MAX_REGION_SIZE), proper_unit_for_byte_size(MAX_REGION_SIZE));
vm_exit_during_initialization("Invalid -XX:ShenandoahRegionSize option", message);
}
region_size = ShenandoahRegionSize;
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp
index e27bbbb737d..7853238f080 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp
@@ -197,11 +197,13 @@ public:
bool is_regular() const { return state() == _regular; }
bool is_humongous_continuation() const { return state() == _humongous_cont; }
bool is_regular_pinned() const { return state() == _pinned; }
- bool is_trash() const { return state() == _trash; }
+ bool is_trash() const { return is_trash(state()); }
// Derived state predicates (boolean combinations of individual states)
+ bool static is_trash(RegionState state) { return state == _trash; }
bool static is_empty_state(RegionState state) { return state == _empty_committed || state == _empty_uncommitted; }
bool static is_humongous_start_state(RegionState state) { return state == _humongous_start || state == _pinned_humongous_start; }
+ bool is_empty_or_trash() const { auto cur_state = state(); return is_empty_state(cur_state) || cur_state == _trash; }
bool is_empty() const { return is_empty_state(this->state()); }
bool is_active() const { auto cur_state = state(); return !is_empty_state(cur_state) && cur_state != _trash; }
bool is_humongous_start() const { return is_humongous_start_state(state()); }
@@ -209,6 +211,7 @@ public:
bool is_committed() const { return !is_empty_uncommitted(); }
bool is_cset() const { auto cur_state = state(); return cur_state == _cset || cur_state == _pinned_cset; }
bool is_pinned() const { auto cur_state = state(); return cur_state == _pinned || cur_state == _pinned_cset || cur_state == _pinned_humongous_start; }
+ bool is_regular_or_regular_pinned() const { auto cur_state = state(); return cur_state == _regular || cur_state == _pinned; }
inline bool is_young() const;
inline bool is_old() const;
@@ -272,12 +275,20 @@ private:
ShenandoahSharedFlag _recycling; // Used to indicate that the region is being recycled; see try_recycle*().
+ // Set when an evacuation failure self-forwarded at least one object in this
+ // region. The drain at degen/full GC entry scans flagged regions and CAS-
+ // clears the self_fwd bits. Safety-net reset on region recycle.
+ ShenandoahSharedFlag _has_self_forwards;
+
bool _needs_bitmap_reset;
public:
ShenandoahHeapRegion(HeapWord* start, size_t index, bool committed);
+ // Absolute minimums and maximums we should not ever break.
static const size_t MIN_NUM_REGIONS = 10;
+ static const size_t MIN_REGION_SIZE = 256*K;
+ static const size_t MAX_REGION_SIZE = 32*M;
// Return adjusted max heap size
static size_t setup_sizes(size_t max_heap_size);
@@ -526,6 +537,13 @@ public:
_needs_bitmap_reset = false;
}
+ // Self-forward accounting: set by an evacuating thread after it successfully
+ // installs a self-forward mark on an object in this region. Tested and cleared
+ // at the drain phase (degen/full GC entry) and again on region recycle.
+ bool has_self_forwards() const { return _has_self_forwards.is_set(); }
+ void set_has_self_forwards() { _has_self_forwards.set(); }
+ void clear_has_self_forwards() { _has_self_forwards.unset(); }
+
private:
void decrement_humongous_waste();
void do_commit();
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp
index aaf152e2890..3e809ea84b9 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp
@@ -77,8 +77,8 @@ ShenandoahHeapRegionCounters::ShenandoahHeapRegionCounters() :
}
ShenandoahHeapRegionCounters::~ShenandoahHeapRegionCounters() {
- if (_name_space != nullptr) FREE_C_HEAP_ARRAY(char, _name_space);
- if (_regions_data != nullptr) FREE_C_HEAP_ARRAY(PerfVariable*, _regions_data);
+ if (_name_space != nullptr) FREE_C_HEAP_ARRAY(_name_space);
+ if (_regions_data != nullptr) FREE_C_HEAP_ARRAY(_regions_data);
}
void ShenandoahHeapRegionCounters::write_snapshot(PerfLongVariable** regions,
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionSet.cpp
index 560de816db9..1d2cd97b75d 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionSet.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionSet.cpp
@@ -47,7 +47,7 @@ ShenandoahHeapRegionSet::ShenandoahHeapRegionSet() :
}
ShenandoahHeapRegionSet::~ShenandoahHeapRegionSet() {
- FREE_C_HEAP_ARRAY(jbyte, _set_map);
+ FREE_C_HEAP_ARRAY(_set_map);
}
void ShenandoahHeapRegionSet::add_region(ShenandoahHeapRegion* r) {
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp
index a3c28e2c6d3..dfd921fdf0b 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp
@@ -199,7 +199,7 @@ void ShenandoahMark::mark_loop_work(T* cl, ShenandoahLiveData* live_data, uint w
if (work == 0) {
// No work encountered in current stride, try to terminate.
// Need to leave the STS here otherwise it might block safepoints.
- ShenandoahSuspendibleThreadSetLeaver stsl(CANCELLABLE);
+ SuspendibleThreadSetLeaver stsl(CANCELLABLE);
ShenandoahTerminatorTerminator tt(heap);
if (terminator->offer_termination(&tt)) return;
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp
index 5867478d734..0b38f6e0cea 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp
@@ -140,13 +140,11 @@ void ShenandoahMmuTracker::record_mixed(size_t gcid) {
update_utilization(gcid, "Mixed Concurrent GC");
}
-void ShenandoahMmuTracker::record_degenerated(size_t gcid, bool is_old_bootstrap) {
+void ShenandoahMmuTracker::record_degenerated(size_t gcid, const char* msg) {
if ((gcid == _most_recent_gcid) && _most_recent_is_full) {
// Do nothing. This is a redundant recording for the full gc that just completed.
- } else if (is_old_bootstrap) {
- update_utilization(gcid, "Degenerated Bootstrap Old GC");
} else {
- update_utilization(gcid, "Degenerated Young GC");
+ update_utilization(gcid, msg);
}
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.hpp
index 89dbf921cd4..cb2db12c233 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.hpp
@@ -72,6 +72,8 @@ private:
ShenandoahMmuTask* _mmu_periodic_task;
TruncatedSeq _mmu_average;
+ // Updates and logs the GC and mutator CPU utilization since the last cycle, where "msg"
+ // identifies the GC cycle type.
void update_utilization(size_t gcid, const char* msg);
static void fetch_cpu_times(double &gc_time, double &mutator_time);
@@ -95,7 +97,8 @@ public:
void record_old_marking_increment(bool old_marking_done);
void record_mixed(size_t gcid);
void record_full(size_t gcid);
- void record_degenerated(size_t gcid, bool is_old_boostrap);
+ // Records GC utilization for a degenerated cycle, where "msg" describes the degeneration type.
+ void record_degenerated(size_t gcid, const char* msg);
// This is called by the periodic task timer. The interval is defined by
// GCPauseIntervalMillis and defaults to 5 seconds. This method computes
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp b/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp
index 594ad614d90..5b24cfc979a 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp
@@ -48,7 +48,7 @@ ShenandoahNMethod::ShenandoahNMethod(nmethod* nm, GrowableArray& oops, boo
ShenandoahNMethod::~ShenandoahNMethod() {
if (_oops != nullptr) {
- FREE_C_HEAP_ARRAY(oop*, _oops);
+ FREE_C_HEAP_ARRAY(_oops);
}
}
@@ -60,7 +60,7 @@ void ShenandoahNMethod::update() {
detect_reloc_oops(nm(), oops, non_immediate_oops);
if (oops.length() != _oops_count) {
if (_oops != nullptr) {
- FREE_C_HEAP_ARRAY(oop*, _oops);
+ FREE_C_HEAP_ARRAY(_oops);
_oops = nullptr;
}
@@ -126,7 +126,6 @@ void ShenandoahNMethod::heal_nmethod(nmethod* nm) {
ShenandoahHeap* const heap = ShenandoahHeap::heap();
if (heap->is_concurrent_weak_root_in_progress() ||
heap->is_concurrent_strong_root_in_progress()) {
- ShenandoahEvacOOMScope evac_scope;
heal_nmethod_metadata(data);
} else if (heap->is_concurrent_mark_in_progress()) {
ShenandoahKeepAliveClosure cl;
@@ -394,7 +393,7 @@ ShenandoahNMethodList::ShenandoahNMethodList(int size) :
ShenandoahNMethodList::~ShenandoahNMethodList() {
assert(_list != nullptr, "Sanity");
assert(_ref_count == 0, "Must be");
- FREE_C_HEAP_ARRAY(ShenandoahNMethod*, _list);
+ FREE_C_HEAP_ARRAY(_list);
}
void ShenandoahNMethodList::transfer(ShenandoahNMethodList* const list, int limit) {
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.cpp b/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.cpp
index 1ddd8e1c032..82b027faca2 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.cpp
@@ -39,10 +39,10 @@ HdrSeq::~HdrSeq() {
for (int c = 0; c < MagBuckets; c++) {
int* sub = _hdr[c];
if (sub != nullptr) {
- FREE_C_HEAP_ARRAY(int, sub);
+ FREE_C_HEAP_ARRAY(sub);
}
}
- FREE_C_HEAP_ARRAY(int*, _hdr);
+ FREE_C_HEAP_ARRAY(_hdr);
}
void HdrSeq::add(double val) {
@@ -191,7 +191,7 @@ BinaryMagnitudeSeq::BinaryMagnitudeSeq() {
}
BinaryMagnitudeSeq::~BinaryMagnitudeSeq() {
- FREE_C_HEAP_ARRAY(size_t, _mags);
+ FREE_C_HEAP_ARRAY(_mags);
}
void BinaryMagnitudeSeq::clear() {
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahObjArrayAllocator.cpp b/src/hotspot/share/gc/shenandoah/shenandoahObjArrayAllocator.cpp
new file mode 100644
index 00000000000..dc9fef16cf6
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/shenandoahObjArrayAllocator.cpp
@@ -0,0 +1,124 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "gc/shared/memAllocator.hpp"
+#include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
+#include "gc/shenandoah/shenandoahHeap.inline.hpp"
+#include "gc/shenandoah/shenandoahHeapRegion.hpp"
+#include "gc/shenandoah/shenandoahObjArrayAllocator.hpp"
+#include "gc/shenandoah/shenandoahThreadLocalData.hpp"
+#include "memory/universe.hpp"
+#include "oops/arrayKlass.hpp"
+#include "oops/arrayOop.hpp"
+#include "runtime/interfaceSupport.inline.hpp"
+#include "utilities/copy.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+ShenandoahObjArrayAllocator::ShenandoahObjArrayAllocator(
+ Klass* klass, size_t word_size, int length, bool do_zero, Thread* thread)
+ : ObjArrayAllocator(klass, word_size, length, do_zero, thread) {
+ assert(_length >= 0, "length should be non-negative");
+}
+
+oop ShenandoahObjArrayAllocator::initialize(HeapWord* mem) const {
+ // threshold of object size, while above this size current mutator will yield to safepoint
+ // when it clears the array content.
+ constexpr size_t THRESHOLD = 64 * K / BytesPerWord;
+
+ ShenandoahHeap* const heap = ShenandoahHeap::heap();
+
+ // Fast path: delegate to base class for small arrays or no-zero case.
+ // In no-zero case(_do_zero is false), the content of the array won't be zeroed, therefore no need to fall into slow-path.
+ if (!_do_zero || _word_size <= THRESHOLD) {
+ return ObjArrayAllocator::initialize(mem);
+ }
+
+ // Slow path: yield to safepoint when clearing for large arrays
+
+ // Compute clearing bounds
+ const BasicType element_type = ArrayKlass::cast(_klass)->element_type();
+ const size_t base_offset_in_bytes = (size_t)arrayOopDesc::base_offset_in_bytes(element_type);
+ const size_t process_start_offset_in_bytes = align_up(base_offset_in_bytes, (size_t)BytesPerWord);
+
+ const size_t process_start = process_start_offset_in_bytes / BytesPerWord;
+ const size_t process_size = _word_size - process_start;
+
+ // Pin the region before clearing to avoid moving the object until it is done
+ ShenandoahHeapRegion* region = heap->heap_region_containing(mem);
+ region->record_pin();
+
+ // Always initialize the mem with primitive array first so GC won't look into the elements in the array.
+ // For obj array, the header will be corrected to object array after clearing the memory.
+ Klass* filling_klass = _klass;
+ const bool is_ref_type = is_reference_type(element_type);
+
+ if (is_ref_type) {
+ filling_klass = LP64_ONLY(UseCompressedOops ? Universe::intArrayKlass() : Universe::longArrayKlass()) NOT_LP64(Universe::intArrayKlass());
+ assert(type2aelembytes(ArrayKlass::cast(filling_klass)->element_type()) == type2aelembytes(element_type), "filling element size must match ref size");
+ }
+ // Use _length directly: it matches the ref count, and the filling element size equals the ref size.
+ ObjArrayAllocator filling_array_allocator(filling_klass, _word_size, _length, /* do_zero */ false);
+ filling_array_allocator.initialize(mem);
+
+ // Invisible roots will be scanned and marked at the end of marking.
+ ShenandoahThreadLocalData::set_invisible_root(_thread, mem, _word_size);
+
+ {
+ // The mem has been initialized as primitive array, the entire clearing work is safe for safepoint
+ ThreadBlockInVM tbivm(JavaThread::cast(_thread)); // Allow safepoint to proceed.
+ // Handle potential 4-byte alignment gap before array data
+ if (process_start_offset_in_bytes != base_offset_in_bytes) {
+ assert(process_start_offset_in_bytes - base_offset_in_bytes == 4, "Must be 4-byte aligned");
+ *reinterpret_cast(reinterpret_cast(mem) + base_offset_in_bytes) = 0;
+ }
+
+ Copy::zero_to_words(mem + process_start, process_size);
+
+ if (!is_ref_type) {
+ // zap paddings
+ mem_zap_start_padding(mem);
+ mem_zap_end_padding(mem);
+ }
+ }
+
+ // reference array, header need to be overridden to its own.
+ if (is_ref_type) {
+ finish(mem);
+ // zap paddings after setting correct klass
+ mem_zap_start_padding(mem);
+ mem_zap_end_padding(mem);
+ }
+
+ oop arrayObj = cast_to_oop(mem);
+ if (heap->is_concurrent_young_mark_in_progress() && !heap->marking_context()->allocated_after_mark_start(arrayObj)) {
+ // Keep the obj alive because we don't know the progress of marking,
+ // current concurrent marking could have done and VM is calling safepoint for final mark.
+ heap->keep_alive(arrayObj);
+ }
+ ShenandoahThreadLocalData::clear_invisible_root(_thread);
+
+ region->record_unpin();
+
+ return arrayObj;
+}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahObjArrayAllocator.hpp b/src/hotspot/share/gc/shenandoah/shenandoahObjArrayAllocator.hpp
new file mode 100644
index 00000000000..7605b69eb7e
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/shenandoahObjArrayAllocator.hpp
@@ -0,0 +1,40 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHOBJARRAYALLOCATOR_HPP
+#define SHARE_GC_SHENANDOAH_SHENANDOAHOBJARRAYALLOCATOR_HPP
+
+#include "gc/shared/memAllocator.hpp"
+
+class ShenandoahObjArrayAllocator : public ObjArrayAllocator {
+private:
+ // Override: clearing with safepoint yields for large arrays
+ oop initialize(HeapWord* mem) const override;
+
+public:
+ ShenandoahObjArrayAllocator(Klass* klass, size_t word_size, int length,
+ bool do_zero, Thread* thread);
+};
+
+#endif // SHARE_GC_SHENANDOAH_SHENANDOAHOBJARRAYALLOCATOR_HPP
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp
index 37de5966554..1dd38deb4d7 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp
@@ -675,9 +675,9 @@ void ShenandoahOldGeneration::log_failed_promotion(LogStream& ls, Thread* thread
void ShenandoahOldGeneration::update_card_table() {
for_each_region([this](ShenandoahHeapRegion* region) {
- if (region->is_regular()) {
+ if (region->is_regular_or_regular_pinned()) {
// Humongous regions are promoted in place, remembered set maintenance is handled there
- // Regular regions that are promoted in place have their rset maintenance handled for
+ // Regular/pinned regions that are promoted in place have their rset maintenance handled for
// the objects in the region when it was promoted. We record TEAS for such a region
// when the in-place-promotion is completed. Such a region may be used for additional
// promotions in the same cycle it was itself promoted.
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahParallelCleaning.cpp b/src/hotspot/share/gc/shenandoah/shenandoahParallelCleaning.cpp
index 6c82f970606..a7a74f30a4b 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahParallelCleaning.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahParallelCleaning.cpp
@@ -26,7 +26,6 @@
#include "gc/shenandoah/shenandoahClosures.inline.hpp"
#include "gc/shenandoah/shenandoahCodeRoots.hpp"
-#include "gc/shenandoah/shenandoahEvacOOMHandler.hpp"
#include "gc/shenandoah/shenandoahParallelCleaning.hpp"
#include "runtime/safepoint.hpp"
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp
index 6a316e2265a..bb21cd5be66 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp
@@ -138,6 +138,7 @@ class outputStream;
\
f(degen_gc_gross, "Pause Degenerated GC (G)") \
f(degen_gc, "Pause Degenerated GC (N)") \
+ f(degen_gc_un_self_forward, " Un-Self-Forward") \
f(degen_gc_stw_mark, " Degen STW Mark") \
SHENANDOAH_PAR_PHASE_DO(degen_gc_stw_mark_, " DSM: ", f) \
f(degen_gc_mark, " Degen Mark") \
@@ -170,6 +171,7 @@ class outputStream;
\
f(full_gc_gross, "Pause Full GC (G)") \
f(full_gc, "Pause Full GC (N)") \
+ f(full_gc_un_self_forward, " Un-Self-Forward") \
f(full_gc_heapdump_pre, " Pre Heap Dump") \
f(full_gc_prepare, " Prepare") \
f(full_gc_update_roots, " Update Roots") \
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.inline.hpp
index 4504ac96819..a62f1c7fd66 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.inline.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.inline.hpp
@@ -141,6 +141,49 @@ public:
}
};
+class ShenandoahInvisibleRootsMarkClosure : public ThreadClosure {
+public:
+ void do_thread(Thread* t) {
+ assert_at_safepoint();
+
+ HeapWord* invisible_root = ShenandoahThreadLocalData::get_invisible_root(t);
+ if (invisible_root == nullptr) {
+ return;
+ }
+ size_t invisible_root_word_size = ShenandoahThreadLocalData::get_invisible_root_word_size(t);
+
+ ShenandoahHeap* const heap = ShenandoahHeap::heap();
+ ShenandoahMarkingContext* const marking_context = heap->marking_context();
+ // Mark the invisible root if it is not marked.
+ if (!marking_context->is_marked(invisible_root)) {
+ bool was_upgraded = false;
+ if (!marking_context->mark_strong(cast_to_oop(invisible_root), was_upgraded)) {
+ return;
+ }
+
+ // Update region liveness data
+ ShenandoahHeapRegion* region = heap->heap_region_containing(invisible_root);
+ if (region->is_regular_or_regular_pinned()) {
+ assert(!ShenandoahHeapRegion::requires_humongous(invisible_root_word_size), "Must not be humongous.");
+ region->increase_live_data_alloc_words(invisible_root_word_size);
+ } else if (region->is_humongous_start()) {
+ DEBUG_ONLY(size_t total_live_words = 0;)
+ do {
+ size_t current = region->get_live_data_words();
+ size_t region_used_words = region->used() >> LogHeapWordSize;
+ DEBUG_ONLY(total_live_words += region_used_words;)
+ assert(current == 0 || current == region_used_words, "Must be");
+ if (current == 0) {
+ region->increase_live_data_alloc_words(region_used_words);
+ }
+ region = heap->get_region(region->index() + 1);
+ } while (region != nullptr && region->is_humongous_continuation());
+ assert(total_live_words == invisible_root_word_size, "Must be");
+ }
+ }
+ }
+};
+
// The rationale for selecting the roots to scan is as follows:
// a. With unload_classes = true, we only want to scan the actual strong roots from the
// code cache. This will allow us to identify the dead classes, unload them, *and*
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp b/src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp
index 0bee8b4cf42..e106cc37627 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp
@@ -49,27 +49,27 @@ JRT_LEAF(void, ShenandoahRuntime::write_barrier_pre(oopDesc* orig))
JRT_END
JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_strong(oopDesc* src, oop* load_addr))
- return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_mutator(src, load_addr);
+ return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_mutator(src, load_addr);
JRT_END
JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_strong_narrow(oopDesc* src, narrowOop* load_addr))
- return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_mutator(src, load_addr);
+ return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_mutator(src, load_addr);
JRT_END
JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_weak(oopDesc* src, oop* load_addr))
- return (oopDesc*) ShenandoahBarrierSet::barrier_set()->load_reference_barrier(ON_WEAK_OOP_REF, oop(src), load_addr);
+ return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_mutator(src, load_addr);
JRT_END
JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_weak_narrow(oopDesc* src, narrowOop* load_addr))
- return (oopDesc*) ShenandoahBarrierSet::barrier_set()->load_reference_barrier(ON_WEAK_OOP_REF, oop(src), load_addr);
+ return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_mutator(src, load_addr);
JRT_END
JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_phantom(oopDesc* src, oop* load_addr))
- return (oopDesc*) ShenandoahBarrierSet::barrier_set()->load_reference_barrier(ON_PHANTOM_OOP_REF, oop(src), load_addr);
+ return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_mutator(src, load_addr);
JRT_END
JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_phantom_narrow(oopDesc* src, narrowOop* load_addr))
- return (oopDesc*) ShenandoahBarrierSet::barrier_set()->load_reference_barrier(ON_PHANTOM_OOP_REF, oop(src), load_addr);
+ return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_mutator(src, load_addr);
JRT_END
JRT_LEAF(void, ShenandoahRuntime::clone_barrier(oopDesc* src))
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp
index 5b4ce6d0bc9..73935ed4e91 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp
@@ -104,6 +104,12 @@ void ShenandoahSTWMark::mark() {
heap->workers()->run_task(&task);
assert(task_queues()->is_empty(), "Should be empty");
+
+ if (!generation()->is_old()) {
+ // Lastly, ensure all the invisible roots are marked.
+ ShenandoahInvisibleRootsMarkClosure cl;
+ Threads::java_threads_do(&cl);
+ }
}
_generation->set_mark_complete();
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp
index 8d7ba2dc46f..7ae148a7144 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp
@@ -798,7 +798,7 @@ void ShenandoahScanRememberedTask::work(uint worker_id) {
if (_is_concurrent) {
// This sets up a thread local reference to the worker_id which is needed by the weak reference processor.
ShenandoahConcurrentWorkerSession worker_session(worker_id);
- ShenandoahSuspendibleThreadSetJoiner stsj;
+ SuspendibleThreadSetJoiner stsj;
do_work(worker_id);
} else {
// This sets up a thread local reference to the worker_id which is needed by the weak reference processor.
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp
index 53f00e64a03..244ed7edd4c 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp
@@ -413,7 +413,7 @@ public:
}
~ShenandoahCardCluster() {
- FREE_C_HEAP_ARRAY(crossing_info, _object_starts);
+ FREE_C_HEAP_ARRAY(_object_starts);
_object_starts = nullptr;
}
@@ -751,7 +751,7 @@ public:
for (uint i = 0; i < ParallelGCThreads; i++) {
delete _card_stats[i];
}
- FREE_C_HEAP_ARRAY(HdrSeq*, _card_stats);
+ FREE_C_HEAP_ARRAY(_card_stats);
_card_stats = nullptr;
}
assert(_card_stats == nullptr, "Error");
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.cpp
index 82a759e34db..d0029b60167 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.cpp
@@ -35,7 +35,7 @@ ShenandoahSimpleBitMap::ShenandoahSimpleBitMap(idx_t num_bits) :
ShenandoahSimpleBitMap::~ShenandoahSimpleBitMap() {
if (_bitmap != nullptr) {
- FREE_C_HEAP_ARRAY(uintx, _bitmap);
+ FREE_C_HEAP_ARRAY(_bitmap);
}
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.hpp
index ea51c5f7f63..d47b6440d36 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.hpp
@@ -27,6 +27,7 @@
#include "cppstdlib/cstddef.hpp"
#include "gc/shenandoah/shenandoahAsserts.hpp"
+#include "utilities/integerCast.hpp"
// TODO: Merge the enhanced capabilities of ShenandoahSimpleBitMap into src/hotspot/share/utilities/bitMap.hpp
// and deprecate ShenandoahSimpleBitMap. The key enhanced capabilities to be integrated include:
@@ -110,7 +111,7 @@ public:
assert((idx >= 0) && (idx < _num_bits), "precondition");
size_t array_idx = idx >> LogBitsPerWord;
uintx bit_number = idx & (BitsPerWord - 1);
- uintx the_bit = nth_bit(bit_number);
+ uintx the_bit = nth_bit(integer_cast(bit_number));
_bitmap[array_idx] |= the_bit;
}
@@ -118,7 +119,7 @@ public:
assert((idx >= 0) && (idx < _num_bits), "precondition");
size_t array_idx = idx >> LogBitsPerWord;
uintx bit_number = idx & (BitsPerWord - 1);
- uintx the_bit = nth_bit(bit_number);
+ uintx the_bit = nth_bit(integer_cast(bit_number));
_bitmap[array_idx] &= ~the_bit;
}
@@ -126,7 +127,7 @@ public:
assert((idx >= 0) && (idx < _num_bits), "precondition");
size_t array_idx = idx >> LogBitsPerWord;
uintx bit_number = idx & (BitsPerWord - 1);
- uintx the_bit = nth_bit(bit_number);
+ uintx the_bit = nth_bit(integer_cast(bit_number));
return (_bitmap[array_idx] & the_bit) != 0;
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahTaskqueue.hpp b/src/hotspot/share/gc/shenandoah/shenandoahTaskqueue.hpp
index dbae1b35c6f..ad4e29a5cc2 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahTaskqueue.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahTaskqueue.hpp
@@ -158,8 +158,8 @@ private:
static const uintptr_t weak_extract_mask = 1 << 1;
static const uintptr_t chunk_pow_extract_mask = ~right_n_bits(oop_bits);
- static const int chunk_range_mask = right_n_bits(chunk_bits);
- static const int pow_range_mask = right_n_bits(pow_bits);
+ static const int chunk_range_mask = right_n_bits(chunk_bits);
+ static const int pow_range_mask = right_n_bits(pow_bits);
inline oop decode_oop(uintptr_t val) const {
STATIC_ASSERT(oop_shift == 0);
@@ -249,7 +249,7 @@ public:
}
static int chunk_size() {
- return nth_bit(chunk_bits);
+ return nth_bit(chunk_bits);
}
};
#else
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.cpp b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.cpp
index 1f3ce76cc1c..4aeab028983 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.cpp
@@ -31,14 +31,14 @@
ShenandoahThreadLocalData::ShenandoahThreadLocalData() :
_gc_state(0),
- _oom_scope_nesting_level(0),
- _oom_during_evac(false),
_satb_mark_queue(&ShenandoahBarrierSet::satb_mark_queue_set()),
_card_table(nullptr),
_gclab(nullptr),
_gclab_size(0),
_shenandoah_plab(nullptr),
- _evacuation_stats(new ShenandoahEvacuationStats()) {
+ _evacuation_stats(new ShenandoahEvacuationStats()),
+ _invisible_root(nullptr),
+ _invisible_root_word_size(0) {
}
ShenandoahThreadLocalData::~ShenandoahThreadLocalData() {
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp
index b1b923bbfce..0ac8f7e0f31 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp
@@ -45,9 +45,6 @@
class ShenandoahThreadLocalData {
private:
char _gc_state;
- // Evacuation OOM state
- uint8_t _oom_scope_nesting_level;
- bool _oom_during_evac;
SATBMarkQueue _satb_mark_queue;
@@ -67,6 +64,9 @@ private:
ShenandoahEvacuationStats* _evacuation_stats;
+ Atomic _invisible_root;
+ Atomic _invisible_root_word_size;
+
ShenandoahThreadLocalData();
~ShenandoahThreadLocalData();
@@ -157,39 +157,6 @@ public:
return data(thread)->_shenandoah_plab;
}
- // Evacuation OOM handling
- static bool is_oom_during_evac(Thread* thread) {
- return data(thread)->_oom_during_evac;
- }
-
- static void set_oom_during_evac(Thread* thread, bool oom) {
- data(thread)->_oom_during_evac = oom;
- }
-
- static uint8_t evac_oom_scope_level(Thread* thread) {
- return data(thread)->_oom_scope_nesting_level;
- }
-
- // Push the scope one level deeper, return previous level
- static uint8_t push_evac_oom_scope(Thread* thread) {
- uint8_t level = evac_oom_scope_level(thread);
- assert(level < 254, "Overflow nesting level"); // UINT8_MAX = 255
- data(thread)->_oom_scope_nesting_level = level + 1;
- return level;
- }
-
- // Pop the scope by one level, return previous level
- static uint8_t pop_evac_oom_scope(Thread* thread) {
- uint8_t level = evac_oom_scope_level(thread);
- assert(level > 0, "Underflow nesting level");
- data(thread)->_oom_scope_nesting_level = level - 1;
- return level;
- }
-
- static bool is_evac_allowed(Thread* thread) {
- return evac_oom_scope_level(thread) > 0;
- }
-
// Offsets
static ByteSize satb_mark_queue_index_offset() {
return satb_mark_queue_offset() + SATBMarkQueue::byte_offset_of_index();
@@ -206,6 +173,25 @@ public:
static ByteSize card_table_offset() {
return Thread::gc_data_offset() + byte_offset_of(ShenandoahThreadLocalData, _card_table);
}
+
+ // invisible root are the partially initialized obj array set by ShenandoahObjArrayAllocator
+ static void set_invisible_root(Thread* thread, HeapWord* invisible_root, size_t word_size) {
+ data(thread)->_invisible_root.store_relaxed(invisible_root);
+ data(thread)->_invisible_root_word_size.store_relaxed(word_size);
+ }
+
+ static void clear_invisible_root(Thread* thread) {
+ data(thread)->_invisible_root.store_relaxed(nullptr);
+ data(thread)->_invisible_root_word_size.store_relaxed(0);
+ }
+
+ static HeapWord* get_invisible_root(Thread* thread) {
+ return data(thread)->_invisible_root.load_relaxed();
+ }
+
+ static size_t get_invisible_root_word_size(Thread* thread) {
+ return data(thread)->_invisible_root_word_size.load_relaxed();
+ }
};
STATIC_ASSERT(sizeof(ShenandoahThreadLocalData) <= sizeof(GCThreadLocalData));
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp b/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp
index 1ed6e43e3e1..b6e084c9091 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp
@@ -221,30 +221,6 @@ public:
~ShenandoahParallelWorkerSession();
};
-class ShenandoahSuspendibleThreadSetJoiner {
-private:
- SuspendibleThreadSetJoiner _joiner;
-public:
- ShenandoahSuspendibleThreadSetJoiner(bool active = true) : _joiner(active) {
- assert(!ShenandoahThreadLocalData::is_evac_allowed(Thread::current()), "STS should be joined before evac scope");
- }
- ~ShenandoahSuspendibleThreadSetJoiner() {
- assert(!ShenandoahThreadLocalData::is_evac_allowed(Thread::current()), "STS should be left after evac scope");
- }
-};
-
-class ShenandoahSuspendibleThreadSetLeaver {
-private:
- SuspendibleThreadSetLeaver _leaver;
-public:
- ShenandoahSuspendibleThreadSetLeaver(bool active = true) : _leaver(active) {
- assert(!ShenandoahThreadLocalData::is_evac_allowed(Thread::current()), "STS should be left after evac scope");
- }
- ~ShenandoahSuspendibleThreadSetLeaver() {
- assert(!ShenandoahThreadLocalData::is_evac_allowed(Thread::current()), "STS should be joined before evac scope");
- }
-};
-
// Regions cannot be uncommitted when concurrent reset is zeroing out the bitmaps.
// This CADR class enforces this by forbidding region uncommits while it is in scope.
class ShenandoahNoUncommitMark : public StackObj {
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp
index 8299cbe62c6..5df88c0fc0a 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp
@@ -1073,7 +1073,7 @@ void ShenandoahVerifier::verify_at_safepoint(ShenandoahGeneration* generation,
log_info(gc)("Verify %s, Level %zd (%zu reachable, %zu marked)",
label, ShenandoahVerifyLevel, count_reachable, count_marked);
- FREE_C_HEAP_ARRAY(ShenandoahLivenessData, ld);
+ FREE_C_HEAP_ARRAY(ld);
}
void ShenandoahVerifier::verify_generic(ShenandoahGeneration* generation, VerifyOption vo) {
@@ -1187,7 +1187,7 @@ void ShenandoahVerifier::verify_after_update_refs(ShenandoahGeneration* generati
"After Updating References",
_verify_remembered_disable, // do not verify remembered set
_verify_forwarded_none, // no forwarded references
- _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well
+ _verify_marked_disable, // no need to check unreachable objects, end of cycle
_verify_cset_none, // no cset references, all updated
_verify_liveness_disable, // no reliable liveness data anymore
_verify_regions_nocset, // no cset regions, trash regions have appeared
@@ -1204,7 +1204,7 @@ void ShenandoahVerifier::verify_after_gc(ShenandoahGeneration* generation) {
"After GC",
_verify_remembered_disable, // do not verify remembered set
_verify_forwarded_none, // no forwarded references
- _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well
+ _verify_marked_disable, // no need to check unreachable objects, end of cycle
_verify_cset_none, // no cset references, all updated
_verify_liveness_disable, // no reliable liveness data anymore
_verify_regions_nocset, // no cset regions, trash regions have appeared
@@ -1220,7 +1220,7 @@ void ShenandoahVerifier::verify_after_degenerated(ShenandoahGeneration* generati
"After Degenerated GC",
_verify_remembered_disable, // do not verify remembered set
_verify_forwarded_none, // all objects are non-forwarded
- _verify_marked_complete, // all objects are marked in complete bitmap
+ _verify_marked_disable, // no need to check unreachable objects, end of cycle
_verify_cset_none, // no cset references
_verify_liveness_disable, // no reliable liveness data anymore
_verify_regions_notrash_nocset, // no trash, no cset
@@ -1248,14 +1248,14 @@ void ShenandoahVerifier::verify_after_fullgc(ShenandoahGeneration* generation) {
verify_at_safepoint(
generation,
"After Full GC",
- _verify_remembered_after_full_gc, // verify read-write remembered set
- _verify_forwarded_none, // all objects are non-forwarded
- _verify_marked_incomplete, // all objects are marked in incomplete bitmap
- _verify_cset_none, // no cset references
- _verify_liveness_disable, // no reliable liveness data anymore
- _verify_regions_notrash_nocset, // no trash, no cset
- _verify_size_exact, // expect generation and heap sizes to match exactly
- _verify_gcstate_stable // full gc cleaned up everything
+ _verify_remembered_after_full_gc, // verify read-write remembered set
+ _verify_forwarded_none, // all objects are non-forwarded
+ _verify_marked_disable, // no need to check unreachable objects, end of cycle
+ _verify_cset_none, // no cset references
+ _verify_liveness_disable, // no reliable liveness data anymore
+ _verify_regions_notrash_nocset, // no trash, no cset
+ _verify_size_exact, // expect generation and heap sizes to match exactly
+ _verify_gcstate_stable // full gc cleaned up everything
);
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp
index 2c5ba726ef2..d26959edf89 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp
@@ -196,14 +196,6 @@
"of regions that would be used, within min/max region size " \
"limits.") \
\
- product(size_t, ShenandoahMinRegionSize, 256 * K, EXPERIMENTAL, \
- "With automatic region sizing, the regions would be at least " \
- "this large.") \
- \
- product(size_t, ShenandoahMaxRegionSize, 32 * M, EXPERIMENTAL, \
- "With automatic region sizing, the regions would be at most " \
- "this large.") \
- \
product(ccstr, ShenandoahGCMode, "satb", \
"GC mode to use. Among other things, this defines which " \
"barriers are in in use. Possible values are:" \
diff --git a/src/hotspot/share/gc/z/zForwardingAllocator.cpp b/src/hotspot/share/gc/z/zForwardingAllocator.cpp
index 451a1d62754..37b5b8f520c 100644
--- a/src/hotspot/share/gc/z/zForwardingAllocator.cpp
+++ b/src/hotspot/share/gc/z/zForwardingAllocator.cpp
@@ -30,11 +30,11 @@ ZForwardingAllocator::ZForwardingAllocator()
_top(nullptr) {}
ZForwardingAllocator::~ZForwardingAllocator() {
- FREE_C_HEAP_ARRAY(char, _start);
+ FREE_C_HEAP_ARRAY(_start);
}
void ZForwardingAllocator::reset(size_t size) {
- _start = REALLOC_C_HEAP_ARRAY(char, _start, size, mtGC);
+ _start = REALLOC_C_HEAP_ARRAY(_start, size, mtGC);
_top.store_relaxed(_start);
_end = _start + size;
}
diff --git a/src/hotspot/share/gc/z/zIterator.hpp b/src/hotspot/share/gc/z/zIterator.hpp
index e048002e52e..0e9ef808dff 100644
--- a/src/hotspot/share/gc/z/zIterator.hpp
+++ b/src/hotspot/share/gc/z/zIterator.hpp
@@ -31,11 +31,14 @@ class ZIterator : AllStatic {
private:
static bool is_invisible_object(oop obj);
static bool is_invisible_object_array(oop obj);
+ static bool is_invisible_object_array(oop obj, Klass* klass);
public:
// This iterator skips invisible roots
template
static void oop_iterate_safe(oop obj, OopClosureT* cl);
+ template
+ static void oop_iterate_safe(oop obj, Klass* klass, OopClosureT* cl);
template
static void oop_iterate(oop obj, OopClosureT* cl);
@@ -46,6 +49,8 @@ public:
// This function skips invisible roots
template
static void basic_oop_iterate_safe(oop obj, Function function);
+ template
+ static void basic_oop_iterate_safe(oop obj, Klass* klass, Function function);
template
static void basic_oop_iterate(oop obj, Function function);
diff --git a/src/hotspot/share/gc/z/zIterator.inline.hpp b/src/hotspot/share/gc/z/zIterator.inline.hpp
index cbfe1a79aaf..6e51929c7b4 100644
--- a/src/hotspot/share/gc/z/zIterator.inline.hpp
+++ b/src/hotspot/share/gc/z/zIterator.inline.hpp
@@ -45,17 +45,27 @@ inline bool ZIterator::is_invisible_object(oop obj) {
}
inline bool ZIterator::is_invisible_object_array(oop obj) {
- return obj->klass()->is_objArray_klass() && is_invisible_object(obj);
+ return is_invisible_object_array(obj, obj->klass());
}
-// This iterator skips invisible object arrays
+inline bool ZIterator::is_invisible_object_array(oop obj, Klass* klass) {
+ return klass->is_objArray_klass() && is_invisible_object(obj);
+}
+
+// These iterators skips invisible object arrays
+
template
void ZIterator::oop_iterate_safe(oop obj, OopClosureT* cl) {
+ oop_iterate_safe(obj, obj->klass(), cl);
+}
+
+template
+void ZIterator::oop_iterate_safe(oop obj, Klass* klass, OopClosureT* cl) {
// Skip invisible object arrays - we only filter out *object* arrays,
// because that check is arguably faster than the is_invisible_object
// check, and primitive arrays are cheap to call oop_iterate on.
- if (!is_invisible_object_array(obj)) {
- obj->oop_iterate(cl);
+ if (!is_invisible_object_array(obj, klass)) {
+ OopIteratorClosureDispatch::oop_oop_iterate(cl, obj, klass);
}
}
@@ -89,11 +99,17 @@ public:
}
};
-// This function skips invisible roots
+// These functions skip invisible roots
+
template
void ZIterator::basic_oop_iterate_safe(oop obj, Function function) {
+ basic_oop_iterate_safe(obj, obj->klass(), function);
+}
+
+template
+void ZIterator::basic_oop_iterate_safe(oop obj, Klass* klass, Function function) {
ZBasicOopIterateClosure cl(function);
- oop_iterate_safe(obj, &cl);
+ oop_iterate_safe(obj, klass, &cl);
}
template
diff --git a/src/hotspot/share/gc/z/zRelocate.cpp b/src/hotspot/share/gc/z/zRelocate.cpp
index d51cf5abbae..1c2a4078904 100644
--- a/src/hotspot/share/gc/z/zRelocate.cpp
+++ b/src/hotspot/share/gc/z/zRelocate.cpp
@@ -1272,7 +1272,7 @@ public:
for (ZPage* page; _iter.next(&page);) {
page->object_iterate([&](oop obj) {
// Remap oops and add remset if needed
- ZIterator::basic_oop_iterate_safe(obj, remap_and_maybe_add_remset);
+ ZIterator::basic_oop_iterate_safe(obj, obj->klass(), remap_and_maybe_add_remset);
// String dedup
string_dedup_context.request(obj);
diff --git a/src/hotspot/share/interpreter/bytecodeTracer.cpp b/src/hotspot/share/interpreter/bytecodeTracer.cpp
index 4578a3eec4e..cbe28e92c51 100644
--- a/src/hotspot/share/interpreter/bytecodeTracer.cpp
+++ b/src/hotspot/share/interpreter/bytecodeTracer.cpp
@@ -40,21 +40,17 @@
#include "utilities/align.hpp"
// Prints the current bytecode and its attributes using bytecode-specific information.
+// Printing bytecodes is not an idempotent operation, as this relies on modifying state.
+// An instance of this class is thus intended to only print a single bytecode. All state
+// shared between multiple bytecodes needs to be in BytecodeTracerData.
class BytecodePrinter {
private:
- // %%% This field is not GC-ed, and so can contain garbage
- // between critical sections. Use only pointer-comparison
- // operations on the pointer, except within a critical section.
- // (Also, ensure that occasional false positives are benign.)
- Method* _current_method;
- bool _is_wide;
- Bytecodes::Code _code;
- address _next_pc; // current decoding position
- int _flags;
- bool _use_cp_cache;
+ BytecodeTracerData* _data;
+ int _flags;
+ Bytecodes::Code _raw_code; // note: some methods translate this to the Java bytecode
+ address _next_pc; // current decoding position, destructive
- bool use_cp_cache() const { return _use_cp_cache; }
void align() { _next_pc = align_up(_next_pc, sizeof(jint)); }
int get_byte() { return *(jbyte*) _next_pc++; } // signed
int get_index_u1() { return *(address)_next_pc++; } // returns 0x00 - 0xff as an int
@@ -65,11 +61,22 @@ class BytecodePrinter {
int get_Java_index_u2() { int i = Bytes::get_Java_u2 (_next_pc); _next_pc += 2; return i; }
int get_Java_index_u4() { int i = Bytes::get_Java_u4 (_next_pc); _next_pc += 4; return i; }
int get_index_special() { return (is_wide()) ? get_Java_index_u2() : get_index_u1(); }
- Method* method() const { return _current_method; }
- bool is_wide() const { return _is_wide; }
- Bytecodes::Code raw_code() const { return Bytecodes::Code(_code); }
- ConstantPool* constants() const { return method()->constants(); }
- ConstantPoolCache* cpcache() const { assert(use_cp_cache(), "must be"); return constants()->cache(); }
+
+ // Warning: only do pointer comparison between critical sections.
+ Method* method() const { return _data->current_method(); }
+ void set_method(Method* current) { _data->set_current_method(current); }
+
+ // Warning: should only be used for comparison and not dereferenced.
+ intptr_t* fp() const { return _data->current_fp(); }
+ void set_fp(intptr_t* current) { _data->set_current_fp(current); }
+
+ bool is_wide() const { return _data->is_wide(); }
+ void set_wide(bool wide) { _data->set_wide(wide); }
+
+ ConstantPool* constants() const { return method()->constants(); }
+ // This may be called during linking after bytecodes are rewritten to point to the cpCache.
+ bool use_cp_cache() const { return constants()->cache() != nullptr; }
+ ConstantPoolCache* cpcache() const { assert(use_cp_cache(), "must be"); return constants()->cache(); }
void print_constant(int i, outputStream* st);
void print_cpcache_entry(int cpc_index, outputStream* st);
@@ -81,35 +88,32 @@ class BytecodePrinter {
void print_method_data_at(int bci, outputStream* st);
public:
- BytecodePrinter(int flags = 0) : _is_wide(false), _code(Bytecodes::_illegal), _flags(flags) {}
+ BytecodePrinter(BytecodeTracerData* data, int flags = 0) :
+ _data(data),
+ _flags(flags),
+ _raw_code(Bytecodes::_illegal),
+ _next_pc(nullptr) {}
#ifndef PRODUCT
- BytecodePrinter(Method* prev_method) : BytecodePrinter(0) {
- _current_method = prev_method;
- }
-
// This method is called while executing the raw bytecodes, so none of
// the adjustments that BytecodeStream performs applies.
- void trace(const methodHandle& method, address bcp, uintptr_t tos, uintptr_t tos2, outputStream* st) {
+ void trace(const methodHandle& method, intptr_t* fp, address bcp, uintptr_t tos, uintptr_t tos2, outputStream* st) {
+ assert(_raw_code == Bytecodes::_illegal, "invariant");
ResourceMark rm;
- bool method_changed = _current_method != method();
- _current_method = method();
- _use_cp_cache = method->constants()->cache() != nullptr;
+ // Method changes can be another method getting called, or a self-recursive call.
+ bool method_changed = (this->method() != method()) || (this->fp() != fp);
+ set_method(method());
+ set_fp(fp);
assert(method->method_holder()->is_linked(),
"this function must be called on methods that are already executing");
-
+ // If the method changed (new method call, return to previous method after call finishes),
+ // the signature needs to be re-printed for interpretability.
if (method_changed) {
- // Note 1: This code will not work as expected with true MT/MP.
- // Need an explicit lock or a different solution.
- // It is possible for this block to be skipped, if a garbage
- // _current_method pointer happens to have the same bits as
- // the incoming method. We could lose a line of trace output.
- // This is acceptable in a debug-only feature.
- st->cr();
st->print("[%zu] ", Thread::current()->osthread()->thread_id_for_printing());
method->print_name(st);
st->cr();
}
+
Bytecodes::Code code;
if (is_wide()) {
// bcp wasn't advanced if previous bytecode was _wide.
@@ -117,14 +121,13 @@ class BytecodePrinter {
} else {
code = Bytecodes::code_at(method(), bcp);
}
- _code = code;
+ _raw_code = code;
_next_pc = is_wide() ? bcp+2 : bcp+1;
+
+ bool is_terminal = Bytecodes::is_terminal(code);
// Trace each bytecode unless we're truncating the tracing output, then only print the first
- // bytecode in every method as well as returns/throws that pop control flow
- if (!TraceBytecodesTruncated || method_changed ||
- code == Bytecodes::_athrow ||
- code == Bytecodes::_return_register_finalizer ||
- (code >= Bytecodes::_ireturn && code <= Bytecodes::_return)) {
+ // bytecode in every method as well as returns/throws that pop control flow.
+ if (!TraceBytecodesTruncated || method_changed || is_terminal) {
int bci = (int)(bcp - method->code_base());
st->print("[%zu] ", Thread::current()->osthread()->thread_id_for_printing());
if (Verbose) {
@@ -138,8 +141,16 @@ class BytecodePrinter {
}
// Set is_wide for the next one, since the caller of this doesn't skip
// the next bytecode.
- _is_wide = (code == Bytecodes::_wide);
- _code = Bytecodes::_illegal;
+ set_wide(code == Bytecodes::_wide);
+ // Finished using the code, reset to illegal.
+ _raw_code = Bytecodes::_illegal;
+
+ // Invalidate the current method to force a signature change. In some
+ // rare cases, the method and frame pointers aren't enough to determine
+ // a new method invocation, so this ensures the signature is re-printed.
+ if (is_terminal) {
+ set_method(nullptr);
+ }
if (TraceBytecodesStopAt != 0 && BytecodeCounter::counter_value() >= TraceBytecodesStopAt) {
TraceBytecodes = false;
@@ -150,17 +161,16 @@ class BytecodePrinter {
// Used for Method::print_codes(). The input bcp comes from
// BytecodeStream, which will skip wide bytecodes.
void trace(const methodHandle& method, address bcp, outputStream* st) {
- _current_method = method();
- // This may be called during linking after bytecodes are rewritten to point to the cpCache.
- _use_cp_cache = method->constants()->cache() != nullptr;
+ assert(_raw_code == Bytecodes::_illegal, "invariant");
+ set_method(method());
ResourceMark rm;
Bytecodes::Code code = Bytecodes::code_at(method(), bcp);
// Set is_wide
- _is_wide = (code == Bytecodes::_wide);
+ set_wide(code == Bytecodes::_wide);
if (is_wide()) {
code = Bytecodes::code_at(method(), bcp+1);
}
- _code = code;
+ _raw_code = code;
int bci = (int)(bcp - method->code_base());
// Print bytecode index and name
if (ClassPrinter::has_mode(_flags, ClassPrinter::PRINT_BYTECODE_ADDR)) {
@@ -180,26 +190,20 @@ class BytecodePrinter {
};
#ifndef PRODUCT
-// We need a global instance to keep track of the method being printed so we can report that
-// the method has changed. If this method is redefined and removed, that's ok because the method passed
-// in won't match, and this will print the method passed in again. Racing threads changing this global
-// will result in reprinting the method passed in again.
-static Method* _method_currently_being_printed = nullptr;
-
-void BytecodeTracer::trace_interpreter(const methodHandle& method, address bcp, uintptr_t tos, uintptr_t tos2, outputStream* st) {
+void BytecodeTracer::trace_interpreter(const methodHandle& method, intptr_t* fp, address bcp, uintptr_t tos, uintptr_t tos2, outputStream* st) {
if (TraceBytecodes && BytecodeCounter::counter_value() >= TraceBytecodesAt) {
- BytecodePrinter printer(AtomicAccess::load_acquire(&_method_currently_being_printed));
- stringStream buf;
- printer.trace(method, bcp, tos, tos2, &buf);
- st->print("%s", buf.freeze());
- // Save method currently being printed to detect when method printing changes.
- AtomicAccess::release_store(&_method_currently_being_printed, method());
+ BytecodeTracerData* data = JavaThread::current()->bytecode_tracer_data();
+ BytecodePrinter printer(data);
+ printer.trace(method, fp, bcp, tos, tos2, st);
}
}
#endif
void BytecodeTracer::print_method_codes(const methodHandle& method, int from, int to, outputStream* st, int flags, bool buffered) {
- BytecodePrinter method_printer(flags);
+ // Debug builds can't re-use the data in the Java Thread as that is used for tracing
+ // the current bytecodes, rather than to print diagnostic information as is the
+ // case here. Always stack-allocate for this printing.
+ BytecodeTracerData data;
BytecodeStream s(method);
s.set_interval(from, to);
@@ -207,6 +211,7 @@ void BytecodeTracer::print_method_codes(const methodHandle& method, int from, in
stringStream ss;
outputStream* out = buffered ? &ss : st;
while (s.next() >= 0) {
+ BytecodePrinter method_printer(&data, flags);
method_printer.trace(method, s.bcp(), out);
}
if (buffered) {
@@ -345,7 +350,7 @@ void BytecodePrinter::print_bsm(int cp_index, outputStream* st) {
void BytecodePrinter::print_attributes(int bci, outputStream* st) {
// Show attributes of pre-rewritten codes
- Bytecodes::Code code = Bytecodes::java_code(raw_code());
+ Bytecodes::Code code = Bytecodes::java_code(_raw_code);
// If the code doesn't have any fields there's nothing to print.
// note this is ==1 because the tableswitch and lookupswitch are
// zero size (for some reason) and we want to print stuff out for them.
@@ -366,7 +371,7 @@ void BytecodePrinter::print_attributes(int bci, outputStream* st) {
case Bytecodes::_ldc:
{
int cp_index;
- if (Bytecodes::uses_cp_cache(raw_code())) {
+ if (Bytecodes::uses_cp_cache(_raw_code)) {
assert(use_cp_cache(), "fast ldc bytecode must be in linked classes");
int obj_index = get_index_u1();
cp_index = constants()->object_to_cp_index(obj_index);
@@ -381,7 +386,7 @@ void BytecodePrinter::print_attributes(int bci, outputStream* st) {
case Bytecodes::_ldc2_w:
{
int cp_index;
- if (Bytecodes::uses_cp_cache(raw_code())) {
+ if (Bytecodes::uses_cp_cache(_raw_code)) {
assert(use_cp_cache(), "fast ldc bytecode must be in linked classes");
int obj_index = get_native_index_u2();
cp_index = constants()->object_to_cp_index(obj_index);
@@ -533,7 +538,7 @@ void BytecodePrinter::print_attributes(int bci, outputStream* st) {
cp_index = method_entry->constant_pool_index();
print_field_or_method(cp_index, st);
- if (raw_code() == Bytecodes::_invokehandle &&
+ if (_raw_code == Bytecodes::_invokehandle &&
ClassPrinter::has_mode(_flags, ClassPrinter::PRINT_METHOD_HANDLE)) {
assert(use_cp_cache(), "invokehandle is only in rewritten methods");
method_entry->print_on(st);
diff --git a/src/hotspot/share/interpreter/bytecodeTracer.hpp b/src/hotspot/share/interpreter/bytecodeTracer.hpp
index ab66030b6cd..03340fbe5bf 100644
--- a/src/hotspot/share/interpreter/bytecodeTracer.hpp
+++ b/src/hotspot/share/interpreter/bytecodeTracer.hpp
@@ -25,21 +25,54 @@
#ifndef SHARE_INTERPRETER_BYTECODETRACER_HPP
#define SHARE_INTERPRETER_BYTECODETRACER_HPP
+#include "interpreter/bytecodes.hpp"
#include "memory/allStatic.hpp"
#include "utilities/globalDefinitions.hpp"
-// The BytecodeTracer is a helper class used by the interpreter for run-time
-// bytecode tracing. If TraceBytecodes turned on, trace_interpreter() will be called
-// for each bytecode.
-
+class Method;
class methodHandle;
class outputStream;
-
class BytecodeClosure;
+
+// The BytecodeTracer is a helper class used by the interpreter for run-time
+// bytecode tracing. If TraceBytecodes is turned on, trace_interpreter() will be called
+// for each bytecode.
class BytecodeTracer: AllStatic {
public:
- NOT_PRODUCT(static void trace_interpreter(const methodHandle& method, address bcp, uintptr_t tos, uintptr_t tos2, outputStream* st);)
+ NOT_PRODUCT(static void trace_interpreter(const methodHandle& method, intptr_t* fp, address bcp, uintptr_t tos, uintptr_t tos2, outputStream* st);)
static void print_method_codes(const methodHandle& method, int from, int to, outputStream* st, int flags, bool buffered = true);
};
+// Provides tracing-centric context whose lifespan exceeds the printing of
+// a single bytecode. For instance, it is needed to determine method switches
+// in order to print the appropriate signature once a switch happens.
+class BytecodeTracerData {
+ private:
+ Method* _current_method; // for method switches
+ intptr_t* _current_fp; // for self-recursion
+ bool _is_wide; // to parse the next bytecode properly
+
+ public:
+ BytecodeTracerData() : _current_method(nullptr),
+ _current_fp(nullptr),
+ _is_wide(false) {}
+
+ // The current method may point to a stale/garbage Method. While pointer
+ // comparison is safe, it should only be dereferenced while guaranteed to
+ // be valid. For example, if the current method is set to the result of a
+ // methodHandle call, current_method() may be dereferenced while the handle
+ // is live. It is always up to the caller to ensure that current_method()
+ // is safe to dereference.
+ Method* current_method() const { return _current_method; }
+ void set_current_method(Method* current) { _current_method = current; }
+
+ // The frame pointer should only ever be used for pointer comparison and may
+ // never be dereferenced.
+ intptr_t* current_fp() const { return _current_fp; }
+ void set_current_fp(intptr_t* current) { _current_fp = current; }
+
+ bool is_wide() const { return _is_wide; }
+ void set_wide(bool wide) { _is_wide = wide; }
+};
+
#endif // SHARE_INTERPRETER_BYTECODETRACER_HPP
diff --git a/src/hotspot/share/interpreter/bytecodes.hpp b/src/hotspot/share/interpreter/bytecodes.hpp
index 629cca706ae..6f77e95f5bd 100644
--- a/src/hotspot/share/interpreter/bytecodes.hpp
+++ b/src/hotspot/share/interpreter/bytecodes.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -418,6 +418,9 @@ class Bytecodes: AllStatic {
static bool is_zero_const (Code code) { return (code == _aconst_null || code == _iconst_0
|| code == _fconst_0 || code == _dconst_0); }
static bool is_return (Code code) { return (_ireturn <= code && code <= _return); }
+ static bool is_terminal (Code code) { return is_return(code) ||
+ code == _return_register_finalizer ||
+ code == _athrow; }
static bool is_invoke (Code code) { return (_invokevirtual <= code && code <= _invokedynamic); }
static bool is_field_code (Code code) { return (_getstatic <= java_code(code) && java_code(code) <= _putfield); }
static bool has_receiver (Code code) { assert(is_invoke(code), ""); return code == _invokevirtual ||
diff --git a/src/hotspot/share/interpreter/interpreterRuntime.cpp b/src/hotspot/share/interpreter/interpreterRuntime.cpp
index cd0a062ebc8..375cb402892 100644
--- a/src/hotspot/share/interpreter/interpreterRuntime.cpp
+++ b/src/hotspot/share/interpreter/interpreterRuntime.cpp
@@ -1487,23 +1487,32 @@ JRT_ENTRY(void, InterpreterRuntime::member_name_arg_or_null(JavaThread* current,
Method* method, address bcp))
Bytecodes::Code code = Bytecodes::code_at(method, bcp);
if (code != Bytecodes::_invokestatic) {
+ current->set_vm_result_oop(nullptr);
return;
}
+
ConstantPool* cpool = method->constants();
int cp_index = Bytes::get_native_u2(bcp + 1);
Symbol* cname = cpool->klass_name_at(cpool->klass_ref_index_at(cp_index, code));
Symbol* mname = cpool->name_ref_at(cp_index, code);
- if (MethodHandles::has_member_arg(cname, mname)) {
- oop member_name_oop = cast_to_oop(member_name);
- if (java_lang_invoke_DirectMethodHandle::is_instance(member_name_oop)) {
- // FIXME: remove after j.l.i.InvokerBytecodeGenerator code shape is updated.
- member_name_oop = java_lang_invoke_DirectMethodHandle::member(member_name_oop);
- }
- current->set_vm_result_oop(member_name_oop);
- } else {
+ if (!MethodHandles::has_member_arg(cname, mname)) {
current->set_vm_result_oop(nullptr);
+ return;
}
+
+ oop member_name_oop = cast_to_oop(member_name);
+
+ guarantee(member_name_oop != nullptr, "member_name_oop should not be nullptr");
+ guarantee(oopDesc::is_oop(member_name_oop), "member_name_oop should be an oop");
+ guarantee(java_lang_invoke_MemberName::is_instance(member_name_oop) ||
+ java_lang_invoke_DirectMethodHandle::is_instance(member_name_oop),
+ "member_name_oop is not MemberName or DMH");
+
+ if (java_lang_invoke_DirectMethodHandle::is_instance(member_name_oop)) {
+ member_name_oop = java_lang_invoke_DirectMethodHandle::member(member_name_oop);
+ }
+ current->set_vm_result_oop(member_name_oop);
JRT_END
#endif // INCLUDE_JVMTI
@@ -1517,7 +1526,9 @@ JRT_LEAF(intptr_t, InterpreterRuntime::trace_bytecode(JavaThread* current, intpt
LastFrameAccessor last_frame(current);
assert(last_frame.is_interpreted_frame(), "must be an interpreted frame");
methodHandle mh(current, last_frame.method());
- BytecodeTracer::trace_interpreter(mh, last_frame.bcp(), tos, tos2, tty);
+ stringStream st;
+ BytecodeTracer::trace_interpreter(mh, last_frame.get_frame().real_fp(), last_frame.bcp(), tos, tos2, &st);
+ tty->print("%s", st.freeze());
return preserve_this_value;
JRT_END
#endif // !PRODUCT
diff --git a/src/hotspot/share/interpreter/oopMapCache.cpp b/src/hotspot/share/interpreter/oopMapCache.cpp
index af45f7f9bed..34e226b00bf 100644
--- a/src/hotspot/share/interpreter/oopMapCache.cpp
+++ b/src/hotspot/share/interpreter/oopMapCache.cpp
@@ -156,7 +156,7 @@ InterpreterOopMap::InterpreterOopMap() {
InterpreterOopMap::~InterpreterOopMap() {
if (has_valid_mask() && mask_size() > small_mask_limit) {
assert(_bit_mask[0] != 0, "should have pointer to C heap");
- FREE_C_HEAP_ARRAY(uintptr_t, _bit_mask[0]);
+ FREE_C_HEAP_ARRAY((uintptr_t*)_bit_mask[0]);
}
}
@@ -288,7 +288,7 @@ void OopMapCacheEntry::deallocate_bit_mask() {
if (mask_size() > small_mask_limit && _bit_mask[0] != 0) {
assert(!Thread::current()->resource_area()->contains((void*)_bit_mask[0]),
"This bit mask should not be in the resource area");
- FREE_C_HEAP_ARRAY(uintptr_t, _bit_mask[0]);
+ FREE_C_HEAP_ARRAY((uintptr_t*)_bit_mask[0]);
DEBUG_ONLY(_bit_mask[0] = 0;)
}
}
diff --git a/src/hotspot/share/jfr/jni/jfrJavaSupport.cpp b/src/hotspot/share/jfr/jni/jfrJavaSupport.cpp
index a6e97ab227a..32710595d27 100644
--- a/src/hotspot/share/jfr/jni/jfrJavaSupport.cpp
+++ b/src/hotspot/share/jfr/jni/jfrJavaSupport.cpp
@@ -528,7 +528,7 @@ const char* JfrJavaSupport::c_str(jstring string, Thread* thread, bool c_heap /*
void JfrJavaSupport::free_c_str(const char* str, bool c_heap) {
if (c_heap) {
- FREE_C_HEAP_ARRAY(char, str);
+ FREE_C_HEAP_ARRAY(str);
}
}
diff --git a/src/hotspot/share/jfr/leakprofiler/leakProfiler.cpp b/src/hotspot/share/jfr/leakprofiler/leakProfiler.cpp
index e8ad79783c0..9098a0af6b3 100644
--- a/src/hotspot/share/jfr/leakprofiler/leakProfiler.cpp
+++ b/src/hotspot/share/jfr/leakprofiler/leakProfiler.cpp
@@ -34,9 +34,15 @@
#include "runtime/vmThread.hpp"
bool LeakProfiler::is_supported() {
- if (UseShenandoahGC) {
+ if (UseShenandoahGC || UseZGC) {
// Leak Profiler uses mark words in the ways that might interfere
// with concurrent GC uses of them. This affects Shenandoah.
+ //
+ // Generational ZGC only does weak reference processing in the old generation.
+ // All objects that would usually die, because we are sampling stuff
+ // that immediately becomes garbage, will be artificially kept alive
+ // until an old-generation collection. This incurs a significant
+ // performance hit by causing allocation stalls.
return false;
}
return true;
@@ -58,7 +64,8 @@ bool LeakProfiler::start(int sample_count) {
// Exit cleanly if not supported
if (!is_supported()) {
- log_trace(jfr, system)("Object sampling is not supported");
+ log_info(jfr, system)("jdk.OldObjectSample event is currently not supported for %s.",
+ UseShenandoahGC ? "ShenandoahGC" : "ZGC");
return false;
}
diff --git a/src/hotspot/share/jfr/leakprofiler/sampling/samplePriorityQueue.cpp b/src/hotspot/share/jfr/leakprofiler/sampling/samplePriorityQueue.cpp
index 644cd25ec8a..5eac068467f 100644
--- a/src/hotspot/share/jfr/leakprofiler/sampling/samplePriorityQueue.cpp
+++ b/src/hotspot/share/jfr/leakprofiler/sampling/samplePriorityQueue.cpp
@@ -36,7 +36,7 @@ SamplePriorityQueue::SamplePriorityQueue(size_t size) :
}
SamplePriorityQueue::~SamplePriorityQueue() {
- FREE_C_HEAP_ARRAY(ObjectSample*, _items);
+ FREE_C_HEAP_ARRAY(_items);
_items = nullptr;
}
diff --git a/src/hotspot/share/jfr/periodic/jfrNetworkUtilization.cpp b/src/hotspot/share/jfr/periodic/jfrNetworkUtilization.cpp
index 11e211f6505..1650ad7d9c0 100644
--- a/src/hotspot/share/jfr/periodic/jfrNetworkUtilization.cpp
+++ b/src/hotspot/share/jfr/periodic/jfrNetworkUtilization.cpp
@@ -46,7 +46,7 @@ static GrowableArray* _interfaces = nullptr;
void JfrNetworkUtilization::destroy() {
if (_interfaces != nullptr) {
for (int i = 0; i < _interfaces->length(); ++i) {
- FREE_C_HEAP_ARRAY(char, _interfaces->at(i).name);
+ FREE_C_HEAP_ARRAY(_interfaces->at(i).name);
}
delete _interfaces;
_interfaces = nullptr;
diff --git a/src/hotspot/share/jfr/periodic/jfrPeriodic.cpp b/src/hotspot/share/jfr/periodic/jfrPeriodic.cpp
index 969c9ca60c1..92f406bd095 100644
--- a/src/hotspot/share/jfr/periodic/jfrPeriodic.cpp
+++ b/src/hotspot/share/jfr/periodic/jfrPeriodic.cpp
@@ -256,10 +256,7 @@ TRACE_REQUEST_FUNC(SystemProcess) {
// feature is implemented, write real event
while (processes != nullptr) {
SystemProcess* tmp = processes;
- const char* info = processes->command_line();
- if (info == nullptr) {
- info = processes->path();
- }
+ const char* info = processes->path();
if (info == nullptr) {
info = processes->name();
}
diff --git a/src/hotspot/share/jfr/recorder/service/jfrOptionSet.cpp b/src/hotspot/share/jfr/recorder/service/jfrOptionSet.cpp
index b212ee4ccba..ec8ab36d75a 100644
--- a/src/hotspot/share/jfr/recorder/service/jfrOptionSet.cpp
+++ b/src/hotspot/share/jfr/recorder/service/jfrOptionSet.cpp
@@ -799,7 +799,7 @@ void JfrOptionSet::release_start_flight_recording_options() {
if (start_flight_recording_options_array != nullptr) {
const int length = start_flight_recording_options_array->length();
for (int i = 0; i < length; ++i) {
- FREE_C_HEAP_ARRAY(char, start_flight_recording_options_array->at(i));
+ FREE_C_HEAP_ARRAY(start_flight_recording_options_array->at(i));
}
delete start_flight_recording_options_array;
start_flight_recording_options_array = nullptr;
diff --git a/src/hotspot/share/jfr/recorder/stacktrace/jfrStackFilter.cpp b/src/hotspot/share/jfr/recorder/stacktrace/jfrStackFilter.cpp
index 1eb057b564e..39a1c621888 100644
--- a/src/hotspot/share/jfr/recorder/stacktrace/jfrStackFilter.cpp
+++ b/src/hotspot/share/jfr/recorder/stacktrace/jfrStackFilter.cpp
@@ -55,6 +55,6 @@ JfrStackFilter::~JfrStackFilter() {
Symbol::maybe_decrement_refcount(_method_names[i]);
Symbol::maybe_decrement_refcount(_class_names[i]);
}
- FREE_C_HEAP_ARRAY(Symbol*, _method_names);
- FREE_C_HEAP_ARRAY(Symbol*, _class_names);
+ FREE_C_HEAP_ARRAY(_method_names);
+ FREE_C_HEAP_ARRAY(_class_names);
}
diff --git a/src/hotspot/share/jfr/recorder/stacktrace/jfrStackFilterRegistry.cpp b/src/hotspot/share/jfr/recorder/stacktrace/jfrStackFilterRegistry.cpp
index 0721604b1c1..cb9811e7cae 100644
--- a/src/hotspot/share/jfr/recorder/stacktrace/jfrStackFilterRegistry.cpp
+++ b/src/hotspot/share/jfr/recorder/stacktrace/jfrStackFilterRegistry.cpp
@@ -43,8 +43,8 @@ int64_t JfrStackFilterRegistry::add(jobjectArray classes, jobjectArray methods,
Symbol** method_names = JfrJavaSupport::symbol_array(methods, jt, &m_size, true);
assert(method_names != nullptr, "invariant");
if (c_size != m_size) {
- FREE_C_HEAP_ARRAY(Symbol*, class_names);
- FREE_C_HEAP_ARRAY(Symbol*, method_names);
+ FREE_C_HEAP_ARRAY(class_names);
+ FREE_C_HEAP_ARRAY(method_names);
JfrJavaSupport::throw_internal_error("Method array size doesn't match class array size", jt);
return STACK_FILTER_ERROR_CODE;
}
diff --git a/src/hotspot/share/jfr/support/methodtracer/jfrFilter.cpp b/src/hotspot/share/jfr/support/methodtracer/jfrFilter.cpp
index 2a977f9e823..767036b57f2 100644
--- a/src/hotspot/share/jfr/support/methodtracer/jfrFilter.cpp
+++ b/src/hotspot/share/jfr/support/methodtracer/jfrFilter.cpp
@@ -51,10 +51,10 @@ JfrFilter::~JfrFilter() {
Symbol::maybe_decrement_refcount(_method_names[i]);
Symbol::maybe_decrement_refcount(_annotation_names[i]);
}
- FREE_C_HEAP_ARRAY(Symbol*, _class_names);
- FREE_C_HEAP_ARRAY(Symbol*, _method_names);
- FREE_C_HEAP_ARRAY(Symbol*, _annotation_names);
- FREE_C_HEAP_ARRAY(int, _modifications);
+ FREE_C_HEAP_ARRAY(_class_names);
+ FREE_C_HEAP_ARRAY(_method_names);
+ FREE_C_HEAP_ARRAY(_annotation_names);
+ FREE_C_HEAP_ARRAY(_modifications);
}
bool JfrFilter::can_instrument_module(const ModuleEntry* module) const {
diff --git a/src/hotspot/share/jfr/support/methodtracer/jfrFilterManager.cpp b/src/hotspot/share/jfr/support/methodtracer/jfrFilterManager.cpp
index d9081efa08c..ea031f9f205 100644
--- a/src/hotspot/share/jfr/support/methodtracer/jfrFilterManager.cpp
+++ b/src/hotspot/share/jfr/support/methodtracer/jfrFilterManager.cpp
@@ -130,10 +130,10 @@ bool JfrFilterManager::install(jobjectArray classes, jobjectArray methods, jobje
modifications[i] = modification_tah->int_at(i);
}
if (class_size != method_size || class_size != annotation_size || class_size != modification_size) {
- FREE_C_HEAP_ARRAY(Symbol*, class_names);
- FREE_C_HEAP_ARRAY(Symbol*, method_names);
- FREE_C_HEAP_ARRAY(Symbol*, annotation_names);
- FREE_C_HEAP_ARRAY(int, modifications);
+ FREE_C_HEAP_ARRAY(class_names);
+ FREE_C_HEAP_ARRAY(method_names);
+ FREE_C_HEAP_ARRAY(annotation_names);
+ FREE_C_HEAP_ARRAY(modifications);
JfrJavaSupport::throw_internal_error("Method array sizes don't match", jt);
return false;
}
diff --git a/src/hotspot/share/jfr/utilities/jfrConcurrentHashtable.inline.hpp b/src/hotspot/share/jfr/utilities/jfrConcurrentHashtable.inline.hpp
index 6d6908234a3..22028a96e55 100644
--- a/src/hotspot/share/jfr/utilities/jfrConcurrentHashtable.inline.hpp
+++ b/src/hotspot/share/jfr/utilities/jfrConcurrentHashtable.inline.hpp
@@ -59,7 +59,7 @@ inline JfrConcurrentHashtable::JfrConcurrentHashtable(uns
template class TableEntry>
inline JfrConcurrentHashtable::~JfrConcurrentHashtable() {
- FREE_C_HEAP_ARRAY(Bucket, _buckets);
+ FREE_C_HEAP_ARRAY(_buckets);
}
template class TableEntry>
diff --git a/src/hotspot/share/jfr/utilities/jfrHashtable.hpp b/src/hotspot/share/jfr/utilities/jfrHashtable.hpp
index 0be2d92ed19..ce4c920cf8b 100644
--- a/src/hotspot/share/jfr/utilities/jfrHashtable.hpp
+++ b/src/hotspot/share/jfr/utilities/jfrHashtable.hpp
@@ -93,7 +93,7 @@ class JfrBasicHashtable : public CHeapObj {
--_number_of_entries;
}
void free_buckets() {
- FREE_C_HEAP_ARRAY(Bucket, _buckets);
+ FREE_C_HEAP_ARRAY(_buckets);
}
TableEntry* bucket(size_t i) { return _buckets[i].get_entry();}
TableEntry** bucket_addr(size_t i) { return _buckets[i].entry_addr(); }
diff --git a/src/hotspot/share/jfr/utilities/jfrSet.hpp b/src/hotspot/share/jfr/utilities/jfrSet.hpp
index c443434800a..3e828d51ec3 100644
--- a/src/hotspot/share/jfr/utilities/jfrSet.hpp
+++ b/src/hotspot/share/jfr/utilities/jfrSet.hpp
@@ -82,7 +82,7 @@ class JfrSetStorage : public AnyObj {
~JfrSetStorage() {
if (CONFIG::alloc_type() == C_HEAP) {
- FREE_C_HEAP_ARRAY(K, _table);
+ FREE_C_HEAP_ARRAY(_table);
}
}
@@ -160,7 +160,7 @@ class JfrSet : public JfrSetStorage {
}
}
if (CONFIG::alloc_type() == AnyObj::C_HEAP) {
- FREE_C_HEAP_ARRAY(K, old_table);
+ FREE_C_HEAP_ARRAY(old_table);
}
assert(_table_mask + 1 == this->_table_size, "invariant");
assert(_resize_threshold << 1 == this->_table_size, "invariant");
diff --git a/src/hotspot/share/jvmci/jvmciCodeInstaller.cpp b/src/hotspot/share/jvmci/jvmciCodeInstaller.cpp
index ce617a4a514..6043b400e3b 100644
--- a/src/hotspot/share/jvmci/jvmciCodeInstaller.cpp
+++ b/src/hotspot/share/jvmci/jvmciCodeInstaller.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -818,9 +818,8 @@ JVMCI::CodeInstallResult CodeInstaller::install(JVMCICompiler* compiler,
cb = nm;
if (compile_state == nullptr) {
// This compile didn't come through the CompileBroker so perform the printing here
- DirectiveSet* directive = DirectivesStack::getMatchingDirective(method, compiler);
- nm->maybe_print_nmethod(directive);
- DirectivesStack::release(directive);
+ CompilerDirectiveMatcher matcher(method, CompLevel_full_optimization);
+ nm->maybe_print_nmethod(matcher.directive_set());
// Since this compilation didn't pass through the broker it wasn't logged yet.
if (PrintCompilation) {
diff --git a/src/hotspot/share/jvmci/jvmciCompilerToVM.cpp b/src/hotspot/share/jvmci/jvmciCompilerToVM.cpp
index a01f95d8cd5..5d0d2aedc62 100644
--- a/src/hotspot/share/jvmci/jvmciCompilerToVM.cpp
+++ b/src/hotspot/share/jvmci/jvmciCompilerToVM.cpp
@@ -585,7 +585,7 @@ C2V_END
C2V_VMENTRY_0(jboolean, hasNeverInlineDirective,(JNIEnv* env, jobject, ARGUMENT_PAIR(method)))
methodHandle method (THREAD, UNPACK_PAIR(Method, method));
- return !Inline || CompilerOracle::should_not_inline(method) || method->dont_inline();
+ return !Inline || CompilerOracle::should_not_inline(method, CompLevel_full_optimization) || method->dont_inline();
C2V_END
C2V_VMENTRY_0(jboolean, shouldInlineMethod,(JNIEnv* env, jobject, ARGUMENT_PAIR(method)))
diff --git a/src/hotspot/share/jvmci/vmStructs_jvmci.cpp b/src/hotspot/share/jvmci/vmStructs_jvmci.cpp
index 1fdf98588fd..4a1ddf13d60 100644
--- a/src/hotspot/share/jvmci/vmStructs_jvmci.cpp
+++ b/src/hotspot/share/jvmci/vmStructs_jvmci.cpp
@@ -1017,7 +1017,7 @@
static_field(VM_Version, _rop_protection, bool) \
volatile_nonstatic_field(JavaFrameAnchor, _last_Java_fp, intptr_t*)
-#define DECLARE_INT_CPU_FEATURE_CONSTANT(id, name, bit) GENERATE_VM_INT_CONSTANT_ENTRY(VM_Version::CPU_##id)
+#define DECLARE_INT_CPU_FEATURE_CONSTANT(id, name) GENERATE_VM_INT_CONSTANT_ENTRY(VM_Version::CPU_##id)
#define VM_INT_CPU_FEATURE_CONSTANTS CPU_FEATURE_FLAGS(DECLARE_INT_CPU_FEATURE_CONSTANT)
#endif
@@ -1037,7 +1037,7 @@
declare_constant(frame::interpreter_frame_sender_sp_offset) \
declare_constant(frame::interpreter_frame_last_sp_offset)
-#define DECLARE_LONG_CPU_FEATURE_CONSTANT(id, name, bit) GENERATE_VM_LONG_CONSTANT_ENTRY(VM_Version::CPU_##id)
+#define DECLARE_LONG_CPU_FEATURE_CONSTANT(id, name) GENERATE_VM_LONG_CONSTANT_ENTRY(VM_Version::CPU_##id)
#define VM_LONG_CPU_FEATURE_CONSTANTS \
CPU_FEATURE_FLAGS(DECLARE_LONG_CPU_FEATURE_CONSTANT)
diff --git a/src/hotspot/share/libadt/vectset.cpp b/src/hotspot/share/libadt/vectset.cpp
index 176660d8302..bd9e97d9877 100644
--- a/src/hotspot/share/libadt/vectset.cpp
+++ b/src/hotspot/share/libadt/vectset.cpp
@@ -51,7 +51,7 @@ void VectorSet::grow(uint new_word_capacity) {
assert(new_word_capacity < (1U << 30), "");
uint x = next_power_of_2(new_word_capacity);
if (x > _data_size) {
- _data = REALLOC_ARENA_ARRAY(_set_arena, uint32_t, _data, _size, x);
+ _data = REALLOC_ARENA_ARRAY(_set_arena, _data, _size, x);
_data_size = x;
}
Copy::zero_to_bytes(_data + _size, (x - _size) * sizeof(uint32_t));
diff --git a/src/hotspot/share/logging/logAsyncWriter.hpp b/src/hotspot/share/logging/logAsyncWriter.hpp
index a93b1ca58f6..933482929c7 100644
--- a/src/hotspot/share/logging/logAsyncWriter.hpp
+++ b/src/hotspot/share/logging/logAsyncWriter.hpp
@@ -124,7 +124,7 @@ class AsyncLogWriter : public NonJavaThread {
}
~Buffer() {
- FREE_C_HEAP_ARRAY(char, _buf);
+ FREE_C_HEAP_ARRAY(_buf);
}
void push_flush_token();
diff --git a/src/hotspot/share/logging/logConfiguration.cpp b/src/hotspot/share/logging/logConfiguration.cpp
index b883dbccacf..1ba45cc050f 100644
--- a/src/hotspot/share/logging/logConfiguration.cpp
+++ b/src/hotspot/share/logging/logConfiguration.cpp
@@ -123,7 +123,7 @@ void LogConfiguration::initialize(jlong vm_start_time) {
void LogConfiguration::finalize() {
disable_outputs();
- FREE_C_HEAP_ARRAY(LogOutput*, _outputs);
+ FREE_C_HEAP_ARRAY(_outputs);
}
// Normalizes the given LogOutput name to type=name form.
@@ -206,7 +206,7 @@ LogOutput* LogConfiguration::new_output(const char* name,
size_t LogConfiguration::add_output(LogOutput* output) {
size_t idx = _n_outputs++;
- _outputs = REALLOC_C_HEAP_ARRAY(LogOutput*, _outputs, _n_outputs, mtLogging);
+ _outputs = REALLOC_C_HEAP_ARRAY(_outputs, _n_outputs, mtLogging);
_outputs[idx] = output;
return idx;
}
@@ -218,7 +218,7 @@ void LogConfiguration::delete_output(size_t idx) {
LogOutput* output = _outputs[idx];
// Swap places with the last output and shrink the array
_outputs[idx] = _outputs[--_n_outputs];
- _outputs = REALLOC_C_HEAP_ARRAY(LogOutput*, _outputs, _n_outputs, mtLogging);
+ _outputs = REALLOC_C_HEAP_ARRAY(_outputs, _n_outputs, mtLogging);
delete output;
}
@@ -546,7 +546,7 @@ bool LogConfiguration::parse_log_arguments(const char* outputstr,
}
}
- FREE_C_HEAP_ARRAY(char, normalized);
+ FREE_C_HEAP_ARRAY(normalized);
if (idx == SIZE_MAX) {
return false;
}
@@ -724,8 +724,7 @@ void LogConfiguration::register_update_listener(UpdateListenerFunction cb) {
assert(cb != nullptr, "Should not register nullptr as listener");
ConfigurationLock cl;
size_t idx = _n_listener_callbacks++;
- _listener_callbacks = REALLOC_C_HEAP_ARRAY(UpdateListenerFunction,
- _listener_callbacks,
+ _listener_callbacks = REALLOC_C_HEAP_ARRAY(_listener_callbacks,
_n_listener_callbacks,
mtLogging);
_listener_callbacks[idx] = cb;
diff --git a/src/hotspot/share/logging/logFileOutput.cpp b/src/hotspot/share/logging/logFileOutput.cpp
index 9d6d19d739e..c805a3e1077 100644
--- a/src/hotspot/share/logging/logFileOutput.cpp
+++ b/src/hotspot/share/logging/logFileOutput.cpp
@@ -160,8 +160,8 @@ static uint next_file_number(const char* filename,
}
}
- FREE_C_HEAP_ARRAY(char, oldest_name);
- FREE_C_HEAP_ARRAY(char, archive_name);
+ FREE_C_HEAP_ARRAY(oldest_name);
+ FREE_C_HEAP_ARRAY(archive_name);
return next_num;
}
diff --git a/src/hotspot/share/logging/logMessageBuffer.cpp b/src/hotspot/share/logging/logMessageBuffer.cpp
index 8f308b1f015..4714cd65f8a 100644
--- a/src/hotspot/share/logging/logMessageBuffer.cpp
+++ b/src/hotspot/share/logging/logMessageBuffer.cpp
@@ -31,7 +31,7 @@ static void grow(T*& buffer, size_t& capacity, size_t minimum_length = 0) {
if (new_size < minimum_length) {
new_size = minimum_length;
}
- buffer = REALLOC_C_HEAP_ARRAY(T, buffer, new_size, mtLogging);
+ buffer = REALLOC_C_HEAP_ARRAY(buffer, new_size, mtLogging);
capacity = new_size;
}
@@ -48,8 +48,8 @@ LogMessageBuffer::LogMessageBuffer() : _message_buffer_size(0),
LogMessageBuffer::~LogMessageBuffer() {
if (_allocated) {
- FREE_C_HEAP_ARRAY(char, _message_buffer);
- FREE_C_HEAP_ARRAY(LogLine, _lines);
+ FREE_C_HEAP_ARRAY(_message_buffer);
+ FREE_C_HEAP_ARRAY(_lines);
}
}
diff --git a/src/hotspot/share/logging/logOutput.cpp b/src/hotspot/share/logging/logOutput.cpp
index e3c68b49b13..f74e614e539 100644
--- a/src/hotspot/share/logging/logOutput.cpp
+++ b/src/hotspot/share/logging/logOutput.cpp
@@ -181,7 +181,7 @@ static void add_selections(LogSelection** selections,
// Ensure there's enough room for both wildcard_match and exact_match
if (*n_selections + 2 > *selections_cap) {
*selections_cap *= 2;
- *selections = REALLOC_C_HEAP_ARRAY(LogSelection, *selections, *selections_cap, mtLogging);
+ *selections = REALLOC_C_HEAP_ARRAY(*selections, *selections_cap, mtLogging);
}
// Add found matching selections to the result array
@@ -317,8 +317,8 @@ void LogOutput::update_config_string(const size_t on_level[LogLevel::Count]) {
break;
}
}
- FREE_C_HEAP_ARRAY(LogTagSet*, deviates);
- FREE_C_HEAP_ARRAY(Selection, selections);
+ FREE_C_HEAP_ARRAY(deviates);
+ FREE_C_HEAP_ARRAY(selections);
}
bool LogOutput::parse_options(const char* options, outputStream* errstream) {
diff --git a/src/hotspot/share/logging/logTagSet.cpp b/src/hotspot/share/logging/logTagSet.cpp
index 667c76ec306..28b0d697b14 100644
--- a/src/hotspot/share/logging/logTagSet.cpp
+++ b/src/hotspot/share/logging/logTagSet.cpp
@@ -215,5 +215,5 @@ void LogTagSet::list_all_tagsets(outputStream* out) {
os::free(tagset_labels[idx]);
}
out->cr();
- FREE_C_HEAP_ARRAY(char*, tagset_labels);
+ FREE_C_HEAP_ARRAY(tagset_labels);
}
diff --git a/src/hotspot/share/memory/allocation.cpp b/src/hotspot/share/memory/allocation.cpp
index 2a109688ca6..f5e4e1f3df6 100644
--- a/src/hotspot/share/memory/allocation.cpp
+++ b/src/hotspot/share/memory/allocation.cpp
@@ -28,6 +28,7 @@
#include "memory/metaspace.hpp"
#include "memory/resourceArea.hpp"
#include "nmt/memTracker.hpp"
+#include "runtime/atomicAccess.hpp"
#include "runtime/os.hpp"
#include "runtime/task.hpp"
#include "utilities/ostream.hpp"
@@ -66,9 +67,24 @@ void FreeHeap(void* p) {
os::free(p);
}
+// These are used by the Serviceability Agent even if CDS is disabled
void* MetaspaceObj::_aot_metaspace_base = nullptr;
void* MetaspaceObj::_aot_metaspace_top = nullptr;
+#if INCLUDE_CDS
+volatile bool MetaspaceObj::_aot_metaspace_range_initialized = false;
+
+void MetaspaceObj::set_aot_metaspace_range(void* base, void* top) {
+ _aot_metaspace_base = base;
+ _aot_metaspace_top = top;
+ AtomicAccess::release_store(&_aot_metaspace_range_initialized, true);
+}
+
+bool MetaspaceObj::aot_metaspace_range_initialized() {
+ return AtomicAccess::load_acquire(&_aot_metaspace_range_initialized);
+}
+#endif
+
void* MetaspaceObj::operator new(size_t size, ClassLoaderData* loader_data,
size_t word_size,
MetaspaceObj::Type type, TRAPS) throw() {
@@ -168,7 +184,8 @@ void AnyObj::set_in_aot_cache() {
}
bool AnyObj::in_aot_cache() const {
- if (AOTMetaspace::in_aot_cache(this)) {
+ if (MetaspaceObj::is_pointer_in_aot_cache(this)) {
+ // "this" can be AOT space only if aot_metaspace_range_initialized()
precond(_allocation_t[0] == 0);
precond(_allocation_t[1] == 0);
return true;
diff --git a/src/hotspot/share/memory/allocation.hpp b/src/hotspot/share/memory/allocation.hpp
index 15b7750a363..a49f9ce9a0b 100644
--- a/src/hotspot/share/memory/allocation.hpp
+++ b/src/hotspot/share/memory/allocation.hpp
@@ -95,7 +95,7 @@ typedef AllocFailStrategy::AllocFailEnum AllocFailType;
//
// char* AllocateHeap(size_t size, MemTag mem_tag, const NativeCallStack& stack, AllocFailType alloc_failmode = AllocFailStrategy::EXIT_OOM);
// char* AllocateHeap(size_t size, MemTag mem_tag, AllocFailType alloc_failmode = AllocFailStrategy::EXIT_OOM);
-// char* ReallocateHeap(char *old, size_t size, MemTag mem_tag, AllocFailType alloc_failmode = AllocFailStrategy::EXIT_OOM);
+// char* ReallocateHeap(char* old, size_t size, MemTag mem_tag, AllocFailType alloc_failmode = AllocFailStrategy::EXIT_OOM);
// void FreeHeap(void* p);
//
@@ -112,7 +112,7 @@ char* AllocateHeap(size_t size,
MemTag mem_tag,
AllocFailType alloc_failmode = AllocFailStrategy::EXIT_OOM);
-char* ReallocateHeap(char *old,
+char* ReallocateHeap(char* old,
size_t size,
MemTag mem_tag,
AllocFailType alloc_failmode = AllocFailStrategy::EXIT_OOM);
@@ -260,11 +260,8 @@ class MetaspaceObj {
// void deallocate_contents(ClassLoaderData* loader_data);
friend class VMStructs;
- // All metsapce objects in the AOT cache (CDS archive) are mapped
- // into a single contiguous memory block, so we can use these
- // two pointers to quickly determine if a MetaspaceObj is in the
- // AOT cache.
- // When AOT/CDS is not enabled, both pointers are set to null.
+
+ // These are used by the Serviceability Agent even if CDS is disabled
static void* _aot_metaspace_base; // (inclusive) low address
static void* _aot_metaspace_top; // (exclusive) high address
@@ -275,28 +272,48 @@ class MetaspaceObj {
// regular- or aot metaspace.
static bool is_valid(const MetaspaceObj* p);
-#if INCLUDE_CDS
- static bool in_aot_cache(const MetaspaceObj* p) {
+#if !INCLUDE_CDS
+ static bool is_pointer_in_aot_cache(const void* p) { return false; }
+ static bool is_pointer_in_aot_cache_no_init_check(const void* p) { return false; }
+#else
+private:
+ // All metsapce objects in the AOT cache (CDS archive) are mapped
+ // into a single contiguous memory block, so we can use these
+ // two pointers to quickly determine if a MetaspaceObj is in the
+ // AOT cache.
+ // When AOT/CDS is not enabled, both pointers are set to null.
+ static volatile bool _aot_metaspace_range_initialized;
+ static bool aot_metaspace_range_initialized();
+
+public:
+ inline static bool is_pointer_in_aot_cache(const void* p) {
+ return aot_metaspace_range_initialized() && is_pointer_in_aot_cache_no_init_check(p);
+ }
+
+ // Call this ONLY if you know that the AOT metaspace has already been initialized.
+ inline static bool is_pointer_in_aot_cache_no_init_check(const void* p) {
+ precond(aot_metaspace_range_initialized());
+
// If no shared metaspace regions are mapped, _aot_metaspace_{base,top} will
// both be null and all values of p will be rejected quickly.
- return (((void*)p) < _aot_metaspace_top &&
- ((void*)p) >= _aot_metaspace_base);
+ return (p < _aot_metaspace_top &&
+ p >= _aot_metaspace_base);
}
- bool in_aot_cache() const { return MetaspaceObj::in_aot_cache(this); }
-#else
- static bool in_aot_cache(const MetaspaceObj* p) { return false; }
- bool in_aot_cache() const { return false; }
-#endif
- void print_address_on(outputStream* st) const; // nonvirtual address printing
-
- static void set_aot_metaspace_range(void* base, void* top) {
- _aot_metaspace_base = base;
- _aot_metaspace_top = top;
- }
+ static void set_aot_metaspace_range(void* base, void* top);
static void* aot_metaspace_base() { return _aot_metaspace_base; }
static void* aot_metaspace_top() { return _aot_metaspace_top; }
+#endif // INCLUDE_CDS
+
+ bool in_aot_cache() const {
+ // MetaspaceObjects are only created or loaded from the AOT cache after
+ // the AOT metaspace has been initialized, so we can skip init checks.
+ return is_pointer_in_aot_cache_no_init_check(this);
+ }
+
+ void print_address_on(outputStream* st) const; // nonvirtual address printing
+
#define METASPACE_OBJ_TYPES_DO(f) \
f(Class) \
@@ -374,9 +391,9 @@ extern char* resource_allocate_bytes(size_t size,
AllocFailType alloc_failmode = AllocFailStrategy::EXIT_OOM);
extern char* resource_allocate_bytes(Thread* thread, size_t size,
AllocFailType alloc_failmode = AllocFailStrategy::EXIT_OOM);
-extern char* resource_reallocate_bytes( char *old, size_t old_size, size_t new_size,
+extern char* resource_reallocate_bytes(char* old, size_t old_size, size_t new_size,
AllocFailType alloc_failmode = AllocFailStrategy::EXIT_OOM);
-extern void resource_free_bytes( Thread* thread, char *old, size_t size );
+extern void resource_free_bytes(Thread* thread, char* obj, size_t size);
//----------------------------------------------------------------------
// Base class for objects allocated in the resource area.
@@ -479,6 +496,8 @@ protected:
#endif // PRODUCT
};
+#define REALLOC_RETURN_TYPE(old) typename std::remove_reference::type
+
// One of the following macros must be used when allocating an array
// or object to determine whether it should reside in the C heap on in
// the resource area.
@@ -495,21 +514,18 @@ protected:
#define NEW_RESOURCE_ARRAY_IN_THREAD_RETURN_NULL(thread, type, size)\
(type*) resource_allocate_bytes(thread, (size) * sizeof(type), AllocFailStrategy::RETURN_NULL)
-#define REALLOC_RESOURCE_ARRAY(type, old, old_size, new_size)\
- (type*) resource_reallocate_bytes((char*)(old), (old_size) * sizeof(type), (new_size) * sizeof(type))
+#define REALLOC_RESOURCE_ARRAY(old, old_size, new_size)\
+ (REALLOC_RETURN_TYPE(old)) resource_reallocate_bytes((char*)(old), (old_size) * sizeof(*old), (new_size) * sizeof(*old))
-#define REALLOC_RESOURCE_ARRAY_RETURN_NULL(type, old, old_size, new_size)\
- (type*) resource_reallocate_bytes((char*)(old), (old_size) * sizeof(type),\
- (new_size) * sizeof(type), AllocFailStrategy::RETURN_NULL)
+#define REALLOC_RESOURCE_ARRAY_RETURN_NULL(old, old_size, new_size)\
+ (REALLOC_RETURN_TYPE(old)) resource_reallocate_bytes((char*)(old), (old_size) * sizeof(*old), \
+ (new_size) * sizeof(*old), AllocFailStrategy::RETURN_NULL)
-#define FREE_RESOURCE_ARRAY(type, old, size)\
- resource_free_bytes(Thread::current(), (char*)(old), (size) * sizeof(type))
+#define FREE_RESOURCE_ARRAY(obj, size)\
+ resource_free_bytes(Thread::current(), (char*)(obj), (size) * sizeof(*obj))
-#define FREE_RESOURCE_ARRAY_IN_THREAD(thread, type, old, size)\
- resource_free_bytes(thread, (char*)(old), (size) * sizeof(type))
-
-#define FREE_FAST(old)\
- /* nop */
+#define FREE_RESOURCE_ARRAY_IN_THREAD(thread, obj, size)\
+ resource_free_bytes(thread, (char*)(obj), (size) * sizeof(*obj))
#define NEW_RESOURCE_OBJ(type)\
NEW_RESOURCE_ARRAY(type, 1)
@@ -532,14 +548,14 @@ protected:
#define NEW_C_HEAP_ARRAY_RETURN_NULL(type, size, mem_tag)\
NEW_C_HEAP_ARRAY2(type, (size), mem_tag, AllocFailStrategy::RETURN_NULL)
-#define REALLOC_C_HEAP_ARRAY(type, old, size, mem_tag)\
- (type*) (ReallocateHeap((char*)(old), (size) * sizeof(type), mem_tag))
+#define REALLOC_C_HEAP_ARRAY(old, size, mem_tag)\
+ (REALLOC_RETURN_TYPE(old)) ReallocateHeap((char*)(old), (size) * sizeof(*old), mem_tag)
-#define REALLOC_C_HEAP_ARRAY_RETURN_NULL(type, old, size, mem_tag)\
- (type*) (ReallocateHeap((char*)(old), (size) * sizeof(type), mem_tag, AllocFailStrategy::RETURN_NULL))
+#define REALLOC_C_HEAP_ARRAY_RETURN_NULL(old, size, mem_tag)\
+ (REALLOC_RETURN_TYPE(old)) ReallocateHeap((char*)(old), (size) * sizeof(*old), mem_tag, AllocFailStrategy::RETURN_NULL)
-#define FREE_C_HEAP_ARRAY(type, old) \
- FreeHeap((char*)(old))
+#define FREE_C_HEAP_ARRAY(obj) \
+ FreeHeap((void*)(obj))
// allocate type in heap without calling ctor
#define NEW_C_HEAP_OBJ(type, mem_tag)\
@@ -548,9 +564,9 @@ protected:
#define NEW_C_HEAP_OBJ_RETURN_NULL(type, mem_tag)\
NEW_C_HEAP_ARRAY_RETURN_NULL(type, 1, mem_tag)
-// deallocate obj of type in heap without calling dtor
-#define FREE_C_HEAP_OBJ(objname)\
- FreeHeap((char*)objname);
+// deallocate obj in heap without calling dtor
+#define FREE_C_HEAP_OBJ(obj)\
+ FREE_C_HEAP_ARRAY(obj)
//------------------------------ReallocMark---------------------------------
diff --git a/src/hotspot/share/memory/arena.hpp b/src/hotspot/share/memory/arena.hpp
index 7d88c79ca52..252e511f615 100644
--- a/src/hotspot/share/memory/arena.hpp
+++ b/src/hotspot/share/memory/arena.hpp
@@ -240,12 +240,12 @@ private:
#define NEW_ARENA_ARRAY(arena, type, size) \
(type*) (arena)->Amalloc((size) * sizeof(type))
-#define REALLOC_ARENA_ARRAY(arena, type, old, old_size, new_size) \
- (type*) (arena)->Arealloc((char*)(old), (old_size) * sizeof(type), \
- (new_size) * sizeof(type) )
+#define REALLOC_ARENA_ARRAY(arena, old, old_size, new_size) \
+ (REALLOC_RETURN_TYPE(old)) (arena)->Arealloc((char*)(old), (old_size) * sizeof(*old), \
+ (new_size) * sizeof(*old) )
-#define FREE_ARENA_ARRAY(arena, type, old, size) \
- (arena)->Afree((char*)(old), (size) * sizeof(type))
+#define FREE_ARENA_ARRAY(arena, obj, size) \
+ (arena)->Afree((char*)(obj), (size) * sizeof(*obj))
#define NEW_ARENA_OBJ(arena, type) \
NEW_ARENA_ARRAY(arena, type, 1)
diff --git a/src/hotspot/share/memory/heapInspection.cpp b/src/hotspot/share/memory/heapInspection.cpp
index aae3c99a634..5abb7e4ddcc 100644
--- a/src/hotspot/share/memory/heapInspection.cpp
+++ b/src/hotspot/share/memory/heapInspection.cpp
@@ -189,7 +189,7 @@ KlassInfoTable::~KlassInfoTable() {
for (int index = 0; index < _num_buckets; index++) {
_buckets[index].empty();
}
- FREE_C_HEAP_ARRAY(KlassInfoBucket, _buckets);
+ FREE_C_HEAP_ARRAY(_buckets);
_buckets = nullptr;
}
}
diff --git a/src/hotspot/share/memory/memRegion.cpp b/src/hotspot/share/memory/memRegion.cpp
index 3c481926025..dde4b0ace9a 100644
--- a/src/hotspot/share/memory/memRegion.cpp
+++ b/src/hotspot/share/memory/memRegion.cpp
@@ -115,5 +115,5 @@ void MemRegion::destroy_array(MemRegion* array, size_t length) {
for (size_t i = 0; i < length; i++) {
array[i].~MemRegion();
}
- FREE_C_HEAP_ARRAY(MemRegion, array);
+ FREE_C_HEAP_ARRAY(array);
}
diff --git a/src/hotspot/share/memory/memoryReserver.cpp b/src/hotspot/share/memory/memoryReserver.cpp
index 1c44f76ed11..f9359420693 100644
--- a/src/hotspot/share/memory/memoryReserver.cpp
+++ b/src/hotspot/share/memory/memoryReserver.cpp
@@ -656,11 +656,12 @@ ReservedHeapSpace HeapReserver::Instance::reserve_compressed_oops_heap(const siz
#endif // _LP64
ReservedHeapSpace HeapReserver::Instance::reserve_heap(size_t size, size_t alignment, size_t page_size) {
- if (UseCompressedOops) {
#ifdef _LP64
+ if (UseCompressedOops) {
return reserve_compressed_oops_heap(size, alignment, page_size);
+ } else
#endif
- } else {
+ {
return reserve_uncompressed_oops_heap(size, alignment, page_size);
}
}
diff --git a/src/hotspot/share/memory/metaspace.cpp b/src/hotspot/share/memory/metaspace.cpp
index da6ebc991c8..b97ae9ab540 100644
--- a/src/hotspot/share/memory/metaspace.cpp
+++ b/src/hotspot/share/memory/metaspace.cpp
@@ -739,6 +739,9 @@ void Metaspace::global_initialize() {
AOTMetaspace::initialize_runtime_shared_and_meta_spaces();
// If any of the archived space fails to map, UseSharedSpaces
// is reset to false.
+ } else {
+ // Trivially set the range to empty to satisfy the assert in MetaspaceObj::is_pointer_in_aot_cache()
+ MetaspaceObj::set_aot_metaspace_range(nullptr, nullptr);
}
#endif // INCLUDE_CDS
diff --git a/src/hotspot/share/memory/metaspace/rootChunkArea.cpp b/src/hotspot/share/memory/metaspace/rootChunkArea.cpp
index a178e122788..39407b46999 100644
--- a/src/hotspot/share/memory/metaspace/rootChunkArea.cpp
+++ b/src/hotspot/share/memory/metaspace/rootChunkArea.cpp
@@ -473,7 +473,7 @@ RootChunkAreaLUT::~RootChunkAreaLUT() {
for (int i = 0; i < _num; i++) {
_arr[i].~RootChunkArea();
}
- FREE_C_HEAP_ARRAY(RootChunkArea, _arr);
+ FREE_C_HEAP_ARRAY(_arr);
}
#ifdef ASSERT
diff --git a/src/hotspot/share/memory/metaspaceClosure.hpp b/src/hotspot/share/memory/metaspaceClosure.hpp
index ac42dd13c6c..20e5528e9bc 100644
--- a/src/hotspot/share/memory/metaspaceClosure.hpp
+++ b/src/hotspot/share/memory/metaspaceClosure.hpp
@@ -37,19 +37,6 @@
#include "utilities/macros.hpp"
#include "utilities/resizableHashTable.hpp"
-// This macro just check for the existence of a member with the name "metaspace_pointers_do". If the
-// parameter list is not (MetaspaceClosure* it), you will get a compilation error.
-#define HAS_METASPACE_POINTERS_DO(T) HasMetaspacePointersDo::value
-
-template
-class HasMetaspacePointersDo {
- template static void* test(decltype(&U::metaspace_pointers_do));
- template static int test(...);
- using test_type = decltype(test(nullptr));
-public:
- static constexpr bool value = std::is_pointer_v;
-};
-
// class MetaspaceClosure --
//
// This class is used for iterating the class metadata objects. It
@@ -98,7 +85,6 @@ public:
// only in the Metadata class.
//
// To work around the lack of a vtable, we use the Ref class with templates
- // (see MSORef, OtherArrayRef, MSOArrayRef, and MSOPointerArrayRef)
// so that we can statically discover the type of a object. The use of Ref
// depends on the fact that:
//
@@ -194,15 +180,6 @@ private:
return obj->size_in_heapwords();
}
- static MetaspaceClosureType as_type(MetaspaceClosureType t) {
- return t;
- }
-
- static MetaspaceClosureType as_type(MetaspaceObj::Type msotype) {
- precond(msotype < MetaspaceObj::_number_of_types);
- return (MetaspaceClosureType)msotype;
- }
-
// MSORef -- iterate an instance of T, where T::metaspace_pointers_do() exists.
template class MSORef : public Ref {
T** _mpp;
@@ -210,99 +187,22 @@ private:
return strip_tags(*_mpp);
}
protected:
- virtual void** mpp() const {
+ virtual void** mpp() const override {
return (void**)_mpp;
}
public:
MSORef(T** mpp, Writability w) : Ref(w), _mpp(mpp) {}
- virtual bool is_read_only_by_default() const { return T::is_read_only_by_default(); }
- virtual bool not_null() const { return dereference() != nullptr; }
- virtual int size() const { return get_size(dereference()); }
- virtual MetaspaceClosureType type() const { return as_type(dereference()->type()); }
-
- virtual void metaspace_pointers_do(MetaspaceClosure *it) const {
+ virtual bool is_read_only_by_default() const override { return T::is_read_only_by_default(); }
+ virtual bool not_null() const override { return dereference() != nullptr; }
+ virtual int size() const override { return get_size(dereference()); }
+ virtual MetaspaceClosureType type() const override { return as_type(dereference()->type()); }
+ virtual void metaspace_pointers_do(MetaspaceClosure *it) const override {
dereference()->metaspace_pointers_do(it);
}
};
- //---------------------
- // Support for Array
- //---------------------
-
- // Abstract base class for MSOArrayRef, MSOPointerArrayRef and OtherArrayRef.
- // These are used for iterating Array.
- template class ArrayRef : public Ref {
- Array** _mpp;
- protected:
- Array* dereference() const {
- return strip_tags(*_mpp);
- }
- virtual void** mpp() const {
- return (void**)_mpp;
- }
-
- ArrayRef(Array** mpp, Writability w) : Ref(w), _mpp(mpp) {}
-
- // all Arrays are read-only by default
- virtual bool is_read_only_by_default() const { return true; }
- virtual bool not_null() const { return dereference() != nullptr; }
- virtual int size() const { return dereference()->size(); }
- virtual MetaspaceClosureType type() const { return as_type(MetaspaceObj::array_type(sizeof(T))); }
- };
-
- // OtherArrayRef -- iterate an instance of Array, where T does NOT have metaspace_pointer_do().
- // T can be a primitive type, such as int, or a structure. However, we do not scan
- // the fields inside T, so you should not embed any pointers inside T.
- template class OtherArrayRef : public ArrayRef {
- public:
- OtherArrayRef(Array** mpp, Writability w) : ArrayRef(mpp, w) {}
-
- virtual void metaspace_pointers_do(MetaspaceClosure *it) const {
- Array* array = ArrayRef::dereference();
- log_trace(aot)("Iter(OtherArray): %p [%d]", array, array->length());
- }
- };
-
- // MSOArrayRef -- iterate an instance of Array, where T has metaspace_pointer_do().
- // We recursively call T::metaspace_pointers_do() for each element in this array.
- template class MSOArrayRef : public ArrayRef {
- public:
- MSOArrayRef(Array** mpp, Writability w) : ArrayRef(mpp, w) {}
-
- virtual void metaspace_pointers_do(MetaspaceClosure *it) const {
- metaspace_pointers_do_at_impl(it, ArrayRef::dereference());
- }
- private:
- void metaspace_pointers_do_at_impl(MetaspaceClosure *it, Array