diff --git a/.github/actions/build-jtreg/action.yml b/.github/actions/build-jtreg/action.yml
index a9c046e9dd9..334812e8341 100644
--- a/.github/actions/build-jtreg/action.yml
+++ b/.github/actions/build-jtreg/action.yml
@@ -37,13 +37,13 @@ runs:
- name: 'Check cache for already built JTReg'
id: get-cached
- uses: actions/cache@v4
+ uses: actions/cache@v5
with:
path: jtreg/installed
key: jtreg-${{ steps.version.outputs.value }}
- name: 'Checkout the JTReg source'
- uses: actions/checkout@v4
+ uses: actions/checkout@v6
with:
repository: openjdk/jtreg
ref: jtreg-${{ steps.version.outputs.value }}
@@ -61,7 +61,7 @@ runs:
if: (steps.get-cached.outputs.cache-hit != 'true')
- name: 'Upload JTReg artifact'
- uses: actions/upload-artifact@v4
+ uses: actions/upload-artifact@v6
with:
name: bundles-jtreg-${{ steps.version.outputs.value }}
path: jtreg/installed
diff --git a/.github/actions/do-build/action.yml b/.github/actions/do-build/action.yml
index 6f2c2ce0218..6f6bbdabb68 100644
--- a/.github/actions/do-build/action.yml
+++ b/.github/actions/do-build/action.yml
@@ -66,7 +66,7 @@ runs:
shell: bash
- name: 'Upload build logs'
- uses: actions/upload-artifact@v4
+ uses: actions/upload-artifact@v6
with:
name: failure-logs-${{ inputs.platform }}${{ inputs.debug-suffix }}
path: failure-logs
@@ -74,7 +74,7 @@ runs:
# This is the best way I found to abort the job with an error message
- name: 'Notify about build failures'
- uses: actions/github-script@v7
+ uses: actions/github-script@v8
with:
script: core.setFailed('Build failed. See summary for details.')
if: steps.check.outputs.failure == 'true'
diff --git a/.github/actions/get-bootjdk/action.yml b/.github/actions/get-bootjdk/action.yml
index 312fb642c82..d531358b7dd 100644
--- a/.github/actions/get-bootjdk/action.yml
+++ b/.github/actions/get-bootjdk/action.yml
@@ -65,7 +65,7 @@ runs:
- name: 'Check cache for BootJDK'
id: get-cached-bootjdk
- uses: actions/cache@v4
+ uses: actions/cache@v5
with:
path: bootjdk/jdk
key: boot-jdk-${{ inputs.platform }}-${{ steps.sha256.outputs.value }}
diff --git a/.github/actions/get-bundles/action.yml b/.github/actions/get-bundles/action.yml
index a356aa9fd8d..55fa0e842d2 100644
--- a/.github/actions/get-bundles/action.yml
+++ b/.github/actions/get-bundles/action.yml
@@ -54,14 +54,14 @@ runs:
steps:
- name: 'Download bundles artifact'
id: download-bundles
- uses: actions/download-artifact@v4
+ uses: actions/download-artifact@v8
with:
name: bundles-${{ inputs.platform }}${{ inputs.debug-suffix }}
path: bundles
continue-on-error: true
- name: 'Download bundles artifact (retry)'
- uses: actions/download-artifact@v4
+ uses: actions/download-artifact@v8
with:
name: bundles-${{ inputs.platform }}${{ inputs.debug-suffix }}
path: bundles
@@ -69,7 +69,7 @@ runs:
- name: 'Download static bundles artifact'
id: download-static-bundles
- uses: actions/download-artifact@v4
+ uses: actions/download-artifact@v8
with:
name: bundles-${{ inputs.platform }}${{ inputs.debug-suffix }}${{ inputs.static-suffix }}
path: bundles
diff --git a/.github/actions/get-gtest/action.yml b/.github/actions/get-gtest/action.yml
index 7a329460a6e..bc53fa2a3b1 100644
--- a/.github/actions/get-gtest/action.yml
+++ b/.github/actions/get-gtest/action.yml
@@ -40,7 +40,7 @@ runs:
var: GTEST_VERSION
- name: 'Checkout GTest source'
- uses: actions/checkout@v4
+ uses: actions/checkout@v6
with:
repository: google/googletest
ref: 'v${{ steps.version.outputs.value }}'
diff --git a/.github/actions/get-jtreg/action.yml b/.github/actions/get-jtreg/action.yml
index 36c895fc59d..8c75ae10c7f 100644
--- a/.github/actions/get-jtreg/action.yml
+++ b/.github/actions/get-jtreg/action.yml
@@ -41,7 +41,7 @@ runs:
- name: 'Download JTReg artifact'
id: download-jtreg
- uses: actions/download-artifact@v4
+ uses: actions/download-artifact@v8
with:
name: bundles-jtreg-${{ steps.version.outputs.value }}
path: jtreg/installed
diff --git a/.github/actions/get-msys2/action.yml b/.github/actions/get-msys2/action.yml
index 308230ebf2e..7351a120ac4 100644
--- a/.github/actions/get-msys2/action.yml
+++ b/.github/actions/get-msys2/action.yml
@@ -31,7 +31,7 @@ runs:
steps:
- name: 'Install MSYS2'
id: msys2
- uses: msys2/setup-msys2@v2.28.0
+ uses: msys2/setup-msys2@v2.31.0
with:
install: 'autoconf tar unzip zip make'
path-type: minimal
diff --git a/.github/actions/upload-bundles/action.yml b/.github/actions/upload-bundles/action.yml
index 78fb0a94bfd..94308002ea7 100644
--- a/.github/actions/upload-bundles/action.yml
+++ b/.github/actions/upload-bundles/action.yml
@@ -87,7 +87,7 @@ runs:
shell: bash
- name: 'Upload bundles artifact'
- uses: actions/upload-artifact@v4
+ uses: actions/upload-artifact@v6
with:
name: bundles-${{ inputs.platform }}${{ inputs.debug-suffix }}${{ inputs.static-suffix }}${{ inputs.bundle-suffix }}
path: bundles
diff --git a/.github/workflows/build-alpine-linux.yml b/.github/workflows/build-alpine-linux.yml
index c39962fa07f..6863da9016e 100644
--- a/.github/workflows/build-alpine-linux.yml
+++ b/.github/workflows/build-alpine-linux.yml
@@ -74,7 +74,7 @@ jobs:
steps:
- name: 'Checkout the JDK source'
- uses: actions/checkout@v4
+ uses: actions/checkout@v6
- name: 'Install toolchain and dependencies'
run: |
diff --git a/.github/workflows/build-cross-compile.yml b/.github/workflows/build-cross-compile.yml
index a0642d469aa..99b6c40606c 100644
--- a/.github/workflows/build-cross-compile.yml
+++ b/.github/workflows/build-cross-compile.yml
@@ -94,7 +94,7 @@ jobs:
steps:
- name: 'Checkout the JDK source'
- uses: actions/checkout@v4
+ uses: actions/checkout@v6
- name: 'Get the BootJDK'
id: bootjdk
@@ -122,7 +122,7 @@ jobs:
- name: 'Check cache for sysroot'
id: get-cached-sysroot
- uses: actions/cache@v4
+ uses: actions/cache@v5
with:
path: sysroot
key: sysroot-${{ matrix.debian-arch }}-${{ hashFiles('./.github/workflows/build-cross-compile.yml') }}
diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml
index 791b53a3f04..c501670439e 100644
--- a/.github/workflows/build-linux.yml
+++ b/.github/workflows/build-linux.yml
@@ -84,7 +84,7 @@ jobs:
steps:
- name: 'Checkout the JDK source'
- uses: actions/checkout@v4
+ uses: actions/checkout@v6
- name: 'Get the BootJDK'
id: bootjdk
diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml
index 484e616fad7..435576f4afd 100644
--- a/.github/workflows/build-macos.yml
+++ b/.github/workflows/build-macos.yml
@@ -75,7 +75,7 @@ jobs:
steps:
- name: 'Checkout the JDK source'
- uses: actions/checkout@v4
+ uses: actions/checkout@v6
- name: 'Get the BootJDK'
id: bootjdk
diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml
index 4dafc016a99..3bb50a137ec 100644
--- a/.github/workflows/build-windows.yml
+++ b/.github/workflows/build-windows.yml
@@ -83,7 +83,7 @@ jobs:
steps:
- name: 'Checkout the JDK source'
- uses: actions/checkout@v4
+ uses: actions/checkout@v6
- name: 'Get MSYS2'
uses: ./.github/actions/get-msys2
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 85ec75f343c..20be196b128 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -75,7 +75,7 @@ jobs:
steps:
- name: 'Checkout the scripts'
- uses: actions/checkout@v4
+ uses: actions/checkout@v6
with:
sparse-checkout: |
.github
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 8f33454305e..b240b42fb97 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -128,7 +128,7 @@ jobs:
steps:
- name: 'Checkout the JDK source'
- uses: actions/checkout@v4
+ uses: actions/checkout@v6
- name: 'Get MSYS2'
uses: ./.github/actions/get-msys2
@@ -239,7 +239,7 @@ jobs:
if: always()
- name: 'Upload test results'
- uses: actions/upload-artifact@v4
+ uses: actions/upload-artifact@v6
with:
path: results
name: ${{ steps.package.outputs.artifact-name }}
@@ -247,7 +247,7 @@ jobs:
# This is the best way I found to abort the job with an error message
- name: 'Notify about test failures'
- uses: actions/github-script@v7
+ uses: actions/github-script@v8
with:
script: core.setFailed('${{ steps.run-tests.outputs.error-message }}')
if: steps.run-tests.outputs.failure == 'true'
diff --git a/.gitignore b/.gitignore
index 0743489f8ec..b6b4a1a559a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,6 +16,7 @@ NashornProfile.txt
**/JTreport/**
**/JTwork/**
/src/utils/LogCompilation/target/
+/src/utils/LogCompilation/logc.jar
/.project/
/.settings/
/compile_commands.json
diff --git a/doc/testing.html b/doc/testing.html
index 195153c8612..c8d0b928bb0 100644
--- a/doc/testing.html
+++ b/doc/testing.html
@@ -284,9 +284,10 @@ possible, or if you want to use a fully qualified test descriptor, add
Gtest
Note: To be able to run the Gtest suite, you need to
configure your build to be able to find a proper version of the gtest
-source. For details, see the section "Running Tests" in the build
-documentation.
+source. For details, see the section "Running Tests" in the
+build documentation (html, markdown).
Since the Hotspot Gtest suite is so quick, the default is to run all
tests. This is specified by just gtest, or as a fully
qualified test descriptor gtest:all.
diff --git a/doc/testing.md b/doc/testing.md
index d0e54aab02b..5f70f2796ad 100644
--- a/doc/testing.md
+++ b/doc/testing.md
@@ -198,8 +198,8 @@ use a fully qualified test descriptor, add `jtreg:`, e.g.
**Note:** To be able to run the Gtest suite, you need to configure your build
to be able to find a proper version of the gtest source. For details, see the
-section ["Running Tests" in the build
-documentation](building.html#running-tests).
+section **"Running Tests" in the build
+documentation** ([html](building.html#running-tests), [markdown](building.md#running-tests)).
Since the Hotspot Gtest suite is so quick, the default is to run all tests.
This is specified by just `gtest`, or as a fully qualified test descriptor
diff --git a/make/CompileInterimLangtools.gmk b/make/CompileInterimLangtools.gmk
index c7d1c3796f6..8254b8fc0a7 100644
--- a/make/CompileInterimLangtools.gmk
+++ b/make/CompileInterimLangtools.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2014, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2014, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -68,17 +68,19 @@ java.compiler.interim_EXTRA_FILES := \
TARGETS += $(BUILDTOOLS_OUTPUTDIR)/gensrc/java.compiler.interim/javax/tools/ToolProvider.java
################################################################################
-# Use the up-to-date PreviewFeature.java and NoPreview.java from the current
-# sources, instead of the versions from the boot JDK, as javac may be referring
-# to constants from the up-to-date versions.
+# Create a hybrid PreviewFeature.java that combines constants
+# from the current sources, as those can be used in javac APIs, and from the
+# bootstrap JDK, as those can be used from bootstrap JDK classfiles.
-$(eval $(call SetupCopyFiles, COPY_PREVIEW_FEATURES, \
- FILES := $(TOPDIR)/src/java.base/share/classes/jdk/internal/javac/PreviewFeature.java \
- $(TOPDIR)/src/java.base/share/classes/jdk/internal/javac/NoPreview.java, \
- DEST := $(BUILDTOOLS_OUTPUTDIR)/gensrc/java.base.interim/jdk/internal/javac/, \
-))
+$(BUILDTOOLS_OUTPUTDIR)/gensrc/java.base.interim/jdk/internal/javac/PreviewFeature.java: \
+ $(TOPDIR)/src/java.base/share/classes/jdk/internal/javac/PreviewFeature.java
+ $(call LogInfo, Generating $@)
+ $(JAVA) $(TOPDIR)/make/langtools/tools/previewfeature/SetupPreviewFeature.java \
+ $(TOPDIR)/src/java.base/share/classes/jdk/internal/javac/PreviewFeature.java \
+ $@
-TARGETS += $(COPY_PREVIEW_FEATURES)
+
+TARGETS += $(BUILDTOOLS_OUTPUTDIR)/gensrc/java.base.interim/jdk/internal/javac/PreviewFeature.java
################################################################################
# Setup the rules to build interim langtools, which is compiled by the boot
@@ -123,7 +125,8 @@ define SetupInterimModule
$1_DEPS_INTERIM := $$(addsuffix .interim, $$(filter \
$$(INTERIM_LANGTOOLS_BASE_MODULES), $$(call FindTransitiveDepsForModule, $1)))
- $$(BUILD_$1.interim): $$(foreach d, $$($1_DEPS_INTERIM), $$(BUILD_$$d)) $(COPY_PREVIEW_FEATURES)
+ $$(BUILD_$1.interim): $$(foreach d, $$($1_DEPS_INTERIM), $$(BUILD_$$d)) \
+ $(BUILDTOOLS_OUTPUTDIR)/gensrc/java.base.interim/jdk/internal/javac/PreviewFeature.java
TARGETS += $$(BUILD_$1.interim)
endef
diff --git a/make/GenerateLinkOptData.gmk b/make/GenerateLinkOptData.gmk
index 6f6e1f29b4c..d615a34e71a 100644
--- a/make/GenerateLinkOptData.gmk
+++ b/make/GenerateLinkOptData.gmk
@@ -70,12 +70,15 @@ CLASSLIST_FILE_VM_OPTS = \
# Save the stderr output of the command and print it along with stdout in case
# something goes wrong.
+# The classlists must be generated with -Xint to avoid non-determinism
+# introduced by JIT compiled code
$(CLASSLIST_FILE): $(INTERIM_IMAGE_DIR)/bin/java$(EXECUTABLE_SUFFIX) $(CLASSLIST_JAR)
$(call MakeDir, $(LINK_OPT_DIR))
$(call LogInfo, Generating $(patsubst $(OUTPUTDIR)/%, %, $@))
$(call LogInfo, Generating $(patsubst $(OUTPUTDIR)/%, %, $(JLI_TRACE_FILE)))
$(FIXPATH) $(INTERIM_IMAGE_DIR)/bin/java -XX:DumpLoadedClassList=$@.raw \
$(CLASSLIST_FILE_VM_OPTS) \
+ -Xint \
-Xlog:aot=off \
-Xlog:cds=off \
-cp $(SUPPORT_OUTPUTDIR)/classlist.jar \
@@ -90,6 +93,7 @@ $(CLASSLIST_FILE): $(INTERIM_IMAGE_DIR)/bin/java$(EXECUTABLE_SUFFIX) $(CLASSLIST
-XX:SharedClassListFile=$@.interim -XX:SharedArchiveFile=$@.jsa \
-Djava.lang.invoke.MethodHandle.TRACE_RESOLVE=true \
$(CLASSLIST_FILE_VM_OPTS) \
+ -Xint \
--module-path $(SUPPORT_OUTPUTDIR)/classlist.jar \
-Xlog:aot=off \
-Xlog:cds=off \
diff --git a/make/PreInit.gmk b/make/PreInit.gmk
index 3df44308dd9..8152587781c 100644
--- a/make/PreInit.gmk
+++ b/make/PreInit.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2012, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2012, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -66,7 +66,8 @@ CALLED_SPEC_TARGETS := $(filter-out $(ALL_GLOBAL_TARGETS), $(CALLED_TARGETS))
ifeq ($(CALLED_SPEC_TARGETS), )
SKIP_SPEC := true
endif
-ifeq ($(findstring p, $(MAKEFLAGS))$(findstring q, $(MAKEFLAGS)), pq)
+MFLAGS_SINGLE := $(filter-out --%, $(MFLAGS))
+ifeq ($(findstring p, $(MFLAGS_SINGLE))$(findstring q, $(MFLAGS_SINGLE)), pq)
SKIP_SPEC := true
endif
diff --git a/make/autoconf/basic_tools.m4 b/make/autoconf/basic_tools.m4
index 8e42f9205a4..66ef94d48a8 100644
--- a/make/autoconf/basic_tools.m4
+++ b/make/autoconf/basic_tools.m4
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -369,6 +369,10 @@ AC_DEFUN_ONCE([BASIC_SETUP_COMPLEX_TOOLS],
IS_GNU_DATE=yes
else
AC_MSG_RESULT([no])
+ # Likely at the AIX provided version of the date utility here, which is not compatible
+ if test "x$OPENJDK_TARGET_OS" = "xaix"; then
+ AC_MSG_ERROR([gnu date from AIX toolbox is required])
+ fi
IS_GNU_DATE=no
fi
AC_SUBST(IS_GNU_DATE)
diff --git a/make/autoconf/flags-cflags.m4 b/make/autoconf/flags-cflags.m4
index 0cdf02c61c6..ab9cd8be19b 100644
--- a/make/autoconf/flags-cflags.m4
+++ b/make/autoconf/flags-cflags.m4
@@ -214,6 +214,7 @@ AC_DEFUN([FLAGS_SETUP_WARNINGS],
WARNINGS_ENABLE_ADDITIONAL_CXX=""
WARNINGS_ENABLE_ADDITIONAL_JVM=""
DISABLED_WARNINGS="4800 5105"
+ CFLAGS_CONVERSION_WARNINGS=
;;
gcc)
@@ -239,6 +240,7 @@ AC_DEFUN([FLAGS_SETUP_WARNINGS],
if test "x$OPENJDK_TARGET_CPU_ARCH" = "xppc"; then
DISABLED_WARNINGS="$DISABLED_WARNINGS psabi"
fi
+ CFLAGS_CONVERSION_WARNINGS="-Wconversion -Wno-float-conversion"
;;
clang)
@@ -258,6 +260,7 @@ AC_DEFUN([FLAGS_SETUP_WARNINGS],
# These warnings will never be turned on, since they generate too many
# false positives.
DISABLED_WARNINGS="unknown-warning-option unused-parameter"
+ CFLAGS_CONVERSION_WARNINGS="-Wimplicit-int-conversion"
;;
esac
WARNINGS_ENABLE_ALL="$WARNINGS_ENABLE_ALL_NORMAL $WARNINGS_ENABLE_ADDITIONAL"
@@ -270,6 +273,7 @@ AC_DEFUN([FLAGS_SETUP_WARNINGS],
AC_SUBST(DISABLED_WARNINGS)
AC_SUBST(DISABLED_WARNINGS_C)
AC_SUBST(DISABLED_WARNINGS_CXX)
+ AC_SUBST(CFLAGS_CONVERSION_WARNINGS)
])
AC_DEFUN([FLAGS_SETUP_QUALITY_CHECKS],
@@ -574,6 +578,11 @@ AC_DEFUN([FLAGS_SETUP_CFLAGS_HELPER],
TOOLCHAIN_CFLAGS_JDK_CONLY="-fno-strict-aliasing" # technically NOT for CXX
fi
+ if test "x$ENABLE_LINKTIME_GC" = xtrue; then
+ TOOLCHAIN_CFLAGS_JDK="$TOOLCHAIN_CFLAGS_JDK -ffunction-sections -fdata-sections"
+ TOOLCHAIN_CFLAGS_JVM="$TOOLCHAIN_CFLAGS_JVM -ffunction-sections -fdata-sections"
+ fi
+
if test "x$OPENJDK_TARGET_OS" = xaix; then
TOOLCHAIN_CFLAGS_JVM="$TOOLCHAIN_CFLAGS_JVM -ffunction-sections -ftls-model -fno-math-errno"
TOOLCHAIN_CFLAGS_JDK="-ffunction-sections -fsigned-char"
diff --git a/make/autoconf/flags-ldflags.m4 b/make/autoconf/flags-ldflags.m4
index 466ff1beaf4..7782735be25 100644
--- a/make/autoconf/flags-ldflags.m4
+++ b/make/autoconf/flags-ldflags.m4
@@ -80,6 +80,10 @@ AC_DEFUN([FLAGS_SETUP_LDFLAGS_HELPER],
if test "x$CXX_IS_USER_SUPPLIED" = xfalse && test "x$CC_IS_USER_SUPPLIED" = xfalse; then
UTIL_REQUIRE_TOOLCHAIN_PROGS(LLD, lld)
fi
+
+ if test "x$ENABLE_LINKTIME_GC" = xtrue; then
+ BASIC_LDFLAGS_JDK_ONLY="$BASIC_LDFLAGS_JDK_ONLY -Wl,--gc-sections"
+ fi
fi
if test "x$OPENJDK_TARGET_OS" = xaix; then
BASIC_LDFLAGS="-Wl,-b64 -Wl,-brtl -Wl,-bnorwexec -Wl,-blibpath:/usr/lib:lib -Wl,-bnoexpall \
diff --git a/make/autoconf/jdk-options.m4 b/make/autoconf/jdk-options.m4
index 87d147d4f07..89fcbc88521 100644
--- a/make/autoconf/jdk-options.m4
+++ b/make/autoconf/jdk-options.m4
@@ -102,9 +102,20 @@ AC_DEFUN_ONCE([JDKOPT_SETUP_JDK_OPTIONS],
CHECKING_MSG: [if we should build headless-only (no GUI)])
AC_SUBST(ENABLE_HEADLESS_ONLY)
+ # Avoid headless-only on macOS and Windows, it is not supported there
+ if test "x$ENABLE_HEADLESS_ONLY" = xtrue; then
+ if test "x$OPENJDK_TARGET_OS" = xwindows || test "x$OPENJDK_TARGET_OS" = xmacosx; then
+ AC_MSG_ERROR([headless-only is not supported on macOS and Windows])
+ fi
+ fi
+
# should we linktime gc unused code sections in the JDK build ?
- if test "x$OPENJDK_TARGET_OS" = "xlinux" && test "x$OPENJDK_TARGET_CPU" = xs390x; then
- LINKTIME_GC_DEFAULT=true
+ if test "x$OPENJDK_TARGET_OS" = "xlinux"; then
+ if test "x$OPENJDK_TARGET_CPU" = "xs390x" || test "x$OPENJDK_TARGET_CPU" = "xppc64le"; then
+ LINKTIME_GC_DEFAULT=true
+ else
+ LINKTIME_GC_DEFAULT=false
+ fi
else
LINKTIME_GC_DEFAULT=false
fi
diff --git a/make/autoconf/lib-bundled.m4 b/make/autoconf/lib-bundled.m4
index a6266bec014..7aa5fdf2589 100644
--- a/make/autoconf/lib-bundled.m4
+++ b/make/autoconf/lib-bundled.m4
@@ -267,8 +267,8 @@ AC_DEFUN_ONCE([LIB_SETUP_ZLIB],
LIBZ_LIBS=""
if test "x$USE_EXTERNAL_LIBZ" = "xfalse"; then
LIBZ_CFLAGS="$LIBZ_CFLAGS -I$TOPDIR/src/java.base/share/native/libzip/zlib"
- if test "x$OPENJDK_TARGET_OS" = xmacosx; then
- LIBZ_CFLAGS="$LIBZ_CFLAGS -DHAVE_UNISTD_H"
+ if test "x$OPENJDK_TARGET_OS" = xmacosx -o "x$OPENJDK_TARGET_OS" = xaix -o "x$OPENJDK_TARGET_OS" = xlinux; then
+ LIBZ_CFLAGS="$LIBZ_CFLAGS -DHAVE_UNISTD_H=1 -DHAVE_STDARG_H=1"
fi
else
LIBZ_LIBS="-lz"
diff --git a/make/autoconf/spec.gmk.template b/make/autoconf/spec.gmk.template
index b3d58704c50..c4e5a23d31a 100644
--- a/make/autoconf/spec.gmk.template
+++ b/make/autoconf/spec.gmk.template
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -529,6 +529,7 @@ CFLAGS_WARNINGS_ARE_ERRORS := @CFLAGS_WARNINGS_ARE_ERRORS@
DISABLED_WARNINGS := @DISABLED_WARNINGS@
DISABLED_WARNINGS_C := @DISABLED_WARNINGS_C@
DISABLED_WARNINGS_CXX := @DISABLED_WARNINGS_CXX@
+CFLAGS_CONVERSION_WARNINGS := @CFLAGS_CONVERSION_WARNINGS@
# A global flag (true or false) determining if native warnings are considered errors.
WARNINGS_AS_ERRORS := @WARNINGS_AS_ERRORS@
diff --git a/make/autoconf/toolchain_microsoft.m4 b/make/autoconf/toolchain_microsoft.m4
index f577cf1a2a1..afe04acf029 100644
--- a/make/autoconf/toolchain_microsoft.m4
+++ b/make/autoconf/toolchain_microsoft.m4
@@ -217,10 +217,12 @@ AC_DEFUN([TOOLCHAIN_FIND_VISUAL_STUDIO_BAT_FILE],
TOOLCHAIN_CHECK_POSSIBLE_VISUAL_STUDIO_ROOT([$TARGET_CPU], [$VS_VERSION],
[$PROGRAMFILES_X86/$VS_INSTALL_DIR], [well-known name])
fi
+ # Derive system drive root from CMD (which is at /windows/system32/cmd.exe)
+ WINSYSDRIVE_ROOT="$(dirname "$(dirname "$(dirname "$CMD")")")"
TOOLCHAIN_CHECK_POSSIBLE_VISUAL_STUDIO_ROOT([$TARGET_CPU], [$VS_VERSION],
- [c:/program files/$VS_INSTALL_DIR], [well-known name])
+ [$WINSYSDRIVE_ROOT/program files/$VS_INSTALL_DIR], [well-known name])
TOOLCHAIN_CHECK_POSSIBLE_VISUAL_STUDIO_ROOT([$TARGET_CPU], [$VS_VERSION],
- [c:/program files (x86)/$VS_INSTALL_DIR], [well-known name])
+ [$WINSYSDRIVE_ROOT/program files (x86)/$VS_INSTALL_DIR], [well-known name])
if test "x$SDK_INSTALL_DIR" != x; then
if test "x$ProgramW6432" != x; then
TOOLCHAIN_CHECK_POSSIBLE_WIN_SDK_ROOT([$TARGET_CPU], [$VS_VERSION],
@@ -235,9 +237,9 @@ AC_DEFUN([TOOLCHAIN_FIND_VISUAL_STUDIO_BAT_FILE],
[$PROGRAMFILES/$SDK_INSTALL_DIR], [well-known name])
fi
TOOLCHAIN_CHECK_POSSIBLE_WIN_SDK_ROOT([$TARGET_CPU], [$VS_VERSION],
- [c:/program files/$SDK_INSTALL_DIR], [well-known name])
+ [$WINSYSDRIVE_ROOT/program files/$SDK_INSTALL_DIR], [well-known name])
TOOLCHAIN_CHECK_POSSIBLE_WIN_SDK_ROOT([$TARGET_CPU], [$VS_VERSION],
- [c:/program files (x86)/$SDK_INSTALL_DIR], [well-known name])
+ [$WINSYSDRIVE_ROOT/program files (x86)/$SDK_INSTALL_DIR], [well-known name])
fi
VCVARS_VER=auto
@@ -338,7 +340,7 @@ AC_DEFUN([TOOLCHAIN_EXTRACT_VISUAL_STUDIO_ENV],
OLDPATH="$PATH"
# Make sure we only capture additions to PATH needed by VS.
# Clear out path, but need system dir present for vsvars cmd file to be able to run
- export PATH=$WINENV_PREFIX/c/windows/system32
+ export PATH="$(dirname "$CMD")"
# The "| cat" is to stop SetEnv.Cmd to mess with system colors on some systems
# We can't pass -vcvars_ver=$VCVARS_VER here because cmd.exe eats all '='
# in bat file arguments. :-(
diff --git a/make/common/MakeBase.gmk b/make/common/MakeBase.gmk
index 97ef88932cb..45cc3c77dea 100644
--- a/make/common/MakeBase.gmk
+++ b/make/common/MakeBase.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -141,6 +141,66 @@ endef
# Make sure logging is setup for everyone that includes MakeBase.gmk.
$(eval $(call SetupLogging))
+################################################################################
+# Make does not have support for VARARGS, you can send variable amount
+# of arguments, but you can for example not append a list at the end.
+# It is therefore not easy to send the elements of a list of unknown
+# length as argument to a function. This can somewhat be worked around
+# by sending a list as an argument, and then interpreting each element
+# of the list as an argument to the function. However, Make is
+# limited, and using this method you can not easily send spaces.
+#
+# We need to quote strings for two reasons when sending them as
+# "variable append packs":
+#
+# 1) variable appends can include spaces, and those must be preserved
+# 2) variable appends can include assignment strings ":=", and those
+# must be quoted to a form so that we can recognise the "append pack".
+# We recognise an "append pack" by its lack of strict assignment ":="
+
+Q := $(HASH)
+SpaceQ := $(Q)s
+AppendQ := $(Q)+
+AssignQ := $(Q)a
+QQ := $(Q)$(Q)
+
+# $(call Quote,echo "#trala:=") -> echo#s"##trala#a"
+Quote = $(subst :=,$(AssignQ),$(subst $(SPACE),$(SpaceQ),$(subst $(Q),$(QQ),$1)))
+
+# $(call Unquote,echo#s"##trala#a") -> echo "#trala:="
+Unquote = $(subst $(QQ),$(Q),$(subst $(SpaceQ),$(SPACE),$(subst $(AssignQ),:=,$1)))
+
+# $(call QuoteAppend,name,some value) -> name#+some#svalue
+# $(call QuoteAppend,bad+=name,some value) -> error
+QuoteAppend = $(if $(findstring +=,$1),$(error you can not have += in a variable name: "$1"),$(call Quote,$1)$(AppendQ)$(call Quote,$2))
+
+# $(call UnquoteAppendIndex,name#+some#svalue,1) -> name
+# $(call UnquoteAppendIndex,name#+some#svalue,2) -> some value
+UnquoteAppendIndex = $(call Unquote,$(word $2,$(subst $(AppendQ),$(SPACE),$1)))
+
+# $(call FilterFiles,dir,%.cpp) -> file1.cpp file2.cpp (without path)
+FilterFiles = $(filter $2,$(notdir $(call FindFiles,$1)))
+
+# $(call Unpack module_,file1.cpp_CXXFLAGS#+-Wconversion file2.cpp_CXXFLAGS#+-Wconversion) -> module_file1.cpp_CXXFLAGS += -Wconversion
+# module_file2.cpp_CXXFLAGS += -Wconversion
+Unpack = $(foreach pair,$2,$1$(call UnquoteAppendIndex,$(pair),1) += $(call UnquoteAppendIndex,$(pair),2)$(NEWLINE))
+
+# This macro takes four arguments:
+# $1: directory where to find files (striped), example: $(TOPDIR)/src/hotspot/share/gc/g1
+# $2: filter to match what to keep (striped), example: g1Concurrent%.cpp
+# $3: what flags to override (striped), example: _CXXFLAGS
+# $4: what value to append to the flag (striped), example: $(CFLAGS_CONVERSION_WARNINGS)
+#
+# The result will be a quoted string that can be unpacked to a list of
+# variable appendings (see macro Unpack above). You do not need to take
+# care of unpacking, it is done in NamedParamsMacroTemplate.
+#
+# This feature should only be used for warnings that we want to
+# incrementally add to the rest of the code base.
+#
+# $(call ExtendFlags,dir,%.cpp,_CXXFLAGS,-Wconversion) -> file1.cpp_CXXFLAGS#+-Wconversion file2.cpp_CXXFLAGS#+-Wconversion
+ExtendFlags = $(foreach file,$(call FilterFiles,$(strip $1),$(strip $2)),$(call QuoteAppend,$(file)$(strip $3),$(strip $4)))
+
################################################################################
MAX_PARAMS := 96
@@ -166,7 +226,10 @@ define NamedParamsMacroTemplate
Too many named arguments to macro, please update MAX_PARAMS in MakeBase.gmk))
# Iterate over 2 3 4... and evaluate the named parameters with $1_ as prefix
$(foreach i, $(PARAM_SEQUENCE), $(if $(strip $($i)), \
- $(strip $1)_$(strip $(call EscapeHash, $(call DoubleDollar, $($i))))$(NEWLINE)))
+ $(if $(findstring :=,$($i)), \
+ $(strip $1)_$(strip $(call EscapeHash, $(call DoubleDollar, $($i))))$(NEWLINE), \
+ $(call Unpack,$(strip $1)_,$($i)))))
+
# Debug print all named parameter names and values
$(if $(findstring $(LOG_LEVEL), trace), \
$(info $0 $(strip $1) $(foreach i, $(PARAM_SEQUENCE), \
diff --git a/make/common/modules/LauncherCommon.gmk b/make/common/modules/LauncherCommon.gmk
index 7682ffbb95c..8d45142ef4a 100644
--- a/make/common/modules/LauncherCommon.gmk
+++ b/make/common/modules/LauncherCommon.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -35,11 +35,17 @@ include ProcessMarkdown.gmk
include $(TOPDIR)/make/ToolsJdk.gmk
LAUNCHER_SRC := $(TOPDIR)/src/java.base/share/native/launcher
+
LAUNCHER_CFLAGS += -I$(TOPDIR)/src/java.base/share/native/launcher \
-I$(TOPDIR)/src/java.base/share/native/libjli \
-I$(TOPDIR)/src/java.base/$(OPENJDK_TARGET_OS_TYPE)/native/libjli \
-I$(TOPDIR)/src/java.base/$(OPENJDK_TARGET_OS)/native/libjli \
#
+
+ifeq ($(call isTargetOs, aix), true)
+ LAUNCHER_CFLAGS += -I$(TOPDIR)/src/java.base/aix/native/include
+endif
+
MACOSX_PLIST_DIR := $(TOPDIR)/src/java.base/macosx/native/launcher
JAVA_MANIFEST := $(TOPDIR)/src/java.base/windows/native/launcher/java.manifest
diff --git a/make/conf/github-actions.conf b/make/conf/github-actions.conf
index ebfc9191535..6771e8923dc 100644
--- a/make/conf/github-actions.conf
+++ b/make/conf/github-actions.conf
@@ -29,21 +29,21 @@ GTEST_VERSION=1.14.0
JTREG_VERSION=8.2.1+1
LINUX_X64_BOOT_JDK_EXT=tar.gz
-LINUX_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk25/bd75d5f9689641da8e1daabeccb5528b/36/GPL/openjdk-25_linux-x64_bin.tar.gz
-LINUX_X64_BOOT_JDK_SHA256=59cdcaf255add4721de38eb411d4ecfe779356b61fb671aee63c7dec78054c2b
+LINUX_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk26/c3cc523845074aa0af4f5e1e1ed4151d/35/GPL/openjdk-26_linux-x64_bin.tar.gz
+LINUX_X64_BOOT_JDK_SHA256=83c78367f8c81257beef72aca4bbbf8e6dac8ca2b3a4546a85879a09e6e4e128
ALPINE_LINUX_X64_BOOT_JDK_EXT=tar.gz
-ALPINE_LINUX_X64_BOOT_JDK_URL=https://github.com/adoptium/temurin25-binaries/releases/download/jdk-25%2B36/OpenJDK25U-jdk_x64_alpine-linux_hotspot_25_36.tar.gz
-ALPINE_LINUX_X64_BOOT_JDK_SHA256=637e47474d411ed86134f413af7d5fef4180ddb0bf556347b7e74a88cf8904c8
+ALPINE_LINUX_X64_BOOT_JDK_URL=https://github.com/adoptium/temurin26-binaries/releases/download/jdk-26%2B35/OpenJDK26U-jdk_x64_alpine-linux_hotspot_26_35.tar.gz
+ALPINE_LINUX_X64_BOOT_JDK_SHA256=c105e581fdccb4e7120d889235d1ad8d5b2bed0af4972bc881e0a8ba687c94a4
MACOS_AARCH64_BOOT_JDK_EXT=tar.gz
-MACOS_AARCH64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk25/bd75d5f9689641da8e1daabeccb5528b/36/GPL/openjdk-25_macos-aarch64_bin.tar.gz
-MACOS_AARCH64_BOOT_JDK_SHA256=2006337bf326fdfdf6117081751ba38c1c8706d63419ecac7ff102ff7c776876
+MACOS_AARCH64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk26/c3cc523845074aa0af4f5e1e1ed4151d/35/GPL/openjdk-26_macos-aarch64_bin.tar.gz
+MACOS_AARCH64_BOOT_JDK_SHA256=254586bcd1bf6dcd125ad667ac32562cb1e2ab1abf3a61fb117b6fabb571e765
MACOS_X64_BOOT_JDK_EXT=tar.gz
-MACOS_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk25/bd75d5f9689641da8e1daabeccb5528b/36/GPL/openjdk-25_macos-x64_bin.tar.gz
-MACOS_X64_BOOT_JDK_SHA256=47482ad9888991ecac9b2bcc131e2b53ff78aff275104cef85f66252308e8a09
+MACOS_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk26/c3cc523845074aa0af4f5e1e1ed4151d/35/GPL/openjdk-26_macos-x64_bin.tar.gz
+MACOS_X64_BOOT_JDK_SHA256=8642b89d889c14ede2c446fd5bbe3621c8a3082e3df02013fd1658e39f52929a
WINDOWS_X64_BOOT_JDK_EXT=zip
-WINDOWS_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk25/bd75d5f9689641da8e1daabeccb5528b/36/GPL/openjdk-25_windows-x64_bin.zip
-WINDOWS_X64_BOOT_JDK_SHA256=85bcc178461e2cb3c549ab9ca9dfa73afd54c09a175d6510d0884071867137d3
+WINDOWS_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk26/c3cc523845074aa0af4f5e1e1ed4151d/35/GPL/openjdk-26_windows-x64_bin.zip
+WINDOWS_X64_BOOT_JDK_SHA256=2dd2d92c9374cd49a120fe9d916732840bf6bb9f0e0cc29794917a3c08b99c5f
diff --git a/make/conf/jib-profiles.js b/make/conf/jib-profiles.js
index 76a94b7789e..4c1d2835054 100644
--- a/make/conf/jib-profiles.js
+++ b/make/conf/jib-profiles.js
@@ -387,8 +387,8 @@ var getJibProfilesCommon = function (input, data) {
};
};
- common.boot_jdk_version = "25";
- common.boot_jdk_build_number = "37";
+ common.boot_jdk_version = "26";
+ common.boot_jdk_build_number = "35";
common.boot_jdk_home = input.get("boot_jdk", "install_path") + "/jdk-"
+ common.boot_jdk_version
+ (input.build_os == "macosx" ? ".jdk/Contents/Home" : "");
diff --git a/make/conf/version-numbers.conf b/make/conf/version-numbers.conf
index 4392d86ac33..4f63179ae05 100644
--- a/make/conf/version-numbers.conf
+++ b/make/conf/version-numbers.conf
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -37,6 +37,6 @@ DEFAULT_VERSION_DATE=2026-09-15
DEFAULT_VERSION_CLASSFILE_MAJOR=71 # "`$EXPR $DEFAULT_VERSION_FEATURE + 44`"
DEFAULT_VERSION_CLASSFILE_MINOR=0
DEFAULT_VERSION_DOCS_API_SINCE=11
-DEFAULT_ACCEPTABLE_BOOT_VERSIONS="25 26 27"
+DEFAULT_ACCEPTABLE_BOOT_VERSIONS="26 27"
DEFAULT_JDK_SOURCE_TARGET_VERSION=27
DEFAULT_PROMOTED_VERSION_PRE=ea
diff --git a/make/hotspot/lib/CompileGtest.gmk b/make/hotspot/lib/CompileGtest.gmk
index 327014b1e9d..4b21d481049 100644
--- a/make/hotspot/lib/CompileGtest.gmk
+++ b/make/hotspot/lib/CompileGtest.gmk
@@ -63,6 +63,10 @@ $(eval $(call SetupJdkLibrary, BUILD_GTEST_LIBGTEST, \
unused-result zero-as-null-pointer-constant, \
DISABLED_WARNINGS_clang := format-nonliteral undef unused-result \
zero-as-null-pointer-constant, \
+ $(comment Disable deprecated-declarations warnings to workaround) \
+ $(comment clang18+glibc12 bug https://github.com/llvm/llvm-project/issues/76515) \
+ $(comment until the clang bug has been fixed) \
+ DISABLED_WARNINGS_clang_gtest-all.cc := deprecated-declarations, \
DISABLED_WARNINGS_microsoft := 4530, \
DEFAULT_CFLAGS := false, \
CFLAGS := $(JVM_CFLAGS) \
diff --git a/make/hotspot/lib/CompileJvm.gmk b/make/hotspot/lib/CompileJvm.gmk
index 39a549b7db0..f41693e05fb 100644
--- a/make/hotspot/lib/CompileJvm.gmk
+++ b/make/hotspot/lib/CompileJvm.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2013, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2013, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -105,7 +105,7 @@ DISABLED_WARNINGS_gcc := array-bounds comment delete-non-virtual-dtor \
DISABLED_WARNINGS_clang := delete-non-abstract-non-virtual-dtor \
invalid-offsetof missing-braces \
sometimes-uninitialized unknown-pragmas unused-but-set-variable \
- unused-function unused-local-typedef unused-private-field unused-variable
+ unused-local-typedef unused-private-field unused-variable
ifneq ($(DEBUG_LEVEL), release)
# Assert macro gives warning
@@ -190,6 +190,8 @@ $(eval $(call SetupJdkLibrary, BUILD_LIBJVM, \
abstract_vm_version.cpp_CXXFLAGS := $(CFLAGS_VM_VERSION), \
arguments.cpp_CXXFLAGS := $(CFLAGS_VM_VERSION), \
whitebox.cpp_CXXFLAGS := $(CFLAGS_SHIP_DEBUGINFO), \
+ $(call ExtendFlags, $(TOPDIR)/src/hotspot/share/gc/g1, \
+ g1Numa.cpp, _CXXFLAGS, $(CFLAGS_CONVERSION_WARNINGS)), \
DISABLED_WARNINGS_gcc := $(DISABLED_WARNINGS_gcc), \
DISABLED_WARNINGS_gcc_ad_$(HOTSPOT_TARGET_CPU_ARCH).cpp := nonnull, \
DISABLED_WARNINGS_gcc_bytecodeInterpreter.cpp := unused-label, \
diff --git a/make/hotspot/lib/JvmFlags.gmk b/make/hotspot/lib/JvmFlags.gmk
index 57b632ee532..27a96cc4865 100644
--- a/make/hotspot/lib/JvmFlags.gmk
+++ b/make/hotspot/lib/JvmFlags.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2013, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2013, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -43,10 +43,15 @@ JVM_SRC_DIRS += $(call uniq, $(wildcard $(foreach d, $(JVM_SRC_ROOTS), \
$(JVM_VARIANT_OUTPUTDIR)/gensrc
#
+ifeq ($(call isTargetOs, aix), true)
+ ADD_PLATFORM_INCLUDE_DIR := -I$(TOPDIR)/src/java.base/aix/native/include
+endif
+
JVM_CFLAGS_INCLUDES += \
$(patsubst %,-I%,$(JVM_SRC_DIRS)) \
-I$(TOPDIR)/src/hotspot/share/include \
-I$(TOPDIR)/src/hotspot/os/$(HOTSPOT_TARGET_OS_TYPE)/include \
+ $(ADD_PLATFORM_INCLUDE_DIR) \
-I$(SUPPORT_OUTPUTDIR)/modules_include/java.base \
-I$(SUPPORT_OUTPUTDIR)/modules_include/java.base/$(OPENJDK_TARGET_OS_INCLUDE_SUBDIR) \
-I$(TOPDIR)/src/java.base/share/native/libjimage \
diff --git a/make/jdk/src/classes/build/tools/intpoly/FieldGen.java b/make/jdk/src/classes/build/tools/intpoly/FieldGen.java
index fcc45db0219..c29a315f368 100644
--- a/make/jdk/src/classes/build/tools/intpoly/FieldGen.java
+++ b/make/jdk/src/classes/build/tools/intpoly/FieldGen.java
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -34,36 +34,6 @@ import java.util.*;
public class FieldGen {
- static FieldParams Curve25519 = new FieldParams(
- "IntegerPolynomial25519", 26, 10, 1, 255,
- Arrays.asList(
- new Term(0, -19)
- ),
- Curve25519CrSequence(), simpleSmallCrSequence(10)
- );
-
- private static List Curve25519CrSequence() {
- List result = new ArrayList();
-
- // reduce(7,2)
- result.add(new Reduce(17));
- result.add(new Reduce(18));
-
- // carry(8,2)
- result.add(new Carry(8));
- result.add(new Carry(9));
-
- // reduce(0,7)
- for (int i = 10; i < 17; i++) {
- result.add(new Reduce(i));
- }
-
- // carry(0,9)
- result.addAll(fullCarry(10));
-
- return result;
- }
-
static FieldParams Curve448 = new FieldParams(
"IntegerPolynomial448", 28, 16, 1, 448,
Arrays.asList(
@@ -224,8 +194,7 @@ public class FieldGen {
}
static final FieldParams[] ALL_FIELDS = {
- Curve25519, Curve448,
- P256, P384, P521, O256, O384, O521, O25519, O448
+ Curve448, P256, P384, P521, O256, O384, O521, O25519, O448
};
public static class Term {
diff --git a/make/jdk/src/classes/build/tools/taglet/SealedGraph.java b/make/jdk/src/classes/build/tools/taglet/SealedGraph.java
index 3e93826c180..2ffd92e3409 100644
--- a/make/jdk/src/classes/build/tools/taglet/SealedGraph.java
+++ b/make/jdk/src/classes/build/tools/taglet/SealedGraph.java
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2022, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -138,20 +138,25 @@ public final class SealedGraph implements Taglet {
// Generates a graph in DOT format
String graph(TypeElement rootClass, Set exports) {
+ if (!isInPublicApi(rootClass, exports)) {
+ // Alternatively we can return "" for the graph since there is no single root to render
+ throw new IllegalArgumentException("Root not in public API: " + rootClass.getQualifiedName());
+ }
final State state = new State(rootClass);
traverse(state, rootClass, exports);
return state.render();
}
static void traverse(State state, TypeElement node, Set exports) {
+ if (!isInPublicApi(node, exports)) {
+ throw new IllegalArgumentException("Bad request, not in public API: " + node.getQualifiedName());
+ }
state.addNode(node);
if (!(node.getModifiers().contains(Modifier.SEALED) || node.getModifiers().contains(Modifier.FINAL))) {
state.addNonSealedEdge(node);
} else {
for (TypeElement subNode : permittedSubclasses(node, exports)) {
- if (isInPublicApi(node, exports) && isInPublicApi(subNode, exports)) {
- state.addEdge(node, subNode);
- }
+ state.addEdge(node, subNode);
traverse(state, subNode, exports);
}
}
@@ -292,14 +297,30 @@ public final class SealedGraph implements Taglet {
}
private static List permittedSubclasses(TypeElement node, Set exports) {
- return node.getPermittedSubclasses().stream()
- .filter(DeclaredType.class::isInstance)
- .map(DeclaredType.class::cast)
- .map(DeclaredType::asElement)
- .filter(TypeElement.class::isInstance)
- .map(TypeElement.class::cast)
- .filter(te -> isInPublicApi(te, exports))
- .toList();
+ List dfsStack = new ArrayList().reversed(); // Faster operations to head
+ SequencedCollection result = new LinkedHashSet<>(); // Deduplicate diamond interface inheritance
+ // The starting node may be in the public API - still expand it
+ prependSubclasses(node, dfsStack);
+
+ while (!dfsStack.isEmpty()) {
+ TypeElement now = dfsStack.removeFirst();
+ if (isInPublicApi(now, exports)) {
+ result.addLast(now);
+ } else {
+ // Skip the non-exported classes in the hierarchy
+ prependSubclasses(now, dfsStack);
+ }
+ }
+
+ return List.copyOf(result);
+ }
+
+ private static void prependSubclasses(TypeElement node, List dfs) {
+ for (var e : node.getPermittedSubclasses().reversed()) {
+ if (e instanceof DeclaredType dt && dt.asElement() instanceof TypeElement te) {
+ dfs.addFirst(te);
+ }
+ }
}
private static boolean isInPublicApi(TypeElement typeElement, Set exports) {
diff --git a/make/langtools/tools/previewfeature/SetupPreviewFeature.java b/make/langtools/tools/previewfeature/SetupPreviewFeature.java
new file mode 100644
index 00000000000..5f9b00edc6d
--- /dev/null
+++ b/make/langtools/tools/previewfeature/SetupPreviewFeature.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation. Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package previewfeature;
+
+import com.sun.source.util.JavacTask;
+import com.sun.source.util.Trees;
+import java.io.StringWriter;
+import java.lang.reflect.Field;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
+import javax.lang.model.element.ElementKind;
+import javax.tools.ToolProvider;
+
+/* Construct a hybrid PreviewFeature.Feature enum that includes constants both
+ * from the current JDK sources (so that they can be used in the javac API sources),
+ * and from the bootstrap JDK (so that they can be used in the bootstrap classfiles).
+ *
+ * This hybrid enum is only used for the interim javac.
+ */
+public class SetupPreviewFeature {
+ public static void main(String... args) throws Exception {
+ Class runtimeFeature = Class.forName("jdk.internal.javac.PreviewFeature$Feature");
+ Set constantsToAdd = new HashSet<>();
+ for (Field runtimeField : runtimeFeature.getDeclaredFields()) {
+ if (runtimeField.isEnumConstant()) {
+ constantsToAdd.add(runtimeField.getName());
+ }
+ }
+ var dummy = new StringWriter();
+ var compiler = ToolProvider.getSystemJavaCompiler();
+ var source = Path.of(args[0]);
+ try (var fm = compiler.getStandardFileManager(null, null, null)) {
+ JavacTask task =
+ (JavacTask) compiler.getTask(dummy, null, null, null, null, fm.getJavaFileObjects(source));
+ task.analyze();
+ var sourceFeature = task.getElements()
+ .getTypeElement("jdk.internal.javac.PreviewFeature.Feature");
+ int insertPosition = -1;
+ for (var el : sourceFeature.getEnclosedElements()) {
+ if (el.getKind() == ElementKind.ENUM_CONSTANT) {
+ constantsToAdd.remove(el.getSimpleName().toString());
+ if (insertPosition == (-1)) {
+ var trees = Trees.instance(task);
+ var elPath = trees.getPath(el);
+ insertPosition = (int) trees.getSourcePositions()
+ .getStartPosition(elPath.getCompilationUnit(),
+ elPath.getLeaf());
+ }
+ }
+ }
+ var target = Path.of(args[1]);
+ Files.createDirectories(target.getParent());
+ if (constantsToAdd.isEmpty()) {
+ Files.copy(source, target, StandardCopyOption.REPLACE_EXISTING);
+ } else {
+ String sourceCode = Files.readString(source);
+ try (var out = Files.newBufferedWriter(target)) {
+ out.write(sourceCode, 0, insertPosition);
+ out.write(constantsToAdd.stream()
+ .collect(Collectors.joining(", ",
+ "/*compatibility constants:*/ ",
+ ",\n")));
+ out.write(sourceCode, insertPosition, sourceCode.length() - insertPosition);
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/make/modules/java.base/Launcher.gmk b/make/modules/java.base/Launcher.gmk
index 3a3920acb12..bfae0925c07 100644
--- a/make/modules/java.base/Launcher.gmk
+++ b/make/modules/java.base/Launcher.gmk
@@ -95,7 +95,8 @@ ifeq ($(call isTargetOsType, unix), true)
CFLAGS := $(VERSION_CFLAGS), \
EXTRA_HEADER_DIRS := libjava, \
EXTRA_OBJECT_FILES := \
- $(SUPPORT_OUTPUTDIR)/native/$(MODULE)/libjava/childproc$(OBJ_SUFFIX), \
+ $(SUPPORT_OUTPUTDIR)/native/$(MODULE)/libjava/childproc$(OBJ_SUFFIX) \
+ $(SUPPORT_OUTPUTDIR)/native/$(MODULE)/libjava/childproc_errorcodes$(OBJ_SUFFIX), \
LD_SET_ORIGIN := false, \
OUTPUT_DIR := $(SUPPORT_OUTPUTDIR)/modules_libs/$(MODULE), \
))
diff --git a/make/modules/java.desktop/lib/AwtLibraries.gmk b/make/modules/java.desktop/lib/AwtLibraries.gmk
index 8b6b50b9e62..887dfab01df 100644
--- a/make/modules/java.desktop/lib/AwtLibraries.gmk
+++ b/make/modules/java.desktop/lib/AwtLibraries.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -99,14 +99,16 @@ ifeq ($(call isTargetOs, windows), true)
$(TOPDIR)/src/$(MODULE)/windows/native/libawt/windows/awt.rc
endif
-# This is the object file to provide the dladdr API, which is not
-# part of AIX. It occurs several times in the jdk code base.
-# Do not include it. When statically linking the java
-# launcher with all JDK and VM static libraries, we use the
-# --whole-archive linker option. The duplicate objects in different
-# static libraries cause linking errors due to duplicate symbols.
ifeq ($(call isTargetOs, aix), true)
+ # This is the object file to provide the dladdr API, which is not
+ # part of AIX. It occurs several times in the jdk code base.
+ # Do not include it. When statically linking the java
+ # launcher with all JDK and VM static libraries, we use the
+ # --whole-archive linker option. The duplicate objects in different
+ # static libraries cause linking errors due to duplicate symbols.
LIBAWT_STATIC_EXCLUDE_OBJS := porting_aix.o
+
+ LIBAWT_CFLAGS += -I$(TOPDIR)/src/java.base/aix/native/include
endif
# -fgcse-after-reload improves performance of MaskFill in Java2D by 20% for
@@ -423,6 +425,9 @@ endif
ifeq ($(call isTargetOs, linux)+$(ENABLE_HEADLESS_ONLY), true+true)
LIBJAWT_CFLAGS += -DHEADLESS
endif
+ifeq ($(call isTargetOs, aix)+$(ENABLE_HEADLESS_ONLY), true+true)
+ LIBJAWT_CFLAGS += -DHEADLESS
+endif
ifeq ($(call isTargetOs, windows)+$(call isTargetCpu, x86), true+true)
LIBJAWT_LIBS_windows := kernel32.lib
diff --git a/make/modules/java.desktop/lib/ClientLibraries.gmk b/make/modules/java.desktop/lib/ClientLibraries.gmk
index b76cb8dc4e3..3e37fe79643 100644
--- a/make/modules/java.desktop/lib/ClientLibraries.gmk
+++ b/make/modules/java.desktop/lib/ClientLibraries.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -257,6 +257,7 @@ ifeq ($(ENABLE_HEADLESS_ONLY), false)
DISABLED_WARNINGS_microsoft_dgif_lib.c := 4018 4267, \
DISABLED_WARNINGS_microsoft_splashscreen_impl.c := 4018 4267 4244, \
DISABLED_WARNINGS_microsoft_splashscreen_png.c := 4267, \
+ DISABLED_WARNINGS_microsoft_pngread.c := 4146, \
DISABLED_WARNINGS_microsoft_splashscreen_sys.c := 4267 4244, \
LDFLAGS := $(ICONV_LDFLAGS), \
LDFLAGS_windows := -delayload:user32.dll, \
@@ -338,11 +339,8 @@ else
# noexcept-type required for GCC 7 builds. Not required for GCC 8+.
# expansion-to-defined required for GCC 9 builds. Not required for GCC 10+.
# maybe-uninitialized required for GCC 8 builds. Not required for GCC 9+.
- # calloc-transposed-args required for GCC 14 builds. (fixed upstream in
- # Harfbuzz 032c931e1c0cfb20f18e5acb8ba005775242bd92)
HARFBUZZ_DISABLED_WARNINGS_CXX_gcc := class-memaccess noexcept-type \
- expansion-to-defined dangling-reference maybe-uninitialized \
- calloc-transposed-args
+ expansion-to-defined dangling-reference maybe-uninitialized
HARFBUZZ_DISABLED_WARNINGS_clang := missing-field-initializers \
range-loop-analysis unused-variable
HARFBUZZ_DISABLED_WARNINGS_microsoft := 4267 4244
diff --git a/make/modules/jdk.jpackage/Java.gmk b/make/modules/jdk.jpackage/Java.gmk
index da66fc14009..1fd4d527217 100644
--- a/make/modules/jdk.jpackage/Java.gmk
+++ b/make/modules/jdk.jpackage/Java.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -29,7 +29,7 @@ DISABLED_WARNINGS_java += dangling-doc-comments
COPY += .gif .png .txt .spec .script .prerm .preinst \
.postrm .postinst .list .sh .desktop .copyright .control .plist .template \
- .icns .scpt .wxs .wxl .wxi .wxf .ico .bmp .tiff .service .xsl
+ .icns .scpt .wxs .wxl .wxi .wxf .ico .bmp .tiff .service .xsl .js
CLEAN += .properties
diff --git a/make/scripts/fixpath.sh b/make/scripts/fixpath.sh
index 6a524df4c68..78690f1f2cc 100644
--- a/make/scripts/fixpath.sh
+++ b/make/scripts/fixpath.sh
@@ -88,7 +88,10 @@ function setup() {
fi
if [[ -z ${CMD+x} ]]; then
- CMD="$DRIVEPREFIX/c/windows/system32/cmd.exe"
+ CMD="$(type -p cmd.exe 2>/dev/null)"
+ if [[ -z "$CMD" ]]; then
+ CMD="$DRIVEPREFIX/c/windows/system32/cmd.exe"
+ fi
fi
if [[ -z ${WINTEMP+x} ]]; then
diff --git a/make/test/JtregNativeJdk.gmk b/make/test/JtregNativeJdk.gmk
index 0482011f561..6774e708f99 100644
--- a/make/test/JtregNativeJdk.gmk
+++ b/make/test/JtregNativeJdk.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2015, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2015, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -63,7 +63,8 @@ ifeq ($(call isTargetOs, windows), true)
BUILD_JDK_JTREG_EXCLUDE += libDirectIO.c libInheritedChannel.c \
libExplicitAttach.c libImplicitAttach.c \
exelauncher.c libFDLeaker.c exeFDLeakTester.c \
- libChangeSignalDisposition.c exePrintSignalDisposition.c
+ libChangeSignalDisposition.c exePrintSignalDisposition.c \
+ libConcNativeFork.c libPipesCloseOnExec.c
BUILD_JDK_JTREG_EXECUTABLES_LIBS_exeNullCallerTest := $(LIBCXX)
BUILD_JDK_JTREG_EXECUTABLES_LIBS_exerevokeall := advapi32.lib
@@ -77,6 +78,9 @@ else
BUILD_JDK_JTREG_LIBRARIES_LDFLAGS_libLinkerInvokerUnnamed := -pthread
BUILD_JDK_JTREG_LIBRARIES_LDFLAGS_libLinkerInvokerModule := -pthread
BUILD_JDK_JTREG_LIBRARIES_LDFLAGS_libLoaderLookupInvoker := -pthread
+ BUILD_JDK_JTREG_LIBRARIES_LDFLAGS_libConcNativeFork := -pthread
+ BUILD_JDK_JTREG_LIBRARIES_LDFLAGS_libPipesCloseOnExec := -pthread
+ BUILD_JDK_JTREG_LIBRARIES_LDFLAGS_libLoaderLookupInvoker := -pthread
BUILD_JDK_JTREG_LIBRARIES_LDFLAGS_libExplicitAttach := -pthread
BUILD_JDK_JTREG_LIBRARIES_LDFLAGS_libImplicitAttach := -pthread
diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad
index a9ca91d9309..53fa4e3066c 100644
--- a/src/hotspot/cpu/aarch64/aarch64.ad
+++ b/src/hotspot/cpu/aarch64/aarch64.ad
@@ -1,5 +1,5 @@
//
-// Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
// Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
// Copyright 2025 Arm Limited and/or its affiliates.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -1182,12 +1182,12 @@ class CallStubImpl {
public:
// Size of call trampoline stub.
static uint size_call_trampoline() {
- return 0; // no call trampolines on this platform
+ return MacroAssembler::max_trampoline_stub_size();
}
// number of relocations needed by a call trampoline stub
static uint reloc_call_trampoline() {
- return 0; // no call trampolines on this platform
+ return 5; // metadata; call dest; trampoline address; trampoline destination; trampoline_owner_metadata
}
};
@@ -2233,15 +2233,9 @@ uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
{
st->print_cr("# MachUEPNode");
- if (UseCompressedClassPointers) {
- st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
- st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
- st->print_cr("\tcmpw rscratch1, r10");
- } else {
- st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
- st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
- st->print_cr("\tcmp rscratch1, r10");
- }
+ st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
+ st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
+ st->print_cr("\tcmpw rscratch1, r10");
st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
}
#endif
@@ -2467,11 +2461,8 @@ bool Matcher::is_generic_vector(MachOper* opnd) {
return opnd->opcode() == VREG;
}
+#ifdef ASSERT
// Return whether or not this register is ever used as an argument.
-// This function is used on startup to build the trampoline stubs in
-// generateOptoStub. Registers not mentioned will be killed by the VM
-// call in the trampoline, and arguments in those registers not be
-// available to the callee.
bool Matcher::can_be_java_arg(int reg)
{
return
@@ -2492,11 +2483,7 @@ bool Matcher::can_be_java_arg(int reg)
reg == V6_num || reg == V6_H_num ||
reg == V7_num || reg == V7_H_num;
}
-
-bool Matcher::is_spillable_arg(int reg)
-{
- return can_be_java_arg(reg);
-}
+#endif
uint Matcher::int_pressure_limit()
{
@@ -2531,10 +2518,6 @@ uint Matcher::float_pressure_limit()
return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
}
-bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
- return false;
-}
-
const RegMask& Matcher::divI_proj_mask() {
ShouldNotReachHere();
return RegMask::EMPTY;
@@ -3403,11 +3386,13 @@ encode %{
} else if (rtype == relocInfo::metadata_type) {
__ mov_metadata(dst_reg, (Metadata*)con);
} else {
- assert(rtype == relocInfo::none, "unexpected reloc type");
+ assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
+ // load fake address constants using a normal move
if (! __ is_valid_AArch64_address(con) ||
con < (address)(uintptr_t)os::vm_page_size()) {
__ mov(dst_reg, con);
} else {
+ // no reloc so just use adrp and add
uint64_t offset;
__ adrp(dst_reg, con, offset);
__ add(dst_reg, dst_reg, offset);
@@ -3812,11 +3797,6 @@ frame %{
// Compiled code's Frame Pointer
frame_pointer(R31);
- // Interpreter stores its frame pointer in a register which is
- // stored to the stack by I2CAdaptors.
- // I2CAdaptors convert from interpreted java to compiled java.
- interpreter_frame_pointer(R29);
-
// Stack alignment requirement
stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
@@ -4535,6 +4515,18 @@ operand immP_1()
interface(CONST_INTER);
%}
+// AOT Runtime Constants Address
+operand immAOTRuntimeConstantsAddress()
+%{
+ // Check if the address is in the range of AOT Runtime Constants
+ predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
+ match(ConP);
+
+ op_cost(0);
+ format %{ %}
+ interface(CONST_INTER);
+%}
+
// Float and Double operands
// Double Immediate
operand immD()
@@ -6898,6 +6890,20 @@ instruct loadConP1(iRegPNoSp dst, immP_1 con)
ins_pipe(ialu_imm);
%}
+instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
+%{
+ match(Set dst con);
+
+ ins_cost(INSN_COST);
+ format %{ "adr $dst, $con\t# AOT Runtime Constants Address" %}
+
+ ins_encode %{
+ __ load_aotrc_address($dst$$Register, (address)$con$$constant);
+ %}
+
+ ins_pipe(ialu_imm);
+%}
+
// Load Narrow Pointer Constant
instruct loadConN(iRegNNoSp dst, immN con)
@@ -8008,6 +8014,21 @@ instruct membar_release_lock() %{
ins_pipe(pipe_serial);
%}
+instruct membar_storeload() %{
+ match(MemBarStoreLoad);
+ ins_cost(VOLATILE_REF_COST*100);
+
+ format %{ "MEMBAR-store-load\n\t"
+ "dmb ish" %}
+
+ ins_encode %{
+ __ block_comment("membar_storeload");
+ __ membar(Assembler::StoreLoad);
+ %}
+
+ ins_pipe(pipe_serial);
+%}
+
instruct unnecessary_membar_volatile() %{
predicate(unnecessary_volatile(n));
match(MemBarVolatile);
@@ -8037,6 +8058,20 @@ instruct membar_volatile() %{
ins_pipe(pipe_serial);
%}
+instruct membar_full() %{
+ match(MemBarFull);
+ ins_cost(VOLATILE_REF_COST*100);
+
+ format %{ "membar_full\n\t"
+ "dmb ish" %}
+ ins_encode %{
+ __ block_comment("membar_full");
+ __ membar(Assembler::AnyAny);
+ %}
+
+ ins_pipe(pipe_serial);
+%}
+
// ============================================================================
// Cast/Convert Instructions
diff --git a/src/hotspot/cpu/aarch64/assembler_aarch64.hpp b/src/hotspot/cpu/aarch64/assembler_aarch64.hpp
index 19b3bb1a65b..ebd8f3a9e03 100644
--- a/src/hotspot/cpu/aarch64/assembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/assembler_aarch64.hpp
@@ -1095,6 +1095,10 @@ public:
#undef INSN
+ void wfet(Register rt) {
+ system(0b00, 0b011, 0b0001, 0b0000, 0b000, rt);
+ }
+
// we only provide mrs and msr for the special purpose system
// registers where op1 (instr[20:19]) == 11
// n.b msr has L (instr[21]) == 0 mrs has L == 1
@@ -3814,8 +3818,8 @@ public:
}
private:
- void sve_cpy(FloatRegister Zd, SIMD_RegVariant T, PRegister Pg, int imm8,
- bool isMerge, bool isFloat) {
+ void _sve_cpy(FloatRegister Zd, SIMD_RegVariant T, PRegister Pg, int imm8,
+ bool isMerge, bool isFloat) {
starti;
assert(T != Q, "invalid size");
int sh = 0;
@@ -3839,11 +3843,11 @@ private:
public:
// SVE copy signed integer immediate to vector elements (predicated)
void sve_cpy(FloatRegister Zd, SIMD_RegVariant T, PRegister Pg, int imm8, bool isMerge) {
- sve_cpy(Zd, T, Pg, imm8, isMerge, /*isFloat*/false);
+ _sve_cpy(Zd, T, Pg, imm8, isMerge, /*isFloat*/false);
}
// SVE copy floating-point immediate to vector elements (predicated)
void sve_cpy(FloatRegister Zd, SIMD_RegVariant T, PRegister Pg, double d) {
- sve_cpy(Zd, T, Pg, checked_cast(pack(d)), /*isMerge*/true, /*isFloat*/true);
+ _sve_cpy(Zd, T, Pg, checked_cast(pack(d)), /*isMerge*/true, /*isFloat*/true);
}
// SVE conditionally select elements from two vectors
diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
index c0621cbd5c2..4de6237304d 100644
--- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -33,6 +33,7 @@
#include "c1/c1_ValueStack.hpp"
#include "ci/ciArrayKlass.hpp"
#include "ci/ciInstance.hpp"
+#include "code/aotCodeCache.hpp"
#include "code/compiledIC.hpp"
#include "gc/shared/collectedHeap.hpp"
#include "gc/shared/gc_globals.hpp"
@@ -41,6 +42,7 @@
#include "runtime/frame.inline.hpp"
#include "runtime/sharedRuntime.hpp"
#include "runtime/stubRoutines.hpp"
+#include "runtime/threadIdentifier.hpp"
#include "utilities/powerOfTwo.hpp"
#include "vmreg_aarch64.inline.hpp"
@@ -58,22 +60,6 @@ const Register SHIFT_count = r0; // where count for shift operations must be
#define __ _masm->
-static void select_different_registers(Register preserve,
- Register extra,
- Register &tmp1,
- Register &tmp2) {
- if (tmp1 == preserve) {
- assert_different_registers(tmp1, tmp2, extra);
- tmp1 = extra;
- } else if (tmp2 == preserve) {
- assert_different_registers(tmp1, tmp2, extra);
- tmp2 = extra;
- }
- assert_different_registers(preserve, tmp1, tmp2);
-}
-
-
-
static void select_different_registers(Register preserve,
Register extra,
Register &tmp1,
@@ -532,6 +518,19 @@ void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_cod
case T_LONG: {
assert(patch_code == lir_patch_none, "no patching handled here");
+#if INCLUDE_CDS
+ if (AOTCodeCache::is_on_for_dump()) {
+ address b = c->as_pointer();
+ if (b == (address)ThreadIdentifier::unsafe_offset()) {
+ __ lea(dest->as_register_lo(), ExternalAddress(b));
+ break;
+ }
+ if (AOTRuntimeConstants::contains(b)) {
+ __ load_aotrc_address(dest->as_register_lo(), b);
+ break;
+ }
+ }
+#endif
__ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
break;
}
@@ -1259,12 +1258,9 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, L
} else if (obj == klass_RInfo) {
klass_RInfo = dst;
}
- if (k->is_loaded() && !UseCompressedClassPointers) {
- select_different_registers(obj, dst, k_RInfo, klass_RInfo);
- } else {
- Rtmp1 = op->tmp3()->as_register();
- select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
- }
+
+ Rtmp1 = op->tmp3()->as_register();
+ select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
assert_different_registers(obj, k_RInfo, klass_RInfo);
diff --git a/src/hotspot/cpu/aarch64/c1_LIRGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRGenerator_aarch64.cpp
index ad26d494b2d..f10c5197d91 100644
--- a/src/hotspot/cpu/aarch64/c1_LIRGenerator_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c1_LIRGenerator_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -1287,9 +1287,7 @@ void LIRGenerator::do_CheckCast(CheckCast* x) {
}
LIR_Opr reg = rlock_result(x);
LIR_Opr tmp3 = LIR_OprFact::illegalOpr;
- if (!x->klass()->is_loaded() || UseCompressedClassPointers) {
- tmp3 = new_register(objectType);
- }
+ tmp3 = new_register(objectType);
__ checkcast(reg, obj.result(), x->klass(),
new_register(objectType), new_register(objectType), tmp3,
x->direct_compare(), info_for_exception, patching_info, stub,
@@ -1308,9 +1306,7 @@ void LIRGenerator::do_InstanceOf(InstanceOf* x) {
}
obj.load_item();
LIR_Opr tmp3 = LIR_OprFact::illegalOpr;
- if (!x->klass()->is_loaded() || UseCompressedClassPointers) {
- tmp3 = new_register(objectType);
- }
+ tmp3 = new_register(objectType);
__ instanceof(reg, obj.result(), x->klass(),
new_register(objectType), new_register(objectType), tmp3,
x->direct_compare(), patching_info, x->profiled_method(), x->profiled_bci());
diff --git a/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp
index e934632715c..89a9422ea48 100644
--- a/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -105,12 +105,8 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
} else {
mov(t1, checked_cast(markWord::prototype().value()));
str(t1, Address(obj, oopDesc::mark_offset_in_bytes()));
- if (UseCompressedClassPointers) { // Take care not to kill klass
- encode_klass_not_null(t1, klass);
- strw(t1, Address(obj, oopDesc::klass_offset_in_bytes()));
- } else {
- str(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
- }
+ encode_klass_not_null(t1, klass); // Take care not to kill klass
+ strw(t1, Address(obj, oopDesc::klass_offset_in_bytes()));
}
if (len->is_valid()) {
@@ -121,7 +117,7 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
// Clear gap/first 4 bytes following the length field.
strw(zr, Address(obj, base_offset));
}
- } else if (UseCompressedClassPointers && !UseCompactObjectHeaders) {
+ } else if (!UseCompactObjectHeaders) {
store_klass_gap(obj, zr);
}
}
diff --git a/src/hotspot/cpu/aarch64/c1_globals_aarch64.hpp b/src/hotspot/cpu/aarch64/c1_globals_aarch64.hpp
index 938a64dd399..bb6b3ce907e 100644
--- a/src/hotspot/cpu/aarch64/c1_globals_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/c1_globals_aarch64.hpp
@@ -42,7 +42,6 @@ define_pd_global(bool, TieredCompilation, false);
define_pd_global(intx, CompileThreshold, 1500 );
define_pd_global(intx, OnStackReplacePercentage, 933 );
-define_pd_global(intx, NewSizeThreadIncrease, 4*K );
define_pd_global(size_t, InitialCodeCacheSize, 160*K);
define_pd_global(size_t, ReservedCodeCacheSize, 32*M );
define_pd_global(size_t, NonProfiledCodeHeapSize, 13*M );
@@ -52,7 +51,6 @@ define_pd_global(bool, ProfileInterpreter, false);
define_pd_global(size_t, CodeCacheExpansionSize, 32*K );
define_pd_global(size_t, CodeCacheMinBlockLength, 1);
define_pd_global(size_t, CodeCacheMinimumUseSpace, 400*K);
-define_pd_global(bool, NeverActAsServerClassMachine, true );
define_pd_global(bool, CICompileOSR, true );
#endif // !COMPILER2
define_pd_global(bool, UseTypeProfile, false);
diff --git a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
index 958855c7685..7aab7d389e1 100644
--- a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
@@ -30,7 +30,9 @@
#include "opto/matcher.hpp"
#include "opto/output.hpp"
#include "opto/subnode.hpp"
+#include "runtime/objectMonitorTable.hpp"
#include "runtime/stubRoutines.hpp"
+#include "runtime/synchronizer.hpp"
#include "utilities/globalDefinitions.hpp"
#include "utilities/powerOfTwo.hpp"
@@ -221,37 +223,52 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box, Register t1,
if (!UseObjectMonitorTable) {
assert(t1_monitor == t1_mark, "should be the same here");
} else {
+ const Register t1_hash = t1;
Label monitor_found;
- // Load cache address
- lea(t3_t, Address(rthread, JavaThread::om_cache_oops_offset()));
+ // Save the mark, we might need it to extract the hash.
+ mov(t3, t1_mark);
- const int num_unrolled = 2;
+ // Look for the monitor in the om_cache.
+
+ ByteSize cache_offset = JavaThread::om_cache_oops_offset();
+ ByteSize monitor_offset = OMCache::oop_to_monitor_difference();
+ const int num_unrolled = OMCache::CAPACITY;
for (int i = 0; i < num_unrolled; i++) {
- ldr(t1, Address(t3_t));
- cmp(obj, t1);
+ ldr(t1_monitor, Address(rthread, cache_offset + monitor_offset));
+ ldr(t2, Address(rthread, cache_offset));
+ cmp(obj, t2);
br(Assembler::EQ, monitor_found);
- increment(t3_t, in_bytes(OMCache::oop_to_oop_difference()));
+ cache_offset = cache_offset + OMCache::oop_to_oop_difference();
}
- Label loop;
+ // Look for the monitor in the table.
- // Search for obj in cache.
- bind(loop);
+ // Get the hash code.
+ ubfx(t1_hash, t3, markWord::hash_shift, markWord::hash_bits);
- // Check for match.
- ldr(t1, Address(t3_t));
- cmp(obj, t1);
- br(Assembler::EQ, monitor_found);
+ // Get the table and calculate the bucket's address
+ lea(t3, ExternalAddress(ObjectMonitorTable::current_table_address()));
+ ldr(t3, Address(t3));
+ ldr(t2, Address(t3, ObjectMonitorTable::table_capacity_mask_offset()));
+ ands(t1_hash, t1_hash, t2);
+ ldr(t3, Address(t3, ObjectMonitorTable::table_buckets_offset()));
- // Search until null encountered, guaranteed _null_sentinel at end.
- increment(t3_t, in_bytes(OMCache::oop_to_oop_difference()));
- cbnz(t1, loop);
- // Cache Miss, NE set from cmp above, cbnz does not set flags
- b(slow_path);
+ // Read the monitor from the bucket.
+ ldr(t1_monitor, Address(t3, t1_hash, Address::lsl(LogBytesPerWord)));
+
+ // Check if the monitor in the bucket is special (empty, tombstone or removed).
+ cmp(t1_monitor, (unsigned char)ObjectMonitorTable::SpecialPointerValues::below_is_special);
+ br(Assembler::LO, slow_path);
+
+ // Check if object matches.
+ ldr(t3, Address(t1_monitor, ObjectMonitor::object_offset()));
+ BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
+ bs_asm->try_resolve_weak_handle_in_c2(this, t3, t2, slow_path);
+ cmp(t3, obj);
+ br(Assembler::NE, slow_path);
bind(monitor_found);
- ldr(t1_monitor, Address(t3_t, OMCache::oop_to_monitor_difference()));
}
const Register t2_owner_addr = t2;
@@ -2858,3 +2875,24 @@ void C2_MacroAssembler::vector_expand_sve(FloatRegister dst, FloatRegister src,
// dst = 00 87 00 65 00 43 00 21
sve_tbl(dst, size, src, dst);
}
+
+// Optimized SVE cpy (imm, zeroing) instruction.
+//
+// `movi; cpy(imm, merging)` and `cpy(imm, zeroing)` have the same
+// functionality, but test results show that `movi; cpy(imm, merging)` has
+// higher throughput on some microarchitectures. This would depend on
+// microarchitecture and so may vary between implementations.
+void C2_MacroAssembler::sve_cpy(FloatRegister dst, SIMD_RegVariant T,
+ PRegister pg, int imm8, bool isMerge) {
+ if (VM_Version::prefer_sve_merging_mode_cpy() && !isMerge) {
+ // Generates a NEON instruction `movi V.2d, #0`.
+ // On AArch64, Z and V registers alias in the low 128 bits, so V is
+ // the low 128 bits of Z. A write to V also clears all bits of
+ // Z above 128, so this `movi` instruction effectively zeroes the
+ // entire Z register. According to the Arm Software Optimization
+ // Guide, `movi` is zero latency.
+ movi(dst, T2D, 0);
+ isMerge = true;
+ }
+ Assembler::sve_cpy(dst, T, pg, imm8, isMerge);
+}
diff --git a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp
index 4f3a41da402..5c05832afbe 100644
--- a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp
@@ -75,6 +75,8 @@
unsigned vector_length_in_bytes);
public:
+ using Assembler::sve_cpy;
+
// jdk.internal.util.ArraysSupport.vectorizedHashCode
address arrays_hashcode(Register ary, Register cnt, Register result, FloatRegister vdata0,
FloatRegister vdata1, FloatRegister vdata2, FloatRegister vdata3,
@@ -244,4 +246,7 @@
void vector_expand_sve(FloatRegister dst, FloatRegister src, PRegister pg,
FloatRegister tmp1, FloatRegister tmp2, BasicType bt,
int vector_length_in_bytes);
+
+ void sve_cpy(FloatRegister dst, SIMD_RegVariant T, PRegister pg, int imm8,
+ bool isMerge);
#endif // CPU_AARCH64_C2_MACROASSEMBLER_AARCH64_HPP
diff --git a/src/hotspot/cpu/aarch64/c2_globals_aarch64.hpp b/src/hotspot/cpu/aarch64/c2_globals_aarch64.hpp
index a0dea3643a1..192461d1a61 100644
--- a/src/hotspot/cpu/aarch64/c2_globals_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/c2_globals_aarch64.hpp
@@ -47,7 +47,6 @@ define_pd_global(intx, ConditionalMoveLimit, 3);
define_pd_global(intx, FreqInlineSize, 325);
define_pd_global(intx, MinJumpTableSize, 10);
define_pd_global(intx, InteriorEntryAlignment, 16);
-define_pd_global(intx, NewSizeThreadIncrease, ScaleForWordSize(4*K));
define_pd_global(intx, LoopUnrollLimit, 60);
define_pd_global(intx, LoopPercentProfileLimit, 10);
// InitialCodeCacheSize derived from specjbb2000 run.
@@ -75,9 +74,6 @@ define_pd_global(size_t, NonNMethodCodeHeapSize, 5*M );
define_pd_global(size_t, CodeCacheMinBlockLength, 6);
define_pd_global(size_t, CodeCacheMinimumUseSpace, 400*K);
-// Ergonomics related flags
-define_pd_global(bool, NeverActAsServerClassMachine, false);
-
define_pd_global(bool, TrapBasedRangeChecks, false); // Not needed.
#endif // CPU_AARCH64_C2_GLOBALS_AARCH64_HPP
diff --git a/src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp b/src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp
index 6fe3315014b..640cd495383 100644
--- a/src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp
@@ -89,16 +89,21 @@ void CompiledDirectCall::set_to_interpreted(const methodHandle& callee, address
NativeMovConstReg* method_holder
= nativeMovConstReg_at(stub + NativeInstruction::instruction_size);
+ // In AOT "production" run we have mixture of AOTed and normal JITed code.
+ // Static call stub in AOTed nmethod always has far jump.
+ // Normal JITed nmethod may have short or far jump depending on distance.
+ // Determine actual jump instruction we have in code.
+ address next_instr = method_holder->next_instruction_address();
+ bool is_general_jump = nativeInstruction_at(next_instr)->is_general_jump();
+
#ifdef ASSERT
- NativeJump* jump = MacroAssembler::codestub_branch_needs_far_jump()
- ? nativeGeneralJump_at(method_holder->next_instruction_address())
- : nativeJump_at(method_holder->next_instruction_address());
+ NativeJump* jump = is_general_jump ? nativeGeneralJump_at(next_instr) : nativeJump_at(next_instr);
verify_mt_safe(callee, entry, method_holder, jump);
#endif
// Update stub.
method_holder->set_data((intptr_t)callee());
- MacroAssembler::pd_patch_instruction(method_holder->next_instruction_address(), entry);
+ MacroAssembler::pd_patch_instruction(next_instr, entry);
ICache::invalidate_range(stub, to_interp_stub_size());
// Update jump to call.
set_destination_mt_safe(stub);
diff --git a/src/hotspot/cpu/aarch64/downcallLinker_aarch64.cpp b/src/hotspot/cpu/aarch64/downcallLinker_aarch64.cpp
index 65d448f908c..130d2949800 100644
--- a/src/hotspot/cpu/aarch64/downcallLinker_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/downcallLinker_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2019, Arm Limited. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -146,10 +146,10 @@ void DowncallLinker::StubGenerator::generate() {
bool should_save_return_value = !_needs_return_buffer;
RegSpiller out_reg_spiller(_output_registers);
- int spill_offset = -1;
+ int out_spill_offset = -1;
if (should_save_return_value) {
- spill_offset = 0;
+ out_spill_offset = 0;
// spill area can be shared with shadow space and out args,
// since they are only used before the call,
// and spill area is only used after.
@@ -174,6 +174,9 @@ void DowncallLinker::StubGenerator::generate() {
// FP-> | |
// |---------------------| = frame_bottom_offset = frame_size
// | (optional) |
+ // | in_reg_spiller area |
+ // |---------------------|
+ // | (optional) |
// | capture state buf |
// |---------------------| = StubLocations::CAPTURED_STATE_BUFFER
// | (optional) |
@@ -187,6 +190,19 @@ void DowncallLinker::StubGenerator::generate() {
GrowableArray out_regs = ForeignGlobals::replace_place_holders(_input_registers, locs);
ArgumentShuffle arg_shuffle(filtered_java_regs, out_regs, shuffle_reg);
+ // Need to spill for state capturing runtime call.
+ // The area spilled into is distinct from the capture state buffer.
+ RegSpiller in_reg_spiller(out_regs);
+ int in_spill_offset = -1;
+ if (_captured_state_mask != 0) {
+ // The spill area cannot be shared with the out_spill since
+ // spilling needs to happen before the call. Allocate a new
+ // region in the stack for this spill space.
+ in_spill_offset = allocated_frame_size;
+ allocated_frame_size += in_reg_spiller.spill_size_bytes();
+ }
+
+
#ifndef PRODUCT
LogTarget(Trace, foreign, downcall) lt;
if (lt.is_enabled()) {
@@ -228,6 +244,20 @@ void DowncallLinker::StubGenerator::generate() {
arg_shuffle.generate(_masm, shuffle_reg, 0, _abi._shadow_space_bytes);
__ block_comment("} argument shuffle");
+ if (_captured_state_mask != 0) {
+ assert(in_spill_offset != -1, "must be");
+ __ block_comment("{ load initial thread local");
+ in_reg_spiller.generate_spill(_masm, in_spill_offset);
+
+ // Copy the contents of the capture state buffer into thread local
+ __ ldr(c_rarg0, Address(sp, locs.data_offset(StubLocations::CAPTURED_STATE_BUFFER)));
+ __ movw(c_rarg1, _captured_state_mask);
+ __ rt_call(CAST_FROM_FN_PTR(address, DowncallLinker::capture_state_pre), tmp1);
+
+ in_reg_spiller.generate_fill(_masm, in_spill_offset);
+ __ block_comment("} load initial thread local");
+ }
+
__ blr(as_Register(locs.get(StubLocations::TARGET_ADDRESS)));
// this call is assumed not to have killed rthread
@@ -254,15 +284,15 @@ void DowncallLinker::StubGenerator::generate() {
__ block_comment("{ save thread local");
if (should_save_return_value) {
- out_reg_spiller.generate_spill(_masm, spill_offset);
+ out_reg_spiller.generate_spill(_masm, out_spill_offset);
}
__ ldr(c_rarg0, Address(sp, locs.data_offset(StubLocations::CAPTURED_STATE_BUFFER)));
__ movw(c_rarg1, _captured_state_mask);
- __ rt_call(CAST_FROM_FN_PTR(address, DowncallLinker::capture_state), tmp1);
+ __ rt_call(CAST_FROM_FN_PTR(address, DowncallLinker::capture_state_post), tmp1);
if (should_save_return_value) {
- out_reg_spiller.generate_fill(_masm, spill_offset);
+ out_reg_spiller.generate_fill(_masm, out_spill_offset);
}
__ block_comment("} save thread local");
@@ -321,7 +351,7 @@ void DowncallLinker::StubGenerator::generate() {
if (should_save_return_value) {
// Need to save the native result registers around any runtime calls.
- out_reg_spiller.generate_spill(_masm, spill_offset);
+ out_reg_spiller.generate_spill(_masm, out_spill_offset);
}
__ mov(c_rarg0, rthread);
@@ -330,7 +360,7 @@ void DowncallLinker::StubGenerator::generate() {
__ blr(tmp1);
if (should_save_return_value) {
- out_reg_spiller.generate_fill(_masm, spill_offset);
+ out_reg_spiller.generate_fill(_masm, out_spill_offset);
}
__ b(L_after_safepoint_poll);
@@ -342,13 +372,13 @@ void DowncallLinker::StubGenerator::generate() {
__ bind(L_reguard);
if (should_save_return_value) {
- out_reg_spiller.generate_spill(_masm, spill_offset);
+ out_reg_spiller.generate_spill(_masm, out_spill_offset);
}
__ rt_call(CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages), tmp1);
if (should_save_return_value) {
- out_reg_spiller.generate_fill(_masm, spill_offset);
+ out_reg_spiller.generate_fill(_masm, out_spill_offset);
}
__ b(L_after_reguard);
diff --git a/src/hotspot/cpu/aarch64/frame_aarch64.inline.hpp b/src/hotspot/cpu/aarch64/frame_aarch64.inline.hpp
index cb53d8663ad..748ab0e0e2b 100644
--- a/src/hotspot/cpu/aarch64/frame_aarch64.inline.hpp
+++ b/src/hotspot/cpu/aarch64/frame_aarch64.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -245,8 +245,8 @@ inline bool frame::equal(frame other) const {
// Return unique id for this frame. The id must have a value where we can distinguish
// identity and younger/older relationship. null represents an invalid (incomparable)
-// frame.
-inline intptr_t* frame::id(void) const { return unextended_sp(); }
+// frame. Should not be called for heap frames.
+inline intptr_t* frame::id(void) const { return real_fp(); }
// Return true if the frame is older (less recent activation) than the frame represented by id
inline bool frame::is_older(intptr_t* id) const { assert(this->id() != nullptr && id != nullptr, "null frame id");
@@ -412,6 +412,9 @@ inline frame frame::sender(RegisterMap* map) const {
StackWatermarkSet::on_iteration(map->thread(), result);
}
+ // Calling frame::id() is currently not supported for heap frames.
+ assert(result._on_heap || this->_on_heap || result.is_older(this->id()), "Must be");
+
return result;
}
diff --git a/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp
index d7884c27a2c..68291720208 100644
--- a/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp
@@ -23,6 +23,7 @@
*/
#include "asm/macroAssembler.inline.hpp"
+#include "code/aotCodeCache.hpp"
#include "gc/g1/g1BarrierSet.hpp"
#include "gc/g1/g1BarrierSetAssembler.hpp"
#include "gc/g1/g1BarrierSetRuntime.hpp"
@@ -243,9 +244,25 @@ static void generate_post_barrier(MacroAssembler* masm,
assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, noreg, rscratch1);
// Does store cross heap regions?
- __ eor(tmp1, store_addr, new_val); // tmp1 := store address ^ new value
- __ lsr(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes); // tmp1 := ((store address ^ new value) >> LogOfHRGrainBytes)
- __ cbz(tmp1, done);
+ #if INCLUDE_CDS
+ // AOT code needs to load the barrier grain shift from the aot
+ // runtime constants area in the code cache otherwise we can compile
+ // it as an immediate operand
+ if (AOTCodeCache::is_on_for_dump()) {
+ address grain_shift_address = (address)AOTRuntimeConstants::grain_shift_address();
+ __ eor(tmp1, store_addr, new_val);
+ __ lea(tmp2, ExternalAddress(grain_shift_address));
+ __ ldrb(tmp2, tmp2);
+ __ lsrv(tmp1, tmp1, tmp2);
+ __ cbz(tmp1, done);
+ } else
+#endif
+ {
+ __ eor(tmp1, store_addr, new_val); // tmp1 := store address ^ new value
+ __ lsr(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes); // tmp1 := ((store address ^ new value) >> LogOfHRGrainBytes)
+ __ cbz(tmp1, done);
+ }
+
// Crosses regions, storing null?
if (new_val_may_be_null) {
__ cbz(new_val, done);
diff --git a/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.cpp
index 021af3e5698..2a78d688097 100644
--- a/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -441,6 +441,11 @@ OptoReg::Name BarrierSetAssembler::refine_register(const Node* node, OptoReg::Na
return opto_reg;
}
+void BarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path) {
+ // Load the oop from the weak handle.
+ __ ldr(obj, Address(obj));
+}
+
#undef __
#define __ _masm->
diff --git a/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.hpp
index e69be999f00..c2581b2f962 100644
--- a/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -135,6 +135,7 @@ public:
OptoReg::Name opto_reg);
OptoReg::Name refine_register(const Node* node,
OptoReg::Name opto_reg);
+ virtual void try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path);
#endif // COMPILER2
};
diff --git a/src/hotspot/cpu/aarch64/gc/shared/cardTableBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shared/cardTableBarrierSetAssembler_aarch64.cpp
index 0bfc320179d..a9c320912cb 100644
--- a/src/hotspot/cpu/aarch64/gc/shared/cardTableBarrierSetAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/gc/shared/cardTableBarrierSetAssembler_aarch64.cpp
@@ -56,8 +56,10 @@ void CardTableBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet d
}
}
-void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj, Address dst) {
-
+void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj, Address dst, Register tmp1, Register tmp2) {
+ precond(tmp1 != noreg);
+ precond(tmp2 != noreg);
+ assert_different_registers(obj, tmp1, tmp2);
BarrierSet* bs = BarrierSet::barrier_set();
assert(bs->kind() == BarrierSet::CardTableBarrierSet, "Wrong barrier set kind");
@@ -65,16 +67,16 @@ void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register ob
assert(CardTable::dirty_card_val() == 0, "must be");
- __ load_byte_map_base(rscratch1);
+ __ load_byte_map_base(tmp1);
if (UseCondCardMark) {
Label L_already_dirty;
- __ ldrb(rscratch2, Address(obj, rscratch1));
- __ cbz(rscratch2, L_already_dirty);
- __ strb(zr, Address(obj, rscratch1));
+ __ ldrb(tmp2, Address(obj, tmp1));
+ __ cbz(tmp2, L_already_dirty);
+ __ strb(zr, Address(obj, tmp1));
__ bind(L_already_dirty);
} else {
- __ strb(zr, Address(obj, rscratch1));
+ __ strb(zr, Address(obj, tmp1));
}
}
@@ -112,10 +114,10 @@ void CardTableBarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorS
if (needs_post_barrier) {
// flatten object address if needed
if (!precise || (dst.index() == noreg && dst.offset() == 0)) {
- store_check(masm, dst.base(), dst);
+ store_check(masm, dst.base(), dst, tmp1, tmp2);
} else {
__ lea(tmp3, dst);
- store_check(masm, tmp3, dst);
+ store_check(masm, tmp3, dst, tmp1, tmp2);
}
}
}
diff --git a/src/hotspot/cpu/aarch64/gc/shared/cardTableBarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/shared/cardTableBarrierSetAssembler_aarch64.hpp
index 07dd8eb5565..e05e9e90e5d 100644
--- a/src/hotspot/cpu/aarch64/gc/shared/cardTableBarrierSetAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/gc/shared/cardTableBarrierSetAssembler_aarch64.hpp
@@ -46,7 +46,7 @@ protected:
virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Address dst, Register val, Register tmp1, Register tmp2, Register tmp3);
- void store_check(MacroAssembler* masm, Register obj, Address dst);
+ void store_check(MacroAssembler* masm, Register obj, Address dst, Register tmp1, Register tmp2);
};
#endif // CPU_AARCH64_GC_SHARED_CARDTABLEBARRIERSETASSEMBLER_AARCH64_HPP
diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp
index ad7bac4e067..2f7707227b4 100644
--- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp
@@ -1,4 +1,5 @@
/*
+ * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, 2022, Red Hat, Inc. All rights reserved.
* Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -442,6 +443,30 @@ void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler
__ bind(done);
}
+#ifdef COMPILER2
+void ShenandoahBarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj,
+ Register tmp, Label& slow_path) {
+ assert_different_registers(obj, tmp);
+
+ Label done;
+
+ // Resolve weak handle using the standard implementation.
+ BarrierSetAssembler::try_resolve_weak_handle_in_c2(masm, obj, tmp, slow_path);
+
+ // Check if the reference is null, and if it is, take the fast path.
+ __ cbz(obj, done);
+
+ Address gc_state(rthread, ShenandoahThreadLocalData::gc_state_offset());
+ __ lea(tmp, gc_state);
+ __ ldrb(tmp, __ legitimize_address(gc_state, 1, tmp));
+
+ // Check if the heap is under weak-reference/roots processing, in
+ // which case we need to take the slow path.
+ __ tbnz(tmp, ShenandoahHeap::WEAK_ROOTS_BITPOS, slow_path);
+ __ bind(done);
+}
+#endif
+
// Special Shenandoah CAS implementation that handles false negatives due
// to concurrent evacuation. The service is more complex than a
// traditional CAS operation because the CAS operation is intended to
diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp
index 362fcae1ccd..d5d5ce8950e 100644
--- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp
@@ -1,4 +1,5 @@
/*
+ * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved.
* Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -79,6 +80,9 @@ public:
Address dst, Register val, Register tmp1, Register tmp2, Register tmp3);
virtual void try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
Register obj, Register tmp, Label& slowpath);
+#ifdef COMPILER2
+ virtual void try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path);
+#endif
void cmpxchg_oop(MacroAssembler* masm, Register addr, Register expected, Register new_val,
bool acquire, bool release, bool is_cae, Register result);
};
diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp
index 07a2d6fbfa0..f0885fee93d 100644
--- a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -879,7 +879,9 @@ void ZBarrierSetAssembler::patch_barrier_relocation(address addr, int format) {
ShouldNotReachHere();
}
- ICache::invalidate_word((address)patch_addr);
+ if (!UseSingleICacheInvalidation) {
+ ICache::invalidate_word((address)patch_addr);
+ }
}
#ifdef COMPILER1
@@ -1326,6 +1328,23 @@ void ZStoreBarrierStubC2Aarch64::emit_code(MacroAssembler& masm) {
register_stub(this);
}
+#undef __
+#define __ masm->
+
+void ZBarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path) {
+ // Resolve weak handle using the standard implementation.
+ BarrierSetAssembler::try_resolve_weak_handle_in_c2(masm, obj, tmp, slow_path);
+
+ // Check if the oop is bad, in which case we need to take the slow path.
+ __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatMarkBadBeforeMov);
+ __ movzw(tmp, barrier_Relocation::unpatched);
+ __ tst(obj, tmp);
+ __ br(Assembler::NE, slow_path);
+
+ // Oop is okay, so we uncolor it.
+ __ lsr(obj, obj, ZPointerLoadShift);
+}
+
#undef __
#endif // COMPILER2
diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp
index 487970ab0c5..fbbc5c1b517 100644
--- a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -191,6 +191,7 @@ public:
ZLoadBarrierStubC2* stub) const;
void generate_c2_store_barrier_stub(MacroAssembler* masm,
ZStoreBarrierStubC2* stub) const;
+ void try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path);
#endif // COMPILER2
void check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error);
diff --git a/src/hotspot/cpu/aarch64/globals_aarch64.hpp b/src/hotspot/cpu/aarch64/globals_aarch64.hpp
index a59e83c4b69..dfeba73bede 100644
--- a/src/hotspot/cpu/aarch64/globals_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/globals_aarch64.hpp
@@ -95,7 +95,7 @@ define_pd_global(intx, InlineSmallCode, 1000);
"Use simplest and shortest implementation for array equals") \
product(bool, UseSIMDForBigIntegerShiftIntrinsics, true, \
"Use SIMD instructions for left/right shift of BigInteger") \
- product(bool, UseSIMDForSHA3Intrinsic, true, \
+ product(bool, UseSIMDForSHA3Intrinsic, false, \
"Use SIMD SHA3 instructions for SHA3 intrinsic") \
product(bool, AvoidUnalignedAccesses, false, \
"Avoid generating unaligned memory accesses") \
@@ -115,18 +115,26 @@ define_pd_global(intx, InlineSmallCode, 1000);
"Value -1 means off.") \
range(-1, 4096) \
product(ccstr, OnSpinWaitInst, "yield", DIAGNOSTIC, \
- "The instruction to use to implement " \
- "java.lang.Thread.onSpinWait()." \
- "Valid values are: none, nop, isb, yield, sb.") \
+ "The instruction to use for java.lang.Thread.onSpinWait(). " \
+ "Valid values are: none, nop, isb, yield, sb, wfet.") \
constraint(OnSpinWaitInstNameConstraintFunc, AtParse) \
product(uint, OnSpinWaitInstCount, 1, DIAGNOSTIC, \
- "The number of OnSpinWaitInst instructions to generate." \
- "It cannot be used with OnSpinWaitInst=none.") \
+ "The number of OnSpinWaitInst instructions to generate. " \
+ "It cannot be used with OnSpinWaitInst=none. " \
+ "For OnSpinWaitInst=wfet it must be 1.") \
range(1, 99) \
+ product(uint, OnSpinWaitDelay, 40, DIAGNOSTIC, \
+ "The minimum delay (in nanoseconds) of the OnSpinWait loop. " \
+ "It can only be used with -XX:OnSpinWaitInst=wfet.") \
+ range(1, 1000) \
product(ccstr, UseBranchProtection, "none", \
"Branch Protection to use: none, standard, pac-ret") \
product(bool, AlwaysMergeDMB, true, DIAGNOSTIC, \
"Always merge DMB instructions in code emission") \
+ product(bool, NeoverseN1ICacheErratumMitigation, false, DIAGNOSTIC, \
+ "Enable workaround for Neoverse N1 erratum 1542419") \
+ product(bool, UseSingleICacheInvalidation, false, DIAGNOSTIC, \
+ "Defer multiple ICache invalidation to single invalidation") \
// end of ARCH_FLAGS
diff --git a/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp b/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp
index 2b506b241e0..980fedb406d 100644
--- a/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -989,26 +989,15 @@ void InterpreterMacroAssembler::profile_final_call(Register mdp) {
void InterpreterMacroAssembler::profile_virtual_call(Register receiver,
- Register mdp,
- bool receiver_can_be_null) {
+ Register mdp) {
if (ProfileInterpreter) {
Label profile_continue;
// If no method data exists, go to profile_continue.
test_method_data_pointer(mdp, profile_continue);
- Label skip_receiver_profile;
- if (receiver_can_be_null) {
- Label not_null;
- // We are making a call. Increment the count for null receiver.
- increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
- b(skip_receiver_profile);
- bind(not_null);
- }
-
// Record the receiver type.
profile_receiver_type(receiver, mdp, 0);
- bind(skip_receiver_profile);
// The method data pointer needs to be updated to reflect the new target.
update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size()));
diff --git a/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp b/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp
index 74d4430000d..9a074f1ce69 100644
--- a/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2015, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -285,8 +285,7 @@ class InterpreterMacroAssembler: public MacroAssembler {
void profile_not_taken_branch(Register mdp);
void profile_call(Register mdp);
void profile_final_call(Register mdp);
- void profile_virtual_call(Register receiver, Register mdp,
- bool receiver_can_be_null = false);
+ void profile_virtual_call(Register receiver, Register mdp);
void profile_ret(Register return_bci, Register mdp);
void profile_null_seen(Register mdp);
void profile_typecheck(Register mdp, Register klass);
diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
index 409343b6b8d..b5e15402941 100644
--- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
@@ -406,7 +406,6 @@ public:
offset <<= shift;
uint64_t target_page = ((uint64_t)insn_addr) + offset;
target_page &= ((uint64_t)-1) << shift;
- uint32_t insn2 = insn_at(insn_addr, 1);
target = address(target_page);
precond(inner != nullptr);
inner(insn_addr, target);
@@ -520,13 +519,6 @@ int MacroAssembler::patch_narrow_klass(address insn_addr, narrowKlass n) {
return 2 * NativeInstruction::instruction_size;
}
-address MacroAssembler::target_addr_for_insn_or_null(address insn_addr) {
- if (NativeInstruction::is_ldrw_to_zr(insn_addr)) {
- return nullptr;
- }
- return MacroAssembler::target_addr_for_insn(insn_addr);
-}
-
void MacroAssembler::safepoint_poll(Label& slow_path, bool at_return, bool in_nmethod, Register tmp) {
ldr(tmp, Address(rthread, JavaThread::polling_word_offset()));
if (at_return) {
@@ -770,7 +762,7 @@ void MacroAssembler::call_VM_base(Register oop_result,
assert(java_thread == rthread, "unexpected register");
#ifdef ASSERT
// TraceBytecodes does not use r12 but saves it over the call, so don't verify
- // if ((UseCompressedOops || UseCompressedClassPointers) && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");
+ // if (!TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");
#endif // ASSERT
assert(java_thread != oop_result , "cannot use the same register for java_thread & oop_result");
@@ -960,7 +952,10 @@ void MacroAssembler::emit_static_call_stub() {
}
int MacroAssembler::static_call_stub_size() {
- if (!codestub_branch_needs_far_jump()) {
+ // During AOT production run AOT and JIT compiled code
+ // are used at the same time. We need this size
+ // to be the same for both types of code.
+ if (!codestub_branch_needs_far_jump() && !AOTCodeCache::is_on_for_use()) {
// isb; movk; movz; movz; b
return 5 * NativeInstruction::instruction_size;
}
@@ -1010,14 +1005,10 @@ int MacroAssembler::ic_check(int end_alignment) {
load_narrow_klass_compact(tmp1, receiver);
ldrw(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
cmpw(tmp1, tmp2);
- } else if (UseCompressedClassPointers) {
+ } else {
ldrw(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes()));
ldrw(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
cmpw(tmp1, tmp2);
- } else {
- ldr(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes()));
- ldr(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
- cmp(tmp1, tmp2);
}
Label dont;
@@ -1960,9 +1951,7 @@ void MacroAssembler::verify_secondary_supers_table(Register r_sub_klass,
const Register
r_array_base = temp1,
- r_array_length = temp2,
- r_array_index = noreg, // unused
- r_bitmap = noreg; // unused
+ r_array_length = temp2;
BLOCK_COMMENT("verify_secondary_supers_table {");
@@ -3288,7 +3277,6 @@ int MacroAssembler::pop_p(unsigned int bitset, Register stack) {
#ifdef ASSERT
void MacroAssembler::verify_heapbase(const char* msg) {
#if 0
- assert (UseCompressedOops || UseCompressedClassPointers, "should be compressed");
assert (Universe::heap() != nullptr, "java heap should be initialized");
if (!UseCompressedOops || Universe::ptr_base() == nullptr) {
// rheapbase is allocated as general register
@@ -3611,9 +3599,8 @@ extern "C" void findpc(intptr_t x);
void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[])
{
// In order to get locks to work, we need to fake a in_VM state
- if (ShowMessageBoxOnError ) {
+ if (ShowMessageBoxOnError) {
JavaThread* thread = JavaThread::current();
- JavaThreadState saved_state = thread->thread_state();
thread->set_thread_state(_thread_in_vm);
#ifndef PRODUCT
if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
@@ -5078,13 +5065,10 @@ void MacroAssembler::load_narrow_klass_compact(Register dst, Register src) {
void MacroAssembler::load_klass(Register dst, Register src) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(dst, src);
- decode_klass_not_null(dst);
- } else if (UseCompressedClassPointers) {
- ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
- decode_klass_not_null(dst);
} else {
- ldr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
+ ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
}
+ decode_klass_not_null(dst);
}
void MacroAssembler::restore_cpu_control_state_after_jni(Register tmp1, Register tmp2) {
@@ -5136,25 +5120,21 @@ void MacroAssembler::load_mirror(Register dst, Register method, Register tmp1, R
void MacroAssembler::cmp_klass(Register obj, Register klass, Register tmp) {
assert_different_registers(obj, klass, tmp);
- if (UseCompressedClassPointers) {
- if (UseCompactObjectHeaders) {
- load_narrow_klass_compact(tmp, obj);
- } else {
- ldrw(tmp, Address(obj, oopDesc::klass_offset_in_bytes()));
- }
- if (CompressedKlassPointers::base() == nullptr) {
- cmp(klass, tmp, LSL, CompressedKlassPointers::shift());
- return;
- } else if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0
- && CompressedKlassPointers::shift() == 0) {
- // Only the bottom 32 bits matter
- cmpw(klass, tmp);
- return;
- }
- decode_klass_not_null(tmp);
+ if (UseCompactObjectHeaders) {
+ load_narrow_klass_compact(tmp, obj);
} else {
- ldr(tmp, Address(obj, oopDesc::klass_offset_in_bytes()));
+ ldrw(tmp, Address(obj, oopDesc::klass_offset_in_bytes()));
}
+ if (CompressedKlassPointers::base() == nullptr) {
+ cmp(klass, tmp, LSL, CompressedKlassPointers::shift());
+ return;
+ } else if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0
+ && CompressedKlassPointers::shift() == 0) {
+ // Only the bottom 32 bits matter
+ cmpw(klass, tmp);
+ return;
+ }
+ decode_klass_not_null(tmp);
cmp(klass, tmp);
}
@@ -5162,36 +5142,25 @@ void MacroAssembler::cmp_klasses_from_objects(Register obj1, Register obj2, Regi
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(tmp1, obj1);
load_narrow_klass_compact(tmp2, obj2);
- cmpw(tmp1, tmp2);
- } else if (UseCompressedClassPointers) {
+ } else {
ldrw(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
ldrw(tmp2, Address(obj2, oopDesc::klass_offset_in_bytes()));
- cmpw(tmp1, tmp2);
- } else {
- ldr(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
- ldr(tmp2, Address(obj2, oopDesc::klass_offset_in_bytes()));
- cmp(tmp1, tmp2);
}
+ cmpw(tmp1, tmp2);
}
void MacroAssembler::store_klass(Register dst, Register src) {
// FIXME: Should this be a store release? concurrent gcs assumes
// klass length is valid if klass field is not null.
assert(!UseCompactObjectHeaders, "not with compact headers");
- if (UseCompressedClassPointers) {
- encode_klass_not_null(src);
- strw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
- } else {
- str(src, Address(dst, oopDesc::klass_offset_in_bytes()));
- }
+ encode_klass_not_null(src);
+ strw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
}
void MacroAssembler::store_klass_gap(Register dst, Register src) {
assert(!UseCompactObjectHeaders, "not with compact headers");
- if (UseCompressedClassPointers) {
- // Store to klass gap in destination
- strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
- }
+ // Store to klass gap in destination
+ strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
}
// Algorithm must match CompressedOops::encode.
@@ -5337,8 +5306,6 @@ MacroAssembler::KlassDecodeMode MacroAssembler::klass_decode_mode() {
}
MacroAssembler::KlassDecodeMode MacroAssembler::klass_decode_mode(address base, int shift, const size_t range) {
- assert(UseCompressedClassPointers, "not using compressed class pointers");
-
// KlassDecodeMode shouldn't be set already.
assert(_klass_decode_mode == KlassDecodeNone, "set once");
@@ -5468,8 +5435,6 @@ void MacroAssembler::decode_klass_not_null_for_aot(Register dst, Register src) {
}
void MacroAssembler::decode_klass_not_null(Register dst, Register src) {
- assert (UseCompressedClassPointers, "should only be used for compressed headers");
-
if (AOTCodeCache::is_on_for_dump()) {
decode_klass_not_null_for_aot(dst, src);
return;
@@ -5536,7 +5501,6 @@ void MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
}
void MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
- assert (UseCompressedClassPointers, "should only be used for compressed headers");
assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
int index = oop_recorder()->find_index(k);
@@ -5738,7 +5702,6 @@ address MacroAssembler::read_polling_page(Register r, relocInfo::relocType rtype
}
void MacroAssembler::adrp(Register reg1, const Address &dest, uint64_t &byte_offset) {
- relocInfo::relocType rtype = dest.rspec().reloc()->type();
uint64_t low_page = (uint64_t)CodeCache::low_bound() >> 12;
uint64_t high_page = (uint64_t)(CodeCache::high_bound()-1) >> 12;
uint64_t dest_page = (uint64_t)dest.target() >> 12;
@@ -5766,6 +5729,14 @@ void MacroAssembler::adrp(Register reg1, const Address &dest, uint64_t &byte_off
}
void MacroAssembler::load_byte_map_base(Register reg) {
+#if INCLUDE_CDS
+ if (AOTCodeCache::is_on_for_dump()) {
+ address byte_map_base_adr = AOTRuntimeConstants::card_table_base_address();
+ lea(reg, ExternalAddress(byte_map_base_adr));
+ ldr(reg, Address(reg));
+ return;
+ }
+#endif
CardTableBarrierSet* ctbs = CardTableBarrierSet::barrier_set();
// Strictly speaking the card table base isn't an address at all, and it might
@@ -5773,6 +5744,20 @@ void MacroAssembler::load_byte_map_base(Register reg) {
mov(reg, (uint64_t)ctbs->card_table_base_const());
}
+void MacroAssembler::load_aotrc_address(Register reg, address a) {
+#if INCLUDE_CDS
+ assert(AOTRuntimeConstants::contains(a), "address out of range for data area");
+ if (AOTCodeCache::is_on_for_dump()) {
+ // all aotrc field addresses should be registered in the AOTCodeCache address table
+ lea(reg, ExternalAddress(a));
+ } else {
+ mov(reg, (uint64_t)a);
+ }
+#else
+ ShouldNotReachHere();
+#endif
+}
+
void MacroAssembler::build_frame(int framesize) {
assert(framesize >= 2 * wordSize, "framesize must include space for FP/LR");
assert(framesize % (2*wordSize) == 0, "must preserve 2*wordSize alignment");
@@ -6114,7 +6099,6 @@ void MacroAssembler::string_equals(Register a1, Register a2,
Label SAME, DONE, SHORT, NEXT_WORD;
Register tmp1 = rscratch1;
Register tmp2 = rscratch2;
- Register cnt2 = tmp2; // cnt2 only used in array length compare
assert_different_registers(a1, a2, result, cnt1, rscratch1, rscratch2);
@@ -6826,6 +6810,9 @@ void MacroAssembler::spin_wait() {
assert(VM_Version::supports_sb(), "current CPU does not support SB instruction");
sb();
break;
+ case SpinWait::WFET:
+ spin_wait_wfet(VM_Version::spin_wait_desc().delay());
+ break;
default:
ShouldNotReachHere();
}
@@ -6833,6 +6820,28 @@ void MacroAssembler::spin_wait() {
block_comment("}");
}
+void MacroAssembler::spin_wait_wfet(int delay_ns) {
+ // The sequence assumes CNTFRQ_EL0 is fixed to 1GHz. The assumption is valid
+ // starting from Armv8.6, according to the "D12.1.2 The system counter" of the
+ // Arm Architecture Reference Manual for A-profile architecture version M.a.a.
+ // This is sufficient because FEAT_WFXT is introduced from Armv8.6.
+ Register target = rscratch1;
+ Register current = rscratch2;
+ get_cntvctss_el0(current);
+ add(target, current, delay_ns);
+
+ Label L_wait_loop;
+ bind(L_wait_loop);
+
+ wfet(target);
+ get_cntvctss_el0(current);
+
+ cmp(current, target);
+ br(LT, L_wait_loop);
+
+ sb();
+}
+
// Stack frame creation/removal
void MacroAssembler::enter(bool strip_ret_addr) {
diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
index 7eb6cea0614..994fbe3c80f 100644
--- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2024, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -660,6 +660,14 @@ public:
msr(0b011, 0b0100, 0b0010, 0b000, reg);
}
+ // CNTVCTSS_EL0: op1 == 011
+ // CRn == 1110
+ // CRm == 0000
+ // op2 == 110
+ inline void get_cntvctss_el0(Register reg) {
+ mrs(0b011, 0b1110, 0b0000, 0b110, reg);
+ }
+
// idiv variant which deals with MINLONG as dividend and -1 as divisor
int corrected_idivl(Register result, Register ra, Register rb,
bool want_remainder, Register tmp = rscratch1);
@@ -678,7 +686,6 @@ public:
static bool uses_implicit_null_check(void* address);
static address target_addr_for_insn(address insn_addr);
- static address target_addr_for_insn_or_null(address insn_addr);
// Required platform-specific helpers for Label::patch_instructions.
// They _shadow_ the declarations in AbstractAssembler, which are undefined.
@@ -1477,6 +1484,9 @@ public:
// Load the base of the cardtable byte map into reg.
void load_byte_map_base(Register reg);
+ // Load a constant address in the AOT Runtime Constants area
+ void load_aotrc_address(Register reg, address a);
+
// Prolog generator routines to support switch between x86 code and
// generated ARM code
@@ -1722,6 +1732,7 @@ public:
// Code for java.lang.Thread::onSpinWait() intrinsic.
void spin_wait();
+ void spin_wait_wfet(int delay_ns);
void fast_lock(Register basic_lock, Register obj, Register t1, Register t2, Register t3, Label& slow);
void fast_unlock(Register obj, Register t1, Register t2, Register t3, Label& slow);
diff --git a/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp b/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp
index 0cdf36f0bc5..8b76b96d345 100644
--- a/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -192,7 +192,6 @@ int NativeMovRegMem::offset() const {
void NativeMovRegMem::set_offset(int x) {
address pc = instruction_address();
- unsigned insn = *(unsigned*)pc;
if (maybe_cpool_ref(pc)) {
address addr = MacroAssembler::target_addr_for_insn(pc);
*(int64_t*)addr = x;
@@ -204,7 +203,7 @@ void NativeMovRegMem::set_offset(int x) {
void NativeMovRegMem::verify() {
#ifdef ASSERT
- address dest = MacroAssembler::target_addr_for_insn_or_null(instruction_address());
+ MacroAssembler::target_addr_for_insn(instruction_address());
#endif
}
@@ -213,7 +212,7 @@ void NativeMovRegMem::verify() {
void NativeJump::verify() { ; }
address NativeJump::jump_destination() const {
- address dest = MacroAssembler::target_addr_for_insn_or_null(instruction_address());
+ address dest = MacroAssembler::target_addr_for_insn(instruction_address());
// We use jump to self as the unresolved address which the inline
// cache code (and relocs) know about
diff --git a/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp b/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp
index 15b6c9ff215..fc7274714ad 100644
--- a/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2025, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -78,7 +78,6 @@ public:
inline bool is_nop() const;
bool is_jump();
bool is_general_jump();
- inline bool is_jump_or_nop();
inline bool is_cond_jump();
bool is_safepoint_poll();
bool is_movz();
@@ -420,10 +419,6 @@ inline bool NativeInstruction::is_jump() {
return false;
}
-inline bool NativeInstruction::is_jump_or_nop() {
- return is_nop() || is_jump();
-}
-
// Call trampoline stubs.
class NativeCallTrampolineStub : public NativeInstruction {
public:
diff --git a/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp b/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp
index dbec2d76d4f..f1b9fb213a2 100644
--- a/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp
@@ -54,7 +54,12 @@ void Relocation::pd_set_data_value(address x, bool verify_only) {
bytes = MacroAssembler::pd_patch_instruction_size(addr(), x);
break;
}
- ICache::invalidate_range(addr(), bytes);
+
+ if (UseSingleICacheInvalidation) {
+ assert(_binding != nullptr, "expect to be called with RelocIterator in use");
+ } else {
+ ICache::invalidate_range(addr(), bytes);
+ }
}
address Relocation::pd_call_destination(address orig_addr) {
diff --git a/src/hotspot/cpu/aarch64/spin_wait_aarch64.cpp b/src/hotspot/cpu/aarch64/spin_wait_aarch64.cpp
index 7da0151d834..97a981ab815 100644
--- a/src/hotspot/cpu/aarch64/spin_wait_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/spin_wait_aarch64.cpp
@@ -32,6 +32,7 @@ bool SpinWait::supports(const char *name) {
strcmp(name, "isb") == 0 ||
strcmp(name, "yield") == 0 ||
strcmp(name, "sb") == 0 ||
+ strcmp(name, "wfet") == 0 ||
strcmp(name, "none") == 0);
}
@@ -46,6 +47,8 @@ SpinWait::Inst SpinWait::from_name(const char* name) {
return SpinWait::YIELD;
} else if (strcmp(name, "sb") == 0) {
return SpinWait::SB;
+ } else if (strcmp(name, "wfet") == 0) {
+ return SpinWait::WFET;
}
return SpinWait::NONE;
diff --git a/src/hotspot/cpu/aarch64/spin_wait_aarch64.hpp b/src/hotspot/cpu/aarch64/spin_wait_aarch64.hpp
index 0e96a4b7157..6ebcd2477a8 100644
--- a/src/hotspot/cpu/aarch64/spin_wait_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/spin_wait_aarch64.hpp
@@ -24,6 +24,8 @@
#ifndef CPU_AARCH64_SPIN_WAIT_AARCH64_HPP
#define CPU_AARCH64_SPIN_WAIT_AARCH64_HPP
+#include "utilities/debug.hpp"
+
class SpinWait {
public:
enum Inst {
@@ -31,21 +33,30 @@ public:
NOP,
ISB,
YIELD,
- SB
+ SB,
+ WFET
};
private:
Inst _inst;
int _count;
+ int _delay;
Inst from_name(const char *name);
public:
- SpinWait(Inst inst = NONE, int count = 0) : _inst(inst), _count(inst == NONE ? 0 : count) {}
- SpinWait(const char *name, int count) : SpinWait(from_name(name), count) {}
+ SpinWait(Inst inst = NONE, int count = 0, int delay = -1)
+ : _inst(inst), _count(inst == NONE ? 0 : count), _delay(delay) {}
+ SpinWait(const char *name, int count, int delay)
+ : SpinWait(from_name(name), count, delay) {}
Inst inst() const { return _inst; }
int inst_count() const { return _count; }
+ int delay() const {
+ assert(_inst == WFET, "Specifying the delay value is only supported for WFET");
+ assert(_delay > 0, "The delay value must be positive");
+ return _delay;
+ }
static bool supports(const char *name);
};
diff --git a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp
index a459a28b09e..21a1124a8ec 100644
--- a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp
@@ -11876,16 +11876,13 @@ class StubGenerator: public StubCodeGenerator {
StubRoutines::_sha512_implCompress = generate_sha512_implCompress(StubId::stubgen_sha512_implCompress_id);
StubRoutines::_sha512_implCompressMB = generate_sha512_implCompress(StubId::stubgen_sha512_implCompressMB_id);
}
- if (UseSHA3Intrinsics) {
-
+ if (UseSHA3Intrinsics && UseSIMDForSHA3Intrinsic) {
StubRoutines::_double_keccak = generate_double_keccak();
- if (UseSIMDForSHA3Intrinsic) {
- StubRoutines::_sha3_implCompress = generate_sha3_implCompress(StubId::stubgen_sha3_implCompress_id);
- StubRoutines::_sha3_implCompressMB = generate_sha3_implCompress(StubId::stubgen_sha3_implCompressMB_id);
- } else {
- StubRoutines::_sha3_implCompress = generate_sha3_implCompress_gpr(StubId::stubgen_sha3_implCompress_id);
- StubRoutines::_sha3_implCompressMB = generate_sha3_implCompress_gpr(StubId::stubgen_sha3_implCompressMB_id);
- }
+ StubRoutines::_sha3_implCompress = generate_sha3_implCompress(StubId::stubgen_sha3_implCompress_id);
+ StubRoutines::_sha3_implCompressMB = generate_sha3_implCompress(StubId::stubgen_sha3_implCompressMB_id);
+ } else if (UseSHA3Intrinsics) {
+ StubRoutines::_sha3_implCompress = generate_sha3_implCompress_gpr(StubId::stubgen_sha3_implCompress_id);
+ StubRoutines::_sha3_implCompressMB = generate_sha3_implCompress_gpr(StubId::stubgen_sha3_implCompressMB_id);
}
if (UsePoly1305Intrinsics) {
diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
index b678921dd97..15af2d5c4e2 100644
--- a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
@@ -1,7 +1,7 @@
/*
* Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2015, 2020, Red Hat Inc. All rights reserved.
- * Copyright 2025 Arm Limited and/or its affiliates.
+ * Copyright 2025, 2026 Arm Limited and/or its affiliates.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -24,6 +24,7 @@
*
*/
+#include "logging/log.hpp"
#include "pauth_aarch64.hpp"
#include "register_aarch64.hpp"
#include "runtime/arguments.hpp"
@@ -52,17 +53,56 @@ uintptr_t VM_Version::_pac_mask;
SpinWait VM_Version::_spin_wait;
+bool VM_Version::_cache_dic_enabled;
+bool VM_Version::_cache_idc_enabled;
+bool VM_Version::_ic_ivau_trapped;
+
const char* VM_Version::_features_names[MAX_CPU_FEATURES] = { nullptr };
static SpinWait get_spin_wait_desc() {
- SpinWait spin_wait(OnSpinWaitInst, OnSpinWaitInstCount);
+ SpinWait spin_wait(OnSpinWaitInst, OnSpinWaitInstCount, OnSpinWaitDelay);
if (spin_wait.inst() == SpinWait::SB && !VM_Version::supports_sb()) {
vm_exit_during_initialization("OnSpinWaitInst is SB but current CPU does not support SB instruction");
}
+ if (spin_wait.inst() == SpinWait::WFET) {
+ if (!VM_Version::supports_wfxt()) {
+ vm_exit_during_initialization("OnSpinWaitInst is WFET but the CPU does not support the WFET instruction");
+ }
+
+ if (!VM_Version::supports_ecv()) {
+ vm_exit_during_initialization("The CPU does not support the FEAT_ECV required by the -XX:OnSpinWaitInst=wfet implementation");
+ }
+
+ if (!VM_Version::supports_sb()) {
+ vm_exit_during_initialization("The CPU does not support the SB instruction required by the -XX:OnSpinWaitInst=wfet implementation");
+ }
+
+ if (OnSpinWaitInstCount != 1) {
+ vm_exit_during_initialization("OnSpinWaitInstCount for OnSpinWaitInst 'wfet' must be 1");
+ }
+ } else {
+ if (!FLAG_IS_DEFAULT(OnSpinWaitDelay)) {
+ vm_exit_during_initialization("OnSpinWaitDelay can only be used with -XX:OnSpinWaitInst=wfet");
+ }
+ }
+
return spin_wait;
}
+static bool has_neoverse_n1_errata_1542419() {
+ const int major_rev_num = VM_Version::cpu_variant();
+ const int minor_rev_num = VM_Version::cpu_revision();
+ // Neoverse N1: 0xd0c
+ // Erratum 1542419 affects r3p0, r3p1 and r4p0.
+ // It is fixed in r4p1 and later revisions, which are not affected.
+ return (VM_Version::cpu_family() == VM_Version::CPU_ARM &&
+ VM_Version::model_is(0xd0c) &&
+ ((major_rev_num == 3 && minor_rev_num == 0) ||
+ (major_rev_num == 3 && minor_rev_num == 1) ||
+ (major_rev_num == 4 && minor_rev_num == 0)));
+}
+
void VM_Version::initialize() {
#define SET_CPU_FEATURE_NAME(id, name, bit) \
_features_names[bit] = XSTR(name);
@@ -74,9 +114,14 @@ void VM_Version::initialize() {
_supports_atomic_getset8 = true;
_supports_atomic_getadd8 = true;
- get_os_cpu_info();
+ _cache_dic_enabled = false;
+ _cache_idc_enabled = false;
+ _ic_ivau_trapped = false;
- int dcache_line = VM_Version::dcache_line_size();
+ get_os_cpu_info();
+ _cpu_features = _features;
+
+ int dcache_line = dcache_line_size();
// Limit AllocatePrefetchDistance so that it does not exceed the
// static constraint of 512 defined in runtime/globals.hpp.
@@ -124,7 +169,7 @@ void VM_Version::initialize() {
// if dcpop is available publish data cache line flush size via
// generic field, otherwise let if default to zero thereby
// disabling writeback
- if (VM_Version::supports_dcpop()) {
+ if (supports_dcpop()) {
_data_cache_line_flush_size = dcache_line;
}
}
@@ -245,14 +290,24 @@ void VM_Version::initialize() {
}
}
- if (FLAG_IS_DEFAULT(UseCRC32)) {
- UseCRC32 = VM_Version::supports_crc32();
+ if (supports_sha1() || supports_sha256() ||
+ supports_sha3() || supports_sha512()) {
+ if (FLAG_IS_DEFAULT(UseSHA)) {
+ FLAG_SET_DEFAULT(UseSHA, true);
+ } else if (!UseSHA) {
+ clear_feature(CPU_SHA1);
+ clear_feature(CPU_SHA2);
+ clear_feature(CPU_SHA3);
+ clear_feature(CPU_SHA512);
+ }
+ } else if (UseSHA) {
+ warning("SHA instructions are not available on this CPU");
+ FLAG_SET_DEFAULT(UseSHA, false);
}
- if (UseCRC32 && !VM_Version::supports_crc32()) {
- warning("UseCRC32 specified, but not supported on this CPU");
- FLAG_SET_DEFAULT(UseCRC32, false);
- }
+ CHECK_CPU_FEATURE(supports_crc32, CRC32);
+ CHECK_CPU_FEATURE(supports_lse, LSE);
+ CHECK_CPU_FEATURE(supports_aes, AES);
if (_cpu == CPU_ARM &&
model_is_in({ CPU_MODEL_ARM_NEOVERSE_V1, CPU_MODEL_ARM_NEOVERSE_V2,
@@ -265,7 +320,7 @@ void VM_Version::initialize() {
}
}
- if (UseCryptoPmullForCRC32 && (!VM_Version::supports_pmull() || !VM_Version::supports_sha3() || !VM_Version::supports_crc32())) {
+ if (UseCryptoPmullForCRC32 && (!supports_pmull() || !supports_sha3() || !supports_crc32())) {
warning("UseCryptoPmullForCRC32 specified, but not supported on this CPU");
FLAG_SET_DEFAULT(UseCryptoPmullForCRC32, false);
}
@@ -279,48 +334,40 @@ void VM_Version::initialize() {
FLAG_SET_DEFAULT(UseVectorizedMismatchIntrinsic, false);
}
- if (VM_Version::supports_lse()) {
- if (FLAG_IS_DEFAULT(UseLSE))
- FLAG_SET_DEFAULT(UseLSE, true);
- } else {
- if (UseLSE) {
- warning("UseLSE specified, but not supported on this CPU");
- FLAG_SET_DEFAULT(UseLSE, false);
- }
- }
-
- if (VM_Version::supports_aes()) {
- UseAES = UseAES || FLAG_IS_DEFAULT(UseAES);
- UseAESIntrinsics =
- UseAESIntrinsics || (UseAES && FLAG_IS_DEFAULT(UseAESIntrinsics));
- if (UseAESIntrinsics && !UseAES) {
- warning("UseAESIntrinsics enabled, but UseAES not, enabling");
- UseAES = true;
+ if (supports_aes()) {
+ if (FLAG_IS_DEFAULT(UseAESIntrinsics)) {
+ FLAG_SET_DEFAULT(UseAESIntrinsics, true);
}
if (FLAG_IS_DEFAULT(UseAESCTRIntrinsics)) {
FLAG_SET_DEFAULT(UseAESCTRIntrinsics, true);
}
} else {
- if (UseAES) {
- warning("AES instructions are not available on this CPU");
- FLAG_SET_DEFAULT(UseAES, false);
- }
- if (UseAESIntrinsics) {
- warning("AES intrinsics are not available on this CPU");
- FLAG_SET_DEFAULT(UseAESIntrinsics, false);
- }
- if (UseAESCTRIntrinsics) {
- warning("AES/CTR intrinsics are not available on this CPU");
- FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
+ if (!UseAES) {
+ if (UseAESIntrinsics) {
+ warning("AES intrinsics require UseAES flag to be enabled. Intrinsics will be disabled.");
+ FLAG_SET_DEFAULT(UseAESIntrinsics, false);
+ }
+ if (UseAESCTRIntrinsics) {
+ warning("AES/CTR intrinsics require UseAES flag to be enabled. Intrinsics will be disabled.");
+ FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
+ }
+ } else if (!cpu_supports_aes()) {
+ if (UseAESIntrinsics) {
+ warning("AES intrinsics are not available on this CPU");
+ FLAG_SET_DEFAULT(UseAESIntrinsics, false);
+ }
+ if (UseAESCTRIntrinsics) {
+ warning("AES/CTR intrinsics are not available on this CPU");
+ FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
+ }
}
}
-
if (FLAG_IS_DEFAULT(UseCRC32Intrinsics)) {
UseCRC32Intrinsics = true;
}
- if (VM_Version::supports_crc32()) {
+ if (supports_crc32()) {
if (FLAG_IS_DEFAULT(UseCRC32CIntrinsics)) {
FLAG_SET_DEFAULT(UseCRC32CIntrinsics, true);
}
@@ -337,17 +384,7 @@ void VM_Version::initialize() {
UseMD5Intrinsics = true;
}
- if (VM_Version::supports_sha1() || VM_Version::supports_sha256() ||
- VM_Version::supports_sha3() || VM_Version::supports_sha512()) {
- if (FLAG_IS_DEFAULT(UseSHA)) {
- FLAG_SET_DEFAULT(UseSHA, true);
- }
- } else if (UseSHA) {
- warning("SHA instructions are not available on this CPU");
- FLAG_SET_DEFAULT(UseSHA, false);
- }
-
- if (UseSHA && VM_Version::supports_sha1()) {
+ if (UseSHA && supports_sha1()) {
if (FLAG_IS_DEFAULT(UseSHA1Intrinsics)) {
FLAG_SET_DEFAULT(UseSHA1Intrinsics, true);
}
@@ -356,7 +393,7 @@ void VM_Version::initialize() {
FLAG_SET_DEFAULT(UseSHA1Intrinsics, false);
}
- if (UseSHA && VM_Version::supports_sha256()) {
+ if (UseSHA && supports_sha256()) {
if (FLAG_IS_DEFAULT(UseSHA256Intrinsics)) {
FLAG_SET_DEFAULT(UseSHA256Intrinsics, true);
}
@@ -365,21 +402,33 @@ void VM_Version::initialize() {
FLAG_SET_DEFAULT(UseSHA256Intrinsics, false);
}
- if (UseSHA && VM_Version::supports_sha3()) {
- // Auto-enable UseSHA3Intrinsics on hardware with performance benefit.
- // Note that the evaluation of UseSHA3Intrinsics shows better performance
+ if (UseSHA) {
+ // No need to check supports_sha3(), since a fallback GPR intrinsic implementation is provided.
+ if (FLAG_IS_DEFAULT(UseSHA3Intrinsics)) {
+ FLAG_SET_DEFAULT(UseSHA3Intrinsics, true);
+ }
+ } else if (UseSHA3Intrinsics) {
+ // Matches the documented and tested behavior: the -UseSHA option disables all SHA intrinsics.
+ warning("UseSHA3Intrinsics requires that UseSHA is enabled.");
+ FLAG_SET_DEFAULT(UseSHA3Intrinsics, false);
+ }
+
+ if (UseSHA3Intrinsics && supports_sha3()) {
+ // Auto-enable UseSIMDForSHA3Intrinsic on hardware with performance benefit.
+ // Note that the evaluation of SHA3 extension Intrinsics shows better performance
// on Apple and Qualcomm silicon but worse performance on Neoverse V1 and N2.
if (_cpu == CPU_APPLE || _cpu == CPU_QUALCOMM) { // Apple or Qualcomm silicon
- if (FLAG_IS_DEFAULT(UseSHA3Intrinsics)) {
- FLAG_SET_DEFAULT(UseSHA3Intrinsics, true);
+ if (FLAG_IS_DEFAULT(UseSIMDForSHA3Intrinsic)) {
+ FLAG_SET_DEFAULT(UseSIMDForSHA3Intrinsic, true);
}
}
- } else if (UseSHA3Intrinsics && UseSIMDForSHA3Intrinsic) {
+ }
+ if (UseSHA3Intrinsics && UseSIMDForSHA3Intrinsic && !supports_sha3()) {
warning("Intrinsics for SHA3-224, SHA3-256, SHA3-384 and SHA3-512 crypto hash functions not available on this CPU.");
FLAG_SET_DEFAULT(UseSHA3Intrinsics, false);
}
- if (UseSHA && VM_Version::supports_sha512()) {
+ if (UseSHA && supports_sha512()) {
if (FLAG_IS_DEFAULT(UseSHA512Intrinsics)) {
FLAG_SET_DEFAULT(UseSHA512Intrinsics, true);
}
@@ -392,7 +441,7 @@ void VM_Version::initialize() {
FLAG_SET_DEFAULT(UseSHA, false);
}
- if (VM_Version::supports_pmull()) {
+ if (supports_pmull()) {
if (FLAG_IS_DEFAULT(UseGHASHIntrinsics)) {
FLAG_SET_DEFAULT(UseGHASHIntrinsics, true);
}
@@ -443,18 +492,20 @@ void VM_Version::initialize() {
FLAG_SET_DEFAULT(UseBlockZeroing, true);
}
if (FLAG_IS_DEFAULT(BlockZeroingLowLimit)) {
- FLAG_SET_DEFAULT(BlockZeroingLowLimit, 4 * VM_Version::zva_length());
+ FLAG_SET_DEFAULT(BlockZeroingLowLimit, 4 * zva_length());
}
} else if (UseBlockZeroing) {
- warning("DC ZVA is not available on this CPU");
+ if (!FLAG_IS_DEFAULT(UseBlockZeroing)) {
+ warning("DC ZVA is not available on this CPU");
+ }
FLAG_SET_DEFAULT(UseBlockZeroing, false);
}
- if (VM_Version::supports_sve2()) {
+ if (supports_sve2()) {
if (FLAG_IS_DEFAULT(UseSVE)) {
FLAG_SET_DEFAULT(UseSVE, 2);
}
- } else if (VM_Version::supports_sve()) {
+ } else if (supports_sve()) {
if (FLAG_IS_DEFAULT(UseSVE)) {
FLAG_SET_DEFAULT(UseSVE, 1);
} else if (UseSVE > 1) {
@@ -505,7 +556,7 @@ void VM_Version::initialize() {
// 1) this code has been built with branch-protection and
// 2) the CPU/OS supports it
#ifdef __ARM_FEATURE_PAC_DEFAULT
- if (!VM_Version::supports_paca()) {
+ if (!supports_paca()) {
// Disable PAC to prevent illegal instruction crashes.
warning("ROP-protection specified, but not supported on this CPU. Disabling ROP-protection.");
} else {
@@ -647,6 +698,43 @@ void VM_Version::initialize() {
clear_feature(CPU_SVE);
}
+ if (FLAG_IS_DEFAULT(UseSingleICacheInvalidation) && is_cache_idc_enabled() && is_cache_dic_enabled()) {
+ FLAG_SET_DEFAULT(UseSingleICacheInvalidation, true);
+ }
+
+ if (FLAG_IS_DEFAULT(NeoverseN1ICacheErratumMitigation) && has_neoverse_n1_errata_1542419()
+ && is_cache_idc_enabled() && !is_cache_dic_enabled()) {
+ if (_ic_ivau_trapped) {
+ FLAG_SET_DEFAULT(NeoverseN1ICacheErratumMitigation, true);
+ } else {
+ log_info(os)("IC IVAU is not trapped; disabling NeoverseN1ICacheErratumMitigation");
+ FLAG_SET_DEFAULT(NeoverseN1ICacheErratumMitigation, false);
+ }
+ }
+
+ if (NeoverseN1ICacheErratumMitigation) {
+ if (!has_neoverse_n1_errata_1542419()) {
+ vm_exit_during_initialization("NeoverseN1ICacheErratumMitigation is set for the CPU not having Neoverse N1 errata 1542419");
+ }
+ // If the user explicitly set the flag, verify the trap is active.
+ if (!FLAG_IS_DEFAULT(NeoverseN1ICacheErratumMitigation) && !_ic_ivau_trapped) {
+ vm_exit_during_initialization("NeoverseN1ICacheErratumMitigation is set but IC IVAU is not trapped. "
+ "The optimization is not safe on this system.");
+ }
+ if (FLAG_IS_DEFAULT(UseSingleICacheInvalidation)) {
+ FLAG_SET_DEFAULT(UseSingleICacheInvalidation, true);
+ }
+
+ if (!UseSingleICacheInvalidation) {
+ vm_exit_during_initialization("NeoverseN1ICacheErratumMitigation is set but UseSingleICacheInvalidation is not enabled");
+ }
+ }
+
+ if (UseSingleICacheInvalidation
+ && (!is_cache_idc_enabled() || (!is_cache_dic_enabled() && !NeoverseN1ICacheErratumMitigation))) {
+ vm_exit_during_initialization("UseSingleICacheInvalidation is set but neither IDC nor DIC nor NeoverseN1ICacheErratumMitigation is enabled");
+ }
+
// Construct the "features" string
stringStream ss(512);
ss.print("0x%02x:0x%x:0x%03x:%d", _cpu, _variant, _model, _revision);
diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp
index 07f6c09e18f..30f1a5d86ca 100644
--- a/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp
@@ -55,6 +55,15 @@ protected:
static int _max_supported_sve_vector_length;
static bool _rop_protection;
static uintptr_t _pac_mask;
+ // When _prefer_sve_merging_mode_cpy is true, `cpy (imm, zeroing)` is
+ // implemented as `movi; cpy(imm, merging)`.
+ static constexpr bool _prefer_sve_merging_mode_cpy = true;
+ static bool _cache_dic_enabled;
+ static bool _cache_idc_enabled;
+
+ // IC IVAU trap probe for Neoverse N1 erratum 1542419.
+ // Set by get_os_cpu_info() on Linux via ic_ivau_probe_linux_aarch64.S.
+ static bool _ic_ivau_trapped;
static SpinWait _spin_wait;
@@ -156,7 +165,9 @@ public:
/* flags above must follow Linux HWCAP */ \
decl(SVEBITPERM, svebitperm, 27) \
decl(SVE2, sve2, 28) \
- decl(A53MAC, a53mac, 31)
+ decl(A53MAC, a53mac, 31) \
+ decl(ECV, ecv, 32) \
+ decl(WFXT, wfxt, 33)
enum Feature_Flag {
#define DECLARE_CPU_FEATURE_FLAG(id, name, bit) CPU_##id = bit,
@@ -188,6 +199,8 @@ public:
return (features & BIT_MASK(flag)) != 0;
}
+ static bool cpu_supports_aes() { return supports_feature(_cpu_features, CPU_AES); }
+
static int cpu_family() { return _cpu; }
static int cpu_model() { return _model; }
static int cpu_model2() { return _model2; }
@@ -207,7 +220,7 @@ public:
return false;
}
- static bool is_zva_enabled() { return 0 <= _zva_length; }
+ static bool is_zva_enabled() { return 0 < _zva_length; }
static int zva_length() {
assert(is_zva_enabled(), "ZVA not available");
return _zva_length;
@@ -242,12 +255,18 @@ public:
static bool use_rop_protection() { return _rop_protection; }
+ static bool prefer_sve_merging_mode_cpy() { return _prefer_sve_merging_mode_cpy; }
+
// For common 64/128-bit unpredicated vector operations, we may prefer
// emitting NEON instructions rather than the corresponding SVE instructions.
static bool use_neon_for_vector(int vector_length_in_bytes) {
return vector_length_in_bytes <= 16;
}
+ static bool is_cache_dic_enabled() { return _cache_dic_enabled; }
+ static bool is_cache_idc_enabled() { return _cache_idc_enabled; }
+ static bool is_ic_ivau_trapped() { return _ic_ivau_trapped; }
+
static void get_cpu_features_name(void* features_buffer, stringStream& ss);
// Returns names of features present in features_set1 but not in features_set2
diff --git a/src/hotspot/cpu/arm/arm.ad b/src/hotspot/cpu/arm/arm.ad
index 606275d7666..60a0ef307b5 100644
--- a/src/hotspot/cpu/arm/arm.ad
+++ b/src/hotspot/cpu/arm/arm.ad
@@ -1,5 +1,5 @@
//
-// Copyright (c) 2008, 2025, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2008, 2026, Oracle and/or its affiliates. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
@@ -1088,10 +1088,8 @@ bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack,
return clone_base_plus_offset_address(m, mstack, address_visited);
}
-// Return whether or not this register is ever used as an argument. This
-// function is used on startup to build the trampoline stubs in generateOptoStub.
-// Registers not mentioned will be killed by the VM call in the trampoline, and
-// arguments in those registers not be available to the callee.
+#ifdef ASSERT
+// Return whether or not this register is ever used as an argument.
bool Matcher::can_be_java_arg( int reg ) {
if (reg == R_R0_num ||
reg == R_R1_num ||
@@ -1102,10 +1100,7 @@ bool Matcher::can_be_java_arg( int reg ) {
reg <= R_S13_num) return true;
return false;
}
-
-bool Matcher::is_spillable_arg( int reg ) {
- return can_be_java_arg(reg);
-}
+#endif
uint Matcher::int_pressure_limit()
{
@@ -1117,10 +1112,6 @@ uint Matcher::float_pressure_limit()
return (FLOATPRESSURE == -1) ? 30 : FLOATPRESSURE;
}
-bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
- return false;
-}
-
// Register for DIVI projection of divmodI
const RegMask& Matcher::divI_proj_mask() {
ShouldNotReachHere();
@@ -4445,6 +4436,18 @@ instruct membar_release_lock() %{
ins_pipe(empty);
%}
+instruct membar_storeload() %{
+ match(MemBarStoreLoad);
+ ins_cost(4*MEMORY_REF_COST);
+
+ size(4);
+ format %{ "MEMBAR-storeload" %}
+ ins_encode %{
+ __ membar(MacroAssembler::StoreLoad, noreg);
+ %}
+ ins_pipe(long_memory_op);
+%}
+
instruct membar_volatile() %{
match(MemBarVolatile);
ins_cost(4*MEMORY_REF_COST);
@@ -4468,6 +4471,18 @@ instruct unnecessary_membar_volatile() %{
ins_pipe(empty);
%}
+instruct membar_full() %{
+ match(MemBarFull);
+ ins_cost(4*MEMORY_REF_COST);
+
+ size(4);
+ format %{ "MEMBAR-full" %}
+ ins_encode %{
+ __ membar(MacroAssembler::StoreLoad, noreg);
+ %}
+ ins_pipe(long_memory_op);
+%}
+
//----------Register Move Instructions-----------------------------------------
// Cast Index to Pointer for unsafe natives
diff --git a/src/hotspot/cpu/arm/c1_globals_arm.hpp b/src/hotspot/cpu/arm/c1_globals_arm.hpp
index 1fe5f1a23ee..9db999e81b3 100644
--- a/src/hotspot/cpu/arm/c1_globals_arm.hpp
+++ b/src/hotspot/cpu/arm/c1_globals_arm.hpp
@@ -43,7 +43,6 @@ define_pd_global(bool, TieredCompilation, false);
define_pd_global(intx, CompileThreshold, 1500 );
define_pd_global(intx, OnStackReplacePercentage, 933 );
-define_pd_global(size_t, NewSizeThreadIncrease, 4*K );
define_pd_global(size_t, InitialCodeCacheSize, 160*K);
define_pd_global(size_t, ReservedCodeCacheSize, 32*M );
define_pd_global(size_t, NonProfiledCodeHeapSize, 13*M );
@@ -53,7 +52,6 @@ define_pd_global(bool, ProfileInterpreter, false);
define_pd_global(size_t, CodeCacheExpansionSize, 32*K );
define_pd_global(size_t, CodeCacheMinBlockLength, 1);
define_pd_global(size_t, CodeCacheMinimumUseSpace, 400*K);
-define_pd_global(bool, NeverActAsServerClassMachine, true);
define_pd_global(bool, CICompileOSR, true );
#endif // COMPILER2
define_pd_global(bool, UseTypeProfile, false);
diff --git a/src/hotspot/cpu/arm/c2_globals_arm.hpp b/src/hotspot/cpu/arm/c2_globals_arm.hpp
index 0849bd594f0..34da47792ae 100644
--- a/src/hotspot/cpu/arm/c2_globals_arm.hpp
+++ b/src/hotspot/cpu/arm/c2_globals_arm.hpp
@@ -47,7 +47,6 @@ define_pd_global(intx, ConditionalMoveLimit, 4);
// C2 gets to use all the float/double registers
define_pd_global(intx, FreqInlineSize, 175);
define_pd_global(intx, InteriorEntryAlignment, 16); // = CodeEntryAlignment
-define_pd_global(size_t, NewSizeThreadIncrease, ScaleForWordSize(4*K));
// The default setting 16/16 seems to work best.
// (For _228_jack 16/16 is 2% better than 4/4, 16/4, 32/32, 32/16, or 16/32.)
//define_pd_global(intx, OptoLoopAlignment, 16); // = 4*wordSize
@@ -94,7 +93,4 @@ define_pd_global(size_t, CodeCacheMinimumUseSpace, 400*K);
define_pd_global(bool, TrapBasedRangeChecks, false); // Not needed
-// Ergonomics related flags
-define_pd_global(bool, NeverActAsServerClassMachine, false);
-
#endif // CPU_ARM_C2_GLOBALS_ARM_HPP
diff --git a/src/hotspot/cpu/arm/interp_masm_arm.cpp b/src/hotspot/cpu/arm/interp_masm_arm.cpp
index 23ecea24eb2..aee407864ee 100644
--- a/src/hotspot/cpu/arm/interp_masm_arm.cpp
+++ b/src/hotspot/cpu/arm/interp_masm_arm.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2008, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1210,7 +1210,7 @@ void InterpreterMacroAssembler::profile_final_call(Register mdp) {
// Sets mdp, blows Rtemp.
-void InterpreterMacroAssembler::profile_virtual_call(Register mdp, Register receiver, bool receiver_can_be_null) {
+void InterpreterMacroAssembler::profile_virtual_call(Register mdp, Register receiver) {
assert_different_registers(mdp, receiver, Rtemp);
if (ProfileInterpreter) {
@@ -1219,19 +1219,8 @@ void InterpreterMacroAssembler::profile_virtual_call(Register mdp, Register rece
// If no method data exists, go to profile_continue.
test_method_data_pointer(mdp, profile_continue);
- Label skip_receiver_profile;
- if (receiver_can_be_null) {
- Label not_null;
- cbnz(receiver, not_null);
- // We are making a call. Increment the count for null receiver.
- increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp);
- b(skip_receiver_profile);
- bind(not_null);
- }
-
// Record the receiver type.
record_klass_in_profile(receiver, mdp, Rtemp, true);
- bind(skip_receiver_profile);
// The method data pointer needs to be updated to reflect the new target.
update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size()));
diff --git a/src/hotspot/cpu/arm/interp_masm_arm.hpp b/src/hotspot/cpu/arm/interp_masm_arm.hpp
index 530be1c577e..147cd252b2c 100644
--- a/src/hotspot/cpu/arm/interp_masm_arm.hpp
+++ b/src/hotspot/cpu/arm/interp_masm_arm.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2008, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -239,8 +239,7 @@ class InterpreterMacroAssembler: public MacroAssembler {
void profile_call(Register mdp); // Sets mdp, blows Rtemp.
void profile_final_call(Register mdp); // Sets mdp, blows Rtemp.
- void profile_virtual_call(Register mdp, Register receiver, // Sets mdp, blows Rtemp.
- bool receiver_can_be_null = false);
+ void profile_virtual_call(Register mdp, Register receiver); // Sets mdp, blows Rtemp.
void profile_ret(Register mdp, Register return_bci); // Sets mdp, blows R0-R3/R0-R18, Rtemp, LR
void profile_null_seen(Register mdp); // Sets mdp.
void profile_typecheck(Register mdp, Register klass); // Sets mdp, blows Rtemp.
diff --git a/src/hotspot/cpu/arm/matcher_arm.hpp b/src/hotspot/cpu/arm/matcher_arm.hpp
index 6c818e1f20d..7978a5b7090 100644
--- a/src/hotspot/cpu/arm/matcher_arm.hpp
+++ b/src/hotspot/cpu/arm/matcher_arm.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -75,7 +75,6 @@
static bool narrow_klass_use_complex_address() {
NOT_LP64(ShouldNotCallThis());
- assert(UseCompressedClassPointers, "only for compressed klass code");
return false;
}
diff --git a/src/hotspot/cpu/arm/methodHandles_arm.cpp b/src/hotspot/cpu/arm/methodHandles_arm.cpp
index 3710fa33f36..2da14d8ffed 100644
--- a/src/hotspot/cpu/arm/methodHandles_arm.cpp
+++ b/src/hotspot/cpu/arm/methodHandles_arm.cpp
@@ -104,14 +104,13 @@ void MethodHandles::verify_ref_kind(MacroAssembler* _masm, int ref_kind, Registe
__ andr(temp, temp, (unsigned)java_lang_invoke_MemberName::MN_REFERENCE_KIND_MASK);
__ cmp(temp, ref_kind);
__ b(L, eq);
- { char* buf = NEW_C_HEAP_ARRAY(char, 100, mtInternal);
- jio_snprintf(buf, 100, "verify_ref_kind expected %x", ref_kind);
+ const char* msg = ref_kind_to_verify_msg(ref_kind);
if (ref_kind == JVM_REF_invokeVirtual ||
- ref_kind == JVM_REF_invokeSpecial)
+ ref_kind == JVM_REF_invokeSpecial) {
// could do this for all ref_kinds, but would explode assembly code size
- trace_method_handle(_masm, buf);
- __ stop(buf);
+ trace_method_handle(_masm, msg);
}
+ __ stop(msg);
BLOCK_COMMENT("} verify_ref_kind");
__ bind(L);
}
diff --git a/src/hotspot/cpu/ppc/assembler_ppc.hpp b/src/hotspot/cpu/ppc/assembler_ppc.hpp
index 23775a3a52e..378e01fc1cc 100644
--- a/src/hotspot/cpu/ppc/assembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/assembler_ppc.hpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2002, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
+ * Copyright (c) 2002, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -599,6 +599,9 @@ class Assembler : public AbstractAssembler {
XVMAXSP_OPCODE = (60u << OPCODE_SHIFT | 192u << 3),
XVMAXDP_OPCODE = (60u << OPCODE_SHIFT | 224u << 3),
+ XSMINJDP_OPCODE = (60u << OPCODE_SHIFT | 152u << 3),
+ XSMAXJDP_OPCODE = (60u << OPCODE_SHIFT | 144u << 3),
+
// Deliver A Random Number (introduced with POWER9)
DARN_OPCODE = (31u << OPCODE_SHIFT | 755u << 1),
@@ -1577,10 +1580,6 @@ class Assembler : public AbstractAssembler {
static bool is_nop(int x) {
return x == 0x60000000;
}
- // endgroup opcode for Power6
- static bool is_endgroup(int x) {
- return is_ori(x) && inv_ra_field(x) == 1 && inv_rs_field(x) == 1 && inv_d1_field(x) == 0;
- }
private:
@@ -1656,9 +1655,6 @@ class Assembler : public AbstractAssembler {
inline void ori_opt( Register d, int ui16);
inline void oris_opt(Register d, int ui16);
- // endgroup opcode for Power6
- inline void endgroup();
-
// count instructions
inline void cntlzw( Register a, Register s);
inline void cntlzw_( Register a, Register s);
@@ -2455,6 +2451,9 @@ class Assembler : public AbstractAssembler {
inline void xvrdpim( VectorSRegister d, VectorSRegister b);
inline void xvrdpip( VectorSRegister d, VectorSRegister b);
+ inline void xsminjdp( VectorSRegister d, VectorSRegister a, VectorSRegister b); // Requires Power 9
+ inline void xsmaxjdp( VectorSRegister d, VectorSRegister a, VectorSRegister b); // Requires Power 9
+
// The following functions do not match exactly the Java.math semantics.
inline void xvminsp( VectorSRegister d, VectorSRegister a, VectorSRegister b);
inline void xvmindp( VectorSRegister d, VectorSRegister a, VectorSRegister b);
diff --git a/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp b/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp
index 4cda782067e..d349bbc6f87 100644
--- a/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp
+++ b/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2002, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
+ * Copyright (c) 2002, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -253,8 +253,6 @@ inline void Assembler::mr( Register d, Register s) { Assembler::orr(d, s,
inline void Assembler::ori_opt( Register d, int ui16) { if (ui16!=0) Assembler::ori( d, d, ui16); }
inline void Assembler::oris_opt(Register d, int ui16) { if (ui16!=0) Assembler::oris(d, d, ui16); }
-inline void Assembler::endgroup() { Assembler::ori(R1, R1, 0); }
-
// count instructions
inline void Assembler::cntlzw( Register a, Register s) { emit_int32(CNTLZW_OPCODE | rta(a) | rs(s) | rc(0)); }
inline void Assembler::cntlzw_( Register a, Register s) { emit_int32(CNTLZW_OPCODE | rta(a) | rs(s) | rc(1)); }
@@ -908,6 +906,9 @@ inline void Assembler::xvrdpic( VectorSRegister d, VectorSRegister b)
inline void Assembler::xvrdpim( VectorSRegister d, VectorSRegister b) { emit_int32( XVRDPIM_OPCODE | vsrt(d) | vsrb(b)); }
inline void Assembler::xvrdpip( VectorSRegister d, VectorSRegister b) { emit_int32( XVRDPIP_OPCODE | vsrt(d) | vsrb(b)); }
+inline void Assembler::xsminjdp(VectorSRegister d, VectorSRegister a, VectorSRegister b) { emit_int32( XSMINJDP_OPCODE | vsrt(d) | vsra(a) | vsrb(b)); }
+inline void Assembler::xsmaxjdp(VectorSRegister d, VectorSRegister a, VectorSRegister b) { emit_int32( XSMAXJDP_OPCODE | vsrt(d) | vsra(a) | vsrb(b)); }
+
inline void Assembler::xvminsp(VectorSRegister d, VectorSRegister a, VectorSRegister b) { emit_int32( XVMINSP_OPCODE | vsrt(d) | vsra(a) | vsrb(b)); }
inline void Assembler::xvmindp(VectorSRegister d, VectorSRegister a, VectorSRegister b) { emit_int32( XVMINDP_OPCODE | vsrt(d) | vsra(a) | vsrb(b)); }
inline void Assembler::xvmaxsp(VectorSRegister d, VectorSRegister a, VectorSRegister b) { emit_int32( XVMAXSP_OPCODE | vsrt(d) | vsra(a) | vsrb(b)); }
diff --git a/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp
index 798451446e5..4d7af0e4a71 100644
--- a/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2025 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -144,7 +144,7 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
if (len->is_valid()) {
stw(len, arrayOopDesc::length_offset_in_bytes(), obj);
- } else if (UseCompressedClassPointers && !UseCompactObjectHeaders) {
+ } else if (!UseCompactObjectHeaders) {
// Otherwise length is in the class gap.
store_klass_gap(obj);
}
diff --git a/src/hotspot/cpu/ppc/c1_globals_ppc.hpp b/src/hotspot/cpu/ppc/c1_globals_ppc.hpp
index 77d9acd1cd1..c6fe15aac07 100644
--- a/src/hotspot/cpu/ppc/c1_globals_ppc.hpp
+++ b/src/hotspot/cpu/ppc/c1_globals_ppc.hpp
@@ -51,8 +51,6 @@ define_pd_global(size_t, NonNMethodCodeHeapSize, 5*M );
define_pd_global(size_t, CodeCacheExpansionSize, 32*K);
define_pd_global(size_t, CodeCacheMinBlockLength, 1);
define_pd_global(size_t, CodeCacheMinimumUseSpace, 400*K);
-define_pd_global(bool, NeverActAsServerClassMachine, true);
-define_pd_global(size_t, NewSizeThreadIncrease, 16*K);
define_pd_global(size_t, InitialCodeCacheSize, 160*K);
#endif // !COMPILER2
diff --git a/src/hotspot/cpu/ppc/c2_MacroAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c2_MacroAssembler_ppc.cpp
index 2d5071ce86d..8bbffc22c54 100644
--- a/src/hotspot/cpu/ppc/c2_MacroAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/c2_MacroAssembler_ppc.cpp
@@ -1,5 +1,6 @@
/*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
diff --git a/src/hotspot/cpu/ppc/c2_globals_ppc.hpp b/src/hotspot/cpu/ppc/c2_globals_ppc.hpp
index caef322d4a1..e4942fa1850 100644
--- a/src/hotspot/cpu/ppc/c2_globals_ppc.hpp
+++ b/src/hotspot/cpu/ppc/c2_globals_ppc.hpp
@@ -47,7 +47,6 @@ define_pd_global(intx, ConditionalMoveLimit, 3);
define_pd_global(intx, FreqInlineSize, 325);
define_pd_global(intx, MinJumpTableSize, 10);
define_pd_global(intx, InteriorEntryAlignment, 16);
-define_pd_global(size_t, NewSizeThreadIncrease, ScaleForWordSize(4*K));
define_pd_global(intx, RegisterCostAreaRatio, 16000);
define_pd_global(intx, LoopUnrollLimit, 60);
define_pd_global(intx, LoopPercentProfileLimit, 10);
@@ -91,7 +90,4 @@ define_pd_global(size_t, CodeCacheMinimumUseSpace, 400*K);
define_pd_global(bool, TrapBasedRangeChecks, true);
-// Ergonomics related flags
-define_pd_global(bool, NeverActAsServerClassMachine, false);
-
#endif // CPU_PPC_C2_GLOBALS_PPC_HPP
diff --git a/src/hotspot/cpu/ppc/disassembler_ppc.cpp b/src/hotspot/cpu/ppc/disassembler_ppc.cpp
index fb3cb50cdec..2e16e1a301f 100644
--- a/src/hotspot/cpu/ppc/disassembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/disassembler_ppc.cpp
@@ -119,9 +119,6 @@ address Disassembler::decode_instruction0(address here, outputStream * st, addre
} else if (instruction == 0xbadbabe) {
st->print(".data 0xbadbabe");
next = here + Assembler::instr_len(here);
- } else if (Assembler::is_endgroup(instruction)) {
- st->print("endgroup");
- next = here + Assembler::instr_len(here);
} else {
next = here;
}
diff --git a/src/hotspot/cpu/ppc/downcallLinker_ppc.cpp b/src/hotspot/cpu/ppc/downcallLinker_ppc.cpp
index f12d25ac611..d149fc33ac3 100644
--- a/src/hotspot/cpu/ppc/downcallLinker_ppc.cpp
+++ b/src/hotspot/cpu/ppc/downcallLinker_ppc.cpp
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2020, 2025 SAP SE. All rights reserved.
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -135,10 +135,10 @@ void DowncallLinker::StubGenerator::generate() {
bool should_save_return_value = !_needs_return_buffer;
RegSpiller out_reg_spiller(_output_registers);
- int spill_offset = -1;
+ int out_spill_offset = -1;
if (should_save_return_value) {
- spill_offset = frame::native_abi_reg_args_size;
+ out_spill_offset = frame::native_abi_reg_args_size;
// Spill area can be shared with additional out args (>8),
// since it is only used after the call.
int frame_size_including_spill_area = frame::native_abi_reg_args_size + out_reg_spiller.spill_size_bytes();
@@ -170,6 +170,18 @@ void DowncallLinker::StubGenerator::generate() {
ArgumentShuffle arg_shuffle(filtered_java_regs, out_regs, _abi._scratch1);
+ // Need to spill for state capturing runtime call.
+ // The area spilled into is distinct from the capture state buffer.
+ RegSpiller in_reg_spiller(out_regs);
+ int in_spill_offset = -1;
+ if (_captured_state_mask != 0) {
+ // The spill area cannot be shared with the out_spill since
+ // spilling needs to happen before the call. Allocate a new
+ // region in the stack for this spill space.
+ in_spill_offset = allocated_frame_size;
+ allocated_frame_size += in_reg_spiller.spill_size_bytes();
+ }
+
#ifndef PRODUCT
LogTarget(Trace, foreign, downcall) lt;
if (lt.is_enabled()) {
@@ -211,6 +223,21 @@ void DowncallLinker::StubGenerator::generate() {
arg_shuffle.generate(_masm, as_VMStorage(callerSP), frame::jit_out_preserve_size, frame::native_abi_minframe_size);
__ block_comment("} argument shuffle");
+ if (_captured_state_mask != 0) {
+ assert(in_spill_offset != -1, "must be");
+ __ block_comment("{ load initial thread local");
+ in_reg_spiller.generate_spill(_masm, in_spill_offset);
+
+ // Copy the contents of the capture state buffer into thread local
+ __ load_const_optimized(call_target_address, CAST_FROM_FN_PTR(uint64_t, DowncallLinker::capture_state_pre), R0);
+ __ ld(R3_ARG1, locs.data_offset(StubLocations::CAPTURED_STATE_BUFFER), R1_SP);
+ __ load_const_optimized(R4_ARG2, _captured_state_mask, R0);
+ __ call_c(call_target_address);
+
+ in_reg_spiller.generate_fill(_masm, in_spill_offset);
+ __ block_comment("} load initial thread local");
+ }
+
__ call_c(call_target_address);
if (_needs_return_buffer) {
@@ -247,16 +274,16 @@ void DowncallLinker::StubGenerator::generate() {
__ block_comment("{ save thread local");
if (should_save_return_value) {
- out_reg_spiller.generate_spill(_masm, spill_offset);
+ out_reg_spiller.generate_spill(_masm, out_spill_offset);
}
- __ load_const_optimized(call_target_address, CAST_FROM_FN_PTR(uint64_t, DowncallLinker::capture_state), R0);
+ __ load_const_optimized(call_target_address, CAST_FROM_FN_PTR(uint64_t, DowncallLinker::capture_state_post), R0);
__ ld(R3_ARG1, locs.data_offset(StubLocations::CAPTURED_STATE_BUFFER), R1_SP);
__ load_const_optimized(R4_ARG2, _captured_state_mask, R0);
__ call_c(call_target_address);
if (should_save_return_value) {
- out_reg_spiller.generate_fill(_masm, spill_offset);
+ out_reg_spiller.generate_fill(_masm, out_spill_offset);
}
__ block_comment("} save thread local");
@@ -310,7 +337,7 @@ void DowncallLinker::StubGenerator::generate() {
if (should_save_return_value) {
// Need to save the native result registers around any runtime calls.
- out_reg_spiller.generate_spill(_masm, spill_offset);
+ out_reg_spiller.generate_spill(_masm, out_spill_offset);
}
__ load_const_optimized(call_target_address, CAST_FROM_FN_PTR(uint64_t, JavaThread::check_special_condition_for_native_trans), R0);
@@ -318,7 +345,7 @@ void DowncallLinker::StubGenerator::generate() {
__ call_c(call_target_address);
if (should_save_return_value) {
- out_reg_spiller.generate_fill(_masm, spill_offset);
+ out_reg_spiller.generate_fill(_masm, out_spill_offset);
}
__ b(L_after_safepoint_poll);
@@ -330,14 +357,14 @@ void DowncallLinker::StubGenerator::generate() {
__ bind(L_reguard);
if (should_save_return_value) {
- out_reg_spiller.generate_spill(_masm, spill_offset);
+ out_reg_spiller.generate_spill(_masm, out_spill_offset);
}
__ load_const_optimized(call_target_address, CAST_FROM_FN_PTR(uint64_t, SharedRuntime::reguard_yellow_pages), R0);
__ call_c(call_target_address);
if (should_save_return_value) {
- out_reg_spiller.generate_fill(_masm, spill_offset);
+ out_reg_spiller.generate_fill(_masm, out_spill_offset);
}
__ b(L_after_reguard);
diff --git a/src/hotspot/cpu/ppc/frame_ppc.inline.hpp b/src/hotspot/cpu/ppc/frame_ppc.inline.hpp
index bb711f2d053..3c05f950d0c 100644
--- a/src/hotspot/cpu/ppc/frame_ppc.inline.hpp
+++ b/src/hotspot/cpu/ppc/frame_ppc.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2025 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -137,10 +137,10 @@ inline frame::frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address
// Return unique id for this frame. The id must have a value where we
// can distinguish identity and younger/older relationship. null
-// represents an invalid (incomparable) frame.
+// represents an invalid (incomparable) frame. Should not be called for heap frames.
inline intptr_t* frame::id(void) const {
// Use _fp. _sp or _unextended_sp wouldn't be correct due to resizing.
- return _fp;
+ return real_fp();
}
// Return true if this frame is older (less recent activation) than
@@ -319,6 +319,9 @@ inline frame frame::sender(RegisterMap* map) const {
StackWatermarkSet::on_iteration(map->thread(), result);
}
+ // Calling frame::id() is currently not supported for heap frames.
+ assert(result._on_heap || this->_on_heap || result.is_older(this->id()), "Must be");
+
return result;
}
diff --git a/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.cpp b/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.cpp
index 8712c75711d..3692b247989 100644
--- a/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.cpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2018, 2025 SAP SE. All rights reserved.
+ * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -179,6 +179,11 @@ void BarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Re
__ ld(dst, 0, dst); // Resolve (untagged) jobject.
}
+void BarrierSetAssembler::try_resolve_weak_handle(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path) {
+ // Load the oop from the weak handle.
+ __ ld(obj, 0, obj);
+}
+
void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm, Register tmp) {
BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
assert_different_registers(tmp, R0);
diff --git a/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.hpp b/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.hpp
index 2bf26bd5010..8112542d761 100644
--- a/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.hpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2018, 2022 SAP SE. All rights reserved.
+ * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -70,6 +70,12 @@ public:
virtual void try_resolve_jobject_in_native(MacroAssembler* masm, Register dst, Register jni_env,
Register obj, Register tmp, Label& slowpath);
+ // Can be used in nmethods including native wrappers.
+ // Attention: obj will only be valid until next safepoint (no SATB barrier).
+ // TODO: maybe rename to try_peek_weak_handle on all platforms (try: operation may fail, peek: obj is not kept alive)
+ // (other platforms currently use it for C2 only: try_resolve_weak_handle_in_c2)
+ virtual void try_resolve_weak_handle(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path);
+
virtual void barrier_stubs_init() {}
virtual NMethodPatchingType nmethod_patching_type() { return NMethodPatchingType::stw_instruction_and_data_patch; }
diff --git a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp
index c3bb1811031..8e99d23cc99 100644
--- a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp
@@ -1,6 +1,7 @@
/*
+ * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, 2025, Red Hat, Inc. All rights reserved.
- * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -662,6 +663,31 @@ void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler
__ block_comment("} try_resolve_jobject_in_native (shenandoahgc)");
}
+void ShenandoahBarrierSetAssembler::try_resolve_weak_handle(MacroAssembler *masm, Register obj,
+ Register tmp, Label &slow_path) {
+ __ block_comment("try_resolve_weak_handle (shenandoahgc) {");
+
+ assert_different_registers(obj, tmp);
+
+ Label done;
+
+ // Resolve weak handle using the standard implementation.
+ BarrierSetAssembler::try_resolve_weak_handle(masm, obj, tmp, slow_path);
+
+ // Check if the reference is null, and if it is, take the fast path.
+ __ cmpdi(CR0, obj, 0);
+ __ beq(CR0, done);
+
+ // Check if the heap is under weak-reference/roots processing, in
+ // which case we need to take the slow path.
+ __ lbz(tmp, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread);
+ __ andi_(tmp, tmp, ShenandoahHeap::WEAK_ROOTS);
+ __ bne(CR0, slow_path);
+ __ bind(done);
+
+ __ block_comment("} try_resolve_weak_handle (shenandoahgc)");
+}
+
// Special shenandoah CAS implementation that handles false negatives due
// to concurrent evacuation. That is, the CAS operation is intended to succeed in
// the following scenarios (success criteria):
diff --git a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.hpp b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.hpp
index 52615a740af..58180c49642 100644
--- a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.hpp
@@ -1,6 +1,7 @@
/*
+ * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, 2022, Red Hat, Inc. All rights reserved.
- * Copyright (c) 2012, 2022 SAP SE. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -121,6 +122,8 @@ public:
virtual void try_resolve_jobject_in_native(MacroAssembler* masm, Register dst, Register jni_env,
Register obj, Register tmp, Label& slowpath);
+
+ virtual void try_resolve_weak_handle(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path);
};
#endif // CPU_PPC_GC_SHENANDOAH_SHENANDOAHBARRIERSETASSEMBLER_PPC_HPP
diff --git a/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.cpp b/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.cpp
index 0aa5858c8e6..3e74dfb88cb 100644
--- a/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.cpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2021, 2025 SAP SE. All rights reserved.
+ * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -627,6 +627,19 @@ void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, R
__ block_comment("} try_resolve_jobject_in_native (zgc)");
}
+void ZBarrierSetAssembler::try_resolve_weak_handle(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path) {
+ // Resolve weak handle using the standard implementation.
+ BarrierSetAssembler::try_resolve_weak_handle(masm, obj, tmp, slow_path);
+
+ // Check if the oop is bad, in which case we need to take the slow path.
+ __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatMarkBadMask);
+ __ andi_(R0, obj, barrier_Relocation::unpatched);
+ __ bne(CR0, slow_path);
+
+ // Oop is okay, so we uncolor it.
+ __ srdi(obj, obj, ZPointerLoadShift);
+}
+
#undef __
#ifdef COMPILER1
diff --git a/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.hpp b/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.hpp
index 27203e7b01c..655184cf6a3 100644
--- a/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.hpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2021, 2024, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2021, 2022 SAP SE. All rights reserved.
+ * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -72,6 +72,8 @@ public:
virtual void try_resolve_jobject_in_native(MacroAssembler* masm, Register dst, Register jni_env,
Register obj, Register tmp, Label& slowpath);
+ virtual void try_resolve_weak_handle(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path);
+
virtual void check_oop(MacroAssembler *masm, Register obj, const char* msg);
virtual NMethodPatchingType nmethod_patching_type() { return NMethodPatchingType::conc_instruction_and_data_patch; }
diff --git a/src/hotspot/cpu/ppc/icache_ppc.cpp b/src/hotspot/cpu/ppc/icache_ppc.cpp
index 05ad3c7a30d..f3d51bad18c 100644
--- a/src/hotspot/cpu/ppc/icache_ppc.cpp
+++ b/src/hotspot/cpu/ppc/icache_ppc.cpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2018 SAP SE. All rights reserved.
+ * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -24,6 +24,7 @@
*/
#include "runtime/icache.hpp"
+#include "runtime/vm_version.hpp"
// Use inline assembler to implement icache flush.
int ICache::ppc64_flush_icache(address start, int lines, int magic) {
@@ -67,6 +68,9 @@ int ICache::ppc64_flush_icache(address start, int lines, int magic) {
void ICacheStubGenerator::generate_icache_flush(ICache::flush_icache_stub_t* flush_icache_stub) {
+ guarantee(VM_Version::get_icache_line_size() >= ICache::line_size,
+ "processors with smaller cache line size are no longer supported");
+
*flush_icache_stub = (ICache::flush_icache_stub_t)ICache::ppc64_flush_icache;
// First call to flush itself.
diff --git a/src/hotspot/cpu/ppc/icache_ppc.hpp b/src/hotspot/cpu/ppc/icache_ppc.hpp
index d348cad1c72..024f706182a 100644
--- a/src/hotspot/cpu/ppc/icache_ppc.hpp
+++ b/src/hotspot/cpu/ppc/icache_ppc.hpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2002, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2013 SAP SE. All rights reserved.
+ * Copyright (c) 2002, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -35,9 +35,8 @@ class ICache : public AbstractICache {
public:
enum {
- // Actually, cache line size is 64, but keeping it as it is to be
- // on the safe side on ALL PPC64 implementations.
- log2_line_size = 5,
+ // Cache line size is 128 on all supported PPC64 implementations.
+ log2_line_size = 7,
line_size = 1 << log2_line_size
};
diff --git a/src/hotspot/cpu/ppc/interp_masm_ppc.hpp b/src/hotspot/cpu/ppc/interp_masm_ppc.hpp
index 4ea33ebaf63..275ff92c699 100644
--- a/src/hotspot/cpu/ppc/interp_masm_ppc.hpp
+++ b/src/hotspot/cpu/ppc/interp_masm_ppc.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2002, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2002, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2025 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -258,7 +258,7 @@ class InterpreterMacroAssembler: public MacroAssembler {
void profile_not_taken_branch(Register scratch1, Register scratch2);
void profile_call(Register scratch1, Register scratch2);
void profile_final_call(Register scratch1, Register scratch2);
- void profile_virtual_call(Register Rreceiver, Register Rscratch1, Register Rscratch2, bool receiver_can_be_null);
+ void profile_virtual_call(Register Rreceiver, Register Rscratch1, Register Rscratch2);
void profile_typecheck(Register Rklass, Register Rscratch1, Register Rscratch2);
void profile_ret(TosState state, Register return_bci, Register scratch1, Register scratch2);
void profile_switch_default(Register scratch1, Register scratch2);
diff --git a/src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp b/src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp
index f7bf457f72c..56eade8e533 100644
--- a/src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp
+++ b/src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2025 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -1340,28 +1340,15 @@ void InterpreterMacroAssembler::profile_final_call(Register scratch1, Register s
// Count a virtual call in the bytecodes.
void InterpreterMacroAssembler::profile_virtual_call(Register Rreceiver,
Register Rscratch1,
- Register Rscratch2,
- bool receiver_can_be_null) {
+ Register Rscratch2) {
if (!ProfileInterpreter) { return; }
Label profile_continue;
// If no method data exists, go to profile_continue.
test_method_data_pointer(profile_continue);
- Label skip_receiver_profile;
- if (receiver_can_be_null) {
- Label not_null;
- cmpdi(CR0, Rreceiver, 0);
- bne(CR0, not_null);
- // We are making a call. Increment the count for null receiver.
- increment_mdp_data_at(in_bytes(CounterData::count_offset()), Rscratch1, Rscratch2);
- b(skip_receiver_profile);
- bind(not_null);
- }
-
// Record the receiver type.
record_klass_in_profile(Rreceiver, Rscratch1, Rscratch2);
- bind(skip_receiver_profile);
// The method data pointer needs to be updated to reflect the new target.
update_mdp_by_constant(in_bytes(VirtualCallData::virtual_call_data_size()));
diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp
index 809285afddb..5fbcce94029 100644
--- a/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -42,6 +42,7 @@
#include "runtime/icache.hpp"
#include "runtime/interfaceSupport.inline.hpp"
#include "runtime/objectMonitor.hpp"
+#include "runtime/objectMonitorTable.hpp"
#include "runtime/os.hpp"
#include "runtime/safepoint.hpp"
#include "runtime/safepointMechanism.hpp"
@@ -482,7 +483,7 @@ void MacroAssembler::set_dest_of_bc_far_at(address instruction_addr, address des
// variant 3, far cond branch to the next instruction, already patched to nops:
//
// nop
- // endgroup
+ // nop
// SKIP/DEST:
//
return;
@@ -499,7 +500,7 @@ void MacroAssembler::set_dest_of_bc_far_at(address instruction_addr, address des
if (is_bc_far_variant2_at(instruction_addr) && dest == instruction_addr + 8) {
// Far branch to next instruction: Optimize it by patching nops (produce variant 3).
masm.nop();
- masm.endgroup();
+ masm.nop();
} else {
if (is_bc_far_variant1_at(instruction_addr)) {
// variant 1, the 1st instruction contains the destination address:
@@ -2756,39 +2757,54 @@ void MacroAssembler::compiler_fast_lock_object(ConditionRegister flag, Register
addi(owner_addr, mark, in_bytes(ObjectMonitor::owner_offset()) - monitor_tag);
mark = noreg;
} else {
+ const Register tmp3_bucket = tmp3;
+ const Register tmp2_hash = tmp2;
Label monitor_found;
- Register cache_addr = tmp2;
- // Load cache address
- addi(cache_addr, R16_thread, in_bytes(JavaThread::om_cache_oops_offset()));
+ // Save the mark, we might need it to extract the hash.
+ mr(tmp2_hash, mark);
- const int num_unrolled = 2;
+ // Look for the monitor in the om_cache.
+
+ ByteSize cache_offset = JavaThread::om_cache_oops_offset();
+ ByteSize monitor_offset = OMCache::oop_to_monitor_difference();
+ const int num_unrolled = OMCache::CAPACITY;
for (int i = 0; i < num_unrolled; i++) {
- ld(R0, 0, cache_addr);
+ ld(R0, in_bytes(cache_offset), R16_thread);
+ ld(monitor, in_bytes(cache_offset + monitor_offset), R16_thread);
cmpd(CR0, R0, obj);
beq(CR0, monitor_found);
- addi(cache_addr, cache_addr, in_bytes(OMCache::oop_to_oop_difference()));
+ cache_offset = cache_offset + OMCache::oop_to_oop_difference();
}
- Label loop;
+ // Look for the monitor in the table.
- // Search for obj in cache.
- bind(loop);
+ // Get the hash code.
+ srdi(tmp2_hash, tmp2_hash, markWord::hash_shift);
- // Check for match.
- ld(R0, 0, cache_addr);
- cmpd(CR0, R0, obj);
- beq(CR0, monitor_found);
+ // Get the table and calculate the bucket's address
+ int simm16_rest = load_const_optimized(tmp3, ObjectMonitorTable::current_table_address(), R0, true);
+ ld_ptr(tmp3, simm16_rest, tmp3);
+ ld(tmp1, in_bytes(ObjectMonitorTable::table_capacity_mask_offset()), tmp3);
+ andr(tmp2_hash, tmp2_hash, tmp1);
+ ld(tmp3_bucket, in_bytes(ObjectMonitorTable::table_buckets_offset()), tmp3);
- // Search until null encountered, guaranteed _null_sentinel at end.
- addi(cache_addr, cache_addr, in_bytes(OMCache::oop_to_oop_difference()));
- cmpdi(CR1, R0, 0);
- bne(CR1, loop);
- // Cache Miss, CR0.NE set from cmp above
- b(slow_path);
+ // Read the monitor from the bucket.
+ sldi(tmp2_hash, tmp2_hash, LogBytesPerWord);
+ ldx(monitor, tmp3_bucket, tmp2_hash);
+
+ // Check if the monitor in the bucket is special (empty, tombstone or removed).
+ cmpldi(CR0, monitor, ObjectMonitorTable::SpecialPointerValues::below_is_special);
+ blt(CR0, slow_path);
+
+ // Check if object matches.
+ ld(tmp3, in_bytes(ObjectMonitor::object_offset()), monitor);
+ BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
+ bs_asm->try_resolve_weak_handle(this, tmp3, tmp2, slow_path);
+ cmpd(CR0, tmp3, obj);
+ bne(CR0, slow_path);
bind(monitor_found);
- ld(monitor, in_bytes(OMCache::oop_to_monitor_difference()), cache_addr);
// Compute owner address.
addi(owner_addr, monitor, in_bytes(ObjectMonitor::owner_offset()));
@@ -3185,23 +3201,17 @@ Register MacroAssembler::encode_klass_not_null(Register dst, Register src) {
void MacroAssembler::store_klass(Register dst_oop, Register klass, Register ck) {
assert(!UseCompactObjectHeaders, "not with compact headers");
- if (UseCompressedClassPointers) {
- Register compressedKlass = encode_klass_not_null(ck, klass);
- stw(compressedKlass, oopDesc::klass_offset_in_bytes(), dst_oop);
- } else {
- std(klass, oopDesc::klass_offset_in_bytes(), dst_oop);
- }
+ Register compressedKlass = encode_klass_not_null(ck, klass);
+ stw(compressedKlass, oopDesc::klass_offset_in_bytes(), dst_oop);
}
void MacroAssembler::store_klass_gap(Register dst_oop, Register val) {
assert(!UseCompactObjectHeaders, "not with compact headers");
- if (UseCompressedClassPointers) {
- if (val == noreg) {
- val = R0;
- li(val, 0);
- }
- stw(val, oopDesc::klass_gap_offset_in_bytes(), dst_oop);
+ if (val == noreg) {
+ val = R0;
+ li(val, 0);
}
+ stw(val, oopDesc::klass_gap_offset_in_bytes(), dst_oop);
}
int MacroAssembler::instr_size_for_decode_klass_not_null() {
@@ -3210,17 +3220,13 @@ int MacroAssembler::instr_size_for_decode_klass_not_null() {
// Not yet computed?
if (computed_size == -1) {
- if (!UseCompressedClassPointers) {
- computed_size = 0;
- } else {
- // Determine by scratch emit.
- ResourceMark rm;
- int code_size = 8 * BytesPerInstWord;
- CodeBuffer cb("decode_klass_not_null scratch buffer", code_size, 0);
- MacroAssembler* a = new MacroAssembler(&cb);
- a->decode_klass_not_null(R11_scratch1);
- computed_size = a->offset();
- }
+ // Determine by scratch emit.
+ ResourceMark rm;
+ int code_size = 8 * BytesPerInstWord;
+ CodeBuffer cb("decode_klass_not_null scratch buffer", code_size, 0);
+ MacroAssembler* a = new MacroAssembler(&cb);
+ a->decode_klass_not_null(R11_scratch1);
+ computed_size = a->offset();
}
return computed_size;
@@ -3243,18 +3249,14 @@ void MacroAssembler::decode_klass_not_null(Register dst, Register src) {
void MacroAssembler::load_klass_no_decode(Register dst, Register src) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(dst, src);
- } else if (UseCompressedClassPointers) {
- lwz(dst, oopDesc::klass_offset_in_bytes(), src);
} else {
- ld(dst, oopDesc::klass_offset_in_bytes(), src);
+ lwz(dst, oopDesc::klass_offset_in_bytes(), src);
}
}
void MacroAssembler::load_klass(Register dst, Register src) {
load_klass_no_decode(dst, src);
- if (UseCompressedClassPointers) { // also true for UseCompactObjectHeaders
- decode_klass_not_null(dst);
- }
+ decode_klass_not_null(dst);
}
// Loads the obj's Klass* into dst.
@@ -3270,18 +3272,13 @@ void MacroAssembler::load_narrow_klass_compact(Register dst, Register src) {
void MacroAssembler::cmp_klass(ConditionRegister dst, Register obj, Register klass, Register tmp, Register tmp2) {
assert_different_registers(obj, klass, tmp);
- if (UseCompressedClassPointers) {
- if (UseCompactObjectHeaders) {
- load_narrow_klass_compact(tmp, obj);
- } else {
- lwz(tmp, oopDesc::klass_offset_in_bytes(), obj);
- }
- Register encoded_klass = encode_klass_not_null(tmp2, klass);
- cmpw(dst, tmp, encoded_klass);
+ if (UseCompactObjectHeaders) {
+ load_narrow_klass_compact(tmp, obj);
} else {
- ld(tmp, oopDesc::klass_offset_in_bytes(), obj);
- cmpd(dst, tmp, klass);
+ lwz(tmp, oopDesc::klass_offset_in_bytes(), obj);
}
+ Register encoded_klass = encode_klass_not_null(tmp2, klass);
+ cmpw(dst, tmp, encoded_klass);
}
void MacroAssembler::cmp_klasses_from_objects(ConditionRegister dst, Register obj1, Register obj2, Register tmp1, Register tmp2) {
@@ -3289,14 +3286,10 @@ void MacroAssembler::cmp_klasses_from_objects(ConditionRegister dst, Register ob
load_narrow_klass_compact(tmp1, obj1);
load_narrow_klass_compact(tmp2, obj2);
cmpw(dst, tmp1, tmp2);
- } else if (UseCompressedClassPointers) {
+ } else {
lwz(tmp1, oopDesc::klass_offset_in_bytes(), obj1);
lwz(tmp2, oopDesc::klass_offset_in_bytes(), obj2);
cmpw(dst, tmp1, tmp2);
- } else {
- ld(tmp1, oopDesc::klass_offset_in_bytes(), obj1);
- ld(tmp2, oopDesc::klass_offset_in_bytes(), obj2);
- cmpd(dst, tmp1, tmp2);
}
}
diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp
index 875602cae58..4be62098bdf 100644
--- a/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2002, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
+ * Copyright (c) 2002, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -70,14 +70,6 @@ class MacroAssembler: public Assembler {
// Move register if destination register and target register are different
inline void mr_if_needed(Register rd, Register rs, bool allow_invalid = false);
inline void fmr_if_needed(FloatRegister rd, FloatRegister rs);
- // This is dedicated for emitting scheduled mach nodes. For better
- // readability of the ad file I put it here.
- // Endgroups are not needed if
- // - the scheduler is off
- // - the scheduler found that there is a natural group end, in that
- // case it reduced the size of the instruction used in the test
- // yielding 'needed'.
- inline void endgroup_if_needed(bool needed);
// Memory barriers.
inline void membar(int bits);
diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.inline.hpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.inline.hpp
index 2b19d84c69c..cdeb8527bea 100644
--- a/src/hotspot/cpu/ppc/macroAssembler_ppc.inline.hpp
+++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.inline.hpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2002, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
+ * Copyright (c) 2002, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -72,11 +72,6 @@ inline void MacroAssembler::mr_if_needed(Register rd, Register rs, bool allow_no
inline void MacroAssembler::fmr_if_needed(FloatRegister rd, FloatRegister rs) {
if (rs != rd) fmr(rd, rs);
}
-inline void MacroAssembler::endgroup_if_needed(bool needed) {
- if (needed) {
- endgroup();
- }
-}
inline void MacroAssembler::membar(int bits) {
// Comment: Usage of elemental_membar(bits) is not recommended for Power 8.
@@ -240,13 +235,13 @@ inline bool MacroAssembler::is_bc_far_variant3_at(address instruction_addr) {
// Variant 3, far cond branch to the next instruction, already patched to nops:
//
// nop
- // endgroup
+ // nop
// SKIP/DEST:
//
const int instruction_1 = *(int*)(instruction_addr);
const int instruction_2 = *(int*)(instruction_addr + 4);
return is_nop(instruction_1) &&
- is_endgroup(instruction_2);
+ is_nop(instruction_2);
}
// set dst to -1, 0, +1 as follows: if CR0bi is "greater than", dst is set to 1,
diff --git a/src/hotspot/cpu/ppc/matcher_ppc.hpp b/src/hotspot/cpu/ppc/matcher_ppc.hpp
index b50de6323de..cbe882648b8 100644
--- a/src/hotspot/cpu/ppc/matcher_ppc.hpp
+++ b/src/hotspot/cpu/ppc/matcher_ppc.hpp
@@ -1,5 +1,6 @@
/*
- * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -86,7 +87,6 @@
static bool narrow_klass_use_complex_address() {
NOT_LP64(ShouldNotCallThis());
- assert(UseCompressedClassPointers, "only for compressed klass code");
// TODO: PPC port if (MatchDecodeNodes) return true;
return false;
}
diff --git a/src/hotspot/cpu/ppc/methodHandles_ppc.cpp b/src/hotspot/cpu/ppc/methodHandles_ppc.cpp
index 45537e0ea96..ae94a9618b5 100644
--- a/src/hotspot/cpu/ppc/methodHandles_ppc.cpp
+++ b/src/hotspot/cpu/ppc/methodHandles_ppc.cpp
@@ -104,14 +104,13 @@ void MethodHandles::verify_ref_kind(MacroAssembler* _masm, int ref_kind, Registe
__ andi(temp, temp, java_lang_invoke_MemberName::MN_REFERENCE_KIND_MASK);
__ cmpwi(CR1, temp, ref_kind);
__ beq(CR1, L);
- { char* buf = NEW_C_HEAP_ARRAY(char, 100, mtInternal);
- jio_snprintf(buf, 100, "verify_ref_kind expected %x", ref_kind);
- if (ref_kind == JVM_REF_invokeVirtual ||
- ref_kind == JVM_REF_invokeSpecial)
- // could do this for all ref_kinds, but would explode assembly code size
- trace_method_handle(_masm, buf);
- __ stop(buf);
+ const char* msg = ref_kind_to_verify_msg(ref_kind);
+ if (ref_kind == JVM_REF_invokeVirtual ||
+ ref_kind == JVM_REF_invokeSpecial) {
+ // could do this for all ref_kinds, but would explode assembly code size
+ trace_method_handle(_masm, msg);
}
+ __ stop(msg);
BLOCK_COMMENT("} verify_ref_kind");
__ BIND(L);
}
diff --git a/src/hotspot/cpu/ppc/ppc.ad b/src/hotspot/cpu/ppc/ppc.ad
index d926fabd353..f3d33b4305d 100644
--- a/src/hotspot/cpu/ppc/ppc.ad
+++ b/src/hotspot/cpu/ppc/ppc.ad
@@ -1,6 +1,6 @@
//
-// Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved.
-// Copyright (c) 2012, 2025 SAP SE. All rights reserved.
+// Copyright (c) 2011, 2026, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2012, 2026 SAP SE. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
@@ -2234,6 +2234,12 @@ bool Matcher::match_rule_supported(int opcode) {
case Op_FmaVD:
return (SuperwordUseVSX && UseFMA);
+ case Op_MinF:
+ case Op_MaxF:
+ case Op_MinD:
+ case Op_MaxD:
+ return (PowerArchitecturePPC64 >= 9);
+
case Op_Digit:
return vmIntrinsics::is_intrinsic_available(vmIntrinsics::_isDigit);
case Op_LowerCase:
@@ -2406,10 +2412,8 @@ bool Matcher::is_generic_vector(MachOper* opnd) {
return false;
}
-// Return whether or not this register is ever used as an argument. This
-// function is used on startup to build the trampoline stubs in generateOptoStub.
-// Registers not mentioned will be killed by the VM call in the trampoline, and
-// arguments in those registers not be available to the callee.
+#ifdef ASSERT
+// Return whether or not this register is ever used as an argument.
bool Matcher::can_be_java_arg(int reg) {
// We must include the virtual halves in order to get STDs and LDs
// instead of STWs and LWs in the trampoline stubs.
@@ -2441,10 +2445,7 @@ bool Matcher::can_be_java_arg(int reg) {
return false;
}
-
-bool Matcher::is_spillable_arg(int reg) {
- return can_be_java_arg(reg);
-}
+#endif
uint Matcher::int_pressure_limit()
{
@@ -2456,10 +2457,6 @@ uint Matcher::float_pressure_limit()
return (FLOATPRESSURE == -1) ? 28 : FLOATPRESSURE;
}
-bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
- return false;
-}
-
// Register for DIVI projection of divmodI.
const RegMask& Matcher::divI_proj_mask() {
ShouldNotReachHere();
@@ -3709,13 +3706,6 @@ frame %{
// Compiled code's Frame Pointer.
frame_pointer(R1); // R1_SP
- // Interpreter stores its frame pointer in a register which is
- // stored to the stack by I2CAdaptors. I2CAdaptors convert from
- // interpreted java to compiled java.
- //
- // R14_state holds pointer to caller's cInterpreter.
- interpreter_frame_pointer(R14); // R14_state
-
stack_alignment(frame::alignment_in_bytes);
// Number of outgoing stack slots killed above the
@@ -6333,36 +6323,8 @@ instruct loadConD_Ex(regD dst, immD src) %{
// Prefetch instructions.
// Must be safe to execute with invalid address (cannot fault).
-// Special prefetch versions which use the dcbz instruction.
-instruct prefetch_alloc_zero(indirectMemory mem, iRegLsrc src) %{
- match(PrefetchAllocation (AddP mem src));
- predicate(AllocatePrefetchStyle == 3);
- ins_cost(MEMORY_REF_COST);
-
- format %{ "PREFETCH $mem, 2, $src \t// Prefetch write-many with zero" %}
- size(4);
- ins_encode %{
- __ dcbz($src$$Register, $mem$$base$$Register);
- %}
- ins_pipe(pipe_class_memory);
-%}
-
-instruct prefetch_alloc_zero_no_offset(indirectMemory mem) %{
- match(PrefetchAllocation mem);
- predicate(AllocatePrefetchStyle == 3);
- ins_cost(MEMORY_REF_COST);
-
- format %{ "PREFETCH $mem, 2 \t// Prefetch write-many with zero" %}
- size(4);
- ins_encode %{
- __ dcbz($mem$$base$$Register);
- %}
- ins_pipe(pipe_class_memory);
-%}
-
instruct prefetch_alloc(indirectMemory mem, iRegLsrc src) %{
match(PrefetchAllocation (AddP mem src));
- predicate(AllocatePrefetchStyle != 3);
ins_cost(MEMORY_REF_COST);
format %{ "PREFETCH $mem, 2, $src \t// Prefetch write-many" %}
@@ -6375,7 +6337,6 @@ instruct prefetch_alloc(indirectMemory mem, iRegLsrc src) %{
instruct prefetch_alloc_no_offset(indirectMemory mem) %{
match(PrefetchAllocation mem);
- predicate(AllocatePrefetchStyle != 3);
ins_cost(MEMORY_REF_COST);
format %{ "PREFETCH $mem, 2 \t// Prefetch write-many" %}
@@ -7169,6 +7130,18 @@ instruct membar_release_lock() %{
ins_pipe(pipe_class_default);
%}
+instruct membar_storeload() %{
+ match(MemBarStoreLoad);
+ ins_cost(4*MEMORY_REF_COST);
+
+ format %{ "MEMBAR-store-load" %}
+ size(4);
+ ins_encode %{
+ __ fence();
+ %}
+ ins_pipe(pipe_class_default);
+%}
+
instruct membar_volatile() %{
match(MemBarVolatile);
ins_cost(4*MEMORY_REF_COST);
@@ -7211,6 +7184,18 @@ instruct membar_volatile() %{
// ins_pipe(pipe_class_default);
//%}
+instruct membar_full() %{
+ match(MemBarFull);
+ ins_cost(4*MEMORY_REF_COST);
+
+ format %{ "MEMBAR-full" %}
+ size(4);
+ ins_encode %{
+ __ fence();
+ %}
+ ins_pipe(pipe_class_default);
+%}
+
instruct membar_CPUOrder() %{
match(MemBarCPUOrder);
ins_cost(0);
@@ -10318,7 +10303,7 @@ instruct cmovI_bso_stackSlotL(iRegIdst dst, flagsRegSrc crx, stackSlotL src) %{
ins_variable_size_depending_on_alignment(true);
- format %{ "cmovI $crx, $dst, $src" %}
+ format %{ "CMOVI $crx, $dst, $src" %}
size(8);
ins_encode( enc_cmove_bso_stackSlotL(dst, crx, src) );
ins_pipe(pipe_class_default);
@@ -10331,7 +10316,7 @@ instruct cmovI_bso_reg(iRegIdst dst, flagsRegSrc crx, regD src) %{
ins_variable_size_depending_on_alignment(true);
- format %{ "cmovI $crx, $dst, $src" %}
+ format %{ "CMOVI $crx, $dst, $src" %}
size(8);
ins_encode( enc_cmove_bso_reg(dst, crx, src) );
ins_pipe(pipe_class_default);
@@ -10343,7 +10328,7 @@ instruct cmovI_bso_reg_conLvalue0_Ex(iRegIdst dst, flagsRegSrc crx, regD src) %{
effect(DEF dst, USE crx, USE src);
predicate(false);
- format %{ "CmovI $dst, $crx, $src \t// postalloc expanded" %}
+ format %{ "CMOVI $dst, $crx, $src \t// postalloc expanded" %}
postalloc_expand %{
//
// replaces
@@ -10493,7 +10478,7 @@ instruct cmovL_bso_stackSlotL(iRegLdst dst, flagsRegSrc crx, stackSlotL src) %{
ins_variable_size_depending_on_alignment(true);
- format %{ "cmovL $crx, $dst, $src" %}
+ format %{ "CMOVL $crx, $dst, $src" %}
size(8);
ins_encode( enc_cmove_bso_stackSlotL(dst, crx, src) );
ins_pipe(pipe_class_default);
@@ -10506,7 +10491,7 @@ instruct cmovL_bso_reg(iRegLdst dst, flagsRegSrc crx, regD src) %{
ins_variable_size_depending_on_alignment(true);
- format %{ "cmovL $crx, $dst, $src" %}
+ format %{ "CMOVL $crx, $dst, $src" %}
size(8);
ins_encode( enc_cmove_bso_reg(dst, crx, src) );
ins_pipe(pipe_class_default);
@@ -10518,7 +10503,7 @@ instruct cmovL_bso_reg_conLvalue0_Ex(iRegLdst dst, flagsRegSrc crx, regD src) %{
effect(DEF dst, USE crx, USE src);
predicate(false);
- format %{ "CmovL $dst, $crx, $src \t// postalloc expanded" %}
+ format %{ "CMOVL $dst, $crx, $src \t// postalloc expanded" %}
postalloc_expand %{
//
// replaces
@@ -10719,9 +10704,9 @@ instruct convF2HF_reg_reg(iRegIdst dst, regF src, regF tmp) %{
effect(TEMP tmp);
ins_cost(3 * DEFAULT_COST);
size(12);
- format %{ "xscvdphp $tmp, $src\t# convert to half precision\n\t"
- "mffprd $dst, $tmp\t# move result from $tmp to $dst\n\t"
- "extsh $dst, $dst\t# make it a proper short"
+ format %{ "XSCVDPHP $tmp, $src\t# convert to half precision\n\t"
+ "MFFPRD $dst, $tmp\t# move result from $tmp to $dst\n\t"
+ "EXTSH $dst, $dst\t# make it a proper short"
%}
ins_encode %{
__ f2hf($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
@@ -10733,8 +10718,8 @@ instruct convHF2F_reg_reg(regF dst, iRegIsrc src) %{
match(Set dst (ConvHF2F src));
ins_cost(2 * DEFAULT_COST);
size(8);
- format %{ "mtfprd $dst, $src\t# move source from $src to $dst\n\t"
- "xscvhpdp $dst, $dst\t# convert from half precision"
+ format %{ "MTFPRD $dst, $src\t# move source from $src to $dst\n\t"
+ "XSCVHPDP $dst, $dst\t# convert from half precision"
%}
ins_encode %{
__ hf2f($dst$$FloatRegister, $src$$Register);
@@ -11132,7 +11117,7 @@ instruct cmov_bns_less(flagsReg crx) %{
ins_variable_size_depending_on_alignment(true);
- format %{ "cmov $crx" %}
+ format %{ "CMOV $crx" %}
size(12);
ins_encode %{
Label done;
@@ -11160,7 +11145,7 @@ instruct cmpF_reg_reg_Ex(flagsReg crx, regF src1, regF src2) %{
match(Set crx (CmpF src1 src2));
ins_cost(DEFAULT_COST+BRANCH_COST);
- format %{ "CmpF $crx, $src1, $src2 \t// postalloc expanded" %}
+ format %{ "CMPF $crx, $src1, $src2 \t// postalloc expanded" %}
postalloc_expand %{
//
// replaces
@@ -12313,6 +12298,58 @@ instruct maxI_reg_reg_isel(iRegIdst dst, iRegIsrc src1, iRegIsrc src2, flagsRegC
ins_pipe(pipe_class_default);
%}
+instruct minF(regF dst, regF src1, regF src2) %{
+ match(Set dst (MinF src1 src2));
+ predicate(PowerArchitecturePPC64 >= 9);
+ ins_cost(DEFAULT_COST);
+
+ format %{ "XSMINJDP $dst, $src1, $src2\t// MinF" %}
+ size(4);
+ ins_encode %{
+ __ xsminjdp($dst$$FloatRegister->to_vsr(), $src1$$FloatRegister->to_vsr(), $src2$$FloatRegister->to_vsr());
+ %}
+ ins_pipe(pipe_class_default);
+%}
+
+instruct minD(regD dst, regD src1, regD src2) %{
+ match(Set dst (MinD src1 src2));
+ predicate(PowerArchitecturePPC64 >= 9);
+ ins_cost(DEFAULT_COST);
+
+ format %{ "XSMINJDP $dst, $src1, $src2\t// MinD" %}
+ size(4);
+ ins_encode %{
+ __ xsminjdp($dst$$FloatRegister->to_vsr(), $src1$$FloatRegister->to_vsr(), $src2$$FloatRegister->to_vsr());
+ %}
+ ins_pipe(pipe_class_default);
+%}
+
+instruct maxF(regF dst, regF src1, regF src2) %{
+ match(Set dst (MaxF src1 src2));
+ predicate(PowerArchitecturePPC64 >= 9);
+ ins_cost(DEFAULT_COST);
+
+ format %{ "XSMAXJDP $dst, $src1, $src2\t// MaxF" %}
+ size(4);
+ ins_encode %{
+ __ xsmaxjdp($dst$$FloatRegister->to_vsr(), $src1$$FloatRegister->to_vsr(), $src2$$FloatRegister->to_vsr());
+ %}
+ ins_pipe(pipe_class_default);
+%}
+
+instruct maxD(regD dst, regD src1, regD src2) %{
+ match(Set dst (MaxD src1 src2));
+ predicate(PowerArchitecturePPC64 >= 9);
+ ins_cost(DEFAULT_COST);
+
+ format %{ "XSMAXJDP $dst, $src1, $src2\t// MaxD" %}
+ size(4);
+ ins_encode %{
+ __ xsmaxjdp($dst$$FloatRegister->to_vsr(), $src1$$FloatRegister->to_vsr(), $src2$$FloatRegister->to_vsr());
+ %}
+ ins_pipe(pipe_class_default);
+%}
+
//---------- Population Count Instructions ------------------------------------
instruct popCountI(iRegIdst dst, iRegIsrc src) %{
@@ -13835,7 +13872,7 @@ instruct vfma2D_neg2(vecX dst, vecX src1, vecX src2) %{
instruct overflowAddL_reg_reg(flagsRegCR0 cr0, iRegLsrc op1, iRegLsrc op2) %{
match(Set cr0 (OverflowAddL op1 op2));
- format %{ "add_ $op1, $op2\t# overflow check long" %}
+ format %{ "ADD_ $op1, $op2\t# overflow check long" %}
size(12);
ins_encode %{
__ li(R0, 0);
@@ -13848,7 +13885,7 @@ instruct overflowAddL_reg_reg(flagsRegCR0 cr0, iRegLsrc op1, iRegLsrc op2) %{
instruct overflowSubL_reg_reg(flagsRegCR0 cr0, iRegLsrc op1, iRegLsrc op2) %{
match(Set cr0 (OverflowSubL op1 op2));
- format %{ "subfo_ R0, $op2, $op1\t# overflow check long" %}
+ format %{ "SUBFO_ R0, $op2, $op1\t# overflow check long" %}
size(12);
ins_encode %{
__ li(R0, 0);
@@ -13861,7 +13898,7 @@ instruct overflowSubL_reg_reg(flagsRegCR0 cr0, iRegLsrc op1, iRegLsrc op2) %{
instruct overflowNegL_reg(flagsRegCR0 cr0, immL_0 zero, iRegLsrc op2) %{
match(Set cr0 (OverflowSubL zero op2));
- format %{ "nego_ R0, $op2\t# overflow check long" %}
+ format %{ "NEGO_ R0, $op2\t# overflow check long" %}
size(12);
ins_encode %{
__ li(R0, 0);
@@ -13874,7 +13911,7 @@ instruct overflowNegL_reg(flagsRegCR0 cr0, immL_0 zero, iRegLsrc op2) %{
instruct overflowMulL_reg_reg(flagsRegCR0 cr0, iRegLsrc op1, iRegLsrc op2) %{
match(Set cr0 (OverflowMulL op1 op2));
- format %{ "mulldo_ R0, $op1, $op2\t# overflow check long" %}
+ format %{ "MULLDO_ R0, $op1, $op2\t# overflow check long" %}
size(12);
ins_encode %{
__ li(R0, 0);
@@ -14251,7 +14288,7 @@ instruct ForwardExceptionjmp()
match(ForwardException);
ins_cost(CALL_COST);
- format %{ "Jmp forward_exception_stub" %}
+ format %{ "JMP forward_exception_stub" %}
ins_encode %{
__ set_inst_mark();
__ b64_patchable(StubRoutines::forward_exception_entry(), relocInfo::runtime_call_type);
@@ -14279,7 +14316,7 @@ instruct RethrowException() %{
match(Rethrow);
ins_cost(CALL_COST);
- format %{ "Jmp rethrow_stub" %}
+ format %{ "JMP rethrow_stub" %}
ins_encode %{
__ set_inst_mark();
__ b64_patchable((address)OptoRuntime::rethrow_stub(), relocInfo::runtime_call_type);
@@ -14321,20 +14358,6 @@ instruct tlsLoadP(threadRegP dst) %{
//---Some PPC specific nodes---------------------------------------------------
-// Stop a group.
-instruct endGroup() %{
- ins_cost(0);
-
- ins_is_nop(true);
-
- format %{ "End Bundle (ori r1, r1, 0)" %}
- size(4);
- ins_encode %{
- __ endgroup();
- %}
- ins_pipe(pipe_class_default);
-%}
-
// Nop instructions
instruct fxNop() %{
diff --git a/src/hotspot/cpu/ppc/registerMap_ppc.cpp b/src/hotspot/cpu/ppc/registerMap_ppc.cpp
new file mode 100644
index 00000000000..2e7f8af89d3
--- /dev/null
+++ b/src/hotspot/cpu/ppc/registerMap_ppc.cpp
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2026 SAP SE. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "runtime/registerMap.hpp"
+
+address RegisterMap::pd_location(VMReg base_reg, int slot_idx) const {
+ if (base_reg->is_VectorRegister()) {
+ // Not all physical slots belonging to a VectorRegister have corresponding
+ // valid VMReg locations in the RegisterMap.
+ // (See RegisterSaver::push_frame_reg_args_and_save_live_registers.)
+ // However, the slots are always saved to the stack in a contiguous region
+ // of memory so we can calculate the address of the upper slots by
+ // offsetting from the base address.
+ assert(base_reg->is_concrete(), "must pass base reg");
+ address base_location = location(base_reg, nullptr);
+ if (base_location != nullptr) {
+ intptr_t offset_in_bytes = slot_idx * VMRegImpl::stack_slot_size;
+ return base_location + offset_in_bytes;
+ } else {
+ return nullptr;
+ }
+ } else {
+ return location(base_reg->next(slot_idx), nullptr);
+ }
+}
diff --git a/src/hotspot/cpu/ppc/registerMap_ppc.hpp b/src/hotspot/cpu/ppc/registerMap_ppc.hpp
index 01eb642107c..607c712d10f 100644
--- a/src/hotspot/cpu/ppc/registerMap_ppc.hpp
+++ b/src/hotspot/cpu/ppc/registerMap_ppc.hpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2013 SAP SE. All rights reserved.
+ * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -35,9 +35,7 @@
// Since there is none, we just return null.
address pd_location(VMReg reg) const { return nullptr; }
- address pd_location(VMReg base_reg, int slot_idx) const {
- return location(base_reg->next(slot_idx), nullptr);
- }
+ address pd_location(VMReg base_reg, int slot_idx) const;
// no PD state to clear or copy:
void pd_clear() {}
diff --git a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
index 5260ed978ff..53644210415 100644
--- a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
+++ b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
@@ -102,7 +102,7 @@ class RegisterSaver {
// During deoptimization only the result registers need to be restored
// all the other values have already been extracted.
- static void restore_result_registers(MacroAssembler* masm, int frame_size_in_bytes);
+ static void restore_result_registers(MacroAssembler* masm, int frame_size_in_bytes, bool save_vectors);
// Constants and data structures:
@@ -349,7 +349,7 @@ OopMap* RegisterSaver::push_frame_reg_args_and_save_live_registers(MacroAssemble
}
// Note that generate_oop_map in the following loop is only used for the
- // polling_page_vectors_safepoint_handler_blob.
+ // polling_page_vectors_safepoint_handler_blob and the deopt_blob.
// The order in which the vector contents are stored depends on Endianess and
// the utilized instructions (PowerArchitecturePPC64).
assert(is_aligned(offset, StackAlignmentInBytes), "should be");
@@ -361,6 +361,7 @@ OopMap* RegisterSaver::push_frame_reg_args_and_save_live_registers(MacroAssemble
__ stxvp(as_VectorRegister(reg_num).to_vsr(), offset, R1_SP);
// Note: The contents were read in the same order (see loadV16_Power9 node in ppc.ad).
+ // RegisterMap::pd_location only uses the first VMReg for each VectorRegister.
if (generate_oop_map) {
map->set_callee_saved(VMRegImpl::stack2reg(offset >> 2),
RegisterSaver_LiveVecRegs[i LITTLE_ENDIAN_ONLY(+1) ].vmreg);
@@ -380,6 +381,7 @@ OopMap* RegisterSaver::push_frame_reg_args_and_save_live_registers(MacroAssemble
__ stxvd2x(as_VectorRegister(reg_num)->to_vsr(), R31, R1_SP);
}
// Note: The contents were read in the same order (see loadV16_Power8 / loadV16_Power9 node in ppc.ad).
+ // RegisterMap::pd_location only uses the first VMReg for each VectorRegister.
if (generate_oop_map) {
VMReg vsr = RegisterSaver_LiveVecRegs[i].vmreg;
map->set_callee_saved(VMRegImpl::stack2reg(offset >> 2), vsr);
@@ -566,10 +568,14 @@ void RegisterSaver::restore_argument_registers_and_pop_frame(MacroAssembler*masm
}
// Restore the registers that might be holding a result.
-void RegisterSaver::restore_result_registers(MacroAssembler* masm, int frame_size_in_bytes) {
+void RegisterSaver::restore_result_registers(MacroAssembler* masm, int frame_size_in_bytes, bool save_vectors) {
const int regstosave_num = sizeof(RegisterSaver_LiveRegs) /
sizeof(RegisterSaver::LiveRegType);
- const int register_save_size = regstosave_num * reg_size; // VS registers not relevant here.
+ const int vecregstosave_num = save_vectors ? (sizeof(RegisterSaver_LiveVecRegs) /
+ sizeof(RegisterSaver::LiveRegType))
+ : 0;
+ const int register_save_size = regstosave_num * reg_size + vecregstosave_num * vec_reg_size;
+
const int register_save_offset = frame_size_in_bytes - register_save_size;
// restore all result registers (ints and floats)
@@ -598,7 +604,7 @@ void RegisterSaver::restore_result_registers(MacroAssembler* masm, int frame_siz
offset += reg_size;
}
- assert(offset == frame_size_in_bytes, "consistency check");
+ assert(offset == frame_size_in_bytes - (save_vectors ? vecregstosave_num * vec_reg_size : 0), "consistency check");
}
// Is vector's size (in bytes) bigger than a size saved by default?
@@ -2909,7 +2915,8 @@ void SharedRuntime::generate_deopt_blob() {
map = RegisterSaver::push_frame_reg_args_and_save_live_registers(masm,
&first_frame_size_in_bytes,
/*generate_oop_map=*/ true,
- RegisterSaver::return_pc_is_lr);
+ RegisterSaver::return_pc_is_lr,
+ /*save_vectors*/ SuperwordUseVSX);
assert(map != nullptr, "OopMap must have been created");
__ li(exec_mode_reg, Deoptimization::Unpack_deopt);
@@ -2943,7 +2950,8 @@ void SharedRuntime::generate_deopt_blob() {
RegisterSaver::push_frame_reg_args_and_save_live_registers(masm,
&first_frame_size_in_bytes,
/*generate_oop_map=*/ false,
- RegisterSaver::return_pc_is_pre_saved);
+ RegisterSaver::return_pc_is_pre_saved,
+ /*save_vectors*/ SuperwordUseVSX);
// Deopt during an exception. Save exec mode for unpack_frames.
__ li(exec_mode_reg, Deoptimization::Unpack_exception);
@@ -2958,7 +2966,8 @@ void SharedRuntime::generate_deopt_blob() {
RegisterSaver::push_frame_reg_args_and_save_live_registers(masm,
&first_frame_size_in_bytes,
/*generate_oop_map=*/ false,
- RegisterSaver::return_pc_is_pre_saved);
+ RegisterSaver::return_pc_is_pre_saved,
+ /*save_vectors*/ SuperwordUseVSX);
__ li(exec_mode_reg, Deoptimization::Unpack_reexecute);
#endif
@@ -2984,7 +2993,7 @@ void SharedRuntime::generate_deopt_blob() {
// Restore only the result registers that have been saved
// by save_volatile_registers(...).
- RegisterSaver::restore_result_registers(masm, first_frame_size_in_bytes);
+ RegisterSaver::restore_result_registers(masm, first_frame_size_in_bytes, /*save_vectors*/ SuperwordUseVSX);
// reload the exec mode from the UnrollBlock (it might have changed)
__ lwz(exec_mode_reg, in_bytes(Deoptimization::UnrollBlock::unpack_kind_offset()), unroll_block_reg);
diff --git a/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp b/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp
index 8a3af748fa1..37f780535b4 100644
--- a/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp
+++ b/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp
@@ -3489,7 +3489,7 @@ void TemplateTable::invokevirtual(int byte_no) {
// Get receiver klass.
__ load_klass_check_null_throw(Rrecv_klass, Rrecv, R11_scratch1);
__ verify_klass_ptr(Rrecv_klass);
- __ profile_virtual_call(Rrecv_klass, R11_scratch1, R12_scratch2, false);
+ __ profile_virtual_call(Rrecv_klass, R11_scratch1, R12_scratch2);
generate_vtable_call(Rrecv_klass, Rvtableindex_or_method, Rret_addr, R11_scratch1);
}
@@ -3596,7 +3596,7 @@ void TemplateTable::invokeinterface_object_method(Register Rrecv_klass,
// Non-final callc case.
__ bind(LnotFinal);
__ lhz(Rindex, in_bytes(ResolvedMethodEntry::table_index_offset()), Rcache);
- __ profile_virtual_call(Rrecv_klass, Rtemp1, Rscratch, false);
+ __ profile_virtual_call(Rrecv_klass, Rtemp1, Rscratch);
generate_vtable_call(Rrecv_klass, Rindex, Rret, Rscratch);
}
@@ -3664,7 +3664,7 @@ void TemplateTable::invokeinterface(int byte_no) {
__ lookup_interface_method(Rrecv_klass, Rinterface_klass, noreg, noreg, Rscratch1, Rscratch2,
L_no_such_interface, /*return_method=*/false);
- __ profile_virtual_call(Rrecv_klass, Rscratch1, Rscratch2, false);
+ __ profile_virtual_call(Rrecv_klass, Rscratch1, Rscratch2);
// Find entry point to call.
diff --git a/src/hotspot/cpu/ppc/vm_version_ppc.cpp b/src/hotspot/cpu/ppc/vm_version_ppc.cpp
index 75feb389298..0b69ef7d25a 100644
--- a/src/hotspot/cpu/ppc/vm_version_ppc.cpp
+++ b/src/hotspot/cpu/ppc/vm_version_ppc.cpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -25,6 +25,7 @@
#include "asm/assembler.inline.hpp"
#include "asm/macroAssembler.inline.hpp"
+#include "compiler/compilerDefinitions.inline.hpp"
#include "compiler/disassembler.hpp"
#include "jvm.h"
#include "memory/resourceArea.hpp"
@@ -105,7 +106,7 @@ void VM_Version::initialize() {
if (PowerArchitecturePPC64 >= 9) {
// Performance is good since Power9.
- if (FLAG_IS_DEFAULT(SuperwordUseVSX)) {
+ if (FLAG_IS_DEFAULT(SuperwordUseVSX) && CompilerConfig::is_c2_enabled()) {
FLAG_SET_ERGO(SuperwordUseVSX, true);
}
}
@@ -475,19 +476,12 @@ void VM_Version::print_features() {
void VM_Version::determine_features() {
#if defined(ABI_ELFv2)
- // 1 InstWord per call for the blr instruction.
- const int code_size = (num_features+1+2*1)*BytesPerInstWord;
+ const int code_size = (num_features + 1 /*blr*/) * BytesPerInstWord;
#else
- // 7 InstWords for each call (function descriptor + blr instruction).
- const int code_size = (num_features+1+2*7)*BytesPerInstWord;
+ const int code_size = (num_features + 1 /*blr*/ + 6 /* fd */) * BytesPerInstWord;
#endif
int features = 0;
- // create test area
- enum { BUFFER_SIZE = 2*4*K }; // Needs to be >=2* max cache line size (cache line size can't exceed min page size).
- char test_area[BUFFER_SIZE];
- char *mid_of_test_area = &test_area[BUFFER_SIZE>>1];
-
// Allocate space for the code.
ResourceMark rm;
CodeBuffer cb("detect_cpu_features", code_size, 0);
@@ -497,20 +491,13 @@ void VM_Version::determine_features() {
_features = VM_Version::all_features_m;
// Emit code.
- void (*test)(address addr, uint64_t offset)=(void(*)(address addr, uint64_t offset))(void *)a->function_entry();
+ void (*test)() = (void(*)())(void *)a->function_entry();
uint32_t *code = (uint32_t *)a->pc();
- // Keep R3_ARG1 unmodified, it contains &field (see below).
- // Keep R4_ARG2 unmodified, it contains offset = 0 (see below).
a->mfdscr(R0);
a->darn(R7);
a->brw(R5, R6);
a->blr();
- // Emit function to set one cache line to zero. Emit function descriptor and get pointer to it.
- void (*zero_cacheline_func_ptr)(char*) = (void(*)(char*))(void *)a->function_entry();
- a->dcbz(R3_ARG1); // R3_ARG1 = addr
- a->blr();
-
uint32_t *code_end = (uint32_t *)a->pc();
a->flush();
_features = VM_Version::unknown_m;
@@ -522,18 +509,9 @@ void VM_Version::determine_features() {
Disassembler::decode((u_char*)code, (u_char*)code_end, tty);
}
- // Measure cache line size.
- memset(test_area, 0xFF, BUFFER_SIZE); // Fill test area with 0xFF.
- (*zero_cacheline_func_ptr)(mid_of_test_area); // Call function which executes dcbz to the middle.
- int count = 0; // count zeroed bytes
- for (int i = 0; i < BUFFER_SIZE; i++) if (test_area[i] == 0) count++;
- guarantee(is_power_of_2(count), "cache line size needs to be a power of 2");
- _L1_data_cache_line_size = count;
-
// Execute code. Illegal instructions will be replaced by 0 in the signal handler.
VM_Version::_is_determine_features_test_running = true;
- // We must align the first argument to 16 bytes because of the lqarx check.
- (*test)(align_up((address)mid_of_test_area, 16), 0);
+ (*test)();
VM_Version::_is_determine_features_test_running = false;
// determine which instructions are legal.
@@ -550,6 +528,10 @@ void VM_Version::determine_features() {
}
_features = features;
+
+ _L1_data_cache_line_size = VM_Version::get_dcache_line_size();
+ assert(_L1_data_cache_line_size >= DEFAULT_CACHE_LINE_SIZE,
+ "processors with smaller cache line size are no longer supported");
}
// Power 8: Configure Data Stream Control Register.
diff --git a/src/hotspot/cpu/ppc/vm_version_ppc.hpp b/src/hotspot/cpu/ppc/vm_version_ppc.hpp
index 11dce83bed0..0f4eb3593a3 100644
--- a/src/hotspot/cpu/ppc/vm_version_ppc.hpp
+++ b/src/hotspot/cpu/ppc/vm_version_ppc.hpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -81,6 +81,9 @@ public:
static uint64_t _dscr_val;
static void initialize_cpu_information(void);
+
+ static int get_dcache_line_size();
+ static int get_icache_line_size();
};
#endif // CPU_PPC_VM_VERSION_PPC_HPP
diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.cpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.cpp
index 819d6c05654..8aced227a06 100644
--- a/src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -196,12 +196,9 @@ void LIR_Assembler::arraycopy_type_check(Register src, Register src_pos, Registe
if (UseCompactObjectHeaders) {
__ load_narrow_klass_compact(tmp, src);
__ load_narrow_klass_compact(t0, dst);
- } else if (UseCompressedClassPointers) {
+ } else {
__ lwu(tmp, Address(src, oopDesc::klass_offset_in_bytes()));
__ lwu(t0, Address(dst, oopDesc::klass_offset_in_bytes()));
- } else {
- __ ld(tmp, Address(src, oopDesc::klass_offset_in_bytes()));
- __ ld(t0, Address(dst, oopDesc::klass_offset_in_bytes()));
}
__ bne(tmp, t0, *stub->entry(), /* is_far */ true);
} else {
@@ -243,37 +240,6 @@ void LIR_Assembler::arraycopy_type_check(Register src, Register src_pos, Registe
}
}
-void LIR_Assembler::arraycopy_assert(Register src, Register dst, Register tmp, ciArrayKlass *default_type, int flags) {
- assert(default_type != nullptr, "null default_type!");
- BasicType basic_type = default_type->element_type()->basic_type();
- if (basic_type == T_ARRAY) { basic_type = T_OBJECT; }
- if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
- // Sanity check the known type with the incoming class. For the
- // primitive case the types must match exactly with src.klass and
- // dst.klass each exactly matching the default type. For the
- // object array case, if no type check is needed then either the
- // dst type is exactly the expected type and the src type is a
- // subtype which we can't check or src is the same array as dst
- // but not necessarily exactly of type default_type.
- Label known_ok, halt;
- __ mov_metadata(tmp, default_type->constant_encoding());
- if (UseCompressedClassPointers) {
- __ encode_klass_not_null(tmp);
- }
-
- if (basic_type != T_OBJECT) {
- __ cmp_klass_compressed(dst, tmp, t0, halt, false);
- __ cmp_klass_compressed(src, tmp, t0, known_ok, true);
- } else {
- __ cmp_klass_compressed(dst, tmp, t0, known_ok, true);
- __ beq(src, dst, known_ok);
- }
- __ bind(halt);
- __ stop("incorrect type information in arraycopy");
- __ bind(known_ok);
- }
-}
-
void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
ciArrayKlass *default_type = op->expected_type();
Register src = op->src()->as_register();
@@ -304,7 +270,28 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
}
#ifdef ASSERT
- arraycopy_assert(src, dst, tmp, default_type, flags);
+ if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
+ // Sanity check the known type with the incoming class. For the
+ // primitive case the types must match exactly with src.klass and
+ // dst.klass each exactly matching the default type. For the
+ // object array case, if no type check is needed then either the
+ // dst type is exactly the expected type and the src type is a
+ // subtype which we can't check or src is the same array as dst
+ // but not necessarily exactly of type default_type.
+ Label known_ok, halt;
+ __ mov_metadata(tmp, default_type->constant_encoding());
+
+ if (basic_type != T_OBJECT) {
+ __ cmp_klass_bne(dst, tmp, t0, t1, halt);
+ __ cmp_klass_beq(src, tmp, t0, t1, known_ok);
+ } else {
+ __ cmp_klass_beq(dst, tmp, t0, t1, known_ok);
+ __ beq(src, dst, known_ok);
+ }
+ __ bind(halt);
+ __ stop("incorrect type information in arraycopy");
+ __ bind(known_ok);
+ }
#endif
#ifndef PRODUCT
diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.hpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.hpp
index 06a0f248ca6..b5452f3e4cd 100644
--- a/src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.hpp
+++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -39,7 +39,6 @@
void arraycopy_type_check(Register src, Register src_pos, Register length,
Register dst, Register dst_pos, Register tmp,
CodeStub *stub, BasicType basic_type, int flags);
- void arraycopy_assert(Register src, Register dst, Register tmp, ciArrayKlass *default_type, int flags);
void arraycopy_prepare_params(Register src, Register src_pos, Register length,
Register dst, Register dst_pos, BasicType basic_type);
void arraycopy_checkcast_prepare_params(Register src, Register src_pos, Register length,
diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp
index 63e2fd015d7..29e5d86d0cc 100644
--- a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -55,20 +55,6 @@ const Register SHIFT_count = x10; // where count for shift operations must be
#define __ _masm->
-static void select_different_registers(Register preserve,
- Register extra,
- Register &tmp1,
- Register &tmp2) {
- if (tmp1 == preserve) {
- assert_different_registers(tmp1, tmp2, extra);
- tmp1 = extra;
- } else if (tmp2 == preserve) {
- assert_different_registers(tmp1, tmp2, extra);
- tmp2 = extra;
- }
- assert_different_registers(preserve, tmp1, tmp2);
-}
-
static void select_different_registers(Register preserve,
Register extra,
Register &tmp1,
@@ -1155,12 +1141,8 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, L
} else if (obj == klass_RInfo) {
klass_RInfo = dst;
}
- if (k->is_loaded() && !UseCompressedClassPointers) {
- select_different_registers(obj, dst, k_RInfo, klass_RInfo);
- } else {
- Rtmp1 = op->tmp3()->as_register();
- select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
- }
+ Rtmp1 = op->tmp3()->as_register();
+ select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
assert_different_registers(obj, k_RInfo, klass_RInfo);
diff --git a/src/hotspot/cpu/riscv/c1_LIRGenerator_riscv.cpp b/src/hotspot/cpu/riscv/c1_LIRGenerator_riscv.cpp
index 88565d9136f..f290708a231 100644
--- a/src/hotspot/cpu/riscv/c1_LIRGenerator_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c1_LIRGenerator_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -1073,9 +1073,7 @@ void LIRGenerator::do_CheckCast(CheckCast* x) {
}
LIR_Opr reg = rlock_result(x);
LIR_Opr tmp3 = LIR_OprFact::illegalOpr;
- if (!x->klass()->is_loaded() || UseCompressedClassPointers) {
- tmp3 = new_register(objectType);
- }
+ tmp3 = new_register(objectType);
__ checkcast(reg, obj.result(), x->klass(),
new_register(objectType), new_register(objectType), tmp3,
x->direct_compare(), info_for_exception, patching_info, stub,
@@ -1094,9 +1092,7 @@ void LIRGenerator::do_InstanceOf(InstanceOf* x) {
}
obj.load_item();
LIR_Opr tmp3 = LIR_OprFact::illegalOpr;
- if (!x->klass()->is_loaded() || UseCompressedClassPointers) {
- tmp3 = new_register(objectType);
- }
+ tmp3 = new_register(objectType);
__ instanceof(reg, obj.result(), x->klass(),
new_register(objectType), new_register(objectType), tmp3,
x->direct_compare(), patching_info, x->profiled_method(), x->profiled_bci());
diff --git a/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp
index aeb077ba0a0..abcc070b253 100644
--- a/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -92,12 +92,8 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
// This assumes that all prototype bits fitr in an int32_t
mv(tmp1, checked_cast(markWord::prototype().value()));
sd(tmp1, Address(obj, oopDesc::mark_offset_in_bytes()));
- if (UseCompressedClassPointers) { // Take care not to kill klass
- encode_klass_not_null(tmp1, klass, tmp2);
- sw(tmp1, Address(obj, oopDesc::klass_offset_in_bytes()));
- } else {
- sd(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
- }
+ encode_klass_not_null(tmp1, klass, tmp2);
+ sw(tmp1, Address(obj, oopDesc::klass_offset_in_bytes()));
}
if (len->is_valid()) {
@@ -108,7 +104,7 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
// Clear gap/first 4 bytes following the length field.
sw(zr, Address(obj, base_offset));
}
- } else if (UseCompressedClassPointers && !UseCompactObjectHeaders) {
+ } else if (!UseCompactObjectHeaders) {
store_klass_gap(obj, zr);
}
}
diff --git a/src/hotspot/cpu/riscv/c1_globals_riscv.hpp b/src/hotspot/cpu/riscv/c1_globals_riscv.hpp
index b15bb5c23c3..b940393f063 100644
--- a/src/hotspot/cpu/riscv/c1_globals_riscv.hpp
+++ b/src/hotspot/cpu/riscv/c1_globals_riscv.hpp
@@ -42,7 +42,6 @@ define_pd_global(bool, TieredCompilation, false);
define_pd_global(intx, CompileThreshold, 1500 );
define_pd_global(intx, OnStackReplacePercentage, 933 );
-define_pd_global(intx, NewSizeThreadIncrease, 4*K );
define_pd_global(size_t, InitialCodeCacheSize, 160*K);
define_pd_global(size_t, ReservedCodeCacheSize, 32*M );
define_pd_global(size_t, NonProfiledCodeHeapSize, 13*M );
@@ -52,7 +51,6 @@ define_pd_global(bool, ProfileInterpreter, false);
define_pd_global(size_t, CodeCacheExpansionSize, 32*K );
define_pd_global(size_t, CodeCacheMinBlockLength, 1);
define_pd_global(size_t, CodeCacheMinimumUseSpace, 400*K);
-define_pd_global(bool, NeverActAsServerClassMachine, true );
define_pd_global(bool, CICompileOSR, true );
#endif // !COMPILER2
define_pd_global(bool, UseTypeProfile, false);
diff --git a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp
index b4e0ba69042..0d06fd469de 100644
--- a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp
@@ -30,7 +30,9 @@
#include "opto/intrinsicnode.hpp"
#include "opto/output.hpp"
#include "opto/subnode.hpp"
+#include "runtime/objectMonitorTable.hpp"
#include "runtime/stubRoutines.hpp"
+#include "runtime/synchronizer.hpp"
#include "utilities/globalDefinitions.hpp"
#ifdef PRODUCT
@@ -123,35 +125,52 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box,
if (!UseObjectMonitorTable) {
assert(tmp1_monitor == tmp1_mark, "should be the same here");
} else {
+ const Register tmp2_hash = tmp2;
+ const Register tmp3_bucket = tmp3;
Label monitor_found;
- // Load cache address
- la(tmp3_t, Address(xthread, JavaThread::om_cache_oops_offset()));
+ // Save the mark, we might need it to extract the hash.
+ mv(tmp2_hash, tmp1_mark);
- const int num_unrolled = 2;
+ // Look for the monitor in the om_cache.
+
+ ByteSize cache_offset = JavaThread::om_cache_oops_offset();
+ ByteSize monitor_offset = OMCache::oop_to_monitor_difference();
+ const int num_unrolled = OMCache::CAPACITY;
for (int i = 0; i < num_unrolled; i++) {
- ld(tmp1, Address(tmp3_t));
- beq(obj, tmp1, monitor_found);
- add(tmp3_t, tmp3_t, in_bytes(OMCache::oop_to_oop_difference()));
+ ld(tmp1_monitor, Address(xthread, cache_offset + monitor_offset));
+ ld(tmp4, Address(xthread, cache_offset));
+ beq(obj, tmp4, monitor_found);
+ cache_offset = cache_offset + OMCache::oop_to_oop_difference();
}
- Label loop;
+ // Look for the monitor in the table.
- // Search for obj in cache.
- bind(loop);
+ // Get the hash code.
+ srli(tmp2_hash, tmp2_hash, markWord::hash_shift);
- // Check for match.
- ld(tmp1, Address(tmp3_t));
- beq(obj, tmp1, monitor_found);
+ // Get the table and calculate the bucket's address.
+ la(tmp3_t, ExternalAddress(ObjectMonitorTable::current_table_address()));
+ ld(tmp3_t, Address(tmp3_t));
+ ld(tmp1, Address(tmp3_t, ObjectMonitorTable::table_capacity_mask_offset()));
+ andr(tmp2_hash, tmp2_hash, tmp1);
+ ld(tmp3_t, Address(tmp3_t, ObjectMonitorTable::table_buckets_offset()));
- // Search until null encountered, guaranteed _null_sentinel at end.
- add(tmp3_t, tmp3_t, in_bytes(OMCache::oop_to_oop_difference()));
- bnez(tmp1, loop);
- // Cache Miss. Take the slowpath.
- j(slow_path);
+ // Read the monitor from the bucket.
+ shadd(tmp3_bucket, tmp2_hash, tmp3_t, tmp4, LogBytesPerWord);
+ ld(tmp1_monitor, Address(tmp3_bucket));
+
+ // Check if the monitor in the bucket is special (empty, tombstone or removed).
+ mv(tmp2, ObjectMonitorTable::SpecialPointerValues::below_is_special);
+ bltu(tmp1_monitor, tmp2, slow_path);
+
+ // Check if object matches.
+ ld(tmp3, Address(tmp1_monitor, ObjectMonitor::object_offset()));
+ BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
+ bs_asm->try_resolve_weak_handle_in_c2(this, tmp3, tmp2, slow_path);
+ bne(tmp3, obj, slow_path);
bind(monitor_found);
- ld(tmp1_monitor, Address(tmp3_t, OMCache::oop_to_monitor_difference()));
}
const Register tmp2_owner_addr = tmp2;
@@ -1156,8 +1175,7 @@ void C2_MacroAssembler::string_compare_long_same_encoding(Register result, Regis
Label TAIL_CHECK, TAIL, NEXT_WORD, DIFFERENCE;
const int base_offset = arrayOopDesc::base_offset_in_bytes(T_BYTE);
- assert((base_offset % (UseCompactObjectHeaders ? 4 :
- (UseCompressedClassPointers ? 8 : 4))) == 0, "Must be");
+ assert((base_offset % (UseCompactObjectHeaders ? 4 : 8)) == 0, "Must be");
const int minCharsInWord = isLL ? wordSize : wordSize / 2;
@@ -1250,8 +1268,7 @@ void C2_MacroAssembler::string_compare_long_different_encoding(Register result,
Label TAIL, NEXT_WORD, DIFFERENCE;
const int base_offset = arrayOopDesc::base_offset_in_bytes(T_BYTE);
- assert((base_offset % (UseCompactObjectHeaders ? 4 :
- (UseCompressedClassPointers ? 8 : 4))) == 0, "Must be");
+ assert((base_offset % (UseCompactObjectHeaders ? 4 : 8)) == 0, "Must be");
Register strL = isLU ? str1 : str2;
Register strU = isLU ? str2 : str1;
@@ -1466,8 +1483,7 @@ void C2_MacroAssembler::arrays_equals(Register a1, Register a2,
int length_offset = arrayOopDesc::length_offset_in_bytes();
int base_offset = arrayOopDesc::base_offset_in_bytes(elem_size == 2 ? T_CHAR : T_BYTE);
- assert((base_offset % (UseCompactObjectHeaders ? 4 :
- (UseCompressedClassPointers ? 8 : 4))) == 0, "Must be");
+ assert((base_offset % (UseCompactObjectHeaders ? 4 : 8)) == 0, "Must be");
Register cnt1 = tmp3;
Register cnt2 = tmp1; // cnt2 only used in array length compare
@@ -1592,8 +1608,7 @@ void C2_MacroAssembler::string_equals(Register a1, Register a2,
int base_offset = arrayOopDesc::base_offset_in_bytes(T_BYTE);
- assert((base_offset % (UseCompactObjectHeaders ? 4 :
- (UseCompressedClassPointers ? 8 : 4))) == 0, "Must be");
+ assert((base_offset % (UseCompactObjectHeaders ? 4 : 8)) == 0, "Must be");
BLOCK_COMMENT("string_equals {");
@@ -2680,8 +2695,7 @@ void C2_MacroAssembler::arrays_equals_v(Register a1, Register a2, Register resul
int length_offset = arrayOopDesc::length_offset_in_bytes();
int base_offset = arrayOopDesc::base_offset_in_bytes(elem_size == 2 ? T_CHAR : T_BYTE);
- assert((base_offset % (UseCompactObjectHeaders ? 4 :
- (UseCompressedClassPointers ? 8 : 4))) == 0, "Must be");
+ assert((base_offset % (UseCompactObjectHeaders ? 4 : 8)) == 0, "Must be");
BLOCK_COMMENT("arrays_equals_v {");
diff --git a/src/hotspot/cpu/riscv/c2_globals_riscv.hpp b/src/hotspot/cpu/riscv/c2_globals_riscv.hpp
index 648c24ee98b..73ef97939ed 100644
--- a/src/hotspot/cpu/riscv/c2_globals_riscv.hpp
+++ b/src/hotspot/cpu/riscv/c2_globals_riscv.hpp
@@ -47,7 +47,6 @@ define_pd_global(intx, ConditionalMoveLimit, 3);
define_pd_global(intx, FreqInlineSize, 325);
define_pd_global(intx, MinJumpTableSize, 10);
define_pd_global(intx, InteriorEntryAlignment, 16);
-define_pd_global(intx, NewSizeThreadIncrease, ScaleForWordSize(4*K));
define_pd_global(intx, LoopUnrollLimit, 60);
define_pd_global(intx, LoopPercentProfileLimit, 10);
// InitialCodeCacheSize derived from specjbb2000 run.
@@ -75,9 +74,6 @@ define_pd_global(size_t, NonNMethodCodeHeapSize, 5*M );
define_pd_global(size_t, CodeCacheMinBlockLength, 6);
define_pd_global(size_t, CodeCacheMinimumUseSpace, 400*K);
-// Ergonomics related flags
-define_pd_global(bool, NeverActAsServerClassMachine, false);
-
define_pd_global(bool, TrapBasedRangeChecks, false); // Not needed.
#endif // CPU_RISCV_C2_GLOBALS_RISCV_HPP
diff --git a/src/hotspot/cpu/riscv/downcallLinker_riscv.cpp b/src/hotspot/cpu/riscv/downcallLinker_riscv.cpp
index cc685645ec5..f9d7ce78ff0 100644
--- a/src/hotspot/cpu/riscv/downcallLinker_riscv.cpp
+++ b/src/hotspot/cpu/riscv/downcallLinker_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -140,10 +140,10 @@ void DowncallLinker::StubGenerator::generate() {
bool should_save_return_value = !_needs_return_buffer;
RegSpiller out_reg_spiller(_output_registers);
- int spill_offset = -1;
+ int out_spill_offset = -1;
if (should_save_return_value) {
- spill_offset = 0;
+ out_spill_offset = 0;
// spill area can be shared with shadow space and out args,
// since they are only used before the call,
// and spill area is only used after.
@@ -168,6 +168,9 @@ void DowncallLinker::StubGenerator::generate() {
// FP-> | |
// |---------------------| = frame_bottom_offset = frame_size
// | (optional) |
+ // | in_reg_spiller area |
+ // |---------------------|
+ // | (optional) |
// | capture state buf |
// |---------------------| = StubLocations::CAPTURED_STATE_BUFFER
// | (optional) |
@@ -181,6 +184,18 @@ void DowncallLinker::StubGenerator::generate() {
GrowableArray out_regs = ForeignGlobals::replace_place_holders(_input_registers, locs);
ArgumentShuffle arg_shuffle(filtered_java_regs, out_regs, shuffle_reg);
+ // Need to spill for state capturing runtime call.
+ // The area spilled into is distinct from the capture state buffer.
+ RegSpiller in_reg_spiller(out_regs);
+ int in_spill_offset = -1;
+ if (_captured_state_mask != 0) {
+ // The spill area cannot be shared with the out_spill since
+ // spilling needs to happen before the call. Allocate a new
+ // region in the stack for this spill space.
+ in_spill_offset = allocated_frame_size;
+ allocated_frame_size += in_reg_spiller.spill_size_bytes();
+ }
+
#ifndef PRODUCT
LogTarget(Trace, foreign, downcall) lt;
if (lt.is_enabled()) {
@@ -226,6 +241,20 @@ void DowncallLinker::StubGenerator::generate() {
arg_shuffle.generate(_masm, shuffle_reg, 0, _abi._shadow_space_bytes);
__ block_comment("} argument shuffle");
+ if (_captured_state_mask != 0) {
+ assert(in_spill_offset != -1, "must be");
+ __ block_comment("{ load initial thread local");
+ in_reg_spiller.generate_spill(_masm, in_spill_offset);
+
+ // Copy the contents of the capture state buffer into thread local
+ __ ld(c_rarg0, Address(sp, locs.data_offset(StubLocations::CAPTURED_STATE_BUFFER)));
+ __ mv(c_rarg1, _captured_state_mask);
+ __ rt_call(CAST_FROM_FN_PTR(address, DowncallLinker::capture_state_pre));
+
+ in_reg_spiller.generate_fill(_masm, in_spill_offset);
+ __ block_comment("} load initial thread local");
+ }
+
__ jalr(as_Register(locs.get(StubLocations::TARGET_ADDRESS)));
// this call is assumed not to have killed xthread
@@ -254,15 +283,15 @@ void DowncallLinker::StubGenerator::generate() {
__ block_comment("{ save thread local");
if (should_save_return_value) {
- out_reg_spiller.generate_spill(_masm, spill_offset);
+ out_reg_spiller.generate_spill(_masm, out_spill_offset);
}
__ ld(c_rarg0, Address(sp, locs.data_offset(StubLocations::CAPTURED_STATE_BUFFER)));
__ mv(c_rarg1, _captured_state_mask);
- __ rt_call(CAST_FROM_FN_PTR(address, DowncallLinker::capture_state));
+ __ rt_call(CAST_FROM_FN_PTR(address, DowncallLinker::capture_state_post));
if (should_save_return_value) {
- out_reg_spiller.generate_fill(_masm, spill_offset);
+ out_reg_spiller.generate_fill(_masm, out_spill_offset);
}
__ block_comment("} save thread local");
@@ -319,7 +348,7 @@ void DowncallLinker::StubGenerator::generate() {
if (should_save_return_value) {
// Need to save the native result registers around any runtime calls.
- out_reg_spiller.generate_spill(_masm, spill_offset);
+ out_reg_spiller.generate_spill(_masm, out_spill_offset);
}
__ mv(c_rarg0, xthread);
@@ -327,7 +356,7 @@ void DowncallLinker::StubGenerator::generate() {
__ rt_call(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans));
if (should_save_return_value) {
- out_reg_spiller.generate_fill(_masm, spill_offset);
+ out_reg_spiller.generate_fill(_masm, out_spill_offset);
}
__ j(L_after_safepoint_poll);
__ block_comment("} L_safepoint_poll_slow_path");
@@ -339,13 +368,13 @@ void DowncallLinker::StubGenerator::generate() {
if (should_save_return_value) {
// Need to save the native result registers around any runtime calls.
- out_reg_spiller.generate_spill(_masm, spill_offset);
+ out_reg_spiller.generate_spill(_masm, out_spill_offset);
}
__ rt_call(CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages));
if (should_save_return_value) {
- out_reg_spiller.generate_fill(_masm, spill_offset);
+ out_reg_spiller.generate_fill(_masm, out_spill_offset);
}
__ j(L_after_reguard);
diff --git a/src/hotspot/cpu/riscv/frame_riscv.inline.hpp b/src/hotspot/cpu/riscv/frame_riscv.inline.hpp
index 51a203c548c..d1841a347e9 100644
--- a/src/hotspot/cpu/riscv/frame_riscv.inline.hpp
+++ b/src/hotspot/cpu/riscv/frame_riscv.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -236,8 +236,8 @@ inline bool frame::equal(frame other) const {
// Return unique id for this frame. The id must have a value where we can distinguish
// identity and younger/older relationship. null represents an invalid (incomparable)
-// frame.
-inline intptr_t* frame::id(void) const { return unextended_sp(); }
+// frame. Should not be called for heap frames.
+inline intptr_t* frame::id(void) const { return real_fp(); }
// Return true if the frame is older (less recent activation) than the frame represented by id
inline bool frame::is_older(intptr_t* id) const { assert(this->id() != nullptr && id != nullptr, "null frame id");
@@ -398,6 +398,9 @@ frame frame::sender(RegisterMap* map) const {
StackWatermarkSet::on_iteration(map->thread(), result);
}
+ // Calling frame::id() is currently not supported for heap frames.
+ assert(result._on_heap || this->_on_heap || result.is_older(this->id()), "Must be");
+
return result;
}
diff --git a/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.cpp b/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.cpp
index f5916000890..aeb9df06de6 100644
--- a/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -369,6 +369,11 @@ OptoReg::Name BarrierSetAssembler::refine_register(const Node* node, OptoReg::Na
return opto_reg;
}
+void BarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path) {
+ // Load the oop from the weak handle.
+ __ ld(obj, Address(obj));
+}
+
#undef __
#define __ _masm->
diff --git a/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.hpp b/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.hpp
index e50fa1dae36..bbb2a5af824 100644
--- a/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -110,6 +110,8 @@ public:
#ifdef COMPILER2
OptoReg::Name refine_register(const Node* node,
OptoReg::Name opto_reg);
+ virtual void try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj,
+ Register tmp, Label& slow_path);
#endif // COMPILER2
};
diff --git a/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.cpp b/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.cpp
index 3cbbb783258..8d530d15ee5 100644
--- a/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.cpp
@@ -1,4 +1,5 @@
/*
+ * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, 2020, Red Hat, Inc. All rights reserved.
* Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -461,6 +462,30 @@ void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler
__ bind(done);
}
+#ifdef COMPILER2
+void ShenandoahBarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler *masm, Register obj,
+ Register tmp, Label& slow_path) {
+ assert_different_registers(obj, tmp);
+
+ Label done;
+
+ // Resolve weak handle using the standard implementation.
+ BarrierSetAssembler::try_resolve_weak_handle_in_c2(masm, obj, tmp, slow_path);
+
+ // Check if the reference is null, and if it is, take the fast path.
+ __ beqz(obj, done);
+
+ Address gc_state(xthread, ShenandoahThreadLocalData::gc_state_offset());
+ __ lbu(tmp, gc_state);
+
+ // Check if the heap is under weak-reference/roots processing, in
+ // which case we need to take the slow path.
+ __ test_bit(tmp, tmp, ShenandoahHeap::WEAK_ROOTS_BITPOS);
+ __ bnez(tmp, slow_path);
+ __ bind(done);
+}
+#endif
+
// Special Shenandoah CAS implementation that handles false negatives due
// to concurrent evacuation. The service is more complex than a
// traditional CAS operation because the CAS operation is intended to
diff --git a/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.hpp b/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.hpp
index 5085be26b2e..e35e09c93da 100644
--- a/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.hpp
@@ -1,4 +1,5 @@
/*
+ * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved.
* Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -84,7 +85,9 @@ public:
virtual void try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
Register obj, Register tmp, Label& slowpath);
-
+#ifdef COMPILER2
+ virtual void try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path);
+#endif
void cmpxchg_oop(MacroAssembler* masm, Register addr, Register expected, Register new_val,
Assembler::Aqrl acquire, Assembler::Aqrl release, bool is_cae, Register result);
};
diff --git a/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.cpp b/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.cpp
index 09dea62b6d1..163271a2f11 100644
--- a/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -602,6 +602,27 @@ void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm,
BLOCK_COMMENT("} ZBarrierSetAssembler::try_resolve_jobject_in_native");
}
+#ifdef COMPILER2
+void ZBarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path) {
+ BLOCK_COMMENT("ZBarrierSetAssembler::try_resolve_weak_handle_in_c2 {");
+
+ // Resolve weak handle using the standard implementation.
+ BarrierSetAssembler::try_resolve_weak_handle_in_c2(masm, obj, tmp, slow_path);
+
+ // Check if the oop is bad, in which case we need to take the slow path.
+ __ relocate(barrier_Relocation::spec(), [&] {
+ __ li16u(tmp, barrier_Relocation::unpatched);
+ }, ZBarrierRelocationFormatMarkBadMask);
+ __ andr(tmp, obj, tmp);
+ __ bnez(tmp, slow_path);
+
+ // Oop is okay, so we uncolor it.
+ __ srli(obj, obj, ZPointerLoadShift);
+
+ BLOCK_COMMENT("} ZBarrierSetAssembler::try_resolve_weak_handle_in_c2");
+}
+#endif
+
static uint16_t patch_barrier_relocation_value(int format) {
switch (format) {
case ZBarrierRelocationFormatLoadBadMask:
diff --git a/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.hpp b/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.hpp
index 190d81acd0c..648cb3bf63d 100644
--- a/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -170,6 +170,10 @@ public:
ZLoadBarrierStubC2* stub) const;
void generate_c2_store_barrier_stub(MacroAssembler* masm,
ZStoreBarrierStubC2* stub) const;
+ void try_resolve_weak_handle_in_c2(MacroAssembler* masm,
+ Register obj,
+ Register tmp,
+ Label& slow_path);
#endif // COMPILER2
void check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error);
diff --git a/src/hotspot/cpu/riscv/interp_masm_riscv.cpp b/src/hotspot/cpu/riscv/interp_masm_riscv.cpp
index 744590bec2b..804c2072ba5 100644
--- a/src/hotspot/cpu/riscv/interp_masm_riscv.cpp
+++ b/src/hotspot/cpu/riscv/interp_masm_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -1040,26 +1040,15 @@ void InterpreterMacroAssembler::profile_final_call(Register mdp) {
void InterpreterMacroAssembler::profile_virtual_call(Register receiver,
- Register mdp,
- bool receiver_can_be_null) {
+ Register mdp) {
if (ProfileInterpreter) {
Label profile_continue;
// If no method data exists, go to profile_continue.
test_method_data_pointer(mdp, profile_continue);
- Label skip_receiver_profile;
- if (receiver_can_be_null) {
- Label not_null;
- // We are making a call. Increment the count for null receiver.
- increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
- j(skip_receiver_profile);
- bind(not_null);
- }
-
// Record the receiver type.
profile_receiver_type(receiver, mdp, 0);
- bind(skip_receiver_profile);
// The method data pointer needs to be updated to reflect the new target.
diff --git a/src/hotspot/cpu/riscv/interp_masm_riscv.hpp b/src/hotspot/cpu/riscv/interp_masm_riscv.hpp
index 59cc76b022f..df86f0dc532 100644
--- a/src/hotspot/cpu/riscv/interp_masm_riscv.hpp
+++ b/src/hotspot/cpu/riscv/interp_masm_riscv.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2015, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -274,8 +274,7 @@ class InterpreterMacroAssembler: public MacroAssembler {
void profile_not_taken_branch(Register mdp);
void profile_call(Register mdp);
void profile_final_call(Register mdp);
- void profile_virtual_call(Register receiver, Register mdp,
- bool receiver_can_be_null = false);
+ void profile_virtual_call(Register receiver, Register mdp);
void profile_ret(Register return_bci, Register mdp);
void profile_null_seen(Register mdp);
void profile_typecheck(Register mdp, Register klass);
diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
index 4f5e7afc166..a2b7970f9f6 100644
--- a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2024, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -3511,19 +3511,30 @@ void MacroAssembler::orptr(Address adr, RegisterOrConstant src, Register tmp1, R
sd(tmp1, adr);
}
-void MacroAssembler::cmp_klass_compressed(Register oop, Register trial_klass, Register tmp, Label &L, bool equal) {
+void MacroAssembler::cmp_klass_beq(Register obj, Register klass,
+ Register tmp1, Register tmp2,
+ Label &L, bool is_far) {
+ assert_different_registers(obj, klass, tmp1, tmp2);
if (UseCompactObjectHeaders) {
- load_narrow_klass_compact(tmp, oop);
- } else if (UseCompressedClassPointers) {
- lwu(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
+ load_narrow_klass_compact(tmp1, obj);
} else {
- ld(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
+ lwu(tmp1, Address(obj, oopDesc::klass_offset_in_bytes()));
}
- if (equal) {
- beq(trial_klass, tmp, L);
+ decode_klass_not_null(tmp1, tmp2);
+ beq(klass, tmp1, L, is_far);
+}
+
+void MacroAssembler::cmp_klass_bne(Register obj, Register klass,
+ Register tmp1, Register tmp2,
+ Label &L, bool is_far) {
+ assert_different_registers(obj, klass, tmp1, tmp2);
+ if (UseCompactObjectHeaders) {
+ load_narrow_klass_compact(tmp1, obj);
} else {
- bne(trial_klass, tmp, L);
+ lwu(tmp1, Address(obj, oopDesc::klass_offset_in_bytes()));
}
+ decode_klass_not_null(tmp1, tmp2);
+ bne(klass, tmp1, L, is_far);
}
// Move an oop into a register.
@@ -3741,11 +3752,9 @@ void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(dst, src);
decode_klass_not_null(dst, tmp);
- } else if (UseCompressedClassPointers) {
+ } else {
lwu(dst, Address(src, oopDesc::klass_offset_in_bytes()));
decode_klass_not_null(dst, tmp);
- } else {
- ld(dst, Address(src, oopDesc::klass_offset_in_bytes()));
}
}
@@ -3753,20 +3762,15 @@ void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
// FIXME: Should this be a store release? concurrent gcs assumes
// klass length is valid if klass field is not null.
assert(!UseCompactObjectHeaders, "not with compact headers");
- if (UseCompressedClassPointers) {
- encode_klass_not_null(src, tmp);
- sw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
- } else {
- sd(src, Address(dst, oopDesc::klass_offset_in_bytes()));
- }
+ encode_klass_not_null(src, tmp);
+ sw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
+
}
void MacroAssembler::store_klass_gap(Register dst, Register src) {
assert(!UseCompactObjectHeaders, "not with compact headers");
- if (UseCompressedClassPointers) {
- // Store to klass gap in destination
- sw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
- }
+ // Store to klass gap in destination
+ sw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
}
void MacroAssembler::decode_klass_not_null(Register r, Register tmp) {
@@ -3775,7 +3779,6 @@ void MacroAssembler::decode_klass_not_null(Register r, Register tmp) {
}
void MacroAssembler::decode_klass_not_null(Register dst, Register src, Register tmp) {
- assert(UseCompressedClassPointers, "should only be used for compressed headers");
assert_different_registers(dst, tmp);
assert_different_registers(src, tmp);
@@ -3806,8 +3809,6 @@ void MacroAssembler::encode_klass_not_null(Register r, Register tmp) {
}
void MacroAssembler::encode_klass_not_null(Register dst, Register src, Register tmp) {
- assert(UseCompressedClassPointers, "should only be used for compressed headers");
-
if (CompressedKlassPointers::base() == nullptr) {
if (CompressedKlassPointers::shift() != 0) {
srli(dst, src, CompressedKlassPointers::shift());
@@ -5337,7 +5338,6 @@ void MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
}
void MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
- assert (UseCompressedClassPointers, "should only be used for compressed headers");
assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
int index = oop_recorder()->find_index(k);
@@ -5417,12 +5417,9 @@ int MacroAssembler::ic_check(int end_alignment) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(tmp1, receiver);
lwu(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
- } else if (UseCompressedClassPointers) {
+ } else {
lwu(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes()));
lwu(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
- } else {
- ld(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes()));
- ld(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
}
Label ic_hit;
@@ -5543,13 +5540,6 @@ void MacroAssembler::decrementw(const Address dst, int32_t value, Register tmp1,
sw(tmp1, adr);
}
-void MacroAssembler::cmpptr(Register src1, const Address &src2, Label& equal, Register tmp) {
- assert_different_registers(src1, tmp);
- assert(src2.getMode() == Address::literal, "must be applied to a literal address");
- ld(tmp, src2);
- beq(src1, tmp, equal);
-}
-
void MacroAssembler::load_method_holder_cld(Register result, Register method) {
load_method_holder(result, method);
ld(result, Address(result, InstanceKlass::class_loader_data_offset()));
diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
index f5e985c28a2..a51a6aea468 100644
--- a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2024, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -198,7 +198,12 @@ class MacroAssembler: public Assembler {
void load_klass(Register dst, Register src, Register tmp = t0);
void load_narrow_klass_compact(Register dst, Register src);
void store_klass(Register dst, Register src, Register tmp = t0);
- void cmp_klass_compressed(Register oop, Register trial_klass, Register tmp, Label &L, bool equal);
+ void cmp_klass_beq(Register obj, Register klass,
+ Register tmp1, Register tmp2,
+ Label &L, bool is_far = false);
+ void cmp_klass_bne(Register obj, Register klass,
+ Register tmp1, Register tmp2,
+ Label &L, bool is_far = false);
void encode_klass_not_null(Register r, Register tmp = t0);
void decode_klass_not_null(Register r, Register tmp = t0);
@@ -1348,9 +1353,8 @@ public:
void decrement(const Address dst, int64_t value = 1, Register tmp1 = t0, Register tmp2 = t1);
void decrementw(const Address dst, int32_t value = 1, Register tmp1 = t0, Register tmp2 = t1);
- void cmpptr(Register src1, const Address &src2, Label& equal, Register tmp = t0);
-
void clinit_barrier(Register klass, Register tmp, Label* L_fast_path = nullptr, Label* L_slow_path = nullptr);
+
void load_method_holder_cld(Register result, Register method);
void load_method_holder(Register holder, Register method);
diff --git a/src/hotspot/cpu/riscv/methodHandles_riscv.cpp b/src/hotspot/cpu/riscv/methodHandles_riscv.cpp
index d770999df96..e80dedf58ed 100644
--- a/src/hotspot/cpu/riscv/methodHandles_riscv.cpp
+++ b/src/hotspot/cpu/riscv/methodHandles_riscv.cpp
@@ -72,17 +72,22 @@ void MethodHandles::verify_klass(MacroAssembler* _masm,
InstanceKlass** klass_addr = vmClasses::klass_addr_at(klass_id);
Klass* klass = vmClasses::klass_at(klass_id);
Register temp1 = t1;
- Register temp2 = t0; // used by MacroAssembler::cmpptr
+ Register temp2 = t0;
Label L_ok, L_bad;
BLOCK_COMMENT("verify_klass {");
__ verify_oop(obj);
__ beqz(obj, L_bad);
+
__ push_reg(RegSet::of(temp1, temp2), sp);
__ load_klass(temp1, obj, temp2);
- __ cmpptr(temp1, ExternalAddress((address) klass_addr), L_ok);
+ __ ld(temp2, ExternalAddress((address)klass_addr));
+ __ beq(temp1, temp2, L_ok);
+
intptr_t super_check_offset = klass->super_check_offset();
__ ld(temp1, Address(temp1, super_check_offset));
- __ cmpptr(temp1, ExternalAddress((address) klass_addr), L_ok);
+ __ ld(temp2, ExternalAddress((address)klass_addr));
+ __ beq(temp1, temp2, L_ok);
+
__ pop_reg(RegSet::of(temp1, temp2), sp);
__ bind(L_bad);
__ stop(error_message);
diff --git a/src/hotspot/cpu/riscv/riscv.ad b/src/hotspot/cpu/riscv/riscv.ad
index 730dd68dd88..e236d03e6d2 100644
--- a/src/hotspot/cpu/riscv/riscv.ad
+++ b/src/hotspot/cpu/riscv/riscv.ad
@@ -1,5 +1,5 @@
//
-// Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
// Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
// Copyright (c) 2020, 2024, Huawei Technologies Co., Ltd. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -1801,13 +1801,8 @@ void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
{
assert_cond(st != nullptr);
st->print_cr("# MachUEPNode");
- if (UseCompressedClassPointers) {
- st->print_cr("\tlwu t1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
- st->print_cr("\tlwu t2, [t0 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
- } else {
- st->print_cr("\tld t1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
- st->print_cr("\tld t2, [t0 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
- }
+ st->print_cr("\tlwu t1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
+ st->print_cr("\tlwu t2, [t0 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
st->print_cr("\tbeq t1, t2, ic_hit");
st->print_cr("\tj, SharedRuntime::_ic_miss_stub\t # Inline cache check");
st->print_cr("\tic_hit:");
@@ -2060,11 +2055,8 @@ bool Matcher::is_generic_vector(MachOper* opnd) {
return false;
}
+#ifdef ASSERT
// Return whether or not this register is ever used as an argument.
-// This function is used on startup to build the trampoline stubs in
-// generateOptoStub. Registers not mentioned will be killed by the VM
-// call in the trampoline, and arguments in those registers not be
-// available to the callee.
bool Matcher::can_be_java_arg(int reg)
{
return
@@ -2085,11 +2077,7 @@ bool Matcher::can_be_java_arg(int reg)
reg == F16_num || reg == F16_H_num ||
reg == F17_num || reg == F17_H_num;
}
-
-bool Matcher::is_spillable_arg(int reg)
-{
- return can_be_java_arg(reg);
-}
+#endif
uint Matcher::int_pressure_limit()
{
@@ -2118,10 +2106,6 @@ uint Matcher::float_pressure_limit()
return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
}
-bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
- return false;
-}
-
const RegMask& Matcher::divI_proj_mask() {
ShouldNotReachHere();
return RegMask::EMPTY;
@@ -2274,7 +2258,7 @@ encode %{
} else if (rtype == relocInfo::metadata_type) {
__ mov_metadata(dst_reg, (Metadata*)con);
} else {
- assert(rtype == relocInfo::none, "unexpected reloc type");
+ assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
__ mv(dst_reg, $src$$constant);
}
}
@@ -2559,11 +2543,6 @@ frame %{
// Compiled code's Frame Pointer
frame_pointer(R2);
- // Interpreter stores its frame pointer in a register which is
- // stored to the stack by I2CAdaptors.
- // I2CAdaptors convert from interpreted java to compiled java.
- interpreter_frame_pointer(R8);
-
// Stack alignment requirement
stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
@@ -8168,6 +8147,22 @@ instruct unnecessary_membar_rvtso() %{
ins_pipe(real_empty);
%}
+instruct membar_storeload_rvtso() %{
+ predicate(UseZtso);
+ match(MemBarStoreLoad);
+ ins_cost(VOLATILE_REF_COST);
+
+ format %{ "#@membar_storeload_rvtso\n\t"
+ "fence w, r"%}
+
+ ins_encode %{
+ __ block_comment("membar_storeload_rvtso");
+ __ membar(MacroAssembler::StoreLoad);
+ %}
+
+ ins_pipe(pipe_slow);
+%}
+
instruct membar_volatile_rvtso() %{
predicate(UseZtso);
match(MemBarVolatile);
@@ -8198,6 +8193,22 @@ instruct unnecessary_membar_volatile_rvtso() %{
ins_pipe(real_empty);
%}
+instruct membar_full_rvtso() %{
+ predicate(UseZtso);
+ match(MemBarFull);
+ ins_cost(VOLATILE_REF_COST);
+
+ format %{ "#@membar_full_rvtso\n\t"
+ "fence rw, rw" %}
+
+ ins_encode %{
+ __ block_comment("membar_full_rvtso");
+ __ membar(MacroAssembler::AnyAny);
+ %}
+
+ ins_pipe(pipe_slow);
+%}
+
// RVWMO
instruct membar_aqcuire_rvwmo() %{
@@ -8247,6 +8258,22 @@ instruct membar_storestore_rvwmo() %{
ins_pipe(pipe_serial);
%}
+instruct membar_storeload_rvwmo() %{
+ predicate(!UseZtso);
+ match(MemBarStoreLoad);
+ ins_cost(VOLATILE_REF_COST);
+
+ format %{ "#@membar_storeload_rvwmo\n\t"
+ "fence w, r"%}
+
+ ins_encode %{
+ __ block_comment("membar_storeload_rvwmo");
+ __ membar(MacroAssembler::StoreLoad);
+ %}
+
+ ins_pipe(pipe_serial);
+%}
+
instruct membar_volatile_rvwmo() %{
predicate(!UseZtso);
match(MemBarVolatile);
@@ -8291,6 +8318,22 @@ instruct unnecessary_membar_volatile_rvwmo() %{
ins_pipe(real_empty);
%}
+instruct membar_full_rvwmo() %{
+ predicate(!UseZtso);
+ match(MemBarFull);
+ ins_cost(VOLATILE_REF_COST);
+
+ format %{ "#@membar_full_rvwmo\n\t"
+ "fence rw, rw" %}
+
+ ins_encode %{
+ __ block_comment("membar_full_rvwmo");
+ __ membar(MacroAssembler::AnyAny);
+ %}
+
+ ins_pipe(pipe_serial);
+%}
+
instruct spin_wait() %{
predicate(UseZihintpause);
match(OnSpinWait);
diff --git a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp
index 127ac9f6951..964c6d98e9c 100644
--- a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp
+++ b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2025, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2025, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -3070,8 +3070,7 @@ class StubGenerator: public StubCodeGenerator {
const Register tmp = x30, tmpLval = x12;
int base_offset = arrayOopDesc::base_offset_in_bytes(T_BYTE);
- assert((base_offset % (UseCompactObjectHeaders ? 4 :
- (UseCompressedClassPointers ? 8 : 4))) == 0, "Must be");
+ assert((base_offset % (UseCompactObjectHeaders ? 4 : 8)) == 0, "Must be");
#ifdef ASSERT
if (AvoidUnalignedAccesses) {
@@ -3128,8 +3127,7 @@ class StubGenerator: public StubCodeGenerator {
tmp1 = x28, tmp2 = x29, tmp3 = x30, tmp4 = x12;
int base_offset = arrayOopDesc::base_offset_in_bytes(T_BYTE);
- assert((base_offset % (UseCompactObjectHeaders ? 4 :
- (UseCompressedClassPointers ? 8 : 4))) == 0, "Must be");
+ assert((base_offset % (UseCompactObjectHeaders ? 4 : 8)) == 0, "Must be");
Register strU = isLU ? str2 : str1,
strL = isLU ? str1 : str2,
diff --git a/src/hotspot/cpu/riscv/vm_version_riscv.hpp b/src/hotspot/cpu/riscv/vm_version_riscv.hpp
index 03c843efc69..11a88dfedd7 100644
--- a/src/hotspot/cpu/riscv/vm_version_riscv.hpp
+++ b/src/hotspot/cpu/riscv/vm_version_riscv.hpp
@@ -55,7 +55,7 @@ class VM_Version : public Abstract_VM_Version {
public:
RVFeatureValue(const char* pretty, int linux_bit_num, bool fstring) :
- _pretty(pretty), _feature_string(fstring), _linux_feature_bit(nth_bit(linux_bit_num)) {
+ _pretty(pretty), _feature_string(fstring), _linux_feature_bit(nth_bit(linux_bit_num)) {
}
virtual void enable_feature(int64_t value = 0) = 0;
virtual void disable_feature() = 0;
diff --git a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp
index 93d6051aa76..e1d8d062c23 100644
--- a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -2251,9 +2251,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
// but not necessarily exactly of type default_type.
NearLabel known_ok, halt;
metadata2reg(default_type->constant_encoding(), tmp);
- if (UseCompressedClassPointers) {
- __ encode_klass_not_null(tmp);
- }
+ __ encode_klass_not_null(tmp);
if (basic_type != T_OBJECT) {
__ cmp_klass(tmp, dst, Z_R1_scratch);
@@ -2540,13 +2538,8 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, L
// Get object class.
// Not a safepoint as obj null check happens earlier.
if (op->fast_check()) {
- if (UseCompressedClassPointers) {
- __ load_klass(klass_RInfo, obj);
- __ compareU64_and_branch(k_RInfo, klass_RInfo, Assembler::bcondNotEqual, *failure_target);
- } else {
- __ z_cg(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
- __ branch_optimized(Assembler::bcondNotEqual, *failure_target);
- }
+ __ load_klass(klass_RInfo, obj);
+ __ compareU64_and_branch(k_RInfo, klass_RInfo, Assembler::bcondNotEqual, *failure_target);
// Successful cast, fall through to profile or jump.
} else {
bool need_slow_path = !k->is_loaded() ||
diff --git a/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp b/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp
index 993c1a1b552..813143938f9 100644
--- a/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -107,10 +107,10 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
}
if (len->is_valid()) {
- // Length will be in the klass gap, if one exists.
+ // Length will be in the klass gap.
z_st(len, Address(obj, arrayOopDesc::length_offset_in_bytes()));
- } else if (UseCompressedClassPointers && !UseCompactObjectHeaders) {
- store_klass_gap(Rzero, obj); // Zero klass gap for compressed oops.
+ } else if (!UseCompactObjectHeaders) {
+ store_klass_gap(Rzero, obj); // Zero klass gap.
}
}
diff --git a/src/hotspot/cpu/s390/c1_globals_s390.hpp b/src/hotspot/cpu/s390/c1_globals_s390.hpp
index 25e46cd1509..64cc239800a 100644
--- a/src/hotspot/cpu/s390/c1_globals_s390.hpp
+++ b/src/hotspot/cpu/s390/c1_globals_s390.hpp
@@ -51,8 +51,6 @@ define_pd_global(size_t, NonNMethodCodeHeapSize, 5*M);
define_pd_global(size_t, CodeCacheExpansionSize, 32*K);
define_pd_global(size_t, CodeCacheMinBlockLength, 1);
define_pd_global(size_t, CodeCacheMinimumUseSpace, 400*K);
-define_pd_global(bool, NeverActAsServerClassMachine, true);
-define_pd_global(size_t, NewSizeThreadIncrease, 16*K);
define_pd_global(size_t, InitialCodeCacheSize, 160*K);
#endif // !COMPILER2
diff --git a/src/hotspot/cpu/s390/c2_globals_s390.hpp b/src/hotspot/cpu/s390/c2_globals_s390.hpp
index 125b317588d..eee3a8588c3 100644
--- a/src/hotspot/cpu/s390/c2_globals_s390.hpp
+++ b/src/hotspot/cpu/s390/c2_globals_s390.hpp
@@ -46,7 +46,6 @@ define_pd_global(intx, OnStackReplacePercentage, 140);
define_pd_global(intx, ConditionalMoveLimit, 4);
define_pd_global(intx, FreqInlineSize, 325);
define_pd_global(intx, InteriorEntryAlignment, 4);
-define_pd_global(size_t, NewSizeThreadIncrease, ScaleForWordSize(4*K));
define_pd_global(intx, RegisterCostAreaRatio, 12000);
define_pd_global(intx, LoopUnrollLimit, 60);
define_pd_global(intx, LoopPercentProfileLimit, 10);
@@ -79,7 +78,4 @@ define_pd_global(size_t, CodeCacheMinimumUseSpace, 400*K);
define_pd_global(bool, TrapBasedRangeChecks, false); // Not needed on z/Architecture.
-// Ergonomics related flags
-define_pd_global(bool, NeverActAsServerClassMachine, false);
-
#endif // CPU_S390_C2_GLOBALS_S390_HPP
diff --git a/src/hotspot/cpu/s390/downcallLinker_s390.cpp b/src/hotspot/cpu/s390/downcallLinker_s390.cpp
index ccd8002da37..f1c41d05b5c 100644
--- a/src/hotspot/cpu/s390/downcallLinker_s390.cpp
+++ b/src/hotspot/cpu/s390/downcallLinker_s390.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2022, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, Red Hat, Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -129,7 +129,7 @@ void DowncallLinker::StubGenerator::generate() {
assert(!_needs_return_buffer, "unexpected needs_return_buffer");
RegSpiller out_reg_spiller(_output_registers);
- int spill_offset = allocated_frame_size;
+ int out_spill_offset = allocated_frame_size;
allocated_frame_size += BytesPerWord;
StubLocations locs;
@@ -153,6 +153,18 @@ void DowncallLinker::StubGenerator::generate() {
GrowableArray out_regs = ForeignGlobals::replace_place_holders(_input_registers, locs);
ArgumentShuffle arg_shuffle(filtered_java_regs, out_regs, _abi._scratch1);
+ // Need to spill for state capturing runtime call.
+ // The area spilled into is distinct from the capture state buffer.
+ RegSpiller in_reg_spiller(out_regs);
+ int in_spill_offset = -1;
+ if (_captured_state_mask != 0) {
+ // The spill area cannot be shared with the out_spill since
+ // spilling needs to happen before the call. Allocate a new
+ // region in the stack for this spill space.
+ in_spill_offset = allocated_frame_size;
+ allocated_frame_size += in_reg_spiller.spill_size_bytes();
+ }
+
#ifndef PRODUCT
LogTarget(Trace, foreign, downcall) lt;
if (lt.is_enabled()) {
@@ -192,6 +204,21 @@ void DowncallLinker::StubGenerator::generate() {
arg_shuffle.generate(_masm, shuffle_reg, frame::z_jit_out_preserve_size, _abi._shadow_space_bytes);
__ block_comment("} argument_shuffle");
+ if (_captured_state_mask != 0) {
+ assert(in_spill_offset != -1, "must be");
+ __ block_comment("{ load initial thread local");
+ in_reg_spiller.generate_spill(_masm, in_spill_offset);
+
+ // Copy the contents of the capture state buffer into thread local
+ __ load_const_optimized(call_target_address, CAST_FROM_FN_PTR(uint64_t, DowncallLinker::capture_state_pre));
+ __ z_lg(Z_ARG1, Address(Z_SP, locs.data_offset(StubLocations::CAPTURED_STATE_BUFFER)));
+ __ load_const_optimized(Z_ARG2, _captured_state_mask);
+ __ call(call_target_address);
+
+ in_reg_spiller.generate_fill(_masm, in_spill_offset);
+ __ block_comment("} load initial thread local");
+ }
+
__ call(as_Register(locs.get(StubLocations::TARGET_ADDRESS)));
//////////////////////////////////////////////////////////////////////////////
@@ -199,14 +226,14 @@ void DowncallLinker::StubGenerator::generate() {
if (_captured_state_mask != 0) {
__ block_comment("save_thread_local {");
- out_reg_spiller.generate_spill(_masm, spill_offset);
+ out_reg_spiller.generate_spill(_masm, out_spill_offset);
- __ load_const_optimized(call_target_address, CAST_FROM_FN_PTR(uint64_t, DowncallLinker::capture_state));
+ __ load_const_optimized(call_target_address, CAST_FROM_FN_PTR(uint64_t, DowncallLinker::capture_state_post));
__ z_lg(Z_ARG1, Address(Z_SP, locs.data_offset(StubLocations::CAPTURED_STATE_BUFFER)));
__ load_const_optimized(Z_ARG2, _captured_state_mask);
__ call(call_target_address);
- out_reg_spiller.generate_fill(_masm, spill_offset);
+ out_reg_spiller.generate_fill(_masm, out_spill_offset);
__ block_comment("} save_thread_local");
}
@@ -259,13 +286,13 @@ void DowncallLinker::StubGenerator::generate() {
__ bind(L_safepoint_poll_slow_path);
// Need to save the native result registers around any runtime calls.
- out_reg_spiller.generate_spill(_masm, spill_offset);
+ out_reg_spiller.generate_spill(_masm, out_spill_offset);
__ load_const_optimized(call_target_address, CAST_FROM_FN_PTR(uint64_t, JavaThread::check_special_condition_for_native_trans));
__ z_lgr(Z_ARG1, Z_thread);
__ call(call_target_address);
- out_reg_spiller.generate_fill(_masm, spill_offset);
+ out_reg_spiller.generate_fill(_masm, out_spill_offset);
__ z_bru(L_after_safepoint_poll);
__ block_comment("} L_safepoint_poll_slow_path");
@@ -275,12 +302,12 @@ void DowncallLinker::StubGenerator::generate() {
__ bind(L_reguard);
// Need to save the native result registers around any runtime calls.
- out_reg_spiller.generate_spill(_masm, spill_offset);
+ out_reg_spiller.generate_spill(_masm, out_spill_offset);
__ load_const_optimized(call_target_address, CAST_FROM_FN_PTR(uint64_t, SharedRuntime::reguard_yellow_pages));
__ call(call_target_address);
- out_reg_spiller.generate_fill(_masm, spill_offset);
+ out_reg_spiller.generate_fill(_masm, out_spill_offset);
__ z_bru(L_after_reguard);
diff --git a/src/hotspot/cpu/s390/frame_s390.hpp b/src/hotspot/cpu/s390/frame_s390.hpp
index ad754706367..bcdeec43e1a 100644
--- a/src/hotspot/cpu/s390/frame_s390.hpp
+++ b/src/hotspot/cpu/s390/frame_s390.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -463,7 +463,7 @@
// Accessors
- inline intptr_t* fp() const { return _fp; }
+ inline intptr_t* fp() const { assert_absolute(); return _fp; }
private:
diff --git a/src/hotspot/cpu/s390/frame_s390.inline.hpp b/src/hotspot/cpu/s390/frame_s390.inline.hpp
index dea0e72581f..6fcd36c57d1 100644
--- a/src/hotspot/cpu/s390/frame_s390.inline.hpp
+++ b/src/hotspot/cpu/s390/frame_s390.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -133,10 +133,10 @@ inline void frame::interpreter_frame_set_monitors(BasicObjectLock* monitors) {
// Return unique id for this frame. The id must have a value where we
// can distinguish identity and younger/older relationship. null
-// represents an invalid (incomparable) frame.
+// represents an invalid (incomparable) frame. Should not be called for heap frames.
inline intptr_t* frame::id(void) const {
// Use _fp. _sp or _unextended_sp wouldn't be correct due to resizing.
- return _fp;
+ return real_fp();
}
// Return true if this frame is older (less recent activation) than
diff --git a/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.cpp b/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.cpp
index c6f5a4e119c..9fac231df47 100644
--- a/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2022, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -169,6 +169,11 @@ void BarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Re
__ z_lg(obj, 0, obj); // Resolve (untagged) jobject.
}
+void BarrierSetAssembler::try_resolve_weak_handle(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path) {
+ // Load the oop from the weak handle.
+ __ z_lg(obj, Address(obj));
+}
+
void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm) {
BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
__ align(4, __ offset() + OFFSET_TO_PATCHABLE_DATA); // must align the following block which requires atomic updates
diff --git a/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.hpp b/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.hpp
index 65db915b672..8e76ec2f4b4 100644
--- a/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.hpp
+++ b/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2022, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -58,6 +58,11 @@ public:
virtual void try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
Register obj, Register tmp, Label& slowpath);
+ // Can be used in nmethods including native wrappers.
+ // Attention: obj will only be valid until next safepoint (no SATB barrier).
+ // (other platforms currently use it for C2 only: try_resolve_weak_handle_in_c2)
+ virtual void try_resolve_weak_handle(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path);
+
virtual void nmethod_entry_barrier(MacroAssembler* masm);
virtual void barrier_stubs_init() {}
diff --git a/src/hotspot/cpu/s390/interp_masm_s390.cpp b/src/hotspot/cpu/s390/interp_masm_s390.cpp
index a80ca26239b..d5239898dd7 100644
--- a/src/hotspot/cpu/s390/interp_masm_s390.cpp
+++ b/src/hotspot/cpu/s390/interp_masm_s390.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -1259,27 +1259,15 @@ void InterpreterMacroAssembler::profile_final_call(Register mdp) {
void InterpreterMacroAssembler::profile_virtual_call(Register receiver,
Register mdp,
- Register reg2,
- bool receiver_can_be_null) {
+ Register reg2) {
if (ProfileInterpreter) {
NearLabel profile_continue;
// If no method data exists, go to profile_continue.
test_method_data_pointer(mdp, profile_continue);
- NearLabel skip_receiver_profile;
- if (receiver_can_be_null) {
- NearLabel not_null;
- compareU64_and_branch(receiver, (intptr_t)0L, bcondNotEqual, not_null);
- // We are making a call. Increment the count for null receiver.
- increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
- z_bru(skip_receiver_profile);
- bind(not_null);
- }
-
// Record the receiver type.
record_klass_in_profile(receiver, mdp, reg2);
- bind(skip_receiver_profile);
// The method data pointer needs to be updated to reflect the new target.
update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size()));
diff --git a/src/hotspot/cpu/s390/interp_masm_s390.hpp b/src/hotspot/cpu/s390/interp_masm_s390.hpp
index d981f9ea01e..b816185b065 100644
--- a/src/hotspot/cpu/s390/interp_masm_s390.hpp
+++ b/src/hotspot/cpu/s390/interp_masm_s390.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -296,8 +296,7 @@ class InterpreterMacroAssembler: public MacroAssembler {
void profile_call(Register mdp);
void profile_final_call(Register mdp);
void profile_virtual_call(Register receiver, Register mdp,
- Register scratch2,
- bool receiver_can_be_null = false);
+ Register scratch2);
void profile_ret(Register return_bci, Register mdp);
void profile_null_seen(Register mdp);
void profile_typecheck(Register mdp, Register klass, Register scratch);
diff --git a/src/hotspot/cpu/s390/macroAssembler_s390.cpp b/src/hotspot/cpu/s390/macroAssembler_s390.cpp
index f35e18c7398..de3608e74ba 100644
--- a/src/hotspot/cpu/s390/macroAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/macroAssembler_s390.cpp
@@ -1,7 +1,7 @@
/*
- * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2024 SAP SE. All rights reserved.
- * Copyright 2024 IBM Corporation. All rights reserved.
+ * Copyright 2024, 2026 IBM Corporation. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -44,6 +44,7 @@
#include "runtime/icache.hpp"
#include "runtime/interfaceSupport.inline.hpp"
#include "runtime/objectMonitor.hpp"
+#include "runtime/objectMonitorTable.hpp"
#include "runtime/os.hpp"
#include "runtime/safepoint.hpp"
#include "runtime/safepointMechanism.hpp"
@@ -1236,7 +1237,6 @@ void MacroAssembler::load_narrow_oop(Register t, narrowOop a) {
// Load narrow klass constant, compression required.
void MacroAssembler::load_narrow_klass(Register t, Klass* k) {
- assert(UseCompressedClassPointers, "must be on to call this method");
narrowKlass encoded_k = CompressedKlassPointers::encode(k);
load_const_32to64(t, encoded_k, false /*sign_extend*/);
}
@@ -1254,7 +1254,6 @@ void MacroAssembler::compare_immediate_narrow_oop(Register oop1, narrowOop oop2)
// Compare narrow oop in reg with narrow oop constant, no decompression.
void MacroAssembler::compare_immediate_narrow_klass(Register klass1, Klass* klass2) {
- assert(UseCompressedClassPointers, "must be on to call this method");
narrowKlass encoded_k = CompressedKlassPointers::encode(klass2);
Assembler::z_clfi(klass1, encoded_k);
@@ -1347,8 +1346,6 @@ int MacroAssembler::patch_load_narrow_oop(address pos, oop o) {
// Patching the immediate value of CPU version dependent load_narrow_klass sequence.
// The passed ptr must NOT be in compressed format!
int MacroAssembler::patch_load_narrow_klass(address pos, Klass* k) {
- assert(UseCompressedClassPointers, "Can only patch compressed klass pointers");
-
narrowKlass nk = CompressedKlassPointers::encode(k);
return patch_load_const_32to64(pos, nk);
}
@@ -1363,8 +1360,6 @@ int MacroAssembler::patch_compare_immediate_narrow_oop(address pos, oop o) {
// Patching the immediate value of CPU version dependent compare_immediate_narrow_klass sequence.
// The passed ptr must NOT be in compressed format!
int MacroAssembler::patch_compare_immediate_narrow_klass(address pos, Klass* k) {
- assert(UseCompressedClassPointers, "Can only patch compressed klass pointers");
-
narrowKlass nk = CompressedKlassPointers::encode(k);
return patch_compare_immediate_32(pos, nk);
}
@@ -2234,10 +2229,8 @@ int MacroAssembler::ic_check(int end_alignment) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(R1_scratch, R2_receiver);
- } else if (UseCompressedClassPointers) {
- z_llgf(R1_scratch, Address(R2_receiver, oopDesc::klass_offset_in_bytes()));
} else {
- z_lg(R1_scratch, Address(R2_receiver, oopDesc::klass_offset_in_bytes()));
+ z_llgf(R1_scratch, Address(R2_receiver, oopDesc::klass_offset_in_bytes()));
}
z_cg(R1_scratch, Address(R9_data, in_bytes(CompiledICData::speculated_klass_offset())));
z_bre(success);
@@ -3915,7 +3908,6 @@ void MacroAssembler::encode_klass_not_null(Register dst, Register src) {
address base = CompressedKlassPointers::base();
int shift = CompressedKlassPointers::shift();
bool need_zero_extend = base != nullptr;
- assert(UseCompressedClassPointers, "only for compressed klass ptrs");
BLOCK_COMMENT("cKlass encoder {");
@@ -4012,7 +4004,6 @@ int MacroAssembler::instr_size_for_decode_klass_not_null() {
address base = CompressedKlassPointers::base();
int shift_size = CompressedKlassPointers::shift() == 0 ? 0 : 6; /* sllg */
int addbase_size = 0;
- assert(UseCompressedClassPointers, "only for compressed klass ptrs");
if (base != nullptr) {
unsigned int base_h = ((unsigned long)base)>>32;
@@ -4042,7 +4033,6 @@ void MacroAssembler::decode_klass_not_null(Register dst) {
address base = CompressedKlassPointers::base();
int shift = CompressedKlassPointers::shift();
int beg_off = offset();
- assert(UseCompressedClassPointers, "only for compressed klass ptrs");
BLOCK_COMMENT("cKlass decoder (const size) {");
@@ -4084,7 +4074,6 @@ void MacroAssembler::decode_klass_not_null(Register dst) {
void MacroAssembler::decode_klass_not_null(Register dst, Register src) {
address base = CompressedKlassPointers::base();
int shift = CompressedKlassPointers::shift();
- assert(UseCompressedClassPointers, "only for compressed klass ptrs");
BLOCK_COMMENT("cKlass decoder {");
@@ -4124,13 +4113,9 @@ void MacroAssembler::decode_klass_not_null(Register dst, Register src) {
}
void MacroAssembler::load_klass(Register klass, Address mem) {
- if (UseCompressedClassPointers) {
- z_llgf(klass, mem);
- // Attention: no null check here!
- decode_klass_not_null(klass);
- } else {
- z_lg(klass, mem);
- }
+ z_llgf(klass, mem);
+ // Attention: no null check here!
+ decode_klass_not_null(klass);
}
// Loads the obj's Klass* into dst.
@@ -4153,10 +4138,8 @@ void MacroAssembler::cmp_klass(Register klass, Register obj, Register tmp) {
assert_different_registers(klass, obj, tmp);
load_narrow_klass_compact(tmp, obj);
z_cr(klass, tmp);
- } else if (UseCompressedClassPointers) {
- z_c(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
} else {
- z_cg(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
+ z_c(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
}
BLOCK_COMMENT("} cmp_klass");
}
@@ -4169,12 +4152,9 @@ void MacroAssembler::cmp_klasses_from_objects(Register obj1, Register obj2, Regi
load_narrow_klass_compact(tmp1, obj1);
load_narrow_klass_compact(tmp2, obj2);
z_cr(tmp1, tmp2);
- } else if (UseCompressedClassPointers) {
+ } else {
z_l(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
z_c(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes()));
- } else {
- z_lg(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
- z_cg(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes()));
}
BLOCK_COMMENT("} cmp_klasses_from_objects");
}
@@ -4183,36 +4163,28 @@ void MacroAssembler::load_klass(Register klass, Register src_oop) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(klass, src_oop);
decode_klass_not_null(klass);
- } else if (UseCompressedClassPointers) {
+ } else {
z_llgf(klass, oopDesc::klass_offset_in_bytes(), src_oop);
decode_klass_not_null(klass);
- } else {
- z_lg(klass, oopDesc::klass_offset_in_bytes(), src_oop);
}
}
void MacroAssembler::store_klass(Register klass, Register dst_oop, Register ck) {
assert(!UseCompactObjectHeaders, "Don't use with compact headers");
- if (UseCompressedClassPointers) {
- assert_different_registers(dst_oop, klass, Z_R0);
- if (ck == noreg) ck = klass;
- encode_klass_not_null(ck, klass);
- z_st(ck, Address(dst_oop, oopDesc::klass_offset_in_bytes()));
- } else {
- z_stg(klass, Address(dst_oop, oopDesc::klass_offset_in_bytes()));
- }
+ assert_different_registers(dst_oop, klass, Z_R0);
+ if (ck == noreg) ck = klass;
+ encode_klass_not_null(ck, klass);
+ z_st(ck, Address(dst_oop, oopDesc::klass_offset_in_bytes()));
}
void MacroAssembler::store_klass_gap(Register s, Register d) {
assert(!UseCompactObjectHeaders, "Don't use with compact headers");
- if (UseCompressedClassPointers) {
- assert(s != d, "not enough registers");
- // Support s = noreg.
- if (s != noreg) {
- z_st(s, Address(d, oopDesc::klass_gap_offset_in_bytes()));
- } else {
- z_mvhi(Address(d, oopDesc::klass_gap_offset_in_bytes()), 0);
- }
+ assert(s != d, "not enough registers");
+ // Support s = noreg.
+ if (s != noreg) {
+ z_st(s, Address(d, oopDesc::klass_gap_offset_in_bytes()));
+ } else {
+ z_mvhi(Address(d, oopDesc::klass_gap_offset_in_bytes()), 0);
}
}
@@ -4226,67 +4198,64 @@ void MacroAssembler::compare_klass_ptr(Register Rop1, int64_t disp, Register Rba
BLOCK_COMMENT("compare klass ptr {");
- if (UseCompressedClassPointers) {
- const int shift = CompressedKlassPointers::shift();
- address base = CompressedKlassPointers::base();
+ const int shift = CompressedKlassPointers::shift();
+ address base = CompressedKlassPointers::base();
- if (UseCompactObjectHeaders) {
- assert(shift >= 3, "cKlass encoder detected bad shift");
- } else {
- assert((shift == 0) || (shift == 3), "cKlass encoder detected bad shift");
- }
- assert_different_registers(Rop1, Z_R0);
- assert_different_registers(Rop1, Rbase, Z_R1);
-
- // First encode register oop and then compare with cOop in memory.
- // This sequence saves an unnecessary cOop load and decode.
- if (base == nullptr) {
- if (shift == 0) {
- z_cl(Rop1, disp, Rbase); // Unscaled
- } else {
- z_srlg(Z_R0, Rop1, shift); // ZeroBased
- z_cl(Z_R0, disp, Rbase);
- }
- } else { // HeapBased
-#ifdef ASSERT
- bool used_R0 = true;
- bool used_R1 = true;
-#endif
- Register current = Rop1;
- Label done;
-
- if (maybenull) { // null pointer must be preserved!
- z_ltgr(Z_R0, current);
- z_bre(done);
- current = Z_R0;
- }
-
- unsigned int base_h = ((unsigned long)base)>>32;
- unsigned int base_l = (unsigned int)((unsigned long)base);
- if ((base_h != 0) && (base_l == 0) && VM_Version::has_HighWordInstr()) {
- lgr_if_needed(Z_R0, current);
- z_aih(Z_R0, -((int)base_h)); // Base has no set bits in lower half.
- } else if ((base_h == 0) && (base_l != 0)) {
- lgr_if_needed(Z_R0, current);
- z_agfi(Z_R0, -(int)base_l);
- } else {
- int pow2_offset = get_oop_base_complement(Z_R1, ((uint64_t)(intptr_t)base));
- add2reg_with_index(Z_R0, pow2_offset, Z_R1, Rop1); // Subtract base by adding complement.
- }
-
- if (shift != 0) {
- z_srlg(Z_R0, Z_R0, shift);
- }
- bind(done);
- z_cl(Z_R0, disp, Rbase);
-#ifdef ASSERT
- if (used_R0) preset_reg(Z_R0, 0xb05bUL, 2);
- if (used_R1) preset_reg(Z_R1, 0xb06bUL, 2);
-#endif
- }
+ if (UseCompactObjectHeaders) {
+ assert(shift >= 3, "cKlass encoder detected bad shift");
} else {
- z_clg(Rop1, disp, Z_R0, Rbase);
+ assert((shift == 0) || (shift == 3), "cKlass encoder detected bad shift");
}
+ assert_different_registers(Rop1, Z_R0);
+ assert_different_registers(Rop1, Rbase, Z_R1);
+
+ // First encode register oop and then compare with cOop in memory.
+ // This sequence saves an unnecessary cOop load and decode.
+ if (base == nullptr) {
+ if (shift == 0) {
+ z_cl(Rop1, disp, Rbase); // Unscaled
+ } else {
+ z_srlg(Z_R0, Rop1, shift); // ZeroBased
+ z_cl(Z_R0, disp, Rbase);
+ }
+ } else { // HeapBased
+#ifdef ASSERT
+ bool used_R0 = true;
+ bool used_R1 = true;
+#endif
+ Register current = Rop1;
+ Label done;
+
+ if (maybenull) { // null pointer must be preserved!
+ z_ltgr(Z_R0, current);
+ z_bre(done);
+ current = Z_R0;
+ }
+
+ unsigned int base_h = ((unsigned long)base)>>32;
+ unsigned int base_l = (unsigned int)((unsigned long)base);
+ if ((base_h != 0) && (base_l == 0) && VM_Version::has_HighWordInstr()) {
+ lgr_if_needed(Z_R0, current);
+ z_aih(Z_R0, -((int)base_h)); // Base has no set bits in lower half.
+ } else if ((base_h == 0) && (base_l != 0)) {
+ lgr_if_needed(Z_R0, current);
+ z_agfi(Z_R0, -(int)base_l);
+ } else {
+ int pow2_offset = get_oop_base_complement(Z_R1, ((uint64_t)(intptr_t)base));
+ add2reg_with_index(Z_R0, pow2_offset, Z_R1, Rop1); // Subtract base by adding complement.
+ }
+
+ if (shift != 0) {
+ z_srlg(Z_R0, Z_R0, shift);
+ }
+ bind(done);
+ z_cl(Z_R0, disp, Rbase);
+#ifdef ASSERT
+ if (used_R0) preset_reg(Z_R0, 0xb05bUL, 2);
+ if (used_R1) preset_reg(Z_R1, 0xb06bUL, 2);
+#endif
+ }
+
BLOCK_COMMENT("} compare klass ptr");
}
@@ -6372,45 +6341,55 @@ void MacroAssembler::compiler_fast_lock_object(Register obj, Register box, Regis
if (!UseObjectMonitorTable) {
assert(tmp1_monitor == mark, "should be the same here");
} else {
+ const Register tmp1_bucket = tmp1;
+ const Register hash = Z_R0_scratch;
NearLabel monitor_found;
- // load cache address
- z_la(tmp1, Address(Z_thread, JavaThread::om_cache_oops_offset()));
+ // Save the mark, we might need it to extract the hash.
+ z_lgr(hash, mark);
- const int num_unrolled = 2;
+ // Look for the monitor in the om_cache.
+
+ ByteSize cache_offset = JavaThread::om_cache_oops_offset();
+ ByteSize monitor_offset = OMCache::oop_to_monitor_difference();
+ const int num_unrolled = OMCache::CAPACITY;
for (int i = 0; i < num_unrolled; i++) {
- z_cg(obj, Address(tmp1));
+ z_lg(tmp1_monitor, Address(Z_thread, cache_offset + monitor_offset));
+ z_cg(obj, Address(Z_thread, cache_offset));
z_bre(monitor_found);
- add2reg(tmp1, in_bytes(OMCache::oop_to_oop_difference()));
+ cache_offset = cache_offset + OMCache::oop_to_oop_difference();
}
- NearLabel loop;
- // Search for obj in cache
+ // Get the hash code.
+ z_srlg(hash, hash, markWord::hash_shift);
- bind(loop);
+ // Get the table and calculate the bucket's address.
+ load_const_optimized(tmp2, ObjectMonitorTable::current_table_address());
+ z_lg(tmp2, Address(tmp2));
+ z_ng(hash, Address(tmp2, ObjectMonitorTable::table_capacity_mask_offset()));
+ z_lg(tmp1_bucket, Address(tmp2, ObjectMonitorTable::table_buckets_offset()));
+ z_sllg(hash, hash, LogBytesPerWord);
+ z_agr(tmp1_bucket, hash);
- // check for match.
- z_cg(obj, Address(tmp1));
- z_bre(monitor_found);
+ // Read the monitor from the bucket.
+ z_lg(tmp1_monitor, Address(tmp1_bucket));
- // search until null encountered, guaranteed _null_sentinel at end.
- add2reg(tmp1, in_bytes(OMCache::oop_to_oop_difference()));
- z_cghsi(0, tmp1, 0);
- z_brne(loop); // if not EQ to 0, go for another loop
+ // Check if the monitor in the bucket is special (empty, tombstone or removed).
+ z_clgfi(tmp1_monitor, ObjectMonitorTable::SpecialPointerValues::below_is_special);
+ z_brl(slow_path);
- // we reached to the end, cache miss
- z_ltgr(obj, obj); // set CC to NE
- z_bru(slow_path);
+ // Check if object matches.
+ z_lg(tmp2, Address(tmp1_monitor, ObjectMonitor::object_offset()));
+ BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
+ bs_asm->try_resolve_weak_handle(this, tmp2, Z_R0_scratch, slow_path);
+ z_cgr(obj, tmp2);
+ z_brne(slow_path);
- // cache hit
bind(monitor_found);
- z_lg(tmp1_monitor, Address(tmp1, OMCache::oop_to_monitor_difference()));
}
NearLabel monitor_locked;
// lock the monitor
- // mark contains the tagged ObjectMonitor*.
- const Register tagged_monitor = mark;
const Register zero = tmp2;
const ByteSize monitor_tag = in_ByteSize(UseObjectMonitorTable ? 0 : checked_cast(markWord::monitor_value));
diff --git a/src/hotspot/cpu/s390/macroAssembler_s390.hpp b/src/hotspot/cpu/s390/macroAssembler_s390.hpp
index da24ae80d45..32e484d4790 100644
--- a/src/hotspot/cpu/s390/macroAssembler_s390.hpp
+++ b/src/hotspot/cpu/s390/macroAssembler_s390.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2024 SAP SE. All rights reserved.
* Copyright (c) 2024 IBM Corporation. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -842,8 +842,7 @@ class MacroAssembler: public Assembler {
void store_klass(Register klass, Register dst_oop, Register ck = noreg); // Klass will get compressed if ck not provided.
void store_klass_gap(Register s, Register dst_oop);
void load_narrow_klass_compact(Register dst, Register src);
- // Compares the Klass pointer of an object to a given Klass (which might be narrow,
- // depending on UseCompressedClassPointers).
+ // Compares the narrow Klass pointer of an object to a given narrow Klass
void cmp_klass(Register klass, Register obj, Register tmp);
// Compares the Klass pointer of two objects obj1 and obj2. Result is in the condition flags.
// Uses tmp1 and tmp2 as temporary registers.
diff --git a/src/hotspot/cpu/s390/matcher_s390.hpp b/src/hotspot/cpu/s390/matcher_s390.hpp
index 99461e33e3c..b04a6566d41 100644
--- a/src/hotspot/cpu/s390/matcher_s390.hpp
+++ b/src/hotspot/cpu/s390/matcher_s390.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -82,7 +82,6 @@
static bool narrow_klass_use_complex_address() {
NOT_LP64(ShouldNotCallThis());
- assert(UseCompressedClassPointers, "only for compressed klass code");
// TODO HS25: z port if (MatchDecodeNodes) return true;
return false;
}
diff --git a/src/hotspot/cpu/s390/methodHandles_s390.cpp b/src/hotspot/cpu/s390/methodHandles_s390.cpp
index e3de6d911be..dfb8ce09b27 100644
--- a/src/hotspot/cpu/s390/methodHandles_s390.cpp
+++ b/src/hotspot/cpu/s390/methodHandles_s390.cpp
@@ -120,16 +120,12 @@ void MethodHandles::verify_ref_kind(MacroAssembler* _masm, int ref_kind,
__ z_nilf(temp, java_lang_invoke_MemberName::MN_REFERENCE_KIND_MASK);
__ compare32_and_branch(temp, constant(ref_kind), Assembler::bcondEqual, L);
- {
- char *buf = NEW_C_HEAP_ARRAY(char, 100, mtInternal);
-
- jio_snprintf(buf, 100, "verify_ref_kind expected %x", ref_kind);
- if (ref_kind == JVM_REF_invokeVirtual || ref_kind == JVM_REF_invokeSpecial) {
- // Could do this for all ref_kinds, but would explode assembly code size.
- trace_method_handle(_masm, buf);
- }
- __ stop(buf);
+ const char* msg = ref_kind_to_verify_msg(ref_kind);
+ if (ref_kind == JVM_REF_invokeVirtual || ref_kind == JVM_REF_invokeSpecial) {
+ // Could do this for all ref_kinds, but would explode assembly code size.
+ trace_method_handle(_masm, msg);
}
+ __ stop(msg);
BLOCK_COMMENT("} verify_ref_kind");
diff --git a/src/hotspot/cpu/s390/s390.ad b/src/hotspot/cpu/s390/s390.ad
index 1a4a21425f1..dcb16e124e6 100644
--- a/src/hotspot/cpu/s390/s390.ad
+++ b/src/hotspot/cpu/s390/s390.ad
@@ -1,5 +1,5 @@
//
-// Copyright (c) 2017, 2025, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2017, 2026, Oracle and/or its affiliates. All rights reserved.
// Copyright (c) 2017, 2024 SAP SE. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
@@ -1890,10 +1890,8 @@ const int z_num_iarg_registers = sizeof(z_iarg_reg) / sizeof(z_iarg_reg[0]);
const int z_num_farg_registers = sizeof(z_farg_reg) / sizeof(z_farg_reg[0]);
-// Return whether or not this register is ever used as an argument. This
-// function is used on startup to build the trampoline stubs in generateOptoStub.
-// Registers not mentioned will be killed by the VM call in the trampoline, and
-// arguments in those registers not be available to the callee.
+#ifdef ASSERT
+// Return whether or not this register is ever used as an argument.
bool Matcher::can_be_java_arg(int reg) {
// We return true for all registers contained in z_iarg_reg[] and
// z_farg_reg[] and their virtual halves.
@@ -1917,10 +1915,7 @@ bool Matcher::can_be_java_arg(int reg) {
return false;
}
-
-bool Matcher::is_spillable_arg(int reg) {
- return can_be_java_arg(reg);
-}
+#endif
uint Matcher::int_pressure_limit()
{
@@ -1934,10 +1929,6 @@ uint Matcher::float_pressure_limit()
return (FLOATPRESSURE == -1) ? 15 : FLOATPRESSURE;
}
-bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
- return false;
-}
-
// Register for DIVI projection of divmodI
const RegMask& Matcher::divI_proj_mask() {
return _Z_RARG4_INT_REG_mask;
@@ -2606,13 +2597,6 @@ frame %{
// z/Architecture stack pointer
frame_pointer(Z_R15); // Z_SP
- // Interpreter stores its frame pointer in a register which is
- // stored to the stack by I2CAdaptors. I2CAdaptors convert from
- // interpreted java to compiled java.
- //
- // Z_state holds pointer to caller's cInterpreter.
- interpreter_frame_pointer(Z_R7); // Z_state
-
// Use alignment_in_bytes instead of log_2_of_alignment_in_bits.
stack_alignment(frame::alignment_in_bytes);
@@ -5251,6 +5235,15 @@ instruct membar_release_lock() %{
ins_pipe(pipe_class_dummy);
%}
+instruct membar_storeload() %{
+ match(MemBarStoreLoad);
+ ins_cost(4 * MEMORY_REF_COST);
+ size(2);
+ format %{ "MEMBAR-storeload" %}
+ ins_encode %{ __ z_fence(); %}
+ ins_pipe(pipe_class_dummy);
+%}
+
instruct membar_volatile() %{
match(MemBarVolatile);
ins_cost(4 * MEMORY_REF_COST);
@@ -5270,6 +5263,15 @@ instruct unnecessary_membar_volatile() %{
ins_pipe(pipe_class_dummy);
%}
+instruct membar_full() %{
+ match(MemBarFull);
+ ins_cost(4 * MEMORY_REF_COST);
+ size(2);
+ format %{ "MEMBAR-full" %}
+ ins_encode %{ __ z_fence(); %}
+ ins_pipe(pipe_class_dummy);
+%}
+
instruct membar_CPUOrder() %{
match(MemBarCPUOrder);
ins_cost(0);
diff --git a/src/hotspot/cpu/x86/assembler_x86.cpp b/src/hotspot/cpu/x86/assembler_x86.cpp
index 3c8defe62d9..a4f2968f0d1 100644
--- a/src/hotspot/cpu/x86/assembler_x86.cpp
+++ b/src/hotspot/cpu/x86/assembler_x86.cpp
@@ -3472,7 +3472,7 @@ void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
emit_int16(0x6F, (0xC0 | encode));
}
-void Assembler::vmovw(XMMRegister dst, Register src) {
+void Assembler::evmovw(XMMRegister dst, Register src) {
assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
attributes.set_is_evex_instruction();
@@ -3480,7 +3480,7 @@ void Assembler::vmovw(XMMRegister dst, Register src) {
emit_int16(0x6E, (0xC0 | encode));
}
-void Assembler::vmovw(Register dst, XMMRegister src) {
+void Assembler::evmovw(Register dst, XMMRegister src) {
assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
attributes.set_is_evex_instruction();
@@ -3488,6 +3488,36 @@ void Assembler::vmovw(Register dst, XMMRegister src) {
emit_int16(0x7E, (0xC0 | encode));
}
+void Assembler::evmovw(XMMRegister dst, Address src) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
+ attributes.set_is_evex_instruction();
+ vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
+ emit_int8(0x6E);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::evmovw(Address dst, XMMRegister src) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
+ attributes.set_is_evex_instruction();
+ vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
+ emit_int8(0x7E);
+ emit_operand(src, dst, 0);
+}
+
+void Assembler::evmovw(XMMRegister dst, XMMRegister src) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ attributes.set_is_evex_instruction();
+ int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
+ emit_int16(0x6E, (0xC0 | encode));
+}
+
void Assembler::vmovdqu(XMMRegister dst, Address src) {
assert(UseAVX > 0, "");
InstructionMark im(this);
@@ -5442,6 +5472,13 @@ void Assembler::pmovsxwd(XMMRegister dst, XMMRegister src) {
emit_int16(0x23, (0xC0 | encode));
}
+void Assembler::pmovzxwd(XMMRegister dst, XMMRegister src) {
+ assert(VM_Version::supports_sse4_1(), "");
+ InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+ emit_int16(0x33, (0xC0 | encode));
+}
+
void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
assert(VM_Version::supports_avx(), "");
InstructionMark im(this);
@@ -7303,6 +7340,42 @@ void Assembler::etzcntq(Register dst, Address src, bool no_flags) {
emit_operand(dst, src, 0);
}
+void Assembler::evucomish(XMMRegister dst, Address src) {
+ InstructionMark im(this);
+ InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
+ attributes.set_is_evex_instruction();
+ vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes);
+ emit_int8(0x2E);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::evucomish(XMMRegister dst, XMMRegister src) {
+ InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ attributes.set_is_evex_instruction();
+ int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes);
+ emit_int16(0x2E, (0xC0 | encode));
+}
+
+void Assembler::evucomxsh(XMMRegister dst, Address src) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
+ attributes.set_is_evex_instruction();
+ vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
+ emit_int8(0x2E);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::evucomxsh(XMMRegister dst, XMMRegister src) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ attributes.set_is_evex_instruction();
+ int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
+ emit_int16(0x2E, (0xC0 | encode));
+}
+
void Assembler::ucomisd(XMMRegister dst, Address src) {
InstructionMark im(this);
InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
@@ -7320,7 +7393,7 @@ void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
emit_int16(0x2E, (0xC0 | encode));
}
-void Assembler::vucomxsd(XMMRegister dst, Address src) {
+void Assembler::evucomxsd(XMMRegister dst, Address src) {
assert(VM_Version::supports_avx10_2(), "");
InstructionMark im(this);
InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
@@ -7331,7 +7404,7 @@ void Assembler::vucomxsd(XMMRegister dst, Address src) {
emit_operand(dst, src, 0);
}
-void Assembler::vucomxsd(XMMRegister dst, XMMRegister src) {
+void Assembler::evucomxsd(XMMRegister dst, XMMRegister src) {
assert(VM_Version::supports_avx10_2(), "");
InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
attributes.set_is_evex_instruction();
@@ -7354,7 +7427,7 @@ void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
emit_int16(0x2E, (0xC0 | encode));
}
-void Assembler::vucomxss(XMMRegister dst, Address src) {
+void Assembler::evucomxss(XMMRegister dst, Address src) {
assert(VM_Version::supports_avx10_2(), "");
InstructionMark im(this);
InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
@@ -7365,7 +7438,7 @@ void Assembler::vucomxss(XMMRegister dst, Address src) {
emit_operand(dst, src, 0);
}
-void Assembler::vucomxss(XMMRegister dst, XMMRegister src) {
+void Assembler::evucomxss(XMMRegister dst, XMMRegister src) {
assert(VM_Version::supports_avx10_2(), "");
InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
attributes.set_is_evex_instruction();
@@ -8404,30 +8477,6 @@ void Assembler::vmulsh(XMMRegister dst, XMMRegister nds, XMMRegister src) {
emit_int16(0x59, (0xC0 | encode));
}
-void Assembler::vmaxsh(XMMRegister dst, XMMRegister nds, XMMRegister src) {
- assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
- InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- attributes.set_is_evex_instruction();
- int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
- emit_int16(0x5F, (0xC0 | encode));
-}
-
-void Assembler::eminmaxsh(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
- assert(VM_Version::supports_avx10_2(), "");
- InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- attributes.set_is_evex_instruction();
- int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, &attributes);
- emit_int24(0x53, (0xC0 | encode), imm8);
-}
-
-void Assembler::vminsh(XMMRegister dst, XMMRegister nds, XMMRegister src) {
- assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
- InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- attributes.set_is_evex_instruction();
- int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
- emit_int16(0x5D, (0xC0 | encode));
-}
-
void Assembler::vsqrtsh(XMMRegister dst, XMMRegister src) {
assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
@@ -13362,48 +13411,38 @@ bool Assembler::is_demotable(bool no_flags, int dst_enc, int nds_enc) {
return (!no_flags && dst_enc == nds_enc);
}
-void Assembler::vmaxss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
- assert(VM_Version::supports_avx(), "");
- InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
- emit_int16(0x5F, (0xC0 | encode));
-}
-
-void Assembler::vmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
- assert(VM_Version::supports_avx(), "");
- InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- attributes.set_rex_vex_w_reverted();
- int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
- emit_int16(0x5F, (0xC0 | encode));
-}
-
-void Assembler::vminss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
- assert(VM_Version::supports_avx(), "");
- InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
- emit_int16(0x5D, (0xC0 | encode));
-}
-
-void Assembler::eminmaxss(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
+void Assembler::evminmaxsh(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int imm8) {
assert(VM_Version::supports_avx10_2(), "");
- InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, &attributes);
+ emit_int24(0x53, (0xC0 | encode), imm8);
+}
+
+void Assembler::evminmaxss(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int imm8) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
+ attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
emit_int24(0x53, (0xC0 | encode), imm8);
}
-void Assembler::vminsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
- assert(VM_Version::supports_avx(), "");
- InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- attributes.set_rex_vex_w_reverted();
- int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
- emit_int16(0x5D, (0xC0 | encode));
-}
-
-void Assembler::eminmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
+void Assembler::evminmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int imm8) {
assert(VM_Version::supports_avx10_2(), "");
- InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
emit_int24(0x53, (0xC0 | encode), imm8);
}
diff --git a/src/hotspot/cpu/x86/assembler_x86.hpp b/src/hotspot/cpu/x86/assembler_x86.hpp
index 97854f712cf..98684752b0c 100644
--- a/src/hotspot/cpu/x86/assembler_x86.hpp
+++ b/src/hotspot/cpu/x86/assembler_x86.hpp
@@ -1694,8 +1694,11 @@ private:
void movsbl(Register dst, Address src);
void movsbl(Register dst, Register src);
- void vmovw(XMMRegister dst, Register src);
- void vmovw(Register dst, XMMRegister src);
+ void evmovw(XMMRegister dst, Register src);
+ void evmovw(Register dst, XMMRegister src);
+ void evmovw(XMMRegister dst, Address src);
+ void evmovw(Address dst, XMMRegister src);
+ void evmovw(XMMRegister dst, XMMRegister src);
void movsbq(Register dst, Address src);
void movsbq(Register dst, Register src);
@@ -1965,6 +1968,7 @@ private:
void pmovsxbq(XMMRegister dst, XMMRegister src);
void pmovsxbw(XMMRegister dst, XMMRegister src);
void pmovsxwd(XMMRegister dst, XMMRegister src);
+ void pmovzxwd(XMMRegister dst, XMMRegister src);
void vpmovsxbd(XMMRegister dst, XMMRegister src, int vector_len);
void vpmovsxbq(XMMRegister dst, XMMRegister src, int vector_len);
void vpmovsxbw(XMMRegister dst, XMMRegister src, int vector_len);
@@ -2328,17 +2332,23 @@ private:
void tzcntq(Register dst, Address src);
void etzcntq(Register dst, Address src, bool no_flags);
+ // Unordered Compare Scalar Half-Precision Floating-Point Values and set EFLAGS
+ void evucomish(XMMRegister dst, Address src);
+ void evucomish(XMMRegister dst, XMMRegister src);
+ void evucomxsh(XMMRegister dst, Address src);
+ void evucomxsh(XMMRegister dst, XMMRegister src);
+
// Unordered Compare Scalar Double-Precision Floating-Point Values and set EFLAGS
void ucomisd(XMMRegister dst, Address src);
void ucomisd(XMMRegister dst, XMMRegister src);
- void vucomxsd(XMMRegister dst, Address src);
- void vucomxsd(XMMRegister dst, XMMRegister src);
+ void evucomxsd(XMMRegister dst, Address src);
+ void evucomxsd(XMMRegister dst, XMMRegister src);
// Unordered Compare Scalar Single-Precision Floating-Point Values and set EFLAGS
void ucomiss(XMMRegister dst, Address src);
void ucomiss(XMMRegister dst, XMMRegister src);
- void vucomxss(XMMRegister dst, Address src);
- void vucomxss(XMMRegister dst, XMMRegister src);
+ void evucomxss(XMMRegister dst, Address src);
+ void evucomxss(XMMRegister dst, XMMRegister src);
void xabort(int8_t imm8);
@@ -2416,11 +2426,6 @@ private:
void vsubss(XMMRegister dst, XMMRegister nds, Address src);
void vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src);
- void vmaxss(XMMRegister dst, XMMRegister nds, XMMRegister src);
- void vmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src);
- void vminss(XMMRegister dst, XMMRegister nds, XMMRegister src);
- void vminsd(XMMRegister dst, XMMRegister nds, XMMRegister src);
-
void sarxl(Register dst, Register src1, Register src2);
void sarxl(Register dst, Address src1, Register src2);
void sarxq(Register dst, Register src1, Register src2);
@@ -2551,8 +2556,6 @@ private:
void vsubsh(XMMRegister dst, XMMRegister nds, XMMRegister src);
void vmulsh(XMMRegister dst, XMMRegister nds, XMMRegister src);
void vdivsh(XMMRegister dst, XMMRegister nds, XMMRegister src);
- void vmaxsh(XMMRegister dst, XMMRegister nds, XMMRegister src);
- void vminsh(XMMRegister dst, XMMRegister nds, XMMRegister src);
void vsqrtsh(XMMRegister dst, XMMRegister src);
void vfmadd132sh(XMMRegister dst, XMMRegister src1, XMMRegister src2);
@@ -2789,9 +2792,9 @@ private:
void vminpd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len);
// AVX10.2 floating point minmax instructions
- void eminmaxsh(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8);
- void eminmaxss(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8);
- void eminmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8);
+ void evminmaxsh(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int imm8);
+ void evminmaxss(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int imm8);
+ void evminmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int imm8);
void evminmaxph(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int imm8, int vector_len);
void evminmaxph(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int imm8, int vector_len);
void evminmaxps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int imm8, int vector_len);
diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp
index 37ee9451405..1242078e11d 100644
--- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -32,6 +32,7 @@
#include "c1/c1_ValueStack.hpp"
#include "ci/ciArrayKlass.hpp"
#include "ci/ciInstance.hpp"
+#include "code/aotCodeCache.hpp"
#include "compiler/oopMap.hpp"
#include "gc/shared/collectedHeap.hpp"
#include "gc/shared/gc_globals.hpp"
@@ -41,6 +42,7 @@
#include "runtime/safepointMechanism.hpp"
#include "runtime/sharedRuntime.hpp"
#include "runtime/stubRoutines.hpp"
+#include "runtime/threadIdentifier.hpp"
#include "utilities/powerOfTwo.hpp"
#include "vmreg_x86.inline.hpp"
@@ -76,23 +78,6 @@ const Register SHIFT_count = rcx; // where count for shift operations must be
#define __ _masm->
-
-static void select_different_registers(Register preserve,
- Register extra,
- Register &tmp1,
- Register &tmp2) {
- if (tmp1 == preserve) {
- assert_different_registers(tmp1, tmp2, extra);
- tmp1 = extra;
- } else if (tmp2 == preserve) {
- assert_different_registers(tmp1, tmp2, extra);
- tmp2 = extra;
- }
- assert_different_registers(preserve, tmp1, tmp2);
-}
-
-
-
static void select_different_registers(Register preserve,
Register extra,
Register &tmp1,
@@ -535,6 +520,19 @@ void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_cod
case T_LONG: {
assert(patch_code == lir_patch_none, "no patching handled here");
+#if INCLUDE_CDS
+ if (AOTCodeCache::is_on_for_dump()) {
+ address b = c->as_pointer();
+ if (b == (address)ThreadIdentifier::unsafe_offset()) {
+ __ lea(dest->as_register_lo(), ExternalAddress(b));
+ break;
+ }
+ if (AOTRuntimeConstants::contains(b)) {
+ __ load_aotrc_address(dest->as_register_lo(), b);
+ break;
+ }
+ }
+#endif
__ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
break;
}
@@ -1299,12 +1297,8 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, L
} else if (obj == klass_RInfo) {
klass_RInfo = dst;
}
- if (k->is_loaded() && !UseCompressedClassPointers) {
- select_different_registers(obj, dst, k_RInfo, klass_RInfo);
- } else {
- Rtmp1 = op->tmp3()->as_register();
- select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
- }
+ Rtmp1 = op->tmp3()->as_register();
+ select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
assert_different_registers(obj, k_RInfo, klass_RInfo);
@@ -1338,12 +1332,8 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, L
if (op->fast_check()) {
// get object class
// not a safepoint as obj null check happens earlier
- if (UseCompressedClassPointers) {
- __ load_klass(Rtmp1, obj, tmp_load_klass);
- __ cmpptr(k_RInfo, Rtmp1);
- } else {
- __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
- }
+ __ load_klass(Rtmp1, obj, tmp_load_klass);
+ __ cmpptr(k_RInfo, Rtmp1);
__ jcc(Assembler::notEqual, *failure_target);
// successful cast, fall through to profile or jump
} else {
@@ -2641,9 +2631,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
// but not necessarily exactly of type default_type.
Label known_ok, halt;
__ mov_metadata(tmp, default_type->constant_encoding());
- if (UseCompressedClassPointers) {
- __ encode_klass_not_null(tmp, rscratch1);
- }
+ __ encode_klass_not_null(tmp, rscratch1);
if (basic_type != T_OBJECT) {
__ cmp_klass(tmp, dst, tmp2);
diff --git a/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp b/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp
index 5459e8df229..f448e4ee17f 100644
--- a/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp
+++ b/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1291,9 +1291,7 @@ void LIRGenerator::do_CheckCast(CheckCast* x) {
}
LIR_Opr reg = rlock_result(x);
LIR_Opr tmp3 = LIR_OprFact::illegalOpr;
- if (!x->klass()->is_loaded() || UseCompressedClassPointers) {
- tmp3 = new_register(objectType);
- }
+ tmp3 = new_register(objectType);
__ checkcast(reg, obj.result(), x->klass(),
new_register(objectType), new_register(objectType), tmp3,
x->direct_compare(), info_for_exception, patching_info, stub,
@@ -1313,9 +1311,7 @@ void LIRGenerator::do_InstanceOf(InstanceOf* x) {
}
obj.load_item();
LIR_Opr tmp3 = LIR_OprFact::illegalOpr;
- if (!x->klass()->is_loaded() || UseCompressedClassPointers) {
- tmp3 = new_register(objectType);
- }
+ tmp3 = new_register(objectType);
__ instanceof(reg, obj.result(), x->klass(),
new_register(objectType), new_register(objectType), tmp3,
x->direct_compare(), patching_info, x->profiled_method(), x->profiled_bci());
diff --git a/src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp
index 88e2e6c8ba9..7adaea48ff1 100644
--- a/src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -85,14 +85,11 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
if (UseCompactObjectHeaders) {
movptr(t1, Address(klass, Klass::prototype_header_offset()));
movptr(Address(obj, oopDesc::mark_offset_in_bytes()), t1);
- } else if (UseCompressedClassPointers) { // Take care not to kill klass
+ } else { // Take care not to kill klass
movptr(Address(obj, oopDesc::mark_offset_in_bytes()), checked_cast(markWord::prototype().value()));
movptr(t1, klass);
encode_klass_not_null(t1, rscratch1);
movl(Address(obj, oopDesc::klass_offset_in_bytes()), t1);
- } else {
- movptr(Address(obj, oopDesc::mark_offset_in_bytes()), checked_cast(markWord::prototype().value()));
- movptr(Address(obj, oopDesc::klass_offset_in_bytes()), klass);
}
if (len->is_valid()) {
@@ -104,7 +101,7 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
xorl(t1, t1);
movl(Address(obj, base_offset), t1);
}
- } else if (UseCompressedClassPointers && !UseCompactObjectHeaders) {
+ } else if (!UseCompactObjectHeaders) {
xorptr(t1, t1);
store_klass_gap(obj, t1);
}
diff --git a/src/hotspot/cpu/x86/c1_globals_x86.hpp b/src/hotspot/cpu/x86/c1_globals_x86.hpp
index 978b233bb63..bb75a31a77c 100644
--- a/src/hotspot/cpu/x86/c1_globals_x86.hpp
+++ b/src/hotspot/cpu/x86/c1_globals_x86.hpp
@@ -41,7 +41,6 @@ define_pd_global(bool, TieredCompilation, false);
define_pd_global(intx, CompileThreshold, 1500 );
define_pd_global(intx, OnStackReplacePercentage, 933 );
-define_pd_global(size_t, NewSizeThreadIncrease, 4*K );
define_pd_global(size_t, InitialCodeCacheSize, 160*K);
define_pd_global(size_t, ReservedCodeCacheSize, 32*M );
define_pd_global(size_t, NonProfiledCodeHeapSize, 13*M );
@@ -51,7 +50,6 @@ define_pd_global(bool, ProfileInterpreter, false);
define_pd_global(size_t, CodeCacheExpansionSize, 32*K );
define_pd_global(size_t, CodeCacheMinBlockLength, 1 );
define_pd_global(size_t, CodeCacheMinimumUseSpace, 400*K);
-define_pd_global(bool, NeverActAsServerClassMachine, true );
define_pd_global(bool, CICompileOSR, true );
#endif // !COMPILER2
define_pd_global(bool, UseTypeProfile, false);
diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
index c65b439604b..f36c816dd5e 100644
--- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
@@ -34,7 +34,9 @@
#include "opto/subnode.hpp"
#include "runtime/globals.hpp"
#include "runtime/objectMonitor.hpp"
+#include "runtime/objectMonitorTable.hpp"
#include "runtime/stubRoutines.hpp"
+#include "runtime/synchronizer.hpp"
#include "utilities/checkedCast.hpp"
#include "utilities/globalDefinitions.hpp"
#include "utilities/powerOfTwo.hpp"
@@ -150,7 +152,7 @@ inline Assembler::AvxVectorLen C2_MacroAssembler::vector_length_encoding(int vle
// Because the transitions from emitted code to the runtime
// monitorenter/exit helper stubs are so slow it's critical that
-// we inline both the stack-locking fast path and the inflated fast path.
+// we inline both the lock-stack fast path and the inflated fast path.
//
// See also: cmpFastLock and cmpFastUnlock.
//
@@ -217,7 +219,6 @@ inline Assembler::AvxVectorLen C2_MacroAssembler::vector_length_encoding(int vle
// In the case of failure, the node will branch directly to the
// FailureLabel
-
// obj: object to lock
// box: on-stack box address -- KILLED
// rax: tmp -- KILLED
@@ -286,7 +287,7 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box, Register rax_reg,
// After successful lock, push object on lock-stack.
movptr(Address(thread, top), obj);
addl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
- jmpb(locked);
+ jmp(locked);
}
{ // Handle inflated monitor.
@@ -297,38 +298,49 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box, Register rax_reg,
if (!UseObjectMonitorTable) {
assert(mark == monitor, "should be the same here");
} else {
- // Uses ObjectMonitorTable. Look for the monitor in the om_cache.
- // Fetch ObjectMonitor* from the cache or take the slow-path.
+ const Register hash = t;
Label monitor_found;
- // Load cache address
- lea(t, Address(thread, JavaThread::om_cache_oops_offset()));
+ // Look for the monitor in the om_cache.
- const int num_unrolled = 2;
+ ByteSize cache_offset = JavaThread::om_cache_oops_offset();
+ ByteSize monitor_offset = OMCache::oop_to_monitor_difference();
+ const int num_unrolled = OMCache::CAPACITY;
for (int i = 0; i < num_unrolled; i++) {
- cmpptr(obj, Address(t));
+ movptr(monitor, Address(thread, cache_offset + monitor_offset));
+ cmpptr(obj, Address(thread, cache_offset));
jccb(Assembler::equal, monitor_found);
- increment(t, in_bytes(OMCache::oop_to_oop_difference()));
+ cache_offset = cache_offset + OMCache::oop_to_oop_difference();
}
- Label loop;
+ // Look for the monitor in the table.
- // Search for obj in cache.
- bind(loop);
+ // Get the hash code.
+ movptr(hash, Address(obj, oopDesc::mark_offset_in_bytes()));
+ shrq(hash, markWord::hash_shift);
+ andq(hash, markWord::hash_mask);
- // Check for match.
- cmpptr(obj, Address(t));
- jccb(Assembler::equal, monitor_found);
+ // Get the table and calculate the bucket's address.
+ lea(rax_reg, ExternalAddress(ObjectMonitorTable::current_table_address()));
+ movptr(rax_reg, Address(rax_reg));
+ andq(hash, Address(rax_reg, ObjectMonitorTable::table_capacity_mask_offset()));
+ movptr(rax_reg, Address(rax_reg, ObjectMonitorTable::table_buckets_offset()));
- // Search until null encountered, guaranteed _null_sentinel at end.
- cmpptr(Address(t), 1);
- jcc(Assembler::below, slow_path); // 0 check, but with ZF=0 when *t == 0
- increment(t, in_bytes(OMCache::oop_to_oop_difference()));
- jmpb(loop);
+ // Read the monitor from the bucket.
+ movptr(monitor, Address(rax_reg, hash, Address::times_ptr));
+
+ // Check if the monitor in the bucket is special (empty, tombstone or removed)
+ cmpptr(monitor, ObjectMonitorTable::SpecialPointerValues::below_is_special);
+ jcc(Assembler::below, slow_path);
+
+ // Check if object matches.
+ movptr(rax_reg, Address(monitor, ObjectMonitor::object_offset()));
+ BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
+ bs_asm->try_resolve_weak_handle_in_c2(this, rax_reg, slow_path);
+ cmpptr(rax_reg, obj);
+ jcc(Assembler::notEqual, slow_path);
- // Cache hit.
bind(monitor_found);
- movptr(monitor, Address(t, OMCache::oop_to_monitor_difference()));
}
const ByteSize monitor_tag = in_ByteSize(UseObjectMonitorTable ? 0 : checked_cast(markWord::monitor_value));
const Address recursions_address(monitor, ObjectMonitor::recursions_offset() - monitor_tag);
@@ -487,14 +499,14 @@ void C2_MacroAssembler::fast_unlock(Register obj, Register reg_rax, Register t,
cmpl(top, in_bytes(JavaThread::lock_stack_base_offset()));
jcc(Assembler::below, check_done);
cmpptr(obj, Address(thread, top));
- jccb(Assembler::notEqual, inflated_check_lock_stack);
+ jcc(Assembler::notEqual, inflated_check_lock_stack);
stop("Fast Unlock lock on stack");
bind(check_done);
if (UseObjectMonitorTable) {
movptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
}
testptr(mark, markWord::monitor_value);
- jccb(Assembler::notZero, inflated);
+ jcc(Assembler::notZero, inflated);
stop("Fast Unlock not monitor");
#endif
@@ -519,7 +531,7 @@ void C2_MacroAssembler::fast_unlock(Register obj, Register reg_rax, Register t,
// Check if recursive.
cmpptr(recursions_address, 0);
- jccb(Assembler::notZero, recursive);
+ jcc(Assembler::notZero, recursive);
// Set owner to null.
// Release to satisfy the JMM
@@ -530,11 +542,11 @@ void C2_MacroAssembler::fast_unlock(Register obj, Register reg_rax, Register t,
// Check if the entry_list is empty.
cmpptr(entry_list_address, NULL_WORD);
- jccb(Assembler::zero, unlocked); // If so we are done.
+ jcc(Assembler::zero, unlocked); // If so we are done.
// Check if there is a successor.
cmpptr(succ_address, NULL_WORD);
- jccb(Assembler::notZero, unlocked); // If so we are done.
+ jcc(Assembler::notZero, unlocked); // If so we are done.
// Save the monitor pointer in the current thread, so we can try to
// reacquire the lock in SharedRuntime::monitor_exit_helper().
@@ -1025,8 +1037,8 @@ void C2_MacroAssembler::evminmax_fp(int opcode, BasicType elem_bt,
}
}
-void C2_MacroAssembler::vminmax_fp(int opc, BasicType elem_bt, XMMRegister dst, KRegister mask,
- XMMRegister src1, XMMRegister src2, int vlen_enc) {
+void C2_MacroAssembler::vminmax_fp_avx10_2(int opc, BasicType elem_bt, XMMRegister dst, KRegister mask,
+ XMMRegister src1, XMMRegister src2, int vlen_enc) {
assert(opc == Op_MinV || opc == Op_MinReductionV ||
opc == Op_MaxV || opc == Op_MaxReductionV, "sanity");
@@ -1040,6 +1052,21 @@ void C2_MacroAssembler::vminmax_fp(int opc, BasicType elem_bt, XMMRegister dst,
}
}
+void C2_MacroAssembler::sminmax_fp_avx10_2(int opc, BasicType elem_bt, XMMRegister dst, KRegister mask,
+ XMMRegister src1, XMMRegister src2) {
+ assert(opc == Op_MinF || opc == Op_MaxF ||
+ opc == Op_MinD || opc == Op_MaxD, "sanity");
+
+ int imm8 = (opc == Op_MinF || opc == Op_MinD) ? AVX10_2_MINMAX_MIN_COMPARE_SIGN
+ : AVX10_2_MINMAX_MAX_COMPARE_SIGN;
+ if (elem_bt == T_FLOAT) {
+ evminmaxss(dst, mask, src1, src2, true, imm8);
+ } else {
+ assert(elem_bt == T_DOUBLE, "");
+ evminmaxsd(dst, mask, src1, src2, true, imm8);
+ }
+}
+
// Float/Double signum
void C2_MacroAssembler::signum_fp(int opcode, XMMRegister dst, XMMRegister zero, XMMRegister one) {
assert(opcode == Op_SignumF || opcode == Op_SignumD, "sanity");
@@ -1051,7 +1078,7 @@ void C2_MacroAssembler::signum_fp(int opcode, XMMRegister dst, XMMRegister zero,
// If other floating point comparison instructions used, ZF=1 for equal and unordered cases
if (opcode == Op_SignumF) {
if (VM_Version::supports_avx10_2()) {
- vucomxss(dst, zero);
+ evucomxss(dst, zero);
jcc(Assembler::negative, DONE_LABEL);
} else {
ucomiss(dst, zero);
@@ -1062,7 +1089,7 @@ void C2_MacroAssembler::signum_fp(int opcode, XMMRegister dst, XMMRegister zero,
xorps(dst, ExternalAddress(StubRoutines::x86::vector_float_sign_flip()), noreg);
} else if (opcode == Op_SignumD) {
if (VM_Version::supports_avx10_2()) {
- vucomxsd(dst, zero);
+ evucomxsd(dst, zero);
jcc(Assembler::negative, DONE_LABEL);
} else {
ucomisd(dst, zero);
@@ -1717,6 +1744,24 @@ void C2_MacroAssembler::reduce_operation_128(BasicType typ, int opcode, XMMRegis
default: assert(false, "wrong type");
}
break;
+ case Op_UMinReductionV:
+ switch (typ) {
+ case T_BYTE: vpminub(dst, dst, src, Assembler::AVX_128bit); break;
+ case T_SHORT: vpminuw(dst, dst, src, Assembler::AVX_128bit); break;
+ case T_INT: vpminud(dst, dst, src, Assembler::AVX_128bit); break;
+ case T_LONG: evpminuq(dst, k0, dst, src, true, Assembler::AVX_128bit); break;
+ default: assert(false, "wrong type");
+ }
+ break;
+ case Op_UMaxReductionV:
+ switch (typ) {
+ case T_BYTE: vpmaxub(dst, dst, src, Assembler::AVX_128bit); break;
+ case T_SHORT: vpmaxuw(dst, dst, src, Assembler::AVX_128bit); break;
+ case T_INT: vpmaxud(dst, dst, src, Assembler::AVX_128bit); break;
+ case T_LONG: evpmaxuq(dst, k0, dst, src, true, Assembler::AVX_128bit); break;
+ default: assert(false, "wrong type");
+ }
+ break;
case Op_AddReductionVF: addss(dst, src); break;
case Op_AddReductionVD: addsd(dst, src); break;
case Op_AddReductionVI:
@@ -1780,6 +1825,24 @@ void C2_MacroAssembler::reduce_operation_256(BasicType typ, int opcode, XMMRegis
default: assert(false, "wrong type");
}
break;
+ case Op_UMinReductionV:
+ switch (typ) {
+ case T_BYTE: vpminub(dst, src1, src2, vector_len); break;
+ case T_SHORT: vpminuw(dst, src1, src2, vector_len); break;
+ case T_INT: vpminud(dst, src1, src2, vector_len); break;
+ case T_LONG: evpminuq(dst, k0, src1, src2, true, vector_len); break;
+ default: assert(false, "wrong type");
+ }
+ break;
+ case Op_UMaxReductionV:
+ switch (typ) {
+ case T_BYTE: vpmaxub(dst, src1, src2, vector_len); break;
+ case T_SHORT: vpmaxuw(dst, src1, src2, vector_len); break;
+ case T_INT: vpmaxud(dst, src1, src2, vector_len); break;
+ case T_LONG: evpmaxuq(dst, k0, src1, src2, true, vector_len); break;
+ default: assert(false, "wrong type");
+ }
+ break;
case Op_AddReductionVI:
switch (typ) {
case T_BYTE: vpaddb(dst, src1, src2, vector_len); break;
@@ -2046,7 +2109,11 @@ void C2_MacroAssembler::reduce8B(int opcode, Register dst, Register src1, XMMReg
psrldq(vtmp2, 1);
reduce_operation_128(T_BYTE, opcode, vtmp1, vtmp2);
movdl(vtmp2, src1);
- pmovsxbd(vtmp1, vtmp1);
+ if (opcode == Op_UMinReductionV || opcode == Op_UMaxReductionV) {
+ pmovzxbd(vtmp1, vtmp1);
+ } else {
+ pmovsxbd(vtmp1, vtmp1);
+ }
reduce_operation_128(T_INT, opcode, vtmp1, vtmp2);
pextrb(dst, vtmp1, 0x0);
movsbl(dst, dst);
@@ -2083,8 +2150,8 @@ void C2_MacroAssembler::mulreduce16B(int opcode, Register dst, Register src1, XM
} else {
pmovsxbw(vtmp2, src2);
reduce8S(opcode, dst, src1, vtmp2, vtmp1, vtmp2);
- pshufd(vtmp2, src2, 0x1);
- pmovsxbw(vtmp2, src2);
+ pshufd(vtmp2, src2, 0xe);
+ pmovsxbw(vtmp2, vtmp2);
reduce8S(opcode, dst, dst, vtmp2, vtmp1, vtmp2);
}
}
@@ -2093,7 +2160,7 @@ void C2_MacroAssembler::mulreduce32B(int opcode, Register dst, Register src1, XM
if (UseAVX > 2 && VM_Version::supports_avx512bw()) {
int vector_len = Assembler::AVX_512bit;
vpmovsxbw(vtmp1, src2, vector_len);
- reduce32S(opcode, dst, src1, vtmp1, vtmp1, vtmp2);
+ reduce32S(opcode, dst, src1, vtmp1, vtmp2, vtmp1);
} else {
assert(UseAVX >= 2,"Should not reach here.");
mulreduce16B(opcode, dst, src1, src2, vtmp1, vtmp2);
@@ -2123,7 +2190,11 @@ void C2_MacroAssembler::reduce4S(int opcode, Register dst, Register src1, XMMReg
reduce_operation_128(T_SHORT, opcode, vtmp1, vtmp2);
}
movdl(vtmp2, src1);
- pmovsxwd(vtmp1, vtmp1);
+ if (opcode == Op_UMinReductionV || opcode == Op_UMaxReductionV) {
+ pmovzxwd(vtmp1, vtmp1);
+ } else {
+ pmovsxwd(vtmp1, vtmp1);
+ }
reduce_operation_128(T_INT, opcode, vtmp1, vtmp2);
pextrw(dst, vtmp1, 0x0);
movswl(dst, dst);
@@ -2136,6 +2207,7 @@ void C2_MacroAssembler::reduce8S(int opcode, Register dst, Register src1, XMMReg
}
phaddw(vtmp1, src2);
} else {
+ assert_different_registers(src2, vtmp1);
pshufd(vtmp1, src2, 0xE);
reduce_operation_128(T_SHORT, opcode, vtmp1, src2);
}
@@ -2148,6 +2220,7 @@ void C2_MacroAssembler::reduce16S(int opcode, Register dst, Register src1, XMMRe
vphaddw(vtmp2, src2, src2, vector_len);
vpermq(vtmp2, vtmp2, 0xD8, vector_len);
} else {
+ assert_different_registers(src2, vtmp2);
vextracti128_high(vtmp2, src2);
reduce_operation_128(T_SHORT, opcode, vtmp2, src2);
}
@@ -2155,6 +2228,7 @@ void C2_MacroAssembler::reduce16S(int opcode, Register dst, Register src1, XMMRe
}
void C2_MacroAssembler::reduce32S(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2) {
+ assert_different_registers(src2, vtmp1);
int vector_len = Assembler::AVX_256bit;
vextracti64x4_high(vtmp1, src2);
reduce_operation_256(T_SHORT, opcode, vtmp1, vtmp1, src2);
@@ -2344,7 +2418,7 @@ void C2_MacroAssembler::reduceFloatMinMax(int opcode, int vlen, bool is_dst_vali
}
if (VM_Version::supports_avx10_2()) {
- vminmax_fp(opcode, T_FLOAT, wdst, k0, wtmp, wsrc, vlen_enc);
+ vminmax_fp_avx10_2(opcode, T_FLOAT, wdst, k0, wtmp, wsrc, vlen_enc);
} else {
vminmax_fp(opcode, T_FLOAT, wdst, wtmp, wsrc, tmp, atmp, btmp, vlen_enc);
}
@@ -2353,7 +2427,7 @@ void C2_MacroAssembler::reduceFloatMinMax(int opcode, int vlen, bool is_dst_vali
}
if (is_dst_valid) {
if (VM_Version::supports_avx10_2()) {
- vminmax_fp(opcode, T_FLOAT, dst, k0, wdst, dst, Assembler::AVX_128bit);
+ vminmax_fp_avx10_2(opcode, T_FLOAT, dst, k0, wdst, dst, Assembler::AVX_128bit);
} else {
vminmax_fp(opcode, T_FLOAT, dst, wdst, dst, tmp, atmp, btmp, Assembler::AVX_128bit);
}
@@ -2384,7 +2458,7 @@ void C2_MacroAssembler::reduceDoubleMinMax(int opcode, int vlen, bool is_dst_val
}
if (VM_Version::supports_avx10_2()) {
- vminmax_fp(opcode, T_DOUBLE, wdst, k0, wtmp, wsrc, vlen_enc);
+ vminmax_fp_avx10_2(opcode, T_DOUBLE, wdst, k0, wtmp, wsrc, vlen_enc);
} else {
vminmax_fp(opcode, T_DOUBLE, wdst, wtmp, wsrc, tmp, atmp, btmp, vlen_enc);
}
@@ -2395,7 +2469,7 @@ void C2_MacroAssembler::reduceDoubleMinMax(int opcode, int vlen, bool is_dst_val
if (is_dst_valid) {
if (VM_Version::supports_avx10_2()) {
- vminmax_fp(opcode, T_DOUBLE, dst, k0, wdst, dst, Assembler::AVX_128bit);
+ vminmax_fp_avx10_2(opcode, T_DOUBLE, dst, k0, wdst, dst, Assembler::AVX_128bit);
} else {
vminmax_fp(opcode, T_DOUBLE, dst, wdst, dst, tmp, atmp, btmp, Assembler::AVX_128bit);
}
@@ -7005,13 +7079,25 @@ void C2_MacroAssembler::evfp16ph(int opcode, XMMRegister dst, XMMRegister src1,
}
}
-void C2_MacroAssembler::scalar_max_min_fp16(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
- KRegister ktmp, XMMRegister xtmp1, XMMRegister xtmp2) {
- vector_max_min_fp16(opcode, dst, src1, src2, ktmp, xtmp1, xtmp2, Assembler::AVX_128bit);
+void C2_MacroAssembler::sminmax_fp16(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ KRegister ktmp, XMMRegister xtmp1, XMMRegister xtmp2) {
+ vminmax_fp16(opcode, dst, src1, src2, ktmp, xtmp1, xtmp2, Assembler::AVX_128bit);
}
-void C2_MacroAssembler::vector_max_min_fp16(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
- KRegister ktmp, XMMRegister xtmp1, XMMRegister xtmp2, int vlen_enc) {
+void C2_MacroAssembler::sminmax_fp16_avx10_2(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ KRegister ktmp) {
+ if (opcode == Op_MaxHF) {
+ // dst = max(src1, src2)
+ evminmaxsh(dst, ktmp, src1, src2, true, AVX10_2_MINMAX_MAX_COMPARE_SIGN);
+ } else {
+ assert(opcode == Op_MinHF, "");
+ // dst = min(src1, src2)
+ evminmaxsh(dst, ktmp, src1, src2, true, AVX10_2_MINMAX_MIN_COMPARE_SIGN);
+ }
+}
+
+void C2_MacroAssembler::vminmax_fp16(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ KRegister ktmp, XMMRegister xtmp1, XMMRegister xtmp2, int vlen_enc) {
if (opcode == Op_MaxVHF || opcode == Op_MaxHF) {
// Move sign bits of src2 to mask register.
evpmovw2m(ktmp, src2, vlen_enc);
@@ -7054,3 +7140,27 @@ void C2_MacroAssembler::vector_max_min_fp16(int opcode, XMMRegister dst, XMMRegi
Assembler::evmovdquw(dst, ktmp, xtmp1, true, vlen_enc);
}
}
+
+void C2_MacroAssembler::vminmax_fp16_avx10_2(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ KRegister ktmp, int vlen_enc) {
+ if (opcode == Op_MaxVHF) {
+ // dst = max(src1, src2)
+ evminmaxph(dst, ktmp, src1, src2, true, AVX10_2_MINMAX_MAX_COMPARE_SIGN, vlen_enc);
+ } else {
+ assert(opcode == Op_MinVHF, "");
+ // dst = min(src1, src2)
+ evminmaxph(dst, ktmp, src1, src2, true, AVX10_2_MINMAX_MIN_COMPARE_SIGN, vlen_enc);
+ }
+}
+
+void C2_MacroAssembler::vminmax_fp16_avx10_2(int opcode, XMMRegister dst, XMMRegister src1, Address src2,
+ KRegister ktmp, int vlen_enc) {
+ if (opcode == Op_MaxVHF) {
+ // dst = max(src1, src2)
+ evminmaxph(dst, ktmp, src1, src2, true, AVX10_2_MINMAX_MAX_COMPARE_SIGN, vlen_enc);
+ } else {
+ assert(opcode == Op_MinVHF, "");
+ // dst = min(src1, src2)
+ evminmaxph(dst, ktmp, src1, src2, true, AVX10_2_MINMAX_MIN_COMPARE_SIGN, vlen_enc);
+ }
+}
diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp
index 6d8b0ceaebe..4e77f8a5f6f 100644
--- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -67,8 +67,11 @@ public:
XMMRegister tmp, XMMRegister atmp, XMMRegister btmp,
int vlen_enc);
- void vminmax_fp(int opc, BasicType elem_bt, XMMRegister dst, KRegister mask,
- XMMRegister src1, XMMRegister src2, int vlen_enc);
+ void vminmax_fp_avx10_2(int opc, BasicType elem_bt, XMMRegister dst, KRegister mask,
+ XMMRegister src1, XMMRegister src2, int vlen_enc);
+
+ void sminmax_fp_avx10_2(int opc, BasicType elem_bt, XMMRegister dst, KRegister mask,
+ XMMRegister src1, XMMRegister src2);
void vpuminmaxq(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister xtmp1, XMMRegister xtmp2, int vlen_enc);
@@ -576,11 +579,20 @@ public:
void evfp16ph(int opcode, XMMRegister dst, XMMRegister src1, Address src2, int vlen_enc);
- void vector_max_min_fp16(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
- KRegister ktmp, XMMRegister xtmp1, XMMRegister xtmp2, int vlen_enc);
+ void vminmax_fp16(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ KRegister ktmp, XMMRegister xtmp1, XMMRegister xtmp2, int vlen_enc);
- void scalar_max_min_fp16(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
- KRegister ktmp, XMMRegister xtmp1, XMMRegister xtmp2);
+ void vminmax_fp16_avx10_2(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ KRegister ktmp, int vlen_enc);
+
+ void vminmax_fp16_avx10_2(int opcode, XMMRegister dst, XMMRegister src1, Address src2,
+ KRegister ktmp, int vlen_enc);
+
+ void sminmax_fp16(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ KRegister ktmp, XMMRegister xtmp1, XMMRegister xtmp2);
+
+ void sminmax_fp16_avx10_2(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ KRegister ktmp);
void reconstruct_frame_pointer(Register rtmp);
diff --git a/src/hotspot/cpu/x86/c2_globals_x86.hpp b/src/hotspot/cpu/x86/c2_globals_x86.hpp
index 3f616cb4578..11d8c03d0ca 100644
--- a/src/hotspot/cpu/x86/c2_globals_x86.hpp
+++ b/src/hotspot/cpu/x86/c2_globals_x86.hpp
@@ -46,7 +46,6 @@ define_pd_global(intx, FreqInlineSize, 325);
define_pd_global(intx, MinJumpTableSize, 10);
define_pd_global(intx, LoopPercentProfileLimit, 10);
define_pd_global(intx, InteriorEntryAlignment, 16);
-define_pd_global(size_t, NewSizeThreadIncrease, ScaleForWordSize(4*K));
define_pd_global(intx, LoopUnrollLimit, 60);
// InitialCodeCacheSize derived from specjbb2000 run.
define_pd_global(size_t, InitialCodeCacheSize, 2496*K); // Integral multiple of CodeCacheExpansionSize
@@ -74,7 +73,4 @@ define_pd_global(size_t, CodeCacheMinimumUseSpace, 400*K);
define_pd_global(bool, TrapBasedRangeChecks, false); // Not needed on x86.
-// Ergonomics related flags
-define_pd_global(bool, NeverActAsServerClassMachine, false);
-
#endif // CPU_X86_C2_GLOBALS_X86_HPP
diff --git a/src/hotspot/cpu/x86/downcallLinker_x86_64.cpp b/src/hotspot/cpu/x86/downcallLinker_x86_64.cpp
index c48940198ea..e3bf5f17fe9 100644
--- a/src/hotspot/cpu/x86/downcallLinker_x86_64.cpp
+++ b/src/hotspot/cpu/x86/downcallLinker_x86_64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -145,10 +145,10 @@ void DowncallLinker::StubGenerator::generate() {
// when we don't use a return buffer we need to spill the return value around our slow path calls
bool should_save_return_value = !_needs_return_buffer;
RegSpiller out_reg_spiller(_output_registers);
- int spill_rsp_offset = -1;
+ int out_spill_rsp_offset = -1;
if (should_save_return_value) {
- spill_rsp_offset = 0;
+ out_spill_rsp_offset = 0;
// spill area can be shared with shadow space and out args,
// since they are only used before the call,
// and spill area is only used after.
@@ -173,6 +173,9 @@ void DowncallLinker::StubGenerator::generate() {
// FP-> | |
// |---------------------| = frame_bottom_offset = frame_size
// | (optional) |
+ // | in_reg_spiller area |
+ // |---------------------|
+ // | (optional) |
// | capture state buf |
// |---------------------| = StubLocations::CAPTURED_STATE_BUFFER
// | (optional) |
@@ -188,6 +191,18 @@ void DowncallLinker::StubGenerator::generate() {
VMStorage shuffle_reg = as_VMStorage(rbx);
ArgumentShuffle arg_shuffle(filtered_java_regs, out_regs, shuffle_reg);
+ // Need to spill for state capturing runtime call.
+ // The area spilled into is distinct from the capture state buffer.
+ RegSpiller in_reg_spiller(out_regs);
+ int in_spill_rsp_offset = -1;
+ if (_captured_state_mask != 0) {
+ // The spill area cannot be shared with the shadow/out args space
+ // since spilling needs to happen before the call. Allocate a new
+ // region in the stack for this spill space.
+ in_spill_rsp_offset = allocated_frame_size;
+ allocated_frame_size += in_reg_spiller.spill_size_bytes();
+ }
+
#ifndef PRODUCT
LogTarget(Trace, foreign, downcall) lt;
if (lt.is_enabled()) {
@@ -232,6 +247,19 @@ void DowncallLinker::StubGenerator::generate() {
arg_shuffle.generate(_masm, shuffle_reg, 0, _abi._shadow_space_bytes);
__ block_comment("} argument shuffle");
+ if (_captured_state_mask != 0) {
+ assert(in_spill_rsp_offset != -1, "must be");
+ __ block_comment("{ load initial thread local");
+ in_reg_spiller.generate_spill(_masm, in_spill_rsp_offset);
+
+ // Copy the contents of the capture state buffer into thread local
+ __ movptr(c_rarg0, Address(rsp, locs.data_offset(StubLocations::CAPTURED_STATE_BUFFER)));
+ __ movl(c_rarg1, _captured_state_mask);
+ runtime_call(_masm, CAST_FROM_FN_PTR(address, DowncallLinker::capture_state_pre));
+
+ in_reg_spiller.generate_fill(_masm, in_spill_rsp_offset);
+ __ block_comment("} load initial thread local");
+ }
__ call(as_Register(locs.get(StubLocations::TARGET_ADDRESS)));
assert(!_abi.is_volatile_reg(r15_thread), "Call assumed not to kill r15");
@@ -258,15 +286,15 @@ void DowncallLinker::StubGenerator::generate() {
__ block_comment("{ save thread local");
if (should_save_return_value) {
- out_reg_spiller.generate_spill(_masm, spill_rsp_offset);
+ out_reg_spiller.generate_spill(_masm, out_spill_rsp_offset);
}
__ movptr(c_rarg0, Address(rsp, locs.data_offset(StubLocations::CAPTURED_STATE_BUFFER)));
__ movl(c_rarg1, _captured_state_mask);
- runtime_call(_masm, CAST_FROM_FN_PTR(address, DowncallLinker::capture_state));
+ runtime_call(_masm, CAST_FROM_FN_PTR(address, DowncallLinker::capture_state_post));
if (should_save_return_value) {
- out_reg_spiller.generate_fill(_masm, spill_rsp_offset);
+ out_reg_spiller.generate_fill(_masm, out_spill_rsp_offset);
}
__ block_comment("} save thread local");
@@ -319,14 +347,14 @@ void DowncallLinker::StubGenerator::generate() {
__ bind(L_safepoint_poll_slow_path);
if (should_save_return_value) {
- out_reg_spiller.generate_spill(_masm, spill_rsp_offset);
+ out_reg_spiller.generate_spill(_masm, out_spill_rsp_offset);
}
__ mov(c_rarg0, r15_thread);
runtime_call(_masm, CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans));
if (should_save_return_value) {
- out_reg_spiller.generate_fill(_masm, spill_rsp_offset);
+ out_reg_spiller.generate_fill(_masm, out_spill_rsp_offset);
}
__ jmp(L_after_safepoint_poll);
@@ -338,13 +366,13 @@ void DowncallLinker::StubGenerator::generate() {
__ bind(L_reguard);
if (should_save_return_value) {
- out_reg_spiller.generate_spill(_masm, spill_rsp_offset);
+ out_reg_spiller.generate_spill(_masm, out_spill_rsp_offset);
}
runtime_call(_masm, CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages));
if (should_save_return_value) {
- out_reg_spiller.generate_fill(_masm, spill_rsp_offset);
+ out_reg_spiller.generate_fill(_masm, out_spill_rsp_offset);
}
__ jmp(L_after_reguard);
diff --git a/src/hotspot/cpu/x86/frame_x86.inline.hpp b/src/hotspot/cpu/x86/frame_x86.inline.hpp
index dcd766545d3..3f3b951edc8 100644
--- a/src/hotspot/cpu/x86/frame_x86.inline.hpp
+++ b/src/hotspot/cpu/x86/frame_x86.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -231,8 +231,8 @@ inline bool frame::equal(frame other) const {
// Return unique id for this frame. The id must have a value where we can distinguish
// identity and younger/older relationship. null represents an invalid (incomparable)
-// frame.
-inline intptr_t* frame::id(void) const { return unextended_sp(); }
+// frame. Should not be called for heap frames.
+inline intptr_t* frame::id(void) const { return real_fp(); }
// Return true if the frame is older (less recent activation) than the frame represented by id
inline bool frame::is_older(intptr_t* id) const { assert(this->id() != nullptr && id != nullptr, "null frame id");
@@ -397,6 +397,9 @@ inline frame frame::sender(RegisterMap* map) const {
StackWatermarkSet::on_iteration(map->thread(), result);
}
+ // Calling frame::id() is currently not supported for heap frames.
+ assert(result._on_heap || this->_on_heap || result.is_older(this->id()), "Must be");
+
return result;
}
diff --git a/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp
index 34de9403ccf..b20d7b5cd07 100644
--- a/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp
@@ -23,6 +23,7 @@
*/
#include "asm/macroAssembler.inline.hpp"
+#include "code/aotCodeCache.hpp"
#include "gc/g1/g1BarrierSet.hpp"
#include "gc/g1/g1BarrierSetAssembler.hpp"
#include "gc/g1/g1BarrierSetRuntime.hpp"
@@ -268,6 +269,16 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
__ bind(done);
}
+#if INCLUDE_CDS
+// return a register that differs from reg1, reg2, reg3 and reg4
+
+static Register pick_different_reg(Register reg1, Register reg2 = noreg, Register reg3= noreg, Register reg4 = noreg) {
+ RegSet available = (RegSet::of(rscratch1, rscratch2, rax, rbx) + rdx -
+ RegSet::of(reg1, reg2, reg3, reg4));
+ return *(available.begin());
+}
+#endif // INCLUDE_CDS
+
static void generate_post_barrier(MacroAssembler* masm,
const Register store_addr,
const Register new_val,
@@ -280,10 +291,32 @@ static void generate_post_barrier(MacroAssembler* masm,
Label L_done;
// Does store cross heap regions?
- __ movptr(tmp1, store_addr); // tmp1 := store address
- __ xorptr(tmp1, new_val); // tmp1 := store address ^ new value
- __ shrptr(tmp1, G1HeapRegion::LogOfHRGrainBytes); // ((store address ^ new value) >> LogOfHRGrainBytes) == 0?
- __ jccb(Assembler::equal, L_done);
+#if INCLUDE_CDS
+ // AOT code needs to load the barrier grain shift from the aot
+ // runtime constants area in the code cache otherwise we can compile
+ // it as an immediate operand
+
+ if (AOTCodeCache::is_on_for_dump()) {
+ address grain_shift_addr = AOTRuntimeConstants::grain_shift_address();
+ Register save = pick_different_reg(rcx, tmp1, new_val, store_addr);
+ __ push(save);
+ __ movptr(save, store_addr);
+ __ xorptr(save, new_val);
+ __ push(rcx);
+ __ lea(rcx, ExternalAddress(grain_shift_addr));
+ __ movl(rcx, Address(rcx, 0));
+ __ shrptr(save);
+ __ pop(rcx);
+ __ pop(save);
+ __ jcc(Assembler::equal, L_done);
+ } else
+#endif // INCLUDE_CDS
+ {
+ __ movptr(tmp1, store_addr); // tmp1 := store address
+ __ xorptr(tmp1, new_val); // tmp1 := store address ^ new value
+ __ shrptr(tmp1, G1HeapRegion::LogOfHRGrainBytes); // ((store address ^ new value) >> LogOfHRGrainBytes) == 0?
+ __ jccb(Assembler::equal, L_done);
+ }
// Crosses regions, storing null?
if (new_val_may_be_null) {
diff --git a/src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.cpp
index 09c5d93dbb3..215dc30f7fd 100644
--- a/src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -395,6 +395,11 @@ OptoReg::Name BarrierSetAssembler::refine_register(const Node* node, OptoReg::Na
extern void vec_spill_helper(C2_MacroAssembler *masm, bool is_load,
int stack_offset, int reg, uint ireg, outputStream* st);
+void BarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Label& slowpath) {
+ // Load the oop from the weak handle.
+ __ movptr(obj, Address(obj));
+}
+
#undef __
#define __ _masm->
diff --git a/src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.hpp
index c5bf17c3b4e..6aff29850e3 100644
--- a/src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -109,6 +109,8 @@ public:
#ifdef COMPILER2
OptoReg::Name refine_register(const Node* node,
OptoReg::Name opto_reg);
+
+ virtual void try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Label& slowpath);
#endif // COMPILER2
};
diff --git a/src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.cpp
index 65e6b4e01fc..1de147926bb 100644
--- a/src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.cpp
@@ -23,6 +23,7 @@
*/
#include "asm/macroAssembler.inline.hpp"
+#include "code/aotCodeCache.hpp"
#include "gc/shared/barrierSet.hpp"
#include "gc/shared/cardTable.hpp"
#include "gc/shared/cardTableBarrierSet.hpp"
@@ -111,7 +112,15 @@ void CardTableBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembl
__ shrptr(end, CardTable::card_shift());
__ subptr(end, addr); // end --> cards count
- __ mov64(tmp, (intptr_t)ctbs->card_table_base_const());
+#if INCLUDE_CDS
+ if (AOTCodeCache::is_on_for_dump()) {
+ __ lea(tmp, ExternalAddress(AOTRuntimeConstants::card_table_base_address()));
+ __ movq(tmp, Address(tmp, 0));
+ } else
+#endif
+ {
+ __ mov64(tmp, (intptr_t)ctbs->card_table_base_const());
+ }
__ addptr(addr, tmp);
__ BIND(L_loop);
__ movb(Address(addr, count, Address::times_1), 0);
@@ -121,7 +130,7 @@ __ BIND(L_loop);
__ BIND(L_done);
}
-void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj, Address dst) {
+void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj, Address dst, Register rscratch) {
// Does a store check for the oop in register obj. The content of
// register obj is destroyed afterwards.
CardTableBarrierSet* ctbs = CardTableBarrierSet::barrier_set();
@@ -129,6 +138,8 @@ void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register ob
__ shrptr(obj, CardTable::card_shift());
Address card_addr;
+ precond(rscratch != noreg);
+ assert_different_registers(obj, rscratch);
// The calculation for byte_map_base is as follows:
// byte_map_base = _byte_map - (uintptr_t(low_bound) >> card_shift);
@@ -136,6 +147,13 @@ void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register ob
// never need to be relocated. On 64bit however the value may be too
// large for a 32bit displacement.
intptr_t byte_map_base = (intptr_t)ctbs->card_table_base_const();
+#if INCLUDE_CDS
+ if (AOTCodeCache::is_on_for_dump()) {
+ __ lea(rscratch, ExternalAddress(AOTRuntimeConstants::card_table_base_address()));
+ __ movq(rscratch, Address(rscratch, 0));
+ card_addr = Address(rscratch, obj, Address::times_1, 0);
+ } else
+#endif
if (__ is_simm32(byte_map_base)) {
card_addr = Address(noreg, obj, Address::times_1, byte_map_base);
} else {
@@ -145,7 +163,7 @@ void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register ob
// entry and that entry is not properly handled by the relocation code.
AddressLiteral cardtable((address)byte_map_base, relocInfo::none);
Address index(noreg, obj, Address::times_1);
- card_addr = __ as_Address(ArrayAddress(cardtable, index), rscratch1);
+ card_addr = __ as_Address(ArrayAddress(cardtable, index), rscratch);
}
int dirty = CardTable::dirty_card_val();
@@ -174,10 +192,10 @@ void CardTableBarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorS
if (needs_post_barrier) {
// flatten object address if needed
if (!precise || (dst.index() == noreg && dst.disp() == 0)) {
- store_check(masm, dst.base(), dst);
+ store_check(masm, dst.base(), dst, tmp2);
} else {
__ lea(tmp1, dst);
- store_check(masm, tmp1, dst);
+ store_check(masm, tmp1, dst, tmp2);
}
}
}
diff --git a/src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.hpp
index 0a36571c757..201c11062f2 100644
--- a/src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.hpp
@@ -33,7 +33,7 @@ protected:
virtual void gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators,
Register addr, Register count) {}
- void store_check(MacroAssembler* masm, Register obj, Address dst);
+ void store_check(MacroAssembler* masm, Register obj, Address dst, Register rscratch);
virtual void gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, Register addr, Register count, Register tmp);
diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp
index 97829a10a3b..67510fac58f 100644
--- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp
@@ -1,4 +1,5 @@
/*
+ * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved.
* Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -618,6 +619,27 @@ void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler
__ bind(done);
}
+#ifdef COMPILER2
+void ShenandoahBarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Label& slowpath) {
+ Label done;
+
+ // Resolve weak handle using the standard implementation.
+ BarrierSetAssembler::try_resolve_weak_handle_in_c2(masm, obj, slowpath);
+
+ // Check if the reference is null, and if it is, take the fast path.
+ __ testptr(obj, obj);
+ __ jcc(Assembler::zero, done);
+
+ Address gc_state(r15_thread, ShenandoahThreadLocalData::gc_state_offset());
+
+ // Check if the heap is under weak-reference/roots processing, in
+ // which case we need to take the slow path.
+ __ testb(gc_state, ShenandoahHeap::WEAK_ROOTS);
+ __ jcc(Assembler::notZero, slowpath);
+ __ bind(done);
+}
+#endif // COMPILER2
+
// Special Shenandoah CAS implementation that handles false negatives
// due to concurrent evacuation.
void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm,
diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp
index b5cc5c8d834..79540aa19e1 100644
--- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp
@@ -1,4 +1,5 @@
/*
+ * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved.
* Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -77,6 +78,9 @@ public:
Address dst, Register val, Register tmp1, Register tmp2, Register tmp3);
virtual void try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
Register obj, Register tmp, Label& slowpath);
+#ifdef COMPILER2
+ virtual void try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Label& slowpath);
+#endif // COMPILER2
};
#endif // CPU_X86_GC_SHENANDOAH_SHENANDOAHBARRIERSETASSEMBLER_X86_HPP
diff --git a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp
index ae93cca8c19..47a3dad54e7 100644
--- a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1328,6 +1328,19 @@ void ZBarrierSetAssembler::generate_c2_store_barrier_stub(MacroAssembler* masm,
__ jmp(slow_continuation);
}
+void ZBarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Label& slow_path) {
+ // Resolve weak handle using the standard implementation.
+ BarrierSetAssembler::try_resolve_weak_handle_in_c2(masm, obj, slow_path);
+
+ // Check if the oop is bad, in which case we need to take the slow path.
+ __ testptr(obj, Address(r15_thread, ZThreadLocalData::mark_bad_mask_offset()));
+ __ jcc(Assembler::notZero, slow_path);
+
+ // Oop is okay, so we uncolor it.
+ __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatLoadGoodBeforeShl);
+ __ shrq(obj, barrier_Relocation::unpatched);
+}
+
#undef __
#endif // COMPILER2
diff --git a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp
index 19902500f93..e91e2b9ea20 100644
--- a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -167,6 +167,8 @@ public:
ZLoadBarrierStubC2* stub) const;
void generate_c2_store_barrier_stub(MacroAssembler* masm,
ZStoreBarrierStubC2* stub) const;
+
+ virtual void try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Label& slow_path);
#endif // COMPILER2
void store_barrier_fast(MacroAssembler* masm,
diff --git a/src/hotspot/cpu/x86/globals_x86.hpp b/src/hotspot/cpu/x86/globals_x86.hpp
index 4f5b6d31e75..6de46752790 100644
--- a/src/hotspot/cpu/x86/globals_x86.hpp
+++ b/src/hotspot/cpu/x86/globals_x86.hpp
@@ -117,9 +117,6 @@ define_pd_global(intx, InitArrayShortSize, 8*BytesPerLong);
product(bool, UseIncDec, true, DIAGNOSTIC, \
"Use INC, DEC instructions on x86") \
\
- product(bool, UseNewLongLShift, false, \
- "Use optimized bitwise shift left") \
- \
product(bool, UseAddressNop, false, \
"Use '0F 1F [addr]' NOP instructions on x86 cpus") \
\
@@ -168,16 +165,27 @@ define_pd_global(intx, InitArrayShortSize, 8*BytesPerLong);
"Perform Ecore Optimization") \
\
/* Minimum array size in bytes to use AVX512 intrinsics */ \
- /* for copy, inflate and fill which don't bail out early based on any */ \
+ /* for inflate and fill which don't bail out early based on any */ \
/* condition. When this value is set to zero compare operations like */ \
/* compare, vectorizedMismatch, compress can also use AVX512 intrinsics.*/\
product(int, AVX3Threshold, 4096, DIAGNOSTIC, \
"Minimum array size in bytes to use AVX512 intrinsics" \
- "for copy, inflate and fill. When this value is set as zero" \
+ "for inflate and fill. When this value is set as zero" \
"compare operations can also use AVX512 intrinsics.") \
range(0, max_jint) \
constraint(AVX3ThresholdConstraintFunc,AfterErgo) \
\
+ /* Minimum array size in bytes to use AVX512 intrinsics */ \
+ /* for copy and fill which don't bail out early based on any */ \
+ /* condition. When this value is set to zero clear operations that */ \
+ /* work on memory blocks can also use AVX512 intrinsics. */ \
+ product(int, CopyAVX3Threshold, 4096, DIAGNOSTIC, \
+ "Minimum array size in bytes to use AVX512 intrinsics" \
+ "for copy and fill. When this value is set as zero" \
+ "clear operations can also use AVX512 intrinsics.") \
+ range(0, max_jint) \
+ constraint(CopyAVX3ThresholdConstraintFunc,AfterErgo) \
+ \
product(bool, IntelJccErratumMitigation, true, DIAGNOSTIC, \
"Turn off JVM mitigations related to Intel micro code " \
"mitigations for the Intel JCC erratum") \
diff --git a/src/hotspot/cpu/x86/interp_masm_x86.cpp b/src/hotspot/cpu/x86/interp_masm_x86.cpp
index b2ea4143ac4..a38971c86fb 100644
--- a/src/hotspot/cpu/x86/interp_masm_x86.cpp
+++ b/src/hotspot/cpu/x86/interp_masm_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1392,28 +1392,15 @@ void InterpreterMacroAssembler::profile_final_call(Register mdp) {
void InterpreterMacroAssembler::profile_virtual_call(Register receiver,
- Register mdp,
- bool receiver_can_be_null) {
+ Register mdp) {
if (ProfileInterpreter) {
Label profile_continue;
// If no method data exists, go to profile_continue.
test_method_data_pointer(mdp, profile_continue);
- Label skip_receiver_profile;
- if (receiver_can_be_null) {
- Label not_null;
- testptr(receiver, receiver);
- jccb(Assembler::notZero, not_null);
- // We are making a call. Increment the count for null receiver.
- increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
- jmp(skip_receiver_profile);
- bind(not_null);
- }
-
// Record the receiver type.
profile_receiver_type(receiver, mdp, 0);
- bind(skip_receiver_profile);
// The method data pointer needs to be updated to reflect the new target.
update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size()));
diff --git a/src/hotspot/cpu/x86/interp_masm_x86.hpp b/src/hotspot/cpu/x86/interp_masm_x86.hpp
index 4114028f78e..dfbd7ab64e0 100644
--- a/src/hotspot/cpu/x86/interp_masm_x86.hpp
+++ b/src/hotspot/cpu/x86/interp_masm_x86.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -243,8 +243,7 @@ class InterpreterMacroAssembler: public MacroAssembler {
void profile_not_taken_branch(Register mdp);
void profile_call(Register mdp);
void profile_final_call(Register mdp);
- void profile_virtual_call(Register receiver, Register mdp,
- bool receiver_can_be_null = false);
+ void profile_virtual_call(Register receiver, Register mdp);
void profile_ret(Register return_bci, Register mdp);
void profile_null_seen(Register mdp);
void profile_typecheck(Register mdp, Register klass);
diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp
index 83169df3456..356bf8af5c0 100644
--- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp
@@ -961,7 +961,7 @@ void MacroAssembler::call(AddressLiteral entry, Register rscratch) {
void MacroAssembler::ic_call(address entry, jint method_index) {
RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index);
// Needs full 64-bit immediate for later patching.
- mov64(rax, (int64_t)Universe::non_oop_word());
+ Assembler::mov64(rax, (int64_t)Universe::non_oop_word());
call(AddressLiteral(entry, rh));
}
@@ -985,12 +985,9 @@ int MacroAssembler::ic_check(int end_alignment) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(temp, receiver);
cmpl(temp, Address(data, CompiledICData::speculated_klass_offset()));
- } else if (UseCompressedClassPointers) {
+ } else {
movl(temp, Address(receiver, oopDesc::klass_offset_in_bytes()));
cmpl(temp, Address(data, CompiledICData::speculated_klass_offset()));
- } else {
- movptr(temp, Address(receiver, oopDesc::klass_offset_in_bytes()));
- cmpptr(temp, Address(data, CompiledICData::speculated_klass_offset()));
}
// if inline cache check fails, then jump to runtime routine
@@ -1961,6 +1958,30 @@ void MacroAssembler::movflt(XMMRegister dst, AddressLiteral src, Register rscrat
}
}
+void MacroAssembler::movhlf(XMMRegister dst, XMMRegister src, Register rscratch) {
+ if (VM_Version::supports_avx10_2()) {
+ evmovw(dst, src);
+ } else {
+ assert(rscratch != noreg, "missing");
+ evmovw(rscratch, src);
+ evmovw(dst, rscratch);
+ }
+}
+
+void MacroAssembler::mov64(Register dst, int64_t imm64) {
+ if (is_uimm32(imm64)) {
+ movl(dst, checked_cast(imm64));
+ } else if (is_simm32(imm64)) {
+ movq(dst, checked_cast(imm64));
+ } else {
+ Assembler::mov64(dst, imm64);
+ }
+}
+
+void MacroAssembler::mov64(Register dst, int64_t imm64, relocInfo::relocType rtype, int format) {
+ Assembler::mov64(dst, imm64, rtype, format);
+}
+
void MacroAssembler::movptr(Register dst, Register src) {
movq(dst, src);
}
@@ -1971,13 +1992,7 @@ void MacroAssembler::movptr(Register dst, Address src) {
// src should NEVER be a real pointer. Use AddressLiteral for true pointers
void MacroAssembler::movptr(Register dst, intptr_t src) {
- if (is_uimm32(src)) {
- movl(dst, checked_cast(src));
- } else if (is_simm32(src)) {
- movq(dst, checked_cast(src));
- } else {
- mov64(dst, src);
- }
+ mov64(dst, src);
}
void MacroAssembler::movptr(Address dst, Register src) {
@@ -2656,14 +2671,14 @@ void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src, Register rscra
}
}
-void MacroAssembler::vucomxsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
+void MacroAssembler::evucomxsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
assert(rscratch != noreg || always_reachable(src), "missing");
if (reachable(src)) {
- Assembler::vucomxsd(dst, as_Address(src));
+ Assembler::evucomxsd(dst, as_Address(src));
} else {
lea(rscratch, src);
- Assembler::vucomxsd(dst, Address(rscratch, 0));
+ Assembler::evucomxsd(dst, Address(rscratch, 0));
}
}
@@ -2678,14 +2693,36 @@ void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src, Register rscra
}
}
-void MacroAssembler::vucomxss(XMMRegister dst, AddressLiteral src, Register rscratch) {
+void MacroAssembler::evucomxss(XMMRegister dst, AddressLiteral src, Register rscratch) {
assert(rscratch != noreg || always_reachable(src), "missing");
if (reachable(src)) {
- Assembler::vucomxss(dst, as_Address(src));
+ Assembler::evucomxss(dst, as_Address(src));
} else {
lea(rscratch, src);
- Assembler::vucomxss(dst, Address(rscratch, 0));
+ Assembler::evucomxss(dst, Address(rscratch, 0));
+ }
+}
+
+void MacroAssembler::evucomish(XMMRegister dst, AddressLiteral src, Register rscratch) {
+ assert(rscratch != noreg || always_reachable(src), "missing");
+
+ if (reachable(src)) {
+ Assembler::evucomish(dst, as_Address(src));
+ } else {
+ lea(rscratch, src);
+ Assembler::evucomish(dst, Address(rscratch, 0));
+ }
+}
+
+void MacroAssembler::evucomxsh(XMMRegister dst, AddressLiteral src, Register rscratch) {
+ assert(rscratch != noreg || always_reachable(src), "missing");
+
+ if (reachable(src)) {
+ Assembler::evucomxsh(dst, as_Address(src));
+ } else {
+ lea(rscratch, src);
+ Assembler::evucomxsh(dst, Address(rscratch, 0));
}
}
@@ -5376,11 +5413,9 @@ void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(dst, src);
decode_klass_not_null(dst, tmp);
- } else if (UseCompressedClassPointers) {
+ } else {
movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
decode_klass_not_null(dst, tmp);
- } else {
- movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
}
}
@@ -5388,12 +5423,8 @@ void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
assert(!UseCompactObjectHeaders, "not with compact headers");
assert_different_registers(src, tmp);
assert_different_registers(dst, tmp);
- if (UseCompressedClassPointers) {
- encode_klass_not_null(src, tmp);
- movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
- } else {
- movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
- }
+ encode_klass_not_null(src, tmp);
+ movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
}
void MacroAssembler::cmp_klass(Register klass, Register obj, Register tmp) {
@@ -5402,10 +5433,8 @@ void MacroAssembler::cmp_klass(Register klass, Register obj, Register tmp) {
assert_different_registers(klass, obj, tmp);
load_narrow_klass_compact(tmp, obj);
cmpl(klass, tmp);
- } else if (UseCompressedClassPointers) {
- cmpl(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
} else {
- cmpptr(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
+ cmpl(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
}
}
@@ -5416,12 +5445,9 @@ void MacroAssembler::cmp_klasses_from_objects(Register obj1, Register obj2, Regi
load_narrow_klass_compact(tmp1, obj1);
load_narrow_klass_compact(tmp2, obj2);
cmpl(tmp1, tmp2);
- } else if (UseCompressedClassPointers) {
+ } else {
movl(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
cmpl(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes()));
- } else {
- movptr(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
- cmpptr(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes()));
}
}
@@ -5470,10 +5496,8 @@ void MacroAssembler::store_heap_oop_null(Address dst) {
void MacroAssembler::store_klass_gap(Register dst, Register src) {
assert(!UseCompactObjectHeaders, "Don't use with compact headers");
- if (UseCompressedClassPointers) {
- // Store to klass gap in destination
- movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src);
- }
+ // Store to klass gap in destination
+ movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src);
}
#ifdef ASSERT
@@ -5663,7 +5687,6 @@ void MacroAssembler::decode_klass_not_null(Register r, Register tmp) {
BLOCK_COMMENT("decode_klass_not_null {");
assert_different_registers(r, tmp);
// Note: it will change flags
- assert(UseCompressedClassPointers, "should only be used for compressed headers");
// Cannot assert, unverified entry point counts instructions (see .ad file)
// vtableStubs also counts instructions in pd_code_size_limit.
// Also do not verify_oop as this is called by verify_oop.
@@ -5685,7 +5708,6 @@ void MacroAssembler::decode_and_move_klass_not_null(Register dst, Register src)
BLOCK_COMMENT("decode_and_move_klass_not_null {");
assert_different_registers(src, dst);
// Note: it will change flags
- assert (UseCompressedClassPointers, "should only be used for compressed headers");
// Cannot assert, unverified entry point counts instructions (see .ad file)
// vtableStubs also counts instructions in pd_code_size_limit.
// Also do not verify_oop as this is called by verify_oop.
@@ -5742,7 +5764,6 @@ void MacroAssembler::set_narrow_oop(Address dst, jobject obj) {
}
void MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
- assert (UseCompressedClassPointers, "should only be used for compressed headers");
assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
int klass_index = oop_recorder()->find_index(k);
RelocationHolder rspec = metadata_Relocation::spec(klass_index);
@@ -5750,7 +5771,6 @@ void MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
}
void MacroAssembler::set_narrow_klass(Address dst, Klass* k) {
- assert (UseCompressedClassPointers, "should only be used for compressed headers");
assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
int klass_index = oop_recorder()->find_index(k);
RelocationHolder rspec = metadata_Relocation::spec(klass_index);
@@ -5776,7 +5796,6 @@ void MacroAssembler::cmp_narrow_oop(Address dst, jobject obj) {
}
void MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) {
- assert (UseCompressedClassPointers, "should only be used for compressed headers");
assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
int klass_index = oop_recorder()->find_index(k);
RelocationHolder rspec = metadata_Relocation::spec(klass_index);
@@ -5784,7 +5803,6 @@ void MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) {
}
void MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) {
- assert (UseCompressedClassPointers, "should only be used for compressed headers");
assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
int klass_index = oop_recorder()->find_index(k);
RelocationHolder rspec = metadata_Relocation::spec(klass_index);
@@ -5812,7 +5830,7 @@ void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register rtmp, X
// cnt - number of qwords (8-byte words).
// base - start address, qword aligned.
Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end;
- bool use64byteVector = (MaxVectorSize == 64) && (VM_Version::avx3_threshold() == 0);
+ bool use64byteVector = (MaxVectorSize == 64) && (CopyAVX3Threshold == 0);
if (use64byteVector) {
vpxor(xtmp, xtmp, xtmp, AVX_512bit);
} else if (MaxVectorSize >= 32) {
@@ -5876,7 +5894,7 @@ void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register rtmp, X
// Clearing constant sized memory using YMM/ZMM registers.
void MacroAssembler::clear_mem(Register base, int cnt, Register rtmp, XMMRegister xtmp, KRegister mask) {
assert(UseAVX > 2 && VM_Version::supports_avx512vl(), "");
- bool use64byteVector = (MaxVectorSize > 32) && (VM_Version::avx3_threshold() == 0);
+ bool use64byteVector = (MaxVectorSize > 32) && (CopyAVX3Threshold == 0);
int vector64_count = (cnt & (~0x7)) >> 3;
cnt = cnt & 0x7;
@@ -6101,8 +6119,8 @@ void MacroAssembler::generate_fill(BasicType t, bool aligned,
// Fill 64-byte chunks
Label L_fill_64_bytes_loop_avx3, L_check_fill_64_bytes_avx2;
- // If number of bytes to fill < VM_Version::avx3_threshold(), perform fill using AVX2
- cmpptr(count, VM_Version::avx3_threshold());
+ // If number of bytes to fill < CopyAVX3Threshold, perform fill using AVX2
+ cmpptr(count, CopyAVX3Threshold);
jccb(Assembler::below, L_check_fill_64_bytes_avx2);
vpbroadcastd(xtmp, xtmp, Assembler::AVX_512bit);
@@ -9177,7 +9195,7 @@ void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XM
case T_FLOAT:
evminmaxps(dst, mask, nds, src, merge, AVX10_2_MINMAX_MAX_COMPARE_SIGN, vector_len); break;
case T_DOUBLE:
- evminmaxps(dst, mask, nds, src, merge, AVX10_2_MINMAX_MAX_COMPARE_SIGN, vector_len); break;
+ evminmaxpd(dst, mask, nds, src, merge, AVX10_2_MINMAX_MAX_COMPARE_SIGN, vector_len); break;
default:
fatal("Unexpected type argument %s", type2name(type)); break;
}
@@ -9475,7 +9493,6 @@ void MacroAssembler::generate_fill_avx3(BasicType type, Register to, Register va
Label L_fill_zmm_sequence;
int shift = -1;
- int avx3threshold = VM_Version::avx3_threshold();
switch(type) {
case T_BYTE: shift = 0;
break;
@@ -9491,10 +9508,10 @@ void MacroAssembler::generate_fill_avx3(BasicType type, Register to, Register va
fatal("Unhandled type: %s\n", type2name(type));
}
- if ((avx3threshold != 0) || (MaxVectorSize == 32)) {
+ if ((CopyAVX3Threshold != 0) || (MaxVectorSize == 32)) {
if (MaxVectorSize == 64) {
- cmpq(count, avx3threshold >> shift);
+ cmpq(count, CopyAVX3Threshold >> shift);
jcc(Assembler::greater, L_fill_zmm_sequence);
}
@@ -10034,6 +10051,20 @@ void MacroAssembler::restore_legacy_gprs() {
addq(rsp, 16 * wordSize);
}
+void MacroAssembler::load_aotrc_address(Register reg, address a) {
+#if INCLUDE_CDS
+ assert(AOTRuntimeConstants::contains(a), "address out of range for data area");
+ if (AOTCodeCache::is_on_for_dump()) {
+ // all aotrc field addresses should be registered in the AOTCodeCache address table
+ lea(reg, ExternalAddress(a));
+ } else {
+ mov64(reg, (uint64_t)a);
+ }
+#else
+ ShouldNotReachHere();
+#endif
+}
+
void MacroAssembler::setcc(Assembler::Condition comparison, Register dst) {
if (VM_Version::supports_apx_f()) {
esetzucc(comparison, dst);
diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.hpp b/src/hotspot/cpu/x86/macroAssembler_x86.hpp
index eb23199ca63..021d2943ee8 100644
--- a/src/hotspot/cpu/x86/macroAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/macroAssembler_x86.hpp
@@ -162,6 +162,8 @@ class MacroAssembler: public Assembler {
void incrementq(AddressLiteral dst, Register rscratch = noreg);
+ void movhlf(XMMRegister dst, XMMRegister src, Register rscratch = noreg);
+
// Support optimal SSE move instructions.
void movflt(XMMRegister dst, XMMRegister src) {
if (dst-> encoding() == src->encoding()) return;
@@ -351,8 +353,7 @@ class MacroAssembler: public Assembler {
void load_klass(Register dst, Register src, Register tmp);
void store_klass(Register dst, Register src, Register tmp);
- // Compares the Klass pointer of an object to a given Klass (which might be narrow,
- // depending on UseCompressedClassPointers).
+ // Compares the narrow Klass pointer of an object to a given narrow Klass.
void cmp_klass(Register klass, Register obj, Register tmp);
// Compares the Klass pointer of two objects obj1 and obj2. Result is in the condition flags.
@@ -1309,21 +1310,29 @@ public:
void subss(XMMRegister dst, Address src) { Assembler::subss(dst, src); }
void subss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
+ void evucomish(XMMRegister dst, XMMRegister src) { Assembler::evucomish(dst, src); }
+ void evucomish(XMMRegister dst, Address src) { Assembler::evucomish(dst, src); }
+ void evucomish(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
+
+ void evucomxsh(XMMRegister dst, XMMRegister src) { Assembler::evucomxsh(dst, src); }
+ void evucomxsh(XMMRegister dst, Address src) { Assembler::evucomxsh(dst, src); }
+ void evucomxsh(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
+
void ucomiss(XMMRegister dst, XMMRegister src) { Assembler::ucomiss(dst, src); }
void ucomiss(XMMRegister dst, Address src) { Assembler::ucomiss(dst, src); }
void ucomiss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
- void vucomxss(XMMRegister dst, XMMRegister src) { Assembler::vucomxss(dst, src); }
- void vucomxss(XMMRegister dst, Address src) { Assembler::vucomxss(dst, src); }
- void vucomxss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
+ void evucomxss(XMMRegister dst, XMMRegister src) { Assembler::evucomxss(dst, src); }
+ void evucomxss(XMMRegister dst, Address src) { Assembler::evucomxss(dst, src); }
+ void evucomxss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
void ucomisd(XMMRegister dst, XMMRegister src) { Assembler::ucomisd(dst, src); }
void ucomisd(XMMRegister dst, Address src) { Assembler::ucomisd(dst, src); }
void ucomisd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
- void vucomxsd(XMMRegister dst, XMMRegister src) { Assembler::vucomxsd(dst, src); }
- void vucomxsd(XMMRegister dst, Address src) { Assembler::vucomxsd(dst, src); }
- void vucomxsd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
+ void evucomxsd(XMMRegister dst, XMMRegister src) { Assembler::evucomxsd(dst, src); }
+ void evucomxsd(XMMRegister dst, Address src) { Assembler::evucomxsd(dst, src); }
+ void evucomxsd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
// Bitwise Logical XOR of Packed Double-Precision Floating-Point Values
void xorpd(XMMRegister dst, XMMRegister src);
@@ -1869,6 +1878,9 @@ public:
void mov_metadata(Register dst, Metadata* obj);
void mov_metadata(Address dst, Metadata* obj, Register rscratch);
+ void mov64(Register dst, int64_t imm64);
+ void mov64(Register dst, int64_t imm64, relocInfo::relocType rtype, int format);
+
void movptr(Register dst, Register src);
void movptr(Register dst, Address src);
void movptr(Register dst, AddressLiteral src);
@@ -2070,6 +2082,7 @@ public:
void save_legacy_gprs();
void restore_legacy_gprs();
+ void load_aotrc_address(Register reg, address a);
void setcc(Assembler::Condition comparison, Register dst);
};
diff --git a/src/hotspot/cpu/x86/matcher_x86.hpp b/src/hotspot/cpu/x86/matcher_x86.hpp
index f7973a8564e..62a5d2827bc 100644
--- a/src/hotspot/cpu/x86/matcher_x86.hpp
+++ b/src/hotspot/cpu/x86/matcher_x86.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -75,7 +75,6 @@
}
static bool narrow_klass_use_complex_address() {
- assert(UseCompressedClassPointers, "only for compressed klass code");
return (CompressedKlassPointers::shift() <= 3);
}
diff --git a/src/hotspot/cpu/x86/methodHandles_x86.cpp b/src/hotspot/cpu/x86/methodHandles_x86.cpp
index 54376c6ad9a..5b15444bc32 100644
--- a/src/hotspot/cpu/x86/methodHandles_x86.cpp
+++ b/src/hotspot/cpu/x86/methodHandles_x86.cpp
@@ -110,14 +110,13 @@ void MethodHandles::verify_ref_kind(MacroAssembler* _masm, int ref_kind, Registe
__ andl(temp, java_lang_invoke_MemberName::MN_REFERENCE_KIND_MASK);
__ cmpl(temp, ref_kind);
__ jcc(Assembler::equal, L);
- { char* buf = NEW_C_HEAP_ARRAY(char, 100, mtInternal);
- jio_snprintf(buf, 100, "verify_ref_kind expected %x", ref_kind);
- if (ref_kind == JVM_REF_invokeVirtual ||
- ref_kind == JVM_REF_invokeSpecial)
- // could do this for all ref_kinds, but would explode assembly code size
- trace_method_handle(_masm, buf);
- __ STOP(buf);
+ const char* msg = ref_kind_to_verify_msg(ref_kind);
+ if (ref_kind == JVM_REF_invokeVirtual ||
+ ref_kind == JVM_REF_invokeSpecial) {
+ // could do this for all ref_kinds, but would explode assembly code size
+ trace_method_handle(_masm, msg);
}
+ __ STOP(msg);
BLOCK_COMMENT("} verify_ref_kind");
__ bind(L);
}
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp
index 36315535d16..332add6dcd4 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -166,12 +166,12 @@ class StubGenerator: public StubCodeGenerator {
// - If target supports AVX3 features (BW+VL+F) then implementation uses 32 byte vectors (YMMs)
// for both special cases (various small block sizes) and aligned copy loop. This is the
// default configuration.
- // - If copy length is above AVX3Threshold, then implementation use 64 byte vectors (ZMMs)
+ // - If copy length is above CopyAVX3Threshold, then implementation use 64 byte vectors (ZMMs)
// for main copy loop (and subsequent tail) since bulk of the cycles will be consumed in it.
// - If user forces MaxVectorSize=32 then above 4096 bytes its seen that REP MOVs shows a
// better performance for disjoint copies. For conjoint/backward copy vector based
// copy performs better.
- // - If user sets AVX3Threshold=0, then special cases for small blocks sizes operate over
+ // - If user sets CopyAVX3Threshold=0, then special cases for small blocks sizes operate over
// 64 byte vector registers (ZMMs).
address generate_disjoint_copy_avx3_masked(StubId stub_id, address* entry);
@@ -330,6 +330,19 @@ class StubGenerator: public StubCodeGenerator {
void aesecb_decrypt(Register source_addr, Register dest_addr, Register key, Register len);
+ // Shared implementation for ECB/AES Encrypt and Decrypt, which does 4 blocks
+ // in a loop at a time to hide instruction latency. Set is_encrypt=true for
+ // encryption, false for decryption.
+ address generate_electronicCodeBook_AESCrypt_Parallel(bool is_encrypt);
+
+ // A version of ECB/AES Encrypt which does 4 blocks in a loop at a time
+ // to hide instruction latency
+ address generate_electronicCodeBook_encryptAESCrypt_Parallel();
+
+ // A version of ECB/AES Decrypt which does 4 blocks in a loop at a time
+ // to hide instruction latency
+ address generate_electronicCodeBook_decryptAESCrypt_Parallel();
+
// Vector AES Galois Counter Mode implementation
address generate_galoisCounterMode_AESCrypt();
void aesgcm_encrypt(Register in, Register len, Register ct, Register out, Register key,
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_adler.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_adler.cpp
index 2799997a761..1d3e7afde1d 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_adler.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_adler.cpp
@@ -144,7 +144,7 @@ address StubGenerator::generate_updateBytesAdler32() {
__ align32();
if (VM_Version::supports_avx512vl()) {
// AVX2 performs better for smaller inputs because of leaner post loop reduction sequence..
- __ cmpl(s, MAX2(128, VM_Version::avx3_threshold()));
+ __ cmpl(s, MAX2(128, CopyAVX3Threshold));
__ jcc(Assembler::belowEqual, SLOOP1A_AVX2);
__ lea(end, Address(s, data, Address::times_1, - (2*CHUNKSIZE -1)));
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_aes.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_aes.cpp
index 24de32a6fe7..162c92d5190 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_aes.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_aes.cpp
@@ -1,5 +1,5 @@
/*
-* Copyright (c) 2019, 2025, Intel Corporation. All rights reserved.
+* Copyright (c) 2019, 2026, Intel Corporation. All rights reserved.
*
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -218,7 +218,9 @@ void StubGenerator::generate_aes_stubs() {
StubRoutines::_galoisCounterMode_AESCrypt = generate_galoisCounterMode_AESCrypt();
} else {
StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt_Parallel();
- if (VM_Version::supports_avx2()) {
+ StubRoutines::_electronicCodeBook_encryptAESCrypt = generate_electronicCodeBook_encryptAESCrypt_Parallel();
+ StubRoutines::_electronicCodeBook_decryptAESCrypt = generate_electronicCodeBook_decryptAESCrypt_Parallel();
+ if (VM_Version::supports_avx2() && VM_Version::supports_clmul()) {
StubRoutines::_galoisCounterMode_AESCrypt = generate_avx2_galoisCounterMode_AESCrypt();
}
}
@@ -1399,6 +1401,200 @@ address StubGenerator::generate_cipherBlockChaining_encryptAESCrypt() {
return start;
}
+// This is a version of ECB/AES Encrypt/Decrypt which does 4 blocks in a loop
+// at a time to hide instruction latency.
+//
+// For encryption (is_encrypt=true):
+// pxor key[0], aesenc key[1..rounds-1], aesenclast key[rounds]
+// For decryption (is_encrypt=false):
+// pxor key[1], aesdec key[2..rounds], aesdeclast key[0]
+//
+// Arguments:
+//
+// Inputs:
+// c_rarg0 - source byte array address
+// c_rarg1 - destination byte array address
+// c_rarg2 - session key (Ke/Kd) in little endian int array
+// c_rarg3 - input length (must be multiple of blocksize 16)
+//
+// Output:
+// rax - input length
+//
+address StubGenerator::generate_electronicCodeBook_AESCrypt_Parallel(bool is_encrypt) {
+ assert(UseAES, "need AES instructions and misaligned SSE support");
+ __ align(CodeEntryAlignment);
+ StubId stub_id = is_encrypt ? StubId::stubgen_electronicCodeBook_encryptAESCrypt_id
+ : StubId::stubgen_electronicCodeBook_decryptAESCrypt_id;
+ StubCodeMark mark(this, stub_id);
+ address start = __ pc();
+
+ const Register from = c_rarg0; // source array address
+ const Register to = c_rarg1; // destination array address
+ const Register key = c_rarg2; // key array address
+ const Register len_reg = c_rarg3; // src len (must be multiple of blocksize 16)
+ const Register pos = rax;
+ const Register keylen = r11;
+
+ const XMMRegister xmm_result0 = xmm0;
+ const XMMRegister xmm_result1 = xmm1;
+ const XMMRegister xmm_result2 = xmm2;
+ const XMMRegister xmm_result3 = xmm3;
+ const XMMRegister xmm_key_shuf_mask = xmm4;
+ const XMMRegister xmm_key_tmp = xmm5;
+ // keys 0-9 pre-loaded into xmm6-xmm15
+ const int XMM_REG_NUM_KEY_FIRST = 6;
+ const int XMM_REG_NUM_KEY_LAST = 15;
+ const XMMRegister xmm_key_first = as_XMMRegister(XMM_REG_NUM_KEY_FIRST);
+
+ // for key_128, key_192, key_256
+ const int ROUNDS[3] = {10, 12, 14};
+
+ Label L_exit;
+ Label L_loop4[3], L_single[3], L_done[3];
+
+#ifdef DoFour
+#undef DoFour
+#endif
+#ifdef DoOne
+#undef DoOne
+#endif
+
+#define DoFour(opc, reg) \
+__ opc(xmm_result0, reg); \
+__ opc(xmm_result1, reg); \
+__ opc(xmm_result2, reg); \
+__ opc(xmm_result3, reg);
+
+#define DoOne(opc, reg) \
+__ opc(xmm_result0, reg);
+
+ __ enter(); // required for proper stackwalking of RuntimeStub frame
+ __ push(len_reg); // save original length for return value
+
+ __ movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
+
+ __ movdqu(xmm_key_shuf_mask, ExternalAddress(key_shuffle_mask_addr()), r10 /*rscratch*/);
+ // load up xmm regs 6 thru 15 with keys 0x00 - 0x90
+ for (int rnum = XMM_REG_NUM_KEY_FIRST, offset = 0x00; rnum <= XMM_REG_NUM_KEY_LAST; rnum++, offset += 0x10) {
+ load_key(as_XMMRegister(rnum), key, offset, xmm_key_shuf_mask);
+ }
+ __ xorptr(pos, pos);
+
+ // key length could be only {11, 13, 15} * 4 = {44, 52, 60}
+ __ cmpl(keylen, 52);
+ __ jcc(Assembler::equal, L_loop4[1]);
+ __ cmpl(keylen, 60);
+ __ jcc(Assembler::equal, L_loop4[2]);
+
+ // k == 0: generate code for key_128
+ // k == 1: generate code for key_192
+ // k == 2: generate code for key_256
+ for (int k = 0; k < 3; ++k) {
+ __ align(OptoLoopAlignment);
+ __ BIND(L_loop4[k]);
+ __ cmpptr(len_reg, 4 * AESBlockSize);
+ __ jcc(Assembler::less, L_single[k]);
+
+ __ movdqu(xmm_result0, Address(from, pos, Address::times_1, 0 * AESBlockSize));
+ __ movdqu(xmm_result1, Address(from, pos, Address::times_1, 1 * AESBlockSize));
+ __ movdqu(xmm_result2, Address(from, pos, Address::times_1, 2 * AESBlockSize));
+ __ movdqu(xmm_result3, Address(from, pos, Address::times_1, 3 * AESBlockSize));
+
+ if (is_encrypt) {
+ DoFour(pxor, xmm_key_first);
+ for (int rnum = 1; rnum < 10; rnum++) {
+ DoFour(aesenc, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST));
+ }
+ for (int i = 10; i < ROUNDS[k]; i++) {
+ load_key(xmm_key_tmp, key, i * 0x10, xmm_key_shuf_mask);
+ DoFour(aesenc, xmm_key_tmp);
+ }
+ load_key(xmm_key_tmp, key, ROUNDS[k] * 0x10, xmm_key_shuf_mask);
+ DoFour(aesenclast, xmm_key_tmp);
+ } else {
+ DoFour(pxor, as_XMMRegister(1 + XMM_REG_NUM_KEY_FIRST));
+ for (int rnum = 2; rnum < 10; rnum++) {
+ DoFour(aesdec, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST));
+ }
+ for (int i = 10; i <= ROUNDS[k]; i++) {
+ load_key(xmm_key_tmp, key, i * 0x10, xmm_key_shuf_mask);
+ DoFour(aesdec, xmm_key_tmp);
+ }
+ DoFour(aesdeclast, xmm_key_first);
+ }
+
+ __ movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0);
+ __ movdqu(Address(to, pos, Address::times_1, 1 * AESBlockSize), xmm_result1);
+ __ movdqu(Address(to, pos, Address::times_1, 2 * AESBlockSize), xmm_result2);
+ __ movdqu(Address(to, pos, Address::times_1, 3 * AESBlockSize), xmm_result3);
+
+ __ addptr(pos, 4 * AESBlockSize);
+ __ subptr(len_reg, 4 * AESBlockSize);
+ __ jmp(L_loop4[k]);
+
+ __ align(OptoLoopAlignment);
+ __ BIND(L_single[k]);
+ __ cmpptr(len_reg, AESBlockSize);
+ __ jcc(Assembler::less, L_done[k]);
+
+ __ movdqu(xmm_result0, Address(from, pos, Address::times_1, 0));
+
+ if (is_encrypt) {
+ DoOne(pxor, xmm_key_first);
+ for (int rnum = 1; rnum < 10; rnum++) {
+ DoOne(aesenc, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST));
+ }
+ for (int i = 10; i < ROUNDS[k]; i++) {
+ load_key(xmm_key_tmp, key, i * 0x10, xmm_key_shuf_mask);
+ DoOne(aesenc, xmm_key_tmp);
+ }
+ load_key(xmm_key_tmp, key, ROUNDS[k] * 0x10, xmm_key_shuf_mask);
+ DoOne(aesenclast, xmm_key_tmp);
+ } else {
+ DoOne(pxor, as_XMMRegister(1 + XMM_REG_NUM_KEY_FIRST));
+ for (int rnum = 2; rnum < 10; rnum++) {
+ DoOne(aesdec, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST));
+ }
+ for (int i = 10; i <= ROUNDS[k]; i++) {
+ load_key(xmm_key_tmp, key, i * 0x10, xmm_key_shuf_mask);
+ DoOne(aesdec, xmm_key_tmp);
+ }
+ DoOne(aesdeclast, xmm_key_first);
+ }
+
+ __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result0);
+ __ addptr(pos, AESBlockSize);
+ __ subptr(len_reg, AESBlockSize);
+ __ jmp(L_single[k]);
+
+ __ BIND(L_done[k]);
+ if (k < 2) __ jmp(L_exit);
+ } //for key_128/192/256
+
+ __ BIND(L_exit);
+ // Clear all XMM registers holding sensitive key material before returning
+ __ pxor(xmm_key_tmp, xmm_key_tmp);
+ for (int rnum = XMM_REG_NUM_KEY_FIRST; rnum <= XMM_REG_NUM_KEY_LAST; rnum++) {
+ __ pxor(as_XMMRegister(rnum), as_XMMRegister(rnum));
+ }
+ __ pop(rax);
+ __ leave(); // required for proper stackwalking of RuntimeStub frame
+ __ ret(0);
+
+ return start;
+
+#undef DoFour
+#undef DoOne
+}
+
+address StubGenerator::generate_electronicCodeBook_encryptAESCrypt_Parallel() {
+ return generate_electronicCodeBook_AESCrypt_Parallel(true);
+}
+
+address StubGenerator::generate_electronicCodeBook_decryptAESCrypt_Parallel() {
+ return generate_electronicCodeBook_AESCrypt_Parallel(false);
+}
+
// This is a version of CBC/AES Decrypt which does 4 blocks in a loop at a time
// to hide instruction latency
//
@@ -1493,7 +1689,7 @@ address StubGenerator::generate_cipherBlockChaining_decryptAESCrypt_Parallel() {
__ opc(xmm_result0, src_reg); \
__ opc(xmm_result1, src_reg); \
__ opc(xmm_result2, src_reg); \
-__ opc(xmm_result3, src_reg); \
+__ opc(xmm_result3, src_reg);
for (int k = 0; k < 3; ++k) {
__ BIND(L_multiBlock_loopTopHead[k]);
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp
index d53fafafdb4..01e004b7b43 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp
@@ -511,12 +511,12 @@ void StubGenerator::copy_bytes_backward(Register from, Register dest,
// - If target supports AVX3 features (BW+VL+F) then implementation uses 32 byte vectors (YMMs)
// for both special cases (various small block sizes) and aligned copy loop. This is the
// default configuration.
-// - If copy length is above AVX3Threshold, then implementation use 64 byte vectors (ZMMs)
+// - If copy length is above CopyAVX3Threshold, then implementation use 64 byte vectors (ZMMs)
// for main copy loop (and subsequent tail) since bulk of the cycles will be consumed in it.
// - If user forces MaxVectorSize=32 then above 4096 bytes its seen that REP MOVs shows a
// better performance for disjoint copies. For conjoint/backward copy vector based
// copy performs better.
-// - If user sets AVX3Threshold=0, then special cases for small blocks sizes operate over
+// - If user sets CopyAVX3Threshold=0, then special cases for small blocks sizes operate over
// 64 byte vector registers (ZMMs).
// Inputs:
@@ -575,8 +575,7 @@ address StubGenerator::generate_disjoint_copy_avx3_masked(StubId stub_id, addres
StubCodeMark mark(this, stub_id);
address start = __ pc();
- int avx3threshold = VM_Version::avx3_threshold();
- bool use64byteVector = (MaxVectorSize > 32) && (avx3threshold == 0);
+ bool use64byteVector = (MaxVectorSize > 32) && (CopyAVX3Threshold == 0);
const int large_threshold = 2621440; // 2.5 MB
Label L_main_loop, L_main_loop_64bytes, L_tail, L_tail64, L_exit, L_entry;
Label L_repmovs, L_main_pre_loop, L_main_pre_loop_64bytes, L_pre_main_post_64;
@@ -647,7 +646,7 @@ address StubGenerator::generate_disjoint_copy_avx3_masked(StubId stub_id, addres
__ cmpq(temp2, large_threshold);
__ jcc(Assembler::greaterEqual, L_copy_large);
}
- if (avx3threshold != 0) {
+ if (CopyAVX3Threshold != 0) {
__ cmpq(count, threshold[shift]);
if (MaxVectorSize == 64) {
// Copy using 64 byte vectors.
@@ -659,7 +658,7 @@ address StubGenerator::generate_disjoint_copy_avx3_masked(StubId stub_id, addres
}
}
- if ((MaxVectorSize < 64) || (avx3threshold != 0)) {
+ if ((MaxVectorSize < 64) || (CopyAVX3Threshold != 0)) {
// Partial copy to make dst address 32 byte aligned.
__ movq(temp2, to);
__ andq(temp2, 31);
@@ -913,8 +912,7 @@ address StubGenerator::generate_conjoint_copy_avx3_masked(StubId stub_id, addres
StubCodeMark mark(this, stub_id);
address start = __ pc();
- int avx3threshold = VM_Version::avx3_threshold();
- bool use64byteVector = (MaxVectorSize > 32) && (avx3threshold == 0);
+ bool use64byteVector = (MaxVectorSize > 32) && (CopyAVX3Threshold == 0);
Label L_main_pre_loop, L_main_pre_loop_64bytes, L_pre_main_post_64;
Label L_main_loop, L_main_loop_64bytes, L_tail, L_tail64, L_exit, L_entry;
@@ -979,12 +977,12 @@ address StubGenerator::generate_conjoint_copy_avx3_masked(StubId stub_id, addres
// PRE-MAIN-POST loop for aligned copy.
__ BIND(L_entry);
- if ((MaxVectorSize > 32) && (avx3threshold != 0)) {
+ if ((MaxVectorSize > 32) && (CopyAVX3Threshold != 0)) {
__ cmpq(temp1, threshold[shift]);
__ jcc(Assembler::greaterEqual, L_pre_main_post_64);
}
- if ((MaxVectorSize < 64) || (avx3threshold != 0)) {
+ if ((MaxVectorSize < 64) || (CopyAVX3Threshold != 0)) {
// Partial copy to make dst address 32 byte aligned.
__ leaq(temp2, Address(to, temp1, (Address::ScaleFactor)(shift), 0));
__ andq(temp2, 31);
@@ -1199,7 +1197,7 @@ void StubGenerator::arraycopy_avx3_special_cases_conjoint(XMMRegister xmm, KRegi
bool use64byteVector, Label& L_entry, Label& L_exit) {
Label L_entry_64, L_entry_96, L_entry_128;
Label L_entry_160, L_entry_192;
- bool avx3 = (MaxVectorSize > 32) && (VM_Version::avx3_threshold() == 0);
+ bool avx3 = (MaxVectorSize > 32) && (CopyAVX3Threshold == 0);
int size_mat[][6] = {
/* T_BYTE */ {32 , 64, 96 , 128 , 160 , 192 },
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_fmod.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_fmod.cpp
index b1eaa4b8031..a0962943556 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_fmod.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_fmod.cpp
@@ -78,7 +78,7 @@ address StubGenerator::generate_libmFmod() {
address start = __ pc();
__ enter(); // required for proper stackwalking of RuntimeStub frame
- if (VM_Version::supports_avx512vlbwdq()) { // AVX512 version
+ if (VM_Version::supports_avx512vlbwdq() && VM_Version::supports_fma()) { // AVX512 version
// Source used to generate the AVX512 fmod assembly below:
//
diff --git a/src/hotspot/cpu/x86/vm_version_x86.cpp b/src/hotspot/cpu/x86/vm_version_x86.cpp
index 78d6dec08cf..7899a2f7e51 100644
--- a/src/hotspot/cpu/x86/vm_version_x86.cpp
+++ b/src/hotspot/cpu/x86/vm_version_x86.cpp
@@ -958,9 +958,17 @@ void VM_Version::get_processor_features() {
if (UseSSE < 1)
_features.clear_feature(CPU_SSE);
- //since AVX instructions is slower than SSE in some ZX cpus, force USEAVX=0.
- if (is_zx() && ((cpu_family() == 6) || (cpu_family() == 7))) {
- UseAVX = 0;
+ // ZX cpus specific settings
+ if (is_zx() && FLAG_IS_DEFAULT(UseAVX)) {
+ if (cpu_family() == 7) {
+ if (extended_cpu_model() == 0x5B || extended_cpu_model() == 0x6B) {
+ UseAVX = 1;
+ } else if (extended_cpu_model() == 0x1B || extended_cpu_model() == 0x3B) {
+ UseAVX = 0;
+ }
+ } else if (cpu_family() == 6) {
+ UseAVX = 0;
+ }
}
// UseSSE is set to the smaller of what hardware supports and what
@@ -1086,15 +1094,35 @@ void VM_Version::get_processor_features() {
}
}
- // Currently APX support is only enabled for targets supporting AVX512VL feature.
- bool apx_supported = os_supports_apx_egprs() && supports_apx_f() && supports_avx512vl();
- if (UseAPX && !apx_supported) {
- warning("UseAPX is not supported on this CPU, setting it to false");
+ // Currently APX support is only enabled for targets supporting AVX512VL feature.
+ if (supports_apx_f() && os_supports_apx_egprs() && supports_avx512vl()) {
+ if (FLAG_IS_DEFAULT(UseAPX)) {
+ UseAPX = false; // by default UseAPX is false
+ } else if (!UseAPX) {
+ _features.clear_feature(CPU_APX_F);
+ }
+ } else if (UseAPX) {
+ if (!FLAG_IS_DEFAULT(UseAPX)) {
+ warning("APX is not supported on this CPU, setting it to false)");
+ }
FLAG_SET_DEFAULT(UseAPX, false);
}
- if (!UseAPX) {
- _features.clear_feature(CPU_APX_F);
+ CHECK_CPU_FEATURE(supports_clmul, CLMUL);
+ CHECK_CPU_FEATURE(supports_aes, AES);
+ CHECK_CPU_FEATURE(supports_fma, FMA);
+
+ if (supports_sha() || (supports_avx2() && supports_bmi2())) {
+ if (FLAG_IS_DEFAULT(UseSHA)) {
+ UseSHA = true;
+ } else if (!UseSHA) {
+ _features.clear_feature(CPU_SHA);
+ }
+ } else if (UseSHA) {
+ if (!FLAG_IS_DEFAULT(UseSHA)) {
+ warning("SHA instructions are not available on this CPU");
+ }
+ FLAG_SET_DEFAULT(UseSHA, false);
}
if (FLAG_IS_DEFAULT(IntelJccErratumMitigation)) {
@@ -1144,9 +1172,40 @@ void VM_Version::get_processor_features() {
// Use AES instructions if available.
if (supports_aes()) {
- if (FLAG_IS_DEFAULT(UseAES)) {
- FLAG_SET_DEFAULT(UseAES, true);
+ if (supports_sse3()) {
+ if (FLAG_IS_DEFAULT(UseAESIntrinsics)) {
+ FLAG_SET_DEFAULT(UseAESIntrinsics, true);
+ }
+ } else if (UseAESIntrinsics) {
+ // The AES intrinsic stubs require AES instruction support (of course)
+ // but also require sse3 mode or higher for instructions it use.
+ if (!FLAG_IS_DEFAULT(UseAESIntrinsics)) {
+ warning("X86 AES intrinsics require SSE3 instructions or higher. Intrinsics will be disabled.");
+ }
+ FLAG_SET_DEFAULT(UseAESIntrinsics, false);
}
+ if (!UseAESIntrinsics) {
+ if (UseAESCTRIntrinsics) {
+ if (FLAG_IS_DEFAULT(UseAESCTRIntrinsics)) {
+ warning("AES-CTR intrinsics require UseAESIntrinsics flag to be enabled. Intrinsics will be disabled.");
+ }
+ FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
+ }
+ } else {
+ if (supports_sse4_1()) {
+ if (FLAG_IS_DEFAULT(UseAESCTRIntrinsics)) {
+ FLAG_SET_DEFAULT(UseAESCTRIntrinsics, true);
+ }
+ } else if (UseAESCTRIntrinsics) {
+ // The AES-CTR intrinsic stubs require AES instruction support (of course)
+ // but also require sse4.1 mode or higher for instructions it use.
+ if (!FLAG_IS_DEFAULT(UseAESCTRIntrinsics)) {
+ warning("X86 AES-CTR intrinsics require SSE4.1 instructions or higher. Intrinsics will be disabled.");
+ }
+ FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
+ }
+ }
+ } else {
if (!UseAES) {
if (UseAESIntrinsics && !FLAG_IS_DEFAULT(UseAESIntrinsics)) {
warning("AES intrinsics require UseAES flag to be enabled. Intrinsics will be disabled.");
@@ -1156,66 +1215,16 @@ void VM_Version::get_processor_features() {
warning("AES_CTR intrinsics require UseAES flag to be enabled. AES_CTR intrinsics will be disabled.");
}
FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
- } else {
- if (UseSSE > 2) {
- if (FLAG_IS_DEFAULT(UseAESIntrinsics)) {
- FLAG_SET_DEFAULT(UseAESIntrinsics, true);
- }
- } else {
- // The AES intrinsic stubs require AES instruction support (of course)
- // but also require sse3 mode or higher for instructions it use.
- if (UseAESIntrinsics && !FLAG_IS_DEFAULT(UseAESIntrinsics)) {
- warning("X86 AES intrinsics require SSE3 instructions or higher. Intrinsics will be disabled.");
- }
- FLAG_SET_DEFAULT(UseAESIntrinsics, false);
+ } else if (!cpu_supports_aes()) {
+ if (UseAESIntrinsics && !FLAG_IS_DEFAULT(UseAESIntrinsics)) {
+ warning("AES intrinsics are not available on this CPU");
}
-
- // --AES-CTR begins--
- if (!UseAESIntrinsics) {
- if (UseAESCTRIntrinsics && !FLAG_IS_DEFAULT(UseAESCTRIntrinsics)) {
- warning("AES-CTR intrinsics require UseAESIntrinsics flag to be enabled. Intrinsics will be disabled.");
- }
- FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
- } else {
- if (supports_sse4_1()) {
- if (FLAG_IS_DEFAULT(UseAESCTRIntrinsics)) {
- FLAG_SET_DEFAULT(UseAESCTRIntrinsics, true);
- }
- } else {
- // The AES-CTR intrinsic stubs require AES instruction support (of course)
- // but also require sse4.1 mode or higher for instructions it use.
- if (UseAESCTRIntrinsics && !FLAG_IS_DEFAULT(UseAESCTRIntrinsics)) {
- warning("X86 AES-CTR intrinsics require SSE4.1 instructions or higher. Intrinsics will be disabled.");
- }
- FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
- }
+ FLAG_SET_DEFAULT(UseAESIntrinsics, false);
+ if (UseAESCTRIntrinsics && !FLAG_IS_DEFAULT(UseAESCTRIntrinsics)) {
+ warning("AES-CTR intrinsics are not available on this CPU");
}
- // --AES-CTR ends--
+ FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
}
- } else if (UseAES || UseAESIntrinsics || UseAESCTRIntrinsics) {
- if (UseAES && !FLAG_IS_DEFAULT(UseAES)) {
- warning("AES instructions are not available on this CPU");
- }
- FLAG_SET_DEFAULT(UseAES, false);
- if (UseAESIntrinsics && !FLAG_IS_DEFAULT(UseAESIntrinsics)) {
- warning("AES intrinsics are not available on this CPU");
- }
- FLAG_SET_DEFAULT(UseAESIntrinsics, false);
- if (UseAESCTRIntrinsics && !FLAG_IS_DEFAULT(UseAESCTRIntrinsics)) {
- warning("AES-CTR intrinsics are not available on this CPU");
- }
- FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
- }
-
- // Use CLMUL instructions if available.
- if (supports_clmul()) {
- if (FLAG_IS_DEFAULT(UseCLMUL)) {
- UseCLMUL = true;
- }
- } else if (UseCLMUL) {
- if (!FLAG_IS_DEFAULT(UseCLMUL))
- warning("CLMUL instructions not available on this CPU (AVX may also be required)");
- FLAG_SET_DEFAULT(UseCLMUL, false);
}
if (UseCLMUL && (UseSSE > 2)) {
@@ -1256,8 +1265,9 @@ void VM_Version::get_processor_features() {
UseGHASHIntrinsics = true;
}
} else if (UseGHASHIntrinsics) {
- if (!FLAG_IS_DEFAULT(UseGHASHIntrinsics))
+ if (!FLAG_IS_DEFAULT(UseGHASHIntrinsics)) {
warning("GHASH intrinsic requires CLMUL and SSE2 instructions on this CPU");
+ }
FLAG_SET_DEFAULT(UseGHASHIntrinsics, false);
}
@@ -1267,26 +1277,27 @@ void VM_Version::get_processor_features() {
// based on the VM capabilities whether to use an AVX2 or AVX512-enabled
// version.
if (UseAVX >= 1) {
- if (FLAG_IS_DEFAULT(UseChaCha20Intrinsics)) {
- UseChaCha20Intrinsics = true;
- }
+ if (FLAG_IS_DEFAULT(UseChaCha20Intrinsics)) {
+ UseChaCha20Intrinsics = true;
+ }
} else if (UseChaCha20Intrinsics) {
- if (!FLAG_IS_DEFAULT(UseChaCha20Intrinsics)) {
- warning("ChaCha20 intrinsic requires AVX instructions");
- }
- FLAG_SET_DEFAULT(UseChaCha20Intrinsics, false);
+ if (!FLAG_IS_DEFAULT(UseChaCha20Intrinsics)) {
+ warning("ChaCha20 intrinsic requires AVX instructions");
+ }
+ FLAG_SET_DEFAULT(UseChaCha20Intrinsics, false);
}
// Kyber Intrinsics
// Currently we only have them for AVX512
if (supports_evex() && supports_avx512bw()) {
- if (FLAG_IS_DEFAULT(UseKyberIntrinsics)) {
- UseKyberIntrinsics = true;
- }
- } else
- if (UseKyberIntrinsics) {
- warning("Intrinsics for ML-KEM are not available on this CPU.");
- FLAG_SET_DEFAULT(UseKyberIntrinsics, false);
+ if (FLAG_IS_DEFAULT(UseKyberIntrinsics)) {
+ UseKyberIntrinsics = true;
+ }
+ } else if (UseKyberIntrinsics) {
+ if (!FLAG_IS_DEFAULT(UseKyberIntrinsics)) {
+ warning("Intrinsics for ML-KEM are not available on this CPU.");
+ }
+ FLAG_SET_DEFAULT(UseKyberIntrinsics, false);
}
// Dilithium Intrinsics
@@ -1295,8 +1306,10 @@ void VM_Version::get_processor_features() {
UseDilithiumIntrinsics = true;
}
} else if (UseDilithiumIntrinsics) {
+ if (!FLAG_IS_DEFAULT(UseDilithiumIntrinsics)) {
warning("Intrinsics for ML-DSA are not available on this CPU.");
- FLAG_SET_DEFAULT(UseDilithiumIntrinsics, false);
+ }
+ FLAG_SET_DEFAULT(UseDilithiumIntrinsics, false);
}
// Base64 Intrinsics (Check the condition for which the intrinsic will be active)
@@ -1305,39 +1318,24 @@ void VM_Version::get_processor_features() {
UseBASE64Intrinsics = true;
}
} else if (UseBASE64Intrinsics) {
- if (!FLAG_IS_DEFAULT(UseBASE64Intrinsics))
+ if (!FLAG_IS_DEFAULT(UseBASE64Intrinsics)) {
warning("Base64 intrinsic requires EVEX instructions on this CPU");
- FLAG_SET_DEFAULT(UseBASE64Intrinsics, false);
- }
-
- if (supports_fma()) {
- if (FLAG_IS_DEFAULT(UseFMA)) {
- UseFMA = true;
}
- } else if (UseFMA) {
- warning("FMA instructions are not available on this CPU");
- FLAG_SET_DEFAULT(UseFMA, false);
+ FLAG_SET_DEFAULT(UseBASE64Intrinsics, false);
}
if (FLAG_IS_DEFAULT(UseMD5Intrinsics)) {
UseMD5Intrinsics = true;
}
- if (supports_sha() || (supports_avx2() && supports_bmi2())) {
- if (FLAG_IS_DEFAULT(UseSHA)) {
- UseSHA = true;
- }
- } else if (UseSHA) {
- warning("SHA instructions are not available on this CPU");
- FLAG_SET_DEFAULT(UseSHA, false);
- }
-
if (supports_sha() && supports_sse4_1() && UseSHA) {
if (FLAG_IS_DEFAULT(UseSHA1Intrinsics)) {
FLAG_SET_DEFAULT(UseSHA1Intrinsics, true);
}
} else if (UseSHA1Intrinsics) {
- warning("Intrinsics for SHA-1 crypto hash functions not available on this CPU.");
+ if (!FLAG_IS_DEFAULT(UseSHA1Intrinsics)) {
+ warning("Intrinsics for SHA-1 crypto hash functions not available on this CPU.");
+ }
FLAG_SET_DEFAULT(UseSHA1Intrinsics, false);
}
@@ -1346,7 +1344,9 @@ void VM_Version::get_processor_features() {
FLAG_SET_DEFAULT(UseSHA256Intrinsics, true);
}
} else if (UseSHA256Intrinsics) {
- warning("Intrinsics for SHA-224 and SHA-256 crypto hash functions not available on this CPU.");
+ if (!FLAG_IS_DEFAULT(UseSHA256Intrinsics)) {
+ warning("Intrinsics for SHA-224 and SHA-256 crypto hash functions not available on this CPU.");
+ }
FLAG_SET_DEFAULT(UseSHA256Intrinsics, false);
}
@@ -1355,7 +1355,9 @@ void VM_Version::get_processor_features() {
FLAG_SET_DEFAULT(UseSHA512Intrinsics, true);
}
} else if (UseSHA512Intrinsics) {
- warning("Intrinsics for SHA-384 and SHA-512 crypto hash functions not available on this CPU.");
+ if (!FLAG_IS_DEFAULT(UseSHA512Intrinsics)) {
+ warning("Intrinsics for SHA-384 and SHA-512 crypto hash functions not available on this CPU.");
+ }
FLAG_SET_DEFAULT(UseSHA512Intrinsics, false);
}
@@ -1364,7 +1366,9 @@ void VM_Version::get_processor_features() {
FLAG_SET_DEFAULT(UseSHA3Intrinsics, true);
}
} else if (UseSHA3Intrinsics) {
- warning("Intrinsics for SHA3-224, SHA3-256, SHA3-384 and SHA3-512 crypto hash functions not available on this CPU.");
+ if (!FLAG_IS_DEFAULT(UseSHA3Intrinsics)) {
+ warning("Intrinsics for SHA3-224, SHA3-256, SHA3-384 and SHA3-512 crypto hash functions not available on this CPU.");
+ }
FLAG_SET_DEFAULT(UseSHA3Intrinsics, false);
}
@@ -1427,7 +1431,9 @@ void VM_Version::get_processor_features() {
FLAG_SET_DEFAULT(UsePoly1305Intrinsics, true);
}
} else if (UsePoly1305Intrinsics) {
- warning("Intrinsics for Poly1305 crypto hash functions not available on this CPU.");
+ if (!FLAG_IS_DEFAULT(UsePoly1305Intrinsics)) {
+ warning("Intrinsics for Poly1305 crypto hash functions not available on this CPU.");
+ }
FLAG_SET_DEFAULT(UsePoly1305Intrinsics, false);
}
@@ -1436,7 +1442,9 @@ void VM_Version::get_processor_features() {
FLAG_SET_DEFAULT(UseIntPolyIntrinsics, true);
}
} else if (UseIntPolyIntrinsics) {
- warning("Intrinsics for Polynomial crypto functions not available on this CPU.");
+ if (!FLAG_IS_DEFAULT(UseIntPolyIntrinsics)) {
+ warning("Intrinsics for Polynomial crypto functions not available on this CPU.");
+ }
FLAG_SET_DEFAULT(UseIntPolyIntrinsics, false);
}
@@ -1500,9 +1508,6 @@ void VM_Version::get_processor_features() {
MaxLoopPad = 11;
}
#endif // COMPILER2
- if (FLAG_IS_DEFAULT(UseXMMForArrayCopy)) {
- UseXMMForArrayCopy = true; // use SSE2 movq on new ZX cpus
- }
if (supports_sse4_2()) { // new ZX cpus
if (FLAG_IS_DEFAULT(UseUnalignedLoadStores)) {
UseUnalignedLoadStores = true; // use movdqu on newest ZX cpus
@@ -1520,10 +1525,6 @@ void VM_Version::get_processor_features() {
// Use it on new AMD cpus starting from Opteron.
UseAddressNop = true;
}
- if (supports_sse2() && FLAG_IS_DEFAULT(UseNewLongLShift)) {
- // Use it on new AMD cpus starting from Opteron.
- UseNewLongLShift = true;
- }
if (FLAG_IS_DEFAULT(UseXmmLoadAndClearUpper)) {
if (supports_sse4a()) {
UseXmmLoadAndClearUpper = true; // use movsd only on '10h' Opteron
@@ -1563,10 +1564,6 @@ void VM_Version::get_processor_features() {
if (FLAG_IS_DEFAULT(AllocatePrefetchInstr)) {
FLAG_SET_DEFAULT(AllocatePrefetchInstr, 3);
}
- // On family 15h processors use XMM and UnalignedLoadStores for Array Copy
- if (supports_sse2() && FLAG_IS_DEFAULT(UseXMMForArrayCopy)) {
- FLAG_SET_DEFAULT(UseXMMForArrayCopy, true);
- }
if (supports_sse2() && FLAG_IS_DEFAULT(UseUnalignedLoadStores)) {
FLAG_SET_DEFAULT(UseUnalignedLoadStores, true);
}
@@ -1583,9 +1580,6 @@ void VM_Version::get_processor_features() {
if (cpu_family() >= 0x17) {
// On family >=17h processors use XMM and UnalignedLoadStores
// for Array Copy
- if (supports_sse2() && FLAG_IS_DEFAULT(UseXMMForArrayCopy)) {
- FLAG_SET_DEFAULT(UseXMMForArrayCopy, true);
- }
if (supports_sse2() && FLAG_IS_DEFAULT(UseUnalignedLoadStores)) {
FLAG_SET_DEFAULT(UseUnalignedLoadStores, true);
}
@@ -1632,10 +1626,7 @@ void VM_Version::get_processor_features() {
}
#endif // COMPILER2
- if (FLAG_IS_DEFAULT(UseXMMForArrayCopy)) {
- UseXMMForArrayCopy = true; // use SSE2 movq on new Intel cpus
- }
- if ((supports_sse4_2() && supports_ht()) || supports_avx()) { // Newest Intel cpus
+ if (is_intel_modern_cpu()) { // Newest Intel cpus
if (FLAG_IS_DEFAULT(UseUnalignedLoadStores)) {
UseUnalignedLoadStores = true; // use movdqu on newest Intel cpus
}
@@ -1703,8 +1694,8 @@ void VM_Version::get_processor_features() {
if (FLAG_IS_DEFAULT(UseSSE42Intrinsics)) {
FLAG_SET_DEFAULT(UseSSE42Intrinsics, true);
}
- } else {
- if (UseSSE42Intrinsics && !FLAG_IS_DEFAULT(UseSSE42Intrinsics)) {
+ } else if (UseSSE42Intrinsics) {
+ if (!FLAG_IS_DEFAULT(UseSSE42Intrinsics)) {
warning("SSE4.2 intrinsics require SSE4.2 instructions or higher. Intrinsics will be disabled.");
}
FLAG_SET_DEFAULT(UseSSE42Intrinsics, false);
@@ -1714,15 +1705,17 @@ void VM_Version::get_processor_features() {
UseVectorizedMismatchIntrinsic = true;
}
} else if (UseVectorizedMismatchIntrinsic) {
- if (!FLAG_IS_DEFAULT(UseVectorizedMismatchIntrinsic))
+ if (!FLAG_IS_DEFAULT(UseVectorizedMismatchIntrinsic)) {
warning("vectorizedMismatch intrinsics are not available on this CPU");
+ }
FLAG_SET_DEFAULT(UseVectorizedMismatchIntrinsic, false);
}
if (UseAVX >= 2) {
FLAG_SET_DEFAULT(UseVectorizedHashCodeIntrinsic, true);
} else if (UseVectorizedHashCodeIntrinsic) {
- if (!FLAG_IS_DEFAULT(UseVectorizedHashCodeIntrinsic))
+ if (!FLAG_IS_DEFAULT(UseVectorizedHashCodeIntrinsic)) {
warning("vectorizedHashCode intrinsics are not available on this CPU");
+ }
FLAG_SET_DEFAULT(UseVectorizedHashCodeIntrinsic, false);
}
@@ -1732,7 +1725,9 @@ void VM_Version::get_processor_features() {
UseCountLeadingZerosInstruction = true;
}
} else if (UseCountLeadingZerosInstruction) {
- warning("lzcnt instruction is not available on this CPU");
+ if (!FLAG_IS_DEFAULT(UseCountLeadingZerosInstruction)) {
+ warning("lzcnt instruction is not available on this CPU");
+ }
FLAG_SET_DEFAULT(UseCountLeadingZerosInstruction, false);
}
@@ -1748,7 +1743,9 @@ void VM_Version::get_processor_features() {
}
}
} else if (UseCountTrailingZerosInstruction) {
- warning("tzcnt instruction is not available on this CPU");
+ if (!FLAG_IS_DEFAULT(UseCountTrailingZerosInstruction)) {
+ warning("tzcnt instruction is not available on this CPU");
+ }
FLAG_SET_DEFAULT(UseCountTrailingZerosInstruction, false);
}
@@ -1759,7 +1756,9 @@ void VM_Version::get_processor_features() {
UseBMI1Instructions = true;
}
} else if (UseBMI1Instructions) {
- warning("BMI1 instructions are not available on this CPU (AVX is also required)");
+ if (!FLAG_IS_DEFAULT(UseBMI1Instructions)) {
+ warning("BMI1 instructions are not available on this CPU (AVX is also required)");
+ }
FLAG_SET_DEFAULT(UseBMI1Instructions, false);
}
@@ -1768,7 +1767,9 @@ void VM_Version::get_processor_features() {
UseBMI2Instructions = true;
}
} else if (UseBMI2Instructions) {
- warning("BMI2 instructions are not available on this CPU (AVX is also required)");
+ if (!FLAG_IS_DEFAULT(UseBMI2Instructions)) {
+ warning("BMI2 instructions are not available on this CPU (AVX is also required)");
+ }
FLAG_SET_DEFAULT(UseBMI2Instructions, false);
}
@@ -1778,7 +1779,9 @@ void VM_Version::get_processor_features() {
UsePopCountInstruction = true;
}
} else if (UsePopCountInstruction) {
- warning("POPCNT instruction is not available on this CPU");
+ if (!FLAG_IS_DEFAULT(UsePopCountInstruction)) {
+ warning("POPCNT instruction is not available on this CPU");
+ }
FLAG_SET_DEFAULT(UsePopCountInstruction, false);
}
@@ -1788,7 +1791,9 @@ void VM_Version::get_processor_features() {
UseFastStosb = true;
}
} else if (UseFastStosb) {
- warning("fast-string operations are not available on this CPU");
+ if (!FLAG_IS_DEFAULT(UseFastStosb)) {
+ warning("fast-string operations are not available on this CPU");
+ }
FLAG_SET_DEFAULT(UseFastStosb, false);
}
@@ -1814,7 +1819,9 @@ void VM_Version::get_processor_features() {
UseXMMForObjInit = true;
}
} else if (UseXMMForObjInit) {
- warning("UseXMMForObjInit requires SSE2 and unaligned load/stores. Feature is switched off.");
+ if (!FLAG_IS_DEFAULT(UseXMMForObjInit)) {
+ warning("UseXMMForObjInit requires SSE2 and unaligned load/stores. Feature is switched off.");
+ }
FLAG_SET_DEFAULT(UseXMMForObjInit, false);
}
@@ -1855,7 +1862,7 @@ void VM_Version::get_processor_features() {
if (is_intel() && is_intel_server_family() && supports_sse3()) {
if (FLAG_IS_DEFAULT(AllocatePrefetchLines) &&
- supports_sse4_2() && supports_ht()) { // Nehalem based cpus
+ is_intel_modern_cpu()) { // Nehalem based cpus
FLAG_SET_DEFAULT(AllocatePrefetchLines, 4);
}
#ifdef COMPILER2
@@ -1894,7 +1901,7 @@ void VM_Version::get_processor_features() {
if (FLAG_IS_DEFAULT(ContendedPaddingWidth) &&
(cache_line_size > ContendedPaddingWidth))
- ContendedPaddingWidth = cache_line_size;
+ ContendedPaddingWidth = cache_line_size;
// This machine allows unaligned memory accesses
if (FLAG_IS_DEFAULT(UseUnalignedAccesses)) {
@@ -1959,6 +1966,18 @@ void VM_Version::get_processor_features() {
if (FLAG_IS_DEFAULT(UseCopySignIntrinsic)) {
FLAG_SET_DEFAULT(UseCopySignIntrinsic, true);
}
+ // CopyAVX3Threshold is the threshold at which 64-byte instructions are used
+ // for implementing the array copy and clear operations.
+ // The Intel platforms that supports the serialize instruction
+ // have improved implementation of 64-byte load/stores and so the default
+ // threshold is set to 0 for these platforms.
+ if (FLAG_IS_DEFAULT(CopyAVX3Threshold)) {
+ if (is_intel() && is_intel_server_family() && supports_serialize()) {
+ FLAG_SET_DEFAULT(CopyAVX3Threshold, 0);
+ } else {
+ FLAG_SET_DEFAULT(CopyAVX3Threshold, AVX3Threshold);
+ }
+ }
}
void VM_Version::print_platform_virtualization_info(outputStream* st) {
@@ -2114,17 +2133,6 @@ bool VM_Version::is_intel_darkmont() {
return is_intel() && is_intel_server_family() && (_model == 0xCC || _model == 0xDD);
}
-// avx3_threshold() sets the threshold at which 64-byte instructions are used
-// for implementing the array copy and clear operations.
-// The Intel platforms that supports the serialize instruction
-// has improved implementation of 64-byte load/stores and so the default
-// threshold is set to 0 for these platforms.
-int VM_Version::avx3_threshold() {
- return (is_intel_server_family() &&
- supports_serialize() &&
- FLAG_IS_DEFAULT(AVX3Threshold)) ? 0 : AVX3Threshold;
-}
-
void VM_Version::clear_apx_test_state() {
clear_apx_test_state_stub();
}
@@ -2623,6 +2631,23 @@ const char* VM_Version::cpu_family_description(void) {
return _family_id_intel[cpu_family_id];
}
}
+ if (is_zx()) {
+ int cpu_model_id = extended_cpu_model();
+ if (cpu_family_id == 7) {
+ switch (cpu_model_id) {
+ case 0x1B:
+ return "wudaokou";
+ case 0x3B:
+ return "lujiazui";
+ case 0x5B:
+ return "yongfeng";
+ case 0x6B:
+ return "shijidadao";
+ }
+ } else if (cpu_family_id == 6) {
+ return "zhangjiang";
+ }
+ }
if (is_hygon()) {
return "Dhyana";
}
@@ -2642,6 +2667,9 @@ int VM_Version::cpu_type_description(char* const buf, size_t buf_len) {
} else if (is_amd()) {
cpu_type = "AMD";
x64 = cpu_is_em64t() ? " AMD64" : "";
+ } else if (is_zx()) {
+ cpu_type = "Zhaoxin";
+ x64 = cpu_is_em64t() ? " x86_64" : "";
} else if (is_hygon()) {
cpu_type = "Hygon";
x64 = cpu_is_em64t() ? " AMD64" : "";
@@ -3259,9 +3287,15 @@ int VM_Version::allocate_prefetch_distance(bool use_watermark_prefetch) {
} else {
return 128; // Athlon
}
+ } else if (is_zx()) {
+ if (supports_sse2()) {
+ return 256;
+ } else {
+ return 128;
+ }
} else { // Intel
if (supports_sse3() && is_intel_server_family()) {
- if (supports_sse4_2() && supports_ht()) { // Nehalem based cpus
+ if (is_intel_modern_cpu()) { // Nehalem based cpus
return 192;
} else if (use_watermark_prefetch) { // watermark prefetching on Core
return 384;
diff --git a/src/hotspot/cpu/x86/vm_version_x86.hpp b/src/hotspot/cpu/x86/vm_version_x86.hpp
index e0a895737b7..f721635a02e 100644
--- a/src/hotspot/cpu/x86/vm_version_x86.hpp
+++ b/src/hotspot/cpu/x86/vm_version_x86.hpp
@@ -535,6 +535,10 @@ protected:
static const char* _features_names[];
+ static void clear_feature(Feature_Flag feature) {
+ _features.clear_feature(feature);
+ }
+
static void clear_cpu_features() {
_features = VM_Features();
_cpu_features = VM_Features();
@@ -828,7 +832,7 @@ public:
static uint32_t cpu_stepping() { return _cpuid_info.cpu_stepping(); }
static int cpu_family() { return _cpu;}
static bool is_P6() { return cpu_family() >= 6; }
- static bool is_intel_server_family() { return cpu_family() == 6 || cpu_family() == 19; }
+ static bool is_intel_server_family() { return cpu_family() == 6 || cpu_family() == 18 || cpu_family() == 19; }
static bool is_amd() { assert_is_initialized(); return _cpuid_info.std_vendor_name_0 == 0x68747541; } // 'htuA'
static bool is_hygon() { assert_is_initialized(); return _cpuid_info.std_vendor_name_0 == 0x6F677948; } // 'ogyH'
static bool is_amd_family() { return is_amd() || is_hygon(); }
@@ -930,6 +934,7 @@ public:
// Feature identification not affected by VM flags
//
static bool cpu_supports_evex() { return _cpu_features.supports_feature(CPU_AVX512F); }
+ static bool cpu_supports_aes() { return _cpu_features.supports_feature(CPU_AES); }
static bool supports_avx512_simd_sort() {
if (supports_avx512dq()) {
@@ -958,7 +963,11 @@ public:
static bool is_intel_darkmont();
- static int avx3_threshold();
+ static bool is_intel_modern_cpu() {
+ precond(is_intel()); // should be called only for intel CPU
+ // Efficient cores in hybrid CPU may not support hyper-threads.
+ return (supports_avx() || (supports_sse4_2() && (supports_ht() || supports_hybrid())));
+ }
static bool is_intel_tsc_synched_at_init();
diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad
index aed54fe93d4..eaa88d900c7 100644
--- a/src/hotspot/cpu/x86/x86.ad
+++ b/src/hotspot/cpu/x86/x86.ad
@@ -1708,84 +1708,99 @@ static void emit_cmpfp3(MacroAssembler* masm, Register dst) {
__ bind(done);
}
-// Math.min() # Math.max()
-// --------------------------
-// ucomis[s/d] #
-// ja -> b # a
-// jp -> NaN # NaN
-// jb -> a # b
-// je #
-// |-jz -> a | b # a & b
-// | -> a #
+enum FP_PREC {
+ fp_prec_hlf,
+ fp_prec_flt,
+ fp_prec_dbl
+};
+
+static inline void emit_fp_ucom(MacroAssembler* masm, enum FP_PREC pt,
+ XMMRegister p, XMMRegister q) {
+ if (pt == fp_prec_hlf) {
+ __ evucomish(p, q);
+ } else if (pt == fp_prec_flt) {
+ __ ucomiss(p, q);
+ } else {
+ __ ucomisd(p, q);
+ }
+}
+
+static inline void movfp(MacroAssembler* masm, enum FP_PREC pt,
+ XMMRegister dst, XMMRegister src, Register scratch) {
+ if (pt == fp_prec_hlf) {
+ __ movhlf(dst, src, scratch);
+ } else if (pt == fp_prec_flt) {
+ __ movflt(dst, src);
+ } else {
+ __ movdbl(dst, src);
+ }
+}
+
+// Math.min() # Math.max()
+// -----------------------------
+// (v)ucomis[h/s/d] #
+// ja -> b # a
+// jp -> NaN # NaN
+// jb -> a # b
+// je #
+// |-jz -> a | b # a & b
+// | -> a #
static void emit_fp_min_max(MacroAssembler* masm, XMMRegister dst,
XMMRegister a, XMMRegister b,
XMMRegister xmmt, Register rt,
- bool min, bool single) {
+ bool min, enum FP_PREC pt) {
Label nan, zero, below, above, done;
- if (single)
- __ ucomiss(a, b);
- else
- __ ucomisd(a, b);
+ emit_fp_ucom(masm, pt, a, b);
- if (dst->encoding() != (min ? b : a)->encoding())
+ if (dst->encoding() != (min ? b : a)->encoding()) {
__ jccb(Assembler::above, above); // CF=0 & ZF=0
- else
+ } else {
__ jccb(Assembler::above, done);
+ }
__ jccb(Assembler::parity, nan); // PF=1
__ jccb(Assembler::below, below); // CF=1
// equal
__ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
- if (single) {
- __ ucomiss(a, xmmt);
- __ jccb(Assembler::equal, zero);
+ emit_fp_ucom(masm, pt, a, xmmt);
- __ movflt(dst, a);
- __ jmp(done);
- }
- else {
- __ ucomisd(a, xmmt);
- __ jccb(Assembler::equal, zero);
+ __ jccb(Assembler::equal, zero);
+ movfp(masm, pt, dst, a, rt);
- __ movdbl(dst, a);
- __ jmp(done);
- }
+ __ jmp(done);
__ bind(zero);
- if (min)
+ if (min) {
__ vpor(dst, a, b, Assembler::AVX_128bit);
- else
+ } else {
__ vpand(dst, a, b, Assembler::AVX_128bit);
+ }
__ jmp(done);
__ bind(above);
- if (single)
- __ movflt(dst, min ? b : a);
- else
- __ movdbl(dst, min ? b : a);
+ movfp(masm, pt, dst, min ? b : a, rt);
__ jmp(done);
__ bind(nan);
- if (single) {
+ if (pt == fp_prec_hlf) {
+ __ movl(rt, 0x00007e00); // Float16.NaN
+ __ evmovw(dst, rt);
+ } else if (pt == fp_prec_flt) {
__ movl(rt, 0x7fc00000); // Float.NaN
__ movdl(dst, rt);
- }
- else {
+ } else {
__ mov64(rt, 0x7ff8000000000000L); // Double.NaN
__ movdq(dst, rt);
}
__ jmp(done);
__ bind(below);
- if (single)
- __ movflt(dst, min ? a : b);
- else
- __ movdbl(dst, min ? a : b);
+ movfp(masm, pt, dst, min ? a : b, rt);
__ bind(done);
}
@@ -2605,13 +2620,8 @@ uint BoxLockNode::size(PhaseRegAlloc *ra_) const
#ifndef PRODUCT
void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
{
- if (UseCompressedClassPointers) {
- st->print_cr("movl rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
- st->print_cr("\tcmpl rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
- } else {
- st->print_cr("movq rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
- st->print_cr("\tcmpq rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
- }
+ st->print_cr("movl rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
+ st->print_cr("\tcmpl rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
st->print_cr("\tjne SharedRuntime::_ic_miss_stub");
}
#endif
@@ -2726,11 +2736,8 @@ bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
return (-128 <= offset && offset <= 127);
}
+#ifdef ASSERT
// Return whether or not this register is ever used as an argument.
-// This function is used on startup to build the trampoline stubs in
-// generateOptoStub. Registers not mentioned will be killed by the VM
-// call in the trampoline, and arguments in those registers not be
-// available to the callee.
bool Matcher::can_be_java_arg(int reg)
{
return
@@ -2750,11 +2757,7 @@ bool Matcher::can_be_java_arg(int reg)
reg == XMM6_num || reg == XMM6b_num ||
reg == XMM7_num || reg == XMM7b_num;
}
-
-bool Matcher::is_spillable_arg(int reg)
-{
- return can_be_java_arg(reg);
-}
+#endif
uint Matcher::int_pressure_limit()
{
@@ -2770,13 +2773,6 @@ uint Matcher::float_pressure_limit()
return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
}
-bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
- // In 64 bit mode a code which use multiply when
- // devisor is constant is faster than hardware
- // DIV instruction (it uses MulHiL).
- return false;
-}
-
// Register for DIVI projection of divmodI
const RegMask& Matcher::divI_proj_mask() {
return INT_RAX_REG_mask();
@@ -3341,6 +3337,18 @@ bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
return false;
}
break;
+ case Op_UMinReductionV:
+ case Op_UMaxReductionV:
+ if (UseAVX == 0) {
+ return false;
+ }
+ if (bt == T_LONG && !VM_Version::supports_avx512vl()) {
+ return false;
+ }
+ if (UseAVX > 2 && size_in_bits == 512 && !VM_Version::supports_avx512vl()) {
+ return false;
+ }
+ break;
case Op_MaxV:
case Op_MinV:
if (UseSSE < 4 && is_integral_type(bt)) {
@@ -4679,11 +4687,6 @@ frame
// Compiled code's Frame Pointer
frame_pointer(RSP);
- // Interpreter stores its frame pointer in a register which is
- // stored to the stack by I2CAdaptors.
- // I2CAdaptors convert from interpreted java to compiled java.
- interpreter_frame_pointer(RBP);
-
// Stack alignment requirement
stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
@@ -5187,6 +5190,18 @@ operand immL_65535()
interface(CONST_INTER);
%}
+// AOT Runtime Constants Address
+operand immAOTRuntimeConstantsAddress()
+%{
+ // Check if the address is in the range of AOT Runtime Constants
+ predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
+ match(ConP);
+
+ op_cost(0);
+ format %{ %}
+ interface(CONST_INTER);
+%}
+
operand kReg()
%{
constraint(ALLOC_IN_RC(vectmask_reg));
@@ -7332,146 +7347,153 @@ instruct loadD(regD dst, memory mem)
ins_pipe(pipe_slow); // XXX
%}
+instruct loadAOTRCAddress(rRegP dst, immAOTRuntimeConstantsAddress con)
+%{
+ match(Set dst con);
+
+ format %{ "leaq $dst, $con\t# AOT Runtime Constants Address" %}
+
+ ins_encode %{
+ __ load_aotrc_address($dst$$Register, (address)$con$$constant);
+ %}
+
+ ins_pipe(ialu_reg_fat);
+%}
+
+// min = java.lang.Math.min(float a, float b)
// max = java.lang.Math.max(float a, float b)
-instruct maxF_reg_avx10_2(regF dst, regF a, regF b) %{
- predicate(VM_Version::supports_avx10_2());
+instruct minmaxF_reg_avx10_2(regF dst, regF a, regF b)
+%{
+ predicate(VM_Version::supports_avx10_2() && !VLoopReductions::is_reduction(n));
match(Set dst (MaxF a b));
- format %{ "maxF $dst, $a, $b" %}
- ins_encode %{
- __ eminmaxss($dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, AVX10_2_MINMAX_MAX_COMPARE_SIGN);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// max = java.lang.Math.max(float a, float b)
-instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
- predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
- match(Set dst (MaxF a b));
- effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
- format %{ "maxF $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
- ins_encode %{
- __ vminmax_fp(Op_MaxV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xtmp, rRegI rtmp, rFlagsReg cr) %{
- predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
- match(Set dst (MaxF a b));
- effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
-
- format %{ "maxF_reduction $dst, $a, $b \t!using $xtmp and $rtmp as TEMP" %}
- ins_encode %{
- emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
- false /*min*/, true /*single*/);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// max = java.lang.Math.max(double a, double b)
-instruct maxD_reg_avx10_2(regD dst, regD a, regD b) %{
- predicate(VM_Version::supports_avx10_2());
- match(Set dst (MaxD a b));
- format %{ "maxD $dst, $a, $b" %}
- ins_encode %{
- __ eminmaxsd($dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, AVX10_2_MINMAX_MAX_COMPARE_SIGN);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// max = java.lang.Math.max(double a, double b)
-instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
- predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
- match(Set dst (MaxD a b));
- effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
- format %{ "maxD $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
- ins_encode %{
- __ vminmax_fp(Op_MaxV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xtmp, rRegL rtmp, rFlagsReg cr) %{
- predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
- match(Set dst (MaxD a b));
- effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
-
- format %{ "maxD_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
- ins_encode %{
- emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
- false /*min*/, false /*single*/);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// max = java.lang.Math.min(float a, float b)
-instruct minF_reg_avx10_2(regF dst, regF a, regF b) %{
- predicate(VM_Version::supports_avx10_2());
match(Set dst (MinF a b));
- format %{ "minF $dst, $a, $b" %}
+
+ format %{ "minmaxF $dst, $a, $b" %}
ins_encode %{
- __ eminmaxss($dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, AVX10_2_MINMAX_MIN_COMPARE_SIGN);
+ int opcode = this->ideal_Opcode();
+ __ sminmax_fp_avx10_2(opcode, T_FLOAT, $dst$$XMMRegister, k0, $a$$XMMRegister, $b$$XMMRegister);
+ %}
+ ins_pipe( pipe_slow );
+%}
+
+instruct minmaxF_reduction_reg_avx10_2(regF dst, regF a, regF b, regF xtmp, rRegI rtmp, rFlagsReg cr)
+%{
+ predicate(VM_Version::supports_avx10_2() && VLoopReductions::is_reduction(n));
+ match(Set dst (MaxF a b));
+ match(Set dst (MinF a b));
+ effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
+
+ format %{ "minmaxF_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
+ ins_encode %{
+ int opcode = this->ideal_Opcode();
+ bool min = (opcode == Op_MinF) ? true : false;
+ emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
+ min, fp_prec_flt /*pt*/);
%}
ins_pipe( pipe_slow );
%}
// min = java.lang.Math.min(float a, float b)
-instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
+// max = java.lang.Math.max(float a, float b)
+instruct minmaxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp)
+%{
predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
+ match(Set dst (MaxF a b));
match(Set dst (MinF a b));
effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
- format %{ "minF $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
+
+ format %{ "minmaxF $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
ins_encode %{
- __ vminmax_fp(Op_MinV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
+ int opcode = this->ideal_Opcode();
+ int param_opcode = (opcode == Op_MinF) ? Op_MinV : Op_MaxV;
+ __ vminmax_fp(param_opcode, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister,
+ $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
%}
ins_pipe( pipe_slow );
%}
-instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xtmp, rRegI rtmp, rFlagsReg cr) %{
+instruct minmaxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xtmp, rRegI rtmp, rFlagsReg cr)
+%{
predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
+ match(Set dst (MaxF a b));
match(Set dst (MinF a b));
effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
- format %{ "minF_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
+ format %{ "minmaxF_reduction $dst, $a, $b \t!using $xtmp and $rtmp as TEMP" %}
ins_encode %{
+ int opcode = this->ideal_Opcode();
+ bool min = (opcode == Op_MinF) ? true : false;
emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
- true /*min*/, true /*single*/);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// max = java.lang.Math.min(double a, double b)
-instruct minD_reg_avx10_2(regD dst, regD a, regD b) %{
- predicate(VM_Version::supports_avx10_2());
- match(Set dst (MinD a b));
- format %{ "minD $dst, $a, $b" %}
- ins_encode %{
- __ eminmaxsd($dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, AVX10_2_MINMAX_MIN_COMPARE_SIGN);
+ min, fp_prec_flt /*pt*/);
%}
ins_pipe( pipe_slow );
%}
// min = java.lang.Math.min(double a, double b)
-instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
- predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
+// max = java.lang.Math.max(double a, double b)
+instruct minmaxD_reg_avx10_2(regD dst, regD a, regD b)
+%{
+ predicate(VM_Version::supports_avx10_2() && !VLoopReductions::is_reduction(n));
+ match(Set dst (MaxD a b));
match(Set dst (MinD a b));
- effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
- format %{ "minD $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
+
+ format %{ "minmaxD $dst, $a, $b" %}
ins_encode %{
- __ vminmax_fp(Op_MinV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
+ int opcode = this->ideal_Opcode();
+ __ sminmax_fp_avx10_2(opcode, T_DOUBLE, $dst$$XMMRegister, k0, $a$$XMMRegister, $b$$XMMRegister);
%}
ins_pipe( pipe_slow );
%}
-instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xtmp, rRegL rtmp, rFlagsReg cr) %{
- predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
+instruct minmaxD_reduction_reg_avx10_2(regD dst, regD a, regD b, regD xtmp, rRegI rtmp, rFlagsReg cr)
+%{
+ predicate(VM_Version::supports_avx10_2() && VLoopReductions::is_reduction(n));
+ match(Set dst (MaxD a b));
match(Set dst (MinD a b));
effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
- format %{ "maxD_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
+ format %{ "minmaxD_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
ins_encode %{
+ int opcode = this->ideal_Opcode();
+ bool min = (opcode == Op_MinD) ? true : false;
emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
- true /*min*/, false /*single*/);
+ min, fp_prec_dbl /*pt*/);
+ %}
+ ins_pipe( pipe_slow );
+%}
+
+// min = java.lang.Math.min(double a, double b)
+// max = java.lang.Math.max(double a, double b)
+instruct minmaxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp)
+%{
+ predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
+ match(Set dst (MaxD a b));
+ match(Set dst (MinD a b));
+ effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
+
+ format %{ "minmaxD $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
+ ins_encode %{
+ int opcode = this->ideal_Opcode();
+ int param_opcode = (opcode == Op_MinD) ? Op_MinV : Op_MaxV;
+ __ vminmax_fp(param_opcode, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister,
+ $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
+ %}
+ ins_pipe( pipe_slow );
+%}
+
+instruct minmaxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xtmp, rRegL rtmp, rFlagsReg cr)
+%{
+ predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
+ match(Set dst (MaxD a b));
+ match(Set dst (MinD a b));
+ effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
+
+ format %{ "minmaxD_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
+ ins_encode %{
+ int opcode = this->ideal_Opcode();
+ bool min = (opcode == Op_MinD) ? true : false;
+ emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
+ min, fp_prec_dbl /*pt*/);
%}
ins_pipe( pipe_slow );
%}
@@ -8827,6 +8849,21 @@ instruct membar_release_lock()
ins_pipe(empty);
%}
+instruct membar_storeload(rFlagsReg cr) %{
+ match(MemBarStoreLoad);
+ effect(KILL cr);
+ ins_cost(400);
+
+ format %{
+ $$template
+ $$emit$$"lock addl [rsp + #0], 0\t! membar_storeload"
+ %}
+ ins_encode %{
+ __ membar(Assembler::StoreLoad);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
instruct membar_volatile(rFlagsReg cr) %{
match(MemBarVolatile);
effect(KILL cr);
@@ -8854,6 +8891,21 @@ instruct unnecessary_membar_volatile()
ins_pipe(empty);
%}
+instruct membar_full(rFlagsReg cr) %{
+ match(MemBarFull);
+ effect(KILL cr);
+ ins_cost(400);
+
+ format %{
+ $$template
+ $$emit$$"lock addl [rsp + #0], 0\t! membar_full"
+ %}
+ ins_encode %{
+ __ membar(Assembler::StoreLoad);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
instruct membar_storestore() %{
match(MemBarStoreStore);
match(StoreStoreFence);
@@ -14351,9 +14403,9 @@ instruct cmpF_cc_regCFE(rFlagsRegUCFE cr, regF src1, regF src2) %{
match(Set cr (CmpF src1 src2));
ins_cost(100);
- format %{ "vucomxss $src1, $src2" %}
+ format %{ "evucomxss $src1, $src2" %}
ins_encode %{
- __ vucomxss($src1$$XMMRegister, $src2$$XMMRegister);
+ __ evucomxss($src1$$XMMRegister, $src2$$XMMRegister);
%}
ins_pipe(pipe_slow);
%}
@@ -14373,9 +14425,9 @@ instruct cmpF_cc_memCFE(rFlagsRegUCFE cr, regF src1, memory src2) %{
match(Set cr (CmpF src1 (LoadF src2)));
ins_cost(100);
- format %{ "vucomxss $src1, $src2" %}
+ format %{ "evucomxss $src1, $src2" %}
ins_encode %{
- __ vucomxss($src1$$XMMRegister, $src2$$Address);
+ __ evucomxss($src1$$XMMRegister, $src2$$Address);
%}
ins_pipe(pipe_slow);
%}
@@ -14395,9 +14447,9 @@ instruct cmpF_cc_immCFE(rFlagsRegUCFE cr, regF src, immF con) %{
match(Set cr (CmpF src con));
ins_cost(100);
- format %{ "vucomxss $src, [$constantaddress]\t# load from constant table: float=$con" %}
+ format %{ "evucomxss $src, [$constantaddress]\t# load from constant table: float=$con" %}
ins_encode %{
- __ vucomxss($src$$XMMRegister, $constantaddress($con));
+ __ evucomxss($src$$XMMRegister, $constantaddress($con));
%}
ins_pipe(pipe_slow);
%}
@@ -14436,9 +14488,9 @@ instruct cmpD_cc_regCFE(rFlagsRegUCFE cr, regD src1, regD src2) %{
match(Set cr (CmpD src1 src2));
ins_cost(100);
- format %{ "vucomxsd $src1, $src2 test" %}
+ format %{ "evucomxsd $src1, $src2 test" %}
ins_encode %{
- __ vucomxsd($src1$$XMMRegister, $src2$$XMMRegister);
+ __ evucomxsd($src1$$XMMRegister, $src2$$XMMRegister);
%}
ins_pipe(pipe_slow);
%}
@@ -14458,9 +14510,9 @@ instruct cmpD_cc_memCFE(rFlagsRegUCFE cr, regD src1, memory src2) %{
match(Set cr (CmpD src1 (LoadD src2)));
ins_cost(100);
- format %{ "vucomxsd $src1, $src2" %}
+ format %{ "evucomxsd $src1, $src2" %}
ins_encode %{
- __ vucomxsd($src1$$XMMRegister, $src2$$Address);
+ __ evucomxsd($src1$$XMMRegister, $src2$$Address);
%}
ins_pipe(pipe_slow);
%}
@@ -14479,9 +14531,9 @@ instruct cmpD_cc_immCFE(rFlagsRegUCFE cr, regD src, immD con) %{
match(Set cr (CmpD src con));
ins_cost(100);
- format %{ "vucomxsd $src, [$constantaddress]\t# load from constant table: double=$con" %}
+ format %{ "evucomxsd $src, [$constantaddress]\t# load from constant table: double=$con" %}
ins_encode %{
- __ vucomxsd($src$$XMMRegister, $constantaddress($con));
+ __ evucomxsd($src$$XMMRegister, $constantaddress($con));
%}
ins_pipe(pipe_slow);
%}
@@ -18789,7 +18841,7 @@ instruct ReplHF_reg(vec dst, regF src, rRegI rtmp) %{
format %{ "replicateHF $dst, $src \t! using $rtmp as TEMP" %}
ins_encode %{
int vlen_enc = vector_length_encoding(this);
- __ vmovw($rtmp$$Register, $src$$XMMRegister);
+ __ evmovw($rtmp$$Register, $src$$XMMRegister);
__ evpbroadcastw($dst$$XMMRegister, $rtmp$$Register, vlen_enc);
%}
ins_pipe( pipe_slow );
@@ -19346,6 +19398,8 @@ instruct reductionI(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtm
match(Set dst (XorReductionV src1 src2));
match(Set dst (MinReductionV src1 src2));
match(Set dst (MaxReductionV src1 src2));
+ match(Set dst (UMinReductionV src1 src2));
+ match(Set dst (UMaxReductionV src1 src2));
effect(TEMP vtmp1, TEMP vtmp2);
format %{ "vector_reduction_int $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
ins_encode %{
@@ -19367,6 +19421,8 @@ instruct reductionL(rRegL dst, rRegL src1, legVec src2, legVec vtmp1, legVec vtm
match(Set dst (XorReductionV src1 src2));
match(Set dst (MinReductionV src1 src2));
match(Set dst (MaxReductionV src1 src2));
+ match(Set dst (UMinReductionV src1 src2));
+ match(Set dst (UMaxReductionV src1 src2));
effect(TEMP vtmp1, TEMP vtmp2);
format %{ "vector_reduction_long $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
ins_encode %{
@@ -19386,6 +19442,8 @@ instruct reductionL_avx512dq(rRegL dst, rRegL src1, vec src2, vec vtmp1, vec vtm
match(Set dst (XorReductionV src1 src2));
match(Set dst (MinReductionV src1 src2));
match(Set dst (MaxReductionV src1 src2));
+ match(Set dst (UMinReductionV src1 src2));
+ match(Set dst (UMaxReductionV src1 src2));
effect(TEMP vtmp1, TEMP vtmp2);
format %{ "vector_reduction_long $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
ins_encode %{
@@ -19614,6 +19672,8 @@ instruct reductionB(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtm
match(Set dst (XorReductionV src1 src2));
match(Set dst (MinReductionV src1 src2));
match(Set dst (MaxReductionV src1 src2));
+ match(Set dst (UMinReductionV src1 src2));
+ match(Set dst (UMaxReductionV src1 src2));
effect(TEMP vtmp1, TEMP vtmp2);
format %{ "vector_reduction_byte $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
ins_encode %{
@@ -19632,6 +19692,8 @@ instruct reductionB_avx512bw(rRegI dst, rRegI src1, vec src2, vec vtmp1, vec vtm
match(Set dst (XorReductionV src1 src2));
match(Set dst (MinReductionV src1 src2));
match(Set dst (MaxReductionV src1 src2));
+ match(Set dst (UMinReductionV src1 src2));
+ match(Set dst (UMaxReductionV src1 src2));
effect(TEMP vtmp1, TEMP vtmp2);
format %{ "vector_reduction_byte $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
ins_encode %{
@@ -19653,6 +19715,8 @@ instruct reductionS(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtm
match(Set dst (XorReductionV src1 src2));
match(Set dst (MinReductionV src1 src2));
match(Set dst (MaxReductionV src1 src2));
+ match(Set dst (UMinReductionV src1 src2));
+ match(Set dst (UMaxReductionV src1 src2));
effect(TEMP vtmp1, TEMP vtmp2);
format %{ "vector_reduction_short $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
ins_encode %{
@@ -20892,7 +20956,7 @@ instruct minmaxFP_reg_avx10_2(vec dst, vec a, vec b) %{
int vlen_enc = vector_length_encoding(this);
int opcode = this->ideal_Opcode();
BasicType elem_bt = Matcher::vector_element_basic_type(this);
- __ vminmax_fp(opcode, elem_bt, $dst$$XMMRegister, k0, $a$$XMMRegister, $b$$XMMRegister, vlen_enc);
+ __ vminmax_fp_avx10_2(opcode, elem_bt, $dst$$XMMRegister, k0, $a$$XMMRegister, $b$$XMMRegister, vlen_enc);
%}
ins_pipe( pipe_slow );
%}
@@ -23908,8 +23972,12 @@ instruct vmask_gen_imm(kReg dst, immL len, rRegL temp) %{
format %{ "vector_mask_gen $len \t! vector mask generator" %}
effect(TEMP temp);
ins_encode %{
- __ mov64($temp$$Register, (0xFFFFFFFFFFFFFFFFUL >> (64 -$len$$constant)));
- __ kmovql($dst$$KRegister, $temp$$Register);
+ if ($len$$constant > 0) {
+ __ mov64($temp$$Register, right_n_bits($len$$constant));
+ __ kmovql($dst$$KRegister, $temp$$Register);
+ } else {
+ __ kxorql($dst$$KRegister, $dst$$KRegister, $dst$$KRegister);
+ }
%}
ins_pipe( pipe_slow );
%}
@@ -25236,9 +25304,9 @@ instruct vector_selectfrom_twovectors_reg_evex(vec index, vec src1, vec src2)
instruct reinterpretS2HF(regF dst, rRegI src)
%{
match(Set dst (ReinterpretS2HF src));
- format %{ "vmovw $dst, $src" %}
+ format %{ "evmovw $dst, $src" %}
ins_encode %{
- __ vmovw($dst$$XMMRegister, $src$$Register);
+ __ evmovw($dst$$XMMRegister, $src$$Register);
%}
ins_pipe(pipe_slow);
%}
@@ -25246,9 +25314,9 @@ instruct reinterpretS2HF(regF dst, rRegI src)
instruct reinterpretHF2S(rRegI dst, regF src)
%{
match(Set dst (ReinterpretHF2S src));
- format %{ "vmovw $dst, $src" %}
+ format %{ "evmovw $dst, $src" %}
ins_encode %{
- __ vmovw($dst$$Register, $src$$XMMRegister);
+ __ evmovw($dst$$Register, $src$$XMMRegister);
%}
ins_pipe(pipe_slow);
%}
@@ -25302,10 +25370,11 @@ instruct scalar_minmax_HF_reg_avx10_2(regF dst, regF src1, regF src2)
predicate(VM_Version::supports_avx10_2());
match(Set dst (MaxHF src1 src2));
match(Set dst (MinHF src1 src2));
+
format %{ "scalar_min_max_fp16 $dst, $src1, $src2" %}
ins_encode %{
- int function = this->ideal_Opcode() == Op_MinHF ? AVX10_2_MINMAX_MIN_COMPARE_SIGN : AVX10_2_MINMAX_MAX_COMPARE_SIGN;
- __ eminmaxsh($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, function);
+ int opcode = this->ideal_Opcode();
+ __ sminmax_fp16_avx10_2(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, k0);
%}
ins_pipe( pipe_slow );
%}
@@ -25316,11 +25385,12 @@ instruct scalar_minmax_HF_reg(regF dst, regF src1, regF src2, kReg ktmp, regF xt
match(Set dst (MaxHF src1 src2));
match(Set dst (MinHF src1 src2));
effect(TEMP_DEF dst, TEMP ktmp, TEMP xtmp1, TEMP xtmp2);
+
format %{ "scalar_min_max_fp16 $dst, $src1, $src2\t using $ktmp, $xtmp1 and $xtmp2 as TEMP" %}
ins_encode %{
int opcode = this->ideal_Opcode();
- __ scalar_max_min_fp16(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $ktmp$$KRegister,
- $xtmp1$$XMMRegister, $xtmp2$$XMMRegister);
+ __ sminmax_fp16(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $ktmp$$KRegister,
+ $xtmp1$$XMMRegister, $xtmp2$$XMMRegister);
%}
ins_pipe( pipe_slow );
%}
@@ -25420,8 +25490,9 @@ instruct vector_minmax_HF_mem_avx10_2(vec dst, vec src1, memory src2)
format %{ "vector_min_max_fp16_mem $dst, $src1, $src2" %}
ins_encode %{
int vlen_enc = vector_length_encoding(this);
- int function = this->ideal_Opcode() == Op_MinVHF ? AVX10_2_MINMAX_MIN_COMPARE_SIGN : AVX10_2_MINMAX_MAX_COMPARE_SIGN;
- __ evminmaxph($dst$$XMMRegister, k0, $src1$$XMMRegister, $src2$$Address, true, function, vlen_enc);
+ int opcode = this->ideal_Opcode();
+ __ vminmax_fp16_avx10_2(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address,
+ k0, vlen_enc);
%}
ins_pipe( pipe_slow );
%}
@@ -25434,8 +25505,9 @@ instruct vector_minmax_HF_reg_avx10_2(vec dst, vec src1, vec src2)
format %{ "vector_min_max_fp16 $dst, $src1, $src2" %}
ins_encode %{
int vlen_enc = vector_length_encoding(this);
- int function = this->ideal_Opcode() == Op_MinVHF ? AVX10_2_MINMAX_MIN_COMPARE_SIGN : AVX10_2_MINMAX_MAX_COMPARE_SIGN;
- __ evminmaxph($dst$$XMMRegister, k0, $src1$$XMMRegister, $src2$$XMMRegister, true, function, vlen_enc);
+ int opcode = this->ideal_Opcode();
+ __ vminmax_fp16_avx10_2(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister,
+ k0, vlen_enc);
%}
ins_pipe( pipe_slow );
%}
@@ -25450,8 +25522,8 @@ instruct vector_minmax_HF_reg(vec dst, vec src1, vec src2, kReg ktmp, vec xtmp1,
ins_encode %{
int vlen_enc = vector_length_encoding(this);
int opcode = this->ideal_Opcode();
- __ vector_max_min_fp16(opcode, $dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, $ktmp$$KRegister,
- $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
+ __ vminmax_fp16(opcode, $dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, $ktmp$$KRegister,
+ $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
%}
ins_pipe( pipe_slow );
%}
diff --git a/src/hotspot/cpu/zero/register_zero.hpp b/src/hotspot/cpu/zero/register_zero.hpp
index fd30f206762..846b649eebd 100644
--- a/src/hotspot/cpu/zero/register_zero.hpp
+++ b/src/hotspot/cpu/zero/register_zero.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright 2007 Red Hat, Inc.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -48,7 +48,6 @@ class RegisterImpl : public AbstractRegisterImpl {
};
// construction
- inline friend Register as_Register(int encoding);
VMReg as_VMReg();
// derived registers, offsets, and addresses
@@ -113,7 +112,6 @@ class ConcreteRegisterImpl : public AbstractRegisterImpl {
static const int max_fpr;
};
-CONSTANT_REGISTER_DECLARATION(Register, noreg, (-1));
-#define noreg ((Register)(noreg_RegisterEnumValue))
+const Register noreg = as_Register(-1);
#endif // CPU_ZERO_REGISTER_ZERO_HPP
diff --git a/src/hotspot/os/aix/globals_aix.hpp b/src/hotspot/os/aix/globals_aix.hpp
index 473d7759063..adc189666ef 100644
--- a/src/hotspot/os/aix/globals_aix.hpp
+++ b/src/hotspot/os/aix/globals_aix.hpp
@@ -37,16 +37,6 @@
range, \
constraint) \
\
- /* Whether to allow the VM to run if EXTSHM=ON. EXTSHM is an environment */ \
- /* variable used on AIX to activate certain hacks which allow more shm segments */\
- /* for 32bit processes. For 64bit processes, it is pointless and may have */ \
- /* harmful side effects (e.g. for some reasonn prevents allocation of 64k pages */\
- /* via shmctl). */ \
- /* Per default we quit with an error if that variable is found; for certain */ \
- /* customer scenarios, we may want to be able to run despite that variable. */ \
- product(bool, AllowExtshm, false, DIAGNOSTIC, \
- "Allow VM to run with EXTSHM=ON.") \
- \
/* Maximum expected size of the data segment. That correlates with the */ \
/* maximum C Heap consumption we expect. */ \
/* We need to leave "breathing space" for the data segment when */ \
diff --git a/src/hotspot/os/aix/libodm_aix.cpp b/src/hotspot/os/aix/libodm_aix.cpp
index 38e8067181a..57eee47c098 100644
--- a/src/hotspot/os/aix/libodm_aix.cpp
+++ b/src/hotspot/os/aix/libodm_aix.cpp
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2015, 2026, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2015, 2019 SAP SE. All rights reserved.
+ * Copyright (c) 2015, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -63,13 +63,12 @@ dynamicOdm::~dynamicOdm() {
void odmWrapper::clean_data() { if (_data) { permit_forbidden_function::free(_data); _data = nullptr; } }
-int odmWrapper::class_offset(const char *field, bool is_aix_5)
+int odmWrapper::class_offset(const char *field)
{
assert(has_class(), "initialization");
for (int i = 0; i < odm_class()->nelem; i++) {
if (strcmp(odm_class()->elem[i].elemname, field) == 0) {
int offset = odm_class()->elem[i].offset;
- if (is_aix_5) { offset += LINK_VAL_OFFSET; }
return offset;
}
}
@@ -88,11 +87,10 @@ void odmWrapper::determine_os_kernel_version(uint32_t* p_ver) {
return;
}
int voff, roff, moff, foff;
- bool is_aix_5 = (major_aix_version == 5);
- voff = odm.class_offset("ver", is_aix_5);
- roff = odm.class_offset("rel", is_aix_5);
- moff = odm.class_offset("mod", is_aix_5);
- foff = odm.class_offset("fix", is_aix_5);
+ voff = odm.class_offset("ver");
+ roff = odm.class_offset("rel");
+ moff = odm.class_offset("mod");
+ foff = odm.class_offset("fix");
if (voff == -1 || roff == -1 || moff == -1 || foff == -1) {
trcVerbose("try_determine_os_kernel_version: could not get offsets");
return;
diff --git a/src/hotspot/os/aix/libodm_aix.hpp b/src/hotspot/os/aix/libodm_aix.hpp
index 924ccaf8c51..11e67a4f5ae 100644
--- a/src/hotspot/os/aix/libodm_aix.hpp
+++ b/src/hotspot/os/aix/libodm_aix.hpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2015, 2024 SAP SE. All rights reserved.
+ * Copyright (c) 2015, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -82,7 +82,7 @@ class odmWrapper : private dynamicOdm {
CLASS_SYMBOL odm_class() { return _odm_class; }
bool has_class() { return odm_class() != (CLASS_SYMBOL)-1; }
- int class_offset(const char *field, bool is_aix_5);
+ int class_offset(const char *field);
char* data() { return _data; }
char* retrieve_obj(const char* name = nullptr) {
diff --git a/src/hotspot/os/aix/os_aix.cpp b/src/hotspot/os/aix/os_aix.cpp
index 61d3f36683c..3cad24d388c 100644
--- a/src/hotspot/os/aix/os_aix.cpp
+++ b/src/hotspot/os/aix/os_aix.cpp
@@ -1,6 +1,6 @@
/*
* Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -122,17 +122,10 @@
extern "C"
int mread_real_time(timebasestruct_t *t, size_t size_of_timebasestruct_t);
-#if !defined(_AIXVERSION_610)
-extern "C" int getthrds64(pid_t, struct thrdentry64*, int, tid64_t*, int);
-extern "C" int getprocs64(procentry64*, int, fdsinfo*, int, pid_t*, int);
-extern "C" int getargs(procsinfo*, int, char*, int);
-#endif
-
#define MAX_PATH (2 * K)
// for multipage initialization error analysis (in 'g_multipage_error')
#define ERROR_MP_OS_TOO_OLD 100
-#define ERROR_MP_EXTSHM_ACTIVE 101
#define ERROR_MP_VMGETINFO_FAILED 102
#define ERROR_MP_VMGETINFO_CLAIMS_NO_SUPPORT_FOR_64K 103
@@ -184,9 +177,6 @@ uint32_t os::Aix::_os_version = 0;
// -1 = uninitialized, 0 - no, 1 - yes
int os::Aix::_xpg_sus_mode = -1;
-// -1 = uninitialized, 0 - no, 1 - yes
-int os::Aix::_extshm = -1;
-
////////////////////////////////////////////////////////////////////////////////
// local variables
@@ -216,7 +206,7 @@ static address g_brk_at_startup = nullptr;
// shmctl(). Different shared memory regions can have different page
// sizes.
//
-// More information can be found at AIBM info center:
+// More information can be found at IBM info center:
// http://publib.boulder.ibm.com/infocenter/aix/v6r1/index.jsp?topic=/com.ibm.aix.prftungd/doc/prftungd/multiple_page_size_app_support.htm
//
static struct {
@@ -1201,13 +1191,6 @@ void os::print_memory_info(outputStream* st) {
const char* const ldr_cntrl = ::getenv("LDR_CNTRL");
st->print_cr(" LDR_CNTRL=%s.", ldr_cntrl ? ldr_cntrl : "");
- // Print out EXTSHM because it is an unsupported setting.
- const char* const extshm = ::getenv("EXTSHM");
- st->print_cr(" EXTSHM=%s.", extshm ? extshm : "");
- if ( (strcmp(extshm, "on") == 0) || (strcmp(extshm, "ON") == 0) ) {
- st->print_cr(" *** Unsupported! Please remove EXTSHM from your environment! ***");
- }
-
// Print out AIXTHREAD_GUARDPAGES because it affects the size of pthread stacks.
const char* const aixthread_guardpages = ::getenv("AIXTHREAD_GUARDPAGES");
st->print_cr(" AIXTHREAD_GUARDPAGES=%s.",
@@ -1956,11 +1939,6 @@ char* os::pd_reserve_memory_special(size_t bytes, size_t alignment, size_t page_
return nullptr;
}
-bool os::pd_release_memory_special(char* base, size_t bytes) {
- fatal("os::release_memory_special should not be called on AIX.");
- return false;
-}
-
size_t os::large_page_size() {
return _large_page_size;
}
@@ -2144,8 +2122,6 @@ void os::init(void) {
// datapsize = 64k. Data segment, thread stacks are 64k paged.
// This normally means that we can allocate 64k pages dynamically.
- // (There is one special case where this may be false: EXTSHM=on.
- // but we decided to not support that mode).
assert0(g_multipage_support.can_use_64K_pages || g_multipage_support.can_use_64K_mmap_pages);
set_page_size(64*K);
@@ -2530,23 +2506,18 @@ void os::Aix::initialize_os_info() {
assert(minor > 0, "invalid OS release");
_os_version = (major << 24) | (minor << 16);
char ver_str[20] = {0};
- const char* name_str = "unknown OS";
- if (strcmp(uts.sysname, "AIX") == 0) {
- // We run on AIX. We do not support versions older than AIX 7.1.
- // Determine detailed AIX version: Version, Release, Modification, Fix Level.
- odmWrapper::determine_os_kernel_version(&_os_version);
- if (os_version_short() < 0x0701) {
- log_warning(os)("AIX releases older than AIX 7.1 are not supported.");
- assert(false, "AIX release too old.");
- }
- name_str = "AIX";
- jio_snprintf(ver_str, sizeof(ver_str), "%u.%u.%u.%u",
- major, minor, (_os_version >> 8) & 0xFF, _os_version & 0xFF);
- } else {
- assert(false, "%s", name_str);
+ // We do not support versions older than AIX 7.2 TL 5.
+ // Determine detailed AIX version: Version, Release, Modification, Fix Level.
+ odmWrapper::determine_os_kernel_version(&_os_version);
+ if (_os_version < 0x07020500) {
+ log_warning(os)("AIX releases older than AIX 7.2 TL 5 are not supported.");
+ assert(false, "AIX release too old.");
}
- log_info(os)("We run on %s %s", name_str, ver_str);
+
+ jio_snprintf(ver_str, sizeof(ver_str), "%u.%u.%u.%u",
+ major, minor, (_os_version >> 8) & 0xFF, _os_version & 0xFF);
+ log_info(os)("We run on AIX %s", ver_str);
}
guarantee(_os_version, "Could not determine AIX release");
@@ -2559,28 +2530,13 @@ void os::Aix::initialize_os_info() {
void os::Aix::scan_environment() {
char* p;
- int rc;
- // Warn explicitly if EXTSHM=ON is used. That switch changes how
- // System V shared memory behaves. One effect is that page size of
- // shared memory cannot be change dynamically, effectivly preventing
- // large pages from working.
- // This switch was needed on AIX 32bit, but on AIX 64bit the general
- // recommendation is (in OSS notes) to switch it off.
+ // Reject EXTSHM=ON. That switch changes how System V shared memory behaves
+ // and prevents allocation of 64k pages for the heap.
p = ::getenv("EXTSHM");
trcVerbose("EXTSHM=%s.", p ? p : "");
if (p && strcasecmp(p, "ON") == 0) {
- _extshm = 1;
- log_warning(os)("*** Unsupported mode! Please remove EXTSHM from your environment! ***");
- if (!AllowExtshm) {
- // We allow under certain conditions the user to continue. However, we want this
- // to be a fatal error by default. On certain AIX systems, leaving EXTSHM=ON means
- // that the VM is not able to allocate 64k pages for the heap.
- // We do not want to run with reduced performance.
- vm_exit_during_initialization("EXTSHM is ON. Please remove EXTSHM from your environment.");
- }
- } else {
- _extshm = 0;
+ vm_exit_during_initialization("EXTSHM is ON. Please remove EXTSHM from your environment.");
}
// SPEC1170 behaviour: will change the behaviour of a number of POSIX APIs.
@@ -2711,3 +2667,7 @@ void os::print_memory_mappings(char* addr, size_t bytes, outputStream* st) {}
void os::jfr_report_memory_info() {}
#endif // INCLUDE_JFR
+
+void os::print_open_file_descriptors(outputStream* st) {
+ // File descriptor counting not implemented on AIX
+}
diff --git a/src/hotspot/os/aix/os_aix.hpp b/src/hotspot/os/aix/os_aix.hpp
index a30e2077fc2..e21d2cf81bb 100644
--- a/src/hotspot/os/aix/os_aix.hpp
+++ b/src/hotspot/os/aix/os_aix.hpp
@@ -49,11 +49,6 @@ class os::Aix {
// 1 - SPEC1170 requested (XPG_SUS_ENV is ON)
static int _xpg_sus_mode;
- // -1 = uninitialized,
- // 0 - EXTSHM=OFF or not set
- // 1 - EXTSHM=ON
- static int _extshm;
-
static bool available_memory(physical_memory_size_type& value);
static bool free_memory(physical_memory_size_type& value);
static physical_memory_size_type physical_memory() { return _physical_memory; }
@@ -111,12 +106,6 @@ class os::Aix {
return _xpg_sus_mode;
}
- // Returns true if EXTSHM=ON.
- static bool extshm() {
- assert(_extshm != -1, "not initialized");
- return _extshm;
- }
-
// result struct for get_meminfo()
struct meminfo_t {
diff --git a/src/hotspot/os/aix/porting_aix.cpp b/src/hotspot/os/aix/porting_aix.cpp
index b3f878fbfdd..f0527136d90 100644
--- a/src/hotspot/os/aix/porting_aix.cpp
+++ b/src/hotspot/os/aix/porting_aix.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2012, 2024 SAP SE. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
* Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -426,6 +426,10 @@ int dladdr(void* addr, Dl_info* info) {
}
+int JVM_dladdr(void* addr, Dl_info* info) {
+ return dladdr(addr, info);
+}
+
/////////////////////////////////////////////////////////////////////////////
// Native callstack dumping
diff --git a/src/hotspot/os/aix/porting_aix.hpp b/src/hotspot/os/aix/porting_aix.hpp
index a1a22d81471..0bd71079d0a 100644
--- a/src/hotspot/os/aix/porting_aix.hpp
+++ b/src/hotspot/os/aix/porting_aix.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2012, 2024 SAP SE. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -37,25 +37,9 @@
// (see http://linux.die.net/man/3/dladdr)
// dladdr(3) is not POSIX but a GNU extension, and is not available on AIX.
//
-// Differences between AIX dladdr and Linux dladdr:
-//
-// 1) Dl_info.dli_fbase: can never work, is disabled.
-// A loaded image on AIX is divided in multiple segments, at least two
-// (text and data) but potentially also far more. This is because the loader may
-// load each member into an own segment, as for instance happens with the libC.a
-// 2) Dl_info.dli_sname: This only works for code symbols (functions); for data, a
-// zero-length string is returned ("").
-// 3) Dl_info.dli_saddr: For code, this will return the entry point of the function,
-// not the function descriptor.
-typedef struct {
- const char *dli_fname; // file path of loaded library
- // void *dli_fbase;
- const char *dli_sname; // symbol name; "" if not known
- void *dli_saddr; // address of *entry* of function; not function descriptor;
-} Dl_info;
+#include "dl_info.h"
-// Note: we export this to use it inside J2se too
#ifdef __cplusplus
extern "C"
#endif
diff --git a/src/hotspot/os/bsd/os_bsd.cpp b/src/hotspot/os/bsd/os_bsd.cpp
index 0ed5335adc3..a4d9a2197a5 100644
--- a/src/hotspot/os/bsd/os_bsd.cpp
+++ b/src/hotspot/os/bsd/os_bsd.cpp
@@ -76,6 +76,7 @@
# include
# include
# include
+# include
# include
# include
# include
@@ -102,6 +103,7 @@
#endif
#ifdef __APPLE__
+ #include
#include
#include
#endif
@@ -1885,11 +1887,6 @@ char* os::pd_reserve_memory_special(size_t bytes, size_t alignment, size_t page_
return nullptr;
}
-bool os::pd_release_memory_special(char* base, size_t bytes) {
- fatal("os::release_memory_special should not be called on BSD.");
- return false;
-}
-
size_t os::large_page_size() {
return _large_page_size;
}
@@ -2601,3 +2598,45 @@ bool os::pd_dll_unload(void* libhandle, char* ebuf, int ebuflen) {
return res;
} // end: os::pd_dll_unload()
+
+void os::print_open_file_descriptors(outputStream* st) {
+#ifdef __APPLE__
+ char buf[1024 * sizeof(struct proc_fdinfo)];
+ os::Bsd::print_open_file_descriptors(st, buf, sizeof(buf));
+#else
+ st->print_cr("Open File Descriptors: unknown");
+#endif
+}
+
+void os::Bsd::print_open_file_descriptors(outputStream* st, char* buf, size_t buflen) {
+#ifdef __APPLE__
+ pid_t my_pid;
+
+ // ensure the scratch buffer is big enough for at least one FD info struct
+ precond(buflen >= sizeof(struct proc_fdinfo));
+ kern_return_t kres = pid_for_task(mach_task_self(), &my_pid);
+ if (kres != KERN_SUCCESS) {
+ st->print_cr("Open File Descriptors: unknown");
+ return;
+ }
+ size_t max_fds = buflen / sizeof(struct proc_fdinfo);
+ struct proc_fdinfo* fds = reinterpret_cast(buf);
+
+ // fill our buffer with FD info, up to the available buffer size
+ int res = proc_pidinfo(my_pid, PROC_PIDLISTFDS, 0, fds, max_fds * sizeof(struct proc_fdinfo));
+ if (res <= 0) {
+ st->print_cr("Open File Descriptors: unknown");
+ return;
+ }
+
+ // print lower threshold if count exceeds buffer size
+ int nfiles = res / sizeof(struct proc_fdinfo);
+ if ((size_t)nfiles >= max_fds) {
+ st->print_cr("Open File Descriptors: > %zu", max_fds);
+ return;
+ }
+ st->print_cr("Open File Descriptors: %d", nfiles);
+#else
+ st->print_cr("Open File Descriptors: unknown");
+#endif
+}
diff --git a/src/hotspot/os/bsd/os_bsd.hpp b/src/hotspot/os/bsd/os_bsd.hpp
index da73211b9a7..e87a680b2d2 100644
--- a/src/hotspot/os/bsd/os_bsd.hpp
+++ b/src/hotspot/os/bsd/os_bsd.hpp
@@ -123,6 +123,8 @@ class os::Bsd {
static int get_node_by_cpu(int cpu_id);
static void print_uptime_info(outputStream* st);
+ static void print_open_file_descriptors(outputStream* st, char* buf, size_t buflen);
+ static void print_open_file_descriptors(outputStream* st);
};
#endif // OS_BSD_OS_BSD_HPP
diff --git a/src/hotspot/os/bsd/semaphore_bsd.cpp b/src/hotspot/os/bsd/semaphore_bsd.cpp
index 827c955677e..c35712ff2da 100644
--- a/src/hotspot/os/bsd/semaphore_bsd.cpp
+++ b/src/hotspot/os/bsd/semaphore_bsd.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -81,27 +81,37 @@ bool OSXSemaphore::timedwait(int64_t millis) {
// kernel semaphores take a relative timeout
mach_timespec_t waitspec;
- int secs = millis / MILLIUNITS;
- int nsecs = millis_to_nanos(millis % MILLIUNITS);
- waitspec.tv_sec = secs;
- waitspec.tv_nsec = nsecs;
+ int64_t starttime;
+ const bool is_trywait = millis == 0;
- int64_t starttime = os::javaTimeNanos();
+ if (!is_trywait) {
+ int secs = millis / MILLIUNITS;
+ int nsecs = millis_to_nanos(millis % MILLIUNITS);
+ waitspec.tv_sec = secs;
+ waitspec.tv_nsec = nsecs;
+
+ starttime = os::javaTimeNanos();
+ } else {
+ waitspec.tv_sec = 0;
+ waitspec.tv_nsec = 0;
+ }
kr = semaphore_timedwait(_semaphore, waitspec);
while (kr == KERN_ABORTED) {
- // reduce the timeout and try again
- int64_t totalwait = millis_to_nanos(millis);
- int64_t current = os::javaTimeNanos();
- int64_t passedtime = current - starttime;
+ if (!is_trywait) {
+ // reduce the timeout and try again
+ int64_t totalwait = millis_to_nanos(millis);
+ int64_t current = os::javaTimeNanos();
+ int64_t passedtime = current - starttime;
- if (passedtime >= totalwait) {
- waitspec.tv_sec = 0;
- waitspec.tv_nsec = 0;
- } else {
- int64_t waittime = totalwait - (current - starttime);
- waitspec.tv_sec = waittime / NANOSECS_PER_SEC;
- waitspec.tv_nsec = waittime % NANOSECS_PER_SEC;
+ if (passedtime >= totalwait) {
+ waitspec.tv_sec = 0;
+ waitspec.tv_nsec = 0;
+ } else {
+ int64_t waittime = totalwait - (current - starttime);
+ waitspec.tv_sec = waittime / NANOSECS_PER_SEC;
+ waitspec.tv_nsec = waittime % NANOSECS_PER_SEC;
+ }
}
kr = semaphore_timedwait(_semaphore, waitspec);
diff --git a/src/hotspot/os/linux/cgroupSubsystem_linux.cpp b/src/hotspot/os/linux/cgroupSubsystem_linux.cpp
index e49d070890e..13a005591fb 100644
--- a/src/hotspot/os/linux/cgroupSubsystem_linux.cpp
+++ b/src/hotspot/os/linux/cgroupSubsystem_linux.cpp
@@ -28,7 +28,6 @@
#include "cgroupV2Subsystem_linux.hpp"
#include "logging/log.hpp"
#include "memory/allocation.hpp"
-#include "os_linux.hpp"
#include "runtime/globals.hpp"
#include "runtime/os.hpp"
#include "utilities/globalDefinitions.hpp"
@@ -605,6 +604,11 @@ void CgroupSubsystemFactory::cleanup(CgroupInfo* cg_infos) {
}
}
+void CgroupSubsystem::adjust_controllers(physical_memory_size_type upper_mem_bound, int upper_cpu_bound) {
+ CgroupUtil::adjust_controller(memory_controller()->controller(), upper_mem_bound);
+ CgroupUtil::adjust_controller(cpu_controller()->controller(), upper_cpu_bound);
+}
+
/* active_processor_count
*
* Calculate an appropriate number of active processors for the
@@ -631,7 +635,7 @@ void CgroupSubsystemFactory::cleanup(CgroupInfo* cg_infos) {
* return:
* true if there were no errors. false otherwise.
*/
-bool CgroupSubsystem::active_processor_count(double& value) {
+bool CgroupSubsystem::active_processor_count(int (*cpu_bound_func)(), double& value) {
// We use a cache with a timeout to avoid performing expensive
// computations in the event this function is called frequently.
// [See 8227006].
@@ -643,7 +647,7 @@ bool CgroupSubsystem::active_processor_count(double& value) {
return true;
}
- int cpu_count = os::Linux::active_processor_count();
+ int cpu_count = cpu_bound_func();
double result = -1;
if (!CgroupUtil::processor_count(contrl->controller(), cpu_count, result)) {
return false;
diff --git a/src/hotspot/os/linux/cgroupSubsystem_linux.hpp b/src/hotspot/os/linux/cgroupSubsystem_linux.hpp
index d083a9985c2..adde37e1c77 100644
--- a/src/hotspot/os/linux/cgroupSubsystem_linux.hpp
+++ b/src/hotspot/os/linux/cgroupSubsystem_linux.hpp
@@ -278,7 +278,7 @@ class CgroupMemoryController: public CHeapObj {
class CgroupSubsystem: public CHeapObj {
public:
bool memory_limit_in_bytes(physical_memory_size_type upper_bound, physical_memory_size_type& value);
- bool active_processor_count(double& value);
+ bool active_processor_count(int (*cpu_bound_func)(), double& value);
virtual bool pids_max(uint64_t& value) = 0;
virtual bool pids_current(uint64_t& value) = 0;
@@ -291,6 +291,8 @@ class CgroupSubsystem: public CHeapObj {
virtual CachingCgroupController* cpu_controller() = 0;
virtual CgroupCpuacctController* cpuacct_controller() = 0;
+ void adjust_controllers(physical_memory_size_type upper_mem_bound, int upper_cpu_bound);
+
bool cpu_quota(int& value);
bool cpu_period(int& value);
bool cpu_shares(int& value);
diff --git a/src/hotspot/os/linux/cgroupUtil_linux.cpp b/src/hotspot/os/linux/cgroupUtil_linux.cpp
index 570b335940b..f166f6cd5e4 100644
--- a/src/hotspot/os/linux/cgroupUtil_linux.cpp
+++ b/src/hotspot/os/linux/cgroupUtil_linux.cpp
@@ -24,7 +24,6 @@
*/
#include "cgroupUtil_linux.hpp"
-#include "os_linux.hpp"
bool CgroupUtil::processor_count(CgroupCpuController* cpu_ctrl, int upper_bound, double& value) {
assert(upper_bound > 0, "upper bound of cpus must be positive");
@@ -82,7 +81,7 @@ double CgroupUtil::get_updated_cpu_limit(CgroupCpuController* cpu,
return lowest;
}
-void CgroupUtil::adjust_controller(CgroupMemoryController* mem) {
+void CgroupUtil::adjust_controller(CgroupMemoryController* mem, physical_memory_size_type upper_bound) {
assert(mem->cgroup_path() != nullptr, "invariant");
if (strstr(mem->cgroup_path(), "../") != nullptr) {
log_warning(os, container)("Cgroup memory controller path at '%s' seems to have moved "
@@ -100,17 +99,16 @@ void CgroupUtil::adjust_controller(CgroupMemoryController* mem) {
char* cg_path = os::strdup(orig);
char* last_slash;
assert(cg_path[0] == '/', "cgroup path must start with '/'");
- physical_memory_size_type phys_mem = os::Linux::physical_memory();
char* limit_cg_path = nullptr;
physical_memory_size_type limit = value_unlimited;
- physical_memory_size_type lowest_limit = phys_mem;
- lowest_limit = get_updated_mem_limit(mem, lowest_limit, phys_mem);
- physical_memory_size_type orig_limit = lowest_limit != phys_mem ? lowest_limit : phys_mem;
+ physical_memory_size_type lowest_limit = upper_bound;
+ lowest_limit = get_updated_mem_limit(mem, lowest_limit, upper_bound);
+ physical_memory_size_type orig_limit = lowest_limit != upper_bound ? lowest_limit : upper_bound;
while ((last_slash = strrchr(cg_path, '/')) != cg_path) {
*last_slash = '\0'; // strip path
// update to shortened path and try again
mem->set_subsystem_path(cg_path);
- limit = get_updated_mem_limit(mem, lowest_limit, phys_mem);
+ limit = get_updated_mem_limit(mem, lowest_limit, upper_bound);
if (limit < lowest_limit) {
lowest_limit = limit;
os::free(limit_cg_path); // handles nullptr
@@ -119,13 +117,13 @@ void CgroupUtil::adjust_controller(CgroupMemoryController* mem) {
}
// need to check limit at mount point
mem->set_subsystem_path("/");
- limit = get_updated_mem_limit(mem, lowest_limit, phys_mem);
+ limit = get_updated_mem_limit(mem, lowest_limit, upper_bound);
if (limit < lowest_limit) {
lowest_limit = limit;
os::free(limit_cg_path); // handles nullptr
limit_cg_path = os::strdup("/");
}
- assert(lowest_limit <= phys_mem, "limit must not exceed host memory");
+ assert(lowest_limit <= upper_bound, "limit must not exceed upper bound");
if (lowest_limit != orig_limit) {
// we've found a lower limit anywhere in the hierarchy,
// set the path to the limit path
@@ -147,7 +145,7 @@ void CgroupUtil::adjust_controller(CgroupMemoryController* mem) {
os::free(limit_cg_path);
}
-void CgroupUtil::adjust_controller(CgroupCpuController* cpu) {
+void CgroupUtil::adjust_controller(CgroupCpuController* cpu, int upper_bound) {
assert(cpu->cgroup_path() != nullptr, "invariant");
if (strstr(cpu->cgroup_path(), "../") != nullptr) {
log_warning(os, container)("Cgroup cpu controller path at '%s' seems to have moved "
@@ -165,17 +163,16 @@ void CgroupUtil::adjust_controller(CgroupCpuController* cpu) {
char* cg_path = os::strdup(orig);
char* last_slash;
assert(cg_path[0] == '/', "cgroup path must start with '/'");
- int host_cpus = os::Linux::active_processor_count();
- int lowest_limit = host_cpus;
- double cpus = get_updated_cpu_limit(cpu, lowest_limit, host_cpus);
- int orig_limit = lowest_limit != host_cpus ? lowest_limit : host_cpus;
+ int lowest_limit = upper_bound;
+ double cpus = get_updated_cpu_limit(cpu, lowest_limit, upper_bound);
+ int orig_limit = lowest_limit != upper_bound ? lowest_limit : upper_bound;
char* limit_cg_path = nullptr;
while ((last_slash = strrchr(cg_path, '/')) != cg_path) {
*last_slash = '\0'; // strip path
// update to shortened path and try again
cpu->set_subsystem_path(cg_path);
- cpus = get_updated_cpu_limit(cpu, lowest_limit, host_cpus);
- if (cpus != host_cpus && cpus < lowest_limit) {
+ cpus = get_updated_cpu_limit(cpu, lowest_limit, upper_bound);
+ if (cpus != upper_bound && cpus < lowest_limit) {
lowest_limit = cpus;
os::free(limit_cg_path); // handles nullptr
limit_cg_path = os::strdup(cg_path);
@@ -183,8 +180,8 @@ void CgroupUtil::adjust_controller(CgroupCpuController* cpu) {
}
// need to check limit at mount point
cpu->set_subsystem_path("/");
- cpus = get_updated_cpu_limit(cpu, lowest_limit, host_cpus);
- if (cpus != host_cpus && cpus < lowest_limit) {
+ cpus = get_updated_cpu_limit(cpu, lowest_limit, upper_bound);
+ if (cpus != upper_bound && cpus < lowest_limit) {
lowest_limit = cpus;
os::free(limit_cg_path); // handles nullptr
limit_cg_path = os::strdup(cg_path);
diff --git a/src/hotspot/os/linux/cgroupUtil_linux.hpp b/src/hotspot/os/linux/cgroupUtil_linux.hpp
index 1fd2a7d872b..68585c22c2d 100644
--- a/src/hotspot/os/linux/cgroupUtil_linux.hpp
+++ b/src/hotspot/os/linux/cgroupUtil_linux.hpp
@@ -35,10 +35,10 @@ class CgroupUtil: AllStatic {
static bool processor_count(CgroupCpuController* cpu, int upper_bound, double& value);
// Given a memory controller, adjust its path to a point in the hierarchy
// that represents the closest memory limit.
- static void adjust_controller(CgroupMemoryController* m);
+ static void adjust_controller(CgroupMemoryController* m, physical_memory_size_type upper_bound);
// Given a cpu controller, adjust its path to a point in the hierarchy
// that represents the closest cpu limit.
- static void adjust_controller(CgroupCpuController* c);
+ static void adjust_controller(CgroupCpuController* c, int upper_bound);
private:
static physical_memory_size_type get_updated_mem_limit(CgroupMemoryController* m,
physical_memory_size_type lowest,
diff --git a/src/hotspot/os/linux/cgroupV1Subsystem_linux.cpp b/src/hotspot/os/linux/cgroupV1Subsystem_linux.cpp
index c8f5a290c99..e42b7a13391 100644
--- a/src/hotspot/os/linux/cgroupV1Subsystem_linux.cpp
+++ b/src/hotspot/os/linux/cgroupV1Subsystem_linux.cpp
@@ -326,8 +326,6 @@ CgroupV1Subsystem::CgroupV1Subsystem(CgroupV1Controller* cpuset,
_cpuset(cpuset),
_cpuacct(cpuacct),
_pids(pids) {
- CgroupUtil::adjust_controller(memory);
- CgroupUtil::adjust_controller(cpu);
_memory = new CachingCgroupController(memory);
_cpu = new CachingCgroupController(cpu);
}
diff --git a/src/hotspot/os/linux/cgroupV2Subsystem_linux.cpp b/src/hotspot/os/linux/cgroupV2Subsystem_linux.cpp
index 30e1affc646..edd80bb7427 100644
--- a/src/hotspot/os/linux/cgroupV2Subsystem_linux.cpp
+++ b/src/hotspot/os/linux/cgroupV2Subsystem_linux.cpp
@@ -154,8 +154,6 @@ CgroupV2Subsystem::CgroupV2Subsystem(CgroupV2MemoryController * memory,
CgroupV2CpuacctController* cpuacct,
CgroupV2Controller unified) :
_unified(unified) {
- CgroupUtil::adjust_controller(memory);
- CgroupUtil::adjust_controller(cpu);
_memory = new CachingCgroupController(memory);
_cpu = new CachingCgroupController(cpu);
_cpuacct = cpuacct;
diff --git a/src/hotspot/os/linux/hugepages.cpp b/src/hotspot/os/linux/hugepages.cpp
index 5472c093d3f..b065f7b1496 100644
--- a/src/hotspot/os/linux/hugepages.cpp
+++ b/src/hotspot/os/linux/hugepages.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2011, 2024, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -35,11 +35,16 @@
#include
ExplicitHugePageSupport::ExplicitHugePageSupport() :
- _initialized(false), _pagesizes(), _default_hugepage_size(SIZE_MAX), _inconsistent(false) {}
+ _initialized{false}, _os_supported{}, _pre_allocated{}, _default_hugepage_size{0}, _inconsistent{false} {}
-os::PageSizes ExplicitHugePageSupport::pagesizes() const {
+os::PageSizes ExplicitHugePageSupport::os_supported() const {
assert(_initialized, "Not initialized");
- return _pagesizes;
+ return _os_supported;
+}
+
+os::PageSizes ExplicitHugePageSupport::pre_allocated() const {
+ assert(_initialized, "Not initialized");
+ return _pre_allocated;
}
size_t ExplicitHugePageSupport::default_hugepage_size() const {
@@ -63,7 +68,7 @@ static size_t scan_default_hugepagesize() {
// format has been changed), we'll set largest page size to 0
FILE *fp = os::fopen("/proc/meminfo", "r");
- if (fp) {
+ if (fp != nullptr) {
while (!feof(fp)) {
int x = 0;
char buf[16];
@@ -76,7 +81,7 @@ static size_t scan_default_hugepagesize() {
// skip to next line
for (;;) {
int ch = fgetc(fp);
- if (ch == EOF || ch == (int)'\n') break;
+ if (ch == EOF || ch == '\n') break;
}
}
}
@@ -129,10 +134,24 @@ static os::PageSizes scan_hugepages() {
return pagesizes;
}
+static os::PageSizes filter_pre_allocated_hugepages(os::PageSizes pagesizes) {
+ os::PageSizes pre_allocated{};
+ char filename[PATH_MAX];
+ for (size_t ps = pagesizes.smallest(); ps != 0; ps = pagesizes.next_larger(ps)) {
+ os::snprintf_checked(filename, sizeof(filename), "%s/hugepages-%zukB/nr_hugepages", sys_hugepages, ps / K);
+ size_t pages;
+ bool read_success = read_number_file(filename, &pages);
+ if (read_success && pages > 0) {
+ pre_allocated.add(ps);
+ }
+ }
+ return pre_allocated;
+}
+
void ExplicitHugePageSupport::print_on(outputStream* os) {
if (_initialized) {
os->print_cr("Explicit hugepage support:");
- for (size_t s = _pagesizes.smallest(); s != 0; s = _pagesizes.next_larger(s)) {
+ for (size_t s = _os_supported.smallest(); s != 0; s = _os_supported.next_larger(s)) {
os->print_cr(" hugepage size: " EXACTFMT, EXACTFMTARGS(s));
}
os->print_cr(" default hugepage size: " EXACTFMT, EXACTFMTARGS(_default_hugepage_size));
@@ -147,14 +166,15 @@ void ExplicitHugePageSupport::print_on(outputStream* os) {
void ExplicitHugePageSupport::scan_os() {
_default_hugepage_size = scan_default_hugepagesize();
if (_default_hugepage_size > 0) {
- _pagesizes = scan_hugepages();
+ _os_supported = scan_hugepages();
+ _pre_allocated = filter_pre_allocated_hugepages(_os_supported);
// See https://www.kernel.org/doc/Documentation/vm/hugetlbpage.txt: /proc/meminfo should match
// /sys/kernel/mm/hugepages/hugepages-xxxx. However, we may run on a broken kernel (e.g. on WSL)
// that only exposes /proc/meminfo but not /sys/kernel/mm/hugepages. In that case, we are not
// sure about the state of hugepage support by the kernel, so we won't use explicit hugepages.
- if (!_pagesizes.contains(_default_hugepage_size)) {
+ if (!_os_supported.contains(_default_hugepage_size)) {
log_info(pagesize)("Unexpected configuration: default pagesize (%zu) "
- "has no associated directory in /sys/kernel/mm/hugepages..", _default_hugepage_size);
+ "has no associated directory in /sys/kernel/mm/hugepages.", _default_hugepage_size);
_inconsistent = true;
}
}
@@ -167,7 +187,7 @@ void ExplicitHugePageSupport::scan_os() {
}
THPSupport::THPSupport() :
- _initialized(false), _mode(THPMode::never), _pagesize(SIZE_MAX) {}
+ _initialized{false}, _mode{THPMode::never}, _pagesize{0} {}
THPMode THPSupport::mode() const {
@@ -201,7 +221,6 @@ void THPSupport::scan_os() {
}
// Scan large page size for THP from hpage_pmd_size
- _pagesize = 0;
if (read_number_file("/sys/kernel/mm/transparent_hugepage/hpage_pmd_size", &_pagesize)) {
assert(_pagesize > 0, "Expected");
}
diff --git a/src/hotspot/os/linux/hugepages.hpp b/src/hotspot/os/linux/hugepages.hpp
index efd27c55fd6..5a9767b4ff8 100644
--- a/src/hotspot/os/linux/hugepages.hpp
+++ b/src/hotspot/os/linux/hugepages.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2011, 2024, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -45,7 +45,10 @@ class ExplicitHugePageSupport {
// All supported hugepage sizes (sizes for which entries exist
// in /sys/kernel/mm/hugepages/hugepage-xxx)
- os::PageSizes _pagesizes;
+ os::PageSizes _os_supported;
+
+ // Above pages filtered for where the contents of file nr_hugepages was larger than zero
+ os::PageSizes _pre_allocated;
// Contains the default hugepage. The "default hugepage size" is the one that
// - is marked in /proc/meminfo as "Hugepagesize"
@@ -60,7 +63,8 @@ public:
void scan_os();
- os::PageSizes pagesizes() const;
+ os::PageSizes os_supported() const;
+ os::PageSizes pre_allocated() const;
size_t default_hugepage_size() const;
void print_on(outputStream* os);
diff --git a/src/hotspot/os/linux/osContainer_linux.cpp b/src/hotspot/os/linux/osContainer_linux.cpp
index b46263efd99..da2cbf381e6 100644
--- a/src/hotspot/os/linux/osContainer_linux.cpp
+++ b/src/hotspot/os/linux/osContainer_linux.cpp
@@ -59,6 +59,11 @@ void OSContainer::init() {
if (cgroup_subsystem == nullptr) {
return; // Required subsystem files not found or other error
}
+ // Adjust controller paths once subsystem is initialized
+ physical_memory_size_type phys_mem = os::Linux::physical_memory();
+ int host_cpus = os::Linux::active_processor_count();
+ cgroup_subsystem->adjust_controllers(phys_mem, host_cpus);
+
/*
* In order to avoid a false positive on is_containerized() on
* Linux systems outside a container *and* to ensure compatibility
@@ -252,7 +257,7 @@ char * OSContainer::cpu_cpuset_memory_nodes() {
bool OSContainer::active_processor_count(double& value) {
assert(cgroup_subsystem != nullptr, "cgroup subsystem not available");
- return cgroup_subsystem->active_processor_count(value);
+ return cgroup_subsystem->active_processor_count(&os::Linux::active_processor_count, value);
}
bool OSContainer::cpu_quota(int& value) {
diff --git a/src/hotspot/os/linux/os_linux.cpp b/src/hotspot/os/linux/os_linux.cpp
index 09c514e3d05..bf096897aa7 100644
--- a/src/hotspot/os/linux/os_linux.cpp
+++ b/src/hotspot/os/linux/os_linux.cpp
@@ -83,6 +83,7 @@
#endif
# include
+# include
# include
# include
# include
@@ -113,6 +114,7 @@
# include
# include
# include
+# include
# include
#ifdef __GLIBC__
# include
@@ -1311,7 +1313,7 @@ bool os::is_primordial_thread(void) {
// Find the virtual memory area that contains addr
static bool find_vma(address addr, address* vma_low, address* vma_high) {
FILE *fp = os::fopen("/proc/self/maps", "r");
- if (fp) {
+ if (fp != nullptr) {
address low, high;
while (!feof(fp)) {
if (fscanf(fp, "%p-%p", &low, &high) == 2) {
@@ -1324,7 +1326,7 @@ static bool find_vma(address addr, address* vma_low, address* vma_high) {
}
for (;;) {
int ch = fgetc(fp);
- if (ch == EOF || ch == (int)'\n') break;
+ if (ch == EOF || ch == '\n') break;
}
}
fclose(fp);
@@ -2161,6 +2163,8 @@ void os::print_os_info(outputStream* st) {
os::Posix::print_rlimit_info(st);
+ os::print_open_file_descriptors(st);
+
os::Posix::print_load_average(st);
st->cr();
@@ -3814,8 +3818,8 @@ static int hugetlbfs_page_size_flag(size_t page_size) {
}
static bool hugetlbfs_sanity_check(size_t page_size) {
- const os::PageSizes page_sizes = HugePages::explicit_hugepage_info().pagesizes();
- assert(page_sizes.contains(page_size), "Invalid page sizes passed");
+ const os::PageSizes os_supported = HugePages::explicit_hugepage_info().os_supported();
+ assert(os_supported.contains(page_size), "Invalid page sizes passed (%zu)", page_size);
// Include the page size flag to ensure we sanity check the correct page size.
int flags = MAP_ANONYMOUS | MAP_PRIVATE | MAP_HUGETLB | hugetlbfs_page_size_flag(page_size);
@@ -3829,16 +3833,16 @@ static bool hugetlbfs_sanity_check(size_t page_size) {
log_info(pagesize)("Large page size (" EXACTFMT ") failed sanity check, "
"checking if smaller large page sizes are usable",
EXACTFMTARGS(page_size));
- for (size_t page_size_ = page_sizes.next_smaller(page_size);
- page_size_ > os::vm_page_size();
- page_size_ = page_sizes.next_smaller(page_size_)) {
- flags = MAP_ANONYMOUS | MAP_PRIVATE | MAP_HUGETLB | hugetlbfs_page_size_flag(page_size_);
- p = mmap(nullptr, page_size_, PROT_READ|PROT_WRITE, flags, -1, 0);
+ for (size_t size = os_supported.next_smaller(page_size);
+ size > os::vm_page_size();
+ size = os_supported.next_smaller(size)) {
+ flags = MAP_ANONYMOUS | MAP_PRIVATE | MAP_HUGETLB | hugetlbfs_page_size_flag(size);
+ p = mmap(nullptr, size, PROT_READ|PROT_WRITE, flags, -1, 0);
if (p != MAP_FAILED) {
// Mapping succeeded, sanity check passed.
- munmap(p, page_size_);
+ munmap(p, size);
log_info(pagesize)("Large page size (" EXACTFMT ") passed sanity check",
- EXACTFMTARGS(page_size_));
+ EXACTFMTARGS(size));
return true;
}
}
@@ -4020,7 +4024,7 @@ void os::Linux::large_page_init() {
// - os::large_page_size() is the default explicit hugepage size (/proc/meminfo "Hugepagesize")
// - os::pagesizes() contains all hugepage sizes the kernel supports, regardless whether there
// are pages configured in the pool or not (from /sys/kernel/hugepages/hugepage-xxxx ...)
- os::PageSizes all_large_pages = HugePages::explicit_hugepage_info().pagesizes();
+ os::PageSizes all_large_pages = HugePages::explicit_hugepage_info().os_supported();
const size_t default_large_page_size = HugePages::default_explicit_hugepage_size();
// 3) Consistency check and post-processing
@@ -4062,10 +4066,10 @@ void os::Linux::large_page_init() {
_large_page_size = large_page_size;
- // Populate _page_sizes with large page sizes less than or equal to
- // _large_page_size.
- for (size_t page_size = _large_page_size; page_size != 0;
- page_size = all_large_pages.next_smaller(page_size)) {
+ // Populate _page_sizes with _large_page_size (default large page size) even if not pre-allocated.
+ // Then, populate _page_sizes with all smaller large page sizes that have been pre-allocated.
+ os::PageSizes pre_allocated = HugePages::explicit_hugepage_info().pre_allocated();
+ for (size_t page_size = _large_page_size; page_size != 0; page_size = pre_allocated.next_smaller(page_size)) {
_page_sizes.add(page_size);
}
}
@@ -4129,12 +4133,12 @@ static char* reserve_memory_special_huge_tlbfs(size_t bytes,
size_t page_size,
char* req_addr,
bool exec) {
- const os::PageSizes page_sizes = HugePages::explicit_hugepage_info().pagesizes();
+ const os::PageSizes os_supported = HugePages::explicit_hugepage_info().os_supported();
assert(UseLargePages, "only for Huge TLBFS large pages");
assert(is_aligned(req_addr, alignment), "Must be");
assert(is_aligned(req_addr, page_size), "Must be");
assert(is_aligned(alignment, os::vm_allocation_granularity()), "Must be");
- assert(page_sizes.contains(page_size), "Must be a valid page size");
+ assert(os_supported.contains(page_size), "Must be a valid page size");
assert(page_size > os::vm_page_size(), "Must be a large page size");
assert(bytes >= page_size, "Shouldn't allocate large pages for small sizes");
@@ -4208,12 +4212,6 @@ char* os::pd_reserve_memory_special(size_t bytes, size_t alignment, size_t page_
return addr;
}
-bool os::pd_release_memory_special(char* base, size_t bytes) {
- assert(UseLargePages, "only for large pages");
- // Plain munmap is sufficient
- return pd_release_memory(base, bytes);
-}
-
size_t os::large_page_size() {
return _large_page_size;
}
@@ -4386,7 +4384,7 @@ int os::Linux::get_namespace_pid(int vmid) {
os::snprintf_checked(fname, sizeof(fname), "/proc/%d/status", vmid);
FILE *fp = os::fopen(fname, "r");
- if (fp) {
+ if (fp != nullptr) {
int pid, nspid;
int ret;
while (!feof(fp) && !ferror(fp)) {
@@ -4400,7 +4398,7 @@ int os::Linux::get_namespace_pid(int vmid) {
}
for (;;) {
int ch = fgetc(fp);
- if (ch == EOF || ch == (int)'\n') break;
+ if (ch == EOF || ch == '\n') break;
}
}
fclose(fp);
@@ -4555,6 +4553,7 @@ void os::Linux::numa_init() {
FLAG_SET_ERGO_IF_DEFAULT(UseNUMAInterleaving, true);
}
+#if INCLUDE_PARALLELGC
if (UseParallelGC && UseNUMA && UseLargePages && !can_commit_large_page_memory()) {
// With static large pages we cannot uncommit a page, so there's no way
// we can make the adaptive lgrp chunk resizing work. If the user specified both
@@ -4566,6 +4565,7 @@ void os::Linux::numa_init() {
UseAdaptiveNUMAChunkSizing = false;
}
}
+#endif
}
void os::Linux::disable_numa(const char* reason, bool warning) {
@@ -5433,3 +5433,31 @@ bool os::pd_dll_unload(void* libhandle, char* ebuf, int ebuflen) {
return res;
} // end: os::pd_dll_unload()
+
+void os::print_open_file_descriptors(outputStream* st) {
+ DIR* dirp = opendir("/proc/self/fd");
+ int fds = 0;
+ struct dirent* dentp;
+ const jlong TIMEOUT_NS = 50000000L; // 50 ms in nanoseconds
+ bool timed_out = false;
+
+ // limit proc file read to 50ms
+ jlong start = os::javaTimeNanos();
+ assert(dirp != nullptr, "No proc fs?");
+ while ((dentp = readdir(dirp)) != nullptr && !timed_out) {
+ if (isdigit(dentp->d_name[0])) fds++;
+ if (fds % 100 == 0) {
+ jlong now = os::javaTimeNanos();
+ if ((now - start) > TIMEOUT_NS) {
+ timed_out = true;
+ }
+ }
+ }
+
+ closedir(dirp);
+ if (timed_out) {
+ st->print_cr("Open File Descriptors: > %d", fds);
+ } else {
+ st->print_cr("Open File Descriptors: %d", fds);
+ }
+}
diff --git a/src/hotspot/os/posix/dtrace/hotspot_jni.d b/src/hotspot/os/posix/dtrace/hotspot_jni.d
index c5676921b37..1937769dcb2 100644
--- a/src/hotspot/os/posix/dtrace/hotspot_jni.d
+++ b/src/hotspot/os/posix/dtrace/hotspot_jni.d
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -366,6 +366,8 @@ provider hotspot_jni {
probe IsInstanceOf__return(uintptr_t);
probe IsSameObject__entry(void*, void*, void*);
probe IsSameObject__return(uintptr_t);
+ probe IsVirtualThread__entry(void*, void*);
+ probe IsVirtualThread__return(uintptr_t);
probe MonitorEnter__entry(void*, void*);
probe MonitorEnter__return(uint32_t);
probe MonitorExit__entry(void*, void*);
diff --git a/src/hotspot/os/posix/include/jvm_md.h b/src/hotspot/os/posix/include/jvm_md.h
index eb8e1f0d7e9..061ef17aaae 100644
--- a/src/hotspot/os/posix/include/jvm_md.h
+++ b/src/hotspot/os/posix/include/jvm_md.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -62,6 +62,19 @@
#define JVM_X_OK X_OK
#define JVM_F_OK F_OK
+#if defined(AIX)
+#include "jni_md.h"
+#include "dl_info.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+JNIEXPORT int JVM_dladdr(void* addr, Dl_info* info);
+#ifdef __cplusplus
+}
+#endif
+#endif
+
/*
* File I/O
*/
diff --git a/src/hotspot/os/posix/os_posix.cpp b/src/hotspot/os/posix/os_posix.cpp
index f147ed4be93..1fb2a248bec 100644
--- a/src/hotspot/os/posix/os_posix.cpp
+++ b/src/hotspot/os/posix/os_posix.cpp
@@ -888,6 +888,14 @@ void* os::lookup_function(const char* name) {
return dlsym(RTLD_DEFAULT, name);
}
+int64_t os::ftell(FILE* file) {
+ return ::ftell(file);
+}
+
+int os::fseek(FILE* file, int64_t offset, int whence) {
+ return ::fseek(file, offset, whence);
+}
+
jlong os::lseek(int fd, jlong offset, int whence) {
return (jlong) ::lseek(fd, offset, whence);
}
diff --git a/src/hotspot/os/posix/perfMemory_posix.cpp b/src/hotspot/os/posix/perfMemory_posix.cpp
index d9bde6fa825..c5046797e02 100644
--- a/src/hotspot/os/posix/perfMemory_posix.cpp
+++ b/src/hotspot/os/posix/perfMemory_posix.cpp
@@ -701,6 +701,39 @@ static void remove_file(const char* path) {
}
}
+// Files newer than this threshold are considered to belong to a JVM that may
+// still be starting up and are therefore not candidates for stale-file
+// cleanup. This avoids racing a concurrent JVM startup while scanning the
+// hsperfdata directory.
+static const time_t cleanup_grace_period_seconds = 5;
+
+static bool is_cleanup_candidate(const char* filename, const char* dirname) {
+ struct stat statbuf;
+ int result;
+
+ RESTARTABLE(::lstat(filename, &statbuf), result);
+ if (result == OS_ERR) {
+ log_debug(perf, memops)("lstat failed for %s/%s: %s", dirname, filename, os::strerror(errno));
+ return false;
+ }
+
+ if (!S_ISREG(statbuf.st_mode)) {
+ return false;
+ }
+
+ const time_t now = time(nullptr);
+ if (now == (time_t)-1) {
+ return false;
+ }
+
+ if (statbuf.st_mtime >= now - cleanup_grace_period_seconds) {
+ log_debug(perf, memops)("Skip cleanup of fresh file %s/%s", dirname, filename);
+ return false;
+ }
+
+ return true;
+}
+
// cleanup stale shared memory files
//
// This method attempts to remove all stale shared memory files in
@@ -744,6 +777,11 @@ static void cleanup_sharedmem_files(const char* dirname) {
continue;
}
+ if (!is_cleanup_candidate(filename, dirname)) {
+ errno = 0;
+ continue;
+ }
+
#if defined(LINUX)
// Special case on Linux, if multiple containers share the
// same /tmp directory:
@@ -872,16 +910,56 @@ static int create_sharedmem_file(const char* dirname, const char* filename, size
return -1;
}
- // Open the filename in the current directory.
- // Cannot use O_TRUNC here; truncation of an existing file has to happen
- // after the is_file_secure() check below.
- int fd;
- RESTARTABLE(os::open(filename, O_RDWR|O_CREAT|O_NOFOLLOW, S_IRUSR|S_IWUSR), fd);
+ int fd = OS_ERR;
+ static const int create_sharedmem_file_retry_count = LINUX_ONLY(3) NOT_LINUX(1);
+ for (int attempt = 0; attempt < create_sharedmem_file_retry_count; attempt++) {
+ // Open the filename in the current directory.
+ // Use O_EXCL so that startup never reuses an existing pid file unless it
+ // has first been proven stale and removed in `cleanup_sharedmem_files`.
+ RESTARTABLE(os::open(filename, O_RDWR|O_CREAT|O_EXCL|O_NOFOLLOW, S_IRUSR|S_IWUSR), fd);
+ if (fd == OS_ERR) {
+ break;
+ }
+
+#if defined(LINUX)
+ // On Linux, different containerized processes that share the same /tmp
+ // directory (e.g., with "docker --volume ...") may have the same pid and
+ // try to use the same file. To avoid conflicts among such processes, we
+ // allow only one of them (the winner of the flock() call) to write to the
+ // file. If we lose the race, assume we may have collided with a concurrent
+ // scavenger briefly holding the lock on a fresh file and retry a few times
+ // before giving up.
+ int n;
+ RESTARTABLE(::flock(fd, LOCK_EX|LOCK_NB), n);
+ if (n == 0) {
+ break;
+ }
+
+ const int flock_errno = errno;
+ ::close(fd);
+ fd = OS_ERR;
+
+ if (attempt + 1 == create_sharedmem_file_retry_count || flock_errno != EWOULDBLOCK) {
+ log_warning(perf, memops)("Cannot use file %s/%s because %s (errno = %d)", dirname, filename,
+ (flock_errno == EWOULDBLOCK) ?
+ "it is locked by another process" :
+ "flock() failed", flock_errno);
+ errno = flock_errno;
+ break;
+ }
+
+ // Short sleep to allow the lock to free up.
+ os::naked_short_sleep(1);
+#endif
+ }
+
if (fd == OS_ERR) {
if (log_is_enabled(Debug, perf)) {
LogStreamHandle(Debug, perf) log;
if (errno == ELOOP) {
log.print_cr("file %s is a symlink and is not secure", filename);
+ } else if (errno == EEXIST) {
+ log.print_cr("could not create file %s: existing file is not provably stale", filename);
} else {
log.print_cr("could not create file %s: %s", filename, os::strerror(errno));
}
@@ -901,27 +979,7 @@ static int create_sharedmem_file(const char* dirname, const char* filename, size
}
#if defined(LINUX)
- // On Linux, different containerized processes that share the same /tmp
- // directory (e.g., with "docker --volume ...") may have the same pid and
- // try to use the same file. To avoid conflicts among such
- // processes, we allow only one of them (the winner of the flock() call)
- // to write to the file. All the other processes will give up and will
- // have perfdata disabled.
- //
- // Note that the flock will be automatically given up when the winner
- // process exits.
- //
- // The locking protocol works only with other JVMs that have the JDK-8286030
- // fix. If you are sharing the /tmp difrectory among different containers,
- // do not use older JVMs that don't have this fix, or the behavior is undefined.
- int n;
- RESTARTABLE(::flock(fd, LOCK_EX|LOCK_NB), n);
- if (n != 0) {
- log_warning(perf, memops)("Cannot use file %s/%s because %s (errno = %d)", dirname, filename,
- (errno == EWOULDBLOCK) ?
- "it is locked by another process" :
- "flock() failed", errno);
- ::close(fd);
+ if (fd == OS_ERR) {
return -1;
}
#endif
@@ -1084,18 +1142,9 @@ static char* mmap_create_shared(size_t size) {
// release a named shared memory region that was mmap-ed.
//
static void unmap_shared(char* addr, size_t bytes) {
- int res;
- if (MemTracker::enabled()) {
- MemTracker::NmtVirtualMemoryLocker nvml;
- res = ::munmap(addr, bytes);
- if (res == 0) {
- MemTracker::record_virtual_memory_release(addr, bytes);
- }
- } else {
- res = ::munmap(addr, bytes);
- }
- if (res != 0) {
- log_info(os)("os::release_memory failed (" PTR_FORMAT ", %zu)", p2i(addr), bytes);
+ MemTracker::record_virtual_memory_release(addr, bytes);
+ if (::munmap(addr, bytes) != 0) {
+ fatal("os::release_memory failed (" PTR_FORMAT ", %zu)", p2i(addr), bytes);
}
}
diff --git a/src/hotspot/os/windows/os_windows.cpp b/src/hotspot/os/windows/os_windows.cpp
index 2e819e26e37..9d8fb45f0d1 100644
--- a/src/hotspot/os/windows/os_windows.cpp
+++ b/src/hotspot/os/windows/os_windows.cpp
@@ -2528,12 +2528,6 @@ LONG Handle_Exception(struct _EXCEPTION_POINTERS* exceptionInfo,
return EXCEPTION_CONTINUE_EXECUTION;
}
-
-// Used for PostMortemDump
-extern "C" void safepoints();
-extern "C" void find(int x);
-extern "C" void events();
-
// According to Windows API documentation, an illegal instruction sequence should generate
// the 0xC000001C exception code. However, real world experience shows that occasionnaly
// the execution of an illegal instruction can generate the exception code 0xC000001E. This
@@ -3517,11 +3511,6 @@ char* os::pd_reserve_memory_special(size_t bytes, size_t alignment, size_t page_
return reserve_large_pages(bytes, addr, exec);
}
-bool os::pd_release_memory_special(char* base, size_t bytes) {
- assert(base != nullptr, "Sanity check");
- return pd_release_memory(base, bytes);
-}
-
static void warn_fail_commit_memory(char* addr, size_t bytes, bool exec) {
int err = os::get_last_error();
char buf[256];
@@ -5119,6 +5108,13 @@ jlong os::seek_to_file_offset(int fd, jlong offset) {
return (jlong)::_lseeki64(fd, (__int64)offset, SEEK_SET);
}
+int64_t os::ftell(FILE* file) {
+ return ::_ftelli64(file);
+}
+
+int os::fseek(FILE* file, int64_t offset, int whence) {
+ return ::_fseeki64(file,offset, whence);
+}
jlong os::lseek(int fd, jlong offset, int whence) {
return (jlong) ::_lseeki64(fd, offset, whence);
@@ -6281,6 +6277,10 @@ const void* os::get_saved_assert_context(const void** sigInfo) {
return nullptr;
}
+void os::print_open_file_descriptors(outputStream* st) {
+ // File descriptor counting not supported on Windows.
+}
+
/*
* Windows/x64 does not use stack frames the way expected by Java:
* [1] in most cases, there is no frame pointer. All locals are addressed via RSP
diff --git a/src/hotspot/os/windows/perfMemory_windows.cpp b/src/hotspot/os/windows/perfMemory_windows.cpp
index f54a2b52cca..dad2804f18a 100644
--- a/src/hotspot/os/windows/perfMemory_windows.cpp
+++ b/src/hotspot/os/windows/perfMemory_windows.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1682,12 +1682,7 @@ void PerfMemory::detach(char* addr, size_t bytes) {
return;
}
- if (MemTracker::enabled()) {
- // it does not go through os api, the operation has to record from here
- MemTracker::NmtVirtualMemoryLocker nvml;
- remove_file_mapping(addr);
- MemTracker::record_virtual_memory_release(addr, bytes);
- } else {
- remove_file_mapping(addr);
- }
+ // it does not go through os api, the operation has to record from here
+ MemTracker::record_virtual_memory_release(addr, bytes);
+ remove_file_mapping(addr);
}
diff --git a/src/hotspot/os_cpu/aix_ppc/vm_version_aix_ppc.cpp b/src/hotspot/os_cpu/aix_ppc/vm_version_aix_ppc.cpp
new file mode 100644
index 00000000000..8cc8b715201
--- /dev/null
+++ b/src/hotspot/os_cpu/aix_ppc/vm_version_aix_ppc.cpp
@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2026 SAP SE. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "runtime/vm_version.hpp"
+
+#include
+
+int VM_Version::get_dcache_line_size() {
+ return _system_configuration.dcache_line;
+}
+
+int VM_Version::get_icache_line_size() {
+ return _system_configuration.icache_line;
+}
diff --git a/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp b/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp
index 36599594842..49d879731ff 100644
--- a/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp
+++ b/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp
@@ -620,6 +620,8 @@ extern "C" {
assert(VM_Version::supports_sb(), "current CPU does not support SB instruction");
asm volatile(".inst 0xd50330ff" : : : "memory");
break;
+ case SpinWait::WFET:
+ ShouldNotReachHere();
#ifdef ASSERT
default:
ShouldNotReachHere();
diff --git a/src/hotspot/os_cpu/bsd_zero/atomicAccess_bsd_zero.hpp b/src/hotspot/os_cpu/bsd_zero/atomicAccess_bsd_zero.hpp
index 6c8684718fc..8e45490e5b6 100644
--- a/src/hotspot/os_cpu/bsd_zero/atomicAccess_bsd_zero.hpp
+++ b/src/hotspot/os_cpu/bsd_zero/atomicAccess_bsd_zero.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright 2007, 2008, 2011, 2015, Red Hat, Inc.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -27,7 +27,6 @@
#define OS_CPU_BSD_ZERO_ATOMICACCESS_BSD_ZERO_HPP
#include "orderAccess_bsd_zero.hpp"
-#include "runtime/os.hpp"
// Implementation of class AtomicAccess
diff --git a/src/hotspot/os_cpu/linux_aarch64/ic_ivau_probe_linux_aarch64.S b/src/hotspot/os_cpu/linux_aarch64/ic_ivau_probe_linux_aarch64.S
new file mode 100644
index 00000000000..b82053d37b9
--- /dev/null
+++ b/src/hotspot/os_cpu/linux_aarch64/ic_ivau_probe_linux_aarch64.S
@@ -0,0 +1,69 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "defs.S.inc"
+
+ # Probe whether IC IVAU is trapped.
+ #
+ # Returns 1 if IC IVAU is trapped (did not fault), 0 if not trapped
+ # (faulted on VA 0x0, signal handler redirected to continuation).
+ #
+ # int ic_ivau_probe(void);
+DECLARE_FUNC(ic_ivau_probe):
+DECLARE_FUNC(_ic_ivau_probe_fault):
+ ic ivau, xzr
+ mov x0, #1
+ ret
+DECLARE_FUNC(_ic_ivau_probe_continuation):
+ mov x0, #0
+ ret
+
+/* Emit .note.gnu.property section in case of PAC or BTI being enabled. */
+#ifdef __ARM_FEATURE_BTI_DEFAULT
+ #ifdef __ARM_FEATURE_PAC_DEFAULT
+ #define GNU_PROPERTY_AARCH64_FEATURE 3
+ #else
+ #define GNU_PROPERTY_AARCH64_FEATURE 1
+ #endif
+#else
+ #ifdef __ARM_FEATURE_PAC_DEFAULT
+ #define GNU_PROPERTY_AARCH64_FEATURE 2
+ #else
+ #define GNU_PROPERTY_AARCH64_FEATURE 0
+ #endif
+#endif
+
+#if (GNU_PROPERTY_AARCH64_FEATURE != 0)
+ .pushsection .note.gnu.property, "a"
+ .align 3
+ .long 4 /* name length */
+ .long 0x10 /* data length */
+ .long 5 /* note type: NT_GNU_PROPERTY_TYPE_0 */
+ .string "GNU" /* vendor name */
+ .long 0xc0000000 /* GNU_PROPERTY_AARCH64_FEATURE_1_AND */
+ .long 4 /* pr_datasze */
+ .long GNU_PROPERTY_AARCH64_FEATURE
+ .long 0
+ .popsection
+#endif
diff --git a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp
new file mode 100644
index 00000000000..41cad5af325
--- /dev/null
+++ b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp
@@ -0,0 +1,28 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "runtime/icache.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+NOT_PRODUCT(THREAD_LOCAL AArch64ICacheInvalidationContext* AArch64ICacheInvalidationContext::_current_context = nullptr;)
diff --git a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp
index 8fbaa7a6b6e..444b3c3ebd6 100644
--- a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp
+++ b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp
@@ -26,6 +26,11 @@
#ifndef OS_CPU_LINUX_AARCH64_ICACHE_AARCH64_HPP
#define OS_CPU_LINUX_AARCH64_ICACHE_AARCH64_HPP
+#include "memory/allocation.hpp"
+#include "runtime/vm_version.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "vm_version_aarch64.hpp"
+
// Interface for updating the instruction cache. Whenever the VM
// modifies code, part of the processor instruction cache potentially
// has to be flushed.
@@ -37,8 +42,105 @@ class ICache : public AbstractICache {
__builtin___clear_cache((char *)addr, (char *)(addr + 4));
}
static void invalidate_range(address start, int nbytes) {
- __builtin___clear_cache((char *)start, (char *)(start + nbytes));
+ if (NeoverseN1ICacheErratumMitigation) {
+ assert(VM_Version::is_cache_idc_enabled(),
+ "Expect CTR_EL0.IDC to be enabled for Neoverse N1 with erratum "
+ "1542419");
+ assert(!VM_Version::is_cache_dic_enabled(),
+ "Expect CTR_EL0.DIC to be disabled for Neoverse N1 with erratum "
+ "1542419");
+ assert(VM_Version::is_ic_ivau_trapped(), "Expect 'ic ivau, xzr' to be trapped");
+ asm volatile("dsb ish \n"
+ "ic ivau, xzr \n"
+ "dsb ish \n"
+ "isb \n"
+ : : : "memory");
+ } else {
+ __builtin___clear_cache((char *)start, (char *)(start + nbytes));
+ }
}
};
+class AArch64ICacheInvalidationContext : StackObj {
+ private:
+
+#ifdef ASSERT
+ static THREAD_LOCAL AArch64ICacheInvalidationContext* _current_context;
+#endif
+
+ bool _has_modified_code;
+
+ public:
+ NONCOPYABLE(AArch64ICacheInvalidationContext);
+
+ AArch64ICacheInvalidationContext()
+ : _has_modified_code(false) {
+ assert(_current_context == nullptr, "nested ICacheInvalidationContext not supported");
+#ifdef ASSERT
+ _current_context = this;
+#endif
+ }
+
+ ~AArch64ICacheInvalidationContext() {
+ NOT_PRODUCT(_current_context = nullptr);
+
+ if (!_has_modified_code || !UseSingleICacheInvalidation) {
+ return;
+ }
+
+ assert(VM_Version::is_cache_idc_enabled(), "Expect CTR_EL0.IDC to be enabled");
+
+ asm volatile("dsb ish" : : : "memory");
+
+ if (NeoverseN1ICacheErratumMitigation) {
+ assert(!VM_Version::is_cache_dic_enabled(),
+ "Expect CTR_EL0.DIC to be disabled for Neoverse N1 with erratum "
+ "1542419");
+ assert(VM_Version::is_ic_ivau_trapped(), "Expect 'ic ivau, xzr' to be trapped");
+
+ // Errata 1542419: Neoverse N1 cores with the 'COHERENT_ICACHE' feature
+ // may fetch stale instructions when software depends on
+ // prefetch-speculation-protection instead of explicit synchronization.
+ //
+ // Neoverse-N1 implementation mitigates the errata 1542419 with a
+ // workaround:
+ // - Disable coherent icache.
+ // - Trap IC IVAU instructions.
+ // - Execute:
+ // - tlbi vae3is, xzr
+ // - dsb sy
+ // - Ignore trapped IC IVAU instructions.
+ //
+ // `tlbi vae3is, xzr` invalidates all translation entries (all VAs, all
+ // possible levels). It waits for all memory accesses using in-scope old
+ // translation information to complete before it is considered complete.
+ //
+ // As this workaround has significant overhead, Arm Neoverse N1 (MP050)
+ // Software Developer Errata Notice version 29.0 suggests:
+ //
+ // "Since one TLB inner-shareable invalidation is enough to avoid this
+ // erratum, the number of injected TLB invalidations should be minimized
+ // in the trap handler to mitigate the performance impact due to this
+ // workaround."
+ // As the address for icache invalidation is not relevant and
+ // IC IVAU instruction is ignored, we use XZR in it.
+ asm volatile(
+ "ic ivau, xzr \n"
+ "dsb ish \n"
+ :
+ :
+ : "memory");
+ } else {
+ assert(VM_Version::is_cache_dic_enabled(), "Expect CTR_EL0.DIC to be enabled");
+ }
+ asm volatile("isb" : : : "memory");
+ }
+
+ void set_has_modified_code() {
+ _has_modified_code = true;
+ }
+};
+
+#define PD_ICACHE_INVALIDATION_CONTEXT AArch64ICacheInvalidationContext
+
#endif // OS_CPU_LINUX_AARCH64_ICACHE_AARCH64_HPP
diff --git a/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp
index da9e7e159f1..67e0569bf31 100644
--- a/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp
+++ b/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp
@@ -77,6 +77,11 @@
#define REG_LR 30
#define REG_BCP 22
+// IC IVAU trap probe.
+// Defined in ic_ivau_probe_linux_aarch64.S.
+extern "C" char _ic_ivau_probe_fault[] __attribute__ ((visibility ("hidden")));
+extern "C" char _ic_ivau_probe_continuation[] __attribute__ ((visibility ("hidden")));
+
NOINLINE address os::current_stack_pointer() {
return (address)__builtin_frame_address(0);
}
@@ -228,6 +233,12 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info,
}
}
+ // IC IVAU trap probe during VM_Version initialization.
+ // If IC IVAU is not trapped, it faults on unmapped VA 0x0.
+ if (sig == SIGSEGV && pc == (address)_ic_ivau_probe_fault) {
+ stub = (address)_ic_ivau_probe_continuation;
+ }
+
if (thread->thread_state() == _thread_in_Java) {
// Java thread running in Java code => find exception handler if any
// a fault inside compiled code, the interpreter, or a stub
diff --git a/src/hotspot/os_cpu/linux_aarch64/vm_version_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/vm_version_linux_aarch64.cpp
index 1fe06dc640d..ee2d3013c4c 100644
--- a/src/hotspot/os_cpu/linux_aarch64/vm_version_linux_aarch64.cpp
+++ b/src/hotspot/os_cpu/linux_aarch64/vm_version_linux_aarch64.cpp
@@ -31,6 +31,10 @@
#include
#include
+// IC IVAU trap probe.
+// Defined in ic_ivau_probe_linux_aarch64.S.
+extern "C" int ic_ivau_probe(void);
+
#ifndef HWCAP_AES
#define HWCAP_AES (1<<3)
#endif
@@ -95,6 +99,13 @@
#define HWCAP2_SVEBITPERM (1 << 4)
#endif
+#ifndef HWCAP2_ECV
+#define HWCAP2_ECV (1 << 19)
+#endif
+
+#ifndef HWCAP2_WFXT
+#define HWCAP2_WFXT (1u << 31)
+#endif
#ifndef PR_SVE_GET_VL
// For old toolchains which do not have SVE related macros defined.
#define PR_SVE_SET_VL 50
@@ -158,6 +169,12 @@ void VM_Version::get_os_cpu_info() {
if (auxv2 & HWCAP2_SVEBITPERM) {
set_feature(CPU_SVEBITPERM);
}
+ if (auxv2 & HWCAP2_ECV) {
+ set_feature(CPU_ECV);
+ }
+ if (auxv2 & HWCAP2_WFXT) {
+ set_feature(CPU_WFXT);
+ }
uint64_t ctr_el0;
uint64_t dczid_el0;
@@ -169,6 +186,12 @@ void VM_Version::get_os_cpu_info() {
_icache_line_size = (1 << (ctr_el0 & 0x0f)) * 4;
_dcache_line_size = (1 << ((ctr_el0 >> 16) & 0x0f)) * 4;
+ _cache_idc_enabled = ((ctr_el0 >> 28) & 0x1) != 0;
+ _cache_dic_enabled = ((ctr_el0 >> 29) & 0x1) != 0;
+
+ // Probe whether IC IVAU is trapped.
+ // Must run before VM_Version::initialize() sets NeoverseN1ICacheErratumMitigation.
+ _ic_ivau_trapped = (ic_ivau_probe() == 1);
if (!(dczid_el0 & 0x10)) {
_zva_length = 4 << (dczid_el0 & 0xf);
diff --git a/src/hotspot/os_cpu/linux_arm/atomicAccess_linux_arm.hpp b/src/hotspot/os_cpu/linux_arm/atomicAccess_linux_arm.hpp
index 390207f9e5e..c03f5ed1c8b 100644
--- a/src/hotspot/os_cpu/linux_arm/atomicAccess_linux_arm.hpp
+++ b/src/hotspot/os_cpu/linux_arm/atomicAccess_linux_arm.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2008, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -26,8 +26,6 @@
#define OS_CPU_LINUX_ARM_ATOMICACCESS_LINUX_ARM_HPP
#include "memory/allStatic.hpp"
-#include "runtime/os.hpp"
-#include "runtime/vm_version.hpp"
// Implementation of class AtomicAccess
diff --git a/src/hotspot/os_cpu/linux_arm/orderAccess_linux_arm.hpp b/src/hotspot/os_cpu/linux_arm/orderAccess_linux_arm.hpp
index 3bb357704fb..49c6942b8e0 100644
--- a/src/hotspot/os_cpu/linux_arm/orderAccess_linux_arm.hpp
+++ b/src/hotspot/os_cpu/linux_arm/orderAccess_linux_arm.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2008, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -27,7 +27,6 @@
// Included in orderAccess.hpp header file.
-#include "runtime/os.hpp"
#include "runtime/vm_version.hpp"
// Implementation of class OrderAccess.
diff --git a/src/hotspot/os_cpu/linux_ppc/vm_version_linux_ppc.cpp b/src/hotspot/os_cpu/linux_ppc/vm_version_linux_ppc.cpp
new file mode 100644
index 00000000000..d64340edf5c
--- /dev/null
+++ b/src/hotspot/os_cpu/linux_ppc/vm_version_linux_ppc.cpp
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2026 SAP SE. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "runtime/vm_version.hpp"
+
+#include
+
+int VM_Version::get_dcache_line_size() {
+ // This should work on all modern linux versions:
+ int size = sysconf(_SC_LEVEL1_DCACHE_LINESIZE);
+ // It may fail with very old linux / glibc versions. We use DEFAULT_CACHE_LINE_SIZE in this case.
+ // That is the correct value for all currently supported processors.
+ return (size <= 0) ? DEFAULT_CACHE_LINE_SIZE : size;
+}
+
+int VM_Version::get_icache_line_size() {
+ // This should work on all modern linux versions:
+ int size = sysconf(_SC_LEVEL1_ICACHE_LINESIZE);
+ // It may fail with very old linux / glibc versions. We use DEFAULT_CACHE_LINE_SIZE in this case.
+ // That is the correct value for all currently supported processors.
+ return (size <= 0) ? DEFAULT_CACHE_LINE_SIZE : size;
+}
diff --git a/src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp b/src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp
index 35cbb75e8ff..648131b94a3 100644
--- a/src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp
+++ b/src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp
@@ -36,40 +36,42 @@
#include
#include
+static constexpr uint64_t feature_bit(int n) { return nth_bit(n); }
+
#ifndef HWCAP_ISA_I
-#define HWCAP_ISA_I nth_bit('I' - 'A')
+#define HWCAP_ISA_I feature_bit('I' - 'A')
#endif
#ifndef HWCAP_ISA_M
-#define HWCAP_ISA_M nth_bit('M' - 'A')
+#define HWCAP_ISA_M feature_bit('M' - 'A')
#endif
#ifndef HWCAP_ISA_A
-#define HWCAP_ISA_A nth_bit('A' - 'A')
+#define HWCAP_ISA_A feature_bit('A' - 'A')
#endif
#ifndef HWCAP_ISA_F
-#define HWCAP_ISA_F nth_bit('F' - 'A')
+#define HWCAP_ISA_F feature_bit('F' - 'A')
#endif
#ifndef HWCAP_ISA_D
-#define HWCAP_ISA_D nth_bit('D' - 'A')
+#define HWCAP_ISA_D feature_bit('D' - 'A')
#endif
#ifndef HWCAP_ISA_C
-#define HWCAP_ISA_C nth_bit('C' - 'A')
+#define HWCAP_ISA_C feature_bit('C' - 'A')
#endif
#ifndef HWCAP_ISA_Q
-#define HWCAP_ISA_Q nth_bit('Q' - 'A')
+#define HWCAP_ISA_Q feature_bit('Q' - 'A')
#endif
#ifndef HWCAP_ISA_H
-#define HWCAP_ISA_H nth_bit('H' - 'A')
+#define HWCAP_ISA_H feature_bit('H' - 'A')
#endif
#ifndef HWCAP_ISA_V
-#define HWCAP_ISA_V nth_bit('V' - 'A')
+#define HWCAP_ISA_V feature_bit('V' - 'A')
#endif
#define read_csr(csr) \
diff --git a/src/hotspot/os_cpu/linux_s390/atomicAccess_linux_s390.hpp b/src/hotspot/os_cpu/linux_s390/atomicAccess_linux_s390.hpp
index f3c1e8f1a2c..492ccf73bdf 100644
--- a/src/hotspot/os_cpu/linux_s390/atomicAccess_linux_s390.hpp
+++ b/src/hotspot/os_cpu/linux_s390/atomicAccess_linux_s390.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2019 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -26,10 +26,6 @@
#ifndef OS_CPU_LINUX_S390_ATOMICACCESS_LINUX_S390_HPP
#define OS_CPU_LINUX_S390_ATOMICACCESS_LINUX_S390_HPP
-#include "runtime/atomicAccess.hpp"
-#include "runtime/os.hpp"
-#include "runtime/vm_version.hpp"
-
// Note that the compare-and-swap instructions on System z perform
// a serialization function before the storage operand is fetched
// and again after the operation is completed.
diff --git a/src/hotspot/os_cpu/windows_aarch64/atomicAccess_windows_aarch64.hpp b/src/hotspot/os_cpu/windows_aarch64/atomicAccess_windows_aarch64.hpp
index f8119654c50..9238043f7a4 100644
--- a/src/hotspot/os_cpu/windows_aarch64/atomicAccess_windows_aarch64.hpp
+++ b/src/hotspot/os_cpu/windows_aarch64/atomicAccess_windows_aarch64.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, Microsoft Corporation. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -27,9 +27,7 @@
#define OS_CPU_WINDOWS_AARCH64_ATOMICACCESS_WINDOWS_AARCH64_HPP
#include
-#include "runtime/os.hpp"
-#include "runtime/vm_version.hpp"
-
+#include
// As per atomicAccess.hpp all read-modify-write operations have to provide two-way
// barriers semantics. The memory_order parameter is ignored - we always provide
diff --git a/src/hotspot/os_cpu/windows_aarch64/prefetch_windows_aarch64.inline.hpp b/src/hotspot/os_cpu/windows_aarch64/prefetch_windows_aarch64.inline.hpp
index a360ee342be..a2c8f0c685c 100644
--- a/src/hotspot/os_cpu/windows_aarch64/prefetch_windows_aarch64.inline.hpp
+++ b/src/hotspot/os_cpu/windows_aarch64/prefetch_windows_aarch64.inline.hpp
@@ -27,10 +27,24 @@
// Included in runtime/prefetch.inline.hpp
+#include
+
+// __prefetch2(addr, prfop) emits a PRFM instruction.
+// The prfop encoding is:
+// type: PLD = 00, PLI = 01, PST = 10
+// target: L1 = 00, L2 = 01, L3 = 10
+// policy: KEEP = 0, STRM = 1
+
inline void Prefetch::read (const void *loc, intx interval) {
+ if (interval >= 0) {
+ __prefetch2((const char*) loc + interval, /* PLD + L1 + KEEP */ 0);
+ }
}
inline void Prefetch::write(void *loc, intx interval) {
+ if (interval >= 0) {
+ __prefetch2((char*) loc + interval, /* PST + L1 + KEEP */ 16);
+ }
}
#endif // OS_CPU_WINDOWS_AARCH64_PREFETCH_WINDOWS_AARCH64_INLINE_HPP
diff --git a/src/hotspot/os_cpu/windows_aarch64/sve_windows_aarch64.S b/src/hotspot/os_cpu/windows_aarch64/sve_windows_aarch64.S
new file mode 100644
index 00000000000..e0c85830bd4
--- /dev/null
+++ b/src/hotspot/os_cpu/windows_aarch64/sve_windows_aarch64.S
@@ -0,0 +1,42 @@
+;
+; Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
+; DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+;
+; This code is free software; you can redistribute it and/or modify it
+; under the terms of the GNU General Public License version 2 only, as
+; published by the Free Software Foundation.
+;
+; This code is distributed in the hope that it will be useful, but WITHOUT
+; ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+; FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+; version 2 for more details (a copy is included in the LICENSE file that
+; accompanied this code).
+;
+; You should have received a copy of the GNU General Public License version
+; 2 along with this work; if not, write to the Free Software Foundation,
+; Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+;
+; Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+; or visit www.oracle.com if you need additional information or have any
+; questions.
+;
+
+ ; Support for int get_sve_vector_length();
+ ;
+ ; Returns the current SVE vector length in bytes.
+ ; This function uses the INCB instruction which increments a register
+ ; by the number of bytes in an SVE vector register.
+ ;
+ ; Note: This function will fault if SVE is not available or enabled.
+ ; The caller must ensure SVE support is detected before calling.
+
+ ALIGN 4
+ EXPORT get_sve_vector_length
+ AREA sve_text, CODE
+
+get_sve_vector_length
+ mov x0, #0
+ incb x0
+ ret
+
+ END
diff --git a/src/hotspot/os_cpu/windows_aarch64/vm_version_windows_aarch64.cpp b/src/hotspot/os_cpu/windows_aarch64/vm_version_windows_aarch64.cpp
index a20feadcba4..e78a37b4178 100644
--- a/src/hotspot/os_cpu/windows_aarch64/vm_version_windows_aarch64.cpp
+++ b/src/hotspot/os_cpu/windows_aarch64/vm_version_windows_aarch64.cpp
@@ -26,24 +26,53 @@
#include "runtime/os.hpp"
#include "runtime/vm_version.hpp"
+// Assembly function to get SVE vector length using INCB instruction
+extern "C" int get_sve_vector_length();
+
int VM_Version::get_current_sve_vector_length() {
- assert(_features & CPU_SVE, "should not call this");
- ShouldNotReachHere();
- return 0;
+ assert(VM_Version::supports_sve(), "should not call this");
+ // Use assembly instruction to get the actual SVE vector length
+ return VM_Version::supports_sve() ? get_sve_vector_length() : 0; // This value is in bytes
}
int VM_Version::set_and_get_current_sve_vector_length(int length) {
- assert(_features & CPU_SVE, "should not call this");
- ShouldNotReachHere();
- return 0;
+ assert(VM_Version::supports_sve(), "should not call this");
+ // Use assembly instruction to get the SVE vector length
+ return VM_Version::supports_sve() ? get_sve_vector_length() : 0; // This value is in bytes
}
void VM_Version::get_os_cpu_info() {
- if (IsProcessorFeaturePresent(PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE)) _features |= CPU_CRC32;
- if (IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE)) _features |= CPU_AES | CPU_SHA1 | CPU_SHA2;
- if (IsProcessorFeaturePresent(PF_ARM_VFP_32_REGISTERS_AVAILABLE)) _features |= CPU_ASIMD;
- // No check for CPU_PMULL, CPU_SVE, CPU_SVE2
+ if (IsProcessorFeaturePresent(PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE)) {
+ set_feature(CPU_CRC32);
+ }
+ if (IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE)) {
+ set_feature(CPU_AES);
+ set_feature(CPU_SHA1);
+ set_feature(CPU_SHA2);
+ set_feature(CPU_PMULL);
+ }
+ if (IsProcessorFeaturePresent(PF_ARM_VFP_32_REGISTERS_AVAILABLE)) {
+ set_feature(CPU_ASIMD);
+ }
+ if (IsProcessorFeaturePresent(PF_ARM_V81_ATOMIC_INSTRUCTIONS_AVAILABLE)) {
+ set_feature(CPU_LSE);
+ }
+ if (IsProcessorFeaturePresent(PF_ARM_SVE_INSTRUCTIONS_AVAILABLE)) {
+ set_feature(CPU_SVE);
+ }
+ if (IsProcessorFeaturePresent(PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE)) {
+ set_feature(CPU_SVE2);
+ }
+ if (IsProcessorFeaturePresent(PF_ARM_SVE_BITPERM_INSTRUCTIONS_AVAILABLE)) {
+ set_feature(CPU_SVEBITPERM);
+ }
+ if (IsProcessorFeaturePresent(PF_ARM_SHA3_INSTRUCTIONS_AVAILABLE)) {
+ set_feature(CPU_SHA3);
+ }
+ if (IsProcessorFeaturePresent(PF_ARM_SHA512_INSTRUCTIONS_AVAILABLE)) {
+ set_feature(CPU_SHA512);
+ }
__int64 dczid_el0 = _ReadStatusReg(0x5807 /* ARM64_DCZID_EL0 */);
@@ -94,8 +123,8 @@ void VM_Version::get_os_cpu_info() {
SYSTEM_INFO si;
GetSystemInfo(&si);
_model = si.wProcessorLevel;
- _variant = si.wProcessorRevision / 0xFF;
- _revision = si.wProcessorRevision & 0xFF;
+ _variant = (si.wProcessorRevision >> 8) & 0xFF; // Variant is the upper byte of wProcessorRevision
+ _revision = si.wProcessorRevision & 0xFF; // Revision is the lower byte of wProcessorRevision
}
}
}
diff --git a/src/hotspot/os_cpu/windows_x86/atomicAccess_windows_x86.hpp b/src/hotspot/os_cpu/windows_x86/atomicAccess_windows_x86.hpp
index aa78a401235..252411f62bc 100644
--- a/src/hotspot/os_cpu/windows_x86/atomicAccess_windows_x86.hpp
+++ b/src/hotspot/os_cpu/windows_x86/atomicAccess_windows_x86.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -26,7 +26,7 @@
#define OS_CPU_WINDOWS_X86_ATOMICACCESS_WINDOWS_X86_HPP
#include
-#include "runtime/os.hpp"
+#include
// Note that in MSVC, volatile memory accesses are explicitly
// guaranteed to have acquire release semantics (w.r.t. compiler
diff --git a/src/hotspot/os_cpu/windows_x86/prefetch_windows_x86.inline.hpp b/src/hotspot/os_cpu/windows_x86/prefetch_windows_x86.inline.hpp
index 645fbe99a22..575eabc97dd 100644
--- a/src/hotspot/os_cpu/windows_x86/prefetch_windows_x86.inline.hpp
+++ b/src/hotspot/os_cpu/windows_x86/prefetch_windows_x86.inline.hpp
@@ -27,7 +27,18 @@
// Included in runtime/prefetch.inline.hpp
-inline void Prefetch::read (const void *loc, intx interval) {}
-inline void Prefetch::write(void *loc, intx interval) {}
+#include
+
+inline void Prefetch::read (const void *loc, intx interval) {
+ if (interval >= 0) {
+ _mm_prefetch((const char*) loc + interval, _MM_HINT_T0);
+ }
+}
+
+inline void Prefetch::write(void *loc, intx interval) {
+ if (interval >= 0) {
+ _mm_prefetch((const char*) loc + interval, _MM_HINT_T0);
+ }
+}
#endif // OS_CPU_WINDOWS_X86_PREFETCH_WINDOWS_X86_INLINE_HPP
diff --git a/src/hotspot/share/adlc/adlparse.cpp b/src/hotspot/share/adlc/adlparse.cpp
index 356c24760e8..b49efa34be8 100644
--- a/src/hotspot/share/adlc/adlparse.cpp
+++ b/src/hotspot/share/adlc/adlparse.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -993,9 +993,6 @@ void ADLParser::frame_parse(void) {
if (strcmp(token,"frame_pointer")==0) {
frame_pointer_parse(frame, false);
}
- if (strcmp(token,"interpreter_frame_pointer")==0) {
- interpreter_frame_pointer_parse(frame, false);
- }
if (strcmp(token,"inline_cache_reg")==0) {
inline_cache_parse(frame, false);
}
@@ -1119,11 +1116,6 @@ void ADLParser::frame_pointer_parse(FrameForm *frame, bool native) {
else { frame->_frame_pointer = frame_pointer; }
}
-//------------------------------interpreter_frame_pointer_parse----------------------------
-void ADLParser::interpreter_frame_pointer_parse(FrameForm *frame, bool native) {
- frame->_interpreter_frame_pointer_reg = parse_one_arg("interpreter frame pointer entry");
-}
-
//------------------------------inline_cache_parse-----------------------------
void ADLParser::inline_cache_parse(FrameForm *frame, bool native) {
frame->_inline_cache_reg = parse_one_arg("inline cache reg entry");
diff --git a/src/hotspot/share/adlc/adlparse.hpp b/src/hotspot/share/adlc/adlparse.hpp
index 02baec53262..89296193612 100644
--- a/src/hotspot/share/adlc/adlparse.hpp
+++ b/src/hotspot/share/adlc/adlparse.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -120,7 +120,6 @@ protected:
// Parse the components of the frame section
void sync_stack_slots_parse(FrameForm *frame);
void frame_pointer_parse(FrameForm *frame, bool native);
- void interpreter_frame_pointer_parse(FrameForm *frame, bool native);
void inline_cache_parse(FrameForm *frame, bool native);
void interpreter_arg_ptr_parse(FrameForm *frame, bool native);
void interpreter_method_parse(FrameForm *frame, bool native);
diff --git a/src/hotspot/share/adlc/formsopt.cpp b/src/hotspot/share/adlc/formsopt.cpp
index fbd1043492e..091e34f40f4 100644
--- a/src/hotspot/share/adlc/formsopt.cpp
+++ b/src/hotspot/share/adlc/formsopt.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -476,7 +476,6 @@ void AllocClass::forms_do(FormClosure* f) {
FrameForm::FrameForm() {
_sync_stack_slots = nullptr;
_inline_cache_reg = nullptr;
- _interpreter_frame_pointer_reg = nullptr;
_cisc_spilling_operand_name = nullptr;
_frame_pointer = nullptr;
_c_frame_pointer = nullptr;
diff --git a/src/hotspot/share/adlc/formsopt.hpp b/src/hotspot/share/adlc/formsopt.hpp
index 9e0c9db854d..087ab1e2653 100644
--- a/src/hotspot/share/adlc/formsopt.hpp
+++ b/src/hotspot/share/adlc/formsopt.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -347,7 +347,6 @@ public:
// Public Data
char *_sync_stack_slots;
char *_inline_cache_reg;
- char *_interpreter_frame_pointer_reg;
char *_cisc_spilling_operand_name;
char *_frame_pointer;
char *_c_frame_pointer;
diff --git a/src/hotspot/share/adlc/formssel.cpp b/src/hotspot/share/adlc/formssel.cpp
index 182587d2f2f..4dd2bff7c89 100644
--- a/src/hotspot/share/adlc/formssel.cpp
+++ b/src/hotspot/share/adlc/formssel.cpp
@@ -4276,7 +4276,9 @@ bool MatchRule::is_ideal_membar() const {
!strcmp(_opType,"LoadFence" ) ||
!strcmp(_opType,"StoreFence") ||
!strcmp(_opType,"StoreStoreFence") ||
+ !strcmp(_opType,"MemBarStoreLoad") ||
!strcmp(_opType,"MemBarVolatile") ||
+ !strcmp(_opType,"MemBarFull") ||
!strcmp(_opType,"MemBarCPUOrder") ||
!strcmp(_opType,"MemBarStoreStore") ||
!strcmp(_opType,"OnSpinWait");
diff --git a/src/hotspot/share/adlc/main.cpp b/src/hotspot/share/adlc/main.cpp
index 4e8a96617e8..8e6ea5bbec9 100644
--- a/src/hotspot/share/adlc/main.cpp
+++ b/src/hotspot/share/adlc/main.cpp
@@ -213,6 +213,7 @@ int main(int argc, char *argv[])
AD.addInclude(AD._CPP_file, "adfiles", get_basename(AD._VM_file._name));
AD.addInclude(AD._CPP_file, "adfiles", get_basename(AD._HPP_file._name));
AD.addInclude(AD._CPP_file, "memory/allocation.inline.hpp");
+ AD.addInclude(AD._CPP_file, "code/aotCodeCache.hpp");
AD.addInclude(AD._CPP_file, "code/codeCache.hpp");
AD.addInclude(AD._CPP_file, "code/compiledIC.hpp");
AD.addInclude(AD._CPP_file, "code/nativeInst.hpp");
@@ -257,6 +258,7 @@ int main(int argc, char *argv[])
AD.addInclude(AD._CPP_PEEPHOLE_file, "adfiles", get_basename(AD._HPP_file._name));
AD.addInclude(AD._CPP_PIPELINE_file, "adfiles", get_basename(AD._HPP_file._name));
AD.addInclude(AD._DFA_file, "adfiles", get_basename(AD._HPP_file._name));
+ AD.addInclude(AD._DFA_file, "code/aotCodeCache.hpp");
AD.addInclude(AD._DFA_file, "oops/compressedOops.hpp");
AD.addInclude(AD._DFA_file, "opto/cfgnode.hpp"); // Use PROB_MAX in predicate.
AD.addInclude(AD._DFA_file, "opto/intrinsicnode.hpp");
diff --git a/src/hotspot/share/adlc/output_c.cpp b/src/hotspot/share/adlc/output_c.cpp
index 9cbd6aaf66f..45b3d6bda63 100644
--- a/src/hotspot/share/adlc/output_c.cpp
+++ b/src/hotspot/share/adlc/output_c.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -4212,14 +4212,6 @@ void ArchDesc::buildFrameMethods(FILE *fp_cpp) {
fprintf(fp_cpp,"int Matcher::inline_cache_reg_encode() {");
fprintf(fp_cpp," return _regEncode[inline_cache_reg()]; }\n\n");
- // Interpreter's Frame Pointer Register
- fprintf(fp_cpp,"OptoReg::Name Matcher::interpreter_frame_pointer_reg() {");
- if (_frame->_interpreter_frame_pointer_reg == nullptr)
- fprintf(fp_cpp," return OptoReg::Bad; }\n\n");
- else
- fprintf(fp_cpp," return OptoReg::Name(%s_num); }\n\n",
- _frame->_interpreter_frame_pointer_reg);
-
// Frame Pointer definition
/* CNC - I can not contemplate having a different frame pointer between
Java and native code; makes my head hurt to think about it.
diff --git a/src/hotspot/share/asm/codeBuffer.cpp b/src/hotspot/share/asm/codeBuffer.cpp
index ba525588f32..c6078c0ceee 100644
--- a/src/hotspot/share/asm/codeBuffer.cpp
+++ b/src/hotspot/share/asm/codeBuffer.cpp
@@ -32,7 +32,6 @@
#include "oops/methodCounters.hpp"
#include "oops/methodData.hpp"
#include "oops/oop.inline.hpp"
-#include "runtime/icache.hpp"
#include "runtime/safepointVerifiers.hpp"
#include "utilities/align.hpp"
#include "utilities/copy.hpp"
@@ -745,9 +744,6 @@ void CodeBuffer::copy_code_to(CodeBlob* dest_blob) {
// Done moving code bytes; were they the right size?
assert((int)align_up(dest.total_content_size(), oopSize) == dest_blob->content_size(), "sanity");
-
- // Flush generated code
- ICache::invalidate_range(dest_blob->code_begin(), dest_blob->code_size());
}
// Move all my code into another code buffer. Consult applicable
diff --git a/src/hotspot/share/asm/register.hpp b/src/hotspot/share/asm/register.hpp
index f406995b8ac..95c7c7922cf 100644
--- a/src/hotspot/share/asm/register.hpp
+++ b/src/hotspot/share/asm/register.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -49,15 +49,7 @@ class AbstractRegisterImpl {
// Macros to help define all kinds of registers
-#ifndef USE_POINTERS_TO_REGISTER_IMPL_ARRAY
-
-#define AS_REGISTER(type,name) ((type)name##_##type##EnumValue)
-
-#define CONSTANT_REGISTER_DECLARATION(type, name, value) \
-const type name = ((type)value); \
-enum { name##_##type##EnumValue = (value) }
-
-#else // USE_POINTERS_TO_REGISTER_IMPL_ARRAY
+#ifdef USE_POINTERS_TO_REGISTER_IMPL_ARRAY
#define REGISTER_IMPL_DECLARATION(type, impl_type, reg_count) \
inline constexpr type as_ ## type(int encoding) { \
@@ -69,16 +61,8 @@ inline constexpr type impl_type::first() { return all_ ## type ## s + 1; }
#define REGISTER_IMPL_DEFINITION(type, impl_type, reg_count) \
impl_type all_ ## type ## s[reg_count + 1];
-#define CONSTANT_REGISTER_DECLARATION(type, name, value) \
-constexpr type name = as_ ## type(value);
-
#endif // USE_POINTERS_TO_REGISTER_IMPL_ARRAY
-
-#define REGISTER_DECLARATION(type, name, value) \
-const type name = ((type)value)
-
-
// For definitions of RegisterImpl* instances. To be redefined in an
// OS-specific way.
#ifdef __GNUC__
diff --git a/src/hotspot/share/cds/aotArtifactFinder.cpp b/src/hotspot/share/cds/aotArtifactFinder.cpp
index f85f1e46520..bd69b18a1aa 100644
--- a/src/hotspot/share/cds/aotArtifactFinder.cpp
+++ b/src/hotspot/share/cds/aotArtifactFinder.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -32,6 +32,7 @@
#include "cds/lambdaProxyClassDictionary.hpp"
#include "cds/regeneratedClasses.hpp"
#include "classfile/systemDictionaryShared.hpp"
+#include "classfile/vmClasses.hpp"
#include "logging/log.hpp"
#include "memory/metaspaceClosure.hpp"
#include "oops/instanceKlass.hpp"
@@ -169,6 +170,7 @@ void AOTArtifactFinder::find_artifacts() {
end_scanning_for_oops();
TrainingData::cleanup_training_data();
+ check_critical_classes();
}
void AOTArtifactFinder::start_scanning_for_oops() {
@@ -233,6 +235,7 @@ void AOTArtifactFinder::add_cached_instance_class(InstanceKlass* ik) {
bool created;
_seen_classes->put_if_absent(ik, &created);
if (created) {
+ check_critical_class(ik);
append_to_all_cached_classes(ik);
// All super types must be added.
@@ -310,3 +313,25 @@ void AOTArtifactFinder::all_cached_classes_do(MetaspaceClosure* it) {
it->push(_all_cached_classes->adr_at(i));
}
}
+
+void AOTArtifactFinder::check_critical_classes() {
+ if (CDSConfig::is_dumping_static_archive()) {
+ // vmClasses are store in the AOT cache (or AOT config file, or static archive).
+ // If any of the vmClasses is excluded, (usually due to incompatible JVMTI agent),
+ // the resulting cache/config/archive is unusable.
+ for (auto id : EnumRange{}) {
+ check_critical_class(vmClasses::klass_at(id));
+ }
+ }
+}
+
+void AOTArtifactFinder::check_critical_class(InstanceKlass* ik) {
+ if (SystemDictionaryShared::is_excluded_class(ik)) {
+ ResourceMark rm;
+ const char* msg = err_msg("Critical class %s has been excluded. %s cannot be written.",
+ ik->external_name(),
+ CDSConfig::type_of_archive_being_written());
+ AOTMetaspace::unrecoverable_writing_error(msg);
+ }
+}
+
diff --git a/src/hotspot/share/cds/aotArtifactFinder.hpp b/src/hotspot/share/cds/aotArtifactFinder.hpp
index 05bcde6b0ac..50057b6caee 100644
--- a/src/hotspot/share/cds/aotArtifactFinder.hpp
+++ b/src/hotspot/share/cds/aotArtifactFinder.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -81,12 +81,14 @@ class AOTArtifactFinder : AllStatic {
static void add_cached_type_array_class(TypeArrayKlass* tak);
static void add_cached_instance_class(InstanceKlass* ik);
static void append_to_all_cached_classes(Klass* k);
+ static void check_critical_class(InstanceKlass* ik);
public:
static void initialize();
static void find_artifacts();
static void add_cached_class(Klass* k);
static void add_aot_inited_class(InstanceKlass* ik);
static void all_cached_classes_do(MetaspaceClosure* it);
+ static void check_critical_classes();
static void dispose();
};
diff --git a/src/hotspot/share/cds/aotClassInitializer.cpp b/src/hotspot/share/cds/aotClassInitializer.cpp
index 06fc3af6f30..9ef96282aeb 100644
--- a/src/hotspot/share/cds/aotClassInitializer.cpp
+++ b/src/hotspot/share/cds/aotClassInitializer.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -59,6 +59,39 @@ bool AOTClassInitializer::can_archive_initialized_mirror(InstanceKlass* ik) {
return false;
}
+#ifdef ASSERT
+ // If code in ik is executed, then ik must be in the state of being_initialized or
+ // fully_initialized.
+ //
+ // Check that no user code is executed during the assembly phase. Otherwise the user
+ // code may introduce undesirable environment dependencies into the heap image.
+ // If any of these two flags are set, we allow user code to be executed
+ // in the assembly phase. Note that these flags are strictly for the purpose
+ // of testing HotSpot and are not available in product builds.
+ if (AOTInitTestClass == nullptr && ArchiveHeapTestClass == nullptr) {
+ if (ik->defined_by_boot_loader()) {
+ // We allow boot classes to be AOT-initialized, except for classes from
+ // -Xbootclasspath (cp index >= 1) be AOT-initialized, as such classes may be
+ // provided by the user application.
+ assert(ik->shared_classpath_index() <= 0,
+ "only boot classed loaded from the modules image can be AOT-initialized");
+ } else {
+ assert(ik->defined_by_platform_loader() || ik->defined_by_app_loader(),
+ "cannot AOT-initialized classed loaded by other loaders");
+
+ // Hidden classes from platform/app loaders need to be AOT-initialized to
+ // support AOT-linking of lambdas. These hidden classes are generated by the
+ // VM and do not contain user code.
+ if (!ik->is_hidden()) {
+ // OK: ik is an interface used by a lambda. When AOT-linking lambdas, we only
+ // support interfaces that are not interface_needs_clinit_execution_as_super().
+ // See AOTConstantPoolResolver::check_lambda_metafactory_signature().
+ assert(ik->is_interface() && !ik->interface_needs_clinit_execution_as_super(), "cannot execute Java code in assembly phase");
+ }
+ }
+ }
+#endif // ASSERT
+
// About "static field that may hold a different value" errors:
//
// Automatic selection for aot-inited classes
@@ -234,7 +267,8 @@ bool AOTClassInitializer::can_archive_initialized_mirror(InstanceKlass* ik) {
}
void AOTClassInitializer::call_runtime_setup(JavaThread* current, InstanceKlass* ik) {
- assert(ik->has_aot_initialized_mirror(), "sanity");
+ precond(ik->has_aot_initialized_mirror());
+ precond(!AOTLinkedClassBulkLoader::is_initializing_classes_early());
if (ik->is_runtime_setup_required()) {
if (log_is_enabled(Info, aot, init)) {
ResourceMark rm;
diff --git a/src/hotspot/share/cds/aotConstantPoolResolver.cpp b/src/hotspot/share/cds/aotConstantPoolResolver.cpp
index 93145940955..f1a704d4bee 100644
--- a/src/hotspot/share/cds/aotConstantPoolResolver.cpp
+++ b/src/hotspot/share/cds/aotConstantPoolResolver.cpp
@@ -81,6 +81,7 @@ bool AOTConstantPoolResolver::is_resolution_deterministic(ConstantPool* cp, int
bool AOTConstantPoolResolver::is_class_resolution_deterministic(InstanceKlass* cp_holder, Klass* resolved_class) {
assert(!is_in_archivebuilder_buffer(cp_holder), "sanity");
assert(!is_in_archivebuilder_buffer(resolved_class), "sanity");
+ assert_at_safepoint(); // try_add_candidate() is called below and requires to be at safepoint.
if (resolved_class->is_instance_klass()) {
InstanceKlass* ik = InstanceKlass::cast(resolved_class);
@@ -346,7 +347,15 @@ void AOTConstantPoolResolver::maybe_resolve_fmi_ref(InstanceKlass* ik, Method* m
break;
case Bytecodes::_invokehandle:
- InterpreterRuntime::cds_resolve_invokehandle(raw_index, cp, CHECK);
+ if (CDSConfig::is_dumping_method_handles()) {
+ ResolvedMethodEntry* method_entry = cp->resolved_method_entry_at(raw_index);
+ int cp_index = method_entry->constant_pool_index();
+ Symbol* sig = cp->uncached_signature_ref_at(cp_index);
+ Klass* k;
+ if (check_methodtype_signature(cp(), sig, &k, true)) {
+ InterpreterRuntime::cds_resolve_invokehandle(raw_index, cp, CHECK);
+ }
+ }
break;
default:
@@ -400,7 +409,7 @@ void AOTConstantPoolResolver::preresolve_indy_cp_entries(JavaThread* current, In
// Check the MethodType signatures used by parameters to the indy BSMs. Make sure we don't
// use types that have been excluded, or else we might end up creating MethodTypes that cannot be stored
// in the AOT cache.
-bool AOTConstantPoolResolver::check_methodtype_signature(ConstantPool* cp, Symbol* sig, Klass** return_type_ret) {
+bool AOTConstantPoolResolver::check_methodtype_signature(ConstantPool* cp, Symbol* sig, Klass** return_type_ret, bool is_invokehandle) {
ResourceMark rm;
for (SignatureStream ss(sig); !ss.is_done(); ss.next()) {
if (ss.is_reference()) {
@@ -413,11 +422,18 @@ bool AOTConstantPoolResolver::check_methodtype_signature(ConstantPool* cp, Symbo
if (SystemDictionaryShared::should_be_excluded(k)) {
if (log_is_enabled(Warning, aot, resolve)) {
ResourceMark rm;
- log_warning(aot, resolve)("Cannot aot-resolve Lambda proxy because %s is excluded", k->external_name());
+ log_warning(aot, resolve)("Cannot aot-resolve %s because %s is excluded",
+ is_invokehandle ? "invokehandle" : "Lambda proxy",
+ k->external_name());
}
return false;
}
+ // cp->pool_holder() must be able to resolve k in production run
+ precond(CDSConfig::is_dumping_aot_linked_classes());
+ precond(SystemDictionaryShared::is_builtin_loader(cp->pool_holder()->class_loader_data()));
+ precond(SystemDictionaryShared::is_builtin_loader(k->class_loader_data()));
+
if (ss.at_return_type() && return_type_ret != nullptr) {
*return_type_ret = k;
}
@@ -475,11 +491,44 @@ bool AOTConstantPoolResolver::check_lambda_metafactory_methodhandle_arg(Constant
return false;
}
+ // klass and sigature of the method (no need to check the method name)
Symbol* sig = cp->method_handle_signature_ref_at(mh_index);
+ Symbol* klass_name = cp->klass_name_at(cp->method_handle_klass_index_at(mh_index));
+
if (log_is_enabled(Debug, aot, resolve)) {
ResourceMark rm;
log_debug(aot, resolve)("Checking MethodType of MethodHandle for LambdaMetafactory BSM arg %d: %s", arg_i, sig->as_C_string());
}
+
+ {
+ Klass* k = find_loaded_class(Thread::current(), cp->pool_holder()->class_loader(), klass_name);
+ if (k == nullptr) {
+ // Dumping AOT cache: all classes should have been loaded by FinalImageRecipes::load_all_classes(). k must have
+ // been a class that was excluded when FinalImageRecipes recorded all classes at the end of the training run.
+ //
+ // Dumping static CDS archive: all classes in the classlist have already been loaded, before we resolve
+ // constants. k must have been a class that was excluded when the classlist was written
+ // at the end of the training run.
+ if (log_is_enabled(Warning, aot, resolve)) {
+ ResourceMark rm;
+ log_warning(aot, resolve)("Cannot aot-resolve Lambda proxy because %s is not loaded", klass_name->as_C_string());
+ }
+ return false;
+ }
+ if (SystemDictionaryShared::should_be_excluded(k)) {
+ if (log_is_enabled(Warning, aot, resolve)) {
+ ResourceMark rm;
+ log_warning(aot, resolve)("Cannot aot-resolve Lambda proxy because %s is excluded", k->external_name());
+ }
+ return false;
+ }
+
+ // cp->pool_holder() must be able to resolve k in production run
+ precond(CDSConfig::is_dumping_aot_linked_classes());
+ precond(SystemDictionaryShared::is_builtin_loader(cp->pool_holder()->class_loader_data()));
+ precond(SystemDictionaryShared::is_builtin_loader(k->class_loader_data()));
+ }
+
return check_methodtype_signature(cp, sig);
}
diff --git a/src/hotspot/share/cds/aotConstantPoolResolver.hpp b/src/hotspot/share/cds/aotConstantPoolResolver.hpp
index e49d9d1ad0b..ecf2ac27061 100644
--- a/src/hotspot/share/cds/aotConstantPoolResolver.hpp
+++ b/src/hotspot/share/cds/aotConstantPoolResolver.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2022, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -74,7 +74,10 @@ class AOTConstantPoolResolver : AllStatic {
static void maybe_resolve_fmi_ref(InstanceKlass* ik, Method* m, Bytecodes::Code bc, int raw_index,
GrowableArray* resolve_fmi_list, TRAPS);
- static bool check_methodtype_signature(ConstantPool* cp, Symbol* sig, Klass** return_type_ret = nullptr);
+public:
+ static bool check_methodtype_signature(ConstantPool* cp, Symbol* sig, Klass** return_type_ret = nullptr, bool is_invokehandle = false);
+
+private:
static bool check_lambda_metafactory_signature(ConstantPool* cp, Symbol* sig);
static bool check_lambda_metafactory_methodtype_arg(ConstantPool* cp, int bsms_attribute_index, int arg_i);
static bool check_lambda_metafactory_methodhandle_arg(ConstantPool* cp, int bsms_attribute_index, int arg_i);
diff --git a/src/hotspot/share/cds/aotLinkedClassBulkLoader.cpp b/src/hotspot/share/cds/aotLinkedClassBulkLoader.cpp
index 3653f9d518c..8129e6a5a81 100644
--- a/src/hotspot/share/cds/aotLinkedClassBulkLoader.cpp
+++ b/src/hotspot/share/cds/aotLinkedClassBulkLoader.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -116,11 +116,24 @@ void AOTLinkedClassBulkLoader::preload_classes_in_table(Array* c
}
}
+#ifdef ASSERT
+// true iff we are inside AOTLinkedClassBulkLoader::link_classes(), when
+// we are moving classes into the fully_initialized state before the
+// JVM is able to execute any bytecodes.
+static bool _is_initializing_classes_early = false;
+bool AOTLinkedClassBulkLoader::is_initializing_classes_early() {
+ return _is_initializing_classes_early;
+}
+#endif
+
// Some cached heap objects may hold references to methods in aot-linked
// classes (via MemberName). We need to make sure all classes are
// linked before executing any bytecode.
void AOTLinkedClassBulkLoader::link_classes(JavaThread* current) {
+ DEBUG_ONLY(_is_initializing_classes_early = true);
link_classes_impl(current);
+ DEBUG_ONLY(_is_initializing_classes_early = false);
+
if (current->has_pending_exception()) {
exit_on_exception(current);
}
@@ -135,6 +148,13 @@ void AOTLinkedClassBulkLoader::link_classes_impl(TRAPS) {
link_classes_in_table(table->boot2(), CHECK);
link_classes_in_table(table->platform(), CHECK);
link_classes_in_table(table->app(), CHECK);
+
+ init_classes_for_loader(Handle(), AOTLinkedClassTable::get()->boot1(), /*early_only=*/true, CHECK);
+ init_classes_for_loader(Handle(), AOTLinkedClassTable::get()->boot2(), /*early_only=*/true, CHECK);
+ init_classes_for_loader(Handle(), AOTLinkedClassTable::get()->platform(), /*early_only=*/true, CHECK);
+ init_classes_for_loader(Handle(), AOTLinkedClassTable::get()->app(), /*early_only=*/true, CHECK);
+
+ log_info(aot, init)("------ finished early class init");
}
void AOTLinkedClassBulkLoader::link_classes_in_table(Array* classes, TRAPS) {
@@ -216,7 +236,7 @@ void AOTLinkedClassBulkLoader::validate_module(Klass* k, const char* category_na
#endif
void AOTLinkedClassBulkLoader::init_javabase_classes(JavaThread* current) {
- init_classes_for_loader(Handle(), AOTLinkedClassTable::get()->boot1(), current);
+ init_classes_for_loader(Handle(), AOTLinkedClassTable::get()->boot1(), /*early_only=*/false, current);
if (current->has_pending_exception()) {
exit_on_exception(current);
}
@@ -246,9 +266,9 @@ void AOTLinkedClassBulkLoader::init_non_javabase_classes_impl(TRAPS) {
assert(h_system_loader() != nullptr, "must be");
AOTLinkedClassTable* table = AOTLinkedClassTable::get();
- init_classes_for_loader(Handle(), table->boot2(), CHECK);
- init_classes_for_loader(h_platform_loader, table->platform(), CHECK);
- init_classes_for_loader(h_system_loader, table->app(), CHECK);
+ init_classes_for_loader(Handle(), table->boot2(), /*early_only=*/false, CHECK);
+ init_classes_for_loader(h_platform_loader, table->platform(), /*early_only=*/false, CHECK);
+ init_classes_for_loader(h_system_loader, table->app(), /*early_only=*/false, CHECK);
if (Universe::is_fully_initialized() && VerifyDuringStartup) {
// Make sure we're still in a clean state.
@@ -260,6 +280,10 @@ void AOTLinkedClassBulkLoader::init_non_javabase_classes_impl(TRAPS) {
tty->print_cr("==================== archived_training_data ** after all classes preloaded ====================");
TrainingData::print_archived_training_data_on(tty);
}
+ LogStreamHandle(Info, aot, training, data) log;
+ if (log.is_enabled()) {
+ TrainingData::print_archived_training_data_on(&log);
+ }
}
// For the AOT cache to function properly, all classes in the AOTLinkedClassTable
@@ -324,22 +348,80 @@ void AOTLinkedClassBulkLoader::initiate_loading(JavaThread* current, const char*
}
}
-// Some AOT-linked classes for must be initialized early. This includes
-// - classes that were AOT-initialized by AOTClassInitializer
-// - the classes of all objects that are reachable from the archived mirrors of
-// the AOT-linked classes for .
-void AOTLinkedClassBulkLoader::init_classes_for_loader(Handle class_loader, Array* classes, TRAPS) {
+// Can we move ik into fully_initialized state before the JVM is able to execute
+// bytecodes?
+static bool is_early_init_possible(InstanceKlass* ik) {
+ if (ik->is_runtime_setup_required()) {
+ // Bytecodes need to be executed in order to initialize this class.
+ if (log_is_enabled(Debug, aot, init)) {
+ ResourceMark rm;
+ log_debug(aot, init)("No early init %s: needs runtimeSetup()",
+ ik->external_name());
+ }
+ return false;
+ }
+
+ if (ik->super() != nullptr && !ik->super()->is_initialized()) {
+ // is_runtime_setup_required() == true for a super type
+ if (log_is_enabled(Debug, aot, init)) {
+ ResourceMark rm;
+ log_debug(aot, init)("No early init %s: super type %s not initialized",
+ ik->external_name(), ik->super()->external_name());
+ }
+ return false;
+ }
+
+ Array* interfaces = ik->local_interfaces();
+ int num_interfaces = interfaces->length();
+ for (int i = 0; i < num_interfaces; i++) {
+ InstanceKlass* intf = interfaces->at(i);
+ if (!intf->is_initialized() && intf->interface_needs_clinit_execution_as_super(/*also_check_supers*/false)) {
+ // is_runtime_setup_required() == true for this interface
+ if (log_is_enabled(Debug, aot, init)) {
+ ResourceMark rm;
+ log_debug(aot, init)("No early init %s: interface type %s not initialized",
+ ik->external_name(), intf->external_name());
+ }
+ return false;
+ }
+ }
+
+ return true;
+}
+
+// Normally, classes are initialized on demand. However, some AOT-linked classes
+// for the class_loader must be proactively intialized, including:
+// - Classes that have an AOT-initialized mirror (they were AOT-initialized by
+// AOTClassInitializer during the assembly phase).
+// - The classes of all objects that are reachable from the archived mirrors of
+// the AOT-linked classes for the class_loader. These are recorded in the special
+// subgraph.
+//
+// (early_only == true) means that this function is called before the JVM
+// is capable of executing Java bytecodes.
+void AOTLinkedClassBulkLoader::init_classes_for_loader(Handle class_loader, Array* classes,
+ bool early_only, TRAPS) {
if (classes != nullptr) {
for (int i = 0; i < classes->length(); i++) {
InstanceKlass* ik = classes->at(i);
assert(ik->class_loader_data() != nullptr, "must be");
- if (ik->has_aot_initialized_mirror()) {
- ik->initialize_with_aot_initialized_mirror(CHECK);
+
+ bool do_init = ik->has_aot_initialized_mirror();
+ if (do_init && early_only && !is_early_init_possible(ik)) {
+ // ik will be proactively initialized later when init_classes_for_loader()
+ // is called again with (early_only == false).
+ do_init = false;
+ }
+
+ if (do_init) {
+ ik->initialize_with_aot_initialized_mirror(early_only, CHECK);
}
}
}
- HeapShared::init_classes_for_special_subgraph(class_loader, CHECK);
+ if (!early_only) {
+ HeapShared::init_classes_for_special_subgraph(class_loader, CHECK);
+ }
}
void AOTLinkedClassBulkLoader::replay_training_at_init(Array* classes, TRAPS) {
diff --git a/src/hotspot/share/cds/aotLinkedClassBulkLoader.hpp b/src/hotspot/share/cds/aotLinkedClassBulkLoader.hpp
index 31fdac386fe..24ff61cea1e 100644
--- a/src/hotspot/share/cds/aotLinkedClassBulkLoader.hpp
+++ b/src/hotspot/share/cds/aotLinkedClassBulkLoader.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -56,7 +56,7 @@ class AOTLinkedClassBulkLoader : AllStatic {
static void link_classes_impl(TRAPS);
static void link_classes_in_table(Array* classes, TRAPS);
static void init_non_javabase_classes_impl(TRAPS);
- static void init_classes_for_loader(Handle class_loader, Array* classes, TRAPS);
+ static void init_classes_for_loader(Handle class_loader, Array* classes, bool early_only, TRAPS);
static void replay_training_at_init(Array* classes, TRAPS) NOT_CDS_RETURN;
#ifdef ASSERT
@@ -73,8 +73,9 @@ public:
static void init_javabase_classes(JavaThread* current) NOT_CDS_RETURN;
static void init_non_javabase_classes(JavaThread* current) NOT_CDS_RETURN;
static void exit_on_exception(JavaThread* current);
-
static void replay_training_at_init_for_preloaded_classes(TRAPS) NOT_CDS_RETURN;
+
+ static bool is_initializing_classes_early() NOT_DEBUG({return false;});
};
#endif // SHARE_CDS_AOTLINKEDCLASSBULKLOADER_HPP
diff --git a/src/hotspot/share/cds/aotMapLogger.cpp b/src/hotspot/share/cds/aotMapLogger.cpp
index fa769aee1bf..9f338826fd6 100644
--- a/src/hotspot/share/cds/aotMapLogger.cpp
+++ b/src/hotspot/share/cds/aotMapLogger.cpp
@@ -98,8 +98,8 @@ void AOTMapLogger::dumptime_log(ArchiveBuilder* builder, FileMapInfo* mapinfo,
DumpRegion* rw_region = &builder->_rw_region;
DumpRegion* ro_region = &builder->_ro_region;
- dumptime_log_metaspace_region("rw region", rw_region, &builder->_rw_src_objs);
- dumptime_log_metaspace_region("ro region", ro_region, &builder->_ro_src_objs);
+ dumptime_log_metaspace_region("rw region", rw_region, &builder->_rw_src_objs, &builder->_ro_src_objs);
+ dumptime_log_metaspace_region("ro region", ro_region, &builder->_rw_src_objs, &builder->_ro_src_objs);
address bitmap_end = address(bitmap + bitmap_size_in_bytes);
log_region_range("bitmap", address(bitmap), bitmap_end, nullptr);
@@ -122,17 +122,6 @@ void AOTMapLogger::dumptime_log(ArchiveBuilder* builder, FileMapInfo* mapinfo,
class AOTMapLogger::RuntimeGatherArchivedMetaspaceObjs : public UniqueMetaspaceClosure {
GrowableArrayCHeap _objs;
- static int compare_objs_by_addr(ArchivedObjInfo* a, ArchivedObjInfo* b) {
- intx diff = a->_src_addr - b->_src_addr;
- if (diff < 0) {
- return -1;
- } else if (diff == 0) {
- return 0;
- } else {
- return 1;
- }
- }
-
public:
GrowableArrayCHeap* objs() { return &_objs; }
@@ -152,7 +141,7 @@ public:
void finish() {
UniqueMetaspaceClosure::finish();
- _objs.sort(compare_objs_by_addr);
+ _objs.sort(compare_by_address);
}
}; // AOTMapLogger::RuntimeGatherArchivedMetaspaceObjs
@@ -203,24 +192,47 @@ void AOTMapLogger::runtime_log(FileMapInfo* mapinfo, GrowableArrayCHeapbase());
address region_top = address(region->top());
log_region_range(name, region_base, region_top, region_base + _buffer_to_requested_delta);
if (log_is_enabled(Debug, aot, map)) {
GrowableArrayCHeap objs;
- for (int i = 0; i < src_objs->objs()->length(); i++) {
- ArchiveBuilder::SourceObjInfo* src_info = src_objs->at(i);
+ // With -XX:+UseCompactObjectHeaders, it's possible for small objects (including some from
+ // ro_objs) to be allocated in the gaps in the RW region.
+ collect_metaspace_objs(&objs, region_base, region_top, rw_objs);
+ collect_metaspace_objs(&objs, region_base, region_top, ro_objs);
+ objs.sort(compare_by_address);
+ log_metaspace_objects_impl(address(region->base()), address(region->end()), &objs, 0, objs.length());
+ }
+}
+
+void AOTMapLogger::collect_metaspace_objs(GrowableArrayCHeap* objs,
+ address region_base, address region_top ,
+ const ArchiveBuilder::SourceObjList* src_objs) {
+ for (int i = 0; i < src_objs->objs()->length(); i++) {
+ ArchiveBuilder::SourceObjInfo* src_info = src_objs->at(i);
+ address buf_addr = src_info->buffered_addr();
+ if (region_base <= buf_addr && buf_addr < region_top) {
ArchivedObjInfo info;
info._src_addr = src_info->source_addr();
- info._buffered_addr = src_info->buffered_addr();
+ info._buffered_addr = buf_addr;
info._requested_addr = info._buffered_addr + _buffer_to_requested_delta;
info._bytes = src_info->size_in_bytes();
info._type = src_info->type();
- objs.append(info);
+ objs->append(info);
}
+ }
+}
- log_metaspace_objects_impl(address(region->base()), address(region->end()), &objs, 0, objs.length());
+int AOTMapLogger::compare_by_address(ArchivedObjInfo* a, ArchivedObjInfo* b) {
+ if (a->_buffered_addr < b->_buffered_addr) {
+ return -1;
+ } else if (a->_buffered_addr > b->_buffered_addr) {
+ return 1;
+ } else {
+ return 0;
}
}
@@ -577,7 +589,6 @@ public:
}
Klass* real_klass() {
- assert(UseCompressedClassPointers, "heap archiving requires UseCompressedClassPointers");
return _data._klass;
}
diff --git a/src/hotspot/share/cds/aotMapLogger.hpp b/src/hotspot/share/cds/aotMapLogger.hpp
index f495ed97f40..0a89f1e5012 100644
--- a/src/hotspot/share/cds/aotMapLogger.hpp
+++ b/src/hotspot/share/cds/aotMapLogger.hpp
@@ -127,7 +127,12 @@ private:
static void runtime_log(FileMapInfo* mapinfo, GrowableArrayCHeap* objs);
static void runtime_log_metaspace_regions(FileMapInfo* mapinfo, GrowableArrayCHeap* objs);
static void dumptime_log_metaspace_region(const char* name, DumpRegion* region,
- const ArchiveBuilder::SourceObjList* src_objs);
+ const ArchiveBuilder::SourceObjList* rw_objs,
+ const ArchiveBuilder::SourceObjList* ro_objs);
+ static void collect_metaspace_objs(GrowableArrayCHeap* objs,
+ address region_base, address region_top ,
+ const ArchiveBuilder::SourceObjList* src_objs);
+ static int compare_by_address(ArchivedObjInfo* a, ArchivedObjInfo* b);
// Common code for dumptime/runtime
static void log_file_header(FileMapInfo* mapinfo);
diff --git a/src/hotspot/share/cds/aotMappedHeapLoader.hpp b/src/hotspot/share/cds/aotMappedHeapLoader.hpp
index 7c5ca1b1f9e..10f5ce3124f 100644
--- a/src/hotspot/share/cds/aotMappedHeapLoader.hpp
+++ b/src/hotspot/share/cds/aotMappedHeapLoader.hpp
@@ -54,7 +54,7 @@ public:
// Can this VM map archived heap region? Currently only G1+compressed{oops,cp}
static bool can_map() {
- CDS_JAVA_HEAP_ONLY(return (UseG1GC && UseCompressedClassPointers);)
+ CDS_JAVA_HEAP_ONLY(return UseG1GC;)
NOT_CDS_JAVA_HEAP(return false;)
}
diff --git a/src/hotspot/share/cds/aotMappedHeapWriter.cpp b/src/hotspot/share/cds/aotMappedHeapWriter.cpp
index 64c0e3c40e8..8f810ef5244 100644
--- a/src/hotspot/share/cds/aotMappedHeapWriter.cpp
+++ b/src/hotspot/share/cds/aotMappedHeapWriter.cpp
@@ -64,6 +64,11 @@ HeapRootSegments AOTMappedHeapWriter::_heap_root_segments;
address AOTMappedHeapWriter::_requested_bottom;
address AOTMappedHeapWriter::_requested_top;
+static size_t _num_strings = 0;
+static size_t _string_bytes = 0;
+static size_t _num_packages = 0;
+static size_t _num_protection_domains = 0;
+
GrowableArrayCHeap* AOTMappedHeapWriter::_native_pointers;
GrowableArrayCHeap* AOTMappedHeapWriter::_source_objs;
GrowableArrayCHeap* AOTMappedHeapWriter::_source_objs_order;
@@ -71,8 +76,6 @@ GrowableArrayCHeap* AOTMappedH
AOTMappedHeapWriter::BufferOffsetToSourceObjectTable*
AOTMappedHeapWriter::_buffer_offset_to_source_obj_table = nullptr;
-DumpedInternedStrings *AOTMappedHeapWriter::_dumped_interned_strings = nullptr;
-
typedef HashTable<
size_t, // offset of a filler from AOTMappedHeapWriter::buffer_bottom()
size_t, // size of this filler (in bytes)
@@ -87,7 +90,6 @@ void AOTMappedHeapWriter::init() {
Universe::heap()->collect(GCCause::_java_lang_system_gc);
_buffer_offset_to_source_obj_table = new (mtClassShared) BufferOffsetToSourceObjectTable(/*size (prime)*/36137, /*max size*/1 * M);
- _dumped_interned_strings = new (mtClass)DumpedInternedStrings(INITIAL_TABLE_SIZE, MAX_TABLE_SIZE);
_fillers = new (mtClassShared) FillersTable();
_requested_bottom = nullptr;
_requested_top = nullptr;
@@ -141,9 +143,6 @@ int AOTMappedHeapWriter::narrow_oop_shift() {
void AOTMappedHeapWriter::delete_tables_with_raw_oops() {
delete _source_objs;
_source_objs = nullptr;
-
- delete _dumped_interned_strings;
- _dumped_interned_strings = nullptr;
}
void AOTMappedHeapWriter::add_source_obj(oop src_obj) {
@@ -181,25 +180,6 @@ bool AOTMappedHeapWriter::is_too_large_to_archive(size_t size) {
}
}
-// Keep track of the contents of the archived interned string table. This table
-// is used only by CDSHeapVerifier.
-void AOTMappedHeapWriter::add_to_dumped_interned_strings(oop string) {
- assert_at_safepoint(); // DumpedInternedStrings uses raw oops
- assert(!is_string_too_large_to_archive(string), "must be");
- bool created;
- _dumped_interned_strings->put_if_absent(string, true, &created);
- if (created) {
- // Prevent string deduplication from changing the value field to
- // something not in the archive.
- java_lang_String::set_deduplication_forbidden(string);
- _dumped_interned_strings->maybe_grow();
- }
-}
-
-bool AOTMappedHeapWriter::is_dumped_interned_string(oop o) {
- return _dumped_interned_strings->get(o) != nullptr;
-}
-
// Various lookup functions between source_obj, buffered_obj and requested_obj
bool AOTMappedHeapWriter::is_in_requested_range(oop o) {
assert(_requested_bottom != nullptr, "do not call before _requested_bottom is initialized");
@@ -430,6 +410,7 @@ void AOTMappedHeapWriter::copy_source_objs_to_buffer(GrowableArrayCHeapset_buffer_offset(buffer_offset);
+ assert(buffer_offset <= 0x7fffffff, "sanity");
OopHandle handle(Universe::vm_global(), src_obj);
_buffer_offset_to_source_obj_table->put_when_absent(buffer_offset, handle);
@@ -442,6 +423,9 @@ void AOTMappedHeapWriter::copy_source_objs_to_buffer(GrowableArrayCHeaplength() + 1, roots->length(), _num_native_ptrs);
+ log_info(aot)(" strings = %8zu (%zu bytes)", _num_strings, _string_bytes);
+ log_info(aot)(" packages = %8zu", _num_packages);
+ log_info(aot)(" protection domains = %8zu", _num_protection_domains);
}
size_t AOTMappedHeapWriter::filler_array_byte_size(int length) {
@@ -466,7 +450,6 @@ int AOTMappedHeapWriter::filler_array_length(size_t fill_bytes) {
}
HeapWord* AOTMappedHeapWriter::init_filler_array_at_buffer_top(int array_length, size_t fill_bytes) {
- assert(UseCompressedClassPointers, "Archived heap only supported for compressed klasses");
Klass* oak = Universe::objectArrayKlass(); // already relocated to point to archived klass
HeapWord* mem = offset_to_buffered_address(_buffer_used);
memset(mem, 0, fill_bytes);
@@ -530,7 +513,25 @@ void update_buffered_object_field(address buffered_obj, int field_offset, T valu
*field_addr = value;
}
+void AOTMappedHeapWriter::update_stats(oop src_obj) {
+ if (java_lang_String::is_instance(src_obj)) {
+ _num_strings ++;
+ _string_bytes += src_obj->size() * HeapWordSize;
+ _string_bytes += java_lang_String::value(src_obj)->size() * HeapWordSize;
+ } else {
+ Klass* k = src_obj->klass();
+ Symbol* name = k->name();
+ if (name->equals("java/lang/NamedPackage") || name->equals("java/lang/Package")) {
+ _num_packages ++;
+ } else if (name->equals("java/security/ProtectionDomain")) {
+ _num_protection_domains ++;
+ }
+ }
+}
+
size_t AOTMappedHeapWriter::copy_one_source_obj_to_buffer(oop src_obj) {
+ update_stats(src_obj);
+
assert(!is_too_large_to_archive(src_obj), "already checked");
size_t byte_size = src_obj->size() * HeapWordSize;
assert(byte_size > 0, "no zero-size objects");
@@ -722,7 +723,6 @@ template void AOTMappedHeapWriter::mark_oop_pointer(T* buffered_add
}
void AOTMappedHeapWriter::update_header_for_requested_obj(oop requested_obj, oop src_obj, Klass* src_klass) {
- assert(UseCompressedClassPointers, "Archived heap only supported for compressed klasses");
narrowKlass nk = ArchiveBuilder::current()->get_requested_narrow_klass(src_klass);
address buffered_addr = requested_addr_to_buffered_addr(cast_from_oop(requested_obj));
@@ -896,8 +896,14 @@ void AOTMappedHeapWriter::compute_ptrmap(AOTMappedHeapInfo* heap_info) {
native_ptr = RegeneratedClasses::get_regenerated_object(native_ptr);
}
- guarantee(ArchiveBuilder::current()->has_been_archived((address)native_ptr),
- "Metadata %p should have been archived", native_ptr);
+ if (!ArchiveBuilder::current()->has_been_archived((address)native_ptr)) {
+ ResourceMark rm;
+ LogStreamHandle(Error, aot) log;
+ log.print("Marking native pointer for oop %p (type = %s, offset = %d)",
+ cast_from_oop(src_obj), src_obj->klass()->external_name(), field_offset);
+ src_obj->print_on(&log);
+ fatal("Metadata %p should have been archived", native_ptr);
+ }
address buffered_native_ptr = ArchiveBuilder::current()->get_buffered_addr((address)native_ptr);
address requested_native_ptr = ArchiveBuilder::current()->to_requested(buffered_native_ptr);
diff --git a/src/hotspot/share/cds/aotMappedHeapWriter.hpp b/src/hotspot/share/cds/aotMappedHeapWriter.hpp
index 7481e7922a0..2420e68d9fe 100644
--- a/src/hotspot/share/cds/aotMappedHeapWriter.hpp
+++ b/src/hotspot/share/cds/aotMappedHeapWriter.hpp
@@ -40,20 +40,6 @@
class MemRegion;
#if INCLUDE_CDS_JAVA_HEAP
-class DumpedInternedStrings :
- public ResizeableHashTable
-{
-public:
- DumpedInternedStrings(unsigned size, unsigned max_size) :
- ResizeableHashTable(size, max_size) {}
-};
-
class AOTMappedHeapWriter : AllStatic {
friend class HeapShared;
friend class AOTMappedHeapLoader;
@@ -131,7 +117,6 @@ private:
static GrowableArrayCHeap* _native_pointers;
static GrowableArrayCHeap* _source_objs;
- static DumpedInternedStrings *_dumped_interned_strings;
// We sort _source_objs_order to minimize the number of bits in ptrmap and oopmap.
// See comments near the body of AOTMappedHeapWriter::compare_objs_by_oop_fields().
@@ -190,6 +175,7 @@ private:
static void copy_roots_to_buffer(GrowableArrayCHeap* roots);
static void copy_source_objs_to_buffer(GrowableArrayCHeap* roots);
static size_t copy_one_source_obj_to_buffer(oop src_obj);
+ static void update_stats(oop src_obj);
static void maybe_fill_gc_region_gap(size_t required_byte_size);
static size_t filler_array_byte_size(int length);
@@ -227,8 +213,6 @@ public:
static bool is_too_large_to_archive(size_t size);
static bool is_too_large_to_archive(oop obj);
static bool is_string_too_large_to_archive(oop string);
- static bool is_dumped_interned_string(oop o);
- static void add_to_dumped_interned_strings(oop string);
static void write(GrowableArrayCHeap*, AOTMappedHeapInfo* heap_info);
static address requested_address(); // requested address of the lowest achived heap object
static size_t get_filler_size_at(address buffered_addr);
diff --git a/src/hotspot/share/cds/aotMetaspace.cpp b/src/hotspot/share/cds/aotMetaspace.cpp
index b75d7628aa9..4c23ede9cb8 100644
--- a/src/hotspot/share/cds/aotMetaspace.cpp
+++ b/src/hotspot/share/cds/aotMetaspace.cpp
@@ -250,9 +250,9 @@ static bool shared_base_too_high(char* specified_base, char* aligned_base, size_
static char* compute_shared_base(size_t cds_max) {
char* specified_base = (char*)SharedBaseAddress;
size_t alignment = AOTMetaspace::core_region_alignment();
- if (UseCompressedClassPointers && CompressedKlassPointers::needs_class_space()) {
- alignment = MAX2(alignment, Metaspace::reserve_alignment());
- }
+#if INCLUDE_CLASS_SPACE
+ alignment = MAX2(alignment, Metaspace::reserve_alignment());
+#endif
if (SharedBaseAddress == 0) {
// Special meaning of -XX:SharedBaseAddress=0 -> Always map archive at os-selected address.
@@ -949,11 +949,18 @@ void AOTMetaspace::dump_static_archive(TRAPS) {
ResourceMark rm(THREAD);
HandleMark hm(THREAD);
- if (CDSConfig::is_dumping_final_static_archive() && AOTPrintTrainingInfo) {
- tty->print_cr("==================== archived_training_data ** before dumping ====================");
- TrainingData::print_archived_training_data_on(tty);
+ if (CDSConfig::is_dumping_final_static_archive()) {
+ if (AOTPrintTrainingInfo) {
+ tty->print_cr("==================== archived_training_data ** before dumping ====================");
+ TrainingData::print_archived_training_data_on(tty);
+ }
+ LogStreamHandle(Info, aot, training, data) log;
+ if (log.is_enabled()) {
+ TrainingData::print_archived_training_data_on(&log);
+ }
}
+
StaticArchiveBuilder builder;
dump_static_archive_impl(builder, THREAD);
if (HAS_PENDING_EXCEPTION) {
@@ -1187,8 +1194,8 @@ void AOTMetaspace::dump_static_archive_impl(StaticArchiveBuilder& builder, TRAPS
CDSConfig::enable_dumping_aot_code();
{
builder.start_ac_region();
- // Write the contents to AOT code region and close AOTCodeCache before packing the region
- AOTCodeCache::close();
+ // Write the contents to AOT code region before packing the region
+ AOTCodeCache::dump();
builder.end_ac_region();
}
CDSConfig::disable_dumping_aot_code();
@@ -1637,32 +1644,29 @@ MapArchiveResult AOTMetaspace::map_archives(FileMapInfo* static_mapinfo, FileMap
aot_log_debug(aot)("Failed to reserve spaces (use_requested_addr=%u)", (unsigned)use_requested_addr);
} else {
- if (Metaspace::using_class_space()) {
- prot_zone_size = protection_zone_size();
- }
+ CLASS_SPACE_ONLY(prot_zone_size = protection_zone_size();)
-#ifdef ASSERT
// Some sanity checks after reserving address spaces for archives
// and class space.
assert(archive_space_rs.is_reserved(), "Sanity");
- if (Metaspace::using_class_space()) {
- assert(archive_space_rs.base() == mapped_base_address &&
- archive_space_rs.size() > protection_zone_size(),
- "Archive space must lead and include the protection zone");
- // Class space must closely follow the archive space. Both spaces
- // must be aligned correctly.
- assert(class_space_rs.is_reserved() && class_space_rs.size() > 0,
- "A class space should have been reserved");
- assert(class_space_rs.base() >= archive_space_rs.end(),
- "class space should follow the cds archive space");
- assert(is_aligned(archive_space_rs.base(),
- core_region_alignment()),
- "Archive space misaligned");
- assert(is_aligned(class_space_rs.base(),
- Metaspace::reserve_alignment()),
- "class space misaligned");
- }
-#endif // ASSERT
+
+#if INCLUDE_CLASS_SPACE
+ assert(archive_space_rs.base() == mapped_base_address &&
+ archive_space_rs.size() > protection_zone_size(),
+ "Archive space must lead and include the protection zone");
+ // Class space must closely follow the archive space. Both spaces
+ // must be aligned correctly.
+ assert(class_space_rs.is_reserved() && class_space_rs.size() > 0,
+ "A class space should have been reserved");
+ assert(class_space_rs.base() >= archive_space_rs.end(),
+ "class space should follow the cds archive space");
+ assert(is_aligned(archive_space_rs.base(),
+ core_region_alignment()),
+ "Archive space misaligned");
+ assert(is_aligned(class_space_rs.base(),
+ Metaspace::reserve_alignment()),
+ "class space misaligned");
+#endif // INCLUDE_CLASS_SPACE
aot_log_info(aot)("Reserved archive_space_rs [" INTPTR_FORMAT " - " INTPTR_FORMAT "] (%zu) bytes%s",
p2i(archive_space_rs.base()), p2i(archive_space_rs.end()), archive_space_rs.size(),
@@ -1764,67 +1768,60 @@ MapArchiveResult AOTMetaspace::map_archives(FileMapInfo* static_mapinfo, FileMap
if (result == MAP_ARCHIVE_SUCCESS) {
SharedBaseAddress = (size_t)mapped_base_address;
-#ifdef _LP64
- if (Metaspace::using_class_space()) {
- assert(prot_zone_size > 0 &&
- *(mapped_base_address) == 'P' &&
- *(mapped_base_address + prot_zone_size - 1) == 'P',
- "Protection zone was overwritten?");
- // Set up ccs in metaspace.
- Metaspace::initialize_class_space(class_space_rs);
+#if INCLUDE_CLASS_SPACE
+ assert(prot_zone_size > 0 &&
+ *(mapped_base_address) == 'P' &&
+ *(mapped_base_address + prot_zone_size - 1) == 'P',
+ "Protection zone was overwritten?");
+ // Set up ccs in metaspace.
+ Metaspace::initialize_class_space(class_space_rs);
- // Set up compressed Klass pointer encoding: the encoding range must
- // cover both archive and class space.
- const address klass_range_start = (address)mapped_base_address;
- const size_t klass_range_size = (address)class_space_rs.end() - klass_range_start;
- if (INCLUDE_CDS_JAVA_HEAP || UseCompactObjectHeaders) {
- // The CDS archive may contain narrow Klass IDs that were precomputed at archive generation time:
- // - every archived java object header (only if INCLUDE_CDS_JAVA_HEAP)
- // - every archived Klass' prototype (only if +UseCompactObjectHeaders)
- //
- // In order for those IDs to still be valid, we need to dictate base and shift: base should be the
- // mapping start (including protection zone), shift should be the shift used at archive generation time.
- CompressedKlassPointers::initialize_for_given_encoding(
- klass_range_start, klass_range_size,
- klass_range_start, ArchiveBuilder::precomputed_narrow_klass_shift() // precomputed encoding, see ArchiveBuilder
- );
- assert(CompressedKlassPointers::base() == klass_range_start, "must be");
- } else {
- // Let JVM freely choose encoding base and shift
- CompressedKlassPointers::initialize(klass_range_start, klass_range_size);
- assert(CompressedKlassPointers::base() == nullptr ||
- CompressedKlassPointers::base() == klass_range_start, "must be");
- }
- // Establish protection zone, but only if we need one
- if (CompressedKlassPointers::base() == klass_range_start) {
- CompressedKlassPointers::establish_protection_zone(klass_range_start, prot_zone_size);
- }
+ // Set up compressed Klass pointer encoding: the encoding range must
+ // cover both archive and class space.
+ const address klass_range_start = (address)mapped_base_address;
+ const size_t klass_range_size = (address)class_space_rs.end() - klass_range_start;
+ if (INCLUDE_CDS_JAVA_HEAP || UseCompactObjectHeaders) {
+ // The CDS archive may contain narrow Klass IDs that were precomputed at archive generation time:
+ // - every archived java object header (only if INCLUDE_CDS_JAVA_HEAP)
+ // - every archived Klass' prototype (only if +UseCompactObjectHeaders)
+ //
+ // In order for those IDs to still be valid, we need to dictate base and shift: base should be the
+ // mapping start (including protection zone), shift should be the shift used at archive generation time.
+ CompressedKlassPointers::initialize_for_given_encoding(
+ klass_range_start, klass_range_size,
+ klass_range_start, ArchiveBuilder::precomputed_narrow_klass_shift() // precomputed encoding, see ArchiveBuilder
+ );
+ assert(CompressedKlassPointers::base() == klass_range_start, "must be");
+ } else {
+ // Let JVM freely choose encoding base and shift
+ CompressedKlassPointers::initialize(klass_range_start, klass_range_size);
+ assert(CompressedKlassPointers::base() == nullptr ||
+ CompressedKlassPointers::base() == klass_range_start, "must be");
+ }
+ // Establish protection zone, but only if we need one
+ if (CompressedKlassPointers::base() == klass_range_start) {
+ CompressedKlassPointers::establish_protection_zone(klass_range_start, prot_zone_size);
+ }
- if (static_mapinfo->can_use_heap_region()) {
- if (static_mapinfo->object_streaming_mode()) {
- HeapShared::initialize_loading_mode(HeapArchiveMode::_streaming);
- } else {
- // map_or_load_heap_region() compares the current narrow oop and klass encodings
- // with the archived ones, so it must be done after all encodings are determined.
- static_mapinfo->map_or_load_heap_region();
- HeapShared::initialize_loading_mode(HeapArchiveMode::_mapping);
- }
+ if (static_mapinfo->can_use_heap_region()) {
+ if (static_mapinfo->object_streaming_mode()) {
+ HeapShared::initialize_loading_mode(HeapArchiveMode::_streaming);
} else {
- FileMapRegion* r = static_mapinfo->region_at(AOTMetaspace::hp);
- if (r->used() > 0) {
- if (static_mapinfo->object_streaming_mode()) {
- AOTMetaspace::report_loading_error("Cannot use CDS heap data.");
- } else {
- if (!UseCompressedOops && !AOTMappedHeapLoader::can_map()) {
- AOTMetaspace::report_loading_error("Cannot use CDS heap data. Selected GC not compatible -XX:-UseCompressedOops");
- } else {
- AOTMetaspace::report_loading_error("Cannot use CDS heap data. UseEpsilonGC, UseG1GC, UseSerialGC, UseParallelGC, or UseShenandoahGC are required.");
- }
- }
- }
+ // map_or_load_heap_region() compares the current narrow oop and klass encodings
+ // with the archived ones, so it must be done after all encodings are determined.
+ static_mapinfo->map_or_load_heap_region();
+ HeapShared::initialize_loading_mode(HeapArchiveMode::_mapping);
+ }
+ } else {
+ FileMapRegion* r = static_mapinfo->region_at(AOTMetaspace::hp);
+ if (r->used() > 0) {
+ AOTMetaspace::report_loading_error("Cannot use CDS heap data.");
+ }
+ if (!CDSConfig::is_dumping_static_archive()) {
+ CDSConfig::stop_using_full_module_graph("No CDS heap data");
}
}
-#endif // _LP64
+#endif // INCLUDE_CLASS_SPACE
log_info(aot)("initial optimized module handling: %s", CDSConfig::is_using_optimized_module_handling() ? "enabled" : "disabled");
log_info(aot)("initial full module graph: %s", CDSConfig::is_using_full_module_graph() ? "enabled" : "disabled");
} else {
@@ -1857,8 +1854,13 @@ MapArchiveResult AOTMetaspace::map_archives(FileMapInfo* static_mapinfo, FileMap
// (The gap may result from different alignment requirements between metaspace
// and CDS)
//
-// If UseCompressedClassPointers is disabled, only one address space will be
-// reserved:
+// The range encompassing both spaces will be suitable to en/decode narrow Klass
+// pointers: the base will be valid for encoding the range [Base, End) and not
+// surpass the max. range for that encoding.
+//
+// On 32-bit, a "narrow" Klass is just the pointer itself, and the Klass encoding
+// range encompasses the whole address range. Consequently, we can "decode" and
+// "encode" any pointer anywhere, and so are free to place the CDS archive anywhere:
//
// +-- Base address End
// | |
@@ -1872,27 +1874,21 @@ MapArchiveResult AOTMetaspace::map_archives(FileMapInfo* static_mapinfo, FileMap
// use_archive_base_addr address is false, this base address is determined
// by the platform.
//
-// If UseCompressedClassPointers=1, the range encompassing both spaces will be
-// suitable to en/decode narrow Klass pointers: the base will be valid for
-// encoding, the range [Base, End) and not surpass the max. range for that encoding.
-//
// Return:
//
// - On success:
// - total_space_rs will be reserved as whole for archive_space_rs and
-// class_space_rs if UseCompressedClassPointers is true.
+// class_space_rs on 64-bit.
// On Windows, try reserve archive_space_rs and class_space_rs
// separately first if use_archive_base_addr is true.
// - archive_space_rs will be reserved and large enough to host static and
// if needed dynamic archive: [Base, A).
// archive_space_rs.base and size will be aligned to CDS reserve
// granularity.
-// - class_space_rs: If UseCompressedClassPointers=1, class_space_rs will
-// be reserved. Its start address will be aligned to metaspace reserve
-// alignment, which may differ from CDS alignment. It will follow the cds
-// archive space, close enough such that narrow class pointer encoding
-// covers both spaces.
-// If UseCompressedClassPointers=0, class_space_rs remains unreserved.
+// - class_space_rs: On 64-bit, class_space_rs will be reserved. Its start
+// address will be aligned to metaspace reserve alignment, which may differ
+// from CDS alignment. It will follow the cds archive space, close enough
+// such that narrow class pointer encoding covers both spaces.
// - On error: null is returned and the spaces remain unreserved.
char* AOTMetaspace::reserve_address_space_for_archives(FileMapInfo* static_mapinfo,
FileMapInfo* dynamic_mapinfo,
@@ -1908,32 +1904,34 @@ char* AOTMetaspace::reserve_address_space_for_archives(FileMapInfo* static_mapin
size_t archive_end_offset = (dynamic_mapinfo == nullptr) ? static_mapinfo->mapping_end_offset() : dynamic_mapinfo->mapping_end_offset();
size_t archive_space_size = align_up(archive_end_offset, archive_space_alignment);
- if (!Metaspace::using_class_space()) {
- // Get the simple case out of the way first:
- // no compressed class space, simple allocation.
+#if !INCLUDE_CLASS_SPACE
- // When running without class space, requested archive base should be aligned to cds core alignment.
- assert(is_aligned(base_address, archive_space_alignment),
- "Archive base address unaligned: " PTR_FORMAT ", needs alignment: %zu.",
- p2i(base_address), archive_space_alignment);
+ // Get the simple case out of the way first:
+ // no compressed class space, simple allocation.
- archive_space_rs = MemoryReserver::reserve((char*)base_address,
- archive_space_size,
- archive_space_alignment,
- os::vm_page_size(),
- mtNone);
- if (archive_space_rs.is_reserved()) {
- assert(base_address == nullptr ||
- (address)archive_space_rs.base() == base_address, "Sanity");
- // Register archive space with NMT.
- MemTracker::record_virtual_memory_tag(archive_space_rs, mtClassShared);
- return archive_space_rs.base();
- }
- return nullptr;
+ // When running without class space, requested archive base should be aligned to cds core alignment.
+ assert(is_aligned(base_address, archive_space_alignment),
+ "Archive base address unaligned: " PTR_FORMAT ", needs alignment: %zu.",
+ p2i(base_address), archive_space_alignment);
+
+ archive_space_rs = MemoryReserver::reserve((char*)base_address,
+ archive_space_size,
+ archive_space_alignment,
+ os::vm_page_size(),
+ mtNone);
+ if (archive_space_rs.is_reserved()) {
+ assert(base_address == nullptr ||
+ (address)archive_space_rs.base() == base_address, "Sanity");
+ // Register archive space with NMT.
+ MemTracker::record_virtual_memory_tag(archive_space_rs, mtClassShared);
+ return archive_space_rs.base();
}
-#ifdef _LP64
+ return nullptr;
+#else
+
+ // INCLUDE_CLASS_SPACE=1
// Complex case: two spaces adjacent to each other, both to be addressable
// with narrow class pointers.
// We reserve the whole range spanning both spaces, then split that range up.
@@ -2045,11 +2043,7 @@ char* AOTMetaspace::reserve_address_space_for_archives(FileMapInfo* static_mapin
return archive_space_rs.base();
-#else
- ShouldNotReachHere();
- return nullptr;
-#endif
-
+#endif // INCLUDE_CLASS_SPACE
}
void AOTMetaspace::release_reserved_spaces(ReservedSpace& total_space_rs,
diff --git a/src/hotspot/share/cds/aotReferenceObjSupport.cpp b/src/hotspot/share/cds/aotReferenceObjSupport.cpp
index 0c27c8ce5f0..2d5fc8c7f21 100644
--- a/src/hotspot/share/cds/aotReferenceObjSupport.cpp
+++ b/src/hotspot/share/cds/aotReferenceObjSupport.cpp
@@ -96,7 +96,7 @@ class KeepAliveObjectsTable : public HashTable {};
+ HeapShared::oop_address_hash> {};
static KeepAliveObjectsTable* _keep_alive_objs_table;
static OopHandle _keep_alive_objs_array;
diff --git a/src/hotspot/share/cds/aotStreamedHeapWriter.cpp b/src/hotspot/share/cds/aotStreamedHeapWriter.cpp
index f52532b2f2a..25bef10a673 100644
--- a/src/hotspot/share/cds/aotStreamedHeapWriter.cpp
+++ b/src/hotspot/share/cds/aotStreamedHeapWriter.cpp
@@ -242,20 +242,6 @@ void AOTStreamedHeapWriter::copy_roots_max_dfs_to_buffer(int roots_length) {
}
}
-static bool is_interned_string(oop obj) {
- if (!java_lang_String::is_instance(obj)) {
- return false;
- }
-
- ResourceMark rm;
- int len;
- jchar* name = java_lang_String::as_unicode_string_or_null(obj, len);
- if (name == nullptr) {
- fatal("Insufficient memory for dumping");
- }
- return StringTable::lookup(name, len) == obj;
-}
-
static BitMap::idx_t bit_idx_for_buffer_offset(size_t buffer_offset) {
if (UseCompressedOops) {
return BitMap::idx_t(buffer_offset / sizeof(narrowOop));
@@ -264,10 +250,6 @@ static BitMap::idx_t bit_idx_for_buffer_offset(size_t buffer_offset) {
}
}
-bool AOTStreamedHeapWriter::is_dumped_interned_string(oop obj) {
- return is_interned_string(obj) && HeapShared::get_cached_oop_info(obj) != nullptr;
-}
-
void AOTStreamedHeapWriter::copy_source_objs_to_buffer(GrowableArrayCHeap* roots) {
for (int i = 0; i < _source_objs->length(); i++) {
oop src_obj = _source_objs->at(i);
@@ -325,7 +307,7 @@ size_t AOTStreamedHeapWriter::copy_one_source_obj_to_buffer(oop src_obj) {
ensure_buffer_space(new_used);
- if (is_interned_string(src_obj)) {
+ if (HeapShared::is_interned_string(src_obj)) {
java_lang_String::hash_code(src_obj); // Sets the hash code field(s)
java_lang_String::set_deduplication_forbidden(src_obj); // Allows faster interning at runtime
assert(java_lang_String::hash_is_set(src_obj), "hash must be set");
@@ -387,7 +369,6 @@ template void AOTStreamedHeapWriter::map_oop_field_in_buffer(oop ob
}
void AOTStreamedHeapWriter::update_header_for_buffered_addr(address buffered_addr, oop src_obj, Klass* src_klass) {
- assert(UseCompressedClassPointers, "Archived heap only supported for compressed klasses");
narrowKlass nk = ArchiveBuilder::current()->get_requested_narrow_klass(src_klass);
markWord mw = markWord::prototype();
@@ -402,7 +383,7 @@ void AOTStreamedHeapWriter::update_header_for_buffered_addr(address buffered_add
mw = mw.copy_set_hash(src_hash);
}
- if (is_interned_string(src_obj)) {
+ if (HeapShared::is_interned_string(src_obj)) {
// Mark the mark word of interned string so the loader knows to link these to
// the string table at runtime.
mw = mw.set_marked();
diff --git a/src/hotspot/share/cds/aotStreamedHeapWriter.hpp b/src/hotspot/share/cds/aotStreamedHeapWriter.hpp
index ab5aec0327b..c3cc9f2c092 100644
--- a/src/hotspot/share/cds/aotStreamedHeapWriter.hpp
+++ b/src/hotspot/share/cds/aotStreamedHeapWriter.hpp
@@ -148,8 +148,6 @@ public:
return size_t(buffered_addr) - size_t(buffer_bottom());
}
- static bool is_dumped_interned_string(oop obj);
-
static size_t source_obj_to_buffered_offset(oop src_obj);
static address source_obj_to_buffered_addr(oop src_obj);
diff --git a/src/hotspot/share/cds/archiveBuilder.cpp b/src/hotspot/share/cds/archiveBuilder.cpp
index 0ea5d6c6ecb..21eef3d7b0b 100644
--- a/src/hotspot/share/cds/archiveBuilder.cpp
+++ b/src/hotspot/share/cds/archiveBuilder.cpp
@@ -627,6 +627,7 @@ void ArchiveBuilder::dump_ro_metadata() {
start_dump_region(&_ro_region);
make_shallow_copies(&_ro_region, &_ro_src_objs);
RegeneratedClasses::record_regenerated_objects();
+ DumpRegion::report_gaps(&_alloc_stats);
}
void ArchiveBuilder::make_shallow_copies(DumpRegion *dump_region,
@@ -639,33 +640,10 @@ void ArchiveBuilder::make_shallow_copies(DumpRegion *dump_region,
void ArchiveBuilder::make_shallow_copy(DumpRegion *dump_region, SourceObjInfo* src_info) {
address src = src_info->source_addr();
- int bytes = src_info->size_in_bytes(); // word-aligned
- size_t alignment = SharedSpaceObjectAlignment; // alignment for the dest pointer
+ int bytes = src_info->size_in_bytes();
+ char* dest = dump_region->allocate_metaspace_obj(bytes, src, src_info->type(),
+ src_info->read_only(), &_alloc_stats);
- char* oldtop = dump_region->top();
- if (src_info->type() == MetaspaceClosureType::ClassType) {
- // Allocate space for a pointer directly in front of the future InstanceKlass, so
- // we can do a quick lookup from InstanceKlass* -> RunTimeClassInfo*
- // without building another hashtable. See RunTimeClassInfo::get_for()
- // in systemDictionaryShared.cpp.
- Klass* klass = (Klass*)src;
- if (klass->is_instance_klass()) {
- SystemDictionaryShared::validate_before_archiving(InstanceKlass::cast(klass));
- dump_region->allocate(sizeof(address));
- }
-#ifdef _LP64
- // More strict alignments needed for UseCompressedClassPointers
- if (UseCompressedClassPointers) {
- alignment = nth_bit(ArchiveBuilder::precomputed_narrow_klass_shift());
- }
-#endif
- } else if (src_info->type() == MetaspaceClosureType::SymbolType) {
- // Symbols may be allocated by using AllocateHeap, so their sizes
- // may be less than size_in_bytes() indicates.
- bytes = ((Symbol*)src)->byte_size();
- }
-
- char* dest = dump_region->allocate(bytes, alignment);
memcpy(dest, src, bytes);
// Update the hash of buffered sorted symbols for static dump so that the symbols have deterministic contents
@@ -692,11 +670,6 @@ void ArchiveBuilder::make_shallow_copy(DumpRegion *dump_region, SourceObjInfo* s
log_trace(aot)("Copy: " PTR_FORMAT " ==> " PTR_FORMAT " %d", p2i(src), p2i(dest), bytes);
src_info->set_buffered_addr((address)dest);
-
- char* newtop = dump_region->top();
- _alloc_stats.record(src_info->type(), int(newtop - oldtop), src_info->read_only());
-
- DEBUG_ONLY(_alloc_stats.verify((int)dump_region->used(), src_info->read_only()));
}
// This is used by code that hand-assembles data structures, such as the LambdaProxyClassKey, that are
@@ -1119,20 +1092,17 @@ class RelocateBufferToRequested : public BitMapClosure {
}
};
-#ifdef _LP64
int ArchiveBuilder::precomputed_narrow_klass_shift() {
- // Legacy Mode:
- // We use 32 bits for narrowKlass, which should cover the full 4G Klass range. Shift can be 0.
+ // Standard Mode:
+ // We use 32 bits for narrowKlass, which should cover a full 4G Klass range. Shift can be 0.
// CompactObjectHeader Mode:
// narrowKlass is much smaller, and we use the highest possible shift value to later get the maximum
// Klass encoding range.
//
// Note that all of this may change in the future, if we decide to correct the pre-calculated
// narrow Klass IDs at archive load time.
- assert(UseCompressedClassPointers, "Only needed for compressed class pointers");
return UseCompactObjectHeaders ? CompressedKlassPointers::max_shift() : 0;
}
-#endif // _LP64
void ArchiveBuilder::relocate_to_requested() {
if (!ro_region()->is_packed()) {
diff --git a/src/hotspot/share/cds/archiveBuilder.hpp b/src/hotspot/share/cds/archiveBuilder.hpp
index b3667ea11b4..6a9df87092b 100644
--- a/src/hotspot/share/cds/archiveBuilder.hpp
+++ b/src/hotspot/share/cds/archiveBuilder.hpp
@@ -484,7 +484,6 @@ public:
void print_stats();
void report_out_of_space(const char* name, size_t needed_bytes);
-#ifdef _LP64
// The CDS archive contains pre-computed narrow Klass IDs. It carries them in the headers of
// archived heap objects. With +UseCompactObjectHeaders, it also carries them in prototypes
// in Klass.
@@ -504,7 +503,6 @@ public:
// TinyClassPointer Mode:
// We use the highest possible shift value to maximize the encoding range size.
static int precomputed_narrow_klass_shift();
-#endif // _LP64
};
diff --git a/src/hotspot/share/cds/archiveUtils.cpp b/src/hotspot/share/cds/archiveUtils.cpp
index ea9bde8eb8d..6e0608e196b 100644
--- a/src/hotspot/share/cds/archiveUtils.cpp
+++ b/src/hotspot/share/cds/archiveUtils.cpp
@@ -30,6 +30,7 @@
#include "cds/cdsConfig.hpp"
#include "cds/classListParser.hpp"
#include "cds/classListWriter.hpp"
+#include "cds/dumpAllocStats.hpp"
#include "cds/dynamicArchive.hpp"
#include "cds/filemap.hpp"
#include "cds/heapShared.hpp"
@@ -46,6 +47,7 @@
#include "utilities/debug.hpp"
#include "utilities/formatBuffer.hpp"
#include "utilities/globalDefinitions.hpp"
+#include "utilities/rbTree.inline.hpp"
#include "utilities/spinYield.hpp"
CHeapBitMap* ArchivePtrMarker::_ptrmap = nullptr;
@@ -116,13 +118,17 @@ void ArchivePtrMarker::mark_pointer(address* ptr_loc) {
if (ptr_base() <= ptr_loc && ptr_loc < ptr_end()) {
address value = *ptr_loc;
- // We don't want any pointer that points to very bottom of the archive, otherwise when
- // AOTMetaspace::default_base_address()==0, we can't distinguish between a pointer
- // to nothing (null) vs a pointer to an objects that happens to be at the very bottom
- // of the archive.
- assert(value != (address)ptr_base(), "don't point to the bottom of the archive");
-
if (value != nullptr) {
+ // We don't want any pointer that points to very bottom of the AOT metaspace, otherwise
+ // when AOTMetaspace::default_base_address()==0, we can't distinguish between a pointer
+ // to nothing (null) vs a pointer to an objects that happens to be at the very bottom
+ // of the AOT metaspace.
+ //
+ // This should never happen because the protection zone prevents any valid objects from
+ // being allocated at the bottom of the AOT metaspace.
+ assert(AOTMetaspace::protection_zone_size() > 0, "must be");
+ assert(ArchiveBuilder::current()->any_to_offset(value) > 0, "cannot point to bottom of AOT metaspace");
+
assert(uintx(ptr_loc) % sizeof(intptr_t) == 0, "pointers must be stored in aligned addresses");
size_t idx = ptr_loc - ptr_base();
if (_ptrmap->size() <= idx) {
@@ -130,7 +136,6 @@ void ArchivePtrMarker::mark_pointer(address* ptr_loc) {
}
assert(idx < _ptrmap->size(), "must be");
_ptrmap->set_bit(idx);
- //tty->print_cr("Marking pointer [" PTR_FORMAT "] -> " PTR_FORMAT " @ %5zu", p2i(ptr_loc), p2i(*ptr_loc), idx);
}
}
}
@@ -144,7 +149,6 @@ void ArchivePtrMarker::clear_pointer(address* ptr_loc) {
size_t idx = ptr_loc - ptr_base();
assert(idx < _ptrmap->size(), "cannot clear pointers that have not been marked");
_ptrmap->clear_bit(idx);
- //tty->print_cr("Clearing pointer [" PTR_FORMAT "] -> " PTR_FORMAT " @ %5zu", p2i(ptr_loc), p2i(*ptr_loc), idx);
}
class ArchivePtrBitmapCleaner: public BitMapClosure {
@@ -249,16 +253,179 @@ void DumpRegion::commit_to(char* newtop) {
which, commit, _vs->actual_committed_size(), _vs->high());
}
+// Basic allocation. Any alignment gaps will be wasted.
char* DumpRegion::allocate(size_t num_bytes, size_t alignment) {
// Always align to at least minimum alignment
alignment = MAX2(SharedSpaceObjectAlignment, alignment);
char* p = (char*)align_up(_top, alignment);
- char* newtop = p + align_up(num_bytes, (size_t)SharedSpaceObjectAlignment);
+ char* newtop = p + align_up(num_bytes, SharedSpaceObjectAlignment);
expand_top_to(newtop);
memset(p, 0, newtop - p);
return p;
}
+class DumpRegion::AllocGap {
+ size_t _gap_bytes; // size of this gap in bytes
+ char* _gap_bottom; // must be SharedSpaceObjectAlignment aligned
+public:
+ size_t gap_bytes() const { return _gap_bytes; }
+ char* gap_bottom() const { return _gap_bottom; }
+
+ AllocGap(size_t bytes, char* bottom) : _gap_bytes(bytes), _gap_bottom(bottom) {
+ precond(is_aligned(gap_bytes(), SharedSpaceObjectAlignment));
+ precond(is_aligned(gap_bottom(), SharedSpaceObjectAlignment));
+ }
+};
+
+struct DumpRegion::AllocGapCmp {
+ static RBTreeOrdering cmp(AllocGap a, AllocGap b) {
+ RBTreeOrdering order = rbtree_primitive_cmp(a.gap_bytes(), b.gap_bytes());
+ if (order == RBTreeOrdering::EQ) {
+ order = rbtree_primitive_cmp(a.gap_bottom(), b.gap_bottom());
+ }
+ return order;
+ }
+};
+
+struct Empty {};
+using AllocGapNode = RBNode;
+
+class DumpRegion::AllocGapTree : public RBTreeCHeap {
+public:
+ size_t add_gap(char* gap_bottom, char* gap_top) {
+ precond(gap_bottom < gap_top);
+ size_t gap_bytes = pointer_delta(gap_top, gap_bottom, 1);
+ precond(gap_bytes > 0);
+
+ _total_gap_bytes += gap_bytes;
+
+ AllocGap gap(gap_bytes, gap_bottom); // constructor checks alignment
+ AllocGapNode* node = allocate_node(gap, Empty{});
+ insert(gap, node);
+
+ log_trace(aot, alloc)("adding a gap of %zu bytes @ %p (total = %zu) in %zu blocks", gap_bytes, gap_bottom, _total_gap_bytes, size());
+ return gap_bytes;
+ }
+
+ char* allocate_from_gap(size_t num_bytes) {
+ // The gaps are sorted in ascending order of their sizes. When two gaps have the same
+ // size, the one with a lower gap_bottom comes first.
+ //
+ // Find the first gap that's big enough, with the lowest gap_bottom.
+ AllocGap target(num_bytes, nullptr);
+ AllocGapNode* node = closest_ge(target);
+ if (node == nullptr) {
+ return nullptr; // Didn't find any usable gap.
+ }
+
+ size_t gap_bytes = node->key().gap_bytes();
+ char* gap_bottom = node->key().gap_bottom();
+ char* result = gap_bottom;
+ precond(is_aligned(result, SharedSpaceObjectAlignment));
+
+ remove(node);
+
+ precond(_total_gap_bytes >= num_bytes);
+ _total_gap_bytes -= num_bytes;
+ _total_gap_bytes_used += num_bytes;
+ _total_gap_allocs++;
+ DEBUG_ONLY(node = nullptr); // Don't use it anymore!
+
+ precond(gap_bytes >= num_bytes);
+ if (gap_bytes > num_bytes) {
+ gap_bytes -= num_bytes;
+ gap_bottom += num_bytes;
+
+ AllocGap gap(gap_bytes, gap_bottom); // constructor checks alignment
+ AllocGapNode* new_node = allocate_node(gap, Empty{});
+ insert(gap, new_node);
+ }
+ log_trace(aot, alloc)("%zu bytes @ %p in a gap of %zu bytes (used gaps %zu times, remain gap = %zu bytes in %zu blocks)",
+ num_bytes, result, gap_bytes, _total_gap_allocs, _total_gap_bytes, size());
+ return result;
+ }
+};
+
+size_t DumpRegion::_total_gap_bytes = 0;
+size_t DumpRegion::_total_gap_bytes_used = 0;
+size_t DumpRegion::_total_gap_allocs = 0;
+DumpRegion::AllocGapTree DumpRegion::_gap_tree;
+
+// Alignment gaps happen only for the RW space. Collect the gaps into the _gap_tree so they can be
+// used for future small object allocation.
+char* DumpRegion::allocate_metaspace_obj(size_t num_bytes, address src, MetaspaceClosureType type, bool read_only, DumpAllocStats* stats) {
+ num_bytes = align_up(num_bytes, SharedSpaceObjectAlignment);
+ size_t alignment = SharedSpaceObjectAlignment; // alignment for the dest pointer
+ bool is_class = (type == MetaspaceClosureType::ClassType);
+ bool is_instance_class = is_class && ((Klass*)src)->is_instance_klass();
+
+#ifdef _LP64
+ // More strict alignments needed for Klass objects
+ if (is_class) {
+ size_t klass_alignment = checked_cast(nth_bit(ArchiveBuilder::precomputed_narrow_klass_shift()));
+ alignment = MAX2(alignment, klass_alignment);
+ precond(is_aligned(alignment, SharedSpaceObjectAlignment));
+ }
+#endif
+
+ if (alignment == SharedSpaceObjectAlignment && type != MetaspaceClosureType::SymbolType) {
+ // The addresses of Symbols must be in the same order as they are in ArchiveBuilder::SourceObjList.
+ // If we put them in gaps, their order will change.
+ //
+ // We have enough small objects that all gaps are usually filled.
+ char* p = _gap_tree.allocate_from_gap(num_bytes);
+ if (p != nullptr) {
+ // Already memset to 0 when adding the gap
+ stats->record(type, checked_cast(num_bytes), /*read_only=*/false); // all gaps are from RW space (for classes)
+ return p;
+ }
+ }
+
+ // Reserve space for a pointer directly in front of the buffered InstanceKlass, so
+ // we can do a quick lookup from InstanceKlass* -> RunTimeClassInfo*
+ // without building another hashtable. See RunTimeClassInfo::get_for()
+ // in systemDictionaryShared.cpp.
+ const size_t RuntimeClassInfoPtrSize = is_instance_class ? sizeof(address) : 0;
+
+ if (is_class && !is_aligned(top() + RuntimeClassInfoPtrSize, alignment)) {
+ // We need to add a gap to align the buffered Klass. Save the gap for future small allocations.
+ assert(read_only == false, "only gaps in RW region are reusable");
+ char* gap_bottom = top();
+ char* gap_top = align_up(gap_bottom + RuntimeClassInfoPtrSize, alignment) - RuntimeClassInfoPtrSize;
+ size_t gap_bytes = _gap_tree.add_gap(gap_bottom, gap_top);
+ allocate(gap_bytes);
+ }
+
+ char* oldtop = top();
+ if (is_instance_class) {
+ SystemDictionaryShared::validate_before_archiving((InstanceKlass*)src);
+ allocate(RuntimeClassInfoPtrSize);
+ }
+
+ precond(is_aligned(top(), alignment));
+ char* result = allocate(num_bytes);
+ log_trace(aot, alloc)("%zu bytes @ %p", num_bytes, result);
+ stats->record(type, pointer_delta_as_int(top(), oldtop), read_only); // includes RuntimeClassInfoPtrSize for classes
+
+ return result;
+}
+
+// Usually we have no gaps left.
+void DumpRegion::report_gaps(DumpAllocStats* stats) {
+ _gap_tree.visit_in_order([&](const AllocGapNode* node) {
+ stats->record_gap(checked_cast(node->key().gap_bytes()));
+ return true;
+ });
+ if (_gap_tree.size() > 0) {
+ log_warning(aot)("Unexpected %zu gaps (%zu bytes) for Klass alignment",
+ _gap_tree.size(), _total_gap_bytes);
+ }
+ if (_total_gap_allocs > 0) {
+ log_info(aot)("Allocated %zu objects of %zu bytes in gaps (remain = %zu bytes)",
+ _total_gap_allocs, _total_gap_bytes_used, _total_gap_bytes);
+ }
+}
+
void DumpRegion::append_intptr_t(intptr_t n, bool need_to_mark) {
assert(is_aligned(_top, sizeof(intptr_t)), "bad alignment");
intptr_t *p = (intptr_t*)_top;
diff --git a/src/hotspot/share/cds/archiveUtils.hpp b/src/hotspot/share/cds/archiveUtils.hpp
index e5d1efa5eab..42455adedd0 100644
--- a/src/hotspot/share/cds/archiveUtils.hpp
+++ b/src/hotspot/share/cds/archiveUtils.hpp
@@ -28,20 +28,24 @@
#include "cds/cds_globals.hpp"
#include "cds/serializeClosure.hpp"
#include "logging/log.hpp"
+#include "memory/allocation.hpp"
#include "memory/metaspace.hpp"
+#include "memory/metaspaceClosureType.hpp"
#include "memory/virtualspace.hpp"
#include "runtime/nonJavaThread.hpp"
#include "runtime/semaphore.hpp"
#include "utilities/bitMap.hpp"
#include "utilities/exceptions.hpp"
+#include "utilities/growableArray.hpp"
+#include "utilities/hashTable.hpp"
#include "utilities/macros.hpp"
class BootstrapInfo;
+class DumpAllocStats;
class ReservedSpace;
class VirtualSpace;
template class Array;
-template class GrowableArray;
// ArchivePtrMarker is used to mark the location of pointers embedded in a CDS archive. E.g., when an
// InstanceKlass k is dumped, we mark the location of the k->_name pointer by effectively calling
@@ -159,6 +163,18 @@ private:
void commit_to(char* newtop);
+public:
+ // Allocation gaps (due to Klass alignment)
+ class AllocGapTree;
+ class AllocGap;
+ struct AllocGapCmp;
+
+private:
+ static AllocGapTree _gap_tree;
+ static size_t _total_gap_bytes;
+ static size_t _total_gap_bytes_used;
+ static size_t _total_gap_allocs;
+
public:
DumpRegion(const char* name)
: _name(name), _base(nullptr), _top(nullptr), _end(nullptr),
@@ -167,6 +183,7 @@ public:
char* expand_top_to(char* newtop);
char* allocate(size_t num_bytes, size_t alignment = 0);
+ char* allocate_metaspace_obj(size_t num_bytes, address src, MetaspaceClosureType type, bool read_only, DumpAllocStats* stats);
void append_intptr_t(intptr_t n, bool need_to_mark = false) NOT_CDS_RETURN;
@@ -191,6 +208,8 @@ public:
bool contains(char* p) {
return base() <= p && p < top();
}
+
+ static void report_gaps(DumpAllocStats* stats);
};
// Closure for serializing initialization data out to a data area to be
@@ -383,4 +402,39 @@ public:
void run_task(ArchiveWorkerTask* task);
};
+// A utility class for writing an array of unique items into the
+// AOT cache. For determinism, the order of the array is the same
+// as calls to add(). I.e., if items are added in the order
+// of A, B, A, C, B, D, then the array will be written as {A, B, C, D}
+template
+class ArchivableTable : public AnyObj {
+ using Table = HashTable;
+ Table* _seen_items;
+ GrowableArray* _ordered_array;
+public:
+ ArchivableTable() {
+ _seen_items = new (mtClassShared)Table();
+ _ordered_array = new (mtClassShared)GrowableArray(128, mtClassShared);
+ }
+
+ ~ArchivableTable() {
+ delete _seen_items;
+ delete _ordered_array;
+ }
+
+ void add(T t) {
+ bool created;
+ _seen_items->put_if_absent(t, &created);
+ if (created) {
+ _ordered_array->append(t);
+ }
+ }
+
+ Array* write_ordered_array() {
+ return ArchiveUtils::archive_array(_ordered_array);
+ }
+};
+
+using ArchivableKlassTable = ArchivableTable;
+
#endif // SHARE_CDS_ARCHIVEUTILS_HPP
diff --git a/src/hotspot/share/cds/cdsConfig.cpp b/src/hotspot/share/cds/cdsConfig.cpp
index f4ef3c66f7a..ecf3c6d2231 100644
--- a/src/hotspot/share/cds/cdsConfig.cpp
+++ b/src/hotspot/share/cds/cdsConfig.cpp
@@ -108,6 +108,8 @@ void CDSConfig::ergo_initialize() {
}
AOTMapLogger::ergo_initialize();
+
+ setup_compiler_args();
}
const char* CDSConfig::default_archive_path() {
@@ -635,8 +637,6 @@ bool CDSConfig::check_vm_args_consistency(bool patch_mod_javabase, bool mode_fla
FLAG_SET_ERGO_IF_DEFAULT(AOTClassLinking, true);
}
- setup_compiler_args();
-
if (AOTClassLinking) {
// If AOTClassLinking is specified, enable all AOT optimizations by default.
FLAG_SET_ERGO_IF_DEFAULT(AOTInvokeDynamicLinking, true);
@@ -891,10 +891,6 @@ const char* CDSConfig::type_of_archive_being_written() {
// If an incompatible VM options is found, return a text message that explains why
static const char* check_options_incompatible_with_dumping_heap() {
#if INCLUDE_CDS_JAVA_HEAP
- if (!UseCompressedClassPointers) {
- return "UseCompressedClassPointers must be true";
- }
-
return nullptr;
#else
return "JVM not configured for writing Java heap objects";
@@ -972,17 +968,27 @@ bool CDSConfig::is_loading_heap() {
}
bool CDSConfig::is_dumping_klass_subgraphs() {
- if (is_dumping_classic_static_archive() || is_dumping_final_static_archive()) {
+ if (is_dumping_aot_linked_classes()) {
// KlassSubGraphs (see heapShared.cpp) is a legacy mechanism for archiving oops. It
// has been superceded by AOT class linking. This feature is used only when
// AOT class linking is disabled.
- //
- // KlassSubGraphs are disabled in the preimage static archive, which contains a very
- // limited set of oops.
- return is_dumping_heap() && !is_dumping_aot_linked_classes();
- } else {
return false;
}
+
+ if (is_dumping_preimage_static_archive()) {
+ // KlassSubGraphs are disabled in the preimage static archive, which contains a very
+ // limited set of oops.
+ return false;
+ }
+
+ if (!is_dumping_full_module_graph()) {
+ // KlassSubGraphs cannot be partially disabled. Since some of the KlassSubGraphs
+ // are used for (legacy support) of the archived full module graph, if
+ // is_dumping_full_module_graph() is calse, we must disable all KlassSubGraphs.
+ return false;
+ }
+
+ return is_dumping_heap();
}
bool CDSConfig::is_using_klass_subgraphs() {
diff --git a/src/hotspot/share/cds/cdsHeapVerifier.hpp b/src/hotspot/share/cds/cdsHeapVerifier.hpp
index 7f1bdb1d249..f8e090801bb 100644
--- a/src/hotspot/share/cds/cdsHeapVerifier.hpp
+++ b/src/hotspot/share/cds/cdsHeapVerifier.hpp
@@ -53,7 +53,7 @@ class CDSHeapVerifier : public KlassClosure {
15889, // prime number
AnyObj::C_HEAP,
mtClassShared,
- HeapShared::oop_hash> _table;
+ HeapShared::oop_address_hash> _table;
GrowableArray _exclusions;
GrowableArray _shared_secret_accessors;
diff --git a/src/hotspot/share/cds/dumpAllocStats.cpp b/src/hotspot/share/cds/dumpAllocStats.cpp
index 5f324566103..ddd4bac6086 100644
--- a/src/hotspot/share/cds/dumpAllocStats.cpp
+++ b/src/hotspot/share/cds/dumpAllocStats.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -129,15 +129,3 @@ void DumpAllocStats::print_stats(int ro_all, int rw_all) {
_bytes [RW][MethodTrainingDataType]);
}
-
-#ifdef ASSERT
-void DumpAllocStats::verify(int expected_byte_size, bool read_only) const {
- int bytes = 0;
- const int what = (int)(read_only ? RO : RW);
- for (int type = 0; type < int(_number_of_types); type ++) {
- bytes += _bytes[what][type];
- }
- assert(bytes == expected_byte_size, "counter mismatch (%s: %d vs %d)",
- (read_only ? "RO" : "RW"), bytes, expected_byte_size);
-}
-#endif // ASSERT
diff --git a/src/hotspot/share/cds/dumpAllocStats.hpp b/src/hotspot/share/cds/dumpAllocStats.hpp
index 4553f0f6a01..4daef9195a6 100644
--- a/src/hotspot/share/cds/dumpAllocStats.hpp
+++ b/src/hotspot/share/cds/dumpAllocStats.hpp
@@ -41,6 +41,7 @@ public:
f(StringHashentry) \
f(StringBucket) \
f(CppVTables) \
+ f(Gap) \
f(Other)
#define DUMPED_TYPE_DECLARE(name) name ## Type,
@@ -111,12 +112,19 @@ public:
_bytes [which][t] += byte_size;
}
+ void record_gap(int byte_size) {
+ _counts[RW][GapType] += 1;
+ _bytes [RW][GapType] += byte_size;
+ }
+
void record_other_type(int byte_size, bool read_only) {
int which = (read_only) ? RO : RW;
+ _counts[which][OtherType] += 1;
_bytes [which][OtherType] += byte_size;
}
void record_cpp_vtables(int byte_size) {
+ _counts[RW][CppVTablesType] += 1;
_bytes[RW][CppVTablesType] += byte_size;
}
@@ -145,9 +153,6 @@ public:
}
void print_stats(int ro_all, int rw_all);
-
- DEBUG_ONLY(void verify(int expected_byte_size, bool read_only) const);
-
};
#endif // SHARE_CDS_DUMPALLOCSTATS_HPP
diff --git a/src/hotspot/share/cds/dumpTimeClassInfo.hpp b/src/hotspot/share/cds/dumpTimeClassInfo.hpp
index c2f83b22337..d2de8148bea 100644
--- a/src/hotspot/share/cds/dumpTimeClassInfo.hpp
+++ b/src/hotspot/share/cds/dumpTimeClassInfo.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -41,7 +41,6 @@ class Symbol;
class DumpTimeClassInfo: public CHeapObj {
bool _excluded;
bool _is_aot_tooling_class;
- bool _is_early_klass;
bool _has_checked_exclusion;
class DTLoaderConstraint {
@@ -143,7 +142,6 @@ public:
_clsfile_crc32 = -1;
_excluded = false;
_is_aot_tooling_class = false;
- _is_early_klass = JvmtiExport::is_early_phase();
_verifier_constraints = nullptr;
_verifier_constraint_flags = nullptr;
_loader_constraints = nullptr;
@@ -219,11 +217,6 @@ public:
_is_aot_tooling_class = true;
}
- // Was this class loaded while JvmtiExport::is_early_phase()==true
- bool is_early_klass() {
- return _is_early_klass;
- }
-
// simple accessors
void set_excluded() { _excluded = true; }
bool has_checked_exclusion() const { return _has_checked_exclusion; }
diff --git a/src/hotspot/share/cds/filemap.cpp b/src/hotspot/share/cds/filemap.cpp
index 91fbce701e5..38502b2b2d8 100644
--- a/src/hotspot/share/cds/filemap.cpp
+++ b/src/hotspot/share/cds/filemap.cpp
@@ -225,15 +225,9 @@ void FileMapHeader::populate(FileMapInfo *info, size_t core_region_alignment,
}
#endif
_compressed_oops = UseCompressedOops;
- _compressed_class_ptrs = UseCompressedClassPointers;
- if (UseCompressedClassPointers) {
-#ifdef _LP64
- _narrow_klass_pointer_bits = CompressedKlassPointers::narrow_klass_pointer_bits();
- _narrow_klass_shift = ArchiveBuilder::precomputed_narrow_klass_shift();
-#endif
- } else {
- _narrow_klass_pointer_bits = _narrow_klass_shift = -1;
- }
+ _narrow_klass_pointer_bits = CompressedKlassPointers::narrow_klass_pointer_bits();
+ _narrow_klass_shift = ArchiveBuilder::precomputed_narrow_klass_shift();
+
// Which JIT compier is used
_compiler_type = (u1)CompilerConfig::compiler_type();
_type_profile_level = TypeProfileLevel;
@@ -295,7 +289,6 @@ void FileMapHeader::print(outputStream* st) {
st->print_cr("- max_heap_size: %zu", _max_heap_size);
st->print_cr("- narrow_oop_mode: %d", _narrow_oop_mode);
st->print_cr("- compressed_oops: %d", _compressed_oops);
- st->print_cr("- compressed_class_ptrs: %d", _compressed_class_ptrs);
st->print_cr("- narrow_klass_pointer_bits: %d", _narrow_klass_pointer_bits);
st->print_cr("- narrow_klass_shift: %d", _narrow_klass_shift);
st->print_cr("- cloned_vtables: %u", cast_to_u4(_cloned_vtables));
@@ -1535,10 +1528,34 @@ bool FileMapInfo::can_use_heap_region() {
if (!has_heap_region()) {
return false;
}
- if (!object_streaming_mode() && !Universe::heap()->can_load_archived_objects() && !UseG1GC) {
- // Incompatible object format
+
+ if (!object_streaming_mode() && !AOTMappedHeapLoader::can_use()) {
+ // Currently this happens only when using ZGC with an AOT cache generated with -XX:-AOTStreamableObjects
+ AOTMetaspace::report_loading_error("CDS heap data cannot be used by the selected GC. "
+ "Please choose a different GC or rebuild AOT cache "
+ "with -XX:+AOTStreamableObjects");
return false;
}
+
+ if (CDSConfig::is_using_aot_linked_classes()) {
+ assert(!JvmtiExport::should_post_class_file_load_hook(), "already checked");
+ assert(CDSConfig::is_using_full_module_graph(), "already checked");
+ } else {
+ if (JvmtiExport::should_post_class_file_load_hook()) {
+ AOTMetaspace::report_loading_error("CDS heap data is disabled because JVMTI ClassFileLoadHook is in use.");
+ return false;
+ }
+ if (!CDSConfig::is_using_full_module_graph()) {
+ if (CDSConfig::is_dumping_final_static_archive()) {
+ // We are loading the preimage static archive, which has no KlassSubGraphs.
+ // See CDSConfig::is_dumping_klass_subgraphs()
+ } else {
+ AOTMetaspace::report_loading_error("CDS heap data is disabled because archived full module graph is not used.");
+ return false;
+ }
+ }
+ }
+
if (JvmtiExport::should_post_class_file_load_hook() && JvmtiExport::has_early_class_hook_env()) {
ShouldNotReachHere(); // CDS should have been disabled.
// The archived objects are mapped at JVM start-up, but we don't know if
@@ -1902,11 +1919,12 @@ bool FileMapHeader::validate() {
_has_platform_or_app_classes = false;
}
- aot_log_info(aot)("The %s was created with UseCompressedOops = %d, UseCompressedClassPointers = %d, UseCompactObjectHeaders = %d",
- file_type, compressed_oops(), compressed_class_pointers(), compact_headers());
- if (compressed_oops() != UseCompressedOops || compressed_class_pointers() != UseCompressedClassPointers) {
- aot_log_warning(aot)("Unable to use %s.\nThe saved state of UseCompressedOops and UseCompressedClassPointers is "
- "different from runtime, CDS will be disabled.", file_type);
+ aot_log_info(aot)("The %s was created with UseCompressedOops = %d, UseCompactObjectHeaders = %d",
+ file_type, compressed_oops(), compact_headers());
+ if (compressed_oops() != UseCompressedOops) {
+ aot_log_warning(aot)("Unable to use %s.\nThe saved state of UseCompressedOops (%d) is "
+ "different from runtime (%d), CDS will be disabled.", file_type,
+ compressed_oops(), UseCompressedOops);
return false;
}
diff --git a/src/hotspot/share/cds/filemap.hpp b/src/hotspot/share/cds/filemap.hpp
index 56b88df378a..bae08bd5bc7 100644
--- a/src/hotspot/share/cds/filemap.hpp
+++ b/src/hotspot/share/cds/filemap.hpp
@@ -120,7 +120,6 @@ private:
CompressedOops::Mode _narrow_oop_mode; // compressed oop encoding mode
bool _object_streaming_mode; // dump was created for object streaming
bool _compressed_oops; // save the flag UseCompressedOops
- bool _compressed_class_ptrs; // save the flag UseCompressedClassPointers
int _narrow_klass_pointer_bits; // save number of bits in narrowKlass
int _narrow_klass_shift; // save shift width used to pre-compute narrowKlass IDs in archived heap objects
narrowPtr _cloned_vtables; // The address of the first cloned vtable
@@ -200,7 +199,6 @@ public:
bool has_platform_or_app_classes() const { return _has_platform_or_app_classes; }
bool has_aot_linked_classes() const { return _has_aot_linked_classes; }
bool compressed_oops() const { return _compressed_oops; }
- bool compressed_class_pointers() const { return _compressed_class_ptrs; }
int narrow_klass_pointer_bits() const { return _narrow_klass_pointer_bits; }
int narrow_klass_shift() const { return _narrow_klass_shift; }
bool has_full_module_graph() const { return _has_full_module_graph; }
diff --git a/src/hotspot/share/cds/heapShared.cpp b/src/hotspot/share/cds/heapShared.cpp
index 0c0f70eac0a..d75816656b0 100644
--- a/src/hotspot/share/cds/heapShared.cpp
+++ b/src/hotspot/share/cds/heapShared.cpp
@@ -112,6 +112,11 @@ static Klass* _test_class = nullptr;
static const ArchivedKlassSubGraphInfoRecord* _test_class_record = nullptr;
#endif
+#ifdef ASSERT
+// All classes that have at least one instance in the cached heap.
+static ArchivableKlassTable* _dumptime_classes_with_cached_oops = nullptr;
+static Array* _runtime_classes_with_cached_oops = nullptr;
+#endif
//
// If you add new entries to the following tables, you should know what you're doing!
@@ -131,17 +136,14 @@ static ArchivableStaticFieldInfo archive_subgraph_entry_fields[] = {
{"java/lang/module/Configuration", "EMPTY_CONFIGURATION"},
{"jdk/internal/math/FDBigInteger", "archivedCaches"},
-#ifndef PRODUCT
- {nullptr, nullptr}, // Extra slot for -XX:ArchiveHeapTestClass
-#endif
- {nullptr, nullptr},
-};
-
-// full module graph
-static ArchivableStaticFieldInfo fmg_archive_subgraph_entry_fields[] = {
+ // full module graph support
{"jdk/internal/loader/ArchivedClassLoaders", "archivedClassLoaders"},
{ARCHIVED_BOOT_LAYER_CLASS, ARCHIVED_BOOT_LAYER_FIELD},
{"java/lang/Module$ArchivedData", "archivedData"},
+
+#ifndef PRODUCT
+ {nullptr, nullptr}, // Extra slot for -XX:ArchiveHeapTestClass
+#endif
{nullptr, nullptr},
};
@@ -164,8 +166,7 @@ bool HeapShared::is_subgraph_root_class(InstanceKlass* ik) {
assert(CDSConfig::is_dumping_heap(), "dump-time only");
if (CDSConfig::is_dumping_klass_subgraphs()) {
// Legacy CDS archive support (to be deprecated)
- return is_subgraph_root_class_of(archive_subgraph_entry_fields, ik) ||
- is_subgraph_root_class_of(fmg_archive_subgraph_entry_fields, ik);
+ return is_subgraph_root_class_of(archive_subgraph_entry_fields, ik);
} else {
return false;
}
@@ -175,23 +176,39 @@ oop HeapShared::CachedOopInfo::orig_referrer() const {
return _orig_referrer.resolve();
}
-unsigned HeapShared::oop_hash(oop const& p) {
+// This is a simple hashing of the oop's address. This function is used
+// while copying the oops into the AOT heap region. We don't want to
+// have any side effects during the copying, so we avoid calling
+// p->identity_hash() which can update the object header.
+unsigned HeapShared::oop_address_hash(oop const& p) {
assert(SafepointSynchronize::is_at_safepoint() ||
JavaThread::current()->is_in_no_safepoint_scope(), "sanity");
- // Do not call p->identity_hash() as that will update the
- // object header.
return primitive_hash(cast_from_oop(p));
}
-unsigned int HeapShared::oop_handle_hash_raw(const OopHandle& oh) {
- return oop_hash(oh.resolve());
-}
-
-unsigned int HeapShared::oop_handle_hash(const OopHandle& oh) {
+// About the hashcode in the cached objects:
+// - If a source object has a hashcode, it must be copied into the cache.
+// That's because some cached hashtables are laid out using this hashcode.
+// - If a source object doesn't have a hashcode, we avoid computing it while
+// copying the objects into the cache. This will allow the hashcode to be
+// dynamically and randomly computed in each production, which generally
+// desirable to make the hashcodes more random between runs.
+unsigned HeapShared::archived_object_cache_hash(OopHandle const& oh) {
oop o = oh.resolve();
if (o == nullptr) {
return 0;
+ }
+ if (!_use_identity_hash_for_archived_object_cache) {
+ // This is called while we are copying the objects. Don't call o->identity_hash()
+ // as that will update the object header.
+ return oop_address_hash(o);
} else {
+ // This is called after all objects are copied. It's OK to update
+ // the object's hashcode.
+ //
+ // This may be called after we have left the AOT dumping safepoint.
+ // Objects in archived_object_cache() may be moved by the GC, so we
+ // can't use the address of o for computing the hash.
return o->identity_hash();
}
}
@@ -271,6 +288,12 @@ void HeapShared::prepare_for_archiving(TRAPS) {
HeapShared::ArchivedObjectCache* HeapShared::_archived_object_cache = nullptr;
+// Controls the hashing method for the _archived_object_cache.
+// Changes from false to true once, after all objects are copied,
+// inside make_archived_object_cache_gc_safe().
+// See archived_object_cache_hash() for more details.
+bool HeapShared::_use_identity_hash_for_archived_object_cache = false;
+
bool HeapShared::is_archived_heap_in_use() {
if (HeapShared::is_loading()) {
if (HeapShared::is_loading_streaming_mode()) {
@@ -373,6 +396,21 @@ void HeapShared::initialize_streaming() {
}
void HeapShared::enable_gc() {
+#ifdef ASSERT
+ // At this point, a GC may start and will be able to see some or all
+ // of the cached oops. The class of each oop seen by the GC must have
+ // already been loaded. One function with such a requirement is
+ // ClaimMetadataVisitingOopIterateClosure::do_klass().
+ if (is_archived_heap_in_use()) {
+ Array* klasses = _runtime_classes_with_cached_oops;
+
+ for (int i = 0; i < klasses->length(); i++) {
+ assert(klasses->at(i)->class_loader_data() != nullptr,
+ "class of cached oop must have been loaded");
+ }
+ }
+#endif
+
if (AOTStreamedHeapLoader::is_in_use()) {
AOTStreamedHeapLoader::enable_gc();
}
@@ -384,9 +422,8 @@ void HeapShared::materialize_thread_object() {
}
}
-void HeapShared::add_to_dumped_interned_strings(oop string) {
+void HeapShared::archive_interned_string(oop string) {
assert(HeapShared::is_writing_mapping_mode(), "Only used by this mode");
- AOTMappedHeapWriter::add_to_dumped_interned_strings(string);
bool success = archive_reachable_objects_from(1, _dump_time_special_subgraph, string);
assert(success, "shared strings array must not point to arrays or strings that are too large to archive");
}
@@ -404,6 +441,24 @@ void HeapShared::finalize_initialization(FileMapInfo* static_mapinfo) {
}
}
+void HeapShared::make_archived_object_cache_gc_safe() {
+ ArchivedObjectCache* new_cache = new (mtClass)ArchivedObjectCache(INITIAL_TABLE_SIZE, MAX_TABLE_SIZE);
+
+ // It's safe to change the behavior of the hash function now, because iterate_all()
+ // doesn't call the hash function.
+ // See archived_object_cache_hash() for more details.
+ assert(_use_identity_hash_for_archived_object_cache == false, "happens only once");
+ _use_identity_hash_for_archived_object_cache = true;
+
+ // Copy all CachedOopInfo into a new table using a different hashing algorithm
+ archived_object_cache()->iterate_all([&] (OopHandle oh, CachedOopInfo info) {
+ new_cache->put_when_absent(oh, info);
+ });
+
+ destroy_archived_object_cache();
+ _archived_object_cache = new_cache;
+}
+
HeapShared::CachedOopInfo* HeapShared::get_cached_oop_info(oop obj) {
OopHandle oh(Universe::vm_global(), obj);
CachedOopInfo* result = _archived_object_cache->get(oh);
@@ -417,14 +472,53 @@ bool HeapShared::has_been_archived(oop obj) {
}
int HeapShared::append_root(oop obj) {
+ assert(SafepointSynchronize::is_at_safepoint(), "sanity");
assert(CDSConfig::is_dumping_heap(), "dump-time only");
- if (obj != nullptr) {
- assert(has_been_archived(obj), "must be");
- }
- // No GC should happen since we aren't scanning _pending_roots.
- assert(Thread::current() == (Thread*)VMThread::vm_thread(), "should be in vm thread");
+ assert(_pending_roots != nullptr, "sanity");
- return _pending_roots->append(obj);
+ if (obj == nullptr) {
+ assert(_pending_roots->at(0) == nullptr, "root index 0 always maps to null");
+ return 0;
+ } else if (CDSConfig::is_dumping_aot_linked_classes()) {
+ // The AOT compiler may refer the same obj many times, so we
+ // should use the same index for this oop to avoid excessive entries
+ // in the roots array.
+ CachedOopInfo* obj_info = get_cached_oop_info(obj);
+ assert(obj_info != nullptr, "must be archived");
+
+ if (obj_info->root_index() > 0) {
+ return obj_info->root_index();
+ } else {
+ assert(obj_info->root_index() < 0, "must not be zero");
+ int i = _pending_roots->append(obj);
+ obj_info->set_root_index(i);
+ return i;
+ }
+ } else {
+ return _pending_roots->append(obj);
+ }
+}
+
+int HeapShared::get_root_index(oop obj) {
+ if (java_lang_Class::is_instance(obj)) {
+ obj = scratch_java_mirror(obj);
+ }
+
+ CachedOopInfo* obj_info = get_cached_oop_info(obj);
+ const char* error = nullptr;
+ if (obj_info == nullptr) {
+ error = "Not a cached oop";
+ } else if (obj_info->root_index() < 0) {
+ error = "Not a cached oop root";
+ } else {
+ return obj_info->root_index();
+ }
+
+ ResourceMark rm;
+ log_debug(aot, codecache, oops)("%s: " INTPTR_FORMAT " (%s)", error,
+ cast_from_oop(obj),
+ obj->klass()->external_name());
+ return -1;
}
oop HeapShared::get_root(int index, bool clear) {
@@ -453,6 +547,13 @@ void HeapShared::finish_materialize_objects() {
}
void HeapShared::clear_root(int index) {
+ if (CDSConfig::is_using_aot_linked_classes()) {
+ // When AOT linked classes are in use, all roots will be in use all
+ // the time, there's no benefit for clearing the roots. Also, we
+ // can't clear the roots as they can be shared.
+ return;
+ }
+
assert(index >= 0, "sanity");
assert(CDSConfig::is_using_archive(), "must be");
if (is_archived_heap_in_use()) {
@@ -483,8 +584,10 @@ bool HeapShared::archive_object(oop obj, oop referrer, KlassSubGraphInfo* subgra
return false;
}
+ AOTArtifactFinder::add_cached_class(obj->klass());
AOTOopChecker::check(obj); // Make sure contents of this oop are safe.
count_allocation(obj->size());
+ DEBUG_ONLY(_dumptime_classes_with_cached_oops->add(obj->klass()));
if (HeapShared::is_writing_streaming_mode()) {
AOTStreamedHeapWriter::add_source_obj(obj);
@@ -586,11 +689,6 @@ public:
};
void HeapShared::add_scratch_resolved_references(ConstantPool* src, objArrayOop dest) {
- if (CDSConfig::is_dumping_preimage_static_archive() && scratch_resolved_references(src) != nullptr) {
- // We are in AOT training run. The class has been redefined and we are giving it a new resolved_reference.
- // Ignore it, as this class will be excluded from the AOT config.
- return;
- }
if (SystemDictionaryShared::is_builtin_loader(src->pool_holder()->class_loader_data())) {
_scratch_objects_table->set_oop(src, dest);
}
@@ -600,9 +698,17 @@ objArrayOop HeapShared::scratch_resolved_references(ConstantPool* src) {
return (objArrayOop)_scratch_objects_table->get_oop(src);
}
- void HeapShared::init_dumping() {
- _scratch_objects_table = new (mtClass)MetaspaceObjToOopHandleTable();
- _pending_roots = new GrowableArrayCHeap(500);
+void HeapShared::remove_scratch_resolved_references(ConstantPool* src) {
+ if (CDSConfig::is_dumping_heap()) {
+ _scratch_objects_table->remove_oop(src);
+ }
+}
+
+void HeapShared::init_dumping() {
+ _scratch_objects_table = new (mtClass)MetaspaceObjToOopHandleTable();
+ _pending_roots = new GrowableArrayCHeap(500);
+ _pending_roots->append(nullptr); // root index 0 represents a null oop
+ DEBUG_ONLY(_dumptime_classes_with_cached_oops = new (mtClassShared)ArchivableKlassTable());
}
void HeapShared::init_scratch_objects_for_basic_type_mirrors(TRAPS) {
@@ -839,7 +945,7 @@ void HeapShared::start_scanning_for_oops() {
// The special subgraph doesn't belong to any class. We use Object_klass() here just
// for convenience.
- _dump_time_special_subgraph = init_subgraph_info(vmClasses::Object_klass(), false);
+ _dump_time_special_subgraph = init_subgraph_info(vmClasses::Object_klass());
// Cache for recording where the archived objects are copied to
create_archived_object_cache();
@@ -883,6 +989,13 @@ void HeapShared::write_heap(AOTMappedHeapInfo* mapped_heap_info, AOTStreamedHeap
ArchiveBuilder::OtherROAllocMark mark;
write_subgraph_info_table();
+
+ DEBUG_ONLY(_runtime_classes_with_cached_oops = _dumptime_classes_with_cached_oops->write_ordered_array());
+
+ delete _pending_roots;
+ _pending_roots = nullptr;
+
+ make_archived_object_cache_gc_safe();
}
void HeapShared::scan_java_mirror(oop orig_mirror) {
@@ -912,12 +1025,7 @@ void HeapShared::archive_subgraphs() {
assert(CDSConfig::is_dumping_heap(), "must be");
if (CDSConfig::is_dumping_klass_subgraphs()) {
- archive_object_subgraphs(archive_subgraph_entry_fields,
- false /* is_full_module_graph */);
- if (CDSConfig::is_dumping_full_module_graph()) {
- archive_object_subgraphs(fmg_archive_subgraph_entry_fields,
- true /* is_full_module_graph */);
- }
+ archive_object_subgraphs(archive_subgraph_entry_fields);
}
}
@@ -930,12 +1038,11 @@ HeapShared::RunTimeKlassSubGraphInfoTable HeapShared::_run_time_subgraph_info_
// Get the subgraph_info for Klass k. A new subgraph_info is created if
// there is no existing one for k. The subgraph_info records the "buffered"
// address of the class.
-KlassSubGraphInfo* HeapShared::init_subgraph_info(Klass* k, bool is_full_module_graph) {
+KlassSubGraphInfo* HeapShared::init_subgraph_info(Klass* k) {
assert(CDSConfig::is_dumping_heap(), "dump time only");
bool created;
KlassSubGraphInfo* info =
- _dump_time_subgraph_info_table->put_if_absent(k, KlassSubGraphInfo(k, is_full_module_graph),
- &created);
+ _dump_time_subgraph_info_table->put_if_absent(k, KlassSubGraphInfo(k), &created);
assert(created, "must not initialize twice");
return info;
}
@@ -1023,7 +1130,6 @@ void KlassSubGraphInfo::add_subgraph_object_klass(Klass* orig_k) {
}
_subgraph_object_klasses->append_if_missing(orig_k);
- _has_non_early_klasses |= is_non_early_klass(orig_k);
}
void KlassSubGraphInfo::check_allowed_klass(InstanceKlass* ik) {
@@ -1066,45 +1172,11 @@ void KlassSubGraphInfo::check_allowed_klass(InstanceKlass* ik) {
AOTMetaspace::unrecoverable_writing_error();
}
-bool KlassSubGraphInfo::is_non_early_klass(Klass* k) {
- if (k->is_objArray_klass()) {
- k = ObjArrayKlass::cast(k)->bottom_klass();
- }
- if (k->is_instance_klass()) {
- if (!SystemDictionaryShared::is_early_klass(InstanceKlass::cast(k))) {
- ResourceMark rm;
- log_info(aot, heap)("non-early: %s", k->external_name());
- return true;
- } else {
- return false;
- }
- } else {
- return false;
- }
-}
-
// Initialize an archived subgraph_info_record from the given KlassSubGraphInfo.
void ArchivedKlassSubGraphInfoRecord::init(KlassSubGraphInfo* info) {
_k = ArchiveBuilder::get_buffered_klass(info->klass());
_entry_field_records = nullptr;
_subgraph_object_klasses = nullptr;
- _is_full_module_graph = info->is_full_module_graph();
-
- if (_is_full_module_graph) {
- // Consider all classes referenced by the full module graph as early -- we will be
- // allocating objects of these classes during JVMTI early phase, so they cannot
- // be processed by (non-early) JVMTI ClassFileLoadHook
- _has_non_early_klasses = false;
- } else {
- _has_non_early_klasses = info->has_non_early_klasses();
- }
-
- if (_has_non_early_klasses) {
- ResourceMark rm;
- log_info(aot, heap)(
- "Subgraph of klass %s has non-early klasses and cannot be used when JVMTI ClassFileLoadHook is enabled",
- _k->external_name());
- }
// populate the entry fields
GrowableArray* entry_fields = info->subgraph_entry_fields();
@@ -1231,6 +1303,7 @@ void HeapShared::serialize_tables(SerializeClosure* soc) {
_run_time_subgraph_info_table.serialize_header(soc);
soc->do_ptr(&_run_time_special_subgraph);
+ DEBUG_ONLY(soc->do_ptr(&_runtime_classes_with_cached_oops));
}
static void verify_the_heap(Klass* k, const char* which) {
@@ -1262,15 +1335,10 @@ static void verify_the_heap(Klass* k, const char* which) {
// Before GC can execute, we must ensure that all oops reachable from HeapShared::roots()
// have a valid klass. I.e., oopDesc::klass() must have already been resolved.
-//
-// Note: if a ArchivedKlassSubGraphInfoRecord contains non-early classes, and JVMTI
-// ClassFileLoadHook is enabled, it's possible for this class to be dynamically replaced. In
-// this case, we will not load the ArchivedKlassSubGraphInfoRecord and will clear its roots.
void HeapShared::resolve_classes(JavaThread* current) {
assert(CDSConfig::is_using_archive(), "runtime only!");
if (CDSConfig::is_using_klass_subgraphs()) {
resolve_classes_for_subgraphs(current, archive_subgraph_entry_fields);
- resolve_classes_for_subgraphs(current, fmg_archive_subgraph_entry_fields);
}
}
@@ -1418,24 +1486,6 @@ HeapShared::resolve_or_init_classes_for_subgraph_of(Klass* k, bool do_init, TRAP
}
return nullptr;
} else {
- if (record->is_full_module_graph() && !CDSConfig::is_using_full_module_graph()) {
- if (log_is_enabled(Info, aot, heap)) {
- ResourceMark rm(THREAD);
- log_info(aot, heap)("subgraph %s cannot be used because full module graph is disabled",
- k->external_name());
- }
- return nullptr;
- }
-
- if (record->has_non_early_klasses() && JvmtiExport::should_post_class_file_load_hook()) {
- if (log_is_enabled(Info, aot, heap)) {
- ResourceMark rm(THREAD);
- log_info(aot, heap)("subgraph %s cannot be used because JVMTI ClassFileLoadHook is enabled",
- k->external_name());
- }
- return nullptr;
- }
-
if (log_is_enabled(Info, aot, heap)) {
ResourceMark rm;
log_info(aot, heap)("%s subgraph %s ", do_init ? "init" : "resolve", k->external_name());
@@ -1517,8 +1567,8 @@ void HeapShared::init_archived_fields_for(Klass* k, const ArchivedKlassSubGraphI
// mirror after this point.
if (log_is_enabled(Info, aot, heap)) {
ResourceMark rm;
- log_info(aot, heap)("initialize_from_archived_subgraph %s " PTR_FORMAT "%s%s",
- k->external_name(), p2i(k), JvmtiExport::is_early_phase() ? " (early)" : "",
+ log_info(aot, heap)("initialize_from_archived_subgraph %s " PTR_FORMAT "%s",
+ k->external_name(), p2i(k),
k->has_aot_initialized_mirror() ? " (aot-inited)" : "");
}
}
@@ -1911,6 +1961,11 @@ void HeapShared::verify_subgraph_from(oop orig_obj) {
void HeapShared::verify_reachable_objects_from(oop obj) {
_num_total_verifications ++;
if (java_lang_Class::is_instance(obj)) {
+ Klass* k = java_lang_Class::as_Klass(obj);
+ if (RegeneratedClasses::has_been_regenerated(k)) {
+ k = RegeneratedClasses::get_regenerated_object(k);
+ obj = k->java_mirror();
+ }
obj = scratch_java_mirror(obj);
assert(obj != nullptr, "must be");
}
@@ -1979,9 +2034,9 @@ void HeapShared::set_has_been_seen_during_subgraph_recording(oop obj) {
++ _num_new_walked_objs;
}
-void HeapShared::start_recording_subgraph(InstanceKlass *k, const char* class_name, bool is_full_module_graph) {
+void HeapShared::start_recording_subgraph(InstanceKlass *k, const char* class_name) {
log_info(aot, heap)("Start recording subgraph(s) for archived fields in %s", class_name);
- init_subgraph_info(k, is_full_module_graph);
+ init_subgraph_info(k);
init_seen_objects_table();
_num_new_walked_objs = 0;
_num_new_archived_objs = 0;
@@ -2113,9 +2168,6 @@ void HeapShared::init_subgraph_entry_fields(TRAPS) {
_dump_time_subgraph_info_table = new (mtClass)DumpTimeKlassSubGraphInfoTable();
if (CDSConfig::is_dumping_klass_subgraphs()) {
init_subgraph_entry_fields(archive_subgraph_entry_fields, CHECK);
- if (CDSConfig::is_dumping_full_module_graph()) {
- init_subgraph_entry_fields(fmg_archive_subgraph_entry_fields, CHECK);
- }
}
}
@@ -2214,8 +2266,7 @@ void HeapShared::init_heap_writer() {
}
}
-void HeapShared::archive_object_subgraphs(ArchivableStaticFieldInfo fields[],
- bool is_full_module_graph) {
+void HeapShared::archive_object_subgraphs(ArchivableStaticFieldInfo fields[]) {
_num_total_subgraph_recordings = 0;
_num_total_walked_objs = 0;
_num_total_archived_objs = 0;
@@ -2231,7 +2282,7 @@ void HeapShared::archive_object_subgraphs(ArchivableStaticFieldInfo fields[],
for (int i = 0; fields[i].valid(); ) {
ArchivableStaticFieldInfo* info = &fields[i];
const char* klass_name = info->klass_name;
- start_recording_subgraph(info->klass, klass_name, is_full_module_graph);
+ start_recording_subgraph(info->klass, klass_name);
// If you have specified consecutive fields of the same klass in
// fields[], these will be archived in the same
@@ -2264,12 +2315,22 @@ void HeapShared::archive_object_subgraphs(ArchivableStaticFieldInfo fields[],
#endif
}
-bool HeapShared::is_dumped_interned_string(oop o) {
- if (is_writing_mapping_mode()) {
- return AOTMappedHeapWriter::is_dumped_interned_string(o);
- } else {
- return AOTStreamedHeapWriter::is_dumped_interned_string(o);
+bool HeapShared::is_interned_string(oop obj) {
+ if (!java_lang_String::is_instance(obj)) {
+ return false;
}
+
+ ResourceMark rm;
+ int len = 0;
+ jchar* name = java_lang_String::as_unicode_string_or_null(obj, len);
+ if (name == nullptr) {
+ fatal("Insufficient memory for dumping");
+ }
+ return StringTable::lookup(name, len) == obj;
+}
+
+bool HeapShared::is_dumped_interned_string(oop o) {
+ return is_interned_string(o) && has_been_archived(o);
}
// These tables should be used only within the CDS safepoint, so
diff --git a/src/hotspot/share/cds/heapShared.hpp b/src/hotspot/share/cds/heapShared.hpp
index 2cb330160e4..10ea35ab56e 100644
--- a/src/hotspot/share/cds/heapShared.hpp
+++ b/src/hotspot/share/cds/heapShared.hpp
@@ -40,7 +40,6 @@
#include "utilities/hashTable.hpp"
#if INCLUDE_CDS_JAVA_HEAP
-class DumpedInternedStrings;
class FileMapInfo;
class KlassSubGraphInfo;
class MetaspaceObjToOopHandleTable;
@@ -67,21 +66,12 @@ class KlassSubGraphInfo: public CHeapObj {
// For each entry field, it is a tuple of field_offset, field_value
GrowableArray* _subgraph_entry_fields;
- // Does this KlassSubGraphInfo belong to the archived full module graph
- bool _is_full_module_graph;
-
- // Does this KlassSubGraphInfo references any classes that were loaded while
- // JvmtiExport::is_early_phase()!=true. If so, this KlassSubGraphInfo cannot be
- // used at runtime if JVMTI ClassFileLoadHook is enabled.
- bool _has_non_early_klasses;
static bool is_non_early_klass(Klass* k);
static void check_allowed_klass(InstanceKlass* ik);
public:
- KlassSubGraphInfo(Klass* k, bool is_full_module_graph) :
+ KlassSubGraphInfo(Klass* k) :
_k(k), _subgraph_object_klasses(nullptr),
- _subgraph_entry_fields(nullptr),
- _is_full_module_graph(is_full_module_graph),
- _has_non_early_klasses(false) {}
+ _subgraph_entry_fields(nullptr) {}
~KlassSubGraphInfo() {
if (_subgraph_object_klasses != nullptr) {
@@ -105,8 +95,6 @@ class KlassSubGraphInfo: public CHeapObj {
return _subgraph_object_klasses == nullptr ? 0 :
_subgraph_object_klasses->length();
}
- bool is_full_module_graph() const { return _is_full_module_graph; }
- bool has_non_early_klasses() const { return _has_non_early_klasses; }
};
// An archived record of object sub-graphs reachable from static
@@ -115,7 +103,6 @@ class KlassSubGraphInfo: public CHeapObj {
class ArchivedKlassSubGraphInfoRecord {
private:
Klass* _k;
- bool _is_full_module_graph;
bool _has_non_early_klasses;
// contains pairs of field offset and value for each subgraph entry field
@@ -131,7 +118,6 @@ class ArchivedKlassSubGraphInfoRecord {
Klass* klass() const { return _k; }
Array* entry_field_records() const { return _entry_field_records; }
Array* subgraph_object_klasses() const { return _subgraph_object_klasses; }
- bool is_full_module_graph() const { return _is_full_module_graph; }
bool has_non_early_klasses() const { return _has_non_early_klasses; }
};
#endif // INCLUDE_CDS_JAVA_HEAP
@@ -176,7 +162,7 @@ public:
static void initialize_streaming() NOT_CDS_JAVA_HEAP_RETURN;
static void enable_gc() NOT_CDS_JAVA_HEAP_RETURN;
static void materialize_thread_object() NOT_CDS_JAVA_HEAP_RETURN;
- static void add_to_dumped_interned_strings(oop string) NOT_CDS_JAVA_HEAP_RETURN;
+ static void archive_interned_string(oop string);
static void finalize_initialization(FileMapInfo* static_mapinfo) NOT_CDS_JAVA_HEAP_RETURN;
private:
@@ -195,13 +181,8 @@ private:
static void print_stats();
public:
static void debug_trace();
- static unsigned oop_hash(oop const& p);
- static unsigned oop_handle_hash(OopHandle const& oh);
- static unsigned oop_handle_hash_raw(OopHandle const& oh);
+ static unsigned oop_address_hash(oop const& p);
static bool oop_handle_equals(const OopHandle& a, const OopHandle& b);
- static unsigned string_oop_hash(oop const& string) {
- return java_lang_String::hash_code(string);
- }
class CopyKlassSubGraphInfoToArchive;
@@ -217,27 +198,37 @@ public:
// One or more fields in this object are pointing to MetaspaceObj
bool _has_native_pointers;
+
+ // >= 0 if this oop has been append to the list of roots
+ int _root_index;
public:
CachedOopInfo(OopHandle orig_referrer, bool has_oop_pointers)
: _orig_referrer(orig_referrer),
_buffer_offset(0),
_has_oop_pointers(has_oop_pointers),
- _has_native_pointers(false) {}
+ _has_native_pointers(false),
+ _root_index(-1) {}
oop orig_referrer() const;
void set_buffer_offset(size_t offset) { _buffer_offset = offset; }
size_t buffer_offset() const { return _buffer_offset; }
bool has_oop_pointers() const { return _has_oop_pointers; }
bool has_native_pointers() const { return _has_native_pointers; }
void set_has_native_pointers() { _has_native_pointers = true; }
+ int root_index() const { return _root_index; }
+ void set_root_index(int i) { _root_index = i; }
};
private:
static const int INITIAL_TABLE_SIZE = 15889; // prime number
static const int MAX_TABLE_SIZE = 1000000;
+ static bool _use_identity_hash_for_archived_object_cache;
+
+ static unsigned archived_object_cache_hash(OopHandle const& oh);
+
typedef ResizeableHashTable ArchivedObjectCache;
static ArchivedObjectCache* _archived_object_cache;
@@ -266,8 +257,7 @@ private:
static CachedOopInfo make_cached_oop_info(oop obj, oop referrer);
static ArchivedKlassSubGraphInfoRecord* archive_subgraph_info(KlassSubGraphInfo* info);
- static void archive_object_subgraphs(ArchivableStaticFieldInfo fields[],
- bool is_full_module_graph);
+ static void archive_object_subgraphs(ArchivableStaticFieldInfo fields[]);
// Archive object sub-graph starting from the given static field
// in Klass k's mirror.
@@ -281,7 +271,7 @@ private:
static void verify_subgraph_from(oop orig_obj) PRODUCT_RETURN;
static void check_special_subgraph_classes();
- static KlassSubGraphInfo* init_subgraph_info(Klass *k, bool is_full_module_graph);
+ static KlassSubGraphInfo* init_subgraph_info(Klass *k);
static KlassSubGraphInfo* get_subgraph_info(Klass *k);
static void init_subgraph_entry_fields(TRAPS) NOT_CDS_JAVA_HEAP_RETURN;
@@ -297,7 +287,7 @@ private:
typedef ResizeableHashTable SeenObjectsTable;
+ HeapShared::oop_address_hash> SeenObjectsTable;
static SeenObjectsTable *_seen_objects_table;
@@ -336,8 +326,7 @@ private:
static size_t _num_total_recorded_klasses;
static size_t _num_total_verifications;
- static void start_recording_subgraph(InstanceKlass *k, const char* klass_name,
- bool is_full_module_graph);
+ static void start_recording_subgraph(InstanceKlass *k, const char* klass_name);
static void done_recording_subgraph(InstanceKlass *k, const char* klass_name);
static bool has_been_seen_during_subgraph_recording(oop obj);
@@ -394,6 +383,7 @@ private:
delete _archived_object_cache;
_archived_object_cache = nullptr;
}
+ static void make_archived_object_cache_gc_safe();
static ArchivedObjectCache* archived_object_cache() {
return _archived_object_cache;
}
@@ -406,6 +396,7 @@ private:
KlassSubGraphInfo* subgraph_info,
oop orig_obj);
+ static bool is_interned_string(oop obj);
static bool is_dumped_interned_string(oop o);
// Scratch objects for archiving Klass::java_mirror()
@@ -437,6 +428,11 @@ private:
// Dump-time only. Returns the index of the root, which can be used at run time to read
// the root using get_root(index, ...).
static int append_root(oop obj);
+
+ // AOT-compile time only.
+ // Returns -1 if obj is not in the heap root set.
+ static int get_root_index(oop obj) NOT_CDS_JAVA_HEAP_RETURN_(-1);
+
static GrowableArrayCHeap* pending_roots() { return _pending_roots; }
// Dump-time and runtime
@@ -445,9 +441,7 @@ private:
// Run-time only
static void clear_root(int index);
-
static void get_segment_indexes(int index, int& segment_index, int& internal_index);
-
static void setup_test_class(const char* test_class_name) PRODUCT_RETURN;
#endif // INCLUDE_CDS_JAVA_HEAP
@@ -457,6 +451,7 @@ private:
static void write_heap(AOTMappedHeapInfo* mapped_heap_info, AOTStreamedHeapInfo* streamed_heap_info) NOT_CDS_JAVA_HEAP_RETURN;
static objArrayOop scratch_resolved_references(ConstantPool* src);
static void add_scratch_resolved_references(ConstantPool* src, objArrayOop dest) NOT_CDS_JAVA_HEAP_RETURN;
+ static void remove_scratch_resolved_references(ConstantPool* src) NOT_CDS_JAVA_HEAP_RETURN;
static void init_dumping() NOT_CDS_JAVA_HEAP_RETURN;
static void init_scratch_objects_for_basic_type_mirrors(TRAPS) NOT_CDS_JAVA_HEAP_RETURN;
static void init_box_classes(TRAPS) NOT_CDS_JAVA_HEAP_RETURN;
diff --git a/src/hotspot/share/classfile/classFileParser.cpp b/src/hotspot/share/classfile/classFileParser.cpp
index c1f00cbe536..a9ea6fbea11 100644
--- a/src/hotspot/share/classfile/classFileParser.cpp
+++ b/src/hotspot/share/classfile/classFileParser.cpp
@@ -34,6 +34,7 @@
#include "classfile/packageEntry.hpp"
#include "classfile/symbolTable.hpp"
#include "classfile/systemDictionary.hpp"
+#include "classfile/systemDictionaryShared.hpp"
#include "classfile/verificationType.hpp"
#include "classfile/verifier.hpp"
#include "classfile/vmClasses.hpp"
@@ -86,9 +87,6 @@
#include "utilities/macros.hpp"
#include "utilities/ostream.hpp"
#include "utilities/utf8.hpp"
-#if INCLUDE_CDS
-#include "classfile/systemDictionaryShared.hpp"
-#endif
// We generally try to create the oops directly when parsing, rather than
// allocating temporary data structures and copying the bytes twice. A
@@ -194,7 +192,7 @@ void ClassFileParser::parse_constant_pool_entries(const ClassFileStream* const s
// so we don't need bounds-check for reading tag.
const u1 tag = cfs->get_u1_fast();
switch (tag) {
- case JVM_CONSTANT_Class : {
+ case JVM_CONSTANT_Class: {
cfs->guarantee_more(3, CHECK); // name_index, tag/access_flags
const u2 name_index = cfs->get_u2_fast();
cp->klass_index_at_put(index, name_index);
@@ -4403,14 +4401,14 @@ void ClassFileParser::verify_legal_field_modifiers(jint flags,
TRAPS) const {
if (!_need_verify) { return; }
- const bool is_public = (flags & JVM_ACC_PUBLIC) != 0;
- const bool is_protected = (flags & JVM_ACC_PROTECTED) != 0;
- const bool is_private = (flags & JVM_ACC_PRIVATE) != 0;
- const bool is_static = (flags & JVM_ACC_STATIC) != 0;
- const bool is_final = (flags & JVM_ACC_FINAL) != 0;
- const bool is_volatile = (flags & JVM_ACC_VOLATILE) != 0;
- const bool is_transient = (flags & JVM_ACC_TRANSIENT) != 0;
- const bool is_enum = (flags & JVM_ACC_ENUM) != 0;
+ const bool is_public = (flags & JVM_ACC_PUBLIC) != 0;
+ const bool is_protected = (flags & JVM_ACC_PROTECTED) != 0;
+ const bool is_private = (flags & JVM_ACC_PRIVATE) != 0;
+ const bool is_static = (flags & JVM_ACC_STATIC) != 0;
+ const bool is_final = (flags & JVM_ACC_FINAL) != 0;
+ const bool is_volatile = (flags & JVM_ACC_VOLATILE) != 0;
+ const bool is_transient = (flags & JVM_ACC_TRANSIENT) != 0;
+ const bool is_enum = (flags & JVM_ACC_ENUM) != 0;
const bool major_gte_1_5 = _major_version >= JAVA_1_5_VERSION;
bool is_illegal = false;
@@ -5256,6 +5254,9 @@ void ClassFileParser::fill_instance_klass(InstanceKlass* ik,
if (!is_internal()) {
ik->print_class_load_logging(_loader_data, module_entry, _stream);
+ if (CDSConfig::is_dumping_archive()) {
+ SystemDictionaryShared::check_code_source(ik, _stream);
+ }
if (ik->minor_version() == JAVA_PREVIEW_MINOR_VERSION &&
ik->major_version() == JVM_CLASSFILE_MAJOR_VERSION &&
diff --git a/src/hotspot/share/classfile/classLoaderData.cpp b/src/hotspot/share/classfile/classLoaderData.cpp
index dfc3b74db96..d1ea9c09d4c 100644
--- a/src/hotspot/share/classfile/classLoaderData.cpp
+++ b/src/hotspot/share/classfile/classLoaderData.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2012, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -46,6 +46,7 @@
// The bootstrap loader (represented by null) also has a ClassLoaderData,
// the singleton class the_null_class_loader_data().
+#include "cds/heapShared.hpp"
#include "classfile/classLoaderData.inline.hpp"
#include "classfile/classLoaderDataGraph.inline.hpp"
#include "classfile/dictionary.hpp"
@@ -899,6 +900,7 @@ void ClassLoaderData::free_deallocate_list() {
if (m->is_method()) {
MetadataFactory::free_metadata(this, (Method*)m);
} else if (m->is_constantPool()) {
+ HeapShared::remove_scratch_resolved_references((ConstantPool*)m);
MetadataFactory::free_metadata(this, (ConstantPool*)m);
} else if (m->is_klass()) {
MetadataFactory::free_metadata(this, (InstanceKlass*)m);
diff --git a/src/hotspot/share/classfile/classPrinter.cpp b/src/hotspot/share/classfile/classPrinter.cpp
index 3ed0a5e9840..6cf89f7357f 100644
--- a/src/hotspot/share/classfile/classPrinter.cpp
+++ b/src/hotspot/share/classfile/classPrinter.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2022, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -224,6 +224,7 @@ void ClassPrinter::print_flags_help(outputStream* os) {
os->print_cr(" 0x%02x - print info for invokehandle", PRINT_METHOD_HANDLE);
os->print_cr(" 0x%02x - print details of the C++ and Java objects that represent classes", PRINT_CLASS_DETAILS);
os->print_cr(" 0x%02x - print details of the C++ objects that represent methods", PRINT_METHOD_DETAILS);
+ os->print_cr(" 0x%02x - print MethodData", PRINT_METHOD_DATA);
os->cr();
}
diff --git a/src/hotspot/share/classfile/classPrinter.hpp b/src/hotspot/share/classfile/classPrinter.hpp
index 470e82ddc0e..b09a1a1ef3b 100644
--- a/src/hotspot/share/classfile/classPrinter.hpp
+++ b/src/hotspot/share/classfile/classPrinter.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2022, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -54,6 +54,7 @@ public:
PRINT_METHOD_HANDLE = 1 << 4, // extra information for invokehandle
PRINT_CLASS_DETAILS = 1 << 5, // print details of the C++ and Java objects that represent classes
PRINT_METHOD_DETAILS = 1 << 6, // print details of the C++ objects that represent methods
+ PRINT_METHOD_DATA = 1 << 7, // print MethodData - requires MDO lock
};
static bool has_mode(int flags, Mode mode) {
return (flags & static_cast(mode)) != 0;
diff --git a/src/hotspot/share/classfile/fieldLayoutBuilder.cpp b/src/hotspot/share/classfile/fieldLayoutBuilder.cpp
index a87e12edc96..adf4e1e63fa 100644
--- a/src/hotspot/share/classfile/fieldLayoutBuilder.cpp
+++ b/src/hotspot/share/classfile/fieldLayoutBuilder.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -37,7 +37,7 @@
LayoutRawBlock::LayoutRawBlock(Kind kind, int size) :
_next_block(nullptr),
_prev_block(nullptr),
- _kind(kind),
+ _block_kind(kind),
_offset(-1),
_alignment(1),
_size(size),
@@ -52,7 +52,7 @@ LayoutRawBlock::LayoutRawBlock(Kind kind, int size) :
LayoutRawBlock::LayoutRawBlock(int index, Kind kind, int size, int alignment, bool is_reference) :
_next_block(nullptr),
_prev_block(nullptr),
- _kind(kind),
+ _block_kind(kind),
_offset(-1),
_alignment(alignment),
_size(size),
@@ -148,8 +148,8 @@ void FieldLayout::initialize_instance_layout(const InstanceKlass* super_klass, b
LayoutRawBlock* FieldLayout::first_field_block() {
LayoutRawBlock* block = _start;
- while (block->kind() != LayoutRawBlock::INHERITED && block->kind() != LayoutRawBlock::REGULAR
- && block->kind() != LayoutRawBlock::FLATTENED && block->kind() != LayoutRawBlock::PADDING) {
+ while (block->block_kind() != LayoutRawBlock::INHERITED && block->block_kind() != LayoutRawBlock::REGULAR
+ && block->block_kind() != LayoutRawBlock::FLATTENED && block->block_kind() != LayoutRawBlock::PADDING) {
block = block->next_block();
}
return block;
@@ -190,7 +190,7 @@ void FieldLayout::add(GrowableArray* list, LayoutRawBlock* star
assert(cursor != nullptr, "Sanity check");
last_search_success = true;
while (cursor != start) {
- if (cursor->kind() == LayoutRawBlock::EMPTY && cursor->fit(b->size(), b->alignment())) {
+ if (cursor->block_kind() == LayoutRawBlock::EMPTY && cursor->fit(b->size(), b->alignment())) {
if (candidate == nullptr || cursor->size() < candidate->size()) {
candidate = cursor;
}
@@ -202,7 +202,7 @@ void FieldLayout::add(GrowableArray* list, LayoutRawBlock* star
last_search_success = false;
}
assert(candidate != nullptr, "Candidate must not be null");
- assert(candidate->kind() == LayoutRawBlock::EMPTY, "Candidate must be an empty block");
+ assert(candidate->block_kind() == LayoutRawBlock::EMPTY, "Candidate must be an empty block");
assert(candidate->fit(b->size(), b->alignment()), "Candidate must be able to store the block");
}
@@ -221,7 +221,7 @@ void FieldLayout::add_field_at_offset(LayoutRawBlock* block, int offset, LayoutR
while (slot != nullptr) {
if ((slot->offset() <= block->offset() && (slot->offset() + slot->size()) > block->offset()) ||
slot == _last){
- assert(slot->kind() == LayoutRawBlock::EMPTY, "Matching slot must be an empty slot");
+ assert(slot->block_kind() == LayoutRawBlock::EMPTY, "Matching slot must be an empty slot");
assert(slot->size() >= block->offset() + block->size() ,"Matching slot must be big enough");
if (slot->offset() < block->offset()) {
int adjustment = block->offset() - slot->offset();
@@ -261,7 +261,7 @@ void FieldLayout::add_contiguously(GrowableArray* list, LayoutR
} else {
LayoutRawBlock* first = list->at(0);
candidate = last_block()->prev_block();
- while (candidate->kind() != LayoutRawBlock::EMPTY || !candidate->fit(size, first->alignment())) {
+ while (candidate->block_kind() != LayoutRawBlock::EMPTY || !candidate->fit(size, first->alignment())) {
if (candidate == start) {
candidate = last_block();
break;
@@ -269,7 +269,7 @@ void FieldLayout::add_contiguously(GrowableArray* list, LayoutR
candidate = candidate->prev_block();
}
assert(candidate != nullptr, "Candidate must not be null");
- assert(candidate->kind() == LayoutRawBlock::EMPTY, "Candidate must be an empty block");
+ assert(candidate->block_kind() == LayoutRawBlock::EMPTY, "Candidate must be an empty block");
assert(candidate->fit(size, first->alignment()), "Candidate must be able to store the whole contiguous block");
}
@@ -281,7 +281,7 @@ void FieldLayout::add_contiguously(GrowableArray* list, LayoutR
}
LayoutRawBlock* FieldLayout::insert_field_block(LayoutRawBlock* slot, LayoutRawBlock* block) {
- assert(slot->kind() == LayoutRawBlock::EMPTY, "Blocks can only be inserted in empty blocks");
+ assert(slot->block_kind() == LayoutRawBlock::EMPTY, "Blocks can only be inserted in empty blocks");
if (slot->offset() % block->alignment() != 0) {
int adjustment = block->alignment() - (slot->offset() % block->alignment());
LayoutRawBlock* adj = new LayoutRawBlock(LayoutRawBlock::EMPTY, adjustment);
@@ -362,7 +362,7 @@ void FieldLayout::fill_holes(const InstanceKlass* super_klass) {
b = b->next_block();
}
assert(b->next_block() == nullptr, "Invariant at this point");
- assert(b->kind() != LayoutRawBlock::EMPTY, "Sanity check");
+ assert(b->block_kind() != LayoutRawBlock::EMPTY, "Sanity check");
// If the super class has @Contended annotation, a padding block is
// inserted at the end to ensure that fields from the subclasses won't share
@@ -384,7 +384,7 @@ void FieldLayout::fill_holes(const InstanceKlass* super_klass) {
}
LayoutRawBlock* FieldLayout::insert(LayoutRawBlock* slot, LayoutRawBlock* block) {
- assert(slot->kind() == LayoutRawBlock::EMPTY, "Blocks can only be inserted in empty blocks");
+ assert(slot->block_kind() == LayoutRawBlock::EMPTY, "Blocks can only be inserted in empty blocks");
assert(slot->offset() % block->alignment() == 0, "Incompatible alignment");
block->set_offset(slot->offset());
slot->set_offset(slot->offset() + block->size());
@@ -425,7 +425,7 @@ void FieldLayout::print(outputStream* output, bool is_static, const InstanceKlas
ResourceMark rm;
LayoutRawBlock* b = _blocks;
while(b != _last) {
- switch(b->kind()) {
+ switch(b->block_kind()) {
case LayoutRawBlock::REGULAR: {
FieldInfo* fi = _field_info->adr_at(b->field_index());
output->print_cr(" @%d \"%s\" %s %d/%d %s",
@@ -596,11 +596,13 @@ void FieldLayoutBuilder::regular_field_sorting() {
}
}
-void FieldLayoutBuilder::insert_contended_padding(LayoutRawBlock* slot) {
+LayoutRawBlock* FieldLayoutBuilder::insert_contended_padding(LayoutRawBlock* slot) {
+ LayoutRawBlock* padding = nullptr;
if (ContendedPaddingWidth > 0) {
- LayoutRawBlock* padding = new LayoutRawBlock(LayoutRawBlock::PADDING, ContendedPaddingWidth);
+ padding = new LayoutRawBlock(LayoutRawBlock::PADDING, ContendedPaddingWidth);
_layout->insert(slot, padding);
}
+ return padding;
}
// Computation of regular classes layout is an evolution of the previous default layout
@@ -620,10 +622,14 @@ void FieldLayoutBuilder::compute_regular_layout() {
regular_field_sorting();
if (_is_contended) {
- _layout->set_start(_layout->last_block());
// insertion is currently easy because the current strategy doesn't try to fill holes
// in super classes layouts => the _start block is by consequence the _last_block
- insert_contended_padding(_layout->start());
+ _layout->set_start(_layout->last_block());
+ LayoutRawBlock* padding = insert_contended_padding(_layout->start());
+ if (padding != nullptr) {
+ // Setting the padding block as start ensures we do not insert past it.
+ _layout->set_start(padding);
+ }
need_tail_padding = true;
}
@@ -639,7 +645,13 @@ void FieldLayoutBuilder::compute_regular_layout() {
for (int i = 0; i < _contended_groups.length(); i++) {
FieldGroup* cg = _contended_groups.at(i);
LayoutRawBlock* start = _layout->last_block();
- insert_contended_padding(start);
+ LayoutRawBlock* padding = insert_contended_padding(start);
+
+ // Do not insert fields past the padding block.
+ if (padding != nullptr) {
+ start = padding;
+ }
+
_layout->add(cg->primitive_fields(), start);
_layout->add(cg->oop_fields(), start);
need_tail_padding = true;
diff --git a/src/hotspot/share/classfile/fieldLayoutBuilder.hpp b/src/hotspot/share/classfile/fieldLayoutBuilder.hpp
index 82bbaefc623..a45131ec9a3 100644
--- a/src/hotspot/share/classfile/fieldLayoutBuilder.hpp
+++ b/src/hotspot/share/classfile/fieldLayoutBuilder.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -65,7 +65,7 @@ class LayoutRawBlock : public ResourceObj {
private:
LayoutRawBlock* _next_block;
LayoutRawBlock* _prev_block;
- Kind _kind;
+ Kind _block_kind;
int _offset;
int _alignment;
int _size;
@@ -79,7 +79,7 @@ class LayoutRawBlock : public ResourceObj {
void set_next_block(LayoutRawBlock* next) { _next_block = next; }
LayoutRawBlock* prev_block() const { return _prev_block; }
void set_prev_block(LayoutRawBlock* prev) { _prev_block = prev; }
- Kind kind() const { return _kind; }
+ Kind block_kind() const { return _block_kind; }
int offset() const {
assert(_offset >= 0, "Must be initialized");
return _offset;
@@ -173,7 +173,7 @@ class FieldLayout : public ResourceObj {
LayoutRawBlock* first_empty_block() {
LayoutRawBlock* block = _start;
- while (block->kind() != LayoutRawBlock::EMPTY) {
+ while (block->block_kind() != LayoutRawBlock::EMPTY) {
block = block->next_block();
}
return block;
@@ -250,7 +250,7 @@ class FieldLayoutBuilder : public ResourceObj {
void build_layout();
void compute_regular_layout();
- void insert_contended_padding(LayoutRawBlock* slot);
+ LayoutRawBlock* insert_contended_padding(LayoutRawBlock* slot);
private:
void prologue();
diff --git a/src/hotspot/share/classfile/javaClasses.cpp b/src/hotspot/share/classfile/javaClasses.cpp
index c6b0fcb90e0..ef1eeec14dd 100644
--- a/src/hotspot/share/classfile/javaClasses.cpp
+++ b/src/hotspot/share/classfile/javaClasses.cpp
@@ -1925,16 +1925,13 @@ oop java_lang_Thread::async_get_stack_trace(jobject jthread, TRAPS) {
public:
const Handle _thread_h;
int _depth;
- GrowableArray* _methods;
- GrowableArray* _bcis;
+ enum InitLength { len = 64 }; // Minimum length that covers most cases
+ GrowableArrayCHeap _methods;
+ GrowableArrayCHeap _bcis;
GetStackTraceHandshakeClosure(Handle thread_h) :
HandshakeClosure("GetStackTraceHandshakeClosure"), _thread_h(thread_h), _depth(0),
- _methods(nullptr), _bcis(nullptr) {
- }
- ~GetStackTraceHandshakeClosure() {
- delete _methods;
- delete _bcis;
+ _methods(InitLength::len), _bcis(InitLength::len) {
}
void do_thread(Thread* th) {
@@ -1950,11 +1947,6 @@ oop java_lang_Thread::async_get_stack_trace(jobject jthread, TRAPS) {
const int max_depth = MaxJavaStackTraceDepth;
const bool skip_hidden = !ShowHiddenFrames;
- // Pick minimum length that will cover most cases
- int init_length = 64;
- _methods = new (mtInternal) GrowableArray(init_length, mtInternal);
- _bcis = new (mtInternal) GrowableArray(init_length, mtInternal);
-
int total_count = 0;
vframeStream vfst(java_thread != nullptr
? vframeStream(java_thread, false, false, vthread_carrier) // we don't process frames as we don't care about oops
@@ -1968,8 +1960,8 @@ oop java_lang_Thread::async_get_stack_trace(jobject jthread, TRAPS) {
continue;
}
- _methods->push(vfst.method());
- _bcis->push(vfst.bci());
+ _methods.push(vfst.method());
+ _bcis.push(vfst.bci());
total_count++;
}
@@ -2001,9 +1993,9 @@ oop java_lang_Thread::async_get_stack_trace(jobject jthread, TRAPS) {
objArrayHandle trace = oopFactory::new_objArray_handle(k, gsthc._depth, CHECK_NULL);
for (int i = 0; i < gsthc._depth; i++) {
- methodHandle method(THREAD, gsthc._methods->at(i));
+ methodHandle method(THREAD, gsthc._methods.at(i));
oop element = java_lang_StackTraceElement::create(method,
- gsthc._bcis->at(i),
+ gsthc._bcis.at(i),
CHECK_NULL);
trace->obj_at_put(i, element);
}
diff --git a/src/hotspot/share/classfile/stackMapTableFormat.hpp b/src/hotspot/share/classfile/stackMapTableFormat.hpp
index 2b89c53278a..4906f4b9d80 100644
--- a/src/hotspot/share/classfile/stackMapTableFormat.hpp
+++ b/src/hotspot/share/classfile/stackMapTableFormat.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2010, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2010, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -221,9 +221,11 @@ class stack_map_frame {
class same_frame : public stack_map_frame {
private:
static int frame_type_to_offset_delta(u1 frame_type) {
- return frame_type + 1; }
+ return frame_type + 1;
+ }
static u1 offset_delta_to_frame_type(int offset_delta) {
- return checked_cast(offset_delta - 1); }
+ return checked_cast(offset_delta - 1);
+ }
public:
@@ -327,9 +329,11 @@ class same_locals_1_stack_item_frame : public stack_map_frame {
address type_addr() const { return frame_type_addr() + sizeof(u1); }
static int frame_type_to_offset_delta(u1 frame_type) {
- return frame_type - 63; }
+ return frame_type - 63;
+ }
static u1 offset_delta_to_frame_type(int offset_delta) {
- return (u1)(offset_delta + 63); }
+ return (u1)(offset_delta + 63);
+ }
public:
static bool is_frame_type(u1 tag) {
@@ -657,9 +661,11 @@ class full_frame : public stack_map_frame {
address num_locals_addr() const { return offset_delta_addr() + sizeof(u2); }
address locals_addr() const { return num_locals_addr() + sizeof(u2); }
address stack_slots_addr(address end_of_locals) const {
- return end_of_locals; }
+ return end_of_locals;
+ }
address stack_addr(address end_of_locals) const {
- return stack_slots_addr(end_of_locals) + sizeof(u2); }
+ return stack_slots_addr(end_of_locals) + sizeof(u2);
+ }
enum { _frame_id = 255 };
@@ -930,11 +936,14 @@ class stack_map_table {
class stack_map_table_attribute {
private:
address name_index_addr() const {
- return (address)this; }
+ return (address)this;
+ }
address attribute_length_addr() const {
- return name_index_addr() + sizeof(u2); }
+ return name_index_addr() + sizeof(u2);
+ }
address stack_map_table_addr() const {
- return attribute_length_addr() + sizeof(u4); }
+ return attribute_length_addr() + sizeof(u4);
+ }
NONCOPYABLE(stack_map_table_attribute);
protected:
@@ -948,9 +957,11 @@ class stack_map_table_attribute {
}
u2 name_index() const {
- return Bytes::get_Java_u2(name_index_addr()); }
+ return Bytes::get_Java_u2(name_index_addr());
+ }
u4 attribute_length() const {
- return Bytes::get_Java_u4(attribute_length_addr()); }
+ return Bytes::get_Java_u4(attribute_length_addr());
+ }
stack_map_table* table() const {
return stack_map_table::at(stack_map_table_addr());
}
diff --git a/src/hotspot/share/classfile/stringTable.cpp b/src/hotspot/share/classfile/stringTable.cpp
index 2b8b7780a41..c3f60487b9c 100644
--- a/src/hotspot/share/classfile/stringTable.cpp
+++ b/src/hotspot/share/classfile/stringTable.cpp
@@ -946,7 +946,7 @@ void StringTable::init_shared_table() {
// so we are all good.
// - If there's a reference to it, we will report an error inside HeapShared.cpp and
// dumping will fail.
- HeapShared::add_to_dumped_interned_strings(string);
+ HeapShared::archive_interned_string(string);
}
n++;
return true;
diff --git a/src/hotspot/share/classfile/systemDictionary.cpp b/src/hotspot/share/classfile/systemDictionary.cpp
index 0b47c749df8..8483551fd4f 100644
--- a/src/hotspot/share/classfile/systemDictionary.cpp
+++ b/src/hotspot/share/classfile/systemDictionary.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -416,7 +416,7 @@ static inline void log_circularity_error(Symbol* name, PlaceholderEntry* probe)
//
// resolve_with_circularity_detection adds a DETECT_CIRCULARITY placeholder to the placeholder table before calling
// resolve_instance_class_or_null. ClassCircularityError is detected when a DETECT_CIRCULARITY or LOAD_INSTANCE
-// placeholder for the same thread, class, classloader is found.
+// placeholder for the same thread, class, and classloader is found.
// This can be seen with logging option: -Xlog:class+load+placeholders=debug.
//
InstanceKlass* SystemDictionary::resolve_with_circularity_detection(Symbol* class_name,
diff --git a/src/hotspot/share/classfile/systemDictionaryShared.cpp b/src/hotspot/share/classfile/systemDictionaryShared.cpp
index cfb20412ab8..fd30fc6766f 100644
--- a/src/hotspot/share/classfile/systemDictionaryShared.cpp
+++ b/src/hotspot/share/classfile/systemDictionaryShared.cpp
@@ -89,11 +89,9 @@ DEBUG_ONLY(bool SystemDictionaryShared::_class_loading_may_happen = true;)
#ifdef ASSERT
static void check_klass_after_loading(const Klass* k) {
-#ifdef _LP64
- if (k != nullptr && UseCompressedClassPointers) {
+ if (k != nullptr) {
CompressedKlassPointers::check_encodable(k);
}
-#endif
}
#endif
@@ -204,6 +202,20 @@ DumpTimeClassInfo* SystemDictionaryShared::get_info_locked(InstanceKlass* k) {
return info;
}
+void SystemDictionaryShared::check_code_source(InstanceKlass* ik, const ClassFileStream* cfs) {
+ if (CDSConfig::is_dumping_preimage_static_archive() && !is_builtin_loader(ik->class_loader_data())) {
+ if (cfs == nullptr || cfs->source() == nullptr || strncmp(cfs->source(), "file:", 5) != 0) {
+ // AOT cache filtering:
+ // For non-built-in loaders, cache only the classes that have a file: code source, so
+ // we can avoid caching dynamically generated classes that are likely to change from
+ // run to run. This is similar to the filtering in ClassListWriter::write_to_stream()
+ // for the classic CDS static archive.
+ SystemDictionaryShared::log_exclusion(ik, "Not loaded from \"file:\" code source");
+ SystemDictionaryShared::set_excluded(ik);
+ }
+ }
+}
+
bool SystemDictionaryShared::should_be_excluded_impl(InstanceKlass* k, DumpTimeClassInfo* info) {
assert_lock_strong(DumpTimeTable_lock);
@@ -373,11 +385,6 @@ bool SystemDictionaryShared::is_jfr_event_class(InstanceKlass *k) {
return false;
}
-bool SystemDictionaryShared::is_early_klass(InstanceKlass* ik) {
- DumpTimeClassInfo* info = _dumptime_table->get(ik);
- return (info != nullptr) ? info->is_early_klass() : false;
-}
-
bool SystemDictionaryShared::check_self_exclusion(InstanceKlass* k) {
bool log_warning = false;
const char* error = check_self_exclusion_helper(k, log_warning);
diff --git a/src/hotspot/share/classfile/systemDictionaryShared.hpp b/src/hotspot/share/classfile/systemDictionaryShared.hpp
index 2619a642fd1..c837a386344 100644
--- a/src/hotspot/share/classfile/systemDictionaryShared.hpp
+++ b/src/hotspot/share/classfile/systemDictionaryShared.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2014, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -199,7 +199,6 @@ private:
static void iterate_verification_constraint_names(InstanceKlass* k, DumpTimeClassInfo* info, Function func);
public:
- static bool is_early_klass(InstanceKlass* k); // Was k loaded while JvmtiExport::is_early_phase()==true
static bool has_archived_enum_objs(InstanceKlass* ik);
static void set_has_archived_enum_objs(InstanceKlass* ik);
@@ -236,6 +235,7 @@ public:
static void update_shared_entry(InstanceKlass* klass, int id);
static void set_shared_class_misc_info(InstanceKlass* k, ClassFileStream* cfs);
+ static void check_code_source(InstanceKlass* ik, const ClassFileStream* cfs) NOT_CDS_RETURN;
static InstanceKlass* lookup_from_stream(Symbol* class_name,
Handle class_loader,
Handle protection_domain,
diff --git a/src/hotspot/share/classfile/verifier.cpp b/src/hotspot/share/classfile/verifier.cpp
index 30f147b9ae7..48be24c20dc 100644
--- a/src/hotspot/share/classfile/verifier.cpp
+++ b/src/hotspot/share/classfile/verifier.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -620,9 +620,6 @@ TypeOrigin ClassVerifier::ref_ctx(const char* sig) {
void ClassVerifier::verify_class(TRAPS) {
log_info(verification)("Verifying class %s with new format", _klass->external_name());
- // Either verifying both local and remote classes or just remote classes.
- assert(BytecodeVerificationRemote, "Should not be here");
-
Array* methods = _klass->methods();
int num_methods = methods->length();
@@ -1610,12 +1607,12 @@ void ClassVerifier::verify_method(const methodHandle& m, TRAPS) {
case Bytecodes::_if_acmpeq :
case Bytecodes::_if_acmpne :
current_frame.pop_stack(
- VerificationType::reference_check(), CHECK_VERIFY(this));
+ object_type(), CHECK_VERIFY(this));
// fall through
case Bytecodes::_ifnull :
case Bytecodes::_ifnonnull :
current_frame.pop_stack(
- VerificationType::reference_check(), CHECK_VERIFY(this));
+ object_type(), CHECK_VERIFY(this));
stackmap_table.check_jump_target
(¤t_frame, bcs.bci(), bcs.get_offset_s2(), CHECK_VERIFY(this));
no_control_flow = false; break;
diff --git a/src/hotspot/share/classfile/vmIntrinsics.hpp b/src/hotspot/share/classfile/vmIntrinsics.hpp
index 29cb5d737d8..e84acd62284 100644
--- a/src/hotspot/share/classfile/vmIntrinsics.hpp
+++ b/src/hotspot/share/classfile/vmIntrinsics.hpp
@@ -368,10 +368,10 @@ class methodHandle;
do_intrinsic(_inflateStringB, java_lang_StringLatin1, inflate_name, inflateB_signature, F_S) \
do_signature(inflateB_signature, "([BI[BII)V") \
do_intrinsic(_toBytesStringU, java_lang_StringUTF16, toBytes_name, toBytesU_signature, F_S) \
- do_name( toBytes_name, "toBytes") \
+ do_name( toBytes_name, "toBytes0") \
do_signature(toBytesU_signature, "([CII)[B") \
do_intrinsic(_getCharsStringU, java_lang_StringUTF16, getCharsU_name, getCharsU_signature, F_S) \
- do_name( getCharsU_name, "getChars") \
+ do_name( getCharsU_name, "getChars0") \
do_signature(getCharsU_signature, "([BII[CI)V") \
do_intrinsic(_getCharStringU, java_lang_StringUTF16, getChar_name, getCharStringU_signature, F_S) \
do_signature(getCharStringU_signature, "([BI)C") \
@@ -1029,7 +1029,7 @@ class methodHandle;
do_intrinsic(_VectorUnaryLibOp, jdk_internal_vm_vector_VectorSupport, vector_unary_lib_op_name, vector_unary_lib_op_sig, F_S) \
do_signature(vector_unary_lib_op_sig,"(J" \
"Ljava/lang/Class;" \
- "Ljava/lang/Class;" \
+ "I" \
"I" \
"Ljava/lang/String;" \
"Ljdk/internal/vm/vector/VectorSupport$Vector;" \
@@ -1040,7 +1040,7 @@ class methodHandle;
do_intrinsic(_VectorBinaryLibOp, jdk_internal_vm_vector_VectorSupport, vector_binary_lib_op_name, vector_binary_lib_op_sig, F_S) \
do_signature(vector_binary_lib_op_sig,"(J" \
"Ljava/lang/Class;" \
- "Ljava/lang/Class;" \
+ "I" \
"I" \
"Ljava/lang/String;" \
"Ljdk/internal/vm/vector/VectorSupport$VectorPayload;" \
diff --git a/src/hotspot/share/classfile/vmSymbols.hpp b/src/hotspot/share/classfile/vmSymbols.hpp
index 0054b7ba3f2..2ae42bebcfd 100644
--- a/src/hotspot/share/classfile/vmSymbols.hpp
+++ b/src/hotspot/share/classfile/vmSymbols.hpp
@@ -98,6 +98,7 @@ class SerializeClosure;
template(jdk_internal_vm_vector_VectorMask, "jdk/internal/vm/vector/VectorSupport$VectorMask") \
template(jdk_internal_vm_vector_VectorShuffle, "jdk/internal/vm/vector/VectorSupport$VectorShuffle") \
template(payload_name, "payload") \
+ template(CTYPE_name, "CTYPE") \
template(ETYPE_name, "ETYPE") \
template(VLENGTH_name, "VLENGTH") \
\
diff --git a/src/hotspot/share/code/aotCodeCache.cpp b/src/hotspot/share/code/aotCodeCache.cpp
index 32a53691f3f..e3cbc3c3c8b 100644
--- a/src/hotspot/share/code/aotCodeCache.cpp
+++ b/src/hotspot/share/code/aotCodeCache.cpp
@@ -29,9 +29,11 @@
#include "cds/cds_globals.hpp"
#include "cds/cdsConfig.hpp"
#include "cds/heapShared.hpp"
+#include "ci/ciUtilities.hpp"
#include "classfile/javaAssertions.hpp"
#include "code/aotCodeCache.hpp"
#include "code/codeCache.hpp"
+#include "gc/shared/cardTableBarrierSet.hpp"
#include "gc/shared/gcConfig.hpp"
#include "logging/logStream.hpp"
#include "memory/memoryReserver.hpp"
@@ -53,6 +55,7 @@
#endif
#if INCLUDE_G1GC
#include "gc/g1/g1BarrierSetRuntime.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#endif
#if INCLUDE_SHENANDOAHGC
#include "gc/shenandoah/shenandoahRuntime.hpp"
@@ -70,11 +73,23 @@ const char* aot_code_entry_kind_name[] = {
#undef DECL_KIND_STRING
};
+// Stream to printing AOTCodeCache loading failure.
+// Print to error channel when -XX:AOTMode is set to "on"
+static LogStream& load_failure_log() {
+ static LogStream err_stream(LogLevel::Error, LogTagSetMapping::tagset());
+ static LogStream dbg_stream(LogLevel::Debug, LogTagSetMapping::tagset());
+ if (RequireSharedSpaces) {
+ return err_stream;
+ } else {
+ return dbg_stream;
+ }
+}
+
static void report_load_failure() {
if (AbortVMOnAOTCodeFailure) {
vm_exit_during_initialization("Unable to use AOT Code Cache.", nullptr);
}
- log_info(aot, codecache, init)("Unable to use AOT Code Cache.");
+ load_failure_log().print_cr("Unable to use AOT Code Cache.");
AOTCodeCache::disable_caching();
}
@@ -83,7 +98,7 @@ static void report_store_failure() {
tty->print_cr("Unable to create AOT Code Cache.");
vm_abort(false);
}
- log_info(aot, codecache, exit)("Unable to create AOT Code Cache.");
+ log_error(aot, codecache, exit)("Unable to create AOT Code Cache.");
AOTCodeCache::disable_caching();
}
@@ -258,6 +273,9 @@ void AOTCodeCache::init2() {
return;
}
+ // initialize aot runtime constants as appropriate to this runtime
+ AOTRuntimeConstants::initialize_from_runtime();
+
// initialize the table of external routines so we can save
// generated code blobs that reference them
AOTCodeAddressTable* table = opened_cache->_table;
@@ -278,11 +296,11 @@ bool AOTCodeCache::open_cache(bool is_dumping, bool is_using) {
return true;
}
-void AOTCodeCache::close() {
+void AOTCodeCache::dump() {
if (is_on()) {
- delete _cache; // Free memory
- _cache = nullptr;
- opened_cache = nullptr;
+ assert(is_on_for_dump(), "should be called only when dumping AOT code");
+ MutexLocker ml(Compile_lock);
+ _cache->finish_write();
}
}
@@ -298,7 +316,6 @@ AOTCodeCache::AOTCodeCache(bool is_dumping, bool is_using) :
_store_size(0),
_for_use(is_using),
_for_dump(is_dumping),
- _closing(false),
_failed(false),
_lookup_failed(false),
_table(nullptr),
@@ -375,61 +392,24 @@ void AOTCodeCache::init_early_c1_table() {
}
}
-AOTCodeCache::~AOTCodeCache() {
- if (_closing) {
- return; // Already closed
- }
- // Stop any further access to cache.
- _closing = true;
-
- MutexLocker ml(Compile_lock);
- if (for_dump()) { // Finalize cache
- finish_write();
- }
- _load_buffer = nullptr;
- if (_C_store_buffer != nullptr) {
- FREE_C_HEAP_ARRAY(char, _C_store_buffer);
- _C_store_buffer = nullptr;
- _store_buffer = nullptr;
- }
- if (_table != nullptr) {
- MutexLocker ml(AOTCodeCStrings_lock, Mutex::_no_safepoint_check_flag);
- delete _table;
- _table = nullptr;
- }
-}
-
void AOTCodeCache::Config::record(uint cpu_features_offset) {
- _flags = 0;
-#ifdef ASSERT
- _flags |= debugVM;
-#endif
- if (UseCompressedOops) {
- _flags |= compressedOops;
- }
- if (UseCompressedClassPointers) {
- _flags |= compressedClassPointers;
- }
- if (UseTLAB) {
- _flags |= useTLAB;
- }
- if (JavaAssertions::systemClassDefault()) {
- _flags |= systemClassAssertions;
- }
- if (JavaAssertions::userClassDefault()) {
- _flags |= userClassAssertions;
- }
- if (EnableContended) {
- _flags |= enableContendedPadding;
- }
- if (RestrictContended) {
- _flags |= restrictContendedPadding;
- }
- _compressedOopShift = CompressedOops::shift();
+
+#define AOTCODECACHE_SAVE_VAR(type, name) _saved_ ## name = name;
+#define AOTCODECACHE_SAVE_FUN(type, name, fun) _saved_ ## name = fun;
+
+ AOTCODECACHE_CONFIGS_DO(AOTCODECACHE_SAVE_VAR, AOTCODECACHE_SAVE_FUN);
+
+ // Special configs that cannot be checked with macros
_compressedOopBase = CompressedOops::base();
- _compressedKlassShift = CompressedKlassPointers::shift();
- _contendedPaddingWidth = ContendedPaddingWidth;
- _gc = (uint)Universe::heap()->kind();
+
+#if defined(X86) && !defined(ZERO)
+ _useUnalignedLoadStores = UseUnalignedLoadStores;
+#endif
+
+#if defined(AARCH64) && !defined(ZERO)
+ _avoidUnalignedAccesses = AvoidUnalignedAccesses;
+#endif
+
_cpu_features_offset = cpu_features_offset;
}
@@ -460,78 +440,114 @@ bool AOTCodeCache::Config::verify_cpu_features(AOTCodeCache* cache) const {
}
}
} else {
- if (log.is_enabled()) {
+ if (load_failure_log().is_enabled()) {
ResourceMark rm; // required for stringStream::as_string()
stringStream ss;
char* runtime_cpu_features = NEW_RESOURCE_ARRAY(char, VM_Version::cpu_features_size());
VM_Version::store_cpu_features(runtime_cpu_features);
VM_Version::get_missing_features_name(cached_cpu_features_buffer, runtime_cpu_features, ss);
- log.print_cr("AOT Code Cache disabled: required cpu features are missing: %s", ss.as_string());
+ load_failure_log().print_cr("AOT Code Cache disabled: required cpu features are missing: %s", ss.as_string());
}
return false;
}
return true;
}
-bool AOTCodeCache::Config::verify(AOTCodeCache* cache) const {
- // First checks affect all cached AOT code
-#ifdef ASSERT
- if ((_flags & debugVM) == 0) {
- log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created by product VM, it can't be used by debug VM");
- return false;
- }
-#else
- if ((_flags & debugVM) != 0) {
- log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created by debug VM, it can't be used by product VM");
- return false;
- }
+#define AOTCODECACHE_DISABLED_MSG "AOT Code Cache disabled: it was created with %s = "
+
+// Special case, print "GC = ..." to be more understandable.
+inline void log_config_mismatch(CollectedHeap::Name saved, CollectedHeap::Name current, const char* name/*unused*/) {
+ load_failure_log().print_cr("AOT Code Cache disabled: it was created with GC = \"%s\" vs current \"%s\"",
+ GCConfig::hs_err_name(saved), GCConfig::hs_err_name(current));
+}
+
+inline void log_config_mismatch(bool saved, bool current, const char* name) {
+ load_failure_log().print_cr(AOTCODECACHE_DISABLED_MSG "%s vs current %s", name,
+ saved ? "true" : "false", current ? "true" : "false");
+}
+
+inline void log_config_mismatch(int saved, int current, const char* name) {
+ load_failure_log().print_cr(AOTCODECACHE_DISABLED_MSG "%d vs current %d", name, saved, current);
+}
+
+inline void log_config_mismatch(uint saved, uint current, const char* name) {
+ load_failure_log().print_cr(AOTCODECACHE_DISABLED_MSG "%u vs current %u", name, saved, current);
+}
+
+#ifdef _LP64
+inline void log_config_mismatch(intx saved, intx current, const char* name) {
+ load_failure_log().print_cr(AOTCODECACHE_DISABLED_MSG "%zd vs current %zd", name, saved, current);
+}
+
+inline void log_config_mismatch(uintx saved, uintx current, const char* name) {
+ load_failure_log().print_cr(AOTCODECACHE_DISABLED_MSG "%zu vs current %zu", name, saved, current);
+}
#endif
- CollectedHeap::Name aot_gc = (CollectedHeap::Name)_gc;
- if (aot_gc != Universe::heap()->kind()) {
- log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with different GC: %s vs current %s", GCConfig::hs_err_name(aot_gc), GCConfig::hs_err_name());
+template
+bool check_config(T saved, T current, const char* name) {
+ if (saved != current) {
+ log_config_mismatch(saved, current, name);
return false;
+ } else {
+ return true;
}
+}
- if (((_flags & compressedClassPointers) != 0) != UseCompressedClassPointers) {
- log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with UseCompressedClassPointers = %s", UseCompressedClassPointers ? "false" : "true");
- return false;
- }
- if (_compressedKlassShift != (uint)CompressedKlassPointers::shift()) {
- log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with CompressedKlassPointers::shift() = %d vs current %d", _compressedKlassShift, CompressedKlassPointers::shift());
- return false;
- }
-
- // The following checks do not affect AOT adapters caching
-
- if (((_flags & compressedOops) != 0) != UseCompressedOops) {
- log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with UseCompressedOops = %s", UseCompressedOops ? "false" : "true");
- AOTStubCaching = false;
- }
- if (_compressedOopShift != (uint)CompressedOops::shift()) {
- log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with different CompressedOops::shift(): %d vs current %d", _compressedOopShift, CompressedOops::shift());
- AOTStubCaching = false;
- }
-
- // This should be the last check as it only disables AOTStubCaching
- if ((_compressedOopBase == nullptr || CompressedOops::base() == nullptr) && (_compressedOopBase != CompressedOops::base())) {
- log_debug(aot, codecache, init)("AOTStubCaching is disabled: incompatible CompressedOops::base(): %p vs current %p", _compressedOopBase, CompressedOops::base());
- AOTStubCaching = false;
- }
-
+bool AOTCodeCache::Config::verify(AOTCodeCache* cache) const {
+ // check CPU features before checking flags that may be
+ // auto-configured in response to them
if (!verify_cpu_features(cache)) {
return false;
}
+
+ // Tests for config options which might affect validity of adapters,
+ // stubs or nmethods. Currently we take a pessemistic stand and
+ // drop the whole cache if any of these are changed.
+
+#define AOTCODECACHE_CHECK_VAR(type, name) \
+ if (!check_config(_saved_ ## name, name, #name)) { return false; }
+#define AOTCODECACHE_CHECK_FUN(type, name, fun) \
+ if (!check_config(_saved_ ## name, fun, #fun)) { return false; }
+
+ AOTCODECACHE_CONFIGS_DO(AOTCODECACHE_CHECK_VAR, AOTCODECACHE_CHECK_FUN);
+
+ // Special configs that cannot be checked with macros
+
+ if ((_compressedOopBase == nullptr || CompressedOops::base() == nullptr) && (_compressedOopBase != CompressedOops::base())) {
+ load_failure_log().print_cr("AOT Code Cache disabled: incompatible CompressedOops::base(): %p vs current %p",
+ _compressedOopBase, CompressedOops::base());
+ return false;
+ }
+
+#if defined(X86) && !defined(ZERO)
+ // switching off UseUnalignedLoadStores can affect validity of fill
+ // stubs
+ if (_useUnalignedLoadStores && !UseUnalignedLoadStores) {
+ log_config_mismatch(_useUnalignedLoadStores, UseUnalignedLoadStores, "UseUnalignedLoadStores");
+ return false;
+ }
+#endif // defined(X86) && !defined(ZERO)
+
+#if defined(AARCH64) && !defined(ZERO)
+ // switching on AvoidUnalignedAccesses may affect validity of array
+ // copy stubs and nmethods
+ if (!_avoidUnalignedAccesses && AvoidUnalignedAccesses) {
+ log_config_mismatch(_avoidUnalignedAccesses, AvoidUnalignedAccesses, "AvoidUnalignedAccesses");
+ return false;
+ }
+#endif // defined(AARCH64) && !defined(ZERO)
+
return true;
}
bool AOTCodeCache::Header::verify(uint load_size) const {
if (_version != AOT_CODE_VERSION) {
- log_debug(aot, codecache, init)("AOT Code Cache disabled: different AOT Code version %d vs %d recorded in AOT Code header", AOT_CODE_VERSION, _version);
+ load_failure_log().print_cr("AOT Code Cache disabled: different AOT Code version %d vs %d recorded in AOT Code header", AOT_CODE_VERSION, _version);
return false;
}
if (load_size < _cache_size) {
- log_debug(aot, codecache, init)("AOT Code Cache disabled: AOT Code Cache size %d < %d recorded in AOT Code header", load_size, _cache_size);
+ load_failure_log().print_cr("AOT Code Cache disabled: AOT Code Cache size %d < %d recorded in AOT Code header", load_size, _cache_size);
return false;
}
return true;
@@ -565,6 +581,9 @@ AOTCodeReader::AOTCodeReader(AOTCodeCache* cache, AOTCodeEntry* entry) {
_load_buffer = cache->cache_buffer();
_read_position = 0;
_lookup_failed = false;
+ _name = nullptr;
+ _reloc_data = nullptr;
+ _oop_maps = nullptr;
}
void AOTCodeReader::set_read_position(uint pos) {
@@ -929,16 +948,6 @@ bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind
has_oop_maps = true;
}
-#ifndef PRODUCT
- // Write asm remarks
- if (!cache->write_asm_remarks(blob)) {
- return false;
- }
- if (!cache->write_dbg_strings(blob)) {
- return false;
- }
-#endif /* PRODUCT */
-
if (!cache->write_relocations(blob)) {
if (!cache->failed()) {
// We may miss an address in AOT table - skip this code blob.
@@ -947,6 +956,16 @@ bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind
return false;
}
+#ifndef PRODUCT
+ // Write asm remarks after relocation info
+ if (!cache->write_asm_remarks(blob)) {
+ return false;
+ }
+ if (!cache->write_dbg_strings(blob)) {
+ return false;
+ }
+#endif /* PRODUCT */
+
uint entry_size = cache->_write_position - entry_position;
AOTCodeEntry* entry = new(cache) AOTCodeEntry(entry_kind, encode_id(entry_kind, id),
entry_position, entry_size, name_offset, name_size,
@@ -1008,39 +1027,28 @@ CodeBlob* AOTCodeReader::compile_code_blob(const char* name) {
set_lookup_failed(); // Skip this blob
return nullptr;
}
+ _name = stored_name;
// Read archived code blob
uint offset = entry_position + _entry->blob_offset();
CodeBlob* archived_blob = (CodeBlob*)addr(offset);
offset += archived_blob->size();
- address reloc_data = (address)addr(offset);
+ _reloc_data = (address)addr(offset);
offset += archived_blob->relocation_size();
set_read_position(offset);
- ImmutableOopMapSet* oop_maps = nullptr;
if (_entry->has_oop_maps()) {
- oop_maps = read_oop_map_set();
+ _oop_maps = read_oop_map_set();
}
- CodeBlob* code_blob = CodeBlob::create(archived_blob,
- stored_name,
- reloc_data,
- oop_maps
- );
+ // CodeBlob::restore() calls AOTCodeReader::restore()
+ CodeBlob* code_blob = CodeBlob::create(archived_blob, this);
+
if (code_blob == nullptr) { // no space left in CodeCache
return nullptr;
}
-#ifndef PRODUCT
- code_blob->asm_remarks().init();
- read_asm_remarks(code_blob->asm_remarks());
- code_blob->dbg_strings().init();
- read_dbg_strings(code_blob->dbg_strings());
-#endif // PRODUCT
-
- fix_relocations(code_blob);
-
#ifdef ASSERT
LogStreamHandle(Trace, aot, codecache, stubs) log;
if (log.is_enabled()) {
@@ -1051,6 +1059,25 @@ CodeBlob* AOTCodeReader::compile_code_blob(const char* name) {
return code_blob;
}
+void AOTCodeReader::restore(CodeBlob* code_blob) {
+ precond(AOTCodeCache::is_on_for_use());
+ precond(_name != nullptr);
+ precond(_reloc_data != nullptr);
+
+ code_blob->set_name(_name);
+ code_blob->restore_mutable_data(_reloc_data);
+ code_blob->set_oop_maps(_oop_maps);
+
+ fix_relocations(code_blob);
+
+#ifndef PRODUCT
+ code_blob->asm_remarks().init();
+ read_asm_remarks(code_blob->asm_remarks());
+ code_blob->dbg_strings().init();
+ read_dbg_strings(code_blob->dbg_strings());
+#endif // PRODUCT
+}
+
// ------------ process code and data --------------
// Can't use -1. It is valid value for jump to iteself destination
@@ -1447,6 +1474,12 @@ void AOTCodeAddressTable::init_extrs() {
#endif
#endif // ZERO
+ // addresses of fields in AOT runtime constants area
+ address* p = AOTRuntimeConstants::field_addresses_list();
+ while (*p != nullptr) {
+ SET_ADDRESS(_extrs, *p++);
+ }
+
_extrs_complete = true;
log_debug(aot, codecache, init)("External addresses recorded");
}
@@ -1533,18 +1566,6 @@ void AOTCodeAddressTable::init_early_c1() {
#undef SET_ADDRESS
-AOTCodeAddressTable::~AOTCodeAddressTable() {
- if (_extrs_addr != nullptr) {
- FREE_C_HEAP_ARRAY(address, _extrs_addr);
- }
- if (_stubs_addr != nullptr) {
- FREE_C_HEAP_ARRAY(address, _stubs_addr);
- }
- if (_shared_blobs_addr != nullptr) {
- FREE_C_HEAP_ARRAY(address, _shared_blobs_addr);
- }
-}
-
#ifdef PRODUCT
#define MAX_STR_COUNT 200
#else
@@ -1729,6 +1750,11 @@ int AOTCodeAddressTable::id_for_address(address addr, RelocIterator reloc, CodeB
if (addr == (address)-1) { // Static call stub has jump to itself
return id;
}
+ // Check card_table_base address first since it can point to any address
+ BarrierSet* bs = BarrierSet::barrier_set();
+ bool is_const_card_table_base = !UseG1GC && !UseShenandoahGC && bs->is_a(BarrierSet::CardTableBarrierSet);
+ guarantee(!is_const_card_table_base || addr != ci_card_table_address_const(), "sanity");
+
// Seach for C string
id = id_for_C_string(addr);
if (id >= 0) {
@@ -1798,6 +1824,44 @@ int AOTCodeAddressTable::id_for_address(address addr, RelocIterator reloc, CodeB
return id;
}
+AOTRuntimeConstants AOTRuntimeConstants::_aot_runtime_constants;
+
+void AOTRuntimeConstants::initialize_from_runtime() {
+ BarrierSet* bs = BarrierSet::barrier_set();
+ address card_table_base = nullptr;
+ uint grain_shift = 0;
+#if INCLUDE_G1GC
+ if (bs->is_a(BarrierSet::G1BarrierSet)) {
+ grain_shift = G1HeapRegion::LogOfHRGrainBytes;
+ } else
+#endif
+#if INCLUDE_SHENANDOAHGC
+ if (bs->is_a(BarrierSet::ShenandoahBarrierSet)) {
+ grain_shift = 0;
+ } else
+#endif
+ if (bs->is_a(BarrierSet::CardTableBarrierSet)) {
+ CardTable::CardValue* base = ci_card_table_address_const();
+ assert(base != nullptr, "unexpected byte_map_base");
+ card_table_base = base;
+ CardTableBarrierSet* ctbs = barrier_set_cast(bs);
+ grain_shift = ctbs->grain_shift();
+ }
+ _aot_runtime_constants._card_table_base = card_table_base;
+ _aot_runtime_constants._grain_shift = grain_shift;
+}
+
+address AOTRuntimeConstants::_field_addresses_list[] = {
+ ((address)&_aot_runtime_constants._card_table_base),
+ ((address)&_aot_runtime_constants._grain_shift),
+ nullptr
+};
+
+address AOTRuntimeConstants::card_table_base_address() {
+ assert(UseSerialGC || UseParallelGC, "Only these GCs have constant card table base");
+ return (address)&_aot_runtime_constants._card_table_base;
+}
+
// This is called after initialize() but before init2()
// and _cache is not set yet.
void AOTCodeCache::print_on(outputStream* st) {
diff --git a/src/hotspot/share/code/aotCodeCache.hpp b/src/hotspot/share/code/aotCodeCache.hpp
index 45b4a15510d..179f131c639 100644
--- a/src/hotspot/share/code/aotCodeCache.hpp
+++ b/src/hotspot/share/code/aotCodeCache.hpp
@@ -25,6 +25,8 @@
#ifndef SHARE_CODE_AOTCODECACHE_HPP
#define SHARE_CODE_AOTCODECACHE_HPP
+#include "gc/shared/collectedHeap.hpp"
+#include "gc/shared/gc_globals.hpp"
#include "runtime/stubInfo.hpp"
/*
@@ -151,7 +153,6 @@ public:
_early_c1_complete(false),
_complete(false)
{ }
- ~AOTCodeAddressTable();
void init_extrs();
void init_early_stubs();
void init_shared_blobs();
@@ -163,30 +164,124 @@ public:
address address_for_id(int id);
};
+#define AOTCODECACHE_CONFIGS_GENERIC_DO(do_var, do_fun) \
+ do_var(int, AllocateInstancePrefetchLines) /* stubs and nmethods */ \
+ do_var(int, AllocatePrefetchDistance) /* stubs and nmethods */ \
+ do_var(int, AllocatePrefetchLines) /* stubs and nmethods */ \
+ do_var(int, AllocatePrefetchStepSize) /* stubs and nmethods */ \
+ do_var(uint, CodeEntryAlignment) /* array copy stubs and nmethods */ \
+ do_var(bool, UseCompressedOops) /* stubs and nmethods */ \
+ do_var(bool, EnableContended) /* nmethods */ \
+ do_var(intx, OptoLoopAlignment) /* array copy stubs and nmethods */ \
+ do_var(bool, RestrictContended) /* nmethods */ \
+ do_var(bool, UseAESCTRIntrinsics) \
+ do_var(bool, UseAESIntrinsics) \
+ do_var(bool, UseBASE64Intrinsics) \
+ do_var(bool, UseChaCha20Intrinsics) \
+ do_var(bool, UseCRC32CIntrinsics) \
+ do_var(bool, UseCRC32Intrinsics) \
+ do_var(bool, UseDilithiumIntrinsics) \
+ do_var(bool, UseGHASHIntrinsics) \
+ do_var(bool, UseKyberIntrinsics) \
+ do_var(bool, UseMD5Intrinsics) \
+ do_var(bool, UsePoly1305Intrinsics) \
+ do_var(bool, UseSecondarySupersTable) \
+ do_var(bool, UseSHA1Intrinsics) \
+ do_var(bool, UseSHA256Intrinsics) \
+ do_var(bool, UseSHA3Intrinsics) \
+ do_var(bool, UseSHA512Intrinsics) \
+ do_var(bool, UseVectorizedMismatchIntrinsic) \
+ do_fun(int, CompressedKlassPointers_shift, CompressedKlassPointers::shift()) \
+ do_fun(int, CompressedOops_shift, CompressedOops::shift()) \
+ do_fun(bool, JavaAssertions_systemClassDefault, JavaAssertions::systemClassDefault()) \
+ do_fun(bool, JavaAssertions_userClassDefault, JavaAssertions::userClassDefault()) \
+ do_fun(CollectedHeap::Name, Universe_heap_kind, Universe::heap()->kind()) \
+ // END
+
+#ifdef COMPILER2
+#define AOTCODECACHE_CONFIGS_COMPILER2_DO(do_var, do_fun) \
+ do_var(intx, ArrayOperationPartialInlineSize) /* array copy stubs and nmethods */ \
+ do_var(intx, MaxVectorSize) /* array copy/fill stubs */ \
+ do_var(bool, UseMontgomeryMultiplyIntrinsic) \
+ do_var(bool, UseMontgomerySquareIntrinsic) \
+ do_var(bool, UseMulAddIntrinsic) \
+ do_var(bool, UseMultiplyToLenIntrinsic) \
+ do_var(bool, UseSquareToLenIntrinsic) \
+ // END
+#else
+#define AOTCODECACHE_CONFIGS_COMPILER2_DO(do_var, do_fun)
+#endif
+
+#if INCLUDE_JVMCI
+#define AOTCODECACHE_CONFIGS_JVMCI_DO(do_var, do_fun) \
+ do_var(bool, EnableJVMCI) /* adapters and nmethods */ \
+ // END
+#else
+#define AOTCODECACHE_CONFIGS_JVMCI_DO(do_var, do_fun)
+#endif
+
+#if defined(AARCH64) && !defined(ZERO)
+#define AOTCODECACHE_CONFIGS_AARCH64_DO(do_var, do_fun) \
+ do_var(intx, BlockZeroingLowLimit) /* array fill stubs */ \
+ do_var(intx, PrefetchCopyIntervalInBytes) /* array copy stubs */ \
+ do_var(int, SoftwarePrefetchHintDistance) /* array fill stubs */ \
+ do_var(bool, UseBlockZeroing) \
+ do_var(bool, UseLSE) /* stubs and nmethods */ \
+ do_var(uint, UseSVE) /* stubs and nmethods */ \
+ do_var(bool, UseSecondarySupersCache) \
+ do_var(bool, UseSIMDForArrayEquals) /* array copy stubs and nmethods */ \
+ do_var(bool, UseSIMDForBigIntegerShiftIntrinsics) \
+ do_var(bool, UseSIMDForMemoryOps) /* array copy stubs and nmethods */ \
+ do_var(bool, UseSIMDForSHA3Intrinsic) /* SHA3 stubs */ \
+ do_var(bool, UseSimpleArrayEquals) \
+ // END
+#else
+#define AOTCODECACHE_CONFIGS_AARCH64_DO(do_var, do_fun)
+#endif
+
+#if defined(X86) && !defined(ZERO)
+#define AOTCODECACHE_CONFIGS_X86_DO(do_var, do_fun) \
+ do_var(int, AVX3Threshold) /* array copy stubs and nmethods */ \
+ do_var(bool, EnableX86ECoreOpts) /* nmethods */ \
+ do_var(int, UseAVX) /* array copy stubs and nmethods */ \
+ do_var(bool, UseAPX) /* nmethods and stubs */ \
+ do_var(bool, UseLibmIntrinsic) \
+ do_var(bool, UseIntPolyIntrinsics) \
+ // END
+#else
+#define AOTCODECACHE_CONFIGS_X86_DO(do_var, do_fun)
+#endif
+
+#define AOTCODECACHE_CONFIGS_DO(do_var, do_fun) \
+ AOTCODECACHE_CONFIGS_GENERIC_DO(do_var, do_fun) \
+ AOTCODECACHE_CONFIGS_COMPILER2_DO(do_var, do_fun) \
+ AOTCODECACHE_CONFIGS_JVMCI_DO(do_var, do_fun) \
+ AOTCODECACHE_CONFIGS_AARCH64_DO(do_var, do_fun) \
+ AOTCODECACHE_CONFIGS_X86_DO(do_var, do_fun) \
+ // END
+
+#define AOTCODECACHE_DECLARE_VAR(type, name) type _saved_ ## name;
+#define AOTCODECACHE_DECLARE_FUN(type, name, func) type _saved_ ## name;
+
class AOTCodeCache : public CHeapObj {
// Classes used to describe AOT code cache.
protected:
class Config {
- address _compressedOopBase;
- uint _compressedOopShift;
- uint _compressedKlassShift;
- uint _contendedPaddingWidth;
- uint _gc;
- enum Flags {
- none = 0,
- debugVM = 1,
- compressedOops = 2,
- compressedClassPointers = 4,
- useTLAB = 8,
- systemClassAssertions = 16,
- userClassAssertions = 32,
- enableContendedPadding = 64,
- restrictContendedPadding = 128
- };
- uint _flags;
- uint _cpu_features_offset; // offset in the cache where cpu features are stored
+ AOTCODECACHE_CONFIGS_DO(AOTCODECACHE_DECLARE_VAR, AOTCODECACHE_DECLARE_FUN)
+ // Special configs that cannot be checked with macros
+ address _compressedOopBase;
+
+#if defined(X86) && !defined(ZERO)
+ bool _useUnalignedLoadStores;
+#endif
+
+#if defined(AARCH64) && !defined(ZERO)
+ bool _avoidUnalignedAccesses;
+#endif
+
+ uint _cpu_features_offset; // offset in the cache where cpu features are stored
public:
void record(uint cpu_features_offset);
bool verify_cpu_features(AOTCodeCache* cache) const;
@@ -259,7 +354,6 @@ private:
uint _store_size; // Used when writing cache
bool _for_use; // AOT cache is open for using AOT code
bool _for_dump; // AOT cache is open for dumping AOT code
- bool _closing; // Closing cache file
bool _failed; // Failed read/write to/from cache (cache is broken?)
bool _lookup_failed; // Failed to lookup for info (skip only this code load)
@@ -289,7 +383,6 @@ private:
public:
AOTCodeCache(bool is_dumping, bool is_using);
- ~AOTCodeCache();
const char* cache_buffer() const { return _load_buffer; }
bool failed() const { return _failed; }
@@ -313,8 +406,6 @@ public:
bool for_use() const { return _for_use && !_failed; }
bool for_dump() const { return _for_dump && !_failed; }
- bool closing() const { return _closing; }
-
AOTCodeEntry* add_entry() {
_store_entries_cnt++;
_store_entries -= 1;
@@ -374,8 +465,8 @@ public:
static AOTCodeCache* cache() { assert(_passed_init2, "Too early to ask"); return _cache; }
static void initialize() NOT_CDS_RETURN;
static void init2() NOT_CDS_RETURN;
- static void close() NOT_CDS_RETURN;
- static bool is_on() CDS_ONLY({ return cache() != nullptr && !_cache->closing(); }) NOT_CDS_RETURN_(false);
+ static void dump() NOT_CDS_RETURN;
+ static bool is_on() CDS_ONLY({ return cache() != nullptr; }) NOT_CDS_RETURN_(false);
static bool is_on_for_use() CDS_ONLY({ return is_on() && _cache->for_use(); }) NOT_CDS_RETURN_(false);
static bool is_on_for_dump() CDS_ONLY({ return is_on() && _cache->for_dump(); }) NOT_CDS_RETURN_(false);
static bool is_dumping_stub() NOT_CDS_RETURN_(false);
@@ -407,11 +498,13 @@ private:
void clear_lookup_failed() { _lookup_failed = false; }
bool lookup_failed() const { return _lookup_failed; }
- AOTCodeEntry* aot_code_entry() { return (AOTCodeEntry*)_entry; }
-public:
- AOTCodeReader(AOTCodeCache* cache, AOTCodeEntry* entry);
+ // Values used by restore(code_blob).
+ // They should be set before calling it.
+ const char* _name;
+ address _reloc_data;
+ ImmutableOopMapSet* _oop_maps;
- CodeBlob* compile_code_blob(const char* name);
+ AOTCodeEntry* aot_code_entry() { return (AOTCodeEntry*)_entry; }
ImmutableOopMapSet* read_oop_map_set();
@@ -420,6 +513,45 @@ public:
void read_asm_remarks(AsmRemarks& asm_remarks);
void read_dbg_strings(DbgStrings& dbg_strings);
#endif // PRODUCT
+
+public:
+ AOTCodeReader(AOTCodeCache* cache, AOTCodeEntry* entry);
+
+ CodeBlob* compile_code_blob(const char* name);
+
+ void restore(CodeBlob* code_blob);
+};
+
+// code cache internal runtime constants area used by AOT code
+class AOTRuntimeConstants {
+ friend class AOTCodeCache;
+ private:
+ address _card_table_base;
+ uint _grain_shift;
+ static address _field_addresses_list[];
+ static AOTRuntimeConstants _aot_runtime_constants;
+ // private constructor for unique singleton
+ AOTRuntimeConstants() { }
+ // private for use by friend class AOTCodeCache
+ static void initialize_from_runtime();
+ public:
+#if INCLUDE_CDS
+ static bool contains(address adr) {
+ address base = (address)&_aot_runtime_constants;
+ address hi = base + sizeof(AOTRuntimeConstants);
+ return (base <= adr && adr < hi);
+ }
+ static address card_table_base_address();
+ static address grain_shift_address() { return (address)&_aot_runtime_constants._grain_shift; }
+ static address* field_addresses_list() {
+ return _field_addresses_list;
+ }
+#else
+ static bool contains(address adr) { return false; }
+ static address card_table_base_address() { return nullptr; }
+ static address grain_shift_address() { return nullptr; }
+ static address* field_addresses_list() { return nullptr; }
+#endif
};
#endif // SHARE_CODE_AOTCODECACHE_HPP
diff --git a/src/hotspot/share/code/codeBlob.cpp b/src/hotspot/share/code/codeBlob.cpp
index fcc0b42a461..e0c286937d0 100644
--- a/src/hotspot/share/code/codeBlob.cpp
+++ b/src/hotspot/share/code/codeBlob.cpp
@@ -22,6 +22,7 @@
*
*/
+#include "code/aotCodeCache.hpp"
#include "code/codeBlob.hpp"
#include "code/codeCache.hpp"
#include "code/relocInfo.hpp"
@@ -39,6 +40,7 @@
#include "prims/forte.hpp"
#include "prims/jvmtiExport.hpp"
#include "runtime/handles.inline.hpp"
+#include "runtime/icache.hpp"
#include "runtime/interfaceSupport.inline.hpp"
#include "runtime/javaFrameAnchor.hpp"
#include "runtime/jniHandles.inline.hpp"
@@ -77,8 +79,10 @@ const BufferBlob::Vptr BufferBlob::_vpntr;
const RuntimeStub::Vptr RuntimeStub::_vpntr;
const SingletonBlob::Vptr SingletonBlob::_vpntr;
const DeoptimizationBlob::Vptr DeoptimizationBlob::_vpntr;
+const SafepointBlob::Vptr SafepointBlob::_vpntr;
#ifdef COMPILER2
const ExceptionBlob::Vptr ExceptionBlob::_vpntr;
+const UncommonTrapBlob::Vptr UncommonTrapBlob::_vpntr;
#endif // COMPILER2
const UpcallStub::Vptr UpcallStub::_vpntr;
@@ -188,22 +192,6 @@ CodeBlob::CodeBlob(const char* name, CodeBlobKind kind, int size, uint16_t heade
assert(_mutable_data == blob_end(), "sanity");
}
-void CodeBlob::restore_mutable_data(address reloc_data) {
- // Relocation data is now stored as part of the mutable data area; allocate it before copy relocations
- if (_mutable_data_size > 0) {
- _mutable_data = (address)os::malloc(_mutable_data_size, mtCode);
- if (_mutable_data == nullptr) {
- vm_exit_out_of_memory(_mutable_data_size, OOM_MALLOC_ERROR, "codebuffer: no space for mutable data");
- }
- } else {
- _mutable_data = blob_end(); // default value
- }
- if (_relocation_size > 0) {
- assert(_mutable_data_size > 0, "relocation is part of mutable data section");
- memcpy((address)relocation_begin(), reloc_data, relocation_size());
- }
-}
-
void CodeBlob::purge() {
assert(_mutable_data != nullptr, "should never be null");
if (_mutable_data != blob_end()) {
@@ -240,6 +228,23 @@ void CodeBlob::print_code_on(outputStream* st) {
Disassembler::decode(this, st);
}
+#if INCLUDE_CDS
+void CodeBlob::restore_mutable_data(address reloc_data) {
+ // Relocation data is now stored as part of the mutable data area; allocate it before copy relocations
+ if (_mutable_data_size > 0) {
+ _mutable_data = (address)os::malloc(_mutable_data_size, mtCode);
+ if (_mutable_data == nullptr) {
+ vm_exit_out_of_memory(_mutable_data_size, OOM_MALLOC_ERROR, "codebuffer: no space for mutable data");
+ }
+ } else {
+ _mutable_data = blob_end(); // default value
+ }
+ if (_relocation_size > 0) {
+ assert(_mutable_data_size > 0, "relocation is part of mutable data section");
+ memcpy((address)relocation_begin(), reloc_data, relocation_size());
+ }
+}
+
void CodeBlob::prepare_for_archiving_impl() {
set_name(nullptr);
_oop_maps = nullptr;
@@ -269,24 +274,15 @@ void CodeBlob::post_restore() {
vptr(_kind)->post_restore(this);
}
-CodeBlob* CodeBlob::restore(address code_cache_buffer,
- const char* name,
- address archived_reloc_data,
- ImmutableOopMapSet* archived_oop_maps)
+CodeBlob* CodeBlob::restore(address code_cache_buffer, AOTCodeReader* reader)
{
copy_to(code_cache_buffer);
CodeBlob* code_blob = (CodeBlob*)code_cache_buffer;
- code_blob->set_name(name);
- code_blob->restore_mutable_data(archived_reloc_data);
- code_blob->set_oop_maps(archived_oop_maps);
+ reader->restore(code_blob);
return code_blob;
}
-CodeBlob* CodeBlob::create(CodeBlob* archived_blob,
- const char* name,
- address archived_reloc_data,
- ImmutableOopMapSet* archived_oop_maps
- )
+CodeBlob* CodeBlob::create(CodeBlob* archived_blob, AOTCodeReader* reader)
{
ThreadInVMfromUnknown __tiv; // get to VM state in case we block on CodeCache_lock
@@ -298,10 +294,7 @@ CodeBlob* CodeBlob::create(CodeBlob* archived_blob,
MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
address code_cache_buffer = (address)CodeCache::allocate(size, CodeBlobType::NonNMethod);
if (code_cache_buffer != nullptr) {
- blob = archived_blob->restore(code_cache_buffer,
- name,
- archived_reloc_data,
- archived_oop_maps);
+ blob = archived_blob->restore(code_cache_buffer, reader);
assert(blob != nullptr, "sanity check");
// Flush the code block
@@ -315,6 +308,8 @@ CodeBlob* CodeBlob::create(CodeBlob* archived_blob,
return blob;
}
+#endif // INCLUDE_CDS
+
//-----------------------------------------------------------------------------------------
// Creates a RuntimeBlob from a CodeBuffer and copy code and relocation info.
@@ -331,7 +326,15 @@ RuntimeBlob::RuntimeBlob(
: CodeBlob(name, kind, cb, size, header_size, frame_complete, frame_size, oop_maps, caller_must_gc_arguments,
align_up(cb->total_relocation_size(), oopSize))
{
+ if (code_size() == 0) {
+ // Nothing to copy
+ return;
+ }
+
cb->copy_code_and_locs_to(this);
+
+ // Flush generated code
+ ICache::invalidate_range(code_begin(), code_size());
}
void RuntimeBlob::free(RuntimeBlob* blob) {
@@ -390,7 +393,7 @@ void RuntimeBlob::trace_new_stub(RuntimeBlob* stub, const char* name1, const cha
// Implementation of BufferBlob
BufferBlob::BufferBlob(const char* name, CodeBlobKind kind, int size, uint16_t header_size)
-: RuntimeBlob(name, kind, size, header_size)
+ : RuntimeBlob(name, kind, size, header_size)
{}
BufferBlob* BufferBlob::create(const char* name, uint buffer_size) {
@@ -625,8 +628,8 @@ DeoptimizationBlob::DeoptimizationBlob(
int unpack_with_reexecution_offset,
int frame_size
)
-: SingletonBlob("DeoptimizationBlob", CodeBlobKind::Deoptimization, cb,
- size, sizeof(DeoptimizationBlob), frame_size, oop_maps)
+ : SingletonBlob("DeoptimizationBlob", CodeBlobKind::Deoptimization, cb,
+ size, sizeof(DeoptimizationBlob), frame_size, oop_maps)
{
_unpack_offset = unpack_offset;
_unpack_with_exception = unpack_with_exception_offset;
@@ -675,8 +678,8 @@ UncommonTrapBlob::UncommonTrapBlob(
OopMapSet* oop_maps,
int frame_size
)
-: SingletonBlob("UncommonTrapBlob", CodeBlobKind::UncommonTrap, cb,
- size, sizeof(UncommonTrapBlob), frame_size, oop_maps)
+ : SingletonBlob("UncommonTrapBlob", CodeBlobKind::UncommonTrap, cb,
+ size, sizeof(UncommonTrapBlob), frame_size, oop_maps)
{}
@@ -707,8 +710,8 @@ ExceptionBlob::ExceptionBlob(
OopMapSet* oop_maps,
int frame_size
)
-: SingletonBlob("ExceptionBlob", CodeBlobKind::Exception, cb,
- size, sizeof(ExceptionBlob), frame_size, oop_maps)
+ : SingletonBlob("ExceptionBlob", CodeBlobKind::Exception, cb,
+ size, sizeof(ExceptionBlob), frame_size, oop_maps)
{}
@@ -741,8 +744,8 @@ SafepointBlob::SafepointBlob(
OopMapSet* oop_maps,
int frame_size
)
-: SingletonBlob("SafepointBlob", CodeBlobKind::Safepoint, cb,
- size, sizeof(SafepointBlob), frame_size, oop_maps)
+ : SingletonBlob(cb->name(), CodeBlobKind::Safepoint, cb,
+ size, sizeof(SafepointBlob), frame_size, oop_maps)
{}
@@ -759,7 +762,7 @@ SafepointBlob* SafepointBlob::create(
blob = new (size) SafepointBlob(cb, size, oop_maps, frame_size);
}
- trace_new_stub(blob, "SafepointBlob");
+ trace_new_stub(blob, "SafepointBlob - ", blob->name());
return blob;
}
@@ -899,7 +902,7 @@ void CodeBlob::dump_for_addr(address addr, outputStream* st, bool verbose) const
}
}
if (is_nmethod()) {
- nmethod* nm = (nmethod*)this;
+ nmethod* nm = as_nmethod();
ResourceMark rm;
st->print(INTPTR_FORMAT " is at entry_point+%d in (nmethod*)" INTPTR_FORMAT,
p2i(addr), (int)(addr - nm->entry_point()), p2i(nm));
@@ -935,7 +938,7 @@ void RuntimeStub::print_on_impl(outputStream* st) const {
RuntimeBlob::print_on_impl(st);
st->print("Runtime Stub (" INTPTR_FORMAT "): ", p2i(this));
st->print_cr("%s", name());
- Disassembler::decode((RuntimeBlob*)this, st);
+ Disassembler::decode((CodeBlob*)this, st);
}
void RuntimeStub::print_value_on_impl(outputStream* st) const {
@@ -946,7 +949,7 @@ void SingletonBlob::print_on_impl(outputStream* st) const {
ttyLocker ttyl;
RuntimeBlob::print_on_impl(st);
st->print_cr("%s", name());
- Disassembler::decode((RuntimeBlob*)this, st);
+ Disassembler::decode((CodeBlob*)this, st);
}
void SingletonBlob::print_value_on_impl(outputStream* st) const {
@@ -964,7 +967,7 @@ void UpcallStub::print_on_impl(outputStream* st) const {
oop recv = JNIHandles::resolve(_receiver);
st->print("Receiver MH=");
recv->print_on(st);
- Disassembler::decode((RuntimeBlob*)this, st);
+ Disassembler::decode((CodeBlob*)this, st);
}
void UpcallStub::print_value_on_impl(outputStream* st) const {
diff --git a/src/hotspot/share/code/codeBlob.hpp b/src/hotspot/share/code/codeBlob.hpp
index 0469b6c71b1..6a1686b80e2 100644
--- a/src/hotspot/share/code/codeBlob.hpp
+++ b/src/hotspot/share/code/codeBlob.hpp
@@ -34,6 +34,7 @@
#include "utilities/align.hpp"
#include "utilities/macros.hpp"
+class AOTCodeReader;
class ImmutableOopMap;
class ImmutableOopMapSet;
class JNIHandleBlock;
@@ -97,7 +98,9 @@ enum class CodeBlobKind : u1 {
class UpcallStub; // for as_upcall_stub()
class RuntimeStub; // for as_runtime_stub()
class JavaFrameAnchor; // for UpcallStub::jfa_for_frame
+class BufferBlob;
class AdapterBlob;
+class SingletonBlob;
class ExceptionBlob;
class DeoptimizationBlob;
class SafepointBlob;
@@ -107,9 +110,6 @@ class CodeBlob {
friend class VMStructs;
friend class JVMCIVMStructs;
-private:
- void restore_mutable_data(address reloc_data);
-
protected:
// order fields from large to small to minimize padding between fields
ImmutableOopMapSet* _oop_maps; // OopMap for this CodeBlob
@@ -169,8 +169,8 @@ protected:
void operator delete(void* p) { }
- void prepare_for_archiving_impl();
- void post_restore_impl();
+ void prepare_for_archiving_impl() NOT_CDS_RETURN;
+ void post_restore_impl() NOT_CDS_RETURN;
public:
@@ -187,8 +187,19 @@ public:
// Typing
bool is_nmethod() const { return _kind == CodeBlobKind::Nmethod; }
- bool is_buffer_blob() const { return _kind == CodeBlobKind::Buffer; }
+ // we may want to check for an actual buffer blob or subtype instance
+ bool is_buffer_blob(bool strict=true) const {
+ if (strict) {
+ return _kind == CodeBlobKind::Buffer;
+ } else {
+ return (_kind == CodeBlobKind::Buffer ||
+ _kind == CodeBlobKind::Adapter ||
+ _kind == CodeBlobKind::Vtable ||
+ _kind == CodeBlobKind::MHAdapter);
+ }
+ }
bool is_runtime_stub() const { return _kind == CodeBlobKind::RuntimeStub; }
+ // singleton blobs are never directly implemented
bool is_deoptimization_stub() const { return _kind == CodeBlobKind::Deoptimization; }
#ifdef COMPILER2
bool is_uncommon_trap_stub() const { return _kind == CodeBlobKind::UncommonTrap; }
@@ -198,6 +209,12 @@ public:
bool is_exception_stub() const { return false; }
#endif
bool is_safepoint_stub() const { return _kind == CodeBlobKind::Safepoint; }
+ bool is_singleton_blob() const {
+ return (is_deoptimization_stub() ||
+ is_uncommon_trap_stub() ||
+ is_exception_stub() ||
+ is_safepoint_stub());
+ }
bool is_adapter_blob() const { return _kind == CodeBlobKind::Adapter; }
bool is_vtable_blob() const { return _kind == CodeBlobKind::Vtable; }
bool is_method_handles_adapter_blob() const { return _kind == CodeBlobKind::MHAdapter; }
@@ -207,8 +224,12 @@ public:
nmethod* as_nmethod_or_null() const { return is_nmethod() ? (nmethod*) this : nullptr; }
nmethod* as_nmethod() const { assert(is_nmethod(), "must be nmethod"); return (nmethod*) this; }
CodeBlob* as_codeblob() const { return (CodeBlob*) this; }
+ // we may want to force an actual buffer blob or subtype instance
+ BufferBlob* as_buffer_blob(bool strict = true) const { assert(is_buffer_blob(), "must be %sbuffer blob", (strict ? "strict " : "")); return (BufferBlob*) this; }
AdapterBlob* as_adapter_blob() const { assert(is_adapter_blob(), "must be adapter blob"); return (AdapterBlob*) this; }
ExceptionBlob* as_exception_blob() const { assert(is_exception_stub(), "must be exception stub"); return (ExceptionBlob*) this; }
+ // this will always return a subtype instance
+ SingletonBlob* as_singleton_blob() const { assert(is_singleton_blob(), "must be singleton blob"); return (SingletonBlob*) this; }
DeoptimizationBlob* as_deoptimization_blob() const { assert(is_deoptimization_stub(), "must be deopt stub"); return (DeoptimizationBlob*) this; }
SafepointBlob* as_safepoint_blob() const { assert(is_safepoint_stub(), "must be safepoint stub"); return (SafepointBlob*) this; }
UpcallStub* as_upcall_stub() const { assert(is_upcall_stub(), "must be upcall stub"); return (UpcallStub*) this; }
@@ -304,6 +325,9 @@ public:
void use_strings(DbgStrings &strings) { _dbg_strings.share(strings); }
#endif
+#if INCLUDE_CDS
+ void restore_mutable_data(address reloc_data);
+
void copy_to(address buffer) {
memcpy(buffer, this, this->size());
}
@@ -314,11 +338,9 @@ public:
// methods to restore a blob from AOT code cache into the CodeCache
void post_restore();
- CodeBlob* restore(address code_cache_buffer, const char* name, address archived_reloc_data, ImmutableOopMapSet* archived_oop_maps);
- static CodeBlob* create(CodeBlob* archived_blob,
- const char* name,
- address archived_reloc_data,
- ImmutableOopMapSet* archived_oop_maps);
+ CodeBlob* restore(address code_cache_buffer, AOTCodeReader* reader);
+ static CodeBlob* create(CodeBlob* archived_blob, AOTCodeReader* reader);
+#endif
};
//----------------------------------------------------------------------------------------------------
@@ -388,10 +410,10 @@ class BufferBlob: public RuntimeBlob {
class Vptr : public RuntimeBlob::Vptr {
void print_on(const CodeBlob* instance, outputStream* st) const override {
- ((const BufferBlob*)instance)->print_on_impl(st);
+ instance->as_buffer_blob(false)->print_on_impl(st);
}
void print_value_on(const CodeBlob* instance, outputStream* st) const override {
- ((const BufferBlob*)instance)->print_value_on_impl(st);
+ instance->as_buffer_blob(false)->print_value_on_impl(st);
}
};
@@ -487,10 +509,17 @@ class RuntimeStub: public RuntimeBlob {
address entry_point() const { return code_begin(); }
+ void post_restore_impl() {
+ trace_new_stub(this, "RuntimeStub - ", name());
+ }
+
void print_on_impl(outputStream* st) const;
void print_value_on_impl(outputStream* st) const;
class Vptr : public RuntimeBlob::Vptr {
+ void post_restore(CodeBlob* instance) const override {
+ instance->as_runtime_stub()->post_restore_impl();
+ }
void print_on(const CodeBlob* instance, outputStream* st) const override {
instance->as_runtime_stub()->print_on_impl(st);
}
@@ -532,10 +561,10 @@ class SingletonBlob: public RuntimeBlob {
class Vptr : public RuntimeBlob::Vptr {
void print_on(const CodeBlob* instance, outputStream* st) const override {
- ((const SingletonBlob*)instance)->print_on_impl(st);
+ instance->as_singleton_blob()->print_on_impl(st);
}
void print_value_on(const CodeBlob* instance, outputStream* st) const override {
- ((const SingletonBlob*)instance)->print_value_on_impl(st);
+ instance->as_singleton_blob()->print_value_on_impl(st);
}
};
@@ -606,20 +635,28 @@ class DeoptimizationBlob: public SingletonBlob {
_uncommon_trap_offset = offset;
assert(contains(code_begin() + _uncommon_trap_offset), "must be PC inside codeblob");
}
- address uncommon_trap() const { return code_begin() + _uncommon_trap_offset; }
+ address uncommon_trap() const { return (EnableJVMCI ? code_begin() + _uncommon_trap_offset : nullptr); }
void set_implicit_exception_uncommon_trap_offset(int offset) {
_implicit_exception_uncommon_trap_offset = offset;
assert(contains(code_begin() + _implicit_exception_uncommon_trap_offset), "must be PC inside codeblob");
}
- address implicit_exception_uncommon_trap() const { return code_begin() + _implicit_exception_uncommon_trap_offset; }
+ address implicit_exception_uncommon_trap() const { return (EnableJVMCI ? code_begin() + _implicit_exception_uncommon_trap_offset : nullptr); }
#endif // INCLUDE_JVMCI
+ void post_restore_impl() {
+ trace_new_stub(this, "DeoptimizationBlob");
+ }
+
void print_value_on_impl(outputStream* st) const;
class Vptr : public SingletonBlob::Vptr {
+ void post_restore(CodeBlob* instance) const override {
+ instance->as_deoptimization_blob()->post_restore_impl();
+ }
+
void print_value_on(const CodeBlob* instance, outputStream* st) const override {
- ((const DeoptimizationBlob*)instance)->print_value_on_impl(st);
+ instance->as_deoptimization_blob()->print_value_on_impl(st);
}
};
@@ -649,6 +686,16 @@ class UncommonTrapBlob: public SingletonBlob {
OopMapSet* oop_maps,
int frame_size
);
+ void post_restore_impl() {
+ trace_new_stub(this, "UncommonTrapBlob");
+ }
+ class Vptr : public SingletonBlob::Vptr {
+ void post_restore(CodeBlob* instance) const override {
+ instance->as_uncommon_trap_blob()->post_restore_impl();
+ }
+ };
+
+ static const Vptr _vpntr;
};
@@ -679,7 +726,7 @@ class ExceptionBlob: public SingletonBlob {
class Vptr : public SingletonBlob::Vptr {
void post_restore(CodeBlob* instance) const override {
- ((ExceptionBlob*)instance)->post_restore_impl();
+ instance->as_exception_blob()->post_restore_impl();
}
};
@@ -709,6 +756,17 @@ class SafepointBlob: public SingletonBlob {
OopMapSet* oop_maps,
int frame_size
);
+
+ void post_restore_impl() {
+ trace_new_stub(this, "SafepointBlob - ", name());
+ }
+ class Vptr : public SingletonBlob::Vptr {
+ void post_restore(CodeBlob* instance) const override {
+ instance->as_safepoint_blob()->post_restore_impl();
+ }
+ };
+
+ static const Vptr _vpntr;
};
//----------------------------------------------------------------------------------------------------
diff --git a/src/hotspot/share/code/compiledIC.cpp b/src/hotspot/share/code/compiledIC.cpp
index 5f5c9711441..07d96d6cd44 100644
--- a/src/hotspot/share/code/compiledIC.cpp
+++ b/src/hotspot/share/code/compiledIC.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -76,11 +76,7 @@ CompiledICData::CompiledICData()
// Inline cache callsite info is initialized once the first time it is resolved
void CompiledICData::initialize(CallInfo* call_info, Klass* receiver_klass) {
_speculated_method = call_info->selected_method();
- if (UseCompressedClassPointers) {
- _speculated_klass = (uintptr_t)CompressedKlassPointers::encode_not_null(receiver_klass);
- } else {
- _speculated_klass = (uintptr_t)receiver_klass;
- }
+ _speculated_klass = (uintptr_t)CompressedKlassPointers::encode_not_null(receiver_klass);
if (call_info->call_kind() == CallInfo::itable_call) {
assert(call_info->resolved_method() != nullptr, "virtual or interface method must be found");
_itable_defc_klass = call_info->resolved_method()->method_holder();
@@ -133,12 +129,7 @@ Klass* CompiledICData::speculated_klass() const {
if (is_speculated_klass_unloaded()) {
return nullptr;
}
-
- if (UseCompressedClassPointers) {
- return CompressedKlassPointers::decode_not_null((narrowKlass)_speculated_klass);
- } else {
- return (Klass*)_speculated_klass;
- }
+ return CompressedKlassPointers::decode_not_null((narrowKlass)_speculated_klass);
}
//-----------------------------------------------------------------------------
diff --git a/src/hotspot/share/code/nmethod.cpp b/src/hotspot/share/code/nmethod.cpp
index 13eb1ff1604..de2c826667f 100644
--- a/src/hotspot/share/code/nmethod.cpp
+++ b/src/hotspot/share/code/nmethod.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -66,6 +66,7 @@
#include "runtime/flags/flagSetting.hpp"
#include "runtime/frame.inline.hpp"
#include "runtime/handles.inline.hpp"
+#include "runtime/icache.hpp"
#include "runtime/jniHandles.inline.hpp"
#include "runtime/orderAccess.hpp"
#include "runtime/os.hpp"
@@ -938,7 +939,8 @@ address nmethod::continuation_for_implicit_exception(address pc, bool for_div0_c
stringStream ss;
ss.print_cr("implicit exception happened at " INTPTR_FORMAT, p2i(pc));
print_on(&ss);
- method()->print_codes_on(&ss);
+ // Buffering to a stringStream, disable internal buffering so it's not done twice.
+ method()->print_codes_on(&ss, 0, false);
print_code_on(&ss);
print_pcs_on(&ss);
tty->print("%s", ss.as_string()); // print all at once
@@ -1252,6 +1254,9 @@ void nmethod::post_init() {
finalize_relocations();
+ // Flush generated code
+ ICache::invalidate_range(code_begin(), code_size());
+
Universe::heap()->register_nmethod(this);
DEBUG_ONLY(Universe::heap()->verify_nmethod(this));
@@ -1305,9 +1310,7 @@ nmethod::nmethod(
_deopt_handler_entry_offset = 0;
_unwind_handler_offset = 0;
- CHECKED_CAST(_oops_size, uint16_t, align_up(code_buffer->total_oop_size(), oopSize));
- uint16_t metadata_size;
- CHECKED_CAST(metadata_size, uint16_t, align_up(code_buffer->total_metadata_size(), wordSize));
+ int metadata_size = align_up(code_buffer->total_metadata_size(), wordSize);
JVMCI_ONLY( _metadata_size = metadata_size; )
assert(_mutable_data_size == _relocation_size + metadata_size,
"wrong mutable data size: %d != %d + %d",
@@ -1329,6 +1332,7 @@ nmethod::nmethod(
code_buffer->copy_values_to(this);
post_init();
+ ICache::invalidate_range(code_begin(), code_size());
}
if (PrintNativeNMethods || PrintDebugInfo || PrintRelocations || PrintDependencies) {
@@ -1445,7 +1449,6 @@ nmethod::nmethod(const nmethod &nm) : CodeBlob(nm._name, nm._kind, nm._size, nm.
_deopt_handler_entry_offset = nm._deopt_handler_entry_offset;
_unwind_handler_offset = nm._unwind_handler_offset;
_num_stack_arg_slots = nm._num_stack_arg_slots;
- _oops_size = nm._oops_size;
#if INCLUDE_JVMCI
_metadata_size = nm._metadata_size;
#endif
@@ -1586,8 +1589,6 @@ nmethod* nmethod::relocate(CodeBlobType code_blob_type) {
// Attempt to start using the copy
if (nm_copy->make_in_use()) {
- ICache::invalidate_range(nm_copy->code_begin(), nm_copy->code_size());
-
methodHandle mh(Thread::current(), nm_copy->method());
nm_copy->method()->set_code(mh, nm_copy);
@@ -1748,9 +1749,7 @@ nmethod::nmethod(
_unwind_handler_offset = -1;
}
- CHECKED_CAST(_oops_size, uint16_t, align_up(code_buffer->total_oop_size(), oopSize));
- uint16_t metadata_size;
- CHECKED_CAST(metadata_size, uint16_t, align_up(code_buffer->total_metadata_size(), wordSize));
+ int metadata_size = align_up(code_buffer->total_metadata_size(), wordSize);
JVMCI_ONLY( _metadata_size = metadata_size; )
int jvmci_data_size = 0 JVMCI_ONLY( + align_up(compiler->is_jvmci() ? jvmci_data->size() : 0, oopSize));
assert(_mutable_data_size == _relocation_size + metadata_size + jvmci_data_size,
@@ -1813,6 +1812,7 @@ nmethod::nmethod(
init_immutable_data_ref_count();
post_init();
+ ICache::invalidate_range(code_begin(), code_size());
// we use the information of entry points to find out if a method is
// static or non static
@@ -2040,7 +2040,7 @@ void nmethod::copy_values(GrowableArray* array) {
// The code and relocations have already been initialized by the
// CodeBlob constructor, so it is valid even at this early point to
// iterate over relocations and patch the code.
- fix_oop_relocations(nullptr, nullptr, /*initialize_immediates=*/ true);
+ fix_oop_relocations(/*initialize_immediates=*/ true);
}
void nmethod::copy_values(GrowableArray* array) {
@@ -2052,24 +2052,42 @@ void nmethod::copy_values(GrowableArray* array) {
}
}
-void nmethod::fix_oop_relocations(address begin, address end, bool initialize_immediates) {
+bool nmethod::fix_oop_relocations(bool initialize_immediates) {
// re-patch all oop-bearing instructions, just in case some oops moved
- RelocIterator iter(this, begin, end);
+ RelocIterator iter(this);
+ bool modified_code = false;
while (iter.next()) {
if (iter.type() == relocInfo::oop_type) {
oop_Relocation* reloc = iter.oop_reloc();
- if (initialize_immediates && reloc->oop_is_immediate()) {
+ if (!reloc->oop_is_immediate()) {
+ // Refresh the oop-related bits of this instruction.
+ reloc->set_value(reloc->value());
+ modified_code = true;
+ } else if (initialize_immediates) {
oop* dest = reloc->oop_addr();
jobject obj = *reinterpret_cast(dest);
initialize_immediate_oop(dest, obj);
}
- // Refresh the oop-related bits of this instruction.
- reloc->fix_oop_relocation();
} else if (iter.type() == relocInfo::metadata_type) {
metadata_Relocation* reloc = iter.metadata_reloc();
reloc->fix_metadata_relocation();
+ modified_code |= !reloc->metadata_is_immediate();
}
}
+ return modified_code;
+}
+
+void nmethod::fix_oop_relocations() {
+ ICacheInvalidationContext icic;
+ fix_oop_relocations(&icic);
+}
+
+void nmethod::fix_oop_relocations(ICacheInvalidationContext* icic) {
+ assert(icic != nullptr, "must provide context to track if code was modified");
+ bool modified_code = fix_oop_relocations(/*initialize_immediates=*/ false);
+ if (modified_code) {
+ icic->set_has_modified_code();
+ }
}
static void install_post_call_nop_displacement(nmethod* nm, address pc) {
diff --git a/src/hotspot/share/code/nmethod.hpp b/src/hotspot/share/code/nmethod.hpp
index 2391bc6d830..ea8c0e2ad5d 100644
--- a/src/hotspot/share/code/nmethod.hpp
+++ b/src/hotspot/share/code/nmethod.hpp
@@ -41,6 +41,7 @@ class Dependencies;
class DirectiveSet;
class DebugInformationRecorder;
class ExceptionHandlerTable;
+class ICacheInvalidationContext;
class ImplicitExceptionTable;
class JvmtiThreadState;
class MetadataClosure;
@@ -235,11 +236,10 @@ class nmethod : public CodeBlob {
// Number of arguments passed on the stack
uint16_t _num_stack_arg_slots;
- uint16_t _oops_size;
#if INCLUDE_JVMCI
// _metadata_size is not specific to JVMCI. In the non-JVMCI case, it can be derived as:
// _metadata_size = mutable_data_size - relocation_size
- uint16_t _metadata_size;
+ int _metadata_size;
#endif
// Offset in immutable data section
@@ -802,15 +802,15 @@ public:
// Relocation support
private:
- void fix_oop_relocations(address begin, address end, bool initialize_immediates);
+ bool fix_oop_relocations(bool initialize_immediates);
inline void initialize_immediate_oop(oop* dest, jobject handle);
protected:
address oops_reloc_begin() const;
public:
- void fix_oop_relocations(address begin, address end) { fix_oop_relocations(begin, end, false); }
- void fix_oop_relocations() { fix_oop_relocations(nullptr, nullptr, false); }
+ void fix_oop_relocations(ICacheInvalidationContext* icic);
+ void fix_oop_relocations();
bool is_at_poll_return(address pc);
bool is_at_poll_or_poll_return(address pc);
diff --git a/src/hotspot/share/code/relocInfo.cpp b/src/hotspot/share/code/relocInfo.cpp
index 2a6335e2118..25d91edc20f 100644
--- a/src/hotspot/share/code/relocInfo.cpp
+++ b/src/hotspot/share/code/relocInfo.cpp
@@ -590,15 +590,6 @@ oop oop_Relocation::oop_value() {
return *oop_addr();
}
-
-void oop_Relocation::fix_oop_relocation() {
- if (!oop_is_immediate()) {
- // get the oop from the pool, and re-insert it into the instruction:
- set_value(value());
- }
-}
-
-
void oop_Relocation::verify_oop_relocation() {
if (!oop_is_immediate()) {
// get the oop from the pool, and re-insert it into the instruction:
diff --git a/src/hotspot/share/code/relocInfo.hpp b/src/hotspot/share/code/relocInfo.hpp
index 6f1778ef479..bb2b2b5693f 100644
--- a/src/hotspot/share/code/relocInfo.hpp
+++ b/src/hotspot/share/code/relocInfo.hpp
@@ -988,8 +988,6 @@ class oop_Relocation : public DataRelocation {
void pack_data_to(CodeSection* dest) override;
void unpack_data() override;
- void fix_oop_relocation(); // reasserts oop value
-
void verify_oop_relocation();
address value() override { return *reinterpret_cast(oop_addr()); }
diff --git a/src/hotspot/share/compiler/compilerDefinitions.cpp b/src/hotspot/share/compiler/compilerDefinitions.cpp
index 9bd6e893bcd..0e4e211453b 100644
--- a/src/hotspot/share/compiler/compilerDefinitions.cpp
+++ b/src/hotspot/share/compiler/compilerDefinitions.cpp
@@ -185,49 +185,6 @@ intx CompilerConfig::scaled_freq_log(intx freq_log, double scale) {
}
}
-void CompilerConfig::set_client_emulation_mode_flags() {
- assert(has_c1(), "Must have C1 compiler present");
- CompilationModeFlag::set_quick_only();
-
- FLAG_SET_ERGO(ProfileInterpreter, false);
-#if INCLUDE_JVMCI
- FLAG_SET_ERGO(EnableJVMCI, false);
- FLAG_SET_ERGO(UseJVMCICompiler, false);
-#endif
- if (FLAG_IS_DEFAULT(NeverActAsServerClassMachine)) {
- FLAG_SET_ERGO(NeverActAsServerClassMachine, true);
- }
- if (FLAG_IS_DEFAULT(InitialCodeCacheSize)) {
- FLAG_SET_ERGO(InitialCodeCacheSize, 160*K);
- }
- if (FLAG_IS_DEFAULT(ReservedCodeCacheSize)) {
- FLAG_SET_ERGO(ReservedCodeCacheSize, 32*M);
- }
- if (FLAG_IS_DEFAULT(NonProfiledCodeHeapSize)) {
- FLAG_SET_ERGO(NonProfiledCodeHeapSize, 27*M);
- }
- if (FLAG_IS_DEFAULT(ProfiledCodeHeapSize)) {
- FLAG_SET_ERGO(ProfiledCodeHeapSize, 0);
- }
- if (FLAG_IS_DEFAULT(NonNMethodCodeHeapSize)) {
- FLAG_SET_ERGO(NonNMethodCodeHeapSize, 5*M);
- }
- if (FLAG_IS_DEFAULT(CodeCacheExpansionSize)) {
- FLAG_SET_ERGO(CodeCacheExpansionSize, 32*K);
- }
- if (FLAG_IS_DEFAULT(CICompilerCount)) {
- FLAG_SET_ERGO(CICompilerCount, 1);
- }
-}
-
-bool CompilerConfig::is_compilation_mode_selected() {
- return !FLAG_IS_DEFAULT(TieredCompilation) ||
- !FLAG_IS_DEFAULT(TieredStopAtLevel) ||
- !FLAG_IS_DEFAULT(CompilationMode)
- JVMCI_ONLY(|| !FLAG_IS_DEFAULT(EnableJVMCI)
- || !FLAG_IS_DEFAULT(UseJVMCICompiler));
-}
-
static bool check_legacy_flags() {
JVMFlag* compile_threshold_flag = JVMFlag::flag_from_enum(FLAG_MEMBER_ENUM(CompileThreshold));
if (JVMFlagAccess::check_constraint(compile_threshold_flag, JVMFlagLimit::get_constraint(compile_threshold_flag)->constraint_func(), false) != JVMFlag::SUCCESS) {
@@ -447,9 +404,6 @@ void CompilerConfig::set_jvmci_specific_flags() {
if (FLAG_IS_DEFAULT(InitialCodeCacheSize)) {
FLAG_SET_DEFAULT(InitialCodeCacheSize, MAX2(16*M, InitialCodeCacheSize));
}
- if (FLAG_IS_DEFAULT(NewSizeThreadIncrease)) {
- FLAG_SET_DEFAULT(NewSizeThreadIncrease, MAX2(4*K, NewSizeThreadIncrease));
- }
if (FLAG_IS_DEFAULT(Tier3DelayOn)) {
// This effectively prevents the compile broker scheduling tier 2
// (i.e., limited C1 profiling) compilations instead of tier 3
@@ -546,36 +500,11 @@ bool CompilerConfig::check_args_consistency(bool status) {
return status;
}
-bool CompilerConfig::should_set_client_emulation_mode_flags() {
-#if !COMPILER1_OR_COMPILER2
- return false;
-#endif
-
- if (has_c1()) {
- if (!is_compilation_mode_selected()) {
- if (NeverActAsServerClassMachine) {
- return true;
- }
- } else if (!has_c2() && !is_jvmci_compiler()) {
- return true;
- }
- }
-
- return false;
-}
-
void CompilerConfig::ergo_initialize() {
#if !COMPILER1_OR_COMPILER2
return;
#endif
- // This property is also checked when selecting the heap size. Since client
- // emulation mode influences Java heap memory usage, part of the logic must
- // occur before choosing the heap size.
- if (should_set_client_emulation_mode_flags()) {
- set_client_emulation_mode_flags();
- }
-
set_legacy_emulation_flags();
set_compilation_policy_flags();
@@ -594,9 +523,6 @@ void CompilerConfig::ergo_initialize() {
}
if (ProfileInterpreter && CompilerConfig::is_c1_simple_only()) {
- if (!FLAG_IS_DEFAULT(ProfileInterpreter)) {
- warning("ProfileInterpreter disabled due to client emulation mode");
- }
FLAG_SET_CMDLINE(ProfileInterpreter, false);
}
diff --git a/src/hotspot/share/compiler/compilerDefinitions.hpp b/src/hotspot/share/compiler/compilerDefinitions.hpp
index a9b052ff782..e8ba977f705 100644
--- a/src/hotspot/share/compiler/compilerDefinitions.hpp
+++ b/src/hotspot/share/compiler/compilerDefinitions.hpp
@@ -151,14 +151,10 @@ public:
inline static CompilerType compiler_type();
- static bool should_set_client_emulation_mode_flags();
-
private:
- static bool is_compilation_mode_selected();
static void set_compilation_policy_flags();
static void set_jvmci_specific_flags();
static void set_legacy_emulation_flags();
- static void set_client_emulation_mode_flags();
};
#endif // SHARE_COMPILER_COMPILERDEFINITIONS_HPP
diff --git a/src/hotspot/share/compiler/compiler_globals_pd.hpp b/src/hotspot/share/compiler/compiler_globals_pd.hpp
index 6a87fdaaaf1..8ac4b53d6cd 100644
--- a/src/hotspot/share/compiler/compiler_globals_pd.hpp
+++ b/src/hotspot/share/compiler/compiler_globals_pd.hpp
@@ -58,7 +58,6 @@ define_pd_global(bool, TieredCompilation, false);
define_pd_global(intx, CompileThreshold, 0);
define_pd_global(intx, OnStackReplacePercentage, 0);
-define_pd_global(size_t, NewSizeThreadIncrease, 4*K);
define_pd_global(bool, InlineClassNatives, true);
define_pd_global(bool, InlineUnsafeOps, true);
define_pd_global(size_t, InitialCodeCacheSize, 160*K);
@@ -70,13 +69,6 @@ define_pd_global(size_t, NonNMethodCodeHeapSize, 32*M);
define_pd_global(size_t, CodeCacheExpansionSize, 32*K);
define_pd_global(size_t, CodeCacheMinBlockLength, 1);
define_pd_global(size_t, CodeCacheMinimumUseSpace, 200*K);
-#ifndef ZERO
-define_pd_global(bool, NeverActAsServerClassMachine, true);
-#else
-// Zero runs without compilers. Do not let this code to force
-// the GC mode and default heap settings.
-define_pd_global(bool, NeverActAsServerClassMachine, false);
-#endif
#define CI_COMPILER_COUNT 0
#else
diff --git a/src/hotspot/share/compiler/disassembler.hpp b/src/hotspot/share/compiler/disassembler.hpp
index db7066c9023..b9de9c3d27d 100644
--- a/src/hotspot/share/compiler/disassembler.hpp
+++ b/src/hotspot/share/compiler/disassembler.hpp
@@ -112,7 +112,7 @@ class Disassembler : public AbstractDisassembler {
// interpreter code, by riding on the customary __ macro in the interpreter generator.
// See templateTable_x86.cpp for an example.
template inline static T* hook(const char* file, int line, T* masm) {
- if (PrintInterpreter) {
+ if (PrintInterpreter NOT_PRODUCT(|| true)) {
_hook(file, line, masm);
}
return masm;
diff --git a/src/hotspot/share/gc/epsilon/epsilonHeap.cpp b/src/hotspot/share/gc/epsilon/epsilonHeap.cpp
index 004a36147fc..e32daa3d79e 100644
--- a/src/hotspot/share/gc/epsilon/epsilonHeap.cpp
+++ b/src/hotspot/share/gc/epsilon/epsilonHeap.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2023, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, 2022, Red Hat, Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -34,7 +34,6 @@
#include "memory/metaspaceUtils.hpp"
#include "memory/resourceArea.hpp"
#include "memory/universe.hpp"
-#include "runtime/atomicAccess.hpp"
#include "runtime/globals.hpp"
#include "utilities/ostream.hpp"
diff --git a/src/hotspot/share/gc/g1/g1Arguments.cpp b/src/hotspot/share/gc/g1/g1Arguments.cpp
index 8bec6e7e86f..c3bbd5a3b52 100644
--- a/src/hotspot/share/gc/g1/g1Arguments.cpp
+++ b/src/hotspot/share/gc/g1/g1Arguments.cpp
@@ -148,8 +148,9 @@ void G1Arguments::initialize_card_set_configuration() {
if (FLAG_IS_DEFAULT(G1RemSetArrayOfCardsEntries)) {
uint max_cards_in_inline_ptr = G1CardSetConfiguration::max_cards_in_inline_ptr(G1HeapRegion::LogCardsPerRegion);
+ const JVMTypedFlagLimit* limit = JVMFlagLimit::get_range_at(FLAG_MEMBER_ENUM(G1RemSetArrayOfCardsEntries))->cast();
FLAG_SET_ERGO(G1RemSetArrayOfCardsEntries, MAX2(max_cards_in_inline_ptr * 2,
- G1RemSetArrayOfCardsEntriesBase << region_size_log_mb));
+ MIN2(G1RemSetArrayOfCardsEntriesBase << region_size_log_mb, limit->max())));
}
// Howl card set container globals.
diff --git a/src/hotspot/share/gc/g1/g1BarrierSet.hpp b/src/hotspot/share/gc/g1/g1BarrierSet.hpp
index 406096acf10..c5c7058471c 100644
--- a/src/hotspot/share/gc/g1/g1BarrierSet.hpp
+++ b/src/hotspot/share/gc/g1/g1BarrierSet.hpp
@@ -25,6 +25,7 @@
#ifndef SHARE_GC_G1_G1BARRIERSET_HPP
#define SHARE_GC_G1_G1BARRIERSET_HPP
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1SATBMarkQueueSet.hpp"
#include "gc/shared/bufferNode.hpp"
#include "gc/shared/cardTable.hpp"
@@ -116,6 +117,8 @@ class G1BarrierSet: public CardTableBarrierSet {
virtual void print_on(outputStream* st) const;
+ virtual uint grain_shift() { return G1HeapRegion::LogOfHRGrainBytes; }
+
// Callbacks for runtime accesses.
template
class AccessBarrier: public CardTableBarrierSet::AccessBarrier {
diff --git a/src/hotspot/share/gc/g1/g1BlockOffsetTable.cpp b/src/hotspot/share/gc/g1/g1BlockOffsetTable.cpp
index fd70796251d..7f0e5e86cd9 100644
--- a/src/hotspot/share/gc/g1/g1BlockOffsetTable.cpp
+++ b/src/hotspot/share/gc/g1/g1BlockOffsetTable.cpp
@@ -73,8 +73,8 @@ void G1BlockOffsetTable::set_offset_array(Atomic* left, Atomic
#ifdef ASSERT
void G1BlockOffsetTable::check_address(Atomic* addr, const char* msg) const {
- Atomic* start_addr = const_cast*>(_offset_base + (uintptr_t(_reserved.start()) >> CardTable::card_shift()));
- Atomic* end_addr = const_cast*>(_offset_base + (uintptr_t(_reserved.end()) >> CardTable::card_shift()));
+ Atomic* start_addr = _offset_base + (uintptr_t(_reserved.start()) >> CardTable::card_shift());
+ Atomic* end_addr = _offset_base + (uintptr_t(_reserved.end()) >> CardTable::card_shift());
assert(addr >= start_addr && addr <= end_addr,
"%s - offset address: " PTR_FORMAT ", start address: " PTR_FORMAT ", end address: " PTR_FORMAT,
msg, (p2i(addr)), (p2i(start_addr)), (p2i(end_addr)));
diff --git a/src/hotspot/share/gc/g1/g1BlockOffsetTable.inline.hpp b/src/hotspot/share/gc/g1/g1BlockOffsetTable.inline.hpp
index b707e310781..1236d24bb03 100644
--- a/src/hotspot/share/gc/g1/g1BlockOffsetTable.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1BlockOffsetTable.inline.hpp
@@ -54,7 +54,7 @@ uint8_t G1BlockOffsetTable::offset_array(Atomic* addr) const {
inline Atomic* G1BlockOffsetTable::entry_for_addr(const void* const p) const {
assert(_reserved.contains(p),
"out of bounds access to block offset table");
- Atomic* result = const_cast*>(&_offset_base[uintptr_t(p) >> CardTable::card_shift()]);
+ Atomic* result = &_offset_base[uintptr_t(p) >> CardTable::card_shift()];
return result;
}
diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp
index 3f5d674c443..fe286793ae7 100644
--- a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp
+++ b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp
@@ -34,7 +34,7 @@
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1CollectionSet.hpp"
#include "gc/g1/g1CollectionSetCandidates.hpp"
-#include "gc/g1/g1CollectorState.hpp"
+#include "gc/g1/g1CollectorState.inline.hpp"
#include "gc/g1/g1ConcurrentMarkThread.inline.hpp"
#include "gc/g1/g1ConcurrentRefine.hpp"
#include "gc/g1/g1ConcurrentRefineThread.hpp"
@@ -42,7 +42,6 @@
#include "gc/g1/g1FullCollector.hpp"
#include "gc/g1/g1GCCounters.hpp"
#include "gc/g1/g1GCParPhaseTimesTracker.hpp"
-#include "gc/g1/g1GCPauseType.hpp"
#include "gc/g1/g1GCPhaseTimes.hpp"
#include "gc/g1/g1HeapRegion.inline.hpp"
#include "gc/g1/g1HeapRegionPrinter.hpp"
@@ -1652,21 +1651,14 @@ jint G1CollectedHeap::initialize() {
return JNI_OK;
}
-bool G1CollectedHeap::concurrent_mark_is_terminating() const {
- assert(_cm != nullptr, "_cm must have been created");
- assert(_cm->is_fully_initialized(), "thread must exist in order to check if mark is terminating");
- return _cm->cm_thread()->should_terminate();
-}
-
void G1CollectedHeap::stop() {
+ assert_not_at_safepoint();
// Stop all concurrent threads. We do this to make sure these threads
// do not continue to execute and access resources (e.g. logging)
// that are destroyed during shutdown.
_cr->stop();
_service_thread->stop();
- if (_cm->is_fully_initialized()) {
- _cm->cm_thread()->stop();
- }
+ _cm->stop();
}
void G1CollectedHeap::safepoint_synchronize_begin() {
@@ -1857,12 +1849,12 @@ void G1CollectedHeap::increment_old_marking_cycles_completed(bool concurrent,
record_whole_heap_examined_timestamp();
}
- // We need to clear the "in_progress" flag in the CM thread before
+ // We need to tell G1ConcurrentMark to update the state before
// we wake up any waiters (especially when ExplicitInvokesConcurrent
// is set) so that if a waiter requests another System.gc() it doesn't
// incorrectly see that a marking cycle is still in progress.
if (concurrent) {
- _cm->cm_thread()->set_idle();
+ _cm->notify_concurrent_cycle_completed();
}
// Notify threads waiting in System.gc() (with ExplicitGCInvokesConcurrent)
@@ -2489,7 +2481,7 @@ void G1CollectedHeap::trace_heap(GCWhen::Type when, const GCTracer* gc_tracer) {
void G1CollectedHeap::gc_prologue(bool full) {
// Update common counters.
increment_total_collections(full /* full gc */);
- if (full || collector_state()->in_concurrent_start_gc()) {
+ if (full || collector_state()->is_in_concurrent_start_gc()) {
increment_old_marking_cycles_started();
}
}
@@ -2562,14 +2554,12 @@ HeapWord* G1CollectedHeap::do_collection_pause(size_t word_size,
void G1CollectedHeap::start_concurrent_cycle(bool concurrent_operation_is_full_mark) {
assert(_cm->is_fully_initialized(), "sanity");
- assert(!_cm->in_progress(), "Can not start concurrent operation while in progress");
+ assert(!collector_state()->is_in_concurrent_cycle(), "Can not start concurrent cycle when already running");
MutexLocker x(G1CGC_lock, Mutex::_no_safepoint_check_flag);
if (concurrent_operation_is_full_mark) {
- _cm->post_concurrent_mark_start();
- _cm->cm_thread()->start_full_mark();
+ _cm->start_full_concurrent_cycle();
} else {
- _cm->post_concurrent_undo_start();
- _cm->cm_thread()->start_undo_mark();
+ _cm->start_undo_concurrent_cycle();
}
G1CGC_lock->notify();
}
@@ -2661,7 +2651,7 @@ void G1CollectedHeap::verify_after_young_collection(G1HeapVerifier::G1VerifyType
verify_numa_regions("GC End");
_verifier->verify_region_sets_optional();
- if (collector_state()->in_concurrent_start_gc()) {
+ if (collector_state()->is_in_concurrent_start_gc()) {
log_debug(gc, verify)("Marking state");
_verifier->verify_marking_state();
}
@@ -2742,7 +2732,7 @@ void G1CollectedHeap::do_collection_pause_at_safepoint(size_t allocation_word_si
// Record whether this pause may need to trigger a concurrent operation. Later,
// when we signal the G1ConcurrentMarkThread, the collector state has already
// been reset for the next pause.
- bool should_start_concurrent_mark_operation = collector_state()->in_concurrent_start_gc();
+ bool should_start_concurrent_mark_operation = collector_state()->is_in_concurrent_start_gc();
// Perform the collection.
G1YoungCollector collector(gc_cause(), allocation_word_size);
@@ -2837,7 +2827,7 @@ bool G1STWSubjectToDiscoveryClosure::do_object_b(oop obj) {
}
void G1CollectedHeap::make_pending_list_reachable() {
- if (collector_state()->in_concurrent_start_gc()) {
+ if (collector_state()->is_in_concurrent_start_gc()) {
oop pll_head = Universe::reference_pending_list();
if (pll_head != nullptr) {
// Any valid worker id is fine here as we are in the VM thread and single-threaded.
@@ -3222,7 +3212,7 @@ void G1CollectedHeap::retire_gc_alloc_region(G1HeapRegion* alloc_region,
_survivor.add_used_bytes(allocated_bytes);
}
- bool const during_im = collector_state()->in_concurrent_start_gc();
+ bool const during_im = collector_state()->is_in_concurrent_start_gc();
if (during_im && allocated_bytes > 0) {
_cm->add_root_region(alloc_region);
}
diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.hpp b/src/hotspot/share/gc/g1/g1CollectedHeap.hpp
index 8ff9d481000..b5cb9167d92 100644
--- a/src/hotspot/share/gc/g1/g1CollectedHeap.hpp
+++ b/src/hotspot/share/gc/g1/g1CollectedHeap.hpp
@@ -34,7 +34,6 @@
#include "gc/g1/g1ConcurrentMark.hpp"
#include "gc/g1/g1EdenRegions.hpp"
#include "gc/g1/g1EvacStats.hpp"
-#include "gc/g1/g1GCPauseType.hpp"
#include "gc/g1/g1HeapRegionAttr.hpp"
#include "gc/g1/g1HeapRegionManager.hpp"
#include "gc/g1/g1HeapRegionSet.hpp"
@@ -915,9 +914,6 @@ public:
// specified by the policy object.
jint initialize() override;
- // Returns whether concurrent mark threads (and the VM) are about to terminate.
- bool concurrent_mark_is_terminating() const;
-
void safepoint_synchronize_begin() override;
void safepoint_synchronize_end() override;
diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp b/src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp
index 8782b65b6f9..bad9ac18eec 100644
--- a/src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp
@@ -28,7 +28,6 @@
#include "gc/g1/g1CollectedHeap.hpp"
#include "gc/g1/g1BarrierSet.hpp"
-#include "gc/g1/g1CollectorState.hpp"
#include "gc/g1/g1ConcurrentMark.inline.hpp"
#include "gc/g1/g1EvacFailureRegions.hpp"
#include "gc/g1/g1EvacStats.inline.hpp"
@@ -47,9 +46,9 @@
#include "utilities/bitMap.inline.hpp"
inline bool G1STWIsAliveClosure::do_object_b(oop p) {
- // An object is reachable if it is outside the collection set,
- // or is inside and copied.
- return !_g1h->is_in_cset(p) || p->is_forwarded();
+ // An object is reachable if it is outside the collection set and not a
+ // humongous candidate, or is inside and copied.
+ return !_g1h->is_in_cset_or_humongous_candidate(p) || p->is_forwarded();
}
inline JavaThread* const* G1JavaThreadsListClaimer::claim(uint& count) {
diff --git a/src/hotspot/share/gc/g1/g1CollectionSet.cpp b/src/hotspot/share/gc/g1/g1CollectionSet.cpp
index abfddf860e6..b3bcf6094ab 100644
--- a/src/hotspot/share/gc/g1/g1CollectionSet.cpp
+++ b/src/hotspot/share/gc/g1/g1CollectionSet.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -26,7 +26,7 @@
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1CollectionSet.inline.hpp"
#include "gc/g1/g1CollectionSetCandidates.inline.hpp"
-#include "gc/g1/g1CollectorState.hpp"
+#include "gc/g1/g1CollectorState.inline.hpp"
#include "gc/g1/g1HeapRegion.inline.hpp"
#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "gc/g1/g1HeapRegionSet.hpp"
@@ -174,7 +174,6 @@ void G1CollectionSet::iterate(G1HeapRegionClosure* cl) const {
G1HeapRegion* r = _g1h->region_at(_regions[i]);
bool result = cl->do_heap_region(r);
if (result) {
- cl->set_incomplete();
return;
}
}
@@ -326,7 +325,7 @@ double G1CollectionSet::finalize_young_part(double target_pause_time_ms, G1Survi
guarantee(target_pause_time_ms > 0.0,
"target_pause_time_ms = %1.6lf should be positive", target_pause_time_ms);
- bool in_young_only_phase = _policy->collector_state()->in_young_only_phase();
+ bool in_young_only_phase = _policy->collector_state()->is_in_young_only_phase();
size_t pending_cards = _policy->analytics()->predict_pending_cards(in_young_only_phase);
log_trace(gc, ergo, cset)("Start choosing CSet. Pending cards: %zu target pause time: %1.2fms",
@@ -379,7 +378,7 @@ void G1CollectionSet::finalize_old_part(double time_remaining_ms) {
if (!candidates()->is_empty()) {
candidates()->verify();
- if (collector_state()->in_mixed_phase()) {
+ if (collector_state()->is_in_mixed_phase()) {
time_remaining_ms = select_candidates_from_marking(time_remaining_ms);
} else {
log_debug(gc, ergo, cset)("Do not add marking candidates to collection set due to pause type.");
diff --git a/src/hotspot/share/gc/g1/g1CollectionSetCandidates.cpp b/src/hotspot/share/gc/g1/g1CollectionSetCandidates.cpp
index d71108d4d0e..2113db1163b 100644
--- a/src/hotspot/share/gc/g1/g1CollectionSetCandidates.cpp
+++ b/src/hotspot/share/gc/g1/g1CollectionSetCandidates.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -23,7 +23,6 @@
*/
#include "gc/g1/g1CollectionSetCandidates.inline.hpp"
-#include "gc/g1/g1CollectionSetChooser.hpp"
#include "gc/g1/g1HeapRegion.inline.hpp"
#include "utilities/growableArray.hpp"
@@ -250,8 +249,9 @@ void G1CollectionSetCandidates::sort_marking_by_efficiency() {
_from_marking_groups.verify();
}
-void G1CollectionSetCandidates::set_candidates_from_marking(G1HeapRegion** candidates,
- uint num_candidates) {
+void G1CollectionSetCandidates::set_candidates_from_marking(GrowableArrayCHeap* candidates) {
+ uint num_candidates = candidates->length();
+
if (num_candidates == 0) {
log_debug(gc, ergo, cset) ("No regions selected from marking.");
return;
@@ -264,7 +264,7 @@ void G1CollectionSetCandidates::set_candidates_from_marking(G1HeapRegion** candi
// During each Mixed GC, we must collect at least G1Policy::calc_min_old_cset_length regions to meet
// the G1MixedGCCountTarget. For the first collection in a Mixed GC cycle, we can add all regions
// required to meet this threshold to the same remset group. We are certain these will be collected in
- // the same MixedGC.
+ // the same Mixed GC.
uint group_limit = p->calc_min_old_cset_length(num_candidates);
G1CSetCandidateGroup::reset_next_group_id();
@@ -273,7 +273,7 @@ void G1CollectionSetCandidates::set_candidates_from_marking(G1HeapRegion** candi
current = new G1CSetCandidateGroup();
for (uint i = 0; i < num_candidates; i++) {
- G1HeapRegion* r = candidates[i];
+ G1HeapRegion* r = candidates->at(i);
assert(!contains(r), "must not contain region %u", r->hrm_index());
_contains_map[r->hrm_index()] = CandidateOrigin::Marking;
diff --git a/src/hotspot/share/gc/g1/g1CollectionSetCandidates.hpp b/src/hotspot/share/gc/g1/g1CollectionSetCandidates.hpp
index 7e882de7e5a..8a2235cf89c 100644
--- a/src/hotspot/share/gc/g1/g1CollectionSetCandidates.hpp
+++ b/src/hotspot/share/gc/g1/g1CollectionSetCandidates.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -245,8 +245,7 @@ public:
// Merge collection set candidates from marking into the current marking candidates
// (which needs to be empty).
- void set_candidates_from_marking(G1HeapRegion** candidates,
- uint num_candidates);
+ void set_candidates_from_marking(GrowableArrayCHeap* selected);
// The most recent length of the list that had been merged last via
// set_candidates_from_marking(). Used for calculating minimum collection set
// regions.
diff --git a/src/hotspot/share/gc/g1/g1CollectionSetChooser.cpp b/src/hotspot/share/gc/g1/g1CollectionSetChooser.cpp
deleted file mode 100644
index e7bab32129e..00000000000
--- a/src/hotspot/share/gc/g1/g1CollectionSetChooser.cpp
+++ /dev/null
@@ -1,296 +0,0 @@
-/*
- * Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#include "gc/g1/g1CollectedHeap.inline.hpp"
-#include "gc/g1/g1CollectionSetCandidates.hpp"
-#include "gc/g1/g1CollectionSetChooser.hpp"
-#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
-#include "gc/shared/space.hpp"
-#include "runtime/atomic.hpp"
-#include "utilities/quickSort.hpp"
-
-// Determine collection set candidates (from marking): For all regions determine
-// whether they should be a collection set candidate. Calculate their efficiency,
-// sort, and put them into the collection set candidates.
-//
-// Threads calculate the GC efficiency of the regions they get to process, and
-// put them into some work area without sorting. At the end that array is sorted and
-// moved to the destination.
-class G1BuildCandidateRegionsTask : public WorkerTask {
- // Work area for building the set of collection set candidates. Contains references
- // to heap regions with their GC efficiencies calculated. To reduce contention
- // on claiming array elements, worker threads claim parts of this array in chunks;
- // Array elements may be null as threads might not get enough regions to fill
- // up their chunks completely.
- // Final sorting will remove them.
- class G1BuildCandidateArray : public StackObj {
- uint const _max_size;
- uint const _chunk_size;
-
- G1HeapRegion** _data;
-
- Atomic _cur_claim_idx;
-
- static int compare_region_gc_efficiency(G1HeapRegion** rr1, G1HeapRegion** rr2) {
- G1HeapRegion* r1 = *rr1;
- G1HeapRegion* r2 = *rr2;
- // Make sure that null entries are moved to the end.
- if (r1 == nullptr) {
- if (r2 == nullptr) {
- return 0;
- } else {
- return 1;
- }
- } else if (r2 == nullptr) {
- return -1;
- }
-
- G1Policy* p = G1CollectedHeap::heap()->policy();
- double gc_efficiency1 = p->predict_gc_efficiency(r1);
- double gc_efficiency2 = p->predict_gc_efficiency(r2);
-
- if (gc_efficiency1 > gc_efficiency2) {
- return -1;
- } else if (gc_efficiency1 < gc_efficiency2) {
- return 1;
- } else {
- return 0;
- }
- }
-
- // Calculates the maximum array size that will be used.
- static uint required_array_size(uint num_regions, uint chunk_size, uint num_workers) {
- uint const max_waste = num_workers * chunk_size;
- // The array should be aligned with respect to chunk_size.
- uint const aligned_num_regions = ((num_regions + chunk_size - 1) / chunk_size) * chunk_size;
-
- return aligned_num_regions + max_waste;
- }
-
- public:
- G1BuildCandidateArray(uint max_num_regions, uint chunk_size, uint num_workers) :
- _max_size(required_array_size(max_num_regions, chunk_size, num_workers)),
- _chunk_size(chunk_size),
- _data(NEW_C_HEAP_ARRAY(G1HeapRegion*, _max_size, mtGC)),
- _cur_claim_idx(0) {
- for (uint i = 0; i < _max_size; i++) {
- _data[i] = nullptr;
- }
- }
-
- ~G1BuildCandidateArray() {
- FREE_C_HEAP_ARRAY(G1HeapRegion*, _data);
- }
-
- // Claim a new chunk, returning its bounds [from, to[.
- void claim_chunk(uint& from, uint& to) {
- uint result = _cur_claim_idx.add_then_fetch(_chunk_size);
- assert(_max_size > result - 1,
- "Array too small, is %u should be %u with chunk size %u.",
- _max_size, result, _chunk_size);
- from = result - _chunk_size;
- to = result;
- }
-
- // Set element in array.
- void set(uint idx, G1HeapRegion* hr) {
- assert(idx < _max_size, "Index %u out of bounds %u", idx, _max_size);
- assert(_data[idx] == nullptr, "Value must not have been set.");
- _data[idx] = hr;
- }
-
- void sort_by_gc_efficiency() {
- uint length = _cur_claim_idx.load_relaxed();
- if (length == 0) {
- return;
- }
- for (uint i = length; i < _max_size; i++) {
- assert(_data[i] == nullptr, "must be");
- }
- qsort(_data, length, sizeof(_data[0]), (_sort_Fn)compare_region_gc_efficiency);
- for (uint i = length; i < _max_size; i++) {
- assert(_data[i] == nullptr, "must be");
- }
- }
-
- G1HeapRegion** array() const { return _data; }
- };
-
- // Per-region closure. In addition to determining whether a region should be
- // added to the candidates, and calculating those regions' gc efficiencies, also
- // gather additional statistics.
- class G1BuildCandidateRegionsClosure : public G1HeapRegionClosure {
- G1BuildCandidateArray* _array;
-
- uint _cur_chunk_idx;
- uint _cur_chunk_end;
-
- uint _regions_added;
-
- void add_region(G1HeapRegion* hr) {
- if (_cur_chunk_idx == _cur_chunk_end) {
- _array->claim_chunk(_cur_chunk_idx, _cur_chunk_end);
- }
- assert(_cur_chunk_idx < _cur_chunk_end, "Must be");
-
- _array->set(_cur_chunk_idx, hr);
- _cur_chunk_idx++;
-
- _regions_added++;
- }
-
- public:
- G1BuildCandidateRegionsClosure(G1BuildCandidateArray* array) :
- _array(array),
- _cur_chunk_idx(0),
- _cur_chunk_end(0),
- _regions_added(0) { }
-
- bool do_heap_region(G1HeapRegion* r) {
- // Candidates from marking are always old; also keep regions that are already
- // collection set candidates (some retained regions) in that list.
- if (!r->is_old() || r->is_collection_set_candidate()) {
- // Keep remembered sets and everything for these regions.
- return false;
- }
-
- // Can not add a region without a remembered set to the candidates.
- if (!r->rem_set()->is_tracked()) {
- return false;
- }
-
- // Skip any region that is currently used as an old GC alloc region. We should
- // not consider those for collection before we fill them up as the effective
- // gain from them is small. I.e. we only actually reclaim from the filled part,
- // as the remainder is still eligible for allocation. These objects are also
- // likely to have already survived a few collections, so they might be longer
- // lived anyway.
- // Otherwise the Old region must satisfy the liveness condition.
- bool should_add = !G1CollectedHeap::heap()->is_old_gc_alloc_region(r) &&
- G1CollectionSetChooser::region_occupancy_low_enough_for_evac(r->live_bytes());
- if (should_add) {
- add_region(r);
- } else {
- r->rem_set()->clear(true /* only_cardset */);
- }
- return false;
- }
-
- uint regions_added() const { return _regions_added; }
- };
-
- G1CollectedHeap* _g1h;
- G1HeapRegionClaimer _hrclaimer;
-
- Atomic _num_regions_added;
-
- G1BuildCandidateArray _result;
-
- void update_totals(uint num_regions) {
- if (num_regions > 0) {
- _num_regions_added.add_then_fetch(num_regions);
- }
- }
-
- // Early prune (remove) regions meeting the G1HeapWastePercent criteria. That
- // is, either until only the minimum amount of old collection set regions are
- // available (for forward progress in evacuation) or the waste accumulated by the
- // removed regions is above the maximum allowed waste.
- // Updates number of candidates and reclaimable bytes given.
- void prune(G1HeapRegion** data) {
- G1Policy* p = G1CollectedHeap::heap()->policy();
-
- uint num_candidates = _num_regions_added.load_relaxed();
-
- uint min_old_cset_length = p->calc_min_old_cset_length(num_candidates);
- uint num_pruned = 0;
- size_t wasted_bytes = 0;
-
- if (min_old_cset_length >= num_candidates) {
- // We take all of the candidate regions to provide some forward progress.
- return;
- }
-
- size_t allowed_waste = p->allowed_waste_in_collection_set();
- uint max_to_prune = num_candidates - min_old_cset_length;
-
- while (true) {
- G1HeapRegion* r = data[num_candidates - num_pruned - 1];
- size_t const reclaimable = r->reclaimable_bytes();
- if (num_pruned >= max_to_prune ||
- wasted_bytes + reclaimable > allowed_waste) {
- break;
- }
- r->rem_set()->clear(true /* cardset_only */);
-
- wasted_bytes += reclaimable;
- num_pruned++;
- }
-
- log_debug(gc, ergo, cset)("Pruned %u regions out of %u, leaving %zu bytes waste (allowed %zu)",
- num_pruned,
- num_candidates,
- wasted_bytes,
- allowed_waste);
-
- _num_regions_added.sub_then_fetch(num_pruned, memory_order_relaxed);
- }
-
-public:
- G1BuildCandidateRegionsTask(uint max_num_regions, uint chunk_size, uint num_workers) :
- WorkerTask("G1 Build Candidate Regions"),
- _g1h(G1CollectedHeap::heap()),
- _hrclaimer(num_workers),
- _num_regions_added(0),
- _result(max_num_regions, chunk_size, num_workers) { }
-
- void work(uint worker_id) {
- G1BuildCandidateRegionsClosure cl(&_result);
- _g1h->heap_region_par_iterate_from_worker_offset(&cl, &_hrclaimer, worker_id);
- update_totals(cl.regions_added());
- }
-
- void sort_and_prune_into(G1CollectionSetCandidates* candidates) {
- _result.sort_by_gc_efficiency();
- prune(_result.array());
- candidates->set_candidates_from_marking(_result.array(),
- _num_regions_added.load_relaxed());
- }
-};
-
-uint G1CollectionSetChooser::calculate_work_chunk_size(uint num_workers, uint num_regions) {
- assert(num_workers > 0, "Active gc workers should be greater than 0");
- return MAX2(num_regions / num_workers, 1U);
-}
-
-void G1CollectionSetChooser::build(WorkerThreads* workers, uint max_num_regions, G1CollectionSetCandidates* candidates) {
- uint num_workers = workers->active_workers();
- uint chunk_size = calculate_work_chunk_size(num_workers, max_num_regions);
-
- G1BuildCandidateRegionsTask cl(max_num_regions, chunk_size, num_workers);
- workers->run_task(&cl, num_workers);
-
- cl.sort_and_prune_into(candidates);
- candidates->verify();
-}
diff --git a/src/hotspot/share/gc/g1/g1CollectionSetChooser.hpp b/src/hotspot/share/gc/g1/g1CollectionSetChooser.hpp
deleted file mode 100644
index 4db8ed23c49..00000000000
--- a/src/hotspot/share/gc/g1/g1CollectionSetChooser.hpp
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef SHARE_GC_G1_G1COLLECTIONSETCHOOSER_HPP
-#define SHARE_GC_G1_G1COLLECTIONSETCHOOSER_HPP
-
-#include "gc/g1/g1HeapRegion.hpp"
-#include "gc/shared/gc_globals.hpp"
-#include "memory/allStatic.hpp"
-#include "runtime/globals.hpp"
-
-class G1CollectionSetCandidates;
-class WorkerThreads;
-
-// Helper class to calculate collection set candidates, and containing some related
-// methods.
-class G1CollectionSetChooser : public AllStatic {
- static uint calculate_work_chunk_size(uint num_workers, uint num_regions);
-
- static size_t mixed_gc_live_threshold_bytes() {
- return G1HeapRegion::GrainBytes * (size_t)G1MixedGCLiveThresholdPercent / 100;
- }
-
-public:
- static bool region_occupancy_low_enough_for_evac(size_t live_bytes) {
- return live_bytes < mixed_gc_live_threshold_bytes();
- }
-
- // Build and return set of collection set candidates sorted by decreasing gc
- // efficiency.
- static void build(WorkerThreads* workers, uint max_num_regions, G1CollectionSetCandidates* candidates);
-};
-
-#endif // SHARE_GC_G1_G1COLLECTIONSETCHOOSER_HPP
diff --git a/src/hotspot/share/gc/g1/g1CollectorState.cpp b/src/hotspot/share/gc/g1/g1CollectorState.cpp
index d41ee22fdce..76de9c65cc8 100644
--- a/src/hotspot/share/gc/g1/g1CollectorState.cpp
+++ b/src/hotspot/share/gc/g1/g1CollectorState.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -22,25 +22,32 @@
*
*/
-#include "gc/g1/g1CollectorState.hpp"
-#include "gc/g1/g1GCPauseType.hpp"
+#include "gc/g1/g1CollectorState.inline.hpp"
+#include "runtime/safepoint.hpp"
+#include "utilities/debug.hpp"
-G1GCPauseType G1CollectorState::young_gc_pause_type(bool concurrent_operation_is_full_mark) const {
- assert(!in_full_gc(), "must be");
- if (in_concurrent_start_gc()) {
- assert(!in_young_gc_before_mixed(), "must be");
- return concurrent_operation_is_full_mark ? G1GCPauseType::ConcurrentStartMarkGC :
- G1GCPauseType::ConcurrentStartUndoGC;
- } else if (in_young_gc_before_mixed()) {
- assert(!in_concurrent_start_gc(), "must be");
- return G1GCPauseType::LastYoungGC;
- } else if (in_mixed_phase()) {
- assert(!in_concurrent_start_gc(), "must be");
- assert(!in_young_gc_before_mixed(), "must be");
- return G1GCPauseType::MixedGC;
- } else {
- assert(!in_concurrent_start_gc(), "must be");
- assert(!in_young_gc_before_mixed(), "must be");
- return G1GCPauseType::YoungGC;
+G1CollectorState::Pause G1CollectorState::gc_pause_type(bool concurrent_operation_is_full_mark) const {
+ assert(SafepointSynchronize::is_at_safepoint(), "must be");
+ switch (_phase) {
+ case Phase::YoungNormal: return Pause::Normal;
+ case Phase::YoungConcurrentStart:
+ return concurrent_operation_is_full_mark ? Pause::ConcurrentStartFull :
+ Pause::ConcurrentStartUndo;
+ case Phase::YoungPrepareMixed: return Pause::PrepareMixed;
+ case Phase::Mixed: return Pause::Mixed;
+ case Phase::FullGC: return Pause::Full;
+ default: ShouldNotReachHere();
}
}
+
+const char* G1CollectorState::to_string(Pause type) {
+ static const char* pause_strings[] = { "Normal",
+ "Concurrent Start", // Do not distinguish between the different
+ "Concurrent Start", // Concurrent Start pauses.
+ "Prepare Mixed",
+ "Cleanup",
+ "Remark",
+ "Mixed",
+ "Full" };
+ return pause_strings[static_cast(type)];
+}
diff --git a/src/hotspot/share/gc/g1/g1CollectorState.hpp b/src/hotspot/share/gc/g1/g1CollectorState.hpp
index fca30792344..42aaeab03b2 100644
--- a/src/hotspot/share/gc/g1/g1CollectorState.hpp
+++ b/src/hotspot/share/gc/g1/g1CollectorState.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -25,98 +25,100 @@
#ifndef SHARE_GC_G1_G1COLLECTORSTATE_HPP
#define SHARE_GC_G1_G1COLLECTORSTATE_HPP
-#include "gc/g1/g1GCPauseType.hpp"
+#include "utilities/debug.hpp"
+#include "utilities/enumIterator.hpp"
#include "utilities/globalDefinitions.hpp"
// State of the G1 collection.
+//
+// The rough phasing is Young-Only, Mixed / Space Reclamation and
+// Full GC "phase".
+//
+// We split the Young-only phase into three parts to cover interesting
+// sub-phases and avoid separate tracking.
class G1CollectorState {
- // Indicates whether we are in the phase where we do partial gcs that only contain
- // the young generation. Not set while _in_full_gc is set.
- bool _in_young_only_phase;
+ enum class Phase {
+ // Indicates that the next GC in the Young-Only phase will (likely) be a "Normal"
+ // young GC.
+ YoungNormal,
+ // We are in a concurrent start GC during the Young-Only phase. This is only set
+ // during that GC because we only decide whether we do this type of GC at the start
+ // of the pause.
+ YoungConcurrentStart,
+ // Indicates that we are about to start or in the prepare mixed gc in the Young-Only
+ // phase before the Mixed phase. This GC is required to keep pause time requirements.
+ YoungPrepareMixed,
+ // Doing extra old generation evacuation.
+ Mixed,
+ // The Full GC phase (that coincides with the Full GC pause).
+ FullGC
+ } _phase;
- // Indicates whether we are in the last young gc before the mixed gc phase. This GC
- // is required to keep pause time requirements.
- bool _in_young_gc_before_mixed;
-
- // If _initiate_conc_mark_if_possible is set at the beginning of a
- // pause, it is a suggestion that the pause should start a marking
- // cycle by doing the concurrent start work. However, it is possible
- // that the concurrent marking thread is still finishing up the
- // previous marking cycle (e.g., clearing the marking bitmap).
- // If that is the case we cannot start a new cycle and
- // we'll have to wait for the concurrent marking thread to finish
- // what it is doing. In this case we will postpone the marking cycle
- // initiation decision for the next pause. When we eventually decide
- // to start a cycle, we will set _in_concurrent_start_gc which
- // will stay true until the end of the concurrent start pause doing the
- // concurrent start work.
- volatile bool _in_concurrent_start_gc;
-
- // At the end of a pause we check the heap occupancy and we decide
- // whether we will start a marking cycle during the next pause. If
- // we decide that we want to do that, set this parameter. This parameter will
- // stay set until the beginning of a subsequent pause (not necessarily
- // the next one) when we decide that we will indeed start a marking cycle and
- // do the concurrent start phase work.
+ // _initiate_conc_mark_if_possible indicates that there has been a request to start
+ // a concurrent cycle but we have not been able to fulfill it because another one
+ // has been in progress when the request came in.
+ //
+ // This flag remembers that there is an unfullfilled request.
volatile bool _initiate_conc_mark_if_possible;
- // Marking is in progress. Set from start of the concurrent start pause to the
- // end of the Remark pause.
- bool _mark_in_progress;
- // Marking or rebuilding remembered set work is in progress. Set from the end
- // of the concurrent start pause to the end of the Cleanup pause.
- bool _mark_or_rebuild_in_progress;
-
- // The marking bitmap is currently being cleared or about to be cleared.
- bool _clear_bitmap_in_progress;
-
- // Set during a full gc pause.
- bool _in_full_gc;
-
public:
G1CollectorState() :
- _in_young_only_phase(true),
- _in_young_gc_before_mixed(false),
-
- _in_concurrent_start_gc(false),
- _initiate_conc_mark_if_possible(false),
-
- _mark_in_progress(false),
- _mark_or_rebuild_in_progress(false),
- _clear_bitmap_in_progress(false),
- _in_full_gc(false) { }
+ _phase(Phase::YoungNormal),
+ _initiate_conc_mark_if_possible(false) { }
// Phase setters
- void set_in_young_only_phase(bool v) { _in_young_only_phase = v; }
+ inline void set_in_normal_young_gc();
+ inline void set_in_space_reclamation_phase();
+ inline void set_in_full_gc();
- // Pause setters
- void set_in_young_gc_before_mixed(bool v) { _in_young_gc_before_mixed = v; }
- void set_in_concurrent_start_gc(bool v) { _in_concurrent_start_gc = v; }
- void set_in_full_gc(bool v) { _in_full_gc = v; }
+ inline void set_in_concurrent_start_gc();
+ inline void set_in_prepare_mixed_gc();
- void set_initiate_conc_mark_if_possible(bool v) { _initiate_conc_mark_if_possible = v; }
-
- void set_mark_in_progress(bool v) { _mark_in_progress = v; }
- void set_mark_or_rebuild_in_progress(bool v) { _mark_or_rebuild_in_progress = v; }
- void set_clear_bitmap_in_progress(bool v) { _clear_bitmap_in_progress = v; }
+ inline void set_initiate_conc_mark_if_possible(bool v);
// Phase getters
- bool in_young_only_phase() const { return _in_young_only_phase && !_in_full_gc; }
- bool in_mixed_phase() const { return !_in_young_only_phase && !_in_full_gc; }
+ inline bool is_in_young_only_phase() const;
+ inline bool is_in_mixed_phase() const;
// Specific pauses
- bool in_young_gc_before_mixed() const { return _in_young_gc_before_mixed; }
- bool in_full_gc() const { return _in_full_gc; }
- bool in_concurrent_start_gc() const { return _in_concurrent_start_gc; }
+ inline bool is_in_concurrent_start_gc() const;
+ inline bool is_in_prepare_mixed_gc() const;
+ inline bool is_in_full_gc() const;
- bool initiate_conc_mark_if_possible() const { return _initiate_conc_mark_if_possible; }
+ inline bool initiate_conc_mark_if_possible() const;
- bool mark_in_progress() const { return _mark_in_progress; }
- bool mark_or_rebuild_in_progress() const { return _mark_or_rebuild_in_progress; }
- bool clear_bitmap_in_progress() const { return _clear_bitmap_in_progress; }
+ bool is_in_concurrent_cycle() const;
+ bool is_in_marking() const;
+ bool is_in_mark_or_rebuild() const;
+ bool is_in_reset_for_next_cycle() const;
+
+ enum class Pause : uint {
+ Normal,
+ ConcurrentStartFull,
+ ConcurrentStartUndo,
+ PrepareMixed,
+ Cleanup,
+ Remark,
+ Mixed,
+ Full
+ };
// Calculate GC Pause Type from internal state.
- G1GCPauseType young_gc_pause_type(bool concurrent_operation_is_full_mark) const;
+ Pause gc_pause_type(bool concurrent_operation_is_full_mark) const;
+
+ static const char* to_string(Pause type);
+
+ // Pause kind queries
+ inline static void assert_is_young_pause(Pause type);
+
+ inline static bool is_young_only_pause(Pause type);
+ inline static bool is_concurrent_start_pause(Pause type);
+ inline static bool is_prepare_mixed_pause(Pause type);
+ inline static bool is_mixed_pause(Pause type);
+
+ inline static bool is_concurrent_cycle_pause(Pause type);
};
+ENUMERATOR_RANGE(G1CollectorState::Pause, G1CollectorState::Pause::Normal, G1CollectorState::Pause::Full)
+
#endif // SHARE_GC_G1_G1COLLECTORSTATE_HPP
diff --git a/src/hotspot/share/gc/g1/g1CollectorState.inline.hpp b/src/hotspot/share/gc/g1/g1CollectorState.inline.hpp
new file mode 100644
index 00000000000..0c6c9c879c3
--- /dev/null
+++ b/src/hotspot/share/gc/g1/g1CollectorState.inline.hpp
@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_G1_G1COLLECTORSTATE_INLINE_HPP
+#define SHARE_GC_G1_G1COLLECTORSTATE_INLINE_HPP
+
+#include "gc/g1/g1CollectorState.hpp"
+
+#include "gc/g1/g1CollectedHeap.hpp"
+#include "gc/g1/g1ConcurrentMark.inline.hpp"
+
+inline void G1CollectorState::set_in_normal_young_gc() {
+ _phase = Phase::YoungNormal;
+}
+inline void G1CollectorState::set_in_space_reclamation_phase() {
+ _phase = Phase::Mixed;
+}
+inline void G1CollectorState::set_in_full_gc() {
+ _phase = Phase::FullGC;
+}
+
+inline void G1CollectorState::set_in_concurrent_start_gc() {
+ _phase = Phase::YoungConcurrentStart;
+ _initiate_conc_mark_if_possible = false;
+}
+inline void G1CollectorState::set_in_prepare_mixed_gc() {
+ _phase = Phase::YoungPrepareMixed;
+}
+
+inline void G1CollectorState::set_initiate_conc_mark_if_possible(bool v) {
+ _initiate_conc_mark_if_possible = v;
+}
+
+inline bool G1CollectorState::is_in_young_only_phase() const {
+ return _phase == Phase::YoungNormal ||
+ _phase == Phase::YoungConcurrentStart ||
+ _phase == Phase::YoungPrepareMixed;
+}
+inline bool G1CollectorState::is_in_mixed_phase() const {
+ return _phase == Phase::Mixed;
+}
+
+inline bool G1CollectorState::is_in_prepare_mixed_gc() const {
+ return _phase == Phase::YoungPrepareMixed;
+}
+inline bool G1CollectorState::is_in_full_gc() const {
+ return _phase == Phase::FullGC;
+}
+inline bool G1CollectorState::is_in_concurrent_start_gc() const {
+ return _phase == Phase::YoungConcurrentStart;
+}
+
+inline bool G1CollectorState::initiate_conc_mark_if_possible() const {
+ return _initiate_conc_mark_if_possible;
+}
+
+inline bool G1CollectorState::is_in_concurrent_cycle() const {
+ G1ConcurrentMark* cm = G1CollectedHeap::heap()->concurrent_mark();
+ return cm->is_in_concurrent_cycle();
+}
+inline bool G1CollectorState::is_in_marking() const {
+ G1ConcurrentMark* cm = G1CollectedHeap::heap()->concurrent_mark();
+ return cm->is_in_marking();
+}
+inline bool G1CollectorState::is_in_mark_or_rebuild() const {
+ G1ConcurrentMark* cm = G1CollectedHeap::heap()->concurrent_mark();
+ return is_in_marking() || cm->is_in_rebuild_or_scrub();
+}
+inline bool G1CollectorState::is_in_reset_for_next_cycle() const {
+ G1ConcurrentMark* cm = G1CollectedHeap::heap()->concurrent_mark();
+ return cm->is_in_reset_for_next_cycle();
+}
+
+inline void G1CollectorState::assert_is_young_pause(Pause type) {
+ assert(type != Pause::Full, "must be");
+ assert(type != Pause::Remark, "must be");
+ assert(type != Pause::Cleanup, "must be");
+}
+
+inline bool G1CollectorState::is_young_only_pause(Pause type) {
+ assert_is_young_pause(type);
+ return type == Pause::ConcurrentStartUndo ||
+ type == Pause::ConcurrentStartFull ||
+ type == Pause::PrepareMixed ||
+ type == Pause::Normal;
+}
+
+inline bool G1CollectorState::is_mixed_pause(Pause type) {
+ assert_is_young_pause(type);
+ return type == Pause::Mixed;
+}
+
+inline bool G1CollectorState::is_prepare_mixed_pause(Pause type) {
+ assert_is_young_pause(type);
+ return type == Pause::PrepareMixed;
+}
+
+inline bool G1CollectorState::is_concurrent_start_pause(Pause type) {
+ assert_is_young_pause(type);
+ return type == Pause::ConcurrentStartFull || type == Pause::ConcurrentStartUndo;
+}
+
+inline bool G1CollectorState::is_concurrent_cycle_pause(Pause type) {
+ return type == Pause::Cleanup || type == Pause::Remark;
+}
+
+#endif // SHARE_GC_G1_G1COLLECTORSTATE_INLINE_HPP
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp b/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp
index ec5649f4fe2..4c646c82b8a 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp
@@ -30,8 +30,7 @@
#include "gc/g1/g1CardSetMemory.hpp"
#include "gc/g1/g1CardTableClaimTable.inline.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
-#include "gc/g1/g1CollectionSetChooser.hpp"
-#include "gc/g1/g1CollectorState.hpp"
+#include "gc/g1/g1CollectorState.inline.hpp"
#include "gc/g1/g1ConcurrentMark.inline.hpp"
#include "gc/g1/g1ConcurrentMarkRemarkTasks.hpp"
#include "gc/g1/g1ConcurrentMarkThread.inline.hpp"
@@ -373,63 +372,56 @@ void G1CMMarkStack::set_empty() {
G1CMRootMemRegions::G1CMRootMemRegions(uint const max_regions) :
_root_regions(MemRegion::create_array(max_regions, mtGC)),
_max_regions(max_regions),
- _num_root_regions(0),
- _claimed_root_regions(0),
- _scan_in_progress(false),
- _should_abort(false) { }
+ _num_regions(0),
+ _num_claimed_regions(0) { }
G1CMRootMemRegions::~G1CMRootMemRegions() {
MemRegion::destroy_array(_root_regions, _max_regions);
}
void G1CMRootMemRegions::reset() {
- _num_root_regions.store_relaxed(0);
+ assert_at_safepoint();
+ assert(G1CollectedHeap::heap()->collector_state()->is_in_concurrent_start_gc(), "must be");
+
+ _num_regions.store_relaxed(0);
+ _num_claimed_regions.store_relaxed(0);
}
void G1CMRootMemRegions::add(HeapWord* start, HeapWord* end) {
assert_at_safepoint();
- size_t idx = _num_root_regions.fetch_then_add(1u);
- assert(idx < _max_regions, "Trying to add more root MemRegions than there is space %zu", _max_regions);
+ uint idx = _num_regions.fetch_then_add(1u);
+ assert(idx < _max_regions, "Trying to add more root MemRegions than there is space %u", _max_regions);
assert(start != nullptr && end != nullptr && start <= end, "Start (" PTR_FORMAT ") should be less or equal to "
"end (" PTR_FORMAT ")", p2i(start), p2i(end));
_root_regions[idx].set_start(start);
_root_regions[idx].set_end(end);
}
-void G1CMRootMemRegions::prepare_for_scan() {
- assert(!scan_in_progress(), "pre-condition");
-
- _scan_in_progress.store_relaxed(num_root_regions() > 0);
-
- _claimed_root_regions.store_relaxed(0);
- _should_abort.store_relaxed(false);
-}
-
const MemRegion* G1CMRootMemRegions::claim_next() {
- if (_should_abort.load_relaxed()) {
- // If someone has set the should_abort flag, we return null to
- // force the caller to bail out of their loop.
+ uint local_num_regions = num_regions();
+ if (num_claimed_regions() >= local_num_regions) {
return nullptr;
}
- uint local_num_root_regions = num_root_regions();
- if (_claimed_root_regions.load_relaxed() >= local_num_root_regions) {
- return nullptr;
- }
-
- size_t claimed_index = _claimed_root_regions.fetch_then_add(1u);
- if (claimed_index < local_num_root_regions) {
+ uint claimed_index = _num_claimed_regions.fetch_then_add(1u);
+ if (claimed_index < local_num_regions) {
return &_root_regions[claimed_index];
}
return nullptr;
}
-uint G1CMRootMemRegions::num_root_regions() const {
- return (uint)_num_root_regions.load_relaxed();
+bool G1CMRootMemRegions::work_completed() const {
+ return num_remaining_regions() == 0;
+}
+
+uint G1CMRootMemRegions::num_remaining_regions() const {
+ uint total = num_regions();
+ uint claimed = num_claimed_regions();
+ return (total > claimed) ? total - claimed : 0;
}
bool G1CMRootMemRegions::contains(const MemRegion mr) const {
- uint local_num_root_regions = num_root_regions();
+ uint local_num_root_regions = num_regions();
for (uint i = 0; i < local_num_root_regions; i++) {
if (_root_regions[i].equals(mr)) {
return true;
@@ -438,42 +430,6 @@ bool G1CMRootMemRegions::contains(const MemRegion mr) const {
return false;
}
-void G1CMRootMemRegions::notify_scan_done() {
- MutexLocker x(G1RootRegionScan_lock, Mutex::_no_safepoint_check_flag);
- _scan_in_progress.store_relaxed(false);
- G1RootRegionScan_lock->notify_all();
-}
-
-void G1CMRootMemRegions::cancel_scan() {
- notify_scan_done();
-}
-
-void G1CMRootMemRegions::scan_finished() {
- assert(scan_in_progress(), "pre-condition");
-
- if (!_should_abort.load_relaxed()) {
- assert(_claimed_root_regions.load_relaxed() >= num_root_regions(),
- "we should have claimed all root regions, claimed %zu, length = %u",
- _claimed_root_regions.load_relaxed(), num_root_regions());
- }
-
- notify_scan_done();
-}
-
-bool G1CMRootMemRegions::wait_until_scan_finished() {
- if (!scan_in_progress()) {
- return false;
- }
-
- {
- MonitorLocker ml(G1RootRegionScan_lock, Mutex::_no_safepoint_check_flag);
- while (scan_in_progress()) {
- ml.wait();
- }
- }
- return true;
-}
-
G1ConcurrentMark::G1ConcurrentMark(G1CollectedHeap* g1h,
G1RegionToSpaceMapper* bitmap_storage) :
_cm_thread(nullptr),
@@ -484,6 +440,7 @@ G1ConcurrentMark::G1ConcurrentMark(G1CollectedHeap* g1h,
_heap(_g1h->reserved()),
_root_regions(_g1h->max_num_regions()),
+ _root_region_scan_aborted(false),
_global_mark_stack(),
@@ -573,14 +530,31 @@ void G1ConcurrentMark::fully_initialize() {
reset_at_marking_complete();
}
-bool G1ConcurrentMark::in_progress() const {
- return is_fully_initialized() ? _cm_thread->in_progress() : false;
+bool G1ConcurrentMark::is_in_concurrent_cycle() const {
+ return is_fully_initialized() ? _cm_thread->is_in_progress() : false;
+}
+
+bool G1ConcurrentMark::is_in_marking() const {
+ return is_fully_initialized() ? cm_thread()->is_in_marking() : false;
+}
+
+bool G1ConcurrentMark::is_in_rebuild_or_scrub() const {
+ return cm_thread()->is_in_rebuild_or_scrub();
+}
+
+bool G1ConcurrentMark::is_in_reset_for_next_cycle() const {
+ return cm_thread()->is_in_reset_for_next_cycle();
}
PartialArrayStateManager* G1ConcurrentMark::partial_array_state_manager() const {
return _partial_array_state_manager;
}
+G1ConcurrentMarkThread* G1ConcurrentMark::cm_thread() const {
+ assert(is_fully_initialized(), "must be");
+ return _cm_thread;
+}
+
void G1ConcurrentMark::reset() {
_has_aborted.store_relaxed(false);
@@ -598,6 +572,7 @@ void G1ConcurrentMark::reset() {
_region_mark_stats[i].clear();
}
+ _root_region_scan_aborted.store_relaxed(false);
_root_regions.reset();
}
@@ -617,7 +592,7 @@ void G1ConcurrentMark::humongous_object_eagerly_reclaimed(G1HeapRegion* r) {
// Need to clear mark bit of the humongous object. Doing this unconditionally is fine.
mark_bitmap()->clear(r->bottom());
- if (!_g1h->collector_state()->mark_or_rebuild_in_progress()) {
+ if (!_g1h->collector_state()->is_in_mark_or_rebuild()) {
return;
}
@@ -677,14 +652,14 @@ void G1ConcurrentMark::set_concurrency_and_phase(uint active_tasks, bool concurr
#if TASKQUEUE_STATS
void G1ConcurrentMark::print_and_reset_taskqueue_stats() {
- _task_queues->print_and_reset_taskqueue_stats("G1ConcurrentMark Oop Queue");
+ _task_queues->print_and_reset_taskqueue_stats("Concurrent Mark");
auto get_pa_stats = [&](uint i) {
return _tasks[i]->partial_array_task_stats();
};
PartialArrayTaskStats::log_set(_max_num_tasks, get_pa_stats,
- "G1ConcurrentMark Partial Array Task Stats");
+ "Concurrent Mark Partial Array");
for (uint i = 0; i < _max_num_tasks; ++i) {
get_pa_stats(i)->reset();
@@ -715,7 +690,6 @@ public:
private:
// Heap region closure used for clearing the _mark_bitmap.
class G1ClearBitmapHRClosure : public G1HeapRegionClosure {
- private:
G1ConcurrentMark* _cm;
G1CMBitMap* _bitmap;
bool _suspendible; // If suspendible, do yield checks.
@@ -725,7 +699,7 @@ private:
}
bool is_clear_concurrent_undo() {
- return suspendible() && _cm->cm_thread()->in_undo_mark();
+ return suspendible() && _cm->cm_thread()->is_in_undo_cycle();
}
bool has_aborted() {
@@ -781,8 +755,7 @@ private:
// as asserts here to minimize their overhead on the product. However, we
// will have them as guarantees at the beginning / end of the bitmap
// clearing to get some checking in the product.
- assert(!suspendible() || _cm->in_progress(), "invariant");
- assert(!suspendible() || !G1CollectedHeap::heap()->collector_state()->mark_or_rebuild_in_progress(), "invariant");
+ assert(!suspendible() || _cm->is_in_reset_for_next_cycle(), "invariant");
// Abort iteration if necessary.
if (has_aborted()) {
@@ -813,10 +786,6 @@ public:
SuspendibleThreadSetJoiner sts_join(_suspendible);
G1CollectedHeap::heap()->heap_region_par_iterate_from_worker_offset(&_cl, &_hr_claimer, worker_id);
}
-
- bool is_complete() {
- return _cl.is_complete();
- }
};
void G1ConcurrentMark::clear_bitmap(WorkerThreads* workers, bool may_yield) {
@@ -831,29 +800,19 @@ void G1ConcurrentMark::clear_bitmap(WorkerThreads* workers, bool may_yield) {
log_debug(gc, ergo)("Running %s with %u workers for %zu work units.", cl.name(), num_workers, num_chunks);
workers->run_task(&cl, num_workers);
- guarantee(may_yield || cl.is_complete(), "Must have completed iteration when not yielding.");
}
void G1ConcurrentMark::cleanup_for_next_mark() {
// Make sure that the concurrent mark thread looks to still be in
// the current cycle.
- guarantee(is_fully_initialized(), "should be initializd");
- guarantee(in_progress(), "invariant");
-
- // We are finishing up the current cycle by clearing the next
- // marking bitmap and getting it ready for the next cycle. During
- // this time no other cycle can start. So, let's make sure that this
- // is the case.
- guarantee(!_g1h->collector_state()->mark_or_rebuild_in_progress(), "invariant");
+ guarantee(is_in_reset_for_next_cycle(), "invariant");
clear_bitmap(_concurrent_workers, true);
reset_partial_array_state_manager();
- // Repeat the asserts from above.
- guarantee(is_fully_initialized(), "should be initializd");
- guarantee(in_progress(), "invariant");
- guarantee(!_g1h->collector_state()->mark_or_rebuild_in_progress(), "invariant");
+ // Should not have changed state yet (even if a Full GC interrupted us).
+ guarantee(is_in_reset_for_next_cycle(), "invariant");
}
void G1ConcurrentMark::reset_partial_array_state_manager() {
@@ -898,9 +857,26 @@ public:
};
class G1PreConcurrentStartTask::NoteStartOfMarkTask : public G1AbstractSubTask {
+
+ class NoteStartOfMarkHRClosure : public G1HeapRegionClosure {
+ G1ConcurrentMark* _cm;
+
+ public:
+ NoteStartOfMarkHRClosure() : G1HeapRegionClosure(), _cm(G1CollectedHeap::heap()->concurrent_mark()) { }
+
+ bool do_heap_region(G1HeapRegion* r) override {
+ if (r->is_old_or_humongous() && !r->is_collection_set_candidate() && !r->in_collection_set()) {
+ _cm->update_top_at_mark_start(r);
+ } else {
+ _cm->reset_top_at_mark_start(r);
+ }
+ return false;
+ }
+ } _region_cl;
+
G1HeapRegionClaimer _claimer;
public:
- NoteStartOfMarkTask() : G1AbstractSubTask(G1GCPhaseTimes::NoteStartOfMark), _claimer(0) { }
+ NoteStartOfMarkTask() : G1AbstractSubTask(G1GCPhaseTimes::NoteStartOfMark), _region_cl(), _claimer(0) { }
double worker_cost() const override {
// The work done per region is very small, therefore we choose this magic number to cap the number
@@ -909,8 +885,13 @@ public:
return _claimer.n_regions() / regions_per_thread;
}
- void set_max_workers(uint max_workers) override;
- void do_work(uint worker_id) override;
+ void set_max_workers(uint max_workers) override {
+ _claimer.set_n_workers(max_workers);
+ }
+
+ void do_work(uint worker_id) override {
+ G1CollectedHeap::heap()->heap_region_par_iterate_from_worker_offset(&_region_cl, &_claimer, worker_id);
+ }
};
void G1PreConcurrentStartTask::ResetMarkingStateTask::do_work(uint worker_id) {
@@ -918,31 +899,6 @@ void G1PreConcurrentStartTask::ResetMarkingStateTask::do_work(uint worker_id) {
_cm->reset();
}
-class NoteStartOfMarkHRClosure : public G1HeapRegionClosure {
- G1ConcurrentMark* _cm;
-
-public:
- NoteStartOfMarkHRClosure() : G1HeapRegionClosure(), _cm(G1CollectedHeap::heap()->concurrent_mark()) { }
-
- bool do_heap_region(G1HeapRegion* r) override {
- if (r->is_old_or_humongous() && !r->is_collection_set_candidate() && !r->in_collection_set()) {
- _cm->update_top_at_mark_start(r);
- } else {
- _cm->reset_top_at_mark_start(r);
- }
- return false;
- }
-};
-
-void G1PreConcurrentStartTask::NoteStartOfMarkTask::do_work(uint worker_id) {
- NoteStartOfMarkHRClosure start_cl;
- G1CollectedHeap::heap()->heap_region_par_iterate_from_worker_offset(&start_cl, &_claimer, worker_id);
-}
-
-void G1PreConcurrentStartTask::NoteStartOfMarkTask::set_max_workers(uint max_workers) {
- _claimer.set_n_workers(max_workers);
-}
-
G1PreConcurrentStartTask::G1PreConcurrentStartTask(GCCause::Cause cause, G1ConcurrentMark* cm) :
G1BatchedTask("Pre Concurrent Start", G1CollectedHeap::heap()->phase_times()) {
add_serial_task(new ResetMarkingStateTask(cm));
@@ -962,8 +918,7 @@ void G1ConcurrentMark::pre_concurrent_start(GCCause::Cause cause) {
_gc_tracer_cm->set_gc_cause(cause);
}
-
-void G1ConcurrentMark::post_concurrent_mark_start() {
+void G1ConcurrentMark::start_full_concurrent_cycle() {
// Start Concurrent Marking weak-reference discovery.
ReferenceProcessor* rp = _g1h->ref_processor_cm();
rp->start_discovery(false /* always_clear */);
@@ -974,16 +929,34 @@ void G1ConcurrentMark::post_concurrent_mark_start() {
satb_mq_set.set_active_all_threads(true, /* new active value */
false /* expected_active */);
- _root_regions.prepare_for_scan();
-
// update_g1_committed() will be called at the end of an evac pause
// when marking is on. So, it's also called at the end of the
// concurrent start pause to update the heap end, if the heap expands
// during it. No need to call it here.
+
+ // Signal the thread to start work.
+ cm_thread()->start_full_cycle();
}
-void G1ConcurrentMark::post_concurrent_undo_start() {
- root_regions()->cancel_scan();
+void G1ConcurrentMark::start_undo_concurrent_cycle() {
+ assert_at_safepoint_on_vm_thread();
+ // At this time this GC is not a concurrent start gc any more, can only check for young only gc/phase.
+ assert(_g1h->collector_state()->is_in_young_only_phase(), "must be");
+
+ abort_root_region_scan_at_safepoint();
+
+ // Signal the thread to start work.
+ cm_thread()->start_undo_cycle();
+}
+
+void G1ConcurrentMark::notify_concurrent_cycle_completed() {
+ cm_thread()->set_idle();
+}
+
+void G1ConcurrentMark::stop() {
+ if (is_fully_initialized()) {
+ cm_thread()->stop();
+ }
}
/*
@@ -1082,6 +1055,16 @@ uint G1ConcurrentMark::calc_active_marking_workers() {
return result;
}
+bool G1ConcurrentMark::has_root_region_scan_aborted() const {
+ return _root_region_scan_aborted.load_relaxed();
+}
+
+#ifndef PRODUCT
+void G1ConcurrentMark::assert_root_region_scan_completed_or_aborted() {
+ assert(root_regions()->work_completed() || has_root_region_scan_aborted(), "must be");
+}
+#endif
+
void G1ConcurrentMark::scan_root_region(const MemRegion* region, uint worker_id) {
#ifdef ASSERT
HeapWord* last = region->last();
@@ -1108,45 +1091,76 @@ void G1ConcurrentMark::scan_root_region(const MemRegion* region, uint worker_id)
class G1CMRootRegionScanTask : public WorkerTask {
G1ConcurrentMark* _cm;
+ bool _should_yield;
+
public:
- G1CMRootRegionScanTask(G1ConcurrentMark* cm) :
- WorkerTask("G1 Root Region Scan"), _cm(cm) { }
+ G1CMRootRegionScanTask(G1ConcurrentMark* cm, bool should_yield) :
+ WorkerTask("G1 Root Region Scan"), _cm(cm), _should_yield(should_yield) { }
void work(uint worker_id) {
- G1CMRootMemRegions* root_regions = _cm->root_regions();
- const MemRegion* region = root_regions->claim_next();
- while (region != nullptr) {
+ SuspendibleThreadSetJoiner sts_join(_should_yield);
+
+ while (true) {
+ if (_cm->has_root_region_scan_aborted()) {
+ return;
+ }
+ G1CMRootMemRegions* root_regions = _cm->root_regions();
+ const MemRegion* region = root_regions->claim_next();
+ if (region == nullptr) {
+ return;
+ }
_cm->scan_root_region(region, worker_id);
- region = root_regions->claim_next();
+ if (_should_yield) {
+ SuspendibleThreadSet::yield();
+ // If we yielded, a GC may have processed all root regions,
+ // so this loop will naturally exit on the next claim_next() call.
+ // Same if a Full GC signalled abort of the concurrent mark.
+ }
}
}
};
-void G1ConcurrentMark::scan_root_regions() {
- // scan_in_progress() will have been set to true only if there was
- // at least one root region to scan. So, if it's false, we
- // should not attempt to do any further work.
- if (root_regions()->scan_in_progress()) {
- assert(!has_aborted(), "Aborting before root region scanning is finished not supported.");
-
+bool G1ConcurrentMark::scan_root_regions(WorkerThreads* workers, bool concurrent) {
+ // We first check whether there is any work to do as we might have already aborted
+ // the concurrent cycle, or ran into a GC that did the actual work when we reach here.
+ // We want to avoid spinning up the worker threads if that happened.
+ // (Note that due to races reading the abort-flag, we might spin up the threads anyway).
+ //
+ // Abort happens if a Full GC occurs right after starting the concurrent cycle or
+ // a young gc doing the work.
+ //
+ // Concurrent gc threads enter an STS when starting the task, so they stop, then
+ // continue after that safepoint.
+ bool do_scan = !root_regions()->work_completed() && !has_root_region_scan_aborted();
+ if (do_scan) {
// Assign one worker to each root-region but subject to the max constraint.
- const uint num_workers = MIN2(root_regions()->num_root_regions(),
+ // The constraint is also important to avoid accesses beyond the allocated per-worker
+ // marking helper data structures. We might get passed different WorkerThreads with
+ // different number of threads (potential worker ids) than helper data structures when
+ // completing this work during GC.
+ const uint num_workers = MIN2(root_regions()->num_remaining_regions(),
_max_concurrent_workers);
+ assert(num_workers > 0, "no more remaining root regions to process");
- G1CMRootRegionScanTask task(this);
+ G1CMRootRegionScanTask task(this, concurrent);
log_debug(gc, ergo)("Running %s using %u workers for %u work units.",
- task.name(), num_workers, root_regions()->num_root_regions());
- _concurrent_workers->run_task(&task, num_workers);
-
- // It's possible that has_aborted() is true here without actually
- // aborting the survivor scan earlier. This is OK as it's
- // mainly used for sanity checking.
- root_regions()->scan_finished();
+ task.name(), num_workers, root_regions()->num_remaining_regions());
+ workers->run_task(&task, num_workers);
}
+
+ assert_root_region_scan_completed_or_aborted();
+
+ return do_scan;
}
-bool G1ConcurrentMark::wait_until_root_region_scan_finished() {
- return root_regions()->wait_until_scan_finished();
+void G1ConcurrentMark::scan_root_regions_concurrently() {
+ assert(Thread::current() == cm_thread(), "must be on Concurrent Mark Thread");
+ scan_root_regions(_concurrent_workers, true /* concurrent */);
+}
+
+bool G1ConcurrentMark::complete_root_regions_scan_in_safepoint() {
+ assert_at_safepoint_on_vm_thread();
+ return scan_root_regions(_g1h->workers(), false /* concurrent */);
}
void G1ConcurrentMark::add_root_region(G1HeapRegion* r) {
@@ -1157,9 +1171,16 @@ bool G1ConcurrentMark::is_root_region(G1HeapRegion* r) {
return root_regions()->contains(MemRegion(top_at_mark_start(r), r->top()));
}
-void G1ConcurrentMark::root_region_scan_abort_and_wait() {
- root_regions()->abort();
- root_regions()->wait_until_scan_finished();
+void G1ConcurrentMark::abort_root_region_scan() {
+ assert_not_at_safepoint();
+
+ _root_region_scan_aborted.store_relaxed(true);
+}
+
+void G1ConcurrentMark::abort_root_region_scan_at_safepoint() {
+ assert_at_safepoint_on_vm_thread();
+
+ _root_region_scan_aborted.store_relaxed(true);
}
void G1ConcurrentMark::concurrent_cycle_start() {
@@ -1175,8 +1196,6 @@ uint G1ConcurrentMark::completed_mark_cycles() const {
}
void G1ConcurrentMark::concurrent_cycle_end(bool mark_cycle_completed) {
- _g1h->collector_state()->set_clear_bitmap_in_progress(false);
-
_g1h->trace_heap_after_gc(_gc_tracer_cm);
if (mark_cycle_completed) {
@@ -1317,14 +1336,13 @@ void G1ConcurrentMark::remark() {
_g1h->workers()->run_task(&cl, num_workers);
log_debug(gc, remset, tracking)("Remembered Set Tracking update regions total %u, selected %u",
- _g1h->num_committed_regions(), cl.total_selected_for_rebuild());
+ _g1h->num_committed_regions(), cl.total_selected_for_rebuild());
_needs_remembered_set_rebuild = (cl.total_selected_for_rebuild() > 0);
if (_needs_remembered_set_rebuild) {
- // Prune rebuild candidates based on G1HeapWastePercent.
- // Improves rebuild time in addition to remembered set memory usage.
- G1CollectionSetChooser::build(_g1h->workers(), _g1h->num_committed_regions(), _g1h->policy()->candidates());
+ GrowableArrayCHeap* selected = cl.sort_and_prune_old_selected();
+ _g1h->policy()->candidates()->set_candidates_from_marking(selected);
}
}
@@ -1361,6 +1379,9 @@ void G1ConcurrentMark::remark() {
G1ObjectCountIsAliveClosure is_alive(_g1h);
_gc_tracer_cm->report_object_count_after_gc(&is_alive, _g1h->workers());
}
+
+ // Successfully completed marking, advance state.
+ cm_thread()->set_full_cycle_rebuild_and_scrub();
} else {
// We overflowed. Restart concurrent marking.
_restart_for_overflow.store_relaxed(true);
@@ -1381,6 +1402,8 @@ void G1ConcurrentMark::remark() {
_g1h->update_perf_counter_cpu_time();
policy->record_concurrent_mark_remark_end();
+
+ return;
}
void G1ConcurrentMark::compute_new_sizes() {
@@ -1443,6 +1466,10 @@ void G1ConcurrentMark::cleanup() {
GCTraceTime(Debug, gc, phases) debug("Finalize Concurrent Mark Cleanup", _gc_timer_cm);
policy->record_concurrent_mark_cleanup_end(needs_remembered_set_rebuild());
}
+
+ // Advance state.
+ cm_thread()->set_full_cycle_reset_for_next_cycle();
+ return;
}
// 'Keep Alive' oop closure used by both serial parallel reference processing.
@@ -1869,7 +1896,7 @@ public:
void G1ConcurrentMark::verify_no_collection_set_oops() {
assert(SafepointSynchronize::is_at_safepoint() || !is_init_completed(),
"should be at a safepoint or initializing");
- if (!_g1h->collector_state()->mark_or_rebuild_in_progress()) {
+ if (!is_fully_initialized() || !_g1h->collector_state()->is_in_mark_or_rebuild()) {
return;
}
@@ -1930,12 +1957,15 @@ void G1ConcurrentMark::print_stats() {
}
bool G1ConcurrentMark::concurrent_cycle_abort() {
+ assert_at_safepoint_on_vm_thread();
+ assert(_g1h->collector_state()->is_in_full_gc(), "must be");
+
// If we start the compaction before the CM threads finish
// scanning the root regions we might trip them over as we'll
- // be moving objects / updating references. So let's wait until
- // they are done. By telling them to abort, they should complete
- // early.
- root_region_scan_abort_and_wait();
+ // be moving objects / updating references. Since the root region
+ // scan synchronized with the safepoint, just tell it to abort.
+ // It will notice when the threads start up again later.
+ abort_root_region_scan_at_safepoint();
// We haven't started a concurrent cycle no need to do anything; we might have
// aborted the marking because of shutting down though. In this case the marking
@@ -1947,7 +1977,7 @@ bool G1ConcurrentMark::concurrent_cycle_abort() {
// has been signalled is already rare), and this work should be negligible compared
// to actual full gc work.
- if (!is_fully_initialized() || (!cm_thread()->in_progress() && !_g1h->concurrent_mark_is_terminating())) {
+ if (!is_fully_initialized() || (!cm_thread()->is_in_progress() && !cm_thread()->should_terminate())) {
return false;
}
@@ -1965,7 +1995,7 @@ bool G1ConcurrentMark::concurrent_cycle_abort() {
}
void G1ConcurrentMark::abort_marking_threads() {
- assert(!_root_regions.scan_in_progress(), "still doing root region scan");
+ assert_root_region_scan_completed_or_aborted();
_has_aborted.store_relaxed(true);
_first_overflow_barrier_sync.abort();
_second_overflow_barrier_sync.abort();
@@ -2311,20 +2341,16 @@ void G1CMTask::drain_local_queue(bool partially) {
}
}
-size_t G1CMTask::start_partial_array_processing(oop obj) {
- assert(should_be_sliced(obj), "Must be an array object %d and large %zu", obj->is_objArray(), obj->size());
-
- objArrayOop obj_array = objArrayOop(obj);
- size_t array_length = obj_array->length();
-
- size_t initial_chunk_size = _partial_array_splitter.start(_task_queue, obj_array, nullptr, array_length);
+size_t G1CMTask::start_partial_array_processing(objArrayOop obj) {
+ assert(obj->length() >= (int)ObjArrayMarkingStride, "Must be a large array object %d", obj->length());
// Mark objArray klass metadata
- if (_cm_oop_closure->do_metadata()) {
- _cm_oop_closure->do_klass(obj_array->klass());
- }
+ process_klass(obj->klass());
- process_array_chunk(obj_array, 0, initial_chunk_size);
+ size_t array_length = obj->length();
+ size_t initial_chunk_size = _partial_array_splitter.start(_task_queue, obj, nullptr, array_length);
+
+ process_array_chunk(obj, 0, initial_chunk_size);
// Include object header size
return objArrayOopDesc::object_size(checked_cast(initial_chunk_size));
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMark.hpp b/src/hotspot/share/gc/g1/g1ConcurrentMark.hpp
index 11da6dae5b3..f9287f673d2 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentMark.hpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentMark.hpp
@@ -288,57 +288,36 @@ private:
class G1CMRootMemRegions {
// The set of root MemRegions.
MemRegion* _root_regions;
- size_t const _max_regions;
+ uint const _max_regions;
- Atomic _num_root_regions; // Actual number of root regions.
+ Atomic _num_regions; // Actual number of root regions.
+ Atomic _num_claimed_regions; // Number of root regions currently claimed.
- Atomic _claimed_root_regions; // Number of root regions currently claimed.
-
- Atomic _scan_in_progress;
- Atomic _should_abort;
-
- void notify_scan_done();
+ uint num_regions() const { return _num_regions.load_relaxed(); }
+ uint num_claimed_regions() const { return _num_claimed_regions.load_relaxed(); }
public:
G1CMRootMemRegions(uint const max_regions);
~G1CMRootMemRegions();
- // Reset the data structure to allow addition of new root regions.
- void reset();
-
void add(HeapWord* start, HeapWord* end);
- // Reset the claiming / scanning of the root regions.
- void prepare_for_scan();
-
- // Forces get_next() to return null so that the iteration aborts early.
- void abort() { _should_abort.store_relaxed(true); }
-
- // Return true if the CM thread are actively scanning root regions,
- // false otherwise.
- bool scan_in_progress() { return _scan_in_progress.load_relaxed(); }
+ // Reset data structure to initial state.
+ void reset();
// Claim the next root MemRegion to scan atomically, or return null if
// all have been claimed.
const MemRegion* claim_next();
- // The number of root regions to scan.
- uint num_root_regions() const;
+ // Number of root regions to still process.
+ uint num_remaining_regions() const;
+
+ // Returns whether all root regions have been processed or the processing been aborted.
+ bool work_completed() const;
// Is the given memregion contained in the root regions; the MemRegion must
// match exactly.
bool contains(const MemRegion mr) const;
-
- void cancel_scan();
-
- // Flag that we're done with root region scanning and notify anyone
- // who's waiting on it. If aborted is false, assume that all regions
- // have been claimed.
- void scan_finished();
-
- // If CM threads are still scanning root regions, wait until they
- // are done. Return true if we had to wait, false otherwise.
- bool wait_until_scan_finished();
};
// This class manages data structures and methods for doing liveness analysis in
@@ -352,6 +331,8 @@ class G1ConcurrentMark : public CHeapObj {
friend class G1CMRemarkTask;
friend class G1CMRootRegionScanTask;
friend class G1CMTask;
+ friend class G1ClearBitMapTask;
+ friend class G1CollectorState;
friend class G1ConcurrentMarkThread;
G1ConcurrentMarkThread* _cm_thread; // The thread doing the work
@@ -365,6 +346,7 @@ class G1ConcurrentMark : public CHeapObj {
// Root region tracking and claiming
G1CMRootMemRegions _root_regions;
+ Atomic _root_region_scan_aborted;
// For grey objects
G1CMMarkStack _global_mark_stack; // Grey objects behind global finger
@@ -524,6 +506,15 @@ class G1ConcurrentMark : public CHeapObj {
Atomic* _top_at_rebuild_starts;
// True when Remark pause selected regions for rebuilding.
bool _needs_remembered_set_rebuild;
+
+ G1ConcurrentMarkThread* cm_thread() const;
+
+ // Concurrent cycle state queries.
+ bool is_in_concurrent_cycle() const;
+ bool is_in_marking() const;
+ bool is_in_rebuild_or_scrub() const;
+ bool is_in_reset_for_next_cycle() const;
+
public:
// To be called when an object is marked the first time, e.g. after a successful
// mark_in_bitmap call. Updates various statistics data.
@@ -557,7 +548,7 @@ public:
void fully_initialize();
bool is_fully_initialized() const { return _cm_thread != nullptr; }
- bool in_progress() const;
+
uint max_num_tasks() const {return _max_num_tasks; }
// Clear statistics gathered during the concurrent cycle for the given region after
@@ -589,7 +580,7 @@ public:
// Notifies marking threads to abort. This is a best-effort notification. Does not
// guarantee or update any state after the call. Root region scan must not be
- // running.
+ // running or being aborted.
void abort_marking_threads();
// Total cpu time spent in mark worker threads in seconds.
@@ -602,8 +593,6 @@ public:
G1RegionToSpaceMapper* bitmap_storage);
~G1ConcurrentMark();
- G1ConcurrentMarkThread* cm_thread() { return _cm_thread; }
-
G1CMBitMap* mark_bitmap() const { return (G1CMBitMap*)&_mark_bitmap; }
// Calculates the number of concurrent GC threads to be used in the marking phase.
@@ -632,20 +621,40 @@ public:
// These two methods do the work that needs to be done at the start and end of the
// concurrent start pause.
void pre_concurrent_start(GCCause::Cause cause);
- void post_concurrent_mark_start();
- void post_concurrent_undo_start();
- // Scan all the root regions and mark everything reachable from
- // them.
- void scan_root_regions();
- bool wait_until_root_region_scan_finished();
+ // Start the particular type of concurrent cycle. After this call threads may be running.
+ void start_full_concurrent_cycle();
+ void start_undo_concurrent_cycle();
+
+ void notify_concurrent_cycle_completed();
+
+ // Stop active components/the concurrent mark thread.
+ void stop();
+
void add_root_region(G1HeapRegion* r);
bool is_root_region(G1HeapRegion* r);
- void root_region_scan_abort_and_wait();
+
+ // Scan all the root regions concurrently and mark everything reachable from
+ // them.
+ void scan_root_regions_concurrently();
+ // Complete root region scan work in the safepoint, return if we did some work.
+ bool complete_root_regions_scan_in_safepoint();
+
+ // Abort an active concurrent root region scan outside safepoint.
+ void abort_root_region_scan();
+
+ bool has_root_region_scan_aborted() const;
private:
+ // Abort an active concurrent root region scan during safepoint.
+ void abort_root_region_scan_at_safepoint();
+
+ void assert_root_region_scan_completed_or_aborted() PRODUCT_RETURN;
G1CMRootMemRegions* root_regions() { return &_root_regions; }
+ // Perform root region scan until all root regions have been processed, or
+ // the process has been aborted. Returns true if we did some work.
+ bool scan_root_regions(WorkerThreads* workers, bool concurrent);
// Scan a single root MemRegion to mark everything reachable from it.
void scan_root_region(const MemRegion* region, uint worker_id);
@@ -657,8 +666,10 @@ public:
// Do concurrent preclean work.
void preclean();
+ // Executes the Remark pause.
void remark();
+ // Executes the Cleanup pause.
void cleanup();
// Mark in the marking bitmap. Used during evacuation failure to
@@ -833,12 +844,10 @@ private:
// mark bitmap scan, and so needs to be pushed onto the mark stack.
bool is_below_finger(oop obj, HeapWord* global_finger) const;
- template void process_grey_task_entry(G1TaskQueueEntry task_entry, bool stolen);
-
static bool should_be_sliced(oop obj);
// Start processing the given objArrayOop by first pushing its continuations and
// then scanning the first chunk including the header.
- size_t start_partial_array_processing(oop obj);
+ size_t start_partial_array_processing(objArrayOop obj);
// Process the given continuation. Returns the number of words scanned.
size_t process_partial_array(const G1TaskQueueEntry& task, bool stolen);
// Apply the closure to the given range of elements in the objArray.
@@ -907,6 +916,9 @@ public:
template
inline bool deal_with_reference(T* p);
+ // Scan the klass and visit its children.
+ inline void process_klass(Klass* klass);
+
// Scans an object and visits its children.
inline void process_entry(G1TaskQueueEntry task_entry, bool stolen);
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMark.inline.hpp b/src/hotspot/share/gc/g1/g1ConcurrentMark.inline.hpp
index 21167d5cae9..094f4dca994 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentMark.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentMark.inline.hpp
@@ -106,9 +106,27 @@ inline void G1CMMarkStack::iterate(Fn fn) const {
}
#endif
+inline void G1CMTask::process_klass(Klass* klass) {
+ _cm_oop_closure->do_klass(klass);
+}
+
// It scans an object and visits its children.
inline void G1CMTask::process_entry(G1TaskQueueEntry task_entry, bool stolen) {
- process_grey_task_entry(task_entry, stolen);
+ assert(task_entry.is_partial_array_state() || _mark_bitmap->is_marked(cast_from_oop(task_entry.to_oop())),
+ "Any stolen object should be a slice or marked");
+
+ if (task_entry.is_partial_array_state()) {
+ _words_scanned += process_partial_array(task_entry, stolen);
+ } else {
+ oop obj = task_entry.to_oop();
+ if (should_be_sliced(obj)) {
+ _words_scanned += start_partial_array_processing(objArrayOop(obj));
+ } else {
+ _words_scanned += obj->oop_iterate_size(_cm_oop_closure);
+ }
+ }
+
+ check_limits();
}
inline void G1CMTask::push(G1TaskQueueEntry task_entry) {
@@ -160,27 +178,6 @@ inline bool G1CMTask::is_below_finger(oop obj, HeapWord* global_finger) const {
return objAddr < global_finger;
}
-template
-inline void G1CMTask::process_grey_task_entry(G1TaskQueueEntry task_entry, bool stolen) {
- assert(scan || (!task_entry.is_partial_array_state() && task_entry.to_oop()->is_typeArray()), "Skipping scan of grey non-typeArray");
- assert(task_entry.is_partial_array_state() || _mark_bitmap->is_marked(cast_from_oop(task_entry.to_oop())),
- "Any stolen object should be a slice or marked");
-
- if (scan) {
- if (task_entry.is_partial_array_state()) {
- _words_scanned += process_partial_array(task_entry, stolen);
- } else {
- oop obj = task_entry.to_oop();
- if (should_be_sliced(obj)) {
- _words_scanned += start_partial_array_processing(obj);
- } else {
- _words_scanned += obj->oop_iterate_size(_cm_oop_closure);
- }
- }
- }
- check_limits();
-}
-
inline bool G1CMTask::should_be_sliced(oop obj) {
return obj->is_objArray() && ((objArrayOop)obj)->length() >= (int)ObjArrayMarkingStride;
}
@@ -272,7 +269,6 @@ inline bool G1CMTask::make_reference_grey(oop obj) {
// be pushed on the stack. So, some duplicate work, but no
// correctness problems.
if (is_below_finger(obj, global_finger)) {
- G1TaskQueueEntry entry(obj);
if (obj->is_typeArray()) {
// Immediately process arrays of primitive types, rather
// than pushing on the mark stack. This keeps us from
@@ -284,8 +280,8 @@ inline bool G1CMTask::make_reference_grey(oop obj) {
// by only doing a bookkeeping update and avoiding the
// actual scan of the object - a typeArray contains no
// references, and the metadata is built-in.
- process_grey_task_entry(entry, false /* stolen */);
} else {
+ G1TaskQueueEntry entry(obj);
push(entry);
}
}
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMarkRemarkTasks.cpp b/src/hotspot/share/gc/g1/g1ConcurrentMarkRemarkTasks.cpp
index 4eb11f6d8f6..3eda7200e25 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentMarkRemarkTasks.cpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentMarkRemarkTasks.cpp
@@ -31,28 +31,32 @@
#include "gc/g1/g1RemSetTrackingPolicy.hpp"
#include "logging/log.hpp"
#include "runtime/mutexLocker.hpp"
+#include "utilities/growableArray.hpp"
struct G1UpdateRegionLivenessAndSelectForRebuildTask::G1OnRegionClosure : public G1HeapRegionClosure {
G1CollectedHeap* _g1h;
G1ConcurrentMark* _cm;
- // The number of regions actually selected for rebuild.
- uint _num_selected_for_rebuild;
size_t _freed_bytes;
uint _num_old_regions_removed;
uint _num_humongous_regions_removed;
- G1FreeRegionList* _local_cleanup_list;
+
+ GrowableArrayCHeap _old_selected_for_rebuild;
+ uint _num_humongous_selected_for_rebuild;
+
+ G1FreeRegionList* _cleanup_list;
G1OnRegionClosure(G1CollectedHeap* g1h,
G1ConcurrentMark* cm,
G1FreeRegionList* local_cleanup_list) :
_g1h(g1h),
_cm(cm),
- _num_selected_for_rebuild(0),
_freed_bytes(0),
_num_old_regions_removed(0),
_num_humongous_regions_removed(0),
- _local_cleanup_list(local_cleanup_list) {}
+ _old_selected_for_rebuild(16),
+ _num_humongous_selected_for_rebuild(0),
+ _cleanup_list(local_cleanup_list) {}
void reclaim_empty_region_common(G1HeapRegion* hr) {
assert(!hr->has_pinned_objects(), "precondition");
@@ -74,7 +78,7 @@ struct G1UpdateRegionLivenessAndSelectForRebuildTask::G1OnRegionClosure : public
_num_humongous_regions_removed++;
reclaim_empty_region_common(hr);
- _g1h->free_humongous_region(hr, _local_cleanup_list);
+ _g1h->free_humongous_region(hr, _cleanup_list);
};
_g1h->humongous_obj_regions_iterate(hr, on_humongous_region);
@@ -85,7 +89,7 @@ struct G1UpdateRegionLivenessAndSelectForRebuildTask::G1OnRegionClosure : public
_num_old_regions_removed++;
reclaim_empty_region_common(hr);
- _g1h->free_region(hr, _local_cleanup_list);
+ _g1h->free_region(hr, _cleanup_list);
}
bool do_heap_region(G1HeapRegion* hr) override {
@@ -98,13 +102,13 @@ struct G1UpdateRegionLivenessAndSelectForRebuildTask::G1OnRegionClosure : public
|| hr->has_pinned_objects();
if (is_live) {
const bool selected_for_rebuild = tracker->update_humongous_before_rebuild(hr);
+
auto on_humongous_region = [&] (G1HeapRegion* hr) {
if (selected_for_rebuild) {
- _num_selected_for_rebuild++;
+ _num_humongous_selected_for_rebuild++;
}
_cm->update_top_at_rebuild_start(hr);
};
-
_g1h->humongous_obj_regions_iterate(hr, on_humongous_region);
} else {
reclaim_empty_humongous_region(hr);
@@ -118,7 +122,7 @@ struct G1UpdateRegionLivenessAndSelectForRebuildTask::G1OnRegionClosure : public
if (is_live) {
const bool selected_for_rebuild = tracker->update_old_before_rebuild(hr);
if (selected_for_rebuild) {
- _num_selected_for_rebuild++;
+ _old_selected_for_rebuild.push(hr);
}
_cm->update_top_at_rebuild_start(hr);
} else {
@@ -137,7 +141,8 @@ G1UpdateRegionLivenessAndSelectForRebuildTask::G1UpdateRegionLivenessAndSelectFo
_g1h(g1h),
_cm(cm),
_hrclaimer(num_workers),
- _total_selected_for_rebuild(0),
+ _old_selected_for_rebuild(128),
+ _num_humongous_selected_for_rebuild(0),
_cleanup_list("Empty Regions After Mark List") {}
G1UpdateRegionLivenessAndSelectForRebuildTask::~G1UpdateRegionLivenessAndSelectForRebuildTask() {
@@ -153,8 +158,6 @@ void G1UpdateRegionLivenessAndSelectForRebuildTask::work(uint worker_id) {
G1OnRegionClosure on_region_cl(_g1h, _cm, &local_cleanup_list);
_g1h->heap_region_par_iterate_from_worker_offset(&on_region_cl, &_hrclaimer, worker_id);
- _total_selected_for_rebuild.add_then_fetch(on_region_cl._num_selected_for_rebuild);
-
// Update the old/humongous region sets
_g1h->remove_from_old_gen_sets(on_region_cl._num_old_regions_removed,
on_region_cl._num_humongous_regions_removed);
@@ -163,6 +166,9 @@ void G1UpdateRegionLivenessAndSelectForRebuildTask::work(uint worker_id) {
MutexLocker x(G1RareEvent_lock, Mutex::_no_safepoint_check_flag);
_g1h->decrement_summary_bytes(on_region_cl._freed_bytes);
+ _old_selected_for_rebuild.appendAll(&on_region_cl._old_selected_for_rebuild);
+ _num_humongous_selected_for_rebuild += on_region_cl._num_humongous_selected_for_rebuild;
+
_cleanup_list.add_ordered(&local_cleanup_list);
assert(local_cleanup_list.is_empty(), "post-condition");
}
@@ -172,3 +178,78 @@ uint G1UpdateRegionLivenessAndSelectForRebuildTask::desired_num_workers(uint num
const uint num_regions_per_worker = 384;
return (num_regions + num_regions_per_worker - 1) / num_regions_per_worker;
}
+
+// Early prune (remove) regions meeting the G1HeapWastePercent criteria. That
+// is, either until only the minimum amount of old collection set regions are
+// available (for forward progress in evacuation) or the waste accumulated by the
+// removed regions is above the maximum allowed waste.
+// Updates number of candidates and reclaimable bytes given.
+void G1UpdateRegionLivenessAndSelectForRebuildTask::prune(GrowableArrayCHeap* old_regions) {
+ G1Policy* p = G1CollectedHeap::heap()->policy();
+
+ uint num_candidates = (uint)old_regions->length();
+
+ uint min_old_cset_length = p->calc_min_old_cset_length(num_candidates);
+ uint num_pruned = 0;
+ size_t wasted_bytes = 0;
+
+ if (min_old_cset_length >= num_candidates) {
+ // We take all of the candidate regions to provide some forward progress.
+ return;
+ }
+
+ size_t allowed_waste = p->allowed_waste_in_collection_set();
+ uint max_to_prune = num_candidates - min_old_cset_length;
+
+ while (true) {
+ G1HeapRegion* r = old_regions->at(num_candidates - num_pruned - 1);
+ size_t const reclaimable = r->reclaimable_bytes();
+ if (num_pruned >= max_to_prune ||
+ wasted_bytes + reclaimable > allowed_waste) {
+ break;
+ }
+ r->rem_set()->clear(true /* cardset_only */);
+
+ wasted_bytes += reclaimable;
+ num_pruned++;
+ }
+
+ log_debug(gc, ergo, cset)("Pruned %u regions out of %u, leaving %zu bytes waste (allowed %zu)",
+ num_pruned,
+ num_candidates,
+ wasted_bytes,
+ allowed_waste);
+
+ old_regions->trunc_to(num_candidates - num_pruned);
+}
+
+static int compare_region_gc_efficiency(G1HeapRegion** rr1, G1HeapRegion** rr2) {
+ G1HeapRegion* r1 = *rr1;
+ G1HeapRegion* r2 = *rr2;
+
+ assert(r1 != nullptr, "must be");
+ assert(r2 != nullptr, "must be");
+
+ G1Policy* p = G1CollectedHeap::heap()->policy();
+ double gc_efficiency1 = p->predict_gc_efficiency(r1);
+ double gc_efficiency2 = p->predict_gc_efficiency(r2);
+
+ if (gc_efficiency1 > gc_efficiency2) {
+ return -1;
+ } else if (gc_efficiency1 < gc_efficiency2) {
+ return 1;
+ } else {
+ return 0;
+ }
+}
+
+GrowableArrayCHeap* G1UpdateRegionLivenessAndSelectForRebuildTask::sort_and_prune_old_selected() {
+ // Nothing to do for the humongous candidates here. Old selected need to be pruned.
+
+ if (_old_selected_for_rebuild.length() != 0) {
+ _old_selected_for_rebuild.sort(compare_region_gc_efficiency);
+ prune(&_old_selected_for_rebuild);
+ }
+
+ return &_old_selected_for_rebuild;
+}
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMarkRemarkTasks.hpp b/src/hotspot/share/gc/g1/g1ConcurrentMarkRemarkTasks.hpp
index a256693ff1d..6905419e2cc 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentMarkRemarkTasks.hpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentMarkRemarkTasks.hpp
@@ -29,7 +29,7 @@
#include "gc/g1/g1HeapRegionManager.hpp"
#include "gc/g1/g1HeapRegionSet.hpp"
#include "gc/shared/workerThread.hpp"
-#include "runtime/atomic.hpp"
+#include "utilities/growableArray.hpp"
class G1CollectedHeap;
class G1ConcurrentMark;
@@ -42,13 +42,15 @@ class G1UpdateRegionLivenessAndSelectForRebuildTask : public WorkerTask {
G1ConcurrentMark* _cm;
G1HeapRegionClaimer _hrclaimer;
- Atomic _total_selected_for_rebuild;
+ GrowableArrayCHeap _old_selected_for_rebuild;
+ uint _num_humongous_selected_for_rebuild;
// Reclaimed empty regions
G1FreeRegionList _cleanup_list;
struct G1OnRegionClosure;
+ void prune(GrowableArrayCHeap* old_regions);
public:
G1UpdateRegionLivenessAndSelectForRebuildTask(G1CollectedHeap* g1h,
G1ConcurrentMark* cm,
@@ -59,9 +61,14 @@ public:
void work(uint worker_id) override;
uint total_selected_for_rebuild() const {
- return _total_selected_for_rebuild.load_relaxed();
+ return (uint)_old_selected_for_rebuild.length() + _num_humongous_selected_for_rebuild;
}
+ // Sort selected old regions by efficiency and prune them based on G1HeapWastePercent.
+ // This pruning improves rebuild time in addition to remembered set memory usage.
+ // Returns the set of regions selected in efficiency order.
+ GrowableArrayCHeap* sort_and_prune_old_selected();
+
static uint desired_num_workers(uint num_regions);
};
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMarkThread.cpp b/src/hotspot/share/gc/g1/g1ConcurrentMarkThread.cpp
index 629cbae935e..b8c97acd1b0 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentMarkThread.cpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentMarkThread.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -112,33 +112,30 @@ class G1ConcPhaseTimer : public GCTraceConcTimeImpl title("Concurrent %s Cycle", _state == FullMark ? "Mark" : "Undo");
+ FormatBuffer<128> title("Concurrent %s Cycle", is_in_full_concurrent_cycle() ? "Mark" : "Undo");
GCTraceConcTime(Info, gc) tt(title);
concurrent_cycle_start();
- if (_state == FullMark) {
+ if (_state == FullCycleMarking) {
concurrent_mark_cycle_do();
} else {
- assert(_state == UndoMark, "Must do undo mark but is %d", _state);
+ assert(_state == UndoCycleResetForNextCycle, "Must do undo mark but is %d", _state);
concurrent_undo_cycle_do();
}
- concurrent_cycle_end(_state == FullMark && !_cm->has_aborted());
+ concurrent_cycle_end(is_in_full_concurrent_cycle() && !_cm->has_aborted());
update_perf_counter_cpu_time();
}
- _cm->root_regions()->cancel_scan();
}
void G1ConcurrentMarkThread::stop_service() {
- if (in_progress()) {
- // We are not allowed to abort the marking threads during root region scan.
- // Needs to be done separately.
- _cm->root_region_scan_abort_and_wait();
+ if (is_in_progress()) {
+ _cm->abort_root_region_scan();
_cm->abort_marking_threads();
}
@@ -149,7 +146,7 @@ void G1ConcurrentMarkThread::stop_service() {
bool G1ConcurrentMarkThread::wait_for_next_cycle() {
MonitorLocker ml(G1CGC_lock, Mutex::_no_safepoint_check_flag);
- while (!in_progress() && !should_terminate()) {
+ while (!is_in_progress() && !should_terminate()) {
ml.wait();
}
@@ -164,7 +161,7 @@ bool G1ConcurrentMarkThread::phase_clear_cld_claimed_marks() {
bool G1ConcurrentMarkThread::phase_scan_root_regions() {
G1ConcPhaseTimer p(_cm, "Concurrent Scan Root Regions");
- _cm->scan_root_regions();
+ _cm->scan_root_regions_concurrently();
update_perf_counter_cpu_time();
return _cm->has_aborted();
}
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMarkThread.hpp b/src/hotspot/share/gc/g1/g1ConcurrentMarkThread.hpp
index 22be7d9ffbb..e75298fdcb4 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentMarkThread.hpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentMarkThread.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -32,25 +32,34 @@ class G1Policy;
// The concurrent mark thread triggers the various steps of the concurrent marking
// cycle, including various marking cleanup.
+//
+// The concurrent cycle may either be "Full" (i.e. include marking, rebuilding and
+// scrubbing, resetting for the next cycle) or "Undo", i.e. shortened to just the
+// reset part.
class G1ConcurrentMarkThread: public ConcurrentGCThread {
G1ConcurrentMark* _cm;
enum ServiceState : uint {
Idle,
- FullMark,
- UndoMark
+ FullCycleMarking,
+ FullCycleRebuildOrScrub,
+ FullCycleResetForNextCycle,
+ UndoCycleResetForNextCycle
};
volatile ServiceState _state;
+ // Returns whether we are in a "Full" cycle.
+ bool is_in_full_concurrent_cycle() const;
+
// Wait for next cycle. Returns the command passed over.
bool wait_for_next_cycle();
bool mark_loop_needs_restart() const;
- // Phases and subphases for the full concurrent marking cycle in order.
+ // Phases and subphases for the full concurrent cycle in order.
//
- // All these methods return true if the marking should be aborted.
+ // All these methods return true if the cycle should be aborted.
bool phase_clear_cld_claimed_marks();
bool phase_scan_root_regions();
@@ -88,22 +97,25 @@ class G1ConcurrentMarkThread: public ConcurrentGCThread {
double total_mark_cpu_time_s();
// Cpu time used by all marking worker threads in seconds.
double worker_threads_cpu_time_s();
-
- G1ConcurrentMark* cm() { return _cm; }
-
+ // State management.
void set_idle();
- void start_full_mark();
- void start_undo_mark();
+ void start_full_cycle();
+ void start_undo_cycle();
- bool idle() const;
+ void set_full_cycle_rebuild_and_scrub();
+ void set_full_cycle_reset_for_next_cycle();
+
+ bool is_idle() const;
// Returns true from the moment a concurrent cycle is
- // initiated (during the concurrent start pause when started() is set)
- // to the moment when the cycle completes (just after the next
- // marking bitmap has been cleared and in_progress() is
- // cleared).
- bool in_progress() const;
+ // initiated (during the concurrent start pause when calling one of the
+ // start_*_cycle() methods) to the moment when the cycle completes.
+ bool is_in_progress() const;
- bool in_undo_mark() const;
+ bool is_in_marking() const;
+ bool is_in_rebuild_or_scrub() const;
+ bool is_in_reset_for_next_cycle() const;
+
+ bool is_in_undo_cycle() const;
// Update the perf data counter for concurrent mark.
void update_perf_counter_cpu_time();
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMarkThread.inline.hpp b/src/hotspot/share/gc/g1/g1ConcurrentMarkThread.inline.hpp
index 254eaf62bb2..8cb7881e000 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentMarkThread.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentMarkThread.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -32,7 +32,7 @@
// Total virtual time so far.
inline double G1ConcurrentMarkThread::total_mark_cpu_time_s() {
- return os::thread_cpu_time(this) + worker_threads_cpu_time_s();
+ return static_cast(os::thread_cpu_time(this)) + worker_threads_cpu_time_s();
}
// Marking virtual time so far
@@ -40,29 +40,64 @@ inline double G1ConcurrentMarkThread::worker_threads_cpu_time_s() {
return _cm->worker_threads_cpu_time_s();
}
+inline bool G1ConcurrentMarkThread::is_in_full_concurrent_cycle() const {
+ ServiceState state = _state;
+ return (state == FullCycleMarking || state == FullCycleRebuildOrScrub || state == FullCycleResetForNextCycle);
+}
+
inline void G1ConcurrentMarkThread::set_idle() {
- assert(_state == FullMark || _state == UndoMark, "must not be starting a new cycle");
+ // Concurrent cycle may be aborted any time.
+ assert(!is_idle(), "must not be idle");
_state = Idle;
}
-inline void G1ConcurrentMarkThread::start_full_mark() {
- assert(_state == Idle, "cycle in progress");
- _state = FullMark;
+inline void G1ConcurrentMarkThread::start_full_cycle() {
+ assert(SafepointSynchronize::is_at_safepoint(), "must be");
+ assert(is_idle(), "cycle in progress");
+ _state = FullCycleMarking;
}
-inline void G1ConcurrentMarkThread::start_undo_mark() {
- assert(_state == Idle, "cycle in progress");
- _state = UndoMark;
+inline void G1ConcurrentMarkThread::start_undo_cycle() {
+ assert(SafepointSynchronize::is_at_safepoint(), "must be");
+ assert(is_idle(), "cycle in progress");
+ _state = UndoCycleResetForNextCycle;
}
-inline bool G1ConcurrentMarkThread::idle() const { return _state == Idle; }
-
-inline bool G1ConcurrentMarkThread::in_progress() const {
- return !idle();
+inline void G1ConcurrentMarkThread::set_full_cycle_rebuild_and_scrub() {
+ assert(SafepointSynchronize::is_at_safepoint(), "must be");
+ assert(_state == FullCycleMarking, "must be");
+ _state = FullCycleRebuildOrScrub;
}
-inline bool G1ConcurrentMarkThread::in_undo_mark() const {
- return _state == UndoMark;
+inline void G1ConcurrentMarkThread::set_full_cycle_reset_for_next_cycle() {
+ assert(SafepointSynchronize::is_at_safepoint(), "must be");
+ assert(_state == FullCycleRebuildOrScrub, "must be");
+ _state = FullCycleResetForNextCycle;
+}
+
+inline bool G1ConcurrentMarkThread::is_in_marking() const {
+ return _state == FullCycleMarking;
+}
+
+inline bool G1ConcurrentMarkThread::is_in_rebuild_or_scrub() const {
+ return _state == FullCycleRebuildOrScrub;
+}
+
+inline bool G1ConcurrentMarkThread::is_in_reset_for_next_cycle() const {
+ ServiceState state = _state;
+ return state == FullCycleResetForNextCycle || state == UndoCycleResetForNextCycle;
+}
+
+inline bool G1ConcurrentMarkThread::is_idle() const {
+ return _state == Idle;
+}
+
+inline bool G1ConcurrentMarkThread::is_in_progress() const {
+ return !is_idle();
+}
+
+inline bool G1ConcurrentMarkThread::is_in_undo_cycle() const {
+ return _state == UndoCycleResetForNextCycle;
}
#endif // SHARE_GC_G1_G1CONCURRENTMARKTHREAD_INLINE_HPP
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentRefineStats.cpp b/src/hotspot/share/gc/g1/g1ConcurrentRefineStats.cpp
index c14c5658127..7da0066e2f1 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentRefineStats.cpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentRefineStats.cpp
@@ -23,7 +23,6 @@
*/
#include "gc/g1/g1ConcurrentRefineStats.inline.hpp"
-#include "runtime/atomicAccess.hpp"
#include "runtime/timer.hpp"
void G1ConcurrentRefineStats::add_atomic(G1ConcurrentRefineStats* other) {
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentStartToMixedTimeTracker.hpp b/src/hotspot/share/gc/g1/g1ConcurrentStartToMixedTimeTracker.hpp
index 57372e695c8..f8bad4bdcd7 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentStartToMixedTimeTracker.hpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentStartToMixedTimeTracker.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -30,7 +30,7 @@
// Used to track time from the end of concurrent start to the first mixed GC.
// After calling the concurrent start/mixed gc notifications, the result can be
-// obtained in last_marking_time() once, after which the tracking resets.
+// obtained in get_and_reset_last_marking_time() once, after which the tracking resets.
// Any pauses recorded by add_pause() will be subtracted from that results.
class G1ConcurrentStartToMixedTimeTracker {
private:
@@ -60,7 +60,7 @@ public:
}
}
- double last_marking_time() {
+ double get_and_reset_last_marking_time() {
assert(has_result(), "Do not have all measurements yet.");
double result = (_mixed_start_time - _concurrent_start_end_time) - _total_pause_time;
reset();
@@ -80,6 +80,8 @@ public:
}
}
+ bool is_active() const { return _active; }
+
// Returns whether we have a result that can be retrieved.
bool has_result() const { return _mixed_start_time > 0.0 && _concurrent_start_end_time > 0.0; }
};
diff --git a/src/hotspot/share/gc/g1/g1FullCollector.cpp b/src/hotspot/share/gc/g1/g1FullCollector.cpp
index e8498250f85..c835dd159a6 100644
--- a/src/hotspot/share/gc/g1/g1FullCollector.cpp
+++ b/src/hotspot/share/gc/g1/g1FullCollector.cpp
@@ -353,7 +353,13 @@ void G1FullCollector::phase1_mark_live_objects() {
scope()->tracer()->report_object_count_after_gc(&_is_alive, _heap->workers());
}
#if TASKQUEUE_STATS
- marking_task_queues()->print_and_reset_taskqueue_stats("Marking Task Queue");
+ marking_task_queues()->print_and_reset_taskqueue_stats("Full GC");
+
+ auto get_stats = [&](uint i) {
+ return marker(i)->partial_array_splitter().stats();
+ };
+ PartialArrayTaskStats::log_set(_num_workers, get_stats,
+ "Full GC Partial Array");
#endif
}
diff --git a/src/hotspot/share/gc/g1/g1FullCollector.inline.hpp b/src/hotspot/share/gc/g1/g1FullCollector.inline.hpp
index 0c201f0e43f..ef6344d349d 100644
--- a/src/hotspot/share/gc/g1/g1FullCollector.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1FullCollector.inline.hpp
@@ -30,7 +30,6 @@
#include "gc/g1/g1FullGCHeapRegionAttr.hpp"
#include "gc/g1/g1HeapRegion.inline.hpp"
#include "oops/oopsHierarchy.hpp"
-#include "runtime/atomicAccess.hpp"
bool G1FullCollector::is_compacting(oop obj) const {
return _region_attr_table.is_compacting(cast_from_oop(obj));
diff --git a/src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp b/src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp
index 5dbf70f36b3..93d8da0d842 100644
--- a/src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp
+++ b/src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp
@@ -22,7 +22,7 @@
*
*/
-#include "gc/g1/g1CollectedHeap.hpp"
+#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1ConcurrentMarkBitMap.inline.hpp"
#include "gc/g1/g1FullCollector.inline.hpp"
#include "gc/g1/g1FullGCCompactionPoint.hpp"
diff --git a/src/hotspot/share/gc/g1/g1FullGCMarker.hpp b/src/hotspot/share/gc/g1/g1FullGCMarker.hpp
index 5973cc841c5..82fe3655319 100644
--- a/src/hotspot/share/gc/g1/g1FullGCMarker.hpp
+++ b/src/hotspot/share/gc/g1/g1FullGCMarker.hpp
@@ -89,6 +89,7 @@ public:
~G1FullGCMarker();
G1MarkTasksQueue* task_queue() { return &_task_queue; }
+ PartialArraySplitter& partial_array_splitter() { return _partial_array_splitter; }
// Marking entry points
template inline void mark_and_push(T* p);
diff --git a/src/hotspot/share/gc/g1/g1GCPauseType.hpp b/src/hotspot/share/gc/g1/g1GCPauseType.hpp
deleted file mode 100644
index 254edb28fea..00000000000
--- a/src/hotspot/share/gc/g1/g1GCPauseType.hpp
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef SHARE_GC_G1_G1GCPAUSETYPES_HPP
-#define SHARE_GC_G1_G1GCPAUSETYPES_HPP
-
-#include "utilities/debug.hpp"
-#include "utilities/enumIterator.hpp"
-
-enum class G1GCPauseType : uint {
- YoungGC,
- LastYoungGC,
- ConcurrentStartMarkGC,
- ConcurrentStartUndoGC,
- Cleanup,
- Remark,
- MixedGC,
- FullGC
-};
-
-ENUMERATOR_RANGE(G1GCPauseType, G1GCPauseType::YoungGC, G1GCPauseType::FullGC)
-
-class G1GCPauseTypeHelper {
-public:
-
- static void assert_is_young_pause(G1GCPauseType type) {
- assert(type != G1GCPauseType::FullGC, "must be");
- assert(type != G1GCPauseType::Remark, "must be");
- assert(type != G1GCPauseType::Cleanup, "must be");
- }
-
- static bool is_young_only_pause(G1GCPauseType type) {
- assert_is_young_pause(type);
- return type == G1GCPauseType::ConcurrentStartUndoGC ||
- type == G1GCPauseType::ConcurrentStartMarkGC ||
- type == G1GCPauseType::LastYoungGC ||
- type == G1GCPauseType::YoungGC;
- }
-
- static bool is_mixed_pause(G1GCPauseType type) {
- assert_is_young_pause(type);
- return type == G1GCPauseType::MixedGC;
- }
-
- static bool is_last_young_pause(G1GCPauseType type) {
- assert_is_young_pause(type);
- return type == G1GCPauseType::LastYoungGC;
- }
-
- static bool is_concurrent_start_pause(G1GCPauseType type) {
- assert_is_young_pause(type);
- return type == G1GCPauseType::ConcurrentStartMarkGC || type == G1GCPauseType::ConcurrentStartUndoGC;
- }
-
- static const char* to_string(G1GCPauseType type) {
- static const char* pause_strings[] = { "Normal",
- "Prepare Mixed",
- "Concurrent Start", // Do not distinguish between the different
- "Concurrent Start", // Concurrent Start pauses.
- "Cleanup",
- "Remark",
- "Mixed",
- "Full" };
- return pause_strings[static_cast(type)];
- }
-};
-
-#endif // SHARE_GC_G1_G1GCPAUSETYPES_HPP
diff --git a/src/hotspot/share/gc/g1/g1GCPhaseTimes.cpp b/src/hotspot/share/gc/g1/g1GCPhaseTimes.cpp
index a5013ddbb40..023790a2422 100644
--- a/src/hotspot/share/gc/g1/g1GCPhaseTimes.cpp
+++ b/src/hotspot/share/gc/g1/g1GCPhaseTimes.cpp
@@ -180,7 +180,7 @@ void G1GCPhaseTimes::reset() {
_cur_post_evacuate_cleanup_2_time_ms = 0.0;
_cur_resize_heap_time_ms = 0.0;
_cur_ref_proc_time_ms = 0.0;
- _root_region_scan_wait_time_ms = 0.0;
+ _root_region_scan_time_ms = 0.0;
_external_accounted_time_ms = 0.0;
_recorded_prepare_heap_roots_time_ms = 0.0;
_recorded_young_cset_choice_time_ms = 0.0;
@@ -549,8 +549,8 @@ void G1GCPhaseTimes::print_other(double accounted_ms) const {
// In addition, these are not included in G1GCPhaseTimes::_gc_pause_time_ms.
// See G1YoungCollector::collect().
void G1GCPhaseTimes::print(bool evacuation_failed) {
- if (_root_region_scan_wait_time_ms > 0.0) {
- debug_time("Root Region Scan Waiting", _root_region_scan_wait_time_ms);
+ if (_root_region_scan_time_ms > 0.0) {
+ debug_time("Root Region Scan", _root_region_scan_time_ms);
}
// Check if some time has been recorded for verification and only then print
diff --git a/src/hotspot/share/gc/g1/g1GCPhaseTimes.hpp b/src/hotspot/share/gc/g1/g1GCPhaseTimes.hpp
index 8223148b791..b57bf0d617e 100644
--- a/src/hotspot/share/gc/g1/g1GCPhaseTimes.hpp
+++ b/src/hotspot/share/gc/g1/g1GCPhaseTimes.hpp
@@ -191,7 +191,7 @@ class G1GCPhaseTimes : public CHeapObj {
double _cur_ref_proc_time_ms;
// Not included in _gc_pause_time_ms
- double _root_region_scan_wait_time_ms;
+ double _root_region_scan_time_ms;
double _external_accounted_time_ms;
@@ -325,8 +325,8 @@ class G1GCPhaseTimes : public CHeapObj {
_cur_prepare_concurrent_task_time_ms = ms;
}
- void record_root_region_scan_wait_time(double time_ms) {
- _root_region_scan_wait_time_ms = time_ms;
+ void record_root_region_scan_time(double time_ms) {
+ _root_region_scan_time_ms = time_ms;
}
void record_serial_free_cset_time_ms(double time_ms) {
@@ -399,8 +399,8 @@ class G1GCPhaseTimes : public CHeapObj {
return _cur_resize_heap_time_ms;
}
- double root_region_scan_wait_time_ms() {
- return _root_region_scan_wait_time_ms;
+ double root_region_scan_time_ms() {
+ return _root_region_scan_time_ms;
}
double young_cset_choice_time_ms() {
diff --git a/src/hotspot/share/gc/g1/g1HeapRegion.hpp b/src/hotspot/share/gc/g1/g1HeapRegion.hpp
index 2b4b640d52b..ec9cab26049 100644
--- a/src/hotspot/share/gc/g1/g1HeapRegion.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegion.hpp
@@ -567,41 +567,15 @@ public:
// G1HeapRegionClosure is used for iterating over regions.
// Terminates the iteration when the "do_heap_region" method returns "true".
class G1HeapRegionClosure : public StackObj {
- friend class G1HeapRegionManager;
- friend class G1CollectionSet;
- friend class G1CollectionSetCandidates;
-
- bool _is_complete;
- void set_incomplete() { _is_complete = false; }
-
public:
- G1HeapRegionClosure(): _is_complete(true) {}
-
// Typically called on each region until it returns true.
virtual bool do_heap_region(G1HeapRegion* r) = 0;
-
- // True after iteration if the closure was applied to all heap regions
- // and returned "false" in all cases.
- bool is_complete() { return _is_complete; }
};
class G1HeapRegionIndexClosure : public StackObj {
- friend class G1HeapRegionManager;
- friend class G1CollectionSet;
- friend class G1CollectionSetCandidates;
-
- bool _is_complete;
- void set_incomplete() { _is_complete = false; }
-
public:
- G1HeapRegionIndexClosure(): _is_complete(true) {}
-
// Typically called on each region until it returns true.
virtual bool do_heap_region_index(uint region_index) = 0;
-
- // True after iteration if the closure was applied to all heap regions
- // and returned "false" in all cases.
- bool is_complete() { return _is_complete; }
};
#endif // SHARE_GC_G1_G1HEAPREGION_HPP
diff --git a/src/hotspot/share/gc/g1/g1HeapRegion.inline.hpp b/src/hotspot/share/gc/g1/g1HeapRegion.inline.hpp
index 4f242b7a537..f92e37fee3c 100644
--- a/src/hotspot/share/gc/g1/g1HeapRegion.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegion.inline.hpp
@@ -42,6 +42,11 @@
#include "utilities/globalDefinitions.hpp"
inline HeapWord* G1HeapRegion::block_start(const void* addr) const {
+ if (is_young()) {
+ // We are here because of BlockLocationPrinter.
+ // Can be invoked in any context, so this region might not be parsable.
+ return nullptr;
+ }
return block_start(addr, parsable_bottom_acquire());
}
@@ -64,6 +69,7 @@ inline HeapWord* G1HeapRegion::advance_to_block_containing_addr(const void* addr
inline HeapWord* G1HeapRegion::block_start(const void* addr, HeapWord* const pb) const {
assert(addr >= bottom() && addr < top(), "invalid address");
+ assert(!is_young(), "Only non-young regions have BOT");
HeapWord* first_block = _bot->block_start_reaching_into_card(addr);
return advance_to_block_containing_addr(addr, pb, first_block);
}
diff --git a/src/hotspot/share/gc/g1/g1HeapRegionManager.cpp b/src/hotspot/share/gc/g1/g1HeapRegionManager.cpp
index 44897c8a277..3c0318827ef 100644
--- a/src/hotspot/share/gc/g1/g1HeapRegionManager.cpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionManager.cpp
@@ -34,7 +34,6 @@
#include "jfr/jfrEvents.hpp"
#include "logging/logStream.hpp"
#include "memory/allocation.hpp"
-#include "runtime/atomicAccess.hpp"
#include "runtime/mutexLocker.hpp"
#include "runtime/orderAccess.hpp"
#include "utilities/bitMap.inline.hpp"
@@ -512,7 +511,6 @@ void G1HeapRegionManager::iterate(G1HeapRegionClosure* blk) const {
guarantee(at(i) != nullptr, "Tried to access region %u that has a null G1HeapRegion*", i);
bool res = blk->do_heap_region(at(i));
if (res) {
- blk->set_incomplete();
return;
}
}
@@ -527,7 +525,6 @@ void G1HeapRegionManager::iterate(G1HeapRegionIndexClosure* blk) const {
}
bool res = blk->do_heap_region_index(i);
if (res) {
- blk->set_incomplete();
return;
}
}
diff --git a/src/hotspot/share/gc/g1/g1HeapSizingPolicy.cpp b/src/hotspot/share/gc/g1/g1HeapSizingPolicy.cpp
index 4dd0a509bcd..1b9704e8ad3 100644
--- a/src/hotspot/share/gc/g1/g1HeapSizingPolicy.cpp
+++ b/src/hotspot/share/gc/g1/g1HeapSizingPolicy.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -366,6 +366,12 @@ static size_t target_heap_capacity(size_t used_bytes, uintx free_ratio) {
}
size_t G1HeapSizingPolicy::full_collection_resize_amount(bool& expand, size_t allocation_word_size) {
+ // User-requested Full GCs introduce GC load unrelated to heap size; reset CPU
+ // usage tracking so heap resizing heuristics are driven only by GC pressure.
+ if (GCCause::is_user_requested_gc(_g1h->gc_cause())) {
+ reset_cpu_usage_tracking_data();
+ }
+
const size_t capacity_after_gc = _g1h->capacity();
// Capacity, free and used after the GC counted as full regions to
// include the waste in the following calculations.
diff --git a/src/hotspot/share/gc/g1/g1HeapVerifier.cpp b/src/hotspot/share/gc/g1/g1HeapVerifier.cpp
index a2a9bc8e857..714a2473a08 100644
--- a/src/hotspot/share/gc/g1/g1HeapVerifier.cpp
+++ b/src/hotspot/share/gc/g1/g1HeapVerifier.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -25,6 +25,7 @@
#include "code/nmethod.hpp"
#include "gc/g1/g1Allocator.inline.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
+#include "gc/g1/g1CollectorState.inline.hpp"
#include "gc/g1/g1ConcurrentMarkThread.hpp"
#include "gc/g1/g1HeapRegion.inline.hpp"
#include "gc/g1/g1HeapRegionRemSet.hpp"
@@ -236,7 +237,7 @@ private:
VerifyOption _vo;
bool _failures;
- bool is_in_full_gc() const { return G1CollectedHeap::heap()->collector_state()->in_full_gc(); }
+ bool is_in_full_gc() const { return G1CollectedHeap::heap()->collector_state()->is_in_full_gc(); }
public:
VerifyRegionClosure(VerifyOption vo)
@@ -349,7 +350,7 @@ void G1HeapVerifier::verify(VerifyOption vo) {
bool failures = rootsCl.failures() || codeRootsCl.failures();
- if (!_g1h->policy()->collector_state()->in_full_gc()) {
+ if (!_g1h->policy()->collector_state()->is_in_full_gc()) {
// If we're verifying during a full GC then the region sets
// will have been torn down at the start of the GC. Therefore
// verifying the region sets will fail. So we only verify
@@ -494,7 +495,7 @@ public:
};
void G1HeapVerifier::verify_marking_state() {
- assert(G1CollectedHeap::heap()->collector_state()->in_concurrent_start_gc(), "must be");
+ assert(G1CollectedHeap::heap()->collector_state()->is_in_concurrent_start_gc(), "must be");
// Verify TAMSes, bitmaps and liveness statistics.
//
diff --git a/src/hotspot/share/gc/g1/g1IHOPControl.cpp b/src/hotspot/share/gc/g1/g1IHOPControl.cpp
index 43698e9f12b..1e1c52477f9 100644
--- a/src/hotspot/share/gc/g1/g1IHOPControl.cpp
+++ b/src/hotspot/share/gc/g1/g1IHOPControl.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -38,18 +38,18 @@ double G1IHOPControl::predict(const TruncatedSeq* seq) const {
bool G1IHOPControl::have_enough_data_for_prediction() const {
assert(_is_adaptive, "precondition");
- return ((size_t)_marking_times_s.num() >= G1AdaptiveIHOPNumInitialSamples) &&
- ((size_t)_allocation_rate_s.num() >= G1AdaptiveIHOPNumInitialSamples);
+ return ((size_t)_marking_start_to_mixed_time_s.num() >= G1AdaptiveIHOPNumInitialSamples) &&
+ ((size_t)_old_gen_alloc_rate.num() >= G1AdaptiveIHOPNumInitialSamples);
}
-double G1IHOPControl::last_marking_length_s() const {
- return _marking_times_s.last();
+double G1IHOPControl::last_marking_start_to_mixed_time_s() const {
+ return _marking_start_to_mixed_time_s.last();
}
-size_t G1IHOPControl::actual_target_threshold() const {
+size_t G1IHOPControl::effective_target_occupancy() const {
assert(_is_adaptive, "precondition");
- // The actual target threshold takes the heap reserve and the expected waste in
+ // The effective target occupancy takes the heap reserve and the expected waste in
// free space into account.
// _heap_reserve is that part of the total heap capacity that is reserved for
// eventual promotion failure.
@@ -79,9 +79,9 @@ G1IHOPControl::G1IHOPControl(double ihop_percent,
_last_allocation_time_s(0.0),
_old_gen_alloc_tracker(old_gen_alloc_tracker),
_predictor(predictor),
- _marking_times_s(10, 0.05),
- _allocation_rate_s(10, 0.05),
- _last_unrestrained_young_size(0) {
+ _marking_start_to_mixed_time_s(10, 0.05),
+ _old_gen_alloc_rate(10, 0.05),
+ _expected_young_gen_at_first_mixed_gc(0) {
assert(_initial_ihop_percent >= 0.0 && _initial_ihop_percent <= 100.0,
"IHOP percent out of range: %.3f", ihop_percent);
assert(!_is_adaptive || _predictor != nullptr, "precondition");
@@ -98,85 +98,104 @@ void G1IHOPControl::report_statistics(G1NewTracer* new_tracer, size_t non_young_
send_trace_event(new_tracer, non_young_occupancy);
}
-void G1IHOPControl::update_allocation_info(double allocation_time_s, size_t additional_buffer_size) {
+void G1IHOPControl::update_allocation_info(double allocation_time_s, size_t expected_young_gen_size) {
assert(allocation_time_s > 0, "Invalid allocation time: %.3f", allocation_time_s);
_last_allocation_time_s = allocation_time_s;
double alloc_rate = _old_gen_alloc_tracker->last_period_old_gen_growth() / allocation_time_s;
- _allocation_rate_s.add(alloc_rate);
- _last_unrestrained_young_size = additional_buffer_size;
+ _old_gen_alloc_rate.add(alloc_rate);
+ _expected_young_gen_at_first_mixed_gc = expected_young_gen_size;
}
-void G1IHOPControl::update_marking_length(double marking_length_s) {
- assert(marking_length_s >= 0.0, "Invalid marking length: %.3f", marking_length_s);
- _marking_times_s.add(marking_length_s);
+void G1IHOPControl::add_marking_start_to_mixed_length(double length_s) {
+ assert(length_s >= 0.0, "Invalid marking length: %.3f", length_s);
+ _marking_start_to_mixed_time_s.add(length_s);
}
-size_t G1IHOPControl::get_conc_mark_start_threshold() {
+// Determine the old generation occupancy threshold at which to start
+// concurrent marking such that reclamation (first Mixed GC) begins
+// before the heap reaches a critical occupancy level.
+size_t G1IHOPControl::old_gen_threshold_for_conc_mark_start() {
guarantee(_target_occupancy > 0, "Target occupancy must be initialized");
if (!_is_adaptive || !have_enough_data_for_prediction()) {
return (size_t)(_initial_ihop_percent * _target_occupancy / 100.0);
}
- double pred_marking_time = predict(&_marking_times_s);
- double pred_rate = predict(&_allocation_rate_s);
- size_t pred_bytes = (size_t)(pred_marking_time * pred_rate);
- size_t predicted_needed = pred_bytes + _last_unrestrained_young_size;
- size_t internal_threshold = actual_target_threshold();
+ // During the time between marking start and the first Mixed GC,
+ // additional memory will be consumed:
+ // - Old gen grows due to allocations:
+ // old_gen_alloc_bytes = old_gen_alloc_rate * marking_start_to_mixed_time
+ // - Young gen will occupy a certain size at the first Mixed GC:
+ // expected_young_gen_at_first_mixed_gc
+ double marking_start_to_mixed_time = predict(&_marking_start_to_mixed_time_s);
+ double old_gen_alloc_rate = predict(&_old_gen_alloc_rate);
+ size_t old_gen_alloc_bytes = (size_t)(marking_start_to_mixed_time * old_gen_alloc_rate);
- return predicted_needed < internal_threshold
- ? internal_threshold - predicted_needed
+ // Therefore, the total heap occupancy at the first Mixed GC is:
+ // current_old_gen + old_gen_growth + expected_young_gen_at_first_mixed_gc
+ //
+ // To ensure this does not exceed the target_heap_occupancy, we work
+ // backwards to compute the old gen occupancy at which marking must start:
+ // mark_start_threshold = target_heap_occupancy -
+ // (old_gen_growth + expected_young_gen_at_first_mixed_gc)
+
+ size_t predicted_needed = old_gen_alloc_bytes + _expected_young_gen_at_first_mixed_gc;
+ size_t target_heap_occupancy = effective_target_occupancy();
+
+ return predicted_needed < target_heap_occupancy
+ ? target_heap_occupancy - predicted_needed
: 0;
}
void G1IHOPControl::print_log(size_t non_young_occupancy) {
assert(_target_occupancy > 0, "Target occupancy still not updated yet.");
- size_t cur_conc_mark_start_threshold = get_conc_mark_start_threshold();
- log_debug(gc, ihop)("Basic information (value update), threshold: %zuB (%1.2f), target occupancy: %zuB, non-young occupancy: %zuB, "
- "recent allocation size: %zuB, recent allocation duration: %1.2fms, recent old gen allocation rate: %1.2fB/s, recent marking phase length: %1.2fms",
- cur_conc_mark_start_threshold,
- percent_of(cur_conc_mark_start_threshold, _target_occupancy),
+ size_t old_gen_mark_start_threshold = old_gen_threshold_for_conc_mark_start();
+ log_debug(gc, ihop)("Basic information (value update), old-gen threshold: %zuB (%1.2f%%), target occupancy: %zuB, old-gen occupancy: %zuB (%1.2f%%), "
+ "recent old-gen allocation size: %zuB, recent allocation duration: %1.2fms, recent old-gen allocation rate: %1.2fB/s, recent marking phase length: %1.2fms",
+ old_gen_mark_start_threshold,
+ percent_of(old_gen_mark_start_threshold, _target_occupancy),
_target_occupancy,
non_young_occupancy,
+ percent_of(non_young_occupancy, _target_occupancy),
_old_gen_alloc_tracker->last_period_old_gen_bytes(),
_last_allocation_time_s * 1000.0,
_last_allocation_time_s > 0.0 ? _old_gen_alloc_tracker->last_period_old_gen_bytes() / _last_allocation_time_s : 0.0,
- last_marking_length_s() * 1000.0);
+ last_marking_start_to_mixed_time_s() * 1000.0);
if (!_is_adaptive) {
return;
}
- size_t actual_threshold = actual_target_threshold();
- log_debug(gc, ihop)("Adaptive IHOP information (value update), threshold: %zuB (%1.2f), internal target threshold: %zuB, "
- "non-young occupancy: %zuB, additional buffer size: %zuB, predicted old gen allocation rate: %1.2fB/s, "
- "predicted marking phase length: %1.2fms, prediction active: %s",
- cur_conc_mark_start_threshold,
- percent_of(cur_conc_mark_start_threshold, actual_threshold),
- actual_threshold,
+ size_t effective_target = effective_target_occupancy();
+ log_debug(gc, ihop)("Adaptive IHOP information (value update), prediction active: %s, old-gen threshold: %zuB (%1.2f%%), internal target occupancy: %zuB, "
+ "old-gen occupancy: %zuB, additional buffer size: %zuB, predicted old-gen allocation rate: %1.2fB/s, "
+ "predicted marking phase length: %1.2fms",
+ BOOL_TO_STR(have_enough_data_for_prediction()),
+ old_gen_mark_start_threshold,
+ percent_of(old_gen_mark_start_threshold, effective_target),
+ effective_target,
non_young_occupancy,
- _last_unrestrained_young_size,
- predict(&_allocation_rate_s),
- predict(&_marking_times_s) * 1000.0,
- have_enough_data_for_prediction() ? "true" : "false");
+ _expected_young_gen_at_first_mixed_gc,
+ predict(&_old_gen_alloc_rate),
+ predict(&_marking_start_to_mixed_time_s) * 1000.0);
}
void G1IHOPControl::send_trace_event(G1NewTracer* tracer, size_t non_young_occupancy) {
assert(_target_occupancy > 0, "Target occupancy still not updated yet.");
- tracer->report_basic_ihop_statistics(get_conc_mark_start_threshold(),
+ tracer->report_basic_ihop_statistics(old_gen_threshold_for_conc_mark_start(),
_target_occupancy,
non_young_occupancy,
_old_gen_alloc_tracker->last_period_old_gen_bytes(),
_last_allocation_time_s,
- last_marking_length_s());
+ last_marking_start_to_mixed_time_s());
if (_is_adaptive) {
- tracer->report_adaptive_ihop_statistics(get_conc_mark_start_threshold(),
- actual_target_threshold(),
+ tracer->report_adaptive_ihop_statistics(old_gen_threshold_for_conc_mark_start(),
+ effective_target_occupancy(),
non_young_occupancy,
- _last_unrestrained_young_size,
- predict(&_allocation_rate_s),
- predict(&_marking_times_s),
+ _expected_young_gen_at_first_mixed_gc,
+ predict(&_old_gen_alloc_rate),
+ predict(&_marking_start_to_mixed_time_s),
have_enough_data_for_prediction());
}
}
diff --git a/src/hotspot/share/gc/g1/g1IHOPControl.hpp b/src/hotspot/share/gc/g1/g1IHOPControl.hpp
index 24061c026d1..ff209012f02 100644
--- a/src/hotspot/share/gc/g1/g1IHOPControl.hpp
+++ b/src/hotspot/share/gc/g1/g1IHOPControl.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -58,8 +58,11 @@ class G1IHOPControl : public CHeapObj {
const G1OldGenAllocationTracker* _old_gen_alloc_tracker;
const G1Predictions* _predictor;
- TruncatedSeq _marking_times_s;
- TruncatedSeq _allocation_rate_s;
+ // Wall-clock time in seconds from marking start to the first mixed GC,
+ // excluding GC Pause time.
+ TruncatedSeq _marking_start_to_mixed_time_s;
+ // Old generation allocation rate in bytes per second.
+ TruncatedSeq _old_gen_alloc_rate;
// The most recent unrestrained size of the young gen. This is used as an additional
// factor in the calculation of the threshold, as the threshold is based on
@@ -68,18 +71,18 @@ class G1IHOPControl : public CHeapObj {
// Since we cannot know what young gen sizes are used in the future, we will just
// use the current one. We expect that this one will be one with a fairly large size,
// as there is no marking or mixed gc that could impact its size too much.
- size_t _last_unrestrained_young_size;
+ size_t _expected_young_gen_at_first_mixed_gc;
// Get a new prediction bounded below by zero from the given sequence.
double predict(const TruncatedSeq* seq) const;
bool have_enough_data_for_prediction() const;
- double last_marking_length_s() const;
+ double last_marking_start_to_mixed_time_s() const;
- // The "actual" target threshold the algorithm wants to keep during and at the
- // end of marking. This is typically lower than the requested threshold, as the
+ // The "effective" target occupancy the algorithm wants to keep until the start
+ // of Mixed GCs. This is typically lower than the target occupancy, as the
// algorithm needs to consider restrictions by the environment.
- size_t actual_target_threshold() const;
+ size_t effective_target_occupancy() const;
void print_log(size_t non_young_occupancy);
void send_trace_event(G1NewTracer* tracer, size_t non_young_occupancy);
@@ -95,22 +98,24 @@ class G1IHOPControl : public CHeapObj {
// Adjust target occupancy.
void update_target_occupancy(size_t new_target_occupancy);
- // Update information about time during which allocations in the Java heap occurred,
- // how large these allocations were in bytes, and an additional buffer.
- // The allocations should contain any amount of space made unusable for further
- // allocation, e.g. any waste caused by TLAB allocation, space at the end of
- // humongous objects that can not be used for allocation, etc.
- // Together with the target occupancy, this additional buffer should contain the
- // difference between old gen size and total heap size at the start of reclamation,
- // and space required for that reclamation.
- void update_allocation_info(double allocation_time_s, size_t additional_buffer_size);
+ void update_target_after_marking_phase();
+
+ // Update allocation rate information and current expected young gen size for the
+ // first mixed gc needed for the predictor. Allocation rate is given as the
+ // separately passed in allocation increment and the time passed (mutator time)
+ // for the latest allocation increment here. Allocation size is the memory needed
+ // during the mutator before and the first mixed gc pause itself.
+ // Contents include young gen at that point, and the memory required for evacuating
+ // the collection set in that first mixed gc (including waste caused by PLAB
+ // allocation etc.).
+ void update_allocation_info(double allocation_time_s, size_t expected_young_gen_size);
// Update the time spent in the mutator beginning from the end of concurrent start to
// the first mixed gc.
- void update_marking_length(double marking_length_s);
+ void add_marking_start_to_mixed_length(double length_s);
// Get the current non-young occupancy at which concurrent marking should start.
- size_t get_conc_mark_start_threshold();
+ size_t old_gen_threshold_for_conc_mark_start();
void report_statistics(G1NewTracer* tracer, size_t non_young_occupancy);
};
diff --git a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
index 75a8ef1a336..cb857dc6eab 100644
--- a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
+++ b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
@@ -741,7 +741,7 @@ void G1ParScanThreadStateSet::print_partial_array_task_stats() {
return state_for_worker(i)->partial_array_task_stats();
};
PartialArrayTaskStats::log_set(_num_workers, get_stats,
- "Partial Array Task Stats");
+ "Young GC Partial Array");
}
#endif // TASKQUEUE_STATS
diff --git a/src/hotspot/share/gc/g1/g1PeriodicGCTask.cpp b/src/hotspot/share/gc/g1/g1PeriodicGCTask.cpp
index f280d76f3c7..b5ff4272764 100644
--- a/src/hotspot/share/gc/g1/g1PeriodicGCTask.cpp
+++ b/src/hotspot/share/gc/g1/g1PeriodicGCTask.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -23,6 +23,7 @@
*/
#include "gc/g1/g1CollectedHeap.inline.hpp"
+#include "gc/g1/g1CollectorState.inline.hpp"
#include "gc/g1/g1ConcurrentMark.inline.hpp"
#include "gc/g1/g1ConcurrentMarkThread.inline.hpp"
#include "gc/g1/g1GCCounters.hpp"
@@ -38,8 +39,8 @@ bool G1PeriodicGCTask::should_start_periodic_gc(G1CollectedHeap* g1h,
// Ensure no GC safepoints while we're doing the checks, to avoid data races.
SuspendibleThreadSetJoiner sts;
- // If we are currently in a concurrent mark we are going to uncommit memory soon.
- if (g1h->concurrent_mark()->in_progress()) {
+ // If we are currently in a concurrent cycle we are going to uncommit memory soon.
+ if (g1h->collector_state()->is_in_concurrent_cycle()) {
log_debug(gc, periodic)("Concurrent cycle in progress. Skipping.");
return false;
}
diff --git a/src/hotspot/share/gc/g1/g1Policy.cpp b/src/hotspot/share/gc/g1/g1Policy.cpp
index 98e6acc1d77..5744bbc2f03 100644
--- a/src/hotspot/share/gc/g1/g1Policy.cpp
+++ b/src/hotspot/share/gc/g1/g1Policy.cpp
@@ -28,7 +28,7 @@
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1CollectionSet.hpp"
#include "gc/g1/g1CollectionSetCandidates.inline.hpp"
-#include "gc/g1/g1CollectionSetChooser.hpp"
+#include "gc/g1/g1CollectorState.inline.hpp"
#include "gc/g1/g1ConcurrentMark.hpp"
#include "gc/g1/g1ConcurrentMarkThread.inline.hpp"
#include "gc/g1/g1ConcurrentRefine.hpp"
@@ -177,7 +177,7 @@ uint G1Policy::calculate_desired_eden_length_by_mmu() const {
void G1Policy::update_young_length_bounds() {
assert(!Universe::is_fully_initialized() || SafepointSynchronize::is_at_safepoint(), "must be");
- bool for_young_only_phase = collector_state()->in_young_only_phase();
+ bool for_young_only_phase = collector_state()->is_in_young_only_phase();
update_young_length_bounds(_analytics->predict_pending_cards(for_young_only_phase),
_analytics->predict_card_rs_length(for_young_only_phase),
_analytics->predict_code_root_rs_length(for_young_only_phase));
@@ -505,7 +505,7 @@ uint G1Policy::calculate_desired_eden_length_before_mixed(double base_time_ms,
double G1Policy::predict_survivor_regions_evac_time() const {
double survivor_regions_evac_time = predict_young_region_other_time_ms(_g1h->survivor()->length());
for (G1HeapRegion* r : _g1h->survivor()->regions()) {
- survivor_regions_evac_time += predict_region_copy_time_ms(r, _g1h->collector_state()->in_young_only_phase());
+ survivor_regions_evac_time += predict_region_copy_time_ms(r, _g1h->collector_state()->is_in_young_only_phase());
}
return survivor_regions_evac_time;
@@ -561,8 +561,7 @@ void G1Policy::revise_young_list_target_length(size_t pending_cards, size_t card
void G1Policy::record_full_collection_start() {
record_pause_start_time();
// Release the future to-space so that it is available for compaction into.
- collector_state()->set_in_young_only_phase(false);
- collector_state()->set_in_full_gc(true);
+ collector_state()->set_in_full_gc();
_collection_set->abandon_all_candidates();
}
@@ -571,17 +570,10 @@ void G1Policy::record_full_collection_end(size_t allocation_word_size) {
// since last pause.
double end_sec = os::elapsedTime();
- collector_state()->set_in_full_gc(false);
-
// "Nuke" the heuristics that control the young/mixed GC
// transitions and make sure we start with young GCs after the Full GC.
- collector_state()->set_in_young_only_phase(true);
- collector_state()->set_in_young_gc_before_mixed(false);
+ collector_state()->set_in_normal_young_gc();
collector_state()->set_initiate_conc_mark_if_possible(need_to_start_conc_mark("end of Full GC", allocation_word_size));
- collector_state()->set_in_concurrent_start_gc(false);
- collector_state()->set_mark_in_progress(false);
- collector_state()->set_mark_or_rebuild_in_progress(false);
- collector_state()->set_clear_bitmap_in_progress(false);
_eden_surv_rate_group->start_adding_regions();
// also call this on any additional surv rate groups
@@ -593,7 +585,7 @@ void G1Policy::record_full_collection_end(size_t allocation_word_size) {
_old_gen_alloc_tracker.reset_after_gc(_g1h->humongous_regions_count() * G1HeapRegion::GrainBytes);
double start_time_sec = cur_pause_start_sec();
- record_pause(G1GCPauseType::FullGC, start_time_sec, end_sec);
+ record_pause(Pause::Full, start_time_sec, end_sec);
}
static void log_refinement_stats(const G1ConcurrentRefineStats& stats) {
@@ -698,18 +690,12 @@ void G1Policy::record_young_collection_start() {
assert(_g1h->collection_set()->verify_young_ages(), "region age verification failed");
}
-void G1Policy::record_concurrent_mark_init_end() {
- assert(!collector_state()->initiate_conc_mark_if_possible(), "we should have cleared it by now");
- collector_state()->set_in_concurrent_start_gc(false);
-}
-
void G1Policy::record_concurrent_mark_remark_end() {
double end_time_sec = os::elapsedTime();
double start_time_sec = cur_pause_start_sec();
double elapsed_time_ms = (end_time_sec - start_time_sec) * 1000.0;
_analytics->report_concurrent_mark_remark_times_ms(elapsed_time_ms);
- record_pause(G1GCPauseType::Remark, start_time_sec, end_time_sec);
- collector_state()->set_mark_in_progress(false);
+ record_pause(Pause::Remark, start_time_sec, end_time_sec);
}
G1CollectionSetCandidates* G1Policy::candidates() const {
@@ -739,7 +725,7 @@ double G1Policy::constant_other_time_ms(double pause_time_ms) const {
}
bool G1Policy::about_to_start_mixed_phase() const {
- return _g1h->concurrent_mark()->in_progress() || collector_state()->in_young_gc_before_mixed();
+ return collector_state()->is_in_concurrent_cycle() || collector_state()->is_in_prepare_mixed_gc();
}
bool G1Policy::need_to_start_conc_mark(const char* source, size_t allocation_word_size) {
@@ -747,21 +733,21 @@ bool G1Policy::need_to_start_conc_mark(const char* source, size_t allocation_wor
return false;
}
- size_t marking_initiating_used_threshold = _ihop_control->get_conc_mark_start_threshold();
+ size_t marking_initiating_old_gen_threshold = _ihop_control->old_gen_threshold_for_conc_mark_start();
size_t non_young_occupancy = _g1h->non_young_occupancy_after_allocation(allocation_word_size);
bool result = false;
- if (non_young_occupancy > marking_initiating_used_threshold) {
- result = collector_state()->in_young_only_phase();
+ if (non_young_occupancy > marking_initiating_old_gen_threshold) {
+ result = collector_state()->is_in_young_only_phase();
log_debug(gc, ergo, ihop)("%s non-young occupancy: %zuB allocation request: %zuB threshold: %zuB (%1.2f) source: %s",
result ? "Request concurrent cycle initiation (occupancy higher than threshold)" : "Do not request concurrent cycle initiation (still doing mixed collections)",
- non_young_occupancy, allocation_word_size * HeapWordSize, marking_initiating_used_threshold, (double) marking_initiating_used_threshold / _g1h->capacity() * 100, source);
+ non_young_occupancy, allocation_word_size * HeapWordSize, marking_initiating_old_gen_threshold, (double) marking_initiating_old_gen_threshold / _g1h->capacity() * 100, source);
}
return result;
}
bool G1Policy::concurrent_operation_is_full_mark(const char* msg, size_t allocation_word_size) {
- return collector_state()->in_concurrent_start_gc() &&
+ return collector_state()->is_in_concurrent_start_gc() &&
((_g1h->gc_cause() != GCCause::_g1_humongous_allocation) || need_to_start_conc_mark(msg, allocation_word_size));
}
@@ -800,11 +786,12 @@ void G1Policy::record_young_collection_end(bool concurrent_operation_is_full_mar
double end_time_sec = Ticks::now().seconds();
double pause_time_ms = (end_time_sec - start_time_sec) * 1000.0;
- G1GCPauseType this_pause = collector_state()->young_gc_pause_type(concurrent_operation_is_full_mark);
- bool is_young_only_pause = G1GCPauseTypeHelper::is_young_only_pause(this_pause);
+ Pause this_pause = collector_state()->gc_pause_type(concurrent_operation_is_full_mark);
+ bool is_young_only_pause = G1CollectorState::is_young_only_pause(this_pause);
- if (G1GCPauseTypeHelper::is_concurrent_start_pause(this_pause)) {
- record_concurrent_mark_init_end();
+ if (G1CollectorState::is_concurrent_start_pause(this_pause)) {
+ assert(!collector_state()->initiate_conc_mark_if_possible(), "we should have cleared it by now");
+ collector_state()->set_in_normal_young_gc();
} else {
maybe_start_marking(allocation_word_size);
}
@@ -945,20 +932,19 @@ void G1Policy::record_young_collection_end(bool concurrent_operation_is_full_mar
record_pause(this_pause, start_time_sec, end_time_sec);
- if (G1GCPauseTypeHelper::is_last_young_pause(this_pause)) {
- assert(!G1GCPauseTypeHelper::is_concurrent_start_pause(this_pause),
+ if (G1CollectorState::is_prepare_mixed_pause(this_pause)) {
+ assert(!G1CollectorState::is_concurrent_start_pause(this_pause),
"The young GC before mixed is not allowed to be concurrent start GC");
// This has been the young GC before we start doing mixed GCs. We already
// decided to start mixed GCs much earlier, so there is nothing to do except
// advancing the state.
- collector_state()->set_in_young_only_phase(false);
- collector_state()->set_in_young_gc_before_mixed(false);
- } else if (G1GCPauseTypeHelper::is_mixed_pause(this_pause)) {
+ collector_state()->set_in_space_reclamation_phase();
+ } else if (G1CollectorState::is_mixed_pause(this_pause)) {
// This is a mixed GC. Here we decide whether to continue doing more
// mixed GCs or not.
if (!next_gc_should_be_mixed()) {
log_debug(gc, ergo)("do not continue mixed GCs (candidate old regions not available)");
- collector_state()->set_in_young_only_phase(true);
+ collector_state()->set_in_normal_young_gc();
assert(!candidates()->has_more_marking_candidates(),
"only end mixed if all candidates from marking were processed");
@@ -971,12 +957,8 @@ void G1Policy::record_young_collection_end(bool concurrent_operation_is_full_mar
_eden_surv_rate_group->start_adding_regions();
- assert(!(G1GCPauseTypeHelper::is_concurrent_start_pause(this_pause) && collector_state()->mark_or_rebuild_in_progress()),
- "If the last pause has been concurrent start, we should not have been in the marking window");
- if (G1GCPauseTypeHelper::is_concurrent_start_pause(this_pause)) {
- collector_state()->set_mark_in_progress(concurrent_operation_is_full_mark);
- collector_state()->set_mark_or_rebuild_in_progress(concurrent_operation_is_full_mark);
- }
+ assert(!(G1CollectorState::is_concurrent_start_pause(this_pause) && collector_state()->is_in_concurrent_cycle()),
+ "If the last pause has been concurrent start, we should not have been in the marking cycle");
_free_regions_at_end_of_collection = _g1h->num_free_regions();
@@ -986,8 +968,7 @@ void G1Policy::record_young_collection_end(bool concurrent_operation_is_full_mar
update_young_length_bounds();
_old_gen_alloc_tracker.reset_after_gc(_g1h->humongous_regions_count() * G1HeapRegion::GrainBytes);
- if (update_ihop_prediction(app_time_ms / 1000.0,
- G1GCPauseTypeHelper::is_young_only_pause(this_pause))) {
+ if (update_ihop_prediction(app_time_ms / 1000.0, is_young_only_pause)) {
_ihop_control->report_statistics(_g1h->gc_tracer_stw(), _g1h->non_young_occupancy_after_allocation(allocation_word_size));
}
} else {
@@ -1044,14 +1025,13 @@ bool G1Policy::update_ihop_prediction(double mutator_time_s,
bool report = false;
- double marking_to_mixed_time = -1.0;
if (!this_gc_was_young_only && _concurrent_start_to_mixed.has_result()) {
- marking_to_mixed_time = _concurrent_start_to_mixed.last_marking_time();
+ double marking_to_mixed_time = _concurrent_start_to_mixed.get_and_reset_last_marking_time();
assert(marking_to_mixed_time > 0.0,
"Concurrent start to mixed time must be larger than zero but is %.3f",
marking_to_mixed_time);
if (marking_to_mixed_time > min_valid_time) {
- _ihop_control->update_marking_length(marking_to_mixed_time);
+ _ihop_control->add_marking_start_to_mixed_length(marking_to_mixed_time);
report = true;
}
}
@@ -1081,7 +1061,7 @@ void G1Policy::record_young_gc_pause_end(bool evacuation_failed) {
double G1Policy::predict_base_time_ms(size_t pending_cards,
size_t card_rs_length,
size_t code_root_rs_length) const {
- bool in_young_only_phase = collector_state()->in_young_only_phase();
+ bool in_young_only_phase = collector_state()->is_in_young_only_phase();
// Cards from the refinement table and the cards from the young gen remset are
// unique to each other as they are located on the card table.
@@ -1105,7 +1085,7 @@ double G1Policy::predict_base_time_ms(size_t pending_cards,
}
double G1Policy::predict_base_time_ms(size_t pending_cards, size_t card_rs_length) const {
- bool for_young_only_phase = collector_state()->in_young_only_phase();
+ bool for_young_only_phase = collector_state()->is_in_young_only_phase();
size_t code_root_rs_length = _analytics->predict_code_root_rs_length(for_young_only_phase);
return predict_base_time_ms(pending_cards, card_rs_length, code_root_rs_length);
}
@@ -1145,7 +1125,18 @@ double G1Policy::predict_eden_copy_time_ms(uint count, size_t* bytes_to_copy) co
if (bytes_to_copy != nullptr) {
*bytes_to_copy = expected_bytes;
}
- return _analytics->predict_object_copy_time_ms(expected_bytes, collector_state()->in_young_only_phase());
+ return _analytics->predict_object_copy_time_ms(expected_bytes, collector_state()->is_in_young_only_phase());
+}
+
+bool G1Policy::should_update_surv_rate_group_predictors() {
+ return collector_state()->is_in_young_only_phase() && !collector_state()->is_in_mark_or_rebuild();
+}
+
+void G1Policy::cset_regions_freed() {
+ bool update = should_update_surv_rate_group_predictors();
+
+ _eden_surv_rate_group->all_surviving_words_recorded(predictor(), update);
+ _survivor_surv_rate_group->all_surviving_words_recorded(predictor(), update);
}
double G1Policy::predict_region_copy_time_ms(G1HeapRegion* hr, bool for_young_only_phase) const {
@@ -1235,7 +1226,7 @@ bool G1Policy::force_concurrent_start_if_outside_cycle(GCCause::Cause gc_cause)
// We actually check whether we are marking here and not if we are in a
// reclamation phase. This means that we will schedule a concurrent mark
// even while we are still in the process of reclaiming memory.
- bool during_cycle = _g1h->concurrent_mark()->in_progress();
+ bool during_cycle = collector_state()->is_in_concurrent_cycle();
if (!during_cycle) {
log_debug(gc, ergo)("Request concurrent cycle initiation (requested by GC cause). "
"GC cause: %s",
@@ -1250,11 +1241,6 @@ bool G1Policy::force_concurrent_start_if_outside_cycle(GCCause::Cause gc_cause)
}
}
-void G1Policy::initiate_conc_mark() {
- collector_state()->set_in_concurrent_start_gc(true);
- collector_state()->set_initiate_conc_mark_if_possible(false);
-}
-
static const char* requester_for_mixed_abort(GCCause::Cause cause) {
if (cause == GCCause::_wb_breakpoint) {
return "run_to breakpoint";
@@ -1270,11 +1256,11 @@ void G1Policy::decide_on_concurrent_start_pause() {
// We are about to decide on whether this pause will be a
// concurrent start pause.
- // First, collector_state()->in_concurrent_start_gc() should not be already set. We
+ // First, collector_state()->is_in_concurrent_start_gc() should not already be set. We
// will set it here if we have to. However, it should be cleared by
// the end of the pause (it's only set for the duration of a
// concurrent start pause).
- assert(!collector_state()->in_concurrent_start_gc(), "pre-condition");
+ assert(!collector_state()->is_in_concurrent_start_gc(), "pre-condition");
if (collector_state()->initiate_conc_mark_if_possible()) {
// We had noticed on a previous pause that the heap occupancy has
@@ -1287,25 +1273,21 @@ void G1Policy::decide_on_concurrent_start_pause() {
if ((cause != GCCause::_wb_breakpoint) &&
ConcurrentGCBreakpoints::is_controlled()) {
log_debug(gc, ergo)("Do not initiate concurrent cycle (whitebox controlled)");
- } else if (!about_to_start_mixed_phase() && collector_state()->in_young_only_phase()) {
+ } else if (!about_to_start_mixed_phase() && collector_state()->is_in_young_only_phase()) {
// Initiate a new concurrent start if there is no marking or reclamation going on.
- initiate_conc_mark();
+ collector_state()->set_in_concurrent_start_gc();
log_debug(gc, ergo)("Initiate concurrent cycle (concurrent cycle initiation requested)");
} else if (_g1h->is_user_requested_concurrent_full_gc(cause) ||
GCCause::is_codecache_requested_gc(cause) ||
(cause == GCCause::_wb_breakpoint)) {
- // Initiate a concurrent start. A concurrent start must be a young only
- // GC, so the collector state must be updated to reflect this.
- collector_state()->set_in_young_only_phase(true);
- collector_state()->set_in_young_gc_before_mixed(false);
-
+ // Force concurrent start.
+ collector_state()->set_in_concurrent_start_gc();
// We might have ended up coming here about to start a mixed phase with a collection set
// active. The following remark might change the change the "evacuation efficiency" of
// the regions in this set, leading to failing asserts later.
// Since the concurrent cycle will recreate the collection set anyway, simply drop it here.
abandon_collection_set_candidates();
abort_time_to_mixed_tracking();
- initiate_conc_mark();
log_debug(gc, ergo)("Initiate concurrent cycle (%s requested concurrent cycle)",
requester_for_mixed_abort(cause));
} else {
@@ -1326,10 +1308,10 @@ void G1Policy::decide_on_concurrent_start_pause() {
}
// Result consistency checks.
// We do not allow concurrent start to be piggy-backed on a mixed GC.
- assert(!collector_state()->in_concurrent_start_gc() ||
- collector_state()->in_young_only_phase(), "sanity");
- // We also do not allow mixed GCs during marking.
- assert(!collector_state()->mark_or_rebuild_in_progress() || collector_state()->in_young_only_phase(), "sanity");
+ assert(!collector_state()->is_in_concurrent_start_gc() ||
+ collector_state()->is_in_young_only_phase(), "sanity");
+ // We also do not allow mixed GCs during marking/rebuilding.
+ assert(!collector_state()->is_in_mark_or_rebuild() || collector_state()->is_in_young_only_phase(), "sanity %d %d", collector_state()->is_in_concurrent_cycle(), collector_state()->is_in_young_only_phase());
}
void G1Policy::record_concurrent_mark_cleanup_end(bool has_rebuilt_remembered_sets) {
@@ -1348,16 +1330,16 @@ void G1Policy::record_concurrent_mark_cleanup_end(bool has_rebuilt_remembered_se
abort_time_to_mixed_tracking();
log_debug(gc, ergo)("request young-only gcs (candidate old regions not available)");
}
- collector_state()->set_in_young_gc_before_mixed(mixed_gc_pending);
- collector_state()->set_mark_or_rebuild_in_progress(false);
- collector_state()->set_clear_bitmap_in_progress(true);
+ if (mixed_gc_pending) {
+ collector_state()->set_in_prepare_mixed_gc();
+ }
double end_sec = os::elapsedTime();
double start_sec = cur_pause_start_sec();
double elapsed_time_ms = (end_sec - start_sec) * 1000.0;
_analytics->report_concurrent_mark_cleanup_times_ms(elapsed_time_ms);
- record_pause(G1GCPauseType::Cleanup, start_sec, end_sec);
+ record_pause(Pause::Cleanup, start_sec, end_sec);
}
void G1Policy::abandon_collection_set_candidates() {
@@ -1373,25 +1355,25 @@ void G1Policy::maybe_start_marking(size_t allocation_word_size) {
}
}
-void G1Policy::update_gc_pause_time_ratios(G1GCPauseType gc_type, double start_time_sec, double end_time_sec) {
+void G1Policy::update_gc_pause_time_ratios(Pause gc_type, double start_time_sec, double end_time_sec) {
double pause_time_sec = end_time_sec - start_time_sec;
double pause_time_ms = pause_time_sec * 1000.0;
_analytics->update_gc_time_ratios(end_time_sec, pause_time_ms);
- if (gc_type == G1GCPauseType::Cleanup || gc_type == G1GCPauseType::Remark) {
+ if (G1CollectorState::is_concurrent_cycle_pause(gc_type)) {
_analytics->append_prev_collection_pause_end_ms(pause_time_ms);
} else {
_analytics->set_prev_collection_pause_end_ms(end_time_sec * 1000.0);
}
}
-void G1Policy::record_pause(G1GCPauseType gc_type,
+void G1Policy::record_pause(Pause gc_type,
double start,
double end) {
// Manage the MMU tracker. For some reason it ignores Full GCs.
- if (gc_type != G1GCPauseType::FullGC) {
+ if (gc_type != Pause::Full) {
_mmu_tracker->add_pause(start, end);
}
@@ -1403,21 +1385,21 @@ void G1Policy::record_pause(G1GCPauseType gc_type,
_analytics->set_gc_cpu_time_at_pause_end_ms(elapsed_gc_cpu_time);
}
-void G1Policy::update_time_to_mixed_tracking(G1GCPauseType gc_type,
+void G1Policy::update_time_to_mixed_tracking(Pause gc_type,
double start,
double end) {
// Manage the mutator time tracking from concurrent start to first mixed gc.
switch (gc_type) {
- case G1GCPauseType::FullGC:
+ case Pause::Full:
abort_time_to_mixed_tracking();
break;
- case G1GCPauseType::Cleanup:
- case G1GCPauseType::Remark:
- case G1GCPauseType::YoungGC:
- case G1GCPauseType::LastYoungGC:
+ case Pause::Cleanup:
+ case Pause::Remark:
+ case Pause::Normal:
+ case Pause::PrepareMixed:
_concurrent_start_to_mixed.add_pause(end - start);
break;
- case G1GCPauseType::ConcurrentStartMarkGC:
+ case Pause::ConcurrentStartFull:
// Do not track time-to-mixed time for periodic collections as they are likely
// to be not representative to regular operation as the mutators are idle at
// that time. Also only track full concurrent mark cycles.
@@ -1425,12 +1407,12 @@ void G1Policy::update_time_to_mixed_tracking(G1GCPauseType gc_type,
_concurrent_start_to_mixed.record_concurrent_start_end(end);
}
break;
- case G1GCPauseType::ConcurrentStartUndoGC:
+ case Pause::ConcurrentStartUndo:
assert(_g1h->gc_cause() == GCCause::_g1_humongous_allocation,
"GC cause must be humongous allocation but is %d",
_g1h->gc_cause());
break;
- case G1GCPauseType::MixedGC:
+ case Pause::Mixed:
_concurrent_start_to_mixed.record_mixed_gc_start(start);
break;
default:
diff --git a/src/hotspot/share/gc/g1/g1Policy.hpp b/src/hotspot/share/gc/g1/g1Policy.hpp
index 9513c79869e..5c5c2bc3572 100644
--- a/src/hotspot/share/gc/g1/g1Policy.hpp
+++ b/src/hotspot/share/gc/g1/g1Policy.hpp
@@ -56,7 +56,7 @@ class GCPolicyCounters;
class STWGCTimer;
class G1Policy: public CHeapObj {
- private:
+ using Pause = G1CollectorState::Pause;
static G1IHOPControl* create_ihop_control(const G1OldGenAllocationTracker* old_gen_alloc_tracker,
const G1Predictions* predictor);
@@ -114,9 +114,7 @@ class G1Policy: public CHeapObj {
G1ConcurrentStartToMixedTimeTracker _concurrent_start_to_mixed;
- bool should_update_surv_rate_group_predictors() {
- return collector_state()->in_young_only_phase() && !collector_state()->mark_or_rebuild_in_progress();
- }
+ bool should_update_surv_rate_group_predictors();
double pending_cards_processing_time() const;
public:
@@ -160,12 +158,7 @@ public:
// bytes_to_copy is non-null.
double predict_eden_copy_time_ms(uint count, size_t* bytes_to_copy = nullptr) const;
- void cset_regions_freed() {
- bool update = should_update_surv_rate_group_predictors();
-
- _eden_surv_rate_group->all_surviving_words_recorded(predictor(), update);
- _survivor_surv_rate_group->all_surviving_words_recorded(predictor(), update);
- }
+ void cset_regions_freed();
G1MMUTracker* mmu_tracker() {
return _mmu_tracker;
@@ -268,13 +261,13 @@ private:
// Sets up marking if proper conditions are met.
void maybe_start_marking(size_t allocation_word_size);
// Manage time-to-mixed tracking.
- void update_time_to_mixed_tracking(G1GCPauseType gc_type, double start, double end);
+ void update_time_to_mixed_tracking(Pause gc_type, double start, double end);
// Record the given STW pause with the given start and end times (in s).
- void record_pause(G1GCPauseType gc_type,
+ void record_pause(Pause gc_type,
double start,
double end);
- void update_gc_pause_time_ratios(G1GCPauseType gc_type, double start_sec, double end_sec);
+ void update_gc_pause_time_ratios(Pause gc_type, double start_sec, double end_sec);
// Indicate that we aborted marking before doing any mixed GCs.
void abort_time_to_mixed_tracking();
@@ -320,9 +313,6 @@ public:
void record_full_collection_start();
void record_full_collection_end(size_t allocation_word_size);
- // Must currently be called while the world is stopped.
- void record_concurrent_mark_init_end();
-
void record_concurrent_mark_remark_end();
// Record start, end, and completion of cleanup.
@@ -339,11 +329,6 @@ private:
// regions and update the associated members.
void update_survival_estimates_for_next_collection();
- // Set the state to start a concurrent marking cycle and clear
- // _initiate_conc_mark_if_possible because it has now been
- // acted on.
- void initiate_conc_mark();
-
public:
// This sets the initiate_conc_mark_if_possible() flag to start a
// new cycle, as long as we are not already in one. It's best if it
diff --git a/src/hotspot/share/gc/g1/g1RegionMarkStatsCache.hpp b/src/hotspot/share/gc/g1/g1RegionMarkStatsCache.hpp
index 4dcdd33846e..df76147f4b1 100644
--- a/src/hotspot/share/gc/g1/g1RegionMarkStatsCache.hpp
+++ b/src/hotspot/share/gc/g1/g1RegionMarkStatsCache.hpp
@@ -95,7 +95,7 @@ private:
// Evict a given element of the statistics cache.
void evict(uint idx);
- size_t _num_cache_entries_mask;
+ const uint _num_cache_entries_mask;
uint hash(uint idx) {
return idx & _num_cache_entries_mask;
diff --git a/src/hotspot/share/gc/g1/g1RemSet.cpp b/src/hotspot/share/gc/g1/g1RemSet.cpp
index 0c9a0fad8f2..9f9f0ecdf3a 100644
--- a/src/hotspot/share/gc/g1/g1RemSet.cpp
+++ b/src/hotspot/share/gc/g1/g1RemSet.cpp
@@ -31,6 +31,7 @@
#include "gc/g1/g1CardTableEntryClosure.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1CollectionSet.inline.hpp"
+#include "gc/g1/g1CollectorState.inline.hpp"
#include "gc/g1/g1ConcurrentRefine.hpp"
#include "gc/g1/g1ConcurrentRefineSweepTask.hpp"
#include "gc/g1/g1FromCardCache.hpp"
@@ -1026,7 +1027,7 @@ class G1MergeHeapRootsTask : public WorkerTask {
// the pause occurs during the Concurrent Cleanup for Next Mark phase.
// Only at that point the region's bitmap may contain marks while being in the collection
// set at the same time.
- return _g1h->collector_state()->clear_bitmap_in_progress() &&
+ return _g1h->collector_state()->is_in_reset_for_next_cycle() &&
hr->is_old();
}
diff --git a/src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.cpp b/src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.cpp
index 0c9973c520d..94f5466b8e0 100644
--- a/src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.cpp
+++ b/src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -23,12 +23,16 @@
*/
#include "gc/g1/g1CollectedHeap.inline.hpp"
-#include "gc/g1/g1CollectionSetChooser.hpp"
#include "gc/g1/g1HeapRegion.inline.hpp"
#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "gc/g1/g1RemSetTrackingPolicy.hpp"
#include "runtime/safepoint.hpp"
+static bool region_occupancy_low_enough_for_evac(size_t live_bytes) {
+ size_t mixed_gc_live_threshold_bytes = G1HeapRegion::GrainBytes * (size_t)G1MixedGCLiveThresholdPercent / 100;
+ return live_bytes < mixed_gc_live_threshold_bytes;
+}
+
void G1RemSetTrackingPolicy::update_at_allocate(G1HeapRegion* r) {
assert(r->is_young() || r->is_humongous() || r->is_old(),
"Region %u with unexpected heap region type %s", r->hrm_index(), r->get_type_str());
@@ -75,7 +79,8 @@ bool G1RemSetTrackingPolicy::update_old_before_rebuild(G1HeapRegion* r) {
bool selected_for_rebuild = false;
- if (G1CollectionSetChooser::region_occupancy_low_enough_for_evac(r->live_bytes()) &&
+ if (region_occupancy_low_enough_for_evac(r->live_bytes()) &&
+ !G1CollectedHeap::heap()->is_old_gc_alloc_region(r) &&
!r->rem_set()->is_tracked()) {
r->rem_set()->set_state_updating();
selected_for_rebuild = true;
diff --git a/src/hotspot/share/gc/g1/g1RootClosures.cpp b/src/hotspot/share/gc/g1/g1RootClosures.cpp
index f03681487cb..16c47cddea1 100644
--- a/src/hotspot/share/gc/g1/g1RootClosures.cpp
+++ b/src/hotspot/share/gc/g1/g1RootClosures.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -22,6 +22,7 @@
*
*/
+#include "gc/g1/g1CollectorState.inline.hpp"
#include "gc/g1/g1OopClosures.inline.hpp"
#include "gc/g1/g1RootClosures.hpp"
#include "gc/g1/g1SharedClosures.hpp"
@@ -72,7 +73,7 @@ G1EvacuationRootClosures* G1EvacuationRootClosures::create_root_closures(G1Colle
G1ParScanThreadState* pss,
bool process_only_dirty_klasses) {
G1EvacuationRootClosures* res = nullptr;
- if (g1h->collector_state()->in_concurrent_start_gc()) {
+ if (g1h->collector_state()->is_in_concurrent_start_gc()) {
if (ClassUnloadingWithConcurrentMark) {
res = new G1ConcurrentStartMarkClosures(g1h, pss);
} else {
diff --git a/src/hotspot/share/gc/g1/g1RootProcessor.cpp b/src/hotspot/share/gc/g1/g1RootProcessor.cpp
index dac237cb277..a534eefb428 100644
--- a/src/hotspot/share/gc/g1/g1RootProcessor.cpp
+++ b/src/hotspot/share/gc/g1/g1RootProcessor.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -27,7 +27,6 @@
#include "code/codeCache.hpp"
#include "gc/g1/g1BarrierSet.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
-#include "gc/g1/g1CollectorState.hpp"
#include "gc/g1/g1GCParPhaseTimesTracker.hpp"
#include "gc/g1/g1GCPhaseTimes.hpp"
#include "gc/g1/g1HeapRegion.inline.hpp"
diff --git a/src/hotspot/share/gc/g1/g1Trace.cpp b/src/hotspot/share/gc/g1/g1Trace.cpp
index ed6a91f41ed..d6eadda5d50 100644
--- a/src/hotspot/share/gc/g1/g1Trace.cpp
+++ b/src/hotspot/share/gc/g1/g1Trace.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -22,8 +22,8 @@
*
*/
+#include "gc/g1/g1CollectorState.inline.hpp"
#include "gc/g1/g1EvacInfo.hpp"
-#include "gc/g1/g1GCPauseType.hpp"
#include "gc/g1/g1HeapRegionTraceType.hpp"
#include "gc/g1/g1Trace.hpp"
#include "gc/shared/gcHeapSummary.hpp"
@@ -48,12 +48,12 @@ public:
class G1YCTypeConstant : public JfrSerializer {
public:
void serialize(JfrCheckpointWriter& writer) {
- constexpr EnumRange types{};
+ constexpr EnumRange types{};
static const u4 nof_entries = static_cast