diff --git a/.github/actions/build-jtreg/action.yml b/.github/actions/build-jtreg/action.yml
index a9c046e9dd9..334812e8341 100644
--- a/.github/actions/build-jtreg/action.yml
+++ b/.github/actions/build-jtreg/action.yml
@@ -37,13 +37,13 @@ runs:
- name: 'Check cache for already built JTReg'
id: get-cached
- uses: actions/cache@v4
+ uses: actions/cache@v5
with:
path: jtreg/installed
key: jtreg-${{ steps.version.outputs.value }}
- name: 'Checkout the JTReg source'
- uses: actions/checkout@v4
+ uses: actions/checkout@v6
with:
repository: openjdk/jtreg
ref: jtreg-${{ steps.version.outputs.value }}
@@ -61,7 +61,7 @@ runs:
if: (steps.get-cached.outputs.cache-hit != 'true')
- name: 'Upload JTReg artifact'
- uses: actions/upload-artifact@v4
+ uses: actions/upload-artifact@v6
with:
name: bundles-jtreg-${{ steps.version.outputs.value }}
path: jtreg/installed
diff --git a/.github/actions/do-build/action.yml b/.github/actions/do-build/action.yml
index 6f2c2ce0218..6f6bbdabb68 100644
--- a/.github/actions/do-build/action.yml
+++ b/.github/actions/do-build/action.yml
@@ -66,7 +66,7 @@ runs:
shell: bash
- name: 'Upload build logs'
- uses: actions/upload-artifact@v4
+ uses: actions/upload-artifact@v6
with:
name: failure-logs-${{ inputs.platform }}${{ inputs.debug-suffix }}
path: failure-logs
@@ -74,7 +74,7 @@ runs:
# This is the best way I found to abort the job with an error message
- name: 'Notify about build failures'
- uses: actions/github-script@v7
+ uses: actions/github-script@v8
with:
script: core.setFailed('Build failed. See summary for details.')
if: steps.check.outputs.failure == 'true'
diff --git a/.github/actions/get-bootjdk/action.yml b/.github/actions/get-bootjdk/action.yml
index 312fb642c82..d531358b7dd 100644
--- a/.github/actions/get-bootjdk/action.yml
+++ b/.github/actions/get-bootjdk/action.yml
@@ -65,7 +65,7 @@ runs:
- name: 'Check cache for BootJDK'
id: get-cached-bootjdk
- uses: actions/cache@v4
+ uses: actions/cache@v5
with:
path: bootjdk/jdk
key: boot-jdk-${{ inputs.platform }}-${{ steps.sha256.outputs.value }}
diff --git a/.github/actions/get-bundles/action.yml b/.github/actions/get-bundles/action.yml
index a356aa9fd8d..55fa0e842d2 100644
--- a/.github/actions/get-bundles/action.yml
+++ b/.github/actions/get-bundles/action.yml
@@ -54,14 +54,14 @@ runs:
steps:
- name: 'Download bundles artifact'
id: download-bundles
- uses: actions/download-artifact@v4
+ uses: actions/download-artifact@v8
with:
name: bundles-${{ inputs.platform }}${{ inputs.debug-suffix }}
path: bundles
continue-on-error: true
- name: 'Download bundles artifact (retry)'
- uses: actions/download-artifact@v4
+ uses: actions/download-artifact@v8
with:
name: bundles-${{ inputs.platform }}${{ inputs.debug-suffix }}
path: bundles
@@ -69,7 +69,7 @@ runs:
- name: 'Download static bundles artifact'
id: download-static-bundles
- uses: actions/download-artifact@v4
+ uses: actions/download-artifact@v8
with:
name: bundles-${{ inputs.platform }}${{ inputs.debug-suffix }}${{ inputs.static-suffix }}
path: bundles
diff --git a/.github/actions/get-gtest/action.yml b/.github/actions/get-gtest/action.yml
index 7a329460a6e..bc53fa2a3b1 100644
--- a/.github/actions/get-gtest/action.yml
+++ b/.github/actions/get-gtest/action.yml
@@ -40,7 +40,7 @@ runs:
var: GTEST_VERSION
- name: 'Checkout GTest source'
- uses: actions/checkout@v4
+ uses: actions/checkout@v6
with:
repository: google/googletest
ref: 'v${{ steps.version.outputs.value }}'
diff --git a/.github/actions/get-jtreg/action.yml b/.github/actions/get-jtreg/action.yml
index 36c895fc59d..8c75ae10c7f 100644
--- a/.github/actions/get-jtreg/action.yml
+++ b/.github/actions/get-jtreg/action.yml
@@ -41,7 +41,7 @@ runs:
- name: 'Download JTReg artifact'
id: download-jtreg
- uses: actions/download-artifact@v4
+ uses: actions/download-artifact@v8
with:
name: bundles-jtreg-${{ steps.version.outputs.value }}
path: jtreg/installed
diff --git a/.github/actions/get-msys2/action.yml b/.github/actions/get-msys2/action.yml
index 308230ebf2e..7351a120ac4 100644
--- a/.github/actions/get-msys2/action.yml
+++ b/.github/actions/get-msys2/action.yml
@@ -31,7 +31,7 @@ runs:
steps:
- name: 'Install MSYS2'
id: msys2
- uses: msys2/setup-msys2@v2.28.0
+ uses: msys2/setup-msys2@v2.31.0
with:
install: 'autoconf tar unzip zip make'
path-type: minimal
diff --git a/.github/actions/upload-bundles/action.yml b/.github/actions/upload-bundles/action.yml
index 78fb0a94bfd..94308002ea7 100644
--- a/.github/actions/upload-bundles/action.yml
+++ b/.github/actions/upload-bundles/action.yml
@@ -87,7 +87,7 @@ runs:
shell: bash
- name: 'Upload bundles artifact'
- uses: actions/upload-artifact@v4
+ uses: actions/upload-artifact@v6
with:
name: bundles-${{ inputs.platform }}${{ inputs.debug-suffix }}${{ inputs.static-suffix }}${{ inputs.bundle-suffix }}
path: bundles
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
new file mode 100644
index 00000000000..d3f63784ece
--- /dev/null
+++ b/.github/pull_request_template.md
@@ -0,0 +1,5 @@
+
+
+
+---------
+- [ ] I confirm that I make this contribution in accordance with the [OpenJDK Interim AI Policy](https://openjdk.org/legal/ai).
diff --git a/.github/workflows/build-alpine-linux.yml b/.github/workflows/build-alpine-linux.yml
index c39962fa07f..6863da9016e 100644
--- a/.github/workflows/build-alpine-linux.yml
+++ b/.github/workflows/build-alpine-linux.yml
@@ -74,7 +74,7 @@ jobs:
steps:
- name: 'Checkout the JDK source'
- uses: actions/checkout@v4
+ uses: actions/checkout@v6
- name: 'Install toolchain and dependencies'
run: |
diff --git a/.github/workflows/build-cross-compile.yml b/.github/workflows/build-cross-compile.yml
index a0642d469aa..99b6c40606c 100644
--- a/.github/workflows/build-cross-compile.yml
+++ b/.github/workflows/build-cross-compile.yml
@@ -94,7 +94,7 @@ jobs:
steps:
- name: 'Checkout the JDK source'
- uses: actions/checkout@v4
+ uses: actions/checkout@v6
- name: 'Get the BootJDK'
id: bootjdk
@@ -122,7 +122,7 @@ jobs:
- name: 'Check cache for sysroot'
id: get-cached-sysroot
- uses: actions/cache@v4
+ uses: actions/cache@v5
with:
path: sysroot
key: sysroot-${{ matrix.debian-arch }}-${{ hashFiles('./.github/workflows/build-cross-compile.yml') }}
diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml
index 791b53a3f04..c501670439e 100644
--- a/.github/workflows/build-linux.yml
+++ b/.github/workflows/build-linux.yml
@@ -84,7 +84,7 @@ jobs:
steps:
- name: 'Checkout the JDK source'
- uses: actions/checkout@v4
+ uses: actions/checkout@v6
- name: 'Get the BootJDK'
id: bootjdk
diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml
index 484e616fad7..435576f4afd 100644
--- a/.github/workflows/build-macos.yml
+++ b/.github/workflows/build-macos.yml
@@ -75,7 +75,7 @@ jobs:
steps:
- name: 'Checkout the JDK source'
- uses: actions/checkout@v4
+ uses: actions/checkout@v6
- name: 'Get the BootJDK'
id: bootjdk
diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml
index 4dafc016a99..3bb50a137ec 100644
--- a/.github/workflows/build-windows.yml
+++ b/.github/workflows/build-windows.yml
@@ -83,7 +83,7 @@ jobs:
steps:
- name: 'Checkout the JDK source'
- uses: actions/checkout@v4
+ uses: actions/checkout@v6
- name: 'Get MSYS2'
uses: ./.github/actions/get-msys2
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 85ec75f343c..20be196b128 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -75,7 +75,7 @@ jobs:
steps:
- name: 'Checkout the scripts'
- uses: actions/checkout@v4
+ uses: actions/checkout@v6
with:
sparse-checkout: |
.github
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 8f33454305e..b240b42fb97 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -128,7 +128,7 @@ jobs:
steps:
- name: 'Checkout the JDK source'
- uses: actions/checkout@v4
+ uses: actions/checkout@v6
- name: 'Get MSYS2'
uses: ./.github/actions/get-msys2
@@ -239,7 +239,7 @@ jobs:
if: always()
- name: 'Upload test results'
- uses: actions/upload-artifact@v4
+ uses: actions/upload-artifact@v6
with:
path: results
name: ${{ steps.package.outputs.artifact-name }}
@@ -247,7 +247,7 @@ jobs:
# This is the best way I found to abort the job with an error message
- name: 'Notify about test failures'
- uses: actions/github-script@v7
+ uses: actions/github-script@v8
with:
script: core.setFailed('${{ steps.run-tests.outputs.error-message }}')
if: steps.run-tests.outputs.failure == 'true'
diff --git a/bin/idea.sh b/bin/idea.sh
index a184884b61a..d9a18956e3b 100644
--- a/bin/idea.sh
+++ b/bin/idea.sh
@@ -187,14 +187,18 @@ fi
SOURCE_PREFIX=""
+# SOURCES is a single string containing embeded newlines.
for root in $MODULE_ROOTS; do
if [ "x$CYGPATH" != "x" ]; then
root=`$CYGPATH -am $root`
elif [ "x$WSL_DISTRO_NAME" != "x" ]; then
root=`wslpath -am $root`
fi
-
- SOURCES=$SOURCES" $SOURCE_PREFIX""$root""$SOURCE_POSTFIX"
+ # Add line termination/indentation for everything after the first entry.
+ if [ "x$SOURCES" != "x" ]; then
+ SOURCES="${SOURCES}\n "
+ fi
+ SOURCES="${SOURCES}${SOURCE_PREFIX}${root}${SOURCE_POSTFIX}"
done
add_replacement "###SOURCE_ROOTS###" "$SOURCES"
diff --git a/doc/testing.html b/doc/testing.html
index 195153c8612..c8d0b928bb0 100644
--- a/doc/testing.html
+++ b/doc/testing.html
@@ -284,9 +284,10 @@ possible, or if you want to use a fully qualified test descriptor, add
Gtest
Note: To be able to run the Gtest suite, you need to
configure your build to be able to find a proper version of the gtest
-source. For details, see the section "Running Tests" in the build
-documentation.
+source. For details, see the section "Running Tests" in the
+build documentation (html, markdown).
Since the Hotspot Gtest suite is so quick, the default is to run all
tests. This is specified by just gtest, or as a fully
qualified test descriptor gtest:all.
diff --git a/doc/testing.md b/doc/testing.md
index d0e54aab02b..5f70f2796ad 100644
--- a/doc/testing.md
+++ b/doc/testing.md
@@ -198,8 +198,8 @@ use a fully qualified test descriptor, add `jtreg:`, e.g.
**Note:** To be able to run the Gtest suite, you need to configure your build
to be able to find a proper version of the gtest source. For details, see the
-section ["Running Tests" in the build
-documentation](building.html#running-tests).
+section **"Running Tests" in the build
+documentation** ([html](building.html#running-tests), [markdown](building.md#running-tests)).
Since the Hotspot Gtest suite is so quick, the default is to run all tests.
This is specified by just `gtest`, or as a fully qualified test descriptor
diff --git a/make/CompileInterimLangtools.gmk b/make/CompileInterimLangtools.gmk
index c7d1c3796f6..8254b8fc0a7 100644
--- a/make/CompileInterimLangtools.gmk
+++ b/make/CompileInterimLangtools.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2014, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2014, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -68,17 +68,19 @@ java.compiler.interim_EXTRA_FILES := \
TARGETS += $(BUILDTOOLS_OUTPUTDIR)/gensrc/java.compiler.interim/javax/tools/ToolProvider.java
################################################################################
-# Use the up-to-date PreviewFeature.java and NoPreview.java from the current
-# sources, instead of the versions from the boot JDK, as javac may be referring
-# to constants from the up-to-date versions.
+# Create a hybrid PreviewFeature.java that combines constants
+# from the current sources, as those can be used in javac APIs, and from the
+# bootstrap JDK, as those can be used from bootstrap JDK classfiles.
-$(eval $(call SetupCopyFiles, COPY_PREVIEW_FEATURES, \
- FILES := $(TOPDIR)/src/java.base/share/classes/jdk/internal/javac/PreviewFeature.java \
- $(TOPDIR)/src/java.base/share/classes/jdk/internal/javac/NoPreview.java, \
- DEST := $(BUILDTOOLS_OUTPUTDIR)/gensrc/java.base.interim/jdk/internal/javac/, \
-))
+$(BUILDTOOLS_OUTPUTDIR)/gensrc/java.base.interim/jdk/internal/javac/PreviewFeature.java: \
+ $(TOPDIR)/src/java.base/share/classes/jdk/internal/javac/PreviewFeature.java
+ $(call LogInfo, Generating $@)
+ $(JAVA) $(TOPDIR)/make/langtools/tools/previewfeature/SetupPreviewFeature.java \
+ $(TOPDIR)/src/java.base/share/classes/jdk/internal/javac/PreviewFeature.java \
+ $@
-TARGETS += $(COPY_PREVIEW_FEATURES)
+
+TARGETS += $(BUILDTOOLS_OUTPUTDIR)/gensrc/java.base.interim/jdk/internal/javac/PreviewFeature.java
################################################################################
# Setup the rules to build interim langtools, which is compiled by the boot
@@ -123,7 +125,8 @@ define SetupInterimModule
$1_DEPS_INTERIM := $$(addsuffix .interim, $$(filter \
$$(INTERIM_LANGTOOLS_BASE_MODULES), $$(call FindTransitiveDepsForModule, $1)))
- $$(BUILD_$1.interim): $$(foreach d, $$($1_DEPS_INTERIM), $$(BUILD_$$d)) $(COPY_PREVIEW_FEATURES)
+ $$(BUILD_$1.interim): $$(foreach d, $$($1_DEPS_INTERIM), $$(BUILD_$$d)) \
+ $(BUILDTOOLS_OUTPUTDIR)/gensrc/java.base.interim/jdk/internal/javac/PreviewFeature.java
TARGETS += $$(BUILD_$1.interim)
endef
diff --git a/make/Docs.gmk b/make/Docs.gmk
index a8d40e078e2..9cee8cd40c1 100644
--- a/make/Docs.gmk
+++ b/make/Docs.gmk
@@ -1,4 +1,4 @@
-# Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -93,19 +93,16 @@ JAVADOC_DISABLED_DOCLINT_WARNINGS := missing
JAVADOC_DISABLED_DOCLINT_PACKAGES := org.w3c.* javax.smartcardio
# The initial set of options for javadoc
-# -XDaccessInternalAPI is a temporary workaround, see 8373909
JAVADOC_OPTIONS := -use -keywords -notimestamp \
-serialwarn -encoding utf-8 -docencoding utf-8 -breakiterator \
-splitIndex --system none -javafx --expand-requires transitive \
- --override-methods=summary \
- -XDaccessInternalAPI
+ --override-methods=summary
# The reference options must stay stable to allow for comparisons across the
# development cycle.
REFERENCE_OPTIONS := -XDignore.symbol.file=true -use -keywords -notimestamp \
-serialwarn -encoding utf-8 -breakiterator -splitIndex --system none \
- -html5 -javafx --expand-requires transitive \
- -XDaccessInternalAPI
+ -html5 -javafx --expand-requires transitive
# Should we add DRAFT stamps to the generated javadoc?
ifeq ($(VERSION_IS_GA), true)
diff --git a/make/GenerateLinkOptData.gmk b/make/GenerateLinkOptData.gmk
index 6f6e1f29b4c..d615a34e71a 100644
--- a/make/GenerateLinkOptData.gmk
+++ b/make/GenerateLinkOptData.gmk
@@ -70,12 +70,15 @@ CLASSLIST_FILE_VM_OPTS = \
# Save the stderr output of the command and print it along with stdout in case
# something goes wrong.
+# The classlists must be generated with -Xint to avoid non-determinism
+# introduced by JIT compiled code
$(CLASSLIST_FILE): $(INTERIM_IMAGE_DIR)/bin/java$(EXECUTABLE_SUFFIX) $(CLASSLIST_JAR)
$(call MakeDir, $(LINK_OPT_DIR))
$(call LogInfo, Generating $(patsubst $(OUTPUTDIR)/%, %, $@))
$(call LogInfo, Generating $(patsubst $(OUTPUTDIR)/%, %, $(JLI_TRACE_FILE)))
$(FIXPATH) $(INTERIM_IMAGE_DIR)/bin/java -XX:DumpLoadedClassList=$@.raw \
$(CLASSLIST_FILE_VM_OPTS) \
+ -Xint \
-Xlog:aot=off \
-Xlog:cds=off \
-cp $(SUPPORT_OUTPUTDIR)/classlist.jar \
@@ -90,6 +93,7 @@ $(CLASSLIST_FILE): $(INTERIM_IMAGE_DIR)/bin/java$(EXECUTABLE_SUFFIX) $(CLASSLIST
-XX:SharedClassListFile=$@.interim -XX:SharedArchiveFile=$@.jsa \
-Djava.lang.invoke.MethodHandle.TRACE_RESOLVE=true \
$(CLASSLIST_FILE_VM_OPTS) \
+ -Xint \
--module-path $(SUPPORT_OUTPUTDIR)/classlist.jar \
-Xlog:aot=off \
-Xlog:cds=off \
diff --git a/make/Hsdis.gmk b/make/Hsdis.gmk
index 469cc488f16..76695fc8dde 100644
--- a/make/Hsdis.gmk
+++ b/make/Hsdis.gmk
@@ -44,6 +44,9 @@ ifeq ($(HSDIS_BACKEND), capstone)
else ifeq ($(call isTargetCpuArch, aarch64), true)
CAPSTONE_ARCH := CS_ARCH_$(CAPSTONE_ARCH_AARCH64_NAME)
CAPSTONE_MODE := CS_MODE_ARM
+ else ifeq ($(call isTargetCpuArch, arm), true)
+ CAPSTONE_ARCH := CS_ARCH_ARM
+ CAPSTONE_MODE := CS_MODE_ARM
else
$(error No support for Capstone on this platform)
endif
diff --git a/make/RunTests.gmk b/make/RunTests.gmk
index 02ea632e3ec..d4be5936c41 100644
--- a/make/RunTests.gmk
+++ b/make/RunTests.gmk
@@ -1020,6 +1020,9 @@ define SetupRunJtregTestBody
VM_OPTIONS := $$(JTREG_ALL_OPTIONS) ))
$$(call LogWarn, AOT_JDK_CACHE=$$($1_AOT_JDK_CACHE))
$1_JTREG_BASIC_OPTIONS += -vmoption:-XX:AOTCache="$$($1_AOT_JDK_CACHE)"
+ $1_JTREG_BASIC_OPTIONS += $$(addprefix $$(JTREG_PROBLEM_LIST_PREFIX), $$(wildcard \
+ $$(addprefix $$($1_TEST_ROOT)/, ProblemList-AotJdk.txt) \
+ ))
endif
diff --git a/make/autoconf/basic_tools.m4 b/make/autoconf/basic_tools.m4
index 8e42f9205a4..66ef94d48a8 100644
--- a/make/autoconf/basic_tools.m4
+++ b/make/autoconf/basic_tools.m4
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -369,6 +369,10 @@ AC_DEFUN_ONCE([BASIC_SETUP_COMPLEX_TOOLS],
IS_GNU_DATE=yes
else
AC_MSG_RESULT([no])
+ # Likely at the AIX provided version of the date utility here, which is not compatible
+ if test "x$OPENJDK_TARGET_OS" = "xaix"; then
+ AC_MSG_ERROR([gnu date from AIX toolbox is required])
+ fi
IS_GNU_DATE=no
fi
AC_SUBST(IS_GNU_DATE)
diff --git a/make/autoconf/flags-cflags.m4 b/make/autoconf/flags-cflags.m4
index 2d39d84f52e..423654cd50a 100644
--- a/make/autoconf/flags-cflags.m4
+++ b/make/autoconf/flags-cflags.m4
@@ -544,12 +544,9 @@ AC_DEFUN([FLAGS_SETUP_CFLAGS_HELPER],
TOOLCHAIN_CFLAGS_JVM="$TOOLCHAIN_CFLAGS_JVM -fstack-protector"
TOOLCHAIN_CFLAGS_JDK="-fvisibility=hidden -pipe -fstack-protector"
# reduce lib size on linux in link step, this needs also special compile flags
- # do this on s390x also for libjvm (where serviceability agent is not supported)
if test "x$ENABLE_LINKTIME_GC" = xtrue; then
TOOLCHAIN_CFLAGS_JDK="$TOOLCHAIN_CFLAGS_JDK -ffunction-sections -fdata-sections"
- if test "x$OPENJDK_TARGET_CPU" = xs390x && test "x$DEBUG_LEVEL" == xrelease; then
- TOOLCHAIN_CFLAGS_JVM="$TOOLCHAIN_CFLAGS_JVM -ffunction-sections -fdata-sections"
- fi
+ TOOLCHAIN_CFLAGS_JVM="$TOOLCHAIN_CFLAGS_JVM -ffunction-sections -fdata-sections"
fi
# technically NOT for CXX (but since this gives *worse* performance, use
# no-strict-aliasing everywhere!)
@@ -578,6 +575,11 @@ AC_DEFUN([FLAGS_SETUP_CFLAGS_HELPER],
TOOLCHAIN_CFLAGS_JDK_CONLY="-fno-strict-aliasing" # technically NOT for CXX
fi
+ if test "x$ENABLE_LINKTIME_GC" = xtrue; then
+ TOOLCHAIN_CFLAGS_JDK="$TOOLCHAIN_CFLAGS_JDK -ffunction-sections -fdata-sections"
+ TOOLCHAIN_CFLAGS_JVM="$TOOLCHAIN_CFLAGS_JVM -ffunction-sections -fdata-sections"
+ fi
+
if test "x$OPENJDK_TARGET_OS" = xaix; then
TOOLCHAIN_CFLAGS_JVM="$TOOLCHAIN_CFLAGS_JVM -ffunction-sections -ftls-model -fno-math-errno"
TOOLCHAIN_CFLAGS_JDK="-ffunction-sections -fsigned-char"
diff --git a/make/autoconf/flags-ldflags.m4 b/make/autoconf/flags-ldflags.m4
index 466ff1beaf4..ff10828731e 100644
--- a/make/autoconf/flags-ldflags.m4
+++ b/make/autoconf/flags-ldflags.m4
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -53,16 +53,15 @@ AC_DEFUN([FLAGS_SETUP_LDFLAGS_HELPER],
# add --icf=all (Identical Code Folding — merges identical functions)
BASIC_LDFLAGS="-Wl,-z,defs -Wl,-z,relro -Wl,-z,now -Wl,--no-as-needed -Wl,--exclude-libs,ALL"
+ BASIC_LDFLAGS_JVM_ONLY=""
# Linux : remove unused code+data in link step
if test "x$ENABLE_LINKTIME_GC" = xtrue; then
- if test "x$OPENJDK_TARGET_CPU" = xs390x; then
- BASIC_LDFLAGS="$BASIC_LDFLAGS -Wl,--gc-sections"
- else
- BASIC_LDFLAGS_JDK_ONLY="$BASIC_LDFLAGS_JDK_ONLY -Wl,--gc-sections"
- fi
+ # keep vtables : -Wl,--undefined-glob=_ZTV* (but this seems not to work with gold ld)
+ # so keep at least the Metadata vtable that is used in the serviceability agent
+ BASIC_LDFLAGS_JVM_ONLY="$BASIC_LDFLAGS_JVM_ONLY -Wl,--gc-sections -Wl,--undefined=_ZTV8Metadata"
+ BASIC_LDFLAGS_JDK_ONLY="$BASIC_LDFLAGS_JDK_ONLY -Wl,--gc-sections"
fi
- BASIC_LDFLAGS_JVM_ONLY=""
LDFLAGS_LTO="-flto=auto -fuse-linker-plugin -fno-strict-aliasing $DEBUG_PREFIX_CFLAGS"
LDFLAGS_CXX_PARTIAL_LINKING="$MACHINE_FLAG -r"
@@ -80,6 +79,10 @@ AC_DEFUN([FLAGS_SETUP_LDFLAGS_HELPER],
if test "x$CXX_IS_USER_SUPPLIED" = xfalse && test "x$CC_IS_USER_SUPPLIED" = xfalse; then
UTIL_REQUIRE_TOOLCHAIN_PROGS(LLD, lld)
fi
+
+ if test "x$ENABLE_LINKTIME_GC" = xtrue; then
+ BASIC_LDFLAGS_JDK_ONLY="$BASIC_LDFLAGS_JDK_ONLY -Wl,--gc-sections"
+ fi
fi
if test "x$OPENJDK_TARGET_OS" = xaix; then
BASIC_LDFLAGS="-Wl,-b64 -Wl,-brtl -Wl,-bnorwexec -Wl,-blibpath:/usr/lib:lib -Wl,-bnoexpall \
diff --git a/make/autoconf/jdk-options.m4 b/make/autoconf/jdk-options.m4
index 87d147d4f07..89fcbc88521 100644
--- a/make/autoconf/jdk-options.m4
+++ b/make/autoconf/jdk-options.m4
@@ -102,9 +102,20 @@ AC_DEFUN_ONCE([JDKOPT_SETUP_JDK_OPTIONS],
CHECKING_MSG: [if we should build headless-only (no GUI)])
AC_SUBST(ENABLE_HEADLESS_ONLY)
+ # Avoid headless-only on macOS and Windows, it is not supported there
+ if test "x$ENABLE_HEADLESS_ONLY" = xtrue; then
+ if test "x$OPENJDK_TARGET_OS" = xwindows || test "x$OPENJDK_TARGET_OS" = xmacosx; then
+ AC_MSG_ERROR([headless-only is not supported on macOS and Windows])
+ fi
+ fi
+
# should we linktime gc unused code sections in the JDK build ?
- if test "x$OPENJDK_TARGET_OS" = "xlinux" && test "x$OPENJDK_TARGET_CPU" = xs390x; then
- LINKTIME_GC_DEFAULT=true
+ if test "x$OPENJDK_TARGET_OS" = "xlinux"; then
+ if test "x$OPENJDK_TARGET_CPU" = "xs390x" || test "x$OPENJDK_TARGET_CPU" = "xppc64le"; then
+ LINKTIME_GC_DEFAULT=true
+ else
+ LINKTIME_GC_DEFAULT=false
+ fi
else
LINKTIME_GC_DEFAULT=false
fi
diff --git a/make/autoconf/toolchain_microsoft.m4 b/make/autoconf/toolchain_microsoft.m4
index f577cf1a2a1..afe04acf029 100644
--- a/make/autoconf/toolchain_microsoft.m4
+++ b/make/autoconf/toolchain_microsoft.m4
@@ -217,10 +217,12 @@ AC_DEFUN([TOOLCHAIN_FIND_VISUAL_STUDIO_BAT_FILE],
TOOLCHAIN_CHECK_POSSIBLE_VISUAL_STUDIO_ROOT([$TARGET_CPU], [$VS_VERSION],
[$PROGRAMFILES_X86/$VS_INSTALL_DIR], [well-known name])
fi
+ # Derive system drive root from CMD (which is at /windows/system32/cmd.exe)
+ WINSYSDRIVE_ROOT="$(dirname "$(dirname "$(dirname "$CMD")")")"
TOOLCHAIN_CHECK_POSSIBLE_VISUAL_STUDIO_ROOT([$TARGET_CPU], [$VS_VERSION],
- [c:/program files/$VS_INSTALL_DIR], [well-known name])
+ [$WINSYSDRIVE_ROOT/program files/$VS_INSTALL_DIR], [well-known name])
TOOLCHAIN_CHECK_POSSIBLE_VISUAL_STUDIO_ROOT([$TARGET_CPU], [$VS_VERSION],
- [c:/program files (x86)/$VS_INSTALL_DIR], [well-known name])
+ [$WINSYSDRIVE_ROOT/program files (x86)/$VS_INSTALL_DIR], [well-known name])
if test "x$SDK_INSTALL_DIR" != x; then
if test "x$ProgramW6432" != x; then
TOOLCHAIN_CHECK_POSSIBLE_WIN_SDK_ROOT([$TARGET_CPU], [$VS_VERSION],
@@ -235,9 +237,9 @@ AC_DEFUN([TOOLCHAIN_FIND_VISUAL_STUDIO_BAT_FILE],
[$PROGRAMFILES/$SDK_INSTALL_DIR], [well-known name])
fi
TOOLCHAIN_CHECK_POSSIBLE_WIN_SDK_ROOT([$TARGET_CPU], [$VS_VERSION],
- [c:/program files/$SDK_INSTALL_DIR], [well-known name])
+ [$WINSYSDRIVE_ROOT/program files/$SDK_INSTALL_DIR], [well-known name])
TOOLCHAIN_CHECK_POSSIBLE_WIN_SDK_ROOT([$TARGET_CPU], [$VS_VERSION],
- [c:/program files (x86)/$SDK_INSTALL_DIR], [well-known name])
+ [$WINSYSDRIVE_ROOT/program files (x86)/$SDK_INSTALL_DIR], [well-known name])
fi
VCVARS_VER=auto
@@ -338,7 +340,7 @@ AC_DEFUN([TOOLCHAIN_EXTRACT_VISUAL_STUDIO_ENV],
OLDPATH="$PATH"
# Make sure we only capture additions to PATH needed by VS.
# Clear out path, but need system dir present for vsvars cmd file to be able to run
- export PATH=$WINENV_PREFIX/c/windows/system32
+ export PATH="$(dirname "$CMD")"
# The "| cat" is to stop SetEnv.Cmd to mess with system colors on some systems
# We can't pass -vcvars_ver=$VCVARS_VER here because cmd.exe eats all '='
# in bat file arguments. :-(
diff --git a/make/common/modules/LauncherCommon.gmk b/make/common/modules/LauncherCommon.gmk
index 7682ffbb95c..8d45142ef4a 100644
--- a/make/common/modules/LauncherCommon.gmk
+++ b/make/common/modules/LauncherCommon.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -35,11 +35,17 @@ include ProcessMarkdown.gmk
include $(TOPDIR)/make/ToolsJdk.gmk
LAUNCHER_SRC := $(TOPDIR)/src/java.base/share/native/launcher
+
LAUNCHER_CFLAGS += -I$(TOPDIR)/src/java.base/share/native/launcher \
-I$(TOPDIR)/src/java.base/share/native/libjli \
-I$(TOPDIR)/src/java.base/$(OPENJDK_TARGET_OS_TYPE)/native/libjli \
-I$(TOPDIR)/src/java.base/$(OPENJDK_TARGET_OS)/native/libjli \
#
+
+ifeq ($(call isTargetOs, aix), true)
+ LAUNCHER_CFLAGS += -I$(TOPDIR)/src/java.base/aix/native/include
+endif
+
MACOSX_PLIST_DIR := $(TOPDIR)/src/java.base/macosx/native/launcher
JAVA_MANIFEST := $(TOPDIR)/src/java.base/windows/native/launcher/java.manifest
diff --git a/make/conf/github-actions.conf b/make/conf/github-actions.conf
index ebfc9191535..6771e8923dc 100644
--- a/make/conf/github-actions.conf
+++ b/make/conf/github-actions.conf
@@ -29,21 +29,21 @@ GTEST_VERSION=1.14.0
JTREG_VERSION=8.2.1+1
LINUX_X64_BOOT_JDK_EXT=tar.gz
-LINUX_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk25/bd75d5f9689641da8e1daabeccb5528b/36/GPL/openjdk-25_linux-x64_bin.tar.gz
-LINUX_X64_BOOT_JDK_SHA256=59cdcaf255add4721de38eb411d4ecfe779356b61fb671aee63c7dec78054c2b
+LINUX_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk26/c3cc523845074aa0af4f5e1e1ed4151d/35/GPL/openjdk-26_linux-x64_bin.tar.gz
+LINUX_X64_BOOT_JDK_SHA256=83c78367f8c81257beef72aca4bbbf8e6dac8ca2b3a4546a85879a09e6e4e128
ALPINE_LINUX_X64_BOOT_JDK_EXT=tar.gz
-ALPINE_LINUX_X64_BOOT_JDK_URL=https://github.com/adoptium/temurin25-binaries/releases/download/jdk-25%2B36/OpenJDK25U-jdk_x64_alpine-linux_hotspot_25_36.tar.gz
-ALPINE_LINUX_X64_BOOT_JDK_SHA256=637e47474d411ed86134f413af7d5fef4180ddb0bf556347b7e74a88cf8904c8
+ALPINE_LINUX_X64_BOOT_JDK_URL=https://github.com/adoptium/temurin26-binaries/releases/download/jdk-26%2B35/OpenJDK26U-jdk_x64_alpine-linux_hotspot_26_35.tar.gz
+ALPINE_LINUX_X64_BOOT_JDK_SHA256=c105e581fdccb4e7120d889235d1ad8d5b2bed0af4972bc881e0a8ba687c94a4
MACOS_AARCH64_BOOT_JDK_EXT=tar.gz
-MACOS_AARCH64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk25/bd75d5f9689641da8e1daabeccb5528b/36/GPL/openjdk-25_macos-aarch64_bin.tar.gz
-MACOS_AARCH64_BOOT_JDK_SHA256=2006337bf326fdfdf6117081751ba38c1c8706d63419ecac7ff102ff7c776876
+MACOS_AARCH64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk26/c3cc523845074aa0af4f5e1e1ed4151d/35/GPL/openjdk-26_macos-aarch64_bin.tar.gz
+MACOS_AARCH64_BOOT_JDK_SHA256=254586bcd1bf6dcd125ad667ac32562cb1e2ab1abf3a61fb117b6fabb571e765
MACOS_X64_BOOT_JDK_EXT=tar.gz
-MACOS_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk25/bd75d5f9689641da8e1daabeccb5528b/36/GPL/openjdk-25_macos-x64_bin.tar.gz
-MACOS_X64_BOOT_JDK_SHA256=47482ad9888991ecac9b2bcc131e2b53ff78aff275104cef85f66252308e8a09
+MACOS_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk26/c3cc523845074aa0af4f5e1e1ed4151d/35/GPL/openjdk-26_macos-x64_bin.tar.gz
+MACOS_X64_BOOT_JDK_SHA256=8642b89d889c14ede2c446fd5bbe3621c8a3082e3df02013fd1658e39f52929a
WINDOWS_X64_BOOT_JDK_EXT=zip
-WINDOWS_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk25/bd75d5f9689641da8e1daabeccb5528b/36/GPL/openjdk-25_windows-x64_bin.zip
-WINDOWS_X64_BOOT_JDK_SHA256=85bcc178461e2cb3c549ab9ca9dfa73afd54c09a175d6510d0884071867137d3
+WINDOWS_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk26/c3cc523845074aa0af4f5e1e1ed4151d/35/GPL/openjdk-26_windows-x64_bin.zip
+WINDOWS_X64_BOOT_JDK_SHA256=2dd2d92c9374cd49a120fe9d916732840bf6bb9f0e0cc29794917a3c08b99c5f
diff --git a/make/conf/jib-profiles.js b/make/conf/jib-profiles.js
index 76a94b7789e..4c1d2835054 100644
--- a/make/conf/jib-profiles.js
+++ b/make/conf/jib-profiles.js
@@ -387,8 +387,8 @@ var getJibProfilesCommon = function (input, data) {
};
};
- common.boot_jdk_version = "25";
- common.boot_jdk_build_number = "37";
+ common.boot_jdk_version = "26";
+ common.boot_jdk_build_number = "35";
common.boot_jdk_home = input.get("boot_jdk", "install_path") + "/jdk-"
+ common.boot_jdk_version
+ (input.build_os == "macosx" ? ".jdk/Contents/Home" : "");
diff --git a/make/conf/version-numbers.conf b/make/conf/version-numbers.conf
index 4392d86ac33..4f63179ae05 100644
--- a/make/conf/version-numbers.conf
+++ b/make/conf/version-numbers.conf
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -37,6 +37,6 @@ DEFAULT_VERSION_DATE=2026-09-15
DEFAULT_VERSION_CLASSFILE_MAJOR=71 # "`$EXPR $DEFAULT_VERSION_FEATURE + 44`"
DEFAULT_VERSION_CLASSFILE_MINOR=0
DEFAULT_VERSION_DOCS_API_SINCE=11
-DEFAULT_ACCEPTABLE_BOOT_VERSIONS="25 26 27"
+DEFAULT_ACCEPTABLE_BOOT_VERSIONS="26 27"
DEFAULT_JDK_SOURCE_TARGET_VERSION=27
DEFAULT_PROMOTED_VERSION_PRE=ea
diff --git a/make/hotspot/lib/CompileGtest.gmk b/make/hotspot/lib/CompileGtest.gmk
index 327014b1e9d..4b21d481049 100644
--- a/make/hotspot/lib/CompileGtest.gmk
+++ b/make/hotspot/lib/CompileGtest.gmk
@@ -63,6 +63,10 @@ $(eval $(call SetupJdkLibrary, BUILD_GTEST_LIBGTEST, \
unused-result zero-as-null-pointer-constant, \
DISABLED_WARNINGS_clang := format-nonliteral undef unused-result \
zero-as-null-pointer-constant, \
+ $(comment Disable deprecated-declarations warnings to workaround) \
+ $(comment clang18+glibc12 bug https://github.com/llvm/llvm-project/issues/76515) \
+ $(comment until the clang bug has been fixed) \
+ DISABLED_WARNINGS_clang_gtest-all.cc := deprecated-declarations, \
DISABLED_WARNINGS_microsoft := 4530, \
DEFAULT_CFLAGS := false, \
CFLAGS := $(JVM_CFLAGS) \
diff --git a/make/hotspot/lib/JvmFlags.gmk b/make/hotspot/lib/JvmFlags.gmk
index 57b632ee532..27a96cc4865 100644
--- a/make/hotspot/lib/JvmFlags.gmk
+++ b/make/hotspot/lib/JvmFlags.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2013, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2013, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -43,10 +43,15 @@ JVM_SRC_DIRS += $(call uniq, $(wildcard $(foreach d, $(JVM_SRC_ROOTS), \
$(JVM_VARIANT_OUTPUTDIR)/gensrc
#
+ifeq ($(call isTargetOs, aix), true)
+ ADD_PLATFORM_INCLUDE_DIR := -I$(TOPDIR)/src/java.base/aix/native/include
+endif
+
JVM_CFLAGS_INCLUDES += \
$(patsubst %,-I%,$(JVM_SRC_DIRS)) \
-I$(TOPDIR)/src/hotspot/share/include \
-I$(TOPDIR)/src/hotspot/os/$(HOTSPOT_TARGET_OS_TYPE)/include \
+ $(ADD_PLATFORM_INCLUDE_DIR) \
-I$(SUPPORT_OUTPUTDIR)/modules_include/java.base \
-I$(SUPPORT_OUTPUTDIR)/modules_include/java.base/$(OPENJDK_TARGET_OS_INCLUDE_SUBDIR) \
-I$(TOPDIR)/src/java.base/share/native/libjimage \
diff --git a/make/jdk/src/classes/build/tools/cldrconverter/CLDRConverter.java b/make/jdk/src/classes/build/tools/cldrconverter/CLDRConverter.java
index ab878a4d2a5..9f42326ef09 100644
--- a/make/jdk/src/classes/build/tools/cldrconverter/CLDRConverter.java
+++ b/make/jdk/src/classes/build/tools/cldrconverter/CLDRConverter.java
@@ -87,6 +87,7 @@ public class CLDRConverter {
static final String EXEMPLAR_CITY_PREFIX = "timezone.excity.";
static final String ZONE_NAME_PREFIX = "timezone.displayname.";
static final String METAZONE_ID_PREFIX = "metazone.id.";
+ static final String METAZONE_DSTOFFSET_PREFIX = "metazone.dstoffset.";
static final String PARENT_LOCALE_PREFIX = "parentLocale.";
static final String LIKELY_SCRIPT_PREFIX = "likelyScript.";
static final String META_EMPTY_ZONE_NAME = "EMPTY_ZONE";
@@ -139,6 +140,11 @@ public class CLDRConverter {
private static final Map tzdbSubstLetters = HashMap.newHashMap(512);
private static final Map tzdbLinks = HashMap.newHashMap(512);
+ // Map of explicit dst offsets for metazones
+ // key: time zone ID
+ // value: explicit dstOffset for the corresponding metazone name
+ static final Map explicitDstOffsets = HashMap.newHashMap(32);
+
static enum DraftType {
UNCONFIRMED,
PROVISIONAL,
@@ -795,10 +801,7 @@ public class CLDRConverter {
String tzKey = Optional.ofNullable((String)handlerSupplMeta.get(tzid))
.orElse(tzid);
// Follow link, if needed
- String tzLink = null;
- for (var k = tzKey; tzdbLinks.containsKey(k);) {
- k = tzLink = tzdbLinks.get(k);
- }
+ String tzLink = getTZDBLink(tzKey);
if (tzLink == null && tzdbLinks.containsValue(tzKey)) {
// reverse link search
// this is needed as in tzdb, "America/Buenos_Aires" links to
@@ -827,7 +830,7 @@ public class CLDRConverter {
} else {
// TZDB short names
tznames = Arrays.copyOf(tznames, tznames.length);
- fillTZDBShortNames(tzid, tznames);
+ fillTZDBShortNames(tzKey, tznames);
names.put(tzid, tznames);
}
} else {
@@ -840,11 +843,13 @@ public class CLDRConverter {
String metaKey = METAZONE_ID_PREFIX + meta;
data = map.get(metaKey);
if (data instanceof String[] tznames) {
- // TZDB short names
- tznames = Arrays.copyOf((String[])names.getOrDefault(metaKey, tznames), 6);
- fillTZDBShortNames(tzid, tznames);
- // Keep the metazone prefix here.
- names.putIfAbsent(metaKey, tznames);
+ if (isDefaultZone(meta, tzKey)) {
+ // Record the metazone names only from the default
+ // (001) zone, with short names filled from TZDB
+ tznames = Arrays.copyOf(tznames, tznames.length);
+ fillTZDBShortNames(tzKey, tznames);
+ names.put(metaKey, tznames);
+ }
names.put(tzid, meta);
if (tzLink != null && availableIds.contains(tzLink)) {
names.put(tzLink, meta);
@@ -867,6 +872,12 @@ public class CLDRConverter {
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
names.putAll(exCities);
+ // Explicit metazone offsets
+ if (id.equals("root")) {
+ explicitDstOffsets.forEach((k, v) ->
+ names.put(METAZONE_DSTOFFSET_PREFIX + k, v));
+ }
+
// If there's no UTC entry at this point, add an empty one
if (!names.isEmpty() && !names.containsKey("UTC")) {
names.putIfAbsent(METAZONE_ID_PREFIX + META_EMPTY_ZONE_NAME, EMPTY_ZONE);
@@ -1492,12 +1503,12 @@ public class CLDRConverter {
* Fill the TZDB short names if there is no name provided by the CLDR
*/
private static void fillTZDBShortNames(String tzid, String[] names) {
- var val = tzdbShortNamesMap.get(tzdbLinks.getOrDefault(tzid, tzid));
+ var val = tzdbShortNamesMap.getOrDefault(tzid, tzdbShortNamesMap.get(getTZDBLink(tzid)));
if (val != null) {
var format = val.split(NBSP)[0];
var rule = val.split(NBSP)[1];
IntStream.of(1, 3, 5).forEach(i -> {
- if (names[i] == null) {
+ if (names[i] == null || names[i].isEmpty()) {
if (format.contains("%s")) {
names[i] = switch (i) {
case 1 -> format.formatted(tzdbSubstLetters.get(rule + NBSP + STD));
@@ -1519,6 +1530,21 @@ public class CLDRConverter {
}
}
+ private static boolean isDefaultZone(String meta, String tzid) {
+ String zone001 = handlerMetaZones.zidMap().get(meta);
+ var tzLink = getTZDBLink(tzid);
+ return canonicalTZMap.getOrDefault(tzid, tzid).equals(zone001) ||
+ tzLink != null && canonicalTZMap.getOrDefault(tzLink, tzLink).equals(zone001);
+ }
+
+ private static String getTZDBLink(String tzid) {
+ String tzLink = null;
+ for (var k = tzid; tzdbLinks.containsKey(k);) {
+ k = tzLink = tzdbLinks.get(k);
+ }
+ return tzLink;
+ }
+
/*
* Convert TZDB offsets to JDK's offsets, eg, "-08" to "GMT-08:00".
* If it cannot recognize the pattern, return the argument as is.
diff --git a/make/jdk/src/classes/build/tools/cldrconverter/MetaZonesParseHandler.java b/make/jdk/src/classes/build/tools/cldrconverter/MetaZonesParseHandler.java
index 2c3757b7a47..45de46d2476 100644
--- a/make/jdk/src/classes/build/tools/cldrconverter/MetaZonesParseHandler.java
+++ b/make/jdk/src/classes/build/tools/cldrconverter/MetaZonesParseHandler.java
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2012, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -84,7 +84,15 @@ class MetaZonesParseHandler extends AbstractLDMLHandler {
if (fromLDT.isBefore(now) && toLDT.isAfter(now)) {
metazone = attributes.getValue("mzone");
+
+ // Explicit metazone DST offsets. Only the "dst" offset is needed,
+ // as "std" is used by default when it doesn't match.
+ String dstOffset = attributes.getValue("dstOffset");
+ if (dstOffset != null) {
+ CLDRConverter.explicitDstOffsets.put(tzid, dstOffset);
+ }
}
+
pushIgnoredContainer(qName);
break;
diff --git a/make/jdk/src/classes/build/tools/cldrconverter/ResourceBundleGenerator.java b/make/jdk/src/classes/build/tools/cldrconverter/ResourceBundleGenerator.java
index 3953f38f653..8278bf6bcfa 100644
--- a/make/jdk/src/classes/build/tools/cldrconverter/ResourceBundleGenerator.java
+++ b/make/jdk/src/classes/build/tools/cldrconverter/ResourceBundleGenerator.java
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2012, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -198,7 +198,8 @@ class ResourceBundleGenerator implements BundleGenerator {
} else if (value instanceof String) {
String valStr = (String)value;
if (type == BundleType.TIMEZONE &&
- !key.startsWith(CLDRConverter.EXEMPLAR_CITY_PREFIX) ||
+ !(key.startsWith(CLDRConverter.EXEMPLAR_CITY_PREFIX) ||
+ key.startsWith(CLDRConverter.METAZONE_DSTOFFSET_PREFIX)) ||
valStr.startsWith(META_VALUE_PREFIX)) {
out.printf(" { \"%s\", %s },\n", key, CLDRConverter.saveConvert(valStr, useJava));
} else {
diff --git a/make/jdk/src/classes/build/tools/intpoly/FieldGen.java b/make/jdk/src/classes/build/tools/intpoly/FieldGen.java
index fcc45db0219..c29a315f368 100644
--- a/make/jdk/src/classes/build/tools/intpoly/FieldGen.java
+++ b/make/jdk/src/classes/build/tools/intpoly/FieldGen.java
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -34,36 +34,6 @@ import java.util.*;
public class FieldGen {
- static FieldParams Curve25519 = new FieldParams(
- "IntegerPolynomial25519", 26, 10, 1, 255,
- Arrays.asList(
- new Term(0, -19)
- ),
- Curve25519CrSequence(), simpleSmallCrSequence(10)
- );
-
- private static List Curve25519CrSequence() {
- List result = new ArrayList();
-
- // reduce(7,2)
- result.add(new Reduce(17));
- result.add(new Reduce(18));
-
- // carry(8,2)
- result.add(new Carry(8));
- result.add(new Carry(9));
-
- // reduce(0,7)
- for (int i = 10; i < 17; i++) {
- result.add(new Reduce(i));
- }
-
- // carry(0,9)
- result.addAll(fullCarry(10));
-
- return result;
- }
-
static FieldParams Curve448 = new FieldParams(
"IntegerPolynomial448", 28, 16, 1, 448,
Arrays.asList(
@@ -224,8 +194,7 @@ public class FieldGen {
}
static final FieldParams[] ALL_FIELDS = {
- Curve25519, Curve448,
- P256, P384, P521, O256, O384, O521, O25519, O448
+ Curve448, P256, P384, P521, O256, O384, O521, O25519, O448
};
public static class Term {
diff --git a/make/jdk/src/classes/build/tools/taglet/JSpec.java b/make/jdk/src/classes/build/tools/taglet/JSpec.java
index 7e1e0ca215e..196aaccb32b 100644
--- a/make/jdk/src/classes/build/tools/taglet/JSpec.java
+++ b/make/jdk/src/classes/build/tools/taglet/JSpec.java
@@ -25,13 +25,13 @@
package build.tools.taglet;
+import java.net.URI;
import java.util.EnumSet;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
-import java.lang.reflect.Field;
import javax.lang.model.element.Element;
@@ -141,6 +141,11 @@ public class JSpec implements Taglet {
@Override
public String toString(List tags, Element elem) {
+ throw new UnsupportedOperationException();
+ }
+
+ // @Override - requires JDK-8373922 in build JDK
+ public String toString(List tags, Element elem, URI docRoot) {
if (tags.isEmpty())
return "";
@@ -177,7 +182,7 @@ public class JSpec implements Taglet {
String preview = m.group("preview"); // null if no preview feature
String chapter = m.group("chapter");
String section = m.group("section");
- String rootParent = currentPath().replaceAll("[^/]+", "..");
+ String rootParent = docRoot.resolve("..").toString();
String url = preview == null ?
String.format("%1$s/specs/%2$s/%2$s-%3$s.html#%2$s-%3$s%4$s",
@@ -230,23 +235,6 @@ public class JSpec implements Taglet {
return sb.toString();
}
- private static ThreadLocal CURRENT_PATH = null;
-
- private String currentPath() {
- if (CURRENT_PATH == null) {
- try {
- Field f = Class.forName("jdk.javadoc.internal.doclets.formats.html.HtmlDocletWriter")
- .getField("CURRENT_PATH");
- @SuppressWarnings("unchecked")
- ThreadLocal tl = (ThreadLocal) f.get(null);
- CURRENT_PATH = tl;
- } catch (ReflectiveOperationException e) {
- throw new RuntimeException("Cannot determine current path", e);
- }
- }
- return CURRENT_PATH.get();
- }
-
private String expand(List trees) {
return (new SimpleDocTreeVisitor() {
public StringBuilder defaultAction(DocTree tree, StringBuilder sb) {
diff --git a/make/jdk/src/classes/build/tools/taglet/SealedGraph.java b/make/jdk/src/classes/build/tools/taglet/SealedGraph.java
index 3e93826c180..300999b77c0 100644
--- a/make/jdk/src/classes/build/tools/taglet/SealedGraph.java
+++ b/make/jdk/src/classes/build/tools/taglet/SealedGraph.java
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2022, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -32,7 +32,9 @@ import jdk.javadoc.doclet.Taglet;
import javax.lang.model.element.*;
import javax.lang.model.type.DeclaredType;
+import javax.lang.model.util.Elements;
import java.io.IOException;
+import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
@@ -78,6 +80,11 @@ public final class SealedGraph implements Taglet {
@Override
public String toString(List tags, Element element) {
+ throw new UnsupportedOperationException();
+ }
+
+ // @Override - requires JDK-8373922 in build JDK
+ public String toString(List tags, Element element, URI docRoot) {
if (sealedDotOutputDir == null || sealedDotOutputDir.isEmpty()) {
return "";
}
@@ -85,9 +92,15 @@ public final class SealedGraph implements Taglet {
return "";
}
- ModuleElement module = docletEnvironment.getElementUtils().getModuleOf(element);
+ Elements util = docletEnvironment.getElementUtils();
+ ModuleElement module = util.getModuleOf(element);
+
+ // '.' in .DOT file name is converted to '/' in .SVG path, so we use '-' as separator for nested classes.
+ // module_package.subpackage.Outer-Inner.dot => module/package/subpackage/Outer-Inner-sealed-graph.svg
Path dotFile = Path.of(sealedDotOutputDir,
- module.getQualifiedName() + "_" + typeElement.getQualifiedName() + ".dot");
+ module.getQualifiedName() + "_"
+ + util.getPackageOf(element).getQualifiedName() + "."
+ + packagelessCanonicalName(typeElement).replace(".", "-") + ".dot");
Set exports = module.getDirectives().stream()
.filter(ModuleElement.ExportsDirective.class::isInstance)
@@ -99,7 +112,7 @@ public final class SealedGraph implements Taglet {
.map(Objects::toString)
.collect(Collectors.toUnmodifiableSet());
- String dotContent = new Renderer().graph(typeElement, exports);
+ String dotContent = new Renderer().graph(typeElement, exports, docRoot);
try {
Files.writeString(dotFile, dotContent, WRITE, CREATE, TRUNCATE_EXISTING);
@@ -107,8 +120,8 @@ public final class SealedGraph implements Taglet {
throw new RuntimeException(e);
}
- String simpleTypeName = packagelessCanonicalName(typeElement).replace('.', '/');
- String imageFile = simpleTypeName + "-sealed-graph.svg";
+ String simpleTypeName = packagelessCanonicalName(typeElement);
+ String imageFile = simpleTypeName.replace(".", "-") + "-sealed-graph.svg";
int thumbnailHeight = 100; // also appears in the stylesheet
String hoverImage = ""
+ getImage(simpleTypeName, imageFile, -1, true)
@@ -137,21 +150,26 @@ public final class SealedGraph implements Taglet {
private final class Renderer {
// Generates a graph in DOT format
- String graph(TypeElement rootClass, Set exports) {
- final State state = new State(rootClass);
+ String graph(TypeElement rootClass, Set exports, URI pathToRoot) {
+ if (!isInPublicApi(rootClass, exports)) {
+ // Alternatively we can return "" for the graph since there is no single root to render
+ throw new IllegalArgumentException("Root not in public API: " + rootClass.getQualifiedName());
+ }
+ final State state = new State(pathToRoot);
traverse(state, rootClass, exports);
return state.render();
}
static void traverse(State state, TypeElement node, Set exports) {
+ if (!isInPublicApi(node, exports)) {
+ throw new IllegalArgumentException("Bad request, not in public API: " + node.getQualifiedName());
+ }
state.addNode(node);
if (!(node.getModifiers().contains(Modifier.SEALED) || node.getModifiers().contains(Modifier.FINAL))) {
state.addNonSealedEdge(node);
} else {
for (TypeElement subNode : permittedSubclasses(node, exports)) {
- if (isInPublicApi(node, exports) && isInPublicApi(subNode, exports)) {
- state.addEdge(node, subNode);
- }
+ state.addEdge(node, subNode);
traverse(state, subNode, exports);
}
}
@@ -163,7 +181,7 @@ public final class SealedGraph implements Taglet {
private static final String TOOLTIP = "tooltip";
private static final String LINK = "href";
- private final TypeElement rootNode;
+ private final URI pathToRoot;
private final StringBuilder builder;
@@ -188,8 +206,8 @@ public final class SealedGraph implements Taglet {
}
}
- public State(TypeElement rootNode) {
- this.rootNode = rootNode;
+ public State(URI pathToRoot) {
+ this.pathToRoot = pathToRoot;
nodeStyleMap = new LinkedHashMap<>();
builder = new StringBuilder()
.append("digraph G {")
@@ -212,24 +230,15 @@ public final class SealedGraph implements Taglet {
var styles = nodeStyleMap.computeIfAbsent(id(node), n -> new LinkedHashMap<>());
styles.put(LABEL, new StyleItem.PlainString(node.getSimpleName().toString()));
styles.put(TOOLTIP, new StyleItem.PlainString(node.getQualifiedName().toString()));
- styles.put(LINK, new StyleItem.PlainString(relativeLink(node)));
+ styles.put(LINK, new StyleItem.PlainString(pathToRoot.resolve(relativeLink(node)).toString()));
}
- // A permitted class must be in the same package or in the same module.
- // This implies the module is always the same.
private String relativeLink(TypeElement node) {
var util = SealedGraph.this.docletEnvironment.getElementUtils();
- var nodePackage = util.getPackageOf(node);
- // Note: SVG files for nested types use the simple names of containing types as parent directories.
- // We therefore need to convert all dots in the qualified name to "../" below.
- var backNavigator = rootNode.getQualifiedName().toString().chars()
- .filter(c -> c == '.')
- .mapToObj(c -> "../")
- .collect(joining());
- var forwardNavigator = nodePackage.getQualifiedName().toString()
- .replace(".", "/");
+ var path = util.getModuleOf(node).getQualifiedName().toString() + "/"
+ + util.getPackageOf(node).getQualifiedName().toString().replace(".", "/");
- return backNavigator + forwardNavigator + "/" + packagelessCanonicalName(node) + ".html";
+ return path + "/" + packagelessCanonicalName(node) + ".html";
}
public void addEdge(TypeElement node, TypeElement subNode) {
@@ -281,25 +290,33 @@ public final class SealedGraph implements Taglet {
private String quotedId(TypeElement node) {
return "\"" + id(node) + "\"";
}
-
- private String simpleName(String name) {
- int lastDot = name.lastIndexOf('.');
- return lastDot < 0
- ? name
- : name.substring(lastDot);
- }
-
}
private static List permittedSubclasses(TypeElement node, Set exports) {
- return node.getPermittedSubclasses().stream()
- .filter(DeclaredType.class::isInstance)
- .map(DeclaredType.class::cast)
- .map(DeclaredType::asElement)
- .filter(TypeElement.class::isInstance)
- .map(TypeElement.class::cast)
- .filter(te -> isInPublicApi(te, exports))
- .toList();
+ List dfsStack = new ArrayList().reversed(); // Faster operations to head
+ SequencedCollection result = new LinkedHashSet<>(); // Deduplicate diamond interface inheritance
+ // The starting node may be in the public API - still expand it
+ prependSubclasses(node, dfsStack);
+
+ while (!dfsStack.isEmpty()) {
+ TypeElement now = dfsStack.removeFirst();
+ if (isInPublicApi(now, exports)) {
+ result.addLast(now);
+ } else {
+ // Skip the non-exported classes in the hierarchy
+ prependSubclasses(now, dfsStack);
+ }
+ }
+
+ return List.copyOf(result);
+ }
+
+ private static void prependSubclasses(TypeElement node, List dfs) {
+ for (var e : node.getPermittedSubclasses().reversed()) {
+ if (e instanceof DeclaredType dt && dt.asElement() instanceof TypeElement te) {
+ dfs.addFirst(te);
+ }
+ }
}
private static boolean isInPublicApi(TypeElement typeElement, Set exports) {
diff --git a/make/jdk/src/classes/build/tools/taglet/ToolGuide.java b/make/jdk/src/classes/build/tools/taglet/ToolGuide.java
index 8db2aee3092..7ad4f6b9b9f 100644
--- a/make/jdk/src/classes/build/tools/taglet/ToolGuide.java
+++ b/make/jdk/src/classes/build/tools/taglet/ToolGuide.java
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -25,13 +25,13 @@
package build.tools.taglet;
+import java.net.URI;
import java.util.EnumSet;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
-import java.lang.reflect.Field;
import javax.lang.model.element.Element;
@@ -91,6 +91,11 @@ public class ToolGuide implements Taglet {
@Override
public String toString(List tags, Element elem) {
+ throw new UnsupportedOperationException();
+ }
+
+ // @Override - requires JDK-8373922 in build JDK
+ public String toString(List tags, Element elem, URI docRoot) {
if (tags.isEmpty())
return "";
@@ -118,7 +123,7 @@ public class ToolGuide implements Taglet {
if (label.isEmpty()) {
label = name;
}
- String rootParent = currentPath().replaceAll("[^/]+", "..");
+ String rootParent = docRoot.resolve("..").toString();
String url = String.format("%s/%s/%s.html",
rootParent, BASE_URL, name);
@@ -141,22 +146,4 @@ public class ToolGuide implements Taglet {
return sb.toString();
}
-
- private static ThreadLocal CURRENT_PATH = null;
-
- private String currentPath() {
- if (CURRENT_PATH == null) {
- try {
- Field f = Class.forName("jdk.javadoc.internal.doclets.formats.html.HtmlDocletWriter")
- .getField("CURRENT_PATH");
- @SuppressWarnings("unchecked")
- ThreadLocal tl = (ThreadLocal) f.get(null);
- CURRENT_PATH = tl;
- } catch (ReflectiveOperationException e) {
- throw new RuntimeException("Cannot determine current path", e);
- }
- }
- return CURRENT_PATH.get();
- }
-
}
diff --git a/make/langtools/tools/previewfeature/SetupPreviewFeature.java b/make/langtools/tools/previewfeature/SetupPreviewFeature.java
new file mode 100644
index 00000000000..5f9b00edc6d
--- /dev/null
+++ b/make/langtools/tools/previewfeature/SetupPreviewFeature.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation. Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package previewfeature;
+
+import com.sun.source.util.JavacTask;
+import com.sun.source.util.Trees;
+import java.io.StringWriter;
+import java.lang.reflect.Field;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
+import javax.lang.model.element.ElementKind;
+import javax.tools.ToolProvider;
+
+/* Construct a hybrid PreviewFeature.Feature enum that includes constants both
+ * from the current JDK sources (so that they can be used in the javac API sources),
+ * and from the bootstrap JDK (so that they can be used in the bootstrap classfiles).
+ *
+ * This hybrid enum is only used for the interim javac.
+ */
+public class SetupPreviewFeature {
+ public static void main(String... args) throws Exception {
+ Class runtimeFeature = Class.forName("jdk.internal.javac.PreviewFeature$Feature");
+ Set constantsToAdd = new HashSet<>();
+ for (Field runtimeField : runtimeFeature.getDeclaredFields()) {
+ if (runtimeField.isEnumConstant()) {
+ constantsToAdd.add(runtimeField.getName());
+ }
+ }
+ var dummy = new StringWriter();
+ var compiler = ToolProvider.getSystemJavaCompiler();
+ var source = Path.of(args[0]);
+ try (var fm = compiler.getStandardFileManager(null, null, null)) {
+ JavacTask task =
+ (JavacTask) compiler.getTask(dummy, null, null, null, null, fm.getJavaFileObjects(source));
+ task.analyze();
+ var sourceFeature = task.getElements()
+ .getTypeElement("jdk.internal.javac.PreviewFeature.Feature");
+ int insertPosition = -1;
+ for (var el : sourceFeature.getEnclosedElements()) {
+ if (el.getKind() == ElementKind.ENUM_CONSTANT) {
+ constantsToAdd.remove(el.getSimpleName().toString());
+ if (insertPosition == (-1)) {
+ var trees = Trees.instance(task);
+ var elPath = trees.getPath(el);
+ insertPosition = (int) trees.getSourcePositions()
+ .getStartPosition(elPath.getCompilationUnit(),
+ elPath.getLeaf());
+ }
+ }
+ }
+ var target = Path.of(args[1]);
+ Files.createDirectories(target.getParent());
+ if (constantsToAdd.isEmpty()) {
+ Files.copy(source, target, StandardCopyOption.REPLACE_EXISTING);
+ } else {
+ String sourceCode = Files.readString(source);
+ try (var out = Files.newBufferedWriter(target)) {
+ out.write(sourceCode, 0, insertPosition);
+ out.write(constantsToAdd.stream()
+ .collect(Collectors.joining(", ",
+ "/*compatibility constants:*/ ",
+ ",\n")));
+ out.write(sourceCode, insertPosition, sourceCode.length() - insertPosition);
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/make/modules/java.base/Launcher.gmk b/make/modules/java.base/Launcher.gmk
index 3a3920acb12..bfae0925c07 100644
--- a/make/modules/java.base/Launcher.gmk
+++ b/make/modules/java.base/Launcher.gmk
@@ -95,7 +95,8 @@ ifeq ($(call isTargetOsType, unix), true)
CFLAGS := $(VERSION_CFLAGS), \
EXTRA_HEADER_DIRS := libjava, \
EXTRA_OBJECT_FILES := \
- $(SUPPORT_OUTPUTDIR)/native/$(MODULE)/libjava/childproc$(OBJ_SUFFIX), \
+ $(SUPPORT_OUTPUTDIR)/native/$(MODULE)/libjava/childproc$(OBJ_SUFFIX) \
+ $(SUPPORT_OUTPUTDIR)/native/$(MODULE)/libjava/childproc_errorcodes$(OBJ_SUFFIX), \
LD_SET_ORIGIN := false, \
OUTPUT_DIR := $(SUPPORT_OUTPUTDIR)/modules_libs/$(MODULE), \
))
diff --git a/make/modules/java.desktop/lib/AwtLibraries.gmk b/make/modules/java.desktop/lib/AwtLibraries.gmk
index 8b6b50b9e62..887dfab01df 100644
--- a/make/modules/java.desktop/lib/AwtLibraries.gmk
+++ b/make/modules/java.desktop/lib/AwtLibraries.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -99,14 +99,16 @@ ifeq ($(call isTargetOs, windows), true)
$(TOPDIR)/src/$(MODULE)/windows/native/libawt/windows/awt.rc
endif
-# This is the object file to provide the dladdr API, which is not
-# part of AIX. It occurs several times in the jdk code base.
-# Do not include it. When statically linking the java
-# launcher with all JDK and VM static libraries, we use the
-# --whole-archive linker option. The duplicate objects in different
-# static libraries cause linking errors due to duplicate symbols.
ifeq ($(call isTargetOs, aix), true)
+ # This is the object file to provide the dladdr API, which is not
+ # part of AIX. It occurs several times in the jdk code base.
+ # Do not include it. When statically linking the java
+ # launcher with all JDK and VM static libraries, we use the
+ # --whole-archive linker option. The duplicate objects in different
+ # static libraries cause linking errors due to duplicate symbols.
LIBAWT_STATIC_EXCLUDE_OBJS := porting_aix.o
+
+ LIBAWT_CFLAGS += -I$(TOPDIR)/src/java.base/aix/native/include
endif
# -fgcse-after-reload improves performance of MaskFill in Java2D by 20% for
@@ -423,6 +425,9 @@ endif
ifeq ($(call isTargetOs, linux)+$(ENABLE_HEADLESS_ONLY), true+true)
LIBJAWT_CFLAGS += -DHEADLESS
endif
+ifeq ($(call isTargetOs, aix)+$(ENABLE_HEADLESS_ONLY), true+true)
+ LIBJAWT_CFLAGS += -DHEADLESS
+endif
ifeq ($(call isTargetOs, windows)+$(call isTargetCpu, x86), true+true)
LIBJAWT_LIBS_windows := kernel32.lib
diff --git a/make/modules/java.desktop/lib/ClientLibraries.gmk b/make/modules/java.desktop/lib/ClientLibraries.gmk
index b76cb8dc4e3..3e37fe79643 100644
--- a/make/modules/java.desktop/lib/ClientLibraries.gmk
+++ b/make/modules/java.desktop/lib/ClientLibraries.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -257,6 +257,7 @@ ifeq ($(ENABLE_HEADLESS_ONLY), false)
DISABLED_WARNINGS_microsoft_dgif_lib.c := 4018 4267, \
DISABLED_WARNINGS_microsoft_splashscreen_impl.c := 4018 4267 4244, \
DISABLED_WARNINGS_microsoft_splashscreen_png.c := 4267, \
+ DISABLED_WARNINGS_microsoft_pngread.c := 4146, \
DISABLED_WARNINGS_microsoft_splashscreen_sys.c := 4267 4244, \
LDFLAGS := $(ICONV_LDFLAGS), \
LDFLAGS_windows := -delayload:user32.dll, \
@@ -338,11 +339,8 @@ else
# noexcept-type required for GCC 7 builds. Not required for GCC 8+.
# expansion-to-defined required for GCC 9 builds. Not required for GCC 10+.
# maybe-uninitialized required for GCC 8 builds. Not required for GCC 9+.
- # calloc-transposed-args required for GCC 14 builds. (fixed upstream in
- # Harfbuzz 032c931e1c0cfb20f18e5acb8ba005775242bd92)
HARFBUZZ_DISABLED_WARNINGS_CXX_gcc := class-memaccess noexcept-type \
- expansion-to-defined dangling-reference maybe-uninitialized \
- calloc-transposed-args
+ expansion-to-defined dangling-reference maybe-uninitialized
HARFBUZZ_DISABLED_WARNINGS_clang := missing-field-initializers \
range-loop-analysis unused-variable
HARFBUZZ_DISABLED_WARNINGS_microsoft := 4267 4244
diff --git a/make/modules/jdk.hotspot.agent/Lib.gmk b/make/modules/jdk.hotspot.agent/Lib.gmk
index ed8de631dc3..da02e0dab39 100644
--- a/make/modules/jdk.hotspot.agent/Lib.gmk
+++ b/make/modules/jdk.hotspot.agent/Lib.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2015, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2015, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -55,6 +55,12 @@ else
LIBSAPROC_LINK_TYPE := C
endif
+# DWARF related sources would be included on supported platforms only.
+LIBSAPROC_EXCLUDE_FILES :=
+ifneq ($(call And, $(call isTargetOs, linux) $(call isTargetCpu, x86_64 aarch64)), true)
+ LIBSAPROC_EXCLUDE_FILES := DwarfParser.cpp dwarf.cpp
+endif
+
$(eval $(call SetupJdkLibrary, BUILD_LIBSAPROC, \
NAME := saproc, \
LINK_TYPE := $(LIBSAPROC_LINK_TYPE), \
@@ -70,6 +76,7 @@ $(eval $(call SetupJdkLibrary, BUILD_LIBSAPROC, \
CFLAGS := $(LIBSAPROC_CFLAGS), \
CXXFLAGS := $(LIBSAPROC_CFLAGS) $(LIBSAPROC_CXXFLAGS), \
EXTRA_SRC := $(LIBSAPROC_EXTRA_SRC), \
+ EXCLUDE_FILES := $(LIBSAPROC_EXCLUDE_FILES), \
JDK_LIBS := java.base:libjava, \
LIBS_linux := $(LIBDL), \
LIBS_macosx := \
diff --git a/make/modules/jdk.jpackage/Java.gmk b/make/modules/jdk.jpackage/Java.gmk
index da66fc14009..1fd4d527217 100644
--- a/make/modules/jdk.jpackage/Java.gmk
+++ b/make/modules/jdk.jpackage/Java.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -29,7 +29,7 @@ DISABLED_WARNINGS_java += dangling-doc-comments
COPY += .gif .png .txt .spec .script .prerm .preinst \
.postrm .postinst .list .sh .desktop .copyright .control .plist .template \
- .icns .scpt .wxs .wxl .wxi .wxf .ico .bmp .tiff .service .xsl
+ .icns .scpt .wxs .wxl .wxi .wxf .ico .bmp .tiff .service .xsl .js
CLEAN += .properties
diff --git a/make/scripts/fixpath.sh b/make/scripts/fixpath.sh
index 6a524df4c68..78690f1f2cc 100644
--- a/make/scripts/fixpath.sh
+++ b/make/scripts/fixpath.sh
@@ -88,7 +88,10 @@ function setup() {
fi
if [[ -z ${CMD+x} ]]; then
- CMD="$DRIVEPREFIX/c/windows/system32/cmd.exe"
+ CMD="$(type -p cmd.exe 2>/dev/null)"
+ if [[ -z "$CMD" ]]; then
+ CMD="$DRIVEPREFIX/c/windows/system32/cmd.exe"
+ fi
fi
if [[ -z ${WINTEMP+x} ]]; then
diff --git a/make/test/JtregNativeJdk.gmk b/make/test/JtregNativeJdk.gmk
index 0482011f561..6774e708f99 100644
--- a/make/test/JtregNativeJdk.gmk
+++ b/make/test/JtregNativeJdk.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2015, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2015, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -63,7 +63,8 @@ ifeq ($(call isTargetOs, windows), true)
BUILD_JDK_JTREG_EXCLUDE += libDirectIO.c libInheritedChannel.c \
libExplicitAttach.c libImplicitAttach.c \
exelauncher.c libFDLeaker.c exeFDLeakTester.c \
- libChangeSignalDisposition.c exePrintSignalDisposition.c
+ libChangeSignalDisposition.c exePrintSignalDisposition.c \
+ libConcNativeFork.c libPipesCloseOnExec.c
BUILD_JDK_JTREG_EXECUTABLES_LIBS_exeNullCallerTest := $(LIBCXX)
BUILD_JDK_JTREG_EXECUTABLES_LIBS_exerevokeall := advapi32.lib
@@ -77,6 +78,9 @@ else
BUILD_JDK_JTREG_LIBRARIES_LDFLAGS_libLinkerInvokerUnnamed := -pthread
BUILD_JDK_JTREG_LIBRARIES_LDFLAGS_libLinkerInvokerModule := -pthread
BUILD_JDK_JTREG_LIBRARIES_LDFLAGS_libLoaderLookupInvoker := -pthread
+ BUILD_JDK_JTREG_LIBRARIES_LDFLAGS_libConcNativeFork := -pthread
+ BUILD_JDK_JTREG_LIBRARIES_LDFLAGS_libPipesCloseOnExec := -pthread
+ BUILD_JDK_JTREG_LIBRARIES_LDFLAGS_libLoaderLookupInvoker := -pthread
BUILD_JDK_JTREG_LIBRARIES_LDFLAGS_libExplicitAttach := -pthread
BUILD_JDK_JTREG_LIBRARIES_LDFLAGS_libImplicitAttach := -pthread
diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad
index 9734c6845ea..53fa4e3066c 100644
--- a/src/hotspot/cpu/aarch64/aarch64.ad
+++ b/src/hotspot/cpu/aarch64/aarch64.ad
@@ -1,5 +1,5 @@
//
-// Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
// Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
// Copyright 2025 Arm Limited and/or its affiliates.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -1182,12 +1182,12 @@ class CallStubImpl {
public:
// Size of call trampoline stub.
static uint size_call_trampoline() {
- return 0; // no call trampolines on this platform
+ return MacroAssembler::max_trampoline_stub_size();
}
// number of relocations needed by a call trampoline stub
static uint reloc_call_trampoline() {
- return 0; // no call trampolines on this platform
+ return 5; // metadata; call dest; trampoline address; trampoline destination; trampoline_owner_metadata
}
};
@@ -2233,15 +2233,9 @@ uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
{
st->print_cr("# MachUEPNode");
- if (UseCompressedClassPointers) {
- st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
- st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
- st->print_cr("\tcmpw rscratch1, r10");
- } else {
- st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
- st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
- st->print_cr("\tcmp rscratch1, r10");
- }
+ st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
+ st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
+ st->print_cr("\tcmpw rscratch1, r10");
st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
}
#endif
@@ -2467,11 +2461,8 @@ bool Matcher::is_generic_vector(MachOper* opnd) {
return opnd->opcode() == VREG;
}
+#ifdef ASSERT
// Return whether or not this register is ever used as an argument.
-// This function is used on startup to build the trampoline stubs in
-// generateOptoStub. Registers not mentioned will be killed by the VM
-// call in the trampoline, and arguments in those registers not be
-// available to the callee.
bool Matcher::can_be_java_arg(int reg)
{
return
@@ -2492,11 +2483,7 @@ bool Matcher::can_be_java_arg(int reg)
reg == V6_num || reg == V6_H_num ||
reg == V7_num || reg == V7_H_num;
}
-
-bool Matcher::is_spillable_arg(int reg)
-{
- return can_be_java_arg(reg);
-}
+#endif
uint Matcher::int_pressure_limit()
{
@@ -2531,10 +2518,6 @@ uint Matcher::float_pressure_limit()
return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
}
-bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
- return false;
-}
-
const RegMask& Matcher::divI_proj_mask() {
ShouldNotReachHere();
return RegMask::EMPTY;
@@ -3814,11 +3797,6 @@ frame %{
// Compiled code's Frame Pointer
frame_pointer(R31);
- // Interpreter stores its frame pointer in a register which is
- // stored to the stack by I2CAdaptors.
- // I2CAdaptors convert from interpreted java to compiled java.
- interpreter_frame_pointer(R29);
-
// Stack alignment requirement
stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
@@ -8036,6 +8014,21 @@ instruct membar_release_lock() %{
ins_pipe(pipe_serial);
%}
+instruct membar_storeload() %{
+ match(MemBarStoreLoad);
+ ins_cost(VOLATILE_REF_COST*100);
+
+ format %{ "MEMBAR-store-load\n\t"
+ "dmb ish" %}
+
+ ins_encode %{
+ __ block_comment("membar_storeload");
+ __ membar(Assembler::StoreLoad);
+ %}
+
+ ins_pipe(pipe_serial);
+%}
+
instruct unnecessary_membar_volatile() %{
predicate(unnecessary_volatile(n));
match(MemBarVolatile);
@@ -8065,6 +8058,20 @@ instruct membar_volatile() %{
ins_pipe(pipe_serial);
%}
+instruct membar_full() %{
+ match(MemBarFull);
+ ins_cost(VOLATILE_REF_COST*100);
+
+ format %{ "membar_full\n\t"
+ "dmb ish" %}
+ ins_encode %{
+ __ block_comment("membar_full");
+ __ membar(Assembler::AnyAny);
+ %}
+
+ ins_pipe(pipe_serial);
+%}
+
// ============================================================================
// Cast/Convert Instructions
diff --git a/src/hotspot/cpu/aarch64/aarch64_vector.ad b/src/hotspot/cpu/aarch64/aarch64_vector.ad
index 19f03d97a72..4c854913e63 100644
--- a/src/hotspot/cpu/aarch64/aarch64_vector.ad
+++ b/src/hotspot/cpu/aarch64/aarch64_vector.ad
@@ -1,6 +1,6 @@
//
// Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
-// Copyright (c) 2020, 2025, Arm Limited. All rights reserved.
+// Copyright (c) 2020, 2026, Arm Limited. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
@@ -247,10 +247,39 @@ source %{
case Op_MinVHF:
case Op_MaxVHF:
case Op_SqrtVHF:
+ if (UseSVE == 0 && !is_feat_fp16_supported()) {
+ return false;
+ }
+ break;
+ // At the time of writing this, the Vector API has no half-float (FP16) species.
+ // Consequently, AddReductionVHF and MulReductionVHF are only produced by the
+ // auto-vectorizer, which requires strictly ordered semantics for FP reductions.
+ //
+ // There is no direct Neon instruction that performs strictly ordered floating
+ // point add reduction. Hence, on Neon only machines, the add reduction operation
+ // is implemented as a scalarized sequence using half-precision scalar instruction
+ // FADD which requires FEAT_FP16 and ASIMDHP to be available on the target.
+ // On SVE machines (UseSVE > 0) however, there is a direct instruction (FADDA) which
+ // implements strictly ordered floating point add reduction which does not require
+ // the FEAT_FP16 and ASIMDHP checks as SVE supports half-precision floats by default.
+ case Op_AddReductionVHF:
// FEAT_FP16 is enabled if both "fphp" and "asimdhp" features are supported.
// Only the Neon instructions need this check. SVE supports half-precision floats
// by default.
- if (UseSVE == 0 && !is_feat_fp16_supported()) {
+ if (length_in_bytes < 8 || (UseSVE == 0 && !is_feat_fp16_supported())) {
+ return false;
+ }
+ break;
+ case Op_MulReductionVHF:
+ // There are no direct Neon/SVE instructions that perform strictly ordered
+ // floating point multiply reduction.
+ // For vector length ≤ 16 bytes, the reduction is implemented as a scalarized
+ // sequence using half-precision scalar instruction FMUL. This path requires
+ // FEAT_FP16 and ASIMDHP to be available on the target.
+ // For vector length > 16 bytes, this operation is disabled because there is no
+ // direct SVE instruction that performs a strictly ordered FP16 multiply
+ // reduction.
+ if (length_in_bytes < 8 || length_in_bytes > 16 || !is_feat_fp16_supported()) {
return false;
}
break;
@@ -300,6 +329,7 @@ source %{
case Op_VectorRearrange:
case Op_MulReductionVD:
case Op_MulReductionVF:
+ case Op_MulReductionVHF:
case Op_MulReductionVI:
case Op_MulReductionVL:
case Op_CompressBitsV:
@@ -364,6 +394,7 @@ source %{
case Op_VectorMaskCmp:
case Op_LoadVectorGather:
case Op_StoreVectorScatter:
+ case Op_AddReductionVHF:
case Op_AddReductionVF:
case Op_AddReductionVD:
case Op_AndReductionV:
@@ -597,13 +628,9 @@ instruct vloadcon(vReg dst, immI0 src) %{
BasicType bt = Matcher::vector_element_basic_type(this);
if (UseSVE == 0) {
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
+ int entry_idx = __ vector_iota_entry_index(bt);
assert(length_in_bytes <= 16, "must be");
- // The iota indices are ordered by type B/S/I/L/F/D, and the offset between two types is 16.
- int offset = exact_log2(type2aelembytes(bt)) << 4;
- if (is_floating_point_type(bt)) {
- offset += 32;
- }
- __ lea(rscratch1, ExternalAddress(StubRoutines::aarch64::vector_iota_indices() + offset));
+ __ lea(rscratch1, ExternalAddress(StubRoutines::aarch64::vector_iota_indices(entry_idx)));
if (length_in_bytes == 16) {
__ ldrq($dst$$FloatRegister, rscratch1);
} else {
@@ -3406,6 +3433,44 @@ instruct reduce_non_strict_order_add4F_neon(vRegF dst, vRegF fsrc, vReg vsrc, vR
ins_pipe(pipe_slow);
%}
+// Add Reduction for Half floats (FP16).
+// Neon does not provide direct instructions for strictly ordered floating-point add reductions.
+// On Neon-only targets (UseSVE = 0), this operation is implemented as a sequence of scalar additions:
+// values equal to the vector width are loaded into a vector register, each lane is extracted,
+// and its value is accumulated into the running sum, producing a final scalar result.
+instruct reduce_addHF_neon(vRegF dst, vRegF fsrc, vReg vsrc, vReg tmp) %{
+ predicate(UseSVE == 0);
+ match(Set dst (AddReductionVHF fsrc vsrc));
+ effect(TEMP_DEF dst, TEMP tmp);
+ format %{ "reduce_addHF $dst, $fsrc, $vsrc\t# 4HF/8HF. KILL $tmp" %}
+ ins_encode %{
+ uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
+ __ neon_reduce_add_fp16($dst$$FloatRegister, $fsrc$$FloatRegister,
+ $vsrc$$FloatRegister, length_in_bytes, $tmp$$FloatRegister);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+// This rule calculates the reduction result in strict order. Two cases will
+// reach here:
+// 1. Non strictly-ordered AddReductionVHF when vector size > 128-bits. For example -
+// AddReductionVHF generated by Vector API. For vector size > 128-bits, it is more
+// beneficial performance-wise to generate direct SVE instruction even if it is
+// strictly ordered.
+// 2. Strictly-ordered AddReductionVHF. For example - AddReductionVHF generated by
+// auto-vectorization on SVE machine.
+instruct reduce_addHF_sve(vRegF dst_src1, vReg src2) %{
+ predicate(UseSVE > 0);
+ match(Set dst_src1 (AddReductionVHF dst_src1 src2));
+ format %{ "reduce_addHF_sve $dst_src1, $dst_src1, $src2" %}
+ ins_encode %{
+ uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src2);
+ assert(length_in_bytes == MaxVectorSize, "invalid vector length");
+ __ sve_fadda($dst_src1$$FloatRegister, __ H, ptrue, $src2$$FloatRegister);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
// This rule calculates the reduction result in strict order. Two cases will
// reach here:
// 1. Non strictly-ordered AddReductionVF when vector size > 128-bits. For example -
@@ -3496,12 +3561,14 @@ instruct reduce_addL_masked(iRegLNoSp dst, iRegL isrc, vReg vsrc, pRegGov pg, vR
ins_pipe(pipe_slow);
%}
-instruct reduce_addF_masked(vRegF dst_src1, vReg src2, pRegGov pg) %{
+instruct reduce_addFHF_masked(vRegF dst_src1, vReg src2, pRegGov pg) %{
predicate(UseSVE > 0);
+ match(Set dst_src1 (AddReductionVHF (Binary dst_src1 src2) pg));
match(Set dst_src1 (AddReductionVF (Binary dst_src1 src2) pg));
- format %{ "reduce_addF_masked $dst_src1, $pg, $dst_src1, $src2" %}
+ format %{ "reduce_addFHF_masked $dst_src1, $pg, $dst_src1, $src2" %}
ins_encode %{
- __ sve_fadda($dst_src1$$FloatRegister, __ S,
+ BasicType bt = Matcher::vector_element_basic_type(this, $src2);
+ __ sve_fadda($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
$pg$$PRegister, $src2$$FloatRegister);
%}
ins_pipe(pipe_slow);
@@ -3549,14 +3616,17 @@ instruct reduce_mulL(iRegLNoSp dst, iRegL isrc, vReg vsrc) %{
ins_pipe(pipe_slow);
%}
-instruct reduce_mulF(vRegF dst, vRegF fsrc, vReg vsrc, vReg tmp) %{
+
+instruct reduce_mulFHF(vRegF dst, vRegF fsrc, vReg vsrc, vReg tmp) %{
predicate(Matcher::vector_length_in_bytes(n->in(2)) <= 16);
+ match(Set dst (MulReductionVHF fsrc vsrc));
match(Set dst (MulReductionVF fsrc vsrc));
effect(TEMP_DEF dst, TEMP tmp);
- format %{ "reduce_mulF $dst, $fsrc, $vsrc\t# 2F/4F. KILL $tmp" %}
+ format %{ "reduce_mulFHF $dst, $fsrc, $vsrc\t# 2F/4F/4HF/8HF. KILL $tmp" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
- __ neon_reduce_mul_fp($dst$$FloatRegister, T_FLOAT, $fsrc$$FloatRegister,
+ BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
+ __ neon_reduce_mul_fp($dst$$FloatRegister, bt, $fsrc$$FloatRegister,
$vsrc$$FloatRegister, length_in_bytes, $tmp$$FloatRegister);
%}
ins_pipe(pipe_slow);
diff --git a/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 b/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4
index 48bffb3cf35..58ed234194a 100644
--- a/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4
+++ b/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4
@@ -1,6 +1,6 @@
//
// Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
-// Copyright (c) 2020, 2025, Arm Limited. All rights reserved.
+// Copyright (c) 2020, 2026, Arm Limited. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
@@ -237,10 +237,39 @@ source %{
case Op_MinVHF:
case Op_MaxVHF:
case Op_SqrtVHF:
+ if (UseSVE == 0 && !is_feat_fp16_supported()) {
+ return false;
+ }
+ break;
+ // At the time of writing this, the Vector API has no half-float (FP16) species.
+ // Consequently, AddReductionVHF and MulReductionVHF are only produced by the
+ // auto-vectorizer, which requires strictly ordered semantics for FP reductions.
+ //
+ // There is no direct Neon instruction that performs strictly ordered floating
+ // point add reduction. Hence, on Neon only machines, the add reduction operation
+ // is implemented as a scalarized sequence using half-precision scalar instruction
+ // FADD which requires FEAT_FP16 and ASIMDHP to be available on the target.
+ // On SVE machines (UseSVE > 0) however, there is a direct instruction (FADDA) which
+ // implements strictly ordered floating point add reduction which does not require
+ // the FEAT_FP16 and ASIMDHP checks as SVE supports half-precision floats by default.
+ case Op_AddReductionVHF:
// FEAT_FP16 is enabled if both "fphp" and "asimdhp" features are supported.
// Only the Neon instructions need this check. SVE supports half-precision floats
// by default.
- if (UseSVE == 0 && !is_feat_fp16_supported()) {
+ if (length_in_bytes < 8 || (UseSVE == 0 && !is_feat_fp16_supported())) {
+ return false;
+ }
+ break;
+ case Op_MulReductionVHF:
+ // There are no direct Neon/SVE instructions that perform strictly ordered
+ // floating point multiply reduction.
+ // For vector length ≤ 16 bytes, the reduction is implemented as a scalarized
+ // sequence using half-precision scalar instruction FMUL. This path requires
+ // FEAT_FP16 and ASIMDHP to be available on the target.
+ // For vector length > 16 bytes, this operation is disabled because there is no
+ // direct SVE instruction that performs a strictly ordered FP16 multiply
+ // reduction.
+ if (length_in_bytes < 8 || length_in_bytes > 16 || !is_feat_fp16_supported()) {
return false;
}
break;
@@ -290,6 +319,7 @@ source %{
case Op_VectorRearrange:
case Op_MulReductionVD:
case Op_MulReductionVF:
+ case Op_MulReductionVHF:
case Op_MulReductionVI:
case Op_MulReductionVL:
case Op_CompressBitsV:
@@ -354,6 +384,7 @@ source %{
case Op_VectorMaskCmp:
case Op_LoadVectorGather:
case Op_StoreVectorScatter:
+ case Op_AddReductionVHF:
case Op_AddReductionVF:
case Op_AddReductionVD:
case Op_AndReductionV:
@@ -2063,6 +2094,25 @@ instruct reduce_non_strict_order_add4F_neon(vRegF dst, vRegF fsrc, vReg vsrc, vR
ins_pipe(pipe_slow);
%}
dnl
+
+// Add Reduction for Half floats (FP16).
+// Neon does not provide direct instructions for strictly ordered floating-point add reductions.
+// On Neon-only targets (UseSVE = 0), this operation is implemented as a sequence of scalar additions:
+// values equal to the vector width are loaded into a vector register, each lane is extracted,
+// and its value is accumulated into the running sum, producing a final scalar result.
+instruct reduce_addHF_neon(vRegF dst, vRegF fsrc, vReg vsrc, vReg tmp) %{
+ predicate(UseSVE == 0);
+ match(Set dst (AddReductionVHF fsrc vsrc));
+ effect(TEMP_DEF dst, TEMP tmp);
+ format %{ "reduce_addHF $dst, $fsrc, $vsrc\t# 4HF/8HF. KILL $tmp" %}
+ ins_encode %{
+ uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
+ __ neon_reduce_add_fp16($dst$$FloatRegister, $fsrc$$FloatRegister,
+ $vsrc$$FloatRegister, length_in_bytes, $tmp$$FloatRegister);
+ %}
+ ins_pipe(pipe_slow);
+%}
+dnl
dnl REDUCE_ADD_FP_SVE($1, $2 )
dnl REDUCE_ADD_FP_SVE(type, size)
define(`REDUCE_ADD_FP_SVE', `
@@ -2074,21 +2124,26 @@ define(`REDUCE_ADD_FP_SVE', `
// strictly ordered.
// 2. Strictly-ordered AddReductionV$1. For example - AddReductionV$1 generated by
// auto-vectorization on SVE machine.
-instruct reduce_add$1_sve(vReg$1 dst_src1, vReg src2) %{
- predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(2))) ||
- n->as_Reduction()->requires_strict_order());
+instruct reduce_add$1_sve(vReg`'ifelse($1, HF, F, $1) dst_src1, vReg src2) %{
+ ifelse($1, HF,
+ `predicate(UseSVE > 0);',
+ `predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(2))) ||
+ n->as_Reduction()->requires_strict_order());')
match(Set dst_src1 (AddReductionV$1 dst_src1 src2));
format %{ "reduce_add$1_sve $dst_src1, $dst_src1, $src2" %}
ins_encode %{
- assert(UseSVE > 0, "must be sve");
- uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src2);
+ ifelse($1, HF, `',
+ `assert(UseSVE > 0, "must be sve");
+ ')dnl
+uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src2);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
__ sve_fadda($dst_src1$$FloatRegister, __ $2, ptrue, $src2$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
-REDUCE_ADD_FP_SVE(F, S)
+REDUCE_ADD_FP_SVE(HF, H)
+REDUCE_ADD_FP_SVE(F, S)
// reduction addD
@@ -2129,21 +2184,30 @@ dnl
dnl REDUCE_ADD_FP_PREDICATE($1, $2 )
dnl REDUCE_ADD_FP_PREDICATE(insn_name, op_name)
define(`REDUCE_ADD_FP_PREDICATE', `
-instruct reduce_add$1_masked(vReg$1 dst_src1, vReg src2, pRegGov pg) %{
+instruct reduce_add$1_masked(vReg$2 dst_src1, vReg src2, pRegGov pg) %{
predicate(UseSVE > 0);
- match(Set dst_src1 (AddReductionV$1 (Binary dst_src1 src2) pg));
+ ifelse($2, F,
+ `match(Set dst_src1 (AddReductionVHF (Binary dst_src1 src2) pg));
+ match(Set dst_src1 (AddReductionV$2 (Binary dst_src1 src2) pg));',
+ `match(Set dst_src1 (AddReductionV$2 (Binary dst_src1 src2) pg));')
format %{ "reduce_add$1_masked $dst_src1, $pg, $dst_src1, $src2" %}
ins_encode %{
- __ sve_fadda($dst_src1$$FloatRegister, __ $2,
- $pg$$PRegister, $src2$$FloatRegister);
+ ifelse($2, F,
+ `BasicType bt = Matcher::vector_element_basic_type(this, $src2);
+ ',)dnl
+ifelse($2, F,
+ `__ sve_fadda($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
+ $pg$$PRegister, $src2$$FloatRegister);',
+ `__ sve_fadda($dst_src1$$FloatRegister, __ $2,
+ $pg$$PRegister, $src2$$FloatRegister);')
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
REDUCE_ADD_INT_PREDICATE(I, iRegIorL2I)
REDUCE_ADD_INT_PREDICATE(L, iRegL)
-REDUCE_ADD_FP_PREDICATE(F, S)
-REDUCE_ADD_FP_PREDICATE(D, D)
+REDUCE_ADD_FP_PREDICATE(FHF, F)
+REDUCE_ADD_FP_PREDICATE(D, D)
// ------------------------------ Vector reduction mul -------------------------
@@ -2176,30 +2240,37 @@ instruct reduce_mulL(iRegLNoSp dst, iRegL isrc, vReg vsrc) %{
ins_pipe(pipe_slow);
%}
-instruct reduce_mulF(vRegF dst, vRegF fsrc, vReg vsrc, vReg tmp) %{
- predicate(Matcher::vector_length_in_bytes(n->in(2)) <= 16);
- match(Set dst (MulReductionVF fsrc vsrc));
+dnl REDUCE_MUL_FP($1, $2 )
+dnl REDUCE_MUL_FP(insn_name, op_name)
+define(`REDUCE_MUL_FP', `
+instruct reduce_mul$1(vReg$2 dst, vReg$2 ifelse($2, F, fsrc, dsrc), vReg vsrc, vReg tmp) %{
+ predicate(Matcher::vector_length_in_bytes(n->in(2)) ifelse($2, F, <=, ==) 16);
+ ifelse($2, F,
+ `match(Set dst (MulReductionVHF fsrc vsrc));
+ match(Set dst (MulReductionV$2 fsrc vsrc));',
+ `match(Set dst (MulReductionV$2 dsrc vsrc));')
effect(TEMP_DEF dst, TEMP tmp);
- format %{ "reduce_mulF $dst, $fsrc, $vsrc\t# 2F/4F. KILL $tmp" %}
+ ifelse($2, F,
+ `format %{ "reduce_mul$1 $dst, $fsrc, $vsrc\t# 2F/4F/4HF/8HF. KILL $tmp" %}',
+ `format %{ "reduce_mul$1 $dst, $dsrc, $vsrc\t# 2D. KILL $tmp" %}')
ins_encode %{
- uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
- __ neon_reduce_mul_fp($dst$$FloatRegister, T_FLOAT, $fsrc$$FloatRegister,
- $vsrc$$FloatRegister, length_in_bytes, $tmp$$FloatRegister);
+ ifelse($2, F,
+ `uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
+ ',)dnl
+ifelse($2, F,
+ `BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
+ ',)dnl
+ifelse($2, F,
+ `__ neon_reduce_mul_fp($dst$$FloatRegister, bt, $fsrc$$FloatRegister,
+ $vsrc$$FloatRegister, length_in_bytes, $tmp$$FloatRegister);',
+ `__ neon_reduce_mul_fp($dst$$FloatRegister, T_DOUBLE, $dsrc$$FloatRegister,
+ $vsrc$$FloatRegister, 16, $tmp$$FloatRegister);')
%}
ins_pipe(pipe_slow);
-%}
-
-instruct reduce_mulD(vRegD dst, vRegD dsrc, vReg vsrc, vReg tmp) %{
- predicate(Matcher::vector_length_in_bytes(n->in(2)) == 16);
- match(Set dst (MulReductionVD dsrc vsrc));
- effect(TEMP_DEF dst, TEMP tmp);
- format %{ "reduce_mulD $dst, $dsrc, $vsrc\t# 2D. KILL $tmp" %}
- ins_encode %{
- __ neon_reduce_mul_fp($dst$$FloatRegister, T_DOUBLE, $dsrc$$FloatRegister,
- $vsrc$$FloatRegister, 16, $tmp$$FloatRegister);
- %}
- ins_pipe(pipe_slow);
-%}
+%}')dnl
+dnl
+REDUCE_MUL_FP(FHF, F)
+REDUCE_MUL_FP(D, D)
dnl
dnl REDUCE_BITWISE_OP_NEON($1, $2 $3 $4 )
diff --git a/src/hotspot/cpu/aarch64/assembler_aarch64.hpp b/src/hotspot/cpu/aarch64/assembler_aarch64.hpp
index 19b3bb1a65b..4c1c8d9bbc8 100644
--- a/src/hotspot/cpu/aarch64/assembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/assembler_aarch64.hpp
@@ -1000,30 +1000,6 @@ public:
f(0b0101010, 31, 25), f(0, 24), sf(offset, 23, 5), f(0, 4), f(cond, 3, 0);
}
-#define INSN(NAME, cond) \
- void NAME(address dest) { \
- br(cond, dest); \
- }
-
- INSN(beq, EQ);
- INSN(bne, NE);
- INSN(bhs, HS);
- INSN(bcs, CS);
- INSN(blo, LO);
- INSN(bcc, CC);
- INSN(bmi, MI);
- INSN(bpl, PL);
- INSN(bvs, VS);
- INSN(bvc, VC);
- INSN(bhi, HI);
- INSN(bls, LS);
- INSN(bge, GE);
- INSN(blt, LT);
- INSN(bgt, GT);
- INSN(ble, LE);
- INSN(bal, AL);
- INSN(bnv, NV);
-
void br(Condition cc, Label &L);
#undef INSN
@@ -1095,6 +1071,10 @@ public:
#undef INSN
+ void wfet(Register rt) {
+ system(0b00, 0b011, 0b0001, 0b0000, 0b000, rt);
+ }
+
// we only provide mrs and msr for the special purpose system
// registers where op1 (instr[20:19]) == 11
// n.b msr has L (instr[21]) == 0 mrs has L == 1
@@ -3814,8 +3794,8 @@ public:
}
private:
- void sve_cpy(FloatRegister Zd, SIMD_RegVariant T, PRegister Pg, int imm8,
- bool isMerge, bool isFloat) {
+ void _sve_cpy(FloatRegister Zd, SIMD_RegVariant T, PRegister Pg, int imm8,
+ bool isMerge, bool isFloat) {
starti;
assert(T != Q, "invalid size");
int sh = 0;
@@ -3839,11 +3819,11 @@ private:
public:
// SVE copy signed integer immediate to vector elements (predicated)
void sve_cpy(FloatRegister Zd, SIMD_RegVariant T, PRegister Pg, int imm8, bool isMerge) {
- sve_cpy(Zd, T, Pg, imm8, isMerge, /*isFloat*/false);
+ _sve_cpy(Zd, T, Pg, imm8, isMerge, /*isFloat*/false);
}
// SVE copy floating-point immediate to vector elements (predicated)
void sve_cpy(FloatRegister Zd, SIMD_RegVariant T, PRegister Pg, double d) {
- sve_cpy(Zd, T, Pg, checked_cast(pack(d)), /*isMerge*/true, /*isFloat*/true);
+ _sve_cpy(Zd, T, Pg, checked_cast(pack(d)), /*isMerge*/true, /*isFloat*/true);
}
// SVE conditionally select elements from two vectors
diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
index 30048a2079d..4de6237304d 100644
--- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -42,6 +42,7 @@
#include "runtime/frame.inline.hpp"
#include "runtime/sharedRuntime.hpp"
#include "runtime/stubRoutines.hpp"
+#include "runtime/threadIdentifier.hpp"
#include "utilities/powerOfTwo.hpp"
#include "vmreg_aarch64.inline.hpp"
@@ -59,22 +60,6 @@ const Register SHIFT_count = r0; // where count for shift operations must be
#define __ _masm->
-static void select_different_registers(Register preserve,
- Register extra,
- Register &tmp1,
- Register &tmp2) {
- if (tmp1 == preserve) {
- assert_different_registers(tmp1, tmp2, extra);
- tmp1 = extra;
- } else if (tmp2 == preserve) {
- assert_different_registers(tmp1, tmp2, extra);
- tmp2 = extra;
- }
- assert_different_registers(preserve, tmp1, tmp2);
-}
-
-
-
static void select_different_registers(Register preserve,
Register extra,
Register &tmp1,
@@ -536,6 +521,10 @@ void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_cod
#if INCLUDE_CDS
if (AOTCodeCache::is_on_for_dump()) {
address b = c->as_pointer();
+ if (b == (address)ThreadIdentifier::unsafe_offset()) {
+ __ lea(dest->as_register_lo(), ExternalAddress(b));
+ break;
+ }
if (AOTRuntimeConstants::contains(b)) {
__ load_aotrc_address(dest->as_register_lo(), b);
break;
@@ -1269,12 +1258,9 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, L
} else if (obj == klass_RInfo) {
klass_RInfo = dst;
}
- if (k->is_loaded() && !UseCompressedClassPointers) {
- select_different_registers(obj, dst, k_RInfo, klass_RInfo);
- } else {
- Rtmp1 = op->tmp3()->as_register();
- select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
- }
+
+ Rtmp1 = op->tmp3()->as_register();
+ select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
assert_different_registers(obj, k_RInfo, klass_RInfo);
diff --git a/src/hotspot/cpu/aarch64/c1_LIRGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRGenerator_aarch64.cpp
index ad26d494b2d..f10c5197d91 100644
--- a/src/hotspot/cpu/aarch64/c1_LIRGenerator_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c1_LIRGenerator_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -1287,9 +1287,7 @@ void LIRGenerator::do_CheckCast(CheckCast* x) {
}
LIR_Opr reg = rlock_result(x);
LIR_Opr tmp3 = LIR_OprFact::illegalOpr;
- if (!x->klass()->is_loaded() || UseCompressedClassPointers) {
- tmp3 = new_register(objectType);
- }
+ tmp3 = new_register(objectType);
__ checkcast(reg, obj.result(), x->klass(),
new_register(objectType), new_register(objectType), tmp3,
x->direct_compare(), info_for_exception, patching_info, stub,
@@ -1308,9 +1306,7 @@ void LIRGenerator::do_InstanceOf(InstanceOf* x) {
}
obj.load_item();
LIR_Opr tmp3 = LIR_OprFact::illegalOpr;
- if (!x->klass()->is_loaded() || UseCompressedClassPointers) {
- tmp3 = new_register(objectType);
- }
+ tmp3 = new_register(objectType);
__ instanceof(reg, obj.result(), x->klass(),
new_register(objectType), new_register(objectType), tmp3,
x->direct_compare(), patching_info, x->profiled_method(), x->profiled_bci());
diff --git a/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp
index e934632715c..89a9422ea48 100644
--- a/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -105,12 +105,8 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
} else {
mov(t1, checked_cast(markWord::prototype().value()));
str(t1, Address(obj, oopDesc::mark_offset_in_bytes()));
- if (UseCompressedClassPointers) { // Take care not to kill klass
- encode_klass_not_null(t1, klass);
- strw(t1, Address(obj, oopDesc::klass_offset_in_bytes()));
- } else {
- str(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
- }
+ encode_klass_not_null(t1, klass); // Take care not to kill klass
+ strw(t1, Address(obj, oopDesc::klass_offset_in_bytes()));
}
if (len->is_valid()) {
@@ -121,7 +117,7 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
// Clear gap/first 4 bytes following the length field.
strw(zr, Address(obj, base_offset));
}
- } else if (UseCompressedClassPointers && !UseCompactObjectHeaders) {
+ } else if (!UseCompactObjectHeaders) {
store_klass_gap(obj, zr);
}
}
diff --git a/src/hotspot/cpu/aarch64/c1_globals_aarch64.hpp b/src/hotspot/cpu/aarch64/c1_globals_aarch64.hpp
index 938a64dd399..bb6b3ce907e 100644
--- a/src/hotspot/cpu/aarch64/c1_globals_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/c1_globals_aarch64.hpp
@@ -42,7 +42,6 @@ define_pd_global(bool, TieredCompilation, false);
define_pd_global(intx, CompileThreshold, 1500 );
define_pd_global(intx, OnStackReplacePercentage, 933 );
-define_pd_global(intx, NewSizeThreadIncrease, 4*K );
define_pd_global(size_t, InitialCodeCacheSize, 160*K);
define_pd_global(size_t, ReservedCodeCacheSize, 32*M );
define_pd_global(size_t, NonProfiledCodeHeapSize, 13*M );
@@ -52,7 +51,6 @@ define_pd_global(bool, ProfileInterpreter, false);
define_pd_global(size_t, CodeCacheExpansionSize, 32*K );
define_pd_global(size_t, CodeCacheMinBlockLength, 1);
define_pd_global(size_t, CodeCacheMinimumUseSpace, 400*K);
-define_pd_global(bool, NeverActAsServerClassMachine, true );
define_pd_global(bool, CICompileOSR, true );
#endif // !COMPILER2
define_pd_global(bool, UseTypeProfile, false);
diff --git a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
index dc0b9eb9546..3c179f21c14 100644
--- a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright 2026 Arm Limited and/or its affiliates.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1883,6 +1884,27 @@ void C2_MacroAssembler::neon_reduce_mul_fp(FloatRegister dst, BasicType bt,
BLOCK_COMMENT("neon_reduce_mul_fp {");
switch(bt) {
+ // The T_SHORT type below is for Float16 type which also uses floating-point
+ // instructions.
+ case T_SHORT:
+ fmulh(dst, fsrc, vsrc);
+ ext(vtmp, T8B, vsrc, vsrc, 2);
+ fmulh(dst, dst, vtmp);
+ ext(vtmp, T8B, vsrc, vsrc, 4);
+ fmulh(dst, dst, vtmp);
+ ext(vtmp, T8B, vsrc, vsrc, 6);
+ fmulh(dst, dst, vtmp);
+ if (isQ) {
+ ext(vtmp, T16B, vsrc, vsrc, 8);
+ fmulh(dst, dst, vtmp);
+ ext(vtmp, T16B, vsrc, vsrc, 10);
+ fmulh(dst, dst, vtmp);
+ ext(vtmp, T16B, vsrc, vsrc, 12);
+ fmulh(dst, dst, vtmp);
+ ext(vtmp, T16B, vsrc, vsrc, 14);
+ fmulh(dst, dst, vtmp);
+ }
+ break;
case T_FLOAT:
fmuls(dst, fsrc, vsrc);
ins(vtmp, S, vsrc, 0, 1);
@@ -1907,6 +1929,33 @@ void C2_MacroAssembler::neon_reduce_mul_fp(FloatRegister dst, BasicType bt,
BLOCK_COMMENT("} neon_reduce_mul_fp");
}
+// Vector reduction add for half float type with ASIMD instructions.
+void C2_MacroAssembler::neon_reduce_add_fp16(FloatRegister dst, FloatRegister fsrc, FloatRegister vsrc,
+ unsigned vector_length_in_bytes, FloatRegister vtmp) {
+ assert(vector_length_in_bytes == 8 || vector_length_in_bytes == 16, "unsupported");
+ bool isQ = vector_length_in_bytes == 16;
+
+ BLOCK_COMMENT("neon_reduce_add_fp16 {");
+ faddh(dst, fsrc, vsrc);
+ ext(vtmp, T8B, vsrc, vsrc, 2);
+ faddh(dst, dst, vtmp);
+ ext(vtmp, T8B, vsrc, vsrc, 4);
+ faddh(dst, dst, vtmp);
+ ext(vtmp, T8B, vsrc, vsrc, 6);
+ faddh(dst, dst, vtmp);
+ if (isQ) {
+ ext(vtmp, T16B, vsrc, vsrc, 8);
+ faddh(dst, dst, vtmp);
+ ext(vtmp, T16B, vsrc, vsrc, 10);
+ faddh(dst, dst, vtmp);
+ ext(vtmp, T16B, vsrc, vsrc, 12);
+ faddh(dst, dst, vtmp);
+ ext(vtmp, T16B, vsrc, vsrc, 14);
+ faddh(dst, dst, vtmp);
+ }
+ BLOCK_COMMENT("} neon_reduce_add_fp16");
+}
+
// Helper to select logical instruction
void C2_MacroAssembler::neon_reduce_logical_helper(int opc, bool is64, Register Rd,
Register Rn, Register Rm,
@@ -2414,17 +2463,17 @@ void C2_MacroAssembler::neon_rearrange_hsd(FloatRegister dst, FloatRegister src,
break;
case T_LONG:
case T_DOUBLE:
- // Load the iota indices for Long type. The indices are ordered by
- // type B/S/I/L/F/D, and the offset between two types is 16; Hence
- // the offset for L is 48.
- lea(rscratch1,
- ExternalAddress(StubRoutines::aarch64::vector_iota_indices() + 48));
- ldrq(tmp, rscratch1);
- // Check whether the input "shuffle" is the same with iota indices.
- // Return "src" if true, otherwise swap the two elements of "src".
- cm(EQ, dst, size2, shuffle, tmp);
- ext(tmp, size1, src, src, 8);
- bsl(dst, size1, src, tmp);
+ {
+ int idx = vector_iota_entry_index(T_LONG);
+ lea(rscratch1,
+ ExternalAddress(StubRoutines::aarch64::vector_iota_indices(idx)));
+ ldrq(tmp, rscratch1);
+ // Check whether the input "shuffle" is the same with iota indices.
+ // Return "src" if true, otherwise swap the two elements of "src".
+ cm(EQ, dst, size2, shuffle, tmp);
+ ext(tmp, size1, src, src, 8);
+ bsl(dst, size1, src, tmp);
+ }
break;
default:
assert(false, "unsupported element type");
@@ -2875,3 +2924,45 @@ void C2_MacroAssembler::vector_expand_sve(FloatRegister dst, FloatRegister src,
// dst = 00 87 00 65 00 43 00 21
sve_tbl(dst, size, src, dst);
}
+
+// Optimized SVE cpy (imm, zeroing) instruction.
+//
+// `movi; cpy(imm, merging)` and `cpy(imm, zeroing)` have the same
+// functionality, but test results show that `movi; cpy(imm, merging)` has
+// higher throughput on some microarchitectures. This would depend on
+// microarchitecture and so may vary between implementations.
+void C2_MacroAssembler::sve_cpy(FloatRegister dst, SIMD_RegVariant T,
+ PRegister pg, int imm8, bool isMerge) {
+ if (VM_Version::prefer_sve_merging_mode_cpy() && !isMerge) {
+ // Generates a NEON instruction `movi V.2d, #0`.
+ // On AArch64, Z and V registers alias in the low 128 bits, so V is
+ // the low 128 bits of Z. A write to V also clears all bits of
+ // Z above 128, so this `movi` instruction effectively zeroes the
+ // entire Z register. According to the Arm Software Optimization
+ // Guide, `movi` is zero latency.
+ movi(dst, T2D, 0);
+ isMerge = true;
+ }
+ Assembler::sve_cpy(dst, T, pg, imm8, isMerge);
+}
+
+int C2_MacroAssembler::vector_iota_entry_index(BasicType bt) {
+ // The vector iota entries array is ordered by type B/S/I/L/F/D, and
+ // the offset between two types is 16.
+ switch(bt) {
+ case T_BYTE:
+ return 0;
+ case T_SHORT:
+ return 1;
+ case T_INT:
+ return 2;
+ case T_LONG:
+ return 3;
+ case T_FLOAT:
+ return 4;
+ case T_DOUBLE:
+ return 5;
+ default:
+ ShouldNotReachHere();
+ }
+}
diff --git a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp
index 4f3a41da402..f96d3ffb863 100644
--- a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp
@@ -75,6 +75,8 @@
unsigned vector_length_in_bytes);
public:
+ using Assembler::sve_cpy;
+
// jdk.internal.util.ArraysSupport.vectorizedHashCode
address arrays_hashcode(Register ary, Register cnt, Register result, FloatRegister vdata0,
FloatRegister vdata1, FloatRegister vdata2, FloatRegister vdata3,
@@ -175,6 +177,9 @@
FloatRegister fsrc, FloatRegister vsrc,
unsigned vector_length_in_bytes, FloatRegister vtmp);
+ void neon_reduce_add_fp16(FloatRegister dst, FloatRegister fsrc, FloatRegister vsrc,
+ unsigned vector_length_in_bytes, FloatRegister vtmp);
+
void neon_reduce_logical(int opc, Register dst, BasicType bt, Register isrc,
FloatRegister vsrc, unsigned vector_length_in_bytes);
@@ -244,4 +249,8 @@
void vector_expand_sve(FloatRegister dst, FloatRegister src, PRegister pg,
FloatRegister tmp1, FloatRegister tmp2, BasicType bt,
int vector_length_in_bytes);
+
+ void sve_cpy(FloatRegister dst, SIMD_RegVariant T, PRegister pg, int imm8,
+ bool isMerge);
+ int vector_iota_entry_index(BasicType bt);
#endif // CPU_AARCH64_C2_MACROASSEMBLER_AARCH64_HPP
diff --git a/src/hotspot/cpu/aarch64/c2_globals_aarch64.hpp b/src/hotspot/cpu/aarch64/c2_globals_aarch64.hpp
index a0dea3643a1..192461d1a61 100644
--- a/src/hotspot/cpu/aarch64/c2_globals_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/c2_globals_aarch64.hpp
@@ -47,7 +47,6 @@ define_pd_global(intx, ConditionalMoveLimit, 3);
define_pd_global(intx, FreqInlineSize, 325);
define_pd_global(intx, MinJumpTableSize, 10);
define_pd_global(intx, InteriorEntryAlignment, 16);
-define_pd_global(intx, NewSizeThreadIncrease, ScaleForWordSize(4*K));
define_pd_global(intx, LoopUnrollLimit, 60);
define_pd_global(intx, LoopPercentProfileLimit, 10);
// InitialCodeCacheSize derived from specjbb2000 run.
@@ -75,9 +74,6 @@ define_pd_global(size_t, NonNMethodCodeHeapSize, 5*M );
define_pd_global(size_t, CodeCacheMinBlockLength, 6);
define_pd_global(size_t, CodeCacheMinimumUseSpace, 400*K);
-// Ergonomics related flags
-define_pd_global(bool, NeverActAsServerClassMachine, false);
-
define_pd_global(bool, TrapBasedRangeChecks, false); // Not needed.
#endif // CPU_AARCH64_C2_GLOBALS_AARCH64_HPP
diff --git a/src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp b/src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp
index 6fe3315014b..640cd495383 100644
--- a/src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp
@@ -89,16 +89,21 @@ void CompiledDirectCall::set_to_interpreted(const methodHandle& callee, address
NativeMovConstReg* method_holder
= nativeMovConstReg_at(stub + NativeInstruction::instruction_size);
+ // In AOT "production" run we have mixture of AOTed and normal JITed code.
+ // Static call stub in AOTed nmethod always has far jump.
+ // Normal JITed nmethod may have short or far jump depending on distance.
+ // Determine actual jump instruction we have in code.
+ address next_instr = method_holder->next_instruction_address();
+ bool is_general_jump = nativeInstruction_at(next_instr)->is_general_jump();
+
#ifdef ASSERT
- NativeJump* jump = MacroAssembler::codestub_branch_needs_far_jump()
- ? nativeGeneralJump_at(method_holder->next_instruction_address())
- : nativeJump_at(method_holder->next_instruction_address());
+ NativeJump* jump = is_general_jump ? nativeGeneralJump_at(next_instr) : nativeJump_at(next_instr);
verify_mt_safe(callee, entry, method_holder, jump);
#endif
// Update stub.
method_holder->set_data((intptr_t)callee());
- MacroAssembler::pd_patch_instruction(method_holder->next_instruction_address(), entry);
+ MacroAssembler::pd_patch_instruction(next_instr, entry);
ICache::invalidate_range(stub, to_interp_stub_size());
// Update jump to call.
set_destination_mt_safe(stub);
diff --git a/src/hotspot/cpu/aarch64/downcallLinker_aarch64.cpp b/src/hotspot/cpu/aarch64/downcallLinker_aarch64.cpp
index 65d448f908c..130d2949800 100644
--- a/src/hotspot/cpu/aarch64/downcallLinker_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/downcallLinker_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2019, Arm Limited. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -146,10 +146,10 @@ void DowncallLinker::StubGenerator::generate() {
bool should_save_return_value = !_needs_return_buffer;
RegSpiller out_reg_spiller(_output_registers);
- int spill_offset = -1;
+ int out_spill_offset = -1;
if (should_save_return_value) {
- spill_offset = 0;
+ out_spill_offset = 0;
// spill area can be shared with shadow space and out args,
// since they are only used before the call,
// and spill area is only used after.
@@ -174,6 +174,9 @@ void DowncallLinker::StubGenerator::generate() {
// FP-> | |
// |---------------------| = frame_bottom_offset = frame_size
// | (optional) |
+ // | in_reg_spiller area |
+ // |---------------------|
+ // | (optional) |
// | capture state buf |
// |---------------------| = StubLocations::CAPTURED_STATE_BUFFER
// | (optional) |
@@ -187,6 +190,19 @@ void DowncallLinker::StubGenerator::generate() {
GrowableArray out_regs = ForeignGlobals::replace_place_holders(_input_registers, locs);
ArgumentShuffle arg_shuffle(filtered_java_regs, out_regs, shuffle_reg);
+ // Need to spill for state capturing runtime call.
+ // The area spilled into is distinct from the capture state buffer.
+ RegSpiller in_reg_spiller(out_regs);
+ int in_spill_offset = -1;
+ if (_captured_state_mask != 0) {
+ // The spill area cannot be shared with the out_spill since
+ // spilling needs to happen before the call. Allocate a new
+ // region in the stack for this spill space.
+ in_spill_offset = allocated_frame_size;
+ allocated_frame_size += in_reg_spiller.spill_size_bytes();
+ }
+
+
#ifndef PRODUCT
LogTarget(Trace, foreign, downcall) lt;
if (lt.is_enabled()) {
@@ -228,6 +244,20 @@ void DowncallLinker::StubGenerator::generate() {
arg_shuffle.generate(_masm, shuffle_reg, 0, _abi._shadow_space_bytes);
__ block_comment("} argument shuffle");
+ if (_captured_state_mask != 0) {
+ assert(in_spill_offset != -1, "must be");
+ __ block_comment("{ load initial thread local");
+ in_reg_spiller.generate_spill(_masm, in_spill_offset);
+
+ // Copy the contents of the capture state buffer into thread local
+ __ ldr(c_rarg0, Address(sp, locs.data_offset(StubLocations::CAPTURED_STATE_BUFFER)));
+ __ movw(c_rarg1, _captured_state_mask);
+ __ rt_call(CAST_FROM_FN_PTR(address, DowncallLinker::capture_state_pre), tmp1);
+
+ in_reg_spiller.generate_fill(_masm, in_spill_offset);
+ __ block_comment("} load initial thread local");
+ }
+
__ blr(as_Register(locs.get(StubLocations::TARGET_ADDRESS)));
// this call is assumed not to have killed rthread
@@ -254,15 +284,15 @@ void DowncallLinker::StubGenerator::generate() {
__ block_comment("{ save thread local");
if (should_save_return_value) {
- out_reg_spiller.generate_spill(_masm, spill_offset);
+ out_reg_spiller.generate_spill(_masm, out_spill_offset);
}
__ ldr(c_rarg0, Address(sp, locs.data_offset(StubLocations::CAPTURED_STATE_BUFFER)));
__ movw(c_rarg1, _captured_state_mask);
- __ rt_call(CAST_FROM_FN_PTR(address, DowncallLinker::capture_state), tmp1);
+ __ rt_call(CAST_FROM_FN_PTR(address, DowncallLinker::capture_state_post), tmp1);
if (should_save_return_value) {
- out_reg_spiller.generate_fill(_masm, spill_offset);
+ out_reg_spiller.generate_fill(_masm, out_spill_offset);
}
__ block_comment("} save thread local");
@@ -321,7 +351,7 @@ void DowncallLinker::StubGenerator::generate() {
if (should_save_return_value) {
// Need to save the native result registers around any runtime calls.
- out_reg_spiller.generate_spill(_masm, spill_offset);
+ out_reg_spiller.generate_spill(_masm, out_spill_offset);
}
__ mov(c_rarg0, rthread);
@@ -330,7 +360,7 @@ void DowncallLinker::StubGenerator::generate() {
__ blr(tmp1);
if (should_save_return_value) {
- out_reg_spiller.generate_fill(_masm, spill_offset);
+ out_reg_spiller.generate_fill(_masm, out_spill_offset);
}
__ b(L_after_safepoint_poll);
@@ -342,13 +372,13 @@ void DowncallLinker::StubGenerator::generate() {
__ bind(L_reguard);
if (should_save_return_value) {
- out_reg_spiller.generate_spill(_masm, spill_offset);
+ out_reg_spiller.generate_spill(_masm, out_spill_offset);
}
__ rt_call(CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages), tmp1);
if (should_save_return_value) {
- out_reg_spiller.generate_fill(_masm, spill_offset);
+ out_reg_spiller.generate_fill(_masm, out_spill_offset);
}
__ b(L_after_reguard);
diff --git a/src/hotspot/cpu/aarch64/frame_aarch64.inline.hpp b/src/hotspot/cpu/aarch64/frame_aarch64.inline.hpp
index cb53d8663ad..748ab0e0e2b 100644
--- a/src/hotspot/cpu/aarch64/frame_aarch64.inline.hpp
+++ b/src/hotspot/cpu/aarch64/frame_aarch64.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -245,8 +245,8 @@ inline bool frame::equal(frame other) const {
// Return unique id for this frame. The id must have a value where we can distinguish
// identity and younger/older relationship. null represents an invalid (incomparable)
-// frame.
-inline intptr_t* frame::id(void) const { return unextended_sp(); }
+// frame. Should not be called for heap frames.
+inline intptr_t* frame::id(void) const { return real_fp(); }
// Return true if the frame is older (less recent activation) than the frame represented by id
inline bool frame::is_older(intptr_t* id) const { assert(this->id() != nullptr && id != nullptr, "null frame id");
@@ -412,6 +412,9 @@ inline frame frame::sender(RegisterMap* map) const {
StackWatermarkSet::on_iteration(map->thread(), result);
}
+ // Calling frame::id() is currently not supported for heap frames.
+ assert(result._on_heap || this->_on_heap || result.is_older(this->id()), "Must be");
+
return result;
}
diff --git a/src/hotspot/cpu/aarch64/gc/shared/cardTableBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shared/cardTableBarrierSetAssembler_aarch64.cpp
index 0bfc320179d..7ce4e0f8aed 100644
--- a/src/hotspot/cpu/aarch64/gc/shared/cardTableBarrierSetAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/gc/shared/cardTableBarrierSetAssembler_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -56,8 +56,10 @@ void CardTableBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet d
}
}
-void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj, Address dst) {
-
+void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2) {
+ precond(tmp1 != noreg);
+ precond(tmp2 != noreg);
+ assert_different_registers(obj, tmp1, tmp2);
BarrierSet* bs = BarrierSet::barrier_set();
assert(bs->kind() == BarrierSet::CardTableBarrierSet, "Wrong barrier set kind");
@@ -65,16 +67,16 @@ void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register ob
assert(CardTable::dirty_card_val() == 0, "must be");
- __ load_byte_map_base(rscratch1);
+ __ load_byte_map_base(tmp1);
if (UseCondCardMark) {
Label L_already_dirty;
- __ ldrb(rscratch2, Address(obj, rscratch1));
- __ cbz(rscratch2, L_already_dirty);
- __ strb(zr, Address(obj, rscratch1));
+ __ ldrb(tmp2, Address(obj, tmp1));
+ __ cbz(tmp2, L_already_dirty);
+ __ strb(zr, Address(obj, tmp1));
__ bind(L_already_dirty);
} else {
- __ strb(zr, Address(obj, rscratch1));
+ __ strb(zr, Address(obj, tmp1));
}
}
@@ -112,10 +114,10 @@ void CardTableBarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorS
if (needs_post_barrier) {
// flatten object address if needed
if (!precise || (dst.index() == noreg && dst.offset() == 0)) {
- store_check(masm, dst.base(), dst);
+ store_check(masm, dst.base(), tmp1, tmp2);
} else {
__ lea(tmp3, dst);
- store_check(masm, tmp3, dst);
+ store_check(masm, tmp3, tmp1, tmp2);
}
}
}
diff --git a/src/hotspot/cpu/aarch64/gc/shared/cardTableBarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/shared/cardTableBarrierSetAssembler_aarch64.hpp
index 07dd8eb5565..07016381f78 100644
--- a/src/hotspot/cpu/aarch64/gc/shared/cardTableBarrierSetAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/gc/shared/cardTableBarrierSetAssembler_aarch64.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -46,7 +46,7 @@ protected:
virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Address dst, Register val, Register tmp1, Register tmp2, Register tmp3);
- void store_check(MacroAssembler* masm, Register obj, Address dst);
+ void store_check(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2);
};
#endif // CPU_AARCH64_GC_SHARED_CARDTABLEBARRIERSETASSEMBLER_AARCH64_HPP
diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp
index 4f0977a414f..f0885fee93d 100644
--- a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp
@@ -879,7 +879,9 @@ void ZBarrierSetAssembler::patch_barrier_relocation(address addr, int format) {
ShouldNotReachHere();
}
- ICache::invalidate_word((address)patch_addr);
+ if (!UseSingleICacheInvalidation) {
+ ICache::invalidate_word((address)patch_addr);
+ }
}
#ifdef COMPILER1
diff --git a/src/hotspot/cpu/aarch64/globals_aarch64.hpp b/src/hotspot/cpu/aarch64/globals_aarch64.hpp
index a59e83c4b69..dfeba73bede 100644
--- a/src/hotspot/cpu/aarch64/globals_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/globals_aarch64.hpp
@@ -95,7 +95,7 @@ define_pd_global(intx, InlineSmallCode, 1000);
"Use simplest and shortest implementation for array equals") \
product(bool, UseSIMDForBigIntegerShiftIntrinsics, true, \
"Use SIMD instructions for left/right shift of BigInteger") \
- product(bool, UseSIMDForSHA3Intrinsic, true, \
+ product(bool, UseSIMDForSHA3Intrinsic, false, \
"Use SIMD SHA3 instructions for SHA3 intrinsic") \
product(bool, AvoidUnalignedAccesses, false, \
"Avoid generating unaligned memory accesses") \
@@ -115,18 +115,26 @@ define_pd_global(intx, InlineSmallCode, 1000);
"Value -1 means off.") \
range(-1, 4096) \
product(ccstr, OnSpinWaitInst, "yield", DIAGNOSTIC, \
- "The instruction to use to implement " \
- "java.lang.Thread.onSpinWait()." \
- "Valid values are: none, nop, isb, yield, sb.") \
+ "The instruction to use for java.lang.Thread.onSpinWait(). " \
+ "Valid values are: none, nop, isb, yield, sb, wfet.") \
constraint(OnSpinWaitInstNameConstraintFunc, AtParse) \
product(uint, OnSpinWaitInstCount, 1, DIAGNOSTIC, \
- "The number of OnSpinWaitInst instructions to generate." \
- "It cannot be used with OnSpinWaitInst=none.") \
+ "The number of OnSpinWaitInst instructions to generate. " \
+ "It cannot be used with OnSpinWaitInst=none. " \
+ "For OnSpinWaitInst=wfet it must be 1.") \
range(1, 99) \
+ product(uint, OnSpinWaitDelay, 40, DIAGNOSTIC, \
+ "The minimum delay (in nanoseconds) of the OnSpinWait loop. " \
+ "It can only be used with -XX:OnSpinWaitInst=wfet.") \
+ range(1, 1000) \
product(ccstr, UseBranchProtection, "none", \
"Branch Protection to use: none, standard, pac-ret") \
product(bool, AlwaysMergeDMB, true, DIAGNOSTIC, \
"Always merge DMB instructions in code emission") \
+ product(bool, NeoverseN1ICacheErratumMitigation, false, DIAGNOSTIC, \
+ "Enable workaround for Neoverse N1 erratum 1542419") \
+ product(bool, UseSingleICacheInvalidation, false, DIAGNOSTIC, \
+ "Defer multiple ICache invalidation to single invalidation") \
// end of ARCH_FLAGS
diff --git a/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp b/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp
index 2b506b241e0..980fedb406d 100644
--- a/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -989,26 +989,15 @@ void InterpreterMacroAssembler::profile_final_call(Register mdp) {
void InterpreterMacroAssembler::profile_virtual_call(Register receiver,
- Register mdp,
- bool receiver_can_be_null) {
+ Register mdp) {
if (ProfileInterpreter) {
Label profile_continue;
// If no method data exists, go to profile_continue.
test_method_data_pointer(mdp, profile_continue);
- Label skip_receiver_profile;
- if (receiver_can_be_null) {
- Label not_null;
- // We are making a call. Increment the count for null receiver.
- increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
- b(skip_receiver_profile);
- bind(not_null);
- }
-
// Record the receiver type.
profile_receiver_type(receiver, mdp, 0);
- bind(skip_receiver_profile);
// The method data pointer needs to be updated to reflect the new target.
update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size()));
diff --git a/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp b/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp
index 74d4430000d..9a074f1ce69 100644
--- a/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2015, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -285,8 +285,7 @@ class InterpreterMacroAssembler: public MacroAssembler {
void profile_not_taken_branch(Register mdp);
void profile_call(Register mdp);
void profile_final_call(Register mdp);
- void profile_virtual_call(Register receiver, Register mdp,
- bool receiver_can_be_null = false);
+ void profile_virtual_call(Register receiver, Register mdp);
void profile_ret(Register return_bci, Register mdp);
void profile_null_seen(Register mdp);
void profile_typecheck(Register mdp, Register klass);
diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
index 3e3e95be07e..7bec0a3c0ca 100644
--- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
@@ -55,6 +55,7 @@
#include "runtime/sharedRuntime.hpp"
#include "runtime/stubRoutines.hpp"
#include "utilities/globalDefinitions.hpp"
+#include "utilities/integerCast.hpp"
#include "utilities/powerOfTwo.hpp"
#ifdef COMPILER1
#include "c1/c1_LIRAssembler.hpp"
@@ -762,7 +763,7 @@ void MacroAssembler::call_VM_base(Register oop_result,
assert(java_thread == rthread, "unexpected register");
#ifdef ASSERT
// TraceBytecodes does not use r12 but saves it over the call, so don't verify
- // if ((UseCompressedOops || UseCompressedClassPointers) && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");
+ // if (!TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");
#endif // ASSERT
assert(java_thread != oop_result , "cannot use the same register for java_thread & oop_result");
@@ -952,7 +953,10 @@ void MacroAssembler::emit_static_call_stub() {
}
int MacroAssembler::static_call_stub_size() {
- if (!codestub_branch_needs_far_jump()) {
+ // During AOT production run AOT and JIT compiled code
+ // are used at the same time. We need this size
+ // to be the same for both types of code.
+ if (!codestub_branch_needs_far_jump() && !AOTCodeCache::is_on_for_use()) {
// isb; movk; movz; movz; b
return 5 * NativeInstruction::instruction_size;
}
@@ -1002,14 +1006,10 @@ int MacroAssembler::ic_check(int end_alignment) {
load_narrow_klass_compact(tmp1, receiver);
ldrw(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
cmpw(tmp1, tmp2);
- } else if (UseCompressedClassPointers) {
+ } else {
ldrw(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes()));
ldrw(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
cmpw(tmp1, tmp2);
- } else {
- ldr(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes()));
- ldr(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
- cmp(tmp1, tmp2);
}
Label dont;
@@ -2917,7 +2917,11 @@ void MacroAssembler::increment(Address dst, int value)
// Push lots of registers in the bit set supplied. Don't push sp.
// Return the number of words pushed
-int MacroAssembler::push(unsigned int bitset, Register stack) {
+int MacroAssembler::push(RegSet regset, Register stack) {
+ if (regset.bits() == 0) {
+ return 0;
+ }
+ auto bitset = integer_cast(regset.bits());
int words_pushed = 0;
// Scan bitset to accumulate register pairs
@@ -2947,7 +2951,11 @@ int MacroAssembler::push(unsigned int bitset, Register stack) {
return count;
}
-int MacroAssembler::pop(unsigned int bitset, Register stack) {
+int MacroAssembler::pop(RegSet regset, Register stack) {
+ if (regset.bits() == 0) {
+ return 0;
+ }
+ auto bitset = integer_cast(regset.bits());
int words_pushed = 0;
// Scan bitset to accumulate register pairs
@@ -2979,7 +2987,11 @@ int MacroAssembler::pop(unsigned int bitset, Register stack) {
// Push lots of registers in the bit set supplied. Don't push sp.
// Return the number of dwords pushed
-int MacroAssembler::push_fp(unsigned int bitset, Register stack, FpPushPopMode mode) {
+int MacroAssembler::push_fp(FloatRegSet regset, Register stack, FpPushPopMode mode) {
+ if (regset.bits() == 0) {
+ return 0;
+ }
+ auto bitset = integer_cast(regset.bits());
int words_pushed = 0;
bool use_sve = false;
int sve_vector_size_in_bytes = 0;
@@ -3092,7 +3104,11 @@ int MacroAssembler::push_fp(unsigned int bitset, Register stack, FpPushPopMode m
}
// Return the number of dwords popped
-int MacroAssembler::pop_fp(unsigned int bitset, Register stack, FpPushPopMode mode) {
+int MacroAssembler::pop_fp(FloatRegSet regset, Register stack, FpPushPopMode mode) {
+ if (regset.bits() == 0) {
+ return 0;
+ }
+ auto bitset = integer_cast(regset.bits());
int words_pushed = 0;
bool use_sve = false;
int sve_vector_size_in_bytes = 0;
@@ -3202,7 +3218,11 @@ int MacroAssembler::pop_fp(unsigned int bitset, Register stack, FpPushPopMode mo
}
// Return the number of dwords pushed
-int MacroAssembler::push_p(unsigned int bitset, Register stack) {
+int MacroAssembler::push_p(PRegSet regset, Register stack) {
+ if (regset.bits() == 0) {
+ return 0;
+ }
+ auto bitset = integer_cast(regset.bits());
bool use_sve = false;
int sve_predicate_size_in_slots = 0;
@@ -3239,7 +3259,11 @@ int MacroAssembler::push_p(unsigned int bitset, Register stack) {
}
// Return the number of dwords popped
-int MacroAssembler::pop_p(unsigned int bitset, Register stack) {
+int MacroAssembler::pop_p(PRegSet regset, Register stack) {
+ if (regset.bits() == 0) {
+ return 0;
+ }
+ auto bitset = integer_cast(regset.bits());
bool use_sve = false;
int sve_predicate_size_in_slots = 0;
@@ -3278,7 +3302,6 @@ int MacroAssembler::pop_p(unsigned int bitset, Register stack) {
#ifdef ASSERT
void MacroAssembler::verify_heapbase(const char* msg) {
#if 0
- assert (UseCompressedOops || UseCompressedClassPointers, "should be compressed");
assert (Universe::heap() != nullptr, "java heap should be initialized");
if (!UseCompressedOops || Universe::ptr_base() == nullptr) {
// rheapbase is allocated as general register
@@ -3456,7 +3479,7 @@ void MacroAssembler::subw(Register Rd, Register Rn, RegisterOrConstant decrement
void MacroAssembler::reinit_heapbase()
{
if (UseCompressedOops) {
- if (Universe::is_fully_initialized()) {
+ if (Universe::is_fully_initialized() && !AOTCodeCache::is_on_for_dump()) {
mov(rheapbase, CompressedOops::base());
} else {
lea(rheapbase, ExternalAddress(CompressedOops::base_addr()));
@@ -5067,13 +5090,10 @@ void MacroAssembler::load_narrow_klass_compact(Register dst, Register src) {
void MacroAssembler::load_klass(Register dst, Register src) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(dst, src);
- decode_klass_not_null(dst);
- } else if (UseCompressedClassPointers) {
- ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
- decode_klass_not_null(dst);
} else {
- ldr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
+ ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
}
+ decode_klass_not_null(dst);
}
void MacroAssembler::restore_cpu_control_state_after_jni(Register tmp1, Register tmp2) {
@@ -5125,25 +5145,22 @@ void MacroAssembler::load_mirror(Register dst, Register method, Register tmp1, R
void MacroAssembler::cmp_klass(Register obj, Register klass, Register tmp) {
assert_different_registers(obj, klass, tmp);
- if (UseCompressedClassPointers) {
- if (UseCompactObjectHeaders) {
- load_narrow_klass_compact(tmp, obj);
- } else {
- ldrw(tmp, Address(obj, oopDesc::klass_offset_in_bytes()));
- }
- if (CompressedKlassPointers::base() == nullptr) {
- cmp(klass, tmp, LSL, CompressedKlassPointers::shift());
- return;
- } else if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0
- && CompressedKlassPointers::shift() == 0) {
- // Only the bottom 32 bits matter
- cmpw(klass, tmp);
- return;
- }
- decode_klass_not_null(tmp);
+ if (UseCompactObjectHeaders) {
+ load_narrow_klass_compact(tmp, obj);
} else {
- ldr(tmp, Address(obj, oopDesc::klass_offset_in_bytes()));
+ ldrw(tmp, Address(obj, oopDesc::klass_offset_in_bytes()));
}
+ if (CompressedKlassPointers::base() == nullptr) {
+ cmp(klass, tmp, LSL, CompressedKlassPointers::shift());
+ return;
+ } else if (!AOTCodeCache::is_on_for_dump() &&
+ ((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0
+ && CompressedKlassPointers::shift() == 0) {
+ // Only the bottom 32 bits matter
+ cmpw(klass, tmp);
+ return;
+ }
+ decode_klass_not_null(tmp);
cmp(klass, tmp);
}
@@ -5151,36 +5168,25 @@ void MacroAssembler::cmp_klasses_from_objects(Register obj1, Register obj2, Regi
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(tmp1, obj1);
load_narrow_klass_compact(tmp2, obj2);
- cmpw(tmp1, tmp2);
- } else if (UseCompressedClassPointers) {
+ } else {
ldrw(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
ldrw(tmp2, Address(obj2, oopDesc::klass_offset_in_bytes()));
- cmpw(tmp1, tmp2);
- } else {
- ldr(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
- ldr(tmp2, Address(obj2, oopDesc::klass_offset_in_bytes()));
- cmp(tmp1, tmp2);
}
+ cmpw(tmp1, tmp2);
}
void MacroAssembler::store_klass(Register dst, Register src) {
// FIXME: Should this be a store release? concurrent gcs assumes
// klass length is valid if klass field is not null.
assert(!UseCompactObjectHeaders, "not with compact headers");
- if (UseCompressedClassPointers) {
- encode_klass_not_null(src);
- strw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
- } else {
- str(src, Address(dst, oopDesc::klass_offset_in_bytes()));
- }
+ encode_klass_not_null(src);
+ strw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
}
void MacroAssembler::store_klass_gap(Register dst, Register src) {
assert(!UseCompactObjectHeaders, "not with compact headers");
- if (UseCompressedClassPointers) {
- // Store to klass gap in destination
- strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
- }
+ // Store to klass gap in destination
+ strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
}
// Algorithm must match CompressedOops::encode.
@@ -5326,8 +5332,6 @@ MacroAssembler::KlassDecodeMode MacroAssembler::klass_decode_mode() {
}
MacroAssembler::KlassDecodeMode MacroAssembler::klass_decode_mode(address base, int shift, const size_t range) {
- assert(UseCompressedClassPointers, "not using compressed class pointers");
-
// KlassDecodeMode shouldn't be set already.
assert(_klass_decode_mode == KlassDecodeNone, "set once");
@@ -5393,7 +5397,7 @@ void MacroAssembler::encode_klass_not_null_for_aot(Register dst, Register src) {
}
void MacroAssembler::encode_klass_not_null(Register dst, Register src) {
- if (AOTCodeCache::is_on_for_dump()) {
+ if (CompressedKlassPointers::base() != nullptr && AOTCodeCache::is_on_for_dump()) {
encode_klass_not_null_for_aot(dst, src);
return;
}
@@ -5457,8 +5461,6 @@ void MacroAssembler::decode_klass_not_null_for_aot(Register dst, Register src) {
}
void MacroAssembler::decode_klass_not_null(Register dst, Register src) {
- assert (UseCompressedClassPointers, "should only be used for compressed headers");
-
if (AOTCodeCache::is_on_for_dump()) {
decode_klass_not_null_for_aot(dst, src);
return;
@@ -5525,7 +5527,6 @@ void MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
}
void MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
- assert (UseCompressedClassPointers, "should only be used for compressed headers");
assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
int index = oop_recorder()->find_index(k);
@@ -6835,6 +6836,9 @@ void MacroAssembler::spin_wait() {
assert(VM_Version::supports_sb(), "current CPU does not support SB instruction");
sb();
break;
+ case SpinWait::WFET:
+ spin_wait_wfet(VM_Version::spin_wait_desc().delay());
+ break;
default:
ShouldNotReachHere();
}
@@ -6842,6 +6846,28 @@ void MacroAssembler::spin_wait() {
block_comment("}");
}
+void MacroAssembler::spin_wait_wfet(int delay_ns) {
+ // The sequence assumes CNTFRQ_EL0 is fixed to 1GHz. The assumption is valid
+ // starting from Armv8.6, according to the "D12.1.2 The system counter" of the
+ // Arm Architecture Reference Manual for A-profile architecture version M.a.a.
+ // This is sufficient because FEAT_WFXT is introduced from Armv8.6.
+ Register target = rscratch1;
+ Register current = rscratch2;
+ get_cntvctss_el0(current);
+ add(target, current, delay_ns);
+
+ Label L_wait_loop;
+ bind(L_wait_loop);
+
+ wfet(target);
+ get_cntvctss_el0(current);
+
+ cmp(current, target);
+ br(LT, L_wait_loop);
+
+ sb();
+}
+
// Stack frame creation/removal
void MacroAssembler::enter(bool strip_ret_addr) {
diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
index fa32f3055b9..a6cc862d05c 100644
--- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
@@ -499,29 +499,20 @@ private:
void mov_immediate64(Register dst, uint64_t imm64);
void mov_immediate32(Register dst, uint32_t imm32);
- int push(unsigned int bitset, Register stack);
- int pop(unsigned int bitset, Register stack);
-
- int push_fp(unsigned int bitset, Register stack, FpPushPopMode mode);
- int pop_fp(unsigned int bitset, Register stack, FpPushPopMode mode);
-
- int push_p(unsigned int bitset, Register stack);
- int pop_p(unsigned int bitset, Register stack);
-
void mov(Register dst, Address a);
public:
- void push(RegSet regs, Register stack) { if (regs.bits()) push(regs.bits(), stack); }
- void pop(RegSet regs, Register stack) { if (regs.bits()) pop(regs.bits(), stack); }
+ int push(RegSet regset, Register stack);
+ int pop(RegSet regset, Register stack);
- void push_fp(FloatRegSet regs, Register stack, FpPushPopMode mode = PushPopFull) { if (regs.bits()) push_fp(regs.bits(), stack, mode); }
- void pop_fp(FloatRegSet regs, Register stack, FpPushPopMode mode = PushPopFull) { if (regs.bits()) pop_fp(regs.bits(), stack, mode); }
+ int push_fp(FloatRegSet regset, Register stack, FpPushPopMode mode = PushPopFull);
+ int pop_fp(FloatRegSet regset, Register stack, FpPushPopMode mode = PushPopFull);
static RegSet call_clobbered_gp_registers();
- void push_p(PRegSet regs, Register stack) { if (regs.bits()) push_p(regs.bits(), stack); }
- void pop_p(PRegSet regs, Register stack) { if (regs.bits()) pop_p(regs.bits(), stack); }
+ int push_p(PRegSet regset, Register stack);
+ int pop_p(PRegSet regset, Register stack);
// Push and pop everything that might be clobbered by a native
// runtime call except rscratch1 and rscratch2. (They are always
@@ -660,6 +651,14 @@ public:
msr(0b011, 0b0100, 0b0010, 0b000, reg);
}
+ // CNTVCTSS_EL0: op1 == 011
+ // CRn == 1110
+ // CRm == 0000
+ // op2 == 110
+ inline void get_cntvctss_el0(Register reg) {
+ mrs(0b011, 0b1110, 0b0000, 0b110, reg);
+ }
+
// idiv variant which deals with MINLONG as dividend and -1 as divisor
int corrected_idivl(Register result, Register ra, Register rb,
bool want_remainder, Register tmp = rscratch1);
@@ -891,10 +890,6 @@ public:
// thread in the default location (rthread)
void reset_last_Java_frame(bool clear_fp);
- // Stores
- void store_check(Register obj); // store check for obj - register is destroyed afterwards
- void store_check(Register obj, Address dst); // same as above, dst is exact store location (reg. is destroyed)
-
void resolve_jobject(Register value, Register tmp1, Register tmp2);
void resolve_global_jobject(Register value, Register tmp1, Register tmp2);
@@ -1724,6 +1719,7 @@ public:
// Code for java.lang.Thread::onSpinWait() intrinsic.
void spin_wait();
+ void spin_wait_wfet(int delay_ns);
void fast_lock(Register basic_lock, Register obj, Register t1, Register t2, Register t3, Label& slow);
void fast_unlock(Register obj, Register t1, Register t2, Register t3, Label& slow);
diff --git a/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp b/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp
index fc7274714ad..ab9896fa426 100644
--- a/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp
@@ -97,7 +97,7 @@ protected:
#define MACOS_WX_WRITE MACOS_AARCH64_ONLY(os::thread_wx_enable_write())
void set_char_at(int offset, char c) { MACOS_WX_WRITE; *addr_at(offset) = (u_char)c; }
void set_int_at(int offset, jint i) { MACOS_WX_WRITE; *(jint*)addr_at(offset) = i; }
- void set_uint_at(int offset, jint i) { MACOS_WX_WRITE; *(juint*)addr_at(offset) = i; }
+ void set_uint_at(int offset, juint i) { MACOS_WX_WRITE; *(juint*)addr_at(offset) = i; }
void set_ptr_at(int offset, address ptr) { MACOS_WX_WRITE; *(address*)addr_at(offset) = ptr; }
void set_oop_at(int offset, oop o) { MACOS_WX_WRITE; *(oop*)addr_at(offset) = o; }
#undef MACOS_WX_WRITE
@@ -178,13 +178,11 @@ public:
address destination() const;
void set_destination(address dest) {
- int offset = dest - instruction_address();
- unsigned int insn = 0b100101 << 26;
+ int64_t offset = dest - instruction_address();
+ juint insn = 0b100101u << 26u;
assert((offset & 3) == 0, "should be");
- offset >>= 2;
- offset &= (1 << 26) - 1; // mask off insn part
- insn |= offset;
- set_int_at(displacement_offset, insn);
+ Instruction_aarch64::spatch(reinterpret_cast(&insn), 25, 0, offset >> 2);
+ set_uint_at(displacement_offset, insn);
}
void verify_alignment() { ; }
diff --git a/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp b/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp
index dbec2d76d4f..f1b9fb213a2 100644
--- a/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp
@@ -54,7 +54,12 @@ void Relocation::pd_set_data_value(address x, bool verify_only) {
bytes = MacroAssembler::pd_patch_instruction_size(addr(), x);
break;
}
- ICache::invalidate_range(addr(), bytes);
+
+ if (UseSingleICacheInvalidation) {
+ assert(_binding != nullptr, "expect to be called with RelocIterator in use");
+ } else {
+ ICache::invalidate_range(addr(), bytes);
+ }
}
address Relocation::pd_call_destination(address orig_addr) {
diff --git a/src/hotspot/cpu/aarch64/runtime_aarch64.cpp b/src/hotspot/cpu/aarch64/runtime_aarch64.cpp
index e36aa21b567..638e57b03fe 100644
--- a/src/hotspot/cpu/aarch64/runtime_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/runtime_aarch64.cpp
@@ -290,7 +290,7 @@ ExceptionBlob* OptoRuntime::generate_exception_blob() {
assert(SimpleRuntimeFrame::framesize % 4 == 0, "sp not 16-byte aligned");
const char* name = OptoRuntime::stub_name(StubId::c2_exception_id);
- CodeBlob* blob = AOTCodeCache::load_code_blob(AOTCodeEntry::C2Blob, (uint)BlobId::c2_exception_id, name);
+ CodeBlob* blob = AOTCodeCache::load_code_blob(AOTCodeEntry::C2Blob, BlobId::c2_exception_id);
if (blob != nullptr) {
return blob->as_exception_blob();
}
diff --git a/src/hotspot/cpu/aarch64/spin_wait_aarch64.cpp b/src/hotspot/cpu/aarch64/spin_wait_aarch64.cpp
index 7da0151d834..97a981ab815 100644
--- a/src/hotspot/cpu/aarch64/spin_wait_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/spin_wait_aarch64.cpp
@@ -32,6 +32,7 @@ bool SpinWait::supports(const char *name) {
strcmp(name, "isb") == 0 ||
strcmp(name, "yield") == 0 ||
strcmp(name, "sb") == 0 ||
+ strcmp(name, "wfet") == 0 ||
strcmp(name, "none") == 0);
}
@@ -46,6 +47,8 @@ SpinWait::Inst SpinWait::from_name(const char* name) {
return SpinWait::YIELD;
} else if (strcmp(name, "sb") == 0) {
return SpinWait::SB;
+ } else if (strcmp(name, "wfet") == 0) {
+ return SpinWait::WFET;
}
return SpinWait::NONE;
diff --git a/src/hotspot/cpu/aarch64/spin_wait_aarch64.hpp b/src/hotspot/cpu/aarch64/spin_wait_aarch64.hpp
index 0e96a4b7157..6ebcd2477a8 100644
--- a/src/hotspot/cpu/aarch64/spin_wait_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/spin_wait_aarch64.hpp
@@ -24,6 +24,8 @@
#ifndef CPU_AARCH64_SPIN_WAIT_AARCH64_HPP
#define CPU_AARCH64_SPIN_WAIT_AARCH64_HPP
+#include "utilities/debug.hpp"
+
class SpinWait {
public:
enum Inst {
@@ -31,21 +33,30 @@ public:
NOP,
ISB,
YIELD,
- SB
+ SB,
+ WFET
};
private:
Inst _inst;
int _count;
+ int _delay;
Inst from_name(const char *name);
public:
- SpinWait(Inst inst = NONE, int count = 0) : _inst(inst), _count(inst == NONE ? 0 : count) {}
- SpinWait(const char *name, int count) : SpinWait(from_name(name), count) {}
+ SpinWait(Inst inst = NONE, int count = 0, int delay = -1)
+ : _inst(inst), _count(inst == NONE ? 0 : count), _delay(delay) {}
+ SpinWait(const char *name, int count, int delay)
+ : SpinWait(from_name(name), count, delay) {}
Inst inst() const { return _inst; }
int inst_count() const { return _count; }
+ int delay() const {
+ assert(_inst == WFET, "Specifying the delay value is only supported for WFET");
+ assert(_delay > 0, "The delay value must be positive");
+ return _delay;
+ }
static bool supports(const char *name);
};
diff --git a/src/hotspot/cpu/aarch64/stubDeclarations_aarch64.hpp b/src/hotspot/cpu/aarch64/stubDeclarations_aarch64.hpp
index 695534604b8..d1f59e479db 100644
--- a/src/hotspot/cpu/aarch64/stubDeclarations_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/stubDeclarations_aarch64.hpp
@@ -29,32 +29,39 @@
#define STUBGEN_PREUNIVERSE_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(preuniverse, 0) \
#define STUBGEN_INITIAL_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(initial, 10000) \
#define STUBGEN_CONTINUATION_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(continuation, 2000) \
+// count needed for declaration of vector_iota_indices stub
+#define VECTOR_IOTA_COUNT 6
#define STUBGEN_COMPILER_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(compiler, 70000) \
do_stub(compiler, vector_iota_indices) \
- do_arch_entry(aarch64, compiler, vector_iota_indices, \
- vector_iota_indices, vector_iota_indices) \
+ do_arch_entry_array(aarch64, compiler, vector_iota_indices, \
+ vector_iota_indices, vector_iota_indices, \
+ VECTOR_IOTA_COUNT) \
do_stub(compiler, large_array_equals) \
do_arch_entry(aarch64, compiler, large_array_equals, \
large_array_equals, large_array_equals) \
@@ -84,8 +91,7 @@
do_stub(compiler, count_positives) \
do_arch_entry(aarch64, compiler, count_positives, count_positives, \
count_positives) \
- do_stub(compiler, count_positives_long) \
- do_arch_entry(aarch64, compiler, count_positives_long, \
+ do_arch_entry(aarch64, compiler, count_positives, \
count_positives_long, count_positives_long) \
do_stub(compiler, compare_long_string_LL) \
do_arch_entry(aarch64, compiler, compare_long_string_LL, \
@@ -108,14 +114,16 @@
do_stub(compiler, string_indexof_linear_ul) \
do_arch_entry(aarch64, compiler, string_indexof_linear_ul, \
string_indexof_linear_ul, string_indexof_linear_ul) \
- /* this uses the entry for ghash_processBlocks */ \
- do_stub(compiler, ghash_processBlocks_wide) \
+ do_stub(compiler, ghash_processBlocks_small) \
+ do_arch_entry(aarch64, compiler, ghash_processBlocks_small, \
+ ghash_processBlocks_small, ghash_processBlocks_small) \
#define STUBGEN_FINAL_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(final, 20000 ZGC_ONLY(+85000)) \
do_stub(final, copy_byte_f) \
do_arch_entry(aarch64, final, copy_byte_f, copy_byte_f, \
@@ -139,9 +147,49 @@
do_stub(final, spin_wait) \
do_arch_entry_init(aarch64, final, spin_wait, spin_wait, \
spin_wait, empty_spin_wait) \
- /* stub only -- entries are not stored in StubRoutines::aarch64 */ \
/* n.b. these are not the same as the generic atomic stubs */ \
do_stub(final, atomic_entry_points) \
+ do_arch_entry(aarch64, final, atomic_entry_points, \
+ atomic_fetch_add_4_impl, atomic_fetch_add_4_impl) \
+ do_arch_entry(aarch64, final, atomic_entry_points, \
+ atomic_fetch_add_8_impl, atomic_fetch_add_8_impl) \
+ do_arch_entry(aarch64, final, atomic_entry_points, \
+ atomic_fetch_add_4_relaxed_impl, \
+ atomic_fetch_add_4_relaxed_impl) \
+ do_arch_entry(aarch64, final, atomic_entry_points, \
+ atomic_fetch_add_8_relaxed_impl, \
+ atomic_fetch_add_8_relaxed_impl) \
+ do_arch_entry(aarch64, final, atomic_entry_points, \
+ atomic_xchg_4_impl, atomic_xchg_4_impl) \
+ do_arch_entry(aarch64, final, atomic_entry_points, \
+ atomic_xchg_8_impl, atomic_xchg_8_impl) \
+ do_arch_entry(aarch64, final, atomic_entry_points, \
+ atomic_cmpxchg_1_impl, atomic_cmpxchg_1_impl) \
+ do_arch_entry(aarch64, final, atomic_entry_points, \
+ atomic_cmpxchg_4_impl, atomic_cmpxchg_4_impl) \
+ do_arch_entry(aarch64, final, atomic_entry_points, \
+ atomic_cmpxchg_8_impl, atomic_cmpxchg_8_impl) \
+ do_arch_entry(aarch64, final, atomic_entry_points, \
+ atomic_cmpxchg_1_relaxed_impl, \
+ atomic_cmpxchg_1_relaxed_impl) \
+ do_arch_entry(aarch64, final, atomic_entry_points, \
+ atomic_cmpxchg_4_relaxed_impl, \
+ atomic_cmpxchg_4_relaxed_impl) \
+ do_arch_entry(aarch64, final, atomic_entry_points, \
+ atomic_cmpxchg_8_relaxed_impl, \
+ atomic_cmpxchg_8_relaxed_impl) \
+ do_arch_entry(aarch64, final, atomic_entry_points, \
+ atomic_cmpxchg_4_release_impl, \
+ atomic_cmpxchg_4_release_impl) \
+ do_arch_entry(aarch64, final, atomic_entry_points, \
+ atomic_cmpxchg_8_release_impl, \
+ atomic_cmpxchg_8_release_impl) \
+ do_arch_entry(aarch64, final, atomic_entry_points, \
+ atomic_cmpxchg_4_seq_cst_impl, \
+ atomic_cmpxchg_4_seq_cst_impl) \
+ do_arch_entry(aarch64, final, atomic_entry_points, \
+ atomic_cmpxchg_8_seq_cst_impl, \
+ atomic_cmpxchg_8_seq_cst_impl) \
#endif // CPU_AARCH64_STUBDECLARATIONS_HPP
diff --git a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp
index a459a28b09e..fddb37b7b8d 100644
--- a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp
@@ -79,6 +79,166 @@
#define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
+// Constant data definitions
+
+static const uint32_t _sha256_round_consts[64] = {
+ 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5,
+ 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
+ 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
+ 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
+ 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc,
+ 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
+ 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
+ 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
+ 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
+ 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
+ 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3,
+ 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
+ 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5,
+ 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
+ 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
+ 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2,
+};
+
+static const uint64_t _sha512_round_consts[80] = {
+ 0x428A2F98D728AE22L, 0x7137449123EF65CDL, 0xB5C0FBCFEC4D3B2FL,
+ 0xE9B5DBA58189DBBCL, 0x3956C25BF348B538L, 0x59F111F1B605D019L,
+ 0x923F82A4AF194F9BL, 0xAB1C5ED5DA6D8118L, 0xD807AA98A3030242L,
+ 0x12835B0145706FBEL, 0x243185BE4EE4B28CL, 0x550C7DC3D5FFB4E2L,
+ 0x72BE5D74F27B896FL, 0x80DEB1FE3B1696B1L, 0x9BDC06A725C71235L,
+ 0xC19BF174CF692694L, 0xE49B69C19EF14AD2L, 0xEFBE4786384F25E3L,
+ 0x0FC19DC68B8CD5B5L, 0x240CA1CC77AC9C65L, 0x2DE92C6F592B0275L,
+ 0x4A7484AA6EA6E483L, 0x5CB0A9DCBD41FBD4L, 0x76F988DA831153B5L,
+ 0x983E5152EE66DFABL, 0xA831C66D2DB43210L, 0xB00327C898FB213FL,
+ 0xBF597FC7BEEF0EE4L, 0xC6E00BF33DA88FC2L, 0xD5A79147930AA725L,
+ 0x06CA6351E003826FL, 0x142929670A0E6E70L, 0x27B70A8546D22FFCL,
+ 0x2E1B21385C26C926L, 0x4D2C6DFC5AC42AEDL, 0x53380D139D95B3DFL,
+ 0x650A73548BAF63DEL, 0x766A0ABB3C77B2A8L, 0x81C2C92E47EDAEE6L,
+ 0x92722C851482353BL, 0xA2BFE8A14CF10364L, 0xA81A664BBC423001L,
+ 0xC24B8B70D0F89791L, 0xC76C51A30654BE30L, 0xD192E819D6EF5218L,
+ 0xD69906245565A910L, 0xF40E35855771202AL, 0x106AA07032BBD1B8L,
+ 0x19A4C116B8D2D0C8L, 0x1E376C085141AB53L, 0x2748774CDF8EEB99L,
+ 0x34B0BCB5E19B48A8L, 0x391C0CB3C5C95A63L, 0x4ED8AA4AE3418ACBL,
+ 0x5B9CCA4F7763E373L, 0x682E6FF3D6B2B8A3L, 0x748F82EE5DEFB2FCL,
+ 0x78A5636F43172F60L, 0x84C87814A1F0AB72L, 0x8CC702081A6439ECL,
+ 0x90BEFFFA23631E28L, 0xA4506CEBDE82BDE9L, 0xBEF9A3F7B2C67915L,
+ 0xC67178F2E372532BL, 0xCA273ECEEA26619CL, 0xD186B8C721C0C207L,
+ 0xEADA7DD6CDE0EB1EL, 0xF57D4F7FEE6ED178L, 0x06F067AA72176FBAL,
+ 0x0A637DC5A2C898A6L, 0x113F9804BEF90DAEL, 0x1B710B35131C471BL,
+ 0x28DB77F523047D84L, 0x32CAAB7B40C72493L, 0x3C9EBE0A15C9BEBCL,
+ 0x431D67C49C100D4CL, 0x4CC5D4BECB3E42B6L, 0x597F299CFC657E2AL,
+ 0x5FCB6FAB3AD6FAECL, 0x6C44198C4A475817L
+};
+
+static const uint64_t _sha3_round_consts[24] = {
+ 0x0000000000000001L, 0x0000000000008082L, 0x800000000000808AL,
+ 0x8000000080008000L, 0x000000000000808BL, 0x0000000080000001L,
+ 0x8000000080008081L, 0x8000000000008009L, 0x000000000000008AL,
+ 0x0000000000000088L, 0x0000000080008009L, 0x000000008000000AL,
+ 0x000000008000808BL, 0x800000000000008BL, 0x8000000000008089L,
+ 0x8000000000008003L, 0x8000000000008002L, 0x8000000000000080L,
+ 0x000000000000800AL, 0x800000008000000AL, 0x8000000080008081L,
+ 0x8000000000008080L, 0x0000000080000001L, 0x8000000080008008L
+};
+
+static const uint64_t _double_keccak_round_consts[24] = {
+ 0x0000000000000001L, 0x0000000000008082L, 0x800000000000808AL,
+ 0x8000000080008000L, 0x000000000000808BL, 0x0000000080000001L,
+ 0x8000000080008081L, 0x8000000000008009L, 0x000000000000008AL,
+ 0x0000000000000088L, 0x0000000080008009L, 0x000000008000000AL,
+ 0x000000008000808BL, 0x800000000000008BL, 0x8000000000008089L,
+ 0x8000000000008003L, 0x8000000000008002L, 0x8000000000000080L,
+ 0x000000000000800AL, 0x800000008000000AL, 0x8000000080008081L,
+ 0x8000000000008080L, 0x0000000080000001L, 0x8000000080008008L
+};
+
+static const char _encodeBlock_toBase64[64] = {
+ 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
+ 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
+ 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
+ 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
+ '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'
+};
+
+static const char _encodeBlock_toBase64URL[64] = {
+ 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
+ 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
+ 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
+ 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
+ '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_'
+};
+
+// Non-SIMD lookup tables are mostly dumped from fromBase64 array used in java.util.Base64,
+// except the trailing character '=' is also treated illegal value in this intrinsic. That
+// is java.util.Base64.fromBase64['='] = -2, while fromBase(URL)64ForNoSIMD['='] = 255 here.
+static const uint8_t _decodeBlock_fromBase64ForNoSIMD[256] = {
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, 255u, 63u,
+ 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, 14u,
+ 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, 40u,
+ 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+};
+
+static const uint8_t _decodeBlock_fromBase64URLForNoSIMD[256] = {
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u,
+ 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, 14u,
+ 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, 63u,
+ 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, 40u,
+ 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+};
+
+// A legal value of base64 code is in range [0, 127]. We need two lookups
+// with tbl/tbx and combine them to get the decode data. The 1st table vector
+// lookup use tbl, out of range indices are set to 0 in destination. The 2nd
+// table vector lookup use tbx, out of range indices are unchanged in
+// destination. Input [64..126] is mapped to index [65, 127] in second lookup.
+// The value of index 64 is set to 0, so that we know that we already get the
+// decoded data with the 1st lookup.
+static const uint8_t _decodeBlock_fromBase64ForSIMD[128] = {
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, 255u, 63u,
+ 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 0u, 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u,
+ 14u, 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u,
+ 40u, 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u,
+};
+
+static const uint8_t _decodeBlock_fromBase64URLForSIMD[128] = {
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u,
+ 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 0u, 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u,
+ 14u, 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u,
+ 63u, 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u,
+ 40u, 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u,
+};
+
+
// Stub Code definitions
class StubGenerator: public StubCodeGenerator {
@@ -203,8 +363,17 @@ class StubGenerator: public StubCodeGenerator {
"adjust this code");
StubId stub_id = StubId::stubgen_call_stub_id;
+ GrowableArray entries;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 2, "sanity check");
+ address start = load_archive_data(stub_id, &entries);
+ if (start != nullptr) {
+ assert(entries.length() == 1, "expected 1 extra entry");
+ return_address = entries.at(0);
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Address sp_after_call (rfp, sp_after_call_off * wordSize);
@@ -323,6 +492,7 @@ class StubGenerator: public StubCodeGenerator {
// save current address for use by exception handling code
return_address = __ pc();
+ entries.append(return_address);
// store result depending on type (everything that is not
// T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
@@ -406,6 +576,9 @@ class StubGenerator: public StubCodeGenerator {
__ strd(j_farg0, Address(j_rarg2, 0));
__ br(Assembler::AL, exit);
+ // record the stub entry and end plus the auxiliary entry
+ store_archive_data(stub_id, start, __ pc(), &entries);
+
return start;
}
@@ -423,8 +596,14 @@ class StubGenerator: public StubCodeGenerator {
address generate_catch_exception() {
StubId stub_id = StubId::stubgen_catch_exception_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
// same as in generate_call_stub():
const Address sp_after_call(rfp, sp_after_call_off * wordSize);
@@ -450,7 +629,9 @@ class StubGenerator: public StubCodeGenerator {
__ verify_oop(r0);
__ str(r0, Address(rthread, Thread::pending_exception_offset()));
- __ mov(rscratch1, (address)__FILE__);
+ // special case -- add file name string to AOT address table
+ address file = (address)AOTCodeCache::add_C_string(__FILE__);
+ __ lea(rscratch1, ExternalAddress(file));
__ str(rscratch1, Address(rthread, Thread::exception_file_offset()));
__ movw(rscratch1, (int)__LINE__);
__ strw(rscratch1, Address(rthread, Thread::exception_line_offset()));
@@ -458,7 +639,10 @@ class StubGenerator: public StubCodeGenerator {
// complete return to VM
assert(StubRoutines::_call_stub_return_address != nullptr,
"_call_stub_return_address must have been generated before");
- __ b(StubRoutines::_call_stub_return_address);
+ __ b(RuntimeAddress(StubRoutines::_call_stub_return_address));
+
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
return start;
}
@@ -479,8 +663,14 @@ class StubGenerator: public StubCodeGenerator {
address generate_forward_exception() {
StubId stub_id = StubId::stubgen_forward_exception_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
// Upon entry, LR points to the return address returning into
// Java (interpreted or compiled) code; i.e., the return address
@@ -551,6 +741,9 @@ class StubGenerator: public StubCodeGenerator {
__ verify_oop(r0);
__ br(r19);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -569,8 +762,14 @@ class StubGenerator: public StubCodeGenerator {
// [tos + 5]: saved rscratch1
address generate_verify_oop() {
StubId stub_id = StubId::stubgen_verify_oop_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label exit, error;
@@ -613,33 +812,64 @@ class StubGenerator: public StubCodeGenerator {
__ blr(rscratch1);
__ hlt(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
// Generate indices for iota vector.
- address generate_iota_indices(StubId stub_id) {
+ void generate_iota_indices(StubId stub_id) {
+ GrowableArray entries;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == VECTOR_IOTA_COUNT, "sanity check");
+ address start = load_archive_data(stub_id, &entries);
+ if (start != nullptr) {
+ assert(entries.length() == entry_count - 1,
+ "unexpected entries count %d", entries.length());
+ StubRoutines::aarch64::_vector_iota_indices[0] = start;
+ for (int i = 1; i < VECTOR_IOTA_COUNT; i++) {
+ StubRoutines::aarch64::_vector_iota_indices[i] = entries.at(i - 1);
+ }
+ return;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
// B
__ emit_data64(0x0706050403020100, relocInfo::none);
__ emit_data64(0x0F0E0D0C0B0A0908, relocInfo::none);
+ entries.append(__ pc());
// H
__ emit_data64(0x0003000200010000, relocInfo::none);
__ emit_data64(0x0007000600050004, relocInfo::none);
+ entries.append(__ pc());
// S
__ emit_data64(0x0000000100000000, relocInfo::none);
__ emit_data64(0x0000000300000002, relocInfo::none);
+ entries.append(__ pc());
// D
__ emit_data64(0x0000000000000000, relocInfo::none);
__ emit_data64(0x0000000000000001, relocInfo::none);
+ entries.append(__ pc());
// S - FP
__ emit_data64(0x3F80000000000000, relocInfo::none); // 0.0f, 1.0f
__ emit_data64(0x4040000040000000, relocInfo::none); // 2.0f, 3.0f
+ entries.append(__ pc());
// D - FP
__ emit_data64(0x0000000000000000, relocInfo::none); // 0.0d
__ emit_data64(0x3FF0000000000000, relocInfo::none); // 1.0d
- return start;
+
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc(), &entries);
+
+ // install the entry addresses in the entry array
+ assert(entries.length() == entry_count - 1,
+ "unexpected entries count %d", entries.length());
+ StubRoutines::aarch64::_vector_iota_indices[0] = start;
+ for (int i = 1; i < VECTOR_IOTA_COUNT; i++) {
+ StubRoutines::aarch64::_vector_iota_indices[i] = entries.at(i - 1);
+ }
}
// The inner part of zero_words(). This is the bulk operation,
@@ -656,15 +886,21 @@ class StubGenerator: public StubCodeGenerator {
// r11 < MacroAssembler::zero_words_block_size.
address generate_zero_blocks() {
+ StubId stub_id = StubId::stubgen_zero_blocks_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
+ StubCodeMark mark(this, stub_id);
Label done;
Label base_aligned;
Register base = r10, cnt = r11;
- __ align(CodeEntryAlignment);
- StubId stub_id = StubId::stubgen_zero_blocks_id;
- StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
if (UseBlockZeroing) {
int zva_length = VM_Version::zva_length();
@@ -707,6 +943,9 @@ class StubGenerator: public StubCodeGenerator {
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -803,6 +1042,12 @@ class StubGenerator: public StubCodeGenerator {
// s and d are adjusted to point to the remaining words to copy
//
address generate_copy_longs(StubId stub_id, DecoratorSet decorators, Register s, Register d, Register count) {
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
BasicType type;
copy_direction direction;
@@ -854,7 +1099,7 @@ class StubGenerator: public StubCodeGenerator {
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label unaligned_copy_long;
if (AvoidUnalignedAccesses) {
@@ -1154,6 +1399,9 @@ class StubGenerator: public StubCodeGenerator {
__ ret(lr);
}
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -1445,19 +1693,25 @@ class StubGenerator: public StubCodeGenerator {
}
if (direction == copy_forwards) {
if (type != T_OBJECT) {
- __ bl(StubRoutines::aarch64::copy_byte_f());
+ __ lea(rscratch1, RuntimeAddress(StubRoutines::aarch64::copy_byte_f()));
+ __ blr(rscratch1);
} else if ((decorators & IS_DEST_UNINITIALIZED) != 0) {
- __ bl(StubRoutines::aarch64::copy_oop_uninit_f());
+ __ lea(rscratch1, RuntimeAddress(StubRoutines::aarch64::copy_oop_uninit_f()));
+ __ blr(rscratch1);
} else {
- __ bl(StubRoutines::aarch64::copy_oop_f());
+ __ lea(rscratch1, RuntimeAddress(StubRoutines::aarch64::copy_oop_f()));
+ __ blr(rscratch1);
}
} else {
if (type != T_OBJECT) {
- __ bl(StubRoutines::aarch64::copy_byte_b());
+ __ lea(rscratch1, RuntimeAddress(StubRoutines::aarch64::copy_byte_b()));
+ __ blr(rscratch1);
} else if ((decorators & IS_DEST_UNINITIALIZED) != 0) {
- __ bl(StubRoutines::aarch64::copy_oop_uninit_b());
+ __ lea(rscratch1, RuntimeAddress(StubRoutines::aarch64::copy_oop_uninit_b()));
+ __ blr(rscratch1);
} else {
- __ bl(StubRoutines::aarch64::copy_oop_b());
+ __ lea(rscratch1, RuntimeAddress(StubRoutines::aarch64::copy_oop_b()));
+ __ blr(rscratch1);
}
}
@@ -1508,8 +1762,8 @@ class StubGenerator: public StubCodeGenerator {
// stub_id - is used to name the stub and identify all details of
// how to perform the copy.
//
- // entry - is assigned to the stub's post push entry point unless
- // it is null
+ // nopush_entry - is assigned to the stub's post push entry point
+ // unless it is null
//
// Inputs:
// c_rarg0 - source array address
@@ -1525,8 +1779,6 @@ class StubGenerator: public StubCodeGenerator {
// copy method
//
address generate_disjoint_copy(StubId stub_id, address *nopush_entry) {
- Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
- RegSet saved_reg = RegSet::of(s, d, count);
int size;
bool aligned;
bool is_oop;
@@ -1607,17 +1859,45 @@ class StubGenerator: public StubCodeGenerator {
ShouldNotReachHere();
break;
}
+ // all stubs provide a 2nd entry which omits the frame push for
+ // use when bailing out from a conjoint copy. However we may also
+ // need some extra addressses for memory access protection.
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 2, "sanity check");
+ assert(nopush_entry != nullptr, "all disjoint copy stubs export a nopush entry");
+
+ bool add_extras = !is_oop && (!aligned || sizeof(jlong) == size);
+ int extra_count = ((add_extras ? 1 : 0) * UnsafeMemoryAccess::COLUMN_COUNT);
+ GrowableArray entries;
+ GrowableArray extras;
+ GrowableArray *extras_ptr = (extra_count > 0 ? &extras : nullptr);
+ address start = load_archive_data(stub_id, &entries, extras_ptr);
+ if (start != nullptr) {
+ assert(entries.length() == entry_count - 1,
+ "unexpected entries count %d", entries.length());
+ *nopush_entry = entries.at(0);
+ assert(extras.length() == extra_count,
+ "unexpected extra count %d", extras.length());
+ if (add_extras) {
+ // register one handler at offset 0
+ register_unsafe_access_handlers(extras, 0, 1);
+ }
+ return start;
+ }
+
+ Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
+ RegSet saved_reg = RegSet::of(s, d, count);
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
- if (nopush_entry != nullptr) {
- *nopush_entry = __ pc();
- // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
- BLOCK_COMMENT("Entry:");
- }
+ *nopush_entry = __ pc();
+ entries.append(*nopush_entry);
+
+ // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
+ BLOCK_COMMENT("Post-Push Entry:");
DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT;
if (dest_uninitialized) {
@@ -1636,8 +1916,7 @@ class StubGenerator: public StubCodeGenerator {
}
{
// UnsafeMemoryAccess page error: continue after unsafe access
- bool add_entry = !is_oop && (!aligned || sizeof(jlong) == size);
- UnsafeMemoryAccessMark umam(this, add_entry, true);
+ UnsafeMemoryAccessMark umam(this, add_extras, true);
copy_memory(decorators, is_oop ? T_OBJECT : T_BYTE, aligned, s, d, count, size);
}
@@ -1652,6 +1931,20 @@ class StubGenerator: public StubCodeGenerator {
__ leave();
__ mov(r0, zr); // return 0
__ ret(lr);
+
+ address end = __ pc();
+
+ if (add_extras) {
+ // retrieve the registered handler addresses
+ retrieve_unsafe_access_handlers(start, end, extras);
+ assert(extras.length() == extra_count
+ , "incorrect handlers count %d", extras.length());
+ }
+
+ // record the stub entry and end plus the no_push entry and any
+ // extra handler addresses
+ store_archive_data(stub_id, start, end, &entries, extras_ptr);
+
return start;
}
@@ -1663,8 +1956,8 @@ class StubGenerator: public StubCodeGenerator {
// corresponding disjoint copy routine which can be
// jumped to if the ranges do not actually overlap
//
- // entry - is assigned to the stub's post push entry point unless
- // it is null
+ // nopush_entry - is assigned to the stub's post push entry point
+ // unless it is null
//
//
// Inputs:
@@ -1681,8 +1974,6 @@ class StubGenerator: public StubCodeGenerator {
// used by some other conjoint copy method
//
address generate_conjoint_copy(StubId stub_id, address nooverlap_target, address *nopush_entry) {
- Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
- RegSet saved_regs = RegSet::of(s, d, count);
int size;
bool aligned;
bool is_oop;
@@ -1762,15 +2053,47 @@ class StubGenerator: public StubCodeGenerator {
default:
ShouldNotReachHere();
}
+ // only some conjoint stubs generate a 2nd entry
+ int entry_count = StubInfo::entry_count(stub_id);
+ int expected_entry_count = (nopush_entry == nullptr ? 1 : 2);
+ assert(entry_count == expected_entry_count,
+ "expected entry count %d does not match declared entry count %d for stub %s",
+ expected_entry_count, entry_count, StubInfo::name(stub_id));
+ // We need to protect memory accesses in certain cases
+ bool add_extras = !is_oop && (!aligned || sizeof(jlong) == size);
+ int extra_count = ((add_extras ? 1 : 0) * UnsafeMemoryAccess::COLUMN_COUNT);
+ GrowableArray entries;
+ GrowableArray extras;
+ GrowableArray *entries_ptr = (nopush_entry != nullptr ? &entries : nullptr);
+ GrowableArray *extras_ptr = (extra_count > 0 ? &extras : nullptr);
+ address start = load_archive_data(stub_id, entries_ptr, extras_ptr);
+ if (start != nullptr) {
+ assert(entries.length() == expected_entry_count - 1,
+ "unexpected entries count %d", entries.length());
+ assert(extras.length() == extra_count,
+ "unexpected extra count %d", extras.length());
+ if (nopush_entry != nullptr) {
+ *nopush_entry = entries.at(0);
+ }
+ if (add_extras) {
+ // register one handler at offset 0
+ register_unsafe_access_handlers(extras, 0, 1);
+ }
+ return start;
+ }
+
+ Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
+ RegSet saved_regs = RegSet::of(s, d, count);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
if (nopush_entry != nullptr) {
*nopush_entry = __ pc();
+ entries.append(*nopush_entry);
// caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
- BLOCK_COMMENT("Entry:");
+ BLOCK_COMMENT("Post-Push Entry:");
}
// use fwd copy when (d-s) above_equal (count*size)
@@ -1798,8 +2121,7 @@ class StubGenerator: public StubCodeGenerator {
}
{
// UnsafeMemoryAccess page error: continue after unsafe access
- bool add_entry = !is_oop && (!aligned || sizeof(jlong) == size);
- UnsafeMemoryAccessMark umam(this, add_entry, true);
+ UnsafeMemoryAccessMark umam(this, add_extras, true);
copy_memory(decorators, is_oop ? T_OBJECT : T_BYTE, aligned, s, d, count, -size);
}
if (is_oop) {
@@ -1811,6 +2133,23 @@ class StubGenerator: public StubCodeGenerator {
__ leave();
__ mov(r0, zr); // return 0
__ ret(lr);
+
+ assert(entries.length() == expected_entry_count - 1,
+ "unexpected entries count %d", entries.length());
+
+ address end = __ pc();
+
+ if (add_extras) {
+ // retrieve the registered handler addresses
+ retrieve_unsafe_access_handlers(start, end, extras);
+ assert(extras.length() == extra_count,
+ "incorrect handlers count %d", extras.length());
+ }
+
+ // record the stub entry and end plus any no_push entry and/or
+ // extra handler addresses
+ store_archive_data(stub_id, start, end, entries_ptr, extras_ptr);
+
return start;
}
@@ -1864,6 +2203,27 @@ class StubGenerator: public StubCodeGenerator {
ShouldNotReachHere();
}
+ // The normal stub provides a 2nd entry which omits the frame push
+ // for use when bailing out from a disjoint copy.
+ // Only some conjoint stubs generate a 2nd entry
+ int entry_count = StubInfo::entry_count(stub_id);
+ int expected_entry_count = (nopush_entry == nullptr ? 1 : 2);
+ GrowableArray entries;
+ GrowableArray *entries_ptr = (expected_entry_count == 1 ? nullptr : &entries);
+ assert(entry_count == expected_entry_count,
+ "expected entry count %d does not match declared entry count %d for stub %s",
+ expected_entry_count, entry_count, StubInfo::name(stub_id));
+ address start = load_archive_data(stub_id, entries_ptr);
+ if (start != nullptr) {
+ assert(entries.length() + 1 == expected_entry_count,
+ "expected entry count %d does not match return entry count %d for stub %s",
+ expected_entry_count, entries.length() + 1, StubInfo::name(stub_id));
+ if (nopush_entry != nullptr) {
+ *nopush_entry = entries.at(0);
+ }
+ return start;
+ }
+
Label L_load_element, L_store_element, L_do_card_marks, L_done, L_done_pop;
// Input registers (after setup_arg_regs)
@@ -1896,7 +2256,7 @@ class StubGenerator: public StubCodeGenerator {
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter(); // required for proper stackwalking of RuntimeStub frame
@@ -1913,6 +2273,7 @@ class StubGenerator: public StubCodeGenerator {
// Caller of this entry point must set up the argument registers.
if (nopush_entry != nullptr) {
*nopush_entry = __ pc();
+ entries.append(*nopush_entry);
BLOCK_COMMENT("Entry:");
}
@@ -2010,6 +2371,8 @@ class StubGenerator: public StubCodeGenerator {
__ leave();
__ ret(lr);
+ // record the stub entry and end plus any no_push entry
+ store_archive_data(stub_id, start, __ pc() , entries_ptr);
return start;
}
@@ -2072,13 +2435,18 @@ class StubGenerator: public StubCodeGenerator {
address int_copy_entry,
address long_copy_entry) {
StubId stub_id = StubId::stubgen_unsafe_arraycopy_id;
-
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
Label L_long_aligned, L_int_aligned, L_short_aligned;
Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter(); // required for proper stackwalking of RuntimeStub frame
// bump this on entry, not on exit:
@@ -2104,6 +2472,9 @@ class StubGenerator: public StubCodeGenerator {
__ lsr(count, count, LogBytesPerLong); // size => long_count
__ b(RuntimeAddress(long_copy_entry));
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -2125,7 +2496,12 @@ class StubGenerator: public StubCodeGenerator {
address int_copy_entry, address oop_copy_entry,
address long_copy_entry, address checkcast_copy_entry) {
StubId stub_id = StubId::stubgen_generic_arraycopy_id;
-
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
Label L_failed, L_objArray;
Label L_copy_bytes, L_copy_shorts, L_copy_ints, L_copy_longs;
@@ -2144,7 +2520,7 @@ class StubGenerator: public StubCodeGenerator {
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter(); // required for proper stackwalking of RuntimeStub frame
@@ -2383,6 +2759,9 @@ class StubGenerator: public StubCodeGenerator {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -2427,10 +2806,15 @@ class StubGenerator: public StubCodeGenerator {
default:
ShouldNotReachHere();
};
-
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
BLOCK_COMMENT("Entry:");
@@ -2563,15 +2947,32 @@ class StubGenerator: public StubCodeGenerator {
__ bind(L_exit2);
__ leave();
__ ret(lr);
+
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address generate_unsafecopy_common_error_exit() {
- address start_pc = __ pc();
+ StubId stub_id = StubId::stubgen_unsafecopy_common_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
+ StubCodeMark mark(this, stub_id);
+ start = __ pc();
__ leave();
__ mov(r0, 0);
__ ret(lr);
- return start_pc;
+
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
+ return start;
}
//
@@ -2589,13 +2990,28 @@ class StubGenerator: public StubCodeGenerator {
// c_rarg2 - byte value
//
address generate_unsafe_setmemory() {
+ StubId stub_id = StubId::stubgen_unsafe_setmemory_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ // we expect one set of extra unsafememory access handler entries
+ GrowableArray extras;
+ int extra_count = 1 * UnsafeMemoryAccess::COLUMN_COUNT;
+ address start = load_archive_data(stub_id, nullptr, &extras);
+ if (start != nullptr) {
+ assert(extras.length() == extra_count,
+ "unexpected extra entry count %d", extras.length());
+ register_unsafe_access_handlers(extras, 0, 1);
+ return start;
+ }
+
__ align(CodeEntryAlignment);
- StubCodeMark mark(this, StubId::stubgen_unsafe_setmemory_id);
- address start = __ pc();
+ StubCodeMark mark(this, stub_id);
+ start = __ pc();
Register dest = c_rarg0, count = c_rarg1, value = c_rarg2;
Label tail;
+ {
UnsafeMemoryAccessMark umam(this, true, false);
__ enter(); // required for proper stackwalking of RuntimeStub frame
@@ -2679,6 +3095,17 @@ class StubGenerator: public StubCodeGenerator {
__ bind(finished);
__ leave();
__ ret(lr);
+ // have to exit the block and destroy the UnsafeMemoryAccessMark
+ // in order to retrieve the handler end address
+ }
+
+ // install saved handler addresses in extras
+ address end = __ pc();
+ retrieve_unsafe_access_handlers(start, end, extras);
+ assert(extras.length() == extra_count,
+ "incorrect handlers count %d", extras.length());
+ // record the stub entry and end plus the extras
+ store_archive_data(stub_id, start, end, nullptr, &extras);
return start;
}
@@ -2686,33 +3113,45 @@ class StubGenerator: public StubCodeGenerator {
address generate_data_cache_writeback() {
const Register line = c_rarg0; // address of line to write back
- __ align(CodeEntryAlignment);
-
StubId stub_id = StubId::stubgen_data_cache_writeback_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
__ cache_wb(Address(line, 0));
__ leave();
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address generate_data_cache_writeback_sync() {
- const Register is_pre = c_rarg0; // pre or post sync
-
- __ align(CodeEntryAlignment);
-
StubId stub_id = StubId::stubgen_data_cache_writeback_sync_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ const Register is_pre = c_rarg0; // pre or post sync
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
// pre wbsync is a no-op
// post wbsync translates to an sfence
Label skip;
- address start = __ pc();
+ start = __ pc();
__ enter();
__ cbnz(is_pre, skip);
__ cache_wbsync(false);
@@ -2720,6 +3159,9 @@ class StubGenerator: public StubCodeGenerator {
__ leave();
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -2882,8 +3324,15 @@ class StubGenerator: public StubCodeGenerator {
// c_rarg2 - sessionKe (key) in little endian int array
//
address generate_aescrypt_encryptBlock() {
- __ align(CodeEntryAlignment);
+ assert(UseAES, "need AES cryptographic extension support");
StubId stub_id = StubId::stubgen_aescrypt_encryptBlock_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
const Register from = c_rarg0; // source array address
@@ -2891,7 +3340,7 @@ class StubGenerator: public StubCodeGenerator {
const Register key = c_rarg2; // key array address
const Register keylen = rscratch1;
- address start = __ pc();
+ start = __ pc();
__ enter();
__ ldrw(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
@@ -2904,6 +3353,9 @@ class StubGenerator: public StubCodeGenerator {
__ leave();
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -2916,8 +3368,14 @@ class StubGenerator: public StubCodeGenerator {
//
address generate_aescrypt_decryptBlock() {
assert(UseAES, "need AES cryptographic extension support");
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_aescrypt_decryptBlock_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
Label L_doLast;
@@ -2926,7 +3384,7 @@ class StubGenerator: public StubCodeGenerator {
const Register key = c_rarg2; // key array address
const Register keylen = rscratch1;
- address start = __ pc();
+ start = __ pc();
__ enter(); // required for proper stackwalking of RuntimeStub frame
__ ldrw(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
@@ -2938,6 +3396,9 @@ class StubGenerator: public StubCodeGenerator {
__ leave();
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -2955,8 +3416,14 @@ class StubGenerator: public StubCodeGenerator {
//
address generate_cipherBlockChaining_encryptAESCrypt() {
assert(UseAES, "need AES cryptographic extension support");
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_cipherBlockChaining_encryptAESCrypt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
Label L_loadkeys_44, L_loadkeys_52, L_aes_loop, L_rounds_44, L_rounds_52;
@@ -2969,7 +3436,7 @@ class StubGenerator: public StubCodeGenerator {
const Register len_reg = c_rarg4; // src len (must be multiple of blocksize 16)
const Register keylen = rscratch1;
- address start = __ pc();
+ start = __ pc();
__ enter();
@@ -3043,6 +3510,9 @@ class StubGenerator: public StubCodeGenerator {
__ leave();
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3060,8 +3530,14 @@ class StubGenerator: public StubCodeGenerator {
//
address generate_cipherBlockChaining_decryptAESCrypt() {
assert(UseAES, "need AES cryptographic extension support");
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_cipherBlockChaining_decryptAESCrypt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
Label L_loadkeys_44, L_loadkeys_52, L_aes_loop, L_rounds_44, L_rounds_52;
@@ -3074,7 +3550,7 @@ class StubGenerator: public StubCodeGenerator {
const Register len_reg = c_rarg4; // src len (must be multiple of blocksize 16)
const Register keylen = rscratch1;
- address start = __ pc();
+ start = __ pc();
__ enter();
@@ -3152,6 +3628,9 @@ class StubGenerator: public StubCodeGenerator {
__ leave();
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3188,6 +3667,13 @@ class StubGenerator: public StubCodeGenerator {
// r0 - input length
//
address generate_counterMode_AESCrypt() {
+ StubId stub_id = StubId::stubgen_counterMode_AESCrypt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
const Register in = c_rarg0;
const Register out = c_rarg1;
const Register key = c_rarg2;
@@ -3248,9 +3734,8 @@ class StubGenerator: public StubCodeGenerator {
// Wide bulk encryption of whole blocks.
__ align(CodeEntryAlignment);
- StubId stub_id = StubId::stubgen_counterMode_AESCrypt_id;
StubCodeMark mark(this, stub_id);
- const address start = __ pc();
+ start = __ pc();
__ enter();
Label DONE, CTR_large_block, large_block_return;
@@ -3435,6 +3920,9 @@ class StubGenerator: public StubCodeGenerator {
__ strw(used, Address(used_ptr));
__ b(large_block_return);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3451,11 +3939,16 @@ class StubGenerator: public StubCodeGenerator {
// return - number of processed bytes
address generate_galoisCounterMode_AESCrypt() {
Label ghash_polynomial; // local data generated after code
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_galoisCounterMode_AESCrypt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register in = c_rarg0;
@@ -3567,6 +4060,9 @@ class StubGenerator: public StubCodeGenerator {
// 128-bit vector
__ emit_int64(0x87);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3685,10 +4181,16 @@ class StubGenerator: public StubCodeGenerator {
default:
ShouldNotReachHere();
}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Register buf = c_rarg0;
Register state = c_rarg1;
@@ -3815,6 +4317,9 @@ class StubGenerator: public StubCodeGenerator {
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3838,11 +4343,16 @@ class StubGenerator: public StubCodeGenerator {
default:
ShouldNotReachHere();
}
-
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Register buf = c_rarg0;
Register state = c_rarg1;
@@ -3919,6 +4429,9 @@ class StubGenerator: public StubCodeGenerator {
__ emit_int32(0x8f1bbcdc);
__ emit_int32(0xca62c1d6);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3943,30 +4456,15 @@ class StubGenerator: public StubCodeGenerator {
default:
ShouldNotReachHere();
}
-
- static const uint32_t round_consts[64] = {
- 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5,
- 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
- 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
- 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
- 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc,
- 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
- 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
- 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
- 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
- 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
- 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3,
- 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
- 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5,
- 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
- 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
- 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2,
- };
-
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
-
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Register buf = c_rarg0;
Register state = c_rarg1;
@@ -3987,7 +4485,7 @@ class StubGenerator: public StubCodeGenerator {
// t1 == v7
// load 16 keys to v16..v31
- __ lea(rscratch1, ExternalAddress((address)round_consts));
+ __ lea(rscratch1, ExternalAddress((address)_sha256_round_consts));
__ ld1(v16, v17, v18, v19, __ T4S, __ post(rscratch1, 64));
__ ld1(v20, v21, v22, v23, __ T4S, __ post(rscratch1, 64));
__ ld1(v24, v25, v26, v27, __ T4S, __ post(rscratch1, 64));
@@ -4048,6 +4546,9 @@ class StubGenerator: public StubCodeGenerator {
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -4099,41 +4600,15 @@ class StubGenerator: public StubCodeGenerator {
default:
ShouldNotReachHere();
}
-
- static const uint64_t round_consts[80] = {
- 0x428A2F98D728AE22L, 0x7137449123EF65CDL, 0xB5C0FBCFEC4D3B2FL,
- 0xE9B5DBA58189DBBCL, 0x3956C25BF348B538L, 0x59F111F1B605D019L,
- 0x923F82A4AF194F9BL, 0xAB1C5ED5DA6D8118L, 0xD807AA98A3030242L,
- 0x12835B0145706FBEL, 0x243185BE4EE4B28CL, 0x550C7DC3D5FFB4E2L,
- 0x72BE5D74F27B896FL, 0x80DEB1FE3B1696B1L, 0x9BDC06A725C71235L,
- 0xC19BF174CF692694L, 0xE49B69C19EF14AD2L, 0xEFBE4786384F25E3L,
- 0x0FC19DC68B8CD5B5L, 0x240CA1CC77AC9C65L, 0x2DE92C6F592B0275L,
- 0x4A7484AA6EA6E483L, 0x5CB0A9DCBD41FBD4L, 0x76F988DA831153B5L,
- 0x983E5152EE66DFABL, 0xA831C66D2DB43210L, 0xB00327C898FB213FL,
- 0xBF597FC7BEEF0EE4L, 0xC6E00BF33DA88FC2L, 0xD5A79147930AA725L,
- 0x06CA6351E003826FL, 0x142929670A0E6E70L, 0x27B70A8546D22FFCL,
- 0x2E1B21385C26C926L, 0x4D2C6DFC5AC42AEDL, 0x53380D139D95B3DFL,
- 0x650A73548BAF63DEL, 0x766A0ABB3C77B2A8L, 0x81C2C92E47EDAEE6L,
- 0x92722C851482353BL, 0xA2BFE8A14CF10364L, 0xA81A664BBC423001L,
- 0xC24B8B70D0F89791L, 0xC76C51A30654BE30L, 0xD192E819D6EF5218L,
- 0xD69906245565A910L, 0xF40E35855771202AL, 0x106AA07032BBD1B8L,
- 0x19A4C116B8D2D0C8L, 0x1E376C085141AB53L, 0x2748774CDF8EEB99L,
- 0x34B0BCB5E19B48A8L, 0x391C0CB3C5C95A63L, 0x4ED8AA4AE3418ACBL,
- 0x5B9CCA4F7763E373L, 0x682E6FF3D6B2B8A3L, 0x748F82EE5DEFB2FCL,
- 0x78A5636F43172F60L, 0x84C87814A1F0AB72L, 0x8CC702081A6439ECL,
- 0x90BEFFFA23631E28L, 0xA4506CEBDE82BDE9L, 0xBEF9A3F7B2C67915L,
- 0xC67178F2E372532BL, 0xCA273ECEEA26619CL, 0xD186B8C721C0C207L,
- 0xEADA7DD6CDE0EB1EL, 0xF57D4F7FEE6ED178L, 0x06F067AA72176FBAL,
- 0x0A637DC5A2C898A6L, 0x113F9804BEF90DAEL, 0x1B710B35131C471BL,
- 0x28DB77F523047D84L, 0x32CAAB7B40C72493L, 0x3C9EBE0A15C9BEBCL,
- 0x431D67C49C100D4CL, 0x4CC5D4BECB3E42B6L, 0x597F299CFC657E2AL,
- 0x5FCB6FAB3AD6FAECL, 0x6C44198C4A475817L
- };
-
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
-
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Register buf = c_rarg0;
Register state = c_rarg1;
@@ -4151,7 +4626,7 @@ class StubGenerator: public StubCodeGenerator {
__ ld1(v8, v9, v10, v11, __ T2D, state);
// load first 4 round constants
- __ lea(rscratch1, ExternalAddress((address)round_consts));
+ __ lea(rscratch1, ExternalAddress((address)_sha512_round_consts));
__ ld1(v20, v21, v22, v23, __ T2D, __ post(rscratch1, 64));
__ BIND(sha512_loop);
@@ -4236,6 +4711,9 @@ class StubGenerator: public StubCodeGenerator {
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -4349,22 +4827,15 @@ class StubGenerator: public StubCodeGenerator {
default:
ShouldNotReachHere();
}
-
- static const uint64_t round_consts[24] = {
- 0x0000000000000001L, 0x0000000000008082L, 0x800000000000808AL,
- 0x8000000080008000L, 0x000000000000808BL, 0x0000000080000001L,
- 0x8000000080008081L, 0x8000000000008009L, 0x000000000000008AL,
- 0x0000000000000088L, 0x0000000080008009L, 0x000000008000000AL,
- 0x000000008000808BL, 0x800000000000008BL, 0x8000000000008089L,
- 0x8000000000008003L, 0x8000000000008002L, 0x8000000000000080L,
- 0x000000000000800AL, 0x800000008000000AL, 0x8000000080008081L,
- 0x8000000000008080L, 0x0000000080000001L, 0x8000000080008008L
- };
-
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
-
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Register buf = c_rarg0;
Register state = c_rarg1;
@@ -4396,7 +4867,7 @@ class StubGenerator: public StubCodeGenerator {
__ movw(rscratch2, 24);
// load round_constants base
- __ lea(rscratch1, ExternalAddress((address) round_consts));
+ __ lea(rscratch1, ExternalAddress((address) _sha3_round_consts));
// load input
__ ld1(v25, v26, v27, v28, __ T8B, __ post(buf, 32));
@@ -4488,6 +4959,9 @@ class StubGenerator: public StubCodeGenerator {
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -4495,22 +4969,18 @@ class StubGenerator: public StubCodeGenerator {
// c_rarg0 - long[] state0
// c_rarg1 - long[] state1
address generate_double_keccak() {
- static const uint64_t round_consts[24] = {
- 0x0000000000000001L, 0x0000000000008082L, 0x800000000000808AL,
- 0x8000000080008000L, 0x000000000000808BL, 0x0000000080000001L,
- 0x8000000080008081L, 0x8000000000008009L, 0x000000000000008AL,
- 0x0000000000000088L, 0x0000000080008009L, 0x000000008000000AL,
- 0x000000008000808BL, 0x800000000000008BL, 0x8000000000008089L,
- 0x8000000000008003L, 0x8000000000008002L, 0x8000000000000080L,
- 0x000000000000800AL, 0x800000008000000AL, 0x8000000080008081L,
- 0x8000000000008080L, 0x0000000080000001L, 0x8000000080008008L
- };
-
+ StubId stub_id = StubId::stubgen_double_keccak_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
// Implements the double_keccak() method of the
// sun.secyrity.provider.SHA3Parallel class
__ align(CodeEntryAlignment);
- StubCodeMark mark(this, "StubRoutines", "double_keccak");
- address start = __ pc();
+ StubCodeMark mark(this, stub_id);
+ start = __ pc();
__ enter();
Register state0 = c_rarg0;
@@ -4546,7 +5016,7 @@ class StubGenerator: public StubCodeGenerator {
__ movw(rscratch2, 24);
// load round_constants base
- __ lea(rscratch1, ExternalAddress((address) round_consts));
+ __ lea(rscratch1, ExternalAddress((address) _double_keccak_round_consts));
__ BIND(rounds24_loop);
__ subw(rscratch2, rscratch2, 1);
@@ -4578,6 +5048,9 @@ class StubGenerator: public StubCodeGenerator {
__ mov(r0, zr); // return 0
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -4611,11 +5084,17 @@ class StubGenerator: public StubCodeGenerator {
// vectors write their first lane back to the keystream buffer, followed
// by the second lane from all vectors and so on.
address generate_chacha20Block_blockpar() {
+ StubId stub_id = StubId::stubgen_chacha20Block_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
Label L_twoRounds, L_cc20_const;
__ align(CodeEntryAlignment);
- StubId stub_id = StubId::stubgen_chacha20Block_id;
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
int i, j;
@@ -4770,6 +5249,9 @@ class StubGenerator: public StubCodeGenerator {
__ emit_int64(0x0605040702010003UL);
__ emit_int64(0x0E0D0C0F0A09080BUL);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -5258,11 +5740,16 @@ class StubGenerator: public StubCodeGenerator {
// coeffs (short[256]) = c_rarg0
// ntt_zetas (short[256]) = c_rarg1
address generate_kyberNtt() {
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_kyberNtt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register coeffs = c_rarg0;
@@ -5486,6 +5973,9 @@ class StubGenerator: public StubCodeGenerator {
__ mov(r0, zr); // return 0
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -5496,11 +5986,16 @@ class StubGenerator: public StubCodeGenerator {
// coeffs (short[256]) = c_rarg0
// ntt_zetas (short[256]) = c_rarg1
address generate_kyberInverseNtt() {
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_kyberInverseNtt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register coeffs = c_rarg0;
@@ -5770,6 +6265,9 @@ class StubGenerator: public StubCodeGenerator {
__ mov(r0, zr); // return 0
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -5783,11 +6281,16 @@ class StubGenerator: public StubCodeGenerator {
// nttb (short[256]) = c_rarg2
// zetas (short[128]) = c_rarg3
address generate_kyberNttMult() {
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_kyberNttMult_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register result = c_rarg0;
@@ -5889,6 +6392,9 @@ class StubGenerator: public StubCodeGenerator {
__ mov(r0, zr); // return 0
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -5900,11 +6406,16 @@ class StubGenerator: public StubCodeGenerator {
// a (short[256]) = c_rarg1
// b (short[256]) = c_rarg2
address generate_kyberAddPoly_2() {
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_kyberAddPoly_2_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register result = c_rarg0;
@@ -5973,6 +6484,9 @@ class StubGenerator: public StubCodeGenerator {
__ mov(r0, zr); // return 0
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -5985,11 +6499,16 @@ class StubGenerator: public StubCodeGenerator {
// b (short[256]) = c_rarg2
// c (short[256]) = c_rarg3
address generate_kyberAddPoly_3() {
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_kyberAddPoly_3_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register result = c_rarg0;
@@ -6072,6 +6591,9 @@ class StubGenerator: public StubCodeGenerator {
__ mov(r0, zr); // return 0
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -6092,12 +6614,18 @@ class StubGenerator: public StubCodeGenerator {
// parsed (short[]) = c_rarg2
// parsedLength = c_rarg3
address generate_kyber12To16() {
+ StubId stub_id = StubId::stubgen_kyber12To16_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
Label L_F00, L_loop;
__ align(CodeEntryAlignment);
- StubId stub_id = StubId::stubgen_kyber12To16_id;
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register condensed = c_rarg0;
@@ -6225,6 +6753,9 @@ class StubGenerator: public StubCodeGenerator {
__ emit_int64(0x0f000f000f000f00);
__ emit_int64(0x0f000f000f000f00);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -6234,11 +6765,16 @@ class StubGenerator: public StubCodeGenerator {
//
// coeffs (short[256]) = c_rarg0
address generate_kyberBarrettReduce() {
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_kyberBarrettReduce_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register coeffs = c_rarg0;
@@ -6318,6 +6854,9 @@ class StubGenerator: public StubCodeGenerator {
__ mov(r0, zr); // return 0
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -6481,11 +7020,16 @@ class StubGenerator: public StubCodeGenerator {
// coeffs (int[256]) = c_rarg0
// zetas (int[256]) = c_rarg1
address generate_dilithiumAlmostNtt() {
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_dilithiumAlmostNtt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register coeffs = c_rarg0;
@@ -6596,6 +7140,9 @@ class StubGenerator: public StubCodeGenerator {
__ mov(r0, zr); // return 0
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -6688,11 +7235,16 @@ class StubGenerator: public StubCodeGenerator {
// coeffs (int[256]) = c_rarg0
// zetas (int[256]) = c_rarg1
address generate_dilithiumAlmostInverseNtt() {
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_dilithiumAlmostInverseNtt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register coeffs = c_rarg0;
@@ -6788,6 +7340,9 @@ class StubGenerator: public StubCodeGenerator {
__ mov(r0, zr); // return 0
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -6801,11 +7356,16 @@ class StubGenerator: public StubCodeGenerator {
// poly1 (int[256]) = c_rarg1
// poly2 (int[256]) = c_rarg2
address generate_dilithiumNttMult() {
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_dilithiumNttMult_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
Label L_loop;
@@ -6854,6 +7414,9 @@ class StubGenerator: public StubCodeGenerator {
__ mov(r0, zr); // return 0
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -6865,11 +7428,16 @@ class StubGenerator: public StubCodeGenerator {
// coeffs (int[256]) = c_rarg0
// constant (int) = c_rarg1
address generate_dilithiumMontMulByConstant() {
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_dilithiumMontMulByConstant_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
Label L_loop;
@@ -6915,6 +7483,9 @@ class StubGenerator: public StubCodeGenerator {
__ mov(r0, zr); // return 0
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -6929,11 +7500,16 @@ class StubGenerator: public StubCodeGenerator {
// twoGamma2 (int) = c_rarg3
// multiplier (int) = c_rarg4
address generate_dilithiumDecomposePoly() {
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_dilithiumDecomposePoly_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_loop;
const Register input = c_rarg0;
@@ -7073,6 +7649,9 @@ class StubGenerator: public StubCodeGenerator {
__ mov(r0, zr); // return 0
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -7212,21 +7791,15 @@ class StubGenerator: public StubCodeGenerator {
default:
ShouldNotReachHere();
}
-
- static const uint64_t round_consts[24] = {
- 0x0000000000000001L, 0x0000000000008082L, 0x800000000000808AL,
- 0x8000000080008000L, 0x000000000000808BL, 0x0000000080000001L,
- 0x8000000080008081L, 0x8000000000008009L, 0x000000000000008AL,
- 0x0000000000000088L, 0x0000000080008009L, 0x000000008000000AL,
- 0x000000008000808BL, 0x800000000000008BL, 0x8000000000008089L,
- 0x8000000000008003L, 0x8000000000008002L, 0x8000000000000080L,
- 0x000000000000800AL, 0x800000008000000AL, 0x8000000080008081L,
- 0x8000000000008080L, 0x0000000080000001L, 0x8000000080008008L
- };
-
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Register buf = c_rarg0;
Register state = c_rarg1;
@@ -7378,7 +7951,7 @@ class StubGenerator: public StubCodeGenerator {
__ fmovs(v1, 1.0); // exact representation
__ str(buf, Address(sp, 16));
- __ lea(tmp3, ExternalAddress((address) round_consts));
+ __ lea(tmp3, ExternalAddress((address) _sha3_round_consts));
__ BIND(loop_body);
keccak_round_gpr(can_use_fp, can_use_r18, tmp3,
@@ -7433,6 +8006,9 @@ class StubGenerator: public StubCodeGenerator {
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -7449,12 +8025,17 @@ class StubGenerator: public StubCodeGenerator {
*/
address generate_updateBytesCRC32() {
assert(UseCRC32Intrinsics, "what are we doing here?");
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_updateBytesCRC32_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register crc = c_rarg0; // crc
const Register buf = c_rarg1; // source java byte array address
@@ -7474,6 +8055,9 @@ class StubGenerator: public StubCodeGenerator {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -7491,12 +8075,17 @@ class StubGenerator: public StubCodeGenerator {
*/
address generate_updateBytesCRC32C() {
assert(UseCRC32CIntrinsics, "what are we doing here?");
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_updateBytesCRC32C_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register crc = c_rarg0; // crc
const Register buf = c_rarg1; // source java byte array address
@@ -7516,6 +8105,9 @@ class StubGenerator: public StubCodeGenerator {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -7531,10 +8123,16 @@ class StubGenerator: public StubCodeGenerator {
* c_rarg0 - int adler result
*/
address generate_updateBytesAdler32() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_updateBytesAdler32_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_simple_by1_loop, L_nmax, L_nmax_loop, L_by16, L_by16_loop, L_by1_loop, L_do_mod, L_combine, L_by1;
@@ -7702,6 +8300,9 @@ class StubGenerator: public StubCodeGenerator {
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -7753,11 +8354,17 @@ class StubGenerator: public StubCodeGenerator {
* c_rarg4 - z address
*/
address generate_multiplyToLen() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_multiplyToLen_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register x = r0;
const Register xlen = r1;
const Register y = r2;
@@ -7779,6 +8386,9 @@ class StubGenerator: public StubCodeGenerator {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -7786,10 +8396,16 @@ class StubGenerator: public StubCodeGenerator {
// squareToLen algorithm for sizes 1..127 described in java code works
// faster than multiply_to_len on some CPUs and slower on others, but
// multiply_to_len shows a bit better overall results
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_squareToLen_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register x = r0;
const Register xlen = r1;
@@ -7816,15 +8432,25 @@ class StubGenerator: public StubCodeGenerator {
__ pop(spilled_regs, sp);
__ leave();
__ ret(lr);
+
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address generate_mulAdd() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_mulAdd_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register out = r0;
const Register in = r1;
@@ -7838,6 +8464,9 @@ class StubGenerator: public StubCodeGenerator {
__ leave();
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -7851,10 +8480,16 @@ class StubGenerator: public StubCodeGenerator {
// c_rarg4 - numIter
//
address generate_bigIntegerRightShift() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_bigIntegerRightShiftWorker_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label ShiftSIMDLoop, ShiftTwoLoop, ShiftThree, ShiftTwo, ShiftOne, Exit;
@@ -7961,6 +8596,9 @@ class StubGenerator: public StubCodeGenerator {
__ BIND(Exit);
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -7974,10 +8612,16 @@ class StubGenerator: public StubCodeGenerator {
// c_rarg4 - numIter
//
address generate_bigIntegerLeftShift() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_bigIntegerLeftShiftWorker_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label ShiftSIMDLoop, ShiftTwoLoop, ShiftThree, ShiftTwo, ShiftOne, Exit;
@@ -8072,10 +8716,25 @@ class StubGenerator: public StubCodeGenerator {
__ BIND(Exit);
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address generate_count_positives(address &count_positives_long) {
+ StubId stub_id = StubId::stubgen_count_positives_id;
+ GrowableArray entries;
+ int entry_count = StubInfo::entry_count(stub_id);
+ // We have an extra entry for count_positives_long.
+ assert(entry_count == 2, "sanity check");
+ address start = load_archive_data(stub_id, &entries);
+ if (start != nullptr) {
+ assert(entries.length() == 1,
+ "unexpected extra entry count %d", entries.length());
+ count_positives_long = entries.at(0);
+ return start;
+ }
const u1 large_loop_size = 64;
const uint64_t UPPER_BIT_MASK=0x8080808080808080;
int dcache_line = VM_Version::dcache_line_size();
@@ -8083,8 +8742,6 @@ class StubGenerator: public StubCodeGenerator {
Register ary1 = r1, len = r2, result = r0;
__ align(CodeEntryAlignment);
-
- StubId stub_id = StubId::stubgen_count_positives_id;
StubCodeMark mark(this, stub_id);
address entry = __ pc();
@@ -8127,6 +8784,7 @@ class StubGenerator: public StubCodeGenerator {
const RegSet spilled_regs = RegSet::range(tmp1, tmp5) + tmp6;
count_positives_long = __ pc(); // 2nd entry point
+ entries.append(count_positives_long);
__ enter();
@@ -8241,6 +8899,9 @@ class StubGenerator: public StubCodeGenerator {
__ sub(result, result, len);
__ ret(lr);
+ // record the stub entry and end plus the extra entry
+ store_archive_data(stub_id, entry, __ pc(), &entries);
+
return entry;
}
@@ -8331,6 +8992,13 @@ class StubGenerator: public StubCodeGenerator {
// r3-r5 are reserved temporary registers
// Clobbers: v0-v7 when UseSIMDForArrayEquals, rscratch1, rscratch2
address generate_large_array_equals() {
+ StubId stub_id = StubId::stubgen_large_array_equals_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
Register a1 = r1, a2 = r2, result = r0, cnt1 = r10, tmp1 = rscratch1,
tmp2 = rscratch2, tmp3 = r3, tmp4 = r4, tmp5 = r5, tmp6 = r11,
tmp7 = r12, tmp8 = r13;
@@ -8346,7 +9014,6 @@ class StubGenerator: public StubCodeGenerator {
__ align(CodeEntryAlignment);
- StubId stub_id = StubId::stubgen_large_array_equals_id;
StubCodeMark mark(this, stub_id);
address entry = __ pc();
@@ -8421,6 +9088,10 @@ class StubGenerator: public StubCodeGenerator {
__ bind(NOT_EQUAL_NO_POP);
__ leave();
__ ret(lr);
+
+ // record the stub entry and end
+ store_archive_data(stub_id, entry, __ pc());
+
return entry;
}
@@ -8429,6 +9100,33 @@ class StubGenerator: public StubCodeGenerator {
// cnt = r2 - elements count
// Clobbers: v0-v13, rscratch1, rscratch2
address generate_large_arrays_hashcode(BasicType eltype) {
+ StubId stub_id;
+ switch (eltype) {
+ case T_BOOLEAN:
+ stub_id = StubId::stubgen_large_arrays_hashcode_boolean_id;
+ break;
+ case T_BYTE:
+ stub_id = StubId::stubgen_large_arrays_hashcode_byte_id;
+ break;
+ case T_CHAR:
+ stub_id = StubId::stubgen_large_arrays_hashcode_char_id;
+ break;
+ case T_SHORT:
+ stub_id = StubId::stubgen_large_arrays_hashcode_short_id;
+ break;
+ case T_INT:
+ stub_id = StubId::stubgen_large_arrays_hashcode_int_id;
+ break;
+ default:
+ stub_id = StubId::NO_STUBID;
+ ShouldNotReachHere();
+ };
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
const Register result = r0, ary = r1, cnt = r2;
const FloatRegister vdata0 = v3, vdata1 = v2, vdata2 = v1, vdata3 = v0;
const FloatRegister vmul0 = v4, vmul1 = v5, vmul2 = v6, vmul3 = v7;
@@ -8472,28 +9170,6 @@ class StubGenerator: public StubCodeGenerator {
__ align(CodeEntryAlignment);
- StubId stub_id;
- switch (eltype) {
- case T_BOOLEAN:
- stub_id = StubId::stubgen_large_arrays_hashcode_boolean_id;
- break;
- case T_BYTE:
- stub_id = StubId::stubgen_large_arrays_hashcode_byte_id;
- break;
- case T_CHAR:
- stub_id = StubId::stubgen_large_arrays_hashcode_char_id;
- break;
- case T_SHORT:
- stub_id = StubId::stubgen_large_arrays_hashcode_short_id;
- break;
- case T_INT:
- stub_id = StubId::stubgen_large_arrays_hashcode_int_id;
- break;
- default:
- stub_id = StubId::NO_STUBID;
- ShouldNotReachHere();
- };
-
StubCodeMark mark(this, stub_id);
address entry = __ pc();
@@ -8728,19 +9404,32 @@ class StubGenerator: public StubCodeGenerator {
__ leave();
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, entry, __ pc());
+
return entry;
}
address generate_dsin_dcos(bool isCos) {
- __ align(CodeEntryAlignment);
StubId stub_id = (isCos ? StubId::stubgen_dcos_id : StubId::stubgen_dsin_id);
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ generate_dsin_dcos(isCos, (address)StubRoutines::aarch64::_npio2_hw,
(address)StubRoutines::aarch64::_two_over_pi,
(address)StubRoutines::aarch64::_pio2,
(address)StubRoutines::aarch64::_dsin_coef,
(address)StubRoutines::aarch64::_dcos_coef);
+
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -8784,8 +9473,14 @@ class StubGenerator: public StubCodeGenerator {
// r10 = tmp1
// r11 = tmp2
address generate_compare_long_string_different_encoding(bool isLU) {
- __ align(CodeEntryAlignment);
StubId stub_id = (isLU ? StubId::stubgen_compare_long_string_LU_id : StubId::stubgen_compare_long_string_UL_id);
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
address entry = __ pc();
Label SMALL_LOOP, TAIL, TAIL_LOAD_16, LOAD_LAST, DIFF1, DIFF2,
@@ -8887,20 +9582,34 @@ class StubGenerator: public StubCodeGenerator {
__ subw(result, tmp1, rscratch1);
__ bind(DONE);
__ ret(lr);
- return entry;
+
+ // record the stub entry and end
+ store_archive_data(stub_id, entry, __ pc());
+
+ return entry;
}
// r0 = input (float16)
// v0 = result (float)
// v1 = temporary float register
address generate_float16ToFloat() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_hf2f_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
address entry = __ pc();
BLOCK_COMMENT("Entry:");
__ flt16_to_flt(v0, r0, v1);
__ ret(lr);
+
+ // record the stub entry and end
+ store_archive_data(stub_id, entry, __ pc());
+
return entry;
}
@@ -8908,24 +9617,40 @@ class StubGenerator: public StubCodeGenerator {
// r0 = result (float16)
// v1 = temporary float register
address generate_floatToFloat16() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_f2hf_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
address entry = __ pc();
BLOCK_COMMENT("Entry:");
__ flt_to_flt16(r0, v0, v1);
__ ret(lr);
+
+ // record the stub entry and end
+ store_archive_data(stub_id, entry, __ pc());
+
return entry;
}
address generate_method_entry_barrier() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_method_entry_barrier_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
Label deoptimize_label;
- address start = __ pc();
+ start = __ pc();
BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
@@ -8974,6 +9699,9 @@ class StubGenerator: public StubCodeGenerator {
__ mov(sp, rscratch1);
__ br(rscratch2);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -8985,8 +9713,14 @@ class StubGenerator: public StubCodeGenerator {
// r10 = tmp1
// r11 = tmp2
address generate_compare_long_string_same_encoding(bool isLL) {
- __ align(CodeEntryAlignment);
StubId stub_id = (isLL ? StubId::stubgen_compare_long_string_LL_id : StubId::stubgen_compare_long_string_UU_id);
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
address entry = __ pc();
Register result = r0, str1 = r1, cnt1 = r2, str2 = r3, cnt2 = r4,
@@ -9094,6 +9828,10 @@ class StubGenerator: public StubCodeGenerator {
__ bind(LENGTH_DIFF);
__ ret(lr);
+
+ // record the stub entry and end
+ store_archive_data(stub_id, entry, __ pc());
+
return entry;
}
@@ -9125,8 +9863,14 @@ class StubGenerator: public StubCodeGenerator {
case UU: stub_id = StubId::stubgen_compare_long_string_UU_id; break;
default: ShouldNotReachHere();
}
-
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
+ StubCodeMark mark(this, stub_id);
address entry = __ pc();
Register result = r0, str1 = r1, cnt1 = r2, str2 = r3, cnt2 = r4,
tmp1 = r10, tmp2 = r11;
@@ -9161,8 +9905,6 @@ class StubGenerator: public StubCodeGenerator {
ShouldNotReachHere(); \
}
- StubCodeMark mark(this, stub_id);
-
__ mov(idx, 0);
__ sve_whilelt(pgtmp1, mode == LL ? __ B : __ H, idx, cnt);
@@ -9206,6 +9948,10 @@ class StubGenerator: public StubCodeGenerator {
__ bind(DONE);
__ ret(lr);
#undef LOAD_PAIR
+
+ // record the stub entry and end
+ store_archive_data(stub_id, entry, __ pc());
+
return entry;
}
@@ -9267,6 +10013,12 @@ class StubGenerator: public StubCodeGenerator {
stub_id = StubId::stubgen_string_indexof_linear_uu_id;
}
}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
address entry = __ pc();
@@ -9535,6 +10287,10 @@ class StubGenerator: public StubCodeGenerator {
__ BIND(DONE);
__ pop(spilled_regs, sp);
__ ret(lr);
+
+ // record the stub entry and end
+ store_archive_data(stub_id, entry, __ pc());
+
return entry;
}
@@ -9565,8 +10321,14 @@ class StubGenerator: public StubCodeGenerator {
// v1 = loaded 8 bytes
// Clobbers: r0, r1, r3, rscratch1, rflags, v0-v6
address generate_large_byte_array_inflate() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_large_byte_array_inflate_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
address entry = __ pc();
Label LOOP, LOOP_START, LOOP_PRFM, LOOP_PRFM_START, DONE;
@@ -9605,6 +10367,10 @@ class StubGenerator: public StubCodeGenerator {
__ br(__ GE, LOOP);
__ bind(DONE);
__ ret(lr);
+
+ // record the stub entry and end
+ store_archive_data(stub_id, entry, __ pc());
+
return entry;
}
@@ -9620,7 +10386,7 @@ class StubGenerator: public StubCodeGenerator {
* Output:
* Updated state at c_rarg0
*/
- address generate_ghash_processBlocks() {
+ address generate_ghash_processBlocks_small() {
// Bafflingly, GCM uses little-endian for the byte order, but
// big-endian for the bit order. For example, the polynomial 1 is
// represented as the 16-byte string 80 00 00 00 | 12 bytes of 00.
@@ -9632,11 +10398,17 @@ class StubGenerator: public StubCodeGenerator {
// that) and keep the data in little-endian bit order through the
// calculation, bit-reversing the inputs and outputs.
- StubId stub_id = StubId::stubgen_ghash_processBlocks_id;
+ StubId stub_id = StubId::stubgen_ghash_processBlocks_small_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
Label polynomial; // local data generated at end of stub
- __ align(CodeEntryAlignment);
- address start = __ pc();
+ start = __ pc();
Register state = c_rarg0;
Register subkeyH = c_rarg1;
@@ -9696,17 +10468,24 @@ class StubGenerator: public StubCodeGenerator {
// 128-bit vector
__ emit_int64(0x87);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
- address generate_ghash_processBlocks_wide() {
- address small = generate_ghash_processBlocks();
-
- StubId stub_id = StubId::stubgen_ghash_processBlocks_wide_id;
- StubCodeMark mark(this, stub_id);
+ address generate_ghash_processBlocks(address small) {
+ StubId stub_id = StubId::stubgen_ghash_processBlocks_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
Label polynomial; // local data generated after stub
__ align(CodeEntryAlignment);
- address start = __ pc();
+ StubCodeMark mark(this, stub_id);
+ start = __ pc();
Register state = c_rarg0;
Register subkeyH = c_rarg1;
@@ -9748,8 +10527,10 @@ class StubGenerator: public StubCodeGenerator {
// 128-bit vector
__ emit_int64(0x87);
- return start;
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+ return start;
}
void generate_base64_encode_simdround(Register src, Register dst,
@@ -9800,26 +10581,16 @@ class StubGenerator: public StubCodeGenerator {
*/
address generate_base64_encodeBlock() {
- static const char toBase64[64] = {
- 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
- 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
- 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
- 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
- '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'
- };
-
- static const char toBase64URL[64] = {
- 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
- 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
- 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
- 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
- '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_'
- };
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_base64_encodeBlock_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Register src = c_rarg0; // source array
Register soff = c_rarg1; // source start offset
@@ -9839,9 +10610,9 @@ class StubGenerator: public StubCodeGenerator {
__ sub(length, send, soff);
// load the codec base address
- __ lea(codec, ExternalAddress((address) toBase64));
+ __ lea(codec, ExternalAddress((address) _encodeBlock_toBase64));
__ cbz(isURL, ProcessData);
- __ lea(codec, ExternalAddress((address) toBase64URL));
+ __ lea(codec, ExternalAddress((address) _encodeBlock_toBase64URL));
__ BIND(ProcessData);
@@ -9894,6 +10665,9 @@ class StubGenerator: public StubCodeGenerator {
__ BIND(Exit);
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -10015,80 +10789,16 @@ class StubGenerator: public StubCodeGenerator {
// on http://0x80.pl/articles/base64-simd-neon.html#encoding-quadwords, in section
// titled "Base64 decoding".
- // Non-SIMD lookup tables are mostly dumped from fromBase64 array used in java.util.Base64,
- // except the trailing character '=' is also treated illegal value in this intrinsic. That
- // is java.util.Base64.fromBase64['='] = -2, while fromBase(URL)64ForNoSIMD['='] = 255 here.
- static const uint8_t fromBase64ForNoSIMD[256] = {
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, 255u, 63u,
- 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, 14u,
- 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, 255u,
- 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, 40u,
- 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- };
-
- static const uint8_t fromBase64URLForNoSIMD[256] = {
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u,
- 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, 14u,
- 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, 63u,
- 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, 40u,
- 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- };
-
- // A legal value of base64 code is in range [0, 127]. We need two lookups
- // with tbl/tbx and combine them to get the decode data. The 1st table vector
- // lookup use tbl, out of range indices are set to 0 in destination. The 2nd
- // table vector lookup use tbx, out of range indices are unchanged in
- // destination. Input [64..126] is mapped to index [65, 127] in second lookup.
- // The value of index 64 is set to 0, so that we know that we already get the
- // decoded data with the 1st lookup.
- static const uint8_t fromBase64ForSIMD[128] = {
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, 255u, 63u,
- 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u,
- 0u, 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u,
- 14u, 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u,
- 255u, 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u,
- 40u, 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u,
- };
-
- static const uint8_t fromBase64URLForSIMD[128] = {
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u,
- 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u,
- 0u, 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u,
- 14u, 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u,
- 63u, 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u,
- 40u, 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u,
- };
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_base64_decodeBlock_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Register src = c_rarg0; // source array
Register soff = c_rarg1; // source start offset
@@ -10115,9 +10825,9 @@ class StubGenerator: public StubCodeGenerator {
__ sub(length, send, soff);
__ bfm(length, zr, 0, 1);
- __ lea(nosimd_codec, ExternalAddress((address) fromBase64ForNoSIMD));
+ __ lea(nosimd_codec, ExternalAddress((address) _decodeBlock_fromBase64ForNoSIMD));
__ cbz(isURL, ProcessData);
- __ lea(nosimd_codec, ExternalAddress((address) fromBase64URLForNoSIMD));
+ __ lea(nosimd_codec, ExternalAddress((address) _decodeBlock_fromBase64URLForNoSIMD));
__ BIND(ProcessData);
__ mov(rscratch1, length);
@@ -10162,9 +10872,9 @@ class StubGenerator: public StubCodeGenerator {
__ cbzw(rscratch1, Exit);
__ sub(length, length, 80);
- __ lea(simd_codec, ExternalAddress((address) fromBase64ForSIMD));
+ __ lea(simd_codec, ExternalAddress((address) _decodeBlock_fromBase64ForSIMD));
__ cbz(isURL, SIMDEnter);
- __ lea(simd_codec, ExternalAddress((address) fromBase64URLForSIMD));
+ __ lea(simd_codec, ExternalAddress((address) _decodeBlock_fromBase64URLForSIMD));
__ BIND(SIMDEnter);
__ ld1(v0, v1, v2, v3, __ T16B, __ post(simd_codec, 64));
@@ -10197,24 +10907,50 @@ class StubGenerator: public StubCodeGenerator {
__ leave();
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
// Support for spin waits.
address generate_spin_wait() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_spin_wait_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ spin_wait();
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
void generate_lookup_secondary_supers_table_stub() {
StubId stub_id = StubId::stubgen_lookup_secondary_supers_table_id;
+ GrowableArray entries;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == Klass::SECONDARY_SUPERS_TABLE_SIZE, "sanity check");
+ address start = load_archive_data(stub_id, &entries);
+ if (start != nullptr) {
+ assert(entries.length() == Klass::SECONDARY_SUPERS_TABLE_SIZE - 1,
+ "unexpected extra entry count %d", entries.length());
+ StubRoutines::_lookup_secondary_supers_table_stubs[0] = start;
+ for (int slot = 1; slot < Klass::SECONDARY_SUPERS_TABLE_SIZE; slot++) {
+ StubRoutines::_lookup_secondary_supers_table_stubs[slot] = entries.at(slot - 1);
+ }
+ return;
+ }
+
StubCodeMark mark(this, stub_id);
const Register
@@ -10229,7 +10965,13 @@ class StubGenerator: public StubCodeGenerator {
vtemp = v0;
for (int slot = 0; slot < Klass::SECONDARY_SUPERS_TABLE_SIZE; slot++) {
- StubRoutines::_lookup_secondary_supers_table_stubs[slot] = __ pc();
+ address next_entry = __ pc();
+ StubRoutines::_lookup_secondary_supers_table_stubs[slot] = next_entry;
+ if (slot == 0) {
+ start = next_entry;
+ } else {
+ entries.append(next_entry);
+ }
Label L_success;
__ enter();
__ lookup_secondary_supers_table_const(r_sub_klass, r_super_klass,
@@ -10239,14 +10981,21 @@ class StubGenerator: public StubCodeGenerator {
__ leave();
__ ret(lr);
}
+ // record the stub entry and end plus all the auxiliary entries
+ store_archive_data(stub_id, start, __ pc(), &entries);
}
// Slow path implementation for UseSecondarySupersTable.
address generate_lookup_secondary_supers_table_slow_path_stub() {
StubId stub_id = StubId::stubgen_lookup_secondary_supers_table_slow_path_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
-
- address start = __ pc();
+ start = __ pc();
const Register
r_super_klass = r0, // argument
r_array_base = r1, // argument
@@ -10258,6 +11007,9 @@ class StubGenerator: public StubCodeGenerator {
__ lookup_secondary_supers_table_slow_path(r_super_klass, r_array_base, r_array_index, r_bitmap, temp1, result);
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -10397,14 +11149,43 @@ class StubGenerator: public StubCodeGenerator {
if (! UseLSE) {
return;
}
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_atomic_entry_points_id;
- StubCodeMark mark(this, stub_id);
- address first_entry = __ pc();
+ GrowableArray entries;
+ int entry_count = StubInfo::entry_count(stub_id);
+ address start = load_archive_data(stub_id, &entries);
+ if (start != nullptr) {
+ assert(entries.length() == entry_count - 1,
+ "unexpected extra entry count %d", entries.length());
+ aarch64_atomic_fetch_add_4_impl = (aarch64_atomic_stub_t)start;
+ int idx = 0;
+ aarch64_atomic_fetch_add_8_impl = (aarch64_atomic_stub_t)entries.at(idx++);
+ aarch64_atomic_fetch_add_4_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++);
+ aarch64_atomic_fetch_add_8_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++);
+ aarch64_atomic_xchg_4_impl = (aarch64_atomic_stub_t)entries.at(idx++);
+ aarch64_atomic_xchg_8_impl = (aarch64_atomic_stub_t)entries.at(idx++);
+ aarch64_atomic_cmpxchg_1_impl = (aarch64_atomic_stub_t)entries.at(idx++);
+ aarch64_atomic_cmpxchg_4_impl = (aarch64_atomic_stub_t)entries.at(idx++);
+ aarch64_atomic_cmpxchg_8_impl = (aarch64_atomic_stub_t)entries.at(idx++);
+ aarch64_atomic_cmpxchg_1_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++);
+ aarch64_atomic_cmpxchg_4_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++);
+ aarch64_atomic_cmpxchg_8_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++);
+ aarch64_atomic_cmpxchg_4_release_impl = (aarch64_atomic_stub_t)entries.at(idx++);
+ aarch64_atomic_cmpxchg_8_release_impl = (aarch64_atomic_stub_t)entries.at(idx++);
+ aarch64_atomic_cmpxchg_4_seq_cst_impl = (aarch64_atomic_stub_t)entries.at(idx++);
+ aarch64_atomic_cmpxchg_8_seq_cst_impl = (aarch64_atomic_stub_t)entries.at(idx++);
+ assert(idx == entries.length(), "sanity!");
+ return;
+ }
+ __ align(CodeEntryAlignment);
+ StubCodeMark mark(this, stub_id);
+ start = __ pc();
+ address end;
+ {
// ADD, memory_order_conservative
AtomicStubMark mark_fetch_add_4(_masm, &aarch64_atomic_fetch_add_4_impl);
gen_ldadd_entry(Assembler::word, memory_order_conservative);
+
AtomicStubMark mark_fetch_add_8(_masm, &aarch64_atomic_fetch_add_8_impl);
gen_ldadd_entry(Assembler::xword, memory_order_conservative);
@@ -10412,6 +11193,7 @@ class StubGenerator: public StubCodeGenerator {
AtomicStubMark mark_fetch_add_4_relaxed
(_masm, &aarch64_atomic_fetch_add_4_relaxed_impl);
gen_ldadd_entry(MacroAssembler::word, memory_order_relaxed);
+
AtomicStubMark mark_fetch_add_8_relaxed
(_masm, &aarch64_atomic_fetch_add_8_relaxed_impl);
gen_ldadd_entry(MacroAssembler::xword, memory_order_relaxed);
@@ -10419,14 +11201,17 @@ class StubGenerator: public StubCodeGenerator {
// XCHG, memory_order_conservative
AtomicStubMark mark_xchg_4(_masm, &aarch64_atomic_xchg_4_impl);
gen_swpal_entry(Assembler::word);
- AtomicStubMark mark_xchg_8_impl(_masm, &aarch64_atomic_xchg_8_impl);
+
+ AtomicStubMark mark_xchg_8(_masm, &aarch64_atomic_xchg_8_impl);
gen_swpal_entry(Assembler::xword);
// CAS, memory_order_conservative
AtomicStubMark mark_cmpxchg_1(_masm, &aarch64_atomic_cmpxchg_1_impl);
gen_cas_entry(MacroAssembler::byte, memory_order_conservative);
+
AtomicStubMark mark_cmpxchg_4(_masm, &aarch64_atomic_cmpxchg_4_impl);
gen_cas_entry(MacroAssembler::word, memory_order_conservative);
+
AtomicStubMark mark_cmpxchg_8(_masm, &aarch64_atomic_cmpxchg_8_impl);
gen_cas_entry(MacroAssembler::xword, memory_order_conservative);
@@ -10434,9 +11219,11 @@ class StubGenerator: public StubCodeGenerator {
AtomicStubMark mark_cmpxchg_1_relaxed
(_masm, &aarch64_atomic_cmpxchg_1_relaxed_impl);
gen_cas_entry(MacroAssembler::byte, memory_order_relaxed);
+
AtomicStubMark mark_cmpxchg_4_relaxed
(_masm, &aarch64_atomic_cmpxchg_4_relaxed_impl);
gen_cas_entry(MacroAssembler::word, memory_order_relaxed);
+
AtomicStubMark mark_cmpxchg_8_relaxed
(_masm, &aarch64_atomic_cmpxchg_8_relaxed_impl);
gen_cas_entry(MacroAssembler::xword, memory_order_relaxed);
@@ -10444,6 +11231,7 @@ class StubGenerator: public StubCodeGenerator {
AtomicStubMark mark_cmpxchg_4_release
(_masm, &aarch64_atomic_cmpxchg_4_release_impl);
gen_cas_entry(MacroAssembler::word, memory_order_release);
+
AtomicStubMark mark_cmpxchg_8_release
(_masm, &aarch64_atomic_cmpxchg_8_release_impl);
gen_cas_entry(MacroAssembler::xword, memory_order_release);
@@ -10451,11 +11239,41 @@ class StubGenerator: public StubCodeGenerator {
AtomicStubMark mark_cmpxchg_4_seq_cst
(_masm, &aarch64_atomic_cmpxchg_4_seq_cst_impl);
gen_cas_entry(MacroAssembler::word, memory_order_seq_cst);
+
AtomicStubMark mark_cmpxchg_8_seq_cst
(_masm, &aarch64_atomic_cmpxchg_8_seq_cst_impl);
gen_cas_entry(MacroAssembler::xword, memory_order_seq_cst);
- ICache::invalidate_range(first_entry, __ pc() - first_entry);
+ end = __ pc();
+
+ ICache::invalidate_range(start, end - start);
+ // exit block to force update of AtomicStubMark targets
+ }
+
+ assert(start == (address)aarch64_atomic_fetch_add_4_impl,
+ "atomic stub should be at start of buffer");
+ // record the stub start and end plus all the entries saved by the
+ // AtomicStubMark destructor
+ entries.append((address)aarch64_atomic_fetch_add_8_impl);
+ entries.append((address)aarch64_atomic_fetch_add_4_relaxed_impl);
+ entries.append((address)aarch64_atomic_fetch_add_8_relaxed_impl);
+ entries.append((address)aarch64_atomic_xchg_4_impl);
+ entries.append((address)aarch64_atomic_xchg_8_impl);
+ entries.append((address)aarch64_atomic_cmpxchg_1_impl);
+ entries.append((address)aarch64_atomic_cmpxchg_4_impl);
+ entries.append((address)aarch64_atomic_cmpxchg_8_impl);
+ entries.append((address)aarch64_atomic_cmpxchg_1_relaxed_impl);
+ entries.append((address)aarch64_atomic_cmpxchg_4_relaxed_impl);
+ entries.append((address)aarch64_atomic_cmpxchg_8_relaxed_impl);
+ entries.append((address)aarch64_atomic_cmpxchg_4_release_impl);
+ entries.append((address)aarch64_atomic_cmpxchg_8_release_impl);
+ entries.append((address)aarch64_atomic_cmpxchg_4_seq_cst_impl);
+ entries.append((address)aarch64_atomic_cmpxchg_8_seq_cst_impl);
+
+ assert(entries.length() == entry_count - 1,
+ "unexpected extra entry count %d", entries.length());
+
+ store_archive_data(stub_id, start, end, &entries);
}
#endif // LINUX
@@ -10559,9 +11377,19 @@ class StubGenerator: public StubCodeGenerator {
if (!Continuations::enabled()) return nullptr;
StubId stub_id = StubId::stubgen_cont_thaw_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
generate_cont_thaw(Continuation::thaw_top);
+
+ // record the stub start and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -10570,11 +11398,20 @@ class StubGenerator: public StubCodeGenerator {
// TODO: will probably need multiple return barriers depending on return type
StubId stub_id = StubId::stubgen_cont_returnBarrier_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
generate_cont_thaw(Continuation::thaw_return_barrier);
+ // record the stub start and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -10582,19 +11419,34 @@ class StubGenerator: public StubCodeGenerator {
if (!Continuations::enabled()) return nullptr;
StubId stub_id = StubId::stubgen_cont_returnBarrierExc_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
generate_cont_thaw(Continuation::thaw_return_barrier_exception);
+ // record the stub start and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address generate_cont_preempt_stub() {
if (!Continuations::enabled()) return nullptr;
StubId stub_id = StubId::stubgen_cont_preempt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ reset_last_Java_frame(true);
@@ -10619,6 +11471,9 @@ class StubGenerator: public StubCodeGenerator {
__ ldr(rscratch1, Address(rscratch1));
__ br(rscratch1);
+ // record the stub start and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -10674,10 +11529,16 @@ class StubGenerator: public StubCodeGenerator {
// computation.
address generate_poly1305_processBlocks() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_poly1305_processBlocks_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label here;
__ enter();
RegSet callee_saved = RegSet::range(r19, r28);
@@ -10785,14 +11646,23 @@ class StubGenerator: public StubCodeGenerator {
__ leave();
__ ret(lr);
+ // record the stub start and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
// exception handler for upcall stubs
address generate_upcall_stub_exception_handler() {
StubId stub_id = StubId::stubgen_upcall_stub_exception_handler_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
// Native caller has no idea how to handle exceptions,
// so we just crash here. Up to callee to catch exceptions.
@@ -10801,6 +11671,9 @@ class StubGenerator: public StubCodeGenerator {
__ blr(rscratch1);
__ should_not_reach_here();
+ // record the stub start and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -10809,8 +11682,14 @@ class StubGenerator: public StubCodeGenerator {
// rmethod = result
address generate_upcall_stub_load_target() {
StubId stub_id = StubId::stubgen_upcall_stub_load_target_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ resolve_global_jobject(j_rarg0, rscratch1, rscratch2);
// Load target method from receiver
@@ -10824,6 +11703,9 @@ class StubGenerator: public StubCodeGenerator {
__ ret(lr);
+ // record the stub start and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -11223,8 +12105,6 @@ class StubGenerator: public StubCodeGenerator {
*/
address generate_multiply() {
Label argh, nothing;
- bind(argh);
- stop("MontgomeryMultiply total_allocation must be <= 8192");
align(CodeEntryAlignment);
address entry = pc();
@@ -11331,6 +12211,10 @@ class StubGenerator: public StubCodeGenerator {
bind(nothing);
ret(lr);
+ // handler for error case
+ bind(argh);
+ stop("MontgomeryMultiply total_allocation must be <= 8192");
+
return entry;
}
// In C, approximately:
@@ -11434,8 +12318,6 @@ class StubGenerator: public StubCodeGenerator {
*/
address generate_square() {
Label argh;
- bind(argh);
- stop("MontgomeryMultiply total_allocation must be <= 8192");
align(CodeEntryAlignment);
address entry = pc();
@@ -11544,6 +12426,10 @@ class StubGenerator: public StubCodeGenerator {
leave();
ret(lr);
+ // handler for error case
+ bind(argh);
+ stop("MontgomeryMultiply total_allocation must be <= 8192");
+
return entry;
}
// In C, approximately:
@@ -11753,7 +12639,7 @@ class StubGenerator: public StubCodeGenerator {
#if COMPILER2_OR_JVMCI
if (UseSVE == 0) {
- StubRoutines::aarch64::_vector_iota_indices = generate_iota_indices(StubId::stubgen_vector_iota_indices_id);
+ generate_iota_indices(StubId::stubgen_vector_iota_indices_id);
}
// array equals stub for large arrays.
@@ -11798,18 +12684,32 @@ class StubGenerator: public StubCodeGenerator {
if (UseMontgomeryMultiplyIntrinsic) {
StubId stub_id = StubId::stubgen_montgomeryMultiply_id;
- StubCodeMark mark(this, stub_id);
- MontgomeryMultiplyGenerator g(_masm, /*squaring*/false);
- StubRoutines::_montgomeryMultiply = g.generate_multiply();
+ address start = load_archive_data(stub_id);
+ if (start == nullptr) {
+ // we have to generate it
+ StubCodeMark mark(this, stub_id);
+ MontgomeryMultiplyGenerator g(_masm, /*squaring*/false);
+ start = g.generate_multiply();
+ // record the stub start and end
+ store_archive_data(stub_id, start, _masm->pc());
+ }
+ StubRoutines::_montgomeryMultiply = start;
}
if (UseMontgomerySquareIntrinsic) {
StubId stub_id = StubId::stubgen_montgomerySquare_id;
- StubCodeMark mark(this, stub_id);
- MontgomeryMultiplyGenerator g(_masm, /*squaring*/true);
- // We use generate_multiply() rather than generate_square()
- // because it's faster for the sizes of modulus we care about.
- StubRoutines::_montgomerySquare = g.generate_multiply();
+ address start = load_archive_data(stub_id);
+ if (start == nullptr) {
+ // we have to generate it
+ StubCodeMark mark(this, stub_id);
+ MontgomeryMultiplyGenerator g(_masm, /*squaring*/true);
+ // We use generate_multiply() rather than generate_square()
+ // because it's faster for the sizes of modulus we care about.
+ start = g.generate_multiply();
+ // record the stub start and end
+ store_archive_data(stub_id, start, _masm->pc());
+ }
+ StubRoutines::_montgomerySquare = start;
}
#endif // COMPILER2
@@ -11854,7 +12754,8 @@ class StubGenerator: public StubCodeGenerator {
}
if (UseGHASHIntrinsics) {
// StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks();
- StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks_wide();
+ StubRoutines::aarch64::_ghash_processBlocks_small = generate_ghash_processBlocks_small();
+ StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks(StubRoutines::aarch64::_ghash_processBlocks_small);
}
if (UseAESIntrinsics && UseGHASHIntrinsics) {
StubRoutines::_galoisCounterMode_AESCrypt = generate_galoisCounterMode_AESCrypt();
@@ -11876,16 +12777,13 @@ class StubGenerator: public StubCodeGenerator {
StubRoutines::_sha512_implCompress = generate_sha512_implCompress(StubId::stubgen_sha512_implCompress_id);
StubRoutines::_sha512_implCompressMB = generate_sha512_implCompress(StubId::stubgen_sha512_implCompressMB_id);
}
- if (UseSHA3Intrinsics) {
-
+ if (UseSHA3Intrinsics && UseSIMDForSHA3Intrinsic) {
StubRoutines::_double_keccak = generate_double_keccak();
- if (UseSIMDForSHA3Intrinsic) {
- StubRoutines::_sha3_implCompress = generate_sha3_implCompress(StubId::stubgen_sha3_implCompress_id);
- StubRoutines::_sha3_implCompressMB = generate_sha3_implCompress(StubId::stubgen_sha3_implCompressMB_id);
- } else {
- StubRoutines::_sha3_implCompress = generate_sha3_implCompress_gpr(StubId::stubgen_sha3_implCompress_id);
- StubRoutines::_sha3_implCompressMB = generate_sha3_implCompress_gpr(StubId::stubgen_sha3_implCompressMB_id);
- }
+ StubRoutines::_sha3_implCompress = generate_sha3_implCompress(StubId::stubgen_sha3_implCompress_id);
+ StubRoutines::_sha3_implCompressMB = generate_sha3_implCompress(StubId::stubgen_sha3_implCompressMB_id);
+ } else if (UseSHA3Intrinsics) {
+ StubRoutines::_sha3_implCompress = generate_sha3_implCompress_gpr(StubId::stubgen_sha3_implCompress_id);
+ StubRoutines::_sha3_implCompressMB = generate_sha3_implCompress_gpr(StubId::stubgen_sha3_implCompressMB_id);
}
if (UsePoly1305Intrinsics) {
@@ -11901,7 +12799,7 @@ class StubGenerator: public StubCodeGenerator {
}
public:
- StubGenerator(CodeBuffer* code, BlobId blob_id) : StubCodeGenerator(code, blob_id) {
+ StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) : StubCodeGenerator(code, blob_id, stub_data) {
switch(blob_id) {
case BlobId::stubgen_preuniverse_id:
generate_preuniverse_stubs();
@@ -11923,12 +12821,35 @@ class StubGenerator: public StubCodeGenerator {
break;
};
}
+
+#if INCLUDE_CDS
+ static void init_AOTAddressTable(GrowableArray& external_addresses) {
+ // external data defined in this file
+#define ADD(addr) external_addresses.append((address)(addr));
+ ADD(_sha256_round_consts);
+ ADD(_sha512_round_consts);
+ ADD(_sha3_round_consts);
+ ADD(_double_keccak_round_consts);
+ ADD(_encodeBlock_toBase64);
+ ADD(_encodeBlock_toBase64URL);
+ ADD(_decodeBlock_fromBase64ForNoSIMD);
+ ADD(_decodeBlock_fromBase64URLForNoSIMD);
+ ADD(_decodeBlock_fromBase64ForSIMD);
+ ADD(_decodeBlock_fromBase64URLForSIMD);
+#undef ADD
+ }
+#endif // INCLUDE_CDS
}; // end class declaration
-void StubGenerator_generate(CodeBuffer* code, BlobId blob_id) {
- StubGenerator g(code, blob_id);
+void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) {
+ StubGenerator g(code, blob_id, stub_data);
}
+#if INCLUDE_CDS
+void StubGenerator_init_AOTAddressTable(GrowableArray& addresses) {
+ StubGenerator::init_AOTAddressTable(addresses);
+}
+#endif // INCLUDE_CDS
#if defined (LINUX)
diff --git a/src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp b/src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp
index 88993818b47..f02b681ca10 100644
--- a/src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp
@@ -41,8 +41,12 @@ static void empty_spin_wait() { }
#define DEFINE_ARCH_ENTRY_INIT(arch, blob_name, stub_name, field_name, getter_name, init_function) \
address StubRoutines:: arch :: STUB_FIELD_NAME(field_name) = CAST_FROM_FN_PTR(address, init_function);
-STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY, DEFINE_ARCH_ENTRY_INIT)
+#define DEFINE_ARCH_ENTRY_ARRAY(arch, blob_name, stub_name, field_name, getter_name, count) \
+ address StubRoutines:: arch :: STUB_FIELD_NAME(field_name) [count];
+STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY, DEFINE_ARCH_ENTRY_INIT, DEFINE_ARCH_ENTRY_ARRAY)
+
+#undef DEFINE_ARCH_ENTRY_ARARAY
#undef DEFINE_ARCH_ENTRY_INIT
#undef DEFINE_ARCH_ENTRY
@@ -413,3 +417,36 @@ ATTRIBUTE_ALIGNED(64) jdouble StubRoutines::aarch64::_pio2[] = {
2.73370053816464559624e-44, // 0x36E3822280000000
2.16741683877804819444e-51, // 0x3569F31D00000000
};
+
+#if INCLUDE_CDS
+extern void StubGenerator_init_AOTAddressTable(GrowableArray& addresses);
+
+void StubRoutines::init_AOTAddressTable() {
+ ResourceMark rm;
+ GrowableArray external_addresses;
+ // publish static addresses referred to by aarch64 generator
+ // n.b. we have to use use an extern call here because class
+ // StubGenerator, which provides the static method that knows how to
+ // add the relevant addresses, is declared in a source file rather
+ // than in a separately includeable header.
+ StubGenerator_init_AOTAddressTable(external_addresses);
+ // publish external data addresses defined in nested aarch64 class
+ StubRoutines::aarch64::init_AOTAddressTable(external_addresses);
+ AOTCodeCache::publish_external_addresses(external_addresses);
+}
+
+void StubRoutines::aarch64::init_AOTAddressTable(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr));
+ ADD(_kyberConsts);
+ ADD(_dilithiumConsts);
+ // this is added in generic code
+ // ADD(_crc_table);
+ ADD(_adler_table);
+ ADD(_npio2_hw);
+ ADD(_dsin_coef);
+ ADD(_dcos_coef);
+ ADD(_two_over_pi);
+ ADD(_pio2);
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp b/src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp
index c35371e1083..6067408ef13 100644
--- a/src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp
@@ -60,9 +60,13 @@ class aarch64 {
#define DECLARE_ARCH_ENTRY_INIT(arch, blob_name, stub_name, field_name, getter_name, init_function) \
DECLARE_ARCH_ENTRY(arch, blob_name, stub_name, field_name, getter_name)
-private:
- STUBGEN_ARCH_ENTRIES_DO(DECLARE_ARCH_ENTRY, DECLARE_ARCH_ENTRY_INIT)
+#define DECLARE_ARCH_ENTRY_ARRAY(arch, blob_name, stub_name, field_name, getter_name, count) \
+ static address STUB_FIELD_NAME(field_name) [count];
+private:
+ STUBGEN_ARCH_ENTRIES_DO(DECLARE_ARCH_ENTRY, DECLARE_ARCH_ENTRY_INIT, DECLARE_ARCH_ENTRY_ARRAY)
+
+#undef DECLARE_ARCH_ENTRY_ARRAY
#undef DECLARE_ARCH_ENTRY_INIT
#undef DECLARE_ARCH_ENTRY
@@ -78,8 +82,15 @@ private:
#define DEFINE_ARCH_ENTRY_GETTER_INIT(arch, blob_name, stub_name, field_name, getter_name, init_function) \
DEFINE_ARCH_ENTRY_GETTER(arch, blob_name, stub_name, field_name, getter_name)
- STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY_GETTER, DEFINE_ARCH_ENTRY_GETTER_INIT)
+#define DEFINE_ARCH_ENTRY_GETTER_ARRAY(arch, blob_name, stub_name, field_name, getter_name, count) \
+ static address getter_name(int idx) { \
+ assert(0 <= idx && idx < count, "entry array index out of range"); \
+ return STUB_FIELD_NAME(field_name) [idx]; \
+ }
+ STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY_GETTER, DEFINE_ARCH_ENTRY_GETTER_INIT, DEFINE_ARCH_ENTRY_GETTER_ARRAY)
+
+#undef DEFINE_ARCH_ENTRY_GETTER_ARRAY
#undef DEFINE_ARCH_ENTRY_GETTER_INIT
#undef DEFINE_ARCH_ENTRY_GETTER
@@ -110,6 +121,11 @@ private:
_completed = true;
}
+#if INCLUDE_CDS
+ static void init_AOTAddressTable(GrowableArray& external_addresses);
+#endif // INCLUDE_CDS
+
+
private:
static uint16_t _kyberConsts[];
static uint32_t _dilithiumConsts[];
diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
index 9b85733ed08..441bd4859fe 100644
--- a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
@@ -1,7 +1,7 @@
/*
* Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2015, 2020, Red Hat Inc. All rights reserved.
- * Copyright 2025 Arm Limited and/or its affiliates.
+ * Copyright 2025, 2026 Arm Limited and/or its affiliates.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -24,6 +24,7 @@
*
*/
+#include "logging/log.hpp"
#include "pauth_aarch64.hpp"
#include "register_aarch64.hpp"
#include "runtime/arguments.hpp"
@@ -52,17 +53,56 @@ uintptr_t VM_Version::_pac_mask;
SpinWait VM_Version::_spin_wait;
+bool VM_Version::_cache_dic_enabled;
+bool VM_Version::_cache_idc_enabled;
+bool VM_Version::_ic_ivau_trapped;
+
const char* VM_Version::_features_names[MAX_CPU_FEATURES] = { nullptr };
static SpinWait get_spin_wait_desc() {
- SpinWait spin_wait(OnSpinWaitInst, OnSpinWaitInstCount);
+ SpinWait spin_wait(OnSpinWaitInst, OnSpinWaitInstCount, OnSpinWaitDelay);
if (spin_wait.inst() == SpinWait::SB && !VM_Version::supports_sb()) {
vm_exit_during_initialization("OnSpinWaitInst is SB but current CPU does not support SB instruction");
}
+ if (spin_wait.inst() == SpinWait::WFET) {
+ if (!VM_Version::supports_wfxt()) {
+ vm_exit_during_initialization("OnSpinWaitInst is WFET but the CPU does not support the WFET instruction");
+ }
+
+ if (!VM_Version::supports_ecv()) {
+ vm_exit_during_initialization("The CPU does not support the FEAT_ECV required by the -XX:OnSpinWaitInst=wfet implementation");
+ }
+
+ if (!VM_Version::supports_sb()) {
+ vm_exit_during_initialization("The CPU does not support the SB instruction required by the -XX:OnSpinWaitInst=wfet implementation");
+ }
+
+ if (OnSpinWaitInstCount != 1) {
+ vm_exit_during_initialization("OnSpinWaitInstCount for OnSpinWaitInst 'wfet' must be 1");
+ }
+ } else {
+ if (!FLAG_IS_DEFAULT(OnSpinWaitDelay)) {
+ vm_exit_during_initialization("OnSpinWaitDelay can only be used with -XX:OnSpinWaitInst=wfet");
+ }
+ }
+
return spin_wait;
}
+static bool has_neoverse_n1_errata_1542419() {
+ const int major_rev_num = VM_Version::cpu_variant();
+ const int minor_rev_num = VM_Version::cpu_revision();
+ // Neoverse N1: 0xd0c
+ // Erratum 1542419 affects r3p0, r3p1 and r4p0.
+ // It is fixed in r4p1 and later revisions, which are not affected.
+ return (VM_Version::cpu_family() == VM_Version::CPU_ARM &&
+ VM_Version::model_is(0xd0c) &&
+ ((major_rev_num == 3 && minor_rev_num == 0) ||
+ (major_rev_num == 3 && minor_rev_num == 1) ||
+ (major_rev_num == 4 && minor_rev_num == 0)));
+}
+
void VM_Version::initialize() {
#define SET_CPU_FEATURE_NAME(id, name, bit) \
_features_names[bit] = XSTR(name);
@@ -74,9 +114,14 @@ void VM_Version::initialize() {
_supports_atomic_getset8 = true;
_supports_atomic_getadd8 = true;
- get_os_cpu_info();
+ _cache_dic_enabled = false;
+ _cache_idc_enabled = false;
+ _ic_ivau_trapped = false;
- int dcache_line = VM_Version::dcache_line_size();
+ get_os_cpu_info();
+ _cpu_features = _features;
+
+ int dcache_line = dcache_line_size();
// Limit AllocatePrefetchDistance so that it does not exceed the
// static constraint of 512 defined in runtime/globals.hpp.
@@ -124,7 +169,7 @@ void VM_Version::initialize() {
// if dcpop is available publish data cache line flush size via
// generic field, otherwise let if default to zero thereby
// disabling writeback
- if (VM_Version::supports_dcpop()) {
+ if (supports_dcpop()) {
_data_cache_line_flush_size = dcache_line;
}
}
@@ -245,14 +290,24 @@ void VM_Version::initialize() {
}
}
- if (FLAG_IS_DEFAULT(UseCRC32)) {
- UseCRC32 = VM_Version::supports_crc32();
+ if (supports_sha1() || supports_sha256() ||
+ supports_sha3() || supports_sha512()) {
+ if (FLAG_IS_DEFAULT(UseSHA)) {
+ FLAG_SET_DEFAULT(UseSHA, true);
+ } else if (!UseSHA) {
+ clear_feature(CPU_SHA1);
+ clear_feature(CPU_SHA2);
+ clear_feature(CPU_SHA3);
+ clear_feature(CPU_SHA512);
+ }
+ } else if (UseSHA) {
+ warning("SHA instructions are not available on this CPU");
+ FLAG_SET_DEFAULT(UseSHA, false);
}
- if (UseCRC32 && !VM_Version::supports_crc32()) {
- warning("UseCRC32 specified, but not supported on this CPU");
- FLAG_SET_DEFAULT(UseCRC32, false);
- }
+ CHECK_CPU_FEATURE(supports_crc32, CRC32);
+ CHECK_CPU_FEATURE(supports_lse, LSE);
+ CHECK_CPU_FEATURE(supports_aes, AES);
if (_cpu == CPU_ARM &&
model_is_in({ CPU_MODEL_ARM_NEOVERSE_V1, CPU_MODEL_ARM_NEOVERSE_V2,
@@ -265,7 +320,7 @@ void VM_Version::initialize() {
}
}
- if (UseCryptoPmullForCRC32 && (!VM_Version::supports_pmull() || !VM_Version::supports_sha3() || !VM_Version::supports_crc32())) {
+ if (UseCryptoPmullForCRC32 && (!supports_pmull() || !supports_sha3() || !supports_crc32())) {
warning("UseCryptoPmullForCRC32 specified, but not supported on this CPU");
FLAG_SET_DEFAULT(UseCryptoPmullForCRC32, false);
}
@@ -279,48 +334,40 @@ void VM_Version::initialize() {
FLAG_SET_DEFAULT(UseVectorizedMismatchIntrinsic, false);
}
- if (VM_Version::supports_lse()) {
- if (FLAG_IS_DEFAULT(UseLSE))
- FLAG_SET_DEFAULT(UseLSE, true);
- } else {
- if (UseLSE) {
- warning("UseLSE specified, but not supported on this CPU");
- FLAG_SET_DEFAULT(UseLSE, false);
- }
- }
-
- if (VM_Version::supports_aes()) {
- UseAES = UseAES || FLAG_IS_DEFAULT(UseAES);
- UseAESIntrinsics =
- UseAESIntrinsics || (UseAES && FLAG_IS_DEFAULT(UseAESIntrinsics));
- if (UseAESIntrinsics && !UseAES) {
- warning("UseAESIntrinsics enabled, but UseAES not, enabling");
- UseAES = true;
+ if (supports_aes()) {
+ if (FLAG_IS_DEFAULT(UseAESIntrinsics)) {
+ FLAG_SET_DEFAULT(UseAESIntrinsics, true);
}
if (FLAG_IS_DEFAULT(UseAESCTRIntrinsics)) {
FLAG_SET_DEFAULT(UseAESCTRIntrinsics, true);
}
} else {
- if (UseAES) {
- warning("AES instructions are not available on this CPU");
- FLAG_SET_DEFAULT(UseAES, false);
- }
- if (UseAESIntrinsics) {
- warning("AES intrinsics are not available on this CPU");
- FLAG_SET_DEFAULT(UseAESIntrinsics, false);
- }
- if (UseAESCTRIntrinsics) {
- warning("AES/CTR intrinsics are not available on this CPU");
- FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
+ if (!UseAES) {
+ if (UseAESIntrinsics) {
+ warning("AES intrinsics require UseAES flag to be enabled. Intrinsics will be disabled.");
+ FLAG_SET_DEFAULT(UseAESIntrinsics, false);
+ }
+ if (UseAESCTRIntrinsics) {
+ warning("AES/CTR intrinsics require UseAES flag to be enabled. Intrinsics will be disabled.");
+ FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
+ }
+ } else if (!cpu_supports_aes()) {
+ if (UseAESIntrinsics) {
+ warning("AES intrinsics are not available on this CPU");
+ FLAG_SET_DEFAULT(UseAESIntrinsics, false);
+ }
+ if (UseAESCTRIntrinsics) {
+ warning("AES/CTR intrinsics are not available on this CPU");
+ FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
+ }
}
}
-
if (FLAG_IS_DEFAULT(UseCRC32Intrinsics)) {
UseCRC32Intrinsics = true;
}
- if (VM_Version::supports_crc32()) {
+ if (supports_crc32()) {
if (FLAG_IS_DEFAULT(UseCRC32CIntrinsics)) {
FLAG_SET_DEFAULT(UseCRC32CIntrinsics, true);
}
@@ -337,17 +384,7 @@ void VM_Version::initialize() {
UseMD5Intrinsics = true;
}
- if (VM_Version::supports_sha1() || VM_Version::supports_sha256() ||
- VM_Version::supports_sha3() || VM_Version::supports_sha512()) {
- if (FLAG_IS_DEFAULT(UseSHA)) {
- FLAG_SET_DEFAULT(UseSHA, true);
- }
- } else if (UseSHA) {
- warning("SHA instructions are not available on this CPU");
- FLAG_SET_DEFAULT(UseSHA, false);
- }
-
- if (UseSHA && VM_Version::supports_sha1()) {
+ if (UseSHA && supports_sha1()) {
if (FLAG_IS_DEFAULT(UseSHA1Intrinsics)) {
FLAG_SET_DEFAULT(UseSHA1Intrinsics, true);
}
@@ -356,7 +393,7 @@ void VM_Version::initialize() {
FLAG_SET_DEFAULT(UseSHA1Intrinsics, false);
}
- if (UseSHA && VM_Version::supports_sha256()) {
+ if (UseSHA && supports_sha256()) {
if (FLAG_IS_DEFAULT(UseSHA256Intrinsics)) {
FLAG_SET_DEFAULT(UseSHA256Intrinsics, true);
}
@@ -365,21 +402,33 @@ void VM_Version::initialize() {
FLAG_SET_DEFAULT(UseSHA256Intrinsics, false);
}
- if (UseSHA && VM_Version::supports_sha3()) {
- // Auto-enable UseSHA3Intrinsics on hardware with performance benefit.
- // Note that the evaluation of UseSHA3Intrinsics shows better performance
+ if (UseSHA) {
+ // No need to check supports_sha3(), since a fallback GPR intrinsic implementation is provided.
+ if (FLAG_IS_DEFAULT(UseSHA3Intrinsics)) {
+ FLAG_SET_DEFAULT(UseSHA3Intrinsics, true);
+ }
+ } else if (UseSHA3Intrinsics) {
+ // Matches the documented and tested behavior: the -UseSHA option disables all SHA intrinsics.
+ warning("UseSHA3Intrinsics requires that UseSHA is enabled.");
+ FLAG_SET_DEFAULT(UseSHA3Intrinsics, false);
+ }
+
+ if (UseSHA3Intrinsics && supports_sha3()) {
+ // Auto-enable UseSIMDForSHA3Intrinsic on hardware with performance benefit.
+ // Note that the evaluation of SHA3 extension Intrinsics shows better performance
// on Apple and Qualcomm silicon but worse performance on Neoverse V1 and N2.
if (_cpu == CPU_APPLE || _cpu == CPU_QUALCOMM) { // Apple or Qualcomm silicon
- if (FLAG_IS_DEFAULT(UseSHA3Intrinsics)) {
- FLAG_SET_DEFAULT(UseSHA3Intrinsics, true);
+ if (FLAG_IS_DEFAULT(UseSIMDForSHA3Intrinsic)) {
+ FLAG_SET_DEFAULT(UseSIMDForSHA3Intrinsic, true);
}
}
- } else if (UseSHA3Intrinsics && UseSIMDForSHA3Intrinsic) {
+ }
+ if (UseSHA3Intrinsics && UseSIMDForSHA3Intrinsic && !supports_sha3()) {
warning("Intrinsics for SHA3-224, SHA3-256, SHA3-384 and SHA3-512 crypto hash functions not available on this CPU.");
FLAG_SET_DEFAULT(UseSHA3Intrinsics, false);
}
- if (UseSHA && VM_Version::supports_sha512()) {
+ if (UseSHA && supports_sha512()) {
if (FLAG_IS_DEFAULT(UseSHA512Intrinsics)) {
FLAG_SET_DEFAULT(UseSHA512Intrinsics, true);
}
@@ -388,11 +437,7 @@ void VM_Version::initialize() {
FLAG_SET_DEFAULT(UseSHA512Intrinsics, false);
}
- if (!(UseSHA1Intrinsics || UseSHA256Intrinsics || UseSHA3Intrinsics || UseSHA512Intrinsics)) {
- FLAG_SET_DEFAULT(UseSHA, false);
- }
-
- if (VM_Version::supports_pmull()) {
+ if (supports_pmull()) {
if (FLAG_IS_DEFAULT(UseGHASHIntrinsics)) {
FLAG_SET_DEFAULT(UseGHASHIntrinsics, true);
}
@@ -443,7 +488,7 @@ void VM_Version::initialize() {
FLAG_SET_DEFAULT(UseBlockZeroing, true);
}
if (FLAG_IS_DEFAULT(BlockZeroingLowLimit)) {
- FLAG_SET_DEFAULT(BlockZeroingLowLimit, 4 * VM_Version::zva_length());
+ FLAG_SET_DEFAULT(BlockZeroingLowLimit, 4 * zva_length());
}
} else if (UseBlockZeroing) {
if (!FLAG_IS_DEFAULT(UseBlockZeroing)) {
@@ -452,11 +497,11 @@ void VM_Version::initialize() {
FLAG_SET_DEFAULT(UseBlockZeroing, false);
}
- if (VM_Version::supports_sve2()) {
+ if (supports_sve2()) {
if (FLAG_IS_DEFAULT(UseSVE)) {
FLAG_SET_DEFAULT(UseSVE, 2);
}
- } else if (VM_Version::supports_sve()) {
+ } else if (supports_sve()) {
if (FLAG_IS_DEFAULT(UseSVE)) {
FLAG_SET_DEFAULT(UseSVE, 1);
} else if (UseSVE > 1) {
@@ -507,7 +552,7 @@ void VM_Version::initialize() {
// 1) this code has been built with branch-protection and
// 2) the CPU/OS supports it
#ifdef __ARM_FEATURE_PAC_DEFAULT
- if (!VM_Version::supports_paca()) {
+ if (!supports_paca()) {
// Disable PAC to prevent illegal instruction crashes.
warning("ROP-protection specified, but not supported on this CPU. Disabling ROP-protection.");
} else {
@@ -649,6 +694,43 @@ void VM_Version::initialize() {
clear_feature(CPU_SVE);
}
+ if (FLAG_IS_DEFAULT(UseSingleICacheInvalidation) && is_cache_idc_enabled() && is_cache_dic_enabled()) {
+ FLAG_SET_DEFAULT(UseSingleICacheInvalidation, true);
+ }
+
+ if (FLAG_IS_DEFAULT(NeoverseN1ICacheErratumMitigation) && has_neoverse_n1_errata_1542419()
+ && is_cache_idc_enabled() && !is_cache_dic_enabled()) {
+ if (_ic_ivau_trapped) {
+ FLAG_SET_DEFAULT(NeoverseN1ICacheErratumMitigation, true);
+ } else {
+ log_info(os)("IC IVAU is not trapped; disabling NeoverseN1ICacheErratumMitigation");
+ FLAG_SET_DEFAULT(NeoverseN1ICacheErratumMitigation, false);
+ }
+ }
+
+ if (NeoverseN1ICacheErratumMitigation) {
+ if (!has_neoverse_n1_errata_1542419()) {
+ vm_exit_during_initialization("NeoverseN1ICacheErratumMitigation is set for the CPU not having Neoverse N1 errata 1542419");
+ }
+ // If the user explicitly set the flag, verify the trap is active.
+ if (!FLAG_IS_DEFAULT(NeoverseN1ICacheErratumMitigation) && !_ic_ivau_trapped) {
+ vm_exit_during_initialization("NeoverseN1ICacheErratumMitigation is set but IC IVAU is not trapped. "
+ "The optimization is not safe on this system.");
+ }
+ if (FLAG_IS_DEFAULT(UseSingleICacheInvalidation)) {
+ FLAG_SET_DEFAULT(UseSingleICacheInvalidation, true);
+ }
+
+ if (!UseSingleICacheInvalidation) {
+ vm_exit_during_initialization("NeoverseN1ICacheErratumMitigation is set but UseSingleICacheInvalidation is not enabled");
+ }
+ }
+
+ if (UseSingleICacheInvalidation
+ && (!is_cache_idc_enabled() || (!is_cache_dic_enabled() && !NeoverseN1ICacheErratumMitigation))) {
+ vm_exit_during_initialization("UseSingleICacheInvalidation is set but neither IDC nor DIC nor NeoverseN1ICacheErratumMitigation is enabled");
+ }
+
// Construct the "features" string
stringStream ss(512);
ss.print("0x%02x:0x%x:0x%03x:%d", _cpu, _variant, _model, _revision);
diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp
index 0213872852b..30f1a5d86ca 100644
--- a/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp
@@ -55,6 +55,15 @@ protected:
static int _max_supported_sve_vector_length;
static bool _rop_protection;
static uintptr_t _pac_mask;
+ // When _prefer_sve_merging_mode_cpy is true, `cpy (imm, zeroing)` is
+ // implemented as `movi; cpy(imm, merging)`.
+ static constexpr bool _prefer_sve_merging_mode_cpy = true;
+ static bool _cache_dic_enabled;
+ static bool _cache_idc_enabled;
+
+ // IC IVAU trap probe for Neoverse N1 erratum 1542419.
+ // Set by get_os_cpu_info() on Linux via ic_ivau_probe_linux_aarch64.S.
+ static bool _ic_ivau_trapped;
static SpinWait _spin_wait;
@@ -156,7 +165,9 @@ public:
/* flags above must follow Linux HWCAP */ \
decl(SVEBITPERM, svebitperm, 27) \
decl(SVE2, sve2, 28) \
- decl(A53MAC, a53mac, 31)
+ decl(A53MAC, a53mac, 31) \
+ decl(ECV, ecv, 32) \
+ decl(WFXT, wfxt, 33)
enum Feature_Flag {
#define DECLARE_CPU_FEATURE_FLAG(id, name, bit) CPU_##id = bit,
@@ -188,6 +199,8 @@ public:
return (features & BIT_MASK(flag)) != 0;
}
+ static bool cpu_supports_aes() { return supports_feature(_cpu_features, CPU_AES); }
+
static int cpu_family() { return _cpu; }
static int cpu_model() { return _model; }
static int cpu_model2() { return _model2; }
@@ -242,12 +255,18 @@ public:
static bool use_rop_protection() { return _rop_protection; }
+ static bool prefer_sve_merging_mode_cpy() { return _prefer_sve_merging_mode_cpy; }
+
// For common 64/128-bit unpredicated vector operations, we may prefer
// emitting NEON instructions rather than the corresponding SVE instructions.
static bool use_neon_for_vector(int vector_length_in_bytes) {
return vector_length_in_bytes <= 16;
}
+ static bool is_cache_dic_enabled() { return _cache_dic_enabled; }
+ static bool is_cache_idc_enabled() { return _cache_idc_enabled; }
+ static bool is_ic_ivau_trapped() { return _ic_ivau_trapped; }
+
static void get_cpu_features_name(void* features_buffer, stringStream& ss);
// Returns names of features present in features_set1 but not in features_set2
diff --git a/src/hotspot/cpu/arm/arm.ad b/src/hotspot/cpu/arm/arm.ad
index 606275d7666..60a0ef307b5 100644
--- a/src/hotspot/cpu/arm/arm.ad
+++ b/src/hotspot/cpu/arm/arm.ad
@@ -1,5 +1,5 @@
//
-// Copyright (c) 2008, 2025, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2008, 2026, Oracle and/or its affiliates. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
@@ -1088,10 +1088,8 @@ bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack,
return clone_base_plus_offset_address(m, mstack, address_visited);
}
-// Return whether or not this register is ever used as an argument. This
-// function is used on startup to build the trampoline stubs in generateOptoStub.
-// Registers not mentioned will be killed by the VM call in the trampoline, and
-// arguments in those registers not be available to the callee.
+#ifdef ASSERT
+// Return whether or not this register is ever used as an argument.
bool Matcher::can_be_java_arg( int reg ) {
if (reg == R_R0_num ||
reg == R_R1_num ||
@@ -1102,10 +1100,7 @@ bool Matcher::can_be_java_arg( int reg ) {
reg <= R_S13_num) return true;
return false;
}
-
-bool Matcher::is_spillable_arg( int reg ) {
- return can_be_java_arg(reg);
-}
+#endif
uint Matcher::int_pressure_limit()
{
@@ -1117,10 +1112,6 @@ uint Matcher::float_pressure_limit()
return (FLOATPRESSURE == -1) ? 30 : FLOATPRESSURE;
}
-bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
- return false;
-}
-
// Register for DIVI projection of divmodI
const RegMask& Matcher::divI_proj_mask() {
ShouldNotReachHere();
@@ -4445,6 +4436,18 @@ instruct membar_release_lock() %{
ins_pipe(empty);
%}
+instruct membar_storeload() %{
+ match(MemBarStoreLoad);
+ ins_cost(4*MEMORY_REF_COST);
+
+ size(4);
+ format %{ "MEMBAR-storeload" %}
+ ins_encode %{
+ __ membar(MacroAssembler::StoreLoad, noreg);
+ %}
+ ins_pipe(long_memory_op);
+%}
+
instruct membar_volatile() %{
match(MemBarVolatile);
ins_cost(4*MEMORY_REF_COST);
@@ -4468,6 +4471,18 @@ instruct unnecessary_membar_volatile() %{
ins_pipe(empty);
%}
+instruct membar_full() %{
+ match(MemBarFull);
+ ins_cost(4*MEMORY_REF_COST);
+
+ size(4);
+ format %{ "MEMBAR-full" %}
+ ins_encode %{
+ __ membar(MacroAssembler::StoreLoad, noreg);
+ %}
+ ins_pipe(long_memory_op);
+%}
+
//----------Register Move Instructions-----------------------------------------
// Cast Index to Pointer for unsafe natives
diff --git a/src/hotspot/cpu/arm/c1_globals_arm.hpp b/src/hotspot/cpu/arm/c1_globals_arm.hpp
index 1fe5f1a23ee..9db999e81b3 100644
--- a/src/hotspot/cpu/arm/c1_globals_arm.hpp
+++ b/src/hotspot/cpu/arm/c1_globals_arm.hpp
@@ -43,7 +43,6 @@ define_pd_global(bool, TieredCompilation, false);
define_pd_global(intx, CompileThreshold, 1500 );
define_pd_global(intx, OnStackReplacePercentage, 933 );
-define_pd_global(size_t, NewSizeThreadIncrease, 4*K );
define_pd_global(size_t, InitialCodeCacheSize, 160*K);
define_pd_global(size_t, ReservedCodeCacheSize, 32*M );
define_pd_global(size_t, NonProfiledCodeHeapSize, 13*M );
@@ -53,7 +52,6 @@ define_pd_global(bool, ProfileInterpreter, false);
define_pd_global(size_t, CodeCacheExpansionSize, 32*K );
define_pd_global(size_t, CodeCacheMinBlockLength, 1);
define_pd_global(size_t, CodeCacheMinimumUseSpace, 400*K);
-define_pd_global(bool, NeverActAsServerClassMachine, true);
define_pd_global(bool, CICompileOSR, true );
#endif // COMPILER2
define_pd_global(bool, UseTypeProfile, false);
diff --git a/src/hotspot/cpu/arm/c2_globals_arm.hpp b/src/hotspot/cpu/arm/c2_globals_arm.hpp
index 0849bd594f0..34da47792ae 100644
--- a/src/hotspot/cpu/arm/c2_globals_arm.hpp
+++ b/src/hotspot/cpu/arm/c2_globals_arm.hpp
@@ -47,7 +47,6 @@ define_pd_global(intx, ConditionalMoveLimit, 4);
// C2 gets to use all the float/double registers
define_pd_global(intx, FreqInlineSize, 175);
define_pd_global(intx, InteriorEntryAlignment, 16); // = CodeEntryAlignment
-define_pd_global(size_t, NewSizeThreadIncrease, ScaleForWordSize(4*K));
// The default setting 16/16 seems to work best.
// (For _228_jack 16/16 is 2% better than 4/4, 16/4, 32/32, 32/16, or 16/32.)
//define_pd_global(intx, OptoLoopAlignment, 16); // = 4*wordSize
@@ -94,7 +93,4 @@ define_pd_global(size_t, CodeCacheMinimumUseSpace, 400*K);
define_pd_global(bool, TrapBasedRangeChecks, false); // Not needed
-// Ergonomics related flags
-define_pd_global(bool, NeverActAsServerClassMachine, false);
-
#endif // CPU_ARM_C2_GLOBALS_ARM_HPP
diff --git a/src/hotspot/cpu/arm/interp_masm_arm.cpp b/src/hotspot/cpu/arm/interp_masm_arm.cpp
index 23ecea24eb2..aee407864ee 100644
--- a/src/hotspot/cpu/arm/interp_masm_arm.cpp
+++ b/src/hotspot/cpu/arm/interp_masm_arm.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2008, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1210,7 +1210,7 @@ void InterpreterMacroAssembler::profile_final_call(Register mdp) {
// Sets mdp, blows Rtemp.
-void InterpreterMacroAssembler::profile_virtual_call(Register mdp, Register receiver, bool receiver_can_be_null) {
+void InterpreterMacroAssembler::profile_virtual_call(Register mdp, Register receiver) {
assert_different_registers(mdp, receiver, Rtemp);
if (ProfileInterpreter) {
@@ -1219,19 +1219,8 @@ void InterpreterMacroAssembler::profile_virtual_call(Register mdp, Register rece
// If no method data exists, go to profile_continue.
test_method_data_pointer(mdp, profile_continue);
- Label skip_receiver_profile;
- if (receiver_can_be_null) {
- Label not_null;
- cbnz(receiver, not_null);
- // We are making a call. Increment the count for null receiver.
- increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp);
- b(skip_receiver_profile);
- bind(not_null);
- }
-
// Record the receiver type.
record_klass_in_profile(receiver, mdp, Rtemp, true);
- bind(skip_receiver_profile);
// The method data pointer needs to be updated to reflect the new target.
update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size()));
diff --git a/src/hotspot/cpu/arm/interp_masm_arm.hpp b/src/hotspot/cpu/arm/interp_masm_arm.hpp
index 530be1c577e..147cd252b2c 100644
--- a/src/hotspot/cpu/arm/interp_masm_arm.hpp
+++ b/src/hotspot/cpu/arm/interp_masm_arm.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2008, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -239,8 +239,7 @@ class InterpreterMacroAssembler: public MacroAssembler {
void profile_call(Register mdp); // Sets mdp, blows Rtemp.
void profile_final_call(Register mdp); // Sets mdp, blows Rtemp.
- void profile_virtual_call(Register mdp, Register receiver, // Sets mdp, blows Rtemp.
- bool receiver_can_be_null = false);
+ void profile_virtual_call(Register mdp, Register receiver); // Sets mdp, blows Rtemp.
void profile_ret(Register mdp, Register return_bci); // Sets mdp, blows R0-R3/R0-R18, Rtemp, LR
void profile_null_seen(Register mdp); // Sets mdp.
void profile_typecheck(Register mdp, Register klass); // Sets mdp, blows Rtemp.
diff --git a/src/hotspot/cpu/arm/matcher_arm.hpp b/src/hotspot/cpu/arm/matcher_arm.hpp
index 6c818e1f20d..7978a5b7090 100644
--- a/src/hotspot/cpu/arm/matcher_arm.hpp
+++ b/src/hotspot/cpu/arm/matcher_arm.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -75,7 +75,6 @@
static bool narrow_klass_use_complex_address() {
NOT_LP64(ShouldNotCallThis());
- assert(UseCompressedClassPointers, "only for compressed klass code");
return false;
}
diff --git a/src/hotspot/cpu/arm/methodHandles_arm.cpp b/src/hotspot/cpu/arm/methodHandles_arm.cpp
index 3710fa33f36..2da14d8ffed 100644
--- a/src/hotspot/cpu/arm/methodHandles_arm.cpp
+++ b/src/hotspot/cpu/arm/methodHandles_arm.cpp
@@ -104,14 +104,13 @@ void MethodHandles::verify_ref_kind(MacroAssembler* _masm, int ref_kind, Registe
__ andr(temp, temp, (unsigned)java_lang_invoke_MemberName::MN_REFERENCE_KIND_MASK);
__ cmp(temp, ref_kind);
__ b(L, eq);
- { char* buf = NEW_C_HEAP_ARRAY(char, 100, mtInternal);
- jio_snprintf(buf, 100, "verify_ref_kind expected %x", ref_kind);
+ const char* msg = ref_kind_to_verify_msg(ref_kind);
if (ref_kind == JVM_REF_invokeVirtual ||
- ref_kind == JVM_REF_invokeSpecial)
+ ref_kind == JVM_REF_invokeSpecial) {
// could do this for all ref_kinds, but would explode assembly code size
- trace_method_handle(_masm, buf);
- __ stop(buf);
+ trace_method_handle(_masm, msg);
}
+ __ stop(msg);
BLOCK_COMMENT("} verify_ref_kind");
__ bind(L);
}
diff --git a/src/hotspot/cpu/arm/stubDeclarations_arm.hpp b/src/hotspot/cpu/arm/stubDeclarations_arm.hpp
index 5f768a205a5..5fb0d4e901f 100644
--- a/src/hotspot/cpu/arm/stubDeclarations_arm.hpp
+++ b/src/hotspot/cpu/arm/stubDeclarations_arm.hpp
@@ -29,7 +29,8 @@
#define STUBGEN_PREUNIVERSE_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(preuniverse, 500) \
do_stub(preuniverse, atomic_load_long) \
do_arch_entry(Arm, preuniverse, atomic_load_long, \
@@ -42,7 +43,8 @@
#define STUBGEN_INITIAL_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(initial, 9000) \
do_stub(initial, idiv_irem) \
do_arch_entry(Arm, initial, idiv_irem, \
@@ -51,14 +53,16 @@
#define STUBGEN_CONTINUATION_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(continuation, 2000) \
#define STUBGEN_COMPILER_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(compiler, 22000) \
do_stub(compiler, partial_subtype_check) \
do_arch_entry(Arm, compiler, partial_subtype_check, \
@@ -68,7 +72,8 @@
#define STUBGEN_FINAL_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(final, 22000) \
diff --git a/src/hotspot/cpu/arm/stubGenerator_arm.cpp b/src/hotspot/cpu/arm/stubGenerator_arm.cpp
index a36ad3a0c47..a705b15eff5 100644
--- a/src/hotspot/cpu/arm/stubGenerator_arm.cpp
+++ b/src/hotspot/cpu/arm/stubGenerator_arm.cpp
@@ -3211,7 +3211,7 @@ class StubGenerator: public StubCodeGenerator {
}
public:
- StubGenerator(CodeBuffer* code, BlobId blob_id) : StubCodeGenerator(code, blob_id) {
+ StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) : StubCodeGenerator(code, blob_id, stub_data) {
switch(blob_id) {
case BlobId::stubgen_preuniverse_id:
generate_preuniverse_stubs();
@@ -3235,8 +3235,8 @@ class StubGenerator: public StubCodeGenerator {
}
}; // end class declaration
-void StubGenerator_generate(CodeBuffer* code, BlobId blob_id) {
- StubGenerator g(code, blob_id);
+void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) {
+ StubGenerator g(code, blob_id, stub_data);
}
// implementation of internal development flag
diff --git a/src/hotspot/cpu/arm/stubRoutines_arm.cpp b/src/hotspot/cpu/arm/stubRoutines_arm.cpp
index a4f2b5e1bd9..38a9b298562 100644
--- a/src/hotspot/cpu/arm/stubRoutines_arm.cpp
+++ b/src/hotspot/cpu/arm/stubRoutines_arm.cpp
@@ -32,10 +32,16 @@
#define DEFINE_ARCH_ENTRY_INIT(arch, blob_name, stub_name, field_name, getter_name, init_function) \
address StubRoutines:: arch :: STUB_FIELD_NAME(field_name) = CAST_FROM_FN_PTR(address, init_function);
-STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY, DEFINE_ARCH_ENTRY_INIT)
+STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY, DEFINE_ARCH_ENTRY_INIT, DEFINE_ARCH_ENTRY_ARRAY)
#undef DEFINE_ARCH_ENTRY_INIT
#undef DEFINE_ARCH_ENTRY
address StubRoutines::crc_table_addr() { ShouldNotCallThis(); return nullptr; }
address StubRoutines::crc32c_table_addr() { ShouldNotCallThis(); return nullptr; }
+
+#if INCLUDE_CDS
+// nothing to do for arm
+void StubRoutines::init_AOTAddressTable() {
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/arm/stubRoutines_arm.hpp b/src/hotspot/cpu/arm/stubRoutines_arm.hpp
index 45ab10d14f9..29d96d0e653 100644
--- a/src/hotspot/cpu/arm/stubRoutines_arm.hpp
+++ b/src/hotspot/cpu/arm/stubRoutines_arm.hpp
@@ -55,9 +55,13 @@ class Arm {
#define DECLARE_ARCH_ENTRY_INIT(arch, blob_name, stub_name, field_name, getter_name, init_function) \
DECLARE_ARCH_ENTRY(arch, blob_name, stub_name, field_name, getter_name)
-private:
- STUBGEN_ARCH_ENTRIES_DO(DECLARE_ARCH_ENTRY, DECLARE_ARCH_ENTRY_INIT)
+#define DECLARE_ARCH_ENTRY_ARRAY(arch, blob_name, stub_name, field_name, getter_name, count) \
+ static address STUB_FIELD_NAME(field_name) [count] ;
+private:
+ STUBGEN_ARCH_ENTRIES_DO(DECLARE_ARCH_ENTRY, DECLARE_ARCH_ENTRY_INIT, DECLARE_ARCH_ENTRY_ARRAY)
+
+#undef DECLARE_ARCH_ENTRY_ARRAY
#undef DECLARE_ARCH_ENTRY_INIT
#undef DECLARE_ARCH_ENTRY
@@ -71,8 +75,12 @@ public:
#define DEFINE_ARCH_ENTRY_GETTER_INIT(arch, blob_name, stub_name, field_name, getter_name, init_function) \
DEFINE_ARCH_ENTRY_GETTER(arch, blob_name, stub_name, field_name, getter_name)
- STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY_GETTER, DEFINE_ARCH_ENTRY_GETTER_INIT)
+#define DEFINE_ARCH_ENTRY_GETTER_ARRAY(arch, blob_name, stub_name, field_name, getter_name, count) \
+ static address getter_name(int idx) { return STUB_FIELD_NAME(field_name) [idx] ; }
+ STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY_GETTER, DEFINE_ARCH_ENTRY_GETTER_INIT, DEFINE_ARCH_ENTRY_GETTER_ARRAY)
+
+#undef DEFINE_ARCH_ENTRY_GETTER_ARRAY
#undef DEFINE_ARCH_ENTRY_GETTER_INIT
#undef DEFINE_ARCH_ENTRY_GETTER
diff --git a/src/hotspot/cpu/ppc/assembler_ppc.hpp b/src/hotspot/cpu/ppc/assembler_ppc.hpp
index da2daffd579..378e01fc1cc 100644
--- a/src/hotspot/cpu/ppc/assembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/assembler_ppc.hpp
@@ -1580,10 +1580,6 @@ class Assembler : public AbstractAssembler {
static bool is_nop(int x) {
return x == 0x60000000;
}
- // endgroup opcode for Power6
- static bool is_endgroup(int x) {
- return is_ori(x) && inv_ra_field(x) == 1 && inv_rs_field(x) == 1 && inv_d1_field(x) == 0;
- }
private:
@@ -1659,9 +1655,6 @@ class Assembler : public AbstractAssembler {
inline void ori_opt( Register d, int ui16);
inline void oris_opt(Register d, int ui16);
- // endgroup opcode for Power6
- inline void endgroup();
-
// count instructions
inline void cntlzw( Register a, Register s);
inline void cntlzw_( Register a, Register s);
diff --git a/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp b/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp
index bd6f3300606..d349bbc6f87 100644
--- a/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp
+++ b/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp
@@ -253,8 +253,6 @@ inline void Assembler::mr( Register d, Register s) { Assembler::orr(d, s,
inline void Assembler::ori_opt( Register d, int ui16) { if (ui16!=0) Assembler::ori( d, d, ui16); }
inline void Assembler::oris_opt(Register d, int ui16) { if (ui16!=0) Assembler::oris(d, d, ui16); }
-inline void Assembler::endgroup() { Assembler::ori(R1, R1, 0); }
-
// count instructions
inline void Assembler::cntlzw( Register a, Register s) { emit_int32(CNTLZW_OPCODE | rta(a) | rs(s) | rc(0)); }
inline void Assembler::cntlzw_( Register a, Register s) { emit_int32(CNTLZW_OPCODE | rta(a) | rs(s) | rc(1)); }
diff --git a/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp
index 798451446e5..4d7af0e4a71 100644
--- a/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2025 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -144,7 +144,7 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
if (len->is_valid()) {
stw(len, arrayOopDesc::length_offset_in_bytes(), obj);
- } else if (UseCompressedClassPointers && !UseCompactObjectHeaders) {
+ } else if (!UseCompactObjectHeaders) {
// Otherwise length is in the class gap.
store_klass_gap(obj);
}
diff --git a/src/hotspot/cpu/ppc/c1_globals_ppc.hpp b/src/hotspot/cpu/ppc/c1_globals_ppc.hpp
index 77d9acd1cd1..c6fe15aac07 100644
--- a/src/hotspot/cpu/ppc/c1_globals_ppc.hpp
+++ b/src/hotspot/cpu/ppc/c1_globals_ppc.hpp
@@ -51,8 +51,6 @@ define_pd_global(size_t, NonNMethodCodeHeapSize, 5*M );
define_pd_global(size_t, CodeCacheExpansionSize, 32*K);
define_pd_global(size_t, CodeCacheMinBlockLength, 1);
define_pd_global(size_t, CodeCacheMinimumUseSpace, 400*K);
-define_pd_global(bool, NeverActAsServerClassMachine, true);
-define_pd_global(size_t, NewSizeThreadIncrease, 16*K);
define_pd_global(size_t, InitialCodeCacheSize, 160*K);
#endif // !COMPILER2
diff --git a/src/hotspot/cpu/ppc/c2_globals_ppc.hpp b/src/hotspot/cpu/ppc/c2_globals_ppc.hpp
index caef322d4a1..e4942fa1850 100644
--- a/src/hotspot/cpu/ppc/c2_globals_ppc.hpp
+++ b/src/hotspot/cpu/ppc/c2_globals_ppc.hpp
@@ -47,7 +47,6 @@ define_pd_global(intx, ConditionalMoveLimit, 3);
define_pd_global(intx, FreqInlineSize, 325);
define_pd_global(intx, MinJumpTableSize, 10);
define_pd_global(intx, InteriorEntryAlignment, 16);
-define_pd_global(size_t, NewSizeThreadIncrease, ScaleForWordSize(4*K));
define_pd_global(intx, RegisterCostAreaRatio, 16000);
define_pd_global(intx, LoopUnrollLimit, 60);
define_pd_global(intx, LoopPercentProfileLimit, 10);
@@ -91,7 +90,4 @@ define_pd_global(size_t, CodeCacheMinimumUseSpace, 400*K);
define_pd_global(bool, TrapBasedRangeChecks, true);
-// Ergonomics related flags
-define_pd_global(bool, NeverActAsServerClassMachine, false);
-
#endif // CPU_PPC_C2_GLOBALS_PPC_HPP
diff --git a/src/hotspot/cpu/ppc/disassembler_ppc.cpp b/src/hotspot/cpu/ppc/disassembler_ppc.cpp
index fb3cb50cdec..2e16e1a301f 100644
--- a/src/hotspot/cpu/ppc/disassembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/disassembler_ppc.cpp
@@ -119,9 +119,6 @@ address Disassembler::decode_instruction0(address here, outputStream * st, addre
} else if (instruction == 0xbadbabe) {
st->print(".data 0xbadbabe");
next = here + Assembler::instr_len(here);
- } else if (Assembler::is_endgroup(instruction)) {
- st->print("endgroup");
- next = here + Assembler::instr_len(here);
} else {
next = here;
}
diff --git a/src/hotspot/cpu/ppc/downcallLinker_ppc.cpp b/src/hotspot/cpu/ppc/downcallLinker_ppc.cpp
index f12d25ac611..d149fc33ac3 100644
--- a/src/hotspot/cpu/ppc/downcallLinker_ppc.cpp
+++ b/src/hotspot/cpu/ppc/downcallLinker_ppc.cpp
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2020, 2025 SAP SE. All rights reserved.
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -135,10 +135,10 @@ void DowncallLinker::StubGenerator::generate() {
bool should_save_return_value = !_needs_return_buffer;
RegSpiller out_reg_spiller(_output_registers);
- int spill_offset = -1;
+ int out_spill_offset = -1;
if (should_save_return_value) {
- spill_offset = frame::native_abi_reg_args_size;
+ out_spill_offset = frame::native_abi_reg_args_size;
// Spill area can be shared with additional out args (>8),
// since it is only used after the call.
int frame_size_including_spill_area = frame::native_abi_reg_args_size + out_reg_spiller.spill_size_bytes();
@@ -170,6 +170,18 @@ void DowncallLinker::StubGenerator::generate() {
ArgumentShuffle arg_shuffle(filtered_java_regs, out_regs, _abi._scratch1);
+ // Need to spill for state capturing runtime call.
+ // The area spilled into is distinct from the capture state buffer.
+ RegSpiller in_reg_spiller(out_regs);
+ int in_spill_offset = -1;
+ if (_captured_state_mask != 0) {
+ // The spill area cannot be shared with the out_spill since
+ // spilling needs to happen before the call. Allocate a new
+ // region in the stack for this spill space.
+ in_spill_offset = allocated_frame_size;
+ allocated_frame_size += in_reg_spiller.spill_size_bytes();
+ }
+
#ifndef PRODUCT
LogTarget(Trace, foreign, downcall) lt;
if (lt.is_enabled()) {
@@ -211,6 +223,21 @@ void DowncallLinker::StubGenerator::generate() {
arg_shuffle.generate(_masm, as_VMStorage(callerSP), frame::jit_out_preserve_size, frame::native_abi_minframe_size);
__ block_comment("} argument shuffle");
+ if (_captured_state_mask != 0) {
+ assert(in_spill_offset != -1, "must be");
+ __ block_comment("{ load initial thread local");
+ in_reg_spiller.generate_spill(_masm, in_spill_offset);
+
+ // Copy the contents of the capture state buffer into thread local
+ __ load_const_optimized(call_target_address, CAST_FROM_FN_PTR(uint64_t, DowncallLinker::capture_state_pre), R0);
+ __ ld(R3_ARG1, locs.data_offset(StubLocations::CAPTURED_STATE_BUFFER), R1_SP);
+ __ load_const_optimized(R4_ARG2, _captured_state_mask, R0);
+ __ call_c(call_target_address);
+
+ in_reg_spiller.generate_fill(_masm, in_spill_offset);
+ __ block_comment("} load initial thread local");
+ }
+
__ call_c(call_target_address);
if (_needs_return_buffer) {
@@ -247,16 +274,16 @@ void DowncallLinker::StubGenerator::generate() {
__ block_comment("{ save thread local");
if (should_save_return_value) {
- out_reg_spiller.generate_spill(_masm, spill_offset);
+ out_reg_spiller.generate_spill(_masm, out_spill_offset);
}
- __ load_const_optimized(call_target_address, CAST_FROM_FN_PTR(uint64_t, DowncallLinker::capture_state), R0);
+ __ load_const_optimized(call_target_address, CAST_FROM_FN_PTR(uint64_t, DowncallLinker::capture_state_post), R0);
__ ld(R3_ARG1, locs.data_offset(StubLocations::CAPTURED_STATE_BUFFER), R1_SP);
__ load_const_optimized(R4_ARG2, _captured_state_mask, R0);
__ call_c(call_target_address);
if (should_save_return_value) {
- out_reg_spiller.generate_fill(_masm, spill_offset);
+ out_reg_spiller.generate_fill(_masm, out_spill_offset);
}
__ block_comment("} save thread local");
@@ -310,7 +337,7 @@ void DowncallLinker::StubGenerator::generate() {
if (should_save_return_value) {
// Need to save the native result registers around any runtime calls.
- out_reg_spiller.generate_spill(_masm, spill_offset);
+ out_reg_spiller.generate_spill(_masm, out_spill_offset);
}
__ load_const_optimized(call_target_address, CAST_FROM_FN_PTR(uint64_t, JavaThread::check_special_condition_for_native_trans), R0);
@@ -318,7 +345,7 @@ void DowncallLinker::StubGenerator::generate() {
__ call_c(call_target_address);
if (should_save_return_value) {
- out_reg_spiller.generate_fill(_masm, spill_offset);
+ out_reg_spiller.generate_fill(_masm, out_spill_offset);
}
__ b(L_after_safepoint_poll);
@@ -330,14 +357,14 @@ void DowncallLinker::StubGenerator::generate() {
__ bind(L_reguard);
if (should_save_return_value) {
- out_reg_spiller.generate_spill(_masm, spill_offset);
+ out_reg_spiller.generate_spill(_masm, out_spill_offset);
}
__ load_const_optimized(call_target_address, CAST_FROM_FN_PTR(uint64_t, SharedRuntime::reguard_yellow_pages), R0);
__ call_c(call_target_address);
if (should_save_return_value) {
- out_reg_spiller.generate_fill(_masm, spill_offset);
+ out_reg_spiller.generate_fill(_masm, out_spill_offset);
}
__ b(L_after_reguard);
diff --git a/src/hotspot/cpu/ppc/frame_ppc.inline.hpp b/src/hotspot/cpu/ppc/frame_ppc.inline.hpp
index bb711f2d053..3c05f950d0c 100644
--- a/src/hotspot/cpu/ppc/frame_ppc.inline.hpp
+++ b/src/hotspot/cpu/ppc/frame_ppc.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2025 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -137,10 +137,10 @@ inline frame::frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address
// Return unique id for this frame. The id must have a value where we
// can distinguish identity and younger/older relationship. null
-// represents an invalid (incomparable) frame.
+// represents an invalid (incomparable) frame. Should not be called for heap frames.
inline intptr_t* frame::id(void) const {
// Use _fp. _sp or _unextended_sp wouldn't be correct due to resizing.
- return _fp;
+ return real_fp();
}
// Return true if this frame is older (less recent activation) than
@@ -319,6 +319,9 @@ inline frame frame::sender(RegisterMap* map) const {
StackWatermarkSet::on_iteration(map->thread(), result);
}
+ // Calling frame::id() is currently not supported for heap frames.
+ assert(result._on_heap || this->_on_heap || result.is_older(this->id()), "Must be");
+
return result;
}
diff --git a/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.cpp b/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.cpp
index 82d06f6c685..3692b247989 100644
--- a/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.cpp
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2018, 2025 SAP SE. All rights reserved.
+ * Copyright (c) 2018, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -179,6 +179,11 @@ void BarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Re
__ ld(dst, 0, dst); // Resolve (untagged) jobject.
}
+void BarrierSetAssembler::try_resolve_weak_handle(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path) {
+ // Load the oop from the weak handle.
+ __ ld(obj, 0, obj);
+}
+
void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm, Register tmp) {
BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
assert_different_registers(tmp, R0);
@@ -275,11 +280,6 @@ OptoReg::Name BarrierSetAssembler::refine_register(const Node* node, OptoReg::Na
return opto_reg;
}
-void BarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path) {
- // Load the oop from the weak handle.
- __ ld(obj, 0, obj);
-}
-
#undef __
#define __ _masm->
diff --git a/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.hpp b/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.hpp
index d78071f2ee0..8112542d761 100644
--- a/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.hpp
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2018, 2022 SAP SE. All rights reserved.
+ * Copyright (c) 2018, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -70,6 +70,12 @@ public:
virtual void try_resolve_jobject_in_native(MacroAssembler* masm, Register dst, Register jni_env,
Register obj, Register tmp, Label& slowpath);
+ // Can be used in nmethods including native wrappers.
+ // Attention: obj will only be valid until next safepoint (no SATB barrier).
+ // TODO: maybe rename to try_peek_weak_handle on all platforms (try: operation may fail, peek: obj is not kept alive)
+ // (other platforms currently use it for C2 only: try_resolve_weak_handle_in_c2)
+ virtual void try_resolve_weak_handle(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path);
+
virtual void barrier_stubs_init() {}
virtual NMethodPatchingType nmethod_patching_type() { return NMethodPatchingType::stw_instruction_and_data_patch; }
@@ -81,8 +87,6 @@ public:
#ifdef COMPILER2
OptoReg::Name refine_register(const Node* node, OptoReg::Name opto_reg) const;
- virtual void try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj,
- Register tmp, Label& slow_path);
#endif // COMPILER2
};
diff --git a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp
index e1f0416d65d..8e99d23cc99 100644
--- a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp
@@ -1,7 +1,7 @@
/*
* Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, 2025, Red Hat, Inc. All rights reserved.
- * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -663,17 +663,16 @@ void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler
__ block_comment("} try_resolve_jobject_in_native (shenandoahgc)");
}
-#ifdef COMPILER2
-void ShenandoahBarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler *masm, Register obj,
- Register tmp, Label &slow_path) {
- __ block_comment("try_resolve_weak_handle_in_c2 (shenandoahgc) {");
+void ShenandoahBarrierSetAssembler::try_resolve_weak_handle(MacroAssembler *masm, Register obj,
+ Register tmp, Label &slow_path) {
+ __ block_comment("try_resolve_weak_handle (shenandoahgc) {");
assert_different_registers(obj, tmp);
Label done;
// Resolve weak handle using the standard implementation.
- BarrierSetAssembler::try_resolve_weak_handle_in_c2(masm, obj, tmp, slow_path);
+ BarrierSetAssembler::try_resolve_weak_handle(masm, obj, tmp, slow_path);
// Check if the reference is null, and if it is, take the fast path.
__ cmpdi(CR0, obj, 0);
@@ -686,9 +685,8 @@ void ShenandoahBarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler
__ bne(CR0, slow_path);
__ bind(done);
- __ block_comment("} try_resolve_weak_handle_in_c2 (shenandoahgc)");
+ __ block_comment("} try_resolve_weak_handle (shenandoahgc)");
}
-#endif
// Special shenandoah CAS implementation that handles false negatives due
// to concurrent evacuation. That is, the CAS operation is intended to succeed in
diff --git a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.hpp b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.hpp
index 672f8122bcb..58180c49642 100644
--- a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.hpp
@@ -1,7 +1,7 @@
/*
* Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, 2022, Red Hat, Inc. All rights reserved.
- * Copyright (c) 2012, 2022 SAP SE. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -122,9 +122,8 @@ public:
virtual void try_resolve_jobject_in_native(MacroAssembler* masm, Register dst, Register jni_env,
Register obj, Register tmp, Label& slowpath);
-#ifdef COMPILER2
- virtual void try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path);
-#endif
+
+ virtual void try_resolve_weak_handle(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path);
};
#endif // CPU_PPC_GC_SHENANDOAH_SHENANDOAHBARRIERSETASSEMBLER_PPC_HPP
diff --git a/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.cpp b/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.cpp
index bfa3c87c179..3e74dfb88cb 100644
--- a/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.cpp
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2021, 2025 SAP SE. All rights reserved.
+ * Copyright (c) 2021, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -627,6 +627,19 @@ void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, R
__ block_comment("} try_resolve_jobject_in_native (zgc)");
}
+void ZBarrierSetAssembler::try_resolve_weak_handle(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path) {
+ // Resolve weak handle using the standard implementation.
+ BarrierSetAssembler::try_resolve_weak_handle(masm, obj, tmp, slow_path);
+
+ // Check if the oop is bad, in which case we need to take the slow path.
+ __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatMarkBadMask);
+ __ andi_(R0, obj, barrier_Relocation::unpatched);
+ __ bne(CR0, slow_path);
+
+ // Oop is okay, so we uncolor it.
+ __ srdi(obj, obj, ZPointerLoadShift);
+}
+
#undef __
#ifdef COMPILER1
@@ -950,19 +963,6 @@ void ZBarrierSetAssembler::generate_c2_store_barrier_stub(MacroAssembler* masm,
__ b(*stub->continuation());
}
-void ZBarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path) {
- // Resolve weak handle using the standard implementation.
- BarrierSetAssembler::try_resolve_weak_handle_in_c2(masm, obj, tmp, slow_path);
-
- // Check if the oop is bad, in which case we need to take the slow path.
- __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatMarkBadMask);
- __ andi_(R0, obj, barrier_Relocation::unpatched);
- __ bne(CR0, slow_path);
-
- // Oop is okay, so we uncolor it.
- __ srdi(obj, obj, ZPointerLoadShift);
-}
-
#undef __
#endif // COMPILER2
diff --git a/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.hpp b/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.hpp
index e31817370d9..655184cf6a3 100644
--- a/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.hpp
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2021, 2022 SAP SE. All rights reserved.
+ * Copyright (c) 2021, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -72,6 +72,8 @@ public:
virtual void try_resolve_jobject_in_native(MacroAssembler* masm, Register dst, Register jni_env,
Register obj, Register tmp, Label& slowpath);
+ virtual void try_resolve_weak_handle(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path);
+
virtual void check_oop(MacroAssembler *masm, Register obj, const char* msg);
virtual NMethodPatchingType nmethod_patching_type() { return NMethodPatchingType::conc_instruction_and_data_patch; }
@@ -108,8 +110,6 @@ public:
void generate_c2_load_barrier_stub(MacroAssembler* masm, ZLoadBarrierStubC2* stub) const;
void generate_c2_store_barrier_stub(MacroAssembler* masm, ZStoreBarrierStubC2* stub) const;
-
- void try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path);
#endif // COMPILER2
void store_barrier_fast(MacroAssembler* masm,
diff --git a/src/hotspot/cpu/ppc/icache_ppc.cpp b/src/hotspot/cpu/ppc/icache_ppc.cpp
index 05ad3c7a30d..f3d51bad18c 100644
--- a/src/hotspot/cpu/ppc/icache_ppc.cpp
+++ b/src/hotspot/cpu/ppc/icache_ppc.cpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2018 SAP SE. All rights reserved.
+ * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -24,6 +24,7 @@
*/
#include "runtime/icache.hpp"
+#include "runtime/vm_version.hpp"
// Use inline assembler to implement icache flush.
int ICache::ppc64_flush_icache(address start, int lines, int magic) {
@@ -67,6 +68,9 @@ int ICache::ppc64_flush_icache(address start, int lines, int magic) {
void ICacheStubGenerator::generate_icache_flush(ICache::flush_icache_stub_t* flush_icache_stub) {
+ guarantee(VM_Version::get_icache_line_size() >= ICache::line_size,
+ "processors with smaller cache line size are no longer supported");
+
*flush_icache_stub = (ICache::flush_icache_stub_t)ICache::ppc64_flush_icache;
// First call to flush itself.
diff --git a/src/hotspot/cpu/ppc/icache_ppc.hpp b/src/hotspot/cpu/ppc/icache_ppc.hpp
index d348cad1c72..024f706182a 100644
--- a/src/hotspot/cpu/ppc/icache_ppc.hpp
+++ b/src/hotspot/cpu/ppc/icache_ppc.hpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2002, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2013 SAP SE. All rights reserved.
+ * Copyright (c) 2002, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -35,9 +35,8 @@ class ICache : public AbstractICache {
public:
enum {
- // Actually, cache line size is 64, but keeping it as it is to be
- // on the safe side on ALL PPC64 implementations.
- log2_line_size = 5,
+ // Cache line size is 128 on all supported PPC64 implementations.
+ log2_line_size = 7,
line_size = 1 << log2_line_size
};
diff --git a/src/hotspot/cpu/ppc/interp_masm_ppc.hpp b/src/hotspot/cpu/ppc/interp_masm_ppc.hpp
index 4ea33ebaf63..275ff92c699 100644
--- a/src/hotspot/cpu/ppc/interp_masm_ppc.hpp
+++ b/src/hotspot/cpu/ppc/interp_masm_ppc.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2002, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2002, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2025 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -258,7 +258,7 @@ class InterpreterMacroAssembler: public MacroAssembler {
void profile_not_taken_branch(Register scratch1, Register scratch2);
void profile_call(Register scratch1, Register scratch2);
void profile_final_call(Register scratch1, Register scratch2);
- void profile_virtual_call(Register Rreceiver, Register Rscratch1, Register Rscratch2, bool receiver_can_be_null);
+ void profile_virtual_call(Register Rreceiver, Register Rscratch1, Register Rscratch2);
void profile_typecheck(Register Rklass, Register Rscratch1, Register Rscratch2);
void profile_ret(TosState state, Register return_bci, Register scratch1, Register scratch2);
void profile_switch_default(Register scratch1, Register scratch2);
diff --git a/src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp b/src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp
index f7bf457f72c..56eade8e533 100644
--- a/src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp
+++ b/src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2025 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -1340,28 +1340,15 @@ void InterpreterMacroAssembler::profile_final_call(Register scratch1, Register s
// Count a virtual call in the bytecodes.
void InterpreterMacroAssembler::profile_virtual_call(Register Rreceiver,
Register Rscratch1,
- Register Rscratch2,
- bool receiver_can_be_null) {
+ Register Rscratch2) {
if (!ProfileInterpreter) { return; }
Label profile_continue;
// If no method data exists, go to profile_continue.
test_method_data_pointer(profile_continue);
- Label skip_receiver_profile;
- if (receiver_can_be_null) {
- Label not_null;
- cmpdi(CR0, Rreceiver, 0);
- bne(CR0, not_null);
- // We are making a call. Increment the count for null receiver.
- increment_mdp_data_at(in_bytes(CounterData::count_offset()), Rscratch1, Rscratch2);
- b(skip_receiver_profile);
- bind(not_null);
- }
-
// Record the receiver type.
record_klass_in_profile(Rreceiver, Rscratch1, Rscratch2);
- bind(skip_receiver_profile);
// The method data pointer needs to be updated to reflect the new target.
update_mdp_by_constant(in_bytes(VirtualCallData::virtual_call_data_size()));
diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp
index 986dd335816..5fbcce94029 100644
--- a/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp
@@ -1,6 +1,6 @@
/*
* Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -483,7 +483,7 @@ void MacroAssembler::set_dest_of_bc_far_at(address instruction_addr, address des
// variant 3, far cond branch to the next instruction, already patched to nops:
//
// nop
- // endgroup
+ // nop
// SKIP/DEST:
//
return;
@@ -500,7 +500,7 @@ void MacroAssembler::set_dest_of_bc_far_at(address instruction_addr, address des
if (is_bc_far_variant2_at(instruction_addr) && dest == instruction_addr + 8) {
// Far branch to next instruction: Optimize it by patching nops (produce variant 3).
masm.nop();
- masm.endgroup();
+ masm.nop();
} else {
if (is_bc_far_variant1_at(instruction_addr)) {
// variant 1, the 1st instruction contains the destination address:
@@ -2800,7 +2800,7 @@ void MacroAssembler::compiler_fast_lock_object(ConditionRegister flag, Register
// Check if object matches.
ld(tmp3, in_bytes(ObjectMonitor::object_offset()), monitor);
BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
- bs_asm->try_resolve_weak_handle_in_c2(this, tmp3, tmp2, slow_path);
+ bs_asm->try_resolve_weak_handle(this, tmp3, tmp2, slow_path);
cmpd(CR0, tmp3, obj);
bne(CR0, slow_path);
@@ -3201,23 +3201,17 @@ Register MacroAssembler::encode_klass_not_null(Register dst, Register src) {
void MacroAssembler::store_klass(Register dst_oop, Register klass, Register ck) {
assert(!UseCompactObjectHeaders, "not with compact headers");
- if (UseCompressedClassPointers) {
- Register compressedKlass = encode_klass_not_null(ck, klass);
- stw(compressedKlass, oopDesc::klass_offset_in_bytes(), dst_oop);
- } else {
- std(klass, oopDesc::klass_offset_in_bytes(), dst_oop);
- }
+ Register compressedKlass = encode_klass_not_null(ck, klass);
+ stw(compressedKlass, oopDesc::klass_offset_in_bytes(), dst_oop);
}
void MacroAssembler::store_klass_gap(Register dst_oop, Register val) {
assert(!UseCompactObjectHeaders, "not with compact headers");
- if (UseCompressedClassPointers) {
- if (val == noreg) {
- val = R0;
- li(val, 0);
- }
- stw(val, oopDesc::klass_gap_offset_in_bytes(), dst_oop);
+ if (val == noreg) {
+ val = R0;
+ li(val, 0);
}
+ stw(val, oopDesc::klass_gap_offset_in_bytes(), dst_oop);
}
int MacroAssembler::instr_size_for_decode_klass_not_null() {
@@ -3226,17 +3220,13 @@ int MacroAssembler::instr_size_for_decode_klass_not_null() {
// Not yet computed?
if (computed_size == -1) {
- if (!UseCompressedClassPointers) {
- computed_size = 0;
- } else {
- // Determine by scratch emit.
- ResourceMark rm;
- int code_size = 8 * BytesPerInstWord;
- CodeBuffer cb("decode_klass_not_null scratch buffer", code_size, 0);
- MacroAssembler* a = new MacroAssembler(&cb);
- a->decode_klass_not_null(R11_scratch1);
- computed_size = a->offset();
- }
+ // Determine by scratch emit.
+ ResourceMark rm;
+ int code_size = 8 * BytesPerInstWord;
+ CodeBuffer cb("decode_klass_not_null scratch buffer", code_size, 0);
+ MacroAssembler* a = new MacroAssembler(&cb);
+ a->decode_klass_not_null(R11_scratch1);
+ computed_size = a->offset();
}
return computed_size;
@@ -3259,18 +3249,14 @@ void MacroAssembler::decode_klass_not_null(Register dst, Register src) {
void MacroAssembler::load_klass_no_decode(Register dst, Register src) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(dst, src);
- } else if (UseCompressedClassPointers) {
- lwz(dst, oopDesc::klass_offset_in_bytes(), src);
} else {
- ld(dst, oopDesc::klass_offset_in_bytes(), src);
+ lwz(dst, oopDesc::klass_offset_in_bytes(), src);
}
}
void MacroAssembler::load_klass(Register dst, Register src) {
load_klass_no_decode(dst, src);
- if (UseCompressedClassPointers) { // also true for UseCompactObjectHeaders
- decode_klass_not_null(dst);
- }
+ decode_klass_not_null(dst);
}
// Loads the obj's Klass* into dst.
@@ -3286,18 +3272,13 @@ void MacroAssembler::load_narrow_klass_compact(Register dst, Register src) {
void MacroAssembler::cmp_klass(ConditionRegister dst, Register obj, Register klass, Register tmp, Register tmp2) {
assert_different_registers(obj, klass, tmp);
- if (UseCompressedClassPointers) {
- if (UseCompactObjectHeaders) {
- load_narrow_klass_compact(tmp, obj);
- } else {
- lwz(tmp, oopDesc::klass_offset_in_bytes(), obj);
- }
- Register encoded_klass = encode_klass_not_null(tmp2, klass);
- cmpw(dst, tmp, encoded_klass);
+ if (UseCompactObjectHeaders) {
+ load_narrow_klass_compact(tmp, obj);
} else {
- ld(tmp, oopDesc::klass_offset_in_bytes(), obj);
- cmpd(dst, tmp, klass);
+ lwz(tmp, oopDesc::klass_offset_in_bytes(), obj);
}
+ Register encoded_klass = encode_klass_not_null(tmp2, klass);
+ cmpw(dst, tmp, encoded_klass);
}
void MacroAssembler::cmp_klasses_from_objects(ConditionRegister dst, Register obj1, Register obj2, Register tmp1, Register tmp2) {
@@ -3305,14 +3286,10 @@ void MacroAssembler::cmp_klasses_from_objects(ConditionRegister dst, Register ob
load_narrow_klass_compact(tmp1, obj1);
load_narrow_klass_compact(tmp2, obj2);
cmpw(dst, tmp1, tmp2);
- } else if (UseCompressedClassPointers) {
+ } else {
lwz(tmp1, oopDesc::klass_offset_in_bytes(), obj1);
lwz(tmp2, oopDesc::klass_offset_in_bytes(), obj2);
cmpw(dst, tmp1, tmp2);
- } else {
- ld(tmp1, oopDesc::klass_offset_in_bytes(), obj1);
- ld(tmp2, oopDesc::klass_offset_in_bytes(), obj2);
- cmpd(dst, tmp1, tmp2);
}
}
diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp
index 58dec702d7a..4be62098bdf 100644
--- a/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp
@@ -70,14 +70,6 @@ class MacroAssembler: public Assembler {
// Move register if destination register and target register are different
inline void mr_if_needed(Register rd, Register rs, bool allow_invalid = false);
inline void fmr_if_needed(FloatRegister rd, FloatRegister rs);
- // This is dedicated for emitting scheduled mach nodes. For better
- // readability of the ad file I put it here.
- // Endgroups are not needed if
- // - the scheduler is off
- // - the scheduler found that there is a natural group end, in that
- // case it reduced the size of the instruction used in the test
- // yielding 'needed'.
- inline void endgroup_if_needed(bool needed);
// Memory barriers.
inline void membar(int bits);
diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.inline.hpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.inline.hpp
index 2b19d84c69c..cdeb8527bea 100644
--- a/src/hotspot/cpu/ppc/macroAssembler_ppc.inline.hpp
+++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.inline.hpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2002, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
+ * Copyright (c) 2002, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -72,11 +72,6 @@ inline void MacroAssembler::mr_if_needed(Register rd, Register rs, bool allow_no
inline void MacroAssembler::fmr_if_needed(FloatRegister rd, FloatRegister rs) {
if (rs != rd) fmr(rd, rs);
}
-inline void MacroAssembler::endgroup_if_needed(bool needed) {
- if (needed) {
- endgroup();
- }
-}
inline void MacroAssembler::membar(int bits) {
// Comment: Usage of elemental_membar(bits) is not recommended for Power 8.
@@ -240,13 +235,13 @@ inline bool MacroAssembler::is_bc_far_variant3_at(address instruction_addr) {
// Variant 3, far cond branch to the next instruction, already patched to nops:
//
// nop
- // endgroup
+ // nop
// SKIP/DEST:
//
const int instruction_1 = *(int*)(instruction_addr);
const int instruction_2 = *(int*)(instruction_addr + 4);
return is_nop(instruction_1) &&
- is_endgroup(instruction_2);
+ is_nop(instruction_2);
}
// set dst to -1, 0, +1 as follows: if CR0bi is "greater than", dst is set to 1,
diff --git a/src/hotspot/cpu/ppc/matcher_ppc.hpp b/src/hotspot/cpu/ppc/matcher_ppc.hpp
index 2ddbec3e48c..cbe882648b8 100644
--- a/src/hotspot/cpu/ppc/matcher_ppc.hpp
+++ b/src/hotspot/cpu/ppc/matcher_ppc.hpp
@@ -87,7 +87,6 @@
static bool narrow_klass_use_complex_address() {
NOT_LP64(ShouldNotCallThis());
- assert(UseCompressedClassPointers, "only for compressed klass code");
// TODO: PPC port if (MatchDecodeNodes) return true;
return false;
}
diff --git a/src/hotspot/cpu/ppc/methodHandles_ppc.cpp b/src/hotspot/cpu/ppc/methodHandles_ppc.cpp
index 45537e0ea96..ae94a9618b5 100644
--- a/src/hotspot/cpu/ppc/methodHandles_ppc.cpp
+++ b/src/hotspot/cpu/ppc/methodHandles_ppc.cpp
@@ -104,14 +104,13 @@ void MethodHandles::verify_ref_kind(MacroAssembler* _masm, int ref_kind, Registe
__ andi(temp, temp, java_lang_invoke_MemberName::MN_REFERENCE_KIND_MASK);
__ cmpwi(CR1, temp, ref_kind);
__ beq(CR1, L);
- { char* buf = NEW_C_HEAP_ARRAY(char, 100, mtInternal);
- jio_snprintf(buf, 100, "verify_ref_kind expected %x", ref_kind);
- if (ref_kind == JVM_REF_invokeVirtual ||
- ref_kind == JVM_REF_invokeSpecial)
- // could do this for all ref_kinds, but would explode assembly code size
- trace_method_handle(_masm, buf);
- __ stop(buf);
+ const char* msg = ref_kind_to_verify_msg(ref_kind);
+ if (ref_kind == JVM_REF_invokeVirtual ||
+ ref_kind == JVM_REF_invokeSpecial) {
+ // could do this for all ref_kinds, but would explode assembly code size
+ trace_method_handle(_masm, msg);
}
+ __ stop(msg);
BLOCK_COMMENT("} verify_ref_kind");
__ BIND(L);
}
diff --git a/src/hotspot/cpu/ppc/ppc.ad b/src/hotspot/cpu/ppc/ppc.ad
index 4cb9f8820a0..f3d33b4305d 100644
--- a/src/hotspot/cpu/ppc/ppc.ad
+++ b/src/hotspot/cpu/ppc/ppc.ad
@@ -2412,10 +2412,8 @@ bool Matcher::is_generic_vector(MachOper* opnd) {
return false;
}
-// Return whether or not this register is ever used as an argument. This
-// function is used on startup to build the trampoline stubs in generateOptoStub.
-// Registers not mentioned will be killed by the VM call in the trampoline, and
-// arguments in those registers not be available to the callee.
+#ifdef ASSERT
+// Return whether or not this register is ever used as an argument.
bool Matcher::can_be_java_arg(int reg) {
// We must include the virtual halves in order to get STDs and LDs
// instead of STWs and LWs in the trampoline stubs.
@@ -2447,10 +2445,7 @@ bool Matcher::can_be_java_arg(int reg) {
return false;
}
-
-bool Matcher::is_spillable_arg(int reg) {
- return can_be_java_arg(reg);
-}
+#endif
uint Matcher::int_pressure_limit()
{
@@ -2462,10 +2457,6 @@ uint Matcher::float_pressure_limit()
return (FLOATPRESSURE == -1) ? 28 : FLOATPRESSURE;
}
-bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
- return false;
-}
-
// Register for DIVI projection of divmodI.
const RegMask& Matcher::divI_proj_mask() {
ShouldNotReachHere();
@@ -3715,13 +3706,6 @@ frame %{
// Compiled code's Frame Pointer.
frame_pointer(R1); // R1_SP
- // Interpreter stores its frame pointer in a register which is
- // stored to the stack by I2CAdaptors. I2CAdaptors convert from
- // interpreted java to compiled java.
- //
- // R14_state holds pointer to caller's cInterpreter.
- interpreter_frame_pointer(R14); // R14_state
-
stack_alignment(frame::alignment_in_bytes);
// Number of outgoing stack slots killed above the
@@ -6339,36 +6323,8 @@ instruct loadConD_Ex(regD dst, immD src) %{
// Prefetch instructions.
// Must be safe to execute with invalid address (cannot fault).
-// Special prefetch versions which use the dcbz instruction.
-instruct prefetch_alloc_zero(indirectMemory mem, iRegLsrc src) %{
- match(PrefetchAllocation (AddP mem src));
- predicate(AllocatePrefetchStyle == 3);
- ins_cost(MEMORY_REF_COST);
-
- format %{ "PREFETCH $mem, 2, $src \t// Prefetch write-many with zero" %}
- size(4);
- ins_encode %{
- __ dcbz($src$$Register, $mem$$base$$Register);
- %}
- ins_pipe(pipe_class_memory);
-%}
-
-instruct prefetch_alloc_zero_no_offset(indirectMemory mem) %{
- match(PrefetchAllocation mem);
- predicate(AllocatePrefetchStyle == 3);
- ins_cost(MEMORY_REF_COST);
-
- format %{ "PREFETCH $mem, 2 \t// Prefetch write-many with zero" %}
- size(4);
- ins_encode %{
- __ dcbz($mem$$base$$Register);
- %}
- ins_pipe(pipe_class_memory);
-%}
-
instruct prefetch_alloc(indirectMemory mem, iRegLsrc src) %{
match(PrefetchAllocation (AddP mem src));
- predicate(AllocatePrefetchStyle != 3);
ins_cost(MEMORY_REF_COST);
format %{ "PREFETCH $mem, 2, $src \t// Prefetch write-many" %}
@@ -6381,7 +6337,6 @@ instruct prefetch_alloc(indirectMemory mem, iRegLsrc src) %{
instruct prefetch_alloc_no_offset(indirectMemory mem) %{
match(PrefetchAllocation mem);
- predicate(AllocatePrefetchStyle != 3);
ins_cost(MEMORY_REF_COST);
format %{ "PREFETCH $mem, 2 \t// Prefetch write-many" %}
@@ -7175,6 +7130,18 @@ instruct membar_release_lock() %{
ins_pipe(pipe_class_default);
%}
+instruct membar_storeload() %{
+ match(MemBarStoreLoad);
+ ins_cost(4*MEMORY_REF_COST);
+
+ format %{ "MEMBAR-store-load" %}
+ size(4);
+ ins_encode %{
+ __ fence();
+ %}
+ ins_pipe(pipe_class_default);
+%}
+
instruct membar_volatile() %{
match(MemBarVolatile);
ins_cost(4*MEMORY_REF_COST);
@@ -7217,6 +7184,18 @@ instruct membar_volatile() %{
// ins_pipe(pipe_class_default);
//%}
+instruct membar_full() %{
+ match(MemBarFull);
+ ins_cost(4*MEMORY_REF_COST);
+
+ format %{ "MEMBAR-full" %}
+ size(4);
+ ins_encode %{
+ __ fence();
+ %}
+ ins_pipe(pipe_class_default);
+%}
+
instruct membar_CPUOrder() %{
match(MemBarCPUOrder);
ins_cost(0);
@@ -10324,7 +10303,7 @@ instruct cmovI_bso_stackSlotL(iRegIdst dst, flagsRegSrc crx, stackSlotL src) %{
ins_variable_size_depending_on_alignment(true);
- format %{ "cmovI $crx, $dst, $src" %}
+ format %{ "CMOVI $crx, $dst, $src" %}
size(8);
ins_encode( enc_cmove_bso_stackSlotL(dst, crx, src) );
ins_pipe(pipe_class_default);
@@ -10337,7 +10316,7 @@ instruct cmovI_bso_reg(iRegIdst dst, flagsRegSrc crx, regD src) %{
ins_variable_size_depending_on_alignment(true);
- format %{ "cmovI $crx, $dst, $src" %}
+ format %{ "CMOVI $crx, $dst, $src" %}
size(8);
ins_encode( enc_cmove_bso_reg(dst, crx, src) );
ins_pipe(pipe_class_default);
@@ -10349,7 +10328,7 @@ instruct cmovI_bso_reg_conLvalue0_Ex(iRegIdst dst, flagsRegSrc crx, regD src) %{
effect(DEF dst, USE crx, USE src);
predicate(false);
- format %{ "CmovI $dst, $crx, $src \t// postalloc expanded" %}
+ format %{ "CMOVI $dst, $crx, $src \t// postalloc expanded" %}
postalloc_expand %{
//
// replaces
@@ -10499,7 +10478,7 @@ instruct cmovL_bso_stackSlotL(iRegLdst dst, flagsRegSrc crx, stackSlotL src) %{
ins_variable_size_depending_on_alignment(true);
- format %{ "cmovL $crx, $dst, $src" %}
+ format %{ "CMOVL $crx, $dst, $src" %}
size(8);
ins_encode( enc_cmove_bso_stackSlotL(dst, crx, src) );
ins_pipe(pipe_class_default);
@@ -10512,7 +10491,7 @@ instruct cmovL_bso_reg(iRegLdst dst, flagsRegSrc crx, regD src) %{
ins_variable_size_depending_on_alignment(true);
- format %{ "cmovL $crx, $dst, $src" %}
+ format %{ "CMOVL $crx, $dst, $src" %}
size(8);
ins_encode( enc_cmove_bso_reg(dst, crx, src) );
ins_pipe(pipe_class_default);
@@ -10524,7 +10503,7 @@ instruct cmovL_bso_reg_conLvalue0_Ex(iRegLdst dst, flagsRegSrc crx, regD src) %{
effect(DEF dst, USE crx, USE src);
predicate(false);
- format %{ "CmovL $dst, $crx, $src \t// postalloc expanded" %}
+ format %{ "CMOVL $dst, $crx, $src \t// postalloc expanded" %}
postalloc_expand %{
//
// replaces
@@ -10725,9 +10704,9 @@ instruct convF2HF_reg_reg(iRegIdst dst, regF src, regF tmp) %{
effect(TEMP tmp);
ins_cost(3 * DEFAULT_COST);
size(12);
- format %{ "xscvdphp $tmp, $src\t# convert to half precision\n\t"
- "mffprd $dst, $tmp\t# move result from $tmp to $dst\n\t"
- "extsh $dst, $dst\t# make it a proper short"
+ format %{ "XSCVDPHP $tmp, $src\t# convert to half precision\n\t"
+ "MFFPRD $dst, $tmp\t# move result from $tmp to $dst\n\t"
+ "EXTSH $dst, $dst\t# make it a proper short"
%}
ins_encode %{
__ f2hf($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
@@ -10739,8 +10718,8 @@ instruct convHF2F_reg_reg(regF dst, iRegIsrc src) %{
match(Set dst (ConvHF2F src));
ins_cost(2 * DEFAULT_COST);
size(8);
- format %{ "mtfprd $dst, $src\t# move source from $src to $dst\n\t"
- "xscvhpdp $dst, $dst\t# convert from half precision"
+ format %{ "MTFPRD $dst, $src\t# move source from $src to $dst\n\t"
+ "XSCVHPDP $dst, $dst\t# convert from half precision"
%}
ins_encode %{
__ hf2f($dst$$FloatRegister, $src$$Register);
@@ -11138,7 +11117,7 @@ instruct cmov_bns_less(flagsReg crx) %{
ins_variable_size_depending_on_alignment(true);
- format %{ "cmov $crx" %}
+ format %{ "CMOV $crx" %}
size(12);
ins_encode %{
Label done;
@@ -11166,7 +11145,7 @@ instruct cmpF_reg_reg_Ex(flagsReg crx, regF src1, regF src2) %{
match(Set crx (CmpF src1 src2));
ins_cost(DEFAULT_COST+BRANCH_COST);
- format %{ "CmpF $crx, $src1, $src2 \t// postalloc expanded" %}
+ format %{ "CMPF $crx, $src1, $src2 \t// postalloc expanded" %}
postalloc_expand %{
//
// replaces
@@ -12324,7 +12303,7 @@ instruct minF(regF dst, regF src1, regF src2) %{
predicate(PowerArchitecturePPC64 >= 9);
ins_cost(DEFAULT_COST);
- format %{ "MinF $dst, $src1, $src2" %}
+ format %{ "XSMINJDP $dst, $src1, $src2\t// MinF" %}
size(4);
ins_encode %{
__ xsminjdp($dst$$FloatRegister->to_vsr(), $src1$$FloatRegister->to_vsr(), $src2$$FloatRegister->to_vsr());
@@ -12337,7 +12316,7 @@ instruct minD(regD dst, regD src1, regD src2) %{
predicate(PowerArchitecturePPC64 >= 9);
ins_cost(DEFAULT_COST);
- format %{ "MinD $dst, $src1, $src2" %}
+ format %{ "XSMINJDP $dst, $src1, $src2\t// MinD" %}
size(4);
ins_encode %{
__ xsminjdp($dst$$FloatRegister->to_vsr(), $src1$$FloatRegister->to_vsr(), $src2$$FloatRegister->to_vsr());
@@ -12350,7 +12329,7 @@ instruct maxF(regF dst, regF src1, regF src2) %{
predicate(PowerArchitecturePPC64 >= 9);
ins_cost(DEFAULT_COST);
- format %{ "MaxF $dst, $src1, $src2" %}
+ format %{ "XSMAXJDP $dst, $src1, $src2\t// MaxF" %}
size(4);
ins_encode %{
__ xsmaxjdp($dst$$FloatRegister->to_vsr(), $src1$$FloatRegister->to_vsr(), $src2$$FloatRegister->to_vsr());
@@ -12363,7 +12342,7 @@ instruct maxD(regD dst, regD src1, regD src2) %{
predicate(PowerArchitecturePPC64 >= 9);
ins_cost(DEFAULT_COST);
- format %{ "MaxD $dst, $src1, $src2" %}
+ format %{ "XSMAXJDP $dst, $src1, $src2\t// MaxD" %}
size(4);
ins_encode %{
__ xsmaxjdp($dst$$FloatRegister->to_vsr(), $src1$$FloatRegister->to_vsr(), $src2$$FloatRegister->to_vsr());
@@ -13893,7 +13872,7 @@ instruct vfma2D_neg2(vecX dst, vecX src1, vecX src2) %{
instruct overflowAddL_reg_reg(flagsRegCR0 cr0, iRegLsrc op1, iRegLsrc op2) %{
match(Set cr0 (OverflowAddL op1 op2));
- format %{ "add_ $op1, $op2\t# overflow check long" %}
+ format %{ "ADD_ $op1, $op2\t# overflow check long" %}
size(12);
ins_encode %{
__ li(R0, 0);
@@ -13906,7 +13885,7 @@ instruct overflowAddL_reg_reg(flagsRegCR0 cr0, iRegLsrc op1, iRegLsrc op2) %{
instruct overflowSubL_reg_reg(flagsRegCR0 cr0, iRegLsrc op1, iRegLsrc op2) %{
match(Set cr0 (OverflowSubL op1 op2));
- format %{ "subfo_ R0, $op2, $op1\t# overflow check long" %}
+ format %{ "SUBFO_ R0, $op2, $op1\t# overflow check long" %}
size(12);
ins_encode %{
__ li(R0, 0);
@@ -13919,7 +13898,7 @@ instruct overflowSubL_reg_reg(flagsRegCR0 cr0, iRegLsrc op1, iRegLsrc op2) %{
instruct overflowNegL_reg(flagsRegCR0 cr0, immL_0 zero, iRegLsrc op2) %{
match(Set cr0 (OverflowSubL zero op2));
- format %{ "nego_ R0, $op2\t# overflow check long" %}
+ format %{ "NEGO_ R0, $op2\t# overflow check long" %}
size(12);
ins_encode %{
__ li(R0, 0);
@@ -13932,7 +13911,7 @@ instruct overflowNegL_reg(flagsRegCR0 cr0, immL_0 zero, iRegLsrc op2) %{
instruct overflowMulL_reg_reg(flagsRegCR0 cr0, iRegLsrc op1, iRegLsrc op2) %{
match(Set cr0 (OverflowMulL op1 op2));
- format %{ "mulldo_ R0, $op1, $op2\t# overflow check long" %}
+ format %{ "MULLDO_ R0, $op1, $op2\t# overflow check long" %}
size(12);
ins_encode %{
__ li(R0, 0);
@@ -14309,7 +14288,7 @@ instruct ForwardExceptionjmp()
match(ForwardException);
ins_cost(CALL_COST);
- format %{ "Jmp forward_exception_stub" %}
+ format %{ "JMP forward_exception_stub" %}
ins_encode %{
__ set_inst_mark();
__ b64_patchable(StubRoutines::forward_exception_entry(), relocInfo::runtime_call_type);
@@ -14337,7 +14316,7 @@ instruct RethrowException() %{
match(Rethrow);
ins_cost(CALL_COST);
- format %{ "Jmp rethrow_stub" %}
+ format %{ "JMP rethrow_stub" %}
ins_encode %{
__ set_inst_mark();
__ b64_patchable((address)OptoRuntime::rethrow_stub(), relocInfo::runtime_call_type);
@@ -14379,20 +14358,6 @@ instruct tlsLoadP(threadRegP dst) %{
//---Some PPC specific nodes---------------------------------------------------
-// Stop a group.
-instruct endGroup() %{
- ins_cost(0);
-
- ins_is_nop(true);
-
- format %{ "End Bundle (ori r1, r1, 0)" %}
- size(4);
- ins_encode %{
- __ endgroup();
- %}
- ins_pipe(pipe_class_default);
-%}
-
// Nop instructions
instruct fxNop() %{
diff --git a/src/hotspot/cpu/ppc/registerMap_ppc.cpp b/src/hotspot/cpu/ppc/registerMap_ppc.cpp
new file mode 100644
index 00000000000..2e7f8af89d3
--- /dev/null
+++ b/src/hotspot/cpu/ppc/registerMap_ppc.cpp
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2026 SAP SE. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "runtime/registerMap.hpp"
+
+address RegisterMap::pd_location(VMReg base_reg, int slot_idx) const {
+ if (base_reg->is_VectorRegister()) {
+ // Not all physical slots belonging to a VectorRegister have corresponding
+ // valid VMReg locations in the RegisterMap.
+ // (See RegisterSaver::push_frame_reg_args_and_save_live_registers.)
+ // However, the slots are always saved to the stack in a contiguous region
+ // of memory so we can calculate the address of the upper slots by
+ // offsetting from the base address.
+ assert(base_reg->is_concrete(), "must pass base reg");
+ address base_location = location(base_reg, nullptr);
+ if (base_location != nullptr) {
+ intptr_t offset_in_bytes = slot_idx * VMRegImpl::stack_slot_size;
+ return base_location + offset_in_bytes;
+ } else {
+ return nullptr;
+ }
+ } else {
+ return location(base_reg->next(slot_idx), nullptr);
+ }
+}
diff --git a/src/hotspot/cpu/ppc/registerMap_ppc.hpp b/src/hotspot/cpu/ppc/registerMap_ppc.hpp
index 01eb642107c..607c712d10f 100644
--- a/src/hotspot/cpu/ppc/registerMap_ppc.hpp
+++ b/src/hotspot/cpu/ppc/registerMap_ppc.hpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2013 SAP SE. All rights reserved.
+ * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -35,9 +35,7 @@
// Since there is none, we just return null.
address pd_location(VMReg reg) const { return nullptr; }
- address pd_location(VMReg base_reg, int slot_idx) const {
- return location(base_reg->next(slot_idx), nullptr);
- }
+ address pd_location(VMReg base_reg, int slot_idx) const;
// no PD state to clear or copy:
void pd_clear() {}
diff --git a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
index 5260ed978ff..53644210415 100644
--- a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
+++ b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
@@ -102,7 +102,7 @@ class RegisterSaver {
// During deoptimization only the result registers need to be restored
// all the other values have already been extracted.
- static void restore_result_registers(MacroAssembler* masm, int frame_size_in_bytes);
+ static void restore_result_registers(MacroAssembler* masm, int frame_size_in_bytes, bool save_vectors);
// Constants and data structures:
@@ -349,7 +349,7 @@ OopMap* RegisterSaver::push_frame_reg_args_and_save_live_registers(MacroAssemble
}
// Note that generate_oop_map in the following loop is only used for the
- // polling_page_vectors_safepoint_handler_blob.
+ // polling_page_vectors_safepoint_handler_blob and the deopt_blob.
// The order in which the vector contents are stored depends on Endianess and
// the utilized instructions (PowerArchitecturePPC64).
assert(is_aligned(offset, StackAlignmentInBytes), "should be");
@@ -361,6 +361,7 @@ OopMap* RegisterSaver::push_frame_reg_args_and_save_live_registers(MacroAssemble
__ stxvp(as_VectorRegister(reg_num).to_vsr(), offset, R1_SP);
// Note: The contents were read in the same order (see loadV16_Power9 node in ppc.ad).
+ // RegisterMap::pd_location only uses the first VMReg for each VectorRegister.
if (generate_oop_map) {
map->set_callee_saved(VMRegImpl::stack2reg(offset >> 2),
RegisterSaver_LiveVecRegs[i LITTLE_ENDIAN_ONLY(+1) ].vmreg);
@@ -380,6 +381,7 @@ OopMap* RegisterSaver::push_frame_reg_args_and_save_live_registers(MacroAssemble
__ stxvd2x(as_VectorRegister(reg_num)->to_vsr(), R31, R1_SP);
}
// Note: The contents were read in the same order (see loadV16_Power8 / loadV16_Power9 node in ppc.ad).
+ // RegisterMap::pd_location only uses the first VMReg for each VectorRegister.
if (generate_oop_map) {
VMReg vsr = RegisterSaver_LiveVecRegs[i].vmreg;
map->set_callee_saved(VMRegImpl::stack2reg(offset >> 2), vsr);
@@ -566,10 +568,14 @@ void RegisterSaver::restore_argument_registers_and_pop_frame(MacroAssembler*masm
}
// Restore the registers that might be holding a result.
-void RegisterSaver::restore_result_registers(MacroAssembler* masm, int frame_size_in_bytes) {
+void RegisterSaver::restore_result_registers(MacroAssembler* masm, int frame_size_in_bytes, bool save_vectors) {
const int regstosave_num = sizeof(RegisterSaver_LiveRegs) /
sizeof(RegisterSaver::LiveRegType);
- const int register_save_size = regstosave_num * reg_size; // VS registers not relevant here.
+ const int vecregstosave_num = save_vectors ? (sizeof(RegisterSaver_LiveVecRegs) /
+ sizeof(RegisterSaver::LiveRegType))
+ : 0;
+ const int register_save_size = regstosave_num * reg_size + vecregstosave_num * vec_reg_size;
+
const int register_save_offset = frame_size_in_bytes - register_save_size;
// restore all result registers (ints and floats)
@@ -598,7 +604,7 @@ void RegisterSaver::restore_result_registers(MacroAssembler* masm, int frame_siz
offset += reg_size;
}
- assert(offset == frame_size_in_bytes, "consistency check");
+ assert(offset == frame_size_in_bytes - (save_vectors ? vecregstosave_num * vec_reg_size : 0), "consistency check");
}
// Is vector's size (in bytes) bigger than a size saved by default?
@@ -2909,7 +2915,8 @@ void SharedRuntime::generate_deopt_blob() {
map = RegisterSaver::push_frame_reg_args_and_save_live_registers(masm,
&first_frame_size_in_bytes,
/*generate_oop_map=*/ true,
- RegisterSaver::return_pc_is_lr);
+ RegisterSaver::return_pc_is_lr,
+ /*save_vectors*/ SuperwordUseVSX);
assert(map != nullptr, "OopMap must have been created");
__ li(exec_mode_reg, Deoptimization::Unpack_deopt);
@@ -2943,7 +2950,8 @@ void SharedRuntime::generate_deopt_blob() {
RegisterSaver::push_frame_reg_args_and_save_live_registers(masm,
&first_frame_size_in_bytes,
/*generate_oop_map=*/ false,
- RegisterSaver::return_pc_is_pre_saved);
+ RegisterSaver::return_pc_is_pre_saved,
+ /*save_vectors*/ SuperwordUseVSX);
// Deopt during an exception. Save exec mode for unpack_frames.
__ li(exec_mode_reg, Deoptimization::Unpack_exception);
@@ -2958,7 +2966,8 @@ void SharedRuntime::generate_deopt_blob() {
RegisterSaver::push_frame_reg_args_and_save_live_registers(masm,
&first_frame_size_in_bytes,
/*generate_oop_map=*/ false,
- RegisterSaver::return_pc_is_pre_saved);
+ RegisterSaver::return_pc_is_pre_saved,
+ /*save_vectors*/ SuperwordUseVSX);
__ li(exec_mode_reg, Deoptimization::Unpack_reexecute);
#endif
@@ -2984,7 +2993,7 @@ void SharedRuntime::generate_deopt_blob() {
// Restore only the result registers that have been saved
// by save_volatile_registers(...).
- RegisterSaver::restore_result_registers(masm, first_frame_size_in_bytes);
+ RegisterSaver::restore_result_registers(masm, first_frame_size_in_bytes, /*save_vectors*/ SuperwordUseVSX);
// reload the exec mode from the UnrollBlock (it might have changed)
__ lwz(exec_mode_reg, in_bytes(Deoptimization::UnrollBlock::unpack_kind_offset()), unroll_block_reg);
diff --git a/src/hotspot/cpu/ppc/stubDeclarations_ppc.hpp b/src/hotspot/cpu/ppc/stubDeclarations_ppc.hpp
index be51afe42a4..41b8b71486d 100644
--- a/src/hotspot/cpu/ppc/stubDeclarations_ppc.hpp
+++ b/src/hotspot/cpu/ppc/stubDeclarations_ppc.hpp
@@ -29,35 +29,40 @@
#define STUBGEN_PREUNIVERSE_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(preuniverse, 0) \
#define STUBGEN_INITIAL_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(initial, 20000) \
#define STUBGEN_CONTINUATION_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(continuation, 2000) \
#define STUBGEN_COMPILER_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(compiler, 24000) \
#define STUBGEN_FINAL_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(final, 24000) \
diff --git a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp
index e48778a8b9f..f528587a8bb 100644
--- a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp
+++ b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp
@@ -5095,7 +5095,7 @@ void generate_lookup_secondary_supers_table_stub() {
}
public:
- StubGenerator(CodeBuffer* code, BlobId blob_id) : StubCodeGenerator(code, blob_id) {
+ StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData *stub_data) : StubCodeGenerator(code, blob_id, stub_data) {
switch(blob_id) {
case BlobId::stubgen_preuniverse_id:
generate_preuniverse_stubs();
@@ -5119,7 +5119,7 @@ void generate_lookup_secondary_supers_table_stub() {
}
};
-void StubGenerator_generate(CodeBuffer* code, BlobId blob_id) {
- StubGenerator g(code, blob_id);
+void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData *stub_data) {
+ StubGenerator g(code, blob_id, stub_data);
}
diff --git a/src/hotspot/cpu/ppc/stubRoutines_ppc_64.cpp b/src/hotspot/cpu/ppc/stubRoutines_ppc_64.cpp
index 914c5a17a19..3b7ee66348a 100644
--- a/src/hotspot/cpu/ppc/stubRoutines_ppc_64.cpp
+++ b/src/hotspot/cpu/ppc/stubRoutines_ppc_64.cpp
@@ -183,3 +183,9 @@ address StubRoutines::ppc::generate_crc_constants(juint reverse_poly) {
return consts;
}
+
+#if INCLUDE_CDS
+// nothing to do for ppc
+void StubRoutines::init_AOTAddressTable() {
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp b/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp
index 8a3af748fa1..37f780535b4 100644
--- a/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp
+++ b/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp
@@ -3489,7 +3489,7 @@ void TemplateTable::invokevirtual(int byte_no) {
// Get receiver klass.
__ load_klass_check_null_throw(Rrecv_klass, Rrecv, R11_scratch1);
__ verify_klass_ptr(Rrecv_klass);
- __ profile_virtual_call(Rrecv_klass, R11_scratch1, R12_scratch2, false);
+ __ profile_virtual_call(Rrecv_klass, R11_scratch1, R12_scratch2);
generate_vtable_call(Rrecv_klass, Rvtableindex_or_method, Rret_addr, R11_scratch1);
}
@@ -3596,7 +3596,7 @@ void TemplateTable::invokeinterface_object_method(Register Rrecv_klass,
// Non-final callc case.
__ bind(LnotFinal);
__ lhz(Rindex, in_bytes(ResolvedMethodEntry::table_index_offset()), Rcache);
- __ profile_virtual_call(Rrecv_klass, Rtemp1, Rscratch, false);
+ __ profile_virtual_call(Rrecv_klass, Rtemp1, Rscratch);
generate_vtable_call(Rrecv_klass, Rindex, Rret, Rscratch);
}
@@ -3664,7 +3664,7 @@ void TemplateTable::invokeinterface(int byte_no) {
__ lookup_interface_method(Rrecv_klass, Rinterface_klass, noreg, noreg, Rscratch1, Rscratch2,
L_no_such_interface, /*return_method=*/false);
- __ profile_virtual_call(Rrecv_klass, Rscratch1, Rscratch2, false);
+ __ profile_virtual_call(Rrecv_klass, Rscratch1, Rscratch2);
// Find entry point to call.
diff --git a/src/hotspot/cpu/ppc/vm_version_ppc.cpp b/src/hotspot/cpu/ppc/vm_version_ppc.cpp
index 75feb389298..3e3b1103c86 100644
--- a/src/hotspot/cpu/ppc/vm_version_ppc.cpp
+++ b/src/hotspot/cpu/ppc/vm_version_ppc.cpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -25,6 +25,7 @@
#include "asm/assembler.inline.hpp"
#include "asm/macroAssembler.inline.hpp"
+#include "compiler/compilerDefinitions.inline.hpp"
#include "compiler/disassembler.hpp"
#include "jvm.h"
#include "memory/resourceArea.hpp"
@@ -105,7 +106,7 @@ void VM_Version::initialize() {
if (PowerArchitecturePPC64 >= 9) {
// Performance is good since Power9.
- if (FLAG_IS_DEFAULT(SuperwordUseVSX)) {
+ if (FLAG_IS_DEFAULT(SuperwordUseVSX) && CompilerConfig::is_c2_enabled()) {
FLAG_SET_ERGO(SuperwordUseVSX, true);
}
}
@@ -310,11 +311,6 @@ void VM_Version::initialize() {
FLAG_SET_DEFAULT(UseSHA3Intrinsics, false);
}
- if (!(UseSHA1Intrinsics || UseSHA256Intrinsics || UseSHA512Intrinsics)) {
- FLAG_SET_DEFAULT(UseSHA, false);
- }
-
-
#ifdef COMPILER2
if (FLAG_IS_DEFAULT(UseSquareToLenIntrinsic)) {
UseSquareToLenIntrinsic = true;
@@ -475,19 +471,12 @@ void VM_Version::print_features() {
void VM_Version::determine_features() {
#if defined(ABI_ELFv2)
- // 1 InstWord per call for the blr instruction.
- const int code_size = (num_features+1+2*1)*BytesPerInstWord;
+ const int code_size = (num_features + 1 /*blr*/) * BytesPerInstWord;
#else
- // 7 InstWords for each call (function descriptor + blr instruction).
- const int code_size = (num_features+1+2*7)*BytesPerInstWord;
+ const int code_size = (num_features + 1 /*blr*/ + 6 /* fd */) * BytesPerInstWord;
#endif
int features = 0;
- // create test area
- enum { BUFFER_SIZE = 2*4*K }; // Needs to be >=2* max cache line size (cache line size can't exceed min page size).
- char test_area[BUFFER_SIZE];
- char *mid_of_test_area = &test_area[BUFFER_SIZE>>1];
-
// Allocate space for the code.
ResourceMark rm;
CodeBuffer cb("detect_cpu_features", code_size, 0);
@@ -497,20 +486,13 @@ void VM_Version::determine_features() {
_features = VM_Version::all_features_m;
// Emit code.
- void (*test)(address addr, uint64_t offset)=(void(*)(address addr, uint64_t offset))(void *)a->function_entry();
+ void (*test)() = (void(*)())(void *)a->function_entry();
uint32_t *code = (uint32_t *)a->pc();
- // Keep R3_ARG1 unmodified, it contains &field (see below).
- // Keep R4_ARG2 unmodified, it contains offset = 0 (see below).
a->mfdscr(R0);
a->darn(R7);
a->brw(R5, R6);
a->blr();
- // Emit function to set one cache line to zero. Emit function descriptor and get pointer to it.
- void (*zero_cacheline_func_ptr)(char*) = (void(*)(char*))(void *)a->function_entry();
- a->dcbz(R3_ARG1); // R3_ARG1 = addr
- a->blr();
-
uint32_t *code_end = (uint32_t *)a->pc();
a->flush();
_features = VM_Version::unknown_m;
@@ -522,18 +504,9 @@ void VM_Version::determine_features() {
Disassembler::decode((u_char*)code, (u_char*)code_end, tty);
}
- // Measure cache line size.
- memset(test_area, 0xFF, BUFFER_SIZE); // Fill test area with 0xFF.
- (*zero_cacheline_func_ptr)(mid_of_test_area); // Call function which executes dcbz to the middle.
- int count = 0; // count zeroed bytes
- for (int i = 0; i < BUFFER_SIZE; i++) if (test_area[i] == 0) count++;
- guarantee(is_power_of_2(count), "cache line size needs to be a power of 2");
- _L1_data_cache_line_size = count;
-
// Execute code. Illegal instructions will be replaced by 0 in the signal handler.
VM_Version::_is_determine_features_test_running = true;
- // We must align the first argument to 16 bytes because of the lqarx check.
- (*test)(align_up((address)mid_of_test_area, 16), 0);
+ (*test)();
VM_Version::_is_determine_features_test_running = false;
// determine which instructions are legal.
@@ -550,6 +523,10 @@ void VM_Version::determine_features() {
}
_features = features;
+
+ _L1_data_cache_line_size = VM_Version::get_dcache_line_size();
+ assert(_L1_data_cache_line_size >= DEFAULT_CACHE_LINE_SIZE,
+ "processors with smaller cache line size are no longer supported");
}
// Power 8: Configure Data Stream Control Register.
diff --git a/src/hotspot/cpu/ppc/vm_version_ppc.hpp b/src/hotspot/cpu/ppc/vm_version_ppc.hpp
index 11dce83bed0..0f4eb3593a3 100644
--- a/src/hotspot/cpu/ppc/vm_version_ppc.hpp
+++ b/src/hotspot/cpu/ppc/vm_version_ppc.hpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -81,6 +81,9 @@ public:
static uint64_t _dscr_val;
static void initialize_cpu_information(void);
+
+ static int get_dcache_line_size();
+ static int get_icache_line_size();
};
#endif // CPU_PPC_VM_VERSION_PPC_HPP
diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.cpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.cpp
index 819d6c05654..8aced227a06 100644
--- a/src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -196,12 +196,9 @@ void LIR_Assembler::arraycopy_type_check(Register src, Register src_pos, Registe
if (UseCompactObjectHeaders) {
__ load_narrow_klass_compact(tmp, src);
__ load_narrow_klass_compact(t0, dst);
- } else if (UseCompressedClassPointers) {
+ } else {
__ lwu(tmp, Address(src, oopDesc::klass_offset_in_bytes()));
__ lwu(t0, Address(dst, oopDesc::klass_offset_in_bytes()));
- } else {
- __ ld(tmp, Address(src, oopDesc::klass_offset_in_bytes()));
- __ ld(t0, Address(dst, oopDesc::klass_offset_in_bytes()));
}
__ bne(tmp, t0, *stub->entry(), /* is_far */ true);
} else {
@@ -243,37 +240,6 @@ void LIR_Assembler::arraycopy_type_check(Register src, Register src_pos, Registe
}
}
-void LIR_Assembler::arraycopy_assert(Register src, Register dst, Register tmp, ciArrayKlass *default_type, int flags) {
- assert(default_type != nullptr, "null default_type!");
- BasicType basic_type = default_type->element_type()->basic_type();
- if (basic_type == T_ARRAY) { basic_type = T_OBJECT; }
- if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
- // Sanity check the known type with the incoming class. For the
- // primitive case the types must match exactly with src.klass and
- // dst.klass each exactly matching the default type. For the
- // object array case, if no type check is needed then either the
- // dst type is exactly the expected type and the src type is a
- // subtype which we can't check or src is the same array as dst
- // but not necessarily exactly of type default_type.
- Label known_ok, halt;
- __ mov_metadata(tmp, default_type->constant_encoding());
- if (UseCompressedClassPointers) {
- __ encode_klass_not_null(tmp);
- }
-
- if (basic_type != T_OBJECT) {
- __ cmp_klass_compressed(dst, tmp, t0, halt, false);
- __ cmp_klass_compressed(src, tmp, t0, known_ok, true);
- } else {
- __ cmp_klass_compressed(dst, tmp, t0, known_ok, true);
- __ beq(src, dst, known_ok);
- }
- __ bind(halt);
- __ stop("incorrect type information in arraycopy");
- __ bind(known_ok);
- }
-}
-
void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
ciArrayKlass *default_type = op->expected_type();
Register src = op->src()->as_register();
@@ -304,7 +270,28 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
}
#ifdef ASSERT
- arraycopy_assert(src, dst, tmp, default_type, flags);
+ if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
+ // Sanity check the known type with the incoming class. For the
+ // primitive case the types must match exactly with src.klass and
+ // dst.klass each exactly matching the default type. For the
+ // object array case, if no type check is needed then either the
+ // dst type is exactly the expected type and the src type is a
+ // subtype which we can't check or src is the same array as dst
+ // but not necessarily exactly of type default_type.
+ Label known_ok, halt;
+ __ mov_metadata(tmp, default_type->constant_encoding());
+
+ if (basic_type != T_OBJECT) {
+ __ cmp_klass_bne(dst, tmp, t0, t1, halt);
+ __ cmp_klass_beq(src, tmp, t0, t1, known_ok);
+ } else {
+ __ cmp_klass_beq(dst, tmp, t0, t1, known_ok);
+ __ beq(src, dst, known_ok);
+ }
+ __ bind(halt);
+ __ stop("incorrect type information in arraycopy");
+ __ bind(known_ok);
+ }
#endif
#ifndef PRODUCT
diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.hpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.hpp
index 06a0f248ca6..b5452f3e4cd 100644
--- a/src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.hpp
+++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -39,7 +39,6 @@
void arraycopy_type_check(Register src, Register src_pos, Register length,
Register dst, Register dst_pos, Register tmp,
CodeStub *stub, BasicType basic_type, int flags);
- void arraycopy_assert(Register src, Register dst, Register tmp, ciArrayKlass *default_type, int flags);
void arraycopy_prepare_params(Register src, Register src_pos, Register length,
Register dst, Register dst_pos, BasicType basic_type);
void arraycopy_checkcast_prepare_params(Register src, Register src_pos, Register length,
diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp
index 63e2fd015d7..29e5d86d0cc 100644
--- a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -55,20 +55,6 @@ const Register SHIFT_count = x10; // where count for shift operations must be
#define __ _masm->
-static void select_different_registers(Register preserve,
- Register extra,
- Register &tmp1,
- Register &tmp2) {
- if (tmp1 == preserve) {
- assert_different_registers(tmp1, tmp2, extra);
- tmp1 = extra;
- } else if (tmp2 == preserve) {
- assert_different_registers(tmp1, tmp2, extra);
- tmp2 = extra;
- }
- assert_different_registers(preserve, tmp1, tmp2);
-}
-
static void select_different_registers(Register preserve,
Register extra,
Register &tmp1,
@@ -1155,12 +1141,8 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, L
} else if (obj == klass_RInfo) {
klass_RInfo = dst;
}
- if (k->is_loaded() && !UseCompressedClassPointers) {
- select_different_registers(obj, dst, k_RInfo, klass_RInfo);
- } else {
- Rtmp1 = op->tmp3()->as_register();
- select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
- }
+ Rtmp1 = op->tmp3()->as_register();
+ select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
assert_different_registers(obj, k_RInfo, klass_RInfo);
diff --git a/src/hotspot/cpu/riscv/c1_LIRGenerator_riscv.cpp b/src/hotspot/cpu/riscv/c1_LIRGenerator_riscv.cpp
index 88565d9136f..f290708a231 100644
--- a/src/hotspot/cpu/riscv/c1_LIRGenerator_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c1_LIRGenerator_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -1073,9 +1073,7 @@ void LIRGenerator::do_CheckCast(CheckCast* x) {
}
LIR_Opr reg = rlock_result(x);
LIR_Opr tmp3 = LIR_OprFact::illegalOpr;
- if (!x->klass()->is_loaded() || UseCompressedClassPointers) {
- tmp3 = new_register(objectType);
- }
+ tmp3 = new_register(objectType);
__ checkcast(reg, obj.result(), x->klass(),
new_register(objectType), new_register(objectType), tmp3,
x->direct_compare(), info_for_exception, patching_info, stub,
@@ -1094,9 +1092,7 @@ void LIRGenerator::do_InstanceOf(InstanceOf* x) {
}
obj.load_item();
LIR_Opr tmp3 = LIR_OprFact::illegalOpr;
- if (!x->klass()->is_loaded() || UseCompressedClassPointers) {
- tmp3 = new_register(objectType);
- }
+ tmp3 = new_register(objectType);
__ instanceof(reg, obj.result(), x->klass(),
new_register(objectType), new_register(objectType), tmp3,
x->direct_compare(), patching_info, x->profiled_method(), x->profiled_bci());
diff --git a/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp
index aeb077ba0a0..abcc070b253 100644
--- a/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -92,12 +92,8 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
// This assumes that all prototype bits fitr in an int32_t
mv(tmp1, checked_cast(markWord::prototype().value()));
sd(tmp1, Address(obj, oopDesc::mark_offset_in_bytes()));
- if (UseCompressedClassPointers) { // Take care not to kill klass
- encode_klass_not_null(tmp1, klass, tmp2);
- sw(tmp1, Address(obj, oopDesc::klass_offset_in_bytes()));
- } else {
- sd(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
- }
+ encode_klass_not_null(tmp1, klass, tmp2);
+ sw(tmp1, Address(obj, oopDesc::klass_offset_in_bytes()));
}
if (len->is_valid()) {
@@ -108,7 +104,7 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
// Clear gap/first 4 bytes following the length field.
sw(zr, Address(obj, base_offset));
}
- } else if (UseCompressedClassPointers && !UseCompactObjectHeaders) {
+ } else if (!UseCompactObjectHeaders) {
store_klass_gap(obj, zr);
}
}
diff --git a/src/hotspot/cpu/riscv/c1_globals_riscv.hpp b/src/hotspot/cpu/riscv/c1_globals_riscv.hpp
index b15bb5c23c3..b940393f063 100644
--- a/src/hotspot/cpu/riscv/c1_globals_riscv.hpp
+++ b/src/hotspot/cpu/riscv/c1_globals_riscv.hpp
@@ -42,7 +42,6 @@ define_pd_global(bool, TieredCompilation, false);
define_pd_global(intx, CompileThreshold, 1500 );
define_pd_global(intx, OnStackReplacePercentage, 933 );
-define_pd_global(intx, NewSizeThreadIncrease, 4*K );
define_pd_global(size_t, InitialCodeCacheSize, 160*K);
define_pd_global(size_t, ReservedCodeCacheSize, 32*M );
define_pd_global(size_t, NonProfiledCodeHeapSize, 13*M );
@@ -52,7 +51,6 @@ define_pd_global(bool, ProfileInterpreter, false);
define_pd_global(size_t, CodeCacheExpansionSize, 32*K );
define_pd_global(size_t, CodeCacheMinBlockLength, 1);
define_pd_global(size_t, CodeCacheMinimumUseSpace, 400*K);
-define_pd_global(bool, NeverActAsServerClassMachine, true );
define_pd_global(bool, CICompileOSR, true );
#endif // !COMPILER2
define_pd_global(bool, UseTypeProfile, false);
diff --git a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp
index 72a90ddde1f..0d06fd469de 100644
--- a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp
@@ -1175,8 +1175,7 @@ void C2_MacroAssembler::string_compare_long_same_encoding(Register result, Regis
Label TAIL_CHECK, TAIL, NEXT_WORD, DIFFERENCE;
const int base_offset = arrayOopDesc::base_offset_in_bytes(T_BYTE);
- assert((base_offset % (UseCompactObjectHeaders ? 4 :
- (UseCompressedClassPointers ? 8 : 4))) == 0, "Must be");
+ assert((base_offset % (UseCompactObjectHeaders ? 4 : 8)) == 0, "Must be");
const int minCharsInWord = isLL ? wordSize : wordSize / 2;
@@ -1269,8 +1268,7 @@ void C2_MacroAssembler::string_compare_long_different_encoding(Register result,
Label TAIL, NEXT_WORD, DIFFERENCE;
const int base_offset = arrayOopDesc::base_offset_in_bytes(T_BYTE);
- assert((base_offset % (UseCompactObjectHeaders ? 4 :
- (UseCompressedClassPointers ? 8 : 4))) == 0, "Must be");
+ assert((base_offset % (UseCompactObjectHeaders ? 4 : 8)) == 0, "Must be");
Register strL = isLU ? str1 : str2;
Register strU = isLU ? str2 : str1;
@@ -1485,8 +1483,7 @@ void C2_MacroAssembler::arrays_equals(Register a1, Register a2,
int length_offset = arrayOopDesc::length_offset_in_bytes();
int base_offset = arrayOopDesc::base_offset_in_bytes(elem_size == 2 ? T_CHAR : T_BYTE);
- assert((base_offset % (UseCompactObjectHeaders ? 4 :
- (UseCompressedClassPointers ? 8 : 4))) == 0, "Must be");
+ assert((base_offset % (UseCompactObjectHeaders ? 4 : 8)) == 0, "Must be");
Register cnt1 = tmp3;
Register cnt2 = tmp1; // cnt2 only used in array length compare
@@ -1611,8 +1608,7 @@ void C2_MacroAssembler::string_equals(Register a1, Register a2,
int base_offset = arrayOopDesc::base_offset_in_bytes(T_BYTE);
- assert((base_offset % (UseCompactObjectHeaders ? 4 :
- (UseCompressedClassPointers ? 8 : 4))) == 0, "Must be");
+ assert((base_offset % (UseCompactObjectHeaders ? 4 : 8)) == 0, "Must be");
BLOCK_COMMENT("string_equals {");
@@ -2699,8 +2695,7 @@ void C2_MacroAssembler::arrays_equals_v(Register a1, Register a2, Register resul
int length_offset = arrayOopDesc::length_offset_in_bytes();
int base_offset = arrayOopDesc::base_offset_in_bytes(elem_size == 2 ? T_CHAR : T_BYTE);
- assert((base_offset % (UseCompactObjectHeaders ? 4 :
- (UseCompressedClassPointers ? 8 : 4))) == 0, "Must be");
+ assert((base_offset % (UseCompactObjectHeaders ? 4 : 8)) == 0, "Must be");
BLOCK_COMMENT("arrays_equals_v {");
diff --git a/src/hotspot/cpu/riscv/c2_globals_riscv.hpp b/src/hotspot/cpu/riscv/c2_globals_riscv.hpp
index 648c24ee98b..73ef97939ed 100644
--- a/src/hotspot/cpu/riscv/c2_globals_riscv.hpp
+++ b/src/hotspot/cpu/riscv/c2_globals_riscv.hpp
@@ -47,7 +47,6 @@ define_pd_global(intx, ConditionalMoveLimit, 3);
define_pd_global(intx, FreqInlineSize, 325);
define_pd_global(intx, MinJumpTableSize, 10);
define_pd_global(intx, InteriorEntryAlignment, 16);
-define_pd_global(intx, NewSizeThreadIncrease, ScaleForWordSize(4*K));
define_pd_global(intx, LoopUnrollLimit, 60);
define_pd_global(intx, LoopPercentProfileLimit, 10);
// InitialCodeCacheSize derived from specjbb2000 run.
@@ -75,9 +74,6 @@ define_pd_global(size_t, NonNMethodCodeHeapSize, 5*M );
define_pd_global(size_t, CodeCacheMinBlockLength, 6);
define_pd_global(size_t, CodeCacheMinimumUseSpace, 400*K);
-// Ergonomics related flags
-define_pd_global(bool, NeverActAsServerClassMachine, false);
-
define_pd_global(bool, TrapBasedRangeChecks, false); // Not needed.
#endif // CPU_RISCV_C2_GLOBALS_RISCV_HPP
diff --git a/src/hotspot/cpu/riscv/downcallLinker_riscv.cpp b/src/hotspot/cpu/riscv/downcallLinker_riscv.cpp
index cc685645ec5..f9d7ce78ff0 100644
--- a/src/hotspot/cpu/riscv/downcallLinker_riscv.cpp
+++ b/src/hotspot/cpu/riscv/downcallLinker_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -140,10 +140,10 @@ void DowncallLinker::StubGenerator::generate() {
bool should_save_return_value = !_needs_return_buffer;
RegSpiller out_reg_spiller(_output_registers);
- int spill_offset = -1;
+ int out_spill_offset = -1;
if (should_save_return_value) {
- spill_offset = 0;
+ out_spill_offset = 0;
// spill area can be shared with shadow space and out args,
// since they are only used before the call,
// and spill area is only used after.
@@ -168,6 +168,9 @@ void DowncallLinker::StubGenerator::generate() {
// FP-> | |
// |---------------------| = frame_bottom_offset = frame_size
// | (optional) |
+ // | in_reg_spiller area |
+ // |---------------------|
+ // | (optional) |
// | capture state buf |
// |---------------------| = StubLocations::CAPTURED_STATE_BUFFER
// | (optional) |
@@ -181,6 +184,18 @@ void DowncallLinker::StubGenerator::generate() {
GrowableArray out_regs = ForeignGlobals::replace_place_holders(_input_registers, locs);
ArgumentShuffle arg_shuffle(filtered_java_regs, out_regs, shuffle_reg);
+ // Need to spill for state capturing runtime call.
+ // The area spilled into is distinct from the capture state buffer.
+ RegSpiller in_reg_spiller(out_regs);
+ int in_spill_offset = -1;
+ if (_captured_state_mask != 0) {
+ // The spill area cannot be shared with the out_spill since
+ // spilling needs to happen before the call. Allocate a new
+ // region in the stack for this spill space.
+ in_spill_offset = allocated_frame_size;
+ allocated_frame_size += in_reg_spiller.spill_size_bytes();
+ }
+
#ifndef PRODUCT
LogTarget(Trace, foreign, downcall) lt;
if (lt.is_enabled()) {
@@ -226,6 +241,20 @@ void DowncallLinker::StubGenerator::generate() {
arg_shuffle.generate(_masm, shuffle_reg, 0, _abi._shadow_space_bytes);
__ block_comment("} argument shuffle");
+ if (_captured_state_mask != 0) {
+ assert(in_spill_offset != -1, "must be");
+ __ block_comment("{ load initial thread local");
+ in_reg_spiller.generate_spill(_masm, in_spill_offset);
+
+ // Copy the contents of the capture state buffer into thread local
+ __ ld(c_rarg0, Address(sp, locs.data_offset(StubLocations::CAPTURED_STATE_BUFFER)));
+ __ mv(c_rarg1, _captured_state_mask);
+ __ rt_call(CAST_FROM_FN_PTR(address, DowncallLinker::capture_state_pre));
+
+ in_reg_spiller.generate_fill(_masm, in_spill_offset);
+ __ block_comment("} load initial thread local");
+ }
+
__ jalr(as_Register(locs.get(StubLocations::TARGET_ADDRESS)));
// this call is assumed not to have killed xthread
@@ -254,15 +283,15 @@ void DowncallLinker::StubGenerator::generate() {
__ block_comment("{ save thread local");
if (should_save_return_value) {
- out_reg_spiller.generate_spill(_masm, spill_offset);
+ out_reg_spiller.generate_spill(_masm, out_spill_offset);
}
__ ld(c_rarg0, Address(sp, locs.data_offset(StubLocations::CAPTURED_STATE_BUFFER)));
__ mv(c_rarg1, _captured_state_mask);
- __ rt_call(CAST_FROM_FN_PTR(address, DowncallLinker::capture_state));
+ __ rt_call(CAST_FROM_FN_PTR(address, DowncallLinker::capture_state_post));
if (should_save_return_value) {
- out_reg_spiller.generate_fill(_masm, spill_offset);
+ out_reg_spiller.generate_fill(_masm, out_spill_offset);
}
__ block_comment("} save thread local");
@@ -319,7 +348,7 @@ void DowncallLinker::StubGenerator::generate() {
if (should_save_return_value) {
// Need to save the native result registers around any runtime calls.
- out_reg_spiller.generate_spill(_masm, spill_offset);
+ out_reg_spiller.generate_spill(_masm, out_spill_offset);
}
__ mv(c_rarg0, xthread);
@@ -327,7 +356,7 @@ void DowncallLinker::StubGenerator::generate() {
__ rt_call(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans));
if (should_save_return_value) {
- out_reg_spiller.generate_fill(_masm, spill_offset);
+ out_reg_spiller.generate_fill(_masm, out_spill_offset);
}
__ j(L_after_safepoint_poll);
__ block_comment("} L_safepoint_poll_slow_path");
@@ -339,13 +368,13 @@ void DowncallLinker::StubGenerator::generate() {
if (should_save_return_value) {
// Need to save the native result registers around any runtime calls.
- out_reg_spiller.generate_spill(_masm, spill_offset);
+ out_reg_spiller.generate_spill(_masm, out_spill_offset);
}
__ rt_call(CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages));
if (should_save_return_value) {
- out_reg_spiller.generate_fill(_masm, spill_offset);
+ out_reg_spiller.generate_fill(_masm, out_spill_offset);
}
__ j(L_after_reguard);
diff --git a/src/hotspot/cpu/riscv/frame_riscv.inline.hpp b/src/hotspot/cpu/riscv/frame_riscv.inline.hpp
index 51a203c548c..d1841a347e9 100644
--- a/src/hotspot/cpu/riscv/frame_riscv.inline.hpp
+++ b/src/hotspot/cpu/riscv/frame_riscv.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -236,8 +236,8 @@ inline bool frame::equal(frame other) const {
// Return unique id for this frame. The id must have a value where we can distinguish
// identity and younger/older relationship. null represents an invalid (incomparable)
-// frame.
-inline intptr_t* frame::id(void) const { return unextended_sp(); }
+// frame. Should not be called for heap frames.
+inline intptr_t* frame::id(void) const { return real_fp(); }
// Return true if the frame is older (less recent activation) than the frame represented by id
inline bool frame::is_older(intptr_t* id) const { assert(this->id() != nullptr && id != nullptr, "null frame id");
@@ -398,6 +398,9 @@ frame frame::sender(RegisterMap* map) const {
StackWatermarkSet::on_iteration(map->thread(), result);
}
+ // Calling frame::id() is currently not supported for heap frames.
+ assert(result._on_heap || this->_on_heap || result.is_older(this->id()), "Must be");
+
return result;
}
diff --git a/src/hotspot/cpu/riscv/gc/shared/cardTableBarrierSetAssembler_riscv.cpp b/src/hotspot/cpu/riscv/gc/shared/cardTableBarrierSetAssembler_riscv.cpp
index d94bf428fd2..9eb546a1888 100644
--- a/src/hotspot/cpu/riscv/gc/shared/cardTableBarrierSetAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/gc/shared/cardTableBarrierSetAssembler_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -56,8 +56,10 @@ void CardTableBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet d
}
}
-void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj, Register tmp) {
- assert_different_registers(obj, tmp);
+void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2) {
+ precond(tmp1 != noreg);
+ precond(tmp2 != noreg);
+ assert_different_registers(obj, tmp1, tmp2);
BarrierSet* bs = BarrierSet::barrier_set();
assert(bs->kind() == BarrierSet::CardTableBarrierSet, "Wrong barrier set kind");
@@ -65,17 +67,17 @@ void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register ob
assert(CardTable::dirty_card_val() == 0, "must be");
- __ load_byte_map_base(tmp);
- __ add(tmp, obj, tmp);
+ __ load_byte_map_base(tmp1);
+ __ add(tmp1, obj, tmp1);
if (UseCondCardMark) {
Label L_already_dirty;
- __ lbu(t1, Address(tmp));
- __ beqz(t1, L_already_dirty);
- __ sb(zr, Address(tmp));
+ __ lbu(tmp2, Address(tmp1));
+ __ beqz(tmp2, L_already_dirty);
+ __ sb(zr, Address(tmp1));
__ bind(L_already_dirty);
} else {
- __ sb(zr, Address(tmp));
+ __ sb(zr, Address(tmp1));
}
}
@@ -119,10 +121,10 @@ void CardTableBarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorS
if (needs_post_barrier) {
// flatten object address if needed
if (!precise || dst.offset() == 0) {
- store_check(masm, dst.base(), tmp3);
+ store_check(masm, dst.base(), tmp1, tmp2);
} else {
__ la(tmp3, dst);
- store_check(masm, tmp3, t0);
+ store_check(masm, tmp3, tmp1, tmp2);
}
}
}
diff --git a/src/hotspot/cpu/riscv/gc/shared/cardTableBarrierSetAssembler_riscv.hpp b/src/hotspot/cpu/riscv/gc/shared/cardTableBarrierSetAssembler_riscv.hpp
index 6f6e9065103..1576f0a6dd8 100644
--- a/src/hotspot/cpu/riscv/gc/shared/cardTableBarrierSetAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/gc/shared/cardTableBarrierSetAssembler_riscv.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -31,7 +31,7 @@
class CardTableBarrierSetAssembler: public BarrierSetAssembler {
protected:
- void store_check(MacroAssembler* masm, Register obj, Register tmp);
+ void store_check(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2);
virtual void gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators,
Register addr, Register count, RegSet saved_regs) {}
diff --git a/src/hotspot/cpu/riscv/interp_masm_riscv.cpp b/src/hotspot/cpu/riscv/interp_masm_riscv.cpp
index 744590bec2b..804c2072ba5 100644
--- a/src/hotspot/cpu/riscv/interp_masm_riscv.cpp
+++ b/src/hotspot/cpu/riscv/interp_masm_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -1040,26 +1040,15 @@ void InterpreterMacroAssembler::profile_final_call(Register mdp) {
void InterpreterMacroAssembler::profile_virtual_call(Register receiver,
- Register mdp,
- bool receiver_can_be_null) {
+ Register mdp) {
if (ProfileInterpreter) {
Label profile_continue;
// If no method data exists, go to profile_continue.
test_method_data_pointer(mdp, profile_continue);
- Label skip_receiver_profile;
- if (receiver_can_be_null) {
- Label not_null;
- // We are making a call. Increment the count for null receiver.
- increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
- j(skip_receiver_profile);
- bind(not_null);
- }
-
// Record the receiver type.
profile_receiver_type(receiver, mdp, 0);
- bind(skip_receiver_profile);
// The method data pointer needs to be updated to reflect the new target.
diff --git a/src/hotspot/cpu/riscv/interp_masm_riscv.hpp b/src/hotspot/cpu/riscv/interp_masm_riscv.hpp
index 59cc76b022f..df86f0dc532 100644
--- a/src/hotspot/cpu/riscv/interp_masm_riscv.hpp
+++ b/src/hotspot/cpu/riscv/interp_masm_riscv.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2015, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -274,8 +274,7 @@ class InterpreterMacroAssembler: public MacroAssembler {
void profile_not_taken_branch(Register mdp);
void profile_call(Register mdp);
void profile_final_call(Register mdp);
- void profile_virtual_call(Register receiver, Register mdp,
- bool receiver_can_be_null = false);
+ void profile_virtual_call(Register receiver, Register mdp);
void profile_ret(Register return_bci, Register mdp);
void profile_null_seen(Register mdp);
void profile_typecheck(Register mdp, Register klass);
diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
index 4f5e7afc166..0e32c602d95 100644
--- a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2024, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -49,6 +49,7 @@
#include "runtime/sharedRuntime.hpp"
#include "runtime/stubRoutines.hpp"
#include "utilities/globalDefinitions.hpp"
+#include "utilities/integerCast.hpp"
#include "utilities/powerOfTwo.hpp"
#ifdef COMPILER2
#include "opto/compile.hpp"
@@ -1947,14 +1948,12 @@ void MacroAssembler::restore_cpu_control_state_after_jni(Register tmp) {
}
}
-void MacroAssembler::push_reg(Register Rs)
-{
+void MacroAssembler::push_reg(Register Rs) {
subi(esp, esp, wordSize);
sd(Rs, Address(esp, 0));
}
-void MacroAssembler::pop_reg(Register Rd)
-{
+void MacroAssembler::pop_reg(Register Rd) {
ld(Rd, Address(esp, 0));
addi(esp, esp, wordSize);
}
@@ -1973,7 +1972,11 @@ int MacroAssembler::bitset_to_regs(unsigned int bitset, unsigned char* regs) {
// Push integer registers in the bitset supplied. Don't push sp.
// Return the number of words pushed
-int MacroAssembler::push_reg(unsigned int bitset, Register stack) {
+int MacroAssembler::push_reg(RegSet regset, Register stack) {
+ if (regset.bits() == 0) {
+ return 0;
+ }
+ auto bitset = integer_cast(regset.bits());
DEBUG_ONLY(int words_pushed = 0;)
unsigned char regs[32];
int count = bitset_to_regs(bitset, regs);
@@ -1993,7 +1996,11 @@ int MacroAssembler::push_reg(unsigned int bitset, Register stack) {
return count;
}
-int MacroAssembler::pop_reg(unsigned int bitset, Register stack) {
+int MacroAssembler::pop_reg(RegSet regset, Register stack) {
+ if (regset.bits() == 0) {
+ return 0;
+ }
+ auto bitset = integer_cast(regset.bits());
DEBUG_ONLY(int words_popped = 0;)
unsigned char regs[32];
int count = bitset_to_regs(bitset, regs);
@@ -2015,7 +2022,11 @@ int MacroAssembler::pop_reg(unsigned int bitset, Register stack) {
// Push floating-point registers in the bitset supplied.
// Return the number of words pushed
-int MacroAssembler::push_fp(unsigned int bitset, Register stack) {
+int MacroAssembler::push_fp(FloatRegSet regset, Register stack) {
+ if (regset.bits() == 0) {
+ return 0;
+ }
+ auto bitset = integer_cast(regset.bits());
DEBUG_ONLY(int words_pushed = 0;)
unsigned char regs[32];
int count = bitset_to_regs(bitset, regs);
@@ -2035,7 +2046,11 @@ int MacroAssembler::push_fp(unsigned int bitset, Register stack) {
return count;
}
-int MacroAssembler::pop_fp(unsigned int bitset, Register stack) {
+int MacroAssembler::pop_fp(FloatRegSet regset, Register stack) {
+ if (regset.bits() == 0) {
+ return 0;
+ }
+ auto bitset = integer_cast(regset.bits());
DEBUG_ONLY(int words_popped = 0;)
unsigned char regs[32];
int count = bitset_to_regs(bitset, regs);
@@ -2721,7 +2736,11 @@ void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len,
#ifdef COMPILER2
// Push vector registers in the bitset supplied.
// Return the number of words pushed
-int MacroAssembler::push_v(unsigned int bitset, Register stack) {
+int MacroAssembler::push_v(VectorRegSet regset, Register stack) {
+ if (regset.bits() == 0) {
+ return 0;
+ }
+ auto bitset = integer_cast(regset.bits());
int vector_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
// Scan bitset to accumulate register pairs
@@ -2736,7 +2755,11 @@ int MacroAssembler::push_v(unsigned int bitset, Register stack) {
return count * vector_size_in_bytes / wordSize;
}
-int MacroAssembler::pop_v(unsigned int bitset, Register stack) {
+int MacroAssembler::pop_v(VectorRegSet regset, Register stack) {
+ if (regset.bits() == 0) {
+ return 0;
+ }
+ auto bitset = integer_cast(regset.bits());
int vector_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
// Scan bitset to accumulate register pairs
@@ -3511,19 +3534,30 @@ void MacroAssembler::orptr(Address adr, RegisterOrConstant src, Register tmp1, R
sd(tmp1, adr);
}
-void MacroAssembler::cmp_klass_compressed(Register oop, Register trial_klass, Register tmp, Label &L, bool equal) {
+void MacroAssembler::cmp_klass_beq(Register obj, Register klass,
+ Register tmp1, Register tmp2,
+ Label &L, bool is_far) {
+ assert_different_registers(obj, klass, tmp1, tmp2);
if (UseCompactObjectHeaders) {
- load_narrow_klass_compact(tmp, oop);
- } else if (UseCompressedClassPointers) {
- lwu(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
+ load_narrow_klass_compact(tmp1, obj);
} else {
- ld(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
+ lwu(tmp1, Address(obj, oopDesc::klass_offset_in_bytes()));
}
- if (equal) {
- beq(trial_klass, tmp, L);
+ decode_klass_not_null(tmp1, tmp2);
+ beq(klass, tmp1, L, is_far);
+}
+
+void MacroAssembler::cmp_klass_bne(Register obj, Register klass,
+ Register tmp1, Register tmp2,
+ Label &L, bool is_far) {
+ assert_different_registers(obj, klass, tmp1, tmp2);
+ if (UseCompactObjectHeaders) {
+ load_narrow_klass_compact(tmp1, obj);
} else {
- bne(trial_klass, tmp, L);
+ lwu(tmp1, Address(obj, oopDesc::klass_offset_in_bytes()));
}
+ decode_klass_not_null(tmp1, tmp2);
+ bne(klass, tmp1, L, is_far);
}
// Move an oop into a register.
@@ -3741,11 +3775,9 @@ void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(dst, src);
decode_klass_not_null(dst, tmp);
- } else if (UseCompressedClassPointers) {
+ } else {
lwu(dst, Address(src, oopDesc::klass_offset_in_bytes()));
decode_klass_not_null(dst, tmp);
- } else {
- ld(dst, Address(src, oopDesc::klass_offset_in_bytes()));
}
}
@@ -3753,20 +3785,15 @@ void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
// FIXME: Should this be a store release? concurrent gcs assumes
// klass length is valid if klass field is not null.
assert(!UseCompactObjectHeaders, "not with compact headers");
- if (UseCompressedClassPointers) {
- encode_klass_not_null(src, tmp);
- sw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
- } else {
- sd(src, Address(dst, oopDesc::klass_offset_in_bytes()));
- }
+ encode_klass_not_null(src, tmp);
+ sw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
+
}
void MacroAssembler::store_klass_gap(Register dst, Register src) {
assert(!UseCompactObjectHeaders, "not with compact headers");
- if (UseCompressedClassPointers) {
- // Store to klass gap in destination
- sw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
- }
+ // Store to klass gap in destination
+ sw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
}
void MacroAssembler::decode_klass_not_null(Register r, Register tmp) {
@@ -3775,7 +3802,6 @@ void MacroAssembler::decode_klass_not_null(Register r, Register tmp) {
}
void MacroAssembler::decode_klass_not_null(Register dst, Register src, Register tmp) {
- assert(UseCompressedClassPointers, "should only be used for compressed headers");
assert_different_registers(dst, tmp);
assert_different_registers(src, tmp);
@@ -3806,8 +3832,6 @@ void MacroAssembler::encode_klass_not_null(Register r, Register tmp) {
}
void MacroAssembler::encode_klass_not_null(Register dst, Register src, Register tmp) {
- assert(UseCompressedClassPointers, "should only be used for compressed headers");
-
if (CompressedKlassPointers::base() == nullptr) {
if (CompressedKlassPointers::shift() != 0) {
srli(dst, src, CompressedKlassPointers::shift());
@@ -5337,7 +5361,6 @@ void MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
}
void MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
- assert (UseCompressedClassPointers, "should only be used for compressed headers");
assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
int index = oop_recorder()->find_index(k);
@@ -5417,12 +5440,9 @@ int MacroAssembler::ic_check(int end_alignment) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(tmp1, receiver);
lwu(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
- } else if (UseCompressedClassPointers) {
+ } else {
lwu(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes()));
lwu(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
- } else {
- ld(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes()));
- ld(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
}
Label ic_hit;
@@ -5543,13 +5563,6 @@ void MacroAssembler::decrementw(const Address dst, int32_t value, Register tmp1,
sw(tmp1, adr);
}
-void MacroAssembler::cmpptr(Register src1, const Address &src2, Label& equal, Register tmp) {
- assert_different_registers(src1, tmp);
- assert(src2.getMode() == Address::literal, "must be applied to a literal address");
- ld(tmp, src2);
- beq(src1, tmp, equal);
-}
-
void MacroAssembler::load_method_holder_cld(Register result, Register method) {
load_method_holder(result, method);
ld(result, Address(result, InstanceKlass::class_loader_data_offset()));
diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
index f5e985c28a2..4cc55e7ae23 100644
--- a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2024, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -198,7 +198,12 @@ class MacroAssembler: public Assembler {
void load_klass(Register dst, Register src, Register tmp = t0);
void load_narrow_klass_compact(Register dst, Register src);
void store_klass(Register dst, Register src, Register tmp = t0);
- void cmp_klass_compressed(Register oop, Register trial_klass, Register tmp, Label &L, bool equal);
+ void cmp_klass_beq(Register obj, Register klass,
+ Register tmp1, Register tmp2,
+ Label &L, bool is_far = false);
+ void cmp_klass_bne(Register obj, Register klass,
+ Register tmp1, Register tmp2,
+ Label &L, bool is_far = false);
void encode_klass_not_null(Register r, Register tmp = t0);
void decode_klass_not_null(Register r, Register tmp = t0);
@@ -813,15 +818,6 @@ class MacroAssembler: public Assembler {
void double_bgt(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far = false, bool is_unordered = false);
private:
- int push_reg(unsigned int bitset, Register stack);
- int pop_reg(unsigned int bitset, Register stack);
- int push_fp(unsigned int bitset, Register stack);
- int pop_fp(unsigned int bitset, Register stack);
-#ifdef COMPILER2
- int push_v(unsigned int bitset, Register stack);
- int pop_v(unsigned int bitset, Register stack);
-#endif // COMPILER2
-
// The signed 20-bit upper imm can materialize at most negative 0xF...F80000000, two G.
// The following signed 12-bit imm can at max subtract 0x800, two K, from that previously loaded two G.
bool is_valid_32bit_offset(int64_t x) {
@@ -839,15 +835,19 @@ private:
}
public:
+ // Stack push and pop individual 64 bit registers
void push_reg(Register Rs);
void pop_reg(Register Rd);
- void push_reg(RegSet regs, Register stack) { if (regs.bits()) push_reg(regs.bits(), stack); }
- void pop_reg(RegSet regs, Register stack) { if (regs.bits()) pop_reg(regs.bits(), stack); }
- void push_fp(FloatRegSet regs, Register stack) { if (regs.bits()) push_fp(regs.bits(), stack); }
- void pop_fp(FloatRegSet regs, Register stack) { if (regs.bits()) pop_fp(regs.bits(), stack); }
+
+ int push_reg(RegSet regset, Register stack);
+ int pop_reg(RegSet regset, Register stack);
+
+ int push_fp(FloatRegSet regset, Register stack);
+ int pop_fp(FloatRegSet regset, Register stack);
+
#ifdef COMPILER2
- void push_v(VectorRegSet regs, Register stack) { if (regs.bits()) push_v(regs.bits(), stack); }
- void pop_v(VectorRegSet regs, Register stack) { if (regs.bits()) pop_v(regs.bits(), stack); }
+ int push_v(VectorRegSet regset, Register stack);
+ int pop_v(VectorRegSet regset, Register stack);
#endif // COMPILER2
// Push and pop everything that might be clobbered by a native
@@ -1348,9 +1348,8 @@ public:
void decrement(const Address dst, int64_t value = 1, Register tmp1 = t0, Register tmp2 = t1);
void decrementw(const Address dst, int32_t value = 1, Register tmp1 = t0, Register tmp2 = t1);
- void cmpptr(Register src1, const Address &src2, Label& equal, Register tmp = t0);
-
void clinit_barrier(Register klass, Register tmp, Label* L_fast_path = nullptr, Label* L_slow_path = nullptr);
+
void load_method_holder_cld(Register result, Register method);
void load_method_holder(Register holder, Register method);
diff --git a/src/hotspot/cpu/riscv/methodHandles_riscv.cpp b/src/hotspot/cpu/riscv/methodHandles_riscv.cpp
index d770999df96..e80dedf58ed 100644
--- a/src/hotspot/cpu/riscv/methodHandles_riscv.cpp
+++ b/src/hotspot/cpu/riscv/methodHandles_riscv.cpp
@@ -72,17 +72,22 @@ void MethodHandles::verify_klass(MacroAssembler* _masm,
InstanceKlass** klass_addr = vmClasses::klass_addr_at(klass_id);
Klass* klass = vmClasses::klass_at(klass_id);
Register temp1 = t1;
- Register temp2 = t0; // used by MacroAssembler::cmpptr
+ Register temp2 = t0;
Label L_ok, L_bad;
BLOCK_COMMENT("verify_klass {");
__ verify_oop(obj);
__ beqz(obj, L_bad);
+
__ push_reg(RegSet::of(temp1, temp2), sp);
__ load_klass(temp1, obj, temp2);
- __ cmpptr(temp1, ExternalAddress((address) klass_addr), L_ok);
+ __ ld(temp2, ExternalAddress((address)klass_addr));
+ __ beq(temp1, temp2, L_ok);
+
intptr_t super_check_offset = klass->super_check_offset();
__ ld(temp1, Address(temp1, super_check_offset));
- __ cmpptr(temp1, ExternalAddress((address) klass_addr), L_ok);
+ __ ld(temp2, ExternalAddress((address)klass_addr));
+ __ beq(temp1, temp2, L_ok);
+
__ pop_reg(RegSet::of(temp1, temp2), sp);
__ bind(L_bad);
__ stop(error_message);
diff --git a/src/hotspot/cpu/riscv/riscv.ad b/src/hotspot/cpu/riscv/riscv.ad
index 730dd68dd88..e236d03e6d2 100644
--- a/src/hotspot/cpu/riscv/riscv.ad
+++ b/src/hotspot/cpu/riscv/riscv.ad
@@ -1,5 +1,5 @@
//
-// Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
// Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
// Copyright (c) 2020, 2024, Huawei Technologies Co., Ltd. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -1801,13 +1801,8 @@ void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
{
assert_cond(st != nullptr);
st->print_cr("# MachUEPNode");
- if (UseCompressedClassPointers) {
- st->print_cr("\tlwu t1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
- st->print_cr("\tlwu t2, [t0 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
- } else {
- st->print_cr("\tld t1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
- st->print_cr("\tld t2, [t0 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
- }
+ st->print_cr("\tlwu t1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
+ st->print_cr("\tlwu t2, [t0 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
st->print_cr("\tbeq t1, t2, ic_hit");
st->print_cr("\tj, SharedRuntime::_ic_miss_stub\t # Inline cache check");
st->print_cr("\tic_hit:");
@@ -2060,11 +2055,8 @@ bool Matcher::is_generic_vector(MachOper* opnd) {
return false;
}
+#ifdef ASSERT
// Return whether or not this register is ever used as an argument.
-// This function is used on startup to build the trampoline stubs in
-// generateOptoStub. Registers not mentioned will be killed by the VM
-// call in the trampoline, and arguments in those registers not be
-// available to the callee.
bool Matcher::can_be_java_arg(int reg)
{
return
@@ -2085,11 +2077,7 @@ bool Matcher::can_be_java_arg(int reg)
reg == F16_num || reg == F16_H_num ||
reg == F17_num || reg == F17_H_num;
}
-
-bool Matcher::is_spillable_arg(int reg)
-{
- return can_be_java_arg(reg);
-}
+#endif
uint Matcher::int_pressure_limit()
{
@@ -2118,10 +2106,6 @@ uint Matcher::float_pressure_limit()
return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
}
-bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
- return false;
-}
-
const RegMask& Matcher::divI_proj_mask() {
ShouldNotReachHere();
return RegMask::EMPTY;
@@ -2274,7 +2258,7 @@ encode %{
} else if (rtype == relocInfo::metadata_type) {
__ mov_metadata(dst_reg, (Metadata*)con);
} else {
- assert(rtype == relocInfo::none, "unexpected reloc type");
+ assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
__ mv(dst_reg, $src$$constant);
}
}
@@ -2559,11 +2543,6 @@ frame %{
// Compiled code's Frame Pointer
frame_pointer(R2);
- // Interpreter stores its frame pointer in a register which is
- // stored to the stack by I2CAdaptors.
- // I2CAdaptors convert from interpreted java to compiled java.
- interpreter_frame_pointer(R8);
-
// Stack alignment requirement
stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
@@ -8168,6 +8147,22 @@ instruct unnecessary_membar_rvtso() %{
ins_pipe(real_empty);
%}
+instruct membar_storeload_rvtso() %{
+ predicate(UseZtso);
+ match(MemBarStoreLoad);
+ ins_cost(VOLATILE_REF_COST);
+
+ format %{ "#@membar_storeload_rvtso\n\t"
+ "fence w, r"%}
+
+ ins_encode %{
+ __ block_comment("membar_storeload_rvtso");
+ __ membar(MacroAssembler::StoreLoad);
+ %}
+
+ ins_pipe(pipe_slow);
+%}
+
instruct membar_volatile_rvtso() %{
predicate(UseZtso);
match(MemBarVolatile);
@@ -8198,6 +8193,22 @@ instruct unnecessary_membar_volatile_rvtso() %{
ins_pipe(real_empty);
%}
+instruct membar_full_rvtso() %{
+ predicate(UseZtso);
+ match(MemBarFull);
+ ins_cost(VOLATILE_REF_COST);
+
+ format %{ "#@membar_full_rvtso\n\t"
+ "fence rw, rw" %}
+
+ ins_encode %{
+ __ block_comment("membar_full_rvtso");
+ __ membar(MacroAssembler::AnyAny);
+ %}
+
+ ins_pipe(pipe_slow);
+%}
+
// RVWMO
instruct membar_aqcuire_rvwmo() %{
@@ -8247,6 +8258,22 @@ instruct membar_storestore_rvwmo() %{
ins_pipe(pipe_serial);
%}
+instruct membar_storeload_rvwmo() %{
+ predicate(!UseZtso);
+ match(MemBarStoreLoad);
+ ins_cost(VOLATILE_REF_COST);
+
+ format %{ "#@membar_storeload_rvwmo\n\t"
+ "fence w, r"%}
+
+ ins_encode %{
+ __ block_comment("membar_storeload_rvwmo");
+ __ membar(MacroAssembler::StoreLoad);
+ %}
+
+ ins_pipe(pipe_serial);
+%}
+
instruct membar_volatile_rvwmo() %{
predicate(!UseZtso);
match(MemBarVolatile);
@@ -8291,6 +8318,22 @@ instruct unnecessary_membar_volatile_rvwmo() %{
ins_pipe(real_empty);
%}
+instruct membar_full_rvwmo() %{
+ predicate(!UseZtso);
+ match(MemBarFull);
+ ins_cost(VOLATILE_REF_COST);
+
+ format %{ "#@membar_full_rvwmo\n\t"
+ "fence rw, rw" %}
+
+ ins_encode %{
+ __ block_comment("membar_full_rvwmo");
+ __ membar(MacroAssembler::AnyAny);
+ %}
+
+ ins_pipe(pipe_serial);
+%}
+
instruct spin_wait() %{
predicate(UseZihintpause);
match(OnSpinWait);
diff --git a/src/hotspot/cpu/riscv/stubDeclarations_riscv.hpp b/src/hotspot/cpu/riscv/stubDeclarations_riscv.hpp
index f977d759d20..890e354fd27 100644
--- a/src/hotspot/cpu/riscv/stubDeclarations_riscv.hpp
+++ b/src/hotspot/cpu/riscv/stubDeclarations_riscv.hpp
@@ -29,28 +29,32 @@
#define STUBGEN_PREUNIVERSE_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(preuniverse, 0) \
#define STUBGEN_INITIAL_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(initial, 10000) \
#define STUBGEN_CONTINUATION_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(continuation, 2000) \
#define STUBGEN_COMPILER_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(compiler, 45000) \
do_stub(compiler, compare_long_string_LL) \
do_arch_entry(riscv, compiler, compare_long_string_LL, \
@@ -81,7 +85,8 @@
#define STUBGEN_FINAL_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(final, 20000 ZGC_ONLY(+10000)) \
do_stub(final, copy_byte_f) \
do_arch_entry(riscv, final, copy_byte_f, copy_byte_f, \
diff --git a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp
index 127ac9f6951..4656b5c0d41 100644
--- a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp
+++ b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2025, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2025, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -3070,8 +3070,7 @@ class StubGenerator: public StubCodeGenerator {
const Register tmp = x30, tmpLval = x12;
int base_offset = arrayOopDesc::base_offset_in_bytes(T_BYTE);
- assert((base_offset % (UseCompactObjectHeaders ? 4 :
- (UseCompressedClassPointers ? 8 : 4))) == 0, "Must be");
+ assert((base_offset % (UseCompactObjectHeaders ? 4 : 8)) == 0, "Must be");
#ifdef ASSERT
if (AvoidUnalignedAccesses) {
@@ -3128,8 +3127,7 @@ class StubGenerator: public StubCodeGenerator {
tmp1 = x28, tmp2 = x29, tmp3 = x30, tmp4 = x12;
int base_offset = arrayOopDesc::base_offset_in_bytes(T_BYTE);
- assert((base_offset % (UseCompactObjectHeaders ? 4 :
- (UseCompressedClassPointers ? 8 : 4))) == 0, "Must be");
+ assert((base_offset % (UseCompactObjectHeaders ? 4 : 8)) == 0, "Must be");
Register strU = isLU ? str2 : str1,
strL = isLU ? str1 : str2,
@@ -7350,7 +7348,7 @@ static const int64_t right_3_bits = right_n_bits(3);
}
public:
- StubGenerator(CodeBuffer* code, BlobId blob_id) : StubCodeGenerator(code, blob_id) {
+ StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) : StubCodeGenerator(code, blob_id, stub_data) {
switch(blob_id) {
case BlobId::stubgen_preuniverse_id:
generate_preuniverse_stubs();
@@ -7374,6 +7372,6 @@ static const int64_t right_3_bits = right_n_bits(3);
}
}; // end class declaration
-void StubGenerator_generate(CodeBuffer* code, BlobId blob_id) {
- StubGenerator g(code, blob_id);
+void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) {
+ StubGenerator g(code, blob_id, stub_data);
}
diff --git a/src/hotspot/cpu/riscv/stubRoutines_riscv.cpp b/src/hotspot/cpu/riscv/stubRoutines_riscv.cpp
index 2aac95d71fa..b7f69eff9fa 100644
--- a/src/hotspot/cpu/riscv/stubRoutines_riscv.cpp
+++ b/src/hotspot/cpu/riscv/stubRoutines_riscv.cpp
@@ -42,8 +42,12 @@
#define DEFINE_ARCH_ENTRY_INIT(arch, blob_name, stub_name, field_name, getter_name, init_function) \
address StubRoutines:: arch :: STUB_FIELD_NAME(field_name) = CAST_FROM_FN_PTR(address, init_function);
-STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY, DEFINE_ARCH_ENTRY_INIT)
+#define DEFINE_ARCH_ENTRY_ARRAY(arch, blob_name, stub_name, field_name, getter_name, count) \
+ address StubRoutines:: arch :: STUB_FIELD_NAME(field_name) [count] ;
+STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY, DEFINE_ARCH_ENTRY_INIT, DEFINE_ARCH_ENTRY_ARRAY)
+
+#undef DEFINE_ARCH_ENTRY_ARRAY
#undef DEFINE_ARCH_ENTRY_INIT
#undef DEFINE_ARCH_ENTRY
@@ -501,3 +505,9 @@ ATTRIBUTE_ALIGNED(4096) juint StubRoutines::riscv::_crc_table[] =
0x751997d0UL, 0x00000001UL,
0xccaa009eUL, 0x00000000UL,
};
+
+#if INCLUDE_CDS
+// nothing to do for riscv
+void StubRoutines::init_AOTAddressTable() {
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/riscv/stubRoutines_riscv.hpp b/src/hotspot/cpu/riscv/stubRoutines_riscv.hpp
index 2c4e7210413..ec67a338052 100644
--- a/src/hotspot/cpu/riscv/stubRoutines_riscv.hpp
+++ b/src/hotspot/cpu/riscv/stubRoutines_riscv.hpp
@@ -61,9 +61,13 @@ class riscv {
#define DECLARE_ARCH_ENTRY_INIT(arch, blob_name, stub_name, field_name, getter_name, init_function) \
DECLARE_ARCH_ENTRY(arch, blob_name, stub_name, field_name, getter_name)
-private:
- STUBGEN_ARCH_ENTRIES_DO(DECLARE_ARCH_ENTRY, DECLARE_ARCH_ENTRY_INIT)
+#define DECLARE_ARCH_ENTRY_ARRAY(arch, blob_name, stub_name, field_name, getter_name, count) \
+ static address STUB_FIELD_NAME(field_name) [count] ;
+private:
+ STUBGEN_ARCH_ENTRIES_DO(DECLARE_ARCH_ENTRY, DECLARE_ARCH_ENTRY_INIT, DECLARE_ARCH_ENTRY_ARRAY)
+
+#undef DECLARE_ARCH_ENTRY_ARRAY
#undef DECLARE_ARCH_ENTRY_INIT
#undef DECLARE_ARCH_ENTRY
@@ -79,8 +83,12 @@ private:
#define DEFINE_ARCH_ENTRY_GETTER_INIT(arch, blob_name, stub_name, field_name, getter_name, init_function) \
DEFINE_ARCH_ENTRY_GETTER(arch, blob_name, stub_name, field_name, getter_name)
- STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY_GETTER, DEFINE_ARCH_ENTRY_GETTER_INIT)
+#define DEFINE_ARCH_ENTRY_GETTER_ARRAY(arch, blob_name, stub_name, field_name, getter_name, count) \
+ static address getter_name(int idx) { return STUB_FIELD_NAME(field_name) [idx] ; }
+ STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY_GETTER, DEFINE_ARCH_ENTRY_GETTER_INIT, DEFINE_ARCH_ENTRY_GETTER_ARRAY)
+
+#undef DEFINE_ARCH_ENTRY_GETTER_ARRAY
#undef DEFINE_ARCH_ENTRY_GETTER_INIT
#undef DEFINE_ARCH_ENTRY_GETTER
diff --git a/src/hotspot/cpu/riscv/vm_version_riscv.cpp b/src/hotspot/cpu/riscv/vm_version_riscv.cpp
index 36f0864da0b..3a6415d52bd 100644
--- a/src/hotspot/cpu/riscv/vm_version_riscv.cpp
+++ b/src/hotspot/cpu/riscv/vm_version_riscv.cpp
@@ -420,11 +420,6 @@ void VM_Version::c2_initialize() {
FLAG_SET_DEFAULT(UseSHA3Intrinsics, false);
}
- // UseSHA
- if (!(UseSHA1Intrinsics || UseSHA256Intrinsics || UseSHA3Intrinsics || UseSHA512Intrinsics)) {
- FLAG_SET_DEFAULT(UseSHA, false);
- }
-
// AES
if (UseZvkn) {
UseAES = UseAES || FLAG_IS_DEFAULT(UseAES);
diff --git a/src/hotspot/cpu/riscv/vm_version_riscv.hpp b/src/hotspot/cpu/riscv/vm_version_riscv.hpp
index 03c843efc69..11a88dfedd7 100644
--- a/src/hotspot/cpu/riscv/vm_version_riscv.hpp
+++ b/src/hotspot/cpu/riscv/vm_version_riscv.hpp
@@ -55,7 +55,7 @@ class VM_Version : public Abstract_VM_Version {
public:
RVFeatureValue(const char* pretty, int linux_bit_num, bool fstring) :
- _pretty(pretty), _feature_string(fstring), _linux_feature_bit(nth_bit(linux_bit_num)) {
+ _pretty(pretty), _feature_string(fstring), _linux_feature_bit(nth_bit(linux_bit_num)) {
}
virtual void enable_feature(int64_t value = 0) = 0;
virtual void disable_feature() = 0;
diff --git a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp
index 93d6051aa76..e1d8d062c23 100644
--- a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -2251,9 +2251,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
// but not necessarily exactly of type default_type.
NearLabel known_ok, halt;
metadata2reg(default_type->constant_encoding(), tmp);
- if (UseCompressedClassPointers) {
- __ encode_klass_not_null(tmp);
- }
+ __ encode_klass_not_null(tmp);
if (basic_type != T_OBJECT) {
__ cmp_klass(tmp, dst, Z_R1_scratch);
@@ -2540,13 +2538,8 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, L
// Get object class.
// Not a safepoint as obj null check happens earlier.
if (op->fast_check()) {
- if (UseCompressedClassPointers) {
- __ load_klass(klass_RInfo, obj);
- __ compareU64_and_branch(k_RInfo, klass_RInfo, Assembler::bcondNotEqual, *failure_target);
- } else {
- __ z_cg(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
- __ branch_optimized(Assembler::bcondNotEqual, *failure_target);
- }
+ __ load_klass(klass_RInfo, obj);
+ __ compareU64_and_branch(k_RInfo, klass_RInfo, Assembler::bcondNotEqual, *failure_target);
// Successful cast, fall through to profile or jump.
} else {
bool need_slow_path = !k->is_loaded() ||
diff --git a/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp b/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp
index 993c1a1b552..813143938f9 100644
--- a/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -107,10 +107,10 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
}
if (len->is_valid()) {
- // Length will be in the klass gap, if one exists.
+ // Length will be in the klass gap.
z_st(len, Address(obj, arrayOopDesc::length_offset_in_bytes()));
- } else if (UseCompressedClassPointers && !UseCompactObjectHeaders) {
- store_klass_gap(Rzero, obj); // Zero klass gap for compressed oops.
+ } else if (!UseCompactObjectHeaders) {
+ store_klass_gap(Rzero, obj); // Zero klass gap.
}
}
diff --git a/src/hotspot/cpu/s390/c1_globals_s390.hpp b/src/hotspot/cpu/s390/c1_globals_s390.hpp
index 25e46cd1509..64cc239800a 100644
--- a/src/hotspot/cpu/s390/c1_globals_s390.hpp
+++ b/src/hotspot/cpu/s390/c1_globals_s390.hpp
@@ -51,8 +51,6 @@ define_pd_global(size_t, NonNMethodCodeHeapSize, 5*M);
define_pd_global(size_t, CodeCacheExpansionSize, 32*K);
define_pd_global(size_t, CodeCacheMinBlockLength, 1);
define_pd_global(size_t, CodeCacheMinimumUseSpace, 400*K);
-define_pd_global(bool, NeverActAsServerClassMachine, true);
-define_pd_global(size_t, NewSizeThreadIncrease, 16*K);
define_pd_global(size_t, InitialCodeCacheSize, 160*K);
#endif // !COMPILER2
diff --git a/src/hotspot/cpu/s390/c2_globals_s390.hpp b/src/hotspot/cpu/s390/c2_globals_s390.hpp
index 125b317588d..eee3a8588c3 100644
--- a/src/hotspot/cpu/s390/c2_globals_s390.hpp
+++ b/src/hotspot/cpu/s390/c2_globals_s390.hpp
@@ -46,7 +46,6 @@ define_pd_global(intx, OnStackReplacePercentage, 140);
define_pd_global(intx, ConditionalMoveLimit, 4);
define_pd_global(intx, FreqInlineSize, 325);
define_pd_global(intx, InteriorEntryAlignment, 4);
-define_pd_global(size_t, NewSizeThreadIncrease, ScaleForWordSize(4*K));
define_pd_global(intx, RegisterCostAreaRatio, 12000);
define_pd_global(intx, LoopUnrollLimit, 60);
define_pd_global(intx, LoopPercentProfileLimit, 10);
@@ -79,7 +78,4 @@ define_pd_global(size_t, CodeCacheMinimumUseSpace, 400*K);
define_pd_global(bool, TrapBasedRangeChecks, false); // Not needed on z/Architecture.
-// Ergonomics related flags
-define_pd_global(bool, NeverActAsServerClassMachine, false);
-
#endif // CPU_S390_C2_GLOBALS_S390_HPP
diff --git a/src/hotspot/cpu/s390/downcallLinker_s390.cpp b/src/hotspot/cpu/s390/downcallLinker_s390.cpp
index ccd8002da37..f1c41d05b5c 100644
--- a/src/hotspot/cpu/s390/downcallLinker_s390.cpp
+++ b/src/hotspot/cpu/s390/downcallLinker_s390.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2022, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, Red Hat, Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -129,7 +129,7 @@ void DowncallLinker::StubGenerator::generate() {
assert(!_needs_return_buffer, "unexpected needs_return_buffer");
RegSpiller out_reg_spiller(_output_registers);
- int spill_offset = allocated_frame_size;
+ int out_spill_offset = allocated_frame_size;
allocated_frame_size += BytesPerWord;
StubLocations locs;
@@ -153,6 +153,18 @@ void DowncallLinker::StubGenerator::generate() {
GrowableArray out_regs = ForeignGlobals::replace_place_holders(_input_registers, locs);
ArgumentShuffle arg_shuffle(filtered_java_regs, out_regs, _abi._scratch1);
+ // Need to spill for state capturing runtime call.
+ // The area spilled into is distinct from the capture state buffer.
+ RegSpiller in_reg_spiller(out_regs);
+ int in_spill_offset = -1;
+ if (_captured_state_mask != 0) {
+ // The spill area cannot be shared with the out_spill since
+ // spilling needs to happen before the call. Allocate a new
+ // region in the stack for this spill space.
+ in_spill_offset = allocated_frame_size;
+ allocated_frame_size += in_reg_spiller.spill_size_bytes();
+ }
+
#ifndef PRODUCT
LogTarget(Trace, foreign, downcall) lt;
if (lt.is_enabled()) {
@@ -192,6 +204,21 @@ void DowncallLinker::StubGenerator::generate() {
arg_shuffle.generate(_masm, shuffle_reg, frame::z_jit_out_preserve_size, _abi._shadow_space_bytes);
__ block_comment("} argument_shuffle");
+ if (_captured_state_mask != 0) {
+ assert(in_spill_offset != -1, "must be");
+ __ block_comment("{ load initial thread local");
+ in_reg_spiller.generate_spill(_masm, in_spill_offset);
+
+ // Copy the contents of the capture state buffer into thread local
+ __ load_const_optimized(call_target_address, CAST_FROM_FN_PTR(uint64_t, DowncallLinker::capture_state_pre));
+ __ z_lg(Z_ARG1, Address(Z_SP, locs.data_offset(StubLocations::CAPTURED_STATE_BUFFER)));
+ __ load_const_optimized(Z_ARG2, _captured_state_mask);
+ __ call(call_target_address);
+
+ in_reg_spiller.generate_fill(_masm, in_spill_offset);
+ __ block_comment("} load initial thread local");
+ }
+
__ call(as_Register(locs.get(StubLocations::TARGET_ADDRESS)));
//////////////////////////////////////////////////////////////////////////////
@@ -199,14 +226,14 @@ void DowncallLinker::StubGenerator::generate() {
if (_captured_state_mask != 0) {
__ block_comment("save_thread_local {");
- out_reg_spiller.generate_spill(_masm, spill_offset);
+ out_reg_spiller.generate_spill(_masm, out_spill_offset);
- __ load_const_optimized(call_target_address, CAST_FROM_FN_PTR(uint64_t, DowncallLinker::capture_state));
+ __ load_const_optimized(call_target_address, CAST_FROM_FN_PTR(uint64_t, DowncallLinker::capture_state_post));
__ z_lg(Z_ARG1, Address(Z_SP, locs.data_offset(StubLocations::CAPTURED_STATE_BUFFER)));
__ load_const_optimized(Z_ARG2, _captured_state_mask);
__ call(call_target_address);
- out_reg_spiller.generate_fill(_masm, spill_offset);
+ out_reg_spiller.generate_fill(_masm, out_spill_offset);
__ block_comment("} save_thread_local");
}
@@ -259,13 +286,13 @@ void DowncallLinker::StubGenerator::generate() {
__ bind(L_safepoint_poll_slow_path);
// Need to save the native result registers around any runtime calls.
- out_reg_spiller.generate_spill(_masm, spill_offset);
+ out_reg_spiller.generate_spill(_masm, out_spill_offset);
__ load_const_optimized(call_target_address, CAST_FROM_FN_PTR(uint64_t, JavaThread::check_special_condition_for_native_trans));
__ z_lgr(Z_ARG1, Z_thread);
__ call(call_target_address);
- out_reg_spiller.generate_fill(_masm, spill_offset);
+ out_reg_spiller.generate_fill(_masm, out_spill_offset);
__ z_bru(L_after_safepoint_poll);
__ block_comment("} L_safepoint_poll_slow_path");
@@ -275,12 +302,12 @@ void DowncallLinker::StubGenerator::generate() {
__ bind(L_reguard);
// Need to save the native result registers around any runtime calls.
- out_reg_spiller.generate_spill(_masm, spill_offset);
+ out_reg_spiller.generate_spill(_masm, out_spill_offset);
__ load_const_optimized(call_target_address, CAST_FROM_FN_PTR(uint64_t, SharedRuntime::reguard_yellow_pages));
__ call(call_target_address);
- out_reg_spiller.generate_fill(_masm, spill_offset);
+ out_reg_spiller.generate_fill(_masm, out_spill_offset);
__ z_bru(L_after_reguard);
diff --git a/src/hotspot/cpu/s390/frame_s390.hpp b/src/hotspot/cpu/s390/frame_s390.hpp
index ad754706367..bcdeec43e1a 100644
--- a/src/hotspot/cpu/s390/frame_s390.hpp
+++ b/src/hotspot/cpu/s390/frame_s390.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -463,7 +463,7 @@
// Accessors
- inline intptr_t* fp() const { return _fp; }
+ inline intptr_t* fp() const { assert_absolute(); return _fp; }
private:
diff --git a/src/hotspot/cpu/s390/frame_s390.inline.hpp b/src/hotspot/cpu/s390/frame_s390.inline.hpp
index dea0e72581f..6fcd36c57d1 100644
--- a/src/hotspot/cpu/s390/frame_s390.inline.hpp
+++ b/src/hotspot/cpu/s390/frame_s390.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -133,10 +133,10 @@ inline void frame::interpreter_frame_set_monitors(BasicObjectLock* monitors) {
// Return unique id for this frame. The id must have a value where we
// can distinguish identity and younger/older relationship. null
-// represents an invalid (incomparable) frame.
+// represents an invalid (incomparable) frame. Should not be called for heap frames.
inline intptr_t* frame::id(void) const {
// Use _fp. _sp or _unextended_sp wouldn't be correct due to resizing.
- return _fp;
+ return real_fp();
}
// Return true if this frame is older (less recent activation) than
diff --git a/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.cpp b/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.cpp
index 7617c7a49e8..9fac231df47 100644
--- a/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.cpp
@@ -169,6 +169,11 @@ void BarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Re
__ z_lg(obj, 0, obj); // Resolve (untagged) jobject.
}
+void BarrierSetAssembler::try_resolve_weak_handle(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path) {
+ // Load the oop from the weak handle.
+ __ z_lg(obj, Address(obj));
+}
+
void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm) {
BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
__ align(4, __ offset() + OFFSET_TO_PATCHABLE_DATA); // must align the following block which requires atomic updates
@@ -206,11 +211,6 @@ OptoReg::Name BarrierSetAssembler::refine_register(const Node* node, OptoReg::Na
return opto_reg;
}
-void BarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path) {
- // Load the oop from the weak handle.
- __ z_lg(obj, Address(obj));
-}
-
#undef __
#define __ _masm->
diff --git a/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.hpp b/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.hpp
index d5682450414..8e76ec2f4b4 100644
--- a/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.hpp
+++ b/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.hpp
@@ -58,6 +58,11 @@ public:
virtual void try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
Register obj, Register tmp, Label& slowpath);
+ // Can be used in nmethods including native wrappers.
+ // Attention: obj will only be valid until next safepoint (no SATB barrier).
+ // (other platforms currently use it for C2 only: try_resolve_weak_handle_in_c2)
+ virtual void try_resolve_weak_handle(MacroAssembler* masm, Register obj, Register tmp, Label& slow_path);
+
virtual void nmethod_entry_barrier(MacroAssembler* masm);
virtual void barrier_stubs_init() {}
@@ -65,8 +70,6 @@ public:
#ifdef COMPILER2
OptoReg::Name refine_register(const Node* node,
OptoReg::Name opto_reg) const;
- virtual void try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj,
- Register tmp, Label& slow_path);
#endif // COMPILER2
static const int OFFSET_TO_PATCHABLE_DATA_INSTRUCTION = 6 + 6 + 6; // iihf(6) + iilf(6) + lg(6)
diff --git a/src/hotspot/cpu/s390/interp_masm_s390.cpp b/src/hotspot/cpu/s390/interp_masm_s390.cpp
index a80ca26239b..d5239898dd7 100644
--- a/src/hotspot/cpu/s390/interp_masm_s390.cpp
+++ b/src/hotspot/cpu/s390/interp_masm_s390.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -1259,27 +1259,15 @@ void InterpreterMacroAssembler::profile_final_call(Register mdp) {
void InterpreterMacroAssembler::profile_virtual_call(Register receiver,
Register mdp,
- Register reg2,
- bool receiver_can_be_null) {
+ Register reg2) {
if (ProfileInterpreter) {
NearLabel profile_continue;
// If no method data exists, go to profile_continue.
test_method_data_pointer(mdp, profile_continue);
- NearLabel skip_receiver_profile;
- if (receiver_can_be_null) {
- NearLabel not_null;
- compareU64_and_branch(receiver, (intptr_t)0L, bcondNotEqual, not_null);
- // We are making a call. Increment the count for null receiver.
- increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
- z_bru(skip_receiver_profile);
- bind(not_null);
- }
-
// Record the receiver type.
record_klass_in_profile(receiver, mdp, reg2);
- bind(skip_receiver_profile);
// The method data pointer needs to be updated to reflect the new target.
update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size()));
diff --git a/src/hotspot/cpu/s390/interp_masm_s390.hpp b/src/hotspot/cpu/s390/interp_masm_s390.hpp
index d981f9ea01e..b816185b065 100644
--- a/src/hotspot/cpu/s390/interp_masm_s390.hpp
+++ b/src/hotspot/cpu/s390/interp_masm_s390.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -296,8 +296,7 @@ class InterpreterMacroAssembler: public MacroAssembler {
void profile_call(Register mdp);
void profile_final_call(Register mdp);
void profile_virtual_call(Register receiver, Register mdp,
- Register scratch2,
- bool receiver_can_be_null = false);
+ Register scratch2);
void profile_ret(Register return_bci, Register mdp);
void profile_null_seen(Register mdp);
void profile_typecheck(Register mdp, Register klass, Register scratch);
diff --git a/src/hotspot/cpu/s390/macroAssembler_s390.cpp b/src/hotspot/cpu/s390/macroAssembler_s390.cpp
index 78779a9098a..de3608e74ba 100644
--- a/src/hotspot/cpu/s390/macroAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/macroAssembler_s390.cpp
@@ -1,7 +1,7 @@
/*
* Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2024 SAP SE. All rights reserved.
- * Copyright 2024 IBM Corporation. All rights reserved.
+ * Copyright 2024, 2026 IBM Corporation. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1237,7 +1237,6 @@ void MacroAssembler::load_narrow_oop(Register t, narrowOop a) {
// Load narrow klass constant, compression required.
void MacroAssembler::load_narrow_klass(Register t, Klass* k) {
- assert(UseCompressedClassPointers, "must be on to call this method");
narrowKlass encoded_k = CompressedKlassPointers::encode(k);
load_const_32to64(t, encoded_k, false /*sign_extend*/);
}
@@ -1255,7 +1254,6 @@ void MacroAssembler::compare_immediate_narrow_oop(Register oop1, narrowOop oop2)
// Compare narrow oop in reg with narrow oop constant, no decompression.
void MacroAssembler::compare_immediate_narrow_klass(Register klass1, Klass* klass2) {
- assert(UseCompressedClassPointers, "must be on to call this method");
narrowKlass encoded_k = CompressedKlassPointers::encode(klass2);
Assembler::z_clfi(klass1, encoded_k);
@@ -1348,8 +1346,6 @@ int MacroAssembler::patch_load_narrow_oop(address pos, oop o) {
// Patching the immediate value of CPU version dependent load_narrow_klass sequence.
// The passed ptr must NOT be in compressed format!
int MacroAssembler::patch_load_narrow_klass(address pos, Klass* k) {
- assert(UseCompressedClassPointers, "Can only patch compressed klass pointers");
-
narrowKlass nk = CompressedKlassPointers::encode(k);
return patch_load_const_32to64(pos, nk);
}
@@ -1364,8 +1360,6 @@ int MacroAssembler::patch_compare_immediate_narrow_oop(address pos, oop o) {
// Patching the immediate value of CPU version dependent compare_immediate_narrow_klass sequence.
// The passed ptr must NOT be in compressed format!
int MacroAssembler::patch_compare_immediate_narrow_klass(address pos, Klass* k) {
- assert(UseCompressedClassPointers, "Can only patch compressed klass pointers");
-
narrowKlass nk = CompressedKlassPointers::encode(k);
return patch_compare_immediate_32(pos, nk);
}
@@ -2235,10 +2229,8 @@ int MacroAssembler::ic_check(int end_alignment) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(R1_scratch, R2_receiver);
- } else if (UseCompressedClassPointers) {
- z_llgf(R1_scratch, Address(R2_receiver, oopDesc::klass_offset_in_bytes()));
} else {
- z_lg(R1_scratch, Address(R2_receiver, oopDesc::klass_offset_in_bytes()));
+ z_llgf(R1_scratch, Address(R2_receiver, oopDesc::klass_offset_in_bytes()));
}
z_cg(R1_scratch, Address(R9_data, in_bytes(CompiledICData::speculated_klass_offset())));
z_bre(success);
@@ -3916,7 +3908,6 @@ void MacroAssembler::encode_klass_not_null(Register dst, Register src) {
address base = CompressedKlassPointers::base();
int shift = CompressedKlassPointers::shift();
bool need_zero_extend = base != nullptr;
- assert(UseCompressedClassPointers, "only for compressed klass ptrs");
BLOCK_COMMENT("cKlass encoder {");
@@ -4013,7 +4004,6 @@ int MacroAssembler::instr_size_for_decode_klass_not_null() {
address base = CompressedKlassPointers::base();
int shift_size = CompressedKlassPointers::shift() == 0 ? 0 : 6; /* sllg */
int addbase_size = 0;
- assert(UseCompressedClassPointers, "only for compressed klass ptrs");
if (base != nullptr) {
unsigned int base_h = ((unsigned long)base)>>32;
@@ -4043,7 +4033,6 @@ void MacroAssembler::decode_klass_not_null(Register dst) {
address base = CompressedKlassPointers::base();
int shift = CompressedKlassPointers::shift();
int beg_off = offset();
- assert(UseCompressedClassPointers, "only for compressed klass ptrs");
BLOCK_COMMENT("cKlass decoder (const size) {");
@@ -4085,7 +4074,6 @@ void MacroAssembler::decode_klass_not_null(Register dst) {
void MacroAssembler::decode_klass_not_null(Register dst, Register src) {
address base = CompressedKlassPointers::base();
int shift = CompressedKlassPointers::shift();
- assert(UseCompressedClassPointers, "only for compressed klass ptrs");
BLOCK_COMMENT("cKlass decoder {");
@@ -4125,13 +4113,9 @@ void MacroAssembler::decode_klass_not_null(Register dst, Register src) {
}
void MacroAssembler::load_klass(Register klass, Address mem) {
- if (UseCompressedClassPointers) {
- z_llgf(klass, mem);
- // Attention: no null check here!
- decode_klass_not_null(klass);
- } else {
- z_lg(klass, mem);
- }
+ z_llgf(klass, mem);
+ // Attention: no null check here!
+ decode_klass_not_null(klass);
}
// Loads the obj's Klass* into dst.
@@ -4154,10 +4138,8 @@ void MacroAssembler::cmp_klass(Register klass, Register obj, Register tmp) {
assert_different_registers(klass, obj, tmp);
load_narrow_klass_compact(tmp, obj);
z_cr(klass, tmp);
- } else if (UseCompressedClassPointers) {
- z_c(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
} else {
- z_cg(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
+ z_c(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
}
BLOCK_COMMENT("} cmp_klass");
}
@@ -4170,12 +4152,9 @@ void MacroAssembler::cmp_klasses_from_objects(Register obj1, Register obj2, Regi
load_narrow_klass_compact(tmp1, obj1);
load_narrow_klass_compact(tmp2, obj2);
z_cr(tmp1, tmp2);
- } else if (UseCompressedClassPointers) {
+ } else {
z_l(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
z_c(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes()));
- } else {
- z_lg(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
- z_cg(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes()));
}
BLOCK_COMMENT("} cmp_klasses_from_objects");
}
@@ -4184,36 +4163,28 @@ void MacroAssembler::load_klass(Register klass, Register src_oop) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(klass, src_oop);
decode_klass_not_null(klass);
- } else if (UseCompressedClassPointers) {
+ } else {
z_llgf(klass, oopDesc::klass_offset_in_bytes(), src_oop);
decode_klass_not_null(klass);
- } else {
- z_lg(klass, oopDesc::klass_offset_in_bytes(), src_oop);
}
}
void MacroAssembler::store_klass(Register klass, Register dst_oop, Register ck) {
assert(!UseCompactObjectHeaders, "Don't use with compact headers");
- if (UseCompressedClassPointers) {
- assert_different_registers(dst_oop, klass, Z_R0);
- if (ck == noreg) ck = klass;
- encode_klass_not_null(ck, klass);
- z_st(ck, Address(dst_oop, oopDesc::klass_offset_in_bytes()));
- } else {
- z_stg(klass, Address(dst_oop, oopDesc::klass_offset_in_bytes()));
- }
+ assert_different_registers(dst_oop, klass, Z_R0);
+ if (ck == noreg) ck = klass;
+ encode_klass_not_null(ck, klass);
+ z_st(ck, Address(dst_oop, oopDesc::klass_offset_in_bytes()));
}
void MacroAssembler::store_klass_gap(Register s, Register d) {
assert(!UseCompactObjectHeaders, "Don't use with compact headers");
- if (UseCompressedClassPointers) {
- assert(s != d, "not enough registers");
- // Support s = noreg.
- if (s != noreg) {
- z_st(s, Address(d, oopDesc::klass_gap_offset_in_bytes()));
- } else {
- z_mvhi(Address(d, oopDesc::klass_gap_offset_in_bytes()), 0);
- }
+ assert(s != d, "not enough registers");
+ // Support s = noreg.
+ if (s != noreg) {
+ z_st(s, Address(d, oopDesc::klass_gap_offset_in_bytes()));
+ } else {
+ z_mvhi(Address(d, oopDesc::klass_gap_offset_in_bytes()), 0);
}
}
@@ -4227,67 +4198,64 @@ void MacroAssembler::compare_klass_ptr(Register Rop1, int64_t disp, Register Rba
BLOCK_COMMENT("compare klass ptr {");
- if (UseCompressedClassPointers) {
- const int shift = CompressedKlassPointers::shift();
- address base = CompressedKlassPointers::base();
+ const int shift = CompressedKlassPointers::shift();
+ address base = CompressedKlassPointers::base();
- if (UseCompactObjectHeaders) {
- assert(shift >= 3, "cKlass encoder detected bad shift");
- } else {
- assert((shift == 0) || (shift == 3), "cKlass encoder detected bad shift");
- }
- assert_different_registers(Rop1, Z_R0);
- assert_different_registers(Rop1, Rbase, Z_R1);
-
- // First encode register oop and then compare with cOop in memory.
- // This sequence saves an unnecessary cOop load and decode.
- if (base == nullptr) {
- if (shift == 0) {
- z_cl(Rop1, disp, Rbase); // Unscaled
- } else {
- z_srlg(Z_R0, Rop1, shift); // ZeroBased
- z_cl(Z_R0, disp, Rbase);
- }
- } else { // HeapBased
-#ifdef ASSERT
- bool used_R0 = true;
- bool used_R1 = true;
-#endif
- Register current = Rop1;
- Label done;
-
- if (maybenull) { // null pointer must be preserved!
- z_ltgr(Z_R0, current);
- z_bre(done);
- current = Z_R0;
- }
-
- unsigned int base_h = ((unsigned long)base)>>32;
- unsigned int base_l = (unsigned int)((unsigned long)base);
- if ((base_h != 0) && (base_l == 0) && VM_Version::has_HighWordInstr()) {
- lgr_if_needed(Z_R0, current);
- z_aih(Z_R0, -((int)base_h)); // Base has no set bits in lower half.
- } else if ((base_h == 0) && (base_l != 0)) {
- lgr_if_needed(Z_R0, current);
- z_agfi(Z_R0, -(int)base_l);
- } else {
- int pow2_offset = get_oop_base_complement(Z_R1, ((uint64_t)(intptr_t)base));
- add2reg_with_index(Z_R0, pow2_offset, Z_R1, Rop1); // Subtract base by adding complement.
- }
-
- if (shift != 0) {
- z_srlg(Z_R0, Z_R0, shift);
- }
- bind(done);
- z_cl(Z_R0, disp, Rbase);
-#ifdef ASSERT
- if (used_R0) preset_reg(Z_R0, 0xb05bUL, 2);
- if (used_R1) preset_reg(Z_R1, 0xb06bUL, 2);
-#endif
- }
+ if (UseCompactObjectHeaders) {
+ assert(shift >= 3, "cKlass encoder detected bad shift");
} else {
- z_clg(Rop1, disp, Z_R0, Rbase);
+ assert((shift == 0) || (shift == 3), "cKlass encoder detected bad shift");
}
+ assert_different_registers(Rop1, Z_R0);
+ assert_different_registers(Rop1, Rbase, Z_R1);
+
+ // First encode register oop and then compare with cOop in memory.
+ // This sequence saves an unnecessary cOop load and decode.
+ if (base == nullptr) {
+ if (shift == 0) {
+ z_cl(Rop1, disp, Rbase); // Unscaled
+ } else {
+ z_srlg(Z_R0, Rop1, shift); // ZeroBased
+ z_cl(Z_R0, disp, Rbase);
+ }
+ } else { // HeapBased
+#ifdef ASSERT
+ bool used_R0 = true;
+ bool used_R1 = true;
+#endif
+ Register current = Rop1;
+ Label done;
+
+ if (maybenull) { // null pointer must be preserved!
+ z_ltgr(Z_R0, current);
+ z_bre(done);
+ current = Z_R0;
+ }
+
+ unsigned int base_h = ((unsigned long)base)>>32;
+ unsigned int base_l = (unsigned int)((unsigned long)base);
+ if ((base_h != 0) && (base_l == 0) && VM_Version::has_HighWordInstr()) {
+ lgr_if_needed(Z_R0, current);
+ z_aih(Z_R0, -((int)base_h)); // Base has no set bits in lower half.
+ } else if ((base_h == 0) && (base_l != 0)) {
+ lgr_if_needed(Z_R0, current);
+ z_agfi(Z_R0, -(int)base_l);
+ } else {
+ int pow2_offset = get_oop_base_complement(Z_R1, ((uint64_t)(intptr_t)base));
+ add2reg_with_index(Z_R0, pow2_offset, Z_R1, Rop1); // Subtract base by adding complement.
+ }
+
+ if (shift != 0) {
+ z_srlg(Z_R0, Z_R0, shift);
+ }
+ bind(done);
+ z_cl(Z_R0, disp, Rbase);
+#ifdef ASSERT
+ if (used_R0) preset_reg(Z_R0, 0xb05bUL, 2);
+ if (used_R1) preset_reg(Z_R1, 0xb06bUL, 2);
+#endif
+ }
+
BLOCK_COMMENT("} compare klass ptr");
}
@@ -6413,7 +6381,7 @@ void MacroAssembler::compiler_fast_lock_object(Register obj, Register box, Regis
// Check if object matches.
z_lg(tmp2, Address(tmp1_monitor, ObjectMonitor::object_offset()));
BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
- bs_asm->try_resolve_weak_handle_in_c2(this, tmp2, Z_R0_scratch, slow_path);
+ bs_asm->try_resolve_weak_handle(this, tmp2, Z_R0_scratch, slow_path);
z_cgr(obj, tmp2);
z_brne(slow_path);
diff --git a/src/hotspot/cpu/s390/macroAssembler_s390.hpp b/src/hotspot/cpu/s390/macroAssembler_s390.hpp
index da24ae80d45..32e484d4790 100644
--- a/src/hotspot/cpu/s390/macroAssembler_s390.hpp
+++ b/src/hotspot/cpu/s390/macroAssembler_s390.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2024 SAP SE. All rights reserved.
* Copyright (c) 2024 IBM Corporation. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -842,8 +842,7 @@ class MacroAssembler: public Assembler {
void store_klass(Register klass, Register dst_oop, Register ck = noreg); // Klass will get compressed if ck not provided.
void store_klass_gap(Register s, Register dst_oop);
void load_narrow_klass_compact(Register dst, Register src);
- // Compares the Klass pointer of an object to a given Klass (which might be narrow,
- // depending on UseCompressedClassPointers).
+ // Compares the narrow Klass pointer of an object to a given narrow Klass
void cmp_klass(Register klass, Register obj, Register tmp);
// Compares the Klass pointer of two objects obj1 and obj2. Result is in the condition flags.
// Uses tmp1 and tmp2 as temporary registers.
diff --git a/src/hotspot/cpu/s390/matcher_s390.hpp b/src/hotspot/cpu/s390/matcher_s390.hpp
index 99461e33e3c..b04a6566d41 100644
--- a/src/hotspot/cpu/s390/matcher_s390.hpp
+++ b/src/hotspot/cpu/s390/matcher_s390.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -82,7 +82,6 @@
static bool narrow_klass_use_complex_address() {
NOT_LP64(ShouldNotCallThis());
- assert(UseCompressedClassPointers, "only for compressed klass code");
// TODO HS25: z port if (MatchDecodeNodes) return true;
return false;
}
diff --git a/src/hotspot/cpu/s390/methodHandles_s390.cpp b/src/hotspot/cpu/s390/methodHandles_s390.cpp
index e3de6d911be..dfb8ce09b27 100644
--- a/src/hotspot/cpu/s390/methodHandles_s390.cpp
+++ b/src/hotspot/cpu/s390/methodHandles_s390.cpp
@@ -120,16 +120,12 @@ void MethodHandles::verify_ref_kind(MacroAssembler* _masm, int ref_kind,
__ z_nilf(temp, java_lang_invoke_MemberName::MN_REFERENCE_KIND_MASK);
__ compare32_and_branch(temp, constant(ref_kind), Assembler::bcondEqual, L);
- {
- char *buf = NEW_C_HEAP_ARRAY(char, 100, mtInternal);
-
- jio_snprintf(buf, 100, "verify_ref_kind expected %x", ref_kind);
- if (ref_kind == JVM_REF_invokeVirtual || ref_kind == JVM_REF_invokeSpecial) {
- // Could do this for all ref_kinds, but would explode assembly code size.
- trace_method_handle(_masm, buf);
- }
- __ stop(buf);
+ const char* msg = ref_kind_to_verify_msg(ref_kind);
+ if (ref_kind == JVM_REF_invokeVirtual || ref_kind == JVM_REF_invokeSpecial) {
+ // Could do this for all ref_kinds, but would explode assembly code size.
+ trace_method_handle(_masm, msg);
}
+ __ stop(msg);
BLOCK_COMMENT("} verify_ref_kind");
diff --git a/src/hotspot/cpu/s390/s390.ad b/src/hotspot/cpu/s390/s390.ad
index 19bd3620228..2208a197ac9 100644
--- a/src/hotspot/cpu/s390/s390.ad
+++ b/src/hotspot/cpu/s390/s390.ad
@@ -1,5 +1,5 @@
//
-// Copyright (c) 2017, 2025, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2017, 2026, Oracle and/or its affiliates. All rights reserved.
// Copyright (c) 2017, 2024 SAP SE. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
@@ -1890,10 +1890,8 @@ const int z_num_iarg_registers = sizeof(z_iarg_reg) / sizeof(z_iarg_reg[0]);
const int z_num_farg_registers = sizeof(z_farg_reg) / sizeof(z_farg_reg[0]);
-// Return whether or not this register is ever used as an argument. This
-// function is used on startup to build the trampoline stubs in generateOptoStub.
-// Registers not mentioned will be killed by the VM call in the trampoline, and
-// arguments in those registers not be available to the callee.
+#ifdef ASSERT
+// Return whether or not this register is ever used as an argument.
bool Matcher::can_be_java_arg(int reg) {
// We return true for all registers contained in z_iarg_reg[] and
// z_farg_reg[] and their virtual halves.
@@ -1917,10 +1915,7 @@ bool Matcher::can_be_java_arg(int reg) {
return false;
}
-
-bool Matcher::is_spillable_arg(int reg) {
- return can_be_java_arg(reg);
-}
+#endif
uint Matcher::int_pressure_limit()
{
@@ -1934,10 +1929,6 @@ uint Matcher::float_pressure_limit()
return (FLOATPRESSURE == -1) ? 15 : FLOATPRESSURE;
}
-bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
- return false;
-}
-
// Register for DIVI projection of divmodI
const RegMask& Matcher::divI_proj_mask() {
return _Z_RARG4_INT_REG_mask;
@@ -2606,13 +2597,6 @@ frame %{
// z/Architecture stack pointer
frame_pointer(Z_R15); // Z_SP
- // Interpreter stores its frame pointer in a register which is
- // stored to the stack by I2CAdaptors. I2CAdaptors convert from
- // interpreted java to compiled java.
- //
- // Z_state holds pointer to caller's cInterpreter.
- interpreter_frame_pointer(Z_R7); // Z_state
-
// Use alignment_in_bytes instead of log_2_of_alignment_in_bits.
stack_alignment(frame::alignment_in_bytes);
@@ -5251,6 +5235,15 @@ instruct membar_release_lock() %{
ins_pipe(pipe_class_dummy);
%}
+instruct membar_storeload() %{
+ match(MemBarStoreLoad);
+ ins_cost(4 * MEMORY_REF_COST);
+ size(2);
+ format %{ "MEMBAR-storeload" %}
+ ins_encode %{ __ z_fence(); %}
+ ins_pipe(pipe_class_dummy);
+%}
+
instruct membar_volatile() %{
match(MemBarVolatile);
ins_cost(4 * MEMORY_REF_COST);
@@ -5270,6 +5263,15 @@ instruct unnecessary_membar_volatile() %{
ins_pipe(pipe_class_dummy);
%}
+instruct membar_full() %{
+ match(MemBarFull);
+ ins_cost(4 * MEMORY_REF_COST);
+ size(2);
+ format %{ "MEMBAR-full" %}
+ ins_encode %{ __ z_fence(); %}
+ ins_pipe(pipe_class_dummy);
+%}
+
instruct membar_CPUOrder() %{
match(MemBarCPUOrder);
ins_cost(0);
diff --git a/src/hotspot/cpu/s390/stubDeclarations_s390.hpp b/src/hotspot/cpu/s390/stubDeclarations_s390.hpp
index c3ad3cefeb9..d0e26beedab 100644
--- a/src/hotspot/cpu/s390/stubDeclarations_s390.hpp
+++ b/src/hotspot/cpu/s390/stubDeclarations_s390.hpp
@@ -29,28 +29,32 @@
#define STUBGEN_PREUNIVERSE_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(preuniverse, 0) \
#define STUBGEN_INITIAL_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(initial, 20000) \
#define STUBGEN_CONTINUATION_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(continuation, 2000) \
#define STUBGEN_COMPILER_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(compiler, 20000 ) \
do_stub(compiler, partial_subtype_check) \
do_arch_entry(zarch, compiler, partial_subtype_check, \
@@ -60,7 +64,8 @@
#define STUBGEN_FINAL_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(final, 20000) \
diff --git a/src/hotspot/cpu/s390/stubGenerator_s390.cpp b/src/hotspot/cpu/s390/stubGenerator_s390.cpp
index 2aa365be999..3f16312eb48 100644
--- a/src/hotspot/cpu/s390/stubGenerator_s390.cpp
+++ b/src/hotspot/cpu/s390/stubGenerator_s390.cpp
@@ -3422,7 +3422,7 @@ class StubGenerator: public StubCodeGenerator {
}
public:
- StubGenerator(CodeBuffer* code, BlobId blob_id) : StubCodeGenerator(code, blob_id) {
+ StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) : StubCodeGenerator(code, blob_id, stub_data) {
switch(blob_id) {
case BlobId::stubgen_preuniverse_id:
generate_preuniverse_stubs();
@@ -3479,6 +3479,6 @@ class StubGenerator: public StubCodeGenerator {
};
-void StubGenerator_generate(CodeBuffer* code, BlobId blob_id) {
- StubGenerator g(code, blob_id);
+void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) {
+ StubGenerator g(code, blob_id, stub_data);
}
diff --git a/src/hotspot/cpu/s390/stubRoutines_s390.cpp b/src/hotspot/cpu/s390/stubRoutines_s390.cpp
index 6feb20f9604..eda0ebfdecc 100644
--- a/src/hotspot/cpu/s390/stubRoutines_s390.cpp
+++ b/src/hotspot/cpu/s390/stubRoutines_s390.cpp
@@ -40,8 +40,12 @@
#define DEFINE_ARCH_ENTRY_INIT(arch, blob_name, stub_name, field_name, getter_name, init_function) \
address StubRoutines:: arch :: STUB_FIELD_NAME(field_name) = CAST_FROM_FN_PTR(address, init_function);
-STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY, DEFINE_ARCH_ENTRY_INIT)
+#define DEFINE_ARCH_ENTRY_ARRAY(arch, blob_name, stub_name, field_name, getter_name, count) \
+ address StubRoutines:: arch :: STUB_FIELD_NAME(field_name) [idx] ;
+STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY, DEFINE_ARCH_ENTRY_INIT, DEFINE_ARCH_ENTRY_ARRAY)
+
+#undef DEFINE_ARCH_ENTRY_ARRAY
#undef DEFINE_ARCH_ENTRY_INIT
#undef DEFINE_ARCH_ENTRY
@@ -736,3 +740,9 @@ juint StubRoutines::zarch::_crc32c_table[CRC32_TABLES][CRC32_COLUMN_SIZE] = {
}
#endif
};
+
+#if INCLUDE_CDS
+// nothing to do for s390
+void StubRoutines::init_AOTAddressTable() {
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/s390/stubRoutines_s390.hpp b/src/hotspot/cpu/s390/stubRoutines_s390.hpp
index 0a07efae46c..e575115b731 100644
--- a/src/hotspot/cpu/s390/stubRoutines_s390.hpp
+++ b/src/hotspot/cpu/s390/stubRoutines_s390.hpp
@@ -81,9 +81,13 @@ class zarch {
#define DECLARE_ARCH_ENTRY_INIT(arch, blob_name, stub_name, field_name, getter_name, init_function) \
DECLARE_ARCH_ENTRY(arch, blob_name, stub_name, field_name, getter_name)
-private:
- STUBGEN_ARCH_ENTRIES_DO(DECLARE_ARCH_ENTRY, DECLARE_ARCH_ENTRY_INIT)
+#define DECLARE_ARCH_ENTRY_ARRAY(arch, blob_name, stub_name, field_name, getter_name, count) \
+ static address STUB_FIELD_NAME(field_name) [count] ;
+private:
+ STUBGEN_ARCH_ENTRIES_DO(DECLARE_ARCH_ENTRY, DECLARE_ARCH_ENTRY_INIT, DECLARE_ARCH_ENTRY_ARRAY)
+
+#undef DECLARE_ARCH_ENTRY_ARRAY
#undef DECLARE_ARCH_ENTRY_INIT
#undef DECLARE_ARCH_ENTRY
@@ -108,8 +112,12 @@ private:
#define DEFINE_ARCH_ENTRY_GETTER_INIT(arch, blob_name, stub_name, field_name, getter_name, init_function) \
DEFINE_ARCH_ENTRY_GETTER(arch, blob_name, stub_name, field_name, getter_name)
- STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY_GETTER, DEFINE_ARCH_ENTRY_GETTER_INIT)
+#define DEFINE_ARCH_ENTRY_GETTER_ARRAY(arch, blob_name, stub_name, field_name, getter_name, count) \
+ static address getter_name(int idx) { return STUB_FIELD_NAME(field_name) [idx] ; }
+ STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY_GETTER, DEFINE_ARCH_ENTRY_GETTER_INIT, DEFINE_ARCH_ENTRY_GETTER_ARRAY)
+
+#undef DEFINE_ARCH_ENTRY_GETTER_ARRAY
#undef DEFINE_ARCH_ENTRY_GETTER_INIT
#undef DEFINE_ARCH_ENTRY_GETTER
diff --git a/src/hotspot/cpu/s390/vm_version_s390.cpp b/src/hotspot/cpu/s390/vm_version_s390.cpp
index 7f5b4870aab..7e9000991ca 100644
--- a/src/hotspot/cpu/s390/vm_version_s390.cpp
+++ b/src/hotspot/cpu/s390/vm_version_s390.cpp
@@ -289,10 +289,6 @@ void VM_Version::initialize() {
FLAG_SET_DEFAULT(UseSHA3Intrinsics, false);
}
- if (!(UseSHA1Intrinsics || UseSHA256Intrinsics || UseSHA512Intrinsics)) {
- FLAG_SET_DEFAULT(UseSHA, false);
- }
-
if (UseSecondarySupersTable && VM_Version::get_model_index() < 5 /* z196/z11 */) {
if (!FLAG_IS_DEFAULT(UseSecondarySupersTable)) {
warning("UseSecondarySupersTable requires z196 or later.");
diff --git a/src/hotspot/cpu/x86/assembler_x86.cpp b/src/hotspot/cpu/x86/assembler_x86.cpp
index 3c8defe62d9..a4f2968f0d1 100644
--- a/src/hotspot/cpu/x86/assembler_x86.cpp
+++ b/src/hotspot/cpu/x86/assembler_x86.cpp
@@ -3472,7 +3472,7 @@ void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
emit_int16(0x6F, (0xC0 | encode));
}
-void Assembler::vmovw(XMMRegister dst, Register src) {
+void Assembler::evmovw(XMMRegister dst, Register src) {
assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
attributes.set_is_evex_instruction();
@@ -3480,7 +3480,7 @@ void Assembler::vmovw(XMMRegister dst, Register src) {
emit_int16(0x6E, (0xC0 | encode));
}
-void Assembler::vmovw(Register dst, XMMRegister src) {
+void Assembler::evmovw(Register dst, XMMRegister src) {
assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
attributes.set_is_evex_instruction();
@@ -3488,6 +3488,36 @@ void Assembler::vmovw(Register dst, XMMRegister src) {
emit_int16(0x7E, (0xC0 | encode));
}
+void Assembler::evmovw(XMMRegister dst, Address src) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
+ attributes.set_is_evex_instruction();
+ vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
+ emit_int8(0x6E);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::evmovw(Address dst, XMMRegister src) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
+ attributes.set_is_evex_instruction();
+ vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
+ emit_int8(0x7E);
+ emit_operand(src, dst, 0);
+}
+
+void Assembler::evmovw(XMMRegister dst, XMMRegister src) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ attributes.set_is_evex_instruction();
+ int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
+ emit_int16(0x6E, (0xC0 | encode));
+}
+
void Assembler::vmovdqu(XMMRegister dst, Address src) {
assert(UseAVX > 0, "");
InstructionMark im(this);
@@ -5442,6 +5472,13 @@ void Assembler::pmovsxwd(XMMRegister dst, XMMRegister src) {
emit_int16(0x23, (0xC0 | encode));
}
+void Assembler::pmovzxwd(XMMRegister dst, XMMRegister src) {
+ assert(VM_Version::supports_sse4_1(), "");
+ InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
+ emit_int16(0x33, (0xC0 | encode));
+}
+
void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
assert(VM_Version::supports_avx(), "");
InstructionMark im(this);
@@ -7303,6 +7340,42 @@ void Assembler::etzcntq(Register dst, Address src, bool no_flags) {
emit_operand(dst, src, 0);
}
+void Assembler::evucomish(XMMRegister dst, Address src) {
+ InstructionMark im(this);
+ InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
+ attributes.set_is_evex_instruction();
+ vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes);
+ emit_int8(0x2E);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::evucomish(XMMRegister dst, XMMRegister src) {
+ InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ attributes.set_is_evex_instruction();
+ int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes);
+ emit_int16(0x2E, (0xC0 | encode));
+}
+
+void Assembler::evucomxsh(XMMRegister dst, Address src) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
+ attributes.set_is_evex_instruction();
+ vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
+ emit_int8(0x2E);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::evucomxsh(XMMRegister dst, XMMRegister src) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ attributes.set_is_evex_instruction();
+ int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
+ emit_int16(0x2E, (0xC0 | encode));
+}
+
void Assembler::ucomisd(XMMRegister dst, Address src) {
InstructionMark im(this);
InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
@@ -7320,7 +7393,7 @@ void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
emit_int16(0x2E, (0xC0 | encode));
}
-void Assembler::vucomxsd(XMMRegister dst, Address src) {
+void Assembler::evucomxsd(XMMRegister dst, Address src) {
assert(VM_Version::supports_avx10_2(), "");
InstructionMark im(this);
InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
@@ -7331,7 +7404,7 @@ void Assembler::vucomxsd(XMMRegister dst, Address src) {
emit_operand(dst, src, 0);
}
-void Assembler::vucomxsd(XMMRegister dst, XMMRegister src) {
+void Assembler::evucomxsd(XMMRegister dst, XMMRegister src) {
assert(VM_Version::supports_avx10_2(), "");
InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
attributes.set_is_evex_instruction();
@@ -7354,7 +7427,7 @@ void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
emit_int16(0x2E, (0xC0 | encode));
}
-void Assembler::vucomxss(XMMRegister dst, Address src) {
+void Assembler::evucomxss(XMMRegister dst, Address src) {
assert(VM_Version::supports_avx10_2(), "");
InstructionMark im(this);
InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
@@ -7365,7 +7438,7 @@ void Assembler::vucomxss(XMMRegister dst, Address src) {
emit_operand(dst, src, 0);
}
-void Assembler::vucomxss(XMMRegister dst, XMMRegister src) {
+void Assembler::evucomxss(XMMRegister dst, XMMRegister src) {
assert(VM_Version::supports_avx10_2(), "");
InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
attributes.set_is_evex_instruction();
@@ -8404,30 +8477,6 @@ void Assembler::vmulsh(XMMRegister dst, XMMRegister nds, XMMRegister src) {
emit_int16(0x59, (0xC0 | encode));
}
-void Assembler::vmaxsh(XMMRegister dst, XMMRegister nds, XMMRegister src) {
- assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
- InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- attributes.set_is_evex_instruction();
- int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
- emit_int16(0x5F, (0xC0 | encode));
-}
-
-void Assembler::eminmaxsh(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
- assert(VM_Version::supports_avx10_2(), "");
- InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- attributes.set_is_evex_instruction();
- int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, &attributes);
- emit_int24(0x53, (0xC0 | encode), imm8);
-}
-
-void Assembler::vminsh(XMMRegister dst, XMMRegister nds, XMMRegister src) {
- assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
- InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- attributes.set_is_evex_instruction();
- int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
- emit_int16(0x5D, (0xC0 | encode));
-}
-
void Assembler::vsqrtsh(XMMRegister dst, XMMRegister src) {
assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
@@ -13362,48 +13411,38 @@ bool Assembler::is_demotable(bool no_flags, int dst_enc, int nds_enc) {
return (!no_flags && dst_enc == nds_enc);
}
-void Assembler::vmaxss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
- assert(VM_Version::supports_avx(), "");
- InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
- emit_int16(0x5F, (0xC0 | encode));
-}
-
-void Assembler::vmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
- assert(VM_Version::supports_avx(), "");
- InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- attributes.set_rex_vex_w_reverted();
- int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
- emit_int16(0x5F, (0xC0 | encode));
-}
-
-void Assembler::vminss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
- assert(VM_Version::supports_avx(), "");
- InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
- emit_int16(0x5D, (0xC0 | encode));
-}
-
-void Assembler::eminmaxss(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
+void Assembler::evminmaxsh(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int imm8) {
assert(VM_Version::supports_avx10_2(), "");
- InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, &attributes);
+ emit_int24(0x53, (0xC0 | encode), imm8);
+}
+
+void Assembler::evminmaxss(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int imm8) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
+ attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
emit_int24(0x53, (0xC0 | encode), imm8);
}
-void Assembler::vminsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
- assert(VM_Version::supports_avx(), "");
- InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- attributes.set_rex_vex_w_reverted();
- int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
- emit_int16(0x5D, (0xC0 | encode));
-}
-
-void Assembler::eminmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
+void Assembler::evminmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int imm8) {
assert(VM_Version::supports_avx10_2(), "");
- InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
emit_int24(0x53, (0xC0 | encode), imm8);
}
diff --git a/src/hotspot/cpu/x86/assembler_x86.hpp b/src/hotspot/cpu/x86/assembler_x86.hpp
index 97854f712cf..98684752b0c 100644
--- a/src/hotspot/cpu/x86/assembler_x86.hpp
+++ b/src/hotspot/cpu/x86/assembler_x86.hpp
@@ -1694,8 +1694,11 @@ private:
void movsbl(Register dst, Address src);
void movsbl(Register dst, Register src);
- void vmovw(XMMRegister dst, Register src);
- void vmovw(Register dst, XMMRegister src);
+ void evmovw(XMMRegister dst, Register src);
+ void evmovw(Register dst, XMMRegister src);
+ void evmovw(XMMRegister dst, Address src);
+ void evmovw(Address dst, XMMRegister src);
+ void evmovw(XMMRegister dst, XMMRegister src);
void movsbq(Register dst, Address src);
void movsbq(Register dst, Register src);
@@ -1965,6 +1968,7 @@ private:
void pmovsxbq(XMMRegister dst, XMMRegister src);
void pmovsxbw(XMMRegister dst, XMMRegister src);
void pmovsxwd(XMMRegister dst, XMMRegister src);
+ void pmovzxwd(XMMRegister dst, XMMRegister src);
void vpmovsxbd(XMMRegister dst, XMMRegister src, int vector_len);
void vpmovsxbq(XMMRegister dst, XMMRegister src, int vector_len);
void vpmovsxbw(XMMRegister dst, XMMRegister src, int vector_len);
@@ -2328,17 +2332,23 @@ private:
void tzcntq(Register dst, Address src);
void etzcntq(Register dst, Address src, bool no_flags);
+ // Unordered Compare Scalar Half-Precision Floating-Point Values and set EFLAGS
+ void evucomish(XMMRegister dst, Address src);
+ void evucomish(XMMRegister dst, XMMRegister src);
+ void evucomxsh(XMMRegister dst, Address src);
+ void evucomxsh(XMMRegister dst, XMMRegister src);
+
// Unordered Compare Scalar Double-Precision Floating-Point Values and set EFLAGS
void ucomisd(XMMRegister dst, Address src);
void ucomisd(XMMRegister dst, XMMRegister src);
- void vucomxsd(XMMRegister dst, Address src);
- void vucomxsd(XMMRegister dst, XMMRegister src);
+ void evucomxsd(XMMRegister dst, Address src);
+ void evucomxsd(XMMRegister dst, XMMRegister src);
// Unordered Compare Scalar Single-Precision Floating-Point Values and set EFLAGS
void ucomiss(XMMRegister dst, Address src);
void ucomiss(XMMRegister dst, XMMRegister src);
- void vucomxss(XMMRegister dst, Address src);
- void vucomxss(XMMRegister dst, XMMRegister src);
+ void evucomxss(XMMRegister dst, Address src);
+ void evucomxss(XMMRegister dst, XMMRegister src);
void xabort(int8_t imm8);
@@ -2416,11 +2426,6 @@ private:
void vsubss(XMMRegister dst, XMMRegister nds, Address src);
void vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src);
- void vmaxss(XMMRegister dst, XMMRegister nds, XMMRegister src);
- void vmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src);
- void vminss(XMMRegister dst, XMMRegister nds, XMMRegister src);
- void vminsd(XMMRegister dst, XMMRegister nds, XMMRegister src);
-
void sarxl(Register dst, Register src1, Register src2);
void sarxl(Register dst, Address src1, Register src2);
void sarxq(Register dst, Register src1, Register src2);
@@ -2551,8 +2556,6 @@ private:
void vsubsh(XMMRegister dst, XMMRegister nds, XMMRegister src);
void vmulsh(XMMRegister dst, XMMRegister nds, XMMRegister src);
void vdivsh(XMMRegister dst, XMMRegister nds, XMMRegister src);
- void vmaxsh(XMMRegister dst, XMMRegister nds, XMMRegister src);
- void vminsh(XMMRegister dst, XMMRegister nds, XMMRegister src);
void vsqrtsh(XMMRegister dst, XMMRegister src);
void vfmadd132sh(XMMRegister dst, XMMRegister src1, XMMRegister src2);
@@ -2789,9 +2792,9 @@ private:
void vminpd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len);
// AVX10.2 floating point minmax instructions
- void eminmaxsh(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8);
- void eminmaxss(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8);
- void eminmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8);
+ void evminmaxsh(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int imm8);
+ void evminmaxss(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int imm8);
+ void evminmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int imm8);
void evminmaxph(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int imm8, int vector_len);
void evminmaxph(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int imm8, int vector_len);
void evminmaxps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int imm8, int vector_len);
diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp
index d9be0fdcc8d..5c05b3702bb 100644
--- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -42,6 +42,7 @@
#include "runtime/safepointMechanism.hpp"
#include "runtime/sharedRuntime.hpp"
#include "runtime/stubRoutines.hpp"
+#include "runtime/threadIdentifier.hpp"
#include "utilities/powerOfTwo.hpp"
#include "vmreg_x86.inline.hpp"
@@ -70,6 +71,17 @@ static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2],
static jlong *float_signflip_pool = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000));
static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000));
+#if INCLUDE_CDS
+// publish external addresses defined in this file
+void LIR_Assembler::init_AOTAddressTable(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr));
+ ADD(float_signmask_pool);
+ ADD(double_signmask_pool);
+ ADD(float_signflip_pool);
+ ADD(double_signflip_pool);
+#undef ADD
+}
+#endif // INCLUDE_CDS
NEEDS_CLEANUP // remove this definitions ?
const Register SYNC_header = rax; // synchronization header
@@ -77,23 +89,6 @@ const Register SHIFT_count = rcx; // where count for shift operations must be
#define __ _masm->
-
-static void select_different_registers(Register preserve,
- Register extra,
- Register &tmp1,
- Register &tmp2) {
- if (tmp1 == preserve) {
- assert_different_registers(tmp1, tmp2, extra);
- tmp1 = extra;
- } else if (tmp2 == preserve) {
- assert_different_registers(tmp1, tmp2, extra);
- tmp2 = extra;
- }
- assert_different_registers(preserve, tmp1, tmp2);
-}
-
-
-
static void select_different_registers(Register preserve,
Register extra,
Register &tmp1,
@@ -535,10 +530,23 @@ void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_cod
}
case T_LONG: {
+#if INCLUDE_CDS
+ if (AOTCodeCache::is_on_for_dump()) {
+ address b = c->as_pointer();
+ if (b == (address)ThreadIdentifier::unsafe_offset()) {
+ __ lea(dest->as_register_lo(), ExternalAddress(b));
+ break;
+ }
+ }
+#endif
assert(patch_code == lir_patch_none, "no patching handled here");
#if INCLUDE_CDS
if (AOTCodeCache::is_on_for_dump()) {
address b = c->as_pointer();
+ if (b == (address)ThreadIdentifier::unsafe_offset()) {
+ __ lea(dest->as_register_lo(), ExternalAddress(b));
+ break;
+ }
if (AOTRuntimeConstants::contains(b)) {
__ load_aotrc_address(dest->as_register_lo(), b);
break;
@@ -1309,12 +1317,8 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, L
} else if (obj == klass_RInfo) {
klass_RInfo = dst;
}
- if (k->is_loaded() && !UseCompressedClassPointers) {
- select_different_registers(obj, dst, k_RInfo, klass_RInfo);
- } else {
- Rtmp1 = op->tmp3()->as_register();
- select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
- }
+ Rtmp1 = op->tmp3()->as_register();
+ select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
assert_different_registers(obj, k_RInfo, klass_RInfo);
@@ -1348,12 +1352,8 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, L
if (op->fast_check()) {
// get object class
// not a safepoint as obj null check happens earlier
- if (UseCompressedClassPointers) {
- __ load_klass(Rtmp1, obj, tmp_load_klass);
- __ cmpptr(k_RInfo, Rtmp1);
- } else {
- __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
- }
+ __ load_klass(Rtmp1, obj, tmp_load_klass);
+ __ cmpptr(k_RInfo, Rtmp1);
__ jcc(Assembler::notEqual, *failure_target);
// successful cast, fall through to profile or jump
} else {
@@ -2651,9 +2651,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
// but not necessarily exactly of type default_type.
Label known_ok, halt;
__ mov_metadata(tmp, default_type->constant_encoding());
- if (UseCompressedClassPointers) {
- __ encode_klass_not_null(tmp, rscratch1);
- }
+ __ encode_klass_not_null(tmp, rscratch1);
if (basic_type != T_OBJECT) {
__ cmp_klass(tmp, dst, tmp2);
diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.hpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.hpp
index c4a368b54d8..6f179255e4a 100644
--- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.hpp
@@ -58,4 +58,7 @@ public:
void store_parameter(jobject c, int offset_from_esp_in_words);
void store_parameter(Metadata* c, int offset_from_esp_in_words);
+#if INCLUDE_CDS
+ void static init_AOTAddressTable(GrowableArray& external_addresses);
+#endif // INCLUDE_CDS
#endif // CPU_X86_C1_LIRASSEMBLER_X86_HPP
diff --git a/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp b/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp
index 5459e8df229..f448e4ee17f 100644
--- a/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp
+++ b/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1291,9 +1291,7 @@ void LIRGenerator::do_CheckCast(CheckCast* x) {
}
LIR_Opr reg = rlock_result(x);
LIR_Opr tmp3 = LIR_OprFact::illegalOpr;
- if (!x->klass()->is_loaded() || UseCompressedClassPointers) {
- tmp3 = new_register(objectType);
- }
+ tmp3 = new_register(objectType);
__ checkcast(reg, obj.result(), x->klass(),
new_register(objectType), new_register(objectType), tmp3,
x->direct_compare(), info_for_exception, patching_info, stub,
@@ -1313,9 +1311,7 @@ void LIRGenerator::do_InstanceOf(InstanceOf* x) {
}
obj.load_item();
LIR_Opr tmp3 = LIR_OprFact::illegalOpr;
- if (!x->klass()->is_loaded() || UseCompressedClassPointers) {
- tmp3 = new_register(objectType);
- }
+ tmp3 = new_register(objectType);
__ instanceof(reg, obj.result(), x->klass(),
new_register(objectType), new_register(objectType), tmp3,
x->direct_compare(), patching_info, x->profiled_method(), x->profiled_bci());
diff --git a/src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp
index 88e2e6c8ba9..7adaea48ff1 100644
--- a/src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -85,14 +85,11 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
if (UseCompactObjectHeaders) {
movptr(t1, Address(klass, Klass::prototype_header_offset()));
movptr(Address(obj, oopDesc::mark_offset_in_bytes()), t1);
- } else if (UseCompressedClassPointers) { // Take care not to kill klass
+ } else { // Take care not to kill klass
movptr(Address(obj, oopDesc::mark_offset_in_bytes()), checked_cast(markWord::prototype().value()));
movptr(t1, klass);
encode_klass_not_null(t1, rscratch1);
movl(Address(obj, oopDesc::klass_offset_in_bytes()), t1);
- } else {
- movptr(Address(obj, oopDesc::mark_offset_in_bytes()), checked_cast(markWord::prototype().value()));
- movptr(Address(obj, oopDesc::klass_offset_in_bytes()), klass);
}
if (len->is_valid()) {
@@ -104,7 +101,7 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
xorl(t1, t1);
movl(Address(obj, base_offset), t1);
}
- } else if (UseCompressedClassPointers && !UseCompactObjectHeaders) {
+ } else if (!UseCompactObjectHeaders) {
xorptr(t1, t1);
store_klass_gap(obj, t1);
}
diff --git a/src/hotspot/cpu/x86/c1_globals_x86.hpp b/src/hotspot/cpu/x86/c1_globals_x86.hpp
index 978b233bb63..bb75a31a77c 100644
--- a/src/hotspot/cpu/x86/c1_globals_x86.hpp
+++ b/src/hotspot/cpu/x86/c1_globals_x86.hpp
@@ -41,7 +41,6 @@ define_pd_global(bool, TieredCompilation, false);
define_pd_global(intx, CompileThreshold, 1500 );
define_pd_global(intx, OnStackReplacePercentage, 933 );
-define_pd_global(size_t, NewSizeThreadIncrease, 4*K );
define_pd_global(size_t, InitialCodeCacheSize, 160*K);
define_pd_global(size_t, ReservedCodeCacheSize, 32*M );
define_pd_global(size_t, NonProfiledCodeHeapSize, 13*M );
@@ -51,7 +50,6 @@ define_pd_global(bool, ProfileInterpreter, false);
define_pd_global(size_t, CodeCacheExpansionSize, 32*K );
define_pd_global(size_t, CodeCacheMinBlockLength, 1 );
define_pd_global(size_t, CodeCacheMinimumUseSpace, 400*K);
-define_pd_global(bool, NeverActAsServerClassMachine, true );
define_pd_global(bool, CICompileOSR, true );
#endif // !COMPILER2
define_pd_global(bool, UseTypeProfile, false);
diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
index a3ccc081b6b..b4d8aa10de2 100644
--- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
@@ -152,7 +152,7 @@ inline Assembler::AvxVectorLen C2_MacroAssembler::vector_length_encoding(int vle
// Because the transitions from emitted code to the runtime
// monitorenter/exit helper stubs are so slow it's critical that
-// we inline both the stack-locking fast path and the inflated fast path.
+// we inline both the lock-stack fast path and the inflated fast path.
//
// See also: cmpFastLock and cmpFastUnlock.
//
@@ -1037,8 +1037,8 @@ void C2_MacroAssembler::evminmax_fp(int opcode, BasicType elem_bt,
}
}
-void C2_MacroAssembler::vminmax_fp(int opc, BasicType elem_bt, XMMRegister dst, KRegister mask,
- XMMRegister src1, XMMRegister src2, int vlen_enc) {
+void C2_MacroAssembler::vminmax_fp_avx10_2(int opc, BasicType elem_bt, XMMRegister dst, KRegister mask,
+ XMMRegister src1, XMMRegister src2, int vlen_enc) {
assert(opc == Op_MinV || opc == Op_MinReductionV ||
opc == Op_MaxV || opc == Op_MaxReductionV, "sanity");
@@ -1052,6 +1052,21 @@ void C2_MacroAssembler::vminmax_fp(int opc, BasicType elem_bt, XMMRegister dst,
}
}
+void C2_MacroAssembler::sminmax_fp_avx10_2(int opc, BasicType elem_bt, XMMRegister dst, KRegister mask,
+ XMMRegister src1, XMMRegister src2) {
+ assert(opc == Op_MinF || opc == Op_MaxF ||
+ opc == Op_MinD || opc == Op_MaxD, "sanity");
+
+ int imm8 = (opc == Op_MinF || opc == Op_MinD) ? AVX10_2_MINMAX_MIN_COMPARE_SIGN
+ : AVX10_2_MINMAX_MAX_COMPARE_SIGN;
+ if (elem_bt == T_FLOAT) {
+ evminmaxss(dst, mask, src1, src2, true, imm8);
+ } else {
+ assert(elem_bt == T_DOUBLE, "");
+ evminmaxsd(dst, mask, src1, src2, true, imm8);
+ }
+}
+
// Float/Double signum
void C2_MacroAssembler::signum_fp(int opcode, XMMRegister dst, XMMRegister zero, XMMRegister one) {
assert(opcode == Op_SignumF || opcode == Op_SignumD, "sanity");
@@ -1063,7 +1078,7 @@ void C2_MacroAssembler::signum_fp(int opcode, XMMRegister dst, XMMRegister zero,
// If other floating point comparison instructions used, ZF=1 for equal and unordered cases
if (opcode == Op_SignumF) {
if (VM_Version::supports_avx10_2()) {
- vucomxss(dst, zero);
+ evucomxss(dst, zero);
jcc(Assembler::negative, DONE_LABEL);
} else {
ucomiss(dst, zero);
@@ -1074,7 +1089,7 @@ void C2_MacroAssembler::signum_fp(int opcode, XMMRegister dst, XMMRegister zero,
xorps(dst, ExternalAddress(StubRoutines::x86::vector_float_sign_flip()), noreg);
} else if (opcode == Op_SignumD) {
if (VM_Version::supports_avx10_2()) {
- vucomxsd(dst, zero);
+ evucomxsd(dst, zero);
jcc(Assembler::negative, DONE_LABEL);
} else {
ucomisd(dst, zero);
@@ -1691,12 +1706,8 @@ void C2_MacroAssembler::load_constant_vector(BasicType bt, XMMRegister dst, Inte
}
void C2_MacroAssembler::load_iota_indices(XMMRegister dst, int vlen_in_bytes, BasicType bt) {
- // The iota indices are ordered by type B/S/I/L/F/D, and the offset between two types is 64.
- int offset = exact_log2(type2aelembytes(bt)) << 6;
- if (is_floating_point_type(bt)) {
- offset += 128;
- }
- ExternalAddress addr(StubRoutines::x86::vector_iota_indices() + offset);
+ int entry_idx = vector_iota_entry_index(bt);
+ ExternalAddress addr(StubRoutines::x86::vector_iota_indices(entry_idx));
load_vector(T_BYTE, dst, addr, vlen_in_bytes);
}
@@ -1729,6 +1740,24 @@ void C2_MacroAssembler::reduce_operation_128(BasicType typ, int opcode, XMMRegis
default: assert(false, "wrong type");
}
break;
+ case Op_UMinReductionV:
+ switch (typ) {
+ case T_BYTE: vpminub(dst, dst, src, Assembler::AVX_128bit); break;
+ case T_SHORT: vpminuw(dst, dst, src, Assembler::AVX_128bit); break;
+ case T_INT: vpminud(dst, dst, src, Assembler::AVX_128bit); break;
+ case T_LONG: evpminuq(dst, k0, dst, src, true, Assembler::AVX_128bit); break;
+ default: assert(false, "wrong type");
+ }
+ break;
+ case Op_UMaxReductionV:
+ switch (typ) {
+ case T_BYTE: vpmaxub(dst, dst, src, Assembler::AVX_128bit); break;
+ case T_SHORT: vpmaxuw(dst, dst, src, Assembler::AVX_128bit); break;
+ case T_INT: vpmaxud(dst, dst, src, Assembler::AVX_128bit); break;
+ case T_LONG: evpmaxuq(dst, k0, dst, src, true, Assembler::AVX_128bit); break;
+ default: assert(false, "wrong type");
+ }
+ break;
case Op_AddReductionVF: addss(dst, src); break;
case Op_AddReductionVD: addsd(dst, src); break;
case Op_AddReductionVI:
@@ -1792,6 +1821,24 @@ void C2_MacroAssembler::reduce_operation_256(BasicType typ, int opcode, XMMRegis
default: assert(false, "wrong type");
}
break;
+ case Op_UMinReductionV:
+ switch (typ) {
+ case T_BYTE: vpminub(dst, src1, src2, vector_len); break;
+ case T_SHORT: vpminuw(dst, src1, src2, vector_len); break;
+ case T_INT: vpminud(dst, src1, src2, vector_len); break;
+ case T_LONG: evpminuq(dst, k0, src1, src2, true, vector_len); break;
+ default: assert(false, "wrong type");
+ }
+ break;
+ case Op_UMaxReductionV:
+ switch (typ) {
+ case T_BYTE: vpmaxub(dst, src1, src2, vector_len); break;
+ case T_SHORT: vpmaxuw(dst, src1, src2, vector_len); break;
+ case T_INT: vpmaxud(dst, src1, src2, vector_len); break;
+ case T_LONG: evpmaxuq(dst, k0, src1, src2, true, vector_len); break;
+ default: assert(false, "wrong type");
+ }
+ break;
case Op_AddReductionVI:
switch (typ) {
case T_BYTE: vpaddb(dst, src1, src2, vector_len); break;
@@ -2058,7 +2105,11 @@ void C2_MacroAssembler::reduce8B(int opcode, Register dst, Register src1, XMMReg
psrldq(vtmp2, 1);
reduce_operation_128(T_BYTE, opcode, vtmp1, vtmp2);
movdl(vtmp2, src1);
- pmovsxbd(vtmp1, vtmp1);
+ if (opcode == Op_UMinReductionV || opcode == Op_UMaxReductionV) {
+ pmovzxbd(vtmp1, vtmp1);
+ } else {
+ pmovsxbd(vtmp1, vtmp1);
+ }
reduce_operation_128(T_INT, opcode, vtmp1, vtmp2);
pextrb(dst, vtmp1, 0x0);
movsbl(dst, dst);
@@ -2095,8 +2146,8 @@ void C2_MacroAssembler::mulreduce16B(int opcode, Register dst, Register src1, XM
} else {
pmovsxbw(vtmp2, src2);
reduce8S(opcode, dst, src1, vtmp2, vtmp1, vtmp2);
- pshufd(vtmp2, src2, 0x1);
- pmovsxbw(vtmp2, src2);
+ pshufd(vtmp2, src2, 0xe);
+ pmovsxbw(vtmp2, vtmp2);
reduce8S(opcode, dst, dst, vtmp2, vtmp1, vtmp2);
}
}
@@ -2105,7 +2156,7 @@ void C2_MacroAssembler::mulreduce32B(int opcode, Register dst, Register src1, XM
if (UseAVX > 2 && VM_Version::supports_avx512bw()) {
int vector_len = Assembler::AVX_512bit;
vpmovsxbw(vtmp1, src2, vector_len);
- reduce32S(opcode, dst, src1, vtmp1, vtmp1, vtmp2);
+ reduce32S(opcode, dst, src1, vtmp1, vtmp2, vtmp1);
} else {
assert(UseAVX >= 2,"Should not reach here.");
mulreduce16B(opcode, dst, src1, src2, vtmp1, vtmp2);
@@ -2135,7 +2186,11 @@ void C2_MacroAssembler::reduce4S(int opcode, Register dst, Register src1, XMMReg
reduce_operation_128(T_SHORT, opcode, vtmp1, vtmp2);
}
movdl(vtmp2, src1);
- pmovsxwd(vtmp1, vtmp1);
+ if (opcode == Op_UMinReductionV || opcode == Op_UMaxReductionV) {
+ pmovzxwd(vtmp1, vtmp1);
+ } else {
+ pmovsxwd(vtmp1, vtmp1);
+ }
reduce_operation_128(T_INT, opcode, vtmp1, vtmp2);
pextrw(dst, vtmp1, 0x0);
movswl(dst, dst);
@@ -2148,6 +2203,7 @@ void C2_MacroAssembler::reduce8S(int opcode, Register dst, Register src1, XMMReg
}
phaddw(vtmp1, src2);
} else {
+ assert_different_registers(src2, vtmp1);
pshufd(vtmp1, src2, 0xE);
reduce_operation_128(T_SHORT, opcode, vtmp1, src2);
}
@@ -2160,6 +2216,7 @@ void C2_MacroAssembler::reduce16S(int opcode, Register dst, Register src1, XMMRe
vphaddw(vtmp2, src2, src2, vector_len);
vpermq(vtmp2, vtmp2, 0xD8, vector_len);
} else {
+ assert_different_registers(src2, vtmp2);
vextracti128_high(vtmp2, src2);
reduce_operation_128(T_SHORT, opcode, vtmp2, src2);
}
@@ -2167,6 +2224,7 @@ void C2_MacroAssembler::reduce16S(int opcode, Register dst, Register src1, XMMRe
}
void C2_MacroAssembler::reduce32S(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2) {
+ assert_different_registers(src2, vtmp1);
int vector_len = Assembler::AVX_256bit;
vextracti64x4_high(vtmp1, src2);
reduce_operation_256(T_SHORT, opcode, vtmp1, vtmp1, src2);
@@ -2356,7 +2414,7 @@ void C2_MacroAssembler::reduceFloatMinMax(int opcode, int vlen, bool is_dst_vali
}
if (VM_Version::supports_avx10_2()) {
- vminmax_fp(opcode, T_FLOAT, wdst, k0, wtmp, wsrc, vlen_enc);
+ vminmax_fp_avx10_2(opcode, T_FLOAT, wdst, k0, wtmp, wsrc, vlen_enc);
} else {
vminmax_fp(opcode, T_FLOAT, wdst, wtmp, wsrc, tmp, atmp, btmp, vlen_enc);
}
@@ -2365,7 +2423,7 @@ void C2_MacroAssembler::reduceFloatMinMax(int opcode, int vlen, bool is_dst_vali
}
if (is_dst_valid) {
if (VM_Version::supports_avx10_2()) {
- vminmax_fp(opcode, T_FLOAT, dst, k0, wdst, dst, Assembler::AVX_128bit);
+ vminmax_fp_avx10_2(opcode, T_FLOAT, dst, k0, wdst, dst, Assembler::AVX_128bit);
} else {
vminmax_fp(opcode, T_FLOAT, dst, wdst, dst, tmp, atmp, btmp, Assembler::AVX_128bit);
}
@@ -2396,7 +2454,7 @@ void C2_MacroAssembler::reduceDoubleMinMax(int opcode, int vlen, bool is_dst_val
}
if (VM_Version::supports_avx10_2()) {
- vminmax_fp(opcode, T_DOUBLE, wdst, k0, wtmp, wsrc, vlen_enc);
+ vminmax_fp_avx10_2(opcode, T_DOUBLE, wdst, k0, wtmp, wsrc, vlen_enc);
} else {
vminmax_fp(opcode, T_DOUBLE, wdst, wtmp, wsrc, tmp, atmp, btmp, vlen_enc);
}
@@ -2407,7 +2465,7 @@ void C2_MacroAssembler::reduceDoubleMinMax(int opcode, int vlen, bool is_dst_val
if (is_dst_valid) {
if (VM_Version::supports_avx10_2()) {
- vminmax_fp(opcode, T_DOUBLE, dst, k0, wdst, dst, Assembler::AVX_128bit);
+ vminmax_fp_avx10_2(opcode, T_DOUBLE, dst, k0, wdst, dst, Assembler::AVX_128bit);
} else {
vminmax_fp(opcode, T_DOUBLE, dst, wdst, dst, tmp, atmp, btmp, Assembler::AVX_128bit);
}
@@ -7017,13 +7075,25 @@ void C2_MacroAssembler::evfp16ph(int opcode, XMMRegister dst, XMMRegister src1,
}
}
-void C2_MacroAssembler::scalar_max_min_fp16(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
- KRegister ktmp, XMMRegister xtmp1, XMMRegister xtmp2) {
- vector_max_min_fp16(opcode, dst, src1, src2, ktmp, xtmp1, xtmp2, Assembler::AVX_128bit);
+void C2_MacroAssembler::sminmax_fp16(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ KRegister ktmp, XMMRegister xtmp1, XMMRegister xtmp2) {
+ vminmax_fp16(opcode, dst, src1, src2, ktmp, xtmp1, xtmp2, Assembler::AVX_128bit);
}
-void C2_MacroAssembler::vector_max_min_fp16(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
- KRegister ktmp, XMMRegister xtmp1, XMMRegister xtmp2, int vlen_enc) {
+void C2_MacroAssembler::sminmax_fp16_avx10_2(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ KRegister ktmp) {
+ if (opcode == Op_MaxHF) {
+ // dst = max(src1, src2)
+ evminmaxsh(dst, ktmp, src1, src2, true, AVX10_2_MINMAX_MAX_COMPARE_SIGN);
+ } else {
+ assert(opcode == Op_MinHF, "");
+ // dst = min(src1, src2)
+ evminmaxsh(dst, ktmp, src1, src2, true, AVX10_2_MINMAX_MIN_COMPARE_SIGN);
+ }
+}
+
+void C2_MacroAssembler::vminmax_fp16(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ KRegister ktmp, XMMRegister xtmp1, XMMRegister xtmp2, int vlen_enc) {
if (opcode == Op_MaxVHF || opcode == Op_MaxHF) {
// Move sign bits of src2 to mask register.
evpmovw2m(ktmp, src2, vlen_enc);
@@ -7066,3 +7136,48 @@ void C2_MacroAssembler::vector_max_min_fp16(int opcode, XMMRegister dst, XMMRegi
Assembler::evmovdquw(dst, ktmp, xtmp1, true, vlen_enc);
}
}
+
+void C2_MacroAssembler::vminmax_fp16_avx10_2(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ KRegister ktmp, int vlen_enc) {
+ if (opcode == Op_MaxVHF) {
+ // dst = max(src1, src2)
+ evminmaxph(dst, ktmp, src1, src2, true, AVX10_2_MINMAX_MAX_COMPARE_SIGN, vlen_enc);
+ } else {
+ assert(opcode == Op_MinVHF, "");
+ // dst = min(src1, src2)
+ evminmaxph(dst, ktmp, src1, src2, true, AVX10_2_MINMAX_MIN_COMPARE_SIGN, vlen_enc);
+ }
+}
+
+void C2_MacroAssembler::vminmax_fp16_avx10_2(int opcode, XMMRegister dst, XMMRegister src1, Address src2,
+ KRegister ktmp, int vlen_enc) {
+ if (opcode == Op_MaxVHF) {
+ // dst = max(src1, src2)
+ evminmaxph(dst, ktmp, src1, src2, true, AVX10_2_MINMAX_MAX_COMPARE_SIGN, vlen_enc);
+ } else {
+ assert(opcode == Op_MinVHF, "");
+ // dst = min(src1, src2)
+ evminmaxph(dst, ktmp, src1, src2, true, AVX10_2_MINMAX_MIN_COMPARE_SIGN, vlen_enc);
+ }
+}
+
+int C2_MacroAssembler::vector_iota_entry_index(BasicType bt) {
+ // The vector iota entries array is ordered by type B/S/I/L/F/D, and
+ // the offset between two types is 16.
+ switch(bt) {
+ case T_BYTE:
+ return 0;
+ case T_SHORT:
+ return 1;
+ case T_INT:
+ return 2;
+ case T_LONG:
+ return 3;
+ case T_FLOAT:
+ return 4;
+ case T_DOUBLE:
+ return 5;
+ default:
+ ShouldNotReachHere();
+ }
+}
diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp
index 6d8b0ceaebe..9b229ad7221 100644
--- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -67,8 +67,11 @@ public:
XMMRegister tmp, XMMRegister atmp, XMMRegister btmp,
int vlen_enc);
- void vminmax_fp(int opc, BasicType elem_bt, XMMRegister dst, KRegister mask,
- XMMRegister src1, XMMRegister src2, int vlen_enc);
+ void vminmax_fp_avx10_2(int opc, BasicType elem_bt, XMMRegister dst, KRegister mask,
+ XMMRegister src1, XMMRegister src2, int vlen_enc);
+
+ void sminmax_fp_avx10_2(int opc, BasicType elem_bt, XMMRegister dst, KRegister mask,
+ XMMRegister src1, XMMRegister src2);
void vpuminmaxq(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister xtmp1, XMMRegister xtmp2, int vlen_enc);
@@ -576,12 +579,22 @@ public:
void evfp16ph(int opcode, XMMRegister dst, XMMRegister src1, Address src2, int vlen_enc);
- void vector_max_min_fp16(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
- KRegister ktmp, XMMRegister xtmp1, XMMRegister xtmp2, int vlen_enc);
+ void vminmax_fp16(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ KRegister ktmp, XMMRegister xtmp1, XMMRegister xtmp2, int vlen_enc);
- void scalar_max_min_fp16(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
- KRegister ktmp, XMMRegister xtmp1, XMMRegister xtmp2);
+ void vminmax_fp16_avx10_2(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ KRegister ktmp, int vlen_enc);
+
+ void vminmax_fp16_avx10_2(int opcode, XMMRegister dst, XMMRegister src1, Address src2,
+ KRegister ktmp, int vlen_enc);
+
+ void sminmax_fp16(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ KRegister ktmp, XMMRegister xtmp1, XMMRegister xtmp2);
+
+ void sminmax_fp16_avx10_2(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ KRegister ktmp);
void reconstruct_frame_pointer(Register rtmp);
+ int vector_iota_entry_index(BasicType bt);
#endif // CPU_X86_C2_MACROASSEMBLER_X86_HPP
diff --git a/src/hotspot/cpu/x86/c2_globals_x86.hpp b/src/hotspot/cpu/x86/c2_globals_x86.hpp
index 3f616cb4578..11d8c03d0ca 100644
--- a/src/hotspot/cpu/x86/c2_globals_x86.hpp
+++ b/src/hotspot/cpu/x86/c2_globals_x86.hpp
@@ -46,7 +46,6 @@ define_pd_global(intx, FreqInlineSize, 325);
define_pd_global(intx, MinJumpTableSize, 10);
define_pd_global(intx, LoopPercentProfileLimit, 10);
define_pd_global(intx, InteriorEntryAlignment, 16);
-define_pd_global(size_t, NewSizeThreadIncrease, ScaleForWordSize(4*K));
define_pd_global(intx, LoopUnrollLimit, 60);
// InitialCodeCacheSize derived from specjbb2000 run.
define_pd_global(size_t, InitialCodeCacheSize, 2496*K); // Integral multiple of CodeCacheExpansionSize
@@ -74,7 +73,4 @@ define_pd_global(size_t, CodeCacheMinimumUseSpace, 400*K);
define_pd_global(bool, TrapBasedRangeChecks, false); // Not needed on x86.
-// Ergonomics related flags
-define_pd_global(bool, NeverActAsServerClassMachine, false);
-
#endif // CPU_X86_C2_GLOBALS_X86_HPP
diff --git a/src/hotspot/cpu/x86/downcallLinker_x86_64.cpp b/src/hotspot/cpu/x86/downcallLinker_x86_64.cpp
index c48940198ea..e3bf5f17fe9 100644
--- a/src/hotspot/cpu/x86/downcallLinker_x86_64.cpp
+++ b/src/hotspot/cpu/x86/downcallLinker_x86_64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -145,10 +145,10 @@ void DowncallLinker::StubGenerator::generate() {
// when we don't use a return buffer we need to spill the return value around our slow path calls
bool should_save_return_value = !_needs_return_buffer;
RegSpiller out_reg_spiller(_output_registers);
- int spill_rsp_offset = -1;
+ int out_spill_rsp_offset = -1;
if (should_save_return_value) {
- spill_rsp_offset = 0;
+ out_spill_rsp_offset = 0;
// spill area can be shared with shadow space and out args,
// since they are only used before the call,
// and spill area is only used after.
@@ -173,6 +173,9 @@ void DowncallLinker::StubGenerator::generate() {
// FP-> | |
// |---------------------| = frame_bottom_offset = frame_size
// | (optional) |
+ // | in_reg_spiller area |
+ // |---------------------|
+ // | (optional) |
// | capture state buf |
// |---------------------| = StubLocations::CAPTURED_STATE_BUFFER
// | (optional) |
@@ -188,6 +191,18 @@ void DowncallLinker::StubGenerator::generate() {
VMStorage shuffle_reg = as_VMStorage(rbx);
ArgumentShuffle arg_shuffle(filtered_java_regs, out_regs, shuffle_reg);
+ // Need to spill for state capturing runtime call.
+ // The area spilled into is distinct from the capture state buffer.
+ RegSpiller in_reg_spiller(out_regs);
+ int in_spill_rsp_offset = -1;
+ if (_captured_state_mask != 0) {
+ // The spill area cannot be shared with the shadow/out args space
+ // since spilling needs to happen before the call. Allocate a new
+ // region in the stack for this spill space.
+ in_spill_rsp_offset = allocated_frame_size;
+ allocated_frame_size += in_reg_spiller.spill_size_bytes();
+ }
+
#ifndef PRODUCT
LogTarget(Trace, foreign, downcall) lt;
if (lt.is_enabled()) {
@@ -232,6 +247,19 @@ void DowncallLinker::StubGenerator::generate() {
arg_shuffle.generate(_masm, shuffle_reg, 0, _abi._shadow_space_bytes);
__ block_comment("} argument shuffle");
+ if (_captured_state_mask != 0) {
+ assert(in_spill_rsp_offset != -1, "must be");
+ __ block_comment("{ load initial thread local");
+ in_reg_spiller.generate_spill(_masm, in_spill_rsp_offset);
+
+ // Copy the contents of the capture state buffer into thread local
+ __ movptr(c_rarg0, Address(rsp, locs.data_offset(StubLocations::CAPTURED_STATE_BUFFER)));
+ __ movl(c_rarg1, _captured_state_mask);
+ runtime_call(_masm, CAST_FROM_FN_PTR(address, DowncallLinker::capture_state_pre));
+
+ in_reg_spiller.generate_fill(_masm, in_spill_rsp_offset);
+ __ block_comment("} load initial thread local");
+ }
__ call(as_Register(locs.get(StubLocations::TARGET_ADDRESS)));
assert(!_abi.is_volatile_reg(r15_thread), "Call assumed not to kill r15");
@@ -258,15 +286,15 @@ void DowncallLinker::StubGenerator::generate() {
__ block_comment("{ save thread local");
if (should_save_return_value) {
- out_reg_spiller.generate_spill(_masm, spill_rsp_offset);
+ out_reg_spiller.generate_spill(_masm, out_spill_rsp_offset);
}
__ movptr(c_rarg0, Address(rsp, locs.data_offset(StubLocations::CAPTURED_STATE_BUFFER)));
__ movl(c_rarg1, _captured_state_mask);
- runtime_call(_masm, CAST_FROM_FN_PTR(address, DowncallLinker::capture_state));
+ runtime_call(_masm, CAST_FROM_FN_PTR(address, DowncallLinker::capture_state_post));
if (should_save_return_value) {
- out_reg_spiller.generate_fill(_masm, spill_rsp_offset);
+ out_reg_spiller.generate_fill(_masm, out_spill_rsp_offset);
}
__ block_comment("} save thread local");
@@ -319,14 +347,14 @@ void DowncallLinker::StubGenerator::generate() {
__ bind(L_safepoint_poll_slow_path);
if (should_save_return_value) {
- out_reg_spiller.generate_spill(_masm, spill_rsp_offset);
+ out_reg_spiller.generate_spill(_masm, out_spill_rsp_offset);
}
__ mov(c_rarg0, r15_thread);
runtime_call(_masm, CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans));
if (should_save_return_value) {
- out_reg_spiller.generate_fill(_masm, spill_rsp_offset);
+ out_reg_spiller.generate_fill(_masm, out_spill_rsp_offset);
}
__ jmp(L_after_safepoint_poll);
@@ -338,13 +366,13 @@ void DowncallLinker::StubGenerator::generate() {
__ bind(L_reguard);
if (should_save_return_value) {
- out_reg_spiller.generate_spill(_masm, spill_rsp_offset);
+ out_reg_spiller.generate_spill(_masm, out_spill_rsp_offset);
}
runtime_call(_masm, CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages));
if (should_save_return_value) {
- out_reg_spiller.generate_fill(_masm, spill_rsp_offset);
+ out_reg_spiller.generate_fill(_masm, out_spill_rsp_offset);
}
__ jmp(L_after_reguard);
diff --git a/src/hotspot/cpu/x86/frame_x86.inline.hpp b/src/hotspot/cpu/x86/frame_x86.inline.hpp
index dcd766545d3..3f3b951edc8 100644
--- a/src/hotspot/cpu/x86/frame_x86.inline.hpp
+++ b/src/hotspot/cpu/x86/frame_x86.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -231,8 +231,8 @@ inline bool frame::equal(frame other) const {
// Return unique id for this frame. The id must have a value where we can distinguish
// identity and younger/older relationship. null represents an invalid (incomparable)
-// frame.
-inline intptr_t* frame::id(void) const { return unextended_sp(); }
+// frame. Should not be called for heap frames.
+inline intptr_t* frame::id(void) const { return real_fp(); }
// Return true if the frame is older (less recent activation) than the frame represented by id
inline bool frame::is_older(intptr_t* id) const { assert(this->id() != nullptr && id != nullptr, "null frame id");
@@ -397,6 +397,9 @@ inline frame frame::sender(RegisterMap* map) const {
StackWatermarkSet::on_iteration(map->thread(), result);
}
+ // Calling frame::id() is currently not supported for heap frames.
+ assert(result._on_heap || this->_on_heap || result.is_older(this->id()), "Must be");
+
return result;
}
diff --git a/src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.cpp
index 0ea769dd488..c05f37a3bea 100644
--- a/src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -130,7 +130,7 @@ __ BIND(L_loop);
__ BIND(L_done);
}
-void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj, Address dst, Register rscratch) {
+void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj, Register rscratch) {
// Does a store check for the oop in register obj. The content of
// register obj is destroyed afterwards.
CardTableBarrierSet* ctbs = CardTableBarrierSet::barrier_set();
@@ -138,6 +138,8 @@ void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register ob
__ shrptr(obj, CardTable::card_shift());
Address card_addr;
+ precond(rscratch != noreg);
+ assert_different_registers(obj, rscratch);
// The calculation for byte_map_base is as follows:
// byte_map_base = _byte_map - (uintptr_t(low_bound) >> card_shift);
@@ -161,7 +163,7 @@ void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register ob
// entry and that entry is not properly handled by the relocation code.
AddressLiteral cardtable((address)byte_map_base, relocInfo::none);
Address index(noreg, obj, Address::times_1);
- card_addr = __ as_Address(ArrayAddress(cardtable, index), rscratch1);
+ card_addr = __ as_Address(ArrayAddress(cardtable, index), rscratch);
}
int dirty = CardTable::dirty_card_val();
@@ -190,10 +192,10 @@ void CardTableBarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorS
if (needs_post_barrier) {
// flatten object address if needed
if (!precise || (dst.index() == noreg && dst.disp() == 0)) {
- store_check(masm, dst.base(), dst, tmp2);
+ store_check(masm, dst.base(), tmp2);
} else {
__ lea(tmp1, dst);
- store_check(masm, tmp1, dst, tmp2);
+ store_check(masm, tmp1, tmp2);
}
}
}
diff --git a/src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.hpp
index 201c11062f2..c38e16d4d5f 100644
--- a/src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -33,7 +33,7 @@ protected:
virtual void gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators,
Register addr, Register count) {}
- void store_check(MacroAssembler* masm, Register obj, Address dst, Register rscratch);
+ void store_check(MacroAssembler* masm, Register obj, Register rscratch);
virtual void gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, Register addr, Register count, Register tmp);
diff --git a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp
index 47a3dad54e7..c20551b5084 100644
--- a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp
@@ -31,6 +31,7 @@
#include "gc/z/zBarrierSetAssembler.hpp"
#include "gc/z/zBarrierSetRuntime.hpp"
#include "gc/z/zThreadLocalData.hpp"
+#include "logging/log.hpp"
#include "memory/resourceArea.hpp"
#include "runtime/jniHandles.hpp"
#include "runtime/sharedRuntime.hpp"
@@ -1391,10 +1392,13 @@ static uint16_t patch_barrier_relocation_value(int format) {
}
}
-void ZBarrierSetAssembler::patch_barrier_relocation(address addr, int format) {
+void ZBarrierSetAssembler::patch_barrier_relocation(address addr, int format, bool log) {
const int offset = patch_barrier_relocation_offset(format);
const uint16_t value = patch_barrier_relocation_value(format);
uint8_t* const patch_addr = (uint8_t*)addr + offset;
+ if (log) {
+ log_trace(aot, codecache, stubs)("patching address " INTPTR_FORMAT " offset %d value 0x%x", p2i(addr), offset, value);
+ }
if (format == ZBarrierRelocationFormatLoadGoodBeforeShl) {
if (VM_Version::supports_apx_f()) {
NativeInstruction* instruction = nativeInstruction_at(addr);
@@ -1426,6 +1430,74 @@ void ZBarrierSetAssembler::patch_barriers() {
#undef __
#define __ masm->
+void ZBarrierSetAssembler::register_reloc_addresses(GrowableArray &entries, int begin, int count) {
+ int formats[] = {
+ ZBarrierRelocationFormatLoadBadAfterTest,
+ ZBarrierRelocationFormatStoreBadAfterTest,
+ ZBarrierRelocationFormatStoreGoodAfterOr,
+ -1
+ };
+ int format_idx = 0;
+ int format = formats[format_idx++];
+ for (int i = begin; i < begin + count; i++) {
+ address addr = entries.at(i);
+ // reloc addresses occur in 3 groups terminated with a nullptr
+ if (addr == nullptr) {
+ assert(format_idx < (int)(sizeof(formats) / sizeof(formats[0])),
+ "too many reloc groups");
+ format = formats[format_idx++];
+ } else {
+ switch(format) {
+ case ZBarrierRelocationFormatLoadBadAfterTest:
+ _load_bad_relocations.append(addr);
+ break;
+ case ZBarrierRelocationFormatStoreBadAfterTest:
+ _store_bad_relocations.append(addr);
+ break;
+ case ZBarrierRelocationFormatStoreGoodAfterOr:
+ _store_good_relocations.append(addr);
+ break;
+ default:
+ ShouldNotReachHere();
+ break;
+ }
+ patch_barrier_relocation(addr, format, true);
+ }
+ }
+ assert(format == -1, "unterminated format list");
+}
+
+void ZBarrierSetAssembler::retrieve_reloc_addresses(address start, address end, GrowableArray &entries) {
+ assert(start != nullptr, "start address must not be null");
+ assert(end != nullptr, "start address must not be null");
+ assert(start < end, "stub range must not be empty");
+ for (int i = 0; i < _load_bad_relocations.length(); i++) {
+ address addr = _load_bad_relocations.at(i);
+ assert(addr != nullptr, "load bad reloc address shoudl not be null!");
+ if (start <= addr && addr < end) {
+ entries.append(addr);
+ }
+ }
+ entries.append(nullptr);
+ for (int i = 0; i < _store_bad_relocations.length(); i++) {
+ address addr = _store_bad_relocations.at(i);
+ assert(addr != nullptr, "store bad reloc address shoudl not be null!");
+ if (start <= addr && addr < end) {
+ entries.append(addr);
+ }
+ }
+ entries.append(nullptr);
+ for (int i = 0; i < _store_good_relocations.length(); i++) {
+ address addr = _store_good_relocations.at(i);
+ assert(addr != nullptr, "store good reloc address shoudl not be null!");
+ if (start <= addr && addr < end) {
+ entries.append(addr);
+ }
+ }
+ entries.append(nullptr);
+}
+
+
void ZBarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) {
// C1 calls verfy_oop in the middle of barriers, before they have been uncolored
// and after being colored. Therefore, we must deal with colored oops as well.
diff --git a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp
index e91e2b9ea20..ce0c4769716 100644
--- a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp
@@ -189,10 +189,14 @@ public:
Label& slow_path,
Label& slow_path_continuation) const;
- void patch_barrier_relocation(address addr, int format);
+ void patch_barrier_relocation(address addr, int format, bool log = false);
void patch_barriers();
+ void register_reloc_addresses(GrowableArray &entries, int begin, int count);
+
+ void retrieve_reloc_addresses(address start, address end, GrowableArray &entries);
+
void check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error);
};
diff --git a/src/hotspot/cpu/x86/globals_x86.hpp b/src/hotspot/cpu/x86/globals_x86.hpp
index 4f5b6d31e75..6de46752790 100644
--- a/src/hotspot/cpu/x86/globals_x86.hpp
+++ b/src/hotspot/cpu/x86/globals_x86.hpp
@@ -117,9 +117,6 @@ define_pd_global(intx, InitArrayShortSize, 8*BytesPerLong);
product(bool, UseIncDec, true, DIAGNOSTIC, \
"Use INC, DEC instructions on x86") \
\
- product(bool, UseNewLongLShift, false, \
- "Use optimized bitwise shift left") \
- \
product(bool, UseAddressNop, false, \
"Use '0F 1F [addr]' NOP instructions on x86 cpus") \
\
@@ -168,16 +165,27 @@ define_pd_global(intx, InitArrayShortSize, 8*BytesPerLong);
"Perform Ecore Optimization") \
\
/* Minimum array size in bytes to use AVX512 intrinsics */ \
- /* for copy, inflate and fill which don't bail out early based on any */ \
+ /* for inflate and fill which don't bail out early based on any */ \
/* condition. When this value is set to zero compare operations like */ \
/* compare, vectorizedMismatch, compress can also use AVX512 intrinsics.*/\
product(int, AVX3Threshold, 4096, DIAGNOSTIC, \
"Minimum array size in bytes to use AVX512 intrinsics" \
- "for copy, inflate and fill. When this value is set as zero" \
+ "for inflate and fill. When this value is set as zero" \
"compare operations can also use AVX512 intrinsics.") \
range(0, max_jint) \
constraint(AVX3ThresholdConstraintFunc,AfterErgo) \
\
+ /* Minimum array size in bytes to use AVX512 intrinsics */ \
+ /* for copy and fill which don't bail out early based on any */ \
+ /* condition. When this value is set to zero clear operations that */ \
+ /* work on memory blocks can also use AVX512 intrinsics. */ \
+ product(int, CopyAVX3Threshold, 4096, DIAGNOSTIC, \
+ "Minimum array size in bytes to use AVX512 intrinsics" \
+ "for copy and fill. When this value is set as zero" \
+ "clear operations can also use AVX512 intrinsics.") \
+ range(0, max_jint) \
+ constraint(CopyAVX3ThresholdConstraintFunc,AfterErgo) \
+ \
product(bool, IntelJccErratumMitigation, true, DIAGNOSTIC, \
"Turn off JVM mitigations related to Intel micro code " \
"mitigations for the Intel JCC erratum") \
diff --git a/src/hotspot/cpu/x86/interp_masm_x86.cpp b/src/hotspot/cpu/x86/interp_masm_x86.cpp
index b2ea4143ac4..a38971c86fb 100644
--- a/src/hotspot/cpu/x86/interp_masm_x86.cpp
+++ b/src/hotspot/cpu/x86/interp_masm_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1392,28 +1392,15 @@ void InterpreterMacroAssembler::profile_final_call(Register mdp) {
void InterpreterMacroAssembler::profile_virtual_call(Register receiver,
- Register mdp,
- bool receiver_can_be_null) {
+ Register mdp) {
if (ProfileInterpreter) {
Label profile_continue;
// If no method data exists, go to profile_continue.
test_method_data_pointer(mdp, profile_continue);
- Label skip_receiver_profile;
- if (receiver_can_be_null) {
- Label not_null;
- testptr(receiver, receiver);
- jccb(Assembler::notZero, not_null);
- // We are making a call. Increment the count for null receiver.
- increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
- jmp(skip_receiver_profile);
- bind(not_null);
- }
-
// Record the receiver type.
profile_receiver_type(receiver, mdp, 0);
- bind(skip_receiver_profile);
// The method data pointer needs to be updated to reflect the new target.
update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size()));
diff --git a/src/hotspot/cpu/x86/interp_masm_x86.hpp b/src/hotspot/cpu/x86/interp_masm_x86.hpp
index 4114028f78e..dfbd7ab64e0 100644
--- a/src/hotspot/cpu/x86/interp_masm_x86.hpp
+++ b/src/hotspot/cpu/x86/interp_masm_x86.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -243,8 +243,7 @@ class InterpreterMacroAssembler: public MacroAssembler {
void profile_not_taken_branch(Register mdp);
void profile_call(Register mdp);
void profile_final_call(Register mdp);
- void profile_virtual_call(Register receiver, Register mdp,
- bool receiver_can_be_null = false);
+ void profile_virtual_call(Register receiver, Register mdp);
void profile_ret(Register return_bci, Register mdp);
void profile_null_seen(Register mdp);
void profile_typecheck(Register mdp, Register klass);
diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp
index b54f6adc263..5ab3ca339aa 100644
--- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp
@@ -385,7 +385,8 @@ void MacroAssembler::warn(const char* msg) {
// Windows always allocates space for its register args
subq(rsp, frame::arg_reg_save_area_bytes);
#endif
- lea(c_rarg0, ExternalAddress((address) msg));
+ const char* str = (code_section()->scratch_emit()) ? msg : AOTCodeCache::add_C_string(msg);
+ lea(c_rarg0, ExternalAddress((address) str));
call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning)));
#ifdef _WIN64
@@ -961,7 +962,7 @@ void MacroAssembler::call(AddressLiteral entry, Register rscratch) {
void MacroAssembler::ic_call(address entry, jint method_index) {
RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index);
// Needs full 64-bit immediate for later patching.
- mov64(rax, (int64_t)Universe::non_oop_word());
+ Assembler::mov64(rax, (int64_t)Universe::non_oop_word());
call(AddressLiteral(entry, rh));
}
@@ -985,12 +986,9 @@ int MacroAssembler::ic_check(int end_alignment) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(temp, receiver);
cmpl(temp, Address(data, CompiledICData::speculated_klass_offset()));
- } else if (UseCompressedClassPointers) {
+ } else {
movl(temp, Address(receiver, oopDesc::klass_offset_in_bytes()));
cmpl(temp, Address(data, CompiledICData::speculated_klass_offset()));
- } else {
- movptr(temp, Address(receiver, oopDesc::klass_offset_in_bytes()));
- cmpptr(temp, Address(data, CompiledICData::speculated_klass_offset()));
}
// if inline cache check fails, then jump to runtime routine
@@ -1961,6 +1959,30 @@ void MacroAssembler::movflt(XMMRegister dst, AddressLiteral src, Register rscrat
}
}
+void MacroAssembler::movhlf(XMMRegister dst, XMMRegister src, Register rscratch) {
+ if (VM_Version::supports_avx10_2()) {
+ evmovw(dst, src);
+ } else {
+ assert(rscratch != noreg, "missing");
+ evmovw(rscratch, src);
+ evmovw(dst, rscratch);
+ }
+}
+
+void MacroAssembler::mov64(Register dst, int64_t imm64) {
+ if (is_uimm32(imm64)) {
+ movl(dst, checked_cast(imm64));
+ } else if (is_simm32(imm64)) {
+ movq(dst, checked_cast(imm64));
+ } else {
+ Assembler::mov64(dst, imm64);
+ }
+}
+
+void MacroAssembler::mov64(Register dst, int64_t imm64, relocInfo::relocType rtype, int format) {
+ Assembler::mov64(dst, imm64, rtype, format);
+}
+
void MacroAssembler::movptr(Register dst, Register src) {
movq(dst, src);
}
@@ -1971,13 +1993,7 @@ void MacroAssembler::movptr(Register dst, Address src) {
// src should NEVER be a real pointer. Use AddressLiteral for true pointers
void MacroAssembler::movptr(Register dst, intptr_t src) {
- if (is_uimm32(src)) {
- movl(dst, checked_cast(src));
- } else if (is_simm32(src)) {
- movq(dst, checked_cast(src));
- } else {
- mov64(dst, src);
- }
+ mov64(dst, src);
}
void MacroAssembler::movptr(Address dst, Register src) {
@@ -2656,14 +2672,14 @@ void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src, Register rscra
}
}
-void MacroAssembler::vucomxsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
+void MacroAssembler::evucomxsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
assert(rscratch != noreg || always_reachable(src), "missing");
if (reachable(src)) {
- Assembler::vucomxsd(dst, as_Address(src));
+ Assembler::evucomxsd(dst, as_Address(src));
} else {
lea(rscratch, src);
- Assembler::vucomxsd(dst, Address(rscratch, 0));
+ Assembler::evucomxsd(dst, Address(rscratch, 0));
}
}
@@ -2678,14 +2694,36 @@ void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src, Register rscra
}
}
-void MacroAssembler::vucomxss(XMMRegister dst, AddressLiteral src, Register rscratch) {
+void MacroAssembler::evucomxss(XMMRegister dst, AddressLiteral src, Register rscratch) {
assert(rscratch != noreg || always_reachable(src), "missing");
if (reachable(src)) {
- Assembler::vucomxss(dst, as_Address(src));
+ Assembler::evucomxss(dst, as_Address(src));
} else {
lea(rscratch, src);
- Assembler::vucomxss(dst, Address(rscratch, 0));
+ Assembler::evucomxss(dst, Address(rscratch, 0));
+ }
+}
+
+void MacroAssembler::evucomish(XMMRegister dst, AddressLiteral src, Register rscratch) {
+ assert(rscratch != noreg || always_reachable(src), "missing");
+
+ if (reachable(src)) {
+ Assembler::evucomish(dst, as_Address(src));
+ } else {
+ lea(rscratch, src);
+ Assembler::evucomish(dst, Address(rscratch, 0));
+ }
+}
+
+void MacroAssembler::evucomxsh(XMMRegister dst, AddressLiteral src, Register rscratch) {
+ assert(rscratch != noreg || always_reachable(src), "missing");
+
+ if (reachable(src)) {
+ Assembler::evucomxsh(dst, as_Address(src));
+ } else {
+ lea(rscratch, src);
+ Assembler::evucomxsh(dst, Address(rscratch, 0));
}
}
@@ -5376,11 +5414,9 @@ void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(dst, src);
decode_klass_not_null(dst, tmp);
- } else if (UseCompressedClassPointers) {
+ } else {
movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
decode_klass_not_null(dst, tmp);
- } else {
- movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
}
}
@@ -5388,12 +5424,8 @@ void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
assert(!UseCompactObjectHeaders, "not with compact headers");
assert_different_registers(src, tmp);
assert_different_registers(dst, tmp);
- if (UseCompressedClassPointers) {
- encode_klass_not_null(src, tmp);
- movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
- } else {
- movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
- }
+ encode_klass_not_null(src, tmp);
+ movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
}
void MacroAssembler::cmp_klass(Register klass, Register obj, Register tmp) {
@@ -5402,10 +5434,8 @@ void MacroAssembler::cmp_klass(Register klass, Register obj, Register tmp) {
assert_different_registers(klass, obj, tmp);
load_narrow_klass_compact(tmp, obj);
cmpl(klass, tmp);
- } else if (UseCompressedClassPointers) {
- cmpl(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
} else {
- cmpptr(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
+ cmpl(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
}
}
@@ -5416,12 +5446,9 @@ void MacroAssembler::cmp_klasses_from_objects(Register obj1, Register obj2, Regi
load_narrow_klass_compact(tmp1, obj1);
load_narrow_klass_compact(tmp2, obj2);
cmpl(tmp1, tmp2);
- } else if (UseCompressedClassPointers) {
+ } else {
movl(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
cmpl(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes()));
- } else {
- movptr(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
- cmpptr(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes()));
}
}
@@ -5470,10 +5497,8 @@ void MacroAssembler::store_heap_oop_null(Address dst) {
void MacroAssembler::store_klass_gap(Register dst, Register src) {
assert(!UseCompactObjectHeaders, "Don't use with compact headers");
- if (UseCompressedClassPointers) {
- // Store to klass gap in destination
- movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src);
- }
+ // Store to klass gap in destination
+ movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src);
}
#ifdef ASSERT
@@ -5648,7 +5673,12 @@ void MacroAssembler::encode_and_move_klass_not_null(Register dst, Register src)
BLOCK_COMMENT("encode_and_move_klass_not_null {");
assert_different_registers(src, dst);
if (CompressedKlassPointers::base() != nullptr) {
- movptr(dst, -(intptr_t)CompressedKlassPointers::base());
+ if (AOTCodeCache::is_on_for_dump()) {
+ movptr(dst, ExternalAddress(CompressedKlassPointers::base_addr()));
+ negq(dst);
+ } else {
+ movptr(dst, -(intptr_t)CompressedKlassPointers::base());
+ }
addq(dst, src);
} else {
movptr(dst, src);
@@ -5663,7 +5693,6 @@ void MacroAssembler::decode_klass_not_null(Register r, Register tmp) {
BLOCK_COMMENT("decode_klass_not_null {");
assert_different_registers(r, tmp);
// Note: it will change flags
- assert(UseCompressedClassPointers, "should only be used for compressed headers");
// Cannot assert, unverified entry point counts instructions (see .ad file)
// vtableStubs also counts instructions in pd_code_size_limit.
// Also do not verify_oop as this is called by verify_oop.
@@ -5685,7 +5714,6 @@ void MacroAssembler::decode_and_move_klass_not_null(Register dst, Register src)
BLOCK_COMMENT("decode_and_move_klass_not_null {");
assert_different_registers(src, dst);
// Note: it will change flags
- assert (UseCompressedClassPointers, "should only be used for compressed headers");
// Cannot assert, unverified entry point counts instructions (see .ad file)
// vtableStubs also counts instructions in pd_code_size_limit.
// Also do not verify_oop as this is called by verify_oop.
@@ -5698,7 +5726,11 @@ void MacroAssembler::decode_and_move_klass_not_null(Register dst, Register src)
} else {
if (CompressedKlassPointers::shift() <= Address::times_8) {
if (CompressedKlassPointers::base() != nullptr) {
- movptr(dst, (intptr_t)CompressedKlassPointers::base());
+ if (AOTCodeCache::is_on_for_dump()) {
+ movptr(dst, ExternalAddress(CompressedKlassPointers::base_addr()));
+ } else {
+ movptr(dst, (intptr_t)CompressedKlassPointers::base());
+ }
} else {
xorq(dst, dst);
}
@@ -5710,9 +5742,14 @@ void MacroAssembler::decode_and_move_klass_not_null(Register dst, Register src)
}
} else {
if (CompressedKlassPointers::base() != nullptr) {
- const intptr_t base_right_shifted =
- (intptr_t)CompressedKlassPointers::base() >> CompressedKlassPointers::shift();
- movptr(dst, base_right_shifted);
+ if (AOTCodeCache::is_on_for_dump()) {
+ movptr(dst, ExternalAddress(CompressedKlassPointers::base_addr()));
+ shrq(dst, CompressedKlassPointers::shift());
+ } else {
+ const intptr_t base_right_shifted =
+ (intptr_t)CompressedKlassPointers::base() >> CompressedKlassPointers::shift();
+ movptr(dst, base_right_shifted);
+ }
} else {
xorq(dst, dst);
}
@@ -5742,7 +5779,6 @@ void MacroAssembler::set_narrow_oop(Address dst, jobject obj) {
}
void MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
- assert (UseCompressedClassPointers, "should only be used for compressed headers");
assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
int klass_index = oop_recorder()->find_index(k);
RelocationHolder rspec = metadata_Relocation::spec(klass_index);
@@ -5750,7 +5786,6 @@ void MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
}
void MacroAssembler::set_narrow_klass(Address dst, Klass* k) {
- assert (UseCompressedClassPointers, "should only be used for compressed headers");
assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
int klass_index = oop_recorder()->find_index(k);
RelocationHolder rspec = metadata_Relocation::spec(klass_index);
@@ -5776,7 +5811,6 @@ void MacroAssembler::cmp_narrow_oop(Address dst, jobject obj) {
}
void MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) {
- assert (UseCompressedClassPointers, "should only be used for compressed headers");
assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
int klass_index = oop_recorder()->find_index(k);
RelocationHolder rspec = metadata_Relocation::spec(klass_index);
@@ -5784,7 +5818,6 @@ void MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) {
}
void MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) {
- assert (UseCompressedClassPointers, "should only be used for compressed headers");
assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
int klass_index = oop_recorder()->find_index(k);
RelocationHolder rspec = metadata_Relocation::spec(klass_index);
@@ -5793,7 +5826,7 @@ void MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) {
void MacroAssembler::reinit_heapbase() {
if (UseCompressedOops) {
- if (Universe::heap() != nullptr) {
+ if (Universe::heap() != nullptr && !AOTCodeCache::is_on_for_dump()) {
if (CompressedOops::base() == nullptr) {
MacroAssembler::xorptr(r12_heapbase, r12_heapbase);
} else {
@@ -5812,7 +5845,7 @@ void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register rtmp, X
// cnt - number of qwords (8-byte words).
// base - start address, qword aligned.
Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end;
- bool use64byteVector = (MaxVectorSize == 64) && (VM_Version::avx3_threshold() == 0);
+ bool use64byteVector = (MaxVectorSize == 64) && (CopyAVX3Threshold == 0);
if (use64byteVector) {
vpxor(xtmp, xtmp, xtmp, AVX_512bit);
} else if (MaxVectorSize >= 32) {
@@ -5876,7 +5909,7 @@ void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register rtmp, X
// Clearing constant sized memory using YMM/ZMM registers.
void MacroAssembler::clear_mem(Register base, int cnt, Register rtmp, XMMRegister xtmp, KRegister mask) {
assert(UseAVX > 2 && VM_Version::supports_avx512vl(), "");
- bool use64byteVector = (MaxVectorSize > 32) && (VM_Version::avx3_threshold() == 0);
+ bool use64byteVector = (MaxVectorSize > 32) && (CopyAVX3Threshold == 0);
int vector64_count = (cnt & (~0x7)) >> 3;
cnt = cnt & 0x7;
@@ -6101,8 +6134,8 @@ void MacroAssembler::generate_fill(BasicType t, bool aligned,
// Fill 64-byte chunks
Label L_fill_64_bytes_loop_avx3, L_check_fill_64_bytes_avx2;
- // If number of bytes to fill < VM_Version::avx3_threshold(), perform fill using AVX2
- cmpptr(count, VM_Version::avx3_threshold());
+ // If number of bytes to fill < CopyAVX3Threshold, perform fill using AVX2
+ cmpptr(count, CopyAVX3Threshold);
jccb(Assembler::below, L_check_fill_64_bytes_avx2);
vpbroadcastd(xtmp, xtmp, Assembler::AVX_512bit);
@@ -9177,7 +9210,7 @@ void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XM
case T_FLOAT:
evminmaxps(dst, mask, nds, src, merge, AVX10_2_MINMAX_MAX_COMPARE_SIGN, vector_len); break;
case T_DOUBLE:
- evminmaxps(dst, mask, nds, src, merge, AVX10_2_MINMAX_MAX_COMPARE_SIGN, vector_len); break;
+ evminmaxpd(dst, mask, nds, src, merge, AVX10_2_MINMAX_MAX_COMPARE_SIGN, vector_len); break;
default:
fatal("Unexpected type argument %s", type2name(type)); break;
}
@@ -9475,7 +9508,6 @@ void MacroAssembler::generate_fill_avx3(BasicType type, Register to, Register va
Label L_fill_zmm_sequence;
int shift = -1;
- int avx3threshold = VM_Version::avx3_threshold();
switch(type) {
case T_BYTE: shift = 0;
break;
@@ -9491,10 +9523,10 @@ void MacroAssembler::generate_fill_avx3(BasicType type, Register to, Register va
fatal("Unhandled type: %s\n", type2name(type));
}
- if ((avx3threshold != 0) || (MaxVectorSize == 32)) {
+ if ((CopyAVX3Threshold != 0) || (MaxVectorSize == 32)) {
if (MaxVectorSize == 64) {
- cmpq(count, avx3threshold >> shift);
+ cmpq(count, CopyAVX3Threshold >> shift);
jcc(Assembler::greater, L_fill_zmm_sequence);
}
diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.hpp b/src/hotspot/cpu/x86/macroAssembler_x86.hpp
index 5c049f710e2..021d2943ee8 100644
--- a/src/hotspot/cpu/x86/macroAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/macroAssembler_x86.hpp
@@ -162,6 +162,8 @@ class MacroAssembler: public Assembler {
void incrementq(AddressLiteral dst, Register rscratch = noreg);
+ void movhlf(XMMRegister dst, XMMRegister src, Register rscratch = noreg);
+
// Support optimal SSE move instructions.
void movflt(XMMRegister dst, XMMRegister src) {
if (dst-> encoding() == src->encoding()) return;
@@ -351,8 +353,7 @@ class MacroAssembler: public Assembler {
void load_klass(Register dst, Register src, Register tmp);
void store_klass(Register dst, Register src, Register tmp);
- // Compares the Klass pointer of an object to a given Klass (which might be narrow,
- // depending on UseCompressedClassPointers).
+ // Compares the narrow Klass pointer of an object to a given narrow Klass.
void cmp_klass(Register klass, Register obj, Register tmp);
// Compares the Klass pointer of two objects obj1 and obj2. Result is in the condition flags.
@@ -1309,21 +1310,29 @@ public:
void subss(XMMRegister dst, Address src) { Assembler::subss(dst, src); }
void subss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
+ void evucomish(XMMRegister dst, XMMRegister src) { Assembler::evucomish(dst, src); }
+ void evucomish(XMMRegister dst, Address src) { Assembler::evucomish(dst, src); }
+ void evucomish(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
+
+ void evucomxsh(XMMRegister dst, XMMRegister src) { Assembler::evucomxsh(dst, src); }
+ void evucomxsh(XMMRegister dst, Address src) { Assembler::evucomxsh(dst, src); }
+ void evucomxsh(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
+
void ucomiss(XMMRegister dst, XMMRegister src) { Assembler::ucomiss(dst, src); }
void ucomiss(XMMRegister dst, Address src) { Assembler::ucomiss(dst, src); }
void ucomiss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
- void vucomxss(XMMRegister dst, XMMRegister src) { Assembler::vucomxss(dst, src); }
- void vucomxss(XMMRegister dst, Address src) { Assembler::vucomxss(dst, src); }
- void vucomxss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
+ void evucomxss(XMMRegister dst, XMMRegister src) { Assembler::evucomxss(dst, src); }
+ void evucomxss(XMMRegister dst, Address src) { Assembler::evucomxss(dst, src); }
+ void evucomxss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
void ucomisd(XMMRegister dst, XMMRegister src) { Assembler::ucomisd(dst, src); }
void ucomisd(XMMRegister dst, Address src) { Assembler::ucomisd(dst, src); }
void ucomisd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
- void vucomxsd(XMMRegister dst, XMMRegister src) { Assembler::vucomxsd(dst, src); }
- void vucomxsd(XMMRegister dst, Address src) { Assembler::vucomxsd(dst, src); }
- void vucomxsd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
+ void evucomxsd(XMMRegister dst, XMMRegister src) { Assembler::evucomxsd(dst, src); }
+ void evucomxsd(XMMRegister dst, Address src) { Assembler::evucomxsd(dst, src); }
+ void evucomxsd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
// Bitwise Logical XOR of Packed Double-Precision Floating-Point Values
void xorpd(XMMRegister dst, XMMRegister src);
@@ -1869,6 +1878,9 @@ public:
void mov_metadata(Register dst, Metadata* obj);
void mov_metadata(Address dst, Metadata* obj, Register rscratch);
+ void mov64(Register dst, int64_t imm64);
+ void mov64(Register dst, int64_t imm64, relocInfo::relocType rtype, int format);
+
void movptr(Register dst, Register src);
void movptr(Register dst, Address src);
void movptr(Register dst, AddressLiteral src);
diff --git a/src/hotspot/cpu/x86/macroAssembler_x86_sha.cpp b/src/hotspot/cpu/x86/macroAssembler_x86_sha.cpp
index 9f0232075cd..401d5dc22cc 100644
--- a/src/hotspot/cpu/x86/macroAssembler_x86_sha.cpp
+++ b/src/hotspot/cpu/x86/macroAssembler_x86_sha.cpp
@@ -242,7 +242,6 @@ void MacroAssembler::fast_sha256(XMMRegister msg, XMMRegister state0, XMMRegiste
Label done_hash, loop0;
address K256 = StubRoutines::x86::k256_addr();
- address pshuffle_byte_flip_mask = StubRoutines::x86::pshuffle_byte_flip_mask_addr();
movdqu(state0, Address(state, 0));
movdqu(state1, Address(state, 16));
@@ -253,7 +252,7 @@ void MacroAssembler::fast_sha256(XMMRegister msg, XMMRegister state0, XMMRegiste
palignr(state0, state1, 8);
pblendw(state1, msgtmp4, 0xF0);
- movdqu(shuf_mask, ExternalAddress(pshuffle_byte_flip_mask));
+ movdqu(shuf_mask, ExternalAddress(StubRoutines::x86::pshuffle_byte_flip_mask_addr()));
lea(rax, ExternalAddress(K256));
bind(loop0);
@@ -661,8 +660,6 @@ void MacroAssembler::sha256_AVX2(XMMRegister msg, XMMRegister state0, XMMRegiste
compute_size1, compute_size_end1;
address K256_W = StubRoutines::x86::k256_W_addr();
- address pshuffle_byte_flip_mask = StubRoutines::x86::pshuffle_byte_flip_mask_addr();
- address pshuffle_byte_flip_mask_addr = nullptr;
const XMMRegister& SHUF_00BA = xmm10; // ymm10: shuffle xBxA -> 00BA
const XMMRegister& SHUF_DC00 = xmm12; // ymm12: shuffle xDxC -> DC00
@@ -791,10 +788,14 @@ enum {
// load g - r10 after it is used as scratch
movl(h, Address(CTX, 4*7));
- pshuffle_byte_flip_mask_addr = pshuffle_byte_flip_mask;
- vmovdqu(BYTE_FLIP_MASK, ExternalAddress(pshuffle_byte_flip_mask_addr + 0)); // [PSHUFFLE_BYTE_FLIP_MASK wrt rip]
- vmovdqu(SHUF_00BA, ExternalAddress(pshuffle_byte_flip_mask_addr + 32)); // [_SHUF_00BA wrt rip]
- vmovdqu(SHUF_DC00, ExternalAddress(pshuffle_byte_flip_mask_addr + 64)); // [_SHUF_DC00 wrt rip]
+ // the three successive pshuffle_byte_flip_mask stub entries should
+ // be offset by 32 bytes
+ assert(StubRoutines::x86::pshuffle_byte_flip_mask_addr() + 32 == StubRoutines::x86::pshuffle_byte_flip_mask_00ba_addr(), "sanity");
+ assert(StubRoutines::x86::pshuffle_byte_flip_mask_addr() + 64 == StubRoutines::x86::pshuffle_byte_flip_mask_dc00_addr(), "sanity");
+
+ vmovdqu(BYTE_FLIP_MASK, ExternalAddress(StubRoutines::x86::pshuffle_byte_flip_mask_addr())); // [PSHUFFLE_BYTE_FLIP_MASK wrt rip]
+ vmovdqu(SHUF_00BA, ExternalAddress(StubRoutines::x86::pshuffle_byte_flip_mask_00ba_addr())); // [_SHUF_00BA wrt rip]
+ vmovdqu(SHUF_DC00, ExternalAddress(StubRoutines::x86::pshuffle_byte_flip_mask_dc00_addr())); // [_SHUF_DC00 wrt rip]
movl(g, Address(CTX, 4*6));
@@ -953,11 +954,9 @@ bind(only_one_block);
// load g - r10 after use as scratch
movl(h, Address(CTX, 4*7)); // 0x5be0cd19
-
- pshuffle_byte_flip_mask_addr = pshuffle_byte_flip_mask;
- vmovdqu(BYTE_FLIP_MASK, ExternalAddress(pshuffle_byte_flip_mask_addr + 0)); // [PSHUFFLE_BYTE_FLIP_MASK wrt rip]
- vmovdqu(SHUF_00BA, ExternalAddress(pshuffle_byte_flip_mask_addr + 32)); // [_SHUF_00BA wrt rip]
- vmovdqu(SHUF_DC00, ExternalAddress(pshuffle_byte_flip_mask_addr + 64)); // [_SHUF_DC00 wrt rip]
+ vmovdqu(BYTE_FLIP_MASK, ExternalAddress(StubRoutines::x86::pshuffle_byte_flip_mask_addr())); // [PSHUFFLE_BYTE_FLIP_MASK wrt rip]
+ vmovdqu(SHUF_00BA, ExternalAddress(StubRoutines::x86::pshuffle_byte_flip_mask_00ba_addr())); // [_SHUF_00BA wrt rip]
+ vmovdqu(SHUF_DC00, ExternalAddress(StubRoutines::x86::pshuffle_byte_flip_mask_dc00_addr())); // [_SHUF_DC00 wrt rip]
movl(g, Address(CTX, 4*6)); // 0x1f83d9ab
@@ -1346,9 +1345,12 @@ void MacroAssembler::sha512_AVX2(XMMRegister msg, XMMRegister state0, XMMRegiste
// load g - r10 after it is used as scratch
movq(h, Address(CTX, 8 * 7));
- pshuffle_byte_flip_mask_addr = pshuffle_byte_flip_mask_sha512;
- vmovdqu(BYTE_FLIP_MASK, ExternalAddress(pshuffle_byte_flip_mask_addr + 0)); // PSHUFFLE_BYTE_FLIP_MASK wrt rip
- vmovdqu(YMM_MASK_LO, ExternalAddress(pshuffle_byte_flip_mask_addr + 32));
+ // the two successive pshuffle_byte_flip_mask_sha512 stub entries should
+ // be offset by 32 bytes
+ assert(StubRoutines::x86::pshuffle_byte_flip_mask_addr_sha512() + 32 == StubRoutines::x86::pshuffle_byte_flip_mask_ymm_lo_addr_sha512(), "sanity");
+
+ vmovdqu(BYTE_FLIP_MASK, ExternalAddress(StubRoutines::x86::pshuffle_byte_flip_mask_addr_sha512())); // PSHUFFLE_BYTE_FLIP_MASK wrt rip
+ vmovdqu(YMM_MASK_LO, ExternalAddress(StubRoutines::x86::pshuffle_byte_flip_mask_ymm_lo_addr_sha512())); // MASK_YMM_LO wrt rip
movq(g, Address(CTX, 8 * 6));
diff --git a/src/hotspot/cpu/x86/matcher_x86.hpp b/src/hotspot/cpu/x86/matcher_x86.hpp
index f7973a8564e..62a5d2827bc 100644
--- a/src/hotspot/cpu/x86/matcher_x86.hpp
+++ b/src/hotspot/cpu/x86/matcher_x86.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -75,7 +75,6 @@
}
static bool narrow_klass_use_complex_address() {
- assert(UseCompressedClassPointers, "only for compressed klass code");
return (CompressedKlassPointers::shift() <= 3);
}
diff --git a/src/hotspot/cpu/x86/methodHandles_x86.cpp b/src/hotspot/cpu/x86/methodHandles_x86.cpp
index 54376c6ad9a..5b15444bc32 100644
--- a/src/hotspot/cpu/x86/methodHandles_x86.cpp
+++ b/src/hotspot/cpu/x86/methodHandles_x86.cpp
@@ -110,14 +110,13 @@ void MethodHandles::verify_ref_kind(MacroAssembler* _masm, int ref_kind, Registe
__ andl(temp, java_lang_invoke_MemberName::MN_REFERENCE_KIND_MASK);
__ cmpl(temp, ref_kind);
__ jcc(Assembler::equal, L);
- { char* buf = NEW_C_HEAP_ARRAY(char, 100, mtInternal);
- jio_snprintf(buf, 100, "verify_ref_kind expected %x", ref_kind);
- if (ref_kind == JVM_REF_invokeVirtual ||
- ref_kind == JVM_REF_invokeSpecial)
- // could do this for all ref_kinds, but would explode assembly code size
- trace_method_handle(_masm, buf);
- __ STOP(buf);
+ const char* msg = ref_kind_to_verify_msg(ref_kind);
+ if (ref_kind == JVM_REF_invokeVirtual ||
+ ref_kind == JVM_REF_invokeSpecial) {
+ // could do this for all ref_kinds, but would explode assembly code size
+ trace_method_handle(_masm, msg);
}
+ __ STOP(msg);
BLOCK_COMMENT("} verify_ref_kind");
__ bind(L);
}
diff --git a/src/hotspot/cpu/x86/stubDeclarations_x86.hpp b/src/hotspot/cpu/x86/stubDeclarations_x86.hpp
index 971c8fd3c44..24886deb3c5 100644
--- a/src/hotspot/cpu/x86/stubDeclarations_x86.hpp
+++ b/src/hotspot/cpu/x86/stubDeclarations_x86.hpp
@@ -29,14 +29,16 @@
#define STUBGEN_PREUNIVERSE_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(preuniverse, 500) \
#define STUBGEN_INITIAL_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(initial, PRODUCT_ONLY(20000) NOT_PRODUCT(21000) WINDOWS_ONLY(+1000)) \
do_stub(initial, verify_mxcsr) \
do_arch_entry(x86, initial, verify_mxcsr, verify_mxcsr_entry, \
@@ -65,14 +67,18 @@
#define STUBGEN_CONTINUATION_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(continuation, 3000) \
+// count needed for declaration of vector_iota_indices stub
+#define VECTOR_IOTA_COUNT 6
#define STUBGEN_COMPILER_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(compiler, 120000 WINDOWS_ONLY(+2000)) \
do_stub(compiler, vector_float_sign_mask) \
do_arch_entry(x86, compiler, vector_float_sign_mask, \
@@ -126,8 +132,9 @@
do_arch_entry(x86, compiler, vector_long_sign_mask, \
vector_long_sign_mask, vector_long_sign_mask) \
do_stub(compiler, vector_iota_indices) \
- do_arch_entry(x86, compiler, vector_iota_indices, \
- vector_iota_indices, vector_iota_indices) \
+ do_arch_entry_array(x86, compiler, vector_iota_indices, \
+ vector_iota_indices, vector_iota_indices, \
+ VECTOR_IOTA_COUNT) \
do_stub(compiler, vector_count_leading_zeros_lut) \
do_arch_entry(x86, compiler, vector_count_leading_zeros_lut, \
vector_count_leading_zeros_lut, \
@@ -161,6 +168,12 @@
do_arch_entry(x86, compiler, pshuffle_byte_flip_mask, \
pshuffle_byte_flip_mask_addr, \
pshuffle_byte_flip_mask_addr) \
+ do_arch_entry(x86, compiler, pshuffle_byte_flip_mask, \
+ pshuffle_byte_flip_mask_00ba_addr, \
+ pshuffle_byte_flip_mask_00ba_addr) \
+ do_arch_entry(x86, compiler, pshuffle_byte_flip_mask, \
+ pshuffle_byte_flip_mask_dc00_addr, \
+ pshuffle_byte_flip_mask_dc00_addr) \
/* x86_64 exposes these 3 stubs via a generic entry array */ \
/* other arches use arch-specific entries */ \
/* this really needs rationalising */ \
@@ -171,6 +184,9 @@
do_arch_entry(x86, compiler, pshuffle_byte_flip_mask_sha512, \
pshuffle_byte_flip_mask_addr_sha512, \
pshuffle_byte_flip_mask_addr_sha512) \
+ do_arch_entry(x86, compiler, pshuffle_byte_flip_mask_sha512, \
+ pshuffle_byte_flip_mask_ymm_lo_addr_sha512, \
+ pshuffle_byte_flip_mask_ymm_lo_addr_sha512) \
do_stub(compiler, compress_perm_table32) \
do_arch_entry(x86, compiler, compress_perm_table32, \
compress_perm_table32, compress_perm_table32) \
@@ -241,7 +257,8 @@
#define STUBGEN_FINAL_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(final, 33000 \
WINDOWS_ONLY(+22000) ZGC_ONLY(+20000)) \
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp
index efb0411aa39..993d1964034 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp
@@ -188,8 +188,18 @@ address StubGenerator::generate_call_stub(address& return_address) {
(int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off,
"adjust this code");
StubId stub_id = StubId::stubgen_call_stub_id;
+ GrowableArray entries;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 2, "sanity check");
+ address start = load_archive_data(stub_id, &entries);
+ if (start != nullptr) {
+ assert(entries.length() == 1, "expected 1 extra entry");
+ return_address = entries.at(0);
+ return start;
+ }
+
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
// same as in generate_catch_exception()!
const Address rsp_after_call(rbp, rsp_after_call_off * wordSize);
@@ -298,6 +308,7 @@ address StubGenerator::generate_call_stub(address& return_address) {
BLOCK_COMMENT("call_stub_return_address:");
return_address = __ pc();
+ entries.append(return_address);
// store result depending on type (everything that is not
// T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
@@ -394,6 +405,9 @@ address StubGenerator::generate_call_stub(address& return_address) {
__ movdbl(Address(c_rarg0, 0), xmm0);
__ jmp(exit);
+ // record the stub entry and end plus the auxiliary entry
+ store_archive_data(stub_id, start, __ pc(), &entries);
+
return start;
}
@@ -411,8 +425,15 @@ address StubGenerator::generate_call_stub(address& return_address) {
address StubGenerator::generate_catch_exception() {
StubId stub_id = StubId::stubgen_catch_exception_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
// same as in generate_call_stub():
const Address rsp_after_call(rbp, rsp_after_call_off * wordSize);
@@ -442,7 +463,9 @@ address StubGenerator::generate_catch_exception() {
__ verify_oop(rax);
__ movptr(Address(r15_thread, Thread::pending_exception_offset()), rax);
- __ lea(rscratch1, ExternalAddress((address)__FILE__));
+ // special case -- add file name string to AOT address table
+ address file = (address)AOTCodeCache::add_C_string(__FILE__);
+ __ lea(rscratch1, ExternalAddress(file));
__ movptr(Address(r15_thread, Thread::exception_file_offset()), rscratch1);
__ movl(Address(r15_thread, Thread::exception_line_offset()), (int) __LINE__);
@@ -451,6 +474,9 @@ address StubGenerator::generate_catch_exception() {
"_call_stub_return_address must have been generated before");
__ jump(RuntimeAddress(StubRoutines::_call_stub_return_address));
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -467,8 +493,14 @@ address StubGenerator::generate_catch_exception() {
address StubGenerator::generate_forward_exception() {
StubId stub_id = StubId::stubgen_forward_exception_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
// Upon entry, the sp points to the return address returning into
// Java (interpreted or compiled) code; i.e., the return address
@@ -521,6 +553,9 @@ address StubGenerator::generate_forward_exception() {
__ verify_oop(rax);
__ jmp(rbx);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -531,12 +566,21 @@ address StubGenerator::generate_forward_exception() {
// Result:
address StubGenerator::generate_orderaccess_fence() {
StubId stub_id = StubId::stubgen_fence_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ membar(Assembler::StoreLoad);
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -550,8 +594,14 @@ address StubGenerator::generate_orderaccess_fence() {
address StubGenerator::generate_verify_mxcsr() {
StubId stub_id = StubId::stubgen_verify_mxcsr_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Address mxcsr_save(rsp, 0);
@@ -574,15 +624,24 @@ address StubGenerator::generate_verify_mxcsr() {
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_f2i_fixup() {
StubId stub_id = StubId::stubgen_f2i_fixup_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
Address inout(rsp, 5 * wordSize); // return address + 4 saves
- address start = __ pc();
+ start = __ pc();
Label L;
@@ -613,14 +672,23 @@ address StubGenerator::generate_f2i_fixup() {
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_f2l_fixup() {
StubId stub_id = StubId::stubgen_f2l_fixup_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
Address inout(rsp, 5 * wordSize); // return address + 4 saves
- address start = __ pc();
+ start = __ pc();
Label L;
@@ -651,15 +719,24 @@ address StubGenerator::generate_f2l_fixup() {
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_d2i_fixup() {
StubId stub_id = StubId::stubgen_d2i_fixup_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
Address inout(rsp, 6 * wordSize); // return address + 5 saves
- address start = __ pc();
+ start = __ pc();
Label L;
@@ -699,15 +776,24 @@ address StubGenerator::generate_d2i_fixup() {
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_d2l_fixup() {
StubId stub_id = StubId::stubgen_d2l_fixup_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
Address inout(rsp, 6 * wordSize); // return address + 5 saves
- address start = __ pc();
+ start = __ pc();
Label L;
@@ -747,14 +833,23 @@ address StubGenerator::generate_d2l_fixup() {
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_count_leading_zeros_lut() {
- __ align64();
StubId stub_id = StubId::stubgen_vector_count_leading_zeros_lut_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align64();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ emit_data64(0x0101010102020304, relocInfo::none);
__ emit_data64(0x0000000000000000, relocInfo::none);
@@ -765,14 +860,23 @@ address StubGenerator::generate_count_leading_zeros_lut() {
__ emit_data64(0x0101010102020304, relocInfo::none);
__ emit_data64(0x0000000000000000, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_popcount_avx_lut() {
- __ align64();
StubId stub_id = StubId::stubgen_vector_popcount_lut_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align64();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ emit_data64(0x0302020102010100, relocInfo::none);
__ emit_data64(0x0403030203020201, relocInfo::none);
@@ -783,14 +887,30 @@ address StubGenerator::generate_popcount_avx_lut() {
__ emit_data64(0x0302020102010100, relocInfo::none);
__ emit_data64(0x0403030203020201, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
-address StubGenerator::generate_iota_indices() {
- __ align(CodeEntryAlignment);
+void StubGenerator::generate_iota_indices() {
StubId stub_id = StubId::stubgen_vector_iota_indices_id;
+ GrowableArray entries;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == VECTOR_IOTA_COUNT, "sanity check");
+ address start = load_archive_data(stub_id, &entries);
+ if (start != nullptr) {
+ assert(entries.length() == VECTOR_IOTA_COUNT - 1,
+ "unexpected extra entry count %d", entries.length());
+ StubRoutines::x86::_vector_iota_indices[0] = start;
+ for (int i = 1; i < VECTOR_IOTA_COUNT; i++) {
+ StubRoutines::x86::_vector_iota_indices[i] = entries.at(i - 1);
+ }
+ return;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
// B
__ emit_data64(0x0706050403020100, relocInfo::none);
__ emit_data64(0x0F0E0D0C0B0A0908, relocInfo::none);
@@ -800,6 +920,7 @@ address StubGenerator::generate_iota_indices() {
__ emit_data64(0x2F2E2D2C2B2A2928, relocInfo::none);
__ emit_data64(0x3736353433323130, relocInfo::none);
__ emit_data64(0x3F3E3D3C3B3A3938, relocInfo::none);
+ entries.append(__ pc());
// W
__ emit_data64(0x0003000200010000, relocInfo::none);
__ emit_data64(0x0007000600050004, relocInfo::none);
@@ -809,6 +930,7 @@ address StubGenerator::generate_iota_indices() {
__ emit_data64(0x0017001600150014, relocInfo::none);
__ emit_data64(0x001B001A00190018, relocInfo::none);
__ emit_data64(0x001F001E001D001C, relocInfo::none);
+ entries.append(__ pc());
// D
__ emit_data64(0x0000000100000000, relocInfo::none);
__ emit_data64(0x0000000300000002, relocInfo::none);
@@ -818,6 +940,7 @@ address StubGenerator::generate_iota_indices() {
__ emit_data64(0x0000000B0000000A, relocInfo::none);
__ emit_data64(0x0000000D0000000C, relocInfo::none);
__ emit_data64(0x0000000F0000000E, relocInfo::none);
+ entries.append(__ pc());
// Q
__ emit_data64(0x0000000000000000, relocInfo::none);
__ emit_data64(0x0000000000000001, relocInfo::none);
@@ -827,6 +950,7 @@ address StubGenerator::generate_iota_indices() {
__ emit_data64(0x0000000000000005, relocInfo::none);
__ emit_data64(0x0000000000000006, relocInfo::none);
__ emit_data64(0x0000000000000007, relocInfo::none);
+ entries.append(__ pc());
// D - FP
__ emit_data64(0x3F80000000000000, relocInfo::none); // 0.0f, 1.0f
__ emit_data64(0x4040000040000000, relocInfo::none); // 2.0f, 3.0f
@@ -836,6 +960,7 @@ address StubGenerator::generate_iota_indices() {
__ emit_data64(0x4130000041200000, relocInfo::none); // 10.0f, 11.0f
__ emit_data64(0x4150000041400000, relocInfo::none); // 12.0f, 13.0f
__ emit_data64(0x4170000041600000, relocInfo::none); // 14.0f, 15.0f
+ entries.append(__ pc());
// Q - FP
__ emit_data64(0x0000000000000000, relocInfo::none); // 0.0d
__ emit_data64(0x3FF0000000000000, relocInfo::none); // 1.0d
@@ -845,14 +970,30 @@ address StubGenerator::generate_iota_indices() {
__ emit_data64(0x4014000000000000, relocInfo::none); // 5.0d
__ emit_data64(0x4018000000000000, relocInfo::none); // 6.0d
__ emit_data64(0x401c000000000000, relocInfo::none); // 7.0d
- return start;
+
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc(), &entries);
+
+ // install the entry addresses in the entry array
+ assert(entries.length() == entry_count - 1,
+ "unexpected entries count %d", entries.length());
+ StubRoutines::x86::_vector_iota_indices[0] = start;
+ for (int i = 1; i < VECTOR_IOTA_COUNT; i++) {
+ StubRoutines::x86::_vector_iota_indices[i] = entries.at(i - 1);
+ }
}
address StubGenerator::generate_vector_reverse_bit_lut() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_vector_reverse_bit_lut_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ emit_data64(0x0E060A020C040800, relocInfo::none);
__ emit_data64(0x0F070B030D050901, relocInfo::none);
@@ -863,14 +1004,23 @@ address StubGenerator::generate_vector_reverse_bit_lut() {
__ emit_data64(0x0E060A020C040800, relocInfo::none);
__ emit_data64(0x0F070B030D050901, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_vector_reverse_byte_perm_mask_long() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_vector_reverse_byte_perm_mask_long_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ emit_data64(0x0001020304050607, relocInfo::none);
__ emit_data64(0x08090A0B0C0D0E0F, relocInfo::none);
@@ -881,14 +1031,23 @@ address StubGenerator::generate_vector_reverse_byte_perm_mask_long() {
__ emit_data64(0x0001020304050607, relocInfo::none);
__ emit_data64(0x08090A0B0C0D0E0F, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_vector_reverse_byte_perm_mask_int() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_vector_reverse_byte_perm_mask_int_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ emit_data64(0x0405060700010203, relocInfo::none);
__ emit_data64(0x0C0D0E0F08090A0B, relocInfo::none);
@@ -899,14 +1058,23 @@ address StubGenerator::generate_vector_reverse_byte_perm_mask_int() {
__ emit_data64(0x0405060700010203, relocInfo::none);
__ emit_data64(0x0C0D0E0F08090A0B, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_vector_reverse_byte_perm_mask_short() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_vector_reverse_byte_perm_mask_short_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ emit_data64(0x0607040502030001, relocInfo::none);
__ emit_data64(0x0E0F0C0D0A0B0809, relocInfo::none);
@@ -917,31 +1085,52 @@ address StubGenerator::generate_vector_reverse_byte_perm_mask_short() {
__ emit_data64(0x0607040502030001, relocInfo::none);
__ emit_data64(0x0E0F0C0D0A0B0809, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_vector_byte_shuffle_mask() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_vector_byte_shuffle_mask_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ emit_data64(0x7070707070707070, relocInfo::none);
__ emit_data64(0x7070707070707070, relocInfo::none);
__ emit_data64(0xF0F0F0F0F0F0F0F0, relocInfo::none);
__ emit_data64(0xF0F0F0F0F0F0F0F0, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_fp_mask(StubId stub_id, int64_t mask) {
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ emit_data64( mask, relocInfo::none );
__ emit_data64( mask, relocInfo::none );
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -957,9 +1146,15 @@ address StubGenerator::generate_compress_perm_table(StubId stub_id) {
default:
ShouldNotReachHere();
}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
if (esize == 32) {
// Loop to generate 256 x 8 int compression permute index table. A row is
// accessed using 8 bit index computed using vector mask. An entry in
@@ -997,6 +1192,9 @@ address StubGenerator::generate_compress_perm_table(StubId stub_id) {
}
}
}
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -1012,9 +1210,15 @@ address StubGenerator::generate_expand_perm_table(StubId stub_id) {
default:
ShouldNotReachHere();
}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
if (esize == 32) {
// Loop to generate 256 x 8 int expand permute index table. A row is accessed
// using 8 bit index computed using vector mask. An entry in a row holds either
@@ -1050,13 +1254,22 @@ address StubGenerator::generate_expand_perm_table(StubId stub_id) {
}
}
}
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_vector_mask(StubId stub_id, int64_t mask) {
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ emit_data64(mask, relocInfo::none);
__ emit_data64(mask, relocInfo::none);
@@ -1067,14 +1280,23 @@ address StubGenerator::generate_vector_mask(StubId stub_id, int64_t mask) {
__ emit_data64(mask, relocInfo::none);
__ emit_data64(mask, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_vector_byte_perm_mask() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_vector_byte_perm_mask_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ emit_data64(0x0000000000000001, relocInfo::none);
__ emit_data64(0x0000000000000003, relocInfo::none);
@@ -1085,13 +1307,22 @@ address StubGenerator::generate_vector_byte_perm_mask() {
__ emit_data64(0x0000000000000004, relocInfo::none);
__ emit_data64(0x0000000000000006, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_vector_fp_mask(StubId stub_id, int64_t mask) {
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ emit_data64(mask, relocInfo::none);
__ emit_data64(mask, relocInfo::none);
@@ -1102,6 +1333,9 @@ address StubGenerator::generate_vector_fp_mask(StubId stub_id, int64_t mask) {
__ emit_data64(mask, relocInfo::none);
__ emit_data64(mask, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -1110,9 +1344,15 @@ address StubGenerator::generate_vector_custom_i32(StubId stub_id, Assembler::Avx
int32_t val4, int32_t val5, int32_t val6, int32_t val7,
int32_t val8, int32_t val9, int32_t val10, int32_t val11,
int32_t val12, int32_t val13, int32_t val14, int32_t val15) {
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
assert(len != Assembler::AVX_NoVec, "vector len must be specified");
__ emit_data(val0, relocInfo::none, 0);
@@ -1135,6 +1375,9 @@ address StubGenerator::generate_vector_custom_i32(StubId stub_id, Assembler::Avx
__ emit_data(val15, relocInfo::none, 0);
}
}
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -1156,8 +1399,14 @@ address StubGenerator::generate_vector_custom_i32(StubId stub_id, Assembler::Avx
// * = popped on exit
address StubGenerator::generate_verify_oop() {
StubId stub_id = StubId::stubgen_verify_oop_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label exit, error;
@@ -1235,6 +1484,9 @@ address StubGenerator::generate_verify_oop() {
__ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)));
__ hlt();
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -1350,35 +1602,46 @@ void StubGenerator::restore_argument_regs(BasicType type) {
address StubGenerator::generate_data_cache_writeback() {
const Register src = c_rarg0; // source address
-
- __ align(CodeEntryAlignment);
-
StubId stub_id = StubId::stubgen_data_cache_writeback_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
__ cache_wb(Address(src, 0));
__ leave();
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_data_cache_writeback_sync() {
const Register is_pre = c_rarg0; // pre or post sync
-
- __ align(CodeEntryAlignment);
-
StubId stub_id = StubId::stubgen_data_cache_writeback_sync_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
// pre wbsync is a no-op
// post wbsync translates to an sfence
Label skip;
- address start = __ pc();
+ start = __ pc();
__ enter();
__ cmpl(is_pre, 0);
@@ -1388,6 +1651,9 @@ address StubGenerator::generate_data_cache_writeback_sync() {
__ leave();
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -1405,9 +1671,15 @@ address StubGenerator::generate_md5_implCompress(StubId stub_id) {
default:
ShouldNotReachHere();
}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register buf_param = r15;
const Address state_param(rsp, 0 * wordSize);
@@ -1437,30 +1709,51 @@ address StubGenerator::generate_md5_implCompress(StubId stub_id) {
__ leave();
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_upper_word_mask() {
- __ align64();
StubId stub_id = StubId::stubgen_upper_word_mask_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align64();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ emit_data64(0x0000000000000000, relocInfo::none);
__ emit_data64(0xFFFFFFFF00000000, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_shuffle_byte_flip_mask() {
- __ align64();
StubId stub_id = StubId::stubgen_shuffle_byte_flip_mask_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align64();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ emit_data64(0x08090a0b0c0d0e0f, relocInfo::none);
__ emit_data64(0x0001020304050607, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -1478,9 +1771,15 @@ address StubGenerator::generate_sha1_implCompress(StubId stub_id) {
default:
ShouldNotReachHere();
}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Register buf = c_rarg0;
Register state = c_rarg1;
@@ -1509,15 +1808,32 @@ address StubGenerator::generate_sha1_implCompress(StubId stub_id) {
__ leave();
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
-address StubGenerator::generate_pshuffle_byte_flip_mask() {
- __ align64();
+address StubGenerator::generate_pshuffle_byte_flip_mask(address& entry_00ba, address& entry_dc00) {
StubId stub_id = StubId::stubgen_pshuffle_byte_flip_mask_id;
+ GrowableArray entries;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 3, "sanity check");
+ address start = load_archive_data(stub_id, &entries);
+ if (start != nullptr) {
+ assert(entries.length() == entry_count - 1,
+ "unexpected extra entry count %d", entries.length());
+ entry_00ba = entries.at(0);
+ entry_dc00 = entries.at(1);
+ assert(VM_Version::supports_avx2() == (entry_00ba != nullptr && entry_dc00 != nullptr),
+ "entries cannot be null when avx2 is enabled");
+ return start;
+ }
+ __ align64();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
-
+ start = __ pc();
+ address entry2 = nullptr;
+ address entry3 = nullptr;
__ emit_data64(0x0405060700010203, relocInfo::none);
__ emit_data64(0x0c0d0e0f08090a0b, relocInfo::none);
@@ -1525,37 +1841,66 @@ address StubGenerator::generate_pshuffle_byte_flip_mask() {
__ emit_data64(0x0405060700010203, relocInfo::none); // second copy
__ emit_data64(0x0c0d0e0f08090a0b, relocInfo::none);
// _SHUF_00BA
+ entry2 = __ pc();
__ emit_data64(0x0b0a090803020100, relocInfo::none);
__ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
__ emit_data64(0x0b0a090803020100, relocInfo::none);
__ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
// _SHUF_DC00
+ entry3 = __ pc();
__ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
__ emit_data64(0x0b0a090803020100, relocInfo::none);
__ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
__ emit_data64(0x0b0a090803020100, relocInfo::none);
}
+ // have to track the 2nd and 3rd entries even if they are null
+ entry_00ba = entry2;
+ entries.push(entry_00ba);
+ entry_dc00 = entry3;
+ entries.push(entry_dc00);
+
+ // record the stub entry and end plus all the auxiliary entries
+ store_archive_data(stub_id, start, __ pc(), &entries);
return start;
}
//Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb.
-address StubGenerator::generate_pshuffle_byte_flip_mask_sha512() {
- __ align32();
+address StubGenerator::generate_pshuffle_byte_flip_mask_sha512(address& entry_ymm_lo) {
StubId stub_id = StubId::stubgen_pshuffle_byte_flip_mask_sha512_id;
+ GrowableArray entries;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 2, "sanity check");
+ address start = load_archive_data(stub_id, &entries);
+ if (start != nullptr) {
+ assert(entries.length() == entry_count - 1,
+ "unexpected extra entry count %d", entries.length());
+ entry_ymm_lo = entries.at(0);
+ assert(VM_Version::supports_avx2() == (entry_ymm_lo != nullptr),
+ "entry cannot be null when avx2 is enabled");
+ return start;
+ }
+ __ align32();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
-
+ start = __ pc();
+ address entry2 = nullptr;
if (VM_Version::supports_avx2()) {
__ emit_data64(0x0001020304050607, relocInfo::none); // PSHUFFLE_BYTE_FLIP_MASK
__ emit_data64(0x08090a0b0c0d0e0f, relocInfo::none);
__ emit_data64(0x1011121314151617, relocInfo::none);
__ emit_data64(0x18191a1b1c1d1e1f, relocInfo::none);
+ // capture 2nd entry
+ entry2 = __ pc();
__ emit_data64(0x0000000000000000, relocInfo::none); //MASK_YMM_LO
__ emit_data64(0x0000000000000000, relocInfo::none);
__ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
__ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
}
+ // have to track the 2nd entry even if it is null
+ entry_ymm_lo = entry2;
+ entries.push(entry2);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc(), &entries);
return start;
}
@@ -1575,9 +1920,15 @@ address StubGenerator::generate_sha256_implCompress(StubId stub_id) {
ShouldNotReachHere();
}
assert(VM_Version::supports_sha() || VM_Version::supports_avx2(), "");
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Register buf = c_rarg0;
Register state = c_rarg1;
@@ -1612,6 +1963,9 @@ address StubGenerator::generate_sha256_implCompress(StubId stub_id) {
__ leave();
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -1629,9 +1983,15 @@ address StubGenerator::generate_sha512_implCompress(StubId stub_id) {
}
assert(VM_Version::supports_avx2(), "");
assert(VM_Version::supports_bmi2() || VM_Version::supports_sha512(), "");
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Register buf = c_rarg0;
Register state = c_rarg1;
@@ -1660,14 +2020,23 @@ address StubGenerator::generate_sha512_implCompress(StubId stub_id) {
__ leave();
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::base64_shuffle_addr() {
- __ align64();
StubId stub_id = StubId::stubgen_shuffle_base64_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align64();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
assert(((unsigned long long)start & 0x3f) == 0,
"Alignment problem (0x%08llx)", (unsigned long long)start);
@@ -1680,42 +2049,69 @@ address StubGenerator::base64_shuffle_addr() {
__ emit_data64(0x2829272825262425, relocInfo::none);
__ emit_data64(0x2e2f2d2e2b2c2a2b, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::base64_avx2_shuffle_addr() {
- __ align32();
StubId stub_id = StubId::stubgen_avx2_shuffle_base64_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align32();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ emit_data64(0x0809070805060405, relocInfo::none);
__ emit_data64(0x0e0f0d0e0b0c0a0b, relocInfo::none);
__ emit_data64(0x0405030401020001, relocInfo::none);
__ emit_data64(0x0a0b090a07080607, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::base64_avx2_input_mask_addr() {
- __ align32();
StubId stub_id = StubId::stubgen_avx2_input_mask_base64_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align32();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ emit_data64(0x8000000000000000, relocInfo::none);
__ emit_data64(0x8000000080000000, relocInfo::none);
__ emit_data64(0x8000000080000000, relocInfo::none);
__ emit_data64(0x8000000080000000, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::base64_avx2_lut_addr() {
- __ align32();
StubId stub_id = StubId::stubgen_avx2_lut_base64_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align32();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ emit_data64(0xfcfcfcfcfcfc4741, relocInfo::none);
__ emit_data64(0x0000f0edfcfcfcfc, relocInfo::none);
@@ -1728,14 +2124,23 @@ address StubGenerator::base64_avx2_lut_addr() {
__ emit_data64(0xfcfcfcfcfcfc4741, relocInfo::none);
__ emit_data64(0x000020effcfcfcfc, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::base64_encoding_table_addr() {
- __ align64();
StubId stub_id = StubId::stubgen_encoding_table_base64_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align64();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
assert(((unsigned long long)start & 0x3f) == 0, "Alignment problem (0x%08llx)", (unsigned long long)start);
__ emit_data64(0x4847464544434241, relocInfo::none);
@@ -1757,6 +2162,9 @@ address StubGenerator::base64_encoding_table_addr() {
__ emit_data64(0x333231307a797877, relocInfo::none);
__ emit_data64(0x5f2d393837363534, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -1766,10 +2174,16 @@ address StubGenerator::base64_encoding_table_addr() {
// boolean isURL) {
address StubGenerator::generate_base64_encodeBlock()
{
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_base64_encodeBlock_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
@@ -2144,15 +2558,24 @@ address StubGenerator::generate_base64_encodeBlock()
__ leave();
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
// base64 AVX512vbmi tables
address StubGenerator::base64_vbmi_lookup_lo_addr() {
- __ align64();
StubId stub_id = StubId::stubgen_lookup_lo_base64_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align64();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
assert(((unsigned long long)start & 0x3f) == 0,
"Alignment problem (0x%08llx)", (unsigned long long)start);
@@ -2165,14 +2588,23 @@ address StubGenerator::base64_vbmi_lookup_lo_addr() {
__ emit_data64(0x3b3a393837363534, relocInfo::none);
__ emit_data64(0x8080808080803d3c, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::base64_vbmi_lookup_hi_addr() {
- __ align64();
StubId stub_id = StubId::stubgen_lookup_hi_base64_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align64();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
assert(((unsigned long long)start & 0x3f) == 0,
"Alignment problem (0x%08llx)", (unsigned long long)start);
@@ -2185,13 +2617,22 @@ address StubGenerator::base64_vbmi_lookup_hi_addr() {
__ emit_data64(0x302f2e2d2c2b2a29, relocInfo::none);
__ emit_data64(0x8080808080333231, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::base64_vbmi_lookup_lo_url_addr() {
- __ align64();
StubId stub_id = StubId::stubgen_lookup_lo_base64url_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align64();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
assert(((unsigned long long)start & 0x3f) == 0,
"Alignment problem (0x%08llx)", (unsigned long long)start);
@@ -2204,14 +2645,23 @@ address StubGenerator::base64_vbmi_lookup_lo_url_addr() {
__ emit_data64(0x3b3a393837363534, relocInfo::none);
__ emit_data64(0x8080808080803d3c, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::base64_vbmi_lookup_hi_url_addr() {
- __ align64();
StubId stub_id = StubId::stubgen_lookup_hi_base64url_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align64();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
assert(((unsigned long long)start & 0x3f) == 0,
"Alignment problem (0x%08llx)", (unsigned long long)start);
@@ -2224,14 +2674,23 @@ address StubGenerator::base64_vbmi_lookup_hi_url_addr() {
__ emit_data64(0x302f2e2d2c2b2a29, relocInfo::none);
__ emit_data64(0x8080808080333231, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::base64_vbmi_pack_vec_addr() {
- __ align64();
StubId stub_id = StubId::stubgen_pack_vec_base64_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align64();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
assert(((unsigned long long)start & 0x3f) == 0,
"Alignment problem (0x%08llx)", (unsigned long long)start);
@@ -2244,14 +2703,23 @@ address StubGenerator::base64_vbmi_pack_vec_addr() {
__ emit_data64(0x0000000000000000, relocInfo::none);
__ emit_data64(0x0000000000000000, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::base64_vbmi_join_0_1_addr() {
- __ align64();
StubId stub_id = StubId::stubgen_join_0_1_base64_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align64();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
assert(((unsigned long long)start & 0x3f) == 0,
"Alignment problem (0x%08llx)", (unsigned long long)start);
@@ -2264,14 +2732,23 @@ address StubGenerator::base64_vbmi_join_0_1_addr() {
__ emit_data64(0x494a444546404142, relocInfo::none);
__ emit_data64(0x565051524c4d4e48, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::base64_vbmi_join_1_2_addr() {
- __ align64();
StubId stub_id = StubId::stubgen_join_1_2_base64_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align64();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
assert(((unsigned long long)start & 0x3f) == 0,
"Alignment problem (0x%08llx)", (unsigned long long)start);
@@ -2284,14 +2761,23 @@ address StubGenerator::base64_vbmi_join_1_2_addr() {
__ emit_data64(0x5c5d5e58595a5455, relocInfo::none);
__ emit_data64(0x696a646566606162, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::base64_vbmi_join_2_3_addr() {
- __ align64();
StubId stub_id = StubId::stubgen_join_2_3_base64_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align64();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
assert(((unsigned long long)start & 0x3f) == 0,
"Alignment problem (0x%08llx)", (unsigned long long)start);
@@ -2304,14 +2790,23 @@ address StubGenerator::base64_vbmi_join_2_3_addr() {
__ emit_data64(0x767071726c6d6e68, relocInfo::none);
__ emit_data64(0x7c7d7e78797a7475, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::base64_AVX2_decode_tables_addr() {
- __ align64();
StubId stub_id = StubId::stubgen_avx2_decode_tables_base64_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align64();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
assert(((unsigned long long)start & 0x3f) == 0,
"Alignment problem (0x%08llx)", (unsigned long long)start);
@@ -2339,14 +2834,23 @@ address StubGenerator::base64_AVX2_decode_tables_addr() {
// merge multiplier
__ emit_data(0x00011000, relocInfo::none, 0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::base64_AVX2_decode_LUT_tables_addr() {
- __ align64();
StubId stub_id = StubId::stubgen_avx2_decode_lut_tables_base64_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align64();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
assert(((unsigned long long)start & 0x3f) == 0,
"Alignment problem (0x%08llx)", (unsigned long long)start);
@@ -2380,13 +2884,22 @@ address StubGenerator::base64_AVX2_decode_LUT_tables_addr() {
__ emit_data64(0x0804080402011010, relocInfo::none);
__ emit_data64(0x1010101010101010, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::base64_decoding_table_addr() {
StubId stub_id = StubId::stubgen_decoding_table_base64_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ emit_data64(0xffffffffffffffff, relocInfo::none);
__ emit_data64(0xffffffffffffffff, relocInfo::none);
@@ -2455,6 +2968,9 @@ address StubGenerator::base64_decoding_table_addr() {
__ emit_data64(0xffffffffffffffff, relocInfo::none);
__ emit_data64(0xffffffffffffffff, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -2466,10 +2982,16 @@ address StubGenerator::base64_decoding_table_addr() {
// Intrinsic function prototype in Base64.java:
// private void decodeBlock(byte[] src, int sp, int sl, byte[] dst, int dp, boolean isURL, isMIME) {
address StubGenerator::generate_base64_decodeBlock() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_base64_decodeBlock_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
@@ -2982,6 +3504,9 @@ address StubGenerator::generate_base64_decodeBlock() {
__ leave();
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3000,11 +3525,17 @@ address StubGenerator::generate_base64_decodeBlock() {
address StubGenerator::generate_updateBytesCRC32() {
assert(UseCRC32Intrinsics, "need AVX and CLMUL instructions");
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_updateBytesCRC32_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
// Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
// Unix: rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...)
@@ -3039,6 +3570,9 @@ address StubGenerator::generate_updateBytesCRC32() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3057,10 +3591,16 @@ address StubGenerator::generate_updateBytesCRC32() {
*/
address StubGenerator::generate_updateBytesCRC32C(bool is_pclmulqdq_supported) {
assert(UseCRC32CIntrinsics, "need SSE4_2");
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_updateBytesCRC32C_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
//reg.arg int#0 int#1 int#2 int#3 int#4 int#5 float regs
//Windows RCX RDX R8 R9 none none XMM0..XMM3
@@ -3120,6 +3660,9 @@ address StubGenerator::generate_updateBytesCRC32C(bool is_pclmulqdq_supported) {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3138,10 +3681,16 @@ address StubGenerator::generate_updateBytesCRC32C(bool is_pclmulqdq_supported) {
* rsp+40 - z address
*/
address StubGenerator::generate_multiplyToLen() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_multiplyToLen_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
// Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
// Unix: rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...)
@@ -3179,6 +3728,9 @@ address StubGenerator::generate_multiplyToLen() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3195,10 +3747,16 @@ address StubGenerator::generate_multiplyToLen() {
* rax - int >= mismatched index, < 0 bitwise complement of tail
*/
address StubGenerator::generate_vectorizedMismatch() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_vectorizedMismatch_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
BLOCK_COMMENT("Entry:");
__ enter();
@@ -3232,6 +3790,9 @@ address StubGenerator::generate_vectorizedMismatch() {
__ leave();
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3247,10 +3808,16 @@ address StubGenerator::generate_vectorizedMismatch() {
*/
address StubGenerator::generate_squareToLen() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_squareToLen_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
// Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
// Unix: rdi, rsi, rdx, rcx (c_rarg0, c_rarg1, ...)
@@ -3279,14 +3846,23 @@ address StubGenerator::generate_squareToLen() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_method_entry_barrier() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_method_entry_barrier_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label deoptimize_label;
@@ -3356,6 +3932,9 @@ address StubGenerator::generate_method_entry_barrier() {
__ movptr(rsp, Address(rsp, 0)); // new rsp was written in the barrier
__ jmp(Address(rsp, -1 * wordSize)); // jmp target should be callers verified_entry_point
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3373,10 +3952,16 @@ address StubGenerator::generate_method_entry_barrier() {
* rsp+40 - k
*/
address StubGenerator::generate_mulAdd() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_mulAdd_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
// Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
// Unix: rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...)
@@ -3411,14 +3996,23 @@ address StubGenerator::generate_mulAdd() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_bigIntegerRightShift() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_bigIntegerRightShiftWorker_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label Shift512Loop, ShiftTwo, ShiftTwoLoop, ShiftOne, Exit;
// For Unix, the arguments are as follows: rdi, rsi, rdx, rcx, r8.
@@ -3534,6 +4128,9 @@ address StubGenerator::generate_bigIntegerRightShift() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3551,10 +4148,16 @@ address StubGenerator::generate_bigIntegerRightShift() {
* rsp40 - numIter
*/
address StubGenerator::generate_bigIntegerLeftShift() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_bigIntegerLeftShiftWorker_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label Shift512Loop, ShiftTwo, ShiftTwoLoop, ShiftOne, Exit;
// For Unix, the arguments are as follows: rdi, rsi, rdx, rcx, r8.
@@ -3659,6 +4262,9 @@ address StubGenerator::generate_bigIntegerLeftShift() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3708,9 +4314,15 @@ void StubGenerator::generate_libm_stubs() {
*/
address StubGenerator::generate_float16ToFloat() {
StubId stub_id = StubId::stubgen_hf2f_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
BLOCK_COMMENT("Entry:");
// No need for RuntimeStub frame since it is called only during JIT compilation
@@ -3720,6 +4332,9 @@ address StubGenerator::generate_float16ToFloat() {
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3734,9 +4349,15 @@ address StubGenerator::generate_float16ToFloat() {
*/
address StubGenerator::generate_floatToFloat16() {
StubId stub_id = StubId::stubgen_f2hf_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
BLOCK_COMMENT("Entry:");
// No need for RuntimeStub frame since it is called only during JIT compilation
@@ -3746,6 +4367,9 @@ address StubGenerator::generate_floatToFloat16() {
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3775,8 +4399,14 @@ address StubGenerator::generate_cont_thaw(StubId stub_id) {
default:
ShouldNotReachHere();
}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
// TODO: Handle Valhalla return types. May require generating different return barriers.
@@ -3889,6 +4519,9 @@ address StubGenerator::generate_cont_thaw(StubId stub_id) {
__ ret(0);
}
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3909,8 +4542,14 @@ address StubGenerator::generate_cont_returnBarrier_exception() {
address StubGenerator::generate_cont_preempt_stub() {
if (!Continuations::enabled()) return nullptr;
StubId stub_id = StubId::stubgen_cont_preempt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ reset_last_Java_frame(true);
@@ -3934,14 +4573,23 @@ address StubGenerator::generate_cont_preempt_stub() {
__ movptr(rscratch1, ExternalAddress(ContinuationEntry::thaw_call_pc_address()));
__ jmp(rscratch1);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
// exception handler for upcall stubs
address StubGenerator::generate_upcall_stub_exception_handler() {
StubId stub_id = StubId::stubgen_upcall_stub_exception_handler_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
// native caller has no idea how to handle exceptions
// we just crash here. Up to callee to catch exceptions.
@@ -3953,6 +4601,9 @@ address StubGenerator::generate_upcall_stub_exception_handler() {
__ call(RuntimeAddress(CAST_FROM_FN_PTR(address, UpcallLinker::handle_uncaught_exception)));
__ should_not_reach_here();
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3961,8 +4612,14 @@ address StubGenerator::generate_upcall_stub_exception_handler() {
// rbx = result
address StubGenerator::generate_upcall_stub_load_target() {
StubId stub_id = StubId::stubgen_upcall_stub_load_target_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ resolve_global_jobject(j_rarg0, rscratch1);
// Load target method from receiver
@@ -3976,11 +4633,27 @@ address StubGenerator::generate_upcall_stub_load_target() {
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
void StubGenerator::generate_lookup_secondary_supers_table_stub() {
StubId stub_id = StubId::stubgen_lookup_secondary_supers_table_id;
+ GrowableArray entries;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == Klass::SECONDARY_SUPERS_TABLE_SIZE, "sanity check");
+ address start = load_archive_data(stub_id, &entries);
+ if (start != nullptr) {
+ assert(entries.length() == Klass::SECONDARY_SUPERS_TABLE_SIZE - 1,
+ "unexpected extra entry count %d", entries.length());
+ StubRoutines::_lookup_secondary_supers_table_stubs[0] = start;
+ for (int slot = 1; slot < Klass::SECONDARY_SUPERS_TABLE_SIZE; slot++) {
+ StubRoutines::_lookup_secondary_supers_table_stubs[slot] = entries.at(slot - 1);
+ }
+ return;
+ }
StubCodeMark mark(this, stub_id);
const Register
@@ -3989,21 +4662,35 @@ void StubGenerator::generate_lookup_secondary_supers_table_stub() {
result = rdi;
for (int slot = 0; slot < Klass::SECONDARY_SUPERS_TABLE_SIZE; slot++) {
- StubRoutines::_lookup_secondary_supers_table_stubs[slot] = __ pc();
+ address next_entry = __ pc();
+ if (slot == 0) {
+ start = next_entry;
+ } else {
+ entries.append(next_entry);
+ }
+ StubRoutines::_lookup_secondary_supers_table_stubs[slot] = next_entry;
__ lookup_secondary_supers_table_const(r_sub_klass, r_super_klass,
rdx, rcx, rbx, r11, // temps
result,
slot);
__ ret(0);
}
+
+ // record the stub entry and end plus all the auxiliary entries
+ store_archive_data(stub_id, start, __ pc(), &entries);
}
// Slow path implementation for UseSecondarySupersTable.
address StubGenerator::generate_lookup_secondary_supers_table_slow_path_stub() {
StubId stub_id = StubId::stubgen_lookup_secondary_supers_table_slow_path_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
-
- address start = __ pc();
+ start = __ pc();
const Register
r_super_klass = rax,
@@ -4025,6 +4712,9 @@ address StubGenerator::generate_lookup_secondary_supers_table_slow_path_stub() {
__ movl(result, 0);
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -4165,7 +4855,7 @@ void StubGenerator::generate_compiler_stubs() {
StubRoutines::x86::_vector_short_shuffle_mask = generate_vector_mask(StubId::stubgen_vector_short_shuffle_mask_id, 0x0100010001000100);
StubRoutines::x86::_vector_long_shuffle_mask = generate_vector_mask(StubId::stubgen_vector_long_shuffle_mask_id, 0x0000000100000000);
StubRoutines::x86::_vector_long_sign_mask = generate_vector_mask(StubId::stubgen_vector_long_sign_mask_id, 0x8000000000000000);
- StubRoutines::x86::_vector_iota_indices = generate_iota_indices();
+ generate_iota_indices();
StubRoutines::x86::_vector_count_leading_zeros_lut = generate_count_leading_zeros_lut();
StubRoutines::x86::_vector_reverse_bit_lut = generate_vector_reverse_bit_lut();
StubRoutines::x86::_vector_reverse_byte_perm_mask_long = generate_vector_reverse_byte_perm_mask_long();
@@ -4232,6 +4922,8 @@ void StubGenerator::generate_compiler_stubs() {
}
if (UseSHA256Intrinsics) {
+ address entry2 = nullptr;
+ address entry3 = nullptr;
StubRoutines::x86::_k256_adr = (address)StubRoutines::x86::_k256;
char* dst = (char*)StubRoutines::x86::_k256_W;
char* src = (char*)StubRoutines::x86::_k256;
@@ -4240,14 +4932,18 @@ void StubGenerator::generate_compiler_stubs() {
memcpy(dst + 32 * ii + 16, src + 16 * ii, 16);
}
StubRoutines::x86::_k256_W_adr = (address)StubRoutines::x86::_k256_W;
- StubRoutines::x86::_pshuffle_byte_flip_mask_addr = generate_pshuffle_byte_flip_mask();
+ StubRoutines::x86::_pshuffle_byte_flip_mask_addr = generate_pshuffle_byte_flip_mask(entry2, entry3);
+ StubRoutines::x86::_pshuffle_byte_flip_mask_00ba_addr = entry2;
+ StubRoutines::x86::_pshuffle_byte_flip_mask_dc00_addr = entry3;
StubRoutines::_sha256_implCompress = generate_sha256_implCompress(StubId::stubgen_sha256_implCompress_id);
StubRoutines::_sha256_implCompressMB = generate_sha256_implCompress(StubId::stubgen_sha256_implCompressMB_id);
}
if (UseSHA512Intrinsics) {
+ address entry2 = nullptr;
StubRoutines::x86::_k512_W_addr = (address)StubRoutines::x86::_k512_W;
- StubRoutines::x86::_pshuffle_byte_flip_mask_addr_sha512 = generate_pshuffle_byte_flip_mask_sha512();
+ StubRoutines::x86::_pshuffle_byte_flip_mask_addr_sha512 = generate_pshuffle_byte_flip_mask_sha512(entry2);
+ StubRoutines::x86::_pshuffle_byte_flip_mask_ymm_lo_addr_sha512 = entry2;
StubRoutines::_sha512_implCompress = generate_sha512_implCompress(StubId::stubgen_sha512_implCompress_id);
StubRoutines::_sha512_implCompressMB = generate_sha512_implCompress(StubId::stubgen_sha512_implCompressMB_id);
}
@@ -4325,7 +5021,7 @@ void StubGenerator::generate_compiler_stubs() {
#endif // COMPILER2_OR_JVMCI
}
-StubGenerator::StubGenerator(CodeBuffer* code, BlobId blob_id) : StubCodeGenerator(code, blob_id) {
+StubGenerator::StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) : StubCodeGenerator(code, blob_id, stub_data) {
switch(blob_id) {
case BlobId::stubgen_preuniverse_id:
generate_preuniverse_stubs();
@@ -4348,8 +5044,35 @@ StubGenerator::StubGenerator(CodeBuffer* code, BlobId blob_id) : StubCodeGenerat
};
}
-void StubGenerator_generate(CodeBuffer* code, BlobId blob_id) {
- StubGenerator g(code, blob_id);
+#if INCLUDE_CDS
+// publish addresses of static data defined in this file and in other
+// stubgen stub generator files
+void StubGenerator::init_AOTAddressTable(GrowableArray& external_addresses) {
+ init_AOTAddressTable_adler(external_addresses);
+ init_AOTAddressTable_aes(external_addresses);
+ init_AOTAddressTable_cbrt(external_addresses);
+ init_AOTAddressTable_chacha(external_addresses);
+ // constants publishes for all of address use by cos and almost all of sin
+ init_AOTAddressTable_constants(external_addresses);
+ init_AOTAddressTable_dilithium(external_addresses);
+ init_AOTAddressTable_exp(external_addresses);
+ init_AOTAddressTable_fmod(external_addresses);
+ init_AOTAddressTable_ghash(external_addresses);
+ init_AOTAddressTable_kyber(external_addresses);
+ init_AOTAddressTable_log(external_addresses);
+ init_AOTAddressTable_poly1305(external_addresses);
+ init_AOTAddressTable_poly_mont(external_addresses);
+ init_AOTAddressTable_pow(external_addresses);
+ init_AOTAddressTable_sha3(external_addresses);
+ init_AOTAddressTable_sin(external_addresses);
+ init_AOTAddressTable_sinh(external_addresses);
+ init_AOTAddressTable_tan(external_addresses);
+ init_AOTAddressTable_tanh(external_addresses);
+}
+#endif // INCLUDE_CDS
+
+void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) {
+ StubGenerator g(code, blob_id, stub_data);
}
#undef __
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp
index 36315535d16..d3823cb559f 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -84,7 +84,7 @@ class StubGenerator: public StubCodeGenerator {
address generate_count_leading_zeros_lut();
address generate_popcount_avx_lut();
- address generate_iota_indices();
+ void generate_iota_indices();
address generate_vector_reverse_bit_lut();
address generate_vector_reverse_byte_perm_mask_long();
@@ -166,12 +166,12 @@ class StubGenerator: public StubCodeGenerator {
// - If target supports AVX3 features (BW+VL+F) then implementation uses 32 byte vectors (YMMs)
// for both special cases (various small block sizes) and aligned copy loop. This is the
// default configuration.
- // - If copy length is above AVX3Threshold, then implementation use 64 byte vectors (ZMMs)
+ // - If copy length is above CopyAVX3Threshold, then implementation use 64 byte vectors (ZMMs)
// for main copy loop (and subsequent tail) since bulk of the cycles will be consumed in it.
// - If user forces MaxVectorSize=32 then above 4096 bytes its seen that REP MOVs shows a
// better performance for disjoint copies. For conjoint/backward copy vector based
// copy performs better.
- // - If user sets AVX3Threshold=0, then special cases for small blocks sizes operate over
+ // - If user sets CopyAVX3Threshold=0, then special cases for small blocks sizes operate over
// 64 byte vector registers (ZMMs).
address generate_disjoint_copy_avx3_masked(StubId stub_id, address* entry);
@@ -303,11 +303,11 @@ class StubGenerator: public StubCodeGenerator {
address generate_sha512_implCompress(StubId stub_id);
// Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb.
- address generate_pshuffle_byte_flip_mask_sha512();
+ address generate_pshuffle_byte_flip_mask_sha512(address& entry_ymm_lo);
address generate_upper_word_mask();
address generate_shuffle_byte_flip_mask();
- address generate_pshuffle_byte_flip_mask();
+ address generate_pshuffle_byte_flip_mask(address& entry_00ba, address& entry_dc0);
// AES intrinsic stubs
@@ -330,6 +330,19 @@ class StubGenerator: public StubCodeGenerator {
void aesecb_decrypt(Register source_addr, Register dest_addr, Register key, Register len);
+ // Shared implementation for ECB/AES Encrypt and Decrypt, which does 4 blocks
+ // in a loop at a time to hide instruction latency. Set is_encrypt=true for
+ // encryption, false for decryption.
+ address generate_electronicCodeBook_AESCrypt_Parallel(bool is_encrypt);
+
+ // A version of ECB/AES Encrypt which does 4 blocks in a loop at a time
+ // to hide instruction latency
+ address generate_electronicCodeBook_encryptAESCrypt_Parallel();
+
+ // A version of ECB/AES Decrypt which does 4 blocks in a loop at a time
+ // to hide instruction latency
+ address generate_electronicCodeBook_decryptAESCrypt_Parallel();
+
// Vector AES Galois Counter Mode implementation
address generate_galoisCounterMode_AESCrypt();
void aesgcm_encrypt(Register in, Register len, Register ct, Register out, Register key,
@@ -637,8 +650,33 @@ class StubGenerator: public StubCodeGenerator {
void generate_compiler_stubs();
void generate_final_stubs();
+#if INCLUDE_CDS
+ static void init_AOTAddressTable_adler(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_aes(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_cbrt(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_chacha(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_constants(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_dilithium(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_exp(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_fmod(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_ghash(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_kyber(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_log(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_poly1305(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_poly_mont(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_pow(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_sha3(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_sin(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_sinh(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_tan(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_tanh(GrowableArray& external_addresses);
+#endif // INCLUDE_CDS
+
public:
- StubGenerator(CodeBuffer* code, BlobId blob_id);
+ StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data);
+#if INCLUDE_CDS
+ static void init_AOTAddressTable(GrowableArray& external_addresses);
+#endif // INCLUDE_CDS
};
#endif // CPU_X86_STUBGENERATOR_X86_64_HPP
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_adler.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_adler.cpp
index 2799997a761..a9424978e0e 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_adler.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_adler.cpp
@@ -67,8 +67,14 @@ address StubGenerator::generate_updateBytesAdler32() {
__ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_updateBytesAdler32_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
// Choose an appropriate LIMIT for inner loop based on the granularity
// of intermediate results. For int, LIMIT of 5552 will ensure intermediate
@@ -144,7 +150,7 @@ address StubGenerator::generate_updateBytesAdler32() {
__ align32();
if (VM_Version::supports_avx512vl()) {
// AVX2 performs better for smaller inputs because of leaner post loop reduction sequence..
- __ cmpl(s, MAX2(128, VM_Version::avx3_threshold()));
+ __ cmpl(s, MAX2(128, CopyAVX3Threshold));
__ jcc(Assembler::belowEqual, SLOOP1A_AVX2);
__ lea(end, Address(s, data, Address::times_1, - (2*CHUNKSIZE -1)));
@@ -334,7 +340,19 @@ address StubGenerator::generate_updateBytesAdler32() {
__ leave();
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
return start;
}
#undef __
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_adler(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr))
+ ADD(ADLER32_ASCALE_TABLE);
+ ADD(ADLER32_SHUF0_TABLE);
+ ADD(ADLER32_SHUF1_TABLE);
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_aes.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_aes.cpp
index 24de32a6fe7..b95aa5f8818 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_aes.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_aes.cpp
@@ -1,5 +1,5 @@
/*
-* Copyright (c) 2019, 2025, Intel Corporation. All rights reserved.
+* Copyright (c) 2019, 2026, Intel Corporation. All rights reserved.
*
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -218,7 +218,9 @@ void StubGenerator::generate_aes_stubs() {
StubRoutines::_galoisCounterMode_AESCrypt = generate_galoisCounterMode_AESCrypt();
} else {
StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt_Parallel();
- if (VM_Version::supports_avx2()) {
+ StubRoutines::_electronicCodeBook_encryptAESCrypt = generate_electronicCodeBook_encryptAESCrypt_Parallel();
+ StubRoutines::_electronicCodeBook_decryptAESCrypt = generate_electronicCodeBook_decryptAESCrypt_Parallel();
+ if (VM_Version::supports_avx2() && VM_Version::supports_clmul()) {
StubRoutines::_galoisCounterMode_AESCrypt = generate_avx2_galoisCounterMode_AESCrypt();
}
}
@@ -248,10 +250,16 @@ void StubGenerator::generate_aes_stubs() {
// Output:
// rax - number of processed bytes
address StubGenerator::generate_galoisCounterMode_AESCrypt() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_galoisCounterMode_AESCrypt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register in = c_rarg0;
const Register len = c_rarg1;
@@ -317,6 +325,9 @@ address StubGenerator::generate_galoisCounterMode_AESCrypt() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -335,10 +346,16 @@ address StubGenerator::generate_galoisCounterMode_AESCrypt() {
// Output:
// rax - number of processed bytes
address StubGenerator::generate_avx2_galoisCounterMode_AESCrypt() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_galoisCounterMode_AESCrypt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register in = c_rarg0;
const Register len = c_rarg1;
@@ -402,15 +419,24 @@ address StubGenerator::generate_avx2_galoisCounterMode_AESCrypt() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
// Vector AES Counter implementation
address StubGenerator::generate_counterMode_VectorAESCrypt() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_counterMode_AESCrypt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register from = c_rarg0; // source array address
const Register to = c_rarg1; // destination array address
@@ -469,6 +495,9 @@ address StubGenerator::generate_counterMode_VectorAESCrypt() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -496,10 +525,16 @@ address StubGenerator::generate_counterMode_VectorAESCrypt() {
//
address StubGenerator::generate_counterMode_AESCrypt_Parallel() {
assert(UseAES, "need AES instructions and misaligned SSE support");
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_counterMode_AESCrypt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register from = c_rarg0; // source array address
const Register to = c_rarg1; // destination array address
@@ -779,15 +814,24 @@ address StubGenerator::generate_counterMode_AESCrypt_Parallel() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_cipherBlockChaining_decryptVectorAESCrypt() {
assert(VM_Version::supports_avx512_vaes(), "need AES instructions and misaligned SSE support");
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_cipherBlockChaining_decryptAESCrypt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register from = c_rarg0; // source array address
const Register to = c_rarg1; // destination array address
@@ -1055,6 +1099,9 @@ address StubGenerator::generate_cipherBlockChaining_decryptVectorAESCrypt() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -1067,11 +1114,17 @@ address StubGenerator::generate_cipherBlockChaining_decryptVectorAESCrypt() {
//
address StubGenerator::generate_aescrypt_encryptBlock() {
assert(UseAES, "need AES instructions and misaligned SSE support");
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_aescrypt_encryptBlock_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
Label L_doLast;
- address start = __ pc();
+ start = __ pc();
const Register from = c_rarg0; // source array address
const Register to = c_rarg1; // destination array address
@@ -1150,6 +1203,9 @@ address StubGenerator::generate_aescrypt_encryptBlock() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -1162,11 +1218,17 @@ address StubGenerator::generate_aescrypt_encryptBlock() {
//
address StubGenerator::generate_aescrypt_decryptBlock() {
assert(UseAES, "need AES instructions and misaligned SSE support");
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_aescrypt_decryptBlock_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
Label L_doLast;
- address start = __ pc();
+ start = __ pc();
const Register from = c_rarg0; // source array address
const Register to = c_rarg1; // destination array address
@@ -1246,6 +1308,9 @@ address StubGenerator::generate_aescrypt_decryptBlock() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -1264,10 +1329,16 @@ address StubGenerator::generate_aescrypt_decryptBlock() {
//
address StubGenerator::generate_cipherBlockChaining_encryptAESCrypt() {
assert(UseAES, "need AES instructions and misaligned SSE support");
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_cipherBlockChaining_encryptAESCrypt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_exit, L_key_192_256, L_key_256, L_loopTop_128, L_loopTop_192, L_loopTop_256;
const Register from = c_rarg0; // source array address
@@ -1396,9 +1467,213 @@ address StubGenerator::generate_cipherBlockChaining_encryptAESCrypt() {
__ jcc(Assembler::notEqual, L_loopTop_256);
__ jmp(L_exit);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
+// This is a version of ECB/AES Encrypt/Decrypt which does 4 blocks in a loop
+// at a time to hide instruction latency.
+//
+// For encryption (is_encrypt=true):
+// pxor key[0], aesenc key[1..rounds-1], aesenclast key[rounds]
+// For decryption (is_encrypt=false):
+// pxor key[1], aesdec key[2..rounds], aesdeclast key[0]
+//
+// Arguments:
+//
+// Inputs:
+// c_rarg0 - source byte array address
+// c_rarg1 - destination byte array address
+// c_rarg2 - session key (Ke/Kd) in little endian int array
+// c_rarg3 - input length (must be multiple of blocksize 16)
+//
+// Output:
+// rax - input length
+//
+address StubGenerator::generate_electronicCodeBook_AESCrypt_Parallel(bool is_encrypt) {
+ assert(UseAES, "need AES instructions and misaligned SSE support");
+ StubId stub_id = is_encrypt ? StubId::stubgen_electronicCodeBook_encryptAESCrypt_id
+ : StubId::stubgen_electronicCodeBook_decryptAESCrypt_id;
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
+ StubCodeMark mark(this, stub_id);
+ start = __ pc();
+
+ const Register from = c_rarg0; // source array address
+ const Register to = c_rarg1; // destination array address
+ const Register key = c_rarg2; // key array address
+ const Register len_reg = c_rarg3; // src len (must be multiple of blocksize 16)
+ const Register pos = rax;
+ const Register keylen = r11;
+
+ const XMMRegister xmm_result0 = xmm0;
+ const XMMRegister xmm_result1 = xmm1;
+ const XMMRegister xmm_result2 = xmm2;
+ const XMMRegister xmm_result3 = xmm3;
+ const XMMRegister xmm_key_shuf_mask = xmm4;
+ const XMMRegister xmm_key_tmp = xmm5;
+ // keys 0-9 pre-loaded into xmm6-xmm15
+ const int XMM_REG_NUM_KEY_FIRST = 6;
+ const int XMM_REG_NUM_KEY_LAST = 15;
+ const XMMRegister xmm_key_first = as_XMMRegister(XMM_REG_NUM_KEY_FIRST);
+
+ // for key_128, key_192, key_256
+ const int ROUNDS[3] = {10, 12, 14};
+
+ Label L_exit;
+ Label L_loop4[3], L_single[3], L_done[3];
+
+#ifdef DoFour
+#undef DoFour
+#endif
+#ifdef DoOne
+#undef DoOne
+#endif
+
+#define DoFour(opc, reg) \
+__ opc(xmm_result0, reg); \
+__ opc(xmm_result1, reg); \
+__ opc(xmm_result2, reg); \
+__ opc(xmm_result3, reg);
+
+#define DoOne(opc, reg) \
+__ opc(xmm_result0, reg);
+
+ __ enter(); // required for proper stackwalking of RuntimeStub frame
+ __ push(len_reg); // save original length for return value
+
+ __ movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
+
+ __ movdqu(xmm_key_shuf_mask, ExternalAddress(key_shuffle_mask_addr()), r10 /*rscratch*/);
+ // load up xmm regs 6 thru 15 with keys 0x00 - 0x90
+ for (int rnum = XMM_REG_NUM_KEY_FIRST, offset = 0x00; rnum <= XMM_REG_NUM_KEY_LAST; rnum++, offset += 0x10) {
+ load_key(as_XMMRegister(rnum), key, offset, xmm_key_shuf_mask);
+ }
+ __ xorptr(pos, pos);
+
+ // key length could be only {11, 13, 15} * 4 = {44, 52, 60}
+ __ cmpl(keylen, 52);
+ __ jcc(Assembler::equal, L_loop4[1]);
+ __ cmpl(keylen, 60);
+ __ jcc(Assembler::equal, L_loop4[2]);
+
+ // k == 0: generate code for key_128
+ // k == 1: generate code for key_192
+ // k == 2: generate code for key_256
+ for (int k = 0; k < 3; ++k) {
+ __ align(OptoLoopAlignment);
+ __ BIND(L_loop4[k]);
+ __ cmpptr(len_reg, 4 * AESBlockSize);
+ __ jcc(Assembler::less, L_single[k]);
+
+ __ movdqu(xmm_result0, Address(from, pos, Address::times_1, 0 * AESBlockSize));
+ __ movdqu(xmm_result1, Address(from, pos, Address::times_1, 1 * AESBlockSize));
+ __ movdqu(xmm_result2, Address(from, pos, Address::times_1, 2 * AESBlockSize));
+ __ movdqu(xmm_result3, Address(from, pos, Address::times_1, 3 * AESBlockSize));
+
+ if (is_encrypt) {
+ DoFour(pxor, xmm_key_first);
+ for (int rnum = 1; rnum < 10; rnum++) {
+ DoFour(aesenc, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST));
+ }
+ for (int i = 10; i < ROUNDS[k]; i++) {
+ load_key(xmm_key_tmp, key, i * 0x10, xmm_key_shuf_mask);
+ DoFour(aesenc, xmm_key_tmp);
+ }
+ load_key(xmm_key_tmp, key, ROUNDS[k] * 0x10, xmm_key_shuf_mask);
+ DoFour(aesenclast, xmm_key_tmp);
+ } else {
+ DoFour(pxor, as_XMMRegister(1 + XMM_REG_NUM_KEY_FIRST));
+ for (int rnum = 2; rnum < 10; rnum++) {
+ DoFour(aesdec, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST));
+ }
+ for (int i = 10; i <= ROUNDS[k]; i++) {
+ load_key(xmm_key_tmp, key, i * 0x10, xmm_key_shuf_mask);
+ DoFour(aesdec, xmm_key_tmp);
+ }
+ DoFour(aesdeclast, xmm_key_first);
+ }
+
+ __ movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0);
+ __ movdqu(Address(to, pos, Address::times_1, 1 * AESBlockSize), xmm_result1);
+ __ movdqu(Address(to, pos, Address::times_1, 2 * AESBlockSize), xmm_result2);
+ __ movdqu(Address(to, pos, Address::times_1, 3 * AESBlockSize), xmm_result3);
+
+ __ addptr(pos, 4 * AESBlockSize);
+ __ subptr(len_reg, 4 * AESBlockSize);
+ __ jmp(L_loop4[k]);
+
+ __ align(OptoLoopAlignment);
+ __ BIND(L_single[k]);
+ __ cmpptr(len_reg, AESBlockSize);
+ __ jcc(Assembler::less, L_done[k]);
+
+ __ movdqu(xmm_result0, Address(from, pos, Address::times_1, 0));
+
+ if (is_encrypt) {
+ DoOne(pxor, xmm_key_first);
+ for (int rnum = 1; rnum < 10; rnum++) {
+ DoOne(aesenc, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST));
+ }
+ for (int i = 10; i < ROUNDS[k]; i++) {
+ load_key(xmm_key_tmp, key, i * 0x10, xmm_key_shuf_mask);
+ DoOne(aesenc, xmm_key_tmp);
+ }
+ load_key(xmm_key_tmp, key, ROUNDS[k] * 0x10, xmm_key_shuf_mask);
+ DoOne(aesenclast, xmm_key_tmp);
+ } else {
+ DoOne(pxor, as_XMMRegister(1 + XMM_REG_NUM_KEY_FIRST));
+ for (int rnum = 2; rnum < 10; rnum++) {
+ DoOne(aesdec, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST));
+ }
+ for (int i = 10; i <= ROUNDS[k]; i++) {
+ load_key(xmm_key_tmp, key, i * 0x10, xmm_key_shuf_mask);
+ DoOne(aesdec, xmm_key_tmp);
+ }
+ DoOne(aesdeclast, xmm_key_first);
+ }
+
+ __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result0);
+ __ addptr(pos, AESBlockSize);
+ __ subptr(len_reg, AESBlockSize);
+ __ jmp(L_single[k]);
+
+ __ BIND(L_done[k]);
+ if (k < 2) __ jmp(L_exit);
+ } //for key_128/192/256
+
+ __ BIND(L_exit);
+ // Clear all XMM registers holding sensitive key material before returning
+ __ pxor(xmm_key_tmp, xmm_key_tmp);
+ for (int rnum = XMM_REG_NUM_KEY_FIRST; rnum <= XMM_REG_NUM_KEY_LAST; rnum++) {
+ __ pxor(as_XMMRegister(rnum), as_XMMRegister(rnum));
+ }
+ __ pop(rax);
+ __ leave(); // required for proper stackwalking of RuntimeStub frame
+ __ ret(0);
+
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
+ return start;
+
+#undef DoFour
+#undef DoOne
+}
+
+address StubGenerator::generate_electronicCodeBook_encryptAESCrypt_Parallel() {
+ return generate_electronicCodeBook_AESCrypt_Parallel(true);
+}
+
+address StubGenerator::generate_electronicCodeBook_decryptAESCrypt_Parallel() {
+ return generate_electronicCodeBook_AESCrypt_Parallel(false);
+}
+
// This is a version of CBC/AES Decrypt which does 4 blocks in a loop at a time
// to hide instruction latency
//
@@ -1416,10 +1691,16 @@ address StubGenerator::generate_cipherBlockChaining_encryptAESCrypt() {
//
address StubGenerator::generate_cipherBlockChaining_decryptAESCrypt_Parallel() {
assert(UseAES, "need AES instructions and misaligned SSE support");
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_cipherBlockChaining_decryptAESCrypt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register from = c_rarg0; // source array address
const Register to = c_rarg1; // destination array address
@@ -1493,7 +1774,7 @@ address StubGenerator::generate_cipherBlockChaining_decryptAESCrypt_Parallel() {
__ opc(xmm_result0, src_reg); \
__ opc(xmm_result1, src_reg); \
__ opc(xmm_result2, src_reg); \
-__ opc(xmm_result3, src_reg); \
+__ opc(xmm_result3, src_reg);
for (int k = 0; k < 3; ++k) {
__ BIND(L_multiBlock_loopTopHead[k]);
@@ -1655,14 +1936,23 @@ __ opc(xmm_result3, src_reg); \
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_electronicCodeBook_encryptAESCrypt() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_electronicCodeBook_encryptAESCrypt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register from = c_rarg0; // source array address
const Register to = c_rarg1; // destination array address
@@ -1676,14 +1966,23 @@ address StubGenerator::generate_electronicCodeBook_encryptAESCrypt() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_electronicCodeBook_decryptAESCrypt() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_electronicCodeBook_decryptAESCrypt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register from = c_rarg0; // source array address
const Register to = c_rarg1; // destination array address
@@ -1697,6 +1996,9 @@ address StubGenerator::generate_electronicCodeBook_decryptAESCrypt() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -4096,3 +4398,27 @@ void StubGenerator::aesgcm_avx2(Register in, Register len, Register ct, Register
}
#undef __
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_aes(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr))
+ ADD(key_shuffle_mask_addr());
+ ADD(counter_shuffle_mask_addr());
+ ADD(counter_mask_linc0_addr());
+ ADD(counter_mask_linc1_addr());
+ ADD(counter_mask_linc1f_addr());
+ ADD(counter_mask_linc2_addr());
+ ADD(counter_mask_linc2f_addr());
+ ADD(counter_mask_linc4_addr());
+ ADD(counter_mask_linc8_addr());
+ ADD(counter_mask_linc16_addr());
+ ADD(counter_mask_linc32_addr());
+ ADD(counter_mask_ones_addr());
+ ADD(ghash_polynomial_reduction_addr());
+ ADD(ghash_polynomial_two_one_addr());
+ ADD(counter_mask_addbe_4444_addr());
+ ADD(counter_mask_addbe_1234_addr());
+ ADD(counter_mask_add_1234_addr());
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp
index d53fafafdb4..5530e5325de 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp
@@ -511,12 +511,12 @@ void StubGenerator::copy_bytes_backward(Register from, Register dest,
// - If target supports AVX3 features (BW+VL+F) then implementation uses 32 byte vectors (YMMs)
// for both special cases (various small block sizes) and aligned copy loop. This is the
// default configuration.
-// - If copy length is above AVX3Threshold, then implementation use 64 byte vectors (ZMMs)
+// - If copy length is above CopyAVX3Threshold, then implementation use 64 byte vectors (ZMMs)
// for main copy loop (and subsequent tail) since bulk of the cycles will be consumed in it.
// - If user forces MaxVectorSize=32 then above 4096 bytes its seen that REP MOVs shows a
// better performance for disjoint copies. For conjoint/backward copy vector based
// copy performs better.
-// - If user sets AVX3Threshold=0, then special cases for small blocks sizes operate over
+// - If user sets CopyAVX3Threshold=0, then special cases for small blocks sizes operate over
// 64 byte vector registers (ZMMs).
// Inputs:
@@ -570,13 +570,47 @@ address StubGenerator::generate_disjoint_copy_avx3_masked(StubId stub_id, addres
default:
ShouldNotReachHere();
}
+ GrowableArray entries;
+ GrowableArray extras;
+ bool add_handlers = !is_oop && !aligned;
+ bool add_relocs = UseZGC && is_oop;
+ bool add_extras = add_handlers || add_relocs;
+ // The stub employs one unsafe handler region by default but has two
+ // when MaxVectorSize == 64 So we may expect 0, 3 or 6 extras.
+ int handlers_count = (MaxVectorSize == 64 ? 2 : 1);
+ int expected_entry_count = (entry != nullptr ? 2 : 1);
+ int expected_extra_count = (add_handlers ? handlers_count : 0) * UnsafeMemoryAccess::COLUMN_COUNT; // 0/1/2 x UMAM {start,end,handler}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == expected_entry_count, "sanity check");
+ GrowableArray* entries_ptr = (entry_count == 1 ? nullptr : &entries);
+ GrowableArray* extras_ptr = (add_extras ? &extras : nullptr);
+ address start = load_archive_data(stub_id, entries_ptr, extras_ptr);
+ if (start != nullptr) {
+ assert(entries.length() == expected_entry_count - 1,
+ "unexpected entry count %d", entries.length());
+ assert(!add_handlers || extras.length() == expected_extra_count,
+ "unexpected handler addresses count %d", extras.length());
+ if (entry != nullptr) {
+ *entry = entries.at(0);
+ }
+ if (add_handlers) {
+ // restore 1/2 x UMAM {start,end,handler} addresses from extras
+ register_unsafe_access_handlers(extras, 0, handlers_count);
+ }
+#if INCLUDE_ZGC
+ // register addresses at which ZGC does colour patching
+ if (add_relocs) {
+ register_reloc_addresses(extras, 0, extras.length());
+ }
+#endif // INCLUDE_ZGC
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
- int avx3threshold = VM_Version::avx3_threshold();
- bool use64byteVector = (MaxVectorSize > 32) && (avx3threshold == 0);
+ bool use64byteVector = (MaxVectorSize > 32) && (CopyAVX3Threshold == 0);
const int large_threshold = 2621440; // 2.5 MB
Label L_main_loop, L_main_loop_64bytes, L_tail, L_tail64, L_exit, L_entry;
Label L_repmovs, L_main_pre_loop, L_main_pre_loop_64bytes, L_pre_main_post_64;
@@ -596,6 +630,7 @@ address StubGenerator::generate_disjoint_copy_avx3_masked(StubId stub_id, addres
if (entry != nullptr) {
*entry = __ pc();
+ entries.append(*entry);
// caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
BLOCK_COMMENT("Entry:");
}
@@ -621,7 +656,7 @@ address StubGenerator::generate_disjoint_copy_avx3_masked(StubId stub_id, addres
int threshold[] = { 4096, 2048, 1024, 512};
// UnsafeMemoryAccess page error: continue after unsafe access
- UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true);
+ UnsafeMemoryAccessMark umam(this, add_handlers, true);
// 'from', 'to' and 'count' are now valid
// temp1 holds remaining count and temp4 holds running count used to compute
@@ -647,7 +682,7 @@ address StubGenerator::generate_disjoint_copy_avx3_masked(StubId stub_id, addres
__ cmpq(temp2, large_threshold);
__ jcc(Assembler::greaterEqual, L_copy_large);
}
- if (avx3threshold != 0) {
+ if (CopyAVX3Threshold != 0) {
__ cmpq(count, threshold[shift]);
if (MaxVectorSize == 64) {
// Copy using 64 byte vectors.
@@ -659,7 +694,7 @@ address StubGenerator::generate_disjoint_copy_avx3_masked(StubId stub_id, addres
}
}
- if ((MaxVectorSize < 64) || (avx3threshold != 0)) {
+ if ((MaxVectorSize < 64) || (CopyAVX3Threshold != 0)) {
// Partial copy to make dst address 32 byte aligned.
__ movq(temp2, to);
__ andq(temp2, 31);
@@ -790,10 +825,28 @@ address StubGenerator::generate_disjoint_copy_avx3_masked(StubId stub_id, addres
if (MaxVectorSize == 64) {
__ BIND(L_copy_large);
- UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, false, ucme_exit_pc);
+ UnsafeMemoryAccessMark umam(this, add_handlers, false, ucme_exit_pc);
arraycopy_avx3_large(to, from, temp1, temp2, temp3, temp4, count, xmm1, xmm2, xmm3, xmm4, shift);
__ jmp(L_finish);
}
+ // retrieve the registered handler addresses
+ address end = __ pc();
+ if (add_handlers) {
+ retrieve_unsafe_access_handlers(start, end, extras);
+ }
+ assert(extras.length() == expected_extra_count,
+ "unexpected handler addresses count %d", extras.length());
+#if INCLUDE_ZGC
+ // retrieve addresses at which ZGC does colour patching
+ if (add_relocs) {
+ retrieve_reloc_addresses(start, end, extras);
+ }
+#endif // INCLUDE_ZGC
+
+ // record the stub entry and end plus the no_push entry and any
+ // extra handler addresses
+ store_archive_data(stub_id, start, end, entries_ptr, extras_ptr);
+
return start;
}
@@ -908,13 +961,43 @@ address StubGenerator::generate_conjoint_copy_avx3_masked(StubId stub_id, addres
default:
ShouldNotReachHere();
}
-
+ GrowableArray entries;
+ GrowableArray extras;
+ bool add_handlers = !is_oop && !aligned;
+ bool add_relocs = UseZGC && is_oop;
+ bool add_extras = add_handlers || add_relocs;
+ int expected_entry_count = (entry != nullptr ? 2 : 1);
+ int expected_handler_count = (add_handlers ? 1 : 0) * UnsafeMemoryAccess::COLUMN_COUNT; // 0/1 x UMAM {start,end,handler}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == expected_entry_count, "sanity check");
+ GrowableArray* entries_ptr = (entry_count == 1 ? nullptr : &entries);
+ GrowableArray* extras_ptr = (add_extras ? &extras : nullptr);
+ address start = load_archive_data(stub_id, entries_ptr, extras_ptr);
+ if (start != nullptr) {
+ assert(entries.length() == expected_entry_count - 1,
+ "unexpected entry count %d", entries.length());
+ assert(!add_handlers || extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+ if (entry != nullptr) {
+ *entry = entries.at(0);
+ }
+ if (add_handlers) {
+ // restore 1 x UMAM {start,end,handler} addresses from extras
+ register_unsafe_access_handlers(extras, 0, 1);
+ }
+#if INCLUDE_ZGC
+ if (add_relocs) {
+ // register addresses at which ZGC does colour patching
+ register_reloc_addresses(extras, 0, extras.length());
+ }
+#endif // INCLUDE_ZGC
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
- int avx3threshold = VM_Version::avx3_threshold();
- bool use64byteVector = (MaxVectorSize > 32) && (avx3threshold == 0);
+ bool use64byteVector = (MaxVectorSize > 32) && (CopyAVX3Threshold == 0);
Label L_main_pre_loop, L_main_pre_loop_64bytes, L_pre_main_post_64;
Label L_main_loop, L_main_loop_64bytes, L_tail, L_tail64, L_exit, L_entry;
@@ -933,6 +1016,7 @@ address StubGenerator::generate_conjoint_copy_avx3_masked(StubId stub_id, addres
if (entry != nullptr) {
*entry = __ pc();
+ entries.append(*entry);
// caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
BLOCK_COMMENT("Entry:");
}
@@ -959,7 +1043,7 @@ address StubGenerator::generate_conjoint_copy_avx3_masked(StubId stub_id, addres
int threshold[] = { 4096, 2048, 1024, 512};
// UnsafeMemoryAccess page error: continue after unsafe access
- UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true);
+ UnsafeMemoryAccessMark umam(this, add_handlers, true);
// 'from', 'to' and 'count' are now valid
// temp1 holds remaining count.
@@ -979,12 +1063,12 @@ address StubGenerator::generate_conjoint_copy_avx3_masked(StubId stub_id, addres
// PRE-MAIN-POST loop for aligned copy.
__ BIND(L_entry);
- if ((MaxVectorSize > 32) && (avx3threshold != 0)) {
+ if ((MaxVectorSize > 32) && (CopyAVX3Threshold != 0)) {
__ cmpq(temp1, threshold[shift]);
__ jcc(Assembler::greaterEqual, L_pre_main_post_64);
}
- if ((MaxVectorSize < 64) || (avx3threshold != 0)) {
+ if ((MaxVectorSize < 64) || (CopyAVX3Threshold != 0)) {
// Partial copy to make dst address 32 byte aligned.
__ leaq(temp2, Address(to, temp1, (Address::ScaleFactor)(shift), 0));
__ andq(temp2, 31);
@@ -1073,6 +1157,23 @@ address StubGenerator::generate_conjoint_copy_avx3_masked(StubId stub_id, addres
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // retrieve the registered handler addresses
+ address end = __ pc();
+ if (add_handlers) {
+ retrieve_unsafe_access_handlers(start, end, extras);
+ }
+ assert(extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+#if INCLUDE_ZGC
+ // retrieve addresses at which ZGC does colour patching
+ if (add_relocs) {
+ retrieve_reloc_addresses(start, end, extras);
+ }
+#endif // INCLUDE_ZGC
+ // record the stub entry and end plus the no_push entry and any
+ // extra handler addresses
+ store_archive_data(stub_id, start, end, entries_ptr, extras_ptr);
+
return start;
}
@@ -1199,7 +1300,7 @@ void StubGenerator::arraycopy_avx3_special_cases_conjoint(XMMRegister xmm, KRegi
bool use64byteVector, Label& L_entry, Label& L_exit) {
Label L_entry_64, L_entry_96, L_entry_128;
Label L_entry_160, L_entry_192;
- bool avx3 = (MaxVectorSize > 32) && (VM_Version::avx3_threshold() == 0);
+ bool avx3 = (MaxVectorSize > 32) && (CopyAVX3Threshold == 0);
int size_mat[][6] = {
/* T_BYTE */ {32 , 64, 96 , 128 , 160 , 192 },
@@ -1387,9 +1488,29 @@ address StubGenerator::generate_disjoint_byte_copy(address* entry) {
return generate_disjoint_copy_avx3_masked(stub_id, entry);
}
#endif
+ GrowableArray entries;
+ GrowableArray extras;
+ int expected_entry_count = (entry != nullptr ? 2 : 1);
+ int expected_handler_count = (2 * UnsafeMemoryAccess::COLUMN_COUNT); // 2 x UMAM {start,end,handler}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == expected_entry_count, "sanity check");
+ GrowableArray* entries_ptr = (entry_count == 1 ? nullptr : &entries);
+ address start = load_archive_data(stub_id, entries_ptr, &extras);
+ if (start != nullptr) {
+ assert(entries.length() == expected_entry_count - 1,
+ "unexpected entry count %d", entries.length());
+ assert(extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+ if (entry != nullptr) {
+ *entry = entries.at(0);
+ }
+ // restore 2 UMAM {start,end,handler} addresses from extras
+ register_unsafe_access_handlers(extras, 0, 2);
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT;
Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_copy_2_bytes;
@@ -1409,6 +1530,7 @@ address StubGenerator::generate_disjoint_byte_copy(address* entry) {
if (entry != nullptr) {
*entry = __ pc();
+ entries.append(*entry);
// caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
BLOCK_COMMENT("Entry:");
}
@@ -1478,6 +1600,17 @@ __ BIND(L_exit);
copy_bytes_forward(end_from, end_to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, T_BYTE);
__ jmp(L_copy_4_bytes);
}
+
+ // retrieve the registered handler addresses
+ address end = __ pc();
+ retrieve_unsafe_access_handlers(start, end, extras);
+ assert(extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+
+ // record the stub entry and end plus the no_push entry and any
+ // extra handler addresses
+ store_archive_data(stub_id, start, end, entries_ptr, &extras);
+
return start;
}
@@ -1505,9 +1638,29 @@ address StubGenerator::generate_conjoint_byte_copy(address nooverlap_target, add
return generate_conjoint_copy_avx3_masked(stub_id, entry, nooverlap_target);
}
#endif
+ GrowableArray entries;
+ GrowableArray extras;
+ int expected_entry_count = (entry != nullptr ? 2 : 1);
+ int expected_handler_count = (2 * UnsafeMemoryAccess::COLUMN_COUNT); // 2 x UMAM {start,end,handler}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == expected_entry_count, "sanity check");
+ GrowableArray* entries_ptr = (entry_count == 1 ? nullptr : &entries);
+ address start = load_archive_data(stub_id, entries_ptr, &extras);
+ if (start != nullptr) {
+ assert(entries.length() == expected_entry_count - 1,
+ "unexpected entry count %d", entries.length());
+ assert(extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+ if (entry != nullptr) {
+ *entry = entries.at(0);
+ }
+ // restore 2 UMAM {start,end,handler} addresses from extras
+ register_unsafe_access_handlers(extras, 0, 2);
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
DecoratorSet decorators = IN_HEAP | IS_ARRAY;
Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_copy_2_bytes;
@@ -1522,6 +1675,7 @@ address StubGenerator::generate_conjoint_byte_copy(address nooverlap_target, add
if (entry != nullptr) {
*entry = __ pc();
+ entries.append(*entry);
// caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
BLOCK_COMMENT("Entry:");
}
@@ -1588,6 +1742,16 @@ address StubGenerator::generate_conjoint_byte_copy(address nooverlap_target, add
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // retrieve the registered handler addresses
+ address end = __ pc();
+ retrieve_unsafe_access_handlers(start, end, extras);
+ assert(extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+
+ // record the stub entry and end plus the no_push entry and any
+ // extra handler addresses
+ store_archive_data(stub_id, start, end, entries_ptr, &extras);
+
return start;
}
@@ -1618,10 +1782,29 @@ address StubGenerator::generate_disjoint_short_copy(address *entry) {
return generate_disjoint_copy_avx3_masked(stub_id, entry);
}
#endif
-
+ GrowableArray entries;
+ GrowableArray extras;
+ int expected_entry_count = (entry != nullptr ? 2 : 1);
+ int expected_handler_count = (2 * UnsafeMemoryAccess::COLUMN_COUNT); // 2 x UMAM {start,end,handler}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == expected_entry_count, "sanity check");
+ GrowableArray* entries_ptr = (entry_count == 1 ? nullptr : &entries);
+ address start = load_archive_data(stub_id, entries_ptr, &extras);
+ if (start != nullptr) {
+ assert(entries.length() == expected_entry_count - 1,
+ "unexpected entry count %d", entries.length());
+ assert(extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+ if (entry != nullptr) {
+ *entry = entries.at(0);
+ }
+ // restore 2 UMAM {start,end,handler} addresses from extras
+ register_unsafe_access_handlers(extras, 0, 2);
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT;
Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes,L_copy_2_bytes,L_exit;
@@ -1640,6 +1823,7 @@ address StubGenerator::generate_disjoint_short_copy(address *entry) {
if (entry != nullptr) {
*entry = __ pc();
+ entries.append(*entry);
// caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
BLOCK_COMMENT("Entry:");
}
@@ -1703,6 +1887,16 @@ __ BIND(L_exit);
__ jmp(L_copy_4_bytes);
}
+ // retrieve the registered handler addresses
+ address end = __ pc();
+ retrieve_unsafe_access_handlers(start, end, extras);
+ assert(extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+
+ // record the stub entry and end plus the no_push entry and any
+ // extra handler addresses
+ store_archive_data(stub_id, start, end, entries_ptr, &extras);
+
return start;
}
@@ -1710,7 +1904,6 @@ __ BIND(L_exit);
address StubGenerator::generate_fill(StubId stub_id) {
BasicType t;
bool aligned;
-
switch (stub_id) {
case StubId::stubgen_jbyte_fill_id:
t = T_BYTE;
@@ -1739,10 +1932,27 @@ address StubGenerator::generate_fill(StubId stub_id) {
default:
ShouldNotReachHere();
}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ GrowableArray extras;
+ bool add_handlers = ((t == T_BYTE) && !aligned);
+ int handlers_count = (add_handlers ? 1 : 0);
+ int expected_extras_count = (handlers_count * UnsafeMemoryAccess::COLUMN_COUNT); // 0/1 x UMAM {start,end,handler}
+ GrowableArray* extras_ptr = (add_handlers ? &extras : nullptr);
+ address start = load_archive_data(stub_id, nullptr, extras_ptr);
+ if (start != nullptr) {
+ assert(extras.length() == expected_extras_count,
+ "unexpected handler addresses count %d", extras.length());
+ if (add_handlers) {
+ // restore 1 x UMAM {start,end,handler} addresses from extras
+ register_unsafe_access_handlers(extras, 0, 1);
+ }
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
BLOCK_COMMENT("Entry:");
@@ -1755,7 +1965,7 @@ address StubGenerator::generate_fill(StubId stub_id) {
{
// Add set memory mark to protect against unsafe accesses faulting
- UnsafeMemoryAccessMark umam(this, ((t == T_BYTE) && !aligned), true);
+ UnsafeMemoryAccessMark umam(this, add_handlers, true);
__ generate_fill(t, aligned, to, value, r11, rax, xmm0);
}
@@ -1763,6 +1973,15 @@ address StubGenerator::generate_fill(StubId stub_id) {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ address end = __ pc();
+ if (add_handlers) {
+ retrieve_unsafe_access_handlers(start, end, extras);
+ }
+ assert(extras.length() == expected_extras_count,
+ "unexpected handler addresses count %d", extras.length());
+ // record the stub entry and end
+ store_archive_data(stub_id, start, end, nullptr, extras_ptr);
+
return start;
}
@@ -1790,10 +2009,29 @@ address StubGenerator::generate_conjoint_short_copy(address nooverlap_target, ad
return generate_conjoint_copy_avx3_masked(stub_id, entry, nooverlap_target);
}
#endif
-
+ GrowableArray entries;
+ GrowableArray extras;
+ int expected_entry_count = (entry != nullptr ? 2 : 1);
+ int expected_handler_count = (2 * UnsafeMemoryAccess::COLUMN_COUNT); // 2 x UMAM {start,end,handler}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == expected_entry_count, "sanity check");
+ GrowableArray* entries_ptr = (entry_count == 1 ? nullptr : &entries);
+ address start = load_archive_data(stub_id, entries_ptr, &extras);
+ if (start != nullptr) {
+ assert(entries.length() == expected_entry_count - 1,
+ "unexpected entry count %d", entries.length());
+ assert(extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+ if (entry != nullptr) {
+ *entry = entries.at(0);
+ }
+ // restore 2 UMAM {start,end,handler} addresses from extras
+ register_unsafe_access_handlers(extras, 0, 2);
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
DecoratorSet decorators = IN_HEAP | IS_ARRAY;
Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes;
@@ -1808,6 +2046,7 @@ address StubGenerator::generate_conjoint_short_copy(address nooverlap_target, ad
if (entry != nullptr) {
*entry = __ pc();
+ entries.append(*entry);
// caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
BLOCK_COMMENT("Entry:");
}
@@ -1866,6 +2105,16 @@ address StubGenerator::generate_conjoint_short_copy(address nooverlap_target, ad
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // retrieve the registered handler addresses
+ address end = __ pc();
+ retrieve_unsafe_access_handlers(start, end, extras);
+ assert(extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+
+ // record the stub entry and end plus the no_push entry and any
+ // extra handler addresses
+ store_archive_data(stub_id, start, end, entries_ptr, &extras);
+
return start;
}
@@ -1918,10 +2167,42 @@ address StubGenerator::generate_disjoint_int_oop_copy(StubId stub_id, address* e
return generate_disjoint_copy_avx3_masked(stub_id, entry);
}
#endif
+ GrowableArray entries;
+ GrowableArray extras;
+ bool add_handlers = !is_oop && !aligned;
+ bool add_relocs = UseZGC && is_oop;
+ bool add_extras = add_handlers || add_relocs;
+ int expected_entry_count = (entry != nullptr ? 2 : 1);
+ int expected_handler_count = (add_handlers ? 2 : 0) * UnsafeMemoryAccess::COLUMN_COUNT; // 0/2 x UMAM {start,end,handler}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == expected_entry_count, "sanity check");
+ GrowableArray* entries_ptr = (entry_count == 1 ? nullptr : &entries);
+ GrowableArray* extras_ptr = (add_extras ? &extras : nullptr);
+ address start = load_archive_data(stub_id, entries_ptr, extras_ptr);
+ if (start != nullptr) {
+ assert(entries.length() == expected_entry_count - 1,
+ "unexpected entry count %d", entries.length());
+ assert(!add_handlers || extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+ if (entry != nullptr) {
+ *entry = entries.at(0);
+ }
+ if (add_handlers) {
+ // restore 2 UMAM {start,end,handler} addresses from extras
+ register_unsafe_access_handlers(extras, 0, 2);
+ }
+#if INCLUDE_ZGC
+ // register addresses at which ZGC does colour patching
+ if (add_relocs) {
+ register_reloc_addresses(extras, 0, extras.length());
+ }
+#endif // INCLUDE_ZGC
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_exit;
const Register from = rdi; // source array address
@@ -1939,6 +2220,7 @@ address StubGenerator::generate_disjoint_int_oop_copy(StubId stub_id, address* e
if (entry != nullptr) {
*entry = __ pc();
+ entries.append(*entry);
// caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
BLOCK_COMMENT("Entry:");
}
@@ -1959,7 +2241,7 @@ address StubGenerator::generate_disjoint_int_oop_copy(StubId stub_id, address* e
{
// UnsafeMemoryAccess page error: continue after unsafe access
- UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true);
+ UnsafeMemoryAccessMark umam(this, add_handlers, true);
// 'from', 'to' and 'count' are now valid
__ movptr(dword_count, count);
__ shrptr(count, 1); // count => qword_count
@@ -1971,20 +2253,20 @@ address StubGenerator::generate_disjoint_int_oop_copy(StubId stub_id, address* e
__ jmp(L_copy_bytes);
// Copy trailing qwords
- __ BIND(L_copy_8_bytes);
+ __ BIND(L_copy_8_bytes);
__ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
__ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
__ increment(qword_count);
__ jcc(Assembler::notZero, L_copy_8_bytes);
// Check for and copy trailing dword
- __ BIND(L_copy_4_bytes);
+ __ BIND(L_copy_4_bytes);
__ testl(dword_count, 1); // Only byte test since the value is 0 or 1
__ jccb(Assembler::zero, L_exit);
__ movl(rax, Address(end_from, 8));
__ movl(Address(end_to, 8), rax);
}
-__ BIND(L_exit);
+ __ BIND(L_exit);
address ucme_exit_pc = __ pc();
bs->arraycopy_epilogue(_masm, decorators, type, from, to, dword_count);
restore_arg_regs_using_thread();
@@ -1995,12 +2277,30 @@ __ BIND(L_exit);
__ ret(0);
{
- UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, false, ucme_exit_pc);
+ UnsafeMemoryAccessMark umam(this, add_handlers, false, ucme_exit_pc);
// Copy in multi-bytes chunks
copy_bytes_forward(end_from, end_to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, is_oop ? T_OBJECT : T_INT);
__ jmp(L_copy_4_bytes);
}
+ // retrieve the registered handler addresses
+ address end = __ pc();
+ if (add_handlers) {
+ retrieve_unsafe_access_handlers(start, end, extras);
+ }
+ assert(extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+#if INCLUDE_ZGC
+ // retrieve addresses at which ZGC does colour patching
+ if (add_relocs) {
+ retrieve_reloc_addresses(start, end, extras);
+ }
+#endif // INCLUDE_ZGC
+
+ // record the stub entry and end plus the no_push entry and any
+ // extra handler addresses
+ store_archive_data(stub_id, start, end, entries_ptr, extras_ptr);
+
return start;
}
@@ -2049,10 +2349,42 @@ address StubGenerator::generate_conjoint_int_oop_copy(StubId stub_id, address no
return generate_conjoint_copy_avx3_masked(stub_id, entry, nooverlap_target);
}
#endif
+ bool add_handlers = !is_oop && !aligned;
+ bool add_relocs = UseZGC && is_oop;
+ bool add_extras = add_handlers || add_relocs;
+ GrowableArray entries;
+ GrowableArray extras;
+ int expected_entry_count = (entry != nullptr ? 2 : 1);
+ int expected_handler_count = (add_handlers ? 2 : 0) * UnsafeMemoryAccess::COLUMN_COUNT; // 0/2 x UMAM {start,end,handler}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == expected_entry_count, "sanity check");
+ GrowableArray* entries_ptr = (entry_count == 1 ? nullptr : &entries);
+ GrowableArray* extras_ptr = (add_extras ? &extras : nullptr);
+ address start = load_archive_data(stub_id, entries_ptr, extras_ptr);
+ if (start != nullptr) {
+ assert(entries.length() == expected_entry_count - 1,
+ "unexpected entry count %d", entries.length());
+ assert(!add_handlers || extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+ if (entry != nullptr) {
+ *entry = entries.at(0);
+ }
+ if (add_handlers) {
+ // restore 2 UMAM {start,end,handler} addresses from extras
+ register_unsafe_access_handlers(extras, 0, 2);
+ }
+#if INCLUDE_ZGC
+ // register addresses at which ZGC does colour patching
+ if (add_relocs) {
+ register_reloc_addresses(extras, 6, extras.length());
+ }
+#endif // INCLUDE_ZGC
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_copy_bytes, L_copy_8_bytes, L_exit;
const Register from = rdi; // source array address
@@ -2066,7 +2398,8 @@ address StubGenerator::generate_conjoint_int_oop_copy(StubId stub_id, address no
if (entry != nullptr) {
*entry = __ pc();
- // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
+ entries.append(*entry);
+ // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
BLOCK_COMMENT("Entry:");
}
@@ -2089,7 +2422,7 @@ address StubGenerator::generate_conjoint_int_oop_copy(StubId stub_id, address no
assert_clean_int(count, rax); // Make sure 'count' is clean int.
{
// UnsafeMemoryAccess page error: continue after unsafe access
- UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true);
+ UnsafeMemoryAccessMark umam(this, add_handlers, true);
// 'from', 'to' and 'count' are now valid
__ movptr(dword_count, count);
__ shrptr(count, 1); // count => qword_count
@@ -2104,7 +2437,7 @@ address StubGenerator::generate_conjoint_int_oop_copy(StubId stub_id, address no
__ jmp(L_copy_bytes);
// Copy trailing qwords
- __ BIND(L_copy_8_bytes);
+ __ BIND(L_copy_8_bytes);
__ movq(rax, Address(from, qword_count, Address::times_8, -8));
__ movq(Address(to, qword_count, Address::times_8, -8), rax);
__ decrement(qword_count);
@@ -2122,12 +2455,12 @@ address StubGenerator::generate_conjoint_int_oop_copy(StubId stub_id, address no
{
// UnsafeMemoryAccess page error: continue after unsafe access
- UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true);
+ UnsafeMemoryAccessMark umam(this, add_handlers, true);
// Copy in multi-bytes chunks
copy_bytes_backward(from, to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, is_oop ? T_OBJECT : T_INT);
}
-__ BIND(L_exit);
+ __ BIND(L_exit);
bs->arraycopy_epilogue(_masm, decorators, type, from, to, dword_count);
restore_arg_regs_using_thread();
INC_COUNTER_NP(SharedRuntime::_jint_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free
@@ -2136,6 +2469,23 @@ __ BIND(L_exit);
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // retrieve the registered handler addresses
+ address end = __ pc();
+ if (add_handlers) {
+ retrieve_unsafe_access_handlers(start, end, extras);
+ }
+ assert(extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+#if INCLUDE_ZGC
+ // retrieve addresses at which ZGC does colour patching
+ if (add_relocs) {
+ retrieve_reloc_addresses(start, end, extras);
+ }
+#endif // INCLUDE_ZGC
+ // record the stub entry and end plus the no_push entry and any
+ // extra handler addresses
+ store_archive_data(stub_id, start, end, entries_ptr, extras_ptr);
+
return start;
}
@@ -2182,10 +2532,42 @@ address StubGenerator::generate_disjoint_long_oop_copy(StubId stub_id, address *
return generate_disjoint_copy_avx3_masked(stub_id, entry);
}
#endif
+ bool add_handlers = !is_oop && !aligned;
+ bool add_relocs = UseZGC && is_oop;
+ bool add_extras = add_handlers || add_relocs;
+ GrowableArray entries;
+ GrowableArray extras;
+ int expected_entry_count = (entry != nullptr ? 2 : 1);
+ int expected_handler_count = (add_handlers ? 2 : 0) * UnsafeMemoryAccess::COLUMN_COUNT; // 0/2 x UMAM {start,end,handler}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == expected_entry_count, "sanity check");
+ GrowableArray* entries_ptr = (entry_count == 1 ? nullptr : &entries);
+ GrowableArray* extras_ptr = (add_extras ? &extras : nullptr);
+ address start = load_archive_data(stub_id, entries_ptr, extras_ptr);
+ if (start != nullptr) {
+ assert(entries.length() == expected_entry_count - 1,
+ "unexpected entry count %d", entries.length());
+ assert(!add_handlers || extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+ if (entry != nullptr) {
+ *entry = entries.at(0);
+ }
+ if (add_handlers) {
+ // restore 2 UMAM {start,end,handler} addresses from extras
+ register_unsafe_access_handlers(extras, 0, 2);
+ }
+#if INCLUDE_ZGC
+ // register addresses at which ZGC does colour patching
+ if (add_relocs) {
+ register_reloc_addresses(extras, 0, extras.length());
+ }
+#endif // INCLUDE_ZGC
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_copy_bytes, L_copy_8_bytes, L_exit;
const Register from = rdi; // source array address
@@ -2203,6 +2585,7 @@ address StubGenerator::generate_disjoint_long_oop_copy(StubId stub_id, address *
if (entry != nullptr) {
*entry = __ pc();
+ entries.append(*entry);
// caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
BLOCK_COMMENT("Entry:");
}
@@ -2223,7 +2606,7 @@ address StubGenerator::generate_disjoint_long_oop_copy(StubId stub_id, address *
bs->arraycopy_prologue(_masm, decorators, type, from, to, qword_count);
{
// UnsafeMemoryAccess page error: continue after unsafe access
- UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true);
+ UnsafeMemoryAccessMark umam(this, add_handlers, true);
// Copy from low to high addresses. Use 'to' as scratch.
__ lea(end_from, Address(from, qword_count, Address::times_8, -8));
@@ -2255,7 +2638,7 @@ address StubGenerator::generate_disjoint_long_oop_copy(StubId stub_id, address *
{
// UnsafeMemoryAccess page error: continue after unsafe access
- UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true);
+ UnsafeMemoryAccessMark umam(this, add_handlers, true);
// Copy in multi-bytes chunks
copy_bytes_forward(end_from, end_to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, is_oop ? T_OBJECT : T_LONG);
}
@@ -2271,6 +2654,23 @@ address StubGenerator::generate_disjoint_long_oop_copy(StubId stub_id, address *
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // retrieve the registered handler addresses
+ address end = __ pc();
+ if (add_handlers) {
+ retrieve_unsafe_access_handlers(start, end, extras);
+ }
+ assert(extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+#if INCLUDE_ZGC
+ // retrieve addresses at which ZGC does colour patching
+ if (add_relocs) {
+ retrieve_reloc_addresses(start, end, extras);
+ }
+#endif // INCLUDE_ZGC
+ // record the stub entry and end plus the no_push entry and any
+ // extra handler addresses
+ store_archive_data(stub_id, start, end, entries_ptr, extras_ptr);
+
return start;
}
@@ -2315,10 +2715,42 @@ address StubGenerator::generate_conjoint_long_oop_copy(StubId stub_id, address n
return generate_conjoint_copy_avx3_masked(stub_id, entry, nooverlap_target);
}
#endif
+ bool add_handlers = !is_oop && !aligned;
+ bool add_relocs = UseZGC && is_oop;
+ bool add_extras = add_handlers || add_relocs;
+ GrowableArray entries;
+ GrowableArray extras;
+ int expected_entry_count = (entry != nullptr ? 2 : 1);
+ int expected_handler_count = (add_handlers ? 2 : 0) * UnsafeMemoryAccess::COLUMN_COUNT; // 0/2 x UMAM {start,end,handler}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == expected_entry_count, "sanity check");
+ GrowableArray* entries_ptr = (entry_count == 1 ? nullptr : &entries);
+ GrowableArray* extras_ptr = (add_extras ? &extras : nullptr);
+ address start = load_archive_data(stub_id, entries_ptr, extras_ptr);
+ if (start != nullptr) {
+ assert(entries.length() == expected_entry_count - 1,
+ "unexpected entry count %d", entries.length());
+ assert(!add_handlers || extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+ if (entry != nullptr) {
+ *entry = entries.at(0);
+ }
+ if (add_handlers) {
+ // restore 2 UMAM {start,end,handler} addresses from extras
+ register_unsafe_access_handlers(extras, 0, 2);
+ }
+#if INCLUDE_ZGC
+ // register addresses at which ZGC does colour patching
+ if (add_relocs) {
+ register_reloc_addresses(extras, 0, extras.length());
+ }
+#endif // INCLUDE_ZGC
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_copy_bytes, L_copy_8_bytes, L_exit;
const Register from = rdi; // source array address
@@ -2331,6 +2763,7 @@ address StubGenerator::generate_conjoint_long_oop_copy(StubId stub_id, address n
if (entry != nullptr) {
*entry = __ pc();
+ entries.append(*entry);
// caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
BLOCK_COMMENT("Entry:");
}
@@ -2352,7 +2785,7 @@ address StubGenerator::generate_conjoint_long_oop_copy(StubId stub_id, address n
bs->arraycopy_prologue(_masm, decorators, type, from, to, qword_count);
{
// UnsafeMemoryAccess page error: continue after unsafe access
- UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true);
+ UnsafeMemoryAccessMark umam(this, add_handlers, true);
__ jmp(L_copy_bytes);
@@ -2379,7 +2812,7 @@ address StubGenerator::generate_conjoint_long_oop_copy(StubId stub_id, address n
}
{
// UnsafeMemoryAccess page error: continue after unsafe access
- UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true);
+ UnsafeMemoryAccessMark umam(this, add_handlers, true);
// Copy in multi-bytes chunks
copy_bytes_backward(from, to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, is_oop ? T_OBJECT : T_LONG);
@@ -2395,6 +2828,24 @@ address StubGenerator::generate_conjoint_long_oop_copy(StubId stub_id, address n
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+
+ // retrieve the registered handler addresses
+ address end = __ pc();
+ if (add_handlers) {
+ retrieve_unsafe_access_handlers(start, end, extras);
+ }
+ assert(extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+#if INCLUDE_ZGC
+ // retrieve addresses at which ZGC does colour patching
+ if ((UseZGC && is_oop)) {
+ retrieve_reloc_addresses(start, end, extras);
+ }
+#endif // INCLUDE_ZGC
+ // record the stub entry and end plus the no_push entry and any
+ // extra handler addresses
+ store_archive_data(stub_id, start, end, entries_ptr, extras_ptr);
+
return start;
}
@@ -2450,6 +2901,28 @@ address StubGenerator::generate_checkcast_copy(StubId stub_id, address *entry) {
ShouldNotReachHere();
}
+ GrowableArray entries;
+ GrowableArray extras;
+ int expected_entry_count = (entry != nullptr ? 2 : 1);
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == expected_entry_count, "sanity check");
+ GrowableArray* entries_ptr = (entry_count == 1 ? nullptr : &entries);
+ GrowableArray* extras_ptr = (UseZGC ? &extras : nullptr);
+ address start = load_archive_data(stub_id, entries_ptr, extras_ptr);
+ if (start != nullptr) {
+ assert(entries.length() == expected_entry_count - 1,
+ "unexpected addresses count %d", entries.length());
+ if (entry != nullptr) {
+ *entry = entries.at(0);
+ }
+#if INCLUDE_ZGC
+ if (UseZGC) {
+ register_reloc_addresses(extras, 0, extras.length());
+ }
+#endif // INCLUDE_ZGC
+ return start;
+ }
+
Label L_load_element, L_store_element, L_do_card_marks, L_done;
// Input registers (after setup_arg_regs)
@@ -2479,7 +2952,7 @@ address StubGenerator::generate_checkcast_copy(StubId stub_id, address *entry) {
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter(); // required for proper stackwalking of RuntimeStub frame
@@ -2504,6 +2977,7 @@ address StubGenerator::generate_checkcast_copy(StubId stub_id, address *entry) {
// Caller of this entry point must set up the argument registers.
if (entry != nullptr) {
*entry = __ pc();
+ entries.append(*entry);
BLOCK_COMMENT("Entry:");
}
@@ -2638,6 +3112,16 @@ address StubGenerator::generate_checkcast_copy(StubId stub_id, address *entry) {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ address end = __ pc();
+#if INCLUDE_ZGC
+ // retrieve addresses at which ZGC does colour patching
+ if (UseZGC) {
+ retrieve_reloc_addresses(start, end, extras);
+ }
+#endif // INCLUDE_ZGC
+ // record the stub entry and end plus the no_push entry
+ store_archive_data(stub_id, start, end, entries_ptr, extras_ptr);
+
return start;
}
@@ -2657,6 +3141,14 @@ address StubGenerator::generate_checkcast_copy(StubId stub_id, address *entry) {
address StubGenerator::generate_unsafe_copy(address byte_copy_entry, address short_copy_entry,
address int_copy_entry, address long_copy_entry) {
+ StubId stub_id = StubId::stubgen_unsafe_arraycopy_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+
Label L_long_aligned, L_int_aligned, L_short_aligned;
// Input registers (before setup_arg_regs)
@@ -2668,9 +3160,8 @@ address StubGenerator::generate_unsafe_copy(address byte_copy_entry, address sho
const Register bits = rax; // test copy of low bits
__ align(CodeEntryAlignment);
- StubId stub_id = StubId::stubgen_unsafe_arraycopy_id;
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter(); // required for proper stackwalking of RuntimeStub frame
@@ -2702,6 +3193,9 @@ address StubGenerator::generate_unsafe_copy(address byte_copy_entry, address sho
__ shrptr(size, LogBytesPerLong); // size => qword_count
__ jump(RuntimeAddress(long_copy_entry));
+ // record the stub entry and end plus
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -2803,10 +3297,23 @@ static void do_setmemory_atomic_loop(USM_TYPE type, Register dest,
// to an int, short, or byte fill loop.
//
address StubGenerator::generate_unsafe_setmemory(address unsafe_byte_fill) {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_unsafe_setmemory_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ // we expect three set of extra unsafememory access handler entries
+ GrowableArray extras;
+ int expected_handler_count = 3 * UnsafeMemoryAccess::COLUMN_COUNT;
+ address start = load_archive_data(stub_id, nullptr, &extras);
+ if (start != nullptr) {
+ assert(extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+ register_unsafe_access_handlers(extras, 0, 3);
+ return start;
+ }
+
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter(); // required for proper stackwalking of RuntimeStub frame
assert(unsafe_byte_fill != nullptr, "Invalid call");
@@ -2896,6 +3403,16 @@ address StubGenerator::generate_unsafe_setmemory(address unsafe_byte_fill) {
__ jump(RuntimeAddress(unsafe_byte_fill));
}
+ // retrieve the registered handler addresses
+ address end = __ pc();
+ retrieve_unsafe_access_handlers(start, end, extras);
+ assert(extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+
+ // record the stub entry and end plus the no_push entry and any
+ // extra handler addresses
+ store_archive_data(stub_id, start, end, nullptr, &extras);
+
return start;
}
@@ -2952,7 +3469,15 @@ address StubGenerator::generate_generic_copy(address byte_copy_entry, address sh
address int_copy_entry, address oop_copy_entry,
address long_copy_entry, address checkcast_copy_entry) {
- Label L_failed, L_failed_0, L_objArray;
+ StubId stub_id = StubId::stubgen_generic_arraycopy_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+
+ Label L_failed, L_failed_0, L_skip_failed_0, L_objArray;
Label L_copy_shorts, L_copy_ints, L_copy_longs;
// Input registers
@@ -2968,22 +3493,9 @@ address StubGenerator::generate_generic_copy(address byte_copy_entry, address sh
const Register rklass_tmp = rdi; // load_klass
#endif
- { int modulus = CodeEntryAlignment;
- int target = modulus - 5; // 5 = sizeof jmp(L_failed)
- int advance = target - (__ offset() % modulus);
- if (advance < 0) advance += modulus;
- if (advance > 0) __ nop(advance);
- }
- StubId stub_id = StubId::stubgen_generic_arraycopy_id;
StubCodeMark mark(this, stub_id);
-
- // Short-hop target to L_failed. Makes for denser prologue code.
- __ BIND(L_failed_0);
- __ jmp(L_failed);
- assert(__ offset() % CodeEntryAlignment == 0, "no further alignment needed");
-
__ align(CodeEntryAlignment);
- address start = __ pc();
+ start = __ pc();
__ enter(); // required for proper stackwalking of RuntimeStub frame
@@ -3024,7 +3536,8 @@ address StubGenerator::generate_generic_copy(address byte_copy_entry, address sh
// if (dst_pos < 0) return -1;
__ testl(dst_pos, dst_pos); // dst_pos (32-bits)
size_t j4off = __ offset();
- __ jccb(Assembler::negative, L_failed_0);
+ // skip over the failure trampoline
+ __ jccb(Assembler::positive, L_skip_failed_0);
// The first four tests are very dense code,
// but not quite dense enough to put four
@@ -3034,6 +3547,13 @@ address StubGenerator::generate_generic_copy(address byte_copy_entry, address sh
// Make sure of this.
guarantee(((j1off ^ j4off) & ~15) != 0, "I$ line of 1st & 4th jumps");
+ // Short-hop target to L_failed. Makes for denser prologue code.
+ __ BIND(L_failed_0);
+ __ jmp(L_failed);
+
+ // continue here if first 4 checks pass
+ __ bind(L_skip_failed_0);
+
// registers used as temp
const Register r11_length = r11; // elements count to copy
const Register r10_src_klass = r10; // array klass
@@ -3256,6 +3776,9 @@ __ BIND(L_failed);
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_cbrt.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_cbrt.cpp
index 73330dedc0f..4c647b7d9dc 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_cbrt.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_cbrt.cpp
@@ -191,8 +191,14 @@ ATTRIBUTE_ALIGNED(4) static const juint _D_table[] =
address StubGenerator::generate_libmCbrt() {
StubId stub_id = StubId::stubgen_dcbrt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_2TAG_PACKET_0_0_1, L_2TAG_PACKET_1_0_1, L_2TAG_PACKET_2_0_1;
Label B1_1, B1_2, B1_4;
@@ -335,7 +341,34 @@ address StubGenerator::generate_libmCbrt() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
#undef __
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_cbrt(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr))
+ ADD(_ABS_MASK);
+ ADD(_SIG_MASK);
+ ADD(_EXP_MASK);
+ ADD(_EXP_MSK2);
+ ADD(_EXP_MSK3);
+ ADD(_SCALE63);
+ ADD(_ZERON);
+ ADD(_INF);
+ ADD(_NEG_INF);
+ address coeff_table = (address)_coeff_table;
+ ADD(coeff_table);
+ ADD(coeff_table + 16);
+ ADD(coeff_table + 32);
+ ADD(coeff_table + 48);
+ ADD(_rcp_table);
+ ADD(_cbrt_table);
+ ADD(_D_table);
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_chacha.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_chacha.cpp
index 7afaf34e031..1fa51cd2f18 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_chacha.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_chacha.cpp
@@ -111,10 +111,16 @@ void StubGenerator::generate_chacha_stubs() {
/* The 2-block AVX/AVX2-enabled ChaCha20 block function implementation */
address StubGenerator::generate_chacha20Block_avx() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_chacha20Block_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_twoRounds;
const Register state = c_rarg0;
@@ -295,15 +301,25 @@ address StubGenerator::generate_chacha20Block_avx() {
}
__ leave();
__ ret(0);
+
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
/* The 4-block AVX512-enabled ChaCha20 block function implementation */
address StubGenerator::generate_chacha20Block_avx512() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_chacha20Block_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_twoRounds;
const Register state = c_rarg0;
@@ -466,6 +482,10 @@ address StubGenerator::generate_chacha20Block_avx512() {
__ vzeroupper();
__ leave();
__ ret(0);
+
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -584,3 +604,13 @@ bVec,
}
#undef __
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_chacha(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr))
+ ADD(CC20_COUNTER_ADD_AVX);
+ ADD(CC20_COUNTER_ADD_AVX512);
+ ADD(CC20_LROT_CONSTS);
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_constants.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_constants.cpp
index 93fa7e650db..19e1ca680b3 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_constants.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_constants.cpp
@@ -233,3 +233,30 @@ ATTRIBUTE_ALIGNED(16) static const juint _Ctable[] = {
};
address StubGenerator::Ctable = (address)_Ctable;
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_constants(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr))
+ ADD(_ONE);
+ ADD(_ONEHALF);
+ ADD(_SIGN_MASK);
+ ADD(_TWO_POW_55);
+ ADD(_TWO_POW_M55);
+ ADD(_SHIFTER);
+ ADD(_ZERO);
+ ADD(_SC_1);
+ ADD(_SC_2);
+ ADD(_SC_3);
+ ADD(_SC_4);
+ // Use value which was already cast to (address): StubGenerator::PI_4;
+ ADD(PI_4);
+ ADD(PI_4 + 8);
+ ADD(_PI32INV);
+ ADD(_NEG_ZERO);
+ ADD(_P_1);
+ ADD(_P_2);
+ ADD(_P_3);
+ ADD(_PI_INV_TABLE);
+ ADD(_Ctable);
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_cos.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_cos.cpp
index 8cb6ead21fd..8dedd50cd97 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_cos.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_cos.cpp
@@ -174,8 +174,14 @@
address StubGenerator::generate_libmCos() {
StubId stub_id = StubId::stubgen_dcos_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_2TAG_PACKET_0_0_1, L_2TAG_PACKET_1_0_1, L_2TAG_PACKET_2_0_1, L_2TAG_PACKET_3_0_1;
Label L_2TAG_PACKET_4_0_1, L_2TAG_PACKET_5_0_1, L_2TAG_PACKET_6_0_1, L_2TAG_PACKET_7_0_1;
@@ -619,6 +625,9 @@ address StubGenerator::generate_libmCos() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_dilithium.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_dilithium.cpp
index b9590939468..de8f52a3c05 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_dilithium.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_dilithium.cpp
@@ -401,10 +401,16 @@ static void storeXmms(Register destination, int offset, const XMMRegister xmmReg
//
static address generate_dilithiumAlmostNtt_avx(StubGenerator *stubgen,
int vector_len, MacroAssembler *_masm) {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_dilithiumAlmostNtt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = stubgen->load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(stubgen, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register coeffs = c_rarg0;
@@ -646,6 +652,9 @@ static address generate_dilithiumAlmostNtt_avx(StubGenerator *stubgen,
__ mov64(rax, 0); // return 0
__ ret(0);
+ // record the stub entry and end
+ stubgen->store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -657,10 +666,16 @@ static address generate_dilithiumAlmostNtt_avx(StubGenerator *stubgen,
// zetas (int[128*8]) = c_rarg1
static address generate_dilithiumAlmostInverseNtt_avx(StubGenerator *stubgen,
int vector_len, MacroAssembler *_masm) {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_dilithiumAlmostInverseNtt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = stubgen->load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(stubgen, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register coeffs = c_rarg0;
@@ -886,6 +901,9 @@ static address generate_dilithiumAlmostInverseNtt_avx(StubGenerator *stubgen,
__ mov64(rax, 0); // return 0
__ ret(0);
+ // record the stub entry and end
+ stubgen->store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -900,10 +918,16 @@ static address generate_dilithiumAlmostInverseNtt_avx(StubGenerator *stubgen,
static address generate_dilithiumNttMult_avx(StubGenerator *stubgen,
int vector_len, MacroAssembler *_masm) {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_dilithiumNttMult_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = stubgen->load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(stubgen, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
Label L_loop;
@@ -972,6 +996,9 @@ static address generate_dilithiumNttMult_avx(StubGenerator *stubgen,
__ mov64(rax, 0); // return 0
__ ret(0);
+ // record the stub entry and end
+ stubgen->store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -984,10 +1011,16 @@ static address generate_dilithiumNttMult_avx(StubGenerator *stubgen,
static address generate_dilithiumMontMulByConstant_avx(StubGenerator *stubgen,
int vector_len, MacroAssembler *_masm) {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_dilithiumMontMulByConstant_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = stubgen->load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(stubgen, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
Label L_loop;
@@ -1059,6 +1092,9 @@ static address generate_dilithiumMontMulByConstant_avx(StubGenerator *stubgen,
__ mov64(rax, 0); // return 0
__ ret(0);
+ // record the stub entry and end
+ stubgen->store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -1073,10 +1109,16 @@ static address generate_dilithiumMontMulByConstant_avx(StubGenerator *stubgen,
// multiplier (int) = c_rarg4
static address generate_dilithiumDecomposePoly_avx(StubGenerator *stubgen,
int vector_len, MacroAssembler *_masm) {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_dilithiumDecomposePoly_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = stubgen->load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(stubgen, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
Label L_loop;
@@ -1318,6 +1360,9 @@ static address generate_dilithiumDecomposePoly_avx(StubGenerator *stubgen,
__ mov64(rax, 0); // return 0
__ ret(0);
+ // record the stub entry and end
+ stubgen->store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -1340,3 +1385,21 @@ void StubGenerator::generate_dilithium_stubs() {
generate_dilithiumDecomposePoly_avx(this, vector_len, _masm);
}
}
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_dilithium(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr))
+ // use accessors to correctly identify the relevant addresses
+ ADD(unshufflePermsAddr(0));
+ ADD(unshufflePermsAddr(1));
+ ADD(unshufflePermsAddr(2));
+ ADD(unshufflePermsAddr(3));
+ ADD(unshufflePermsAddr(4));
+ ADD(unshufflePermsAddr(5));
+ ADD(dilithiumAvx512ConstsAddr(montQInvModRIdx));
+ ADD(dilithiumAvx512ConstsAddr(dilithium_qIdx));
+ ADD(dilithiumAvx512ConstsAddr(montRSquareModQIdx));
+ ADD(dilithiumAvx512ConstsAddr(barrettAddendIdx));
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_exp.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_exp.cpp
index 5130fd2c9d2..3c8babcbecf 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_exp.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_exp.cpp
@@ -166,8 +166,14 @@ ATTRIBUTE_ALIGNED(4) static const juint _INF[] =
address StubGenerator::generate_libmExp() {
StubId stub_id = StubId::stubgen_dexp_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_2TAG_PACKET_0_0_2, L_2TAG_PACKET_1_0_2, L_2TAG_PACKET_2_0_2, L_2TAG_PACKET_3_0_2;
Label L_2TAG_PACKET_4_0_2, L_2TAG_PACKET_5_0_2, L_2TAG_PACKET_6_0_2, L_2TAG_PACKET_7_0_2;
@@ -381,7 +387,32 @@ address StubGenerator::generate_libmExp() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
#undef __
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_exp(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr));
+ address cv = (address)_cv;
+ ADD(cv);
+ ADD(cv + 16);
+ ADD(cv + 32);
+ ADD(cv + 48);
+ ADD(cv + 64);
+ ADD(cv + 80);
+ ADD(_mmask);
+ ADD(_bias);
+ ADD(_Tbl_addr);
+ ADD(_ALLONES);
+ ADD(_ebias);
+ ADD(_XMAX);
+ ADD(_XMIN);
+ ADD(_INF);
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_fmod.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_fmod.cpp
index b1eaa4b8031..f53985a13b7 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_fmod.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_fmod.cpp
@@ -72,13 +72,19 @@ ATTRIBUTE_ALIGNED(32) static const uint64_t CONST_e307[] = {
};
address StubGenerator::generate_libmFmod() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_fmod_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter(); // required for proper stackwalking of RuntimeStub frame
- if (VM_Version::supports_avx512vlbwdq()) { // AVX512 version
+ if (VM_Version::supports_avx512vlbwdq() && VM_Version::supports_fma()) { // AVX512 version
// Source used to generate the AVX512 fmod assembly below:
//
@@ -521,7 +527,22 @@ address StubGenerator::generate_libmFmod() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
#undef __
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_fmod(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr));
+ ADD(CONST_NaN);
+ ADD(CONST_1p260);
+ ADD(CONST_MAX);
+ ADD(CONST_INF);
+ ADD(CONST_e307);
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_ghash.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_ghash.cpp
index 6f05b1ab5e6..9ebab07589e 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_ghash.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_ghash.cpp
@@ -80,11 +80,17 @@ void StubGenerator::generate_ghash_stubs() {
// Single and multi-block ghash operations.
address StubGenerator::generate_ghash_processBlocks() {
- __ align(CodeEntryAlignment);
- Label L_ghash_loop, L_exit;
StubId stub_id = StubId::stubgen_ghash_processBlocks_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ Label L_ghash_loop, L_exit;
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register state = c_rarg0;
const Register subkeyH = c_rarg1;
@@ -211,17 +217,25 @@ address StubGenerator::generate_ghash_processBlocks() {
__ leave();
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
// Ghash single and multi block operations using AVX instructions
address StubGenerator::generate_avx_ghash_processBlocks() {
- __ align(CodeEntryAlignment);
-
StubId stub_id = StubId::stubgen_ghash_processBlocks_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
// arguments
const Register state = c_rarg0;
@@ -237,6 +251,9 @@ address StubGenerator::generate_avx_ghash_processBlocks() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -538,3 +555,14 @@ void StubGenerator::generateHtbl_eight_blocks(Register htbl) {
}
#undef __
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_ghash(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr));
+ ADD(GHASH_SHUFFLE_MASK);
+ ADD(GHASH_LONG_SWAP_MASK);
+ ADD(GHASH_BYTE_SWAP_MASK);
+ ADD(GHASH_POLYNOMIAL);
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_kyber.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_kyber.cpp
index 7d5dee6a5df..347a9b936a8 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_kyber.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_kyber.cpp
@@ -400,10 +400,16 @@ static int xmm29_29[] = {29, 29, 29, 29};
// ntt_zetas (short[256]) = c_rarg1
address generate_kyberNtt_avx512(StubGenerator *stubgen,
MacroAssembler *_masm) {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_kyberNtt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = stubgen->load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(stubgen, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register coeffs = c_rarg0;
@@ -487,6 +493,9 @@ address generate_kyberNtt_avx512(StubGenerator *stubgen,
__ mov64(rax, 0); // return 0
__ ret(0);
+ // record the stub entry and end
+ stubgen->store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -496,11 +505,16 @@ address generate_kyberNtt_avx512(StubGenerator *stubgen,
// ntt_zetas (short[256]) = c_rarg1
address generate_kyberInverseNtt_avx512(StubGenerator *stubgen,
MacroAssembler *_masm) {
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_kyberInverseNtt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = stubgen->load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(stubgen, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register coeffs = c_rarg0;
@@ -610,6 +624,9 @@ address generate_kyberInverseNtt_avx512(StubGenerator *stubgen,
__ mov64(rax, 0); // return 0
__ ret(0);
+ // record the stub entry and end
+ stubgen->store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -621,11 +638,16 @@ address generate_kyberInverseNtt_avx512(StubGenerator *stubgen,
// zetas (short[128]) = c_rarg3
address generate_kyberNttMult_avx512(StubGenerator *stubgen,
MacroAssembler *_masm) {
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_kyberNttMult_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = stubgen->load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(stubgen, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register result = c_rarg0;
@@ -731,6 +753,9 @@ address generate_kyberNttMult_avx512(StubGenerator *stubgen,
__ mov64(rax, 0); // return 0
__ ret(0);
+ // record the stub entry and end
+ stubgen->store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -741,11 +766,16 @@ address generate_kyberNttMult_avx512(StubGenerator *stubgen,
// b (short[256]) = c_rarg2
address generate_kyberAddPoly_2_avx512(StubGenerator *stubgen,
MacroAssembler *_masm) {
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_kyberAddPoly_2_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = stubgen->load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(stubgen, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register result = c_rarg0;
@@ -776,6 +806,9 @@ address generate_kyberAddPoly_2_avx512(StubGenerator *stubgen,
__ mov64(rax, 0); // return 0
__ ret(0);
+ // record the stub entry and end
+ stubgen->store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -787,11 +820,16 @@ address generate_kyberAddPoly_2_avx512(StubGenerator *stubgen,
// c (short[256]) = c_rarg3
address generate_kyberAddPoly_3_avx512(StubGenerator *stubgen,
MacroAssembler *_masm) {
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_kyberAddPoly_3_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = stubgen->load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(stubgen, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register result = c_rarg0;
@@ -830,6 +868,9 @@ address generate_kyberAddPoly_3_avx512(StubGenerator *stubgen,
__ mov64(rax, 0); // return 0
__ ret(0);
+ // record the stub entry and end
+ stubgen->store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -841,11 +882,16 @@ address generate_kyberAddPoly_3_avx512(StubGenerator *stubgen,
// parsedLength (int) = c_rarg3
address generate_kyber12To16_avx512(StubGenerator *stubgen,
MacroAssembler *_masm) {
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_kyber12To16_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = stubgen->load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(stubgen, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register condensed = c_rarg0;
@@ -984,6 +1030,9 @@ address generate_kyber12To16_avx512(StubGenerator *stubgen,
__ mov64(rax, 0); // return 0
__ ret(0);
+ // record the stub entry and end
+ stubgen->store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -993,11 +1042,16 @@ address generate_kyber12To16_avx512(StubGenerator *stubgen,
// coeffs (short[256]) = c_rarg0
address generate_kyberBarrettReduce_avx512(StubGenerator *stubgen,
MacroAssembler *_masm) {
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_kyberBarrettReduce_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = stubgen->load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(stubgen, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register coeffs = c_rarg0;
@@ -1021,6 +1075,9 @@ address generate_kyberBarrettReduce_avx512(StubGenerator *stubgen,
__ mov64(rax, 0); // return 0
__ ret(0);
+ // record the stub entry and end
+ stubgen->store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -1038,3 +1095,24 @@ void StubGenerator::generate_kyber_stubs() {
}
}
}
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_kyber(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr))
+ // use accessors to correctly identify the relevant addresses
+ ADD(kyberAvx512NttPermsAddr());
+ ADD(kyberAvx512InverseNttPermsAddr());
+ ADD(kyberAvx512_nttMultPermsAddr());
+ ADD(kyberAvx512_12To16PermsAddr());
+ ADD(kyberAvx512_12To16DupAddr());
+ ADD(kyberAvx512_12To16ShiftAddr());
+ ADD(kyberAvx512_12To16AndAddr());
+ ADD(kyberAvx512ConstsAddr(qOffset));
+ ADD(kyberAvx512ConstsAddr(qInvModROffset));
+ ADD(kyberAvx512ConstsAddr(dimHalfInverseOffset));
+ ADD(kyberAvx512ConstsAddr(barretMultiplierOffset));
+ ADD(kyberAvx512ConstsAddr(montRSquareModqOffset));
+ ADD(kyberAvx512ConstsAddr(f00Offset));
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_log.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_log.cpp
index 6b5b4d704e3..07683a51e3d 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_log.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_log.cpp
@@ -177,8 +177,14 @@ ATTRIBUTE_ALIGNED(16) static const juint _coeff[] =
address StubGenerator::generate_libmLog() {
StubId stub_id = StubId::stubgen_dlog_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_2TAG_PACKET_0_0_2, L_2TAG_PACKET_1_0_2, L_2TAG_PACKET_2_0_2, L_2TAG_PACKET_3_0_2;
Label L_2TAG_PACKET_4_0_2, L_2TAG_PACKET_5_0_2, L_2TAG_PACKET_6_0_2, L_2TAG_PACKET_7_0_2;
@@ -359,6 +365,9 @@ address StubGenerator::generate_libmLog() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -516,8 +525,14 @@ ATTRIBUTE_ALIGNED(16) static const juint _coeff_log10[] =
address StubGenerator::generate_libmLog10() {
StubId stub_id = StubId::stubgen_dlog10_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_2TAG_PACKET_0_0_2, L_2TAG_PACKET_1_0_2, L_2TAG_PACKET_2_0_2, L_2TAG_PACKET_3_0_2;
Label L_2TAG_PACKET_4_0_2, L_2TAG_PACKET_5_0_2, L_2TAG_PACKET_6_0_2, L_2TAG_PACKET_7_0_2;
@@ -704,7 +719,38 @@ address StubGenerator::generate_libmLog10() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
#undef __
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_log(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr));
+ address log2 = (address)_log2;
+ address coeff = (address)_coeff;
+ address LOG10_E = (address)_LOG10_E;
+ address log2_log10 = (address)_log2_log10;
+ address coeff_log10 = (address)_coeff_log10;
+
+ ADD(_L_tbl);
+ ADD(log2);
+ ADD(log2 + 8);
+ ADD(coeff);
+ ADD(coeff + 16);
+ ADD(coeff + 32);
+ ADD(_HIGHSIGMASK_log10);
+ ADD(LOG10_E);
+ ADD(LOG10_E + 8);
+ ADD(_L_tbl_log10);
+ ADD(log2_log10);
+ ADD(log2_log10 + 8);
+ ADD(coeff_log10);
+ ADD(coeff_log10 + 16);
+ ADD(coeff_log10 + 32);
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_poly1305.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_poly1305.cpp
index c80b2d16181..ea7e6d64254 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_poly1305.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_poly1305.cpp
@@ -909,10 +909,16 @@ void StubGenerator::poly1305_process_blocks_avx512(
// After execution, input and length will point at remaining (unprocessed) data
// and accumulator will point to the current accumulator value
address StubGenerator::generate_poly1305_processBlocks() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_poly1305_processBlocks_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
// Save all 'SOE' registers
@@ -1028,6 +1034,10 @@ address StubGenerator::generate_poly1305_processBlocks() {
__ leave();
__ ret(0);
+
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -1695,3 +1705,14 @@ void StubGenerator::poly1305_msg_mul_reduce_vec4_avx2(
__ vpaddq(A1, A1, YTMP2, Assembler::AVX_256bit); //Add medium 42-bit bits from new blocks to accumulator
__ vpaddq(A1, A1, YTMP5, Assembler::AVX_256bit);
}
+#undef __
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_poly1305(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr));
+ ADD(POLY1305_PAD_MSG);
+ ADD(POLY1305_MASK42);
+ ADD(POLY1305_MASK44);
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_poly_mont.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_poly_mont.cpp
index c439e0b370f..308a8042993 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_poly_mont.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_poly_mont.cpp
@@ -558,10 +558,16 @@ void montgomeryMultiplyAVX2(const Register aLimbs, const Register bLimbs, const
}
address StubGenerator::generate_intpoly_montgomeryMult_P256() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_intpoly_montgomeryMult_P256_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
if (VM_Version::supports_avx512ifma() && VM_Version::supports_avx512vlbw()) {
@@ -620,6 +626,10 @@ address StubGenerator::generate_intpoly_montgomeryMult_P256() {
__ leave();
__ ret(0);
+
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -680,10 +690,16 @@ address StubGenerator::generate_intpoly_assign() {
// P521OrderField: 19 = 8 + 8 + 2 + 1
// Special Cases 5, 10, 14, 16, 19
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_intpoly_assign_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
// Inputs
@@ -762,5 +778,24 @@ address StubGenerator::generate_intpoly_assign() {
__ bind(L_Done);
__ leave();
__ ret(0);
+
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
+#undef __
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_poly_mont(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr));
+ // use accessors to retrieve all correct addresses
+ ADD(shift_1L());
+ ADD(shift_1R());
+ ADD(p256_mask52());
+ ADD(mask_limb5());
+ ADD(modulus_p256());
+ ADD(modulus_p256(1));
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_pow.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_pow.cpp
index 3c3df7e6ac4..a9a6dc10da4 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_pow.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_pow.cpp
@@ -760,8 +760,14 @@ ATTRIBUTE_ALIGNED(8) static const juint _DOUBLE0DOT5[] = {
address StubGenerator::generate_libmPow() {
StubId stub_id = StubId::stubgen_dpow_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_2TAG_PACKET_0_0_2, L_2TAG_PACKET_1_0_2, L_2TAG_PACKET_2_0_2, L_2TAG_PACKET_3_0_2;
Label L_2TAG_PACKET_4_0_2, L_2TAG_PACKET_5_0_2, L_2TAG_PACKET_6_0_2, L_2TAG_PACKET_7_0_2;
@@ -1859,7 +1865,45 @@ address StubGenerator::generate_libmPow() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
#undef __
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_pow(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr));
+ address HIGHMASK_Y = (address)_HIGHMASK_Y;
+ address e_coeff = (address)_e_coeff;
+ address coeff_h = (address)_coeff_h;
+ address coeff_pow = (address)_coeff_pow;
+
+ ADD(_HIGHSIGMASK);
+ ADD(_LOG2_E);
+ ADD(HIGHMASK_Y);
+ ADD(HIGHMASK_Y + 8);
+ ADD(_T_exp);
+ ADD(e_coeff);
+ ADD(e_coeff + 16);
+ ADD(e_coeff + 32);
+ ADD(coeff_h);
+ ADD(coeff_h + 8);
+ ADD(_HIGHMASK_LOG_X);
+ ADD(_HALFMASK);
+ ADD(coeff_pow);
+ ADD(coeff_pow + 16);
+ ADD(coeff_pow + 32);
+ ADD(coeff_pow + 48);
+ ADD(coeff_pow + 64);
+ ADD(coeff_pow + 80);
+ ADD(_L_tbl_pow);
+ ADD(_log2_pow);
+ ADD(_DOUBLE2);
+ ADD(_DOUBLE0);
+ ADD(_DOUBLE0DOT5);
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_sha3.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_sha3.cpp
index f9d876f34f3..58f81652a0c 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_sha3.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_sha3.cpp
@@ -104,10 +104,15 @@ static address generate_sha3_implCompress(StubId stub_id,
default:
ShouldNotReachHere();
}
-
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = stubgen->load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(stubgen, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register buf = c_rarg0;
const Register state = c_rarg1;
@@ -316,6 +321,9 @@ static address generate_sha3_implCompress(StubId stub_id,
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ stubgen->store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -326,10 +334,16 @@ static address generate_sha3_implCompress(StubId stub_id,
// Performs two keccak() computations in parallel. The steps of the
// two computations are executed interleaved.
static address generate_double_keccak(StubGenerator *stubgen, MacroAssembler *_masm) {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_double_keccak_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = stubgen->load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(stubgen, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register state0 = c_rarg0;
const Register state1 = c_rarg1;
@@ -495,6 +509,9 @@ static address generate_double_keccak(StubGenerator *stubgen, MacroAssembler *_m
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ stubgen->store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -508,3 +525,14 @@ void StubGenerator::generate_sha3_stubs() {
generate_sha3_implCompress(StubId::stubgen_sha3_implCompressMB_id, this, _masm);
}
}
+
+#undef __
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_sha3(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr));
+ ADD(round_constsAddr());
+ ADD(permsAndRotsAddr());
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_sin.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_sin.cpp
index 5290e737581..00c759a369b 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_sin.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_sin.cpp
@@ -181,8 +181,14 @@ ATTRIBUTE_ALIGNED(8) static const juint _ALL_ONES[] =
address StubGenerator::generate_libmSin() {
StubId stub_id = StubId::stubgen_dsin_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_2TAG_PACKET_0_0_1, L_2TAG_PACKET_1_0_1, L_2TAG_PACKET_2_0_1, L_2TAG_PACKET_3_0_1;
Label L_2TAG_PACKET_4_0_1, L_2TAG_PACKET_5_0_1, L_2TAG_PACKET_6_0_1, L_2TAG_PACKET_7_0_1;
@@ -645,7 +651,18 @@ address StubGenerator::generate_libmSin() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
#undef __
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_sin(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr));
+ ADD(_ALL_ONES);
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_sinh.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_sinh.cpp
index 86e4ac20176..9969866cfc7 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_sinh.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_sinh.cpp
@@ -290,8 +290,14 @@ ATTRIBUTE_ALIGNED(16) static const juint _T2_neg_f[] =
address StubGenerator::generate_libmSinh() {
StubId stub_id = StubId::stubgen_dsinh_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_2TAG_PACKET_0_0_2, L_2TAG_PACKET_1_0_2, L_2TAG_PACKET_3_0_2, L_2TAG_PACKET_4_0_2;
Label L_2TAG_PACKET_5_0_2, L_2TAG_PACKET_6_0_2;
@@ -519,7 +525,36 @@ address StubGenerator::generate_libmSinh() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
#undef __
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_sinh(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr));
+ address L2E = (address)_L2E;
+ address cv = (address)_cv;
+ address pv = (address)_pv;
+
+ ADD(L2E);
+ ADD(L2E + 8);
+ ADD(_HALFMASK);
+ ADD(_Shifter);
+ ADD(cv);
+ ADD(cv + 16);
+ ADD(cv + 32);
+ ADD(cv + 48);
+ ADD(cv + 64);
+ ADD(_T2f);
+ ADD(_T2_neg_f);
+ ADD(pv);
+ ADD(pv + 16);
+ ADD(pv + 32);
+ ADD(_MASK3);
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_tan.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_tan.cpp
index 4f14414652c..9f91b9e8f84 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_tan.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_tan.cpp
@@ -456,8 +456,14 @@ ATTRIBUTE_ALIGNED(8) static const juint _QQ_2_tan[] =
address StubGenerator::generate_libmTan() {
StubId stub_id = StubId::stubgen_dtan_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_2TAG_PACKET_0_0_1, L_2TAG_PACKET_1_0_1, L_2TAG_PACKET_2_0_1, L_2TAG_PACKET_3_0_1;
Label L_2TAG_PACKET_4_0_1, L_2TAG_PACKET_5_0_1, L_2TAG_PACKET_6_0_1, L_2TAG_PACKET_7_0_1;
@@ -1025,7 +1031,35 @@ address StubGenerator::generate_libmTan() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
#undef __
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_tan(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr));
+ address PI_4_tan = (address)_PI_4_tan;
+
+ ADD(_MUL16);
+ ADD(_sign_mask_tan);
+ ADD(_PI32INV_tan);
+ ADD(_P_1_tan);
+ ADD(_P_2_tan);
+ ADD(_P_3_tan);
+ ADD(_Ctable_tan);
+ ADD(_MASK_35_tan);
+ ADD(_Q_11_tan);
+ ADD(_Q_9_tan);
+ ADD(_Q_7_tan);
+ ADD(_Q_5_tan);
+ ADD(_Q_3_tan);
+ ADD(PI_4_tan);
+ ADD(PI_4_tan + 8);
+ ADD(_QQ_2_tan);
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_tanh.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_tanh.cpp
index dce4fbfc455..4f2fe8a460b 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_tanh.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_tanh.cpp
@@ -303,8 +303,14 @@ ATTRIBUTE_ALIGNED(16) static const juint _T2_neg_f[] =
address StubGenerator::generate_libmTanh() {
StubId stub_id = StubId::stubgen_dtanh_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_2TAG_PACKET_0_0_1, L_2TAG_PACKET_1_0_1, L_2TAG_PACKET_2_0_1, L_2TAG_PACKET_3_0_1;
Label L_2TAG_PACKET_4_0_1, L_2TAG_PACKET_5_0_1;
@@ -495,7 +501,36 @@ address StubGenerator::generate_libmTanh() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
#undef __
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_tanh(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr));
+ address L2E = (address)_L2E;
+ address cv = (address)_cv;
+ address pv = (address)_pv;
+
+ ADD(L2E);
+ ADD(L2E + 8);
+ ADD(_HALFMASK);
+ ADD(_ONEMASK);
+ ADD(_TWOMASK);
+ ADD(_Shifter);
+ ADD(cv);
+ ADD(cv + 16);
+ ADD(cv + 32);
+ ADD(_T2_neg_f);
+ ADD(pv);
+ ADD(pv + 16);
+ ADD(pv + 32);
+ ADD(_MASK3);
+ ADD(_RMASK);
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubRoutines_x86.cpp b/src/hotspot/cpu/x86/stubRoutines_x86.cpp
index ee9cea08e64..ce11925dde2 100644
--- a/src/hotspot/cpu/x86/stubRoutines_x86.cpp
+++ b/src/hotspot/cpu/x86/stubRoutines_x86.cpp
@@ -28,6 +28,10 @@
#include "runtime/stubRoutines.hpp"
#include "utilities/globalDefinitions.hpp"
#include "crc32c.h"
+#include "stubGenerator_x86_64.hpp"
+#ifdef COMPILER1
+#include "c1/c1_LIRAssembler.hpp"
+#endif
// Implementation of the platform-specific part of StubRoutines - for
// a description of how to extend it, see the stubRoutines.hpp file.
@@ -40,8 +44,12 @@
#define DEFINE_ARCH_ENTRY_INIT(arch, blob_name, stub_name, field_name, getter_name, init_function) \
address StubRoutines:: arch :: STUB_FIELD_NAME(field_name) = CAST_FROM_FN_PTR(address, init_function);
-STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY, DEFINE_ARCH_ENTRY_INIT)
+#define DEFINE_ARCH_ENTRY_ARRAY(arch, blob_name, stub_name, field_name, getter_name, count) \
+ address StubRoutines:: arch :: STUB_FIELD_NAME(field_name) [count];
+STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY, DEFINE_ARCH_ENTRY_INIT, DEFINE_ARCH_ENTRY_ARRAY)
+
+#undef DEFINE_ARCH_ENTRY_ARRAY
#undef DEFINE_ARCH_ENTRY_INIT
#undef DEFINE_ARCH_ENTRY
@@ -411,3 +419,46 @@ ATTRIBUTE_ALIGNED(64) const julong StubRoutines::x86::_k512_W[] =
0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL,
0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL,
};
+
+#if INCLUDE_CDS
+
+void StubRoutines::init_AOTAddressTable() {
+ ResourceMark rm;
+ GrowableArray external_addresses;
+ // publish static addresses referred to by main x86 generator and
+ // auxiliary x86 generators
+ StubGenerator::init_AOTAddressTable(external_addresses);
+ // publish external data addresses defined in nested x86 class
+ StubRoutines::x86::init_AOTAddressTable(external_addresses);
+#ifdef COMPILER1
+ LIR_Assembler::init_AOTAddressTable(external_addresses);
+#endif
+ AOTCodeCache::publish_external_addresses(external_addresses);
+}
+
+// publish addresses of external data defined in this file which may
+// be referenced from stub or code
+void StubRoutines::x86::init_AOTAddressTable(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr));
+ ADD(&_mxcsr_std);
+ ADD(&_mxcsr_rz);
+ ADD(crc_by128_masks_addr());
+ ADD(crc_by128_masks_addr() + 16);
+ ADD(crc_by128_masks_addr() + 32);
+ // this is added in generic code
+ // ADD(_crc_table);
+ ADD(crc_by128_masks_avx512_addr());
+ ADD(crc_by128_masks_avx512_addr() + 16);
+ ADD(crc_by128_masks_avx512_addr() + 32);
+ ADD(_crc_table_avx512);
+ ADD(_crc32c_table_avx512);
+ ADD(_shuf_table_crc32_avx512);
+ // n.b. call accessor for this one to ensure the table is generated
+ ADD(crc32c_table_addr());
+ ADD(_arrays_hashcode_powers_of_31);
+ ADD(_k256);
+ ADD(_k256_W);
+ ADD(_k512_W);
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubRoutines_x86.hpp b/src/hotspot/cpu/x86/stubRoutines_x86.hpp
index 3654b644131..7283798888b 100644
--- a/src/hotspot/cpu/x86/stubRoutines_x86.hpp
+++ b/src/hotspot/cpu/x86/stubRoutines_x86.hpp
@@ -55,9 +55,13 @@ class x86 {
#define DECLARE_ARCH_ENTRY_INIT(arch, blob_name, stub_name, field_name, getter_name, init_function) \
DECLARE_ARCH_ENTRY(arch, blob_name, stub_name, field_name, getter_name)
-private:
- STUBGEN_ARCH_ENTRIES_DO(DECLARE_ARCH_ENTRY, DECLARE_ARCH_ENTRY_INIT)
+#define DECLARE_ARCH_ENTRY_ARRAY(arch, blob_name, stub_name, field_name, getter_name, count) \
+ static address STUB_FIELD_NAME(field_name) [count] ;
+private:
+ STUBGEN_ARCH_ENTRIES_DO(DECLARE_ARCH_ENTRY, DECLARE_ARCH_ENTRY_INIT, DECLARE_ARCH_ENTRY_ARRAY)
+
+#undef DECLARE_ARCH_ENTRY_ARRAY
#undef DECLARE_ARCH_ENTRY_INIT
#undef DECLARE_ARCH_ENTRY
@@ -70,9 +74,13 @@ private:
#define DEFINE_ARCH_ENTRY_GETTER_INIT(arch, blob_name, stub_name, field_name, getter_name, init_function) \
DEFINE_ARCH_ENTRY_GETTER(arch, blob_name, stub_name, field_name, getter_name)
-public:
- STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY_GETTER, DEFINE_ARCH_ENTRY_GETTER_INIT)
+#define DEFINE_ARCH_ENTRY_GETTER_ARRAY(arch, blob_name, stub_name, field_name, getter_name, count) \
+ static address getter_name(int idx) { return STUB_FIELD_NAME(field_name) [idx]; }
+public:
+ STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY_GETTER, DEFINE_ARCH_ENTRY_GETTER_INIT, DEFINE_ARCH_ENTRY_GETTER_ARRAY)
+
+#undef DEFINE_ARCH_ENTRY_GETTER_ARRAY
#undef DEFINE_ARCH_ENTRY_GETTER_INIT
#undef DEFINE_ARCH_GETTER_ENTRY
@@ -112,6 +120,8 @@ public:
static address arrays_hashcode_powers_of_31() { return (address)_arrays_hashcode_powers_of_31; }
static void generate_CRC32C_table(bool is_pclmulqdq_supported);
+
+ static void init_AOTAddressTable(GrowableArray& external_addresses);
};
#endif // CPU_X86_STUBROUTINES_X86_HPP
diff --git a/src/hotspot/cpu/x86/vm_version_x86.cpp b/src/hotspot/cpu/x86/vm_version_x86.cpp
index 78d6dec08cf..cf9de40a237 100644
--- a/src/hotspot/cpu/x86/vm_version_x86.cpp
+++ b/src/hotspot/cpu/x86/vm_version_x86.cpp
@@ -958,9 +958,17 @@ void VM_Version::get_processor_features() {
if (UseSSE < 1)
_features.clear_feature(CPU_SSE);
- //since AVX instructions is slower than SSE in some ZX cpus, force USEAVX=0.
- if (is_zx() && ((cpu_family() == 6) || (cpu_family() == 7))) {
- UseAVX = 0;
+ // ZX cpus specific settings
+ if (is_zx() && FLAG_IS_DEFAULT(UseAVX)) {
+ if (cpu_family() == 7) {
+ if (extended_cpu_model() == 0x5B || extended_cpu_model() == 0x6B) {
+ UseAVX = 1;
+ } else if (extended_cpu_model() == 0x1B || extended_cpu_model() == 0x3B) {
+ UseAVX = 0;
+ }
+ } else if (cpu_family() == 6) {
+ UseAVX = 0;
+ }
}
// UseSSE is set to the smaller of what hardware supports and what
@@ -1086,15 +1094,36 @@ void VM_Version::get_processor_features() {
}
}
- // Currently APX support is only enabled for targets supporting AVX512VL feature.
- bool apx_supported = os_supports_apx_egprs() && supports_apx_f() && supports_avx512vl();
- if (UseAPX && !apx_supported) {
- warning("UseAPX is not supported on this CPU, setting it to false");
+ // Currently APX support is only enabled for targets supporting AVX512VL feature.
+ if (supports_apx_f() && os_supports_apx_egprs() && supports_avx512vl()) {
+ if (FLAG_IS_DEFAULT(UseAPX)) {
+ UseAPX = false; // by default UseAPX is false
+ _features.clear_feature(CPU_APX_F);
+ } else if (!UseAPX) {
+ _features.clear_feature(CPU_APX_F);
+ }
+ } else if (UseAPX) {
+ if (!FLAG_IS_DEFAULT(UseAPX)) {
+ warning("APX is not supported on this CPU, setting it to false)");
+ }
FLAG_SET_DEFAULT(UseAPX, false);
}
- if (!UseAPX) {
- _features.clear_feature(CPU_APX_F);
+ CHECK_CPU_FEATURE(supports_clmul, CLMUL);
+ CHECK_CPU_FEATURE(supports_aes, AES);
+ CHECK_CPU_FEATURE(supports_fma, FMA);
+
+ if (supports_sha() || (supports_avx2() && supports_bmi2())) {
+ if (FLAG_IS_DEFAULT(UseSHA)) {
+ UseSHA = true;
+ } else if (!UseSHA) {
+ _features.clear_feature(CPU_SHA);
+ }
+ } else if (UseSHA) {
+ if (!FLAG_IS_DEFAULT(UseSHA)) {
+ warning("SHA instructions are not available on this CPU");
+ }
+ FLAG_SET_DEFAULT(UseSHA, false);
}
if (FLAG_IS_DEFAULT(IntelJccErratumMitigation)) {
@@ -1144,10 +1173,50 @@ void VM_Version::get_processor_features() {
// Use AES instructions if available.
if (supports_aes()) {
- if (FLAG_IS_DEFAULT(UseAES)) {
- FLAG_SET_DEFAULT(UseAES, true);
+ if (supports_sse3()) {
+ if (FLAG_IS_DEFAULT(UseAESIntrinsics)) {
+ FLAG_SET_DEFAULT(UseAESIntrinsics, true);
+ }
+ } else if (UseAESIntrinsics) {
+ // The AES intrinsic stubs require AES instruction support (of course)
+ // but also require sse3 mode or higher for instructions it use.
+ if (!FLAG_IS_DEFAULT(UseAESIntrinsics)) {
+ warning("X86 AES intrinsics require SSE3 instructions or higher. Intrinsics will be disabled.");
+ }
+ FLAG_SET_DEFAULT(UseAESIntrinsics, false);
}
- if (!UseAES) {
+ if (!UseAESIntrinsics) {
+ if (UseAESCTRIntrinsics) {
+ if (!FLAG_IS_DEFAULT(UseAESCTRIntrinsics)) {
+ warning("AES-CTR intrinsics require UseAESIntrinsics flag to be enabled. Intrinsics will be disabled.");
+ }
+ FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
+ }
+ } else {
+ if (supports_sse4_1()) {
+ if (FLAG_IS_DEFAULT(UseAESCTRIntrinsics)) {
+ FLAG_SET_DEFAULT(UseAESCTRIntrinsics, true);
+ }
+ } else if (UseAESCTRIntrinsics) {
+ // The AES-CTR intrinsic stubs require AES instruction support (of course)
+ // but also require sse4.1 mode or higher for instructions it use.
+ if (!FLAG_IS_DEFAULT(UseAESCTRIntrinsics)) {
+ warning("X86 AES-CTR intrinsics require SSE4.1 instructions or higher. Intrinsics will be disabled.");
+ }
+ FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
+ }
+ }
+ } else {
+ if (!cpu_supports_aes()) {
+ if (UseAESIntrinsics && !FLAG_IS_DEFAULT(UseAESIntrinsics)) {
+ warning("AES intrinsics are not available on this CPU");
+ }
+ FLAG_SET_DEFAULT(UseAESIntrinsics, false);
+ if (UseAESCTRIntrinsics && !FLAG_IS_DEFAULT(UseAESCTRIntrinsics)) {
+ warning("AES-CTR intrinsics are not available on this CPU");
+ }
+ FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
+ } else if (!UseAES) {
if (UseAESIntrinsics && !FLAG_IS_DEFAULT(UseAESIntrinsics)) {
warning("AES intrinsics require UseAES flag to be enabled. Intrinsics will be disabled.");
}
@@ -1156,66 +1225,7 @@ void VM_Version::get_processor_features() {
warning("AES_CTR intrinsics require UseAES flag to be enabled. AES_CTR intrinsics will be disabled.");
}
FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
- } else {
- if (UseSSE > 2) {
- if (FLAG_IS_DEFAULT(UseAESIntrinsics)) {
- FLAG_SET_DEFAULT(UseAESIntrinsics, true);
- }
- } else {
- // The AES intrinsic stubs require AES instruction support (of course)
- // but also require sse3 mode or higher for instructions it use.
- if (UseAESIntrinsics && !FLAG_IS_DEFAULT(UseAESIntrinsics)) {
- warning("X86 AES intrinsics require SSE3 instructions or higher. Intrinsics will be disabled.");
- }
- FLAG_SET_DEFAULT(UseAESIntrinsics, false);
- }
-
- // --AES-CTR begins--
- if (!UseAESIntrinsics) {
- if (UseAESCTRIntrinsics && !FLAG_IS_DEFAULT(UseAESCTRIntrinsics)) {
- warning("AES-CTR intrinsics require UseAESIntrinsics flag to be enabled. Intrinsics will be disabled.");
- }
- FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
- } else {
- if (supports_sse4_1()) {
- if (FLAG_IS_DEFAULT(UseAESCTRIntrinsics)) {
- FLAG_SET_DEFAULT(UseAESCTRIntrinsics, true);
- }
- } else {
- // The AES-CTR intrinsic stubs require AES instruction support (of course)
- // but also require sse4.1 mode or higher for instructions it use.
- if (UseAESCTRIntrinsics && !FLAG_IS_DEFAULT(UseAESCTRIntrinsics)) {
- warning("X86 AES-CTR intrinsics require SSE4.1 instructions or higher. Intrinsics will be disabled.");
- }
- FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
- }
- }
- // --AES-CTR ends--
}
- } else if (UseAES || UseAESIntrinsics || UseAESCTRIntrinsics) {
- if (UseAES && !FLAG_IS_DEFAULT(UseAES)) {
- warning("AES instructions are not available on this CPU");
- }
- FLAG_SET_DEFAULT(UseAES, false);
- if (UseAESIntrinsics && !FLAG_IS_DEFAULT(UseAESIntrinsics)) {
- warning("AES intrinsics are not available on this CPU");
- }
- FLAG_SET_DEFAULT(UseAESIntrinsics, false);
- if (UseAESCTRIntrinsics && !FLAG_IS_DEFAULT(UseAESCTRIntrinsics)) {
- warning("AES-CTR intrinsics are not available on this CPU");
- }
- FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
- }
-
- // Use CLMUL instructions if available.
- if (supports_clmul()) {
- if (FLAG_IS_DEFAULT(UseCLMUL)) {
- UseCLMUL = true;
- }
- } else if (UseCLMUL) {
- if (!FLAG_IS_DEFAULT(UseCLMUL))
- warning("CLMUL instructions not available on this CPU (AVX may also be required)");
- FLAG_SET_DEFAULT(UseCLMUL, false);
}
if (UseCLMUL && (UseSSE > 2)) {
@@ -1256,8 +1266,9 @@ void VM_Version::get_processor_features() {
UseGHASHIntrinsics = true;
}
} else if (UseGHASHIntrinsics) {
- if (!FLAG_IS_DEFAULT(UseGHASHIntrinsics))
+ if (!FLAG_IS_DEFAULT(UseGHASHIntrinsics)) {
warning("GHASH intrinsic requires CLMUL and SSE2 instructions on this CPU");
+ }
FLAG_SET_DEFAULT(UseGHASHIntrinsics, false);
}
@@ -1267,26 +1278,27 @@ void VM_Version::get_processor_features() {
// based on the VM capabilities whether to use an AVX2 or AVX512-enabled
// version.
if (UseAVX >= 1) {
- if (FLAG_IS_DEFAULT(UseChaCha20Intrinsics)) {
- UseChaCha20Intrinsics = true;
- }
+ if (FLAG_IS_DEFAULT(UseChaCha20Intrinsics)) {
+ UseChaCha20Intrinsics = true;
+ }
} else if (UseChaCha20Intrinsics) {
- if (!FLAG_IS_DEFAULT(UseChaCha20Intrinsics)) {
- warning("ChaCha20 intrinsic requires AVX instructions");
- }
- FLAG_SET_DEFAULT(UseChaCha20Intrinsics, false);
+ if (!FLAG_IS_DEFAULT(UseChaCha20Intrinsics)) {
+ warning("ChaCha20 intrinsic requires AVX instructions");
+ }
+ FLAG_SET_DEFAULT(UseChaCha20Intrinsics, false);
}
// Kyber Intrinsics
// Currently we only have them for AVX512
if (supports_evex() && supports_avx512bw()) {
- if (FLAG_IS_DEFAULT(UseKyberIntrinsics)) {
- UseKyberIntrinsics = true;
- }
- } else
- if (UseKyberIntrinsics) {
- warning("Intrinsics for ML-KEM are not available on this CPU.");
- FLAG_SET_DEFAULT(UseKyberIntrinsics, false);
+ if (FLAG_IS_DEFAULT(UseKyberIntrinsics)) {
+ UseKyberIntrinsics = true;
+ }
+ } else if (UseKyberIntrinsics) {
+ if (!FLAG_IS_DEFAULT(UseKyberIntrinsics)) {
+ warning("Intrinsics for ML-KEM are not available on this CPU.");
+ }
+ FLAG_SET_DEFAULT(UseKyberIntrinsics, false);
}
// Dilithium Intrinsics
@@ -1295,8 +1307,10 @@ void VM_Version::get_processor_features() {
UseDilithiumIntrinsics = true;
}
} else if (UseDilithiumIntrinsics) {
+ if (!FLAG_IS_DEFAULT(UseDilithiumIntrinsics)) {
warning("Intrinsics for ML-DSA are not available on this CPU.");
- FLAG_SET_DEFAULT(UseDilithiumIntrinsics, false);
+ }
+ FLAG_SET_DEFAULT(UseDilithiumIntrinsics, false);
}
// Base64 Intrinsics (Check the condition for which the intrinsic will be active)
@@ -1305,39 +1319,24 @@ void VM_Version::get_processor_features() {
UseBASE64Intrinsics = true;
}
} else if (UseBASE64Intrinsics) {
- if (!FLAG_IS_DEFAULT(UseBASE64Intrinsics))
+ if (!FLAG_IS_DEFAULT(UseBASE64Intrinsics)) {
warning("Base64 intrinsic requires EVEX instructions on this CPU");
- FLAG_SET_DEFAULT(UseBASE64Intrinsics, false);
- }
-
- if (supports_fma()) {
- if (FLAG_IS_DEFAULT(UseFMA)) {
- UseFMA = true;
}
- } else if (UseFMA) {
- warning("FMA instructions are not available on this CPU");
- FLAG_SET_DEFAULT(UseFMA, false);
+ FLAG_SET_DEFAULT(UseBASE64Intrinsics, false);
}
if (FLAG_IS_DEFAULT(UseMD5Intrinsics)) {
UseMD5Intrinsics = true;
}
- if (supports_sha() || (supports_avx2() && supports_bmi2())) {
- if (FLAG_IS_DEFAULT(UseSHA)) {
- UseSHA = true;
- }
- } else if (UseSHA) {
- warning("SHA instructions are not available on this CPU");
- FLAG_SET_DEFAULT(UseSHA, false);
- }
-
if (supports_sha() && supports_sse4_1() && UseSHA) {
if (FLAG_IS_DEFAULT(UseSHA1Intrinsics)) {
FLAG_SET_DEFAULT(UseSHA1Intrinsics, true);
}
} else if (UseSHA1Intrinsics) {
- warning("Intrinsics for SHA-1 crypto hash functions not available on this CPU.");
+ if (!FLAG_IS_DEFAULT(UseSHA1Intrinsics)) {
+ warning("Intrinsics for SHA-1 crypto hash functions not available on this CPU.");
+ }
FLAG_SET_DEFAULT(UseSHA1Intrinsics, false);
}
@@ -1346,7 +1345,9 @@ void VM_Version::get_processor_features() {
FLAG_SET_DEFAULT(UseSHA256Intrinsics, true);
}
} else if (UseSHA256Intrinsics) {
- warning("Intrinsics for SHA-224 and SHA-256 crypto hash functions not available on this CPU.");
+ if (!FLAG_IS_DEFAULT(UseSHA256Intrinsics)) {
+ warning("Intrinsics for SHA-224 and SHA-256 crypto hash functions not available on this CPU.");
+ }
FLAG_SET_DEFAULT(UseSHA256Intrinsics, false);
}
@@ -1355,7 +1356,9 @@ void VM_Version::get_processor_features() {
FLAG_SET_DEFAULT(UseSHA512Intrinsics, true);
}
} else if (UseSHA512Intrinsics) {
- warning("Intrinsics for SHA-384 and SHA-512 crypto hash functions not available on this CPU.");
+ if (!FLAG_IS_DEFAULT(UseSHA512Intrinsics)) {
+ warning("Intrinsics for SHA-384 and SHA-512 crypto hash functions not available on this CPU.");
+ }
FLAG_SET_DEFAULT(UseSHA512Intrinsics, false);
}
@@ -1364,14 +1367,12 @@ void VM_Version::get_processor_features() {
FLAG_SET_DEFAULT(UseSHA3Intrinsics, true);
}
} else if (UseSHA3Intrinsics) {
- warning("Intrinsics for SHA3-224, SHA3-256, SHA3-384 and SHA3-512 crypto hash functions not available on this CPU.");
+ if (!FLAG_IS_DEFAULT(UseSHA3Intrinsics)) {
+ warning("Intrinsics for SHA3-224, SHA3-256, SHA3-384 and SHA3-512 crypto hash functions not available on this CPU.");
+ }
FLAG_SET_DEFAULT(UseSHA3Intrinsics, false);
}
- if (!(UseSHA1Intrinsics || UseSHA256Intrinsics || UseSHA512Intrinsics || UseSHA3Intrinsics)) {
- FLAG_SET_DEFAULT(UseSHA, false);
- }
-
#if COMPILER2_OR_JVMCI
int max_vector_size = 0;
if (UseAVX == 0 || !os_supports_avx_vectors()) {
@@ -1427,7 +1428,9 @@ void VM_Version::get_processor_features() {
FLAG_SET_DEFAULT(UsePoly1305Intrinsics, true);
}
} else if (UsePoly1305Intrinsics) {
- warning("Intrinsics for Poly1305 crypto hash functions not available on this CPU.");
+ if (!FLAG_IS_DEFAULT(UsePoly1305Intrinsics)) {
+ warning("Intrinsics for Poly1305 crypto hash functions not available on this CPU.");
+ }
FLAG_SET_DEFAULT(UsePoly1305Intrinsics, false);
}
@@ -1436,7 +1439,9 @@ void VM_Version::get_processor_features() {
FLAG_SET_DEFAULT(UseIntPolyIntrinsics, true);
}
} else if (UseIntPolyIntrinsics) {
- warning("Intrinsics for Polynomial crypto functions not available on this CPU.");
+ if (!FLAG_IS_DEFAULT(UseIntPolyIntrinsics)) {
+ warning("Intrinsics for Polynomial crypto functions not available on this CPU.");
+ }
FLAG_SET_DEFAULT(UseIntPolyIntrinsics, false);
}
@@ -1500,9 +1505,6 @@ void VM_Version::get_processor_features() {
MaxLoopPad = 11;
}
#endif // COMPILER2
- if (FLAG_IS_DEFAULT(UseXMMForArrayCopy)) {
- UseXMMForArrayCopy = true; // use SSE2 movq on new ZX cpus
- }
if (supports_sse4_2()) { // new ZX cpus
if (FLAG_IS_DEFAULT(UseUnalignedLoadStores)) {
UseUnalignedLoadStores = true; // use movdqu on newest ZX cpus
@@ -1520,10 +1522,6 @@ void VM_Version::get_processor_features() {
// Use it on new AMD cpus starting from Opteron.
UseAddressNop = true;
}
- if (supports_sse2() && FLAG_IS_DEFAULT(UseNewLongLShift)) {
- // Use it on new AMD cpus starting from Opteron.
- UseNewLongLShift = true;
- }
if (FLAG_IS_DEFAULT(UseXmmLoadAndClearUpper)) {
if (supports_sse4a()) {
UseXmmLoadAndClearUpper = true; // use movsd only on '10h' Opteron
@@ -1563,10 +1561,6 @@ void VM_Version::get_processor_features() {
if (FLAG_IS_DEFAULT(AllocatePrefetchInstr)) {
FLAG_SET_DEFAULT(AllocatePrefetchInstr, 3);
}
- // On family 15h processors use XMM and UnalignedLoadStores for Array Copy
- if (supports_sse2() && FLAG_IS_DEFAULT(UseXMMForArrayCopy)) {
- FLAG_SET_DEFAULT(UseXMMForArrayCopy, true);
- }
if (supports_sse2() && FLAG_IS_DEFAULT(UseUnalignedLoadStores)) {
FLAG_SET_DEFAULT(UseUnalignedLoadStores, true);
}
@@ -1583,9 +1577,6 @@ void VM_Version::get_processor_features() {
if (cpu_family() >= 0x17) {
// On family >=17h processors use XMM and UnalignedLoadStores
// for Array Copy
- if (supports_sse2() && FLAG_IS_DEFAULT(UseXMMForArrayCopy)) {
- FLAG_SET_DEFAULT(UseXMMForArrayCopy, true);
- }
if (supports_sse2() && FLAG_IS_DEFAULT(UseUnalignedLoadStores)) {
FLAG_SET_DEFAULT(UseUnalignedLoadStores, true);
}
@@ -1632,10 +1623,7 @@ void VM_Version::get_processor_features() {
}
#endif // COMPILER2
- if (FLAG_IS_DEFAULT(UseXMMForArrayCopy)) {
- UseXMMForArrayCopy = true; // use SSE2 movq on new Intel cpus
- }
- if ((supports_sse4_2() && supports_ht()) || supports_avx()) { // Newest Intel cpus
+ if (is_intel_modern_cpu()) { // Newest Intel cpus
if (FLAG_IS_DEFAULT(UseUnalignedLoadStores)) {
UseUnalignedLoadStores = true; // use movdqu on newest Intel cpus
}
@@ -1703,8 +1691,8 @@ void VM_Version::get_processor_features() {
if (FLAG_IS_DEFAULT(UseSSE42Intrinsics)) {
FLAG_SET_DEFAULT(UseSSE42Intrinsics, true);
}
- } else {
- if (UseSSE42Intrinsics && !FLAG_IS_DEFAULT(UseSSE42Intrinsics)) {
+ } else if (UseSSE42Intrinsics) {
+ if (!FLAG_IS_DEFAULT(UseSSE42Intrinsics)) {
warning("SSE4.2 intrinsics require SSE4.2 instructions or higher. Intrinsics will be disabled.");
}
FLAG_SET_DEFAULT(UseSSE42Intrinsics, false);
@@ -1714,15 +1702,17 @@ void VM_Version::get_processor_features() {
UseVectorizedMismatchIntrinsic = true;
}
} else if (UseVectorizedMismatchIntrinsic) {
- if (!FLAG_IS_DEFAULT(UseVectorizedMismatchIntrinsic))
+ if (!FLAG_IS_DEFAULT(UseVectorizedMismatchIntrinsic)) {
warning("vectorizedMismatch intrinsics are not available on this CPU");
+ }
FLAG_SET_DEFAULT(UseVectorizedMismatchIntrinsic, false);
}
if (UseAVX >= 2) {
FLAG_SET_DEFAULT(UseVectorizedHashCodeIntrinsic, true);
} else if (UseVectorizedHashCodeIntrinsic) {
- if (!FLAG_IS_DEFAULT(UseVectorizedHashCodeIntrinsic))
+ if (!FLAG_IS_DEFAULT(UseVectorizedHashCodeIntrinsic)) {
warning("vectorizedHashCode intrinsics are not available on this CPU");
+ }
FLAG_SET_DEFAULT(UseVectorizedHashCodeIntrinsic, false);
}
@@ -1732,7 +1722,9 @@ void VM_Version::get_processor_features() {
UseCountLeadingZerosInstruction = true;
}
} else if (UseCountLeadingZerosInstruction) {
- warning("lzcnt instruction is not available on this CPU");
+ if (!FLAG_IS_DEFAULT(UseCountLeadingZerosInstruction)) {
+ warning("lzcnt instruction is not available on this CPU");
+ }
FLAG_SET_DEFAULT(UseCountLeadingZerosInstruction, false);
}
@@ -1748,7 +1740,9 @@ void VM_Version::get_processor_features() {
}
}
} else if (UseCountTrailingZerosInstruction) {
- warning("tzcnt instruction is not available on this CPU");
+ if (!FLAG_IS_DEFAULT(UseCountTrailingZerosInstruction)) {
+ warning("tzcnt instruction is not available on this CPU");
+ }
FLAG_SET_DEFAULT(UseCountTrailingZerosInstruction, false);
}
@@ -1759,7 +1753,9 @@ void VM_Version::get_processor_features() {
UseBMI1Instructions = true;
}
} else if (UseBMI1Instructions) {
- warning("BMI1 instructions are not available on this CPU (AVX is also required)");
+ if (!FLAG_IS_DEFAULT(UseBMI1Instructions)) {
+ warning("BMI1 instructions are not available on this CPU (AVX is also required)");
+ }
FLAG_SET_DEFAULT(UseBMI1Instructions, false);
}
@@ -1768,7 +1764,9 @@ void VM_Version::get_processor_features() {
UseBMI2Instructions = true;
}
} else if (UseBMI2Instructions) {
- warning("BMI2 instructions are not available on this CPU (AVX is also required)");
+ if (!FLAG_IS_DEFAULT(UseBMI2Instructions)) {
+ warning("BMI2 instructions are not available on this CPU (AVX is also required)");
+ }
FLAG_SET_DEFAULT(UseBMI2Instructions, false);
}
@@ -1778,7 +1776,9 @@ void VM_Version::get_processor_features() {
UsePopCountInstruction = true;
}
} else if (UsePopCountInstruction) {
- warning("POPCNT instruction is not available on this CPU");
+ if (!FLAG_IS_DEFAULT(UsePopCountInstruction)) {
+ warning("POPCNT instruction is not available on this CPU");
+ }
FLAG_SET_DEFAULT(UsePopCountInstruction, false);
}
@@ -1788,7 +1788,9 @@ void VM_Version::get_processor_features() {
UseFastStosb = true;
}
} else if (UseFastStosb) {
- warning("fast-string operations are not available on this CPU");
+ if (!FLAG_IS_DEFAULT(UseFastStosb)) {
+ warning("fast-string operations are not available on this CPU");
+ }
FLAG_SET_DEFAULT(UseFastStosb, false);
}
@@ -1814,7 +1816,9 @@ void VM_Version::get_processor_features() {
UseXMMForObjInit = true;
}
} else if (UseXMMForObjInit) {
- warning("UseXMMForObjInit requires SSE2 and unaligned load/stores. Feature is switched off.");
+ if (!FLAG_IS_DEFAULT(UseXMMForObjInit)) {
+ warning("UseXMMForObjInit requires SSE2 and unaligned load/stores. Feature is switched off.");
+ }
FLAG_SET_DEFAULT(UseXMMForObjInit, false);
}
@@ -1855,7 +1859,7 @@ void VM_Version::get_processor_features() {
if (is_intel() && is_intel_server_family() && supports_sse3()) {
if (FLAG_IS_DEFAULT(AllocatePrefetchLines) &&
- supports_sse4_2() && supports_ht()) { // Nehalem based cpus
+ is_intel_modern_cpu()) { // Nehalem based cpus
FLAG_SET_DEFAULT(AllocatePrefetchLines, 4);
}
#ifdef COMPILER2
@@ -1894,7 +1898,7 @@ void VM_Version::get_processor_features() {
if (FLAG_IS_DEFAULT(ContendedPaddingWidth) &&
(cache_line_size > ContendedPaddingWidth))
- ContendedPaddingWidth = cache_line_size;
+ ContendedPaddingWidth = cache_line_size;
// This machine allows unaligned memory accesses
if (FLAG_IS_DEFAULT(UseUnalignedAccesses)) {
@@ -1959,6 +1963,18 @@ void VM_Version::get_processor_features() {
if (FLAG_IS_DEFAULT(UseCopySignIntrinsic)) {
FLAG_SET_DEFAULT(UseCopySignIntrinsic, true);
}
+ // CopyAVX3Threshold is the threshold at which 64-byte instructions are used
+ // for implementing the array copy and clear operations.
+ // The Intel platforms that supports the serialize instruction
+ // have improved implementation of 64-byte load/stores and so the default
+ // threshold is set to 0 for these platforms.
+ if (FLAG_IS_DEFAULT(CopyAVX3Threshold)) {
+ if (is_intel() && is_intel_server_family() && supports_serialize()) {
+ FLAG_SET_DEFAULT(CopyAVX3Threshold, 0);
+ } else {
+ FLAG_SET_DEFAULT(CopyAVX3Threshold, AVX3Threshold);
+ }
+ }
}
void VM_Version::print_platform_virtualization_info(outputStream* st) {
@@ -2114,17 +2130,6 @@ bool VM_Version::is_intel_darkmont() {
return is_intel() && is_intel_server_family() && (_model == 0xCC || _model == 0xDD);
}
-// avx3_threshold() sets the threshold at which 64-byte instructions are used
-// for implementing the array copy and clear operations.
-// The Intel platforms that supports the serialize instruction
-// has improved implementation of 64-byte load/stores and so the default
-// threshold is set to 0 for these platforms.
-int VM_Version::avx3_threshold() {
- return (is_intel_server_family() &&
- supports_serialize() &&
- FLAG_IS_DEFAULT(AVX3Threshold)) ? 0 : AVX3Threshold;
-}
-
void VM_Version::clear_apx_test_state() {
clear_apx_test_state_stub();
}
@@ -2623,6 +2628,23 @@ const char* VM_Version::cpu_family_description(void) {
return _family_id_intel[cpu_family_id];
}
}
+ if (is_zx()) {
+ int cpu_model_id = extended_cpu_model();
+ if (cpu_family_id == 7) {
+ switch (cpu_model_id) {
+ case 0x1B:
+ return "wudaokou";
+ case 0x3B:
+ return "lujiazui";
+ case 0x5B:
+ return "yongfeng";
+ case 0x6B:
+ return "shijidadao";
+ }
+ } else if (cpu_family_id == 6) {
+ return "zhangjiang";
+ }
+ }
if (is_hygon()) {
return "Dhyana";
}
@@ -2642,6 +2664,9 @@ int VM_Version::cpu_type_description(char* const buf, size_t buf_len) {
} else if (is_amd()) {
cpu_type = "AMD";
x64 = cpu_is_em64t() ? " AMD64" : "";
+ } else if (is_zx()) {
+ cpu_type = "Zhaoxin";
+ x64 = cpu_is_em64t() ? " x86_64" : "";
} else if (is_hygon()) {
cpu_type = "Hygon";
x64 = cpu_is_em64t() ? " AMD64" : "";
@@ -3259,9 +3284,15 @@ int VM_Version::allocate_prefetch_distance(bool use_watermark_prefetch) {
} else {
return 128; // Athlon
}
+ } else if (is_zx()) {
+ if (supports_sse2()) {
+ return 256;
+ } else {
+ return 128;
+ }
} else { // Intel
if (supports_sse3() && is_intel_server_family()) {
- if (supports_sse4_2() && supports_ht()) { // Nehalem based cpus
+ if (is_intel_modern_cpu()) { // Nehalem based cpus
return 192;
} else if (use_watermark_prefetch) { // watermark prefetching on Core
return 384;
diff --git a/src/hotspot/cpu/x86/vm_version_x86.hpp b/src/hotspot/cpu/x86/vm_version_x86.hpp
index e0a895737b7..f721635a02e 100644
--- a/src/hotspot/cpu/x86/vm_version_x86.hpp
+++ b/src/hotspot/cpu/x86/vm_version_x86.hpp
@@ -535,6 +535,10 @@ protected:
static const char* _features_names[];
+ static void clear_feature(Feature_Flag feature) {
+ _features.clear_feature(feature);
+ }
+
static void clear_cpu_features() {
_features = VM_Features();
_cpu_features = VM_Features();
@@ -828,7 +832,7 @@ public:
static uint32_t cpu_stepping() { return _cpuid_info.cpu_stepping(); }
static int cpu_family() { return _cpu;}
static bool is_P6() { return cpu_family() >= 6; }
- static bool is_intel_server_family() { return cpu_family() == 6 || cpu_family() == 19; }
+ static bool is_intel_server_family() { return cpu_family() == 6 || cpu_family() == 18 || cpu_family() == 19; }
static bool is_amd() { assert_is_initialized(); return _cpuid_info.std_vendor_name_0 == 0x68747541; } // 'htuA'
static bool is_hygon() { assert_is_initialized(); return _cpuid_info.std_vendor_name_0 == 0x6F677948; } // 'ogyH'
static bool is_amd_family() { return is_amd() || is_hygon(); }
@@ -930,6 +934,7 @@ public:
// Feature identification not affected by VM flags
//
static bool cpu_supports_evex() { return _cpu_features.supports_feature(CPU_AVX512F); }
+ static bool cpu_supports_aes() { return _cpu_features.supports_feature(CPU_AES); }
static bool supports_avx512_simd_sort() {
if (supports_avx512dq()) {
@@ -958,7 +963,11 @@ public:
static bool is_intel_darkmont();
- static int avx3_threshold();
+ static bool is_intel_modern_cpu() {
+ precond(is_intel()); // should be called only for intel CPU
+ // Efficient cores in hybrid CPU may not support hyper-threads.
+ return (supports_avx() || (supports_sse4_2() && (supports_ht() || supports_hybrid())));
+ }
static bool is_intel_tsc_synched_at_init();
diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad
index 0ffa4c2031c..eaa88d900c7 100644
--- a/src/hotspot/cpu/x86/x86.ad
+++ b/src/hotspot/cpu/x86/x86.ad
@@ -1708,84 +1708,99 @@ static void emit_cmpfp3(MacroAssembler* masm, Register dst) {
__ bind(done);
}
-// Math.min() # Math.max()
-// --------------------------
-// ucomis[s/d] #
-// ja -> b # a
-// jp -> NaN # NaN
-// jb -> a # b
-// je #
-// |-jz -> a | b # a & b
-// | -> a #
+enum FP_PREC {
+ fp_prec_hlf,
+ fp_prec_flt,
+ fp_prec_dbl
+};
+
+static inline void emit_fp_ucom(MacroAssembler* masm, enum FP_PREC pt,
+ XMMRegister p, XMMRegister q) {
+ if (pt == fp_prec_hlf) {
+ __ evucomish(p, q);
+ } else if (pt == fp_prec_flt) {
+ __ ucomiss(p, q);
+ } else {
+ __ ucomisd(p, q);
+ }
+}
+
+static inline void movfp(MacroAssembler* masm, enum FP_PREC pt,
+ XMMRegister dst, XMMRegister src, Register scratch) {
+ if (pt == fp_prec_hlf) {
+ __ movhlf(dst, src, scratch);
+ } else if (pt == fp_prec_flt) {
+ __ movflt(dst, src);
+ } else {
+ __ movdbl(dst, src);
+ }
+}
+
+// Math.min() # Math.max()
+// -----------------------------
+// (v)ucomis[h/s/d] #
+// ja -> b # a
+// jp -> NaN # NaN
+// jb -> a # b
+// je #
+// |-jz -> a | b # a & b
+// | -> a #
static void emit_fp_min_max(MacroAssembler* masm, XMMRegister dst,
XMMRegister a, XMMRegister b,
XMMRegister xmmt, Register rt,
- bool min, bool single) {
+ bool min, enum FP_PREC pt) {
Label nan, zero, below, above, done;
- if (single)
- __ ucomiss(a, b);
- else
- __ ucomisd(a, b);
+ emit_fp_ucom(masm, pt, a, b);
- if (dst->encoding() != (min ? b : a)->encoding())
+ if (dst->encoding() != (min ? b : a)->encoding()) {
__ jccb(Assembler::above, above); // CF=0 & ZF=0
- else
+ } else {
__ jccb(Assembler::above, done);
+ }
__ jccb(Assembler::parity, nan); // PF=1
__ jccb(Assembler::below, below); // CF=1
// equal
__ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
- if (single) {
- __ ucomiss(a, xmmt);
- __ jccb(Assembler::equal, zero);
+ emit_fp_ucom(masm, pt, a, xmmt);
- __ movflt(dst, a);
- __ jmp(done);
- }
- else {
- __ ucomisd(a, xmmt);
- __ jccb(Assembler::equal, zero);
+ __ jccb(Assembler::equal, zero);
+ movfp(masm, pt, dst, a, rt);
- __ movdbl(dst, a);
- __ jmp(done);
- }
+ __ jmp(done);
__ bind(zero);
- if (min)
+ if (min) {
__ vpor(dst, a, b, Assembler::AVX_128bit);
- else
+ } else {
__ vpand(dst, a, b, Assembler::AVX_128bit);
+ }
__ jmp(done);
__ bind(above);
- if (single)
- __ movflt(dst, min ? b : a);
- else
- __ movdbl(dst, min ? b : a);
+ movfp(masm, pt, dst, min ? b : a, rt);
__ jmp(done);
__ bind(nan);
- if (single) {
+ if (pt == fp_prec_hlf) {
+ __ movl(rt, 0x00007e00); // Float16.NaN
+ __ evmovw(dst, rt);
+ } else if (pt == fp_prec_flt) {
__ movl(rt, 0x7fc00000); // Float.NaN
__ movdl(dst, rt);
- }
- else {
+ } else {
__ mov64(rt, 0x7ff8000000000000L); // Double.NaN
__ movdq(dst, rt);
}
__ jmp(done);
__ bind(below);
- if (single)
- __ movflt(dst, min ? a : b);
- else
- __ movdbl(dst, min ? a : b);
+ movfp(masm, pt, dst, min ? a : b, rt);
__ bind(done);
}
@@ -2605,13 +2620,8 @@ uint BoxLockNode::size(PhaseRegAlloc *ra_) const
#ifndef PRODUCT
void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
{
- if (UseCompressedClassPointers) {
- st->print_cr("movl rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
- st->print_cr("\tcmpl rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
- } else {
- st->print_cr("movq rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
- st->print_cr("\tcmpq rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
- }
+ st->print_cr("movl rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
+ st->print_cr("\tcmpl rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
st->print_cr("\tjne SharedRuntime::_ic_miss_stub");
}
#endif
@@ -2726,11 +2736,8 @@ bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
return (-128 <= offset && offset <= 127);
}
+#ifdef ASSERT
// Return whether or not this register is ever used as an argument.
-// This function is used on startup to build the trampoline stubs in
-// generateOptoStub. Registers not mentioned will be killed by the VM
-// call in the trampoline, and arguments in those registers not be
-// available to the callee.
bool Matcher::can_be_java_arg(int reg)
{
return
@@ -2750,11 +2757,7 @@ bool Matcher::can_be_java_arg(int reg)
reg == XMM6_num || reg == XMM6b_num ||
reg == XMM7_num || reg == XMM7b_num;
}
-
-bool Matcher::is_spillable_arg(int reg)
-{
- return can_be_java_arg(reg);
-}
+#endif
uint Matcher::int_pressure_limit()
{
@@ -2770,13 +2773,6 @@ uint Matcher::float_pressure_limit()
return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
}
-bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
- // In 64 bit mode a code which use multiply when
- // devisor is constant is faster than hardware
- // DIV instruction (it uses MulHiL).
- return false;
-}
-
// Register for DIVI projection of divmodI
const RegMask& Matcher::divI_proj_mask() {
return INT_RAX_REG_mask();
@@ -3341,6 +3337,18 @@ bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
return false;
}
break;
+ case Op_UMinReductionV:
+ case Op_UMaxReductionV:
+ if (UseAVX == 0) {
+ return false;
+ }
+ if (bt == T_LONG && !VM_Version::supports_avx512vl()) {
+ return false;
+ }
+ if (UseAVX > 2 && size_in_bits == 512 && !VM_Version::supports_avx512vl()) {
+ return false;
+ }
+ break;
case Op_MaxV:
case Op_MinV:
if (UseSSE < 4 && is_integral_type(bt)) {
@@ -4679,11 +4687,6 @@ frame
// Compiled code's Frame Pointer
frame_pointer(RSP);
- // Interpreter stores its frame pointer in a register which is
- // stored to the stack by I2CAdaptors.
- // I2CAdaptors convert from interpreted java to compiled java.
- interpreter_frame_pointer(RBP);
-
// Stack alignment requirement
stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
@@ -7357,146 +7360,140 @@ instruct loadAOTRCAddress(rRegP dst, immAOTRuntimeConstantsAddress con)
ins_pipe(ialu_reg_fat);
%}
+// min = java.lang.Math.min(float a, float b)
// max = java.lang.Math.max(float a, float b)
-instruct maxF_reg_avx10_2(regF dst, regF a, regF b) %{
- predicate(VM_Version::supports_avx10_2());
+instruct minmaxF_reg_avx10_2(regF dst, regF a, regF b)
+%{
+ predicate(VM_Version::supports_avx10_2() && !VLoopReductions::is_reduction(n));
match(Set dst (MaxF a b));
- format %{ "maxF $dst, $a, $b" %}
- ins_encode %{
- __ eminmaxss($dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, AVX10_2_MINMAX_MAX_COMPARE_SIGN);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// max = java.lang.Math.max(float a, float b)
-instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
- predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
- match(Set dst (MaxF a b));
- effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
- format %{ "maxF $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
- ins_encode %{
- __ vminmax_fp(Op_MaxV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xtmp, rRegI rtmp, rFlagsReg cr) %{
- predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
- match(Set dst (MaxF a b));
- effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
-
- format %{ "maxF_reduction $dst, $a, $b \t!using $xtmp and $rtmp as TEMP" %}
- ins_encode %{
- emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
- false /*min*/, true /*single*/);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// max = java.lang.Math.max(double a, double b)
-instruct maxD_reg_avx10_2(regD dst, regD a, regD b) %{
- predicate(VM_Version::supports_avx10_2());
- match(Set dst (MaxD a b));
- format %{ "maxD $dst, $a, $b" %}
- ins_encode %{
- __ eminmaxsd($dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, AVX10_2_MINMAX_MAX_COMPARE_SIGN);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// max = java.lang.Math.max(double a, double b)
-instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
- predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
- match(Set dst (MaxD a b));
- effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
- format %{ "maxD $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
- ins_encode %{
- __ vminmax_fp(Op_MaxV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xtmp, rRegL rtmp, rFlagsReg cr) %{
- predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
- match(Set dst (MaxD a b));
- effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
-
- format %{ "maxD_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
- ins_encode %{
- emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
- false /*min*/, false /*single*/);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// max = java.lang.Math.min(float a, float b)
-instruct minF_reg_avx10_2(regF dst, regF a, regF b) %{
- predicate(VM_Version::supports_avx10_2());
match(Set dst (MinF a b));
- format %{ "minF $dst, $a, $b" %}
+
+ format %{ "minmaxF $dst, $a, $b" %}
ins_encode %{
- __ eminmaxss($dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, AVX10_2_MINMAX_MIN_COMPARE_SIGN);
+ int opcode = this->ideal_Opcode();
+ __ sminmax_fp_avx10_2(opcode, T_FLOAT, $dst$$XMMRegister, k0, $a$$XMMRegister, $b$$XMMRegister);
+ %}
+ ins_pipe( pipe_slow );
+%}
+
+instruct minmaxF_reduction_reg_avx10_2(regF dst, regF a, regF b, regF xtmp, rRegI rtmp, rFlagsReg cr)
+%{
+ predicate(VM_Version::supports_avx10_2() && VLoopReductions::is_reduction(n));
+ match(Set dst (MaxF a b));
+ match(Set dst (MinF a b));
+ effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
+
+ format %{ "minmaxF_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
+ ins_encode %{
+ int opcode = this->ideal_Opcode();
+ bool min = (opcode == Op_MinF) ? true : false;
+ emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
+ min, fp_prec_flt /*pt*/);
%}
ins_pipe( pipe_slow );
%}
// min = java.lang.Math.min(float a, float b)
-instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
+// max = java.lang.Math.max(float a, float b)
+instruct minmaxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp)
+%{
predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
+ match(Set dst (MaxF a b));
match(Set dst (MinF a b));
effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
- format %{ "minF $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
+
+ format %{ "minmaxF $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
ins_encode %{
- __ vminmax_fp(Op_MinV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
+ int opcode = this->ideal_Opcode();
+ int param_opcode = (opcode == Op_MinF) ? Op_MinV : Op_MaxV;
+ __ vminmax_fp(param_opcode, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister,
+ $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
%}
ins_pipe( pipe_slow );
%}
-instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xtmp, rRegI rtmp, rFlagsReg cr) %{
+instruct minmaxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xtmp, rRegI rtmp, rFlagsReg cr)
+%{
predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
+ match(Set dst (MaxF a b));
match(Set dst (MinF a b));
effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
- format %{ "minF_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
+ format %{ "minmaxF_reduction $dst, $a, $b \t!using $xtmp and $rtmp as TEMP" %}
ins_encode %{
+ int opcode = this->ideal_Opcode();
+ bool min = (opcode == Op_MinF) ? true : false;
emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
- true /*min*/, true /*single*/);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// max = java.lang.Math.min(double a, double b)
-instruct minD_reg_avx10_2(regD dst, regD a, regD b) %{
- predicate(VM_Version::supports_avx10_2());
- match(Set dst (MinD a b));
- format %{ "minD $dst, $a, $b" %}
- ins_encode %{
- __ eminmaxsd($dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, AVX10_2_MINMAX_MIN_COMPARE_SIGN);
+ min, fp_prec_flt /*pt*/);
%}
ins_pipe( pipe_slow );
%}
// min = java.lang.Math.min(double a, double b)
-instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
- predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
+// max = java.lang.Math.max(double a, double b)
+instruct minmaxD_reg_avx10_2(regD dst, regD a, regD b)
+%{
+ predicate(VM_Version::supports_avx10_2() && !VLoopReductions::is_reduction(n));
+ match(Set dst (MaxD a b));
match(Set dst (MinD a b));
- effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
- format %{ "minD $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
+
+ format %{ "minmaxD $dst, $a, $b" %}
ins_encode %{
- __ vminmax_fp(Op_MinV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
+ int opcode = this->ideal_Opcode();
+ __ sminmax_fp_avx10_2(opcode, T_DOUBLE, $dst$$XMMRegister, k0, $a$$XMMRegister, $b$$XMMRegister);
%}
ins_pipe( pipe_slow );
%}
-instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xtmp, rRegL rtmp, rFlagsReg cr) %{
- predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
+instruct minmaxD_reduction_reg_avx10_2(regD dst, regD a, regD b, regD xtmp, rRegI rtmp, rFlagsReg cr)
+%{
+ predicate(VM_Version::supports_avx10_2() && VLoopReductions::is_reduction(n));
+ match(Set dst (MaxD a b));
match(Set dst (MinD a b));
effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
- format %{ "maxD_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
+ format %{ "minmaxD_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
ins_encode %{
+ int opcode = this->ideal_Opcode();
+ bool min = (opcode == Op_MinD) ? true : false;
emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
- true /*min*/, false /*single*/);
+ min, fp_prec_dbl /*pt*/);
+ %}
+ ins_pipe( pipe_slow );
+%}
+
+// min = java.lang.Math.min(double a, double b)
+// max = java.lang.Math.max(double a, double b)
+instruct minmaxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp)
+%{
+ predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
+ match(Set dst (MaxD a b));
+ match(Set dst (MinD a b));
+ effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
+
+ format %{ "minmaxD $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
+ ins_encode %{
+ int opcode = this->ideal_Opcode();
+ int param_opcode = (opcode == Op_MinD) ? Op_MinV : Op_MaxV;
+ __ vminmax_fp(param_opcode, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister,
+ $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
+ %}
+ ins_pipe( pipe_slow );
+%}
+
+instruct minmaxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xtmp, rRegL rtmp, rFlagsReg cr)
+%{
+ predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
+ match(Set dst (MaxD a b));
+ match(Set dst (MinD a b));
+ effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
+
+ format %{ "minmaxD_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
+ ins_encode %{
+ int opcode = this->ideal_Opcode();
+ bool min = (opcode == Op_MinD) ? true : false;
+ emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
+ min, fp_prec_dbl /*pt*/);
%}
ins_pipe( pipe_slow );
%}
@@ -8852,6 +8849,21 @@ instruct membar_release_lock()
ins_pipe(empty);
%}
+instruct membar_storeload(rFlagsReg cr) %{
+ match(MemBarStoreLoad);
+ effect(KILL cr);
+ ins_cost(400);
+
+ format %{
+ $$template
+ $$emit$$"lock addl [rsp + #0], 0\t! membar_storeload"
+ %}
+ ins_encode %{
+ __ membar(Assembler::StoreLoad);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
instruct membar_volatile(rFlagsReg cr) %{
match(MemBarVolatile);
effect(KILL cr);
@@ -8879,6 +8891,21 @@ instruct unnecessary_membar_volatile()
ins_pipe(empty);
%}
+instruct membar_full(rFlagsReg cr) %{
+ match(MemBarFull);
+ effect(KILL cr);
+ ins_cost(400);
+
+ format %{
+ $$template
+ $$emit$$"lock addl [rsp + #0], 0\t! membar_full"
+ %}
+ ins_encode %{
+ __ membar(Assembler::StoreLoad);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
instruct membar_storestore() %{
match(MemBarStoreStore);
match(StoreStoreFence);
@@ -14376,9 +14403,9 @@ instruct cmpF_cc_regCFE(rFlagsRegUCFE cr, regF src1, regF src2) %{
match(Set cr (CmpF src1 src2));
ins_cost(100);
- format %{ "vucomxss $src1, $src2" %}
+ format %{ "evucomxss $src1, $src2" %}
ins_encode %{
- __ vucomxss($src1$$XMMRegister, $src2$$XMMRegister);
+ __ evucomxss($src1$$XMMRegister, $src2$$XMMRegister);
%}
ins_pipe(pipe_slow);
%}
@@ -14398,9 +14425,9 @@ instruct cmpF_cc_memCFE(rFlagsRegUCFE cr, regF src1, memory src2) %{
match(Set cr (CmpF src1 (LoadF src2)));
ins_cost(100);
- format %{ "vucomxss $src1, $src2" %}
+ format %{ "evucomxss $src1, $src2" %}
ins_encode %{
- __ vucomxss($src1$$XMMRegister, $src2$$Address);
+ __ evucomxss($src1$$XMMRegister, $src2$$Address);
%}
ins_pipe(pipe_slow);
%}
@@ -14420,9 +14447,9 @@ instruct cmpF_cc_immCFE(rFlagsRegUCFE cr, regF src, immF con) %{
match(Set cr (CmpF src con));
ins_cost(100);
- format %{ "vucomxss $src, [$constantaddress]\t# load from constant table: float=$con" %}
+ format %{ "evucomxss $src, [$constantaddress]\t# load from constant table: float=$con" %}
ins_encode %{
- __ vucomxss($src$$XMMRegister, $constantaddress($con));
+ __ evucomxss($src$$XMMRegister, $constantaddress($con));
%}
ins_pipe(pipe_slow);
%}
@@ -14461,9 +14488,9 @@ instruct cmpD_cc_regCFE(rFlagsRegUCFE cr, regD src1, regD src2) %{
match(Set cr (CmpD src1 src2));
ins_cost(100);
- format %{ "vucomxsd $src1, $src2 test" %}
+ format %{ "evucomxsd $src1, $src2 test" %}
ins_encode %{
- __ vucomxsd($src1$$XMMRegister, $src2$$XMMRegister);
+ __ evucomxsd($src1$$XMMRegister, $src2$$XMMRegister);
%}
ins_pipe(pipe_slow);
%}
@@ -14483,9 +14510,9 @@ instruct cmpD_cc_memCFE(rFlagsRegUCFE cr, regD src1, memory src2) %{
match(Set cr (CmpD src1 (LoadD src2)));
ins_cost(100);
- format %{ "vucomxsd $src1, $src2" %}
+ format %{ "evucomxsd $src1, $src2" %}
ins_encode %{
- __ vucomxsd($src1$$XMMRegister, $src2$$Address);
+ __ evucomxsd($src1$$XMMRegister, $src2$$Address);
%}
ins_pipe(pipe_slow);
%}
@@ -14504,9 +14531,9 @@ instruct cmpD_cc_immCFE(rFlagsRegUCFE cr, regD src, immD con) %{
match(Set cr (CmpD src con));
ins_cost(100);
- format %{ "vucomxsd $src, [$constantaddress]\t# load from constant table: double=$con" %}
+ format %{ "evucomxsd $src, [$constantaddress]\t# load from constant table: double=$con" %}
ins_encode %{
- __ vucomxsd($src$$XMMRegister, $constantaddress($con));
+ __ evucomxsd($src$$XMMRegister, $constantaddress($con));
%}
ins_pipe(pipe_slow);
%}
@@ -18814,7 +18841,7 @@ instruct ReplHF_reg(vec dst, regF src, rRegI rtmp) %{
format %{ "replicateHF $dst, $src \t! using $rtmp as TEMP" %}
ins_encode %{
int vlen_enc = vector_length_encoding(this);
- __ vmovw($rtmp$$Register, $src$$XMMRegister);
+ __ evmovw($rtmp$$Register, $src$$XMMRegister);
__ evpbroadcastw($dst$$XMMRegister, $rtmp$$Register, vlen_enc);
%}
ins_pipe( pipe_slow );
@@ -19371,6 +19398,8 @@ instruct reductionI(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtm
match(Set dst (XorReductionV src1 src2));
match(Set dst (MinReductionV src1 src2));
match(Set dst (MaxReductionV src1 src2));
+ match(Set dst (UMinReductionV src1 src2));
+ match(Set dst (UMaxReductionV src1 src2));
effect(TEMP vtmp1, TEMP vtmp2);
format %{ "vector_reduction_int $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
ins_encode %{
@@ -19392,6 +19421,8 @@ instruct reductionL(rRegL dst, rRegL src1, legVec src2, legVec vtmp1, legVec vtm
match(Set dst (XorReductionV src1 src2));
match(Set dst (MinReductionV src1 src2));
match(Set dst (MaxReductionV src1 src2));
+ match(Set dst (UMinReductionV src1 src2));
+ match(Set dst (UMaxReductionV src1 src2));
effect(TEMP vtmp1, TEMP vtmp2);
format %{ "vector_reduction_long $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
ins_encode %{
@@ -19411,6 +19442,8 @@ instruct reductionL_avx512dq(rRegL dst, rRegL src1, vec src2, vec vtmp1, vec vtm
match(Set dst (XorReductionV src1 src2));
match(Set dst (MinReductionV src1 src2));
match(Set dst (MaxReductionV src1 src2));
+ match(Set dst (UMinReductionV src1 src2));
+ match(Set dst (UMaxReductionV src1 src2));
effect(TEMP vtmp1, TEMP vtmp2);
format %{ "vector_reduction_long $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
ins_encode %{
@@ -19639,6 +19672,8 @@ instruct reductionB(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtm
match(Set dst (XorReductionV src1 src2));
match(Set dst (MinReductionV src1 src2));
match(Set dst (MaxReductionV src1 src2));
+ match(Set dst (UMinReductionV src1 src2));
+ match(Set dst (UMaxReductionV src1 src2));
effect(TEMP vtmp1, TEMP vtmp2);
format %{ "vector_reduction_byte $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
ins_encode %{
@@ -19657,6 +19692,8 @@ instruct reductionB_avx512bw(rRegI dst, rRegI src1, vec src2, vec vtmp1, vec vtm
match(Set dst (XorReductionV src1 src2));
match(Set dst (MinReductionV src1 src2));
match(Set dst (MaxReductionV src1 src2));
+ match(Set dst (UMinReductionV src1 src2));
+ match(Set dst (UMaxReductionV src1 src2));
effect(TEMP vtmp1, TEMP vtmp2);
format %{ "vector_reduction_byte $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
ins_encode %{
@@ -19678,6 +19715,8 @@ instruct reductionS(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtm
match(Set dst (XorReductionV src1 src2));
match(Set dst (MinReductionV src1 src2));
match(Set dst (MaxReductionV src1 src2));
+ match(Set dst (UMinReductionV src1 src2));
+ match(Set dst (UMaxReductionV src1 src2));
effect(TEMP vtmp1, TEMP vtmp2);
format %{ "vector_reduction_short $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
ins_encode %{
@@ -20917,7 +20956,7 @@ instruct minmaxFP_reg_avx10_2(vec dst, vec a, vec b) %{
int vlen_enc = vector_length_encoding(this);
int opcode = this->ideal_Opcode();
BasicType elem_bt = Matcher::vector_element_basic_type(this);
- __ vminmax_fp(opcode, elem_bt, $dst$$XMMRegister, k0, $a$$XMMRegister, $b$$XMMRegister, vlen_enc);
+ __ vminmax_fp_avx10_2(opcode, elem_bt, $dst$$XMMRegister, k0, $a$$XMMRegister, $b$$XMMRegister, vlen_enc);
%}
ins_pipe( pipe_slow );
%}
@@ -23933,8 +23972,12 @@ instruct vmask_gen_imm(kReg dst, immL len, rRegL temp) %{
format %{ "vector_mask_gen $len \t! vector mask generator" %}
effect(TEMP temp);
ins_encode %{
- __ mov64($temp$$Register, (0xFFFFFFFFFFFFFFFFUL >> (64 -$len$$constant)));
- __ kmovql($dst$$KRegister, $temp$$Register);
+ if ($len$$constant > 0) {
+ __ mov64($temp$$Register, right_n_bits($len$$constant));
+ __ kmovql($dst$$KRegister, $temp$$Register);
+ } else {
+ __ kxorql($dst$$KRegister, $dst$$KRegister, $dst$$KRegister);
+ }
%}
ins_pipe( pipe_slow );
%}
@@ -25261,9 +25304,9 @@ instruct vector_selectfrom_twovectors_reg_evex(vec index, vec src1, vec src2)
instruct reinterpretS2HF(regF dst, rRegI src)
%{
match(Set dst (ReinterpretS2HF src));
- format %{ "vmovw $dst, $src" %}
+ format %{ "evmovw $dst, $src" %}
ins_encode %{
- __ vmovw($dst$$XMMRegister, $src$$Register);
+ __ evmovw($dst$$XMMRegister, $src$$Register);
%}
ins_pipe(pipe_slow);
%}
@@ -25271,9 +25314,9 @@ instruct reinterpretS2HF(regF dst, rRegI src)
instruct reinterpretHF2S(rRegI dst, regF src)
%{
match(Set dst (ReinterpretHF2S src));
- format %{ "vmovw $dst, $src" %}
+ format %{ "evmovw $dst, $src" %}
ins_encode %{
- __ vmovw($dst$$Register, $src$$XMMRegister);
+ __ evmovw($dst$$Register, $src$$XMMRegister);
%}
ins_pipe(pipe_slow);
%}
@@ -25327,10 +25370,11 @@ instruct scalar_minmax_HF_reg_avx10_2(regF dst, regF src1, regF src2)
predicate(VM_Version::supports_avx10_2());
match(Set dst (MaxHF src1 src2));
match(Set dst (MinHF src1 src2));
+
format %{ "scalar_min_max_fp16 $dst, $src1, $src2" %}
ins_encode %{
- int function = this->ideal_Opcode() == Op_MinHF ? AVX10_2_MINMAX_MIN_COMPARE_SIGN : AVX10_2_MINMAX_MAX_COMPARE_SIGN;
- __ eminmaxsh($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, function);
+ int opcode = this->ideal_Opcode();
+ __ sminmax_fp16_avx10_2(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, k0);
%}
ins_pipe( pipe_slow );
%}
@@ -25341,11 +25385,12 @@ instruct scalar_minmax_HF_reg(regF dst, regF src1, regF src2, kReg ktmp, regF xt
match(Set dst (MaxHF src1 src2));
match(Set dst (MinHF src1 src2));
effect(TEMP_DEF dst, TEMP ktmp, TEMP xtmp1, TEMP xtmp2);
+
format %{ "scalar_min_max_fp16 $dst, $src1, $src2\t using $ktmp, $xtmp1 and $xtmp2 as TEMP" %}
ins_encode %{
int opcode = this->ideal_Opcode();
- __ scalar_max_min_fp16(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $ktmp$$KRegister,
- $xtmp1$$XMMRegister, $xtmp2$$XMMRegister);
+ __ sminmax_fp16(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $ktmp$$KRegister,
+ $xtmp1$$XMMRegister, $xtmp2$$XMMRegister);
%}
ins_pipe( pipe_slow );
%}
@@ -25445,8 +25490,9 @@ instruct vector_minmax_HF_mem_avx10_2(vec dst, vec src1, memory src2)
format %{ "vector_min_max_fp16_mem $dst, $src1, $src2" %}
ins_encode %{
int vlen_enc = vector_length_encoding(this);
- int function = this->ideal_Opcode() == Op_MinVHF ? AVX10_2_MINMAX_MIN_COMPARE_SIGN : AVX10_2_MINMAX_MAX_COMPARE_SIGN;
- __ evminmaxph($dst$$XMMRegister, k0, $src1$$XMMRegister, $src2$$Address, true, function, vlen_enc);
+ int opcode = this->ideal_Opcode();
+ __ vminmax_fp16_avx10_2(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address,
+ k0, vlen_enc);
%}
ins_pipe( pipe_slow );
%}
@@ -25459,8 +25505,9 @@ instruct vector_minmax_HF_reg_avx10_2(vec dst, vec src1, vec src2)
format %{ "vector_min_max_fp16 $dst, $src1, $src2" %}
ins_encode %{
int vlen_enc = vector_length_encoding(this);
- int function = this->ideal_Opcode() == Op_MinVHF ? AVX10_2_MINMAX_MIN_COMPARE_SIGN : AVX10_2_MINMAX_MAX_COMPARE_SIGN;
- __ evminmaxph($dst$$XMMRegister, k0, $src1$$XMMRegister, $src2$$XMMRegister, true, function, vlen_enc);
+ int opcode = this->ideal_Opcode();
+ __ vminmax_fp16_avx10_2(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister,
+ k0, vlen_enc);
%}
ins_pipe( pipe_slow );
%}
@@ -25475,8 +25522,8 @@ instruct vector_minmax_HF_reg(vec dst, vec src1, vec src2, kReg ktmp, vec xtmp1,
ins_encode %{
int vlen_enc = vector_length_encoding(this);
int opcode = this->ideal_Opcode();
- __ vector_max_min_fp16(opcode, $dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, $ktmp$$KRegister,
- $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
+ __ vminmax_fp16(opcode, $dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, $ktmp$$KRegister,
+ $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
%}
ins_pipe( pipe_slow );
%}
diff --git a/src/hotspot/cpu/zero/stubDeclarations_zero.hpp b/src/hotspot/cpu/zero/stubDeclarations_zero.hpp
index 2357bbb5169..9abe313b3a7 100644
--- a/src/hotspot/cpu/zero/stubDeclarations_zero.hpp
+++ b/src/hotspot/cpu/zero/stubDeclarations_zero.hpp
@@ -29,35 +29,40 @@
#define STUBGEN_PREUNIVERSE_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(preuniverse, 0) \
#define STUBGEN_INITIAL_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(initial, 0) \
#define STUBGEN_CONTINUATION_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(continuation, 0) \
#define STUBGEN_COMPILER_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(compiler, 0) \
#define STUBGEN_FINAL_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(final, 0) \
diff --git a/src/hotspot/cpu/zero/stubGenerator_zero.cpp b/src/hotspot/cpu/zero/stubGenerator_zero.cpp
index 08cb173b507..569a2fa8ca9 100644
--- a/src/hotspot/cpu/zero/stubGenerator_zero.cpp
+++ b/src/hotspot/cpu/zero/stubGenerator_zero.cpp
@@ -213,7 +213,7 @@ class StubGenerator: public StubCodeGenerator {
}
public:
- StubGenerator(CodeBuffer* code, BlobId blob_id) : StubCodeGenerator(code, blob_id) {
+ StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData *stub_data) : StubCodeGenerator(code, blob_id, stub_data) {
switch(blob_id) {
case BlobId::stubgen_preuniverse_id:
generate_preuniverse_stubs();
@@ -237,8 +237,8 @@ class StubGenerator: public StubCodeGenerator {
}
};
-void StubGenerator_generate(CodeBuffer* code, BlobId blob_id) {
- StubGenerator g(code, blob_id);
+void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData *stub_data) {
+ StubGenerator g(code, blob_id, stub_data);
}
EntryFrame *EntryFrame::build(const intptr_t* parameters,
diff --git a/src/hotspot/cpu/zero/stubRoutines_zero.cpp b/src/hotspot/cpu/zero/stubRoutines_zero.cpp
index 9b53f09be5d..196907b061f 100644
--- a/src/hotspot/cpu/zero/stubRoutines_zero.cpp
+++ b/src/hotspot/cpu/zero/stubRoutines_zero.cpp
@@ -30,3 +30,9 @@
address StubRoutines::crc_table_addr() { ShouldNotCallThis(); return nullptr; }
address StubRoutines::crc32c_table_addr() { ShouldNotCallThis(); return nullptr; }
+
+#if INCLUDE_CDS
+// nothing to do for zero
+void StubRoutines::init_AOTAddressTable() {
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/os/aix/globals_aix.hpp b/src/hotspot/os/aix/globals_aix.hpp
index 473d7759063..adc189666ef 100644
--- a/src/hotspot/os/aix/globals_aix.hpp
+++ b/src/hotspot/os/aix/globals_aix.hpp
@@ -37,16 +37,6 @@
range, \
constraint) \
\
- /* Whether to allow the VM to run if EXTSHM=ON. EXTSHM is an environment */ \
- /* variable used on AIX to activate certain hacks which allow more shm segments */\
- /* for 32bit processes. For 64bit processes, it is pointless and may have */ \
- /* harmful side effects (e.g. for some reasonn prevents allocation of 64k pages */\
- /* via shmctl). */ \
- /* Per default we quit with an error if that variable is found; for certain */ \
- /* customer scenarios, we may want to be able to run despite that variable. */ \
- product(bool, AllowExtshm, false, DIAGNOSTIC, \
- "Allow VM to run with EXTSHM=ON.") \
- \
/* Maximum expected size of the data segment. That correlates with the */ \
/* maximum C Heap consumption we expect. */ \
/* We need to leave "breathing space" for the data segment when */ \
diff --git a/src/hotspot/os/aix/os_aix.cpp b/src/hotspot/os/aix/os_aix.cpp
index af743dc7484..3cad24d388c 100644
--- a/src/hotspot/os/aix/os_aix.cpp
+++ b/src/hotspot/os/aix/os_aix.cpp
@@ -126,7 +126,6 @@ int mread_real_time(timebasestruct_t *t, size_t size_of_timebasestruct_t);
// for multipage initialization error analysis (in 'g_multipage_error')
#define ERROR_MP_OS_TOO_OLD 100
-#define ERROR_MP_EXTSHM_ACTIVE 101
#define ERROR_MP_VMGETINFO_FAILED 102
#define ERROR_MP_VMGETINFO_CLAIMS_NO_SUPPORT_FOR_64K 103
@@ -178,9 +177,6 @@ uint32_t os::Aix::_os_version = 0;
// -1 = uninitialized, 0 - no, 1 - yes
int os::Aix::_xpg_sus_mode = -1;
-// -1 = uninitialized, 0 - no, 1 - yes
-int os::Aix::_extshm = -1;
-
////////////////////////////////////////////////////////////////////////////////
// local variables
@@ -1195,13 +1191,6 @@ void os::print_memory_info(outputStream* st) {
const char* const ldr_cntrl = ::getenv("LDR_CNTRL");
st->print_cr(" LDR_CNTRL=%s.", ldr_cntrl ? ldr_cntrl : "");
- // Print out EXTSHM because it is an unsupported setting.
- const char* const extshm = ::getenv("EXTSHM");
- st->print_cr(" EXTSHM=%s.", extshm ? extshm : "");
- if ( (strcmp(extshm, "on") == 0) || (strcmp(extshm, "ON") == 0) ) {
- st->print_cr(" *** Unsupported! Please remove EXTSHM from your environment! ***");
- }
-
// Print out AIXTHREAD_GUARDPAGES because it affects the size of pthread stacks.
const char* const aixthread_guardpages = ::getenv("AIXTHREAD_GUARDPAGES");
st->print_cr(" AIXTHREAD_GUARDPAGES=%s.",
@@ -2133,8 +2122,6 @@ void os::init(void) {
// datapsize = 64k. Data segment, thread stacks are 64k paged.
// This normally means that we can allocate 64k pages dynamically.
- // (There is one special case where this may be false: EXTSHM=on.
- // but we decided to not support that mode).
assert0(g_multipage_support.can_use_64K_pages || g_multipage_support.can_use_64K_mmap_pages);
set_page_size(64*K);
@@ -2543,28 +2530,13 @@ void os::Aix::initialize_os_info() {
void os::Aix::scan_environment() {
char* p;
- int rc;
- // Warn explicitly if EXTSHM=ON is used. That switch changes how
- // System V shared memory behaves. One effect is that page size of
- // shared memory cannot be change dynamically, effectivly preventing
- // large pages from working.
- // This switch was needed on AIX 32bit, but on AIX 64bit the general
- // recommendation is (in OSS notes) to switch it off.
+ // Reject EXTSHM=ON. That switch changes how System V shared memory behaves
+ // and prevents allocation of 64k pages for the heap.
p = ::getenv("EXTSHM");
trcVerbose("EXTSHM=%s.", p ? p : "");
if (p && strcasecmp(p, "ON") == 0) {
- _extshm = 1;
- log_warning(os)("*** Unsupported mode! Please remove EXTSHM from your environment! ***");
- if (!AllowExtshm) {
- // We allow under certain conditions the user to continue. However, we want this
- // to be a fatal error by default. On certain AIX systems, leaving EXTSHM=ON means
- // that the VM is not able to allocate 64k pages for the heap.
- // We do not want to run with reduced performance.
- vm_exit_during_initialization("EXTSHM is ON. Please remove EXTSHM from your environment.");
- }
- } else {
- _extshm = 0;
+ vm_exit_during_initialization("EXTSHM is ON. Please remove EXTSHM from your environment.");
}
// SPEC1170 behaviour: will change the behaviour of a number of POSIX APIs.
@@ -2695,3 +2667,7 @@ void os::print_memory_mappings(char* addr, size_t bytes, outputStream* st) {}
void os::jfr_report_memory_info() {}
#endif // INCLUDE_JFR
+
+void os::print_open_file_descriptors(outputStream* st) {
+ // File descriptor counting not implemented on AIX
+}
diff --git a/src/hotspot/os/aix/os_aix.hpp b/src/hotspot/os/aix/os_aix.hpp
index a30e2077fc2..e21d2cf81bb 100644
--- a/src/hotspot/os/aix/os_aix.hpp
+++ b/src/hotspot/os/aix/os_aix.hpp
@@ -49,11 +49,6 @@ class os::Aix {
// 1 - SPEC1170 requested (XPG_SUS_ENV is ON)
static int _xpg_sus_mode;
- // -1 = uninitialized,
- // 0 - EXTSHM=OFF or not set
- // 1 - EXTSHM=ON
- static int _extshm;
-
static bool available_memory(physical_memory_size_type& value);
static bool free_memory(physical_memory_size_type& value);
static physical_memory_size_type physical_memory() { return _physical_memory; }
@@ -111,12 +106,6 @@ class os::Aix {
return _xpg_sus_mode;
}
- // Returns true if EXTSHM=ON.
- static bool extshm() {
- assert(_extshm != -1, "not initialized");
- return _extshm;
- }
-
// result struct for get_meminfo()
struct meminfo_t {
diff --git a/src/hotspot/os/aix/porting_aix.cpp b/src/hotspot/os/aix/porting_aix.cpp
index b3f878fbfdd..f0527136d90 100644
--- a/src/hotspot/os/aix/porting_aix.cpp
+++ b/src/hotspot/os/aix/porting_aix.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2012, 2024 SAP SE. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
* Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -426,6 +426,10 @@ int dladdr(void* addr, Dl_info* info) {
}
+int JVM_dladdr(void* addr, Dl_info* info) {
+ return dladdr(addr, info);
+}
+
/////////////////////////////////////////////////////////////////////////////
// Native callstack dumping
diff --git a/src/hotspot/os/aix/porting_aix.hpp b/src/hotspot/os/aix/porting_aix.hpp
index a1a22d81471..0bd71079d0a 100644
--- a/src/hotspot/os/aix/porting_aix.hpp
+++ b/src/hotspot/os/aix/porting_aix.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2012, 2024 SAP SE. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -37,25 +37,9 @@
// (see http://linux.die.net/man/3/dladdr)
// dladdr(3) is not POSIX but a GNU extension, and is not available on AIX.
//
-// Differences between AIX dladdr and Linux dladdr:
-//
-// 1) Dl_info.dli_fbase: can never work, is disabled.
-// A loaded image on AIX is divided in multiple segments, at least two
-// (text and data) but potentially also far more. This is because the loader may
-// load each member into an own segment, as for instance happens with the libC.a
-// 2) Dl_info.dli_sname: This only works for code symbols (functions); for data, a
-// zero-length string is returned ("").
-// 3) Dl_info.dli_saddr: For code, this will return the entry point of the function,
-// not the function descriptor.
-typedef struct {
- const char *dli_fname; // file path of loaded library
- // void *dli_fbase;
- const char *dli_sname; // symbol name; "" if not known
- void *dli_saddr; // address of *entry* of function; not function descriptor;
-} Dl_info;
+#include "dl_info.h"
-// Note: we export this to use it inside J2se too
#ifdef __cplusplus
extern "C"
#endif
diff --git a/src/hotspot/os/bsd/os_bsd.cpp b/src/hotspot/os/bsd/os_bsd.cpp
index 29ebe65e0db..a4d9a2197a5 100644
--- a/src/hotspot/os/bsd/os_bsd.cpp
+++ b/src/hotspot/os/bsd/os_bsd.cpp
@@ -76,6 +76,7 @@
# include
# include
# include
+# include
# include
# include
# include
@@ -102,6 +103,7 @@
#endif
#ifdef __APPLE__
+ #include
#include
#include
#endif
@@ -2596,3 +2598,45 @@ bool os::pd_dll_unload(void* libhandle, char* ebuf, int ebuflen) {
return res;
} // end: os::pd_dll_unload()
+
+void os::print_open_file_descriptors(outputStream* st) {
+#ifdef __APPLE__
+ char buf[1024 * sizeof(struct proc_fdinfo)];
+ os::Bsd::print_open_file_descriptors(st, buf, sizeof(buf));
+#else
+ st->print_cr("Open File Descriptors: unknown");
+#endif
+}
+
+void os::Bsd::print_open_file_descriptors(outputStream* st, char* buf, size_t buflen) {
+#ifdef __APPLE__
+ pid_t my_pid;
+
+ // ensure the scratch buffer is big enough for at least one FD info struct
+ precond(buflen >= sizeof(struct proc_fdinfo));
+ kern_return_t kres = pid_for_task(mach_task_self(), &my_pid);
+ if (kres != KERN_SUCCESS) {
+ st->print_cr("Open File Descriptors: unknown");
+ return;
+ }
+ size_t max_fds = buflen / sizeof(struct proc_fdinfo);
+ struct proc_fdinfo* fds = reinterpret_cast(buf);
+
+ // fill our buffer with FD info, up to the available buffer size
+ int res = proc_pidinfo(my_pid, PROC_PIDLISTFDS, 0, fds, max_fds * sizeof(struct proc_fdinfo));
+ if (res <= 0) {
+ st->print_cr("Open File Descriptors: unknown");
+ return;
+ }
+
+ // print lower threshold if count exceeds buffer size
+ int nfiles = res / sizeof(struct proc_fdinfo);
+ if ((size_t)nfiles >= max_fds) {
+ st->print_cr("Open File Descriptors: > %zu", max_fds);
+ return;
+ }
+ st->print_cr("Open File Descriptors: %d", nfiles);
+#else
+ st->print_cr("Open File Descriptors: unknown");
+#endif
+}
diff --git a/src/hotspot/os/bsd/os_bsd.hpp b/src/hotspot/os/bsd/os_bsd.hpp
index da73211b9a7..e87a680b2d2 100644
--- a/src/hotspot/os/bsd/os_bsd.hpp
+++ b/src/hotspot/os/bsd/os_bsd.hpp
@@ -123,6 +123,8 @@ class os::Bsd {
static int get_node_by_cpu(int cpu_id);
static void print_uptime_info(outputStream* st);
+ static void print_open_file_descriptors(outputStream* st, char* buf, size_t buflen);
+ static void print_open_file_descriptors(outputStream* st);
};
#endif // OS_BSD_OS_BSD_HPP
diff --git a/src/hotspot/os/bsd/semaphore_bsd.cpp b/src/hotspot/os/bsd/semaphore_bsd.cpp
index 827c955677e..c35712ff2da 100644
--- a/src/hotspot/os/bsd/semaphore_bsd.cpp
+++ b/src/hotspot/os/bsd/semaphore_bsd.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -81,27 +81,37 @@ bool OSXSemaphore::timedwait(int64_t millis) {
// kernel semaphores take a relative timeout
mach_timespec_t waitspec;
- int secs = millis / MILLIUNITS;
- int nsecs = millis_to_nanos(millis % MILLIUNITS);
- waitspec.tv_sec = secs;
- waitspec.tv_nsec = nsecs;
+ int64_t starttime;
+ const bool is_trywait = millis == 0;
- int64_t starttime = os::javaTimeNanos();
+ if (!is_trywait) {
+ int secs = millis / MILLIUNITS;
+ int nsecs = millis_to_nanos(millis % MILLIUNITS);
+ waitspec.tv_sec = secs;
+ waitspec.tv_nsec = nsecs;
+
+ starttime = os::javaTimeNanos();
+ } else {
+ waitspec.tv_sec = 0;
+ waitspec.tv_nsec = 0;
+ }
kr = semaphore_timedwait(_semaphore, waitspec);
while (kr == KERN_ABORTED) {
- // reduce the timeout and try again
- int64_t totalwait = millis_to_nanos(millis);
- int64_t current = os::javaTimeNanos();
- int64_t passedtime = current - starttime;
+ if (!is_trywait) {
+ // reduce the timeout and try again
+ int64_t totalwait = millis_to_nanos(millis);
+ int64_t current = os::javaTimeNanos();
+ int64_t passedtime = current - starttime;
- if (passedtime >= totalwait) {
- waitspec.tv_sec = 0;
- waitspec.tv_nsec = 0;
- } else {
- int64_t waittime = totalwait - (current - starttime);
- waitspec.tv_sec = waittime / NANOSECS_PER_SEC;
- waitspec.tv_nsec = waittime % NANOSECS_PER_SEC;
+ if (passedtime >= totalwait) {
+ waitspec.tv_sec = 0;
+ waitspec.tv_nsec = 0;
+ } else {
+ int64_t waittime = totalwait - (current - starttime);
+ waitspec.tv_sec = waittime / NANOSECS_PER_SEC;
+ waitspec.tv_nsec = waittime % NANOSECS_PER_SEC;
+ }
}
kr = semaphore_timedwait(_semaphore, waitspec);
diff --git a/src/hotspot/os/linux/cgroupSubsystem_linux.cpp b/src/hotspot/os/linux/cgroupSubsystem_linux.cpp
index e49d070890e..4a2d75ecdf3 100644
--- a/src/hotspot/os/linux/cgroupSubsystem_linux.cpp
+++ b/src/hotspot/os/linux/cgroupSubsystem_linux.cpp
@@ -28,7 +28,6 @@
#include "cgroupV2Subsystem_linux.hpp"
#include "logging/log.hpp"
#include "memory/allocation.hpp"
-#include "os_linux.hpp"
#include "runtime/globals.hpp"
#include "runtime/os.hpp"
#include "utilities/globalDefinitions.hpp"
@@ -41,6 +40,8 @@
// Inlined from for portability.
#ifndef CGROUP2_SUPER_MAGIC
# define CGROUP2_SUPER_MAGIC 0x63677270
+#else
+ STATIC_ASSERT(CGROUP2_SUPER_MAGIC == 0x63677270);
#endif
// controller names have to match the *_IDX indices
@@ -605,6 +606,11 @@ void CgroupSubsystemFactory::cleanup(CgroupInfo* cg_infos) {
}
}
+void CgroupSubsystem::adjust_controllers(physical_memory_size_type upper_mem_bound, int upper_cpu_bound) {
+ CgroupUtil::adjust_controller(memory_controller()->controller(), upper_mem_bound);
+ CgroupUtil::adjust_controller(cpu_controller()->controller(), upper_cpu_bound);
+}
+
/* active_processor_count
*
* Calculate an appropriate number of active processors for the
@@ -631,7 +637,7 @@ void CgroupSubsystemFactory::cleanup(CgroupInfo* cg_infos) {
* return:
* true if there were no errors. false otherwise.
*/
-bool CgroupSubsystem::active_processor_count(double& value) {
+bool CgroupSubsystem::active_processor_count(int (*cpu_bound_func)(), double& value) {
// We use a cache with a timeout to avoid performing expensive
// computations in the event this function is called frequently.
// [See 8227006].
@@ -643,7 +649,7 @@ bool CgroupSubsystem::active_processor_count(double& value) {
return true;
}
- int cpu_count = os::Linux::active_processor_count();
+ int cpu_count = cpu_bound_func();
double result = -1;
if (!CgroupUtil::processor_count(contrl->controller(), cpu_count, result)) {
return false;
diff --git a/src/hotspot/os/linux/cgroupSubsystem_linux.hpp b/src/hotspot/os/linux/cgroupSubsystem_linux.hpp
index d083a9985c2..adde37e1c77 100644
--- a/src/hotspot/os/linux/cgroupSubsystem_linux.hpp
+++ b/src/hotspot/os/linux/cgroupSubsystem_linux.hpp
@@ -278,7 +278,7 @@ class CgroupMemoryController: public CHeapObj {
class CgroupSubsystem: public CHeapObj {
public:
bool memory_limit_in_bytes(physical_memory_size_type upper_bound, physical_memory_size_type& value);
- bool active_processor_count(double& value);
+ bool active_processor_count(int (*cpu_bound_func)(), double& value);
virtual bool pids_max(uint64_t& value) = 0;
virtual bool pids_current(uint64_t& value) = 0;
@@ -291,6 +291,8 @@ class CgroupSubsystem: public CHeapObj {
virtual CachingCgroupController* cpu_controller() = 0;
virtual CgroupCpuacctController* cpuacct_controller() = 0;
+ void adjust_controllers(physical_memory_size_type upper_mem_bound, int upper_cpu_bound);
+
bool cpu_quota(int& value);
bool cpu_period(int& value);
bool cpu_shares(int& value);
diff --git a/src/hotspot/os/linux/cgroupUtil_linux.cpp b/src/hotspot/os/linux/cgroupUtil_linux.cpp
index 570b335940b..f166f6cd5e4 100644
--- a/src/hotspot/os/linux/cgroupUtil_linux.cpp
+++ b/src/hotspot/os/linux/cgroupUtil_linux.cpp
@@ -24,7 +24,6 @@
*/
#include "cgroupUtil_linux.hpp"
-#include "os_linux.hpp"
bool CgroupUtil::processor_count(CgroupCpuController* cpu_ctrl, int upper_bound, double& value) {
assert(upper_bound > 0, "upper bound of cpus must be positive");
@@ -82,7 +81,7 @@ double CgroupUtil::get_updated_cpu_limit(CgroupCpuController* cpu,
return lowest;
}
-void CgroupUtil::adjust_controller(CgroupMemoryController* mem) {
+void CgroupUtil::adjust_controller(CgroupMemoryController* mem, physical_memory_size_type upper_bound) {
assert(mem->cgroup_path() != nullptr, "invariant");
if (strstr(mem->cgroup_path(), "../") != nullptr) {
log_warning(os, container)("Cgroup memory controller path at '%s' seems to have moved "
@@ -100,17 +99,16 @@ void CgroupUtil::adjust_controller(CgroupMemoryController* mem) {
char* cg_path = os::strdup(orig);
char* last_slash;
assert(cg_path[0] == '/', "cgroup path must start with '/'");
- physical_memory_size_type phys_mem = os::Linux::physical_memory();
char* limit_cg_path = nullptr;
physical_memory_size_type limit = value_unlimited;
- physical_memory_size_type lowest_limit = phys_mem;
- lowest_limit = get_updated_mem_limit(mem, lowest_limit, phys_mem);
- physical_memory_size_type orig_limit = lowest_limit != phys_mem ? lowest_limit : phys_mem;
+ physical_memory_size_type lowest_limit = upper_bound;
+ lowest_limit = get_updated_mem_limit(mem, lowest_limit, upper_bound);
+ physical_memory_size_type orig_limit = lowest_limit != upper_bound ? lowest_limit : upper_bound;
while ((last_slash = strrchr(cg_path, '/')) != cg_path) {
*last_slash = '\0'; // strip path
// update to shortened path and try again
mem->set_subsystem_path(cg_path);
- limit = get_updated_mem_limit(mem, lowest_limit, phys_mem);
+ limit = get_updated_mem_limit(mem, lowest_limit, upper_bound);
if (limit < lowest_limit) {
lowest_limit = limit;
os::free(limit_cg_path); // handles nullptr
@@ -119,13 +117,13 @@ void CgroupUtil::adjust_controller(CgroupMemoryController* mem) {
}
// need to check limit at mount point
mem->set_subsystem_path("/");
- limit = get_updated_mem_limit(mem, lowest_limit, phys_mem);
+ limit = get_updated_mem_limit(mem, lowest_limit, upper_bound);
if (limit < lowest_limit) {
lowest_limit = limit;
os::free(limit_cg_path); // handles nullptr
limit_cg_path = os::strdup("/");
}
- assert(lowest_limit <= phys_mem, "limit must not exceed host memory");
+ assert(lowest_limit <= upper_bound, "limit must not exceed upper bound");
if (lowest_limit != orig_limit) {
// we've found a lower limit anywhere in the hierarchy,
// set the path to the limit path
@@ -147,7 +145,7 @@ void CgroupUtil::adjust_controller(CgroupMemoryController* mem) {
os::free(limit_cg_path);
}
-void CgroupUtil::adjust_controller(CgroupCpuController* cpu) {
+void CgroupUtil::adjust_controller(CgroupCpuController* cpu, int upper_bound) {
assert(cpu->cgroup_path() != nullptr, "invariant");
if (strstr(cpu->cgroup_path(), "../") != nullptr) {
log_warning(os, container)("Cgroup cpu controller path at '%s' seems to have moved "
@@ -165,17 +163,16 @@ void CgroupUtil::adjust_controller(CgroupCpuController* cpu) {
char* cg_path = os::strdup(orig);
char* last_slash;
assert(cg_path[0] == '/', "cgroup path must start with '/'");
- int host_cpus = os::Linux::active_processor_count();
- int lowest_limit = host_cpus;
- double cpus = get_updated_cpu_limit(cpu, lowest_limit, host_cpus);
- int orig_limit = lowest_limit != host_cpus ? lowest_limit : host_cpus;
+ int lowest_limit = upper_bound;
+ double cpus = get_updated_cpu_limit(cpu, lowest_limit, upper_bound);
+ int orig_limit = lowest_limit != upper_bound ? lowest_limit : upper_bound;
char* limit_cg_path = nullptr;
while ((last_slash = strrchr(cg_path, '/')) != cg_path) {
*last_slash = '\0'; // strip path
// update to shortened path and try again
cpu->set_subsystem_path(cg_path);
- cpus = get_updated_cpu_limit(cpu, lowest_limit, host_cpus);
- if (cpus != host_cpus && cpus < lowest_limit) {
+ cpus = get_updated_cpu_limit(cpu, lowest_limit, upper_bound);
+ if (cpus != upper_bound && cpus < lowest_limit) {
lowest_limit = cpus;
os::free(limit_cg_path); // handles nullptr
limit_cg_path = os::strdup(cg_path);
@@ -183,8 +180,8 @@ void CgroupUtil::adjust_controller(CgroupCpuController* cpu) {
}
// need to check limit at mount point
cpu->set_subsystem_path("/");
- cpus = get_updated_cpu_limit(cpu, lowest_limit, host_cpus);
- if (cpus != host_cpus && cpus < lowest_limit) {
+ cpus = get_updated_cpu_limit(cpu, lowest_limit, upper_bound);
+ if (cpus != upper_bound && cpus < lowest_limit) {
lowest_limit = cpus;
os::free(limit_cg_path); // handles nullptr
limit_cg_path = os::strdup(cg_path);
diff --git a/src/hotspot/os/linux/cgroupUtil_linux.hpp b/src/hotspot/os/linux/cgroupUtil_linux.hpp
index 1fd2a7d872b..68585c22c2d 100644
--- a/src/hotspot/os/linux/cgroupUtil_linux.hpp
+++ b/src/hotspot/os/linux/cgroupUtil_linux.hpp
@@ -35,10 +35,10 @@ class CgroupUtil: AllStatic {
static bool processor_count(CgroupCpuController* cpu, int upper_bound, double& value);
// Given a memory controller, adjust its path to a point in the hierarchy
// that represents the closest memory limit.
- static void adjust_controller(CgroupMemoryController* m);
+ static void adjust_controller(CgroupMemoryController* m, physical_memory_size_type upper_bound);
// Given a cpu controller, adjust its path to a point in the hierarchy
// that represents the closest cpu limit.
- static void adjust_controller(CgroupCpuController* c);
+ static void adjust_controller(CgroupCpuController* c, int upper_bound);
private:
static physical_memory_size_type get_updated_mem_limit(CgroupMemoryController* m,
physical_memory_size_type lowest,
diff --git a/src/hotspot/os/linux/cgroupV1Subsystem_linux.cpp b/src/hotspot/os/linux/cgroupV1Subsystem_linux.cpp
index c8f5a290c99..e42b7a13391 100644
--- a/src/hotspot/os/linux/cgroupV1Subsystem_linux.cpp
+++ b/src/hotspot/os/linux/cgroupV1Subsystem_linux.cpp
@@ -326,8 +326,6 @@ CgroupV1Subsystem::CgroupV1Subsystem(CgroupV1Controller* cpuset,
_cpuset(cpuset),
_cpuacct(cpuacct),
_pids(pids) {
- CgroupUtil::adjust_controller(memory);
- CgroupUtil::adjust_controller(cpu);
_memory = new CachingCgroupController(memory);
_cpu = new CachingCgroupController(cpu);
}
diff --git a/src/hotspot/os/linux/cgroupV2Subsystem_linux.cpp b/src/hotspot/os/linux/cgroupV2Subsystem_linux.cpp
index 30e1affc646..edd80bb7427 100644
--- a/src/hotspot/os/linux/cgroupV2Subsystem_linux.cpp
+++ b/src/hotspot/os/linux/cgroupV2Subsystem_linux.cpp
@@ -154,8 +154,6 @@ CgroupV2Subsystem::CgroupV2Subsystem(CgroupV2MemoryController * memory,
CgroupV2CpuacctController* cpuacct,
CgroupV2Controller unified) :
_unified(unified) {
- CgroupUtil::adjust_controller(memory);
- CgroupUtil::adjust_controller(cpu);
_memory = new CachingCgroupController(memory);
_cpu = new CachingCgroupController(cpu);
_cpuacct = cpuacct;
diff --git a/src/hotspot/os/linux/hugepages.cpp b/src/hotspot/os/linux/hugepages.cpp
index 5472c093d3f..b065f7b1496 100644
--- a/src/hotspot/os/linux/hugepages.cpp
+++ b/src/hotspot/os/linux/hugepages.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2011, 2024, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -35,11 +35,16 @@
#include
ExplicitHugePageSupport::ExplicitHugePageSupport() :
- _initialized(false), _pagesizes(), _default_hugepage_size(SIZE_MAX), _inconsistent(false) {}
+ _initialized{false}, _os_supported{}, _pre_allocated{}, _default_hugepage_size{0}, _inconsistent{false} {}
-os::PageSizes ExplicitHugePageSupport::pagesizes() const {
+os::PageSizes ExplicitHugePageSupport::os_supported() const {
assert(_initialized, "Not initialized");
- return _pagesizes;
+ return _os_supported;
+}
+
+os::PageSizes ExplicitHugePageSupport::pre_allocated() const {
+ assert(_initialized, "Not initialized");
+ return _pre_allocated;
}
size_t ExplicitHugePageSupport::default_hugepage_size() const {
@@ -63,7 +68,7 @@ static size_t scan_default_hugepagesize() {
// format has been changed), we'll set largest page size to 0
FILE *fp = os::fopen("/proc/meminfo", "r");
- if (fp) {
+ if (fp != nullptr) {
while (!feof(fp)) {
int x = 0;
char buf[16];
@@ -76,7 +81,7 @@ static size_t scan_default_hugepagesize() {
// skip to next line
for (;;) {
int ch = fgetc(fp);
- if (ch == EOF || ch == (int)'\n') break;
+ if (ch == EOF || ch == '\n') break;
}
}
}
@@ -129,10 +134,24 @@ static os::PageSizes scan_hugepages() {
return pagesizes;
}
+static os::PageSizes filter_pre_allocated_hugepages(os::PageSizes pagesizes) {
+ os::PageSizes pre_allocated{};
+ char filename[PATH_MAX];
+ for (size_t ps = pagesizes.smallest(); ps != 0; ps = pagesizes.next_larger(ps)) {
+ os::snprintf_checked(filename, sizeof(filename), "%s/hugepages-%zukB/nr_hugepages", sys_hugepages, ps / K);
+ size_t pages;
+ bool read_success = read_number_file(filename, &pages);
+ if (read_success && pages > 0) {
+ pre_allocated.add(ps);
+ }
+ }
+ return pre_allocated;
+}
+
void ExplicitHugePageSupport::print_on(outputStream* os) {
if (_initialized) {
os->print_cr("Explicit hugepage support:");
- for (size_t s = _pagesizes.smallest(); s != 0; s = _pagesizes.next_larger(s)) {
+ for (size_t s = _os_supported.smallest(); s != 0; s = _os_supported.next_larger(s)) {
os->print_cr(" hugepage size: " EXACTFMT, EXACTFMTARGS(s));
}
os->print_cr(" default hugepage size: " EXACTFMT, EXACTFMTARGS(_default_hugepage_size));
@@ -147,14 +166,15 @@ void ExplicitHugePageSupport::print_on(outputStream* os) {
void ExplicitHugePageSupport::scan_os() {
_default_hugepage_size = scan_default_hugepagesize();
if (_default_hugepage_size > 0) {
- _pagesizes = scan_hugepages();
+ _os_supported = scan_hugepages();
+ _pre_allocated = filter_pre_allocated_hugepages(_os_supported);
// See https://www.kernel.org/doc/Documentation/vm/hugetlbpage.txt: /proc/meminfo should match
// /sys/kernel/mm/hugepages/hugepages-xxxx. However, we may run on a broken kernel (e.g. on WSL)
// that only exposes /proc/meminfo but not /sys/kernel/mm/hugepages. In that case, we are not
// sure about the state of hugepage support by the kernel, so we won't use explicit hugepages.
- if (!_pagesizes.contains(_default_hugepage_size)) {
+ if (!_os_supported.contains(_default_hugepage_size)) {
log_info(pagesize)("Unexpected configuration: default pagesize (%zu) "
- "has no associated directory in /sys/kernel/mm/hugepages..", _default_hugepage_size);
+ "has no associated directory in /sys/kernel/mm/hugepages.", _default_hugepage_size);
_inconsistent = true;
}
}
@@ -167,7 +187,7 @@ void ExplicitHugePageSupport::scan_os() {
}
THPSupport::THPSupport() :
- _initialized(false), _mode(THPMode::never), _pagesize(SIZE_MAX) {}
+ _initialized{false}, _mode{THPMode::never}, _pagesize{0} {}
THPMode THPSupport::mode() const {
@@ -201,7 +221,6 @@ void THPSupport::scan_os() {
}
// Scan large page size for THP from hpage_pmd_size
- _pagesize = 0;
if (read_number_file("/sys/kernel/mm/transparent_hugepage/hpage_pmd_size", &_pagesize)) {
assert(_pagesize > 0, "Expected");
}
diff --git a/src/hotspot/os/linux/hugepages.hpp b/src/hotspot/os/linux/hugepages.hpp
index efd27c55fd6..5a9767b4ff8 100644
--- a/src/hotspot/os/linux/hugepages.hpp
+++ b/src/hotspot/os/linux/hugepages.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2011, 2024, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -45,7 +45,10 @@ class ExplicitHugePageSupport {
// All supported hugepage sizes (sizes for which entries exist
// in /sys/kernel/mm/hugepages/hugepage-xxx)
- os::PageSizes _pagesizes;
+ os::PageSizes _os_supported;
+
+ // Above pages filtered for where the contents of file nr_hugepages was larger than zero
+ os::PageSizes _pre_allocated;
// Contains the default hugepage. The "default hugepage size" is the one that
// - is marked in /proc/meminfo as "Hugepagesize"
@@ -60,7 +63,8 @@ public:
void scan_os();
- os::PageSizes pagesizes() const;
+ os::PageSizes os_supported() const;
+ os::PageSizes pre_allocated() const;
size_t default_hugepage_size() const;
void print_on(outputStream* os);
diff --git a/src/hotspot/os/linux/osContainer_linux.cpp b/src/hotspot/os/linux/osContainer_linux.cpp
index b46263efd99..da2cbf381e6 100644
--- a/src/hotspot/os/linux/osContainer_linux.cpp
+++ b/src/hotspot/os/linux/osContainer_linux.cpp
@@ -59,6 +59,11 @@ void OSContainer::init() {
if (cgroup_subsystem == nullptr) {
return; // Required subsystem files not found or other error
}
+ // Adjust controller paths once subsystem is initialized
+ physical_memory_size_type phys_mem = os::Linux::physical_memory();
+ int host_cpus = os::Linux::active_processor_count();
+ cgroup_subsystem->adjust_controllers(phys_mem, host_cpus);
+
/*
* In order to avoid a false positive on is_containerized() on
* Linux systems outside a container *and* to ensure compatibility
@@ -252,7 +257,7 @@ char * OSContainer::cpu_cpuset_memory_nodes() {
bool OSContainer::active_processor_count(double& value) {
assert(cgroup_subsystem != nullptr, "cgroup subsystem not available");
- return cgroup_subsystem->active_processor_count(value);
+ return cgroup_subsystem->active_processor_count(&os::Linux::active_processor_count, value);
}
bool OSContainer::cpu_quota(int& value) {
diff --git a/src/hotspot/os/linux/os_linux.cpp b/src/hotspot/os/linux/os_linux.cpp
index 9c2fbab7535..a87c0ab33fa 100644
--- a/src/hotspot/os/linux/os_linux.cpp
+++ b/src/hotspot/os/linux/os_linux.cpp
@@ -83,6 +83,7 @@
#endif
# include
+# include
# include
# include
# include
@@ -113,6 +114,7 @@
# include
# include
# include
+# include
# include
#ifdef __GLIBC__
# include
@@ -1311,7 +1313,7 @@ bool os::is_primordial_thread(void) {
// Find the virtual memory area that contains addr
static bool find_vma(address addr, address* vma_low, address* vma_high) {
FILE *fp = os::fopen("/proc/self/maps", "r");
- if (fp) {
+ if (fp != nullptr) {
address low, high;
while (!feof(fp)) {
if (fscanf(fp, "%p-%p", &low, &high) == 2) {
@@ -1324,7 +1326,7 @@ static bool find_vma(address addr, address* vma_low, address* vma_high) {
}
for (;;) {
int ch = fgetc(fp);
- if (ch == EOF || ch == (int)'\n') break;
+ if (ch == EOF || ch == '\n') break;
}
}
fclose(fp);
@@ -3814,8 +3816,8 @@ static int hugetlbfs_page_size_flag(size_t page_size) {
}
static bool hugetlbfs_sanity_check(size_t page_size) {
- const os::PageSizes page_sizes = HugePages::explicit_hugepage_info().pagesizes();
- assert(page_sizes.contains(page_size), "Invalid page sizes passed");
+ const os::PageSizes os_supported = HugePages::explicit_hugepage_info().os_supported();
+ assert(os_supported.contains(page_size), "Invalid page sizes passed (%zu)", page_size);
// Include the page size flag to ensure we sanity check the correct page size.
int flags = MAP_ANONYMOUS | MAP_PRIVATE | MAP_HUGETLB | hugetlbfs_page_size_flag(page_size);
@@ -3829,16 +3831,16 @@ static bool hugetlbfs_sanity_check(size_t page_size) {
log_info(pagesize)("Large page size (" EXACTFMT ") failed sanity check, "
"checking if smaller large page sizes are usable",
EXACTFMTARGS(page_size));
- for (size_t page_size_ = page_sizes.next_smaller(page_size);
- page_size_ > os::vm_page_size();
- page_size_ = page_sizes.next_smaller(page_size_)) {
- flags = MAP_ANONYMOUS | MAP_PRIVATE | MAP_HUGETLB | hugetlbfs_page_size_flag(page_size_);
- p = mmap(nullptr, page_size_, PROT_READ|PROT_WRITE, flags, -1, 0);
+ for (size_t size = os_supported.next_smaller(page_size);
+ size > os::vm_page_size();
+ size = os_supported.next_smaller(size)) {
+ flags = MAP_ANONYMOUS | MAP_PRIVATE | MAP_HUGETLB | hugetlbfs_page_size_flag(size);
+ p = mmap(nullptr, size, PROT_READ|PROT_WRITE, flags, -1, 0);
if (p != MAP_FAILED) {
// Mapping succeeded, sanity check passed.
- munmap(p, page_size_);
+ munmap(p, size);
log_info(pagesize)("Large page size (" EXACTFMT ") passed sanity check",
- EXACTFMTARGS(page_size_));
+ EXACTFMTARGS(size));
return true;
}
}
@@ -4020,7 +4022,7 @@ void os::Linux::large_page_init() {
// - os::large_page_size() is the default explicit hugepage size (/proc/meminfo "Hugepagesize")
// - os::pagesizes() contains all hugepage sizes the kernel supports, regardless whether there
// are pages configured in the pool or not (from /sys/kernel/hugepages/hugepage-xxxx ...)
- os::PageSizes all_large_pages = HugePages::explicit_hugepage_info().pagesizes();
+ os::PageSizes all_large_pages = HugePages::explicit_hugepage_info().os_supported();
const size_t default_large_page_size = HugePages::default_explicit_hugepage_size();
// 3) Consistency check and post-processing
@@ -4062,10 +4064,10 @@ void os::Linux::large_page_init() {
_large_page_size = large_page_size;
- // Populate _page_sizes with large page sizes less than or equal to
- // _large_page_size.
- for (size_t page_size = _large_page_size; page_size != 0;
- page_size = all_large_pages.next_smaller(page_size)) {
+ // Populate _page_sizes with _large_page_size (default large page size) even if not pre-allocated.
+ // Then, populate _page_sizes with all smaller large page sizes that have been pre-allocated.
+ os::PageSizes pre_allocated = HugePages::explicit_hugepage_info().pre_allocated();
+ for (size_t page_size = _large_page_size; page_size != 0; page_size = pre_allocated.next_smaller(page_size)) {
_page_sizes.add(page_size);
}
}
@@ -4129,12 +4131,12 @@ static char* reserve_memory_special_huge_tlbfs(size_t bytes,
size_t page_size,
char* req_addr,
bool exec) {
- const os::PageSizes page_sizes = HugePages::explicit_hugepage_info().pagesizes();
+ const os::PageSizes os_supported = HugePages::explicit_hugepage_info().os_supported();
assert(UseLargePages, "only for Huge TLBFS large pages");
assert(is_aligned(req_addr, alignment), "Must be");
assert(is_aligned(req_addr, page_size), "Must be");
assert(is_aligned(alignment, os::vm_allocation_granularity()), "Must be");
- assert(page_sizes.contains(page_size), "Must be a valid page size");
+ assert(os_supported.contains(page_size), "Must be a valid page size");
assert(page_size > os::vm_page_size(), "Must be a large page size");
assert(bytes >= page_size, "Shouldn't allocate large pages for small sizes");
@@ -4380,7 +4382,7 @@ int os::Linux::get_namespace_pid(int vmid) {
os::snprintf_checked(fname, sizeof(fname), "/proc/%d/status", vmid);
FILE *fp = os::fopen(fname, "r");
- if (fp) {
+ if (fp != nullptr) {
int pid, nspid;
int ret;
while (!feof(fp) && !ferror(fp)) {
@@ -4394,7 +4396,7 @@ int os::Linux::get_namespace_pid(int vmid) {
}
for (;;) {
int ch = fgetc(fp);
- if (ch == EOF || ch == (int)'\n') break;
+ if (ch == EOF || ch == '\n') break;
}
}
fclose(fp);
@@ -4549,6 +4551,7 @@ void os::Linux::numa_init() {
FLAG_SET_ERGO_IF_DEFAULT(UseNUMAInterleaving, true);
}
+#if INCLUDE_PARALLELGC
if (UseParallelGC && UseNUMA && UseLargePages && !can_commit_large_page_memory()) {
// With static large pages we cannot uncommit a page, so there's no way
// we can make the adaptive lgrp chunk resizing work. If the user specified both
@@ -4560,6 +4563,7 @@ void os::Linux::numa_init() {
UseAdaptiveNUMAChunkSizing = false;
}
}
+#endif
}
void os::Linux::disable_numa(const char* reason, bool warning) {
@@ -5427,3 +5431,31 @@ bool os::pd_dll_unload(void* libhandle, char* ebuf, int ebuflen) {
return res;
} // end: os::pd_dll_unload()
+
+void os::print_open_file_descriptors(outputStream* st) {
+ DIR* dirp = opendir("/proc/self/fd");
+ int fds = 0;
+ struct dirent* dentp;
+ const jlong TIMEOUT_NS = 50000000L; // 50 ms in nanoseconds
+ bool timed_out = false;
+
+ // limit proc file read to 50ms
+ jlong start = os::javaTimeNanos();
+ assert(dirp != nullptr, "No proc fs?");
+ while ((dentp = readdir(dirp)) != nullptr && !timed_out) {
+ if (isdigit(dentp->d_name[0])) fds++;
+ if (fds % 100 == 0) {
+ jlong now = os::javaTimeNanos();
+ if ((now - start) > TIMEOUT_NS) {
+ timed_out = true;
+ }
+ }
+ }
+
+ closedir(dirp);
+ if (timed_out) {
+ st->print_cr("Open File Descriptors: > %d", fds);
+ } else {
+ st->print_cr("Open File Descriptors: %d", fds);
+ }
+}
diff --git a/src/hotspot/os/posix/dtrace/hotspot_jni.d b/src/hotspot/os/posix/dtrace/hotspot_jni.d
index c5676921b37..1937769dcb2 100644
--- a/src/hotspot/os/posix/dtrace/hotspot_jni.d
+++ b/src/hotspot/os/posix/dtrace/hotspot_jni.d
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -366,6 +366,8 @@ provider hotspot_jni {
probe IsInstanceOf__return(uintptr_t);
probe IsSameObject__entry(void*, void*, void*);
probe IsSameObject__return(uintptr_t);
+ probe IsVirtualThread__entry(void*, void*);
+ probe IsVirtualThread__return(uintptr_t);
probe MonitorEnter__entry(void*, void*);
probe MonitorEnter__return(uint32_t);
probe MonitorExit__entry(void*, void*);
diff --git a/src/hotspot/os/posix/include/jvm_md.h b/src/hotspot/os/posix/include/jvm_md.h
index eb8e1f0d7e9..061ef17aaae 100644
--- a/src/hotspot/os/posix/include/jvm_md.h
+++ b/src/hotspot/os/posix/include/jvm_md.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -62,6 +62,19 @@
#define JVM_X_OK X_OK
#define JVM_F_OK F_OK
+#if defined(AIX)
+#include "jni_md.h"
+#include "dl_info.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+JNIEXPORT int JVM_dladdr(void* addr, Dl_info* info);
+#ifdef __cplusplus
+}
+#endif
+#endif
+
/*
* File I/O
*/
diff --git a/src/hotspot/os/posix/os_posix.cpp b/src/hotspot/os/posix/os_posix.cpp
index f147ed4be93..1fb2a248bec 100644
--- a/src/hotspot/os/posix/os_posix.cpp
+++ b/src/hotspot/os/posix/os_posix.cpp
@@ -888,6 +888,14 @@ void* os::lookup_function(const char* name) {
return dlsym(RTLD_DEFAULT, name);
}
+int64_t os::ftell(FILE* file) {
+ return ::ftell(file);
+}
+
+int os::fseek(FILE* file, int64_t offset, int whence) {
+ return ::fseek(file, offset, whence);
+}
+
jlong os::lseek(int fd, jlong offset, int whence) {
return (jlong) ::lseek(fd, offset, whence);
}
diff --git a/src/hotspot/os/posix/perfMemory_posix.cpp b/src/hotspot/os/posix/perfMemory_posix.cpp
index d9bde6fa825..c5046797e02 100644
--- a/src/hotspot/os/posix/perfMemory_posix.cpp
+++ b/src/hotspot/os/posix/perfMemory_posix.cpp
@@ -701,6 +701,39 @@ static void remove_file(const char* path) {
}
}
+// Files newer than this threshold are considered to belong to a JVM that may
+// still be starting up and are therefore not candidates for stale-file
+// cleanup. This avoids racing a concurrent JVM startup while scanning the
+// hsperfdata directory.
+static const time_t cleanup_grace_period_seconds = 5;
+
+static bool is_cleanup_candidate(const char* filename, const char* dirname) {
+ struct stat statbuf;
+ int result;
+
+ RESTARTABLE(::lstat(filename, &statbuf), result);
+ if (result == OS_ERR) {
+ log_debug(perf, memops)("lstat failed for %s/%s: %s", dirname, filename, os::strerror(errno));
+ return false;
+ }
+
+ if (!S_ISREG(statbuf.st_mode)) {
+ return false;
+ }
+
+ const time_t now = time(nullptr);
+ if (now == (time_t)-1) {
+ return false;
+ }
+
+ if (statbuf.st_mtime >= now - cleanup_grace_period_seconds) {
+ log_debug(perf, memops)("Skip cleanup of fresh file %s/%s", dirname, filename);
+ return false;
+ }
+
+ return true;
+}
+
// cleanup stale shared memory files
//
// This method attempts to remove all stale shared memory files in
@@ -744,6 +777,11 @@ static void cleanup_sharedmem_files(const char* dirname) {
continue;
}
+ if (!is_cleanup_candidate(filename, dirname)) {
+ errno = 0;
+ continue;
+ }
+
#if defined(LINUX)
// Special case on Linux, if multiple containers share the
// same /tmp directory:
@@ -872,16 +910,56 @@ static int create_sharedmem_file(const char* dirname, const char* filename, size
return -1;
}
- // Open the filename in the current directory.
- // Cannot use O_TRUNC here; truncation of an existing file has to happen
- // after the is_file_secure() check below.
- int fd;
- RESTARTABLE(os::open(filename, O_RDWR|O_CREAT|O_NOFOLLOW, S_IRUSR|S_IWUSR), fd);
+ int fd = OS_ERR;
+ static const int create_sharedmem_file_retry_count = LINUX_ONLY(3) NOT_LINUX(1);
+ for (int attempt = 0; attempt < create_sharedmem_file_retry_count; attempt++) {
+ // Open the filename in the current directory.
+ // Use O_EXCL so that startup never reuses an existing pid file unless it
+ // has first been proven stale and removed in `cleanup_sharedmem_files`.
+ RESTARTABLE(os::open(filename, O_RDWR|O_CREAT|O_EXCL|O_NOFOLLOW, S_IRUSR|S_IWUSR), fd);
+ if (fd == OS_ERR) {
+ break;
+ }
+
+#if defined(LINUX)
+ // On Linux, different containerized processes that share the same /tmp
+ // directory (e.g., with "docker --volume ...") may have the same pid and
+ // try to use the same file. To avoid conflicts among such processes, we
+ // allow only one of them (the winner of the flock() call) to write to the
+ // file. If we lose the race, assume we may have collided with a concurrent
+ // scavenger briefly holding the lock on a fresh file and retry a few times
+ // before giving up.
+ int n;
+ RESTARTABLE(::flock(fd, LOCK_EX|LOCK_NB), n);
+ if (n == 0) {
+ break;
+ }
+
+ const int flock_errno = errno;
+ ::close(fd);
+ fd = OS_ERR;
+
+ if (attempt + 1 == create_sharedmem_file_retry_count || flock_errno != EWOULDBLOCK) {
+ log_warning(perf, memops)("Cannot use file %s/%s because %s (errno = %d)", dirname, filename,
+ (flock_errno == EWOULDBLOCK) ?
+ "it is locked by another process" :
+ "flock() failed", flock_errno);
+ errno = flock_errno;
+ break;
+ }
+
+ // Short sleep to allow the lock to free up.
+ os::naked_short_sleep(1);
+#endif
+ }
+
if (fd == OS_ERR) {
if (log_is_enabled(Debug, perf)) {
LogStreamHandle(Debug, perf) log;
if (errno == ELOOP) {
log.print_cr("file %s is a symlink and is not secure", filename);
+ } else if (errno == EEXIST) {
+ log.print_cr("could not create file %s: existing file is not provably stale", filename);
} else {
log.print_cr("could not create file %s: %s", filename, os::strerror(errno));
}
@@ -901,27 +979,7 @@ static int create_sharedmem_file(const char* dirname, const char* filename, size
}
#if defined(LINUX)
- // On Linux, different containerized processes that share the same /tmp
- // directory (e.g., with "docker --volume ...") may have the same pid and
- // try to use the same file. To avoid conflicts among such
- // processes, we allow only one of them (the winner of the flock() call)
- // to write to the file. All the other processes will give up and will
- // have perfdata disabled.
- //
- // Note that the flock will be automatically given up when the winner
- // process exits.
- //
- // The locking protocol works only with other JVMs that have the JDK-8286030
- // fix. If you are sharing the /tmp difrectory among different containers,
- // do not use older JVMs that don't have this fix, or the behavior is undefined.
- int n;
- RESTARTABLE(::flock(fd, LOCK_EX|LOCK_NB), n);
- if (n != 0) {
- log_warning(perf, memops)("Cannot use file %s/%s because %s (errno = %d)", dirname, filename,
- (errno == EWOULDBLOCK) ?
- "it is locked by another process" :
- "flock() failed", errno);
- ::close(fd);
+ if (fd == OS_ERR) {
return -1;
}
#endif
@@ -1084,18 +1142,9 @@ static char* mmap_create_shared(size_t size) {
// release a named shared memory region that was mmap-ed.
//
static void unmap_shared(char* addr, size_t bytes) {
- int res;
- if (MemTracker::enabled()) {
- MemTracker::NmtVirtualMemoryLocker nvml;
- res = ::munmap(addr, bytes);
- if (res == 0) {
- MemTracker::record_virtual_memory_release(addr, bytes);
- }
- } else {
- res = ::munmap(addr, bytes);
- }
- if (res != 0) {
- log_info(os)("os::release_memory failed (" PTR_FORMAT ", %zu)", p2i(addr), bytes);
+ MemTracker::record_virtual_memory_release(addr, bytes);
+ if (::munmap(addr, bytes) != 0) {
+ fatal("os::release_memory failed (" PTR_FORMAT ", %zu)", p2i(addr), bytes);
}
}
diff --git a/src/hotspot/os/windows/os_windows.cpp b/src/hotspot/os/windows/os_windows.cpp
index 76f47640e5a..9d8fb45f0d1 100644
--- a/src/hotspot/os/windows/os_windows.cpp
+++ b/src/hotspot/os/windows/os_windows.cpp
@@ -2528,12 +2528,6 @@ LONG Handle_Exception(struct _EXCEPTION_POINTERS* exceptionInfo,
return EXCEPTION_CONTINUE_EXECUTION;
}
-
-// Used for PostMortemDump
-extern "C" void safepoints();
-extern "C" void find(int x);
-extern "C" void events();
-
// According to Windows API documentation, an illegal instruction sequence should generate
// the 0xC000001C exception code. However, real world experience shows that occasionnaly
// the execution of an illegal instruction can generate the exception code 0xC000001E. This
@@ -5114,6 +5108,13 @@ jlong os::seek_to_file_offset(int fd, jlong offset) {
return (jlong)::_lseeki64(fd, (__int64)offset, SEEK_SET);
}
+int64_t os::ftell(FILE* file) {
+ return ::_ftelli64(file);
+}
+
+int os::fseek(FILE* file, int64_t offset, int whence) {
+ return ::_fseeki64(file,offset, whence);
+}
jlong os::lseek(int fd, jlong offset, int whence) {
return (jlong) ::_lseeki64(fd, offset, whence);
@@ -6276,6 +6277,10 @@ const void* os::get_saved_assert_context(const void** sigInfo) {
return nullptr;
}
+void os::print_open_file_descriptors(outputStream* st) {
+ // File descriptor counting not supported on Windows.
+}
+
/*
* Windows/x64 does not use stack frames the way expected by Java:
* [1] in most cases, there is no frame pointer. All locals are addressed via RSP
diff --git a/src/hotspot/os/windows/perfMemory_windows.cpp b/src/hotspot/os/windows/perfMemory_windows.cpp
index f54a2b52cca..dad2804f18a 100644
--- a/src/hotspot/os/windows/perfMemory_windows.cpp
+++ b/src/hotspot/os/windows/perfMemory_windows.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1682,12 +1682,7 @@ void PerfMemory::detach(char* addr, size_t bytes) {
return;
}
- if (MemTracker::enabled()) {
- // it does not go through os api, the operation has to record from here
- MemTracker::NmtVirtualMemoryLocker nvml;
- remove_file_mapping(addr);
- MemTracker::record_virtual_memory_release(addr, bytes);
- } else {
- remove_file_mapping(addr);
- }
+ // it does not go through os api, the operation has to record from here
+ MemTracker::record_virtual_memory_release(addr, bytes);
+ remove_file_mapping(addr);
}
diff --git a/src/hotspot/share/cds/aotGrowableArray.cpp b/src/hotspot/os_cpu/aix_ppc/vm_version_aix_ppc.cpp
similarity index 78%
rename from src/hotspot/share/cds/aotGrowableArray.cpp
rename to src/hotspot/os_cpu/aix_ppc/vm_version_aix_ppc.cpp
index ec63e7aa57f..8cc8b715201 100644
--- a/src/hotspot/share/cds/aotGrowableArray.cpp
+++ b/src/hotspot/os_cpu/aix_ppc/vm_version_aix_ppc.cpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -22,13 +23,14 @@
*
*/
-#include "cds/aotGrowableArray.hpp"
-#include "cds/aotMetaspace.hpp"
-#include "memory/allocation.inline.hpp"
-#include "utilities/growableArray.hpp"
+#include "runtime/vm_version.hpp"
-void AOTGrowableArrayHelper::deallocate(void* mem) {
- if (!AOTMetaspace::in_aot_cache(mem)) {
- GrowableArrayCHeapAllocator::deallocate(mem);
- }
+#include
+
+int VM_Version::get_dcache_line_size() {
+ return _system_configuration.dcache_line;
+}
+
+int VM_Version::get_icache_line_size() {
+ return _system_configuration.icache_line;
}
diff --git a/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp b/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp
index 36599594842..49d879731ff 100644
--- a/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp
+++ b/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp
@@ -620,6 +620,8 @@ extern "C" {
assert(VM_Version::supports_sb(), "current CPU does not support SB instruction");
asm volatile(".inst 0xd50330ff" : : : "memory");
break;
+ case SpinWait::WFET:
+ ShouldNotReachHere();
#ifdef ASSERT
default:
ShouldNotReachHere();
diff --git a/src/hotspot/os_cpu/bsd_zero/atomicAccess_bsd_zero.hpp b/src/hotspot/os_cpu/bsd_zero/atomicAccess_bsd_zero.hpp
index 6c8684718fc..8e45490e5b6 100644
--- a/src/hotspot/os_cpu/bsd_zero/atomicAccess_bsd_zero.hpp
+++ b/src/hotspot/os_cpu/bsd_zero/atomicAccess_bsd_zero.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright 2007, 2008, 2011, 2015, Red Hat, Inc.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -27,7 +27,6 @@
#define OS_CPU_BSD_ZERO_ATOMICACCESS_BSD_ZERO_HPP
#include "orderAccess_bsd_zero.hpp"
-#include "runtime/os.hpp"
// Implementation of class AtomicAccess
diff --git a/src/hotspot/os_cpu/linux_aarch64/ic_ivau_probe_linux_aarch64.S b/src/hotspot/os_cpu/linux_aarch64/ic_ivau_probe_linux_aarch64.S
new file mode 100644
index 00000000000..b82053d37b9
--- /dev/null
+++ b/src/hotspot/os_cpu/linux_aarch64/ic_ivau_probe_linux_aarch64.S
@@ -0,0 +1,69 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "defs.S.inc"
+
+ # Probe whether IC IVAU is trapped.
+ #
+ # Returns 1 if IC IVAU is trapped (did not fault), 0 if not trapped
+ # (faulted on VA 0x0, signal handler redirected to continuation).
+ #
+ # int ic_ivau_probe(void);
+DECLARE_FUNC(ic_ivau_probe):
+DECLARE_FUNC(_ic_ivau_probe_fault):
+ ic ivau, xzr
+ mov x0, #1
+ ret
+DECLARE_FUNC(_ic_ivau_probe_continuation):
+ mov x0, #0
+ ret
+
+/* Emit .note.gnu.property section in case of PAC or BTI being enabled. */
+#ifdef __ARM_FEATURE_BTI_DEFAULT
+ #ifdef __ARM_FEATURE_PAC_DEFAULT
+ #define GNU_PROPERTY_AARCH64_FEATURE 3
+ #else
+ #define GNU_PROPERTY_AARCH64_FEATURE 1
+ #endif
+#else
+ #ifdef __ARM_FEATURE_PAC_DEFAULT
+ #define GNU_PROPERTY_AARCH64_FEATURE 2
+ #else
+ #define GNU_PROPERTY_AARCH64_FEATURE 0
+ #endif
+#endif
+
+#if (GNU_PROPERTY_AARCH64_FEATURE != 0)
+ .pushsection .note.gnu.property, "a"
+ .align 3
+ .long 4 /* name length */
+ .long 0x10 /* data length */
+ .long 5 /* note type: NT_GNU_PROPERTY_TYPE_0 */
+ .string "GNU" /* vendor name */
+ .long 0xc0000000 /* GNU_PROPERTY_AARCH64_FEATURE_1_AND */
+ .long 4 /* pr_datasze */
+ .long GNU_PROPERTY_AARCH64_FEATURE
+ .long 0
+ .popsection
+#endif
diff --git a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp
new file mode 100644
index 00000000000..11911a48e06
--- /dev/null
+++ b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp
@@ -0,0 +1,28 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "runtime/icache.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+DEBUG_ONLY(THREAD_LOCAL AArch64ICacheInvalidationContext* AArch64ICacheInvalidationContext::_current_context = nullptr;)
diff --git a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp
index 8fbaa7a6b6e..5121a875701 100644
--- a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp
+++ b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp
@@ -26,6 +26,11 @@
#ifndef OS_CPU_LINUX_AARCH64_ICACHE_AARCH64_HPP
#define OS_CPU_LINUX_AARCH64_ICACHE_AARCH64_HPP
+#include "memory/allocation.hpp"
+#include "runtime/vm_version.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "vm_version_aarch64.hpp"
+
// Interface for updating the instruction cache. Whenever the VM
// modifies code, part of the processor instruction cache potentially
// has to be flushed.
@@ -37,8 +42,105 @@ class ICache : public AbstractICache {
__builtin___clear_cache((char *)addr, (char *)(addr + 4));
}
static void invalidate_range(address start, int nbytes) {
- __builtin___clear_cache((char *)start, (char *)(start + nbytes));
+ if (NeoverseN1ICacheErratumMitigation) {
+ assert(VM_Version::is_cache_idc_enabled(),
+ "Expect CTR_EL0.IDC to be enabled for Neoverse N1 with erratum "
+ "1542419");
+ assert(!VM_Version::is_cache_dic_enabled(),
+ "Expect CTR_EL0.DIC to be disabled for Neoverse N1 with erratum "
+ "1542419");
+ assert(VM_Version::is_ic_ivau_trapped(), "Expect 'ic ivau, xzr' to be trapped");
+ asm volatile("dsb ish \n"
+ "ic ivau, xzr \n"
+ "dsb ish \n"
+ "isb \n"
+ : : : "memory");
+ } else {
+ __builtin___clear_cache((char *)start, (char *)(start + nbytes));
+ }
}
};
+class AArch64ICacheInvalidationContext : StackObj {
+ private:
+
+#ifdef ASSERT
+ static THREAD_LOCAL AArch64ICacheInvalidationContext* _current_context;
+#endif
+
+ bool _has_modified_code;
+
+ public:
+ NONCOPYABLE(AArch64ICacheInvalidationContext);
+
+ AArch64ICacheInvalidationContext()
+ : _has_modified_code(false) {
+ assert(_current_context == nullptr, "nested ICacheInvalidationContext not supported");
+#ifdef ASSERT
+ _current_context = this;
+#endif
+ }
+
+ ~AArch64ICacheInvalidationContext() {
+ DEBUG_ONLY(_current_context = nullptr);
+
+ if (!_has_modified_code || !UseSingleICacheInvalidation) {
+ return;
+ }
+
+ assert(VM_Version::is_cache_idc_enabled(), "Expect CTR_EL0.IDC to be enabled");
+
+ asm volatile("dsb ish" : : : "memory");
+
+ if (NeoverseN1ICacheErratumMitigation) {
+ assert(!VM_Version::is_cache_dic_enabled(),
+ "Expect CTR_EL0.DIC to be disabled for Neoverse N1 with erratum "
+ "1542419");
+ assert(VM_Version::is_ic_ivau_trapped(), "Expect 'ic ivau, xzr' to be trapped");
+
+ // Errata 1542419: Neoverse N1 cores with the 'COHERENT_ICACHE' feature
+ // may fetch stale instructions when software depends on
+ // prefetch-speculation-protection instead of explicit synchronization.
+ //
+ // Neoverse-N1 implementation mitigates the errata 1542419 with a
+ // workaround:
+ // - Disable coherent icache.
+ // - Trap IC IVAU instructions.
+ // - Execute:
+ // - tlbi vae3is, xzr
+ // - dsb sy
+ // - Ignore trapped IC IVAU instructions.
+ //
+ // `tlbi vae3is, xzr` invalidates all translation entries (all VAs, all
+ // possible levels). It waits for all memory accesses using in-scope old
+ // translation information to complete before it is considered complete.
+ //
+ // As this workaround has significant overhead, Arm Neoverse N1 (MP050)
+ // Software Developer Errata Notice version 29.0 suggests:
+ //
+ // "Since one TLB inner-shareable invalidation is enough to avoid this
+ // erratum, the number of injected TLB invalidations should be minimized
+ // in the trap handler to mitigate the performance impact due to this
+ // workaround."
+ // As the address for icache invalidation is not relevant and
+ // IC IVAU instruction is ignored, we use XZR in it.
+ asm volatile(
+ "ic ivau, xzr \n"
+ "dsb ish \n"
+ :
+ :
+ : "memory");
+ } else {
+ assert(VM_Version::is_cache_dic_enabled(), "Expect CTR_EL0.DIC to be enabled");
+ }
+ asm volatile("isb" : : : "memory");
+ }
+
+ void set_has_modified_code() {
+ _has_modified_code = true;
+ }
+};
+
+#define PD_ICACHE_INVALIDATION_CONTEXT AArch64ICacheInvalidationContext
+
#endif // OS_CPU_LINUX_AARCH64_ICACHE_AARCH64_HPP
diff --git a/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp
index da9e7e159f1..67e0569bf31 100644
--- a/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp
+++ b/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp
@@ -77,6 +77,11 @@
#define REG_LR 30
#define REG_BCP 22
+// IC IVAU trap probe.
+// Defined in ic_ivau_probe_linux_aarch64.S.
+extern "C" char _ic_ivau_probe_fault[] __attribute__ ((visibility ("hidden")));
+extern "C" char _ic_ivau_probe_continuation[] __attribute__ ((visibility ("hidden")));
+
NOINLINE address os::current_stack_pointer() {
return (address)__builtin_frame_address(0);
}
@@ -228,6 +233,12 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info,
}
}
+ // IC IVAU trap probe during VM_Version initialization.
+ // If IC IVAU is not trapped, it faults on unmapped VA 0x0.
+ if (sig == SIGSEGV && pc == (address)_ic_ivau_probe_fault) {
+ stub = (address)_ic_ivau_probe_continuation;
+ }
+
if (thread->thread_state() == _thread_in_Java) {
// Java thread running in Java code => find exception handler if any
// a fault inside compiled code, the interpreter, or a stub
diff --git a/src/hotspot/os_cpu/linux_aarch64/vm_version_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/vm_version_linux_aarch64.cpp
index 1fe06dc640d..ee2d3013c4c 100644
--- a/src/hotspot/os_cpu/linux_aarch64/vm_version_linux_aarch64.cpp
+++ b/src/hotspot/os_cpu/linux_aarch64/vm_version_linux_aarch64.cpp
@@ -31,6 +31,10 @@
#include
#include
+// IC IVAU trap probe.
+// Defined in ic_ivau_probe_linux_aarch64.S.
+extern "C" int ic_ivau_probe(void);
+
#ifndef HWCAP_AES
#define HWCAP_AES (1<<3)
#endif
@@ -95,6 +99,13 @@
#define HWCAP2_SVEBITPERM (1 << 4)
#endif
+#ifndef HWCAP2_ECV
+#define HWCAP2_ECV (1 << 19)
+#endif
+
+#ifndef HWCAP2_WFXT
+#define HWCAP2_WFXT (1u << 31)
+#endif
#ifndef PR_SVE_GET_VL
// For old toolchains which do not have SVE related macros defined.
#define PR_SVE_SET_VL 50
@@ -158,6 +169,12 @@ void VM_Version::get_os_cpu_info() {
if (auxv2 & HWCAP2_SVEBITPERM) {
set_feature(CPU_SVEBITPERM);
}
+ if (auxv2 & HWCAP2_ECV) {
+ set_feature(CPU_ECV);
+ }
+ if (auxv2 & HWCAP2_WFXT) {
+ set_feature(CPU_WFXT);
+ }
uint64_t ctr_el0;
uint64_t dczid_el0;
@@ -169,6 +186,12 @@ void VM_Version::get_os_cpu_info() {
_icache_line_size = (1 << (ctr_el0 & 0x0f)) * 4;
_dcache_line_size = (1 << ((ctr_el0 >> 16) & 0x0f)) * 4;
+ _cache_idc_enabled = ((ctr_el0 >> 28) & 0x1) != 0;
+ _cache_dic_enabled = ((ctr_el0 >> 29) & 0x1) != 0;
+
+ // Probe whether IC IVAU is trapped.
+ // Must run before VM_Version::initialize() sets NeoverseN1ICacheErratumMitigation.
+ _ic_ivau_trapped = (ic_ivau_probe() == 1);
if (!(dczid_el0 & 0x10)) {
_zva_length = 4 << (dczid_el0 & 0xf);
diff --git a/src/hotspot/os_cpu/linux_arm/atomicAccess_linux_arm.hpp b/src/hotspot/os_cpu/linux_arm/atomicAccess_linux_arm.hpp
index 390207f9e5e..c03f5ed1c8b 100644
--- a/src/hotspot/os_cpu/linux_arm/atomicAccess_linux_arm.hpp
+++ b/src/hotspot/os_cpu/linux_arm/atomicAccess_linux_arm.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2008, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -26,8 +26,6 @@
#define OS_CPU_LINUX_ARM_ATOMICACCESS_LINUX_ARM_HPP
#include "memory/allStatic.hpp"
-#include "runtime/os.hpp"
-#include "runtime/vm_version.hpp"
// Implementation of class AtomicAccess
diff --git a/src/hotspot/os_cpu/linux_arm/orderAccess_linux_arm.hpp b/src/hotspot/os_cpu/linux_arm/orderAccess_linux_arm.hpp
index 3bb357704fb..49c6942b8e0 100644
--- a/src/hotspot/os_cpu/linux_arm/orderAccess_linux_arm.hpp
+++ b/src/hotspot/os_cpu/linux_arm/orderAccess_linux_arm.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2008, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -27,7 +27,6 @@
// Included in orderAccess.hpp header file.
-#include "runtime/os.hpp"
#include "runtime/vm_version.hpp"
// Implementation of class OrderAccess.
diff --git a/src/hotspot/os_cpu/linux_ppc/vm_version_linux_ppc.cpp b/src/hotspot/os_cpu/linux_ppc/vm_version_linux_ppc.cpp
new file mode 100644
index 00000000000..d64340edf5c
--- /dev/null
+++ b/src/hotspot/os_cpu/linux_ppc/vm_version_linux_ppc.cpp
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2026 SAP SE. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "runtime/vm_version.hpp"
+
+#include
+
+int VM_Version::get_dcache_line_size() {
+ // This should work on all modern linux versions:
+ int size = sysconf(_SC_LEVEL1_DCACHE_LINESIZE);
+ // It may fail with very old linux / glibc versions. We use DEFAULT_CACHE_LINE_SIZE in this case.
+ // That is the correct value for all currently supported processors.
+ return (size <= 0) ? DEFAULT_CACHE_LINE_SIZE : size;
+}
+
+int VM_Version::get_icache_line_size() {
+ // This should work on all modern linux versions:
+ int size = sysconf(_SC_LEVEL1_ICACHE_LINESIZE);
+ // It may fail with very old linux / glibc versions. We use DEFAULT_CACHE_LINE_SIZE in this case.
+ // That is the correct value for all currently supported processors.
+ return (size <= 0) ? DEFAULT_CACHE_LINE_SIZE : size;
+}
diff --git a/src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp b/src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp
index 35cbb75e8ff..648131b94a3 100644
--- a/src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp
+++ b/src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp
@@ -36,40 +36,42 @@
#include
#include
+static constexpr uint64_t feature_bit(int n) { return nth_bit(n); }
+
#ifndef HWCAP_ISA_I
-#define HWCAP_ISA_I nth_bit('I' - 'A')
+#define HWCAP_ISA_I feature_bit('I' - 'A')
#endif
#ifndef HWCAP_ISA_M
-#define HWCAP_ISA_M nth_bit('M' - 'A')
+#define HWCAP_ISA_M feature_bit('M' - 'A')
#endif
#ifndef HWCAP_ISA_A
-#define HWCAP_ISA_A nth_bit('A' - 'A')
+#define HWCAP_ISA_A feature_bit('A' - 'A')
#endif
#ifndef HWCAP_ISA_F
-#define HWCAP_ISA_F nth_bit('F' - 'A')
+#define HWCAP_ISA_F feature_bit('F' - 'A')
#endif
#ifndef HWCAP_ISA_D
-#define HWCAP_ISA_D nth_bit('D' - 'A')
+#define HWCAP_ISA_D feature_bit('D' - 'A')
#endif
#ifndef HWCAP_ISA_C
-#define HWCAP_ISA_C nth_bit('C' - 'A')
+#define HWCAP_ISA_C feature_bit('C' - 'A')
#endif
#ifndef HWCAP_ISA_Q
-#define HWCAP_ISA_Q nth_bit('Q' - 'A')
+#define HWCAP_ISA_Q feature_bit('Q' - 'A')
#endif
#ifndef HWCAP_ISA_H
-#define HWCAP_ISA_H nth_bit('H' - 'A')
+#define HWCAP_ISA_H feature_bit('H' - 'A')
#endif
#ifndef HWCAP_ISA_V
-#define HWCAP_ISA_V nth_bit('V' - 'A')
+#define HWCAP_ISA_V feature_bit('V' - 'A')
#endif
#define read_csr(csr) \
diff --git a/src/hotspot/os_cpu/linux_s390/atomicAccess_linux_s390.hpp b/src/hotspot/os_cpu/linux_s390/atomicAccess_linux_s390.hpp
index f3c1e8f1a2c..492ccf73bdf 100644
--- a/src/hotspot/os_cpu/linux_s390/atomicAccess_linux_s390.hpp
+++ b/src/hotspot/os_cpu/linux_s390/atomicAccess_linux_s390.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2019 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -26,10 +26,6 @@
#ifndef OS_CPU_LINUX_S390_ATOMICACCESS_LINUX_S390_HPP
#define OS_CPU_LINUX_S390_ATOMICACCESS_LINUX_S390_HPP
-#include "runtime/atomicAccess.hpp"
-#include "runtime/os.hpp"
-#include "runtime/vm_version.hpp"
-
// Note that the compare-and-swap instructions on System z perform
// a serialization function before the storage operand is fetched
// and again after the operation is completed.
diff --git a/src/hotspot/os_cpu/windows_aarch64/atomicAccess_windows_aarch64.hpp b/src/hotspot/os_cpu/windows_aarch64/atomicAccess_windows_aarch64.hpp
index f8119654c50..9238043f7a4 100644
--- a/src/hotspot/os_cpu/windows_aarch64/atomicAccess_windows_aarch64.hpp
+++ b/src/hotspot/os_cpu/windows_aarch64/atomicAccess_windows_aarch64.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, Microsoft Corporation. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -27,9 +27,7 @@
#define OS_CPU_WINDOWS_AARCH64_ATOMICACCESS_WINDOWS_AARCH64_HPP
#include
-#include "runtime/os.hpp"
-#include "runtime/vm_version.hpp"
-
+#include
// As per atomicAccess.hpp all read-modify-write operations have to provide two-way
// barriers semantics. The memory_order parameter is ignored - we always provide
diff --git a/src/hotspot/os_cpu/windows_aarch64/prefetch_windows_aarch64.inline.hpp b/src/hotspot/os_cpu/windows_aarch64/prefetch_windows_aarch64.inline.hpp
index a360ee342be..a2c8f0c685c 100644
--- a/src/hotspot/os_cpu/windows_aarch64/prefetch_windows_aarch64.inline.hpp
+++ b/src/hotspot/os_cpu/windows_aarch64/prefetch_windows_aarch64.inline.hpp
@@ -27,10 +27,24 @@
// Included in runtime/prefetch.inline.hpp
+#include
+
+// __prefetch2(addr, prfop) emits a PRFM instruction.
+// The prfop encoding is:
+// type: PLD = 00, PLI = 01, PST = 10
+// target: L1 = 00, L2 = 01, L3 = 10
+// policy: KEEP = 0, STRM = 1
+
inline void Prefetch::read (const void *loc, intx interval) {
+ if (interval >= 0) {
+ __prefetch2((const char*) loc + interval, /* PLD + L1 + KEEP */ 0);
+ }
}
inline void Prefetch::write(void *loc, intx interval) {
+ if (interval >= 0) {
+ __prefetch2((char*) loc + interval, /* PST + L1 + KEEP */ 16);
+ }
}
#endif // OS_CPU_WINDOWS_AARCH64_PREFETCH_WINDOWS_AARCH64_INLINE_HPP
diff --git a/src/hotspot/os_cpu/windows_aarch64/sve_windows_aarch64.S b/src/hotspot/os_cpu/windows_aarch64/sve_windows_aarch64.S
new file mode 100644
index 00000000000..e0c85830bd4
--- /dev/null
+++ b/src/hotspot/os_cpu/windows_aarch64/sve_windows_aarch64.S
@@ -0,0 +1,42 @@
+;
+; Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
+; DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+;
+; This code is free software; you can redistribute it and/or modify it
+; under the terms of the GNU General Public License version 2 only, as
+; published by the Free Software Foundation.
+;
+; This code is distributed in the hope that it will be useful, but WITHOUT
+; ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+; FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+; version 2 for more details (a copy is included in the LICENSE file that
+; accompanied this code).
+;
+; You should have received a copy of the GNU General Public License version
+; 2 along with this work; if not, write to the Free Software Foundation,
+; Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+;
+; Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+; or visit www.oracle.com if you need additional information or have any
+; questions.
+;
+
+ ; Support for int get_sve_vector_length();
+ ;
+ ; Returns the current SVE vector length in bytes.
+ ; This function uses the INCB instruction which increments a register
+ ; by the number of bytes in an SVE vector register.
+ ;
+ ; Note: This function will fault if SVE is not available or enabled.
+ ; The caller must ensure SVE support is detected before calling.
+
+ ALIGN 4
+ EXPORT get_sve_vector_length
+ AREA sve_text, CODE
+
+get_sve_vector_length
+ mov x0, #0
+ incb x0
+ ret
+
+ END
diff --git a/src/hotspot/os_cpu/windows_aarch64/vm_version_windows_aarch64.cpp b/src/hotspot/os_cpu/windows_aarch64/vm_version_windows_aarch64.cpp
index 93beb549366..e78a37b4178 100644
--- a/src/hotspot/os_cpu/windows_aarch64/vm_version_windows_aarch64.cpp
+++ b/src/hotspot/os_cpu/windows_aarch64/vm_version_windows_aarch64.cpp
@@ -26,16 +26,19 @@
#include "runtime/os.hpp"
#include "runtime/vm_version.hpp"
+// Assembly function to get SVE vector length using INCB instruction
+extern "C" int get_sve_vector_length();
+
int VM_Version::get_current_sve_vector_length() {
assert(VM_Version::supports_sve(), "should not call this");
- ShouldNotReachHere();
- return 0;
+ // Use assembly instruction to get the actual SVE vector length
+ return VM_Version::supports_sve() ? get_sve_vector_length() : 0; // This value is in bytes
}
int VM_Version::set_and_get_current_sve_vector_length(int length) {
assert(VM_Version::supports_sve(), "should not call this");
- ShouldNotReachHere();
- return 0;
+ // Use assembly instruction to get the SVE vector length
+ return VM_Version::supports_sve() ? get_sve_vector_length() : 0; // This value is in bytes
}
void VM_Version::get_os_cpu_info() {
@@ -47,11 +50,29 @@ void VM_Version::get_os_cpu_info() {
set_feature(CPU_AES);
set_feature(CPU_SHA1);
set_feature(CPU_SHA2);
+ set_feature(CPU_PMULL);
}
if (IsProcessorFeaturePresent(PF_ARM_VFP_32_REGISTERS_AVAILABLE)) {
set_feature(CPU_ASIMD);
}
- // No check for CPU_PMULL, CPU_SVE, CPU_SVE2
+ if (IsProcessorFeaturePresent(PF_ARM_V81_ATOMIC_INSTRUCTIONS_AVAILABLE)) {
+ set_feature(CPU_LSE);
+ }
+ if (IsProcessorFeaturePresent(PF_ARM_SVE_INSTRUCTIONS_AVAILABLE)) {
+ set_feature(CPU_SVE);
+ }
+ if (IsProcessorFeaturePresent(PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE)) {
+ set_feature(CPU_SVE2);
+ }
+ if (IsProcessorFeaturePresent(PF_ARM_SVE_BITPERM_INSTRUCTIONS_AVAILABLE)) {
+ set_feature(CPU_SVEBITPERM);
+ }
+ if (IsProcessorFeaturePresent(PF_ARM_SHA3_INSTRUCTIONS_AVAILABLE)) {
+ set_feature(CPU_SHA3);
+ }
+ if (IsProcessorFeaturePresent(PF_ARM_SHA512_INSTRUCTIONS_AVAILABLE)) {
+ set_feature(CPU_SHA512);
+ }
__int64 dczid_el0 = _ReadStatusReg(0x5807 /* ARM64_DCZID_EL0 */);
@@ -102,8 +123,8 @@ void VM_Version::get_os_cpu_info() {
SYSTEM_INFO si;
GetSystemInfo(&si);
_model = si.wProcessorLevel;
- _variant = si.wProcessorRevision / 0xFF;
- _revision = si.wProcessorRevision & 0xFF;
+ _variant = (si.wProcessorRevision >> 8) & 0xFF; // Variant is the upper byte of wProcessorRevision
+ _revision = si.wProcessorRevision & 0xFF; // Revision is the lower byte of wProcessorRevision
}
}
}
diff --git a/src/hotspot/os_cpu/windows_x86/atomicAccess_windows_x86.hpp b/src/hotspot/os_cpu/windows_x86/atomicAccess_windows_x86.hpp
index aa78a401235..252411f62bc 100644
--- a/src/hotspot/os_cpu/windows_x86/atomicAccess_windows_x86.hpp
+++ b/src/hotspot/os_cpu/windows_x86/atomicAccess_windows_x86.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -26,7 +26,7 @@
#define OS_CPU_WINDOWS_X86_ATOMICACCESS_WINDOWS_X86_HPP
#include
-#include "runtime/os.hpp"
+#include
// Note that in MSVC, volatile memory accesses are explicitly
// guaranteed to have acquire release semantics (w.r.t. compiler
diff --git a/src/hotspot/os_cpu/windows_x86/prefetch_windows_x86.inline.hpp b/src/hotspot/os_cpu/windows_x86/prefetch_windows_x86.inline.hpp
index 645fbe99a22..575eabc97dd 100644
--- a/src/hotspot/os_cpu/windows_x86/prefetch_windows_x86.inline.hpp
+++ b/src/hotspot/os_cpu/windows_x86/prefetch_windows_x86.inline.hpp
@@ -27,7 +27,18 @@
// Included in runtime/prefetch.inline.hpp
-inline void Prefetch::read (const void *loc, intx interval) {}
-inline void Prefetch::write(void *loc, intx interval) {}
+#include
+
+inline void Prefetch::read (const void *loc, intx interval) {
+ if (interval >= 0) {
+ _mm_prefetch((const char*) loc + interval, _MM_HINT_T0);
+ }
+}
+
+inline void Prefetch::write(void *loc, intx interval) {
+ if (interval >= 0) {
+ _mm_prefetch((const char*) loc + interval, _MM_HINT_T0);
+ }
+}
#endif // OS_CPU_WINDOWS_X86_PREFETCH_WINDOWS_X86_INLINE_HPP
diff --git a/src/hotspot/share/adlc/adlparse.cpp b/src/hotspot/share/adlc/adlparse.cpp
index 356c24760e8..b49efa34be8 100644
--- a/src/hotspot/share/adlc/adlparse.cpp
+++ b/src/hotspot/share/adlc/adlparse.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -993,9 +993,6 @@ void ADLParser::frame_parse(void) {
if (strcmp(token,"frame_pointer")==0) {
frame_pointer_parse(frame, false);
}
- if (strcmp(token,"interpreter_frame_pointer")==0) {
- interpreter_frame_pointer_parse(frame, false);
- }
if (strcmp(token,"inline_cache_reg")==0) {
inline_cache_parse(frame, false);
}
@@ -1119,11 +1116,6 @@ void ADLParser::frame_pointer_parse(FrameForm *frame, bool native) {
else { frame->_frame_pointer = frame_pointer; }
}
-//------------------------------interpreter_frame_pointer_parse----------------------------
-void ADLParser::interpreter_frame_pointer_parse(FrameForm *frame, bool native) {
- frame->_interpreter_frame_pointer_reg = parse_one_arg("interpreter frame pointer entry");
-}
-
//------------------------------inline_cache_parse-----------------------------
void ADLParser::inline_cache_parse(FrameForm *frame, bool native) {
frame->_inline_cache_reg = parse_one_arg("inline cache reg entry");
diff --git a/src/hotspot/share/adlc/adlparse.hpp b/src/hotspot/share/adlc/adlparse.hpp
index 02baec53262..89296193612 100644
--- a/src/hotspot/share/adlc/adlparse.hpp
+++ b/src/hotspot/share/adlc/adlparse.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -120,7 +120,6 @@ protected:
// Parse the components of the frame section
void sync_stack_slots_parse(FrameForm *frame);
void frame_pointer_parse(FrameForm *frame, bool native);
- void interpreter_frame_pointer_parse(FrameForm *frame, bool native);
void inline_cache_parse(FrameForm *frame, bool native);
void interpreter_arg_ptr_parse(FrameForm *frame, bool native);
void interpreter_method_parse(FrameForm *frame, bool native);
diff --git a/src/hotspot/share/adlc/formsopt.cpp b/src/hotspot/share/adlc/formsopt.cpp
index fbd1043492e..091e34f40f4 100644
--- a/src/hotspot/share/adlc/formsopt.cpp
+++ b/src/hotspot/share/adlc/formsopt.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -476,7 +476,6 @@ void AllocClass::forms_do(FormClosure* f) {
FrameForm::FrameForm() {
_sync_stack_slots = nullptr;
_inline_cache_reg = nullptr;
- _interpreter_frame_pointer_reg = nullptr;
_cisc_spilling_operand_name = nullptr;
_frame_pointer = nullptr;
_c_frame_pointer = nullptr;
diff --git a/src/hotspot/share/adlc/formsopt.hpp b/src/hotspot/share/adlc/formsopt.hpp
index 9e0c9db854d..087ab1e2653 100644
--- a/src/hotspot/share/adlc/formsopt.hpp
+++ b/src/hotspot/share/adlc/formsopt.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -347,7 +347,6 @@ public:
// Public Data
char *_sync_stack_slots;
char *_inline_cache_reg;
- char *_interpreter_frame_pointer_reg;
char *_cisc_spilling_operand_name;
char *_frame_pointer;
char *_c_frame_pointer;
diff --git a/src/hotspot/share/adlc/formssel.cpp b/src/hotspot/share/adlc/formssel.cpp
index 182587d2f2f..5802217c1c1 100644
--- a/src/hotspot/share/adlc/formssel.cpp
+++ b/src/hotspot/share/adlc/formssel.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -4233,11 +4233,13 @@ int MatchRule::is_expensive() const {
strcmp(opType,"PopulateIndex")==0 ||
strcmp(opType,"AddReductionVI")==0 ||
strcmp(opType,"AddReductionVL")==0 ||
+ strcmp(opType,"AddReductionVHF")==0 ||
strcmp(opType,"AddReductionVF")==0 ||
strcmp(opType,"AddReductionVD")==0 ||
strcmp(opType,"MulReductionVI")==0 ||
strcmp(opType,"MulReductionVL")==0 ||
strcmp(opType,"MulReductionVF")==0 ||
+ strcmp(opType,"MulReductionVHF")==0 ||
strcmp(opType,"MulReductionVD")==0 ||
strcmp(opType,"MinReductionV")==0 ||
strcmp(opType,"MaxReductionV")==0 ||
@@ -4276,7 +4278,9 @@ bool MatchRule::is_ideal_membar() const {
!strcmp(_opType,"LoadFence" ) ||
!strcmp(_opType,"StoreFence") ||
!strcmp(_opType,"StoreStoreFence") ||
+ !strcmp(_opType,"MemBarStoreLoad") ||
!strcmp(_opType,"MemBarVolatile") ||
+ !strcmp(_opType,"MemBarFull") ||
!strcmp(_opType,"MemBarCPUOrder") ||
!strcmp(_opType,"MemBarStoreStore") ||
!strcmp(_opType,"OnSpinWait");
@@ -4346,9 +4350,9 @@ bool MatchRule::is_vector() const {
"MaxV", "MinV", "MinVHF", "MaxVHF", "UMinV", "UMaxV",
"CompressV", "ExpandV", "CompressM", "CompressBitsV", "ExpandBitsV",
"AddReductionVI", "AddReductionVL",
- "AddReductionVF", "AddReductionVD",
+ "AddReductionVHF", "AddReductionVF", "AddReductionVD",
"MulReductionVI", "MulReductionVL",
- "MulReductionVF", "MulReductionVD",
+ "MulReductionVHF", "MulReductionVF", "MulReductionVD",
"MaxReductionV", "MinReductionV",
"AndReductionV", "OrReductionV", "XorReductionV",
"MulAddVS2VI", "MacroLogicV",
diff --git a/src/hotspot/share/adlc/output_c.cpp b/src/hotspot/share/adlc/output_c.cpp
index 9cbd6aaf66f..45b3d6bda63 100644
--- a/src/hotspot/share/adlc/output_c.cpp
+++ b/src/hotspot/share/adlc/output_c.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -4212,14 +4212,6 @@ void ArchDesc::buildFrameMethods(FILE *fp_cpp) {
fprintf(fp_cpp,"int Matcher::inline_cache_reg_encode() {");
fprintf(fp_cpp," return _regEncode[inline_cache_reg()]; }\n\n");
- // Interpreter's Frame Pointer Register
- fprintf(fp_cpp,"OptoReg::Name Matcher::interpreter_frame_pointer_reg() {");
- if (_frame->_interpreter_frame_pointer_reg == nullptr)
- fprintf(fp_cpp," return OptoReg::Bad; }\n\n");
- else
- fprintf(fp_cpp," return OptoReg::Name(%s_num); }\n\n",
- _frame->_interpreter_frame_pointer_reg);
-
// Frame Pointer definition
/* CNC - I can not contemplate having a different frame pointer between
Java and native code; makes my head hurt to think about it.
diff --git a/src/hotspot/share/asm/codeBuffer.cpp b/src/hotspot/share/asm/codeBuffer.cpp
index ba525588f32..854cf73049b 100644
--- a/src/hotspot/share/asm/codeBuffer.cpp
+++ b/src/hotspot/share/asm/codeBuffer.cpp
@@ -32,7 +32,6 @@
#include "oops/methodCounters.hpp"
#include "oops/methodData.hpp"
#include "oops/oop.inline.hpp"
-#include "runtime/icache.hpp"
#include "runtime/safepointVerifiers.hpp"
#include "utilities/align.hpp"
#include "utilities/copy.hpp"
@@ -745,9 +744,6 @@ void CodeBuffer::copy_code_to(CodeBlob* dest_blob) {
// Done moving code bytes; were they the right size?
assert((int)align_up(dest.total_content_size(), oopSize) == dest_blob->content_size(), "sanity");
-
- // Flush generated code
- ICache::invalidate_range(dest_blob->code_begin(), dest_blob->code_size());
}
// Move all my code into another code buffer. Consult applicable
@@ -862,6 +858,13 @@ csize_t CodeBuffer::figure_expanded_capacities(CodeSection* which_cs,
}
void CodeBuffer::expand(CodeSection* which_cs, csize_t amount) {
+#ifdef ASSERT
+ // The code below copies contents across temp buffers. The following
+ // sizes relate to buffer contents, and should not be changed by buffer
+ // expansion.
+ int old_total_skipped = total_skipped_instructions_size();
+#endif
+
#ifndef PRODUCT
if (PrintNMethods && (WizardMode || Verbose)) {
tty->print("expanding CodeBuffer:");
@@ -920,6 +923,7 @@ void CodeBuffer::expand(CodeSection* which_cs, csize_t amount) {
assert(cb_sect->capacity() >= new_capacity[n], "big enough");
address cb_start = cb_sect->start();
cb_sect->set_end(cb_start + this_sect->size());
+ cb_sect->register_skipped(this_sect->_skipped_instructions_size);
if (this_sect->mark() == nullptr) {
cb_sect->clear_mark();
} else {
@@ -956,6 +960,9 @@ void CodeBuffer::expand(CodeSection* which_cs, csize_t amount) {
this->print_on(tty);
}
#endif //PRODUCT
+
+ assert(old_total_skipped == total_skipped_instructions_size(),
+ "Should match: %d == %d", old_total_skipped, total_skipped_instructions_size());
}
void CodeBuffer::adjust_internal_address(address from, address to) {
@@ -1140,7 +1147,7 @@ void AsmRemarks::clear() {
uint AsmRemarks::print(uint offset, outputStream* strm) const {
uint count = 0;
const char* prefix = " ;; ";
- const char* remstr = _remarks->lookup(offset);
+ const char* remstr = (_remarks ? _remarks->lookup(offset) : nullptr);
while (remstr != nullptr) {
strm->bol();
strm->print("%s", prefix);
diff --git a/src/hotspot/share/c1/c1_Runtime1.cpp b/src/hotspot/share/c1/c1_Runtime1.cpp
index 63764dd113a..38f563935e0 100644
--- a/src/hotspot/share/c1/c1_Runtime1.cpp
+++ b/src/hotspot/share/c1/c1_Runtime1.cpp
@@ -278,11 +278,9 @@ bool Runtime1::initialize(BufferBlob* blob) {
if (!generate_blob_for(blob, id)) {
return false;
}
- if (id == StubId::c1_forward_exception_id) {
- // publish early c1 stubs at this point so later stubs can refer to them
- AOTCodeCache::init_early_c1_table();
- }
}
+ // disallow any further c1 stub generation
+ AOTCodeCache::set_c1_stubs_complete();
// printing
#ifndef PRODUCT
if (PrintSimpleStubs) {
diff --git a/src/hotspot/share/cds/aotArtifactFinder.cpp b/src/hotspot/share/cds/aotArtifactFinder.cpp
index f85f1e46520..bd69b18a1aa 100644
--- a/src/hotspot/share/cds/aotArtifactFinder.cpp
+++ b/src/hotspot/share/cds/aotArtifactFinder.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -32,6 +32,7 @@
#include "cds/lambdaProxyClassDictionary.hpp"
#include "cds/regeneratedClasses.hpp"
#include "classfile/systemDictionaryShared.hpp"
+#include "classfile/vmClasses.hpp"
#include "logging/log.hpp"
#include "memory/metaspaceClosure.hpp"
#include "oops/instanceKlass.hpp"
@@ -169,6 +170,7 @@ void AOTArtifactFinder::find_artifacts() {
end_scanning_for_oops();
TrainingData::cleanup_training_data();
+ check_critical_classes();
}
void AOTArtifactFinder::start_scanning_for_oops() {
@@ -233,6 +235,7 @@ void AOTArtifactFinder::add_cached_instance_class(InstanceKlass* ik) {
bool created;
_seen_classes->put_if_absent(ik, &created);
if (created) {
+ check_critical_class(ik);
append_to_all_cached_classes(ik);
// All super types must be added.
@@ -310,3 +313,25 @@ void AOTArtifactFinder::all_cached_classes_do(MetaspaceClosure* it) {
it->push(_all_cached_classes->adr_at(i));
}
}
+
+void AOTArtifactFinder::check_critical_classes() {
+ if (CDSConfig::is_dumping_static_archive()) {
+ // vmClasses are store in the AOT cache (or AOT config file, or static archive).
+ // If any of the vmClasses is excluded, (usually due to incompatible JVMTI agent),
+ // the resulting cache/config/archive is unusable.
+ for (auto id : EnumRange{}) {
+ check_critical_class(vmClasses::klass_at(id));
+ }
+ }
+}
+
+void AOTArtifactFinder::check_critical_class(InstanceKlass* ik) {
+ if (SystemDictionaryShared::is_excluded_class(ik)) {
+ ResourceMark rm;
+ const char* msg = err_msg("Critical class %s has been excluded. %s cannot be written.",
+ ik->external_name(),
+ CDSConfig::type_of_archive_being_written());
+ AOTMetaspace::unrecoverable_writing_error(msg);
+ }
+}
+
diff --git a/src/hotspot/share/cds/aotArtifactFinder.hpp b/src/hotspot/share/cds/aotArtifactFinder.hpp
index 05bcde6b0ac..50057b6caee 100644
--- a/src/hotspot/share/cds/aotArtifactFinder.hpp
+++ b/src/hotspot/share/cds/aotArtifactFinder.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -81,12 +81,14 @@ class AOTArtifactFinder : AllStatic {
static void add_cached_type_array_class(TypeArrayKlass* tak);
static void add_cached_instance_class(InstanceKlass* ik);
static void append_to_all_cached_classes(Klass* k);
+ static void check_critical_class(InstanceKlass* ik);
public:
static void initialize();
static void find_artifacts();
static void add_cached_class(Klass* k);
static void add_aot_inited_class(InstanceKlass* ik);
static void all_cached_classes_do(MetaspaceClosure* it);
+ static void check_critical_classes();
static void dispose();
};
diff --git a/src/hotspot/share/cds/aotClassInitializer.cpp b/src/hotspot/share/cds/aotClassInitializer.cpp
index 06fc3af6f30..9ef96282aeb 100644
--- a/src/hotspot/share/cds/aotClassInitializer.cpp
+++ b/src/hotspot/share/cds/aotClassInitializer.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -59,6 +59,39 @@ bool AOTClassInitializer::can_archive_initialized_mirror(InstanceKlass* ik) {
return false;
}
+#ifdef ASSERT
+ // If code in ik is executed, then ik must be in the state of being_initialized or
+ // fully_initialized.
+ //
+ // Check that no user code is executed during the assembly phase. Otherwise the user
+ // code may introduce undesirable environment dependencies into the heap image.
+ // If any of these two flags are set, we allow user code to be executed
+ // in the assembly phase. Note that these flags are strictly for the purpose
+ // of testing HotSpot and are not available in product builds.
+ if (AOTInitTestClass == nullptr && ArchiveHeapTestClass == nullptr) {
+ if (ik->defined_by_boot_loader()) {
+ // We allow boot classes to be AOT-initialized, except for classes from
+ // -Xbootclasspath (cp index >= 1) be AOT-initialized, as such classes may be
+ // provided by the user application.
+ assert(ik->shared_classpath_index() <= 0,
+ "only boot classed loaded from the modules image can be AOT-initialized");
+ } else {
+ assert(ik->defined_by_platform_loader() || ik->defined_by_app_loader(),
+ "cannot AOT-initialized classed loaded by other loaders");
+
+ // Hidden classes from platform/app loaders need to be AOT-initialized to
+ // support AOT-linking of lambdas. These hidden classes are generated by the
+ // VM and do not contain user code.
+ if (!ik->is_hidden()) {
+ // OK: ik is an interface used by a lambda. When AOT-linking lambdas, we only
+ // support interfaces that are not interface_needs_clinit_execution_as_super().
+ // See AOTConstantPoolResolver::check_lambda_metafactory_signature().
+ assert(ik->is_interface() && !ik->interface_needs_clinit_execution_as_super(), "cannot execute Java code in assembly phase");
+ }
+ }
+ }
+#endif // ASSERT
+
// About "static field that may hold a different value" errors:
//
// Automatic selection for aot-inited classes
@@ -234,7 +267,8 @@ bool AOTClassInitializer::can_archive_initialized_mirror(InstanceKlass* ik) {
}
void AOTClassInitializer::call_runtime_setup(JavaThread* current, InstanceKlass* ik) {
- assert(ik->has_aot_initialized_mirror(), "sanity");
+ precond(ik->has_aot_initialized_mirror());
+ precond(!AOTLinkedClassBulkLoader::is_initializing_classes_early());
if (ik->is_runtime_setup_required()) {
if (log_is_enabled(Info, aot, init)) {
ResourceMark rm;
diff --git a/src/hotspot/share/cds/aotConstantPoolResolver.cpp b/src/hotspot/share/cds/aotConstantPoolResolver.cpp
index 93145940955..f1a704d4bee 100644
--- a/src/hotspot/share/cds/aotConstantPoolResolver.cpp
+++ b/src/hotspot/share/cds/aotConstantPoolResolver.cpp
@@ -81,6 +81,7 @@ bool AOTConstantPoolResolver::is_resolution_deterministic(ConstantPool* cp, int
bool AOTConstantPoolResolver::is_class_resolution_deterministic(InstanceKlass* cp_holder, Klass* resolved_class) {
assert(!is_in_archivebuilder_buffer(cp_holder), "sanity");
assert(!is_in_archivebuilder_buffer(resolved_class), "sanity");
+ assert_at_safepoint(); // try_add_candidate() is called below and requires to be at safepoint.
if (resolved_class->is_instance_klass()) {
InstanceKlass* ik = InstanceKlass::cast(resolved_class);
@@ -346,7 +347,15 @@ void AOTConstantPoolResolver::maybe_resolve_fmi_ref(InstanceKlass* ik, Method* m
break;
case Bytecodes::_invokehandle:
- InterpreterRuntime::cds_resolve_invokehandle(raw_index, cp, CHECK);
+ if (CDSConfig::is_dumping_method_handles()) {
+ ResolvedMethodEntry* method_entry = cp->resolved_method_entry_at(raw_index);
+ int cp_index = method_entry->constant_pool_index();
+ Symbol* sig = cp->uncached_signature_ref_at(cp_index);
+ Klass* k;
+ if (check_methodtype_signature(cp(), sig, &k, true)) {
+ InterpreterRuntime::cds_resolve_invokehandle(raw_index, cp, CHECK);
+ }
+ }
break;
default:
@@ -400,7 +409,7 @@ void AOTConstantPoolResolver::preresolve_indy_cp_entries(JavaThread* current, In
// Check the MethodType signatures used by parameters to the indy BSMs. Make sure we don't
// use types that have been excluded, or else we might end up creating MethodTypes that cannot be stored
// in the AOT cache.
-bool AOTConstantPoolResolver::check_methodtype_signature(ConstantPool* cp, Symbol* sig, Klass** return_type_ret) {
+bool AOTConstantPoolResolver::check_methodtype_signature(ConstantPool* cp, Symbol* sig, Klass** return_type_ret, bool is_invokehandle) {
ResourceMark rm;
for (SignatureStream ss(sig); !ss.is_done(); ss.next()) {
if (ss.is_reference()) {
@@ -413,11 +422,18 @@ bool AOTConstantPoolResolver::check_methodtype_signature(ConstantPool* cp, Symbo
if (SystemDictionaryShared::should_be_excluded(k)) {
if (log_is_enabled(Warning, aot, resolve)) {
ResourceMark rm;
- log_warning(aot, resolve)("Cannot aot-resolve Lambda proxy because %s is excluded", k->external_name());
+ log_warning(aot, resolve)("Cannot aot-resolve %s because %s is excluded",
+ is_invokehandle ? "invokehandle" : "Lambda proxy",
+ k->external_name());
}
return false;
}
+ // cp->pool_holder() must be able to resolve k in production run
+ precond(CDSConfig::is_dumping_aot_linked_classes());
+ precond(SystemDictionaryShared::is_builtin_loader(cp->pool_holder()->class_loader_data()));
+ precond(SystemDictionaryShared::is_builtin_loader(k->class_loader_data()));
+
if (ss.at_return_type() && return_type_ret != nullptr) {
*return_type_ret = k;
}
@@ -475,11 +491,44 @@ bool AOTConstantPoolResolver::check_lambda_metafactory_methodhandle_arg(Constant
return false;
}
+ // klass and sigature of the method (no need to check the method name)
Symbol* sig = cp->method_handle_signature_ref_at(mh_index);
+ Symbol* klass_name = cp->klass_name_at(cp->method_handle_klass_index_at(mh_index));
+
if (log_is_enabled(Debug, aot, resolve)) {
ResourceMark rm;
log_debug(aot, resolve)("Checking MethodType of MethodHandle for LambdaMetafactory BSM arg %d: %s", arg_i, sig->as_C_string());
}
+
+ {
+ Klass* k = find_loaded_class(Thread::current(), cp->pool_holder()->class_loader(), klass_name);
+ if (k == nullptr) {
+ // Dumping AOT cache: all classes should have been loaded by FinalImageRecipes::load_all_classes(). k must have
+ // been a class that was excluded when FinalImageRecipes recorded all classes at the end of the training run.
+ //
+ // Dumping static CDS archive: all classes in the classlist have already been loaded, before we resolve
+ // constants. k must have been a class that was excluded when the classlist was written
+ // at the end of the training run.
+ if (log_is_enabled(Warning, aot, resolve)) {
+ ResourceMark rm;
+ log_warning(aot, resolve)("Cannot aot-resolve Lambda proxy because %s is not loaded", klass_name->as_C_string());
+ }
+ return false;
+ }
+ if (SystemDictionaryShared::should_be_excluded(k)) {
+ if (log_is_enabled(Warning, aot, resolve)) {
+ ResourceMark rm;
+ log_warning(aot, resolve)("Cannot aot-resolve Lambda proxy because %s is excluded", k->external_name());
+ }
+ return false;
+ }
+
+ // cp->pool_holder() must be able to resolve k in production run
+ precond(CDSConfig::is_dumping_aot_linked_classes());
+ precond(SystemDictionaryShared::is_builtin_loader(cp->pool_holder()->class_loader_data()));
+ precond(SystemDictionaryShared::is_builtin_loader(k->class_loader_data()));
+ }
+
return check_methodtype_signature(cp, sig);
}
diff --git a/src/hotspot/share/cds/aotConstantPoolResolver.hpp b/src/hotspot/share/cds/aotConstantPoolResolver.hpp
index e49d9d1ad0b..ecf2ac27061 100644
--- a/src/hotspot/share/cds/aotConstantPoolResolver.hpp
+++ b/src/hotspot/share/cds/aotConstantPoolResolver.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2022, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -74,7 +74,10 @@ class AOTConstantPoolResolver : AllStatic {
static void maybe_resolve_fmi_ref(InstanceKlass* ik, Method* m, Bytecodes::Code bc, int raw_index,
GrowableArray* resolve_fmi_list, TRAPS);
- static bool check_methodtype_signature(ConstantPool* cp, Symbol* sig, Klass** return_type_ret = nullptr);
+public:
+ static bool check_methodtype_signature(ConstantPool* cp, Symbol* sig, Klass** return_type_ret = nullptr, bool is_invokehandle = false);
+
+private:
static bool check_lambda_metafactory_signature(ConstantPool* cp, Symbol* sig);
static bool check_lambda_metafactory_methodtype_arg(ConstantPool* cp, int bsms_attribute_index, int arg_i);
static bool check_lambda_metafactory_methodhandle_arg(ConstantPool* cp, int bsms_attribute_index, int arg_i);
diff --git a/src/hotspot/share/cds/aotGrowableArray.hpp b/src/hotspot/share/cds/aotGrowableArray.hpp
deleted file mode 100644
index 0a0c137ed07..00000000000
--- a/src/hotspot/share/cds/aotGrowableArray.hpp
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef SHARE_AOT_AOTGROWABLEARRAY_HPP
-#define SHARE_AOT_AOTGROWABLEARRAY_HPP
-
-#include
-#include
-
-class AOTGrowableArrayHelper {
-public:
- static void deallocate(void* mem);
-};
-
-// An AOTGrowableArray provides the same functionality as a GrowableArray that
-// uses the C heap allocator. In addition, AOTGrowableArray can be iterated with
-// MetaspaceClosure. This type should be used for growable arrays that need to be
-// stored in the AOT cache. See ModuleEntry::_reads for an example.
-template
-class AOTGrowableArray : public GrowableArrayWithAllocator> {
- friend class VMStructs;
- friend class GrowableArrayWithAllocator;
-
- static E* allocate(int max, MemTag mem_tag) {
- return (E*)GrowableArrayCHeapAllocator::allocate(max, sizeof(E), mem_tag);
- }
-
- E* allocate() {
- return allocate(this->_capacity, mtClass);
- }
-
- void deallocate(E* mem) {
-#if INCLUDE_CDS
- AOTGrowableArrayHelper::deallocate(mem);
-#else
- GrowableArrayCHeapAllocator::deallocate(mem);
-#endif
- }
-
-public:
- AOTGrowableArray(int initial_capacity, MemTag mem_tag) :
- GrowableArrayWithAllocator(
- allocate(initial_capacity, mem_tag),
- initial_capacity) {}
-
- AOTGrowableArray() : AOTGrowableArray(0, mtClassShared) {}
-
- // methods required by MetaspaceClosure
- void metaspace_pointers_do(MetaspaceClosure* it);
- int size_in_heapwords() const { return (int)heap_word_size(sizeof(*this)); }
- MetaspaceClosureType type() const { return MetaspaceClosureType::GrowableArrayType; }
- static bool is_read_only_by_default() { return false; }
-};
-
-#endif // SHARE_AOT_AOTGROWABLEARRAY_HPP
diff --git a/src/hotspot/share/cds/aotLinkedClassBulkLoader.cpp b/src/hotspot/share/cds/aotLinkedClassBulkLoader.cpp
index 3653f9d518c..8129e6a5a81 100644
--- a/src/hotspot/share/cds/aotLinkedClassBulkLoader.cpp
+++ b/src/hotspot/share/cds/aotLinkedClassBulkLoader.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -116,11 +116,24 @@ void AOTLinkedClassBulkLoader::preload_classes_in_table(Array* c
}
}
+#ifdef ASSERT
+// true iff we are inside AOTLinkedClassBulkLoader::link_classes(), when
+// we are moving classes into the fully_initialized state before the
+// JVM is able to execute any bytecodes.
+static bool _is_initializing_classes_early = false;
+bool AOTLinkedClassBulkLoader::is_initializing_classes_early() {
+ return _is_initializing_classes_early;
+}
+#endif
+
// Some cached heap objects may hold references to methods in aot-linked
// classes (via MemberName). We need to make sure all classes are
// linked before executing any bytecode.
void AOTLinkedClassBulkLoader::link_classes(JavaThread* current) {
+ DEBUG_ONLY(_is_initializing_classes_early = true);
link_classes_impl(current);
+ DEBUG_ONLY(_is_initializing_classes_early = false);
+
if (current->has_pending_exception()) {
exit_on_exception(current);
}
@@ -135,6 +148,13 @@ void AOTLinkedClassBulkLoader::link_classes_impl(TRAPS) {
link_classes_in_table(table->boot2(), CHECK);
link_classes_in_table(table->platform(), CHECK);
link_classes_in_table(table->app(), CHECK);
+
+ init_classes_for_loader(Handle(), AOTLinkedClassTable::get()->boot1(), /*early_only=*/true, CHECK);
+ init_classes_for_loader(Handle(), AOTLinkedClassTable::get()->boot2(), /*early_only=*/true, CHECK);
+ init_classes_for_loader(Handle(), AOTLinkedClassTable::get()->platform(), /*early_only=*/true, CHECK);
+ init_classes_for_loader(Handle(), AOTLinkedClassTable::get()->app(), /*early_only=*/true, CHECK);
+
+ log_info(aot, init)("------ finished early class init");
}
void AOTLinkedClassBulkLoader::link_classes_in_table(Array* classes, TRAPS) {
@@ -216,7 +236,7 @@ void AOTLinkedClassBulkLoader::validate_module(Klass* k, const char* category_na
#endif
void AOTLinkedClassBulkLoader::init_javabase_classes(JavaThread* current) {
- init_classes_for_loader(Handle(), AOTLinkedClassTable::get()->boot1(), current);
+ init_classes_for_loader(Handle(), AOTLinkedClassTable::get()->boot1(), /*early_only=*/false, current);
if (current->has_pending_exception()) {
exit_on_exception(current);
}
@@ -246,9 +266,9 @@ void AOTLinkedClassBulkLoader::init_non_javabase_classes_impl(TRAPS) {
assert(h_system_loader() != nullptr, "must be");
AOTLinkedClassTable* table = AOTLinkedClassTable::get();
- init_classes_for_loader(Handle(), table->boot2(), CHECK);
- init_classes_for_loader(h_platform_loader, table->platform(), CHECK);
- init_classes_for_loader(h_system_loader, table->app(), CHECK);
+ init_classes_for_loader(Handle(), table->boot2(), /*early_only=*/false, CHECK);
+ init_classes_for_loader(h_platform_loader, table->platform(), /*early_only=*/false, CHECK);
+ init_classes_for_loader(h_system_loader, table->app(), /*early_only=*/false, CHECK);
if (Universe::is_fully_initialized() && VerifyDuringStartup) {
// Make sure we're still in a clean state.
@@ -260,6 +280,10 @@ void AOTLinkedClassBulkLoader::init_non_javabase_classes_impl(TRAPS) {
tty->print_cr("==================== archived_training_data ** after all classes preloaded ====================");
TrainingData::print_archived_training_data_on(tty);
}
+ LogStreamHandle(Info, aot, training, data) log;
+ if (log.is_enabled()) {
+ TrainingData::print_archived_training_data_on(&log);
+ }
}
// For the AOT cache to function properly, all classes in the AOTLinkedClassTable
@@ -324,22 +348,80 @@ void AOTLinkedClassBulkLoader::initiate_loading(JavaThread* current, const char*
}
}
-// Some AOT-linked classes for must be initialized early. This includes
-// - classes that were AOT-initialized by AOTClassInitializer
-// - the classes of all objects that are reachable from the archived mirrors of
-// the AOT-linked classes for .
-void AOTLinkedClassBulkLoader::init_classes_for_loader(Handle class_loader, Array* classes, TRAPS) {
+// Can we move ik into fully_initialized state before the JVM is able to execute
+// bytecodes?
+static bool is_early_init_possible(InstanceKlass* ik) {
+ if (ik->is_runtime_setup_required()) {
+ // Bytecodes need to be executed in order to initialize this class.
+ if (log_is_enabled(Debug, aot, init)) {
+ ResourceMark rm;
+ log_debug(aot, init)("No early init %s: needs runtimeSetup()",
+ ik->external_name());
+ }
+ return false;
+ }
+
+ if (ik->super() != nullptr && !ik->super()->is_initialized()) {
+ // is_runtime_setup_required() == true for a super type
+ if (log_is_enabled(Debug, aot, init)) {
+ ResourceMark rm;
+ log_debug(aot, init)("No early init %s: super type %s not initialized",
+ ik->external_name(), ik->super()->external_name());
+ }
+ return false;
+ }
+
+ Array* interfaces = ik->local_interfaces();
+ int num_interfaces = interfaces->length();
+ for (int i = 0; i < num_interfaces; i++) {
+ InstanceKlass* intf = interfaces->at(i);
+ if (!intf->is_initialized() && intf->interface_needs_clinit_execution_as_super(/*also_check_supers*/false)) {
+ // is_runtime_setup_required() == true for this interface
+ if (log_is_enabled(Debug, aot, init)) {
+ ResourceMark rm;
+ log_debug(aot, init)("No early init %s: interface type %s not initialized",
+ ik->external_name(), intf->external_name());
+ }
+ return false;
+ }
+ }
+
+ return true;
+}
+
+// Normally, classes are initialized on demand. However, some AOT-linked classes
+// for the class_loader must be proactively intialized, including:
+// - Classes that have an AOT-initialized mirror (they were AOT-initialized by
+// AOTClassInitializer during the assembly phase).
+// - The classes of all objects that are reachable from the archived mirrors of
+// the AOT-linked classes for the class_loader. These are recorded in the special
+// subgraph.
+//
+// (early_only == true) means that this function is called before the JVM
+// is capable of executing Java bytecodes.
+void AOTLinkedClassBulkLoader::init_classes_for_loader(Handle class_loader, Array* classes,
+ bool early_only, TRAPS) {
if (classes != nullptr) {
for (int i = 0; i < classes->length(); i++) {
InstanceKlass* ik = classes->at(i);
assert(ik->class_loader_data() != nullptr, "must be");
- if (ik->has_aot_initialized_mirror()) {
- ik->initialize_with_aot_initialized_mirror(CHECK);
+
+ bool do_init = ik->has_aot_initialized_mirror();
+ if (do_init && early_only && !is_early_init_possible(ik)) {
+ // ik will be proactively initialized later when init_classes_for_loader()
+ // is called again with (early_only == false).
+ do_init = false;
+ }
+
+ if (do_init) {
+ ik->initialize_with_aot_initialized_mirror(early_only, CHECK);
}
}
}
- HeapShared::init_classes_for_special_subgraph(class_loader, CHECK);
+ if (!early_only) {
+ HeapShared::init_classes_for_special_subgraph(class_loader, CHECK);
+ }
}
void AOTLinkedClassBulkLoader::replay_training_at_init(Array* classes, TRAPS) {
diff --git a/src/hotspot/share/cds/aotLinkedClassBulkLoader.hpp b/src/hotspot/share/cds/aotLinkedClassBulkLoader.hpp
index 31fdac386fe..24ff61cea1e 100644
--- a/src/hotspot/share/cds/aotLinkedClassBulkLoader.hpp
+++ b/src/hotspot/share/cds/aotLinkedClassBulkLoader.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -56,7 +56,7 @@ class AOTLinkedClassBulkLoader : AllStatic {
static void link_classes_impl(TRAPS);
static void link_classes_in_table(Array* classes, TRAPS);
static void init_non_javabase_classes_impl(TRAPS);
- static void init_classes_for_loader(Handle class_loader, Array* classes, TRAPS);
+ static void init_classes_for_loader(Handle class_loader, Array* classes, bool early_only, TRAPS);
static void replay_training_at_init(Array