diff --git a/.github/actions/build-jtreg/action.yml b/.github/actions/build-jtreg/action.yml
index a9c046e9dd9..334812e8341 100644
--- a/.github/actions/build-jtreg/action.yml
+++ b/.github/actions/build-jtreg/action.yml
@@ -37,13 +37,13 @@ runs:
- name: 'Check cache for already built JTReg'
id: get-cached
- uses: actions/cache@v4
+ uses: actions/cache@v5
with:
path: jtreg/installed
key: jtreg-${{ steps.version.outputs.value }}
- name: 'Checkout the JTReg source'
- uses: actions/checkout@v4
+ uses: actions/checkout@v6
with:
repository: openjdk/jtreg
ref: jtreg-${{ steps.version.outputs.value }}
@@ -61,7 +61,7 @@ runs:
if: (steps.get-cached.outputs.cache-hit != 'true')
- name: 'Upload JTReg artifact'
- uses: actions/upload-artifact@v4
+ uses: actions/upload-artifact@v6
with:
name: bundles-jtreg-${{ steps.version.outputs.value }}
path: jtreg/installed
diff --git a/.github/actions/do-build/action.yml b/.github/actions/do-build/action.yml
index 6f2c2ce0218..6f6bbdabb68 100644
--- a/.github/actions/do-build/action.yml
+++ b/.github/actions/do-build/action.yml
@@ -66,7 +66,7 @@ runs:
shell: bash
- name: 'Upload build logs'
- uses: actions/upload-artifact@v4
+ uses: actions/upload-artifact@v6
with:
name: failure-logs-${{ inputs.platform }}${{ inputs.debug-suffix }}
path: failure-logs
@@ -74,7 +74,7 @@ runs:
# This is the best way I found to abort the job with an error message
- name: 'Notify about build failures'
- uses: actions/github-script@v7
+ uses: actions/github-script@v8
with:
script: core.setFailed('Build failed. See summary for details.')
if: steps.check.outputs.failure == 'true'
diff --git a/.github/actions/get-bootjdk/action.yml b/.github/actions/get-bootjdk/action.yml
index 312fb642c82..d531358b7dd 100644
--- a/.github/actions/get-bootjdk/action.yml
+++ b/.github/actions/get-bootjdk/action.yml
@@ -65,7 +65,7 @@ runs:
- name: 'Check cache for BootJDK'
id: get-cached-bootjdk
- uses: actions/cache@v4
+ uses: actions/cache@v5
with:
path: bootjdk/jdk
key: boot-jdk-${{ inputs.platform }}-${{ steps.sha256.outputs.value }}
diff --git a/.github/actions/get-bundles/action.yml b/.github/actions/get-bundles/action.yml
index a356aa9fd8d..55fa0e842d2 100644
--- a/.github/actions/get-bundles/action.yml
+++ b/.github/actions/get-bundles/action.yml
@@ -54,14 +54,14 @@ runs:
steps:
- name: 'Download bundles artifact'
id: download-bundles
- uses: actions/download-artifact@v4
+ uses: actions/download-artifact@v8
with:
name: bundles-${{ inputs.platform }}${{ inputs.debug-suffix }}
path: bundles
continue-on-error: true
- name: 'Download bundles artifact (retry)'
- uses: actions/download-artifact@v4
+ uses: actions/download-artifact@v8
with:
name: bundles-${{ inputs.platform }}${{ inputs.debug-suffix }}
path: bundles
@@ -69,7 +69,7 @@ runs:
- name: 'Download static bundles artifact'
id: download-static-bundles
- uses: actions/download-artifact@v4
+ uses: actions/download-artifact@v8
with:
name: bundles-${{ inputs.platform }}${{ inputs.debug-suffix }}${{ inputs.static-suffix }}
path: bundles
diff --git a/.github/actions/get-gtest/action.yml b/.github/actions/get-gtest/action.yml
index 7a329460a6e..bc53fa2a3b1 100644
--- a/.github/actions/get-gtest/action.yml
+++ b/.github/actions/get-gtest/action.yml
@@ -40,7 +40,7 @@ runs:
var: GTEST_VERSION
- name: 'Checkout GTest source'
- uses: actions/checkout@v4
+ uses: actions/checkout@v6
with:
repository: google/googletest
ref: 'v${{ steps.version.outputs.value }}'
diff --git a/.github/actions/get-jtreg/action.yml b/.github/actions/get-jtreg/action.yml
index 36c895fc59d..8c75ae10c7f 100644
--- a/.github/actions/get-jtreg/action.yml
+++ b/.github/actions/get-jtreg/action.yml
@@ -41,7 +41,7 @@ runs:
- name: 'Download JTReg artifact'
id: download-jtreg
- uses: actions/download-artifact@v4
+ uses: actions/download-artifact@v8
with:
name: bundles-jtreg-${{ steps.version.outputs.value }}
path: jtreg/installed
diff --git a/.github/actions/get-msys2/action.yml b/.github/actions/get-msys2/action.yml
index 308230ebf2e..7351a120ac4 100644
--- a/.github/actions/get-msys2/action.yml
+++ b/.github/actions/get-msys2/action.yml
@@ -31,7 +31,7 @@ runs:
steps:
- name: 'Install MSYS2'
id: msys2
- uses: msys2/setup-msys2@v2.28.0
+ uses: msys2/setup-msys2@v2.31.0
with:
install: 'autoconf tar unzip zip make'
path-type: minimal
diff --git a/.github/actions/upload-bundles/action.yml b/.github/actions/upload-bundles/action.yml
index 78fb0a94bfd..94308002ea7 100644
--- a/.github/actions/upload-bundles/action.yml
+++ b/.github/actions/upload-bundles/action.yml
@@ -87,7 +87,7 @@ runs:
shell: bash
- name: 'Upload bundles artifact'
- uses: actions/upload-artifact@v4
+ uses: actions/upload-artifact@v6
with:
name: bundles-${{ inputs.platform }}${{ inputs.debug-suffix }}${{ inputs.static-suffix }}${{ inputs.bundle-suffix }}
path: bundles
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
new file mode 100644
index 00000000000..d3f63784ece
--- /dev/null
+++ b/.github/pull_request_template.md
@@ -0,0 +1,5 @@
+
+
+
+---------
+- [ ] I confirm that I make this contribution in accordance with the [OpenJDK Interim AI Policy](https://openjdk.org/legal/ai).
diff --git a/.github/workflows/build-alpine-linux.yml b/.github/workflows/build-alpine-linux.yml
index c39962fa07f..6863da9016e 100644
--- a/.github/workflows/build-alpine-linux.yml
+++ b/.github/workflows/build-alpine-linux.yml
@@ -74,7 +74,7 @@ jobs:
steps:
- name: 'Checkout the JDK source'
- uses: actions/checkout@v4
+ uses: actions/checkout@v6
- name: 'Install toolchain and dependencies'
run: |
diff --git a/.github/workflows/build-cross-compile.yml b/.github/workflows/build-cross-compile.yml
index a0642d469aa..99b6c40606c 100644
--- a/.github/workflows/build-cross-compile.yml
+++ b/.github/workflows/build-cross-compile.yml
@@ -94,7 +94,7 @@ jobs:
steps:
- name: 'Checkout the JDK source'
- uses: actions/checkout@v4
+ uses: actions/checkout@v6
- name: 'Get the BootJDK'
id: bootjdk
@@ -122,7 +122,7 @@ jobs:
- name: 'Check cache for sysroot'
id: get-cached-sysroot
- uses: actions/cache@v4
+ uses: actions/cache@v5
with:
path: sysroot
key: sysroot-${{ matrix.debian-arch }}-${{ hashFiles('./.github/workflows/build-cross-compile.yml') }}
diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml
index 791b53a3f04..c501670439e 100644
--- a/.github/workflows/build-linux.yml
+++ b/.github/workflows/build-linux.yml
@@ -84,7 +84,7 @@ jobs:
steps:
- name: 'Checkout the JDK source'
- uses: actions/checkout@v4
+ uses: actions/checkout@v6
- name: 'Get the BootJDK'
id: bootjdk
diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml
index 484e616fad7..435576f4afd 100644
--- a/.github/workflows/build-macos.yml
+++ b/.github/workflows/build-macos.yml
@@ -75,7 +75,7 @@ jobs:
steps:
- name: 'Checkout the JDK source'
- uses: actions/checkout@v4
+ uses: actions/checkout@v6
- name: 'Get the BootJDK'
id: bootjdk
diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml
index 4dafc016a99..3bb50a137ec 100644
--- a/.github/workflows/build-windows.yml
+++ b/.github/workflows/build-windows.yml
@@ -83,7 +83,7 @@ jobs:
steps:
- name: 'Checkout the JDK source'
- uses: actions/checkout@v4
+ uses: actions/checkout@v6
- name: 'Get MSYS2'
uses: ./.github/actions/get-msys2
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 85ec75f343c..20be196b128 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -75,7 +75,7 @@ jobs:
steps:
- name: 'Checkout the scripts'
- uses: actions/checkout@v4
+ uses: actions/checkout@v6
with:
sparse-checkout: |
.github
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 8f33454305e..b240b42fb97 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -128,7 +128,7 @@ jobs:
steps:
- name: 'Checkout the JDK source'
- uses: actions/checkout@v4
+ uses: actions/checkout@v6
- name: 'Get MSYS2'
uses: ./.github/actions/get-msys2
@@ -239,7 +239,7 @@ jobs:
if: always()
- name: 'Upload test results'
- uses: actions/upload-artifact@v4
+ uses: actions/upload-artifact@v6
with:
path: results
name: ${{ steps.package.outputs.artifact-name }}
@@ -247,7 +247,7 @@ jobs:
# This is the best way I found to abort the job with an error message
- name: 'Notify about test failures'
- uses: actions/github-script@v7
+ uses: actions/github-script@v8
with:
script: core.setFailed('${{ steps.run-tests.outputs.error-message }}')
if: steps.run-tests.outputs.failure == 'true'
diff --git a/.gitignore b/.gitignore
index b6b4a1a559a..a45e2113756 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,8 +24,6 @@ NashornProfile.txt
/.gdbinit
/.lldbinit
**/core.[0-9]*
-*.rej
-*.orig
test/benchmarks/**/target
/src/hotspot/CMakeLists.txt
/src/hotspot/compile_commands.json
diff --git a/bin/idea.sh b/bin/idea.sh
index a184884b61a..d9a18956e3b 100644
--- a/bin/idea.sh
+++ b/bin/idea.sh
@@ -187,14 +187,18 @@ fi
SOURCE_PREFIX=""
+# SOURCES is a single string containing embeded newlines.
for root in $MODULE_ROOTS; do
if [ "x$CYGPATH" != "x" ]; then
root=`$CYGPATH -am $root`
elif [ "x$WSL_DISTRO_NAME" != "x" ]; then
root=`wslpath -am $root`
fi
-
- SOURCES=$SOURCES" $SOURCE_PREFIX""$root""$SOURCE_POSTFIX"
+ # Add line termination/indentation for everything after the first entry.
+ if [ "x$SOURCES" != "x" ]; then
+ SOURCES="${SOURCES}\n "
+ fi
+ SOURCES="${SOURCES}${SOURCE_PREFIX}${root}${SOURCE_POSTFIX}"
done
add_replacement "###SOURCE_ROOTS###" "$SOURCES"
diff --git a/doc/testing.html b/doc/testing.html
index 195153c8612..c8d0b928bb0 100644
--- a/doc/testing.html
+++ b/doc/testing.html
@@ -284,9 +284,10 @@ possible, or if you want to use a fully qualified test descriptor, add
Gtest
Note: To be able to run the Gtest suite, you need to
configure your build to be able to find a proper version of the gtest
-source. For details, see the section "Running Tests" in the build
-documentation.
+source. For details, see the section "Running Tests" in the
+build documentation (html, markdown).
Since the Hotspot Gtest suite is so quick, the default is to run all
tests. This is specified by just gtest, or as a fully
qualified test descriptor gtest:all.
diff --git a/doc/testing.md b/doc/testing.md
index d0e54aab02b..5f70f2796ad 100644
--- a/doc/testing.md
+++ b/doc/testing.md
@@ -198,8 +198,8 @@ use a fully qualified test descriptor, add `jtreg:`, e.g.
**Note:** To be able to run the Gtest suite, you need to configure your build
to be able to find a proper version of the gtest source. For details, see the
-section ["Running Tests" in the build
-documentation](building.html#running-tests).
+section **"Running Tests" in the build
+documentation** ([html](building.html#running-tests), [markdown](building.md#running-tests)).
Since the Hotspot Gtest suite is so quick, the default is to run all tests.
This is specified by just `gtest`, or as a fully qualified test descriptor
diff --git a/make/Docs.gmk b/make/Docs.gmk
index a8d40e078e2..9cee8cd40c1 100644
--- a/make/Docs.gmk
+++ b/make/Docs.gmk
@@ -1,4 +1,4 @@
-# Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -93,19 +93,16 @@ JAVADOC_DISABLED_DOCLINT_WARNINGS := missing
JAVADOC_DISABLED_DOCLINT_PACKAGES := org.w3c.* javax.smartcardio
# The initial set of options for javadoc
-# -XDaccessInternalAPI is a temporary workaround, see 8373909
JAVADOC_OPTIONS := -use -keywords -notimestamp \
-serialwarn -encoding utf-8 -docencoding utf-8 -breakiterator \
-splitIndex --system none -javafx --expand-requires transitive \
- --override-methods=summary \
- -XDaccessInternalAPI
+ --override-methods=summary
# The reference options must stay stable to allow for comparisons across the
# development cycle.
REFERENCE_OPTIONS := -XDignore.symbol.file=true -use -keywords -notimestamp \
-serialwarn -encoding utf-8 -breakiterator -splitIndex --system none \
- -html5 -javafx --expand-requires transitive \
- -XDaccessInternalAPI
+ -html5 -javafx --expand-requires transitive
# Should we add DRAFT stamps to the generated javadoc?
ifeq ($(VERSION_IS_GA), true)
diff --git a/make/Hsdis.gmk b/make/Hsdis.gmk
index 469cc488f16..76695fc8dde 100644
--- a/make/Hsdis.gmk
+++ b/make/Hsdis.gmk
@@ -44,6 +44,9 @@ ifeq ($(HSDIS_BACKEND), capstone)
else ifeq ($(call isTargetCpuArch, aarch64), true)
CAPSTONE_ARCH := CS_ARCH_$(CAPSTONE_ARCH_AARCH64_NAME)
CAPSTONE_MODE := CS_MODE_ARM
+ else ifeq ($(call isTargetCpuArch, arm), true)
+ CAPSTONE_ARCH := CS_ARCH_ARM
+ CAPSTONE_MODE := CS_MODE_ARM
else
$(error No support for Capstone on this platform)
endif
diff --git a/make/RunTests.gmk b/make/RunTests.gmk
index 02ea632e3ec..d4be5936c41 100644
--- a/make/RunTests.gmk
+++ b/make/RunTests.gmk
@@ -1020,6 +1020,9 @@ define SetupRunJtregTestBody
VM_OPTIONS := $$(JTREG_ALL_OPTIONS) ))
$$(call LogWarn, AOT_JDK_CACHE=$$($1_AOT_JDK_CACHE))
$1_JTREG_BASIC_OPTIONS += -vmoption:-XX:AOTCache="$$($1_AOT_JDK_CACHE)"
+ $1_JTREG_BASIC_OPTIONS += $$(addprefix $$(JTREG_PROBLEM_LIST_PREFIX), $$(wildcard \
+ $$(addprefix $$($1_TEST_ROOT)/, ProblemList-AotJdk.txt) \
+ ))
endif
diff --git a/make/autoconf/flags-cflags.m4 b/make/autoconf/flags-cflags.m4
index ab9cd8be19b..423654cd50a 100644
--- a/make/autoconf/flags-cflags.m4
+++ b/make/autoconf/flags-cflags.m4
@@ -544,12 +544,9 @@ AC_DEFUN([FLAGS_SETUP_CFLAGS_HELPER],
TOOLCHAIN_CFLAGS_JVM="$TOOLCHAIN_CFLAGS_JVM -fstack-protector"
TOOLCHAIN_CFLAGS_JDK="-fvisibility=hidden -pipe -fstack-protector"
# reduce lib size on linux in link step, this needs also special compile flags
- # do this on s390x also for libjvm (where serviceability agent is not supported)
if test "x$ENABLE_LINKTIME_GC" = xtrue; then
TOOLCHAIN_CFLAGS_JDK="$TOOLCHAIN_CFLAGS_JDK -ffunction-sections -fdata-sections"
- if test "x$OPENJDK_TARGET_CPU" = xs390x && test "x$DEBUG_LEVEL" == xrelease; then
- TOOLCHAIN_CFLAGS_JVM="$TOOLCHAIN_CFLAGS_JVM -ffunction-sections -fdata-sections"
- fi
+ TOOLCHAIN_CFLAGS_JVM="$TOOLCHAIN_CFLAGS_JVM -ffunction-sections -fdata-sections"
fi
# technically NOT for CXX (but since this gives *worse* performance, use
# no-strict-aliasing everywhere!)
diff --git a/make/autoconf/flags-ldflags.m4 b/make/autoconf/flags-ldflags.m4
index 7782735be25..ff10828731e 100644
--- a/make/autoconf/flags-ldflags.m4
+++ b/make/autoconf/flags-ldflags.m4
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -53,16 +53,15 @@ AC_DEFUN([FLAGS_SETUP_LDFLAGS_HELPER],
# add --icf=all (Identical Code Folding — merges identical functions)
BASIC_LDFLAGS="-Wl,-z,defs -Wl,-z,relro -Wl,-z,now -Wl,--no-as-needed -Wl,--exclude-libs,ALL"
+ BASIC_LDFLAGS_JVM_ONLY=""
# Linux : remove unused code+data in link step
if test "x$ENABLE_LINKTIME_GC" = xtrue; then
- if test "x$OPENJDK_TARGET_CPU" = xs390x; then
- BASIC_LDFLAGS="$BASIC_LDFLAGS -Wl,--gc-sections"
- else
- BASIC_LDFLAGS_JDK_ONLY="$BASIC_LDFLAGS_JDK_ONLY -Wl,--gc-sections"
- fi
+ # keep vtables : -Wl,--undefined-glob=_ZTV* (but this seems not to work with gold ld)
+ # so keep at least the Metadata vtable that is used in the serviceability agent
+ BASIC_LDFLAGS_JVM_ONLY="$BASIC_LDFLAGS_JVM_ONLY -Wl,--gc-sections -Wl,--undefined=_ZTV8Metadata"
+ BASIC_LDFLAGS_JDK_ONLY="$BASIC_LDFLAGS_JDK_ONLY -Wl,--gc-sections"
fi
- BASIC_LDFLAGS_JVM_ONLY=""
LDFLAGS_LTO="-flto=auto -fuse-linker-plugin -fno-strict-aliasing $DEBUG_PREFIX_CFLAGS"
LDFLAGS_CXX_PARTIAL_LINKING="$MACHINE_FLAG -r"
diff --git a/make/autoconf/toolchain_microsoft.m4 b/make/autoconf/toolchain_microsoft.m4
index f577cf1a2a1..afe04acf029 100644
--- a/make/autoconf/toolchain_microsoft.m4
+++ b/make/autoconf/toolchain_microsoft.m4
@@ -217,10 +217,12 @@ AC_DEFUN([TOOLCHAIN_FIND_VISUAL_STUDIO_BAT_FILE],
TOOLCHAIN_CHECK_POSSIBLE_VISUAL_STUDIO_ROOT([$TARGET_CPU], [$VS_VERSION],
[$PROGRAMFILES_X86/$VS_INSTALL_DIR], [well-known name])
fi
+ # Derive system drive root from CMD (which is at /windows/system32/cmd.exe)
+ WINSYSDRIVE_ROOT="$(dirname "$(dirname "$(dirname "$CMD")")")"
TOOLCHAIN_CHECK_POSSIBLE_VISUAL_STUDIO_ROOT([$TARGET_CPU], [$VS_VERSION],
- [c:/program files/$VS_INSTALL_DIR], [well-known name])
+ [$WINSYSDRIVE_ROOT/program files/$VS_INSTALL_DIR], [well-known name])
TOOLCHAIN_CHECK_POSSIBLE_VISUAL_STUDIO_ROOT([$TARGET_CPU], [$VS_VERSION],
- [c:/program files (x86)/$VS_INSTALL_DIR], [well-known name])
+ [$WINSYSDRIVE_ROOT/program files (x86)/$VS_INSTALL_DIR], [well-known name])
if test "x$SDK_INSTALL_DIR" != x; then
if test "x$ProgramW6432" != x; then
TOOLCHAIN_CHECK_POSSIBLE_WIN_SDK_ROOT([$TARGET_CPU], [$VS_VERSION],
@@ -235,9 +237,9 @@ AC_DEFUN([TOOLCHAIN_FIND_VISUAL_STUDIO_BAT_FILE],
[$PROGRAMFILES/$SDK_INSTALL_DIR], [well-known name])
fi
TOOLCHAIN_CHECK_POSSIBLE_WIN_SDK_ROOT([$TARGET_CPU], [$VS_VERSION],
- [c:/program files/$SDK_INSTALL_DIR], [well-known name])
+ [$WINSYSDRIVE_ROOT/program files/$SDK_INSTALL_DIR], [well-known name])
TOOLCHAIN_CHECK_POSSIBLE_WIN_SDK_ROOT([$TARGET_CPU], [$VS_VERSION],
- [c:/program files (x86)/$SDK_INSTALL_DIR], [well-known name])
+ [$WINSYSDRIVE_ROOT/program files (x86)/$SDK_INSTALL_DIR], [well-known name])
fi
VCVARS_VER=auto
@@ -338,7 +340,7 @@ AC_DEFUN([TOOLCHAIN_EXTRACT_VISUAL_STUDIO_ENV],
OLDPATH="$PATH"
# Make sure we only capture additions to PATH needed by VS.
# Clear out path, but need system dir present for vsvars cmd file to be able to run
- export PATH=$WINENV_PREFIX/c/windows/system32
+ export PATH="$(dirname "$CMD")"
# The "| cat" is to stop SetEnv.Cmd to mess with system colors on some systems
# We can't pass -vcvars_ver=$VCVARS_VER here because cmd.exe eats all '='
# in bat file arguments. :-(
diff --git a/make/common/modules/LauncherCommon.gmk b/make/common/modules/LauncherCommon.gmk
index 859494861b2..8d45142ef4a 100644
--- a/make/common/modules/LauncherCommon.gmk
+++ b/make/common/modules/LauncherCommon.gmk
@@ -36,16 +36,16 @@ include $(TOPDIR)/make/ToolsJdk.gmk
LAUNCHER_SRC := $(TOPDIR)/src/java.base/share/native/launcher
-ifeq ($(call isTargetOs, aix), true)
- ADD_PLATFORM_INCLUDE_DIR := -I$(TOPDIR)/src/java.base/aix/native/include
-endif
-
LAUNCHER_CFLAGS += -I$(TOPDIR)/src/java.base/share/native/launcher \
-I$(TOPDIR)/src/java.base/share/native/libjli \
- $(ADD_PLATFORM_INCLUDE_DIR) \
-I$(TOPDIR)/src/java.base/$(OPENJDK_TARGET_OS_TYPE)/native/libjli \
-I$(TOPDIR)/src/java.base/$(OPENJDK_TARGET_OS)/native/libjli \
#
+
+ifeq ($(call isTargetOs, aix), true)
+ LAUNCHER_CFLAGS += -I$(TOPDIR)/src/java.base/aix/native/include
+endif
+
MACOSX_PLIST_DIR := $(TOPDIR)/src/java.base/macosx/native/launcher
JAVA_MANIFEST := $(TOPDIR)/src/java.base/windows/native/launcher/java.manifest
diff --git a/make/conf/github-actions.conf b/make/conf/github-actions.conf
index ebfc9191535..6771e8923dc 100644
--- a/make/conf/github-actions.conf
+++ b/make/conf/github-actions.conf
@@ -29,21 +29,21 @@ GTEST_VERSION=1.14.0
JTREG_VERSION=8.2.1+1
LINUX_X64_BOOT_JDK_EXT=tar.gz
-LINUX_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk25/bd75d5f9689641da8e1daabeccb5528b/36/GPL/openjdk-25_linux-x64_bin.tar.gz
-LINUX_X64_BOOT_JDK_SHA256=59cdcaf255add4721de38eb411d4ecfe779356b61fb671aee63c7dec78054c2b
+LINUX_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk26/c3cc523845074aa0af4f5e1e1ed4151d/35/GPL/openjdk-26_linux-x64_bin.tar.gz
+LINUX_X64_BOOT_JDK_SHA256=83c78367f8c81257beef72aca4bbbf8e6dac8ca2b3a4546a85879a09e6e4e128
ALPINE_LINUX_X64_BOOT_JDK_EXT=tar.gz
-ALPINE_LINUX_X64_BOOT_JDK_URL=https://github.com/adoptium/temurin25-binaries/releases/download/jdk-25%2B36/OpenJDK25U-jdk_x64_alpine-linux_hotspot_25_36.tar.gz
-ALPINE_LINUX_X64_BOOT_JDK_SHA256=637e47474d411ed86134f413af7d5fef4180ddb0bf556347b7e74a88cf8904c8
+ALPINE_LINUX_X64_BOOT_JDK_URL=https://github.com/adoptium/temurin26-binaries/releases/download/jdk-26%2B35/OpenJDK26U-jdk_x64_alpine-linux_hotspot_26_35.tar.gz
+ALPINE_LINUX_X64_BOOT_JDK_SHA256=c105e581fdccb4e7120d889235d1ad8d5b2bed0af4972bc881e0a8ba687c94a4
MACOS_AARCH64_BOOT_JDK_EXT=tar.gz
-MACOS_AARCH64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk25/bd75d5f9689641da8e1daabeccb5528b/36/GPL/openjdk-25_macos-aarch64_bin.tar.gz
-MACOS_AARCH64_BOOT_JDK_SHA256=2006337bf326fdfdf6117081751ba38c1c8706d63419ecac7ff102ff7c776876
+MACOS_AARCH64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk26/c3cc523845074aa0af4f5e1e1ed4151d/35/GPL/openjdk-26_macos-aarch64_bin.tar.gz
+MACOS_AARCH64_BOOT_JDK_SHA256=254586bcd1bf6dcd125ad667ac32562cb1e2ab1abf3a61fb117b6fabb571e765
MACOS_X64_BOOT_JDK_EXT=tar.gz
-MACOS_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk25/bd75d5f9689641da8e1daabeccb5528b/36/GPL/openjdk-25_macos-x64_bin.tar.gz
-MACOS_X64_BOOT_JDK_SHA256=47482ad9888991ecac9b2bcc131e2b53ff78aff275104cef85f66252308e8a09
+MACOS_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk26/c3cc523845074aa0af4f5e1e1ed4151d/35/GPL/openjdk-26_macos-x64_bin.tar.gz
+MACOS_X64_BOOT_JDK_SHA256=8642b89d889c14ede2c446fd5bbe3621c8a3082e3df02013fd1658e39f52929a
WINDOWS_X64_BOOT_JDK_EXT=zip
-WINDOWS_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk25/bd75d5f9689641da8e1daabeccb5528b/36/GPL/openjdk-25_windows-x64_bin.zip
-WINDOWS_X64_BOOT_JDK_SHA256=85bcc178461e2cb3c549ab9ca9dfa73afd54c09a175d6510d0884071867137d3
+WINDOWS_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk26/c3cc523845074aa0af4f5e1e1ed4151d/35/GPL/openjdk-26_windows-x64_bin.zip
+WINDOWS_X64_BOOT_JDK_SHA256=2dd2d92c9374cd49a120fe9d916732840bf6bb9f0e0cc29794917a3c08b99c5f
diff --git a/make/conf/jib-profiles.js b/make/conf/jib-profiles.js
index 76a94b7789e..4c1d2835054 100644
--- a/make/conf/jib-profiles.js
+++ b/make/conf/jib-profiles.js
@@ -387,8 +387,8 @@ var getJibProfilesCommon = function (input, data) {
};
};
- common.boot_jdk_version = "25";
- common.boot_jdk_build_number = "37";
+ common.boot_jdk_version = "26";
+ common.boot_jdk_build_number = "35";
common.boot_jdk_home = input.get("boot_jdk", "install_path") + "/jdk-"
+ common.boot_jdk_version
+ (input.build_os == "macosx" ? ".jdk/Contents/Home" : "");
diff --git a/make/conf/version-numbers.conf b/make/conf/version-numbers.conf
index 4392d86ac33..4f63179ae05 100644
--- a/make/conf/version-numbers.conf
+++ b/make/conf/version-numbers.conf
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -37,6 +37,6 @@ DEFAULT_VERSION_DATE=2026-09-15
DEFAULT_VERSION_CLASSFILE_MAJOR=71 # "`$EXPR $DEFAULT_VERSION_FEATURE + 44`"
DEFAULT_VERSION_CLASSFILE_MINOR=0
DEFAULT_VERSION_DOCS_API_SINCE=11
-DEFAULT_ACCEPTABLE_BOOT_VERSIONS="25 26 27"
+DEFAULT_ACCEPTABLE_BOOT_VERSIONS="26 27"
DEFAULT_JDK_SOURCE_TARGET_VERSION=27
DEFAULT_PROMOTED_VERSION_PRE=ea
diff --git a/make/jdk/src/classes/build/tools/cldrconverter/CLDRConverter.java b/make/jdk/src/classes/build/tools/cldrconverter/CLDRConverter.java
index ab878a4d2a5..9f42326ef09 100644
--- a/make/jdk/src/classes/build/tools/cldrconverter/CLDRConverter.java
+++ b/make/jdk/src/classes/build/tools/cldrconverter/CLDRConverter.java
@@ -87,6 +87,7 @@ public class CLDRConverter {
static final String EXEMPLAR_CITY_PREFIX = "timezone.excity.";
static final String ZONE_NAME_PREFIX = "timezone.displayname.";
static final String METAZONE_ID_PREFIX = "metazone.id.";
+ static final String METAZONE_DSTOFFSET_PREFIX = "metazone.dstoffset.";
static final String PARENT_LOCALE_PREFIX = "parentLocale.";
static final String LIKELY_SCRIPT_PREFIX = "likelyScript.";
static final String META_EMPTY_ZONE_NAME = "EMPTY_ZONE";
@@ -139,6 +140,11 @@ public class CLDRConverter {
private static final Map tzdbSubstLetters = HashMap.newHashMap(512);
private static final Map tzdbLinks = HashMap.newHashMap(512);
+ // Map of explicit dst offsets for metazones
+ // key: time zone ID
+ // value: explicit dstOffset for the corresponding metazone name
+ static final Map explicitDstOffsets = HashMap.newHashMap(32);
+
static enum DraftType {
UNCONFIRMED,
PROVISIONAL,
@@ -795,10 +801,7 @@ public class CLDRConverter {
String tzKey = Optional.ofNullable((String)handlerSupplMeta.get(tzid))
.orElse(tzid);
// Follow link, if needed
- String tzLink = null;
- for (var k = tzKey; tzdbLinks.containsKey(k);) {
- k = tzLink = tzdbLinks.get(k);
- }
+ String tzLink = getTZDBLink(tzKey);
if (tzLink == null && tzdbLinks.containsValue(tzKey)) {
// reverse link search
// this is needed as in tzdb, "America/Buenos_Aires" links to
@@ -827,7 +830,7 @@ public class CLDRConverter {
} else {
// TZDB short names
tznames = Arrays.copyOf(tznames, tznames.length);
- fillTZDBShortNames(tzid, tznames);
+ fillTZDBShortNames(tzKey, tznames);
names.put(tzid, tznames);
}
} else {
@@ -840,11 +843,13 @@ public class CLDRConverter {
String metaKey = METAZONE_ID_PREFIX + meta;
data = map.get(metaKey);
if (data instanceof String[] tznames) {
- // TZDB short names
- tznames = Arrays.copyOf((String[])names.getOrDefault(metaKey, tznames), 6);
- fillTZDBShortNames(tzid, tznames);
- // Keep the metazone prefix here.
- names.putIfAbsent(metaKey, tznames);
+ if (isDefaultZone(meta, tzKey)) {
+ // Record the metazone names only from the default
+ // (001) zone, with short names filled from TZDB
+ tznames = Arrays.copyOf(tznames, tznames.length);
+ fillTZDBShortNames(tzKey, tznames);
+ names.put(metaKey, tznames);
+ }
names.put(tzid, meta);
if (tzLink != null && availableIds.contains(tzLink)) {
names.put(tzLink, meta);
@@ -867,6 +872,12 @@ public class CLDRConverter {
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
names.putAll(exCities);
+ // Explicit metazone offsets
+ if (id.equals("root")) {
+ explicitDstOffsets.forEach((k, v) ->
+ names.put(METAZONE_DSTOFFSET_PREFIX + k, v));
+ }
+
// If there's no UTC entry at this point, add an empty one
if (!names.isEmpty() && !names.containsKey("UTC")) {
names.putIfAbsent(METAZONE_ID_PREFIX + META_EMPTY_ZONE_NAME, EMPTY_ZONE);
@@ -1492,12 +1503,12 @@ public class CLDRConverter {
* Fill the TZDB short names if there is no name provided by the CLDR
*/
private static void fillTZDBShortNames(String tzid, String[] names) {
- var val = tzdbShortNamesMap.get(tzdbLinks.getOrDefault(tzid, tzid));
+ var val = tzdbShortNamesMap.getOrDefault(tzid, tzdbShortNamesMap.get(getTZDBLink(tzid)));
if (val != null) {
var format = val.split(NBSP)[0];
var rule = val.split(NBSP)[1];
IntStream.of(1, 3, 5).forEach(i -> {
- if (names[i] == null) {
+ if (names[i] == null || names[i].isEmpty()) {
if (format.contains("%s")) {
names[i] = switch (i) {
case 1 -> format.formatted(tzdbSubstLetters.get(rule + NBSP + STD));
@@ -1519,6 +1530,21 @@ public class CLDRConverter {
}
}
+ private static boolean isDefaultZone(String meta, String tzid) {
+ String zone001 = handlerMetaZones.zidMap().get(meta);
+ var tzLink = getTZDBLink(tzid);
+ return canonicalTZMap.getOrDefault(tzid, tzid).equals(zone001) ||
+ tzLink != null && canonicalTZMap.getOrDefault(tzLink, tzLink).equals(zone001);
+ }
+
+ private static String getTZDBLink(String tzid) {
+ String tzLink = null;
+ for (var k = tzid; tzdbLinks.containsKey(k);) {
+ k = tzLink = tzdbLinks.get(k);
+ }
+ return tzLink;
+ }
+
/*
* Convert TZDB offsets to JDK's offsets, eg, "-08" to "GMT-08:00".
* If it cannot recognize the pattern, return the argument as is.
diff --git a/make/jdk/src/classes/build/tools/cldrconverter/MetaZonesParseHandler.java b/make/jdk/src/classes/build/tools/cldrconverter/MetaZonesParseHandler.java
index 2c3757b7a47..45de46d2476 100644
--- a/make/jdk/src/classes/build/tools/cldrconverter/MetaZonesParseHandler.java
+++ b/make/jdk/src/classes/build/tools/cldrconverter/MetaZonesParseHandler.java
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2012, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -84,7 +84,15 @@ class MetaZonesParseHandler extends AbstractLDMLHandler {
if (fromLDT.isBefore(now) && toLDT.isAfter(now)) {
metazone = attributes.getValue("mzone");
+
+ // Explicit metazone DST offsets. Only the "dst" offset is needed,
+ // as "std" is used by default when it doesn't match.
+ String dstOffset = attributes.getValue("dstOffset");
+ if (dstOffset != null) {
+ CLDRConverter.explicitDstOffsets.put(tzid, dstOffset);
+ }
}
+
pushIgnoredContainer(qName);
break;
diff --git a/make/jdk/src/classes/build/tools/cldrconverter/ResourceBundleGenerator.java b/make/jdk/src/classes/build/tools/cldrconverter/ResourceBundleGenerator.java
index 3953f38f653..8278bf6bcfa 100644
--- a/make/jdk/src/classes/build/tools/cldrconverter/ResourceBundleGenerator.java
+++ b/make/jdk/src/classes/build/tools/cldrconverter/ResourceBundleGenerator.java
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2012, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -198,7 +198,8 @@ class ResourceBundleGenerator implements BundleGenerator {
} else if (value instanceof String) {
String valStr = (String)value;
if (type == BundleType.TIMEZONE &&
- !key.startsWith(CLDRConverter.EXEMPLAR_CITY_PREFIX) ||
+ !(key.startsWith(CLDRConverter.EXEMPLAR_CITY_PREFIX) ||
+ key.startsWith(CLDRConverter.METAZONE_DSTOFFSET_PREFIX)) ||
valStr.startsWith(META_VALUE_PREFIX)) {
out.printf(" { \"%s\", %s },\n", key, CLDRConverter.saveConvert(valStr, useJava));
} else {
diff --git a/make/jdk/src/classes/build/tools/taglet/JSpec.java b/make/jdk/src/classes/build/tools/taglet/JSpec.java
index 7e1e0ca215e..196aaccb32b 100644
--- a/make/jdk/src/classes/build/tools/taglet/JSpec.java
+++ b/make/jdk/src/classes/build/tools/taglet/JSpec.java
@@ -25,13 +25,13 @@
package build.tools.taglet;
+import java.net.URI;
import java.util.EnumSet;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
-import java.lang.reflect.Field;
import javax.lang.model.element.Element;
@@ -141,6 +141,11 @@ public class JSpec implements Taglet {
@Override
public String toString(List tags, Element elem) {
+ throw new UnsupportedOperationException();
+ }
+
+ // @Override - requires JDK-8373922 in build JDK
+ public String toString(List tags, Element elem, URI docRoot) {
if (tags.isEmpty())
return "";
@@ -177,7 +182,7 @@ public class JSpec implements Taglet {
String preview = m.group("preview"); // null if no preview feature
String chapter = m.group("chapter");
String section = m.group("section");
- String rootParent = currentPath().replaceAll("[^/]+", "..");
+ String rootParent = docRoot.resolve("..").toString();
String url = preview == null ?
String.format("%1$s/specs/%2$s/%2$s-%3$s.html#%2$s-%3$s%4$s",
@@ -230,23 +235,6 @@ public class JSpec implements Taglet {
return sb.toString();
}
- private static ThreadLocal CURRENT_PATH = null;
-
- private String currentPath() {
- if (CURRENT_PATH == null) {
- try {
- Field f = Class.forName("jdk.javadoc.internal.doclets.formats.html.HtmlDocletWriter")
- .getField("CURRENT_PATH");
- @SuppressWarnings("unchecked")
- ThreadLocal tl = (ThreadLocal) f.get(null);
- CURRENT_PATH = tl;
- } catch (ReflectiveOperationException e) {
- throw new RuntimeException("Cannot determine current path", e);
- }
- }
- return CURRENT_PATH.get();
- }
-
private String expand(List trees) {
return (new SimpleDocTreeVisitor() {
public StringBuilder defaultAction(DocTree tree, StringBuilder sb) {
diff --git a/make/jdk/src/classes/build/tools/taglet/SealedGraph.java b/make/jdk/src/classes/build/tools/taglet/SealedGraph.java
index 3e93826c180..300999b77c0 100644
--- a/make/jdk/src/classes/build/tools/taglet/SealedGraph.java
+++ b/make/jdk/src/classes/build/tools/taglet/SealedGraph.java
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2022, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -32,7 +32,9 @@ import jdk.javadoc.doclet.Taglet;
import javax.lang.model.element.*;
import javax.lang.model.type.DeclaredType;
+import javax.lang.model.util.Elements;
import java.io.IOException;
+import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
@@ -78,6 +80,11 @@ public final class SealedGraph implements Taglet {
@Override
public String toString(List tags, Element element) {
+ throw new UnsupportedOperationException();
+ }
+
+ // @Override - requires JDK-8373922 in build JDK
+ public String toString(List tags, Element element, URI docRoot) {
if (sealedDotOutputDir == null || sealedDotOutputDir.isEmpty()) {
return "";
}
@@ -85,9 +92,15 @@ public final class SealedGraph implements Taglet {
return "";
}
- ModuleElement module = docletEnvironment.getElementUtils().getModuleOf(element);
+ Elements util = docletEnvironment.getElementUtils();
+ ModuleElement module = util.getModuleOf(element);
+
+ // '.' in .DOT file name is converted to '/' in .SVG path, so we use '-' as separator for nested classes.
+ // module_package.subpackage.Outer-Inner.dot => module/package/subpackage/Outer-Inner-sealed-graph.svg
Path dotFile = Path.of(sealedDotOutputDir,
- module.getQualifiedName() + "_" + typeElement.getQualifiedName() + ".dot");
+ module.getQualifiedName() + "_"
+ + util.getPackageOf(element).getQualifiedName() + "."
+ + packagelessCanonicalName(typeElement).replace(".", "-") + ".dot");
Set exports = module.getDirectives().stream()
.filter(ModuleElement.ExportsDirective.class::isInstance)
@@ -99,7 +112,7 @@ public final class SealedGraph implements Taglet {
.map(Objects::toString)
.collect(Collectors.toUnmodifiableSet());
- String dotContent = new Renderer().graph(typeElement, exports);
+ String dotContent = new Renderer().graph(typeElement, exports, docRoot);
try {
Files.writeString(dotFile, dotContent, WRITE, CREATE, TRUNCATE_EXISTING);
@@ -107,8 +120,8 @@ public final class SealedGraph implements Taglet {
throw new RuntimeException(e);
}
- String simpleTypeName = packagelessCanonicalName(typeElement).replace('.', '/');
- String imageFile = simpleTypeName + "-sealed-graph.svg";
+ String simpleTypeName = packagelessCanonicalName(typeElement);
+ String imageFile = simpleTypeName.replace(".", "-") + "-sealed-graph.svg";
int thumbnailHeight = 100; // also appears in the stylesheet
String hoverImage = ""
+ getImage(simpleTypeName, imageFile, -1, true)
@@ -137,21 +150,26 @@ public final class SealedGraph implements Taglet {
private final class Renderer {
// Generates a graph in DOT format
- String graph(TypeElement rootClass, Set exports) {
- final State state = new State(rootClass);
+ String graph(TypeElement rootClass, Set exports, URI pathToRoot) {
+ if (!isInPublicApi(rootClass, exports)) {
+ // Alternatively we can return "" for the graph since there is no single root to render
+ throw new IllegalArgumentException("Root not in public API: " + rootClass.getQualifiedName());
+ }
+ final State state = new State(pathToRoot);
traverse(state, rootClass, exports);
return state.render();
}
static void traverse(State state, TypeElement node, Set exports) {
+ if (!isInPublicApi(node, exports)) {
+ throw new IllegalArgumentException("Bad request, not in public API: " + node.getQualifiedName());
+ }
state.addNode(node);
if (!(node.getModifiers().contains(Modifier.SEALED) || node.getModifiers().contains(Modifier.FINAL))) {
state.addNonSealedEdge(node);
} else {
for (TypeElement subNode : permittedSubclasses(node, exports)) {
- if (isInPublicApi(node, exports) && isInPublicApi(subNode, exports)) {
- state.addEdge(node, subNode);
- }
+ state.addEdge(node, subNode);
traverse(state, subNode, exports);
}
}
@@ -163,7 +181,7 @@ public final class SealedGraph implements Taglet {
private static final String TOOLTIP = "tooltip";
private static final String LINK = "href";
- private final TypeElement rootNode;
+ private final URI pathToRoot;
private final StringBuilder builder;
@@ -188,8 +206,8 @@ public final class SealedGraph implements Taglet {
}
}
- public State(TypeElement rootNode) {
- this.rootNode = rootNode;
+ public State(URI pathToRoot) {
+ this.pathToRoot = pathToRoot;
nodeStyleMap = new LinkedHashMap<>();
builder = new StringBuilder()
.append("digraph G {")
@@ -212,24 +230,15 @@ public final class SealedGraph implements Taglet {
var styles = nodeStyleMap.computeIfAbsent(id(node), n -> new LinkedHashMap<>());
styles.put(LABEL, new StyleItem.PlainString(node.getSimpleName().toString()));
styles.put(TOOLTIP, new StyleItem.PlainString(node.getQualifiedName().toString()));
- styles.put(LINK, new StyleItem.PlainString(relativeLink(node)));
+ styles.put(LINK, new StyleItem.PlainString(pathToRoot.resolve(relativeLink(node)).toString()));
}
- // A permitted class must be in the same package or in the same module.
- // This implies the module is always the same.
private String relativeLink(TypeElement node) {
var util = SealedGraph.this.docletEnvironment.getElementUtils();
- var nodePackage = util.getPackageOf(node);
- // Note: SVG files for nested types use the simple names of containing types as parent directories.
- // We therefore need to convert all dots in the qualified name to "../" below.
- var backNavigator = rootNode.getQualifiedName().toString().chars()
- .filter(c -> c == '.')
- .mapToObj(c -> "../")
- .collect(joining());
- var forwardNavigator = nodePackage.getQualifiedName().toString()
- .replace(".", "/");
+ var path = util.getModuleOf(node).getQualifiedName().toString() + "/"
+ + util.getPackageOf(node).getQualifiedName().toString().replace(".", "/");
- return backNavigator + forwardNavigator + "/" + packagelessCanonicalName(node) + ".html";
+ return path + "/" + packagelessCanonicalName(node) + ".html";
}
public void addEdge(TypeElement node, TypeElement subNode) {
@@ -281,25 +290,33 @@ public final class SealedGraph implements Taglet {
private String quotedId(TypeElement node) {
return "\"" + id(node) + "\"";
}
-
- private String simpleName(String name) {
- int lastDot = name.lastIndexOf('.');
- return lastDot < 0
- ? name
- : name.substring(lastDot);
- }
-
}
private static List permittedSubclasses(TypeElement node, Set exports) {
- return node.getPermittedSubclasses().stream()
- .filter(DeclaredType.class::isInstance)
- .map(DeclaredType.class::cast)
- .map(DeclaredType::asElement)
- .filter(TypeElement.class::isInstance)
- .map(TypeElement.class::cast)
- .filter(te -> isInPublicApi(te, exports))
- .toList();
+ List dfsStack = new ArrayList().reversed(); // Faster operations to head
+ SequencedCollection result = new LinkedHashSet<>(); // Deduplicate diamond interface inheritance
+ // The starting node may be in the public API - still expand it
+ prependSubclasses(node, dfsStack);
+
+ while (!dfsStack.isEmpty()) {
+ TypeElement now = dfsStack.removeFirst();
+ if (isInPublicApi(now, exports)) {
+ result.addLast(now);
+ } else {
+ // Skip the non-exported classes in the hierarchy
+ prependSubclasses(now, dfsStack);
+ }
+ }
+
+ return List.copyOf(result);
+ }
+
+ private static void prependSubclasses(TypeElement node, List dfs) {
+ for (var e : node.getPermittedSubclasses().reversed()) {
+ if (e instanceof DeclaredType dt && dt.asElement() instanceof TypeElement te) {
+ dfs.addFirst(te);
+ }
+ }
}
private static boolean isInPublicApi(TypeElement typeElement, Set exports) {
diff --git a/make/jdk/src/classes/build/tools/taglet/ToolGuide.java b/make/jdk/src/classes/build/tools/taglet/ToolGuide.java
index 8db2aee3092..7ad4f6b9b9f 100644
--- a/make/jdk/src/classes/build/tools/taglet/ToolGuide.java
+++ b/make/jdk/src/classes/build/tools/taglet/ToolGuide.java
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -25,13 +25,13 @@
package build.tools.taglet;
+import java.net.URI;
import java.util.EnumSet;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
-import java.lang.reflect.Field;
import javax.lang.model.element.Element;
@@ -91,6 +91,11 @@ public class ToolGuide implements Taglet {
@Override
public String toString(List tags, Element elem) {
+ throw new UnsupportedOperationException();
+ }
+
+ // @Override - requires JDK-8373922 in build JDK
+ public String toString(List tags, Element elem, URI docRoot) {
if (tags.isEmpty())
return "";
@@ -118,7 +123,7 @@ public class ToolGuide implements Taglet {
if (label.isEmpty()) {
label = name;
}
- String rootParent = currentPath().replaceAll("[^/]+", "..");
+ String rootParent = docRoot.resolve("..").toString();
String url = String.format("%s/%s/%s.html",
rootParent, BASE_URL, name);
@@ -141,22 +146,4 @@ public class ToolGuide implements Taglet {
return sb.toString();
}
-
- private static ThreadLocal CURRENT_PATH = null;
-
- private String currentPath() {
- if (CURRENT_PATH == null) {
- try {
- Field f = Class.forName("jdk.javadoc.internal.doclets.formats.html.HtmlDocletWriter")
- .getField("CURRENT_PATH");
- @SuppressWarnings("unchecked")
- ThreadLocal tl = (ThreadLocal) f.get(null);
- CURRENT_PATH = tl;
- } catch (ReflectiveOperationException e) {
- throw new RuntimeException("Cannot determine current path", e);
- }
- }
- return CURRENT_PATH.get();
- }
-
}
diff --git a/make/langtools/tools/previewfeature/SetupPreviewFeature.java b/make/langtools/tools/previewfeature/SetupPreviewFeature.java
index 2d5207f0e17..5f9b00edc6d 100644
--- a/make/langtools/tools/previewfeature/SetupPreviewFeature.java
+++ b/make/langtools/tools/previewfeature/SetupPreviewFeature.java
@@ -30,6 +30,7 @@ import java.io.StringWriter;
import java.lang.reflect.Field;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
import java.util.HashSet;
import java.util.Set;
import java.util.stream.Collectors;
@@ -76,7 +77,7 @@ public class SetupPreviewFeature {
var target = Path.of(args[1]);
Files.createDirectories(target.getParent());
if (constantsToAdd.isEmpty()) {
- Files.copy(source, target);
+ Files.copy(source, target, StandardCopyOption.REPLACE_EXISTING);
} else {
String sourceCode = Files.readString(source);
try (var out = Files.newBufferedWriter(target)) {
diff --git a/make/modules/java.desktop/lib/ClientLibraries.gmk b/make/modules/java.desktop/lib/ClientLibraries.gmk
index b76cb8dc4e3..2326505d11c 100644
--- a/make/modules/java.desktop/lib/ClientLibraries.gmk
+++ b/make/modules/java.desktop/lib/ClientLibraries.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -257,6 +257,7 @@ ifeq ($(ENABLE_HEADLESS_ONLY), false)
DISABLED_WARNINGS_microsoft_dgif_lib.c := 4018 4267, \
DISABLED_WARNINGS_microsoft_splashscreen_impl.c := 4018 4267 4244, \
DISABLED_WARNINGS_microsoft_splashscreen_png.c := 4267, \
+ DISABLED_WARNINGS_microsoft_pngread.c := 4146, \
DISABLED_WARNINGS_microsoft_splashscreen_sys.c := 4267 4244, \
LDFLAGS := $(ICONV_LDFLAGS), \
LDFLAGS_windows := -delayload:user32.dll, \
@@ -338,11 +339,8 @@ else
# noexcept-type required for GCC 7 builds. Not required for GCC 8+.
# expansion-to-defined required for GCC 9 builds. Not required for GCC 10+.
# maybe-uninitialized required for GCC 8 builds. Not required for GCC 9+.
- # calloc-transposed-args required for GCC 14 builds. (fixed upstream in
- # Harfbuzz 032c931e1c0cfb20f18e5acb8ba005775242bd92)
HARFBUZZ_DISABLED_WARNINGS_CXX_gcc := class-memaccess noexcept-type \
- expansion-to-defined dangling-reference maybe-uninitialized \
- calloc-transposed-args
+ expansion-to-defined dangling-reference maybe-uninitialized
HARFBUZZ_DISABLED_WARNINGS_clang := missing-field-initializers \
range-loop-analysis unused-variable
HARFBUZZ_DISABLED_WARNINGS_microsoft := 4267 4244
@@ -397,6 +395,8 @@ $(eval $(call SetupJdkLibrary, BUILD_LIBFONTMANAGER, \
AccelGlyphCache.c, \
CFLAGS := $(LIBFONTMANAGER_CFLAGS), \
CXXFLAGS := $(LIBFONTMANAGER_CFLAGS), \
+ CXXFLAGS_gcc := -fno-rtti -fno-exceptions, \
+ CXXFLAGS_clang := -fno-rtti -fno-exceptions, \
OPTIMIZATION := HIGHEST, \
CFLAGS_windows = -DCC_NOEX, \
EXTRA_HEADER_DIRS := $(LIBFONTMANAGER_EXTRA_HEADER_DIRS), \
diff --git a/make/modules/jdk.hotspot.agent/Lib.gmk b/make/modules/jdk.hotspot.agent/Lib.gmk
index ed8de631dc3..da02e0dab39 100644
--- a/make/modules/jdk.hotspot.agent/Lib.gmk
+++ b/make/modules/jdk.hotspot.agent/Lib.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2015, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2015, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -55,6 +55,12 @@ else
LIBSAPROC_LINK_TYPE := C
endif
+# DWARF related sources would be included on supported platforms only.
+LIBSAPROC_EXCLUDE_FILES :=
+ifneq ($(call And, $(call isTargetOs, linux) $(call isTargetCpu, x86_64 aarch64)), true)
+ LIBSAPROC_EXCLUDE_FILES := DwarfParser.cpp dwarf.cpp
+endif
+
$(eval $(call SetupJdkLibrary, BUILD_LIBSAPROC, \
NAME := saproc, \
LINK_TYPE := $(LIBSAPROC_LINK_TYPE), \
@@ -70,6 +76,7 @@ $(eval $(call SetupJdkLibrary, BUILD_LIBSAPROC, \
CFLAGS := $(LIBSAPROC_CFLAGS), \
CXXFLAGS := $(LIBSAPROC_CFLAGS) $(LIBSAPROC_CXXFLAGS), \
EXTRA_SRC := $(LIBSAPROC_EXTRA_SRC), \
+ EXCLUDE_FILES := $(LIBSAPROC_EXCLUDE_FILES), \
JDK_LIBS := java.base:libjava, \
LIBS_linux := $(LIBDL), \
LIBS_macosx := \
diff --git a/make/scripts/fixpath.sh b/make/scripts/fixpath.sh
index 6a524df4c68..78690f1f2cc 100644
--- a/make/scripts/fixpath.sh
+++ b/make/scripts/fixpath.sh
@@ -88,7 +88,10 @@ function setup() {
fi
if [[ -z ${CMD+x} ]]; then
- CMD="$DRIVEPREFIX/c/windows/system32/cmd.exe"
+ CMD="$(type -p cmd.exe 2>/dev/null)"
+ if [[ -z "$CMD" ]]; then
+ CMD="$DRIVEPREFIX/c/windows/system32/cmd.exe"
+ fi
fi
if [[ -z ${WINTEMP+x} ]]; then
diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad
index b79030f07e7..53fa4e3066c 100644
--- a/src/hotspot/cpu/aarch64/aarch64.ad
+++ b/src/hotspot/cpu/aarch64/aarch64.ad
@@ -1182,12 +1182,12 @@ class CallStubImpl {
public:
// Size of call trampoline stub.
static uint size_call_trampoline() {
- return 0; // no call trampolines on this platform
+ return MacroAssembler::max_trampoline_stub_size();
}
// number of relocations needed by a call trampoline stub
static uint reloc_call_trampoline() {
- return 0; // no call trampolines on this platform
+ return 5; // metadata; call dest; trampoline address; trampoline destination; trampoline_owner_metadata
}
};
@@ -2233,15 +2233,9 @@ uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
{
st->print_cr("# MachUEPNode");
- if (UseCompressedClassPointers) {
- st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
- st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
- st->print_cr("\tcmpw rscratch1, r10");
- } else {
- st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
- st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
- st->print_cr("\tcmp rscratch1, r10");
- }
+ st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
+ st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
+ st->print_cr("\tcmpw rscratch1, r10");
st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
}
#endif
diff --git a/src/hotspot/cpu/aarch64/aarch64_vector.ad b/src/hotspot/cpu/aarch64/aarch64_vector.ad
index 19f03d97a72..4c854913e63 100644
--- a/src/hotspot/cpu/aarch64/aarch64_vector.ad
+++ b/src/hotspot/cpu/aarch64/aarch64_vector.ad
@@ -1,6 +1,6 @@
//
// Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
-// Copyright (c) 2020, 2025, Arm Limited. All rights reserved.
+// Copyright (c) 2020, 2026, Arm Limited. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
@@ -247,10 +247,39 @@ source %{
case Op_MinVHF:
case Op_MaxVHF:
case Op_SqrtVHF:
+ if (UseSVE == 0 && !is_feat_fp16_supported()) {
+ return false;
+ }
+ break;
+ // At the time of writing this, the Vector API has no half-float (FP16) species.
+ // Consequently, AddReductionVHF and MulReductionVHF are only produced by the
+ // auto-vectorizer, which requires strictly ordered semantics for FP reductions.
+ //
+ // There is no direct Neon instruction that performs strictly ordered floating
+ // point add reduction. Hence, on Neon only machines, the add reduction operation
+ // is implemented as a scalarized sequence using half-precision scalar instruction
+ // FADD which requires FEAT_FP16 and ASIMDHP to be available on the target.
+ // On SVE machines (UseSVE > 0) however, there is a direct instruction (FADDA) which
+ // implements strictly ordered floating point add reduction which does not require
+ // the FEAT_FP16 and ASIMDHP checks as SVE supports half-precision floats by default.
+ case Op_AddReductionVHF:
// FEAT_FP16 is enabled if both "fphp" and "asimdhp" features are supported.
// Only the Neon instructions need this check. SVE supports half-precision floats
// by default.
- if (UseSVE == 0 && !is_feat_fp16_supported()) {
+ if (length_in_bytes < 8 || (UseSVE == 0 && !is_feat_fp16_supported())) {
+ return false;
+ }
+ break;
+ case Op_MulReductionVHF:
+ // There are no direct Neon/SVE instructions that perform strictly ordered
+ // floating point multiply reduction.
+ // For vector length ≤ 16 bytes, the reduction is implemented as a scalarized
+ // sequence using half-precision scalar instruction FMUL. This path requires
+ // FEAT_FP16 and ASIMDHP to be available on the target.
+ // For vector length > 16 bytes, this operation is disabled because there is no
+ // direct SVE instruction that performs a strictly ordered FP16 multiply
+ // reduction.
+ if (length_in_bytes < 8 || length_in_bytes > 16 || !is_feat_fp16_supported()) {
return false;
}
break;
@@ -300,6 +329,7 @@ source %{
case Op_VectorRearrange:
case Op_MulReductionVD:
case Op_MulReductionVF:
+ case Op_MulReductionVHF:
case Op_MulReductionVI:
case Op_MulReductionVL:
case Op_CompressBitsV:
@@ -364,6 +394,7 @@ source %{
case Op_VectorMaskCmp:
case Op_LoadVectorGather:
case Op_StoreVectorScatter:
+ case Op_AddReductionVHF:
case Op_AddReductionVF:
case Op_AddReductionVD:
case Op_AndReductionV:
@@ -597,13 +628,9 @@ instruct vloadcon(vReg dst, immI0 src) %{
BasicType bt = Matcher::vector_element_basic_type(this);
if (UseSVE == 0) {
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
+ int entry_idx = __ vector_iota_entry_index(bt);
assert(length_in_bytes <= 16, "must be");
- // The iota indices are ordered by type B/S/I/L/F/D, and the offset between two types is 16.
- int offset = exact_log2(type2aelembytes(bt)) << 4;
- if (is_floating_point_type(bt)) {
- offset += 32;
- }
- __ lea(rscratch1, ExternalAddress(StubRoutines::aarch64::vector_iota_indices() + offset));
+ __ lea(rscratch1, ExternalAddress(StubRoutines::aarch64::vector_iota_indices(entry_idx)));
if (length_in_bytes == 16) {
__ ldrq($dst$$FloatRegister, rscratch1);
} else {
@@ -3406,6 +3433,44 @@ instruct reduce_non_strict_order_add4F_neon(vRegF dst, vRegF fsrc, vReg vsrc, vR
ins_pipe(pipe_slow);
%}
+// Add Reduction for Half floats (FP16).
+// Neon does not provide direct instructions for strictly ordered floating-point add reductions.
+// On Neon-only targets (UseSVE = 0), this operation is implemented as a sequence of scalar additions:
+// values equal to the vector width are loaded into a vector register, each lane is extracted,
+// and its value is accumulated into the running sum, producing a final scalar result.
+instruct reduce_addHF_neon(vRegF dst, vRegF fsrc, vReg vsrc, vReg tmp) %{
+ predicate(UseSVE == 0);
+ match(Set dst (AddReductionVHF fsrc vsrc));
+ effect(TEMP_DEF dst, TEMP tmp);
+ format %{ "reduce_addHF $dst, $fsrc, $vsrc\t# 4HF/8HF. KILL $tmp" %}
+ ins_encode %{
+ uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
+ __ neon_reduce_add_fp16($dst$$FloatRegister, $fsrc$$FloatRegister,
+ $vsrc$$FloatRegister, length_in_bytes, $tmp$$FloatRegister);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+// This rule calculates the reduction result in strict order. Two cases will
+// reach here:
+// 1. Non strictly-ordered AddReductionVHF when vector size > 128-bits. For example -
+// AddReductionVHF generated by Vector API. For vector size > 128-bits, it is more
+// beneficial performance-wise to generate direct SVE instruction even if it is
+// strictly ordered.
+// 2. Strictly-ordered AddReductionVHF. For example - AddReductionVHF generated by
+// auto-vectorization on SVE machine.
+instruct reduce_addHF_sve(vRegF dst_src1, vReg src2) %{
+ predicate(UseSVE > 0);
+ match(Set dst_src1 (AddReductionVHF dst_src1 src2));
+ format %{ "reduce_addHF_sve $dst_src1, $dst_src1, $src2" %}
+ ins_encode %{
+ uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src2);
+ assert(length_in_bytes == MaxVectorSize, "invalid vector length");
+ __ sve_fadda($dst_src1$$FloatRegister, __ H, ptrue, $src2$$FloatRegister);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
// This rule calculates the reduction result in strict order. Two cases will
// reach here:
// 1. Non strictly-ordered AddReductionVF when vector size > 128-bits. For example -
@@ -3496,12 +3561,14 @@ instruct reduce_addL_masked(iRegLNoSp dst, iRegL isrc, vReg vsrc, pRegGov pg, vR
ins_pipe(pipe_slow);
%}
-instruct reduce_addF_masked(vRegF dst_src1, vReg src2, pRegGov pg) %{
+instruct reduce_addFHF_masked(vRegF dst_src1, vReg src2, pRegGov pg) %{
predicate(UseSVE > 0);
+ match(Set dst_src1 (AddReductionVHF (Binary dst_src1 src2) pg));
match(Set dst_src1 (AddReductionVF (Binary dst_src1 src2) pg));
- format %{ "reduce_addF_masked $dst_src1, $pg, $dst_src1, $src2" %}
+ format %{ "reduce_addFHF_masked $dst_src1, $pg, $dst_src1, $src2" %}
ins_encode %{
- __ sve_fadda($dst_src1$$FloatRegister, __ S,
+ BasicType bt = Matcher::vector_element_basic_type(this, $src2);
+ __ sve_fadda($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
$pg$$PRegister, $src2$$FloatRegister);
%}
ins_pipe(pipe_slow);
@@ -3549,14 +3616,17 @@ instruct reduce_mulL(iRegLNoSp dst, iRegL isrc, vReg vsrc) %{
ins_pipe(pipe_slow);
%}
-instruct reduce_mulF(vRegF dst, vRegF fsrc, vReg vsrc, vReg tmp) %{
+
+instruct reduce_mulFHF(vRegF dst, vRegF fsrc, vReg vsrc, vReg tmp) %{
predicate(Matcher::vector_length_in_bytes(n->in(2)) <= 16);
+ match(Set dst (MulReductionVHF fsrc vsrc));
match(Set dst (MulReductionVF fsrc vsrc));
effect(TEMP_DEF dst, TEMP tmp);
- format %{ "reduce_mulF $dst, $fsrc, $vsrc\t# 2F/4F. KILL $tmp" %}
+ format %{ "reduce_mulFHF $dst, $fsrc, $vsrc\t# 2F/4F/4HF/8HF. KILL $tmp" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
- __ neon_reduce_mul_fp($dst$$FloatRegister, T_FLOAT, $fsrc$$FloatRegister,
+ BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
+ __ neon_reduce_mul_fp($dst$$FloatRegister, bt, $fsrc$$FloatRegister,
$vsrc$$FloatRegister, length_in_bytes, $tmp$$FloatRegister);
%}
ins_pipe(pipe_slow);
diff --git a/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 b/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4
index 48bffb3cf35..58ed234194a 100644
--- a/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4
+++ b/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4
@@ -1,6 +1,6 @@
//
// Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
-// Copyright (c) 2020, 2025, Arm Limited. All rights reserved.
+// Copyright (c) 2020, 2026, Arm Limited. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
@@ -237,10 +237,39 @@ source %{
case Op_MinVHF:
case Op_MaxVHF:
case Op_SqrtVHF:
+ if (UseSVE == 0 && !is_feat_fp16_supported()) {
+ return false;
+ }
+ break;
+ // At the time of writing this, the Vector API has no half-float (FP16) species.
+ // Consequently, AddReductionVHF and MulReductionVHF are only produced by the
+ // auto-vectorizer, which requires strictly ordered semantics for FP reductions.
+ //
+ // There is no direct Neon instruction that performs strictly ordered floating
+ // point add reduction. Hence, on Neon only machines, the add reduction operation
+ // is implemented as a scalarized sequence using half-precision scalar instruction
+ // FADD which requires FEAT_FP16 and ASIMDHP to be available on the target.
+ // On SVE machines (UseSVE > 0) however, there is a direct instruction (FADDA) which
+ // implements strictly ordered floating point add reduction which does not require
+ // the FEAT_FP16 and ASIMDHP checks as SVE supports half-precision floats by default.
+ case Op_AddReductionVHF:
// FEAT_FP16 is enabled if both "fphp" and "asimdhp" features are supported.
// Only the Neon instructions need this check. SVE supports half-precision floats
// by default.
- if (UseSVE == 0 && !is_feat_fp16_supported()) {
+ if (length_in_bytes < 8 || (UseSVE == 0 && !is_feat_fp16_supported())) {
+ return false;
+ }
+ break;
+ case Op_MulReductionVHF:
+ // There are no direct Neon/SVE instructions that perform strictly ordered
+ // floating point multiply reduction.
+ // For vector length ≤ 16 bytes, the reduction is implemented as a scalarized
+ // sequence using half-precision scalar instruction FMUL. This path requires
+ // FEAT_FP16 and ASIMDHP to be available on the target.
+ // For vector length > 16 bytes, this operation is disabled because there is no
+ // direct SVE instruction that performs a strictly ordered FP16 multiply
+ // reduction.
+ if (length_in_bytes < 8 || length_in_bytes > 16 || !is_feat_fp16_supported()) {
return false;
}
break;
@@ -290,6 +319,7 @@ source %{
case Op_VectorRearrange:
case Op_MulReductionVD:
case Op_MulReductionVF:
+ case Op_MulReductionVHF:
case Op_MulReductionVI:
case Op_MulReductionVL:
case Op_CompressBitsV:
@@ -354,6 +384,7 @@ source %{
case Op_VectorMaskCmp:
case Op_LoadVectorGather:
case Op_StoreVectorScatter:
+ case Op_AddReductionVHF:
case Op_AddReductionVF:
case Op_AddReductionVD:
case Op_AndReductionV:
@@ -2063,6 +2094,25 @@ instruct reduce_non_strict_order_add4F_neon(vRegF dst, vRegF fsrc, vReg vsrc, vR
ins_pipe(pipe_slow);
%}
dnl
+
+// Add Reduction for Half floats (FP16).
+// Neon does not provide direct instructions for strictly ordered floating-point add reductions.
+// On Neon-only targets (UseSVE = 0), this operation is implemented as a sequence of scalar additions:
+// values equal to the vector width are loaded into a vector register, each lane is extracted,
+// and its value is accumulated into the running sum, producing a final scalar result.
+instruct reduce_addHF_neon(vRegF dst, vRegF fsrc, vReg vsrc, vReg tmp) %{
+ predicate(UseSVE == 0);
+ match(Set dst (AddReductionVHF fsrc vsrc));
+ effect(TEMP_DEF dst, TEMP tmp);
+ format %{ "reduce_addHF $dst, $fsrc, $vsrc\t# 4HF/8HF. KILL $tmp" %}
+ ins_encode %{
+ uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
+ __ neon_reduce_add_fp16($dst$$FloatRegister, $fsrc$$FloatRegister,
+ $vsrc$$FloatRegister, length_in_bytes, $tmp$$FloatRegister);
+ %}
+ ins_pipe(pipe_slow);
+%}
+dnl
dnl REDUCE_ADD_FP_SVE($1, $2 )
dnl REDUCE_ADD_FP_SVE(type, size)
define(`REDUCE_ADD_FP_SVE', `
@@ -2074,21 +2124,26 @@ define(`REDUCE_ADD_FP_SVE', `
// strictly ordered.
// 2. Strictly-ordered AddReductionV$1. For example - AddReductionV$1 generated by
// auto-vectorization on SVE machine.
-instruct reduce_add$1_sve(vReg$1 dst_src1, vReg src2) %{
- predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(2))) ||
- n->as_Reduction()->requires_strict_order());
+instruct reduce_add$1_sve(vReg`'ifelse($1, HF, F, $1) dst_src1, vReg src2) %{
+ ifelse($1, HF,
+ `predicate(UseSVE > 0);',
+ `predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(2))) ||
+ n->as_Reduction()->requires_strict_order());')
match(Set dst_src1 (AddReductionV$1 dst_src1 src2));
format %{ "reduce_add$1_sve $dst_src1, $dst_src1, $src2" %}
ins_encode %{
- assert(UseSVE > 0, "must be sve");
- uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src2);
+ ifelse($1, HF, `',
+ `assert(UseSVE > 0, "must be sve");
+ ')dnl
+uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src2);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
__ sve_fadda($dst_src1$$FloatRegister, __ $2, ptrue, $src2$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
-REDUCE_ADD_FP_SVE(F, S)
+REDUCE_ADD_FP_SVE(HF, H)
+REDUCE_ADD_FP_SVE(F, S)
// reduction addD
@@ -2129,21 +2184,30 @@ dnl
dnl REDUCE_ADD_FP_PREDICATE($1, $2 )
dnl REDUCE_ADD_FP_PREDICATE(insn_name, op_name)
define(`REDUCE_ADD_FP_PREDICATE', `
-instruct reduce_add$1_masked(vReg$1 dst_src1, vReg src2, pRegGov pg) %{
+instruct reduce_add$1_masked(vReg$2 dst_src1, vReg src2, pRegGov pg) %{
predicate(UseSVE > 0);
- match(Set dst_src1 (AddReductionV$1 (Binary dst_src1 src2) pg));
+ ifelse($2, F,
+ `match(Set dst_src1 (AddReductionVHF (Binary dst_src1 src2) pg));
+ match(Set dst_src1 (AddReductionV$2 (Binary dst_src1 src2) pg));',
+ `match(Set dst_src1 (AddReductionV$2 (Binary dst_src1 src2) pg));')
format %{ "reduce_add$1_masked $dst_src1, $pg, $dst_src1, $src2" %}
ins_encode %{
- __ sve_fadda($dst_src1$$FloatRegister, __ $2,
- $pg$$PRegister, $src2$$FloatRegister);
+ ifelse($2, F,
+ `BasicType bt = Matcher::vector_element_basic_type(this, $src2);
+ ',)dnl
+ifelse($2, F,
+ `__ sve_fadda($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
+ $pg$$PRegister, $src2$$FloatRegister);',
+ `__ sve_fadda($dst_src1$$FloatRegister, __ $2,
+ $pg$$PRegister, $src2$$FloatRegister);')
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
REDUCE_ADD_INT_PREDICATE(I, iRegIorL2I)
REDUCE_ADD_INT_PREDICATE(L, iRegL)
-REDUCE_ADD_FP_PREDICATE(F, S)
-REDUCE_ADD_FP_PREDICATE(D, D)
+REDUCE_ADD_FP_PREDICATE(FHF, F)
+REDUCE_ADD_FP_PREDICATE(D, D)
// ------------------------------ Vector reduction mul -------------------------
@@ -2176,30 +2240,37 @@ instruct reduce_mulL(iRegLNoSp dst, iRegL isrc, vReg vsrc) %{
ins_pipe(pipe_slow);
%}
-instruct reduce_mulF(vRegF dst, vRegF fsrc, vReg vsrc, vReg tmp) %{
- predicate(Matcher::vector_length_in_bytes(n->in(2)) <= 16);
- match(Set dst (MulReductionVF fsrc vsrc));
+dnl REDUCE_MUL_FP($1, $2 )
+dnl REDUCE_MUL_FP(insn_name, op_name)
+define(`REDUCE_MUL_FP', `
+instruct reduce_mul$1(vReg$2 dst, vReg$2 ifelse($2, F, fsrc, dsrc), vReg vsrc, vReg tmp) %{
+ predicate(Matcher::vector_length_in_bytes(n->in(2)) ifelse($2, F, <=, ==) 16);
+ ifelse($2, F,
+ `match(Set dst (MulReductionVHF fsrc vsrc));
+ match(Set dst (MulReductionV$2 fsrc vsrc));',
+ `match(Set dst (MulReductionV$2 dsrc vsrc));')
effect(TEMP_DEF dst, TEMP tmp);
- format %{ "reduce_mulF $dst, $fsrc, $vsrc\t# 2F/4F. KILL $tmp" %}
+ ifelse($2, F,
+ `format %{ "reduce_mul$1 $dst, $fsrc, $vsrc\t# 2F/4F/4HF/8HF. KILL $tmp" %}',
+ `format %{ "reduce_mul$1 $dst, $dsrc, $vsrc\t# 2D. KILL $tmp" %}')
ins_encode %{
- uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
- __ neon_reduce_mul_fp($dst$$FloatRegister, T_FLOAT, $fsrc$$FloatRegister,
- $vsrc$$FloatRegister, length_in_bytes, $tmp$$FloatRegister);
+ ifelse($2, F,
+ `uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
+ ',)dnl
+ifelse($2, F,
+ `BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
+ ',)dnl
+ifelse($2, F,
+ `__ neon_reduce_mul_fp($dst$$FloatRegister, bt, $fsrc$$FloatRegister,
+ $vsrc$$FloatRegister, length_in_bytes, $tmp$$FloatRegister);',
+ `__ neon_reduce_mul_fp($dst$$FloatRegister, T_DOUBLE, $dsrc$$FloatRegister,
+ $vsrc$$FloatRegister, 16, $tmp$$FloatRegister);')
%}
ins_pipe(pipe_slow);
-%}
-
-instruct reduce_mulD(vRegD dst, vRegD dsrc, vReg vsrc, vReg tmp) %{
- predicate(Matcher::vector_length_in_bytes(n->in(2)) == 16);
- match(Set dst (MulReductionVD dsrc vsrc));
- effect(TEMP_DEF dst, TEMP tmp);
- format %{ "reduce_mulD $dst, $dsrc, $vsrc\t# 2D. KILL $tmp" %}
- ins_encode %{
- __ neon_reduce_mul_fp($dst$$FloatRegister, T_DOUBLE, $dsrc$$FloatRegister,
- $vsrc$$FloatRegister, 16, $tmp$$FloatRegister);
- %}
- ins_pipe(pipe_slow);
-%}
+%}')dnl
+dnl
+REDUCE_MUL_FP(FHF, F)
+REDUCE_MUL_FP(D, D)
dnl
dnl REDUCE_BITWISE_OP_NEON($1, $2 $3 $4 )
diff --git a/src/hotspot/cpu/aarch64/assembler_aarch64.hpp b/src/hotspot/cpu/aarch64/assembler_aarch64.hpp
index 67cf77989d2..c8d5ee2eaeb 100644
--- a/src/hotspot/cpu/aarch64/assembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/assembler_aarch64.hpp
@@ -1,6 +1,7 @@
/*
* Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2024, Red Hat Inc. All rights reserved.
+ * Copyright 2026 Arm Limited and/or its affiliates.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1000,30 +1001,6 @@ public:
f(0b0101010, 31, 25), f(0, 24), sf(offset, 23, 5), f(0, 4), f(cond, 3, 0);
}
-#define INSN(NAME, cond) \
- void NAME(address dest) { \
- br(cond, dest); \
- }
-
- INSN(beq, EQ);
- INSN(bne, NE);
- INSN(bhs, HS);
- INSN(bcs, CS);
- INSN(blo, LO);
- INSN(bcc, CC);
- INSN(bmi, MI);
- INSN(bpl, PL);
- INSN(bvs, VS);
- INSN(bvc, VC);
- INSN(bhi, HI);
- INSN(bls, LS);
- INSN(bge, GE);
- INSN(blt, LT);
- INSN(bgt, GT);
- INSN(ble, LE);
- INSN(bal, AL);
- INSN(bnv, NV);
-
void br(Condition cc, Label &L);
#undef INSN
@@ -1095,6 +1072,10 @@ public:
#undef INSN
+ void wfet(Register rt) {
+ system(0b00, 0b011, 0b0001, 0b0000, 0b000, rt);
+ }
+
// we only provide mrs and msr for the special purpose system
// registers where op1 (instr[20:19]) == 11
// n.b msr has L (instr[21]) == 0 mrs has L == 1
@@ -1274,6 +1255,13 @@ public:
sz, 0b000, ordered);
}
+ void load_store_volatile(Register data, BasicType type, Register addr,
+ bool is_load) {
+ load_store_exclusive(dummy_reg, data, dummy_reg, addr,
+ (Assembler::operand_size)exact_log2(type2aelembytes(type)),
+ is_load ? 0b110 : 0b100, /* ordered = */ true);
+ }
+
#define INSN4(NAME, sz, op, o0) /* Four registers */ \
void NAME(Register Rs, Register Rt1, Register Rt2, Register Rn) { \
guarantee(Rs != Rn && Rs != Rt1 && Rs != Rt2, "unpredictable instruction"); \
diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
index 30048a2079d..4eb4e3d5ac7 100644
--- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
@@ -1,6 +1,7 @@
/*
- * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
+ * Copyright 2026 Arm Limited and/or its affiliates.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -42,6 +43,7 @@
#include "runtime/frame.inline.hpp"
#include "runtime/sharedRuntime.hpp"
#include "runtime/stubRoutines.hpp"
+#include "runtime/threadIdentifier.hpp"
#include "utilities/powerOfTwo.hpp"
#include "vmreg_aarch64.inline.hpp"
@@ -59,22 +61,6 @@ const Register SHIFT_count = r0; // where count for shift operations must be
#define __ _masm->
-static void select_different_registers(Register preserve,
- Register extra,
- Register &tmp1,
- Register &tmp2) {
- if (tmp1 == preserve) {
- assert_different_registers(tmp1, tmp2, extra);
- tmp1 = extra;
- } else if (tmp2 == preserve) {
- assert_different_registers(tmp1, tmp2, extra);
- tmp2 = extra;
- }
- assert_different_registers(preserve, tmp1, tmp2);
-}
-
-
-
static void select_different_registers(Register preserve,
Register extra,
Register &tmp1,
@@ -536,6 +522,10 @@ void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_cod
#if INCLUDE_CDS
if (AOTCodeCache::is_on_for_dump()) {
address b = c->as_pointer();
+ if (b == (address)ThreadIdentifier::unsafe_offset()) {
+ __ lea(dest->as_register_lo(), ExternalAddress(b));
+ break;
+ }
if (AOTRuntimeConstants::contains(b)) {
__ load_aotrc_address(dest->as_register_lo(), b);
break;
@@ -922,8 +912,15 @@ void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
reg2stack(temp, dest, dest->type());
}
+void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type,
+ LIR_PatchCode patch_code, CodeEmitInfo* info,
+ bool wide) {
+ mem2reg(src, dest, type, patch_code, info, wide, false);
+}
-void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
+void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type,
+ LIR_PatchCode patch_code, CodeEmitInfo* info,
+ bool wide, bool is_volatile) {
LIR_Address* addr = src->as_address_ptr();
LIR_Address* from_addr = src->as_address_ptr();
@@ -936,10 +933,27 @@ void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_Patch
return;
}
+ if (is_volatile) {
+ load_volatile(from_addr, dest, type, info);
+ } else {
+ load_unordered(from_addr, dest, type, wide, info);
+ }
+
+ if (is_reference_type(type)) {
+ if (UseCompressedOops && !wide) {
+ __ decode_heap_oop(dest->as_register());
+ }
+
+ __ verify_oop(dest->as_register());
+ }
+}
+
+void LIR_Assembler::load_unordered(LIR_Address *from_addr, LIR_Opr dest,
+ BasicType type, bool wide, CodeEmitInfo* info) {
if (info != nullptr) {
add_debug_info_for_null_check_here(info);
}
- int null_check_here = code_offset();
+
switch (type) {
case T_FLOAT: {
__ ldrs(dest->as_float_reg(), as_Address(from_addr));
@@ -997,16 +1011,44 @@ void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_Patch
default:
ShouldNotReachHere();
}
-
- if (is_reference_type(type)) {
- if (UseCompressedOops && !wide) {
- __ decode_heap_oop(dest->as_register());
- }
-
- __ verify_oop(dest->as_register());
- }
}
+void LIR_Assembler::load_volatile(LIR_Address *from_addr, LIR_Opr dest,
+ BasicType type, CodeEmitInfo* info) {
+ __ lea(rscratch1, as_Address(from_addr));
+
+ Register dest_reg = rscratch2;
+ if (!is_floating_point_type(type)) {
+ dest_reg = (dest->is_single_cpu()
+ ? dest->as_register() : dest->as_register_lo());
+ }
+
+ if (info != nullptr) {
+ add_debug_info_for_null_check_here(info);
+ }
+
+ // Uses LDAR to ensure memory ordering.
+ __ load_store_volatile(dest_reg, type, rscratch1, /*is_load*/true);
+
+ switch (type) {
+ // LDAR is unsigned so need to sign-extend for byte and short
+ case T_BYTE:
+ __ sxtb(dest_reg, dest_reg);
+ break;
+ case T_SHORT:
+ __ sxth(dest_reg, dest_reg);
+ break;
+ // need to move from GPR to FPR after LDAR with FMOV for floating types
+ case T_FLOAT:
+ __ fmovs(dest->as_float_reg(), dest_reg);
+ break;
+ case T_DOUBLE:
+ __ fmovd(dest->as_double_reg(), dest_reg);
+ break;
+ default:
+ break;
+ }
+}
int LIR_Assembler::array_element_size(BasicType type) const {
int elem_size = type2aelembytes(type);
@@ -1269,12 +1311,9 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, L
} else if (obj == klass_RInfo) {
klass_RInfo = dst;
}
- if (k->is_loaded() && !UseCompressedClassPointers) {
- select_different_registers(obj, dst, k_RInfo, klass_RInfo);
- } else {
- Rtmp1 = op->tmp3()->as_register();
- select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
- }
+
+ Rtmp1 = op->tmp3()->as_register();
+ select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
assert_different_registers(obj, k_RInfo, klass_RInfo);
@@ -2778,7 +2817,9 @@ void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* arg
}
void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
- if (dest->is_address() || src->is_address()) {
+ if (src->is_address()) {
+ mem2reg(src, dest, type, lir_patch_none, info, /*wide*/false, /*is_volatile*/true);
+ } else if (dest->is_address()) {
move_op(src, dest, type, lir_patch_none, info, /*wide*/false);
} else {
ShouldNotReachHere();
diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.hpp
index 5af06fc6a1c..367256d2f69 100644
--- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.hpp
@@ -57,6 +57,12 @@ friend class ArrayCopyStub;
void casw(Register addr, Register newval, Register cmpval);
void casl(Register addr, Register newval, Register cmpval);
+ void mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type,
+ LIR_PatchCode patch_code,
+ CodeEmitInfo* info, bool wide, bool is_volatile);
+ void load_unordered(LIR_Address *from_addr, LIR_Opr dest, BasicType type, bool wide, CodeEmitInfo* info);
+ void load_volatile(LIR_Address *from_addr, LIR_Opr dest, BasicType type, CodeEmitInfo* info);
+
static const int max_tableswitches = 20;
struct tableswitch switches[max_tableswitches];
int tableswitch_count;
diff --git a/src/hotspot/cpu/aarch64/c1_LIRGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRGenerator_aarch64.cpp
index ad26d494b2d..7e82f410a95 100644
--- a/src/hotspot/cpu/aarch64/c1_LIRGenerator_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c1_LIRGenerator_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -1287,9 +1287,7 @@ void LIRGenerator::do_CheckCast(CheckCast* x) {
}
LIR_Opr reg = rlock_result(x);
LIR_Opr tmp3 = LIR_OprFact::illegalOpr;
- if (!x->klass()->is_loaded() || UseCompressedClassPointers) {
- tmp3 = new_register(objectType);
- }
+ tmp3 = new_register(objectType);
__ checkcast(reg, obj.result(), x->klass(),
new_register(objectType), new_register(objectType), tmp3,
x->direct_compare(), info_for_exception, patching_info, stub,
@@ -1308,9 +1306,7 @@ void LIRGenerator::do_InstanceOf(InstanceOf* x) {
}
obj.load_item();
LIR_Opr tmp3 = LIR_OprFact::illegalOpr;
- if (!x->klass()->is_loaded() || UseCompressedClassPointers) {
- tmp3 = new_register(objectType);
- }
+ tmp3 = new_register(objectType);
__ instanceof(reg, obj.result(), x->klass(),
new_register(objectType), new_register(objectType), tmp3,
x->direct_compare(), patching_info, x->profiled_method(), x->profiled_bci());
@@ -1402,14 +1398,5 @@ void LIRGenerator::volatile_field_store(LIR_Opr value, LIR_Address* address,
void LIRGenerator::volatile_field_load(LIR_Address* address, LIR_Opr result,
CodeEmitInfo* info) {
- // 8179954: We need to make sure that the code generated for
- // volatile accesses forms a sequentially-consistent set of
- // operations when combined with STLR and LDAR. Without a leading
- // membar it's possible for a simple Dekker test to fail if loads
- // use LD;DMB but stores use STLR. This can happen if C2 compiles
- // the stores in one method and C1 compiles the loads in another.
- if (!CompilerConfig::is_c1_only_no_jvmci()) {
- __ membar();
- }
__ volatile_load_mem_reg(address, result, info);
}
diff --git a/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp
index e934632715c..89a9422ea48 100644
--- a/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -105,12 +105,8 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
} else {
mov(t1, checked_cast(markWord::prototype().value()));
str(t1, Address(obj, oopDesc::mark_offset_in_bytes()));
- if (UseCompressedClassPointers) { // Take care not to kill klass
- encode_klass_not_null(t1, klass);
- strw(t1, Address(obj, oopDesc::klass_offset_in_bytes()));
- } else {
- str(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
- }
+ encode_klass_not_null(t1, klass); // Take care not to kill klass
+ strw(t1, Address(obj, oopDesc::klass_offset_in_bytes()));
}
if (len->is_valid()) {
@@ -121,7 +117,7 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
// Clear gap/first 4 bytes following the length field.
strw(zr, Address(obj, base_offset));
}
- } else if (UseCompressedClassPointers && !UseCompactObjectHeaders) {
+ } else if (!UseCompactObjectHeaders) {
store_klass_gap(obj, zr);
}
}
diff --git a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
index 7aab7d389e1..3c179f21c14 100644
--- a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright 2026 Arm Limited and/or its affiliates.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1883,6 +1884,27 @@ void C2_MacroAssembler::neon_reduce_mul_fp(FloatRegister dst, BasicType bt,
BLOCK_COMMENT("neon_reduce_mul_fp {");
switch(bt) {
+ // The T_SHORT type below is for Float16 type which also uses floating-point
+ // instructions.
+ case T_SHORT:
+ fmulh(dst, fsrc, vsrc);
+ ext(vtmp, T8B, vsrc, vsrc, 2);
+ fmulh(dst, dst, vtmp);
+ ext(vtmp, T8B, vsrc, vsrc, 4);
+ fmulh(dst, dst, vtmp);
+ ext(vtmp, T8B, vsrc, vsrc, 6);
+ fmulh(dst, dst, vtmp);
+ if (isQ) {
+ ext(vtmp, T16B, vsrc, vsrc, 8);
+ fmulh(dst, dst, vtmp);
+ ext(vtmp, T16B, vsrc, vsrc, 10);
+ fmulh(dst, dst, vtmp);
+ ext(vtmp, T16B, vsrc, vsrc, 12);
+ fmulh(dst, dst, vtmp);
+ ext(vtmp, T16B, vsrc, vsrc, 14);
+ fmulh(dst, dst, vtmp);
+ }
+ break;
case T_FLOAT:
fmuls(dst, fsrc, vsrc);
ins(vtmp, S, vsrc, 0, 1);
@@ -1907,6 +1929,33 @@ void C2_MacroAssembler::neon_reduce_mul_fp(FloatRegister dst, BasicType bt,
BLOCK_COMMENT("} neon_reduce_mul_fp");
}
+// Vector reduction add for half float type with ASIMD instructions.
+void C2_MacroAssembler::neon_reduce_add_fp16(FloatRegister dst, FloatRegister fsrc, FloatRegister vsrc,
+ unsigned vector_length_in_bytes, FloatRegister vtmp) {
+ assert(vector_length_in_bytes == 8 || vector_length_in_bytes == 16, "unsupported");
+ bool isQ = vector_length_in_bytes == 16;
+
+ BLOCK_COMMENT("neon_reduce_add_fp16 {");
+ faddh(dst, fsrc, vsrc);
+ ext(vtmp, T8B, vsrc, vsrc, 2);
+ faddh(dst, dst, vtmp);
+ ext(vtmp, T8B, vsrc, vsrc, 4);
+ faddh(dst, dst, vtmp);
+ ext(vtmp, T8B, vsrc, vsrc, 6);
+ faddh(dst, dst, vtmp);
+ if (isQ) {
+ ext(vtmp, T16B, vsrc, vsrc, 8);
+ faddh(dst, dst, vtmp);
+ ext(vtmp, T16B, vsrc, vsrc, 10);
+ faddh(dst, dst, vtmp);
+ ext(vtmp, T16B, vsrc, vsrc, 12);
+ faddh(dst, dst, vtmp);
+ ext(vtmp, T16B, vsrc, vsrc, 14);
+ faddh(dst, dst, vtmp);
+ }
+ BLOCK_COMMENT("} neon_reduce_add_fp16");
+}
+
// Helper to select logical instruction
void C2_MacroAssembler::neon_reduce_logical_helper(int opc, bool is64, Register Rd,
Register Rn, Register Rm,
@@ -2414,17 +2463,17 @@ void C2_MacroAssembler::neon_rearrange_hsd(FloatRegister dst, FloatRegister src,
break;
case T_LONG:
case T_DOUBLE:
- // Load the iota indices for Long type. The indices are ordered by
- // type B/S/I/L/F/D, and the offset between two types is 16; Hence
- // the offset for L is 48.
- lea(rscratch1,
- ExternalAddress(StubRoutines::aarch64::vector_iota_indices() + 48));
- ldrq(tmp, rscratch1);
- // Check whether the input "shuffle" is the same with iota indices.
- // Return "src" if true, otherwise swap the two elements of "src".
- cm(EQ, dst, size2, shuffle, tmp);
- ext(tmp, size1, src, src, 8);
- bsl(dst, size1, src, tmp);
+ {
+ int idx = vector_iota_entry_index(T_LONG);
+ lea(rscratch1,
+ ExternalAddress(StubRoutines::aarch64::vector_iota_indices(idx)));
+ ldrq(tmp, rscratch1);
+ // Check whether the input "shuffle" is the same with iota indices.
+ // Return "src" if true, otherwise swap the two elements of "src".
+ cm(EQ, dst, size2, shuffle, tmp);
+ ext(tmp, size1, src, src, 8);
+ bsl(dst, size1, src, tmp);
+ }
break;
default:
assert(false, "unsupported element type");
@@ -2896,3 +2945,24 @@ void C2_MacroAssembler::sve_cpy(FloatRegister dst, SIMD_RegVariant T,
}
Assembler::sve_cpy(dst, T, pg, imm8, isMerge);
}
+
+int C2_MacroAssembler::vector_iota_entry_index(BasicType bt) {
+ // The vector iota entries array is ordered by type B/S/I/L/F/D, and
+ // the offset between two types is 16.
+ switch(bt) {
+ case T_BYTE:
+ return 0;
+ case T_SHORT:
+ return 1;
+ case T_INT:
+ return 2;
+ case T_LONG:
+ return 3;
+ case T_FLOAT:
+ return 4;
+ case T_DOUBLE:
+ return 5;
+ default:
+ ShouldNotReachHere();
+ }
+}
diff --git a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp
index 5c05832afbe..f96d3ffb863 100644
--- a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp
@@ -177,6 +177,9 @@
FloatRegister fsrc, FloatRegister vsrc,
unsigned vector_length_in_bytes, FloatRegister vtmp);
+ void neon_reduce_add_fp16(FloatRegister dst, FloatRegister fsrc, FloatRegister vsrc,
+ unsigned vector_length_in_bytes, FloatRegister vtmp);
+
void neon_reduce_logical(int opc, Register dst, BasicType bt, Register isrc,
FloatRegister vsrc, unsigned vector_length_in_bytes);
@@ -249,4 +252,5 @@
void sve_cpy(FloatRegister dst, SIMD_RegVariant T, PRegister pg, int imm8,
bool isMerge);
+ int vector_iota_entry_index(BasicType bt);
#endif // CPU_AARCH64_C2_MACROASSEMBLER_AARCH64_HPP
diff --git a/src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp b/src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp
index 6fe3315014b..640cd495383 100644
--- a/src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp
@@ -89,16 +89,21 @@ void CompiledDirectCall::set_to_interpreted(const methodHandle& callee, address
NativeMovConstReg* method_holder
= nativeMovConstReg_at(stub + NativeInstruction::instruction_size);
+ // In AOT "production" run we have mixture of AOTed and normal JITed code.
+ // Static call stub in AOTed nmethod always has far jump.
+ // Normal JITed nmethod may have short or far jump depending on distance.
+ // Determine actual jump instruction we have in code.
+ address next_instr = method_holder->next_instruction_address();
+ bool is_general_jump = nativeInstruction_at(next_instr)->is_general_jump();
+
#ifdef ASSERT
- NativeJump* jump = MacroAssembler::codestub_branch_needs_far_jump()
- ? nativeGeneralJump_at(method_holder->next_instruction_address())
- : nativeJump_at(method_holder->next_instruction_address());
+ NativeJump* jump = is_general_jump ? nativeGeneralJump_at(next_instr) : nativeJump_at(next_instr);
verify_mt_safe(callee, entry, method_holder, jump);
#endif
// Update stub.
method_holder->set_data((intptr_t)callee());
- MacroAssembler::pd_patch_instruction(method_holder->next_instruction_address(), entry);
+ MacroAssembler::pd_patch_instruction(next_instr, entry);
ICache::invalidate_range(stub, to_interp_stub_size());
// Update jump to call.
set_destination_mt_safe(stub);
diff --git a/src/hotspot/cpu/aarch64/gc/shared/cardTableBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shared/cardTableBarrierSetAssembler_aarch64.cpp
index 0bfc320179d..7ce4e0f8aed 100644
--- a/src/hotspot/cpu/aarch64/gc/shared/cardTableBarrierSetAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/gc/shared/cardTableBarrierSetAssembler_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -56,8 +56,10 @@ void CardTableBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet d
}
}
-void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj, Address dst) {
-
+void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2) {
+ precond(tmp1 != noreg);
+ precond(tmp2 != noreg);
+ assert_different_registers(obj, tmp1, tmp2);
BarrierSet* bs = BarrierSet::barrier_set();
assert(bs->kind() == BarrierSet::CardTableBarrierSet, "Wrong barrier set kind");
@@ -65,16 +67,16 @@ void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register ob
assert(CardTable::dirty_card_val() == 0, "must be");
- __ load_byte_map_base(rscratch1);
+ __ load_byte_map_base(tmp1);
if (UseCondCardMark) {
Label L_already_dirty;
- __ ldrb(rscratch2, Address(obj, rscratch1));
- __ cbz(rscratch2, L_already_dirty);
- __ strb(zr, Address(obj, rscratch1));
+ __ ldrb(tmp2, Address(obj, tmp1));
+ __ cbz(tmp2, L_already_dirty);
+ __ strb(zr, Address(obj, tmp1));
__ bind(L_already_dirty);
} else {
- __ strb(zr, Address(obj, rscratch1));
+ __ strb(zr, Address(obj, tmp1));
}
}
@@ -112,10 +114,10 @@ void CardTableBarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorS
if (needs_post_barrier) {
// flatten object address if needed
if (!precise || (dst.index() == noreg && dst.offset() == 0)) {
- store_check(masm, dst.base(), dst);
+ store_check(masm, dst.base(), tmp1, tmp2);
} else {
__ lea(tmp3, dst);
- store_check(masm, tmp3, dst);
+ store_check(masm, tmp3, tmp1, tmp2);
}
}
}
diff --git a/src/hotspot/cpu/aarch64/gc/shared/cardTableBarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/shared/cardTableBarrierSetAssembler_aarch64.hpp
index 07dd8eb5565..07016381f78 100644
--- a/src/hotspot/cpu/aarch64/gc/shared/cardTableBarrierSetAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/gc/shared/cardTableBarrierSetAssembler_aarch64.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -46,7 +46,7 @@ protected:
virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Address dst, Register val, Register tmp1, Register tmp2, Register tmp3);
- void store_check(MacroAssembler* masm, Register obj, Address dst);
+ void store_check(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2);
};
#endif // CPU_AARCH64_GC_SHARED_CARDTABLEBARRIERSETASSEMBLER_AARCH64_HPP
diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/c1/shenandoahBarrierSetC1_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shenandoah/c1/shenandoahBarrierSetC1_aarch64.cpp
index e4db8a9ab1f..e31a58243b5 100644
--- a/src/hotspot/cpu/aarch64/gc/shenandoah/c1/shenandoahBarrierSetC1_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/c1/shenandoahBarrierSetC1_aarch64.cpp
@@ -50,14 +50,10 @@ void LIR_OpShenandoahCompareAndSwap::emit_code(LIR_Assembler* masm) {
ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm->masm(), addr, cmpval, newval, /*acquire*/ true, /*release*/ true, /*is_cae*/ false, result);
- if (CompilerConfig::is_c1_only_no_jvmci()) {
- // The membar here is necessary to prevent reordering between the
- // release store in the CAS above and a subsequent volatile load.
- // However for tiered compilation C1 inserts a full barrier before
- // volatile loads which means we don't need an additional barrier
- // here (see LIRGenerator::volatile_field_load()).
- __ membar(__ AnyAny);
- }
+ // The membar here is necessary to prevent reordering between the
+ // release store in the CAS above and a subsequent volatile load.
+ // See also: LIR_Assembler::casw, LIR_Assembler::casl.
+ __ membar(__ AnyAny);
}
#undef __
diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp
index 4f0977a414f..f0885fee93d 100644
--- a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp
@@ -879,7 +879,9 @@ void ZBarrierSetAssembler::patch_barrier_relocation(address addr, int format) {
ShouldNotReachHere();
}
- ICache::invalidate_word((address)patch_addr);
+ if (!UseSingleICacheInvalidation) {
+ ICache::invalidate_word((address)patch_addr);
+ }
}
#ifdef COMPILER1
diff --git a/src/hotspot/cpu/aarch64/globals_aarch64.hpp b/src/hotspot/cpu/aarch64/globals_aarch64.hpp
index e6de2c798b1..dfeba73bede 100644
--- a/src/hotspot/cpu/aarch64/globals_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/globals_aarch64.hpp
@@ -115,18 +115,26 @@ define_pd_global(intx, InlineSmallCode, 1000);
"Value -1 means off.") \
range(-1, 4096) \
product(ccstr, OnSpinWaitInst, "yield", DIAGNOSTIC, \
- "The instruction to use to implement " \
- "java.lang.Thread.onSpinWait()." \
- "Valid values are: none, nop, isb, yield, sb.") \
+ "The instruction to use for java.lang.Thread.onSpinWait(). " \
+ "Valid values are: none, nop, isb, yield, sb, wfet.") \
constraint(OnSpinWaitInstNameConstraintFunc, AtParse) \
product(uint, OnSpinWaitInstCount, 1, DIAGNOSTIC, \
- "The number of OnSpinWaitInst instructions to generate." \
- "It cannot be used with OnSpinWaitInst=none.") \
+ "The number of OnSpinWaitInst instructions to generate. " \
+ "It cannot be used with OnSpinWaitInst=none. " \
+ "For OnSpinWaitInst=wfet it must be 1.") \
range(1, 99) \
+ product(uint, OnSpinWaitDelay, 40, DIAGNOSTIC, \
+ "The minimum delay (in nanoseconds) of the OnSpinWait loop. " \
+ "It can only be used with -XX:OnSpinWaitInst=wfet.") \
+ range(1, 1000) \
product(ccstr, UseBranchProtection, "none", \
"Branch Protection to use: none, standard, pac-ret") \
product(bool, AlwaysMergeDMB, true, DIAGNOSTIC, \
"Always merge DMB instructions in code emission") \
+ product(bool, NeoverseN1ICacheErratumMitigation, false, DIAGNOSTIC, \
+ "Enable workaround for Neoverse N1 erratum 1542419") \
+ product(bool, UseSingleICacheInvalidation, false, DIAGNOSTIC, \
+ "Defer multiple ICache invalidation to single invalidation") \
// end of ARCH_FLAGS
diff --git a/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp b/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp
index 2b506b241e0..980fedb406d 100644
--- a/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -989,26 +989,15 @@ void InterpreterMacroAssembler::profile_final_call(Register mdp) {
void InterpreterMacroAssembler::profile_virtual_call(Register receiver,
- Register mdp,
- bool receiver_can_be_null) {
+ Register mdp) {
if (ProfileInterpreter) {
Label profile_continue;
// If no method data exists, go to profile_continue.
test_method_data_pointer(mdp, profile_continue);
- Label skip_receiver_profile;
- if (receiver_can_be_null) {
- Label not_null;
- // We are making a call. Increment the count for null receiver.
- increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
- b(skip_receiver_profile);
- bind(not_null);
- }
-
// Record the receiver type.
profile_receiver_type(receiver, mdp, 0);
- bind(skip_receiver_profile);
// The method data pointer needs to be updated to reflect the new target.
update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size()));
diff --git a/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp b/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp
index 74d4430000d..9a074f1ce69 100644
--- a/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2015, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -285,8 +285,7 @@ class InterpreterMacroAssembler: public MacroAssembler {
void profile_not_taken_branch(Register mdp);
void profile_call(Register mdp);
void profile_final_call(Register mdp);
- void profile_virtual_call(Register receiver, Register mdp,
- bool receiver_can_be_null = false);
+ void profile_virtual_call(Register receiver, Register mdp);
void profile_ret(Register return_bci, Register mdp);
void profile_null_seen(Register mdp);
void profile_typecheck(Register mdp, Register klass);
diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
index 3e3e95be07e..7bec0a3c0ca 100644
--- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
@@ -55,6 +55,7 @@
#include "runtime/sharedRuntime.hpp"
#include "runtime/stubRoutines.hpp"
#include "utilities/globalDefinitions.hpp"
+#include "utilities/integerCast.hpp"
#include "utilities/powerOfTwo.hpp"
#ifdef COMPILER1
#include "c1/c1_LIRAssembler.hpp"
@@ -762,7 +763,7 @@ void MacroAssembler::call_VM_base(Register oop_result,
assert(java_thread == rthread, "unexpected register");
#ifdef ASSERT
// TraceBytecodes does not use r12 but saves it over the call, so don't verify
- // if ((UseCompressedOops || UseCompressedClassPointers) && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");
+ // if (!TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");
#endif // ASSERT
assert(java_thread != oop_result , "cannot use the same register for java_thread & oop_result");
@@ -952,7 +953,10 @@ void MacroAssembler::emit_static_call_stub() {
}
int MacroAssembler::static_call_stub_size() {
- if (!codestub_branch_needs_far_jump()) {
+ // During AOT production run AOT and JIT compiled code
+ // are used at the same time. We need this size
+ // to be the same for both types of code.
+ if (!codestub_branch_needs_far_jump() && !AOTCodeCache::is_on_for_use()) {
// isb; movk; movz; movz; b
return 5 * NativeInstruction::instruction_size;
}
@@ -1002,14 +1006,10 @@ int MacroAssembler::ic_check(int end_alignment) {
load_narrow_klass_compact(tmp1, receiver);
ldrw(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
cmpw(tmp1, tmp2);
- } else if (UseCompressedClassPointers) {
+ } else {
ldrw(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes()));
ldrw(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
cmpw(tmp1, tmp2);
- } else {
- ldr(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes()));
- ldr(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
- cmp(tmp1, tmp2);
}
Label dont;
@@ -2917,7 +2917,11 @@ void MacroAssembler::increment(Address dst, int value)
// Push lots of registers in the bit set supplied. Don't push sp.
// Return the number of words pushed
-int MacroAssembler::push(unsigned int bitset, Register stack) {
+int MacroAssembler::push(RegSet regset, Register stack) {
+ if (regset.bits() == 0) {
+ return 0;
+ }
+ auto bitset = integer_cast(regset.bits());
int words_pushed = 0;
// Scan bitset to accumulate register pairs
@@ -2947,7 +2951,11 @@ int MacroAssembler::push(unsigned int bitset, Register stack) {
return count;
}
-int MacroAssembler::pop(unsigned int bitset, Register stack) {
+int MacroAssembler::pop(RegSet regset, Register stack) {
+ if (regset.bits() == 0) {
+ return 0;
+ }
+ auto bitset = integer_cast(regset.bits());
int words_pushed = 0;
// Scan bitset to accumulate register pairs
@@ -2979,7 +2987,11 @@ int MacroAssembler::pop(unsigned int bitset, Register stack) {
// Push lots of registers in the bit set supplied. Don't push sp.
// Return the number of dwords pushed
-int MacroAssembler::push_fp(unsigned int bitset, Register stack, FpPushPopMode mode) {
+int MacroAssembler::push_fp(FloatRegSet regset, Register stack, FpPushPopMode mode) {
+ if (regset.bits() == 0) {
+ return 0;
+ }
+ auto bitset = integer_cast(regset.bits());
int words_pushed = 0;
bool use_sve = false;
int sve_vector_size_in_bytes = 0;
@@ -3092,7 +3104,11 @@ int MacroAssembler::push_fp(unsigned int bitset, Register stack, FpPushPopMode m
}
// Return the number of dwords popped
-int MacroAssembler::pop_fp(unsigned int bitset, Register stack, FpPushPopMode mode) {
+int MacroAssembler::pop_fp(FloatRegSet regset, Register stack, FpPushPopMode mode) {
+ if (regset.bits() == 0) {
+ return 0;
+ }
+ auto bitset = integer_cast(regset.bits());
int words_pushed = 0;
bool use_sve = false;
int sve_vector_size_in_bytes = 0;
@@ -3202,7 +3218,11 @@ int MacroAssembler::pop_fp(unsigned int bitset, Register stack, FpPushPopMode mo
}
// Return the number of dwords pushed
-int MacroAssembler::push_p(unsigned int bitset, Register stack) {
+int MacroAssembler::push_p(PRegSet regset, Register stack) {
+ if (regset.bits() == 0) {
+ return 0;
+ }
+ auto bitset = integer_cast(regset.bits());
bool use_sve = false;
int sve_predicate_size_in_slots = 0;
@@ -3239,7 +3259,11 @@ int MacroAssembler::push_p(unsigned int bitset, Register stack) {
}
// Return the number of dwords popped
-int MacroAssembler::pop_p(unsigned int bitset, Register stack) {
+int MacroAssembler::pop_p(PRegSet regset, Register stack) {
+ if (regset.bits() == 0) {
+ return 0;
+ }
+ auto bitset = integer_cast(regset.bits());
bool use_sve = false;
int sve_predicate_size_in_slots = 0;
@@ -3278,7 +3302,6 @@ int MacroAssembler::pop_p(unsigned int bitset, Register stack) {
#ifdef ASSERT
void MacroAssembler::verify_heapbase(const char* msg) {
#if 0
- assert (UseCompressedOops || UseCompressedClassPointers, "should be compressed");
assert (Universe::heap() != nullptr, "java heap should be initialized");
if (!UseCompressedOops || Universe::ptr_base() == nullptr) {
// rheapbase is allocated as general register
@@ -3456,7 +3479,7 @@ void MacroAssembler::subw(Register Rd, Register Rn, RegisterOrConstant decrement
void MacroAssembler::reinit_heapbase()
{
if (UseCompressedOops) {
- if (Universe::is_fully_initialized()) {
+ if (Universe::is_fully_initialized() && !AOTCodeCache::is_on_for_dump()) {
mov(rheapbase, CompressedOops::base());
} else {
lea(rheapbase, ExternalAddress(CompressedOops::base_addr()));
@@ -5067,13 +5090,10 @@ void MacroAssembler::load_narrow_klass_compact(Register dst, Register src) {
void MacroAssembler::load_klass(Register dst, Register src) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(dst, src);
- decode_klass_not_null(dst);
- } else if (UseCompressedClassPointers) {
- ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
- decode_klass_not_null(dst);
} else {
- ldr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
+ ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
}
+ decode_klass_not_null(dst);
}
void MacroAssembler::restore_cpu_control_state_after_jni(Register tmp1, Register tmp2) {
@@ -5125,25 +5145,22 @@ void MacroAssembler::load_mirror(Register dst, Register method, Register tmp1, R
void MacroAssembler::cmp_klass(Register obj, Register klass, Register tmp) {
assert_different_registers(obj, klass, tmp);
- if (UseCompressedClassPointers) {
- if (UseCompactObjectHeaders) {
- load_narrow_klass_compact(tmp, obj);
- } else {
- ldrw(tmp, Address(obj, oopDesc::klass_offset_in_bytes()));
- }
- if (CompressedKlassPointers::base() == nullptr) {
- cmp(klass, tmp, LSL, CompressedKlassPointers::shift());
- return;
- } else if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0
- && CompressedKlassPointers::shift() == 0) {
- // Only the bottom 32 bits matter
- cmpw(klass, tmp);
- return;
- }
- decode_klass_not_null(tmp);
+ if (UseCompactObjectHeaders) {
+ load_narrow_klass_compact(tmp, obj);
} else {
- ldr(tmp, Address(obj, oopDesc::klass_offset_in_bytes()));
+ ldrw(tmp, Address(obj, oopDesc::klass_offset_in_bytes()));
}
+ if (CompressedKlassPointers::base() == nullptr) {
+ cmp(klass, tmp, LSL, CompressedKlassPointers::shift());
+ return;
+ } else if (!AOTCodeCache::is_on_for_dump() &&
+ ((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0
+ && CompressedKlassPointers::shift() == 0) {
+ // Only the bottom 32 bits matter
+ cmpw(klass, tmp);
+ return;
+ }
+ decode_klass_not_null(tmp);
cmp(klass, tmp);
}
@@ -5151,36 +5168,25 @@ void MacroAssembler::cmp_klasses_from_objects(Register obj1, Register obj2, Regi
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(tmp1, obj1);
load_narrow_klass_compact(tmp2, obj2);
- cmpw(tmp1, tmp2);
- } else if (UseCompressedClassPointers) {
+ } else {
ldrw(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
ldrw(tmp2, Address(obj2, oopDesc::klass_offset_in_bytes()));
- cmpw(tmp1, tmp2);
- } else {
- ldr(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
- ldr(tmp2, Address(obj2, oopDesc::klass_offset_in_bytes()));
- cmp(tmp1, tmp2);
}
+ cmpw(tmp1, tmp2);
}
void MacroAssembler::store_klass(Register dst, Register src) {
// FIXME: Should this be a store release? concurrent gcs assumes
// klass length is valid if klass field is not null.
assert(!UseCompactObjectHeaders, "not with compact headers");
- if (UseCompressedClassPointers) {
- encode_klass_not_null(src);
- strw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
- } else {
- str(src, Address(dst, oopDesc::klass_offset_in_bytes()));
- }
+ encode_klass_not_null(src);
+ strw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
}
void MacroAssembler::store_klass_gap(Register dst, Register src) {
assert(!UseCompactObjectHeaders, "not with compact headers");
- if (UseCompressedClassPointers) {
- // Store to klass gap in destination
- strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
- }
+ // Store to klass gap in destination
+ strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
}
// Algorithm must match CompressedOops::encode.
@@ -5326,8 +5332,6 @@ MacroAssembler::KlassDecodeMode MacroAssembler::klass_decode_mode() {
}
MacroAssembler::KlassDecodeMode MacroAssembler::klass_decode_mode(address base, int shift, const size_t range) {
- assert(UseCompressedClassPointers, "not using compressed class pointers");
-
// KlassDecodeMode shouldn't be set already.
assert(_klass_decode_mode == KlassDecodeNone, "set once");
@@ -5393,7 +5397,7 @@ void MacroAssembler::encode_klass_not_null_for_aot(Register dst, Register src) {
}
void MacroAssembler::encode_klass_not_null(Register dst, Register src) {
- if (AOTCodeCache::is_on_for_dump()) {
+ if (CompressedKlassPointers::base() != nullptr && AOTCodeCache::is_on_for_dump()) {
encode_klass_not_null_for_aot(dst, src);
return;
}
@@ -5457,8 +5461,6 @@ void MacroAssembler::decode_klass_not_null_for_aot(Register dst, Register src) {
}
void MacroAssembler::decode_klass_not_null(Register dst, Register src) {
- assert (UseCompressedClassPointers, "should only be used for compressed headers");
-
if (AOTCodeCache::is_on_for_dump()) {
decode_klass_not_null_for_aot(dst, src);
return;
@@ -5525,7 +5527,6 @@ void MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
}
void MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
- assert (UseCompressedClassPointers, "should only be used for compressed headers");
assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
int index = oop_recorder()->find_index(k);
@@ -6835,6 +6836,9 @@ void MacroAssembler::spin_wait() {
assert(VM_Version::supports_sb(), "current CPU does not support SB instruction");
sb();
break;
+ case SpinWait::WFET:
+ spin_wait_wfet(VM_Version::spin_wait_desc().delay());
+ break;
default:
ShouldNotReachHere();
}
@@ -6842,6 +6846,28 @@ void MacroAssembler::spin_wait() {
block_comment("}");
}
+void MacroAssembler::spin_wait_wfet(int delay_ns) {
+ // The sequence assumes CNTFRQ_EL0 is fixed to 1GHz. The assumption is valid
+ // starting from Armv8.6, according to the "D12.1.2 The system counter" of the
+ // Arm Architecture Reference Manual for A-profile architecture version M.a.a.
+ // This is sufficient because FEAT_WFXT is introduced from Armv8.6.
+ Register target = rscratch1;
+ Register current = rscratch2;
+ get_cntvctss_el0(current);
+ add(target, current, delay_ns);
+
+ Label L_wait_loop;
+ bind(L_wait_loop);
+
+ wfet(target);
+ get_cntvctss_el0(current);
+
+ cmp(current, target);
+ br(LT, L_wait_loop);
+
+ sb();
+}
+
// Stack frame creation/removal
void MacroAssembler::enter(bool strip_ret_addr) {
diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
index fa32f3055b9..a6cc862d05c 100644
--- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
@@ -499,29 +499,20 @@ private:
void mov_immediate64(Register dst, uint64_t imm64);
void mov_immediate32(Register dst, uint32_t imm32);
- int push(unsigned int bitset, Register stack);
- int pop(unsigned int bitset, Register stack);
-
- int push_fp(unsigned int bitset, Register stack, FpPushPopMode mode);
- int pop_fp(unsigned int bitset, Register stack, FpPushPopMode mode);
-
- int push_p(unsigned int bitset, Register stack);
- int pop_p(unsigned int bitset, Register stack);
-
void mov(Register dst, Address a);
public:
- void push(RegSet regs, Register stack) { if (regs.bits()) push(regs.bits(), stack); }
- void pop(RegSet regs, Register stack) { if (regs.bits()) pop(regs.bits(), stack); }
+ int push(RegSet regset, Register stack);
+ int pop(RegSet regset, Register stack);
- void push_fp(FloatRegSet regs, Register stack, FpPushPopMode mode = PushPopFull) { if (regs.bits()) push_fp(regs.bits(), stack, mode); }
- void pop_fp(FloatRegSet regs, Register stack, FpPushPopMode mode = PushPopFull) { if (regs.bits()) pop_fp(regs.bits(), stack, mode); }
+ int push_fp(FloatRegSet regset, Register stack, FpPushPopMode mode = PushPopFull);
+ int pop_fp(FloatRegSet regset, Register stack, FpPushPopMode mode = PushPopFull);
static RegSet call_clobbered_gp_registers();
- void push_p(PRegSet regs, Register stack) { if (regs.bits()) push_p(regs.bits(), stack); }
- void pop_p(PRegSet regs, Register stack) { if (regs.bits()) pop_p(regs.bits(), stack); }
+ int push_p(PRegSet regset, Register stack);
+ int pop_p(PRegSet regset, Register stack);
// Push and pop everything that might be clobbered by a native
// runtime call except rscratch1 and rscratch2. (They are always
@@ -660,6 +651,14 @@ public:
msr(0b011, 0b0100, 0b0010, 0b000, reg);
}
+ // CNTVCTSS_EL0: op1 == 011
+ // CRn == 1110
+ // CRm == 0000
+ // op2 == 110
+ inline void get_cntvctss_el0(Register reg) {
+ mrs(0b011, 0b1110, 0b0000, 0b110, reg);
+ }
+
// idiv variant which deals with MINLONG as dividend and -1 as divisor
int corrected_idivl(Register result, Register ra, Register rb,
bool want_remainder, Register tmp = rscratch1);
@@ -891,10 +890,6 @@ public:
// thread in the default location (rthread)
void reset_last_Java_frame(bool clear_fp);
- // Stores
- void store_check(Register obj); // store check for obj - register is destroyed afterwards
- void store_check(Register obj, Address dst); // same as above, dst is exact store location (reg. is destroyed)
-
void resolve_jobject(Register value, Register tmp1, Register tmp2);
void resolve_global_jobject(Register value, Register tmp1, Register tmp2);
@@ -1724,6 +1719,7 @@ public:
// Code for java.lang.Thread::onSpinWait() intrinsic.
void spin_wait();
+ void spin_wait_wfet(int delay_ns);
void fast_lock(Register basic_lock, Register obj, Register t1, Register t2, Register t3, Label& slow);
void fast_unlock(Register obj, Register t1, Register t2, Register t3, Label& slow);
diff --git a/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp b/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp
index fc7274714ad..ab9896fa426 100644
--- a/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp
@@ -97,7 +97,7 @@ protected:
#define MACOS_WX_WRITE MACOS_AARCH64_ONLY(os::thread_wx_enable_write())
void set_char_at(int offset, char c) { MACOS_WX_WRITE; *addr_at(offset) = (u_char)c; }
void set_int_at(int offset, jint i) { MACOS_WX_WRITE; *(jint*)addr_at(offset) = i; }
- void set_uint_at(int offset, jint i) { MACOS_WX_WRITE; *(juint*)addr_at(offset) = i; }
+ void set_uint_at(int offset, juint i) { MACOS_WX_WRITE; *(juint*)addr_at(offset) = i; }
void set_ptr_at(int offset, address ptr) { MACOS_WX_WRITE; *(address*)addr_at(offset) = ptr; }
void set_oop_at(int offset, oop o) { MACOS_WX_WRITE; *(oop*)addr_at(offset) = o; }
#undef MACOS_WX_WRITE
@@ -178,13 +178,11 @@ public:
address destination() const;
void set_destination(address dest) {
- int offset = dest - instruction_address();
- unsigned int insn = 0b100101 << 26;
+ int64_t offset = dest - instruction_address();
+ juint insn = 0b100101u << 26u;
assert((offset & 3) == 0, "should be");
- offset >>= 2;
- offset &= (1 << 26) - 1; // mask off insn part
- insn |= offset;
- set_int_at(displacement_offset, insn);
+ Instruction_aarch64::spatch(reinterpret_cast(&insn), 25, 0, offset >> 2);
+ set_uint_at(displacement_offset, insn);
}
void verify_alignment() { ; }
diff --git a/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp b/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp
index dbec2d76d4f..f1b9fb213a2 100644
--- a/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp
@@ -54,7 +54,12 @@ void Relocation::pd_set_data_value(address x, bool verify_only) {
bytes = MacroAssembler::pd_patch_instruction_size(addr(), x);
break;
}
- ICache::invalidate_range(addr(), bytes);
+
+ if (UseSingleICacheInvalidation) {
+ assert(_binding != nullptr, "expect to be called with RelocIterator in use");
+ } else {
+ ICache::invalidate_range(addr(), bytes);
+ }
}
address Relocation::pd_call_destination(address orig_addr) {
diff --git a/src/hotspot/cpu/aarch64/runtime_aarch64.cpp b/src/hotspot/cpu/aarch64/runtime_aarch64.cpp
index e36aa21b567..638e57b03fe 100644
--- a/src/hotspot/cpu/aarch64/runtime_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/runtime_aarch64.cpp
@@ -290,7 +290,7 @@ ExceptionBlob* OptoRuntime::generate_exception_blob() {
assert(SimpleRuntimeFrame::framesize % 4 == 0, "sp not 16-byte aligned");
const char* name = OptoRuntime::stub_name(StubId::c2_exception_id);
- CodeBlob* blob = AOTCodeCache::load_code_blob(AOTCodeEntry::C2Blob, (uint)BlobId::c2_exception_id, name);
+ CodeBlob* blob = AOTCodeCache::load_code_blob(AOTCodeEntry::C2Blob, BlobId::c2_exception_id);
if (blob != nullptr) {
return blob->as_exception_blob();
}
diff --git a/src/hotspot/cpu/aarch64/spin_wait_aarch64.cpp b/src/hotspot/cpu/aarch64/spin_wait_aarch64.cpp
index 7da0151d834..97a981ab815 100644
--- a/src/hotspot/cpu/aarch64/spin_wait_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/spin_wait_aarch64.cpp
@@ -32,6 +32,7 @@ bool SpinWait::supports(const char *name) {
strcmp(name, "isb") == 0 ||
strcmp(name, "yield") == 0 ||
strcmp(name, "sb") == 0 ||
+ strcmp(name, "wfet") == 0 ||
strcmp(name, "none") == 0);
}
@@ -46,6 +47,8 @@ SpinWait::Inst SpinWait::from_name(const char* name) {
return SpinWait::YIELD;
} else if (strcmp(name, "sb") == 0) {
return SpinWait::SB;
+ } else if (strcmp(name, "wfet") == 0) {
+ return SpinWait::WFET;
}
return SpinWait::NONE;
diff --git a/src/hotspot/cpu/aarch64/spin_wait_aarch64.hpp b/src/hotspot/cpu/aarch64/spin_wait_aarch64.hpp
index 0e96a4b7157..6ebcd2477a8 100644
--- a/src/hotspot/cpu/aarch64/spin_wait_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/spin_wait_aarch64.hpp
@@ -24,6 +24,8 @@
#ifndef CPU_AARCH64_SPIN_WAIT_AARCH64_HPP
#define CPU_AARCH64_SPIN_WAIT_AARCH64_HPP
+#include "utilities/debug.hpp"
+
class SpinWait {
public:
enum Inst {
@@ -31,21 +33,30 @@ public:
NOP,
ISB,
YIELD,
- SB
+ SB,
+ WFET
};
private:
Inst _inst;
int _count;
+ int _delay;
Inst from_name(const char *name);
public:
- SpinWait(Inst inst = NONE, int count = 0) : _inst(inst), _count(inst == NONE ? 0 : count) {}
- SpinWait(const char *name, int count) : SpinWait(from_name(name), count) {}
+ SpinWait(Inst inst = NONE, int count = 0, int delay = -1)
+ : _inst(inst), _count(inst == NONE ? 0 : count), _delay(delay) {}
+ SpinWait(const char *name, int count, int delay)
+ : SpinWait(from_name(name), count, delay) {}
Inst inst() const { return _inst; }
int inst_count() const { return _count; }
+ int delay() const {
+ assert(_inst == WFET, "Specifying the delay value is only supported for WFET");
+ assert(_delay > 0, "The delay value must be positive");
+ return _delay;
+ }
static bool supports(const char *name);
};
diff --git a/src/hotspot/cpu/aarch64/stubDeclarations_aarch64.hpp b/src/hotspot/cpu/aarch64/stubDeclarations_aarch64.hpp
index 695534604b8..d1f59e479db 100644
--- a/src/hotspot/cpu/aarch64/stubDeclarations_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/stubDeclarations_aarch64.hpp
@@ -29,32 +29,39 @@
#define STUBGEN_PREUNIVERSE_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(preuniverse, 0) \
#define STUBGEN_INITIAL_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(initial, 10000) \
#define STUBGEN_CONTINUATION_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(continuation, 2000) \
+// count needed for declaration of vector_iota_indices stub
+#define VECTOR_IOTA_COUNT 6
#define STUBGEN_COMPILER_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(compiler, 70000) \
do_stub(compiler, vector_iota_indices) \
- do_arch_entry(aarch64, compiler, vector_iota_indices, \
- vector_iota_indices, vector_iota_indices) \
+ do_arch_entry_array(aarch64, compiler, vector_iota_indices, \
+ vector_iota_indices, vector_iota_indices, \
+ VECTOR_IOTA_COUNT) \
do_stub(compiler, large_array_equals) \
do_arch_entry(aarch64, compiler, large_array_equals, \
large_array_equals, large_array_equals) \
@@ -84,8 +91,7 @@
do_stub(compiler, count_positives) \
do_arch_entry(aarch64, compiler, count_positives, count_positives, \
count_positives) \
- do_stub(compiler, count_positives_long) \
- do_arch_entry(aarch64, compiler, count_positives_long, \
+ do_arch_entry(aarch64, compiler, count_positives, \
count_positives_long, count_positives_long) \
do_stub(compiler, compare_long_string_LL) \
do_arch_entry(aarch64, compiler, compare_long_string_LL, \
@@ -108,14 +114,16 @@
do_stub(compiler, string_indexof_linear_ul) \
do_arch_entry(aarch64, compiler, string_indexof_linear_ul, \
string_indexof_linear_ul, string_indexof_linear_ul) \
- /* this uses the entry for ghash_processBlocks */ \
- do_stub(compiler, ghash_processBlocks_wide) \
+ do_stub(compiler, ghash_processBlocks_small) \
+ do_arch_entry(aarch64, compiler, ghash_processBlocks_small, \
+ ghash_processBlocks_small, ghash_processBlocks_small) \
#define STUBGEN_FINAL_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(final, 20000 ZGC_ONLY(+85000)) \
do_stub(final, copy_byte_f) \
do_arch_entry(aarch64, final, copy_byte_f, copy_byte_f, \
@@ -139,9 +147,49 @@
do_stub(final, spin_wait) \
do_arch_entry_init(aarch64, final, spin_wait, spin_wait, \
spin_wait, empty_spin_wait) \
- /* stub only -- entries are not stored in StubRoutines::aarch64 */ \
/* n.b. these are not the same as the generic atomic stubs */ \
do_stub(final, atomic_entry_points) \
+ do_arch_entry(aarch64, final, atomic_entry_points, \
+ atomic_fetch_add_4_impl, atomic_fetch_add_4_impl) \
+ do_arch_entry(aarch64, final, atomic_entry_points, \
+ atomic_fetch_add_8_impl, atomic_fetch_add_8_impl) \
+ do_arch_entry(aarch64, final, atomic_entry_points, \
+ atomic_fetch_add_4_relaxed_impl, \
+ atomic_fetch_add_4_relaxed_impl) \
+ do_arch_entry(aarch64, final, atomic_entry_points, \
+ atomic_fetch_add_8_relaxed_impl, \
+ atomic_fetch_add_8_relaxed_impl) \
+ do_arch_entry(aarch64, final, atomic_entry_points, \
+ atomic_xchg_4_impl, atomic_xchg_4_impl) \
+ do_arch_entry(aarch64, final, atomic_entry_points, \
+ atomic_xchg_8_impl, atomic_xchg_8_impl) \
+ do_arch_entry(aarch64, final, atomic_entry_points, \
+ atomic_cmpxchg_1_impl, atomic_cmpxchg_1_impl) \
+ do_arch_entry(aarch64, final, atomic_entry_points, \
+ atomic_cmpxchg_4_impl, atomic_cmpxchg_4_impl) \
+ do_arch_entry(aarch64, final, atomic_entry_points, \
+ atomic_cmpxchg_8_impl, atomic_cmpxchg_8_impl) \
+ do_arch_entry(aarch64, final, atomic_entry_points, \
+ atomic_cmpxchg_1_relaxed_impl, \
+ atomic_cmpxchg_1_relaxed_impl) \
+ do_arch_entry(aarch64, final, atomic_entry_points, \
+ atomic_cmpxchg_4_relaxed_impl, \
+ atomic_cmpxchg_4_relaxed_impl) \
+ do_arch_entry(aarch64, final, atomic_entry_points, \
+ atomic_cmpxchg_8_relaxed_impl, \
+ atomic_cmpxchg_8_relaxed_impl) \
+ do_arch_entry(aarch64, final, atomic_entry_points, \
+ atomic_cmpxchg_4_release_impl, \
+ atomic_cmpxchg_4_release_impl) \
+ do_arch_entry(aarch64, final, atomic_entry_points, \
+ atomic_cmpxchg_8_release_impl, \
+ atomic_cmpxchg_8_release_impl) \
+ do_arch_entry(aarch64, final, atomic_entry_points, \
+ atomic_cmpxchg_4_seq_cst_impl, \
+ atomic_cmpxchg_4_seq_cst_impl) \
+ do_arch_entry(aarch64, final, atomic_entry_points, \
+ atomic_cmpxchg_8_seq_cst_impl, \
+ atomic_cmpxchg_8_seq_cst_impl) \
#endif // CPU_AARCH64_STUBDECLARATIONS_HPP
diff --git a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp
index 21a1124a8ec..fddb37b7b8d 100644
--- a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp
@@ -79,6 +79,166 @@
#define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
+// Constant data definitions
+
+static const uint32_t _sha256_round_consts[64] = {
+ 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5,
+ 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
+ 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
+ 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
+ 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc,
+ 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
+ 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
+ 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
+ 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
+ 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
+ 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3,
+ 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
+ 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5,
+ 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
+ 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
+ 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2,
+};
+
+static const uint64_t _sha512_round_consts[80] = {
+ 0x428A2F98D728AE22L, 0x7137449123EF65CDL, 0xB5C0FBCFEC4D3B2FL,
+ 0xE9B5DBA58189DBBCL, 0x3956C25BF348B538L, 0x59F111F1B605D019L,
+ 0x923F82A4AF194F9BL, 0xAB1C5ED5DA6D8118L, 0xD807AA98A3030242L,
+ 0x12835B0145706FBEL, 0x243185BE4EE4B28CL, 0x550C7DC3D5FFB4E2L,
+ 0x72BE5D74F27B896FL, 0x80DEB1FE3B1696B1L, 0x9BDC06A725C71235L,
+ 0xC19BF174CF692694L, 0xE49B69C19EF14AD2L, 0xEFBE4786384F25E3L,
+ 0x0FC19DC68B8CD5B5L, 0x240CA1CC77AC9C65L, 0x2DE92C6F592B0275L,
+ 0x4A7484AA6EA6E483L, 0x5CB0A9DCBD41FBD4L, 0x76F988DA831153B5L,
+ 0x983E5152EE66DFABL, 0xA831C66D2DB43210L, 0xB00327C898FB213FL,
+ 0xBF597FC7BEEF0EE4L, 0xC6E00BF33DA88FC2L, 0xD5A79147930AA725L,
+ 0x06CA6351E003826FL, 0x142929670A0E6E70L, 0x27B70A8546D22FFCL,
+ 0x2E1B21385C26C926L, 0x4D2C6DFC5AC42AEDL, 0x53380D139D95B3DFL,
+ 0x650A73548BAF63DEL, 0x766A0ABB3C77B2A8L, 0x81C2C92E47EDAEE6L,
+ 0x92722C851482353BL, 0xA2BFE8A14CF10364L, 0xA81A664BBC423001L,
+ 0xC24B8B70D0F89791L, 0xC76C51A30654BE30L, 0xD192E819D6EF5218L,
+ 0xD69906245565A910L, 0xF40E35855771202AL, 0x106AA07032BBD1B8L,
+ 0x19A4C116B8D2D0C8L, 0x1E376C085141AB53L, 0x2748774CDF8EEB99L,
+ 0x34B0BCB5E19B48A8L, 0x391C0CB3C5C95A63L, 0x4ED8AA4AE3418ACBL,
+ 0x5B9CCA4F7763E373L, 0x682E6FF3D6B2B8A3L, 0x748F82EE5DEFB2FCL,
+ 0x78A5636F43172F60L, 0x84C87814A1F0AB72L, 0x8CC702081A6439ECL,
+ 0x90BEFFFA23631E28L, 0xA4506CEBDE82BDE9L, 0xBEF9A3F7B2C67915L,
+ 0xC67178F2E372532BL, 0xCA273ECEEA26619CL, 0xD186B8C721C0C207L,
+ 0xEADA7DD6CDE0EB1EL, 0xF57D4F7FEE6ED178L, 0x06F067AA72176FBAL,
+ 0x0A637DC5A2C898A6L, 0x113F9804BEF90DAEL, 0x1B710B35131C471BL,
+ 0x28DB77F523047D84L, 0x32CAAB7B40C72493L, 0x3C9EBE0A15C9BEBCL,
+ 0x431D67C49C100D4CL, 0x4CC5D4BECB3E42B6L, 0x597F299CFC657E2AL,
+ 0x5FCB6FAB3AD6FAECL, 0x6C44198C4A475817L
+};
+
+static const uint64_t _sha3_round_consts[24] = {
+ 0x0000000000000001L, 0x0000000000008082L, 0x800000000000808AL,
+ 0x8000000080008000L, 0x000000000000808BL, 0x0000000080000001L,
+ 0x8000000080008081L, 0x8000000000008009L, 0x000000000000008AL,
+ 0x0000000000000088L, 0x0000000080008009L, 0x000000008000000AL,
+ 0x000000008000808BL, 0x800000000000008BL, 0x8000000000008089L,
+ 0x8000000000008003L, 0x8000000000008002L, 0x8000000000000080L,
+ 0x000000000000800AL, 0x800000008000000AL, 0x8000000080008081L,
+ 0x8000000000008080L, 0x0000000080000001L, 0x8000000080008008L
+};
+
+static const uint64_t _double_keccak_round_consts[24] = {
+ 0x0000000000000001L, 0x0000000000008082L, 0x800000000000808AL,
+ 0x8000000080008000L, 0x000000000000808BL, 0x0000000080000001L,
+ 0x8000000080008081L, 0x8000000000008009L, 0x000000000000008AL,
+ 0x0000000000000088L, 0x0000000080008009L, 0x000000008000000AL,
+ 0x000000008000808BL, 0x800000000000008BL, 0x8000000000008089L,
+ 0x8000000000008003L, 0x8000000000008002L, 0x8000000000000080L,
+ 0x000000000000800AL, 0x800000008000000AL, 0x8000000080008081L,
+ 0x8000000000008080L, 0x0000000080000001L, 0x8000000080008008L
+};
+
+static const char _encodeBlock_toBase64[64] = {
+ 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
+ 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
+ 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
+ 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
+ '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'
+};
+
+static const char _encodeBlock_toBase64URL[64] = {
+ 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
+ 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
+ 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
+ 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
+ '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_'
+};
+
+// Non-SIMD lookup tables are mostly dumped from fromBase64 array used in java.util.Base64,
+// except the trailing character '=' is also treated illegal value in this intrinsic. That
+// is java.util.Base64.fromBase64['='] = -2, while fromBase(URL)64ForNoSIMD['='] = 255 here.
+static const uint8_t _decodeBlock_fromBase64ForNoSIMD[256] = {
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, 255u, 63u,
+ 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, 14u,
+ 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, 40u,
+ 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+};
+
+static const uint8_t _decodeBlock_fromBase64URLForNoSIMD[256] = {
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u,
+ 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, 14u,
+ 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, 63u,
+ 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, 40u,
+ 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+};
+
+// A legal value of base64 code is in range [0, 127]. We need two lookups
+// with tbl/tbx and combine them to get the decode data. The 1st table vector
+// lookup use tbl, out of range indices are set to 0 in destination. The 2nd
+// table vector lookup use tbx, out of range indices are unchanged in
+// destination. Input [64..126] is mapped to index [65, 127] in second lookup.
+// The value of index 64 is set to 0, so that we know that we already get the
+// decoded data with the 1st lookup.
+static const uint8_t _decodeBlock_fromBase64ForSIMD[128] = {
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, 255u, 63u,
+ 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 0u, 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u,
+ 14u, 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u,
+ 40u, 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u,
+};
+
+static const uint8_t _decodeBlock_fromBase64URLForSIMD[128] = {
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u,
+ 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u,
+ 0u, 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u,
+ 14u, 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u,
+ 63u, 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u,
+ 40u, 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u,
+};
+
+
// Stub Code definitions
class StubGenerator: public StubCodeGenerator {
@@ -203,8 +363,17 @@ class StubGenerator: public StubCodeGenerator {
"adjust this code");
StubId stub_id = StubId::stubgen_call_stub_id;
+ GrowableArray entries;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 2, "sanity check");
+ address start = load_archive_data(stub_id, &entries);
+ if (start != nullptr) {
+ assert(entries.length() == 1, "expected 1 extra entry");
+ return_address = entries.at(0);
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Address sp_after_call (rfp, sp_after_call_off * wordSize);
@@ -323,6 +492,7 @@ class StubGenerator: public StubCodeGenerator {
// save current address for use by exception handling code
return_address = __ pc();
+ entries.append(return_address);
// store result depending on type (everything that is not
// T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
@@ -406,6 +576,9 @@ class StubGenerator: public StubCodeGenerator {
__ strd(j_farg0, Address(j_rarg2, 0));
__ br(Assembler::AL, exit);
+ // record the stub entry and end plus the auxiliary entry
+ store_archive_data(stub_id, start, __ pc(), &entries);
+
return start;
}
@@ -423,8 +596,14 @@ class StubGenerator: public StubCodeGenerator {
address generate_catch_exception() {
StubId stub_id = StubId::stubgen_catch_exception_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
// same as in generate_call_stub():
const Address sp_after_call(rfp, sp_after_call_off * wordSize);
@@ -450,7 +629,9 @@ class StubGenerator: public StubCodeGenerator {
__ verify_oop(r0);
__ str(r0, Address(rthread, Thread::pending_exception_offset()));
- __ mov(rscratch1, (address)__FILE__);
+ // special case -- add file name string to AOT address table
+ address file = (address)AOTCodeCache::add_C_string(__FILE__);
+ __ lea(rscratch1, ExternalAddress(file));
__ str(rscratch1, Address(rthread, Thread::exception_file_offset()));
__ movw(rscratch1, (int)__LINE__);
__ strw(rscratch1, Address(rthread, Thread::exception_line_offset()));
@@ -458,7 +639,10 @@ class StubGenerator: public StubCodeGenerator {
// complete return to VM
assert(StubRoutines::_call_stub_return_address != nullptr,
"_call_stub_return_address must have been generated before");
- __ b(StubRoutines::_call_stub_return_address);
+ __ b(RuntimeAddress(StubRoutines::_call_stub_return_address));
+
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
return start;
}
@@ -479,8 +663,14 @@ class StubGenerator: public StubCodeGenerator {
address generate_forward_exception() {
StubId stub_id = StubId::stubgen_forward_exception_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
// Upon entry, LR points to the return address returning into
// Java (interpreted or compiled) code; i.e., the return address
@@ -551,6 +741,9 @@ class StubGenerator: public StubCodeGenerator {
__ verify_oop(r0);
__ br(r19);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -569,8 +762,14 @@ class StubGenerator: public StubCodeGenerator {
// [tos + 5]: saved rscratch1
address generate_verify_oop() {
StubId stub_id = StubId::stubgen_verify_oop_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label exit, error;
@@ -613,33 +812,64 @@ class StubGenerator: public StubCodeGenerator {
__ blr(rscratch1);
__ hlt(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
// Generate indices for iota vector.
- address generate_iota_indices(StubId stub_id) {
+ void generate_iota_indices(StubId stub_id) {
+ GrowableArray entries;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == VECTOR_IOTA_COUNT, "sanity check");
+ address start = load_archive_data(stub_id, &entries);
+ if (start != nullptr) {
+ assert(entries.length() == entry_count - 1,
+ "unexpected entries count %d", entries.length());
+ StubRoutines::aarch64::_vector_iota_indices[0] = start;
+ for (int i = 1; i < VECTOR_IOTA_COUNT; i++) {
+ StubRoutines::aarch64::_vector_iota_indices[i] = entries.at(i - 1);
+ }
+ return;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
// B
__ emit_data64(0x0706050403020100, relocInfo::none);
__ emit_data64(0x0F0E0D0C0B0A0908, relocInfo::none);
+ entries.append(__ pc());
// H
__ emit_data64(0x0003000200010000, relocInfo::none);
__ emit_data64(0x0007000600050004, relocInfo::none);
+ entries.append(__ pc());
// S
__ emit_data64(0x0000000100000000, relocInfo::none);
__ emit_data64(0x0000000300000002, relocInfo::none);
+ entries.append(__ pc());
// D
__ emit_data64(0x0000000000000000, relocInfo::none);
__ emit_data64(0x0000000000000001, relocInfo::none);
+ entries.append(__ pc());
// S - FP
__ emit_data64(0x3F80000000000000, relocInfo::none); // 0.0f, 1.0f
__ emit_data64(0x4040000040000000, relocInfo::none); // 2.0f, 3.0f
+ entries.append(__ pc());
// D - FP
__ emit_data64(0x0000000000000000, relocInfo::none); // 0.0d
__ emit_data64(0x3FF0000000000000, relocInfo::none); // 1.0d
- return start;
+
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc(), &entries);
+
+ // install the entry addresses in the entry array
+ assert(entries.length() == entry_count - 1,
+ "unexpected entries count %d", entries.length());
+ StubRoutines::aarch64::_vector_iota_indices[0] = start;
+ for (int i = 1; i < VECTOR_IOTA_COUNT; i++) {
+ StubRoutines::aarch64::_vector_iota_indices[i] = entries.at(i - 1);
+ }
}
// The inner part of zero_words(). This is the bulk operation,
@@ -656,15 +886,21 @@ class StubGenerator: public StubCodeGenerator {
// r11 < MacroAssembler::zero_words_block_size.
address generate_zero_blocks() {
+ StubId stub_id = StubId::stubgen_zero_blocks_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
+ StubCodeMark mark(this, stub_id);
Label done;
Label base_aligned;
Register base = r10, cnt = r11;
- __ align(CodeEntryAlignment);
- StubId stub_id = StubId::stubgen_zero_blocks_id;
- StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
if (UseBlockZeroing) {
int zva_length = VM_Version::zva_length();
@@ -707,6 +943,9 @@ class StubGenerator: public StubCodeGenerator {
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -803,6 +1042,12 @@ class StubGenerator: public StubCodeGenerator {
// s and d are adjusted to point to the remaining words to copy
//
address generate_copy_longs(StubId stub_id, DecoratorSet decorators, Register s, Register d, Register count) {
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
BasicType type;
copy_direction direction;
@@ -854,7 +1099,7 @@ class StubGenerator: public StubCodeGenerator {
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label unaligned_copy_long;
if (AvoidUnalignedAccesses) {
@@ -1154,6 +1399,9 @@ class StubGenerator: public StubCodeGenerator {
__ ret(lr);
}
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -1445,19 +1693,25 @@ class StubGenerator: public StubCodeGenerator {
}
if (direction == copy_forwards) {
if (type != T_OBJECT) {
- __ bl(StubRoutines::aarch64::copy_byte_f());
+ __ lea(rscratch1, RuntimeAddress(StubRoutines::aarch64::copy_byte_f()));
+ __ blr(rscratch1);
} else if ((decorators & IS_DEST_UNINITIALIZED) != 0) {
- __ bl(StubRoutines::aarch64::copy_oop_uninit_f());
+ __ lea(rscratch1, RuntimeAddress(StubRoutines::aarch64::copy_oop_uninit_f()));
+ __ blr(rscratch1);
} else {
- __ bl(StubRoutines::aarch64::copy_oop_f());
+ __ lea(rscratch1, RuntimeAddress(StubRoutines::aarch64::copy_oop_f()));
+ __ blr(rscratch1);
}
} else {
if (type != T_OBJECT) {
- __ bl(StubRoutines::aarch64::copy_byte_b());
+ __ lea(rscratch1, RuntimeAddress(StubRoutines::aarch64::copy_byte_b()));
+ __ blr(rscratch1);
} else if ((decorators & IS_DEST_UNINITIALIZED) != 0) {
- __ bl(StubRoutines::aarch64::copy_oop_uninit_b());
+ __ lea(rscratch1, RuntimeAddress(StubRoutines::aarch64::copy_oop_uninit_b()));
+ __ blr(rscratch1);
} else {
- __ bl(StubRoutines::aarch64::copy_oop_b());
+ __ lea(rscratch1, RuntimeAddress(StubRoutines::aarch64::copy_oop_b()));
+ __ blr(rscratch1);
}
}
@@ -1508,8 +1762,8 @@ class StubGenerator: public StubCodeGenerator {
// stub_id - is used to name the stub and identify all details of
// how to perform the copy.
//
- // entry - is assigned to the stub's post push entry point unless
- // it is null
+ // nopush_entry - is assigned to the stub's post push entry point
+ // unless it is null
//
// Inputs:
// c_rarg0 - source array address
@@ -1525,8 +1779,6 @@ class StubGenerator: public StubCodeGenerator {
// copy method
//
address generate_disjoint_copy(StubId stub_id, address *nopush_entry) {
- Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
- RegSet saved_reg = RegSet::of(s, d, count);
int size;
bool aligned;
bool is_oop;
@@ -1607,17 +1859,45 @@ class StubGenerator: public StubCodeGenerator {
ShouldNotReachHere();
break;
}
+ // all stubs provide a 2nd entry which omits the frame push for
+ // use when bailing out from a conjoint copy. However we may also
+ // need some extra addressses for memory access protection.
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 2, "sanity check");
+ assert(nopush_entry != nullptr, "all disjoint copy stubs export a nopush entry");
+
+ bool add_extras = !is_oop && (!aligned || sizeof(jlong) == size);
+ int extra_count = ((add_extras ? 1 : 0) * UnsafeMemoryAccess::COLUMN_COUNT);
+ GrowableArray entries;
+ GrowableArray extras;
+ GrowableArray *extras_ptr = (extra_count > 0 ? &extras : nullptr);
+ address start = load_archive_data(stub_id, &entries, extras_ptr);
+ if (start != nullptr) {
+ assert(entries.length() == entry_count - 1,
+ "unexpected entries count %d", entries.length());
+ *nopush_entry = entries.at(0);
+ assert(extras.length() == extra_count,
+ "unexpected extra count %d", extras.length());
+ if (add_extras) {
+ // register one handler at offset 0
+ register_unsafe_access_handlers(extras, 0, 1);
+ }
+ return start;
+ }
+
+ Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
+ RegSet saved_reg = RegSet::of(s, d, count);
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
- if (nopush_entry != nullptr) {
- *nopush_entry = __ pc();
- // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
- BLOCK_COMMENT("Entry:");
- }
+ *nopush_entry = __ pc();
+ entries.append(*nopush_entry);
+
+ // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
+ BLOCK_COMMENT("Post-Push Entry:");
DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT;
if (dest_uninitialized) {
@@ -1636,8 +1916,7 @@ class StubGenerator: public StubCodeGenerator {
}
{
// UnsafeMemoryAccess page error: continue after unsafe access
- bool add_entry = !is_oop && (!aligned || sizeof(jlong) == size);
- UnsafeMemoryAccessMark umam(this, add_entry, true);
+ UnsafeMemoryAccessMark umam(this, add_extras, true);
copy_memory(decorators, is_oop ? T_OBJECT : T_BYTE, aligned, s, d, count, size);
}
@@ -1652,6 +1931,20 @@ class StubGenerator: public StubCodeGenerator {
__ leave();
__ mov(r0, zr); // return 0
__ ret(lr);
+
+ address end = __ pc();
+
+ if (add_extras) {
+ // retrieve the registered handler addresses
+ retrieve_unsafe_access_handlers(start, end, extras);
+ assert(extras.length() == extra_count
+ , "incorrect handlers count %d", extras.length());
+ }
+
+ // record the stub entry and end plus the no_push entry and any
+ // extra handler addresses
+ store_archive_data(stub_id, start, end, &entries, extras_ptr);
+
return start;
}
@@ -1663,8 +1956,8 @@ class StubGenerator: public StubCodeGenerator {
// corresponding disjoint copy routine which can be
// jumped to if the ranges do not actually overlap
//
- // entry - is assigned to the stub's post push entry point unless
- // it is null
+ // nopush_entry - is assigned to the stub's post push entry point
+ // unless it is null
//
//
// Inputs:
@@ -1681,8 +1974,6 @@ class StubGenerator: public StubCodeGenerator {
// used by some other conjoint copy method
//
address generate_conjoint_copy(StubId stub_id, address nooverlap_target, address *nopush_entry) {
- Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
- RegSet saved_regs = RegSet::of(s, d, count);
int size;
bool aligned;
bool is_oop;
@@ -1762,15 +2053,47 @@ class StubGenerator: public StubCodeGenerator {
default:
ShouldNotReachHere();
}
+ // only some conjoint stubs generate a 2nd entry
+ int entry_count = StubInfo::entry_count(stub_id);
+ int expected_entry_count = (nopush_entry == nullptr ? 1 : 2);
+ assert(entry_count == expected_entry_count,
+ "expected entry count %d does not match declared entry count %d for stub %s",
+ expected_entry_count, entry_count, StubInfo::name(stub_id));
+ // We need to protect memory accesses in certain cases
+ bool add_extras = !is_oop && (!aligned || sizeof(jlong) == size);
+ int extra_count = ((add_extras ? 1 : 0) * UnsafeMemoryAccess::COLUMN_COUNT);
+ GrowableArray entries;
+ GrowableArray extras;
+ GrowableArray *entries_ptr = (nopush_entry != nullptr ? &entries : nullptr);
+ GrowableArray *extras_ptr = (extra_count > 0 ? &extras : nullptr);
+ address start = load_archive_data(stub_id, entries_ptr, extras_ptr);
+ if (start != nullptr) {
+ assert(entries.length() == expected_entry_count - 1,
+ "unexpected entries count %d", entries.length());
+ assert(extras.length() == extra_count,
+ "unexpected extra count %d", extras.length());
+ if (nopush_entry != nullptr) {
+ *nopush_entry = entries.at(0);
+ }
+ if (add_extras) {
+ // register one handler at offset 0
+ register_unsafe_access_handlers(extras, 0, 1);
+ }
+ return start;
+ }
+
+ Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
+ RegSet saved_regs = RegSet::of(s, d, count);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
if (nopush_entry != nullptr) {
*nopush_entry = __ pc();
+ entries.append(*nopush_entry);
// caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
- BLOCK_COMMENT("Entry:");
+ BLOCK_COMMENT("Post-Push Entry:");
}
// use fwd copy when (d-s) above_equal (count*size)
@@ -1798,8 +2121,7 @@ class StubGenerator: public StubCodeGenerator {
}
{
// UnsafeMemoryAccess page error: continue after unsafe access
- bool add_entry = !is_oop && (!aligned || sizeof(jlong) == size);
- UnsafeMemoryAccessMark umam(this, add_entry, true);
+ UnsafeMemoryAccessMark umam(this, add_extras, true);
copy_memory(decorators, is_oop ? T_OBJECT : T_BYTE, aligned, s, d, count, -size);
}
if (is_oop) {
@@ -1811,6 +2133,23 @@ class StubGenerator: public StubCodeGenerator {
__ leave();
__ mov(r0, zr); // return 0
__ ret(lr);
+
+ assert(entries.length() == expected_entry_count - 1,
+ "unexpected entries count %d", entries.length());
+
+ address end = __ pc();
+
+ if (add_extras) {
+ // retrieve the registered handler addresses
+ retrieve_unsafe_access_handlers(start, end, extras);
+ assert(extras.length() == extra_count,
+ "incorrect handlers count %d", extras.length());
+ }
+
+ // record the stub entry and end plus any no_push entry and/or
+ // extra handler addresses
+ store_archive_data(stub_id, start, end, entries_ptr, extras_ptr);
+
return start;
}
@@ -1864,6 +2203,27 @@ class StubGenerator: public StubCodeGenerator {
ShouldNotReachHere();
}
+ // The normal stub provides a 2nd entry which omits the frame push
+ // for use when bailing out from a disjoint copy.
+ // Only some conjoint stubs generate a 2nd entry
+ int entry_count = StubInfo::entry_count(stub_id);
+ int expected_entry_count = (nopush_entry == nullptr ? 1 : 2);
+ GrowableArray entries;
+ GrowableArray *entries_ptr = (expected_entry_count == 1 ? nullptr : &entries);
+ assert(entry_count == expected_entry_count,
+ "expected entry count %d does not match declared entry count %d for stub %s",
+ expected_entry_count, entry_count, StubInfo::name(stub_id));
+ address start = load_archive_data(stub_id, entries_ptr);
+ if (start != nullptr) {
+ assert(entries.length() + 1 == expected_entry_count,
+ "expected entry count %d does not match return entry count %d for stub %s",
+ expected_entry_count, entries.length() + 1, StubInfo::name(stub_id));
+ if (nopush_entry != nullptr) {
+ *nopush_entry = entries.at(0);
+ }
+ return start;
+ }
+
Label L_load_element, L_store_element, L_do_card_marks, L_done, L_done_pop;
// Input registers (after setup_arg_regs)
@@ -1896,7 +2256,7 @@ class StubGenerator: public StubCodeGenerator {
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter(); // required for proper stackwalking of RuntimeStub frame
@@ -1913,6 +2273,7 @@ class StubGenerator: public StubCodeGenerator {
// Caller of this entry point must set up the argument registers.
if (nopush_entry != nullptr) {
*nopush_entry = __ pc();
+ entries.append(*nopush_entry);
BLOCK_COMMENT("Entry:");
}
@@ -2010,6 +2371,8 @@ class StubGenerator: public StubCodeGenerator {
__ leave();
__ ret(lr);
+ // record the stub entry and end plus any no_push entry
+ store_archive_data(stub_id, start, __ pc() , entries_ptr);
return start;
}
@@ -2072,13 +2435,18 @@ class StubGenerator: public StubCodeGenerator {
address int_copy_entry,
address long_copy_entry) {
StubId stub_id = StubId::stubgen_unsafe_arraycopy_id;
-
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
Label L_long_aligned, L_int_aligned, L_short_aligned;
Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter(); // required for proper stackwalking of RuntimeStub frame
// bump this on entry, not on exit:
@@ -2104,6 +2472,9 @@ class StubGenerator: public StubCodeGenerator {
__ lsr(count, count, LogBytesPerLong); // size => long_count
__ b(RuntimeAddress(long_copy_entry));
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -2125,7 +2496,12 @@ class StubGenerator: public StubCodeGenerator {
address int_copy_entry, address oop_copy_entry,
address long_copy_entry, address checkcast_copy_entry) {
StubId stub_id = StubId::stubgen_generic_arraycopy_id;
-
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
Label L_failed, L_objArray;
Label L_copy_bytes, L_copy_shorts, L_copy_ints, L_copy_longs;
@@ -2144,7 +2520,7 @@ class StubGenerator: public StubCodeGenerator {
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter(); // required for proper stackwalking of RuntimeStub frame
@@ -2383,6 +2759,9 @@ class StubGenerator: public StubCodeGenerator {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -2427,10 +2806,15 @@ class StubGenerator: public StubCodeGenerator {
default:
ShouldNotReachHere();
};
-
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
BLOCK_COMMENT("Entry:");
@@ -2563,15 +2947,32 @@ class StubGenerator: public StubCodeGenerator {
__ bind(L_exit2);
__ leave();
__ ret(lr);
+
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address generate_unsafecopy_common_error_exit() {
- address start_pc = __ pc();
+ StubId stub_id = StubId::stubgen_unsafecopy_common_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
+ StubCodeMark mark(this, stub_id);
+ start = __ pc();
__ leave();
__ mov(r0, 0);
__ ret(lr);
- return start_pc;
+
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
+ return start;
}
//
@@ -2589,13 +2990,28 @@ class StubGenerator: public StubCodeGenerator {
// c_rarg2 - byte value
//
address generate_unsafe_setmemory() {
+ StubId stub_id = StubId::stubgen_unsafe_setmemory_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ // we expect one set of extra unsafememory access handler entries
+ GrowableArray extras;
+ int extra_count = 1 * UnsafeMemoryAccess::COLUMN_COUNT;
+ address start = load_archive_data(stub_id, nullptr, &extras);
+ if (start != nullptr) {
+ assert(extras.length() == extra_count,
+ "unexpected extra entry count %d", extras.length());
+ register_unsafe_access_handlers(extras, 0, 1);
+ return start;
+ }
+
__ align(CodeEntryAlignment);
- StubCodeMark mark(this, StubId::stubgen_unsafe_setmemory_id);
- address start = __ pc();
+ StubCodeMark mark(this, stub_id);
+ start = __ pc();
Register dest = c_rarg0, count = c_rarg1, value = c_rarg2;
Label tail;
+ {
UnsafeMemoryAccessMark umam(this, true, false);
__ enter(); // required for proper stackwalking of RuntimeStub frame
@@ -2679,6 +3095,17 @@ class StubGenerator: public StubCodeGenerator {
__ bind(finished);
__ leave();
__ ret(lr);
+ // have to exit the block and destroy the UnsafeMemoryAccessMark
+ // in order to retrieve the handler end address
+ }
+
+ // install saved handler addresses in extras
+ address end = __ pc();
+ retrieve_unsafe_access_handlers(start, end, extras);
+ assert(extras.length() == extra_count,
+ "incorrect handlers count %d", extras.length());
+ // record the stub entry and end plus the extras
+ store_archive_data(stub_id, start, end, nullptr, &extras);
return start;
}
@@ -2686,33 +3113,45 @@ class StubGenerator: public StubCodeGenerator {
address generate_data_cache_writeback() {
const Register line = c_rarg0; // address of line to write back
- __ align(CodeEntryAlignment);
-
StubId stub_id = StubId::stubgen_data_cache_writeback_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
__ cache_wb(Address(line, 0));
__ leave();
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address generate_data_cache_writeback_sync() {
- const Register is_pre = c_rarg0; // pre or post sync
-
- __ align(CodeEntryAlignment);
-
StubId stub_id = StubId::stubgen_data_cache_writeback_sync_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ const Register is_pre = c_rarg0; // pre or post sync
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
// pre wbsync is a no-op
// post wbsync translates to an sfence
Label skip;
- address start = __ pc();
+ start = __ pc();
__ enter();
__ cbnz(is_pre, skip);
__ cache_wbsync(false);
@@ -2720,6 +3159,9 @@ class StubGenerator: public StubCodeGenerator {
__ leave();
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -2882,8 +3324,15 @@ class StubGenerator: public StubCodeGenerator {
// c_rarg2 - sessionKe (key) in little endian int array
//
address generate_aescrypt_encryptBlock() {
- __ align(CodeEntryAlignment);
+ assert(UseAES, "need AES cryptographic extension support");
StubId stub_id = StubId::stubgen_aescrypt_encryptBlock_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
const Register from = c_rarg0; // source array address
@@ -2891,7 +3340,7 @@ class StubGenerator: public StubCodeGenerator {
const Register key = c_rarg2; // key array address
const Register keylen = rscratch1;
- address start = __ pc();
+ start = __ pc();
__ enter();
__ ldrw(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
@@ -2904,6 +3353,9 @@ class StubGenerator: public StubCodeGenerator {
__ leave();
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -2916,8 +3368,14 @@ class StubGenerator: public StubCodeGenerator {
//
address generate_aescrypt_decryptBlock() {
assert(UseAES, "need AES cryptographic extension support");
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_aescrypt_decryptBlock_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
Label L_doLast;
@@ -2926,7 +3384,7 @@ class StubGenerator: public StubCodeGenerator {
const Register key = c_rarg2; // key array address
const Register keylen = rscratch1;
- address start = __ pc();
+ start = __ pc();
__ enter(); // required for proper stackwalking of RuntimeStub frame
__ ldrw(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
@@ -2938,6 +3396,9 @@ class StubGenerator: public StubCodeGenerator {
__ leave();
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -2955,8 +3416,14 @@ class StubGenerator: public StubCodeGenerator {
//
address generate_cipherBlockChaining_encryptAESCrypt() {
assert(UseAES, "need AES cryptographic extension support");
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_cipherBlockChaining_encryptAESCrypt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
Label L_loadkeys_44, L_loadkeys_52, L_aes_loop, L_rounds_44, L_rounds_52;
@@ -2969,7 +3436,7 @@ class StubGenerator: public StubCodeGenerator {
const Register len_reg = c_rarg4; // src len (must be multiple of blocksize 16)
const Register keylen = rscratch1;
- address start = __ pc();
+ start = __ pc();
__ enter();
@@ -3043,6 +3510,9 @@ class StubGenerator: public StubCodeGenerator {
__ leave();
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3060,8 +3530,14 @@ class StubGenerator: public StubCodeGenerator {
//
address generate_cipherBlockChaining_decryptAESCrypt() {
assert(UseAES, "need AES cryptographic extension support");
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_cipherBlockChaining_decryptAESCrypt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
Label L_loadkeys_44, L_loadkeys_52, L_aes_loop, L_rounds_44, L_rounds_52;
@@ -3074,7 +3550,7 @@ class StubGenerator: public StubCodeGenerator {
const Register len_reg = c_rarg4; // src len (must be multiple of blocksize 16)
const Register keylen = rscratch1;
- address start = __ pc();
+ start = __ pc();
__ enter();
@@ -3152,6 +3628,9 @@ class StubGenerator: public StubCodeGenerator {
__ leave();
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3188,6 +3667,13 @@ class StubGenerator: public StubCodeGenerator {
// r0 - input length
//
address generate_counterMode_AESCrypt() {
+ StubId stub_id = StubId::stubgen_counterMode_AESCrypt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
const Register in = c_rarg0;
const Register out = c_rarg1;
const Register key = c_rarg2;
@@ -3248,9 +3734,8 @@ class StubGenerator: public StubCodeGenerator {
// Wide bulk encryption of whole blocks.
__ align(CodeEntryAlignment);
- StubId stub_id = StubId::stubgen_counterMode_AESCrypt_id;
StubCodeMark mark(this, stub_id);
- const address start = __ pc();
+ start = __ pc();
__ enter();
Label DONE, CTR_large_block, large_block_return;
@@ -3435,6 +3920,9 @@ class StubGenerator: public StubCodeGenerator {
__ strw(used, Address(used_ptr));
__ b(large_block_return);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3451,11 +3939,16 @@ class StubGenerator: public StubCodeGenerator {
// return - number of processed bytes
address generate_galoisCounterMode_AESCrypt() {
Label ghash_polynomial; // local data generated after code
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_galoisCounterMode_AESCrypt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register in = c_rarg0;
@@ -3567,6 +4060,9 @@ class StubGenerator: public StubCodeGenerator {
// 128-bit vector
__ emit_int64(0x87);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3685,10 +4181,16 @@ class StubGenerator: public StubCodeGenerator {
default:
ShouldNotReachHere();
}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Register buf = c_rarg0;
Register state = c_rarg1;
@@ -3815,6 +4317,9 @@ class StubGenerator: public StubCodeGenerator {
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3838,11 +4343,16 @@ class StubGenerator: public StubCodeGenerator {
default:
ShouldNotReachHere();
}
-
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Register buf = c_rarg0;
Register state = c_rarg1;
@@ -3919,6 +4429,9 @@ class StubGenerator: public StubCodeGenerator {
__ emit_int32(0x8f1bbcdc);
__ emit_int32(0xca62c1d6);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3943,30 +4456,15 @@ class StubGenerator: public StubCodeGenerator {
default:
ShouldNotReachHere();
}
-
- static const uint32_t round_consts[64] = {
- 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5,
- 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
- 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
- 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
- 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc,
- 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
- 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
- 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
- 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
- 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
- 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3,
- 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
- 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5,
- 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
- 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
- 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2,
- };
-
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
-
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Register buf = c_rarg0;
Register state = c_rarg1;
@@ -3987,7 +4485,7 @@ class StubGenerator: public StubCodeGenerator {
// t1 == v7
// load 16 keys to v16..v31
- __ lea(rscratch1, ExternalAddress((address)round_consts));
+ __ lea(rscratch1, ExternalAddress((address)_sha256_round_consts));
__ ld1(v16, v17, v18, v19, __ T4S, __ post(rscratch1, 64));
__ ld1(v20, v21, v22, v23, __ T4S, __ post(rscratch1, 64));
__ ld1(v24, v25, v26, v27, __ T4S, __ post(rscratch1, 64));
@@ -4048,6 +4546,9 @@ class StubGenerator: public StubCodeGenerator {
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -4099,41 +4600,15 @@ class StubGenerator: public StubCodeGenerator {
default:
ShouldNotReachHere();
}
-
- static const uint64_t round_consts[80] = {
- 0x428A2F98D728AE22L, 0x7137449123EF65CDL, 0xB5C0FBCFEC4D3B2FL,
- 0xE9B5DBA58189DBBCL, 0x3956C25BF348B538L, 0x59F111F1B605D019L,
- 0x923F82A4AF194F9BL, 0xAB1C5ED5DA6D8118L, 0xD807AA98A3030242L,
- 0x12835B0145706FBEL, 0x243185BE4EE4B28CL, 0x550C7DC3D5FFB4E2L,
- 0x72BE5D74F27B896FL, 0x80DEB1FE3B1696B1L, 0x9BDC06A725C71235L,
- 0xC19BF174CF692694L, 0xE49B69C19EF14AD2L, 0xEFBE4786384F25E3L,
- 0x0FC19DC68B8CD5B5L, 0x240CA1CC77AC9C65L, 0x2DE92C6F592B0275L,
- 0x4A7484AA6EA6E483L, 0x5CB0A9DCBD41FBD4L, 0x76F988DA831153B5L,
- 0x983E5152EE66DFABL, 0xA831C66D2DB43210L, 0xB00327C898FB213FL,
- 0xBF597FC7BEEF0EE4L, 0xC6E00BF33DA88FC2L, 0xD5A79147930AA725L,
- 0x06CA6351E003826FL, 0x142929670A0E6E70L, 0x27B70A8546D22FFCL,
- 0x2E1B21385C26C926L, 0x4D2C6DFC5AC42AEDL, 0x53380D139D95B3DFL,
- 0x650A73548BAF63DEL, 0x766A0ABB3C77B2A8L, 0x81C2C92E47EDAEE6L,
- 0x92722C851482353BL, 0xA2BFE8A14CF10364L, 0xA81A664BBC423001L,
- 0xC24B8B70D0F89791L, 0xC76C51A30654BE30L, 0xD192E819D6EF5218L,
- 0xD69906245565A910L, 0xF40E35855771202AL, 0x106AA07032BBD1B8L,
- 0x19A4C116B8D2D0C8L, 0x1E376C085141AB53L, 0x2748774CDF8EEB99L,
- 0x34B0BCB5E19B48A8L, 0x391C0CB3C5C95A63L, 0x4ED8AA4AE3418ACBL,
- 0x5B9CCA4F7763E373L, 0x682E6FF3D6B2B8A3L, 0x748F82EE5DEFB2FCL,
- 0x78A5636F43172F60L, 0x84C87814A1F0AB72L, 0x8CC702081A6439ECL,
- 0x90BEFFFA23631E28L, 0xA4506CEBDE82BDE9L, 0xBEF9A3F7B2C67915L,
- 0xC67178F2E372532BL, 0xCA273ECEEA26619CL, 0xD186B8C721C0C207L,
- 0xEADA7DD6CDE0EB1EL, 0xF57D4F7FEE6ED178L, 0x06F067AA72176FBAL,
- 0x0A637DC5A2C898A6L, 0x113F9804BEF90DAEL, 0x1B710B35131C471BL,
- 0x28DB77F523047D84L, 0x32CAAB7B40C72493L, 0x3C9EBE0A15C9BEBCL,
- 0x431D67C49C100D4CL, 0x4CC5D4BECB3E42B6L, 0x597F299CFC657E2AL,
- 0x5FCB6FAB3AD6FAECL, 0x6C44198C4A475817L
- };
-
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
-
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Register buf = c_rarg0;
Register state = c_rarg1;
@@ -4151,7 +4626,7 @@ class StubGenerator: public StubCodeGenerator {
__ ld1(v8, v9, v10, v11, __ T2D, state);
// load first 4 round constants
- __ lea(rscratch1, ExternalAddress((address)round_consts));
+ __ lea(rscratch1, ExternalAddress((address)_sha512_round_consts));
__ ld1(v20, v21, v22, v23, __ T2D, __ post(rscratch1, 64));
__ BIND(sha512_loop);
@@ -4236,6 +4711,9 @@ class StubGenerator: public StubCodeGenerator {
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -4349,22 +4827,15 @@ class StubGenerator: public StubCodeGenerator {
default:
ShouldNotReachHere();
}
-
- static const uint64_t round_consts[24] = {
- 0x0000000000000001L, 0x0000000000008082L, 0x800000000000808AL,
- 0x8000000080008000L, 0x000000000000808BL, 0x0000000080000001L,
- 0x8000000080008081L, 0x8000000000008009L, 0x000000000000008AL,
- 0x0000000000000088L, 0x0000000080008009L, 0x000000008000000AL,
- 0x000000008000808BL, 0x800000000000008BL, 0x8000000000008089L,
- 0x8000000000008003L, 0x8000000000008002L, 0x8000000000000080L,
- 0x000000000000800AL, 0x800000008000000AL, 0x8000000080008081L,
- 0x8000000000008080L, 0x0000000080000001L, 0x8000000080008008L
- };
-
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
-
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Register buf = c_rarg0;
Register state = c_rarg1;
@@ -4396,7 +4867,7 @@ class StubGenerator: public StubCodeGenerator {
__ movw(rscratch2, 24);
// load round_constants base
- __ lea(rscratch1, ExternalAddress((address) round_consts));
+ __ lea(rscratch1, ExternalAddress((address) _sha3_round_consts));
// load input
__ ld1(v25, v26, v27, v28, __ T8B, __ post(buf, 32));
@@ -4488,6 +4959,9 @@ class StubGenerator: public StubCodeGenerator {
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -4495,22 +4969,18 @@ class StubGenerator: public StubCodeGenerator {
// c_rarg0 - long[] state0
// c_rarg1 - long[] state1
address generate_double_keccak() {
- static const uint64_t round_consts[24] = {
- 0x0000000000000001L, 0x0000000000008082L, 0x800000000000808AL,
- 0x8000000080008000L, 0x000000000000808BL, 0x0000000080000001L,
- 0x8000000080008081L, 0x8000000000008009L, 0x000000000000008AL,
- 0x0000000000000088L, 0x0000000080008009L, 0x000000008000000AL,
- 0x000000008000808BL, 0x800000000000008BL, 0x8000000000008089L,
- 0x8000000000008003L, 0x8000000000008002L, 0x8000000000000080L,
- 0x000000000000800AL, 0x800000008000000AL, 0x8000000080008081L,
- 0x8000000000008080L, 0x0000000080000001L, 0x8000000080008008L
- };
-
+ StubId stub_id = StubId::stubgen_double_keccak_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
// Implements the double_keccak() method of the
// sun.secyrity.provider.SHA3Parallel class
__ align(CodeEntryAlignment);
- StubCodeMark mark(this, "StubRoutines", "double_keccak");
- address start = __ pc();
+ StubCodeMark mark(this, stub_id);
+ start = __ pc();
__ enter();
Register state0 = c_rarg0;
@@ -4546,7 +5016,7 @@ class StubGenerator: public StubCodeGenerator {
__ movw(rscratch2, 24);
// load round_constants base
- __ lea(rscratch1, ExternalAddress((address) round_consts));
+ __ lea(rscratch1, ExternalAddress((address) _double_keccak_round_consts));
__ BIND(rounds24_loop);
__ subw(rscratch2, rscratch2, 1);
@@ -4578,6 +5048,9 @@ class StubGenerator: public StubCodeGenerator {
__ mov(r0, zr); // return 0
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -4611,11 +5084,17 @@ class StubGenerator: public StubCodeGenerator {
// vectors write their first lane back to the keystream buffer, followed
// by the second lane from all vectors and so on.
address generate_chacha20Block_blockpar() {
+ StubId stub_id = StubId::stubgen_chacha20Block_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
Label L_twoRounds, L_cc20_const;
__ align(CodeEntryAlignment);
- StubId stub_id = StubId::stubgen_chacha20Block_id;
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
int i, j;
@@ -4770,6 +5249,9 @@ class StubGenerator: public StubCodeGenerator {
__ emit_int64(0x0605040702010003UL);
__ emit_int64(0x0E0D0C0F0A09080BUL);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -5258,11 +5740,16 @@ class StubGenerator: public StubCodeGenerator {
// coeffs (short[256]) = c_rarg0
// ntt_zetas (short[256]) = c_rarg1
address generate_kyberNtt() {
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_kyberNtt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register coeffs = c_rarg0;
@@ -5486,6 +5973,9 @@ class StubGenerator: public StubCodeGenerator {
__ mov(r0, zr); // return 0
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -5496,11 +5986,16 @@ class StubGenerator: public StubCodeGenerator {
// coeffs (short[256]) = c_rarg0
// ntt_zetas (short[256]) = c_rarg1
address generate_kyberInverseNtt() {
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_kyberInverseNtt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register coeffs = c_rarg0;
@@ -5770,6 +6265,9 @@ class StubGenerator: public StubCodeGenerator {
__ mov(r0, zr); // return 0
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -5783,11 +6281,16 @@ class StubGenerator: public StubCodeGenerator {
// nttb (short[256]) = c_rarg2
// zetas (short[128]) = c_rarg3
address generate_kyberNttMult() {
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_kyberNttMult_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register result = c_rarg0;
@@ -5889,6 +6392,9 @@ class StubGenerator: public StubCodeGenerator {
__ mov(r0, zr); // return 0
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -5900,11 +6406,16 @@ class StubGenerator: public StubCodeGenerator {
// a (short[256]) = c_rarg1
// b (short[256]) = c_rarg2
address generate_kyberAddPoly_2() {
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_kyberAddPoly_2_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register result = c_rarg0;
@@ -5973,6 +6484,9 @@ class StubGenerator: public StubCodeGenerator {
__ mov(r0, zr); // return 0
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -5985,11 +6499,16 @@ class StubGenerator: public StubCodeGenerator {
// b (short[256]) = c_rarg2
// c (short[256]) = c_rarg3
address generate_kyberAddPoly_3() {
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_kyberAddPoly_3_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register result = c_rarg0;
@@ -6072,6 +6591,9 @@ class StubGenerator: public StubCodeGenerator {
__ mov(r0, zr); // return 0
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -6092,12 +6614,18 @@ class StubGenerator: public StubCodeGenerator {
// parsed (short[]) = c_rarg2
// parsedLength = c_rarg3
address generate_kyber12To16() {
+ StubId stub_id = StubId::stubgen_kyber12To16_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
Label L_F00, L_loop;
__ align(CodeEntryAlignment);
- StubId stub_id = StubId::stubgen_kyber12To16_id;
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register condensed = c_rarg0;
@@ -6225,6 +6753,9 @@ class StubGenerator: public StubCodeGenerator {
__ emit_int64(0x0f000f000f000f00);
__ emit_int64(0x0f000f000f000f00);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -6234,11 +6765,16 @@ class StubGenerator: public StubCodeGenerator {
//
// coeffs (short[256]) = c_rarg0
address generate_kyberBarrettReduce() {
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_kyberBarrettReduce_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register coeffs = c_rarg0;
@@ -6318,6 +6854,9 @@ class StubGenerator: public StubCodeGenerator {
__ mov(r0, zr); // return 0
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -6481,11 +7020,16 @@ class StubGenerator: public StubCodeGenerator {
// coeffs (int[256]) = c_rarg0
// zetas (int[256]) = c_rarg1
address generate_dilithiumAlmostNtt() {
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_dilithiumAlmostNtt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register coeffs = c_rarg0;
@@ -6596,6 +7140,9 @@ class StubGenerator: public StubCodeGenerator {
__ mov(r0, zr); // return 0
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -6688,11 +7235,16 @@ class StubGenerator: public StubCodeGenerator {
// coeffs (int[256]) = c_rarg0
// zetas (int[256]) = c_rarg1
address generate_dilithiumAlmostInverseNtt() {
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_dilithiumAlmostInverseNtt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register coeffs = c_rarg0;
@@ -6788,6 +7340,9 @@ class StubGenerator: public StubCodeGenerator {
__ mov(r0, zr); // return 0
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -6801,11 +7356,16 @@ class StubGenerator: public StubCodeGenerator {
// poly1 (int[256]) = c_rarg1
// poly2 (int[256]) = c_rarg2
address generate_dilithiumNttMult() {
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_dilithiumNttMult_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
Label L_loop;
@@ -6854,6 +7414,9 @@ class StubGenerator: public StubCodeGenerator {
__ mov(r0, zr); // return 0
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -6865,11 +7428,16 @@ class StubGenerator: public StubCodeGenerator {
// coeffs (int[256]) = c_rarg0
// constant (int) = c_rarg1
address generate_dilithiumMontMulByConstant() {
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_dilithiumMontMulByConstant_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
Label L_loop;
@@ -6915,6 +7483,9 @@ class StubGenerator: public StubCodeGenerator {
__ mov(r0, zr); // return 0
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -6929,11 +7500,16 @@ class StubGenerator: public StubCodeGenerator {
// twoGamma2 (int) = c_rarg3
// multiplier (int) = c_rarg4
address generate_dilithiumDecomposePoly() {
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_dilithiumDecomposePoly_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_loop;
const Register input = c_rarg0;
@@ -7073,6 +7649,9 @@ class StubGenerator: public StubCodeGenerator {
__ mov(r0, zr); // return 0
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -7212,21 +7791,15 @@ class StubGenerator: public StubCodeGenerator {
default:
ShouldNotReachHere();
}
-
- static const uint64_t round_consts[24] = {
- 0x0000000000000001L, 0x0000000000008082L, 0x800000000000808AL,
- 0x8000000080008000L, 0x000000000000808BL, 0x0000000080000001L,
- 0x8000000080008081L, 0x8000000000008009L, 0x000000000000008AL,
- 0x0000000000000088L, 0x0000000080008009L, 0x000000008000000AL,
- 0x000000008000808BL, 0x800000000000008BL, 0x8000000000008089L,
- 0x8000000000008003L, 0x8000000000008002L, 0x8000000000000080L,
- 0x000000000000800AL, 0x800000008000000AL, 0x8000000080008081L,
- 0x8000000000008080L, 0x0000000080000001L, 0x8000000080008008L
- };
-
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Register buf = c_rarg0;
Register state = c_rarg1;
@@ -7378,7 +7951,7 @@ class StubGenerator: public StubCodeGenerator {
__ fmovs(v1, 1.0); // exact representation
__ str(buf, Address(sp, 16));
- __ lea(tmp3, ExternalAddress((address) round_consts));
+ __ lea(tmp3, ExternalAddress((address) _sha3_round_consts));
__ BIND(loop_body);
keccak_round_gpr(can_use_fp, can_use_r18, tmp3,
@@ -7433,6 +8006,9 @@ class StubGenerator: public StubCodeGenerator {
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -7449,12 +8025,17 @@ class StubGenerator: public StubCodeGenerator {
*/
address generate_updateBytesCRC32() {
assert(UseCRC32Intrinsics, "what are we doing here?");
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_updateBytesCRC32_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register crc = c_rarg0; // crc
const Register buf = c_rarg1; // source java byte array address
@@ -7474,6 +8055,9 @@ class StubGenerator: public StubCodeGenerator {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -7491,12 +8075,17 @@ class StubGenerator: public StubCodeGenerator {
*/
address generate_updateBytesCRC32C() {
assert(UseCRC32CIntrinsics, "what are we doing here?");
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_updateBytesCRC32C_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register crc = c_rarg0; // crc
const Register buf = c_rarg1; // source java byte array address
@@ -7516,6 +8105,9 @@ class StubGenerator: public StubCodeGenerator {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -7531,10 +8123,16 @@ class StubGenerator: public StubCodeGenerator {
* c_rarg0 - int adler result
*/
address generate_updateBytesAdler32() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_updateBytesAdler32_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_simple_by1_loop, L_nmax, L_nmax_loop, L_by16, L_by16_loop, L_by1_loop, L_do_mod, L_combine, L_by1;
@@ -7702,6 +8300,9 @@ class StubGenerator: public StubCodeGenerator {
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -7753,11 +8354,17 @@ class StubGenerator: public StubCodeGenerator {
* c_rarg4 - z address
*/
address generate_multiplyToLen() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_multiplyToLen_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register x = r0;
const Register xlen = r1;
const Register y = r2;
@@ -7779,6 +8386,9 @@ class StubGenerator: public StubCodeGenerator {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -7786,10 +8396,16 @@ class StubGenerator: public StubCodeGenerator {
// squareToLen algorithm for sizes 1..127 described in java code works
// faster than multiply_to_len on some CPUs and slower on others, but
// multiply_to_len shows a bit better overall results
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_squareToLen_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register x = r0;
const Register xlen = r1;
@@ -7816,15 +8432,25 @@ class StubGenerator: public StubCodeGenerator {
__ pop(spilled_regs, sp);
__ leave();
__ ret(lr);
+
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address generate_mulAdd() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_mulAdd_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register out = r0;
const Register in = r1;
@@ -7838,6 +8464,9 @@ class StubGenerator: public StubCodeGenerator {
__ leave();
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -7851,10 +8480,16 @@ class StubGenerator: public StubCodeGenerator {
// c_rarg4 - numIter
//
address generate_bigIntegerRightShift() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_bigIntegerRightShiftWorker_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label ShiftSIMDLoop, ShiftTwoLoop, ShiftThree, ShiftTwo, ShiftOne, Exit;
@@ -7961,6 +8596,9 @@ class StubGenerator: public StubCodeGenerator {
__ BIND(Exit);
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -7974,10 +8612,16 @@ class StubGenerator: public StubCodeGenerator {
// c_rarg4 - numIter
//
address generate_bigIntegerLeftShift() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_bigIntegerLeftShiftWorker_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label ShiftSIMDLoop, ShiftTwoLoop, ShiftThree, ShiftTwo, ShiftOne, Exit;
@@ -8072,10 +8716,25 @@ class StubGenerator: public StubCodeGenerator {
__ BIND(Exit);
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address generate_count_positives(address &count_positives_long) {
+ StubId stub_id = StubId::stubgen_count_positives_id;
+ GrowableArray entries;
+ int entry_count = StubInfo::entry_count(stub_id);
+ // We have an extra entry for count_positives_long.
+ assert(entry_count == 2, "sanity check");
+ address start = load_archive_data(stub_id, &entries);
+ if (start != nullptr) {
+ assert(entries.length() == 1,
+ "unexpected extra entry count %d", entries.length());
+ count_positives_long = entries.at(0);
+ return start;
+ }
const u1 large_loop_size = 64;
const uint64_t UPPER_BIT_MASK=0x8080808080808080;
int dcache_line = VM_Version::dcache_line_size();
@@ -8083,8 +8742,6 @@ class StubGenerator: public StubCodeGenerator {
Register ary1 = r1, len = r2, result = r0;
__ align(CodeEntryAlignment);
-
- StubId stub_id = StubId::stubgen_count_positives_id;
StubCodeMark mark(this, stub_id);
address entry = __ pc();
@@ -8127,6 +8784,7 @@ class StubGenerator: public StubCodeGenerator {
const RegSet spilled_regs = RegSet::range(tmp1, tmp5) + tmp6;
count_positives_long = __ pc(); // 2nd entry point
+ entries.append(count_positives_long);
__ enter();
@@ -8241,6 +8899,9 @@ class StubGenerator: public StubCodeGenerator {
__ sub(result, result, len);
__ ret(lr);
+ // record the stub entry and end plus the extra entry
+ store_archive_data(stub_id, entry, __ pc(), &entries);
+
return entry;
}
@@ -8331,6 +8992,13 @@ class StubGenerator: public StubCodeGenerator {
// r3-r5 are reserved temporary registers
// Clobbers: v0-v7 when UseSIMDForArrayEquals, rscratch1, rscratch2
address generate_large_array_equals() {
+ StubId stub_id = StubId::stubgen_large_array_equals_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
Register a1 = r1, a2 = r2, result = r0, cnt1 = r10, tmp1 = rscratch1,
tmp2 = rscratch2, tmp3 = r3, tmp4 = r4, tmp5 = r5, tmp6 = r11,
tmp7 = r12, tmp8 = r13;
@@ -8346,7 +9014,6 @@ class StubGenerator: public StubCodeGenerator {
__ align(CodeEntryAlignment);
- StubId stub_id = StubId::stubgen_large_array_equals_id;
StubCodeMark mark(this, stub_id);
address entry = __ pc();
@@ -8421,6 +9088,10 @@ class StubGenerator: public StubCodeGenerator {
__ bind(NOT_EQUAL_NO_POP);
__ leave();
__ ret(lr);
+
+ // record the stub entry and end
+ store_archive_data(stub_id, entry, __ pc());
+
return entry;
}
@@ -8429,6 +9100,33 @@ class StubGenerator: public StubCodeGenerator {
// cnt = r2 - elements count
// Clobbers: v0-v13, rscratch1, rscratch2
address generate_large_arrays_hashcode(BasicType eltype) {
+ StubId stub_id;
+ switch (eltype) {
+ case T_BOOLEAN:
+ stub_id = StubId::stubgen_large_arrays_hashcode_boolean_id;
+ break;
+ case T_BYTE:
+ stub_id = StubId::stubgen_large_arrays_hashcode_byte_id;
+ break;
+ case T_CHAR:
+ stub_id = StubId::stubgen_large_arrays_hashcode_char_id;
+ break;
+ case T_SHORT:
+ stub_id = StubId::stubgen_large_arrays_hashcode_short_id;
+ break;
+ case T_INT:
+ stub_id = StubId::stubgen_large_arrays_hashcode_int_id;
+ break;
+ default:
+ stub_id = StubId::NO_STUBID;
+ ShouldNotReachHere();
+ };
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
const Register result = r0, ary = r1, cnt = r2;
const FloatRegister vdata0 = v3, vdata1 = v2, vdata2 = v1, vdata3 = v0;
const FloatRegister vmul0 = v4, vmul1 = v5, vmul2 = v6, vmul3 = v7;
@@ -8472,28 +9170,6 @@ class StubGenerator: public StubCodeGenerator {
__ align(CodeEntryAlignment);
- StubId stub_id;
- switch (eltype) {
- case T_BOOLEAN:
- stub_id = StubId::stubgen_large_arrays_hashcode_boolean_id;
- break;
- case T_BYTE:
- stub_id = StubId::stubgen_large_arrays_hashcode_byte_id;
- break;
- case T_CHAR:
- stub_id = StubId::stubgen_large_arrays_hashcode_char_id;
- break;
- case T_SHORT:
- stub_id = StubId::stubgen_large_arrays_hashcode_short_id;
- break;
- case T_INT:
- stub_id = StubId::stubgen_large_arrays_hashcode_int_id;
- break;
- default:
- stub_id = StubId::NO_STUBID;
- ShouldNotReachHere();
- };
-
StubCodeMark mark(this, stub_id);
address entry = __ pc();
@@ -8728,19 +9404,32 @@ class StubGenerator: public StubCodeGenerator {
__ leave();
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, entry, __ pc());
+
return entry;
}
address generate_dsin_dcos(bool isCos) {
- __ align(CodeEntryAlignment);
StubId stub_id = (isCos ? StubId::stubgen_dcos_id : StubId::stubgen_dsin_id);
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ generate_dsin_dcos(isCos, (address)StubRoutines::aarch64::_npio2_hw,
(address)StubRoutines::aarch64::_two_over_pi,
(address)StubRoutines::aarch64::_pio2,
(address)StubRoutines::aarch64::_dsin_coef,
(address)StubRoutines::aarch64::_dcos_coef);
+
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -8784,8 +9473,14 @@ class StubGenerator: public StubCodeGenerator {
// r10 = tmp1
// r11 = tmp2
address generate_compare_long_string_different_encoding(bool isLU) {
- __ align(CodeEntryAlignment);
StubId stub_id = (isLU ? StubId::stubgen_compare_long_string_LU_id : StubId::stubgen_compare_long_string_UL_id);
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
address entry = __ pc();
Label SMALL_LOOP, TAIL, TAIL_LOAD_16, LOAD_LAST, DIFF1, DIFF2,
@@ -8887,20 +9582,34 @@ class StubGenerator: public StubCodeGenerator {
__ subw(result, tmp1, rscratch1);
__ bind(DONE);
__ ret(lr);
- return entry;
+
+ // record the stub entry and end
+ store_archive_data(stub_id, entry, __ pc());
+
+ return entry;
}
// r0 = input (float16)
// v0 = result (float)
// v1 = temporary float register
address generate_float16ToFloat() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_hf2f_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
address entry = __ pc();
BLOCK_COMMENT("Entry:");
__ flt16_to_flt(v0, r0, v1);
__ ret(lr);
+
+ // record the stub entry and end
+ store_archive_data(stub_id, entry, __ pc());
+
return entry;
}
@@ -8908,24 +9617,40 @@ class StubGenerator: public StubCodeGenerator {
// r0 = result (float16)
// v1 = temporary float register
address generate_floatToFloat16() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_f2hf_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
address entry = __ pc();
BLOCK_COMMENT("Entry:");
__ flt_to_flt16(r0, v0, v1);
__ ret(lr);
+
+ // record the stub entry and end
+ store_archive_data(stub_id, entry, __ pc());
+
return entry;
}
address generate_method_entry_barrier() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_method_entry_barrier_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
Label deoptimize_label;
- address start = __ pc();
+ start = __ pc();
BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
@@ -8974,6 +9699,9 @@ class StubGenerator: public StubCodeGenerator {
__ mov(sp, rscratch1);
__ br(rscratch2);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -8985,8 +9713,14 @@ class StubGenerator: public StubCodeGenerator {
// r10 = tmp1
// r11 = tmp2
address generate_compare_long_string_same_encoding(bool isLL) {
- __ align(CodeEntryAlignment);
StubId stub_id = (isLL ? StubId::stubgen_compare_long_string_LL_id : StubId::stubgen_compare_long_string_UU_id);
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
address entry = __ pc();
Register result = r0, str1 = r1, cnt1 = r2, str2 = r3, cnt2 = r4,
@@ -9094,6 +9828,10 @@ class StubGenerator: public StubCodeGenerator {
__ bind(LENGTH_DIFF);
__ ret(lr);
+
+ // record the stub entry and end
+ store_archive_data(stub_id, entry, __ pc());
+
return entry;
}
@@ -9125,8 +9863,14 @@ class StubGenerator: public StubCodeGenerator {
case UU: stub_id = StubId::stubgen_compare_long_string_UU_id; break;
default: ShouldNotReachHere();
}
-
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
+ StubCodeMark mark(this, stub_id);
address entry = __ pc();
Register result = r0, str1 = r1, cnt1 = r2, str2 = r3, cnt2 = r4,
tmp1 = r10, tmp2 = r11;
@@ -9161,8 +9905,6 @@ class StubGenerator: public StubCodeGenerator {
ShouldNotReachHere(); \
}
- StubCodeMark mark(this, stub_id);
-
__ mov(idx, 0);
__ sve_whilelt(pgtmp1, mode == LL ? __ B : __ H, idx, cnt);
@@ -9206,6 +9948,10 @@ class StubGenerator: public StubCodeGenerator {
__ bind(DONE);
__ ret(lr);
#undef LOAD_PAIR
+
+ // record the stub entry and end
+ store_archive_data(stub_id, entry, __ pc());
+
return entry;
}
@@ -9267,6 +10013,12 @@ class StubGenerator: public StubCodeGenerator {
stub_id = StubId::stubgen_string_indexof_linear_uu_id;
}
}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
address entry = __ pc();
@@ -9535,6 +10287,10 @@ class StubGenerator: public StubCodeGenerator {
__ BIND(DONE);
__ pop(spilled_regs, sp);
__ ret(lr);
+
+ // record the stub entry and end
+ store_archive_data(stub_id, entry, __ pc());
+
return entry;
}
@@ -9565,8 +10321,14 @@ class StubGenerator: public StubCodeGenerator {
// v1 = loaded 8 bytes
// Clobbers: r0, r1, r3, rscratch1, rflags, v0-v6
address generate_large_byte_array_inflate() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_large_byte_array_inflate_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
address entry = __ pc();
Label LOOP, LOOP_START, LOOP_PRFM, LOOP_PRFM_START, DONE;
@@ -9605,6 +10367,10 @@ class StubGenerator: public StubCodeGenerator {
__ br(__ GE, LOOP);
__ bind(DONE);
__ ret(lr);
+
+ // record the stub entry and end
+ store_archive_data(stub_id, entry, __ pc());
+
return entry;
}
@@ -9620,7 +10386,7 @@ class StubGenerator: public StubCodeGenerator {
* Output:
* Updated state at c_rarg0
*/
- address generate_ghash_processBlocks() {
+ address generate_ghash_processBlocks_small() {
// Bafflingly, GCM uses little-endian for the byte order, but
// big-endian for the bit order. For example, the polynomial 1 is
// represented as the 16-byte string 80 00 00 00 | 12 bytes of 00.
@@ -9632,11 +10398,17 @@ class StubGenerator: public StubCodeGenerator {
// that) and keep the data in little-endian bit order through the
// calculation, bit-reversing the inputs and outputs.
- StubId stub_id = StubId::stubgen_ghash_processBlocks_id;
+ StubId stub_id = StubId::stubgen_ghash_processBlocks_small_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
Label polynomial; // local data generated at end of stub
- __ align(CodeEntryAlignment);
- address start = __ pc();
+ start = __ pc();
Register state = c_rarg0;
Register subkeyH = c_rarg1;
@@ -9696,17 +10468,24 @@ class StubGenerator: public StubCodeGenerator {
// 128-bit vector
__ emit_int64(0x87);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
- address generate_ghash_processBlocks_wide() {
- address small = generate_ghash_processBlocks();
-
- StubId stub_id = StubId::stubgen_ghash_processBlocks_wide_id;
- StubCodeMark mark(this, stub_id);
+ address generate_ghash_processBlocks(address small) {
+ StubId stub_id = StubId::stubgen_ghash_processBlocks_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
Label polynomial; // local data generated after stub
__ align(CodeEntryAlignment);
- address start = __ pc();
+ StubCodeMark mark(this, stub_id);
+ start = __ pc();
Register state = c_rarg0;
Register subkeyH = c_rarg1;
@@ -9748,8 +10527,10 @@ class StubGenerator: public StubCodeGenerator {
// 128-bit vector
__ emit_int64(0x87);
- return start;
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+ return start;
}
void generate_base64_encode_simdround(Register src, Register dst,
@@ -9800,26 +10581,16 @@ class StubGenerator: public StubCodeGenerator {
*/
address generate_base64_encodeBlock() {
- static const char toBase64[64] = {
- 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
- 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
- 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
- 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
- '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'
- };
-
- static const char toBase64URL[64] = {
- 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
- 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
- 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
- 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
- '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_'
- };
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_base64_encodeBlock_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Register src = c_rarg0; // source array
Register soff = c_rarg1; // source start offset
@@ -9839,9 +10610,9 @@ class StubGenerator: public StubCodeGenerator {
__ sub(length, send, soff);
// load the codec base address
- __ lea(codec, ExternalAddress((address) toBase64));
+ __ lea(codec, ExternalAddress((address) _encodeBlock_toBase64));
__ cbz(isURL, ProcessData);
- __ lea(codec, ExternalAddress((address) toBase64URL));
+ __ lea(codec, ExternalAddress((address) _encodeBlock_toBase64URL));
__ BIND(ProcessData);
@@ -9894,6 +10665,9 @@ class StubGenerator: public StubCodeGenerator {
__ BIND(Exit);
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -10015,80 +10789,16 @@ class StubGenerator: public StubCodeGenerator {
// on http://0x80.pl/articles/base64-simd-neon.html#encoding-quadwords, in section
// titled "Base64 decoding".
- // Non-SIMD lookup tables are mostly dumped from fromBase64 array used in java.util.Base64,
- // except the trailing character '=' is also treated illegal value in this intrinsic. That
- // is java.util.Base64.fromBase64['='] = -2, while fromBase(URL)64ForNoSIMD['='] = 255 here.
- static const uint8_t fromBase64ForNoSIMD[256] = {
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, 255u, 63u,
- 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, 14u,
- 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, 255u,
- 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, 40u,
- 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- };
-
- static const uint8_t fromBase64URLForNoSIMD[256] = {
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u,
- 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, 14u,
- 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, 63u,
- 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, 40u,
- 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- };
-
- // A legal value of base64 code is in range [0, 127]. We need two lookups
- // with tbl/tbx and combine them to get the decode data. The 1st table vector
- // lookup use tbl, out of range indices are set to 0 in destination. The 2nd
- // table vector lookup use tbx, out of range indices are unchanged in
- // destination. Input [64..126] is mapped to index [65, 127] in second lookup.
- // The value of index 64 is set to 0, so that we know that we already get the
- // decoded data with the 1st lookup.
- static const uint8_t fromBase64ForSIMD[128] = {
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, 255u, 63u,
- 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u,
- 0u, 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u,
- 14u, 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u,
- 255u, 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u,
- 40u, 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u,
- };
-
- static const uint8_t fromBase64URLForSIMD[128] = {
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
- 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u,
- 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u,
- 0u, 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u,
- 14u, 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u,
- 63u, 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u,
- 40u, 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u,
- };
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_base64_decodeBlock_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Register src = c_rarg0; // source array
Register soff = c_rarg1; // source start offset
@@ -10115,9 +10825,9 @@ class StubGenerator: public StubCodeGenerator {
__ sub(length, send, soff);
__ bfm(length, zr, 0, 1);
- __ lea(nosimd_codec, ExternalAddress((address) fromBase64ForNoSIMD));
+ __ lea(nosimd_codec, ExternalAddress((address) _decodeBlock_fromBase64ForNoSIMD));
__ cbz(isURL, ProcessData);
- __ lea(nosimd_codec, ExternalAddress((address) fromBase64URLForNoSIMD));
+ __ lea(nosimd_codec, ExternalAddress((address) _decodeBlock_fromBase64URLForNoSIMD));
__ BIND(ProcessData);
__ mov(rscratch1, length);
@@ -10162,9 +10872,9 @@ class StubGenerator: public StubCodeGenerator {
__ cbzw(rscratch1, Exit);
__ sub(length, length, 80);
- __ lea(simd_codec, ExternalAddress((address) fromBase64ForSIMD));
+ __ lea(simd_codec, ExternalAddress((address) _decodeBlock_fromBase64ForSIMD));
__ cbz(isURL, SIMDEnter);
- __ lea(simd_codec, ExternalAddress((address) fromBase64URLForSIMD));
+ __ lea(simd_codec, ExternalAddress((address) _decodeBlock_fromBase64URLForSIMD));
__ BIND(SIMDEnter);
__ ld1(v0, v1, v2, v3, __ T16B, __ post(simd_codec, 64));
@@ -10197,24 +10907,50 @@ class StubGenerator: public StubCodeGenerator {
__ leave();
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
// Support for spin waits.
address generate_spin_wait() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_spin_wait_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ spin_wait();
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
void generate_lookup_secondary_supers_table_stub() {
StubId stub_id = StubId::stubgen_lookup_secondary_supers_table_id;
+ GrowableArray entries;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == Klass::SECONDARY_SUPERS_TABLE_SIZE, "sanity check");
+ address start = load_archive_data(stub_id, &entries);
+ if (start != nullptr) {
+ assert(entries.length() == Klass::SECONDARY_SUPERS_TABLE_SIZE - 1,
+ "unexpected extra entry count %d", entries.length());
+ StubRoutines::_lookup_secondary_supers_table_stubs[0] = start;
+ for (int slot = 1; slot < Klass::SECONDARY_SUPERS_TABLE_SIZE; slot++) {
+ StubRoutines::_lookup_secondary_supers_table_stubs[slot] = entries.at(slot - 1);
+ }
+ return;
+ }
+
StubCodeMark mark(this, stub_id);
const Register
@@ -10229,7 +10965,13 @@ class StubGenerator: public StubCodeGenerator {
vtemp = v0;
for (int slot = 0; slot < Klass::SECONDARY_SUPERS_TABLE_SIZE; slot++) {
- StubRoutines::_lookup_secondary_supers_table_stubs[slot] = __ pc();
+ address next_entry = __ pc();
+ StubRoutines::_lookup_secondary_supers_table_stubs[slot] = next_entry;
+ if (slot == 0) {
+ start = next_entry;
+ } else {
+ entries.append(next_entry);
+ }
Label L_success;
__ enter();
__ lookup_secondary_supers_table_const(r_sub_klass, r_super_klass,
@@ -10239,14 +10981,21 @@ class StubGenerator: public StubCodeGenerator {
__ leave();
__ ret(lr);
}
+ // record the stub entry and end plus all the auxiliary entries
+ store_archive_data(stub_id, start, __ pc(), &entries);
}
// Slow path implementation for UseSecondarySupersTable.
address generate_lookup_secondary_supers_table_slow_path_stub() {
StubId stub_id = StubId::stubgen_lookup_secondary_supers_table_slow_path_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
-
- address start = __ pc();
+ start = __ pc();
const Register
r_super_klass = r0, // argument
r_array_base = r1, // argument
@@ -10258,6 +11007,9 @@ class StubGenerator: public StubCodeGenerator {
__ lookup_secondary_supers_table_slow_path(r_super_klass, r_array_base, r_array_index, r_bitmap, temp1, result);
__ ret(lr);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -10397,14 +11149,43 @@ class StubGenerator: public StubCodeGenerator {
if (! UseLSE) {
return;
}
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_atomic_entry_points_id;
- StubCodeMark mark(this, stub_id);
- address first_entry = __ pc();
+ GrowableArray entries;
+ int entry_count = StubInfo::entry_count(stub_id);
+ address start = load_archive_data(stub_id, &entries);
+ if (start != nullptr) {
+ assert(entries.length() == entry_count - 1,
+ "unexpected extra entry count %d", entries.length());
+ aarch64_atomic_fetch_add_4_impl = (aarch64_atomic_stub_t)start;
+ int idx = 0;
+ aarch64_atomic_fetch_add_8_impl = (aarch64_atomic_stub_t)entries.at(idx++);
+ aarch64_atomic_fetch_add_4_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++);
+ aarch64_atomic_fetch_add_8_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++);
+ aarch64_atomic_xchg_4_impl = (aarch64_atomic_stub_t)entries.at(idx++);
+ aarch64_atomic_xchg_8_impl = (aarch64_atomic_stub_t)entries.at(idx++);
+ aarch64_atomic_cmpxchg_1_impl = (aarch64_atomic_stub_t)entries.at(idx++);
+ aarch64_atomic_cmpxchg_4_impl = (aarch64_atomic_stub_t)entries.at(idx++);
+ aarch64_atomic_cmpxchg_8_impl = (aarch64_atomic_stub_t)entries.at(idx++);
+ aarch64_atomic_cmpxchg_1_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++);
+ aarch64_atomic_cmpxchg_4_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++);
+ aarch64_atomic_cmpxchg_8_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++);
+ aarch64_atomic_cmpxchg_4_release_impl = (aarch64_atomic_stub_t)entries.at(idx++);
+ aarch64_atomic_cmpxchg_8_release_impl = (aarch64_atomic_stub_t)entries.at(idx++);
+ aarch64_atomic_cmpxchg_4_seq_cst_impl = (aarch64_atomic_stub_t)entries.at(idx++);
+ aarch64_atomic_cmpxchg_8_seq_cst_impl = (aarch64_atomic_stub_t)entries.at(idx++);
+ assert(idx == entries.length(), "sanity!");
+ return;
+ }
+ __ align(CodeEntryAlignment);
+ StubCodeMark mark(this, stub_id);
+ start = __ pc();
+ address end;
+ {
// ADD, memory_order_conservative
AtomicStubMark mark_fetch_add_4(_masm, &aarch64_atomic_fetch_add_4_impl);
gen_ldadd_entry(Assembler::word, memory_order_conservative);
+
AtomicStubMark mark_fetch_add_8(_masm, &aarch64_atomic_fetch_add_8_impl);
gen_ldadd_entry(Assembler::xword, memory_order_conservative);
@@ -10412,6 +11193,7 @@ class StubGenerator: public StubCodeGenerator {
AtomicStubMark mark_fetch_add_4_relaxed
(_masm, &aarch64_atomic_fetch_add_4_relaxed_impl);
gen_ldadd_entry(MacroAssembler::word, memory_order_relaxed);
+
AtomicStubMark mark_fetch_add_8_relaxed
(_masm, &aarch64_atomic_fetch_add_8_relaxed_impl);
gen_ldadd_entry(MacroAssembler::xword, memory_order_relaxed);
@@ -10419,14 +11201,17 @@ class StubGenerator: public StubCodeGenerator {
// XCHG, memory_order_conservative
AtomicStubMark mark_xchg_4(_masm, &aarch64_atomic_xchg_4_impl);
gen_swpal_entry(Assembler::word);
- AtomicStubMark mark_xchg_8_impl(_masm, &aarch64_atomic_xchg_8_impl);
+
+ AtomicStubMark mark_xchg_8(_masm, &aarch64_atomic_xchg_8_impl);
gen_swpal_entry(Assembler::xword);
// CAS, memory_order_conservative
AtomicStubMark mark_cmpxchg_1(_masm, &aarch64_atomic_cmpxchg_1_impl);
gen_cas_entry(MacroAssembler::byte, memory_order_conservative);
+
AtomicStubMark mark_cmpxchg_4(_masm, &aarch64_atomic_cmpxchg_4_impl);
gen_cas_entry(MacroAssembler::word, memory_order_conservative);
+
AtomicStubMark mark_cmpxchg_8(_masm, &aarch64_atomic_cmpxchg_8_impl);
gen_cas_entry(MacroAssembler::xword, memory_order_conservative);
@@ -10434,9 +11219,11 @@ class StubGenerator: public StubCodeGenerator {
AtomicStubMark mark_cmpxchg_1_relaxed
(_masm, &aarch64_atomic_cmpxchg_1_relaxed_impl);
gen_cas_entry(MacroAssembler::byte, memory_order_relaxed);
+
AtomicStubMark mark_cmpxchg_4_relaxed
(_masm, &aarch64_atomic_cmpxchg_4_relaxed_impl);
gen_cas_entry(MacroAssembler::word, memory_order_relaxed);
+
AtomicStubMark mark_cmpxchg_8_relaxed
(_masm, &aarch64_atomic_cmpxchg_8_relaxed_impl);
gen_cas_entry(MacroAssembler::xword, memory_order_relaxed);
@@ -10444,6 +11231,7 @@ class StubGenerator: public StubCodeGenerator {
AtomicStubMark mark_cmpxchg_4_release
(_masm, &aarch64_atomic_cmpxchg_4_release_impl);
gen_cas_entry(MacroAssembler::word, memory_order_release);
+
AtomicStubMark mark_cmpxchg_8_release
(_masm, &aarch64_atomic_cmpxchg_8_release_impl);
gen_cas_entry(MacroAssembler::xword, memory_order_release);
@@ -10451,11 +11239,41 @@ class StubGenerator: public StubCodeGenerator {
AtomicStubMark mark_cmpxchg_4_seq_cst
(_masm, &aarch64_atomic_cmpxchg_4_seq_cst_impl);
gen_cas_entry(MacroAssembler::word, memory_order_seq_cst);
+
AtomicStubMark mark_cmpxchg_8_seq_cst
(_masm, &aarch64_atomic_cmpxchg_8_seq_cst_impl);
gen_cas_entry(MacroAssembler::xword, memory_order_seq_cst);
- ICache::invalidate_range(first_entry, __ pc() - first_entry);
+ end = __ pc();
+
+ ICache::invalidate_range(start, end - start);
+ // exit block to force update of AtomicStubMark targets
+ }
+
+ assert(start == (address)aarch64_atomic_fetch_add_4_impl,
+ "atomic stub should be at start of buffer");
+ // record the stub start and end plus all the entries saved by the
+ // AtomicStubMark destructor
+ entries.append((address)aarch64_atomic_fetch_add_8_impl);
+ entries.append((address)aarch64_atomic_fetch_add_4_relaxed_impl);
+ entries.append((address)aarch64_atomic_fetch_add_8_relaxed_impl);
+ entries.append((address)aarch64_atomic_xchg_4_impl);
+ entries.append((address)aarch64_atomic_xchg_8_impl);
+ entries.append((address)aarch64_atomic_cmpxchg_1_impl);
+ entries.append((address)aarch64_atomic_cmpxchg_4_impl);
+ entries.append((address)aarch64_atomic_cmpxchg_8_impl);
+ entries.append((address)aarch64_atomic_cmpxchg_1_relaxed_impl);
+ entries.append((address)aarch64_atomic_cmpxchg_4_relaxed_impl);
+ entries.append((address)aarch64_atomic_cmpxchg_8_relaxed_impl);
+ entries.append((address)aarch64_atomic_cmpxchg_4_release_impl);
+ entries.append((address)aarch64_atomic_cmpxchg_8_release_impl);
+ entries.append((address)aarch64_atomic_cmpxchg_4_seq_cst_impl);
+ entries.append((address)aarch64_atomic_cmpxchg_8_seq_cst_impl);
+
+ assert(entries.length() == entry_count - 1,
+ "unexpected extra entry count %d", entries.length());
+
+ store_archive_data(stub_id, start, end, &entries);
}
#endif // LINUX
@@ -10559,9 +11377,19 @@ class StubGenerator: public StubCodeGenerator {
if (!Continuations::enabled()) return nullptr;
StubId stub_id = StubId::stubgen_cont_thaw_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
generate_cont_thaw(Continuation::thaw_top);
+
+ // record the stub start and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -10570,11 +11398,20 @@ class StubGenerator: public StubCodeGenerator {
// TODO: will probably need multiple return barriers depending on return type
StubId stub_id = StubId::stubgen_cont_returnBarrier_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
generate_cont_thaw(Continuation::thaw_return_barrier);
+ // record the stub start and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -10582,19 +11419,34 @@ class StubGenerator: public StubCodeGenerator {
if (!Continuations::enabled()) return nullptr;
StubId stub_id = StubId::stubgen_cont_returnBarrierExc_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
generate_cont_thaw(Continuation::thaw_return_barrier_exception);
+ // record the stub start and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address generate_cont_preempt_stub() {
if (!Continuations::enabled()) return nullptr;
StubId stub_id = StubId::stubgen_cont_preempt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ reset_last_Java_frame(true);
@@ -10619,6 +11471,9 @@ class StubGenerator: public StubCodeGenerator {
__ ldr(rscratch1, Address(rscratch1));
__ br(rscratch1);
+ // record the stub start and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -10674,10 +11529,16 @@ class StubGenerator: public StubCodeGenerator {
// computation.
address generate_poly1305_processBlocks() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_poly1305_processBlocks_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label here;
__ enter();
RegSet callee_saved = RegSet::range(r19, r28);
@@ -10785,14 +11646,23 @@ class StubGenerator: public StubCodeGenerator {
__ leave();
__ ret(lr);
+ // record the stub start and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
// exception handler for upcall stubs
address generate_upcall_stub_exception_handler() {
StubId stub_id = StubId::stubgen_upcall_stub_exception_handler_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
// Native caller has no idea how to handle exceptions,
// so we just crash here. Up to callee to catch exceptions.
@@ -10801,6 +11671,9 @@ class StubGenerator: public StubCodeGenerator {
__ blr(rscratch1);
__ should_not_reach_here();
+ // record the stub start and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -10809,8 +11682,14 @@ class StubGenerator: public StubCodeGenerator {
// rmethod = result
address generate_upcall_stub_load_target() {
StubId stub_id = StubId::stubgen_upcall_stub_load_target_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ resolve_global_jobject(j_rarg0, rscratch1, rscratch2);
// Load target method from receiver
@@ -10824,6 +11703,9 @@ class StubGenerator: public StubCodeGenerator {
__ ret(lr);
+ // record the stub start and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -11223,8 +12105,6 @@ class StubGenerator: public StubCodeGenerator {
*/
address generate_multiply() {
Label argh, nothing;
- bind(argh);
- stop("MontgomeryMultiply total_allocation must be <= 8192");
align(CodeEntryAlignment);
address entry = pc();
@@ -11331,6 +12211,10 @@ class StubGenerator: public StubCodeGenerator {
bind(nothing);
ret(lr);
+ // handler for error case
+ bind(argh);
+ stop("MontgomeryMultiply total_allocation must be <= 8192");
+
return entry;
}
// In C, approximately:
@@ -11434,8 +12318,6 @@ class StubGenerator: public StubCodeGenerator {
*/
address generate_square() {
Label argh;
- bind(argh);
- stop("MontgomeryMultiply total_allocation must be <= 8192");
align(CodeEntryAlignment);
address entry = pc();
@@ -11544,6 +12426,10 @@ class StubGenerator: public StubCodeGenerator {
leave();
ret(lr);
+ // handler for error case
+ bind(argh);
+ stop("MontgomeryMultiply total_allocation must be <= 8192");
+
return entry;
}
// In C, approximately:
@@ -11753,7 +12639,7 @@ class StubGenerator: public StubCodeGenerator {
#if COMPILER2_OR_JVMCI
if (UseSVE == 0) {
- StubRoutines::aarch64::_vector_iota_indices = generate_iota_indices(StubId::stubgen_vector_iota_indices_id);
+ generate_iota_indices(StubId::stubgen_vector_iota_indices_id);
}
// array equals stub for large arrays.
@@ -11798,18 +12684,32 @@ class StubGenerator: public StubCodeGenerator {
if (UseMontgomeryMultiplyIntrinsic) {
StubId stub_id = StubId::stubgen_montgomeryMultiply_id;
- StubCodeMark mark(this, stub_id);
- MontgomeryMultiplyGenerator g(_masm, /*squaring*/false);
- StubRoutines::_montgomeryMultiply = g.generate_multiply();
+ address start = load_archive_data(stub_id);
+ if (start == nullptr) {
+ // we have to generate it
+ StubCodeMark mark(this, stub_id);
+ MontgomeryMultiplyGenerator g(_masm, /*squaring*/false);
+ start = g.generate_multiply();
+ // record the stub start and end
+ store_archive_data(stub_id, start, _masm->pc());
+ }
+ StubRoutines::_montgomeryMultiply = start;
}
if (UseMontgomerySquareIntrinsic) {
StubId stub_id = StubId::stubgen_montgomerySquare_id;
- StubCodeMark mark(this, stub_id);
- MontgomeryMultiplyGenerator g(_masm, /*squaring*/true);
- // We use generate_multiply() rather than generate_square()
- // because it's faster for the sizes of modulus we care about.
- StubRoutines::_montgomerySquare = g.generate_multiply();
+ address start = load_archive_data(stub_id);
+ if (start == nullptr) {
+ // we have to generate it
+ StubCodeMark mark(this, stub_id);
+ MontgomeryMultiplyGenerator g(_masm, /*squaring*/true);
+ // We use generate_multiply() rather than generate_square()
+ // because it's faster for the sizes of modulus we care about.
+ start = g.generate_multiply();
+ // record the stub start and end
+ store_archive_data(stub_id, start, _masm->pc());
+ }
+ StubRoutines::_montgomerySquare = start;
}
#endif // COMPILER2
@@ -11854,7 +12754,8 @@ class StubGenerator: public StubCodeGenerator {
}
if (UseGHASHIntrinsics) {
// StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks();
- StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks_wide();
+ StubRoutines::aarch64::_ghash_processBlocks_small = generate_ghash_processBlocks_small();
+ StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks(StubRoutines::aarch64::_ghash_processBlocks_small);
}
if (UseAESIntrinsics && UseGHASHIntrinsics) {
StubRoutines::_galoisCounterMode_AESCrypt = generate_galoisCounterMode_AESCrypt();
@@ -11898,7 +12799,7 @@ class StubGenerator: public StubCodeGenerator {
}
public:
- StubGenerator(CodeBuffer* code, BlobId blob_id) : StubCodeGenerator(code, blob_id) {
+ StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) : StubCodeGenerator(code, blob_id, stub_data) {
switch(blob_id) {
case BlobId::stubgen_preuniverse_id:
generate_preuniverse_stubs();
@@ -11920,12 +12821,35 @@ class StubGenerator: public StubCodeGenerator {
break;
};
}
+
+#if INCLUDE_CDS
+ static void init_AOTAddressTable(GrowableArray& external_addresses) {
+ // external data defined in this file
+#define ADD(addr) external_addresses.append((address)(addr));
+ ADD(_sha256_round_consts);
+ ADD(_sha512_round_consts);
+ ADD(_sha3_round_consts);
+ ADD(_double_keccak_round_consts);
+ ADD(_encodeBlock_toBase64);
+ ADD(_encodeBlock_toBase64URL);
+ ADD(_decodeBlock_fromBase64ForNoSIMD);
+ ADD(_decodeBlock_fromBase64URLForNoSIMD);
+ ADD(_decodeBlock_fromBase64ForSIMD);
+ ADD(_decodeBlock_fromBase64URLForSIMD);
+#undef ADD
+ }
+#endif // INCLUDE_CDS
}; // end class declaration
-void StubGenerator_generate(CodeBuffer* code, BlobId blob_id) {
- StubGenerator g(code, blob_id);
+void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) {
+ StubGenerator g(code, blob_id, stub_data);
}
+#if INCLUDE_CDS
+void StubGenerator_init_AOTAddressTable(GrowableArray& addresses) {
+ StubGenerator::init_AOTAddressTable(addresses);
+}
+#endif // INCLUDE_CDS
#if defined (LINUX)
diff --git a/src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp b/src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp
index 88993818b47..f02b681ca10 100644
--- a/src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp
@@ -41,8 +41,12 @@ static void empty_spin_wait() { }
#define DEFINE_ARCH_ENTRY_INIT(arch, blob_name, stub_name, field_name, getter_name, init_function) \
address StubRoutines:: arch :: STUB_FIELD_NAME(field_name) = CAST_FROM_FN_PTR(address, init_function);
-STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY, DEFINE_ARCH_ENTRY_INIT)
+#define DEFINE_ARCH_ENTRY_ARRAY(arch, blob_name, stub_name, field_name, getter_name, count) \
+ address StubRoutines:: arch :: STUB_FIELD_NAME(field_name) [count];
+STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY, DEFINE_ARCH_ENTRY_INIT, DEFINE_ARCH_ENTRY_ARRAY)
+
+#undef DEFINE_ARCH_ENTRY_ARARAY
#undef DEFINE_ARCH_ENTRY_INIT
#undef DEFINE_ARCH_ENTRY
@@ -413,3 +417,36 @@ ATTRIBUTE_ALIGNED(64) jdouble StubRoutines::aarch64::_pio2[] = {
2.73370053816464559624e-44, // 0x36E3822280000000
2.16741683877804819444e-51, // 0x3569F31D00000000
};
+
+#if INCLUDE_CDS
+extern void StubGenerator_init_AOTAddressTable(GrowableArray& addresses);
+
+void StubRoutines::init_AOTAddressTable() {
+ ResourceMark rm;
+ GrowableArray external_addresses;
+ // publish static addresses referred to by aarch64 generator
+ // n.b. we have to use use an extern call here because class
+ // StubGenerator, which provides the static method that knows how to
+ // add the relevant addresses, is declared in a source file rather
+ // than in a separately includeable header.
+ StubGenerator_init_AOTAddressTable(external_addresses);
+ // publish external data addresses defined in nested aarch64 class
+ StubRoutines::aarch64::init_AOTAddressTable(external_addresses);
+ AOTCodeCache::publish_external_addresses(external_addresses);
+}
+
+void StubRoutines::aarch64::init_AOTAddressTable(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr));
+ ADD(_kyberConsts);
+ ADD(_dilithiumConsts);
+ // this is added in generic code
+ // ADD(_crc_table);
+ ADD(_adler_table);
+ ADD(_npio2_hw);
+ ADD(_dsin_coef);
+ ADD(_dcos_coef);
+ ADD(_two_over_pi);
+ ADD(_pio2);
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp b/src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp
index c35371e1083..6067408ef13 100644
--- a/src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp
@@ -60,9 +60,13 @@ class aarch64 {
#define DECLARE_ARCH_ENTRY_INIT(arch, blob_name, stub_name, field_name, getter_name, init_function) \
DECLARE_ARCH_ENTRY(arch, blob_name, stub_name, field_name, getter_name)
-private:
- STUBGEN_ARCH_ENTRIES_DO(DECLARE_ARCH_ENTRY, DECLARE_ARCH_ENTRY_INIT)
+#define DECLARE_ARCH_ENTRY_ARRAY(arch, blob_name, stub_name, field_name, getter_name, count) \
+ static address STUB_FIELD_NAME(field_name) [count];
+private:
+ STUBGEN_ARCH_ENTRIES_DO(DECLARE_ARCH_ENTRY, DECLARE_ARCH_ENTRY_INIT, DECLARE_ARCH_ENTRY_ARRAY)
+
+#undef DECLARE_ARCH_ENTRY_ARRAY
#undef DECLARE_ARCH_ENTRY_INIT
#undef DECLARE_ARCH_ENTRY
@@ -78,8 +82,15 @@ private:
#define DEFINE_ARCH_ENTRY_GETTER_INIT(arch, blob_name, stub_name, field_name, getter_name, init_function) \
DEFINE_ARCH_ENTRY_GETTER(arch, blob_name, stub_name, field_name, getter_name)
- STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY_GETTER, DEFINE_ARCH_ENTRY_GETTER_INIT)
+#define DEFINE_ARCH_ENTRY_GETTER_ARRAY(arch, blob_name, stub_name, field_name, getter_name, count) \
+ static address getter_name(int idx) { \
+ assert(0 <= idx && idx < count, "entry array index out of range"); \
+ return STUB_FIELD_NAME(field_name) [idx]; \
+ }
+ STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY_GETTER, DEFINE_ARCH_ENTRY_GETTER_INIT, DEFINE_ARCH_ENTRY_GETTER_ARRAY)
+
+#undef DEFINE_ARCH_ENTRY_GETTER_ARRAY
#undef DEFINE_ARCH_ENTRY_GETTER_INIT
#undef DEFINE_ARCH_ENTRY_GETTER
@@ -110,6 +121,11 @@ private:
_completed = true;
}
+#if INCLUDE_CDS
+ static void init_AOTAddressTable(GrowableArray& external_addresses);
+#endif // INCLUDE_CDS
+
+
private:
static uint16_t _kyberConsts[];
static uint32_t _dilithiumConsts[];
diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
index 4423d9c5b58..441bd4859fe 100644
--- a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
@@ -1,7 +1,7 @@
/*
* Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2015, 2020, Red Hat Inc. All rights reserved.
- * Copyright 2025 Arm Limited and/or its affiliates.
+ * Copyright 2025, 2026 Arm Limited and/or its affiliates.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -24,6 +24,7 @@
*
*/
+#include "logging/log.hpp"
#include "pauth_aarch64.hpp"
#include "register_aarch64.hpp"
#include "runtime/arguments.hpp"
@@ -52,17 +53,56 @@ uintptr_t VM_Version::_pac_mask;
SpinWait VM_Version::_spin_wait;
+bool VM_Version::_cache_dic_enabled;
+bool VM_Version::_cache_idc_enabled;
+bool VM_Version::_ic_ivau_trapped;
+
const char* VM_Version::_features_names[MAX_CPU_FEATURES] = { nullptr };
static SpinWait get_spin_wait_desc() {
- SpinWait spin_wait(OnSpinWaitInst, OnSpinWaitInstCount);
+ SpinWait spin_wait(OnSpinWaitInst, OnSpinWaitInstCount, OnSpinWaitDelay);
if (spin_wait.inst() == SpinWait::SB && !VM_Version::supports_sb()) {
vm_exit_during_initialization("OnSpinWaitInst is SB but current CPU does not support SB instruction");
}
+ if (spin_wait.inst() == SpinWait::WFET) {
+ if (!VM_Version::supports_wfxt()) {
+ vm_exit_during_initialization("OnSpinWaitInst is WFET but the CPU does not support the WFET instruction");
+ }
+
+ if (!VM_Version::supports_ecv()) {
+ vm_exit_during_initialization("The CPU does not support the FEAT_ECV required by the -XX:OnSpinWaitInst=wfet implementation");
+ }
+
+ if (!VM_Version::supports_sb()) {
+ vm_exit_during_initialization("The CPU does not support the SB instruction required by the -XX:OnSpinWaitInst=wfet implementation");
+ }
+
+ if (OnSpinWaitInstCount != 1) {
+ vm_exit_during_initialization("OnSpinWaitInstCount for OnSpinWaitInst 'wfet' must be 1");
+ }
+ } else {
+ if (!FLAG_IS_DEFAULT(OnSpinWaitDelay)) {
+ vm_exit_during_initialization("OnSpinWaitDelay can only be used with -XX:OnSpinWaitInst=wfet");
+ }
+ }
+
return spin_wait;
}
+static bool has_neoverse_n1_errata_1542419() {
+ const int major_rev_num = VM_Version::cpu_variant();
+ const int minor_rev_num = VM_Version::cpu_revision();
+ // Neoverse N1: 0xd0c
+ // Erratum 1542419 affects r3p0, r3p1 and r4p0.
+ // It is fixed in r4p1 and later revisions, which are not affected.
+ return (VM_Version::cpu_family() == VM_Version::CPU_ARM &&
+ VM_Version::model_is(0xd0c) &&
+ ((major_rev_num == 3 && minor_rev_num == 0) ||
+ (major_rev_num == 3 && minor_rev_num == 1) ||
+ (major_rev_num == 4 && minor_rev_num == 0)));
+}
+
void VM_Version::initialize() {
#define SET_CPU_FEATURE_NAME(id, name, bit) \
_features_names[bit] = XSTR(name);
@@ -74,9 +114,14 @@ void VM_Version::initialize() {
_supports_atomic_getset8 = true;
_supports_atomic_getadd8 = true;
- get_os_cpu_info();
+ _cache_dic_enabled = false;
+ _cache_idc_enabled = false;
+ _ic_ivau_trapped = false;
- int dcache_line = VM_Version::dcache_line_size();
+ get_os_cpu_info();
+ _cpu_features = _features;
+
+ int dcache_line = dcache_line_size();
// Limit AllocatePrefetchDistance so that it does not exceed the
// static constraint of 512 defined in runtime/globals.hpp.
@@ -124,7 +169,7 @@ void VM_Version::initialize() {
// if dcpop is available publish data cache line flush size via
// generic field, otherwise let if default to zero thereby
// disabling writeback
- if (VM_Version::supports_dcpop()) {
+ if (supports_dcpop()) {
_data_cache_line_flush_size = dcache_line;
}
}
@@ -245,14 +290,24 @@ void VM_Version::initialize() {
}
}
- if (FLAG_IS_DEFAULT(UseCRC32)) {
- UseCRC32 = VM_Version::supports_crc32();
+ if (supports_sha1() || supports_sha256() ||
+ supports_sha3() || supports_sha512()) {
+ if (FLAG_IS_DEFAULT(UseSHA)) {
+ FLAG_SET_DEFAULT(UseSHA, true);
+ } else if (!UseSHA) {
+ clear_feature(CPU_SHA1);
+ clear_feature(CPU_SHA2);
+ clear_feature(CPU_SHA3);
+ clear_feature(CPU_SHA512);
+ }
+ } else if (UseSHA) {
+ warning("SHA instructions are not available on this CPU");
+ FLAG_SET_DEFAULT(UseSHA, false);
}
- if (UseCRC32 && !VM_Version::supports_crc32()) {
- warning("UseCRC32 specified, but not supported on this CPU");
- FLAG_SET_DEFAULT(UseCRC32, false);
- }
+ CHECK_CPU_FEATURE(supports_crc32, CRC32);
+ CHECK_CPU_FEATURE(supports_lse, LSE);
+ CHECK_CPU_FEATURE(supports_aes, AES);
if (_cpu == CPU_ARM &&
model_is_in({ CPU_MODEL_ARM_NEOVERSE_V1, CPU_MODEL_ARM_NEOVERSE_V2,
@@ -265,7 +320,7 @@ void VM_Version::initialize() {
}
}
- if (UseCryptoPmullForCRC32 && (!VM_Version::supports_pmull() || !VM_Version::supports_sha3() || !VM_Version::supports_crc32())) {
+ if (UseCryptoPmullForCRC32 && (!supports_pmull() || !supports_sha3() || !supports_crc32())) {
warning("UseCryptoPmullForCRC32 specified, but not supported on this CPU");
FLAG_SET_DEFAULT(UseCryptoPmullForCRC32, false);
}
@@ -279,48 +334,40 @@ void VM_Version::initialize() {
FLAG_SET_DEFAULT(UseVectorizedMismatchIntrinsic, false);
}
- if (VM_Version::supports_lse()) {
- if (FLAG_IS_DEFAULT(UseLSE))
- FLAG_SET_DEFAULT(UseLSE, true);
- } else {
- if (UseLSE) {
- warning("UseLSE specified, but not supported on this CPU");
- FLAG_SET_DEFAULT(UseLSE, false);
- }
- }
-
- if (VM_Version::supports_aes()) {
- UseAES = UseAES || FLAG_IS_DEFAULT(UseAES);
- UseAESIntrinsics =
- UseAESIntrinsics || (UseAES && FLAG_IS_DEFAULT(UseAESIntrinsics));
- if (UseAESIntrinsics && !UseAES) {
- warning("UseAESIntrinsics enabled, but UseAES not, enabling");
- UseAES = true;
+ if (supports_aes()) {
+ if (FLAG_IS_DEFAULT(UseAESIntrinsics)) {
+ FLAG_SET_DEFAULT(UseAESIntrinsics, true);
}
if (FLAG_IS_DEFAULT(UseAESCTRIntrinsics)) {
FLAG_SET_DEFAULT(UseAESCTRIntrinsics, true);
}
} else {
- if (UseAES) {
- warning("AES instructions are not available on this CPU");
- FLAG_SET_DEFAULT(UseAES, false);
- }
- if (UseAESIntrinsics) {
- warning("AES intrinsics are not available on this CPU");
- FLAG_SET_DEFAULT(UseAESIntrinsics, false);
- }
- if (UseAESCTRIntrinsics) {
- warning("AES/CTR intrinsics are not available on this CPU");
- FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
+ if (!UseAES) {
+ if (UseAESIntrinsics) {
+ warning("AES intrinsics require UseAES flag to be enabled. Intrinsics will be disabled.");
+ FLAG_SET_DEFAULT(UseAESIntrinsics, false);
+ }
+ if (UseAESCTRIntrinsics) {
+ warning("AES/CTR intrinsics require UseAES flag to be enabled. Intrinsics will be disabled.");
+ FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
+ }
+ } else if (!cpu_supports_aes()) {
+ if (UseAESIntrinsics) {
+ warning("AES intrinsics are not available on this CPU");
+ FLAG_SET_DEFAULT(UseAESIntrinsics, false);
+ }
+ if (UseAESCTRIntrinsics) {
+ warning("AES/CTR intrinsics are not available on this CPU");
+ FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
+ }
}
}
-
if (FLAG_IS_DEFAULT(UseCRC32Intrinsics)) {
UseCRC32Intrinsics = true;
}
- if (VM_Version::supports_crc32()) {
+ if (supports_crc32()) {
if (FLAG_IS_DEFAULT(UseCRC32CIntrinsics)) {
FLAG_SET_DEFAULT(UseCRC32CIntrinsics, true);
}
@@ -337,17 +384,7 @@ void VM_Version::initialize() {
UseMD5Intrinsics = true;
}
- if (VM_Version::supports_sha1() || VM_Version::supports_sha256() ||
- VM_Version::supports_sha3() || VM_Version::supports_sha512()) {
- if (FLAG_IS_DEFAULT(UseSHA)) {
- FLAG_SET_DEFAULT(UseSHA, true);
- }
- } else if (UseSHA) {
- warning("SHA instructions are not available on this CPU");
- FLAG_SET_DEFAULT(UseSHA, false);
- }
-
- if (UseSHA && VM_Version::supports_sha1()) {
+ if (UseSHA && supports_sha1()) {
if (FLAG_IS_DEFAULT(UseSHA1Intrinsics)) {
FLAG_SET_DEFAULT(UseSHA1Intrinsics, true);
}
@@ -356,7 +393,7 @@ void VM_Version::initialize() {
FLAG_SET_DEFAULT(UseSHA1Intrinsics, false);
}
- if (UseSHA && VM_Version::supports_sha256()) {
+ if (UseSHA && supports_sha256()) {
if (FLAG_IS_DEFAULT(UseSHA256Intrinsics)) {
FLAG_SET_DEFAULT(UseSHA256Intrinsics, true);
}
@@ -366,7 +403,7 @@ void VM_Version::initialize() {
}
if (UseSHA) {
- // No need to check VM_Version::supports_sha3(), since a fallback GPR intrinsic implementation is provided.
+ // No need to check supports_sha3(), since a fallback GPR intrinsic implementation is provided.
if (FLAG_IS_DEFAULT(UseSHA3Intrinsics)) {
FLAG_SET_DEFAULT(UseSHA3Intrinsics, true);
}
@@ -376,7 +413,7 @@ void VM_Version::initialize() {
FLAG_SET_DEFAULT(UseSHA3Intrinsics, false);
}
- if (UseSHA3Intrinsics && VM_Version::supports_sha3()) {
+ if (UseSHA3Intrinsics && supports_sha3()) {
// Auto-enable UseSIMDForSHA3Intrinsic on hardware with performance benefit.
// Note that the evaluation of SHA3 extension Intrinsics shows better performance
// on Apple and Qualcomm silicon but worse performance on Neoverse V1 and N2.
@@ -386,12 +423,12 @@ void VM_Version::initialize() {
}
}
}
- if (UseSHA3Intrinsics && UseSIMDForSHA3Intrinsic && !VM_Version::supports_sha3()) {
+ if (UseSHA3Intrinsics && UseSIMDForSHA3Intrinsic && !supports_sha3()) {
warning("Intrinsics for SHA3-224, SHA3-256, SHA3-384 and SHA3-512 crypto hash functions not available on this CPU.");
FLAG_SET_DEFAULT(UseSHA3Intrinsics, false);
}
- if (UseSHA && VM_Version::supports_sha512()) {
+ if (UseSHA && supports_sha512()) {
if (FLAG_IS_DEFAULT(UseSHA512Intrinsics)) {
FLAG_SET_DEFAULT(UseSHA512Intrinsics, true);
}
@@ -400,11 +437,7 @@ void VM_Version::initialize() {
FLAG_SET_DEFAULT(UseSHA512Intrinsics, false);
}
- if (!(UseSHA1Intrinsics || UseSHA256Intrinsics || UseSHA3Intrinsics || UseSHA512Intrinsics)) {
- FLAG_SET_DEFAULT(UseSHA, false);
- }
-
- if (VM_Version::supports_pmull()) {
+ if (supports_pmull()) {
if (FLAG_IS_DEFAULT(UseGHASHIntrinsics)) {
FLAG_SET_DEFAULT(UseGHASHIntrinsics, true);
}
@@ -455,7 +488,7 @@ void VM_Version::initialize() {
FLAG_SET_DEFAULT(UseBlockZeroing, true);
}
if (FLAG_IS_DEFAULT(BlockZeroingLowLimit)) {
- FLAG_SET_DEFAULT(BlockZeroingLowLimit, 4 * VM_Version::zva_length());
+ FLAG_SET_DEFAULT(BlockZeroingLowLimit, 4 * zva_length());
}
} else if (UseBlockZeroing) {
if (!FLAG_IS_DEFAULT(UseBlockZeroing)) {
@@ -464,11 +497,11 @@ void VM_Version::initialize() {
FLAG_SET_DEFAULT(UseBlockZeroing, false);
}
- if (VM_Version::supports_sve2()) {
+ if (supports_sve2()) {
if (FLAG_IS_DEFAULT(UseSVE)) {
FLAG_SET_DEFAULT(UseSVE, 2);
}
- } else if (VM_Version::supports_sve()) {
+ } else if (supports_sve()) {
if (FLAG_IS_DEFAULT(UseSVE)) {
FLAG_SET_DEFAULT(UseSVE, 1);
} else if (UseSVE > 1) {
@@ -519,7 +552,7 @@ void VM_Version::initialize() {
// 1) this code has been built with branch-protection and
// 2) the CPU/OS supports it
#ifdef __ARM_FEATURE_PAC_DEFAULT
- if (!VM_Version::supports_paca()) {
+ if (!supports_paca()) {
// Disable PAC to prevent illegal instruction crashes.
warning("ROP-protection specified, but not supported on this CPU. Disabling ROP-protection.");
} else {
@@ -661,6 +694,43 @@ void VM_Version::initialize() {
clear_feature(CPU_SVE);
}
+ if (FLAG_IS_DEFAULT(UseSingleICacheInvalidation) && is_cache_idc_enabled() && is_cache_dic_enabled()) {
+ FLAG_SET_DEFAULT(UseSingleICacheInvalidation, true);
+ }
+
+ if (FLAG_IS_DEFAULT(NeoverseN1ICacheErratumMitigation) && has_neoverse_n1_errata_1542419()
+ && is_cache_idc_enabled() && !is_cache_dic_enabled()) {
+ if (_ic_ivau_trapped) {
+ FLAG_SET_DEFAULT(NeoverseN1ICacheErratumMitigation, true);
+ } else {
+ log_info(os)("IC IVAU is not trapped; disabling NeoverseN1ICacheErratumMitigation");
+ FLAG_SET_DEFAULT(NeoverseN1ICacheErratumMitigation, false);
+ }
+ }
+
+ if (NeoverseN1ICacheErratumMitigation) {
+ if (!has_neoverse_n1_errata_1542419()) {
+ vm_exit_during_initialization("NeoverseN1ICacheErratumMitigation is set for the CPU not having Neoverse N1 errata 1542419");
+ }
+ // If the user explicitly set the flag, verify the trap is active.
+ if (!FLAG_IS_DEFAULT(NeoverseN1ICacheErratumMitigation) && !_ic_ivau_trapped) {
+ vm_exit_during_initialization("NeoverseN1ICacheErratumMitigation is set but IC IVAU is not trapped. "
+ "The optimization is not safe on this system.");
+ }
+ if (FLAG_IS_DEFAULT(UseSingleICacheInvalidation)) {
+ FLAG_SET_DEFAULT(UseSingleICacheInvalidation, true);
+ }
+
+ if (!UseSingleICacheInvalidation) {
+ vm_exit_during_initialization("NeoverseN1ICacheErratumMitigation is set but UseSingleICacheInvalidation is not enabled");
+ }
+ }
+
+ if (UseSingleICacheInvalidation
+ && (!is_cache_idc_enabled() || (!is_cache_dic_enabled() && !NeoverseN1ICacheErratumMitigation))) {
+ vm_exit_during_initialization("UseSingleICacheInvalidation is set but neither IDC nor DIC nor NeoverseN1ICacheErratumMitigation is enabled");
+ }
+
// Construct the "features" string
stringStream ss(512);
ss.print("0x%02x:0x%x:0x%03x:%d", _cpu, _variant, _model, _revision);
diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp
index e8681611234..30f1a5d86ca 100644
--- a/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp
@@ -58,6 +58,12 @@ protected:
// When _prefer_sve_merging_mode_cpy is true, `cpy (imm, zeroing)` is
// implemented as `movi; cpy(imm, merging)`.
static constexpr bool _prefer_sve_merging_mode_cpy = true;
+ static bool _cache_dic_enabled;
+ static bool _cache_idc_enabled;
+
+ // IC IVAU trap probe for Neoverse N1 erratum 1542419.
+ // Set by get_os_cpu_info() on Linux via ic_ivau_probe_linux_aarch64.S.
+ static bool _ic_ivau_trapped;
static SpinWait _spin_wait;
@@ -159,7 +165,9 @@ public:
/* flags above must follow Linux HWCAP */ \
decl(SVEBITPERM, svebitperm, 27) \
decl(SVE2, sve2, 28) \
- decl(A53MAC, a53mac, 31)
+ decl(A53MAC, a53mac, 31) \
+ decl(ECV, ecv, 32) \
+ decl(WFXT, wfxt, 33)
enum Feature_Flag {
#define DECLARE_CPU_FEATURE_FLAG(id, name, bit) CPU_##id = bit,
@@ -191,6 +199,8 @@ public:
return (features & BIT_MASK(flag)) != 0;
}
+ static bool cpu_supports_aes() { return supports_feature(_cpu_features, CPU_AES); }
+
static int cpu_family() { return _cpu; }
static int cpu_model() { return _model; }
static int cpu_model2() { return _model2; }
@@ -253,6 +263,10 @@ public:
return vector_length_in_bytes <= 16;
}
+ static bool is_cache_dic_enabled() { return _cache_dic_enabled; }
+ static bool is_cache_idc_enabled() { return _cache_idc_enabled; }
+ static bool is_ic_ivau_trapped() { return _ic_ivau_trapped; }
+
static void get_cpu_features_name(void* features_buffer, stringStream& ss);
// Returns names of features present in features_set1 but not in features_set2
diff --git a/src/hotspot/cpu/arm/c1_LIRGenerator_arm.cpp b/src/hotspot/cpu/arm/c1_LIRGenerator_arm.cpp
index 4c339968f85..46ec87290ae 100644
--- a/src/hotspot/cpu/arm/c1_LIRGenerator_arm.cpp
+++ b/src/hotspot/cpu/arm/c1_LIRGenerator_arm.cpp
@@ -1332,7 +1332,8 @@ void LIRGenerator::volatile_field_load(LIR_Address* address, LIR_Opr result,
load_addr = address;
}
__ volatile_load_mem_reg(load_addr, result, info);
- return;
+ } else {
+ __ load(address, result, info, lir_patch_none);
}
- __ load(address, result, info, lir_patch_none);
+ __ membar_acquire();
}
diff --git a/src/hotspot/cpu/arm/interp_masm_arm.cpp b/src/hotspot/cpu/arm/interp_masm_arm.cpp
index 23ecea24eb2..aee407864ee 100644
--- a/src/hotspot/cpu/arm/interp_masm_arm.cpp
+++ b/src/hotspot/cpu/arm/interp_masm_arm.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2008, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1210,7 +1210,7 @@ void InterpreterMacroAssembler::profile_final_call(Register mdp) {
// Sets mdp, blows Rtemp.
-void InterpreterMacroAssembler::profile_virtual_call(Register mdp, Register receiver, bool receiver_can_be_null) {
+void InterpreterMacroAssembler::profile_virtual_call(Register mdp, Register receiver) {
assert_different_registers(mdp, receiver, Rtemp);
if (ProfileInterpreter) {
@@ -1219,19 +1219,8 @@ void InterpreterMacroAssembler::profile_virtual_call(Register mdp, Register rece
// If no method data exists, go to profile_continue.
test_method_data_pointer(mdp, profile_continue);
- Label skip_receiver_profile;
- if (receiver_can_be_null) {
- Label not_null;
- cbnz(receiver, not_null);
- // We are making a call. Increment the count for null receiver.
- increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp);
- b(skip_receiver_profile);
- bind(not_null);
- }
-
// Record the receiver type.
record_klass_in_profile(receiver, mdp, Rtemp, true);
- bind(skip_receiver_profile);
// The method data pointer needs to be updated to reflect the new target.
update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size()));
diff --git a/src/hotspot/cpu/arm/interp_masm_arm.hpp b/src/hotspot/cpu/arm/interp_masm_arm.hpp
index 530be1c577e..147cd252b2c 100644
--- a/src/hotspot/cpu/arm/interp_masm_arm.hpp
+++ b/src/hotspot/cpu/arm/interp_masm_arm.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2008, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -239,8 +239,7 @@ class InterpreterMacroAssembler: public MacroAssembler {
void profile_call(Register mdp); // Sets mdp, blows Rtemp.
void profile_final_call(Register mdp); // Sets mdp, blows Rtemp.
- void profile_virtual_call(Register mdp, Register receiver, // Sets mdp, blows Rtemp.
- bool receiver_can_be_null = false);
+ void profile_virtual_call(Register mdp, Register receiver); // Sets mdp, blows Rtemp.
void profile_ret(Register mdp, Register return_bci); // Sets mdp, blows R0-R3/R0-R18, Rtemp, LR
void profile_null_seen(Register mdp); // Sets mdp.
void profile_typecheck(Register mdp, Register klass); // Sets mdp, blows Rtemp.
diff --git a/src/hotspot/cpu/arm/matcher_arm.hpp b/src/hotspot/cpu/arm/matcher_arm.hpp
index 6c818e1f20d..7978a5b7090 100644
--- a/src/hotspot/cpu/arm/matcher_arm.hpp
+++ b/src/hotspot/cpu/arm/matcher_arm.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -75,7 +75,6 @@
static bool narrow_klass_use_complex_address() {
NOT_LP64(ShouldNotCallThis());
- assert(UseCompressedClassPointers, "only for compressed klass code");
return false;
}
diff --git a/src/hotspot/cpu/arm/methodHandles_arm.cpp b/src/hotspot/cpu/arm/methodHandles_arm.cpp
index 3710fa33f36..2da14d8ffed 100644
--- a/src/hotspot/cpu/arm/methodHandles_arm.cpp
+++ b/src/hotspot/cpu/arm/methodHandles_arm.cpp
@@ -104,14 +104,13 @@ void MethodHandles::verify_ref_kind(MacroAssembler* _masm, int ref_kind, Registe
__ andr(temp, temp, (unsigned)java_lang_invoke_MemberName::MN_REFERENCE_KIND_MASK);
__ cmp(temp, ref_kind);
__ b(L, eq);
- { char* buf = NEW_C_HEAP_ARRAY(char, 100, mtInternal);
- jio_snprintf(buf, 100, "verify_ref_kind expected %x", ref_kind);
+ const char* msg = ref_kind_to_verify_msg(ref_kind);
if (ref_kind == JVM_REF_invokeVirtual ||
- ref_kind == JVM_REF_invokeSpecial)
+ ref_kind == JVM_REF_invokeSpecial) {
// could do this for all ref_kinds, but would explode assembly code size
- trace_method_handle(_masm, buf);
- __ stop(buf);
+ trace_method_handle(_masm, msg);
}
+ __ stop(msg);
BLOCK_COMMENT("} verify_ref_kind");
__ bind(L);
}
diff --git a/src/hotspot/cpu/arm/stubDeclarations_arm.hpp b/src/hotspot/cpu/arm/stubDeclarations_arm.hpp
index 5f768a205a5..5fb0d4e901f 100644
--- a/src/hotspot/cpu/arm/stubDeclarations_arm.hpp
+++ b/src/hotspot/cpu/arm/stubDeclarations_arm.hpp
@@ -29,7 +29,8 @@
#define STUBGEN_PREUNIVERSE_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(preuniverse, 500) \
do_stub(preuniverse, atomic_load_long) \
do_arch_entry(Arm, preuniverse, atomic_load_long, \
@@ -42,7 +43,8 @@
#define STUBGEN_INITIAL_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(initial, 9000) \
do_stub(initial, idiv_irem) \
do_arch_entry(Arm, initial, idiv_irem, \
@@ -51,14 +53,16 @@
#define STUBGEN_CONTINUATION_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(continuation, 2000) \
#define STUBGEN_COMPILER_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(compiler, 22000) \
do_stub(compiler, partial_subtype_check) \
do_arch_entry(Arm, compiler, partial_subtype_check, \
@@ -68,7 +72,8 @@
#define STUBGEN_FINAL_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(final, 22000) \
diff --git a/src/hotspot/cpu/arm/stubGenerator_arm.cpp b/src/hotspot/cpu/arm/stubGenerator_arm.cpp
index a36ad3a0c47..a705b15eff5 100644
--- a/src/hotspot/cpu/arm/stubGenerator_arm.cpp
+++ b/src/hotspot/cpu/arm/stubGenerator_arm.cpp
@@ -3211,7 +3211,7 @@ class StubGenerator: public StubCodeGenerator {
}
public:
- StubGenerator(CodeBuffer* code, BlobId blob_id) : StubCodeGenerator(code, blob_id) {
+ StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) : StubCodeGenerator(code, blob_id, stub_data) {
switch(blob_id) {
case BlobId::stubgen_preuniverse_id:
generate_preuniverse_stubs();
@@ -3235,8 +3235,8 @@ class StubGenerator: public StubCodeGenerator {
}
}; // end class declaration
-void StubGenerator_generate(CodeBuffer* code, BlobId blob_id) {
- StubGenerator g(code, blob_id);
+void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) {
+ StubGenerator g(code, blob_id, stub_data);
}
// implementation of internal development flag
diff --git a/src/hotspot/cpu/arm/stubRoutines_arm.cpp b/src/hotspot/cpu/arm/stubRoutines_arm.cpp
index a4f2b5e1bd9..38a9b298562 100644
--- a/src/hotspot/cpu/arm/stubRoutines_arm.cpp
+++ b/src/hotspot/cpu/arm/stubRoutines_arm.cpp
@@ -32,10 +32,16 @@
#define DEFINE_ARCH_ENTRY_INIT(arch, blob_name, stub_name, field_name, getter_name, init_function) \
address StubRoutines:: arch :: STUB_FIELD_NAME(field_name) = CAST_FROM_FN_PTR(address, init_function);
-STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY, DEFINE_ARCH_ENTRY_INIT)
+STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY, DEFINE_ARCH_ENTRY_INIT, DEFINE_ARCH_ENTRY_ARRAY)
#undef DEFINE_ARCH_ENTRY_INIT
#undef DEFINE_ARCH_ENTRY
address StubRoutines::crc_table_addr() { ShouldNotCallThis(); return nullptr; }
address StubRoutines::crc32c_table_addr() { ShouldNotCallThis(); return nullptr; }
+
+#if INCLUDE_CDS
+// nothing to do for arm
+void StubRoutines::init_AOTAddressTable() {
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/arm/stubRoutines_arm.hpp b/src/hotspot/cpu/arm/stubRoutines_arm.hpp
index 45ab10d14f9..29d96d0e653 100644
--- a/src/hotspot/cpu/arm/stubRoutines_arm.hpp
+++ b/src/hotspot/cpu/arm/stubRoutines_arm.hpp
@@ -55,9 +55,13 @@ class Arm {
#define DECLARE_ARCH_ENTRY_INIT(arch, blob_name, stub_name, field_name, getter_name, init_function) \
DECLARE_ARCH_ENTRY(arch, blob_name, stub_name, field_name, getter_name)
-private:
- STUBGEN_ARCH_ENTRIES_DO(DECLARE_ARCH_ENTRY, DECLARE_ARCH_ENTRY_INIT)
+#define DECLARE_ARCH_ENTRY_ARRAY(arch, blob_name, stub_name, field_name, getter_name, count) \
+ static address STUB_FIELD_NAME(field_name) [count] ;
+private:
+ STUBGEN_ARCH_ENTRIES_DO(DECLARE_ARCH_ENTRY, DECLARE_ARCH_ENTRY_INIT, DECLARE_ARCH_ENTRY_ARRAY)
+
+#undef DECLARE_ARCH_ENTRY_ARRAY
#undef DECLARE_ARCH_ENTRY_INIT
#undef DECLARE_ARCH_ENTRY
@@ -71,8 +75,12 @@ public:
#define DEFINE_ARCH_ENTRY_GETTER_INIT(arch, blob_name, stub_name, field_name, getter_name, init_function) \
DEFINE_ARCH_ENTRY_GETTER(arch, blob_name, stub_name, field_name, getter_name)
- STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY_GETTER, DEFINE_ARCH_ENTRY_GETTER_INIT)
+#define DEFINE_ARCH_ENTRY_GETTER_ARRAY(arch, blob_name, stub_name, field_name, getter_name, count) \
+ static address getter_name(int idx) { return STUB_FIELD_NAME(field_name) [idx] ; }
+ STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY_GETTER, DEFINE_ARCH_ENTRY_GETTER_INIT, DEFINE_ARCH_ENTRY_GETTER_ARRAY)
+
+#undef DEFINE_ARCH_ENTRY_GETTER_ARRAY
#undef DEFINE_ARCH_ENTRY_GETTER_INIT
#undef DEFINE_ARCH_ENTRY_GETTER
diff --git a/src/hotspot/cpu/ppc/c1_LIRGenerator_ppc.cpp b/src/hotspot/cpu/ppc/c1_LIRGenerator_ppc.cpp
index 5f030676bcb..a652a155f62 100644
--- a/src/hotspot/cpu/ppc/c1_LIRGenerator_ppc.cpp
+++ b/src/hotspot/cpu/ppc/c1_LIRGenerator_ppc.cpp
@@ -1143,6 +1143,7 @@ void LIRGenerator::volatile_field_load(LIR_Address* address, LIR_Opr result,
Unimplemented();
// __ volatile_load_mem_reg(address, result, info);
#endif
+ __ membar_acquire();
}
diff --git a/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp
index 798451446e5..4d7af0e4a71 100644
--- a/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2025 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -144,7 +144,7 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
if (len->is_valid()) {
stw(len, arrayOopDesc::length_offset_in_bytes(), obj);
- } else if (UseCompressedClassPointers && !UseCompactObjectHeaders) {
+ } else if (!UseCompactObjectHeaders) {
// Otherwise length is in the class gap.
store_klass_gap(obj);
}
diff --git a/src/hotspot/cpu/ppc/interp_masm_ppc.hpp b/src/hotspot/cpu/ppc/interp_masm_ppc.hpp
index 4ea33ebaf63..275ff92c699 100644
--- a/src/hotspot/cpu/ppc/interp_masm_ppc.hpp
+++ b/src/hotspot/cpu/ppc/interp_masm_ppc.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2002, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2002, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2025 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -258,7 +258,7 @@ class InterpreterMacroAssembler: public MacroAssembler {
void profile_not_taken_branch(Register scratch1, Register scratch2);
void profile_call(Register scratch1, Register scratch2);
void profile_final_call(Register scratch1, Register scratch2);
- void profile_virtual_call(Register Rreceiver, Register Rscratch1, Register Rscratch2, bool receiver_can_be_null);
+ void profile_virtual_call(Register Rreceiver, Register Rscratch1, Register Rscratch2);
void profile_typecheck(Register Rklass, Register Rscratch1, Register Rscratch2);
void profile_ret(TosState state, Register return_bci, Register scratch1, Register scratch2);
void profile_switch_default(Register scratch1, Register scratch2);
diff --git a/src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp b/src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp
index f7bf457f72c..56eade8e533 100644
--- a/src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp
+++ b/src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2025 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -1340,28 +1340,15 @@ void InterpreterMacroAssembler::profile_final_call(Register scratch1, Register s
// Count a virtual call in the bytecodes.
void InterpreterMacroAssembler::profile_virtual_call(Register Rreceiver,
Register Rscratch1,
- Register Rscratch2,
- bool receiver_can_be_null) {
+ Register Rscratch2) {
if (!ProfileInterpreter) { return; }
Label profile_continue;
// If no method data exists, go to profile_continue.
test_method_data_pointer(profile_continue);
- Label skip_receiver_profile;
- if (receiver_can_be_null) {
- Label not_null;
- cmpdi(CR0, Rreceiver, 0);
- bne(CR0, not_null);
- // We are making a call. Increment the count for null receiver.
- increment_mdp_data_at(in_bytes(CounterData::count_offset()), Rscratch1, Rscratch2);
- b(skip_receiver_profile);
- bind(not_null);
- }
-
// Record the receiver type.
record_klass_in_profile(Rreceiver, Rscratch1, Rscratch2);
- bind(skip_receiver_profile);
// The method data pointer needs to be updated to reflect the new target.
update_mdp_by_constant(in_bytes(VirtualCallData::virtual_call_data_size()));
diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp
index 14e90ddf185..5fbcce94029 100644
--- a/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp
@@ -3201,23 +3201,17 @@ Register MacroAssembler::encode_klass_not_null(Register dst, Register src) {
void MacroAssembler::store_klass(Register dst_oop, Register klass, Register ck) {
assert(!UseCompactObjectHeaders, "not with compact headers");
- if (UseCompressedClassPointers) {
- Register compressedKlass = encode_klass_not_null(ck, klass);
- stw(compressedKlass, oopDesc::klass_offset_in_bytes(), dst_oop);
- } else {
- std(klass, oopDesc::klass_offset_in_bytes(), dst_oop);
- }
+ Register compressedKlass = encode_klass_not_null(ck, klass);
+ stw(compressedKlass, oopDesc::klass_offset_in_bytes(), dst_oop);
}
void MacroAssembler::store_klass_gap(Register dst_oop, Register val) {
assert(!UseCompactObjectHeaders, "not with compact headers");
- if (UseCompressedClassPointers) {
- if (val == noreg) {
- val = R0;
- li(val, 0);
- }
- stw(val, oopDesc::klass_gap_offset_in_bytes(), dst_oop);
+ if (val == noreg) {
+ val = R0;
+ li(val, 0);
}
+ stw(val, oopDesc::klass_gap_offset_in_bytes(), dst_oop);
}
int MacroAssembler::instr_size_for_decode_klass_not_null() {
@@ -3226,17 +3220,13 @@ int MacroAssembler::instr_size_for_decode_klass_not_null() {
// Not yet computed?
if (computed_size == -1) {
- if (!UseCompressedClassPointers) {
- computed_size = 0;
- } else {
- // Determine by scratch emit.
- ResourceMark rm;
- int code_size = 8 * BytesPerInstWord;
- CodeBuffer cb("decode_klass_not_null scratch buffer", code_size, 0);
- MacroAssembler* a = new MacroAssembler(&cb);
- a->decode_klass_not_null(R11_scratch1);
- computed_size = a->offset();
- }
+ // Determine by scratch emit.
+ ResourceMark rm;
+ int code_size = 8 * BytesPerInstWord;
+ CodeBuffer cb("decode_klass_not_null scratch buffer", code_size, 0);
+ MacroAssembler* a = new MacroAssembler(&cb);
+ a->decode_klass_not_null(R11_scratch1);
+ computed_size = a->offset();
}
return computed_size;
@@ -3259,18 +3249,14 @@ void MacroAssembler::decode_klass_not_null(Register dst, Register src) {
void MacroAssembler::load_klass_no_decode(Register dst, Register src) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(dst, src);
- } else if (UseCompressedClassPointers) {
- lwz(dst, oopDesc::klass_offset_in_bytes(), src);
} else {
- ld(dst, oopDesc::klass_offset_in_bytes(), src);
+ lwz(dst, oopDesc::klass_offset_in_bytes(), src);
}
}
void MacroAssembler::load_klass(Register dst, Register src) {
load_klass_no_decode(dst, src);
- if (UseCompressedClassPointers) { // also true for UseCompactObjectHeaders
- decode_klass_not_null(dst);
- }
+ decode_klass_not_null(dst);
}
// Loads the obj's Klass* into dst.
@@ -3286,18 +3272,13 @@ void MacroAssembler::load_narrow_klass_compact(Register dst, Register src) {
void MacroAssembler::cmp_klass(ConditionRegister dst, Register obj, Register klass, Register tmp, Register tmp2) {
assert_different_registers(obj, klass, tmp);
- if (UseCompressedClassPointers) {
- if (UseCompactObjectHeaders) {
- load_narrow_klass_compact(tmp, obj);
- } else {
- lwz(tmp, oopDesc::klass_offset_in_bytes(), obj);
- }
- Register encoded_klass = encode_klass_not_null(tmp2, klass);
- cmpw(dst, tmp, encoded_klass);
+ if (UseCompactObjectHeaders) {
+ load_narrow_klass_compact(tmp, obj);
} else {
- ld(tmp, oopDesc::klass_offset_in_bytes(), obj);
- cmpd(dst, tmp, klass);
+ lwz(tmp, oopDesc::klass_offset_in_bytes(), obj);
}
+ Register encoded_klass = encode_klass_not_null(tmp2, klass);
+ cmpw(dst, tmp, encoded_klass);
}
void MacroAssembler::cmp_klasses_from_objects(ConditionRegister dst, Register obj1, Register obj2, Register tmp1, Register tmp2) {
@@ -3305,14 +3286,10 @@ void MacroAssembler::cmp_klasses_from_objects(ConditionRegister dst, Register ob
load_narrow_klass_compact(tmp1, obj1);
load_narrow_klass_compact(tmp2, obj2);
cmpw(dst, tmp1, tmp2);
- } else if (UseCompressedClassPointers) {
+ } else {
lwz(tmp1, oopDesc::klass_offset_in_bytes(), obj1);
lwz(tmp2, oopDesc::klass_offset_in_bytes(), obj2);
cmpw(dst, tmp1, tmp2);
- } else {
- ld(tmp1, oopDesc::klass_offset_in_bytes(), obj1);
- ld(tmp2, oopDesc::klass_offset_in_bytes(), obj2);
- cmpd(dst, tmp1, tmp2);
}
}
diff --git a/src/hotspot/cpu/ppc/matcher_ppc.hpp b/src/hotspot/cpu/ppc/matcher_ppc.hpp
index 2ddbec3e48c..cbe882648b8 100644
--- a/src/hotspot/cpu/ppc/matcher_ppc.hpp
+++ b/src/hotspot/cpu/ppc/matcher_ppc.hpp
@@ -87,7 +87,6 @@
static bool narrow_klass_use_complex_address() {
NOT_LP64(ShouldNotCallThis());
- assert(UseCompressedClassPointers, "only for compressed klass code");
// TODO: PPC port if (MatchDecodeNodes) return true;
return false;
}
diff --git a/src/hotspot/cpu/ppc/methodHandles_ppc.cpp b/src/hotspot/cpu/ppc/methodHandles_ppc.cpp
index 45537e0ea96..ae94a9618b5 100644
--- a/src/hotspot/cpu/ppc/methodHandles_ppc.cpp
+++ b/src/hotspot/cpu/ppc/methodHandles_ppc.cpp
@@ -104,14 +104,13 @@ void MethodHandles::verify_ref_kind(MacroAssembler* _masm, int ref_kind, Registe
__ andi(temp, temp, java_lang_invoke_MemberName::MN_REFERENCE_KIND_MASK);
__ cmpwi(CR1, temp, ref_kind);
__ beq(CR1, L);
- { char* buf = NEW_C_HEAP_ARRAY(char, 100, mtInternal);
- jio_snprintf(buf, 100, "verify_ref_kind expected %x", ref_kind);
- if (ref_kind == JVM_REF_invokeVirtual ||
- ref_kind == JVM_REF_invokeSpecial)
- // could do this for all ref_kinds, but would explode assembly code size
- trace_method_handle(_masm, buf);
- __ stop(buf);
+ const char* msg = ref_kind_to_verify_msg(ref_kind);
+ if (ref_kind == JVM_REF_invokeVirtual ||
+ ref_kind == JVM_REF_invokeSpecial) {
+ // could do this for all ref_kinds, but would explode assembly code size
+ trace_method_handle(_masm, msg);
}
+ __ stop(msg);
BLOCK_COMMENT("} verify_ref_kind");
__ BIND(L);
}
diff --git a/src/hotspot/cpu/ppc/registerMap_ppc.cpp b/src/hotspot/cpu/ppc/registerMap_ppc.cpp
new file mode 100644
index 00000000000..2e7f8af89d3
--- /dev/null
+++ b/src/hotspot/cpu/ppc/registerMap_ppc.cpp
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2026 SAP SE. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "runtime/registerMap.hpp"
+
+address RegisterMap::pd_location(VMReg base_reg, int slot_idx) const {
+ if (base_reg->is_VectorRegister()) {
+ // Not all physical slots belonging to a VectorRegister have corresponding
+ // valid VMReg locations in the RegisterMap.
+ // (See RegisterSaver::push_frame_reg_args_and_save_live_registers.)
+ // However, the slots are always saved to the stack in a contiguous region
+ // of memory so we can calculate the address of the upper slots by
+ // offsetting from the base address.
+ assert(base_reg->is_concrete(), "must pass base reg");
+ address base_location = location(base_reg, nullptr);
+ if (base_location != nullptr) {
+ intptr_t offset_in_bytes = slot_idx * VMRegImpl::stack_slot_size;
+ return base_location + offset_in_bytes;
+ } else {
+ return nullptr;
+ }
+ } else {
+ return location(base_reg->next(slot_idx), nullptr);
+ }
+}
diff --git a/src/hotspot/cpu/ppc/registerMap_ppc.hpp b/src/hotspot/cpu/ppc/registerMap_ppc.hpp
index 01eb642107c..607c712d10f 100644
--- a/src/hotspot/cpu/ppc/registerMap_ppc.hpp
+++ b/src/hotspot/cpu/ppc/registerMap_ppc.hpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2013 SAP SE. All rights reserved.
+ * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -35,9 +35,7 @@
// Since there is none, we just return null.
address pd_location(VMReg reg) const { return nullptr; }
- address pd_location(VMReg base_reg, int slot_idx) const {
- return location(base_reg->next(slot_idx), nullptr);
- }
+ address pd_location(VMReg base_reg, int slot_idx) const;
// no PD state to clear or copy:
void pd_clear() {}
diff --git a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
index 5260ed978ff..53644210415 100644
--- a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
+++ b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
@@ -102,7 +102,7 @@ class RegisterSaver {
// During deoptimization only the result registers need to be restored
// all the other values have already been extracted.
- static void restore_result_registers(MacroAssembler* masm, int frame_size_in_bytes);
+ static void restore_result_registers(MacroAssembler* masm, int frame_size_in_bytes, bool save_vectors);
// Constants and data structures:
@@ -349,7 +349,7 @@ OopMap* RegisterSaver::push_frame_reg_args_and_save_live_registers(MacroAssemble
}
// Note that generate_oop_map in the following loop is only used for the
- // polling_page_vectors_safepoint_handler_blob.
+ // polling_page_vectors_safepoint_handler_blob and the deopt_blob.
// The order in which the vector contents are stored depends on Endianess and
// the utilized instructions (PowerArchitecturePPC64).
assert(is_aligned(offset, StackAlignmentInBytes), "should be");
@@ -361,6 +361,7 @@ OopMap* RegisterSaver::push_frame_reg_args_and_save_live_registers(MacroAssemble
__ stxvp(as_VectorRegister(reg_num).to_vsr(), offset, R1_SP);
// Note: The contents were read in the same order (see loadV16_Power9 node in ppc.ad).
+ // RegisterMap::pd_location only uses the first VMReg for each VectorRegister.
if (generate_oop_map) {
map->set_callee_saved(VMRegImpl::stack2reg(offset >> 2),
RegisterSaver_LiveVecRegs[i LITTLE_ENDIAN_ONLY(+1) ].vmreg);
@@ -380,6 +381,7 @@ OopMap* RegisterSaver::push_frame_reg_args_and_save_live_registers(MacroAssemble
__ stxvd2x(as_VectorRegister(reg_num)->to_vsr(), R31, R1_SP);
}
// Note: The contents were read in the same order (see loadV16_Power8 / loadV16_Power9 node in ppc.ad).
+ // RegisterMap::pd_location only uses the first VMReg for each VectorRegister.
if (generate_oop_map) {
VMReg vsr = RegisterSaver_LiveVecRegs[i].vmreg;
map->set_callee_saved(VMRegImpl::stack2reg(offset >> 2), vsr);
@@ -566,10 +568,14 @@ void RegisterSaver::restore_argument_registers_and_pop_frame(MacroAssembler*masm
}
// Restore the registers that might be holding a result.
-void RegisterSaver::restore_result_registers(MacroAssembler* masm, int frame_size_in_bytes) {
+void RegisterSaver::restore_result_registers(MacroAssembler* masm, int frame_size_in_bytes, bool save_vectors) {
const int regstosave_num = sizeof(RegisterSaver_LiveRegs) /
sizeof(RegisterSaver::LiveRegType);
- const int register_save_size = regstosave_num * reg_size; // VS registers not relevant here.
+ const int vecregstosave_num = save_vectors ? (sizeof(RegisterSaver_LiveVecRegs) /
+ sizeof(RegisterSaver::LiveRegType))
+ : 0;
+ const int register_save_size = regstosave_num * reg_size + vecregstosave_num * vec_reg_size;
+
const int register_save_offset = frame_size_in_bytes - register_save_size;
// restore all result registers (ints and floats)
@@ -598,7 +604,7 @@ void RegisterSaver::restore_result_registers(MacroAssembler* masm, int frame_siz
offset += reg_size;
}
- assert(offset == frame_size_in_bytes, "consistency check");
+ assert(offset == frame_size_in_bytes - (save_vectors ? vecregstosave_num * vec_reg_size : 0), "consistency check");
}
// Is vector's size (in bytes) bigger than a size saved by default?
@@ -2909,7 +2915,8 @@ void SharedRuntime::generate_deopt_blob() {
map = RegisterSaver::push_frame_reg_args_and_save_live_registers(masm,
&first_frame_size_in_bytes,
/*generate_oop_map=*/ true,
- RegisterSaver::return_pc_is_lr);
+ RegisterSaver::return_pc_is_lr,
+ /*save_vectors*/ SuperwordUseVSX);
assert(map != nullptr, "OopMap must have been created");
__ li(exec_mode_reg, Deoptimization::Unpack_deopt);
@@ -2943,7 +2950,8 @@ void SharedRuntime::generate_deopt_blob() {
RegisterSaver::push_frame_reg_args_and_save_live_registers(masm,
&first_frame_size_in_bytes,
/*generate_oop_map=*/ false,
- RegisterSaver::return_pc_is_pre_saved);
+ RegisterSaver::return_pc_is_pre_saved,
+ /*save_vectors*/ SuperwordUseVSX);
// Deopt during an exception. Save exec mode for unpack_frames.
__ li(exec_mode_reg, Deoptimization::Unpack_exception);
@@ -2958,7 +2966,8 @@ void SharedRuntime::generate_deopt_blob() {
RegisterSaver::push_frame_reg_args_and_save_live_registers(masm,
&first_frame_size_in_bytes,
/*generate_oop_map=*/ false,
- RegisterSaver::return_pc_is_pre_saved);
+ RegisterSaver::return_pc_is_pre_saved,
+ /*save_vectors*/ SuperwordUseVSX);
__ li(exec_mode_reg, Deoptimization::Unpack_reexecute);
#endif
@@ -2984,7 +2993,7 @@ void SharedRuntime::generate_deopt_blob() {
// Restore only the result registers that have been saved
// by save_volatile_registers(...).
- RegisterSaver::restore_result_registers(masm, first_frame_size_in_bytes);
+ RegisterSaver::restore_result_registers(masm, first_frame_size_in_bytes, /*save_vectors*/ SuperwordUseVSX);
// reload the exec mode from the UnrollBlock (it might have changed)
__ lwz(exec_mode_reg, in_bytes(Deoptimization::UnrollBlock::unpack_kind_offset()), unroll_block_reg);
diff --git a/src/hotspot/cpu/ppc/stubDeclarations_ppc.hpp b/src/hotspot/cpu/ppc/stubDeclarations_ppc.hpp
index be51afe42a4..41b8b71486d 100644
--- a/src/hotspot/cpu/ppc/stubDeclarations_ppc.hpp
+++ b/src/hotspot/cpu/ppc/stubDeclarations_ppc.hpp
@@ -29,35 +29,40 @@
#define STUBGEN_PREUNIVERSE_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(preuniverse, 0) \
#define STUBGEN_INITIAL_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(initial, 20000) \
#define STUBGEN_CONTINUATION_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(continuation, 2000) \
#define STUBGEN_COMPILER_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(compiler, 24000) \
#define STUBGEN_FINAL_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(final, 24000) \
diff --git a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp
index e48778a8b9f..f528587a8bb 100644
--- a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp
+++ b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp
@@ -5095,7 +5095,7 @@ void generate_lookup_secondary_supers_table_stub() {
}
public:
- StubGenerator(CodeBuffer* code, BlobId blob_id) : StubCodeGenerator(code, blob_id) {
+ StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData *stub_data) : StubCodeGenerator(code, blob_id, stub_data) {
switch(blob_id) {
case BlobId::stubgen_preuniverse_id:
generate_preuniverse_stubs();
@@ -5119,7 +5119,7 @@ void generate_lookup_secondary_supers_table_stub() {
}
};
-void StubGenerator_generate(CodeBuffer* code, BlobId blob_id) {
- StubGenerator g(code, blob_id);
+void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData *stub_data) {
+ StubGenerator g(code, blob_id, stub_data);
}
diff --git a/src/hotspot/cpu/ppc/stubRoutines_ppc_64.cpp b/src/hotspot/cpu/ppc/stubRoutines_ppc_64.cpp
index 914c5a17a19..3b7ee66348a 100644
--- a/src/hotspot/cpu/ppc/stubRoutines_ppc_64.cpp
+++ b/src/hotspot/cpu/ppc/stubRoutines_ppc_64.cpp
@@ -183,3 +183,9 @@ address StubRoutines::ppc::generate_crc_constants(juint reverse_poly) {
return consts;
}
+
+#if INCLUDE_CDS
+// nothing to do for ppc
+void StubRoutines::init_AOTAddressTable() {
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp b/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp
index 8a3af748fa1..37f780535b4 100644
--- a/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp
+++ b/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp
@@ -3489,7 +3489,7 @@ void TemplateTable::invokevirtual(int byte_no) {
// Get receiver klass.
__ load_klass_check_null_throw(Rrecv_klass, Rrecv, R11_scratch1);
__ verify_klass_ptr(Rrecv_klass);
- __ profile_virtual_call(Rrecv_klass, R11_scratch1, R12_scratch2, false);
+ __ profile_virtual_call(Rrecv_klass, R11_scratch1, R12_scratch2);
generate_vtable_call(Rrecv_klass, Rvtableindex_or_method, Rret_addr, R11_scratch1);
}
@@ -3596,7 +3596,7 @@ void TemplateTable::invokeinterface_object_method(Register Rrecv_klass,
// Non-final callc case.
__ bind(LnotFinal);
__ lhz(Rindex, in_bytes(ResolvedMethodEntry::table_index_offset()), Rcache);
- __ profile_virtual_call(Rrecv_klass, Rtemp1, Rscratch, false);
+ __ profile_virtual_call(Rrecv_klass, Rtemp1, Rscratch);
generate_vtable_call(Rrecv_klass, Rindex, Rret, Rscratch);
}
@@ -3664,7 +3664,7 @@ void TemplateTable::invokeinterface(int byte_no) {
__ lookup_interface_method(Rrecv_klass, Rinterface_klass, noreg, noreg, Rscratch1, Rscratch2,
L_no_such_interface, /*return_method=*/false);
- __ profile_virtual_call(Rrecv_klass, Rscratch1, Rscratch2, false);
+ __ profile_virtual_call(Rrecv_klass, Rscratch1, Rscratch2);
// Find entry point to call.
diff --git a/src/hotspot/cpu/ppc/vm_version_ppc.cpp b/src/hotspot/cpu/ppc/vm_version_ppc.cpp
index e471f5a6e4f..3e3b1103c86 100644
--- a/src/hotspot/cpu/ppc/vm_version_ppc.cpp
+++ b/src/hotspot/cpu/ppc/vm_version_ppc.cpp
@@ -25,6 +25,7 @@
#include "asm/assembler.inline.hpp"
#include "asm/macroAssembler.inline.hpp"
+#include "compiler/compilerDefinitions.inline.hpp"
#include "compiler/disassembler.hpp"
#include "jvm.h"
#include "memory/resourceArea.hpp"
@@ -105,7 +106,7 @@ void VM_Version::initialize() {
if (PowerArchitecturePPC64 >= 9) {
// Performance is good since Power9.
- if (FLAG_IS_DEFAULT(SuperwordUseVSX)) {
+ if (FLAG_IS_DEFAULT(SuperwordUseVSX) && CompilerConfig::is_c2_enabled()) {
FLAG_SET_ERGO(SuperwordUseVSX, true);
}
}
@@ -310,11 +311,6 @@ void VM_Version::initialize() {
FLAG_SET_DEFAULT(UseSHA3Intrinsics, false);
}
- if (!(UseSHA1Intrinsics || UseSHA256Intrinsics || UseSHA512Intrinsics)) {
- FLAG_SET_DEFAULT(UseSHA, false);
- }
-
-
#ifdef COMPILER2
if (FLAG_IS_DEFAULT(UseSquareToLenIntrinsic)) {
UseSquareToLenIntrinsic = true;
diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.cpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.cpp
index 819d6c05654..8aced227a06 100644
--- a/src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -196,12 +196,9 @@ void LIR_Assembler::arraycopy_type_check(Register src, Register src_pos, Registe
if (UseCompactObjectHeaders) {
__ load_narrow_klass_compact(tmp, src);
__ load_narrow_klass_compact(t0, dst);
- } else if (UseCompressedClassPointers) {
+ } else {
__ lwu(tmp, Address(src, oopDesc::klass_offset_in_bytes()));
__ lwu(t0, Address(dst, oopDesc::klass_offset_in_bytes()));
- } else {
- __ ld(tmp, Address(src, oopDesc::klass_offset_in_bytes()));
- __ ld(t0, Address(dst, oopDesc::klass_offset_in_bytes()));
}
__ bne(tmp, t0, *stub->entry(), /* is_far */ true);
} else {
@@ -243,37 +240,6 @@ void LIR_Assembler::arraycopy_type_check(Register src, Register src_pos, Registe
}
}
-void LIR_Assembler::arraycopy_assert(Register src, Register dst, Register tmp, ciArrayKlass *default_type, int flags) {
- assert(default_type != nullptr, "null default_type!");
- BasicType basic_type = default_type->element_type()->basic_type();
- if (basic_type == T_ARRAY) { basic_type = T_OBJECT; }
- if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
- // Sanity check the known type with the incoming class. For the
- // primitive case the types must match exactly with src.klass and
- // dst.klass each exactly matching the default type. For the
- // object array case, if no type check is needed then either the
- // dst type is exactly the expected type and the src type is a
- // subtype which we can't check or src is the same array as dst
- // but not necessarily exactly of type default_type.
- Label known_ok, halt;
- __ mov_metadata(tmp, default_type->constant_encoding());
- if (UseCompressedClassPointers) {
- __ encode_klass_not_null(tmp);
- }
-
- if (basic_type != T_OBJECT) {
- __ cmp_klass_compressed(dst, tmp, t0, halt, false);
- __ cmp_klass_compressed(src, tmp, t0, known_ok, true);
- } else {
- __ cmp_klass_compressed(dst, tmp, t0, known_ok, true);
- __ beq(src, dst, known_ok);
- }
- __ bind(halt);
- __ stop("incorrect type information in arraycopy");
- __ bind(known_ok);
- }
-}
-
void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
ciArrayKlass *default_type = op->expected_type();
Register src = op->src()->as_register();
@@ -304,7 +270,28 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
}
#ifdef ASSERT
- arraycopy_assert(src, dst, tmp, default_type, flags);
+ if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
+ // Sanity check the known type with the incoming class. For the
+ // primitive case the types must match exactly with src.klass and
+ // dst.klass each exactly matching the default type. For the
+ // object array case, if no type check is needed then either the
+ // dst type is exactly the expected type and the src type is a
+ // subtype which we can't check or src is the same array as dst
+ // but not necessarily exactly of type default_type.
+ Label known_ok, halt;
+ __ mov_metadata(tmp, default_type->constant_encoding());
+
+ if (basic_type != T_OBJECT) {
+ __ cmp_klass_bne(dst, tmp, t0, t1, halt);
+ __ cmp_klass_beq(src, tmp, t0, t1, known_ok);
+ } else {
+ __ cmp_klass_beq(dst, tmp, t0, t1, known_ok);
+ __ beq(src, dst, known_ok);
+ }
+ __ bind(halt);
+ __ stop("incorrect type information in arraycopy");
+ __ bind(known_ok);
+ }
#endif
#ifndef PRODUCT
diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.hpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.hpp
index 06a0f248ca6..b5452f3e4cd 100644
--- a/src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.hpp
+++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -39,7 +39,6 @@
void arraycopy_type_check(Register src, Register src_pos, Register length,
Register dst, Register dst_pos, Register tmp,
CodeStub *stub, BasicType basic_type, int flags);
- void arraycopy_assert(Register src, Register dst, Register tmp, ciArrayKlass *default_type, int flags);
void arraycopy_prepare_params(Register src, Register src_pos, Register length,
Register dst, Register dst_pos, BasicType basic_type);
void arraycopy_checkcast_prepare_params(Register src, Register src_pos, Register length,
diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp
index 63e2fd015d7..29e5d86d0cc 100644
--- a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -55,20 +55,6 @@ const Register SHIFT_count = x10; // where count for shift operations must be
#define __ _masm->
-static void select_different_registers(Register preserve,
- Register extra,
- Register &tmp1,
- Register &tmp2) {
- if (tmp1 == preserve) {
- assert_different_registers(tmp1, tmp2, extra);
- tmp1 = extra;
- } else if (tmp2 == preserve) {
- assert_different_registers(tmp1, tmp2, extra);
- tmp2 = extra;
- }
- assert_different_registers(preserve, tmp1, tmp2);
-}
-
static void select_different_registers(Register preserve,
Register extra,
Register &tmp1,
@@ -1155,12 +1141,8 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, L
} else if (obj == klass_RInfo) {
klass_RInfo = dst;
}
- if (k->is_loaded() && !UseCompressedClassPointers) {
- select_different_registers(obj, dst, k_RInfo, klass_RInfo);
- } else {
- Rtmp1 = op->tmp3()->as_register();
- select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
- }
+ Rtmp1 = op->tmp3()->as_register();
+ select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
assert_different_registers(obj, k_RInfo, klass_RInfo);
diff --git a/src/hotspot/cpu/riscv/c1_LIRGenerator_riscv.cpp b/src/hotspot/cpu/riscv/c1_LIRGenerator_riscv.cpp
index 88565d9136f..5e0deb84a14 100644
--- a/src/hotspot/cpu/riscv/c1_LIRGenerator_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c1_LIRGenerator_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -1073,9 +1073,7 @@ void LIRGenerator::do_CheckCast(CheckCast* x) {
}
LIR_Opr reg = rlock_result(x);
LIR_Opr tmp3 = LIR_OprFact::illegalOpr;
- if (!x->klass()->is_loaded() || UseCompressedClassPointers) {
- tmp3 = new_register(objectType);
- }
+ tmp3 = new_register(objectType);
__ checkcast(reg, obj.result(), x->klass(),
new_register(objectType), new_register(objectType), tmp3,
x->direct_compare(), info_for_exception, patching_info, stub,
@@ -1094,9 +1092,7 @@ void LIRGenerator::do_InstanceOf(InstanceOf* x) {
}
obj.load_item();
LIR_Opr tmp3 = LIR_OprFact::illegalOpr;
- if (!x->klass()->is_loaded() || UseCompressedClassPointers) {
- tmp3 = new_register(objectType);
- }
+ tmp3 = new_register(objectType);
__ instanceof(reg, obj.result(), x->klass(),
new_register(objectType), new_register(objectType), tmp3,
x->direct_compare(), patching_info, x->profiled_method(), x->profiled_bci());
@@ -1173,4 +1169,5 @@ void LIRGenerator::volatile_field_store(LIR_Opr value, LIR_Address* address,
void LIRGenerator::volatile_field_load(LIR_Address* address, LIR_Opr result,
CodeEmitInfo* info) {
__ volatile_load_mem_reg(address, result, info);
+ __ membar_acquire();
}
diff --git a/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp
index aeb077ba0a0..abcc070b253 100644
--- a/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -92,12 +92,8 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
// This assumes that all prototype bits fitr in an int32_t
mv(tmp1, checked_cast(markWord::prototype().value()));
sd(tmp1, Address(obj, oopDesc::mark_offset_in_bytes()));
- if (UseCompressedClassPointers) { // Take care not to kill klass
- encode_klass_not_null(tmp1, klass, tmp2);
- sw(tmp1, Address(obj, oopDesc::klass_offset_in_bytes()));
- } else {
- sd(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
- }
+ encode_klass_not_null(tmp1, klass, tmp2);
+ sw(tmp1, Address(obj, oopDesc::klass_offset_in_bytes()));
}
if (len->is_valid()) {
@@ -108,7 +104,7 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
// Clear gap/first 4 bytes following the length field.
sw(zr, Address(obj, base_offset));
}
- } else if (UseCompressedClassPointers && !UseCompactObjectHeaders) {
+ } else if (!UseCompactObjectHeaders) {
store_klass_gap(obj, zr);
}
}
diff --git a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp
index 72a90ddde1f..c9cd8220551 100644
--- a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp
@@ -1175,8 +1175,7 @@ void C2_MacroAssembler::string_compare_long_same_encoding(Register result, Regis
Label TAIL_CHECK, TAIL, NEXT_WORD, DIFFERENCE;
const int base_offset = arrayOopDesc::base_offset_in_bytes(T_BYTE);
- assert((base_offset % (UseCompactObjectHeaders ? 4 :
- (UseCompressedClassPointers ? 8 : 4))) == 0, "Must be");
+ assert((base_offset % (UseCompactObjectHeaders ? 4 : 8)) == 0, "Must be");
const int minCharsInWord = isLL ? wordSize : wordSize / 2;
@@ -1269,8 +1268,7 @@ void C2_MacroAssembler::string_compare_long_different_encoding(Register result,
Label TAIL, NEXT_WORD, DIFFERENCE;
const int base_offset = arrayOopDesc::base_offset_in_bytes(T_BYTE);
- assert((base_offset % (UseCompactObjectHeaders ? 4 :
- (UseCompressedClassPointers ? 8 : 4))) == 0, "Must be");
+ assert((base_offset % (UseCompactObjectHeaders ? 4 : 8)) == 0, "Must be");
Register strL = isLU ? str1 : str2;
Register strU = isLU ? str2 : str1;
@@ -1485,8 +1483,7 @@ void C2_MacroAssembler::arrays_equals(Register a1, Register a2,
int length_offset = arrayOopDesc::length_offset_in_bytes();
int base_offset = arrayOopDesc::base_offset_in_bytes(elem_size == 2 ? T_CHAR : T_BYTE);
- assert((base_offset % (UseCompactObjectHeaders ? 4 :
- (UseCompressedClassPointers ? 8 : 4))) == 0, "Must be");
+ assert((base_offset % (UseCompactObjectHeaders ? 4 : 8)) == 0, "Must be");
Register cnt1 = tmp3;
Register cnt2 = tmp1; // cnt2 only used in array length compare
@@ -1611,8 +1608,7 @@ void C2_MacroAssembler::string_equals(Register a1, Register a2,
int base_offset = arrayOopDesc::base_offset_in_bytes(T_BYTE);
- assert((base_offset % (UseCompactObjectHeaders ? 4 :
- (UseCompressedClassPointers ? 8 : 4))) == 0, "Must be");
+ assert((base_offset % (UseCompactObjectHeaders ? 4 : 8)) == 0, "Must be");
BLOCK_COMMENT("string_equals {");
@@ -2699,8 +2695,7 @@ void C2_MacroAssembler::arrays_equals_v(Register a1, Register a2, Register resul
int length_offset = arrayOopDesc::length_offset_in_bytes();
int base_offset = arrayOopDesc::base_offset_in_bytes(elem_size == 2 ? T_CHAR : T_BYTE);
- assert((base_offset % (UseCompactObjectHeaders ? 4 :
- (UseCompressedClassPointers ? 8 : 4))) == 0, "Must be");
+ assert((base_offset % (UseCompactObjectHeaders ? 4 : 8)) == 0, "Must be");
BLOCK_COMMENT("arrays_equals_v {");
@@ -3074,12 +3069,12 @@ void C2_MacroAssembler::reduce_mul_integral_v(Register dst, Register src1, Vecto
// If the operation is MUL, then the identity value is one.
vmv_v_i(vtmp1, 1);
vmerge_vvm(vtmp2, vtmp1, src2); // vm == v0
- vslidedown_vi(vtmp1, vtmp2, vector_length);
+ slidedown_v(vtmp1, vtmp2, vector_length);
vsetvli_helper(bt, vector_length);
vmul_vv(vtmp1, vtmp1, vtmp2);
} else {
- vslidedown_vi(vtmp1, src2, vector_length);
+ slidedown_v(vtmp1, src2, vector_length);
vsetvli_helper(bt, vector_length);
vmul_vv(vtmp1, vtmp1, src2);
@@ -3087,7 +3082,7 @@ void C2_MacroAssembler::reduce_mul_integral_v(Register dst, Register src1, Vecto
while (vector_length > 1) {
vector_length /= 2;
- vslidedown_vi(vtmp2, vtmp1, vector_length);
+ slidedown_v(vtmp2, vtmp1, vector_length);
vsetvli_helper(bt, vector_length);
vmul_vv(vtmp1, vtmp1, vtmp2);
}
@@ -3286,40 +3281,44 @@ VFCVT_SAFE(vfcvt_rtz_x_f_v);
// Extract a scalar element from an vector at position 'idx'.
// The input elements in src are expected to be of integral type.
-void C2_MacroAssembler::extract_v(Register dst, VectorRegister src, BasicType bt,
- int idx, VectorRegister tmp) {
+void C2_MacroAssembler::extract_v(Register dst, VectorRegister src,
+ BasicType bt, int idx, VectorRegister vtmp) {
assert(is_integral_type(bt), "unsupported element type");
assert(idx >= 0, "idx cannot be negative");
// Only need the first element after vector slidedown
vsetvli_helper(bt, 1);
if (idx == 0) {
vmv_x_s(dst, src);
- } else if (idx <= 31) {
- vslidedown_vi(tmp, src, idx);
- vmv_x_s(dst, tmp);
} else {
- mv(t0, idx);
- vslidedown_vx(tmp, src, t0);
- vmv_x_s(dst, tmp);
+ slidedown_v(vtmp, src, idx);
+ vmv_x_s(dst, vtmp);
}
}
// Extract a scalar element from an vector at position 'idx'.
// The input elements in src are expected to be of floating point type.
-void C2_MacroAssembler::extract_fp_v(FloatRegister dst, VectorRegister src, BasicType bt,
- int idx, VectorRegister tmp) {
+void C2_MacroAssembler::extract_fp_v(FloatRegister dst, VectorRegister src,
+ BasicType bt, int idx, VectorRegister vtmp) {
assert(is_floating_point_type(bt), "unsupported element type");
assert(idx >= 0, "idx cannot be negative");
// Only need the first element after vector slidedown
vsetvli_helper(bt, 1);
if (idx == 0) {
vfmv_f_s(dst, src);
- } else if (idx <= 31) {
- vslidedown_vi(tmp, src, idx);
- vfmv_f_s(dst, tmp);
} else {
- mv(t0, idx);
- vslidedown_vx(tmp, src, t0);
- vfmv_f_s(dst, tmp);
+ slidedown_v(vtmp, src, idx);
+ vfmv_f_s(dst, vtmp);
+ }
+}
+
+// Move elements down a vector register group.
+// Offset is the start index (offset) for the source.
+void C2_MacroAssembler::slidedown_v(VectorRegister dst, VectorRegister src,
+ uint32_t offset, Register tmp) {
+ if (is_uimm5(offset)) {
+ vslidedown_vi(dst, src, offset);
+ } else {
+ mv(tmp, offset);
+ vslidedown_vx(dst, src, tmp);
}
}
diff --git a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp
index fa87ceba295..468d53b1a54 100644
--- a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -296,7 +296,13 @@
void vfcvt_rtz_x_f_v_safe(VectorRegister dst, VectorRegister src);
- void extract_v(Register dst, VectorRegister src, BasicType bt, int idx, VectorRegister tmp);
- void extract_fp_v(FloatRegister dst, VectorRegister src, BasicType bt, int idx, VectorRegister tmp);
+ void extract_v(Register dst, VectorRegister src,
+ BasicType bt, int idx, VectorRegister vtmp);
+
+ void extract_fp_v(FloatRegister dst, VectorRegister src,
+ BasicType bt, int idx, VectorRegister vtmp);
+
+ void slidedown_v(VectorRegister dst, VectorRegister src,
+ uint32_t offset, Register tmp = t0);
#endif // CPU_RISCV_C2_MACROASSEMBLER_RISCV_HPP
diff --git a/src/hotspot/cpu/riscv/gc/shared/cardTableBarrierSetAssembler_riscv.cpp b/src/hotspot/cpu/riscv/gc/shared/cardTableBarrierSetAssembler_riscv.cpp
index d94bf428fd2..9eb546a1888 100644
--- a/src/hotspot/cpu/riscv/gc/shared/cardTableBarrierSetAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/gc/shared/cardTableBarrierSetAssembler_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -56,8 +56,10 @@ void CardTableBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet d
}
}
-void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj, Register tmp) {
- assert_different_registers(obj, tmp);
+void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2) {
+ precond(tmp1 != noreg);
+ precond(tmp2 != noreg);
+ assert_different_registers(obj, tmp1, tmp2);
BarrierSet* bs = BarrierSet::barrier_set();
assert(bs->kind() == BarrierSet::CardTableBarrierSet, "Wrong barrier set kind");
@@ -65,17 +67,17 @@ void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register ob
assert(CardTable::dirty_card_val() == 0, "must be");
- __ load_byte_map_base(tmp);
- __ add(tmp, obj, tmp);
+ __ load_byte_map_base(tmp1);
+ __ add(tmp1, obj, tmp1);
if (UseCondCardMark) {
Label L_already_dirty;
- __ lbu(t1, Address(tmp));
- __ beqz(t1, L_already_dirty);
- __ sb(zr, Address(tmp));
+ __ lbu(tmp2, Address(tmp1));
+ __ beqz(tmp2, L_already_dirty);
+ __ sb(zr, Address(tmp1));
__ bind(L_already_dirty);
} else {
- __ sb(zr, Address(tmp));
+ __ sb(zr, Address(tmp1));
}
}
@@ -119,10 +121,10 @@ void CardTableBarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorS
if (needs_post_barrier) {
// flatten object address if needed
if (!precise || dst.offset() == 0) {
- store_check(masm, dst.base(), tmp3);
+ store_check(masm, dst.base(), tmp1, tmp2);
} else {
__ la(tmp3, dst);
- store_check(masm, tmp3, t0);
+ store_check(masm, tmp3, tmp1, tmp2);
}
}
}
diff --git a/src/hotspot/cpu/riscv/gc/shared/cardTableBarrierSetAssembler_riscv.hpp b/src/hotspot/cpu/riscv/gc/shared/cardTableBarrierSetAssembler_riscv.hpp
index 6f6e9065103..1576f0a6dd8 100644
--- a/src/hotspot/cpu/riscv/gc/shared/cardTableBarrierSetAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/gc/shared/cardTableBarrierSetAssembler_riscv.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -31,7 +31,7 @@
class CardTableBarrierSetAssembler: public BarrierSetAssembler {
protected:
- void store_check(MacroAssembler* masm, Register obj, Register tmp);
+ void store_check(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2);
virtual void gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators,
Register addr, Register count, RegSet saved_regs) {}
diff --git a/src/hotspot/cpu/riscv/interp_masm_riscv.cpp b/src/hotspot/cpu/riscv/interp_masm_riscv.cpp
index 744590bec2b..804c2072ba5 100644
--- a/src/hotspot/cpu/riscv/interp_masm_riscv.cpp
+++ b/src/hotspot/cpu/riscv/interp_masm_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -1040,26 +1040,15 @@ void InterpreterMacroAssembler::profile_final_call(Register mdp) {
void InterpreterMacroAssembler::profile_virtual_call(Register receiver,
- Register mdp,
- bool receiver_can_be_null) {
+ Register mdp) {
if (ProfileInterpreter) {
Label profile_continue;
// If no method data exists, go to profile_continue.
test_method_data_pointer(mdp, profile_continue);
- Label skip_receiver_profile;
- if (receiver_can_be_null) {
- Label not_null;
- // We are making a call. Increment the count for null receiver.
- increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
- j(skip_receiver_profile);
- bind(not_null);
- }
-
// Record the receiver type.
profile_receiver_type(receiver, mdp, 0);
- bind(skip_receiver_profile);
// The method data pointer needs to be updated to reflect the new target.
diff --git a/src/hotspot/cpu/riscv/interp_masm_riscv.hpp b/src/hotspot/cpu/riscv/interp_masm_riscv.hpp
index 59cc76b022f..df86f0dc532 100644
--- a/src/hotspot/cpu/riscv/interp_masm_riscv.hpp
+++ b/src/hotspot/cpu/riscv/interp_masm_riscv.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2015, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -274,8 +274,7 @@ class InterpreterMacroAssembler: public MacroAssembler {
void profile_not_taken_branch(Register mdp);
void profile_call(Register mdp);
void profile_final_call(Register mdp);
- void profile_virtual_call(Register receiver, Register mdp,
- bool receiver_can_be_null = false);
+ void profile_virtual_call(Register receiver, Register mdp);
void profile_ret(Register return_bci, Register mdp);
void profile_null_seen(Register mdp);
void profile_typecheck(Register mdp, Register klass);
diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
index 4f5e7afc166..0e32c602d95 100644
--- a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2024, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -49,6 +49,7 @@
#include "runtime/sharedRuntime.hpp"
#include "runtime/stubRoutines.hpp"
#include "utilities/globalDefinitions.hpp"
+#include "utilities/integerCast.hpp"
#include "utilities/powerOfTwo.hpp"
#ifdef COMPILER2
#include "opto/compile.hpp"
@@ -1947,14 +1948,12 @@ void MacroAssembler::restore_cpu_control_state_after_jni(Register tmp) {
}
}
-void MacroAssembler::push_reg(Register Rs)
-{
+void MacroAssembler::push_reg(Register Rs) {
subi(esp, esp, wordSize);
sd(Rs, Address(esp, 0));
}
-void MacroAssembler::pop_reg(Register Rd)
-{
+void MacroAssembler::pop_reg(Register Rd) {
ld(Rd, Address(esp, 0));
addi(esp, esp, wordSize);
}
@@ -1973,7 +1972,11 @@ int MacroAssembler::bitset_to_regs(unsigned int bitset, unsigned char* regs) {
// Push integer registers in the bitset supplied. Don't push sp.
// Return the number of words pushed
-int MacroAssembler::push_reg(unsigned int bitset, Register stack) {
+int MacroAssembler::push_reg(RegSet regset, Register stack) {
+ if (regset.bits() == 0) {
+ return 0;
+ }
+ auto bitset = integer_cast(regset.bits());
DEBUG_ONLY(int words_pushed = 0;)
unsigned char regs[32];
int count = bitset_to_regs(bitset, regs);
@@ -1993,7 +1996,11 @@ int MacroAssembler::push_reg(unsigned int bitset, Register stack) {
return count;
}
-int MacroAssembler::pop_reg(unsigned int bitset, Register stack) {
+int MacroAssembler::pop_reg(RegSet regset, Register stack) {
+ if (regset.bits() == 0) {
+ return 0;
+ }
+ auto bitset = integer_cast(regset.bits());
DEBUG_ONLY(int words_popped = 0;)
unsigned char regs[32];
int count = bitset_to_regs(bitset, regs);
@@ -2015,7 +2022,11 @@ int MacroAssembler::pop_reg(unsigned int bitset, Register stack) {
// Push floating-point registers in the bitset supplied.
// Return the number of words pushed
-int MacroAssembler::push_fp(unsigned int bitset, Register stack) {
+int MacroAssembler::push_fp(FloatRegSet regset, Register stack) {
+ if (regset.bits() == 0) {
+ return 0;
+ }
+ auto bitset = integer_cast(regset.bits());
DEBUG_ONLY(int words_pushed = 0;)
unsigned char regs[32];
int count = bitset_to_regs(bitset, regs);
@@ -2035,7 +2046,11 @@ int MacroAssembler::push_fp(unsigned int bitset, Register stack) {
return count;
}
-int MacroAssembler::pop_fp(unsigned int bitset, Register stack) {
+int MacroAssembler::pop_fp(FloatRegSet regset, Register stack) {
+ if (regset.bits() == 0) {
+ return 0;
+ }
+ auto bitset = integer_cast(regset.bits());
DEBUG_ONLY(int words_popped = 0;)
unsigned char regs[32];
int count = bitset_to_regs(bitset, regs);
@@ -2721,7 +2736,11 @@ void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len,
#ifdef COMPILER2
// Push vector registers in the bitset supplied.
// Return the number of words pushed
-int MacroAssembler::push_v(unsigned int bitset, Register stack) {
+int MacroAssembler::push_v(VectorRegSet regset, Register stack) {
+ if (regset.bits() == 0) {
+ return 0;
+ }
+ auto bitset = integer_cast(regset.bits());
int vector_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
// Scan bitset to accumulate register pairs
@@ -2736,7 +2755,11 @@ int MacroAssembler::push_v(unsigned int bitset, Register stack) {
return count * vector_size_in_bytes / wordSize;
}
-int MacroAssembler::pop_v(unsigned int bitset, Register stack) {
+int MacroAssembler::pop_v(VectorRegSet regset, Register stack) {
+ if (regset.bits() == 0) {
+ return 0;
+ }
+ auto bitset = integer_cast(regset.bits());
int vector_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
// Scan bitset to accumulate register pairs
@@ -3511,19 +3534,30 @@ void MacroAssembler::orptr(Address adr, RegisterOrConstant src, Register tmp1, R
sd(tmp1, adr);
}
-void MacroAssembler::cmp_klass_compressed(Register oop, Register trial_klass, Register tmp, Label &L, bool equal) {
+void MacroAssembler::cmp_klass_beq(Register obj, Register klass,
+ Register tmp1, Register tmp2,
+ Label &L, bool is_far) {
+ assert_different_registers(obj, klass, tmp1, tmp2);
if (UseCompactObjectHeaders) {
- load_narrow_klass_compact(tmp, oop);
- } else if (UseCompressedClassPointers) {
- lwu(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
+ load_narrow_klass_compact(tmp1, obj);
} else {
- ld(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
+ lwu(tmp1, Address(obj, oopDesc::klass_offset_in_bytes()));
}
- if (equal) {
- beq(trial_klass, tmp, L);
+ decode_klass_not_null(tmp1, tmp2);
+ beq(klass, tmp1, L, is_far);
+}
+
+void MacroAssembler::cmp_klass_bne(Register obj, Register klass,
+ Register tmp1, Register tmp2,
+ Label &L, bool is_far) {
+ assert_different_registers(obj, klass, tmp1, tmp2);
+ if (UseCompactObjectHeaders) {
+ load_narrow_klass_compact(tmp1, obj);
} else {
- bne(trial_klass, tmp, L);
+ lwu(tmp1, Address(obj, oopDesc::klass_offset_in_bytes()));
}
+ decode_klass_not_null(tmp1, tmp2);
+ bne(klass, tmp1, L, is_far);
}
// Move an oop into a register.
@@ -3741,11 +3775,9 @@ void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(dst, src);
decode_klass_not_null(dst, tmp);
- } else if (UseCompressedClassPointers) {
+ } else {
lwu(dst, Address(src, oopDesc::klass_offset_in_bytes()));
decode_klass_not_null(dst, tmp);
- } else {
- ld(dst, Address(src, oopDesc::klass_offset_in_bytes()));
}
}
@@ -3753,20 +3785,15 @@ void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
// FIXME: Should this be a store release? concurrent gcs assumes
// klass length is valid if klass field is not null.
assert(!UseCompactObjectHeaders, "not with compact headers");
- if (UseCompressedClassPointers) {
- encode_klass_not_null(src, tmp);
- sw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
- } else {
- sd(src, Address(dst, oopDesc::klass_offset_in_bytes()));
- }
+ encode_klass_not_null(src, tmp);
+ sw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
+
}
void MacroAssembler::store_klass_gap(Register dst, Register src) {
assert(!UseCompactObjectHeaders, "not with compact headers");
- if (UseCompressedClassPointers) {
- // Store to klass gap in destination
- sw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
- }
+ // Store to klass gap in destination
+ sw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
}
void MacroAssembler::decode_klass_not_null(Register r, Register tmp) {
@@ -3775,7 +3802,6 @@ void MacroAssembler::decode_klass_not_null(Register r, Register tmp) {
}
void MacroAssembler::decode_klass_not_null(Register dst, Register src, Register tmp) {
- assert(UseCompressedClassPointers, "should only be used for compressed headers");
assert_different_registers(dst, tmp);
assert_different_registers(src, tmp);
@@ -3806,8 +3832,6 @@ void MacroAssembler::encode_klass_not_null(Register r, Register tmp) {
}
void MacroAssembler::encode_klass_not_null(Register dst, Register src, Register tmp) {
- assert(UseCompressedClassPointers, "should only be used for compressed headers");
-
if (CompressedKlassPointers::base() == nullptr) {
if (CompressedKlassPointers::shift() != 0) {
srli(dst, src, CompressedKlassPointers::shift());
@@ -5337,7 +5361,6 @@ void MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
}
void MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
- assert (UseCompressedClassPointers, "should only be used for compressed headers");
assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
int index = oop_recorder()->find_index(k);
@@ -5417,12 +5440,9 @@ int MacroAssembler::ic_check(int end_alignment) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(tmp1, receiver);
lwu(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
- } else if (UseCompressedClassPointers) {
+ } else {
lwu(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes()));
lwu(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
- } else {
- ld(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes()));
- ld(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
}
Label ic_hit;
@@ -5543,13 +5563,6 @@ void MacroAssembler::decrementw(const Address dst, int32_t value, Register tmp1,
sw(tmp1, adr);
}
-void MacroAssembler::cmpptr(Register src1, const Address &src2, Label& equal, Register tmp) {
- assert_different_registers(src1, tmp);
- assert(src2.getMode() == Address::literal, "must be applied to a literal address");
- ld(tmp, src2);
- beq(src1, tmp, equal);
-}
-
void MacroAssembler::load_method_holder_cld(Register result, Register method) {
load_method_holder(result, method);
ld(result, Address(result, InstanceKlass::class_loader_data_offset()));
diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
index f5e985c28a2..4cc55e7ae23 100644
--- a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2024, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -198,7 +198,12 @@ class MacroAssembler: public Assembler {
void load_klass(Register dst, Register src, Register tmp = t0);
void load_narrow_klass_compact(Register dst, Register src);
void store_klass(Register dst, Register src, Register tmp = t0);
- void cmp_klass_compressed(Register oop, Register trial_klass, Register tmp, Label &L, bool equal);
+ void cmp_klass_beq(Register obj, Register klass,
+ Register tmp1, Register tmp2,
+ Label &L, bool is_far = false);
+ void cmp_klass_bne(Register obj, Register klass,
+ Register tmp1, Register tmp2,
+ Label &L, bool is_far = false);
void encode_klass_not_null(Register r, Register tmp = t0);
void decode_klass_not_null(Register r, Register tmp = t0);
@@ -813,15 +818,6 @@ class MacroAssembler: public Assembler {
void double_bgt(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far = false, bool is_unordered = false);
private:
- int push_reg(unsigned int bitset, Register stack);
- int pop_reg(unsigned int bitset, Register stack);
- int push_fp(unsigned int bitset, Register stack);
- int pop_fp(unsigned int bitset, Register stack);
-#ifdef COMPILER2
- int push_v(unsigned int bitset, Register stack);
- int pop_v(unsigned int bitset, Register stack);
-#endif // COMPILER2
-
// The signed 20-bit upper imm can materialize at most negative 0xF...F80000000, two G.
// The following signed 12-bit imm can at max subtract 0x800, two K, from that previously loaded two G.
bool is_valid_32bit_offset(int64_t x) {
@@ -839,15 +835,19 @@ private:
}
public:
+ // Stack push and pop individual 64 bit registers
void push_reg(Register Rs);
void pop_reg(Register Rd);
- void push_reg(RegSet regs, Register stack) { if (regs.bits()) push_reg(regs.bits(), stack); }
- void pop_reg(RegSet regs, Register stack) { if (regs.bits()) pop_reg(regs.bits(), stack); }
- void push_fp(FloatRegSet regs, Register stack) { if (regs.bits()) push_fp(regs.bits(), stack); }
- void pop_fp(FloatRegSet regs, Register stack) { if (regs.bits()) pop_fp(regs.bits(), stack); }
+
+ int push_reg(RegSet regset, Register stack);
+ int pop_reg(RegSet regset, Register stack);
+
+ int push_fp(FloatRegSet regset, Register stack);
+ int pop_fp(FloatRegSet regset, Register stack);
+
#ifdef COMPILER2
- void push_v(VectorRegSet regs, Register stack) { if (regs.bits()) push_v(regs.bits(), stack); }
- void pop_v(VectorRegSet regs, Register stack) { if (regs.bits()) pop_v(regs.bits(), stack); }
+ int push_v(VectorRegSet regset, Register stack);
+ int pop_v(VectorRegSet regset, Register stack);
#endif // COMPILER2
// Push and pop everything that might be clobbered by a native
@@ -1348,9 +1348,8 @@ public:
void decrement(const Address dst, int64_t value = 1, Register tmp1 = t0, Register tmp2 = t1);
void decrementw(const Address dst, int32_t value = 1, Register tmp1 = t0, Register tmp2 = t1);
- void cmpptr(Register src1, const Address &src2, Label& equal, Register tmp = t0);
-
void clinit_barrier(Register klass, Register tmp, Label* L_fast_path = nullptr, Label* L_slow_path = nullptr);
+
void load_method_holder_cld(Register result, Register method);
void load_method_holder(Register holder, Register method);
diff --git a/src/hotspot/cpu/riscv/methodHandles_riscv.cpp b/src/hotspot/cpu/riscv/methodHandles_riscv.cpp
index d770999df96..e80dedf58ed 100644
--- a/src/hotspot/cpu/riscv/methodHandles_riscv.cpp
+++ b/src/hotspot/cpu/riscv/methodHandles_riscv.cpp
@@ -72,17 +72,22 @@ void MethodHandles::verify_klass(MacroAssembler* _masm,
InstanceKlass** klass_addr = vmClasses::klass_addr_at(klass_id);
Klass* klass = vmClasses::klass_at(klass_id);
Register temp1 = t1;
- Register temp2 = t0; // used by MacroAssembler::cmpptr
+ Register temp2 = t0;
Label L_ok, L_bad;
BLOCK_COMMENT("verify_klass {");
__ verify_oop(obj);
__ beqz(obj, L_bad);
+
__ push_reg(RegSet::of(temp1, temp2), sp);
__ load_klass(temp1, obj, temp2);
- __ cmpptr(temp1, ExternalAddress((address) klass_addr), L_ok);
+ __ ld(temp2, ExternalAddress((address)klass_addr));
+ __ beq(temp1, temp2, L_ok);
+
intptr_t super_check_offset = klass->super_check_offset();
__ ld(temp1, Address(temp1, super_check_offset));
- __ cmpptr(temp1, ExternalAddress((address) klass_addr), L_ok);
+ __ ld(temp2, ExternalAddress((address)klass_addr));
+ __ beq(temp1, temp2, L_ok);
+
__ pop_reg(RegSet::of(temp1, temp2), sp);
__ bind(L_bad);
__ stop(error_message);
diff --git a/src/hotspot/cpu/riscv/riscv.ad b/src/hotspot/cpu/riscv/riscv.ad
index 54c0d9c0955..e236d03e6d2 100644
--- a/src/hotspot/cpu/riscv/riscv.ad
+++ b/src/hotspot/cpu/riscv/riscv.ad
@@ -1801,13 +1801,8 @@ void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
{
assert_cond(st != nullptr);
st->print_cr("# MachUEPNode");
- if (UseCompressedClassPointers) {
- st->print_cr("\tlwu t1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
- st->print_cr("\tlwu t2, [t0 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
- } else {
- st->print_cr("\tld t1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
- st->print_cr("\tld t2, [t0 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
- }
+ st->print_cr("\tlwu t1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
+ st->print_cr("\tlwu t2, [t0 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
st->print_cr("\tbeq t1, t2, ic_hit");
st->print_cr("\tj, SharedRuntime::_ic_miss_stub\t # Inline cache check");
st->print_cr("\tic_hit:");
diff --git a/src/hotspot/cpu/riscv/stubDeclarations_riscv.hpp b/src/hotspot/cpu/riscv/stubDeclarations_riscv.hpp
index f977d759d20..890e354fd27 100644
--- a/src/hotspot/cpu/riscv/stubDeclarations_riscv.hpp
+++ b/src/hotspot/cpu/riscv/stubDeclarations_riscv.hpp
@@ -29,28 +29,32 @@
#define STUBGEN_PREUNIVERSE_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(preuniverse, 0) \
#define STUBGEN_INITIAL_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(initial, 10000) \
#define STUBGEN_CONTINUATION_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(continuation, 2000) \
#define STUBGEN_COMPILER_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(compiler, 45000) \
do_stub(compiler, compare_long_string_LL) \
do_arch_entry(riscv, compiler, compare_long_string_LL, \
@@ -81,7 +85,8 @@
#define STUBGEN_FINAL_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(final, 20000 ZGC_ONLY(+10000)) \
do_stub(final, copy_byte_f) \
do_arch_entry(riscv, final, copy_byte_f, copy_byte_f, \
diff --git a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp
index 127ac9f6951..4656b5c0d41 100644
--- a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp
+++ b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2025, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2025, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -3070,8 +3070,7 @@ class StubGenerator: public StubCodeGenerator {
const Register tmp = x30, tmpLval = x12;
int base_offset = arrayOopDesc::base_offset_in_bytes(T_BYTE);
- assert((base_offset % (UseCompactObjectHeaders ? 4 :
- (UseCompressedClassPointers ? 8 : 4))) == 0, "Must be");
+ assert((base_offset % (UseCompactObjectHeaders ? 4 : 8)) == 0, "Must be");
#ifdef ASSERT
if (AvoidUnalignedAccesses) {
@@ -3128,8 +3127,7 @@ class StubGenerator: public StubCodeGenerator {
tmp1 = x28, tmp2 = x29, tmp3 = x30, tmp4 = x12;
int base_offset = arrayOopDesc::base_offset_in_bytes(T_BYTE);
- assert((base_offset % (UseCompactObjectHeaders ? 4 :
- (UseCompressedClassPointers ? 8 : 4))) == 0, "Must be");
+ assert((base_offset % (UseCompactObjectHeaders ? 4 : 8)) == 0, "Must be");
Register strU = isLU ? str2 : str1,
strL = isLU ? str1 : str2,
@@ -7350,7 +7348,7 @@ static const int64_t right_3_bits = right_n_bits(3);
}
public:
- StubGenerator(CodeBuffer* code, BlobId blob_id) : StubCodeGenerator(code, blob_id) {
+ StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) : StubCodeGenerator(code, blob_id, stub_data) {
switch(blob_id) {
case BlobId::stubgen_preuniverse_id:
generate_preuniverse_stubs();
@@ -7374,6 +7372,6 @@ static const int64_t right_3_bits = right_n_bits(3);
}
}; // end class declaration
-void StubGenerator_generate(CodeBuffer* code, BlobId blob_id) {
- StubGenerator g(code, blob_id);
+void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) {
+ StubGenerator g(code, blob_id, stub_data);
}
diff --git a/src/hotspot/cpu/riscv/stubRoutines_riscv.cpp b/src/hotspot/cpu/riscv/stubRoutines_riscv.cpp
index 2aac95d71fa..b7f69eff9fa 100644
--- a/src/hotspot/cpu/riscv/stubRoutines_riscv.cpp
+++ b/src/hotspot/cpu/riscv/stubRoutines_riscv.cpp
@@ -42,8 +42,12 @@
#define DEFINE_ARCH_ENTRY_INIT(arch, blob_name, stub_name, field_name, getter_name, init_function) \
address StubRoutines:: arch :: STUB_FIELD_NAME(field_name) = CAST_FROM_FN_PTR(address, init_function);
-STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY, DEFINE_ARCH_ENTRY_INIT)
+#define DEFINE_ARCH_ENTRY_ARRAY(arch, blob_name, stub_name, field_name, getter_name, count) \
+ address StubRoutines:: arch :: STUB_FIELD_NAME(field_name) [count] ;
+STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY, DEFINE_ARCH_ENTRY_INIT, DEFINE_ARCH_ENTRY_ARRAY)
+
+#undef DEFINE_ARCH_ENTRY_ARRAY
#undef DEFINE_ARCH_ENTRY_INIT
#undef DEFINE_ARCH_ENTRY
@@ -501,3 +505,9 @@ ATTRIBUTE_ALIGNED(4096) juint StubRoutines::riscv::_crc_table[] =
0x751997d0UL, 0x00000001UL,
0xccaa009eUL, 0x00000000UL,
};
+
+#if INCLUDE_CDS
+// nothing to do for riscv
+void StubRoutines::init_AOTAddressTable() {
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/riscv/stubRoutines_riscv.hpp b/src/hotspot/cpu/riscv/stubRoutines_riscv.hpp
index 2c4e7210413..ec67a338052 100644
--- a/src/hotspot/cpu/riscv/stubRoutines_riscv.hpp
+++ b/src/hotspot/cpu/riscv/stubRoutines_riscv.hpp
@@ -61,9 +61,13 @@ class riscv {
#define DECLARE_ARCH_ENTRY_INIT(arch, blob_name, stub_name, field_name, getter_name, init_function) \
DECLARE_ARCH_ENTRY(arch, blob_name, stub_name, field_name, getter_name)
-private:
- STUBGEN_ARCH_ENTRIES_DO(DECLARE_ARCH_ENTRY, DECLARE_ARCH_ENTRY_INIT)
+#define DECLARE_ARCH_ENTRY_ARRAY(arch, blob_name, stub_name, field_name, getter_name, count) \
+ static address STUB_FIELD_NAME(field_name) [count] ;
+private:
+ STUBGEN_ARCH_ENTRIES_DO(DECLARE_ARCH_ENTRY, DECLARE_ARCH_ENTRY_INIT, DECLARE_ARCH_ENTRY_ARRAY)
+
+#undef DECLARE_ARCH_ENTRY_ARRAY
#undef DECLARE_ARCH_ENTRY_INIT
#undef DECLARE_ARCH_ENTRY
@@ -79,8 +83,12 @@ private:
#define DEFINE_ARCH_ENTRY_GETTER_INIT(arch, blob_name, stub_name, field_name, getter_name, init_function) \
DEFINE_ARCH_ENTRY_GETTER(arch, blob_name, stub_name, field_name, getter_name)
- STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY_GETTER, DEFINE_ARCH_ENTRY_GETTER_INIT)
+#define DEFINE_ARCH_ENTRY_GETTER_ARRAY(arch, blob_name, stub_name, field_name, getter_name, count) \
+ static address getter_name(int idx) { return STUB_FIELD_NAME(field_name) [idx] ; }
+ STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY_GETTER, DEFINE_ARCH_ENTRY_GETTER_INIT, DEFINE_ARCH_ENTRY_GETTER_ARRAY)
+
+#undef DEFINE_ARCH_ENTRY_GETTER_ARRAY
#undef DEFINE_ARCH_ENTRY_GETTER_INIT
#undef DEFINE_ARCH_ENTRY_GETTER
diff --git a/src/hotspot/cpu/riscv/vm_version_riscv.cpp b/src/hotspot/cpu/riscv/vm_version_riscv.cpp
index 36f0864da0b..3a6415d52bd 100644
--- a/src/hotspot/cpu/riscv/vm_version_riscv.cpp
+++ b/src/hotspot/cpu/riscv/vm_version_riscv.cpp
@@ -420,11 +420,6 @@ void VM_Version::c2_initialize() {
FLAG_SET_DEFAULT(UseSHA3Intrinsics, false);
}
- // UseSHA
- if (!(UseSHA1Intrinsics || UseSHA256Intrinsics || UseSHA3Intrinsics || UseSHA512Intrinsics)) {
- FLAG_SET_DEFAULT(UseSHA, false);
- }
-
// AES
if (UseZvkn) {
UseAES = UseAES || FLAG_IS_DEFAULT(UseAES);
diff --git a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp
index 93d6051aa76..e1d8d062c23 100644
--- a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -2251,9 +2251,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
// but not necessarily exactly of type default_type.
NearLabel known_ok, halt;
metadata2reg(default_type->constant_encoding(), tmp);
- if (UseCompressedClassPointers) {
- __ encode_klass_not_null(tmp);
- }
+ __ encode_klass_not_null(tmp);
if (basic_type != T_OBJECT) {
__ cmp_klass(tmp, dst, Z_R1_scratch);
@@ -2540,13 +2538,8 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, L
// Get object class.
// Not a safepoint as obj null check happens earlier.
if (op->fast_check()) {
- if (UseCompressedClassPointers) {
- __ load_klass(klass_RInfo, obj);
- __ compareU64_and_branch(k_RInfo, klass_RInfo, Assembler::bcondNotEqual, *failure_target);
- } else {
- __ z_cg(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
- __ branch_optimized(Assembler::bcondNotEqual, *failure_target);
- }
+ __ load_klass(klass_RInfo, obj);
+ __ compareU64_and_branch(k_RInfo, klass_RInfo, Assembler::bcondNotEqual, *failure_target);
// Successful cast, fall through to profile or jump.
} else {
bool need_slow_path = !k->is_loaded() ||
diff --git a/src/hotspot/cpu/s390/c1_LIRGenerator_s390.cpp b/src/hotspot/cpu/s390/c1_LIRGenerator_s390.cpp
index 5a0fd5f9561..1ffd172df8f 100644
--- a/src/hotspot/cpu/s390/c1_LIRGenerator_s390.cpp
+++ b/src/hotspot/cpu/s390/c1_LIRGenerator_s390.cpp
@@ -1046,6 +1046,7 @@ void LIRGenerator::volatile_field_store(LIR_Opr value, LIR_Address* address,
void LIRGenerator::volatile_field_load(LIR_Address* address, LIR_Opr result,
CodeEmitInfo* info) {
__ load(address, result, info);
+ __ membar_acquire();
}
void LIRGenerator::do_update_CRC32(Intrinsic* x) {
diff --git a/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp b/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp
index 993c1a1b552..813143938f9 100644
--- a/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -107,10 +107,10 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
}
if (len->is_valid()) {
- // Length will be in the klass gap, if one exists.
+ // Length will be in the klass gap.
z_st(len, Address(obj, arrayOopDesc::length_offset_in_bytes()));
- } else if (UseCompressedClassPointers && !UseCompactObjectHeaders) {
- store_klass_gap(Rzero, obj); // Zero klass gap for compressed oops.
+ } else if (!UseCompactObjectHeaders) {
+ store_klass_gap(Rzero, obj); // Zero klass gap.
}
}
diff --git a/src/hotspot/cpu/s390/interp_masm_s390.cpp b/src/hotspot/cpu/s390/interp_masm_s390.cpp
index a80ca26239b..d5239898dd7 100644
--- a/src/hotspot/cpu/s390/interp_masm_s390.cpp
+++ b/src/hotspot/cpu/s390/interp_masm_s390.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -1259,27 +1259,15 @@ void InterpreterMacroAssembler::profile_final_call(Register mdp) {
void InterpreterMacroAssembler::profile_virtual_call(Register receiver,
Register mdp,
- Register reg2,
- bool receiver_can_be_null) {
+ Register reg2) {
if (ProfileInterpreter) {
NearLabel profile_continue;
// If no method data exists, go to profile_continue.
test_method_data_pointer(mdp, profile_continue);
- NearLabel skip_receiver_profile;
- if (receiver_can_be_null) {
- NearLabel not_null;
- compareU64_and_branch(receiver, (intptr_t)0L, bcondNotEqual, not_null);
- // We are making a call. Increment the count for null receiver.
- increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
- z_bru(skip_receiver_profile);
- bind(not_null);
- }
-
// Record the receiver type.
record_klass_in_profile(receiver, mdp, reg2);
- bind(skip_receiver_profile);
// The method data pointer needs to be updated to reflect the new target.
update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size()));
diff --git a/src/hotspot/cpu/s390/interp_masm_s390.hpp b/src/hotspot/cpu/s390/interp_masm_s390.hpp
index d981f9ea01e..b816185b065 100644
--- a/src/hotspot/cpu/s390/interp_masm_s390.hpp
+++ b/src/hotspot/cpu/s390/interp_masm_s390.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -296,8 +296,7 @@ class InterpreterMacroAssembler: public MacroAssembler {
void profile_call(Register mdp);
void profile_final_call(Register mdp);
void profile_virtual_call(Register receiver, Register mdp,
- Register scratch2,
- bool receiver_can_be_null = false);
+ Register scratch2);
void profile_ret(Register return_bci, Register mdp);
void profile_null_seen(Register mdp);
void profile_typecheck(Register mdp, Register klass, Register scratch);
diff --git a/src/hotspot/cpu/s390/macroAssembler_s390.cpp b/src/hotspot/cpu/s390/macroAssembler_s390.cpp
index 6e132f895bd..de3608e74ba 100644
--- a/src/hotspot/cpu/s390/macroAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/macroAssembler_s390.cpp
@@ -1237,7 +1237,6 @@ void MacroAssembler::load_narrow_oop(Register t, narrowOop a) {
// Load narrow klass constant, compression required.
void MacroAssembler::load_narrow_klass(Register t, Klass* k) {
- assert(UseCompressedClassPointers, "must be on to call this method");
narrowKlass encoded_k = CompressedKlassPointers::encode(k);
load_const_32to64(t, encoded_k, false /*sign_extend*/);
}
@@ -1255,7 +1254,6 @@ void MacroAssembler::compare_immediate_narrow_oop(Register oop1, narrowOop oop2)
// Compare narrow oop in reg with narrow oop constant, no decompression.
void MacroAssembler::compare_immediate_narrow_klass(Register klass1, Klass* klass2) {
- assert(UseCompressedClassPointers, "must be on to call this method");
narrowKlass encoded_k = CompressedKlassPointers::encode(klass2);
Assembler::z_clfi(klass1, encoded_k);
@@ -1348,8 +1346,6 @@ int MacroAssembler::patch_load_narrow_oop(address pos, oop o) {
// Patching the immediate value of CPU version dependent load_narrow_klass sequence.
// The passed ptr must NOT be in compressed format!
int MacroAssembler::patch_load_narrow_klass(address pos, Klass* k) {
- assert(UseCompressedClassPointers, "Can only patch compressed klass pointers");
-
narrowKlass nk = CompressedKlassPointers::encode(k);
return patch_load_const_32to64(pos, nk);
}
@@ -1364,8 +1360,6 @@ int MacroAssembler::patch_compare_immediate_narrow_oop(address pos, oop o) {
// Patching the immediate value of CPU version dependent compare_immediate_narrow_klass sequence.
// The passed ptr must NOT be in compressed format!
int MacroAssembler::patch_compare_immediate_narrow_klass(address pos, Klass* k) {
- assert(UseCompressedClassPointers, "Can only patch compressed klass pointers");
-
narrowKlass nk = CompressedKlassPointers::encode(k);
return patch_compare_immediate_32(pos, nk);
}
@@ -2235,10 +2229,8 @@ int MacroAssembler::ic_check(int end_alignment) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(R1_scratch, R2_receiver);
- } else if (UseCompressedClassPointers) {
- z_llgf(R1_scratch, Address(R2_receiver, oopDesc::klass_offset_in_bytes()));
} else {
- z_lg(R1_scratch, Address(R2_receiver, oopDesc::klass_offset_in_bytes()));
+ z_llgf(R1_scratch, Address(R2_receiver, oopDesc::klass_offset_in_bytes()));
}
z_cg(R1_scratch, Address(R9_data, in_bytes(CompiledICData::speculated_klass_offset())));
z_bre(success);
@@ -3916,7 +3908,6 @@ void MacroAssembler::encode_klass_not_null(Register dst, Register src) {
address base = CompressedKlassPointers::base();
int shift = CompressedKlassPointers::shift();
bool need_zero_extend = base != nullptr;
- assert(UseCompressedClassPointers, "only for compressed klass ptrs");
BLOCK_COMMENT("cKlass encoder {");
@@ -4013,7 +4004,6 @@ int MacroAssembler::instr_size_for_decode_klass_not_null() {
address base = CompressedKlassPointers::base();
int shift_size = CompressedKlassPointers::shift() == 0 ? 0 : 6; /* sllg */
int addbase_size = 0;
- assert(UseCompressedClassPointers, "only for compressed klass ptrs");
if (base != nullptr) {
unsigned int base_h = ((unsigned long)base)>>32;
@@ -4043,7 +4033,6 @@ void MacroAssembler::decode_klass_not_null(Register dst) {
address base = CompressedKlassPointers::base();
int shift = CompressedKlassPointers::shift();
int beg_off = offset();
- assert(UseCompressedClassPointers, "only for compressed klass ptrs");
BLOCK_COMMENT("cKlass decoder (const size) {");
@@ -4085,7 +4074,6 @@ void MacroAssembler::decode_klass_not_null(Register dst) {
void MacroAssembler::decode_klass_not_null(Register dst, Register src) {
address base = CompressedKlassPointers::base();
int shift = CompressedKlassPointers::shift();
- assert(UseCompressedClassPointers, "only for compressed klass ptrs");
BLOCK_COMMENT("cKlass decoder {");
@@ -4125,13 +4113,9 @@ void MacroAssembler::decode_klass_not_null(Register dst, Register src) {
}
void MacroAssembler::load_klass(Register klass, Address mem) {
- if (UseCompressedClassPointers) {
- z_llgf(klass, mem);
- // Attention: no null check here!
- decode_klass_not_null(klass);
- } else {
- z_lg(klass, mem);
- }
+ z_llgf(klass, mem);
+ // Attention: no null check here!
+ decode_klass_not_null(klass);
}
// Loads the obj's Klass* into dst.
@@ -4154,10 +4138,8 @@ void MacroAssembler::cmp_klass(Register klass, Register obj, Register tmp) {
assert_different_registers(klass, obj, tmp);
load_narrow_klass_compact(tmp, obj);
z_cr(klass, tmp);
- } else if (UseCompressedClassPointers) {
- z_c(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
} else {
- z_cg(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
+ z_c(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
}
BLOCK_COMMENT("} cmp_klass");
}
@@ -4170,12 +4152,9 @@ void MacroAssembler::cmp_klasses_from_objects(Register obj1, Register obj2, Regi
load_narrow_klass_compact(tmp1, obj1);
load_narrow_klass_compact(tmp2, obj2);
z_cr(tmp1, tmp2);
- } else if (UseCompressedClassPointers) {
+ } else {
z_l(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
z_c(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes()));
- } else {
- z_lg(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
- z_cg(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes()));
}
BLOCK_COMMENT("} cmp_klasses_from_objects");
}
@@ -4184,36 +4163,28 @@ void MacroAssembler::load_klass(Register klass, Register src_oop) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(klass, src_oop);
decode_klass_not_null(klass);
- } else if (UseCompressedClassPointers) {
+ } else {
z_llgf(klass, oopDesc::klass_offset_in_bytes(), src_oop);
decode_klass_not_null(klass);
- } else {
- z_lg(klass, oopDesc::klass_offset_in_bytes(), src_oop);
}
}
void MacroAssembler::store_klass(Register klass, Register dst_oop, Register ck) {
assert(!UseCompactObjectHeaders, "Don't use with compact headers");
- if (UseCompressedClassPointers) {
- assert_different_registers(dst_oop, klass, Z_R0);
- if (ck == noreg) ck = klass;
- encode_klass_not_null(ck, klass);
- z_st(ck, Address(dst_oop, oopDesc::klass_offset_in_bytes()));
- } else {
- z_stg(klass, Address(dst_oop, oopDesc::klass_offset_in_bytes()));
- }
+ assert_different_registers(dst_oop, klass, Z_R0);
+ if (ck == noreg) ck = klass;
+ encode_klass_not_null(ck, klass);
+ z_st(ck, Address(dst_oop, oopDesc::klass_offset_in_bytes()));
}
void MacroAssembler::store_klass_gap(Register s, Register d) {
assert(!UseCompactObjectHeaders, "Don't use with compact headers");
- if (UseCompressedClassPointers) {
- assert(s != d, "not enough registers");
- // Support s = noreg.
- if (s != noreg) {
- z_st(s, Address(d, oopDesc::klass_gap_offset_in_bytes()));
- } else {
- z_mvhi(Address(d, oopDesc::klass_gap_offset_in_bytes()), 0);
- }
+ assert(s != d, "not enough registers");
+ // Support s = noreg.
+ if (s != noreg) {
+ z_st(s, Address(d, oopDesc::klass_gap_offset_in_bytes()));
+ } else {
+ z_mvhi(Address(d, oopDesc::klass_gap_offset_in_bytes()), 0);
}
}
@@ -4227,67 +4198,64 @@ void MacroAssembler::compare_klass_ptr(Register Rop1, int64_t disp, Register Rba
BLOCK_COMMENT("compare klass ptr {");
- if (UseCompressedClassPointers) {
- const int shift = CompressedKlassPointers::shift();
- address base = CompressedKlassPointers::base();
+ const int shift = CompressedKlassPointers::shift();
+ address base = CompressedKlassPointers::base();
- if (UseCompactObjectHeaders) {
- assert(shift >= 3, "cKlass encoder detected bad shift");
- } else {
- assert((shift == 0) || (shift == 3), "cKlass encoder detected bad shift");
- }
- assert_different_registers(Rop1, Z_R0);
- assert_different_registers(Rop1, Rbase, Z_R1);
-
- // First encode register oop and then compare with cOop in memory.
- // This sequence saves an unnecessary cOop load and decode.
- if (base == nullptr) {
- if (shift == 0) {
- z_cl(Rop1, disp, Rbase); // Unscaled
- } else {
- z_srlg(Z_R0, Rop1, shift); // ZeroBased
- z_cl(Z_R0, disp, Rbase);
- }
- } else { // HeapBased
-#ifdef ASSERT
- bool used_R0 = true;
- bool used_R1 = true;
-#endif
- Register current = Rop1;
- Label done;
-
- if (maybenull) { // null pointer must be preserved!
- z_ltgr(Z_R0, current);
- z_bre(done);
- current = Z_R0;
- }
-
- unsigned int base_h = ((unsigned long)base)>>32;
- unsigned int base_l = (unsigned int)((unsigned long)base);
- if ((base_h != 0) && (base_l == 0) && VM_Version::has_HighWordInstr()) {
- lgr_if_needed(Z_R0, current);
- z_aih(Z_R0, -((int)base_h)); // Base has no set bits in lower half.
- } else if ((base_h == 0) && (base_l != 0)) {
- lgr_if_needed(Z_R0, current);
- z_agfi(Z_R0, -(int)base_l);
- } else {
- int pow2_offset = get_oop_base_complement(Z_R1, ((uint64_t)(intptr_t)base));
- add2reg_with_index(Z_R0, pow2_offset, Z_R1, Rop1); // Subtract base by adding complement.
- }
-
- if (shift != 0) {
- z_srlg(Z_R0, Z_R0, shift);
- }
- bind(done);
- z_cl(Z_R0, disp, Rbase);
-#ifdef ASSERT
- if (used_R0) preset_reg(Z_R0, 0xb05bUL, 2);
- if (used_R1) preset_reg(Z_R1, 0xb06bUL, 2);
-#endif
- }
+ if (UseCompactObjectHeaders) {
+ assert(shift >= 3, "cKlass encoder detected bad shift");
} else {
- z_clg(Rop1, disp, Z_R0, Rbase);
+ assert((shift == 0) || (shift == 3), "cKlass encoder detected bad shift");
}
+ assert_different_registers(Rop1, Z_R0);
+ assert_different_registers(Rop1, Rbase, Z_R1);
+
+ // First encode register oop and then compare with cOop in memory.
+ // This sequence saves an unnecessary cOop load and decode.
+ if (base == nullptr) {
+ if (shift == 0) {
+ z_cl(Rop1, disp, Rbase); // Unscaled
+ } else {
+ z_srlg(Z_R0, Rop1, shift); // ZeroBased
+ z_cl(Z_R0, disp, Rbase);
+ }
+ } else { // HeapBased
+#ifdef ASSERT
+ bool used_R0 = true;
+ bool used_R1 = true;
+#endif
+ Register current = Rop1;
+ Label done;
+
+ if (maybenull) { // null pointer must be preserved!
+ z_ltgr(Z_R0, current);
+ z_bre(done);
+ current = Z_R0;
+ }
+
+ unsigned int base_h = ((unsigned long)base)>>32;
+ unsigned int base_l = (unsigned int)((unsigned long)base);
+ if ((base_h != 0) && (base_l == 0) && VM_Version::has_HighWordInstr()) {
+ lgr_if_needed(Z_R0, current);
+ z_aih(Z_R0, -((int)base_h)); // Base has no set bits in lower half.
+ } else if ((base_h == 0) && (base_l != 0)) {
+ lgr_if_needed(Z_R0, current);
+ z_agfi(Z_R0, -(int)base_l);
+ } else {
+ int pow2_offset = get_oop_base_complement(Z_R1, ((uint64_t)(intptr_t)base));
+ add2reg_with_index(Z_R0, pow2_offset, Z_R1, Rop1); // Subtract base by adding complement.
+ }
+
+ if (shift != 0) {
+ z_srlg(Z_R0, Z_R0, shift);
+ }
+ bind(done);
+ z_cl(Z_R0, disp, Rbase);
+#ifdef ASSERT
+ if (used_R0) preset_reg(Z_R0, 0xb05bUL, 2);
+ if (used_R1) preset_reg(Z_R1, 0xb06bUL, 2);
+#endif
+ }
+
BLOCK_COMMENT("} compare klass ptr");
}
diff --git a/src/hotspot/cpu/s390/macroAssembler_s390.hpp b/src/hotspot/cpu/s390/macroAssembler_s390.hpp
index da24ae80d45..32e484d4790 100644
--- a/src/hotspot/cpu/s390/macroAssembler_s390.hpp
+++ b/src/hotspot/cpu/s390/macroAssembler_s390.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2024 SAP SE. All rights reserved.
* Copyright (c) 2024 IBM Corporation. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -842,8 +842,7 @@ class MacroAssembler: public Assembler {
void store_klass(Register klass, Register dst_oop, Register ck = noreg); // Klass will get compressed if ck not provided.
void store_klass_gap(Register s, Register dst_oop);
void load_narrow_klass_compact(Register dst, Register src);
- // Compares the Klass pointer of an object to a given Klass (which might be narrow,
- // depending on UseCompressedClassPointers).
+ // Compares the narrow Klass pointer of an object to a given narrow Klass
void cmp_klass(Register klass, Register obj, Register tmp);
// Compares the Klass pointer of two objects obj1 and obj2. Result is in the condition flags.
// Uses tmp1 and tmp2 as temporary registers.
diff --git a/src/hotspot/cpu/s390/matcher_s390.hpp b/src/hotspot/cpu/s390/matcher_s390.hpp
index 99461e33e3c..b04a6566d41 100644
--- a/src/hotspot/cpu/s390/matcher_s390.hpp
+++ b/src/hotspot/cpu/s390/matcher_s390.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -82,7 +82,6 @@
static bool narrow_klass_use_complex_address() {
NOT_LP64(ShouldNotCallThis());
- assert(UseCompressedClassPointers, "only for compressed klass code");
// TODO HS25: z port if (MatchDecodeNodes) return true;
return false;
}
diff --git a/src/hotspot/cpu/s390/methodHandles_s390.cpp b/src/hotspot/cpu/s390/methodHandles_s390.cpp
index e3de6d911be..dfb8ce09b27 100644
--- a/src/hotspot/cpu/s390/methodHandles_s390.cpp
+++ b/src/hotspot/cpu/s390/methodHandles_s390.cpp
@@ -120,16 +120,12 @@ void MethodHandles::verify_ref_kind(MacroAssembler* _masm, int ref_kind,
__ z_nilf(temp, java_lang_invoke_MemberName::MN_REFERENCE_KIND_MASK);
__ compare32_and_branch(temp, constant(ref_kind), Assembler::bcondEqual, L);
- {
- char *buf = NEW_C_HEAP_ARRAY(char, 100, mtInternal);
-
- jio_snprintf(buf, 100, "verify_ref_kind expected %x", ref_kind);
- if (ref_kind == JVM_REF_invokeVirtual || ref_kind == JVM_REF_invokeSpecial) {
- // Could do this for all ref_kinds, but would explode assembly code size.
- trace_method_handle(_masm, buf);
- }
- __ stop(buf);
+ const char* msg = ref_kind_to_verify_msg(ref_kind);
+ if (ref_kind == JVM_REF_invokeVirtual || ref_kind == JVM_REF_invokeSpecial) {
+ // Could do this for all ref_kinds, but would explode assembly code size.
+ trace_method_handle(_masm, msg);
}
+ __ stop(msg);
BLOCK_COMMENT("} verify_ref_kind");
diff --git a/src/hotspot/cpu/s390/stubDeclarations_s390.hpp b/src/hotspot/cpu/s390/stubDeclarations_s390.hpp
index c3ad3cefeb9..d0e26beedab 100644
--- a/src/hotspot/cpu/s390/stubDeclarations_s390.hpp
+++ b/src/hotspot/cpu/s390/stubDeclarations_s390.hpp
@@ -29,28 +29,32 @@
#define STUBGEN_PREUNIVERSE_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(preuniverse, 0) \
#define STUBGEN_INITIAL_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(initial, 20000) \
#define STUBGEN_CONTINUATION_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(continuation, 2000) \
#define STUBGEN_COMPILER_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(compiler, 20000 ) \
do_stub(compiler, partial_subtype_check) \
do_arch_entry(zarch, compiler, partial_subtype_check, \
@@ -60,7 +64,8 @@
#define STUBGEN_FINAL_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(final, 20000) \
diff --git a/src/hotspot/cpu/s390/stubGenerator_s390.cpp b/src/hotspot/cpu/s390/stubGenerator_s390.cpp
index 2aa365be999..3f16312eb48 100644
--- a/src/hotspot/cpu/s390/stubGenerator_s390.cpp
+++ b/src/hotspot/cpu/s390/stubGenerator_s390.cpp
@@ -3422,7 +3422,7 @@ class StubGenerator: public StubCodeGenerator {
}
public:
- StubGenerator(CodeBuffer* code, BlobId blob_id) : StubCodeGenerator(code, blob_id) {
+ StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) : StubCodeGenerator(code, blob_id, stub_data) {
switch(blob_id) {
case BlobId::stubgen_preuniverse_id:
generate_preuniverse_stubs();
@@ -3479,6 +3479,6 @@ class StubGenerator: public StubCodeGenerator {
};
-void StubGenerator_generate(CodeBuffer* code, BlobId blob_id) {
- StubGenerator g(code, blob_id);
+void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) {
+ StubGenerator g(code, blob_id, stub_data);
}
diff --git a/src/hotspot/cpu/s390/stubRoutines_s390.cpp b/src/hotspot/cpu/s390/stubRoutines_s390.cpp
index 6feb20f9604..eda0ebfdecc 100644
--- a/src/hotspot/cpu/s390/stubRoutines_s390.cpp
+++ b/src/hotspot/cpu/s390/stubRoutines_s390.cpp
@@ -40,8 +40,12 @@
#define DEFINE_ARCH_ENTRY_INIT(arch, blob_name, stub_name, field_name, getter_name, init_function) \
address StubRoutines:: arch :: STUB_FIELD_NAME(field_name) = CAST_FROM_FN_PTR(address, init_function);
-STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY, DEFINE_ARCH_ENTRY_INIT)
+#define DEFINE_ARCH_ENTRY_ARRAY(arch, blob_name, stub_name, field_name, getter_name, count) \
+ address StubRoutines:: arch :: STUB_FIELD_NAME(field_name) [idx] ;
+STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY, DEFINE_ARCH_ENTRY_INIT, DEFINE_ARCH_ENTRY_ARRAY)
+
+#undef DEFINE_ARCH_ENTRY_ARRAY
#undef DEFINE_ARCH_ENTRY_INIT
#undef DEFINE_ARCH_ENTRY
@@ -736,3 +740,9 @@ juint StubRoutines::zarch::_crc32c_table[CRC32_TABLES][CRC32_COLUMN_SIZE] = {
}
#endif
};
+
+#if INCLUDE_CDS
+// nothing to do for s390
+void StubRoutines::init_AOTAddressTable() {
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/s390/stubRoutines_s390.hpp b/src/hotspot/cpu/s390/stubRoutines_s390.hpp
index 0a07efae46c..e575115b731 100644
--- a/src/hotspot/cpu/s390/stubRoutines_s390.hpp
+++ b/src/hotspot/cpu/s390/stubRoutines_s390.hpp
@@ -81,9 +81,13 @@ class zarch {
#define DECLARE_ARCH_ENTRY_INIT(arch, blob_name, stub_name, field_name, getter_name, init_function) \
DECLARE_ARCH_ENTRY(arch, blob_name, stub_name, field_name, getter_name)
-private:
- STUBGEN_ARCH_ENTRIES_DO(DECLARE_ARCH_ENTRY, DECLARE_ARCH_ENTRY_INIT)
+#define DECLARE_ARCH_ENTRY_ARRAY(arch, blob_name, stub_name, field_name, getter_name, count) \
+ static address STUB_FIELD_NAME(field_name) [count] ;
+private:
+ STUBGEN_ARCH_ENTRIES_DO(DECLARE_ARCH_ENTRY, DECLARE_ARCH_ENTRY_INIT, DECLARE_ARCH_ENTRY_ARRAY)
+
+#undef DECLARE_ARCH_ENTRY_ARRAY
#undef DECLARE_ARCH_ENTRY_INIT
#undef DECLARE_ARCH_ENTRY
@@ -108,8 +112,12 @@ private:
#define DEFINE_ARCH_ENTRY_GETTER_INIT(arch, blob_name, stub_name, field_name, getter_name, init_function) \
DEFINE_ARCH_ENTRY_GETTER(arch, blob_name, stub_name, field_name, getter_name)
- STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY_GETTER, DEFINE_ARCH_ENTRY_GETTER_INIT)
+#define DEFINE_ARCH_ENTRY_GETTER_ARRAY(arch, blob_name, stub_name, field_name, getter_name, count) \
+ static address getter_name(int idx) { return STUB_FIELD_NAME(field_name) [idx] ; }
+ STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY_GETTER, DEFINE_ARCH_ENTRY_GETTER_INIT, DEFINE_ARCH_ENTRY_GETTER_ARRAY)
+
+#undef DEFINE_ARCH_ENTRY_GETTER_ARRAY
#undef DEFINE_ARCH_ENTRY_GETTER_INIT
#undef DEFINE_ARCH_ENTRY_GETTER
diff --git a/src/hotspot/cpu/s390/vm_version_s390.cpp b/src/hotspot/cpu/s390/vm_version_s390.cpp
index 7f5b4870aab..7e9000991ca 100644
--- a/src/hotspot/cpu/s390/vm_version_s390.cpp
+++ b/src/hotspot/cpu/s390/vm_version_s390.cpp
@@ -289,10 +289,6 @@ void VM_Version::initialize() {
FLAG_SET_DEFAULT(UseSHA3Intrinsics, false);
}
- if (!(UseSHA1Intrinsics || UseSHA256Intrinsics || UseSHA512Intrinsics)) {
- FLAG_SET_DEFAULT(UseSHA, false);
- }
-
if (UseSecondarySupersTable && VM_Version::get_model_index() < 5 /* z196/z11 */) {
if (!FLAG_IS_DEFAULT(UseSecondarySupersTable)) {
warning("UseSecondarySupersTable requires z196 or later.");
diff --git a/src/hotspot/cpu/x86/assembler_x86.cpp b/src/hotspot/cpu/x86/assembler_x86.cpp
index 38a28a6ec49..a4f2968f0d1 100644
--- a/src/hotspot/cpu/x86/assembler_x86.cpp
+++ b/src/hotspot/cpu/x86/assembler_x86.cpp
@@ -3472,7 +3472,7 @@ void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
emit_int16(0x6F, (0xC0 | encode));
}
-void Assembler::vmovw(XMMRegister dst, Register src) {
+void Assembler::evmovw(XMMRegister dst, Register src) {
assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
attributes.set_is_evex_instruction();
@@ -3480,7 +3480,7 @@ void Assembler::vmovw(XMMRegister dst, Register src) {
emit_int16(0x6E, (0xC0 | encode));
}
-void Assembler::vmovw(Register dst, XMMRegister src) {
+void Assembler::evmovw(Register dst, XMMRegister src) {
assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
attributes.set_is_evex_instruction();
@@ -3488,6 +3488,36 @@ void Assembler::vmovw(Register dst, XMMRegister src) {
emit_int16(0x7E, (0xC0 | encode));
}
+void Assembler::evmovw(XMMRegister dst, Address src) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
+ attributes.set_is_evex_instruction();
+ vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
+ emit_int8(0x6E);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::evmovw(Address dst, XMMRegister src) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
+ attributes.set_is_evex_instruction();
+ vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
+ emit_int8(0x7E);
+ emit_operand(src, dst, 0);
+}
+
+void Assembler::evmovw(XMMRegister dst, XMMRegister src) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ attributes.set_is_evex_instruction();
+ int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
+ emit_int16(0x6E, (0xC0 | encode));
+}
+
void Assembler::vmovdqu(XMMRegister dst, Address src) {
assert(UseAVX > 0, "");
InstructionMark im(this);
@@ -7310,6 +7340,42 @@ void Assembler::etzcntq(Register dst, Address src, bool no_flags) {
emit_operand(dst, src, 0);
}
+void Assembler::evucomish(XMMRegister dst, Address src) {
+ InstructionMark im(this);
+ InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
+ attributes.set_is_evex_instruction();
+ vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes);
+ emit_int8(0x2E);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::evucomish(XMMRegister dst, XMMRegister src) {
+ InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ attributes.set_is_evex_instruction();
+ int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes);
+ emit_int16(0x2E, (0xC0 | encode));
+}
+
+void Assembler::evucomxsh(XMMRegister dst, Address src) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
+ attributes.set_is_evex_instruction();
+ vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
+ emit_int8(0x2E);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::evucomxsh(XMMRegister dst, XMMRegister src) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ attributes.set_is_evex_instruction();
+ int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
+ emit_int16(0x2E, (0xC0 | encode));
+}
+
void Assembler::ucomisd(XMMRegister dst, Address src) {
InstructionMark im(this);
InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
@@ -7327,7 +7393,7 @@ void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
emit_int16(0x2E, (0xC0 | encode));
}
-void Assembler::vucomxsd(XMMRegister dst, Address src) {
+void Assembler::evucomxsd(XMMRegister dst, Address src) {
assert(VM_Version::supports_avx10_2(), "");
InstructionMark im(this);
InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
@@ -7338,7 +7404,7 @@ void Assembler::vucomxsd(XMMRegister dst, Address src) {
emit_operand(dst, src, 0);
}
-void Assembler::vucomxsd(XMMRegister dst, XMMRegister src) {
+void Assembler::evucomxsd(XMMRegister dst, XMMRegister src) {
assert(VM_Version::supports_avx10_2(), "");
InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
attributes.set_is_evex_instruction();
@@ -7361,7 +7427,7 @@ void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
emit_int16(0x2E, (0xC0 | encode));
}
-void Assembler::vucomxss(XMMRegister dst, Address src) {
+void Assembler::evucomxss(XMMRegister dst, Address src) {
assert(VM_Version::supports_avx10_2(), "");
InstructionMark im(this);
InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
@@ -7372,7 +7438,7 @@ void Assembler::vucomxss(XMMRegister dst, Address src) {
emit_operand(dst, src, 0);
}
-void Assembler::vucomxss(XMMRegister dst, XMMRegister src) {
+void Assembler::evucomxss(XMMRegister dst, XMMRegister src) {
assert(VM_Version::supports_avx10_2(), "");
InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
attributes.set_is_evex_instruction();
@@ -8411,30 +8477,6 @@ void Assembler::vmulsh(XMMRegister dst, XMMRegister nds, XMMRegister src) {
emit_int16(0x59, (0xC0 | encode));
}
-void Assembler::vmaxsh(XMMRegister dst, XMMRegister nds, XMMRegister src) {
- assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
- InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- attributes.set_is_evex_instruction();
- int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
- emit_int16(0x5F, (0xC0 | encode));
-}
-
-void Assembler::eminmaxsh(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
- assert(VM_Version::supports_avx10_2(), "");
- InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- attributes.set_is_evex_instruction();
- int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, &attributes);
- emit_int24(0x53, (0xC0 | encode), imm8);
-}
-
-void Assembler::vminsh(XMMRegister dst, XMMRegister nds, XMMRegister src) {
- assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
- InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- attributes.set_is_evex_instruction();
- int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
- emit_int16(0x5D, (0xC0 | encode));
-}
-
void Assembler::vsqrtsh(XMMRegister dst, XMMRegister src) {
assert(VM_Version::supports_avx512_fp16(), "requires AVX512-FP16");
InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
@@ -13369,48 +13411,38 @@ bool Assembler::is_demotable(bool no_flags, int dst_enc, int nds_enc) {
return (!no_flags && dst_enc == nds_enc);
}
-void Assembler::vmaxss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
- assert(VM_Version::supports_avx(), "");
- InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
- emit_int16(0x5F, (0xC0 | encode));
-}
-
-void Assembler::vmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
- assert(VM_Version::supports_avx(), "");
- InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- attributes.set_rex_vex_w_reverted();
- int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
- emit_int16(0x5F, (0xC0 | encode));
-}
-
-void Assembler::vminss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
- assert(VM_Version::supports_avx(), "");
- InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
- emit_int16(0x5D, (0xC0 | encode));
-}
-
-void Assembler::eminmaxss(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
+void Assembler::evminmaxsh(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int imm8) {
assert(VM_Version::supports_avx10_2(), "");
- InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
+ int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, &attributes);
+ emit_int24(0x53, (0xC0 | encode), imm8);
+}
+
+void Assembler::evminmaxss(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int imm8) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
+ attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
emit_int24(0x53, (0xC0 | encode), imm8);
}
-void Assembler::vminsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
- assert(VM_Version::supports_avx(), "");
- InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- attributes.set_rex_vex_w_reverted();
- int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
- emit_int16(0x5D, (0xC0 | encode));
-}
-
-void Assembler::eminmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
+void Assembler::evminmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int imm8) {
assert(VM_Version::supports_avx10_2(), "");
- InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
attributes.set_is_evex_instruction();
+ attributes.set_embedded_opmask_register_specifier(mask);
+ if (merge) {
+ attributes.reset_is_clear_context();
+ }
int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
emit_int24(0x53, (0xC0 | encode), imm8);
}
diff --git a/src/hotspot/cpu/x86/assembler_x86.hpp b/src/hotspot/cpu/x86/assembler_x86.hpp
index 57a5e25d7a6..98684752b0c 100644
--- a/src/hotspot/cpu/x86/assembler_x86.hpp
+++ b/src/hotspot/cpu/x86/assembler_x86.hpp
@@ -1694,8 +1694,11 @@ private:
void movsbl(Register dst, Address src);
void movsbl(Register dst, Register src);
- void vmovw(XMMRegister dst, Register src);
- void vmovw(Register dst, XMMRegister src);
+ void evmovw(XMMRegister dst, Register src);
+ void evmovw(Register dst, XMMRegister src);
+ void evmovw(XMMRegister dst, Address src);
+ void evmovw(Address dst, XMMRegister src);
+ void evmovw(XMMRegister dst, XMMRegister src);
void movsbq(Register dst, Address src);
void movsbq(Register dst, Register src);
@@ -2329,17 +2332,23 @@ private:
void tzcntq(Register dst, Address src);
void etzcntq(Register dst, Address src, bool no_flags);
+ // Unordered Compare Scalar Half-Precision Floating-Point Values and set EFLAGS
+ void evucomish(XMMRegister dst, Address src);
+ void evucomish(XMMRegister dst, XMMRegister src);
+ void evucomxsh(XMMRegister dst, Address src);
+ void evucomxsh(XMMRegister dst, XMMRegister src);
+
// Unordered Compare Scalar Double-Precision Floating-Point Values and set EFLAGS
void ucomisd(XMMRegister dst, Address src);
void ucomisd(XMMRegister dst, XMMRegister src);
- void vucomxsd(XMMRegister dst, Address src);
- void vucomxsd(XMMRegister dst, XMMRegister src);
+ void evucomxsd(XMMRegister dst, Address src);
+ void evucomxsd(XMMRegister dst, XMMRegister src);
// Unordered Compare Scalar Single-Precision Floating-Point Values and set EFLAGS
void ucomiss(XMMRegister dst, Address src);
void ucomiss(XMMRegister dst, XMMRegister src);
- void vucomxss(XMMRegister dst, Address src);
- void vucomxss(XMMRegister dst, XMMRegister src);
+ void evucomxss(XMMRegister dst, Address src);
+ void evucomxss(XMMRegister dst, XMMRegister src);
void xabort(int8_t imm8);
@@ -2417,11 +2426,6 @@ private:
void vsubss(XMMRegister dst, XMMRegister nds, Address src);
void vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src);
- void vmaxss(XMMRegister dst, XMMRegister nds, XMMRegister src);
- void vmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src);
- void vminss(XMMRegister dst, XMMRegister nds, XMMRegister src);
- void vminsd(XMMRegister dst, XMMRegister nds, XMMRegister src);
-
void sarxl(Register dst, Register src1, Register src2);
void sarxl(Register dst, Address src1, Register src2);
void sarxq(Register dst, Register src1, Register src2);
@@ -2552,8 +2556,6 @@ private:
void vsubsh(XMMRegister dst, XMMRegister nds, XMMRegister src);
void vmulsh(XMMRegister dst, XMMRegister nds, XMMRegister src);
void vdivsh(XMMRegister dst, XMMRegister nds, XMMRegister src);
- void vmaxsh(XMMRegister dst, XMMRegister nds, XMMRegister src);
- void vminsh(XMMRegister dst, XMMRegister nds, XMMRegister src);
void vsqrtsh(XMMRegister dst, XMMRegister src);
void vfmadd132sh(XMMRegister dst, XMMRegister src1, XMMRegister src2);
@@ -2790,9 +2792,9 @@ private:
void vminpd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len);
// AVX10.2 floating point minmax instructions
- void eminmaxsh(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8);
- void eminmaxss(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8);
- void eminmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8);
+ void evminmaxsh(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int imm8);
+ void evminmaxss(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int imm8);
+ void evminmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int imm8);
void evminmaxph(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int imm8, int vector_len);
void evminmaxph(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int imm8, int vector_len);
void evminmaxps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int imm8, int vector_len);
diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp
index d9be0fdcc8d..5c05b3702bb 100644
--- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -42,6 +42,7 @@
#include "runtime/safepointMechanism.hpp"
#include "runtime/sharedRuntime.hpp"
#include "runtime/stubRoutines.hpp"
+#include "runtime/threadIdentifier.hpp"
#include "utilities/powerOfTwo.hpp"
#include "vmreg_x86.inline.hpp"
@@ -70,6 +71,17 @@ static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2],
static jlong *float_signflip_pool = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000));
static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000));
+#if INCLUDE_CDS
+// publish external addresses defined in this file
+void LIR_Assembler::init_AOTAddressTable(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr));
+ ADD(float_signmask_pool);
+ ADD(double_signmask_pool);
+ ADD(float_signflip_pool);
+ ADD(double_signflip_pool);
+#undef ADD
+}
+#endif // INCLUDE_CDS
NEEDS_CLEANUP // remove this definitions ?
const Register SYNC_header = rax; // synchronization header
@@ -77,23 +89,6 @@ const Register SHIFT_count = rcx; // where count for shift operations must be
#define __ _masm->
-
-static void select_different_registers(Register preserve,
- Register extra,
- Register &tmp1,
- Register &tmp2) {
- if (tmp1 == preserve) {
- assert_different_registers(tmp1, tmp2, extra);
- tmp1 = extra;
- } else if (tmp2 == preserve) {
- assert_different_registers(tmp1, tmp2, extra);
- tmp2 = extra;
- }
- assert_different_registers(preserve, tmp1, tmp2);
-}
-
-
-
static void select_different_registers(Register preserve,
Register extra,
Register &tmp1,
@@ -535,10 +530,23 @@ void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_cod
}
case T_LONG: {
+#if INCLUDE_CDS
+ if (AOTCodeCache::is_on_for_dump()) {
+ address b = c->as_pointer();
+ if (b == (address)ThreadIdentifier::unsafe_offset()) {
+ __ lea(dest->as_register_lo(), ExternalAddress(b));
+ break;
+ }
+ }
+#endif
assert(patch_code == lir_patch_none, "no patching handled here");
#if INCLUDE_CDS
if (AOTCodeCache::is_on_for_dump()) {
address b = c->as_pointer();
+ if (b == (address)ThreadIdentifier::unsafe_offset()) {
+ __ lea(dest->as_register_lo(), ExternalAddress(b));
+ break;
+ }
if (AOTRuntimeConstants::contains(b)) {
__ load_aotrc_address(dest->as_register_lo(), b);
break;
@@ -1309,12 +1317,8 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, L
} else if (obj == klass_RInfo) {
klass_RInfo = dst;
}
- if (k->is_loaded() && !UseCompressedClassPointers) {
- select_different_registers(obj, dst, k_RInfo, klass_RInfo);
- } else {
- Rtmp1 = op->tmp3()->as_register();
- select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
- }
+ Rtmp1 = op->tmp3()->as_register();
+ select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
assert_different_registers(obj, k_RInfo, klass_RInfo);
@@ -1348,12 +1352,8 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, L
if (op->fast_check()) {
// get object class
// not a safepoint as obj null check happens earlier
- if (UseCompressedClassPointers) {
- __ load_klass(Rtmp1, obj, tmp_load_klass);
- __ cmpptr(k_RInfo, Rtmp1);
- } else {
- __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
- }
+ __ load_klass(Rtmp1, obj, tmp_load_klass);
+ __ cmpptr(k_RInfo, Rtmp1);
__ jcc(Assembler::notEqual, *failure_target);
// successful cast, fall through to profile or jump
} else {
@@ -2651,9 +2651,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
// but not necessarily exactly of type default_type.
Label known_ok, halt;
__ mov_metadata(tmp, default_type->constant_encoding());
- if (UseCompressedClassPointers) {
- __ encode_klass_not_null(tmp, rscratch1);
- }
+ __ encode_klass_not_null(tmp, rscratch1);
if (basic_type != T_OBJECT) {
__ cmp_klass(tmp, dst, tmp2);
diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.hpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.hpp
index c4a368b54d8..6f179255e4a 100644
--- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.hpp
@@ -58,4 +58,7 @@ public:
void store_parameter(jobject c, int offset_from_esp_in_words);
void store_parameter(Metadata* c, int offset_from_esp_in_words);
+#if INCLUDE_CDS
+ void static init_AOTAddressTable(GrowableArray& external_addresses);
+#endif // INCLUDE_CDS
#endif // CPU_X86_C1_LIRASSEMBLER_X86_HPP
diff --git a/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp b/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp
index 5459e8df229..cc068cda7a9 100644
--- a/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp
+++ b/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1291,9 +1291,7 @@ void LIRGenerator::do_CheckCast(CheckCast* x) {
}
LIR_Opr reg = rlock_result(x);
LIR_Opr tmp3 = LIR_OprFact::illegalOpr;
- if (!x->klass()->is_loaded() || UseCompressedClassPointers) {
- tmp3 = new_register(objectType);
- }
+ tmp3 = new_register(objectType);
__ checkcast(reg, obj.result(), x->klass(),
new_register(objectType), new_register(objectType), tmp3,
x->direct_compare(), info_for_exception, patching_info, stub,
@@ -1313,9 +1311,7 @@ void LIRGenerator::do_InstanceOf(InstanceOf* x) {
}
obj.load_item();
LIR_Opr tmp3 = LIR_OprFact::illegalOpr;
- if (!x->klass()->is_loaded() || UseCompressedClassPointers) {
- tmp3 = new_register(objectType);
- }
+ tmp3 = new_register(objectType);
__ instanceof(reg, obj.result(), x->klass(),
new_register(objectType), new_register(objectType), tmp3,
x->direct_compare(), patching_info, x->profiled_method(), x->profiled_bci());
@@ -1436,4 +1432,5 @@ void LIRGenerator::volatile_field_load(LIR_Address* address, LIR_Opr result,
} else {
__ load(address, result, info);
}
+ __ membar_acquire();
}
diff --git a/src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp
index 88e2e6c8ba9..7adaea48ff1 100644
--- a/src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -85,14 +85,11 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
if (UseCompactObjectHeaders) {
movptr(t1, Address(klass, Klass::prototype_header_offset()));
movptr(Address(obj, oopDesc::mark_offset_in_bytes()), t1);
- } else if (UseCompressedClassPointers) { // Take care not to kill klass
+ } else { // Take care not to kill klass
movptr(Address(obj, oopDesc::mark_offset_in_bytes()), checked_cast(markWord::prototype().value()));
movptr(t1, klass);
encode_klass_not_null(t1, rscratch1);
movl(Address(obj, oopDesc::klass_offset_in_bytes()), t1);
- } else {
- movptr(Address(obj, oopDesc::mark_offset_in_bytes()), checked_cast(markWord::prototype().value()));
- movptr(Address(obj, oopDesc::klass_offset_in_bytes()), klass);
}
if (len->is_valid()) {
@@ -104,7 +101,7 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
xorl(t1, t1);
movl(Address(obj, base_offset), t1);
}
- } else if (UseCompressedClassPointers && !UseCompactObjectHeaders) {
+ } else if (!UseCompactObjectHeaders) {
xorptr(t1, t1);
store_klass_gap(obj, t1);
}
diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
index 5b5fb02967c..b4d8aa10de2 100644
--- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
@@ -152,7 +152,7 @@ inline Assembler::AvxVectorLen C2_MacroAssembler::vector_length_encoding(int vle
// Because the transitions from emitted code to the runtime
// monitorenter/exit helper stubs are so slow it's critical that
-// we inline both the stack-locking fast path and the inflated fast path.
+// we inline both the lock-stack fast path and the inflated fast path.
//
// See also: cmpFastLock and cmpFastUnlock.
//
@@ -1037,8 +1037,8 @@ void C2_MacroAssembler::evminmax_fp(int opcode, BasicType elem_bt,
}
}
-void C2_MacroAssembler::vminmax_fp(int opc, BasicType elem_bt, XMMRegister dst, KRegister mask,
- XMMRegister src1, XMMRegister src2, int vlen_enc) {
+void C2_MacroAssembler::vminmax_fp_avx10_2(int opc, BasicType elem_bt, XMMRegister dst, KRegister mask,
+ XMMRegister src1, XMMRegister src2, int vlen_enc) {
assert(opc == Op_MinV || opc == Op_MinReductionV ||
opc == Op_MaxV || opc == Op_MaxReductionV, "sanity");
@@ -1052,6 +1052,21 @@ void C2_MacroAssembler::vminmax_fp(int opc, BasicType elem_bt, XMMRegister dst,
}
}
+void C2_MacroAssembler::sminmax_fp_avx10_2(int opc, BasicType elem_bt, XMMRegister dst, KRegister mask,
+ XMMRegister src1, XMMRegister src2) {
+ assert(opc == Op_MinF || opc == Op_MaxF ||
+ opc == Op_MinD || opc == Op_MaxD, "sanity");
+
+ int imm8 = (opc == Op_MinF || opc == Op_MinD) ? AVX10_2_MINMAX_MIN_COMPARE_SIGN
+ : AVX10_2_MINMAX_MAX_COMPARE_SIGN;
+ if (elem_bt == T_FLOAT) {
+ evminmaxss(dst, mask, src1, src2, true, imm8);
+ } else {
+ assert(elem_bt == T_DOUBLE, "");
+ evminmaxsd(dst, mask, src1, src2, true, imm8);
+ }
+}
+
// Float/Double signum
void C2_MacroAssembler::signum_fp(int opcode, XMMRegister dst, XMMRegister zero, XMMRegister one) {
assert(opcode == Op_SignumF || opcode == Op_SignumD, "sanity");
@@ -1063,7 +1078,7 @@ void C2_MacroAssembler::signum_fp(int opcode, XMMRegister dst, XMMRegister zero,
// If other floating point comparison instructions used, ZF=1 for equal and unordered cases
if (opcode == Op_SignumF) {
if (VM_Version::supports_avx10_2()) {
- vucomxss(dst, zero);
+ evucomxss(dst, zero);
jcc(Assembler::negative, DONE_LABEL);
} else {
ucomiss(dst, zero);
@@ -1074,7 +1089,7 @@ void C2_MacroAssembler::signum_fp(int opcode, XMMRegister dst, XMMRegister zero,
xorps(dst, ExternalAddress(StubRoutines::x86::vector_float_sign_flip()), noreg);
} else if (opcode == Op_SignumD) {
if (VM_Version::supports_avx10_2()) {
- vucomxsd(dst, zero);
+ evucomxsd(dst, zero);
jcc(Assembler::negative, DONE_LABEL);
} else {
ucomisd(dst, zero);
@@ -1691,12 +1706,8 @@ void C2_MacroAssembler::load_constant_vector(BasicType bt, XMMRegister dst, Inte
}
void C2_MacroAssembler::load_iota_indices(XMMRegister dst, int vlen_in_bytes, BasicType bt) {
- // The iota indices are ordered by type B/S/I/L/F/D, and the offset between two types is 64.
- int offset = exact_log2(type2aelembytes(bt)) << 6;
- if (is_floating_point_type(bt)) {
- offset += 128;
- }
- ExternalAddress addr(StubRoutines::x86::vector_iota_indices() + offset);
+ int entry_idx = vector_iota_entry_index(bt);
+ ExternalAddress addr(StubRoutines::x86::vector_iota_indices(entry_idx));
load_vector(T_BYTE, dst, addr, vlen_in_bytes);
}
@@ -2135,8 +2146,8 @@ void C2_MacroAssembler::mulreduce16B(int opcode, Register dst, Register src1, XM
} else {
pmovsxbw(vtmp2, src2);
reduce8S(opcode, dst, src1, vtmp2, vtmp1, vtmp2);
- pshufd(vtmp2, src2, 0x1);
- pmovsxbw(vtmp2, src2);
+ pshufd(vtmp2, src2, 0xe);
+ pmovsxbw(vtmp2, vtmp2);
reduce8S(opcode, dst, dst, vtmp2, vtmp1, vtmp2);
}
}
@@ -2145,7 +2156,7 @@ void C2_MacroAssembler::mulreduce32B(int opcode, Register dst, Register src1, XM
if (UseAVX > 2 && VM_Version::supports_avx512bw()) {
int vector_len = Assembler::AVX_512bit;
vpmovsxbw(vtmp1, src2, vector_len);
- reduce32S(opcode, dst, src1, vtmp1, vtmp1, vtmp2);
+ reduce32S(opcode, dst, src1, vtmp1, vtmp2, vtmp1);
} else {
assert(UseAVX >= 2,"Should not reach here.");
mulreduce16B(opcode, dst, src1, src2, vtmp1, vtmp2);
@@ -2192,6 +2203,7 @@ void C2_MacroAssembler::reduce8S(int opcode, Register dst, Register src1, XMMReg
}
phaddw(vtmp1, src2);
} else {
+ assert_different_registers(src2, vtmp1);
pshufd(vtmp1, src2, 0xE);
reduce_operation_128(T_SHORT, opcode, vtmp1, src2);
}
@@ -2204,6 +2216,7 @@ void C2_MacroAssembler::reduce16S(int opcode, Register dst, Register src1, XMMRe
vphaddw(vtmp2, src2, src2, vector_len);
vpermq(vtmp2, vtmp2, 0xD8, vector_len);
} else {
+ assert_different_registers(src2, vtmp2);
vextracti128_high(vtmp2, src2);
reduce_operation_128(T_SHORT, opcode, vtmp2, src2);
}
@@ -2211,6 +2224,7 @@ void C2_MacroAssembler::reduce16S(int opcode, Register dst, Register src1, XMMRe
}
void C2_MacroAssembler::reduce32S(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2) {
+ assert_different_registers(src2, vtmp1);
int vector_len = Assembler::AVX_256bit;
vextracti64x4_high(vtmp1, src2);
reduce_operation_256(T_SHORT, opcode, vtmp1, vtmp1, src2);
@@ -2400,7 +2414,7 @@ void C2_MacroAssembler::reduceFloatMinMax(int opcode, int vlen, bool is_dst_vali
}
if (VM_Version::supports_avx10_2()) {
- vminmax_fp(opcode, T_FLOAT, wdst, k0, wtmp, wsrc, vlen_enc);
+ vminmax_fp_avx10_2(opcode, T_FLOAT, wdst, k0, wtmp, wsrc, vlen_enc);
} else {
vminmax_fp(opcode, T_FLOAT, wdst, wtmp, wsrc, tmp, atmp, btmp, vlen_enc);
}
@@ -2409,7 +2423,7 @@ void C2_MacroAssembler::reduceFloatMinMax(int opcode, int vlen, bool is_dst_vali
}
if (is_dst_valid) {
if (VM_Version::supports_avx10_2()) {
- vminmax_fp(opcode, T_FLOAT, dst, k0, wdst, dst, Assembler::AVX_128bit);
+ vminmax_fp_avx10_2(opcode, T_FLOAT, dst, k0, wdst, dst, Assembler::AVX_128bit);
} else {
vminmax_fp(opcode, T_FLOAT, dst, wdst, dst, tmp, atmp, btmp, Assembler::AVX_128bit);
}
@@ -2440,7 +2454,7 @@ void C2_MacroAssembler::reduceDoubleMinMax(int opcode, int vlen, bool is_dst_val
}
if (VM_Version::supports_avx10_2()) {
- vminmax_fp(opcode, T_DOUBLE, wdst, k0, wtmp, wsrc, vlen_enc);
+ vminmax_fp_avx10_2(opcode, T_DOUBLE, wdst, k0, wtmp, wsrc, vlen_enc);
} else {
vminmax_fp(opcode, T_DOUBLE, wdst, wtmp, wsrc, tmp, atmp, btmp, vlen_enc);
}
@@ -2451,7 +2465,7 @@ void C2_MacroAssembler::reduceDoubleMinMax(int opcode, int vlen, bool is_dst_val
if (is_dst_valid) {
if (VM_Version::supports_avx10_2()) {
- vminmax_fp(opcode, T_DOUBLE, dst, k0, wdst, dst, Assembler::AVX_128bit);
+ vminmax_fp_avx10_2(opcode, T_DOUBLE, dst, k0, wdst, dst, Assembler::AVX_128bit);
} else {
vminmax_fp(opcode, T_DOUBLE, dst, wdst, dst, tmp, atmp, btmp, Assembler::AVX_128bit);
}
@@ -7061,13 +7075,25 @@ void C2_MacroAssembler::evfp16ph(int opcode, XMMRegister dst, XMMRegister src1,
}
}
-void C2_MacroAssembler::scalar_max_min_fp16(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
- KRegister ktmp, XMMRegister xtmp1, XMMRegister xtmp2) {
- vector_max_min_fp16(opcode, dst, src1, src2, ktmp, xtmp1, xtmp2, Assembler::AVX_128bit);
+void C2_MacroAssembler::sminmax_fp16(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ KRegister ktmp, XMMRegister xtmp1, XMMRegister xtmp2) {
+ vminmax_fp16(opcode, dst, src1, src2, ktmp, xtmp1, xtmp2, Assembler::AVX_128bit);
}
-void C2_MacroAssembler::vector_max_min_fp16(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
- KRegister ktmp, XMMRegister xtmp1, XMMRegister xtmp2, int vlen_enc) {
+void C2_MacroAssembler::sminmax_fp16_avx10_2(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ KRegister ktmp) {
+ if (opcode == Op_MaxHF) {
+ // dst = max(src1, src2)
+ evminmaxsh(dst, ktmp, src1, src2, true, AVX10_2_MINMAX_MAX_COMPARE_SIGN);
+ } else {
+ assert(opcode == Op_MinHF, "");
+ // dst = min(src1, src2)
+ evminmaxsh(dst, ktmp, src1, src2, true, AVX10_2_MINMAX_MIN_COMPARE_SIGN);
+ }
+}
+
+void C2_MacroAssembler::vminmax_fp16(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ KRegister ktmp, XMMRegister xtmp1, XMMRegister xtmp2, int vlen_enc) {
if (opcode == Op_MaxVHF || opcode == Op_MaxHF) {
// Move sign bits of src2 to mask register.
evpmovw2m(ktmp, src2, vlen_enc);
@@ -7110,3 +7136,48 @@ void C2_MacroAssembler::vector_max_min_fp16(int opcode, XMMRegister dst, XMMRegi
Assembler::evmovdquw(dst, ktmp, xtmp1, true, vlen_enc);
}
}
+
+void C2_MacroAssembler::vminmax_fp16_avx10_2(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ KRegister ktmp, int vlen_enc) {
+ if (opcode == Op_MaxVHF) {
+ // dst = max(src1, src2)
+ evminmaxph(dst, ktmp, src1, src2, true, AVX10_2_MINMAX_MAX_COMPARE_SIGN, vlen_enc);
+ } else {
+ assert(opcode == Op_MinVHF, "");
+ // dst = min(src1, src2)
+ evminmaxph(dst, ktmp, src1, src2, true, AVX10_2_MINMAX_MIN_COMPARE_SIGN, vlen_enc);
+ }
+}
+
+void C2_MacroAssembler::vminmax_fp16_avx10_2(int opcode, XMMRegister dst, XMMRegister src1, Address src2,
+ KRegister ktmp, int vlen_enc) {
+ if (opcode == Op_MaxVHF) {
+ // dst = max(src1, src2)
+ evminmaxph(dst, ktmp, src1, src2, true, AVX10_2_MINMAX_MAX_COMPARE_SIGN, vlen_enc);
+ } else {
+ assert(opcode == Op_MinVHF, "");
+ // dst = min(src1, src2)
+ evminmaxph(dst, ktmp, src1, src2, true, AVX10_2_MINMAX_MIN_COMPARE_SIGN, vlen_enc);
+ }
+}
+
+int C2_MacroAssembler::vector_iota_entry_index(BasicType bt) {
+ // The vector iota entries array is ordered by type B/S/I/L/F/D, and
+ // the offset between two types is 16.
+ switch(bt) {
+ case T_BYTE:
+ return 0;
+ case T_SHORT:
+ return 1;
+ case T_INT:
+ return 2;
+ case T_LONG:
+ return 3;
+ case T_FLOAT:
+ return 4;
+ case T_DOUBLE:
+ return 5;
+ default:
+ ShouldNotReachHere();
+ }
+}
diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp
index 6d8b0ceaebe..9b229ad7221 100644
--- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -67,8 +67,11 @@ public:
XMMRegister tmp, XMMRegister atmp, XMMRegister btmp,
int vlen_enc);
- void vminmax_fp(int opc, BasicType elem_bt, XMMRegister dst, KRegister mask,
- XMMRegister src1, XMMRegister src2, int vlen_enc);
+ void vminmax_fp_avx10_2(int opc, BasicType elem_bt, XMMRegister dst, KRegister mask,
+ XMMRegister src1, XMMRegister src2, int vlen_enc);
+
+ void sminmax_fp_avx10_2(int opc, BasicType elem_bt, XMMRegister dst, KRegister mask,
+ XMMRegister src1, XMMRegister src2);
void vpuminmaxq(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister xtmp1, XMMRegister xtmp2, int vlen_enc);
@@ -576,12 +579,22 @@ public:
void evfp16ph(int opcode, XMMRegister dst, XMMRegister src1, Address src2, int vlen_enc);
- void vector_max_min_fp16(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
- KRegister ktmp, XMMRegister xtmp1, XMMRegister xtmp2, int vlen_enc);
+ void vminmax_fp16(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ KRegister ktmp, XMMRegister xtmp1, XMMRegister xtmp2, int vlen_enc);
- void scalar_max_min_fp16(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
- KRegister ktmp, XMMRegister xtmp1, XMMRegister xtmp2);
+ void vminmax_fp16_avx10_2(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ KRegister ktmp, int vlen_enc);
+
+ void vminmax_fp16_avx10_2(int opcode, XMMRegister dst, XMMRegister src1, Address src2,
+ KRegister ktmp, int vlen_enc);
+
+ void sminmax_fp16(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ KRegister ktmp, XMMRegister xtmp1, XMMRegister xtmp2);
+
+ void sminmax_fp16_avx10_2(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
+ KRegister ktmp);
void reconstruct_frame_pointer(Register rtmp);
+ int vector_iota_entry_index(BasicType bt);
#endif // CPU_X86_C2_MACROASSEMBLER_X86_HPP
diff --git a/src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.cpp
index 0ea769dd488..c05f37a3bea 100644
--- a/src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -130,7 +130,7 @@ __ BIND(L_loop);
__ BIND(L_done);
}
-void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj, Address dst, Register rscratch) {
+void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj, Register rscratch) {
// Does a store check for the oop in register obj. The content of
// register obj is destroyed afterwards.
CardTableBarrierSet* ctbs = CardTableBarrierSet::barrier_set();
@@ -138,6 +138,8 @@ void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register ob
__ shrptr(obj, CardTable::card_shift());
Address card_addr;
+ precond(rscratch != noreg);
+ assert_different_registers(obj, rscratch);
// The calculation for byte_map_base is as follows:
// byte_map_base = _byte_map - (uintptr_t(low_bound) >> card_shift);
@@ -161,7 +163,7 @@ void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register ob
// entry and that entry is not properly handled by the relocation code.
AddressLiteral cardtable((address)byte_map_base, relocInfo::none);
Address index(noreg, obj, Address::times_1);
- card_addr = __ as_Address(ArrayAddress(cardtable, index), rscratch1);
+ card_addr = __ as_Address(ArrayAddress(cardtable, index), rscratch);
}
int dirty = CardTable::dirty_card_val();
@@ -190,10 +192,10 @@ void CardTableBarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorS
if (needs_post_barrier) {
// flatten object address if needed
if (!precise || (dst.index() == noreg && dst.disp() == 0)) {
- store_check(masm, dst.base(), dst, tmp2);
+ store_check(masm, dst.base(), tmp2);
} else {
__ lea(tmp1, dst);
- store_check(masm, tmp1, dst, tmp2);
+ store_check(masm, tmp1, tmp2);
}
}
}
diff --git a/src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.hpp
index 201c11062f2..c38e16d4d5f 100644
--- a/src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -33,7 +33,7 @@ protected:
virtual void gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators,
Register addr, Register count) {}
- void store_check(MacroAssembler* masm, Register obj, Address dst, Register rscratch);
+ void store_check(MacroAssembler* masm, Register obj, Register rscratch);
virtual void gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, Register addr, Register count, Register tmp);
diff --git a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp
index 47a3dad54e7..c20551b5084 100644
--- a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp
@@ -31,6 +31,7 @@
#include "gc/z/zBarrierSetAssembler.hpp"
#include "gc/z/zBarrierSetRuntime.hpp"
#include "gc/z/zThreadLocalData.hpp"
+#include "logging/log.hpp"
#include "memory/resourceArea.hpp"
#include "runtime/jniHandles.hpp"
#include "runtime/sharedRuntime.hpp"
@@ -1391,10 +1392,13 @@ static uint16_t patch_barrier_relocation_value(int format) {
}
}
-void ZBarrierSetAssembler::patch_barrier_relocation(address addr, int format) {
+void ZBarrierSetAssembler::patch_barrier_relocation(address addr, int format, bool log) {
const int offset = patch_barrier_relocation_offset(format);
const uint16_t value = patch_barrier_relocation_value(format);
uint8_t* const patch_addr = (uint8_t*)addr + offset;
+ if (log) {
+ log_trace(aot, codecache, stubs)("patching address " INTPTR_FORMAT " offset %d value 0x%x", p2i(addr), offset, value);
+ }
if (format == ZBarrierRelocationFormatLoadGoodBeforeShl) {
if (VM_Version::supports_apx_f()) {
NativeInstruction* instruction = nativeInstruction_at(addr);
@@ -1426,6 +1430,74 @@ void ZBarrierSetAssembler::patch_barriers() {
#undef __
#define __ masm->
+void ZBarrierSetAssembler::register_reloc_addresses(GrowableArray &entries, int begin, int count) {
+ int formats[] = {
+ ZBarrierRelocationFormatLoadBadAfterTest,
+ ZBarrierRelocationFormatStoreBadAfterTest,
+ ZBarrierRelocationFormatStoreGoodAfterOr,
+ -1
+ };
+ int format_idx = 0;
+ int format = formats[format_idx++];
+ for (int i = begin; i < begin + count; i++) {
+ address addr = entries.at(i);
+ // reloc addresses occur in 3 groups terminated with a nullptr
+ if (addr == nullptr) {
+ assert(format_idx < (int)(sizeof(formats) / sizeof(formats[0])),
+ "too many reloc groups");
+ format = formats[format_idx++];
+ } else {
+ switch(format) {
+ case ZBarrierRelocationFormatLoadBadAfterTest:
+ _load_bad_relocations.append(addr);
+ break;
+ case ZBarrierRelocationFormatStoreBadAfterTest:
+ _store_bad_relocations.append(addr);
+ break;
+ case ZBarrierRelocationFormatStoreGoodAfterOr:
+ _store_good_relocations.append(addr);
+ break;
+ default:
+ ShouldNotReachHere();
+ break;
+ }
+ patch_barrier_relocation(addr, format, true);
+ }
+ }
+ assert(format == -1, "unterminated format list");
+}
+
+void ZBarrierSetAssembler::retrieve_reloc_addresses(address start, address end, GrowableArray &entries) {
+ assert(start != nullptr, "start address must not be null");
+ assert(end != nullptr, "start address must not be null");
+ assert(start < end, "stub range must not be empty");
+ for (int i = 0; i < _load_bad_relocations.length(); i++) {
+ address addr = _load_bad_relocations.at(i);
+ assert(addr != nullptr, "load bad reloc address shoudl not be null!");
+ if (start <= addr && addr < end) {
+ entries.append(addr);
+ }
+ }
+ entries.append(nullptr);
+ for (int i = 0; i < _store_bad_relocations.length(); i++) {
+ address addr = _store_bad_relocations.at(i);
+ assert(addr != nullptr, "store bad reloc address shoudl not be null!");
+ if (start <= addr && addr < end) {
+ entries.append(addr);
+ }
+ }
+ entries.append(nullptr);
+ for (int i = 0; i < _store_good_relocations.length(); i++) {
+ address addr = _store_good_relocations.at(i);
+ assert(addr != nullptr, "store good reloc address shoudl not be null!");
+ if (start <= addr && addr < end) {
+ entries.append(addr);
+ }
+ }
+ entries.append(nullptr);
+}
+
+
void ZBarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) {
// C1 calls verfy_oop in the middle of barriers, before they have been uncolored
// and after being colored. Therefore, we must deal with colored oops as well.
diff --git a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp
index e91e2b9ea20..ce0c4769716 100644
--- a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp
@@ -189,10 +189,14 @@ public:
Label& slow_path,
Label& slow_path_continuation) const;
- void patch_barrier_relocation(address addr, int format);
+ void patch_barrier_relocation(address addr, int format, bool log = false);
void patch_barriers();
+ void register_reloc_addresses(GrowableArray &entries, int begin, int count);
+
+ void retrieve_reloc_addresses(address start, address end, GrowableArray &entries);
+
void check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error);
};
diff --git a/src/hotspot/cpu/x86/interp_masm_x86.cpp b/src/hotspot/cpu/x86/interp_masm_x86.cpp
index b2ea4143ac4..a38971c86fb 100644
--- a/src/hotspot/cpu/x86/interp_masm_x86.cpp
+++ b/src/hotspot/cpu/x86/interp_masm_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1392,28 +1392,15 @@ void InterpreterMacroAssembler::profile_final_call(Register mdp) {
void InterpreterMacroAssembler::profile_virtual_call(Register receiver,
- Register mdp,
- bool receiver_can_be_null) {
+ Register mdp) {
if (ProfileInterpreter) {
Label profile_continue;
// If no method data exists, go to profile_continue.
test_method_data_pointer(mdp, profile_continue);
- Label skip_receiver_profile;
- if (receiver_can_be_null) {
- Label not_null;
- testptr(receiver, receiver);
- jccb(Assembler::notZero, not_null);
- // We are making a call. Increment the count for null receiver.
- increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()));
- jmp(skip_receiver_profile);
- bind(not_null);
- }
-
// Record the receiver type.
profile_receiver_type(receiver, mdp, 0);
- bind(skip_receiver_profile);
// The method data pointer needs to be updated to reflect the new target.
update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size()));
diff --git a/src/hotspot/cpu/x86/interp_masm_x86.hpp b/src/hotspot/cpu/x86/interp_masm_x86.hpp
index 4114028f78e..dfbd7ab64e0 100644
--- a/src/hotspot/cpu/x86/interp_masm_x86.hpp
+++ b/src/hotspot/cpu/x86/interp_masm_x86.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -243,8 +243,7 @@ class InterpreterMacroAssembler: public MacroAssembler {
void profile_not_taken_branch(Register mdp);
void profile_call(Register mdp);
void profile_final_call(Register mdp);
- void profile_virtual_call(Register receiver, Register mdp,
- bool receiver_can_be_null = false);
+ void profile_virtual_call(Register receiver, Register mdp);
void profile_ret(Register return_bci, Register mdp);
void profile_null_seen(Register mdp);
void profile_typecheck(Register mdp, Register klass);
diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp
index 1d77be26bd9..5ab3ca339aa 100644
--- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp
@@ -385,7 +385,8 @@ void MacroAssembler::warn(const char* msg) {
// Windows always allocates space for its register args
subq(rsp, frame::arg_reg_save_area_bytes);
#endif
- lea(c_rarg0, ExternalAddress((address) msg));
+ const char* str = (code_section()->scratch_emit()) ? msg : AOTCodeCache::add_C_string(msg);
+ lea(c_rarg0, ExternalAddress((address) str));
call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning)));
#ifdef _WIN64
@@ -985,12 +986,9 @@ int MacroAssembler::ic_check(int end_alignment) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(temp, receiver);
cmpl(temp, Address(data, CompiledICData::speculated_klass_offset()));
- } else if (UseCompressedClassPointers) {
+ } else {
movl(temp, Address(receiver, oopDesc::klass_offset_in_bytes()));
cmpl(temp, Address(data, CompiledICData::speculated_klass_offset()));
- } else {
- movptr(temp, Address(receiver, oopDesc::klass_offset_in_bytes()));
- cmpptr(temp, Address(data, CompiledICData::speculated_klass_offset()));
}
// if inline cache check fails, then jump to runtime routine
@@ -1961,6 +1959,16 @@ void MacroAssembler::movflt(XMMRegister dst, AddressLiteral src, Register rscrat
}
}
+void MacroAssembler::movhlf(XMMRegister dst, XMMRegister src, Register rscratch) {
+ if (VM_Version::supports_avx10_2()) {
+ evmovw(dst, src);
+ } else {
+ assert(rscratch != noreg, "missing");
+ evmovw(rscratch, src);
+ evmovw(dst, rscratch);
+ }
+}
+
void MacroAssembler::mov64(Register dst, int64_t imm64) {
if (is_uimm32(imm64)) {
movl(dst, checked_cast(imm64));
@@ -2664,14 +2672,14 @@ void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src, Register rscra
}
}
-void MacroAssembler::vucomxsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
+void MacroAssembler::evucomxsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
assert(rscratch != noreg || always_reachable(src), "missing");
if (reachable(src)) {
- Assembler::vucomxsd(dst, as_Address(src));
+ Assembler::evucomxsd(dst, as_Address(src));
} else {
lea(rscratch, src);
- Assembler::vucomxsd(dst, Address(rscratch, 0));
+ Assembler::evucomxsd(dst, Address(rscratch, 0));
}
}
@@ -2686,14 +2694,36 @@ void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src, Register rscra
}
}
-void MacroAssembler::vucomxss(XMMRegister dst, AddressLiteral src, Register rscratch) {
+void MacroAssembler::evucomxss(XMMRegister dst, AddressLiteral src, Register rscratch) {
assert(rscratch != noreg || always_reachable(src), "missing");
if (reachable(src)) {
- Assembler::vucomxss(dst, as_Address(src));
+ Assembler::evucomxss(dst, as_Address(src));
} else {
lea(rscratch, src);
- Assembler::vucomxss(dst, Address(rscratch, 0));
+ Assembler::evucomxss(dst, Address(rscratch, 0));
+ }
+}
+
+void MacroAssembler::evucomish(XMMRegister dst, AddressLiteral src, Register rscratch) {
+ assert(rscratch != noreg || always_reachable(src), "missing");
+
+ if (reachable(src)) {
+ Assembler::evucomish(dst, as_Address(src));
+ } else {
+ lea(rscratch, src);
+ Assembler::evucomish(dst, Address(rscratch, 0));
+ }
+}
+
+void MacroAssembler::evucomxsh(XMMRegister dst, AddressLiteral src, Register rscratch) {
+ assert(rscratch != noreg || always_reachable(src), "missing");
+
+ if (reachable(src)) {
+ Assembler::evucomxsh(dst, as_Address(src));
+ } else {
+ lea(rscratch, src);
+ Assembler::evucomxsh(dst, Address(rscratch, 0));
}
}
@@ -5384,11 +5414,9 @@ void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(dst, src);
decode_klass_not_null(dst, tmp);
- } else if (UseCompressedClassPointers) {
+ } else {
movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
decode_klass_not_null(dst, tmp);
- } else {
- movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
}
}
@@ -5396,12 +5424,8 @@ void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
assert(!UseCompactObjectHeaders, "not with compact headers");
assert_different_registers(src, tmp);
assert_different_registers(dst, tmp);
- if (UseCompressedClassPointers) {
- encode_klass_not_null(src, tmp);
- movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
- } else {
- movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
- }
+ encode_klass_not_null(src, tmp);
+ movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
}
void MacroAssembler::cmp_klass(Register klass, Register obj, Register tmp) {
@@ -5410,10 +5434,8 @@ void MacroAssembler::cmp_klass(Register klass, Register obj, Register tmp) {
assert_different_registers(klass, obj, tmp);
load_narrow_klass_compact(tmp, obj);
cmpl(klass, tmp);
- } else if (UseCompressedClassPointers) {
- cmpl(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
} else {
- cmpptr(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
+ cmpl(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
}
}
@@ -5424,12 +5446,9 @@ void MacroAssembler::cmp_klasses_from_objects(Register obj1, Register obj2, Regi
load_narrow_klass_compact(tmp1, obj1);
load_narrow_klass_compact(tmp2, obj2);
cmpl(tmp1, tmp2);
- } else if (UseCompressedClassPointers) {
+ } else {
movl(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
cmpl(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes()));
- } else {
- movptr(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
- cmpptr(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes()));
}
}
@@ -5478,10 +5497,8 @@ void MacroAssembler::store_heap_oop_null(Address dst) {
void MacroAssembler::store_klass_gap(Register dst, Register src) {
assert(!UseCompactObjectHeaders, "Don't use with compact headers");
- if (UseCompressedClassPointers) {
- // Store to klass gap in destination
- movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src);
- }
+ // Store to klass gap in destination
+ movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src);
}
#ifdef ASSERT
@@ -5656,7 +5673,12 @@ void MacroAssembler::encode_and_move_klass_not_null(Register dst, Register src)
BLOCK_COMMENT("encode_and_move_klass_not_null {");
assert_different_registers(src, dst);
if (CompressedKlassPointers::base() != nullptr) {
- movptr(dst, -(intptr_t)CompressedKlassPointers::base());
+ if (AOTCodeCache::is_on_for_dump()) {
+ movptr(dst, ExternalAddress(CompressedKlassPointers::base_addr()));
+ negq(dst);
+ } else {
+ movptr(dst, -(intptr_t)CompressedKlassPointers::base());
+ }
addq(dst, src);
} else {
movptr(dst, src);
@@ -5671,7 +5693,6 @@ void MacroAssembler::decode_klass_not_null(Register r, Register tmp) {
BLOCK_COMMENT("decode_klass_not_null {");
assert_different_registers(r, tmp);
// Note: it will change flags
- assert(UseCompressedClassPointers, "should only be used for compressed headers");
// Cannot assert, unverified entry point counts instructions (see .ad file)
// vtableStubs also counts instructions in pd_code_size_limit.
// Also do not verify_oop as this is called by verify_oop.
@@ -5693,7 +5714,6 @@ void MacroAssembler::decode_and_move_klass_not_null(Register dst, Register src)
BLOCK_COMMENT("decode_and_move_klass_not_null {");
assert_different_registers(src, dst);
// Note: it will change flags
- assert (UseCompressedClassPointers, "should only be used for compressed headers");
// Cannot assert, unverified entry point counts instructions (see .ad file)
// vtableStubs also counts instructions in pd_code_size_limit.
// Also do not verify_oop as this is called by verify_oop.
@@ -5706,7 +5726,11 @@ void MacroAssembler::decode_and_move_klass_not_null(Register dst, Register src)
} else {
if (CompressedKlassPointers::shift() <= Address::times_8) {
if (CompressedKlassPointers::base() != nullptr) {
- movptr(dst, (intptr_t)CompressedKlassPointers::base());
+ if (AOTCodeCache::is_on_for_dump()) {
+ movptr(dst, ExternalAddress(CompressedKlassPointers::base_addr()));
+ } else {
+ movptr(dst, (intptr_t)CompressedKlassPointers::base());
+ }
} else {
xorq(dst, dst);
}
@@ -5718,9 +5742,14 @@ void MacroAssembler::decode_and_move_klass_not_null(Register dst, Register src)
}
} else {
if (CompressedKlassPointers::base() != nullptr) {
- const intptr_t base_right_shifted =
- (intptr_t)CompressedKlassPointers::base() >> CompressedKlassPointers::shift();
- movptr(dst, base_right_shifted);
+ if (AOTCodeCache::is_on_for_dump()) {
+ movptr(dst, ExternalAddress(CompressedKlassPointers::base_addr()));
+ shrq(dst, CompressedKlassPointers::shift());
+ } else {
+ const intptr_t base_right_shifted =
+ (intptr_t)CompressedKlassPointers::base() >> CompressedKlassPointers::shift();
+ movptr(dst, base_right_shifted);
+ }
} else {
xorq(dst, dst);
}
@@ -5750,7 +5779,6 @@ void MacroAssembler::set_narrow_oop(Address dst, jobject obj) {
}
void MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
- assert (UseCompressedClassPointers, "should only be used for compressed headers");
assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
int klass_index = oop_recorder()->find_index(k);
RelocationHolder rspec = metadata_Relocation::spec(klass_index);
@@ -5758,7 +5786,6 @@ void MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
}
void MacroAssembler::set_narrow_klass(Address dst, Klass* k) {
- assert (UseCompressedClassPointers, "should only be used for compressed headers");
assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
int klass_index = oop_recorder()->find_index(k);
RelocationHolder rspec = metadata_Relocation::spec(klass_index);
@@ -5784,7 +5811,6 @@ void MacroAssembler::cmp_narrow_oop(Address dst, jobject obj) {
}
void MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) {
- assert (UseCompressedClassPointers, "should only be used for compressed headers");
assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
int klass_index = oop_recorder()->find_index(k);
RelocationHolder rspec = metadata_Relocation::spec(klass_index);
@@ -5792,7 +5818,6 @@ void MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) {
}
void MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) {
- assert (UseCompressedClassPointers, "should only be used for compressed headers");
assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
int klass_index = oop_recorder()->find_index(k);
RelocationHolder rspec = metadata_Relocation::spec(klass_index);
@@ -5801,7 +5826,7 @@ void MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) {
void MacroAssembler::reinit_heapbase() {
if (UseCompressedOops) {
- if (Universe::heap() != nullptr) {
+ if (Universe::heap() != nullptr && !AOTCodeCache::is_on_for_dump()) {
if (CompressedOops::base() == nullptr) {
MacroAssembler::xorptr(r12_heapbase, r12_heapbase);
} else {
@@ -9185,7 +9210,7 @@ void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XM
case T_FLOAT:
evminmaxps(dst, mask, nds, src, merge, AVX10_2_MINMAX_MAX_COMPARE_SIGN, vector_len); break;
case T_DOUBLE:
- evminmaxps(dst, mask, nds, src, merge, AVX10_2_MINMAX_MAX_COMPARE_SIGN, vector_len); break;
+ evminmaxpd(dst, mask, nds, src, merge, AVX10_2_MINMAX_MAX_COMPARE_SIGN, vector_len); break;
default:
fatal("Unexpected type argument %s", type2name(type)); break;
}
diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.hpp b/src/hotspot/cpu/x86/macroAssembler_x86.hpp
index 8469deaa8be..021d2943ee8 100644
--- a/src/hotspot/cpu/x86/macroAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/macroAssembler_x86.hpp
@@ -162,6 +162,8 @@ class MacroAssembler: public Assembler {
void incrementq(AddressLiteral dst, Register rscratch = noreg);
+ void movhlf(XMMRegister dst, XMMRegister src, Register rscratch = noreg);
+
// Support optimal SSE move instructions.
void movflt(XMMRegister dst, XMMRegister src) {
if (dst-> encoding() == src->encoding()) return;
@@ -351,8 +353,7 @@ class MacroAssembler: public Assembler {
void load_klass(Register dst, Register src, Register tmp);
void store_klass(Register dst, Register src, Register tmp);
- // Compares the Klass pointer of an object to a given Klass (which might be narrow,
- // depending on UseCompressedClassPointers).
+ // Compares the narrow Klass pointer of an object to a given narrow Klass.
void cmp_klass(Register klass, Register obj, Register tmp);
// Compares the Klass pointer of two objects obj1 and obj2. Result is in the condition flags.
@@ -1309,21 +1310,29 @@ public:
void subss(XMMRegister dst, Address src) { Assembler::subss(dst, src); }
void subss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
+ void evucomish(XMMRegister dst, XMMRegister src) { Assembler::evucomish(dst, src); }
+ void evucomish(XMMRegister dst, Address src) { Assembler::evucomish(dst, src); }
+ void evucomish(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
+
+ void evucomxsh(XMMRegister dst, XMMRegister src) { Assembler::evucomxsh(dst, src); }
+ void evucomxsh(XMMRegister dst, Address src) { Assembler::evucomxsh(dst, src); }
+ void evucomxsh(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
+
void ucomiss(XMMRegister dst, XMMRegister src) { Assembler::ucomiss(dst, src); }
void ucomiss(XMMRegister dst, Address src) { Assembler::ucomiss(dst, src); }
void ucomiss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
- void vucomxss(XMMRegister dst, XMMRegister src) { Assembler::vucomxss(dst, src); }
- void vucomxss(XMMRegister dst, Address src) { Assembler::vucomxss(dst, src); }
- void vucomxss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
+ void evucomxss(XMMRegister dst, XMMRegister src) { Assembler::evucomxss(dst, src); }
+ void evucomxss(XMMRegister dst, Address src) { Assembler::evucomxss(dst, src); }
+ void evucomxss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
void ucomisd(XMMRegister dst, XMMRegister src) { Assembler::ucomisd(dst, src); }
void ucomisd(XMMRegister dst, Address src) { Assembler::ucomisd(dst, src); }
void ucomisd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
- void vucomxsd(XMMRegister dst, XMMRegister src) { Assembler::vucomxsd(dst, src); }
- void vucomxsd(XMMRegister dst, Address src) { Assembler::vucomxsd(dst, src); }
- void vucomxsd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
+ void evucomxsd(XMMRegister dst, XMMRegister src) { Assembler::evucomxsd(dst, src); }
+ void evucomxsd(XMMRegister dst, Address src) { Assembler::evucomxsd(dst, src); }
+ void evucomxsd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
// Bitwise Logical XOR of Packed Double-Precision Floating-Point Values
void xorpd(XMMRegister dst, XMMRegister src);
diff --git a/src/hotspot/cpu/x86/macroAssembler_x86_sha.cpp b/src/hotspot/cpu/x86/macroAssembler_x86_sha.cpp
index 9f0232075cd..401d5dc22cc 100644
--- a/src/hotspot/cpu/x86/macroAssembler_x86_sha.cpp
+++ b/src/hotspot/cpu/x86/macroAssembler_x86_sha.cpp
@@ -242,7 +242,6 @@ void MacroAssembler::fast_sha256(XMMRegister msg, XMMRegister state0, XMMRegiste
Label done_hash, loop0;
address K256 = StubRoutines::x86::k256_addr();
- address pshuffle_byte_flip_mask = StubRoutines::x86::pshuffle_byte_flip_mask_addr();
movdqu(state0, Address(state, 0));
movdqu(state1, Address(state, 16));
@@ -253,7 +252,7 @@ void MacroAssembler::fast_sha256(XMMRegister msg, XMMRegister state0, XMMRegiste
palignr(state0, state1, 8);
pblendw(state1, msgtmp4, 0xF0);
- movdqu(shuf_mask, ExternalAddress(pshuffle_byte_flip_mask));
+ movdqu(shuf_mask, ExternalAddress(StubRoutines::x86::pshuffle_byte_flip_mask_addr()));
lea(rax, ExternalAddress(K256));
bind(loop0);
@@ -661,8 +660,6 @@ void MacroAssembler::sha256_AVX2(XMMRegister msg, XMMRegister state0, XMMRegiste
compute_size1, compute_size_end1;
address K256_W = StubRoutines::x86::k256_W_addr();
- address pshuffle_byte_flip_mask = StubRoutines::x86::pshuffle_byte_flip_mask_addr();
- address pshuffle_byte_flip_mask_addr = nullptr;
const XMMRegister& SHUF_00BA = xmm10; // ymm10: shuffle xBxA -> 00BA
const XMMRegister& SHUF_DC00 = xmm12; // ymm12: shuffle xDxC -> DC00
@@ -791,10 +788,14 @@ enum {
// load g - r10 after it is used as scratch
movl(h, Address(CTX, 4*7));
- pshuffle_byte_flip_mask_addr = pshuffle_byte_flip_mask;
- vmovdqu(BYTE_FLIP_MASK, ExternalAddress(pshuffle_byte_flip_mask_addr + 0)); // [PSHUFFLE_BYTE_FLIP_MASK wrt rip]
- vmovdqu(SHUF_00BA, ExternalAddress(pshuffle_byte_flip_mask_addr + 32)); // [_SHUF_00BA wrt rip]
- vmovdqu(SHUF_DC00, ExternalAddress(pshuffle_byte_flip_mask_addr + 64)); // [_SHUF_DC00 wrt rip]
+ // the three successive pshuffle_byte_flip_mask stub entries should
+ // be offset by 32 bytes
+ assert(StubRoutines::x86::pshuffle_byte_flip_mask_addr() + 32 == StubRoutines::x86::pshuffle_byte_flip_mask_00ba_addr(), "sanity");
+ assert(StubRoutines::x86::pshuffle_byte_flip_mask_addr() + 64 == StubRoutines::x86::pshuffle_byte_flip_mask_dc00_addr(), "sanity");
+
+ vmovdqu(BYTE_FLIP_MASK, ExternalAddress(StubRoutines::x86::pshuffle_byte_flip_mask_addr())); // [PSHUFFLE_BYTE_FLIP_MASK wrt rip]
+ vmovdqu(SHUF_00BA, ExternalAddress(StubRoutines::x86::pshuffle_byte_flip_mask_00ba_addr())); // [_SHUF_00BA wrt rip]
+ vmovdqu(SHUF_DC00, ExternalAddress(StubRoutines::x86::pshuffle_byte_flip_mask_dc00_addr())); // [_SHUF_DC00 wrt rip]
movl(g, Address(CTX, 4*6));
@@ -953,11 +954,9 @@ bind(only_one_block);
// load g - r10 after use as scratch
movl(h, Address(CTX, 4*7)); // 0x5be0cd19
-
- pshuffle_byte_flip_mask_addr = pshuffle_byte_flip_mask;
- vmovdqu(BYTE_FLIP_MASK, ExternalAddress(pshuffle_byte_flip_mask_addr + 0)); // [PSHUFFLE_BYTE_FLIP_MASK wrt rip]
- vmovdqu(SHUF_00BA, ExternalAddress(pshuffle_byte_flip_mask_addr + 32)); // [_SHUF_00BA wrt rip]
- vmovdqu(SHUF_DC00, ExternalAddress(pshuffle_byte_flip_mask_addr + 64)); // [_SHUF_DC00 wrt rip]
+ vmovdqu(BYTE_FLIP_MASK, ExternalAddress(StubRoutines::x86::pshuffle_byte_flip_mask_addr())); // [PSHUFFLE_BYTE_FLIP_MASK wrt rip]
+ vmovdqu(SHUF_00BA, ExternalAddress(StubRoutines::x86::pshuffle_byte_flip_mask_00ba_addr())); // [_SHUF_00BA wrt rip]
+ vmovdqu(SHUF_DC00, ExternalAddress(StubRoutines::x86::pshuffle_byte_flip_mask_dc00_addr())); // [_SHUF_DC00 wrt rip]
movl(g, Address(CTX, 4*6)); // 0x1f83d9ab
@@ -1346,9 +1345,12 @@ void MacroAssembler::sha512_AVX2(XMMRegister msg, XMMRegister state0, XMMRegiste
// load g - r10 after it is used as scratch
movq(h, Address(CTX, 8 * 7));
- pshuffle_byte_flip_mask_addr = pshuffle_byte_flip_mask_sha512;
- vmovdqu(BYTE_FLIP_MASK, ExternalAddress(pshuffle_byte_flip_mask_addr + 0)); // PSHUFFLE_BYTE_FLIP_MASK wrt rip
- vmovdqu(YMM_MASK_LO, ExternalAddress(pshuffle_byte_flip_mask_addr + 32));
+ // the two successive pshuffle_byte_flip_mask_sha512 stub entries should
+ // be offset by 32 bytes
+ assert(StubRoutines::x86::pshuffle_byte_flip_mask_addr_sha512() + 32 == StubRoutines::x86::pshuffle_byte_flip_mask_ymm_lo_addr_sha512(), "sanity");
+
+ vmovdqu(BYTE_FLIP_MASK, ExternalAddress(StubRoutines::x86::pshuffle_byte_flip_mask_addr_sha512())); // PSHUFFLE_BYTE_FLIP_MASK wrt rip
+ vmovdqu(YMM_MASK_LO, ExternalAddress(StubRoutines::x86::pshuffle_byte_flip_mask_ymm_lo_addr_sha512())); // MASK_YMM_LO wrt rip
movq(g, Address(CTX, 8 * 6));
diff --git a/src/hotspot/cpu/x86/matcher_x86.hpp b/src/hotspot/cpu/x86/matcher_x86.hpp
index f7973a8564e..62a5d2827bc 100644
--- a/src/hotspot/cpu/x86/matcher_x86.hpp
+++ b/src/hotspot/cpu/x86/matcher_x86.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -75,7 +75,6 @@
}
static bool narrow_klass_use_complex_address() {
- assert(UseCompressedClassPointers, "only for compressed klass code");
return (CompressedKlassPointers::shift() <= 3);
}
diff --git a/src/hotspot/cpu/x86/methodHandles_x86.cpp b/src/hotspot/cpu/x86/methodHandles_x86.cpp
index 54376c6ad9a..5b15444bc32 100644
--- a/src/hotspot/cpu/x86/methodHandles_x86.cpp
+++ b/src/hotspot/cpu/x86/methodHandles_x86.cpp
@@ -110,14 +110,13 @@ void MethodHandles::verify_ref_kind(MacroAssembler* _masm, int ref_kind, Registe
__ andl(temp, java_lang_invoke_MemberName::MN_REFERENCE_KIND_MASK);
__ cmpl(temp, ref_kind);
__ jcc(Assembler::equal, L);
- { char* buf = NEW_C_HEAP_ARRAY(char, 100, mtInternal);
- jio_snprintf(buf, 100, "verify_ref_kind expected %x", ref_kind);
- if (ref_kind == JVM_REF_invokeVirtual ||
- ref_kind == JVM_REF_invokeSpecial)
- // could do this for all ref_kinds, but would explode assembly code size
- trace_method_handle(_masm, buf);
- __ STOP(buf);
+ const char* msg = ref_kind_to_verify_msg(ref_kind);
+ if (ref_kind == JVM_REF_invokeVirtual ||
+ ref_kind == JVM_REF_invokeSpecial) {
+ // could do this for all ref_kinds, but would explode assembly code size
+ trace_method_handle(_masm, msg);
}
+ __ STOP(msg);
BLOCK_COMMENT("} verify_ref_kind");
__ bind(L);
}
diff --git a/src/hotspot/cpu/x86/stubDeclarations_x86.hpp b/src/hotspot/cpu/x86/stubDeclarations_x86.hpp
index 971c8fd3c44..24886deb3c5 100644
--- a/src/hotspot/cpu/x86/stubDeclarations_x86.hpp
+++ b/src/hotspot/cpu/x86/stubDeclarations_x86.hpp
@@ -29,14 +29,16 @@
#define STUBGEN_PREUNIVERSE_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(preuniverse, 500) \
#define STUBGEN_INITIAL_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(initial, PRODUCT_ONLY(20000) NOT_PRODUCT(21000) WINDOWS_ONLY(+1000)) \
do_stub(initial, verify_mxcsr) \
do_arch_entry(x86, initial, verify_mxcsr, verify_mxcsr_entry, \
@@ -65,14 +67,18 @@
#define STUBGEN_CONTINUATION_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(continuation, 3000) \
+// count needed for declaration of vector_iota_indices stub
+#define VECTOR_IOTA_COUNT 6
#define STUBGEN_COMPILER_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(compiler, 120000 WINDOWS_ONLY(+2000)) \
do_stub(compiler, vector_float_sign_mask) \
do_arch_entry(x86, compiler, vector_float_sign_mask, \
@@ -126,8 +132,9 @@
do_arch_entry(x86, compiler, vector_long_sign_mask, \
vector_long_sign_mask, vector_long_sign_mask) \
do_stub(compiler, vector_iota_indices) \
- do_arch_entry(x86, compiler, vector_iota_indices, \
- vector_iota_indices, vector_iota_indices) \
+ do_arch_entry_array(x86, compiler, vector_iota_indices, \
+ vector_iota_indices, vector_iota_indices, \
+ VECTOR_IOTA_COUNT) \
do_stub(compiler, vector_count_leading_zeros_lut) \
do_arch_entry(x86, compiler, vector_count_leading_zeros_lut, \
vector_count_leading_zeros_lut, \
@@ -161,6 +168,12 @@
do_arch_entry(x86, compiler, pshuffle_byte_flip_mask, \
pshuffle_byte_flip_mask_addr, \
pshuffle_byte_flip_mask_addr) \
+ do_arch_entry(x86, compiler, pshuffle_byte_flip_mask, \
+ pshuffle_byte_flip_mask_00ba_addr, \
+ pshuffle_byte_flip_mask_00ba_addr) \
+ do_arch_entry(x86, compiler, pshuffle_byte_flip_mask, \
+ pshuffle_byte_flip_mask_dc00_addr, \
+ pshuffle_byte_flip_mask_dc00_addr) \
/* x86_64 exposes these 3 stubs via a generic entry array */ \
/* other arches use arch-specific entries */ \
/* this really needs rationalising */ \
@@ -171,6 +184,9 @@
do_arch_entry(x86, compiler, pshuffle_byte_flip_mask_sha512, \
pshuffle_byte_flip_mask_addr_sha512, \
pshuffle_byte_flip_mask_addr_sha512) \
+ do_arch_entry(x86, compiler, pshuffle_byte_flip_mask_sha512, \
+ pshuffle_byte_flip_mask_ymm_lo_addr_sha512, \
+ pshuffle_byte_flip_mask_ymm_lo_addr_sha512) \
do_stub(compiler, compress_perm_table32) \
do_arch_entry(x86, compiler, compress_perm_table32, \
compress_perm_table32, compress_perm_table32) \
@@ -241,7 +257,8 @@
#define STUBGEN_FINAL_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(final, 33000 \
WINDOWS_ONLY(+22000) ZGC_ONLY(+20000)) \
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp
index efb0411aa39..993d1964034 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp
@@ -188,8 +188,18 @@ address StubGenerator::generate_call_stub(address& return_address) {
(int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off,
"adjust this code");
StubId stub_id = StubId::stubgen_call_stub_id;
+ GrowableArray entries;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 2, "sanity check");
+ address start = load_archive_data(stub_id, &entries);
+ if (start != nullptr) {
+ assert(entries.length() == 1, "expected 1 extra entry");
+ return_address = entries.at(0);
+ return start;
+ }
+
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
// same as in generate_catch_exception()!
const Address rsp_after_call(rbp, rsp_after_call_off * wordSize);
@@ -298,6 +308,7 @@ address StubGenerator::generate_call_stub(address& return_address) {
BLOCK_COMMENT("call_stub_return_address:");
return_address = __ pc();
+ entries.append(return_address);
// store result depending on type (everything that is not
// T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
@@ -394,6 +405,9 @@ address StubGenerator::generate_call_stub(address& return_address) {
__ movdbl(Address(c_rarg0, 0), xmm0);
__ jmp(exit);
+ // record the stub entry and end plus the auxiliary entry
+ store_archive_data(stub_id, start, __ pc(), &entries);
+
return start;
}
@@ -411,8 +425,15 @@ address StubGenerator::generate_call_stub(address& return_address) {
address StubGenerator::generate_catch_exception() {
StubId stub_id = StubId::stubgen_catch_exception_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
// same as in generate_call_stub():
const Address rsp_after_call(rbp, rsp_after_call_off * wordSize);
@@ -442,7 +463,9 @@ address StubGenerator::generate_catch_exception() {
__ verify_oop(rax);
__ movptr(Address(r15_thread, Thread::pending_exception_offset()), rax);
- __ lea(rscratch1, ExternalAddress((address)__FILE__));
+ // special case -- add file name string to AOT address table
+ address file = (address)AOTCodeCache::add_C_string(__FILE__);
+ __ lea(rscratch1, ExternalAddress(file));
__ movptr(Address(r15_thread, Thread::exception_file_offset()), rscratch1);
__ movl(Address(r15_thread, Thread::exception_line_offset()), (int) __LINE__);
@@ -451,6 +474,9 @@ address StubGenerator::generate_catch_exception() {
"_call_stub_return_address must have been generated before");
__ jump(RuntimeAddress(StubRoutines::_call_stub_return_address));
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -467,8 +493,14 @@ address StubGenerator::generate_catch_exception() {
address StubGenerator::generate_forward_exception() {
StubId stub_id = StubId::stubgen_forward_exception_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
// Upon entry, the sp points to the return address returning into
// Java (interpreted or compiled) code; i.e., the return address
@@ -521,6 +553,9 @@ address StubGenerator::generate_forward_exception() {
__ verify_oop(rax);
__ jmp(rbx);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -531,12 +566,21 @@ address StubGenerator::generate_forward_exception() {
// Result:
address StubGenerator::generate_orderaccess_fence() {
StubId stub_id = StubId::stubgen_fence_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ membar(Assembler::StoreLoad);
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -550,8 +594,14 @@ address StubGenerator::generate_orderaccess_fence() {
address StubGenerator::generate_verify_mxcsr() {
StubId stub_id = StubId::stubgen_verify_mxcsr_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Address mxcsr_save(rsp, 0);
@@ -574,15 +624,24 @@ address StubGenerator::generate_verify_mxcsr() {
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_f2i_fixup() {
StubId stub_id = StubId::stubgen_f2i_fixup_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
Address inout(rsp, 5 * wordSize); // return address + 4 saves
- address start = __ pc();
+ start = __ pc();
Label L;
@@ -613,14 +672,23 @@ address StubGenerator::generate_f2i_fixup() {
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_f2l_fixup() {
StubId stub_id = StubId::stubgen_f2l_fixup_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
Address inout(rsp, 5 * wordSize); // return address + 4 saves
- address start = __ pc();
+ start = __ pc();
Label L;
@@ -651,15 +719,24 @@ address StubGenerator::generate_f2l_fixup() {
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_d2i_fixup() {
StubId stub_id = StubId::stubgen_d2i_fixup_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
Address inout(rsp, 6 * wordSize); // return address + 5 saves
- address start = __ pc();
+ start = __ pc();
Label L;
@@ -699,15 +776,24 @@ address StubGenerator::generate_d2i_fixup() {
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_d2l_fixup() {
StubId stub_id = StubId::stubgen_d2l_fixup_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
Address inout(rsp, 6 * wordSize); // return address + 5 saves
- address start = __ pc();
+ start = __ pc();
Label L;
@@ -747,14 +833,23 @@ address StubGenerator::generate_d2l_fixup() {
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_count_leading_zeros_lut() {
- __ align64();
StubId stub_id = StubId::stubgen_vector_count_leading_zeros_lut_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align64();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ emit_data64(0x0101010102020304, relocInfo::none);
__ emit_data64(0x0000000000000000, relocInfo::none);
@@ -765,14 +860,23 @@ address StubGenerator::generate_count_leading_zeros_lut() {
__ emit_data64(0x0101010102020304, relocInfo::none);
__ emit_data64(0x0000000000000000, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_popcount_avx_lut() {
- __ align64();
StubId stub_id = StubId::stubgen_vector_popcount_lut_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align64();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ emit_data64(0x0302020102010100, relocInfo::none);
__ emit_data64(0x0403030203020201, relocInfo::none);
@@ -783,14 +887,30 @@ address StubGenerator::generate_popcount_avx_lut() {
__ emit_data64(0x0302020102010100, relocInfo::none);
__ emit_data64(0x0403030203020201, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
-address StubGenerator::generate_iota_indices() {
- __ align(CodeEntryAlignment);
+void StubGenerator::generate_iota_indices() {
StubId stub_id = StubId::stubgen_vector_iota_indices_id;
+ GrowableArray entries;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == VECTOR_IOTA_COUNT, "sanity check");
+ address start = load_archive_data(stub_id, &entries);
+ if (start != nullptr) {
+ assert(entries.length() == VECTOR_IOTA_COUNT - 1,
+ "unexpected extra entry count %d", entries.length());
+ StubRoutines::x86::_vector_iota_indices[0] = start;
+ for (int i = 1; i < VECTOR_IOTA_COUNT; i++) {
+ StubRoutines::x86::_vector_iota_indices[i] = entries.at(i - 1);
+ }
+ return;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
// B
__ emit_data64(0x0706050403020100, relocInfo::none);
__ emit_data64(0x0F0E0D0C0B0A0908, relocInfo::none);
@@ -800,6 +920,7 @@ address StubGenerator::generate_iota_indices() {
__ emit_data64(0x2F2E2D2C2B2A2928, relocInfo::none);
__ emit_data64(0x3736353433323130, relocInfo::none);
__ emit_data64(0x3F3E3D3C3B3A3938, relocInfo::none);
+ entries.append(__ pc());
// W
__ emit_data64(0x0003000200010000, relocInfo::none);
__ emit_data64(0x0007000600050004, relocInfo::none);
@@ -809,6 +930,7 @@ address StubGenerator::generate_iota_indices() {
__ emit_data64(0x0017001600150014, relocInfo::none);
__ emit_data64(0x001B001A00190018, relocInfo::none);
__ emit_data64(0x001F001E001D001C, relocInfo::none);
+ entries.append(__ pc());
// D
__ emit_data64(0x0000000100000000, relocInfo::none);
__ emit_data64(0x0000000300000002, relocInfo::none);
@@ -818,6 +940,7 @@ address StubGenerator::generate_iota_indices() {
__ emit_data64(0x0000000B0000000A, relocInfo::none);
__ emit_data64(0x0000000D0000000C, relocInfo::none);
__ emit_data64(0x0000000F0000000E, relocInfo::none);
+ entries.append(__ pc());
// Q
__ emit_data64(0x0000000000000000, relocInfo::none);
__ emit_data64(0x0000000000000001, relocInfo::none);
@@ -827,6 +950,7 @@ address StubGenerator::generate_iota_indices() {
__ emit_data64(0x0000000000000005, relocInfo::none);
__ emit_data64(0x0000000000000006, relocInfo::none);
__ emit_data64(0x0000000000000007, relocInfo::none);
+ entries.append(__ pc());
// D - FP
__ emit_data64(0x3F80000000000000, relocInfo::none); // 0.0f, 1.0f
__ emit_data64(0x4040000040000000, relocInfo::none); // 2.0f, 3.0f
@@ -836,6 +960,7 @@ address StubGenerator::generate_iota_indices() {
__ emit_data64(0x4130000041200000, relocInfo::none); // 10.0f, 11.0f
__ emit_data64(0x4150000041400000, relocInfo::none); // 12.0f, 13.0f
__ emit_data64(0x4170000041600000, relocInfo::none); // 14.0f, 15.0f
+ entries.append(__ pc());
// Q - FP
__ emit_data64(0x0000000000000000, relocInfo::none); // 0.0d
__ emit_data64(0x3FF0000000000000, relocInfo::none); // 1.0d
@@ -845,14 +970,30 @@ address StubGenerator::generate_iota_indices() {
__ emit_data64(0x4014000000000000, relocInfo::none); // 5.0d
__ emit_data64(0x4018000000000000, relocInfo::none); // 6.0d
__ emit_data64(0x401c000000000000, relocInfo::none); // 7.0d
- return start;
+
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc(), &entries);
+
+ // install the entry addresses in the entry array
+ assert(entries.length() == entry_count - 1,
+ "unexpected entries count %d", entries.length());
+ StubRoutines::x86::_vector_iota_indices[0] = start;
+ for (int i = 1; i < VECTOR_IOTA_COUNT; i++) {
+ StubRoutines::x86::_vector_iota_indices[i] = entries.at(i - 1);
+ }
}
address StubGenerator::generate_vector_reverse_bit_lut() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_vector_reverse_bit_lut_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ emit_data64(0x0E060A020C040800, relocInfo::none);
__ emit_data64(0x0F070B030D050901, relocInfo::none);
@@ -863,14 +1004,23 @@ address StubGenerator::generate_vector_reverse_bit_lut() {
__ emit_data64(0x0E060A020C040800, relocInfo::none);
__ emit_data64(0x0F070B030D050901, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_vector_reverse_byte_perm_mask_long() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_vector_reverse_byte_perm_mask_long_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ emit_data64(0x0001020304050607, relocInfo::none);
__ emit_data64(0x08090A0B0C0D0E0F, relocInfo::none);
@@ -881,14 +1031,23 @@ address StubGenerator::generate_vector_reverse_byte_perm_mask_long() {
__ emit_data64(0x0001020304050607, relocInfo::none);
__ emit_data64(0x08090A0B0C0D0E0F, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_vector_reverse_byte_perm_mask_int() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_vector_reverse_byte_perm_mask_int_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ emit_data64(0x0405060700010203, relocInfo::none);
__ emit_data64(0x0C0D0E0F08090A0B, relocInfo::none);
@@ -899,14 +1058,23 @@ address StubGenerator::generate_vector_reverse_byte_perm_mask_int() {
__ emit_data64(0x0405060700010203, relocInfo::none);
__ emit_data64(0x0C0D0E0F08090A0B, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_vector_reverse_byte_perm_mask_short() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_vector_reverse_byte_perm_mask_short_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ emit_data64(0x0607040502030001, relocInfo::none);
__ emit_data64(0x0E0F0C0D0A0B0809, relocInfo::none);
@@ -917,31 +1085,52 @@ address StubGenerator::generate_vector_reverse_byte_perm_mask_short() {
__ emit_data64(0x0607040502030001, relocInfo::none);
__ emit_data64(0x0E0F0C0D0A0B0809, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_vector_byte_shuffle_mask() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_vector_byte_shuffle_mask_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ emit_data64(0x7070707070707070, relocInfo::none);
__ emit_data64(0x7070707070707070, relocInfo::none);
__ emit_data64(0xF0F0F0F0F0F0F0F0, relocInfo::none);
__ emit_data64(0xF0F0F0F0F0F0F0F0, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_fp_mask(StubId stub_id, int64_t mask) {
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ emit_data64( mask, relocInfo::none );
__ emit_data64( mask, relocInfo::none );
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -957,9 +1146,15 @@ address StubGenerator::generate_compress_perm_table(StubId stub_id) {
default:
ShouldNotReachHere();
}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
if (esize == 32) {
// Loop to generate 256 x 8 int compression permute index table. A row is
// accessed using 8 bit index computed using vector mask. An entry in
@@ -997,6 +1192,9 @@ address StubGenerator::generate_compress_perm_table(StubId stub_id) {
}
}
}
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -1012,9 +1210,15 @@ address StubGenerator::generate_expand_perm_table(StubId stub_id) {
default:
ShouldNotReachHere();
}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
if (esize == 32) {
// Loop to generate 256 x 8 int expand permute index table. A row is accessed
// using 8 bit index computed using vector mask. An entry in a row holds either
@@ -1050,13 +1254,22 @@ address StubGenerator::generate_expand_perm_table(StubId stub_id) {
}
}
}
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_vector_mask(StubId stub_id, int64_t mask) {
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ emit_data64(mask, relocInfo::none);
__ emit_data64(mask, relocInfo::none);
@@ -1067,14 +1280,23 @@ address StubGenerator::generate_vector_mask(StubId stub_id, int64_t mask) {
__ emit_data64(mask, relocInfo::none);
__ emit_data64(mask, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_vector_byte_perm_mask() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_vector_byte_perm_mask_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ emit_data64(0x0000000000000001, relocInfo::none);
__ emit_data64(0x0000000000000003, relocInfo::none);
@@ -1085,13 +1307,22 @@ address StubGenerator::generate_vector_byte_perm_mask() {
__ emit_data64(0x0000000000000004, relocInfo::none);
__ emit_data64(0x0000000000000006, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_vector_fp_mask(StubId stub_id, int64_t mask) {
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ emit_data64(mask, relocInfo::none);
__ emit_data64(mask, relocInfo::none);
@@ -1102,6 +1333,9 @@ address StubGenerator::generate_vector_fp_mask(StubId stub_id, int64_t mask) {
__ emit_data64(mask, relocInfo::none);
__ emit_data64(mask, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -1110,9 +1344,15 @@ address StubGenerator::generate_vector_custom_i32(StubId stub_id, Assembler::Avx
int32_t val4, int32_t val5, int32_t val6, int32_t val7,
int32_t val8, int32_t val9, int32_t val10, int32_t val11,
int32_t val12, int32_t val13, int32_t val14, int32_t val15) {
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
assert(len != Assembler::AVX_NoVec, "vector len must be specified");
__ emit_data(val0, relocInfo::none, 0);
@@ -1135,6 +1375,9 @@ address StubGenerator::generate_vector_custom_i32(StubId stub_id, Assembler::Avx
__ emit_data(val15, relocInfo::none, 0);
}
}
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -1156,8 +1399,14 @@ address StubGenerator::generate_vector_custom_i32(StubId stub_id, Assembler::Avx
// * = popped on exit
address StubGenerator::generate_verify_oop() {
StubId stub_id = StubId::stubgen_verify_oop_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label exit, error;
@@ -1235,6 +1484,9 @@ address StubGenerator::generate_verify_oop() {
__ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)));
__ hlt();
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -1350,35 +1602,46 @@ void StubGenerator::restore_argument_regs(BasicType type) {
address StubGenerator::generate_data_cache_writeback() {
const Register src = c_rarg0; // source address
-
- __ align(CodeEntryAlignment);
-
StubId stub_id = StubId::stubgen_data_cache_writeback_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
__ cache_wb(Address(src, 0));
__ leave();
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_data_cache_writeback_sync() {
const Register is_pre = c_rarg0; // pre or post sync
-
- __ align(CodeEntryAlignment);
-
StubId stub_id = StubId::stubgen_data_cache_writeback_sync_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
// pre wbsync is a no-op
// post wbsync translates to an sfence
Label skip;
- address start = __ pc();
+ start = __ pc();
__ enter();
__ cmpl(is_pre, 0);
@@ -1388,6 +1651,9 @@ address StubGenerator::generate_data_cache_writeback_sync() {
__ leave();
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -1405,9 +1671,15 @@ address StubGenerator::generate_md5_implCompress(StubId stub_id) {
default:
ShouldNotReachHere();
}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register buf_param = r15;
const Address state_param(rsp, 0 * wordSize);
@@ -1437,30 +1709,51 @@ address StubGenerator::generate_md5_implCompress(StubId stub_id) {
__ leave();
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_upper_word_mask() {
- __ align64();
StubId stub_id = StubId::stubgen_upper_word_mask_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align64();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ emit_data64(0x0000000000000000, relocInfo::none);
__ emit_data64(0xFFFFFFFF00000000, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_shuffle_byte_flip_mask() {
- __ align64();
StubId stub_id = StubId::stubgen_shuffle_byte_flip_mask_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align64();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ emit_data64(0x08090a0b0c0d0e0f, relocInfo::none);
__ emit_data64(0x0001020304050607, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -1478,9 +1771,15 @@ address StubGenerator::generate_sha1_implCompress(StubId stub_id) {
default:
ShouldNotReachHere();
}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Register buf = c_rarg0;
Register state = c_rarg1;
@@ -1509,15 +1808,32 @@ address StubGenerator::generate_sha1_implCompress(StubId stub_id) {
__ leave();
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
-address StubGenerator::generate_pshuffle_byte_flip_mask() {
- __ align64();
+address StubGenerator::generate_pshuffle_byte_flip_mask(address& entry_00ba, address& entry_dc00) {
StubId stub_id = StubId::stubgen_pshuffle_byte_flip_mask_id;
+ GrowableArray entries;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 3, "sanity check");
+ address start = load_archive_data(stub_id, &entries);
+ if (start != nullptr) {
+ assert(entries.length() == entry_count - 1,
+ "unexpected extra entry count %d", entries.length());
+ entry_00ba = entries.at(0);
+ entry_dc00 = entries.at(1);
+ assert(VM_Version::supports_avx2() == (entry_00ba != nullptr && entry_dc00 != nullptr),
+ "entries cannot be null when avx2 is enabled");
+ return start;
+ }
+ __ align64();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
-
+ start = __ pc();
+ address entry2 = nullptr;
+ address entry3 = nullptr;
__ emit_data64(0x0405060700010203, relocInfo::none);
__ emit_data64(0x0c0d0e0f08090a0b, relocInfo::none);
@@ -1525,37 +1841,66 @@ address StubGenerator::generate_pshuffle_byte_flip_mask() {
__ emit_data64(0x0405060700010203, relocInfo::none); // second copy
__ emit_data64(0x0c0d0e0f08090a0b, relocInfo::none);
// _SHUF_00BA
+ entry2 = __ pc();
__ emit_data64(0x0b0a090803020100, relocInfo::none);
__ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
__ emit_data64(0x0b0a090803020100, relocInfo::none);
__ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
// _SHUF_DC00
+ entry3 = __ pc();
__ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
__ emit_data64(0x0b0a090803020100, relocInfo::none);
__ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
__ emit_data64(0x0b0a090803020100, relocInfo::none);
}
+ // have to track the 2nd and 3rd entries even if they are null
+ entry_00ba = entry2;
+ entries.push(entry_00ba);
+ entry_dc00 = entry3;
+ entries.push(entry_dc00);
+
+ // record the stub entry and end plus all the auxiliary entries
+ store_archive_data(stub_id, start, __ pc(), &entries);
return start;
}
//Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb.
-address StubGenerator::generate_pshuffle_byte_flip_mask_sha512() {
- __ align32();
+address StubGenerator::generate_pshuffle_byte_flip_mask_sha512(address& entry_ymm_lo) {
StubId stub_id = StubId::stubgen_pshuffle_byte_flip_mask_sha512_id;
+ GrowableArray entries;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 2, "sanity check");
+ address start = load_archive_data(stub_id, &entries);
+ if (start != nullptr) {
+ assert(entries.length() == entry_count - 1,
+ "unexpected extra entry count %d", entries.length());
+ entry_ymm_lo = entries.at(0);
+ assert(VM_Version::supports_avx2() == (entry_ymm_lo != nullptr),
+ "entry cannot be null when avx2 is enabled");
+ return start;
+ }
+ __ align32();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
-
+ start = __ pc();
+ address entry2 = nullptr;
if (VM_Version::supports_avx2()) {
__ emit_data64(0x0001020304050607, relocInfo::none); // PSHUFFLE_BYTE_FLIP_MASK
__ emit_data64(0x08090a0b0c0d0e0f, relocInfo::none);
__ emit_data64(0x1011121314151617, relocInfo::none);
__ emit_data64(0x18191a1b1c1d1e1f, relocInfo::none);
+ // capture 2nd entry
+ entry2 = __ pc();
__ emit_data64(0x0000000000000000, relocInfo::none); //MASK_YMM_LO
__ emit_data64(0x0000000000000000, relocInfo::none);
__ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
__ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
}
+ // have to track the 2nd entry even if it is null
+ entry_ymm_lo = entry2;
+ entries.push(entry2);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc(), &entries);
return start;
}
@@ -1575,9 +1920,15 @@ address StubGenerator::generate_sha256_implCompress(StubId stub_id) {
ShouldNotReachHere();
}
assert(VM_Version::supports_sha() || VM_Version::supports_avx2(), "");
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Register buf = c_rarg0;
Register state = c_rarg1;
@@ -1612,6 +1963,9 @@ address StubGenerator::generate_sha256_implCompress(StubId stub_id) {
__ leave();
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -1629,9 +1983,15 @@ address StubGenerator::generate_sha512_implCompress(StubId stub_id) {
}
assert(VM_Version::supports_avx2(), "");
assert(VM_Version::supports_bmi2() || VM_Version::supports_sha512(), "");
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Register buf = c_rarg0;
Register state = c_rarg1;
@@ -1660,14 +2020,23 @@ address StubGenerator::generate_sha512_implCompress(StubId stub_id) {
__ leave();
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::base64_shuffle_addr() {
- __ align64();
StubId stub_id = StubId::stubgen_shuffle_base64_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align64();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
assert(((unsigned long long)start & 0x3f) == 0,
"Alignment problem (0x%08llx)", (unsigned long long)start);
@@ -1680,42 +2049,69 @@ address StubGenerator::base64_shuffle_addr() {
__ emit_data64(0x2829272825262425, relocInfo::none);
__ emit_data64(0x2e2f2d2e2b2c2a2b, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::base64_avx2_shuffle_addr() {
- __ align32();
StubId stub_id = StubId::stubgen_avx2_shuffle_base64_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align32();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ emit_data64(0x0809070805060405, relocInfo::none);
__ emit_data64(0x0e0f0d0e0b0c0a0b, relocInfo::none);
__ emit_data64(0x0405030401020001, relocInfo::none);
__ emit_data64(0x0a0b090a07080607, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::base64_avx2_input_mask_addr() {
- __ align32();
StubId stub_id = StubId::stubgen_avx2_input_mask_base64_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align32();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ emit_data64(0x8000000000000000, relocInfo::none);
__ emit_data64(0x8000000080000000, relocInfo::none);
__ emit_data64(0x8000000080000000, relocInfo::none);
__ emit_data64(0x8000000080000000, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::base64_avx2_lut_addr() {
- __ align32();
StubId stub_id = StubId::stubgen_avx2_lut_base64_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align32();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ emit_data64(0xfcfcfcfcfcfc4741, relocInfo::none);
__ emit_data64(0x0000f0edfcfcfcfc, relocInfo::none);
@@ -1728,14 +2124,23 @@ address StubGenerator::base64_avx2_lut_addr() {
__ emit_data64(0xfcfcfcfcfcfc4741, relocInfo::none);
__ emit_data64(0x000020effcfcfcfc, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::base64_encoding_table_addr() {
- __ align64();
StubId stub_id = StubId::stubgen_encoding_table_base64_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align64();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
assert(((unsigned long long)start & 0x3f) == 0, "Alignment problem (0x%08llx)", (unsigned long long)start);
__ emit_data64(0x4847464544434241, relocInfo::none);
@@ -1757,6 +2162,9 @@ address StubGenerator::base64_encoding_table_addr() {
__ emit_data64(0x333231307a797877, relocInfo::none);
__ emit_data64(0x5f2d393837363534, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -1766,10 +2174,16 @@ address StubGenerator::base64_encoding_table_addr() {
// boolean isURL) {
address StubGenerator::generate_base64_encodeBlock()
{
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_base64_encodeBlock_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
@@ -2144,15 +2558,24 @@ address StubGenerator::generate_base64_encodeBlock()
__ leave();
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
// base64 AVX512vbmi tables
address StubGenerator::base64_vbmi_lookup_lo_addr() {
- __ align64();
StubId stub_id = StubId::stubgen_lookup_lo_base64_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align64();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
assert(((unsigned long long)start & 0x3f) == 0,
"Alignment problem (0x%08llx)", (unsigned long long)start);
@@ -2165,14 +2588,23 @@ address StubGenerator::base64_vbmi_lookup_lo_addr() {
__ emit_data64(0x3b3a393837363534, relocInfo::none);
__ emit_data64(0x8080808080803d3c, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::base64_vbmi_lookup_hi_addr() {
- __ align64();
StubId stub_id = StubId::stubgen_lookup_hi_base64_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align64();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
assert(((unsigned long long)start & 0x3f) == 0,
"Alignment problem (0x%08llx)", (unsigned long long)start);
@@ -2185,13 +2617,22 @@ address StubGenerator::base64_vbmi_lookup_hi_addr() {
__ emit_data64(0x302f2e2d2c2b2a29, relocInfo::none);
__ emit_data64(0x8080808080333231, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::base64_vbmi_lookup_lo_url_addr() {
- __ align64();
StubId stub_id = StubId::stubgen_lookup_lo_base64url_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align64();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
assert(((unsigned long long)start & 0x3f) == 0,
"Alignment problem (0x%08llx)", (unsigned long long)start);
@@ -2204,14 +2645,23 @@ address StubGenerator::base64_vbmi_lookup_lo_url_addr() {
__ emit_data64(0x3b3a393837363534, relocInfo::none);
__ emit_data64(0x8080808080803d3c, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::base64_vbmi_lookup_hi_url_addr() {
- __ align64();
StubId stub_id = StubId::stubgen_lookup_hi_base64url_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align64();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
assert(((unsigned long long)start & 0x3f) == 0,
"Alignment problem (0x%08llx)", (unsigned long long)start);
@@ -2224,14 +2674,23 @@ address StubGenerator::base64_vbmi_lookup_hi_url_addr() {
__ emit_data64(0x302f2e2d2c2b2a29, relocInfo::none);
__ emit_data64(0x8080808080333231, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::base64_vbmi_pack_vec_addr() {
- __ align64();
StubId stub_id = StubId::stubgen_pack_vec_base64_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align64();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
assert(((unsigned long long)start & 0x3f) == 0,
"Alignment problem (0x%08llx)", (unsigned long long)start);
@@ -2244,14 +2703,23 @@ address StubGenerator::base64_vbmi_pack_vec_addr() {
__ emit_data64(0x0000000000000000, relocInfo::none);
__ emit_data64(0x0000000000000000, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::base64_vbmi_join_0_1_addr() {
- __ align64();
StubId stub_id = StubId::stubgen_join_0_1_base64_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align64();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
assert(((unsigned long long)start & 0x3f) == 0,
"Alignment problem (0x%08llx)", (unsigned long long)start);
@@ -2264,14 +2732,23 @@ address StubGenerator::base64_vbmi_join_0_1_addr() {
__ emit_data64(0x494a444546404142, relocInfo::none);
__ emit_data64(0x565051524c4d4e48, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::base64_vbmi_join_1_2_addr() {
- __ align64();
StubId stub_id = StubId::stubgen_join_1_2_base64_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align64();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
assert(((unsigned long long)start & 0x3f) == 0,
"Alignment problem (0x%08llx)", (unsigned long long)start);
@@ -2284,14 +2761,23 @@ address StubGenerator::base64_vbmi_join_1_2_addr() {
__ emit_data64(0x5c5d5e58595a5455, relocInfo::none);
__ emit_data64(0x696a646566606162, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::base64_vbmi_join_2_3_addr() {
- __ align64();
StubId stub_id = StubId::stubgen_join_2_3_base64_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align64();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
assert(((unsigned long long)start & 0x3f) == 0,
"Alignment problem (0x%08llx)", (unsigned long long)start);
@@ -2304,14 +2790,23 @@ address StubGenerator::base64_vbmi_join_2_3_addr() {
__ emit_data64(0x767071726c6d6e68, relocInfo::none);
__ emit_data64(0x7c7d7e78797a7475, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::base64_AVX2_decode_tables_addr() {
- __ align64();
StubId stub_id = StubId::stubgen_avx2_decode_tables_base64_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align64();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
assert(((unsigned long long)start & 0x3f) == 0,
"Alignment problem (0x%08llx)", (unsigned long long)start);
@@ -2339,14 +2834,23 @@ address StubGenerator::base64_AVX2_decode_tables_addr() {
// merge multiplier
__ emit_data(0x00011000, relocInfo::none, 0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::base64_AVX2_decode_LUT_tables_addr() {
- __ align64();
StubId stub_id = StubId::stubgen_avx2_decode_lut_tables_base64_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align64();
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
assert(((unsigned long long)start & 0x3f) == 0,
"Alignment problem (0x%08llx)", (unsigned long long)start);
@@ -2380,13 +2884,22 @@ address StubGenerator::base64_AVX2_decode_LUT_tables_addr() {
__ emit_data64(0x0804080402011010, relocInfo::none);
__ emit_data64(0x1010101010101010, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::base64_decoding_table_addr() {
StubId stub_id = StubId::stubgen_decoding_table_base64_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ emit_data64(0xffffffffffffffff, relocInfo::none);
__ emit_data64(0xffffffffffffffff, relocInfo::none);
@@ -2455,6 +2968,9 @@ address StubGenerator::base64_decoding_table_addr() {
__ emit_data64(0xffffffffffffffff, relocInfo::none);
__ emit_data64(0xffffffffffffffff, relocInfo::none);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -2466,10 +2982,16 @@ address StubGenerator::base64_decoding_table_addr() {
// Intrinsic function prototype in Base64.java:
// private void decodeBlock(byte[] src, int sp, int sl, byte[] dst, int dp, boolean isURL, isMIME) {
address StubGenerator::generate_base64_decodeBlock() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_base64_decodeBlock_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
@@ -2982,6 +3504,9 @@ address StubGenerator::generate_base64_decodeBlock() {
__ leave();
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3000,11 +3525,17 @@ address StubGenerator::generate_base64_decodeBlock() {
address StubGenerator::generate_updateBytesCRC32() {
assert(UseCRC32Intrinsics, "need AVX and CLMUL instructions");
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_updateBytesCRC32_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
// Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
// Unix: rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...)
@@ -3039,6 +3570,9 @@ address StubGenerator::generate_updateBytesCRC32() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3057,10 +3591,16 @@ address StubGenerator::generate_updateBytesCRC32() {
*/
address StubGenerator::generate_updateBytesCRC32C(bool is_pclmulqdq_supported) {
assert(UseCRC32CIntrinsics, "need SSE4_2");
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_updateBytesCRC32C_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
//reg.arg int#0 int#1 int#2 int#3 int#4 int#5 float regs
//Windows RCX RDX R8 R9 none none XMM0..XMM3
@@ -3120,6 +3660,9 @@ address StubGenerator::generate_updateBytesCRC32C(bool is_pclmulqdq_supported) {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3138,10 +3681,16 @@ address StubGenerator::generate_updateBytesCRC32C(bool is_pclmulqdq_supported) {
* rsp+40 - z address
*/
address StubGenerator::generate_multiplyToLen() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_multiplyToLen_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
// Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
// Unix: rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...)
@@ -3179,6 +3728,9 @@ address StubGenerator::generate_multiplyToLen() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3195,10 +3747,16 @@ address StubGenerator::generate_multiplyToLen() {
* rax - int >= mismatched index, < 0 bitwise complement of tail
*/
address StubGenerator::generate_vectorizedMismatch() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_vectorizedMismatch_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
BLOCK_COMMENT("Entry:");
__ enter();
@@ -3232,6 +3790,9 @@ address StubGenerator::generate_vectorizedMismatch() {
__ leave();
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3247,10 +3808,16 @@ address StubGenerator::generate_vectorizedMismatch() {
*/
address StubGenerator::generate_squareToLen() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_squareToLen_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
// Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
// Unix: rdi, rsi, rdx, rcx (c_rarg0, c_rarg1, ...)
@@ -3279,14 +3846,23 @@ address StubGenerator::generate_squareToLen() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_method_entry_barrier() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_method_entry_barrier_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label deoptimize_label;
@@ -3356,6 +3932,9 @@ address StubGenerator::generate_method_entry_barrier() {
__ movptr(rsp, Address(rsp, 0)); // new rsp was written in the barrier
__ jmp(Address(rsp, -1 * wordSize)); // jmp target should be callers verified_entry_point
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3373,10 +3952,16 @@ address StubGenerator::generate_method_entry_barrier() {
* rsp+40 - k
*/
address StubGenerator::generate_mulAdd() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_mulAdd_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
// Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
// Unix: rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...)
@@ -3411,14 +3996,23 @@ address StubGenerator::generate_mulAdd() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_bigIntegerRightShift() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_bigIntegerRightShiftWorker_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label Shift512Loop, ShiftTwo, ShiftTwoLoop, ShiftOne, Exit;
// For Unix, the arguments are as follows: rdi, rsi, rdx, rcx, r8.
@@ -3534,6 +4128,9 @@ address StubGenerator::generate_bigIntegerRightShift() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3551,10 +4148,16 @@ address StubGenerator::generate_bigIntegerRightShift() {
* rsp40 - numIter
*/
address StubGenerator::generate_bigIntegerLeftShift() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_bigIntegerLeftShiftWorker_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label Shift512Loop, ShiftTwo, ShiftTwoLoop, ShiftOne, Exit;
// For Unix, the arguments are as follows: rdi, rsi, rdx, rcx, r8.
@@ -3659,6 +4262,9 @@ address StubGenerator::generate_bigIntegerLeftShift() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3708,9 +4314,15 @@ void StubGenerator::generate_libm_stubs() {
*/
address StubGenerator::generate_float16ToFloat() {
StubId stub_id = StubId::stubgen_hf2f_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
BLOCK_COMMENT("Entry:");
// No need for RuntimeStub frame since it is called only during JIT compilation
@@ -3720,6 +4332,9 @@ address StubGenerator::generate_float16ToFloat() {
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3734,9 +4349,15 @@ address StubGenerator::generate_float16ToFloat() {
*/
address StubGenerator::generate_floatToFloat16() {
StubId stub_id = StubId::stubgen_f2hf_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
BLOCK_COMMENT("Entry:");
// No need for RuntimeStub frame since it is called only during JIT compilation
@@ -3746,6 +4367,9 @@ address StubGenerator::generate_floatToFloat16() {
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3775,8 +4399,14 @@ address StubGenerator::generate_cont_thaw(StubId stub_id) {
default:
ShouldNotReachHere();
}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
// TODO: Handle Valhalla return types. May require generating different return barriers.
@@ -3889,6 +4519,9 @@ address StubGenerator::generate_cont_thaw(StubId stub_id) {
__ ret(0);
}
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3909,8 +4542,14 @@ address StubGenerator::generate_cont_returnBarrier_exception() {
address StubGenerator::generate_cont_preempt_stub() {
if (!Continuations::enabled()) return nullptr;
StubId stub_id = StubId::stubgen_cont_preempt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ reset_last_Java_frame(true);
@@ -3934,14 +4573,23 @@ address StubGenerator::generate_cont_preempt_stub() {
__ movptr(rscratch1, ExternalAddress(ContinuationEntry::thaw_call_pc_address()));
__ jmp(rscratch1);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
// exception handler for upcall stubs
address StubGenerator::generate_upcall_stub_exception_handler() {
StubId stub_id = StubId::stubgen_upcall_stub_exception_handler_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
// native caller has no idea how to handle exceptions
// we just crash here. Up to callee to catch exceptions.
@@ -3953,6 +4601,9 @@ address StubGenerator::generate_upcall_stub_exception_handler() {
__ call(RuntimeAddress(CAST_FROM_FN_PTR(address, UpcallLinker::handle_uncaught_exception)));
__ should_not_reach_here();
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -3961,8 +4612,14 @@ address StubGenerator::generate_upcall_stub_exception_handler() {
// rbx = result
address StubGenerator::generate_upcall_stub_load_target() {
StubId stub_id = StubId::stubgen_upcall_stub_load_target_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ resolve_global_jobject(j_rarg0, rscratch1);
// Load target method from receiver
@@ -3976,11 +4633,27 @@ address StubGenerator::generate_upcall_stub_load_target() {
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
void StubGenerator::generate_lookup_secondary_supers_table_stub() {
StubId stub_id = StubId::stubgen_lookup_secondary_supers_table_id;
+ GrowableArray entries;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == Klass::SECONDARY_SUPERS_TABLE_SIZE, "sanity check");
+ address start = load_archive_data(stub_id, &entries);
+ if (start != nullptr) {
+ assert(entries.length() == Klass::SECONDARY_SUPERS_TABLE_SIZE - 1,
+ "unexpected extra entry count %d", entries.length());
+ StubRoutines::_lookup_secondary_supers_table_stubs[0] = start;
+ for (int slot = 1; slot < Klass::SECONDARY_SUPERS_TABLE_SIZE; slot++) {
+ StubRoutines::_lookup_secondary_supers_table_stubs[slot] = entries.at(slot - 1);
+ }
+ return;
+ }
StubCodeMark mark(this, stub_id);
const Register
@@ -3989,21 +4662,35 @@ void StubGenerator::generate_lookup_secondary_supers_table_stub() {
result = rdi;
for (int slot = 0; slot < Klass::SECONDARY_SUPERS_TABLE_SIZE; slot++) {
- StubRoutines::_lookup_secondary_supers_table_stubs[slot] = __ pc();
+ address next_entry = __ pc();
+ if (slot == 0) {
+ start = next_entry;
+ } else {
+ entries.append(next_entry);
+ }
+ StubRoutines::_lookup_secondary_supers_table_stubs[slot] = next_entry;
__ lookup_secondary_supers_table_const(r_sub_klass, r_super_klass,
rdx, rcx, rbx, r11, // temps
result,
slot);
__ ret(0);
}
+
+ // record the stub entry and end plus all the auxiliary entries
+ store_archive_data(stub_id, start, __ pc(), &entries);
}
// Slow path implementation for UseSecondarySupersTable.
address StubGenerator::generate_lookup_secondary_supers_table_slow_path_stub() {
StubId stub_id = StubId::stubgen_lookup_secondary_supers_table_slow_path_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
-
- address start = __ pc();
+ start = __ pc();
const Register
r_super_klass = rax,
@@ -4025,6 +4712,9 @@ address StubGenerator::generate_lookup_secondary_supers_table_slow_path_stub() {
__ movl(result, 0);
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -4165,7 +4855,7 @@ void StubGenerator::generate_compiler_stubs() {
StubRoutines::x86::_vector_short_shuffle_mask = generate_vector_mask(StubId::stubgen_vector_short_shuffle_mask_id, 0x0100010001000100);
StubRoutines::x86::_vector_long_shuffle_mask = generate_vector_mask(StubId::stubgen_vector_long_shuffle_mask_id, 0x0000000100000000);
StubRoutines::x86::_vector_long_sign_mask = generate_vector_mask(StubId::stubgen_vector_long_sign_mask_id, 0x8000000000000000);
- StubRoutines::x86::_vector_iota_indices = generate_iota_indices();
+ generate_iota_indices();
StubRoutines::x86::_vector_count_leading_zeros_lut = generate_count_leading_zeros_lut();
StubRoutines::x86::_vector_reverse_bit_lut = generate_vector_reverse_bit_lut();
StubRoutines::x86::_vector_reverse_byte_perm_mask_long = generate_vector_reverse_byte_perm_mask_long();
@@ -4232,6 +4922,8 @@ void StubGenerator::generate_compiler_stubs() {
}
if (UseSHA256Intrinsics) {
+ address entry2 = nullptr;
+ address entry3 = nullptr;
StubRoutines::x86::_k256_adr = (address)StubRoutines::x86::_k256;
char* dst = (char*)StubRoutines::x86::_k256_W;
char* src = (char*)StubRoutines::x86::_k256;
@@ -4240,14 +4932,18 @@ void StubGenerator::generate_compiler_stubs() {
memcpy(dst + 32 * ii + 16, src + 16 * ii, 16);
}
StubRoutines::x86::_k256_W_adr = (address)StubRoutines::x86::_k256_W;
- StubRoutines::x86::_pshuffle_byte_flip_mask_addr = generate_pshuffle_byte_flip_mask();
+ StubRoutines::x86::_pshuffle_byte_flip_mask_addr = generate_pshuffle_byte_flip_mask(entry2, entry3);
+ StubRoutines::x86::_pshuffle_byte_flip_mask_00ba_addr = entry2;
+ StubRoutines::x86::_pshuffle_byte_flip_mask_dc00_addr = entry3;
StubRoutines::_sha256_implCompress = generate_sha256_implCompress(StubId::stubgen_sha256_implCompress_id);
StubRoutines::_sha256_implCompressMB = generate_sha256_implCompress(StubId::stubgen_sha256_implCompressMB_id);
}
if (UseSHA512Intrinsics) {
+ address entry2 = nullptr;
StubRoutines::x86::_k512_W_addr = (address)StubRoutines::x86::_k512_W;
- StubRoutines::x86::_pshuffle_byte_flip_mask_addr_sha512 = generate_pshuffle_byte_flip_mask_sha512();
+ StubRoutines::x86::_pshuffle_byte_flip_mask_addr_sha512 = generate_pshuffle_byte_flip_mask_sha512(entry2);
+ StubRoutines::x86::_pshuffle_byte_flip_mask_ymm_lo_addr_sha512 = entry2;
StubRoutines::_sha512_implCompress = generate_sha512_implCompress(StubId::stubgen_sha512_implCompress_id);
StubRoutines::_sha512_implCompressMB = generate_sha512_implCompress(StubId::stubgen_sha512_implCompressMB_id);
}
@@ -4325,7 +5021,7 @@ void StubGenerator::generate_compiler_stubs() {
#endif // COMPILER2_OR_JVMCI
}
-StubGenerator::StubGenerator(CodeBuffer* code, BlobId blob_id) : StubCodeGenerator(code, blob_id) {
+StubGenerator::StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) : StubCodeGenerator(code, blob_id, stub_data) {
switch(blob_id) {
case BlobId::stubgen_preuniverse_id:
generate_preuniverse_stubs();
@@ -4348,8 +5044,35 @@ StubGenerator::StubGenerator(CodeBuffer* code, BlobId blob_id) : StubCodeGenerat
};
}
-void StubGenerator_generate(CodeBuffer* code, BlobId blob_id) {
- StubGenerator g(code, blob_id);
+#if INCLUDE_CDS
+// publish addresses of static data defined in this file and in other
+// stubgen stub generator files
+void StubGenerator::init_AOTAddressTable(GrowableArray& external_addresses) {
+ init_AOTAddressTable_adler(external_addresses);
+ init_AOTAddressTable_aes(external_addresses);
+ init_AOTAddressTable_cbrt(external_addresses);
+ init_AOTAddressTable_chacha(external_addresses);
+ // constants publishes for all of address use by cos and almost all of sin
+ init_AOTAddressTable_constants(external_addresses);
+ init_AOTAddressTable_dilithium(external_addresses);
+ init_AOTAddressTable_exp(external_addresses);
+ init_AOTAddressTable_fmod(external_addresses);
+ init_AOTAddressTable_ghash(external_addresses);
+ init_AOTAddressTable_kyber(external_addresses);
+ init_AOTAddressTable_log(external_addresses);
+ init_AOTAddressTable_poly1305(external_addresses);
+ init_AOTAddressTable_poly_mont(external_addresses);
+ init_AOTAddressTable_pow(external_addresses);
+ init_AOTAddressTable_sha3(external_addresses);
+ init_AOTAddressTable_sin(external_addresses);
+ init_AOTAddressTable_sinh(external_addresses);
+ init_AOTAddressTable_tan(external_addresses);
+ init_AOTAddressTable_tanh(external_addresses);
+}
+#endif // INCLUDE_CDS
+
+void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) {
+ StubGenerator g(code, blob_id, stub_data);
}
#undef __
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp
index 332add6dcd4..d3823cb559f 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp
@@ -84,7 +84,7 @@ class StubGenerator: public StubCodeGenerator {
address generate_count_leading_zeros_lut();
address generate_popcount_avx_lut();
- address generate_iota_indices();
+ void generate_iota_indices();
address generate_vector_reverse_bit_lut();
address generate_vector_reverse_byte_perm_mask_long();
@@ -303,11 +303,11 @@ class StubGenerator: public StubCodeGenerator {
address generate_sha512_implCompress(StubId stub_id);
// Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb.
- address generate_pshuffle_byte_flip_mask_sha512();
+ address generate_pshuffle_byte_flip_mask_sha512(address& entry_ymm_lo);
address generate_upper_word_mask();
address generate_shuffle_byte_flip_mask();
- address generate_pshuffle_byte_flip_mask();
+ address generate_pshuffle_byte_flip_mask(address& entry_00ba, address& entry_dc0);
// AES intrinsic stubs
@@ -650,8 +650,33 @@ class StubGenerator: public StubCodeGenerator {
void generate_compiler_stubs();
void generate_final_stubs();
+#if INCLUDE_CDS
+ static void init_AOTAddressTable_adler(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_aes(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_cbrt(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_chacha(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_constants(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_dilithium(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_exp(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_fmod(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_ghash(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_kyber(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_log(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_poly1305(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_poly_mont(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_pow(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_sha3(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_sin(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_sinh(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_tan(GrowableArray& external_addresses);
+ static void init_AOTAddressTable_tanh(GrowableArray& external_addresses);
+#endif // INCLUDE_CDS
+
public:
- StubGenerator(CodeBuffer* code, BlobId blob_id);
+ StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data);
+#if INCLUDE_CDS
+ static void init_AOTAddressTable(GrowableArray& external_addresses);
+#endif // INCLUDE_CDS
};
#endif // CPU_X86_STUBGENERATOR_X86_64_HPP
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_adler.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_adler.cpp
index 1d3e7afde1d..a9424978e0e 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_adler.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_adler.cpp
@@ -67,8 +67,14 @@ address StubGenerator::generate_updateBytesAdler32() {
__ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_updateBytesAdler32_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
// Choose an appropriate LIMIT for inner loop based on the granularity
// of intermediate results. For int, LIMIT of 5552 will ensure intermediate
@@ -334,7 +340,19 @@ address StubGenerator::generate_updateBytesAdler32() {
__ leave();
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
return start;
}
#undef __
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_adler(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr))
+ ADD(ADLER32_ASCALE_TABLE);
+ ADD(ADLER32_SHUF0_TABLE);
+ ADD(ADLER32_SHUF1_TABLE);
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_aes.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_aes.cpp
index 1fa80c9d967..b95aa5f8818 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_aes.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_aes.cpp
@@ -220,7 +220,7 @@ void StubGenerator::generate_aes_stubs() {
StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt_Parallel();
StubRoutines::_electronicCodeBook_encryptAESCrypt = generate_electronicCodeBook_encryptAESCrypt_Parallel();
StubRoutines::_electronicCodeBook_decryptAESCrypt = generate_electronicCodeBook_decryptAESCrypt_Parallel();
- if (VM_Version::supports_avx2()) {
+ if (VM_Version::supports_avx2() && VM_Version::supports_clmul()) {
StubRoutines::_galoisCounterMode_AESCrypt = generate_avx2_galoisCounterMode_AESCrypt();
}
}
@@ -250,10 +250,16 @@ void StubGenerator::generate_aes_stubs() {
// Output:
// rax - number of processed bytes
address StubGenerator::generate_galoisCounterMode_AESCrypt() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_galoisCounterMode_AESCrypt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register in = c_rarg0;
const Register len = c_rarg1;
@@ -319,6 +325,9 @@ address StubGenerator::generate_galoisCounterMode_AESCrypt() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -337,10 +346,16 @@ address StubGenerator::generate_galoisCounterMode_AESCrypt() {
// Output:
// rax - number of processed bytes
address StubGenerator::generate_avx2_galoisCounterMode_AESCrypt() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_galoisCounterMode_AESCrypt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register in = c_rarg0;
const Register len = c_rarg1;
@@ -404,15 +419,24 @@ address StubGenerator::generate_avx2_galoisCounterMode_AESCrypt() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
// Vector AES Counter implementation
address StubGenerator::generate_counterMode_VectorAESCrypt() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_counterMode_AESCrypt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register from = c_rarg0; // source array address
const Register to = c_rarg1; // destination array address
@@ -471,6 +495,9 @@ address StubGenerator::generate_counterMode_VectorAESCrypt() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -498,10 +525,16 @@ address StubGenerator::generate_counterMode_VectorAESCrypt() {
//
address StubGenerator::generate_counterMode_AESCrypt_Parallel() {
assert(UseAES, "need AES instructions and misaligned SSE support");
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_counterMode_AESCrypt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register from = c_rarg0; // source array address
const Register to = c_rarg1; // destination array address
@@ -781,15 +814,24 @@ address StubGenerator::generate_counterMode_AESCrypt_Parallel() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_cipherBlockChaining_decryptVectorAESCrypt() {
assert(VM_Version::supports_avx512_vaes(), "need AES instructions and misaligned SSE support");
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_cipherBlockChaining_decryptAESCrypt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register from = c_rarg0; // source array address
const Register to = c_rarg1; // destination array address
@@ -1057,6 +1099,9 @@ address StubGenerator::generate_cipherBlockChaining_decryptVectorAESCrypt() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -1069,11 +1114,17 @@ address StubGenerator::generate_cipherBlockChaining_decryptVectorAESCrypt() {
//
address StubGenerator::generate_aescrypt_encryptBlock() {
assert(UseAES, "need AES instructions and misaligned SSE support");
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_aescrypt_encryptBlock_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
Label L_doLast;
- address start = __ pc();
+ start = __ pc();
const Register from = c_rarg0; // source array address
const Register to = c_rarg1; // destination array address
@@ -1152,6 +1203,9 @@ address StubGenerator::generate_aescrypt_encryptBlock() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -1164,11 +1218,17 @@ address StubGenerator::generate_aescrypt_encryptBlock() {
//
address StubGenerator::generate_aescrypt_decryptBlock() {
assert(UseAES, "need AES instructions and misaligned SSE support");
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_aescrypt_decryptBlock_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
Label L_doLast;
- address start = __ pc();
+ start = __ pc();
const Register from = c_rarg0; // source array address
const Register to = c_rarg1; // destination array address
@@ -1248,6 +1308,9 @@ address StubGenerator::generate_aescrypt_decryptBlock() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -1266,10 +1329,16 @@ address StubGenerator::generate_aescrypt_decryptBlock() {
//
address StubGenerator::generate_cipherBlockChaining_encryptAESCrypt() {
assert(UseAES, "need AES instructions and misaligned SSE support");
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_cipherBlockChaining_encryptAESCrypt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_exit, L_key_192_256, L_key_256, L_loopTop_128, L_loopTop_192, L_loopTop_256;
const Register from = c_rarg0; // source array address
@@ -1398,6 +1467,9 @@ address StubGenerator::generate_cipherBlockChaining_encryptAESCrypt() {
__ jcc(Assembler::notEqual, L_loopTop_256);
__ jmp(L_exit);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -1422,11 +1494,15 @@ address StubGenerator::generate_cipherBlockChaining_encryptAESCrypt() {
//
address StubGenerator::generate_electronicCodeBook_AESCrypt_Parallel(bool is_encrypt) {
assert(UseAES, "need AES instructions and misaligned SSE support");
- __ align(CodeEntryAlignment);
StubId stub_id = is_encrypt ? StubId::stubgen_electronicCodeBook_encryptAESCrypt_id
: StubId::stubgen_electronicCodeBook_decryptAESCrypt_id;
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register from = c_rarg0; // source array address
const Register to = c_rarg1; // destination array address
@@ -1581,6 +1657,9 @@ __ opc(xmm_result0, reg);
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
#undef DoFour
@@ -1612,10 +1691,16 @@ address StubGenerator::generate_electronicCodeBook_decryptAESCrypt_Parallel() {
//
address StubGenerator::generate_cipherBlockChaining_decryptAESCrypt_Parallel() {
assert(UseAES, "need AES instructions and misaligned SSE support");
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_cipherBlockChaining_decryptAESCrypt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register from = c_rarg0; // source array address
const Register to = c_rarg1; // destination array address
@@ -1851,14 +1936,23 @@ __ opc(xmm_result3, src_reg);
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_electronicCodeBook_encryptAESCrypt() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_electronicCodeBook_encryptAESCrypt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register from = c_rarg0; // source array address
const Register to = c_rarg1; // destination array address
@@ -1872,14 +1966,23 @@ address StubGenerator::generate_electronicCodeBook_encryptAESCrypt() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
address StubGenerator::generate_electronicCodeBook_decryptAESCrypt() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_electronicCodeBook_decryptAESCrypt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register from = c_rarg0; // source array address
const Register to = c_rarg1; // destination array address
@@ -1893,6 +1996,9 @@ address StubGenerator::generate_electronicCodeBook_decryptAESCrypt() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -4292,3 +4398,27 @@ void StubGenerator::aesgcm_avx2(Register in, Register len, Register ct, Register
}
#undef __
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_aes(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr))
+ ADD(key_shuffle_mask_addr());
+ ADD(counter_shuffle_mask_addr());
+ ADD(counter_mask_linc0_addr());
+ ADD(counter_mask_linc1_addr());
+ ADD(counter_mask_linc1f_addr());
+ ADD(counter_mask_linc2_addr());
+ ADD(counter_mask_linc2f_addr());
+ ADD(counter_mask_linc4_addr());
+ ADD(counter_mask_linc8_addr());
+ ADD(counter_mask_linc16_addr());
+ ADD(counter_mask_linc32_addr());
+ ADD(counter_mask_ones_addr());
+ ADD(ghash_polynomial_reduction_addr());
+ ADD(ghash_polynomial_two_one_addr());
+ ADD(counter_mask_addbe_4444_addr());
+ ADD(counter_mask_addbe_1234_addr());
+ ADD(counter_mask_add_1234_addr());
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp
index 01e004b7b43..5530e5325de 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp
@@ -570,10 +570,45 @@ address StubGenerator::generate_disjoint_copy_avx3_masked(StubId stub_id, addres
default:
ShouldNotReachHere();
}
+ GrowableArray entries;
+ GrowableArray extras;
+ bool add_handlers = !is_oop && !aligned;
+ bool add_relocs = UseZGC && is_oop;
+ bool add_extras = add_handlers || add_relocs;
+ // The stub employs one unsafe handler region by default but has two
+ // when MaxVectorSize == 64 So we may expect 0, 3 or 6 extras.
+ int handlers_count = (MaxVectorSize == 64 ? 2 : 1);
+ int expected_entry_count = (entry != nullptr ? 2 : 1);
+ int expected_extra_count = (add_handlers ? handlers_count : 0) * UnsafeMemoryAccess::COLUMN_COUNT; // 0/1/2 x UMAM {start,end,handler}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == expected_entry_count, "sanity check");
+ GrowableArray* entries_ptr = (entry_count == 1 ? nullptr : &entries);
+ GrowableArray* extras_ptr = (add_extras ? &extras : nullptr);
+ address start = load_archive_data(stub_id, entries_ptr, extras_ptr);
+ if (start != nullptr) {
+ assert(entries.length() == expected_entry_count - 1,
+ "unexpected entry count %d", entries.length());
+ assert(!add_handlers || extras.length() == expected_extra_count,
+ "unexpected handler addresses count %d", extras.length());
+ if (entry != nullptr) {
+ *entry = entries.at(0);
+ }
+ if (add_handlers) {
+ // restore 1/2 x UMAM {start,end,handler} addresses from extras
+ register_unsafe_access_handlers(extras, 0, handlers_count);
+ }
+#if INCLUDE_ZGC
+ // register addresses at which ZGC does colour patching
+ if (add_relocs) {
+ register_reloc_addresses(extras, 0, extras.length());
+ }
+#endif // INCLUDE_ZGC
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
bool use64byteVector = (MaxVectorSize > 32) && (CopyAVX3Threshold == 0);
const int large_threshold = 2621440; // 2.5 MB
@@ -595,6 +630,7 @@ address StubGenerator::generate_disjoint_copy_avx3_masked(StubId stub_id, addres
if (entry != nullptr) {
*entry = __ pc();
+ entries.append(*entry);
// caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
BLOCK_COMMENT("Entry:");
}
@@ -620,7 +656,7 @@ address StubGenerator::generate_disjoint_copy_avx3_masked(StubId stub_id, addres
int threshold[] = { 4096, 2048, 1024, 512};
// UnsafeMemoryAccess page error: continue after unsafe access
- UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true);
+ UnsafeMemoryAccessMark umam(this, add_handlers, true);
// 'from', 'to' and 'count' are now valid
// temp1 holds remaining count and temp4 holds running count used to compute
@@ -789,10 +825,28 @@ address StubGenerator::generate_disjoint_copy_avx3_masked(StubId stub_id, addres
if (MaxVectorSize == 64) {
__ BIND(L_copy_large);
- UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, false, ucme_exit_pc);
+ UnsafeMemoryAccessMark umam(this, add_handlers, false, ucme_exit_pc);
arraycopy_avx3_large(to, from, temp1, temp2, temp3, temp4, count, xmm1, xmm2, xmm3, xmm4, shift);
__ jmp(L_finish);
}
+ // retrieve the registered handler addresses
+ address end = __ pc();
+ if (add_handlers) {
+ retrieve_unsafe_access_handlers(start, end, extras);
+ }
+ assert(extras.length() == expected_extra_count,
+ "unexpected handler addresses count %d", extras.length());
+#if INCLUDE_ZGC
+ // retrieve addresses at which ZGC does colour patching
+ if (add_relocs) {
+ retrieve_reloc_addresses(start, end, extras);
+ }
+#endif // INCLUDE_ZGC
+
+ // record the stub entry and end plus the no_push entry and any
+ // extra handler addresses
+ store_archive_data(stub_id, start, end, entries_ptr, extras_ptr);
+
return start;
}
@@ -907,10 +961,41 @@ address StubGenerator::generate_conjoint_copy_avx3_masked(StubId stub_id, addres
default:
ShouldNotReachHere();
}
-
+ GrowableArray entries;
+ GrowableArray extras;
+ bool add_handlers = !is_oop && !aligned;
+ bool add_relocs = UseZGC && is_oop;
+ bool add_extras = add_handlers || add_relocs;
+ int expected_entry_count = (entry != nullptr ? 2 : 1);
+ int expected_handler_count = (add_handlers ? 1 : 0) * UnsafeMemoryAccess::COLUMN_COUNT; // 0/1 x UMAM {start,end,handler}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == expected_entry_count, "sanity check");
+ GrowableArray* entries_ptr = (entry_count == 1 ? nullptr : &entries);
+ GrowableArray* extras_ptr = (add_extras ? &extras : nullptr);
+ address start = load_archive_data(stub_id, entries_ptr, extras_ptr);
+ if (start != nullptr) {
+ assert(entries.length() == expected_entry_count - 1,
+ "unexpected entry count %d", entries.length());
+ assert(!add_handlers || extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+ if (entry != nullptr) {
+ *entry = entries.at(0);
+ }
+ if (add_handlers) {
+ // restore 1 x UMAM {start,end,handler} addresses from extras
+ register_unsafe_access_handlers(extras, 0, 1);
+ }
+#if INCLUDE_ZGC
+ if (add_relocs) {
+ // register addresses at which ZGC does colour patching
+ register_reloc_addresses(extras, 0, extras.length());
+ }
+#endif // INCLUDE_ZGC
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
bool use64byteVector = (MaxVectorSize > 32) && (CopyAVX3Threshold == 0);
@@ -931,6 +1016,7 @@ address StubGenerator::generate_conjoint_copy_avx3_masked(StubId stub_id, addres
if (entry != nullptr) {
*entry = __ pc();
+ entries.append(*entry);
// caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
BLOCK_COMMENT("Entry:");
}
@@ -957,7 +1043,7 @@ address StubGenerator::generate_conjoint_copy_avx3_masked(StubId stub_id, addres
int threshold[] = { 4096, 2048, 1024, 512};
// UnsafeMemoryAccess page error: continue after unsafe access
- UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true);
+ UnsafeMemoryAccessMark umam(this, add_handlers, true);
// 'from', 'to' and 'count' are now valid
// temp1 holds remaining count.
@@ -1071,6 +1157,23 @@ address StubGenerator::generate_conjoint_copy_avx3_masked(StubId stub_id, addres
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // retrieve the registered handler addresses
+ address end = __ pc();
+ if (add_handlers) {
+ retrieve_unsafe_access_handlers(start, end, extras);
+ }
+ assert(extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+#if INCLUDE_ZGC
+ // retrieve addresses at which ZGC does colour patching
+ if (add_relocs) {
+ retrieve_reloc_addresses(start, end, extras);
+ }
+#endif // INCLUDE_ZGC
+ // record the stub entry and end plus the no_push entry and any
+ // extra handler addresses
+ store_archive_data(stub_id, start, end, entries_ptr, extras_ptr);
+
return start;
}
@@ -1385,9 +1488,29 @@ address StubGenerator::generate_disjoint_byte_copy(address* entry) {
return generate_disjoint_copy_avx3_masked(stub_id, entry);
}
#endif
+ GrowableArray entries;
+ GrowableArray extras;
+ int expected_entry_count = (entry != nullptr ? 2 : 1);
+ int expected_handler_count = (2 * UnsafeMemoryAccess::COLUMN_COUNT); // 2 x UMAM {start,end,handler}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == expected_entry_count, "sanity check");
+ GrowableArray* entries_ptr = (entry_count == 1 ? nullptr : &entries);
+ address start = load_archive_data(stub_id, entries_ptr, &extras);
+ if (start != nullptr) {
+ assert(entries.length() == expected_entry_count - 1,
+ "unexpected entry count %d", entries.length());
+ assert(extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+ if (entry != nullptr) {
+ *entry = entries.at(0);
+ }
+ // restore 2 UMAM {start,end,handler} addresses from extras
+ register_unsafe_access_handlers(extras, 0, 2);
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT;
Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_copy_2_bytes;
@@ -1407,6 +1530,7 @@ address StubGenerator::generate_disjoint_byte_copy(address* entry) {
if (entry != nullptr) {
*entry = __ pc();
+ entries.append(*entry);
// caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
BLOCK_COMMENT("Entry:");
}
@@ -1476,6 +1600,17 @@ __ BIND(L_exit);
copy_bytes_forward(end_from, end_to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, T_BYTE);
__ jmp(L_copy_4_bytes);
}
+
+ // retrieve the registered handler addresses
+ address end = __ pc();
+ retrieve_unsafe_access_handlers(start, end, extras);
+ assert(extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+
+ // record the stub entry and end plus the no_push entry and any
+ // extra handler addresses
+ store_archive_data(stub_id, start, end, entries_ptr, &extras);
+
return start;
}
@@ -1503,9 +1638,29 @@ address StubGenerator::generate_conjoint_byte_copy(address nooverlap_target, add
return generate_conjoint_copy_avx3_masked(stub_id, entry, nooverlap_target);
}
#endif
+ GrowableArray entries;
+ GrowableArray extras;
+ int expected_entry_count = (entry != nullptr ? 2 : 1);
+ int expected_handler_count = (2 * UnsafeMemoryAccess::COLUMN_COUNT); // 2 x UMAM {start,end,handler}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == expected_entry_count, "sanity check");
+ GrowableArray* entries_ptr = (entry_count == 1 ? nullptr : &entries);
+ address start = load_archive_data(stub_id, entries_ptr, &extras);
+ if (start != nullptr) {
+ assert(entries.length() == expected_entry_count - 1,
+ "unexpected entry count %d", entries.length());
+ assert(extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+ if (entry != nullptr) {
+ *entry = entries.at(0);
+ }
+ // restore 2 UMAM {start,end,handler} addresses from extras
+ register_unsafe_access_handlers(extras, 0, 2);
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
DecoratorSet decorators = IN_HEAP | IS_ARRAY;
Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_copy_2_bytes;
@@ -1520,6 +1675,7 @@ address StubGenerator::generate_conjoint_byte_copy(address nooverlap_target, add
if (entry != nullptr) {
*entry = __ pc();
+ entries.append(*entry);
// caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
BLOCK_COMMENT("Entry:");
}
@@ -1586,6 +1742,16 @@ address StubGenerator::generate_conjoint_byte_copy(address nooverlap_target, add
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // retrieve the registered handler addresses
+ address end = __ pc();
+ retrieve_unsafe_access_handlers(start, end, extras);
+ assert(extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+
+ // record the stub entry and end plus the no_push entry and any
+ // extra handler addresses
+ store_archive_data(stub_id, start, end, entries_ptr, &extras);
+
return start;
}
@@ -1616,10 +1782,29 @@ address StubGenerator::generate_disjoint_short_copy(address *entry) {
return generate_disjoint_copy_avx3_masked(stub_id, entry);
}
#endif
-
+ GrowableArray entries;
+ GrowableArray extras;
+ int expected_entry_count = (entry != nullptr ? 2 : 1);
+ int expected_handler_count = (2 * UnsafeMemoryAccess::COLUMN_COUNT); // 2 x UMAM {start,end,handler}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == expected_entry_count, "sanity check");
+ GrowableArray* entries_ptr = (entry_count == 1 ? nullptr : &entries);
+ address start = load_archive_data(stub_id, entries_ptr, &extras);
+ if (start != nullptr) {
+ assert(entries.length() == expected_entry_count - 1,
+ "unexpected entry count %d", entries.length());
+ assert(extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+ if (entry != nullptr) {
+ *entry = entries.at(0);
+ }
+ // restore 2 UMAM {start,end,handler} addresses from extras
+ register_unsafe_access_handlers(extras, 0, 2);
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT;
Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes,L_copy_2_bytes,L_exit;
@@ -1638,6 +1823,7 @@ address StubGenerator::generate_disjoint_short_copy(address *entry) {
if (entry != nullptr) {
*entry = __ pc();
+ entries.append(*entry);
// caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
BLOCK_COMMENT("Entry:");
}
@@ -1701,6 +1887,16 @@ __ BIND(L_exit);
__ jmp(L_copy_4_bytes);
}
+ // retrieve the registered handler addresses
+ address end = __ pc();
+ retrieve_unsafe_access_handlers(start, end, extras);
+ assert(extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+
+ // record the stub entry and end plus the no_push entry and any
+ // extra handler addresses
+ store_archive_data(stub_id, start, end, entries_ptr, &extras);
+
return start;
}
@@ -1708,7 +1904,6 @@ __ BIND(L_exit);
address StubGenerator::generate_fill(StubId stub_id) {
BasicType t;
bool aligned;
-
switch (stub_id) {
case StubId::stubgen_jbyte_fill_id:
t = T_BYTE;
@@ -1737,10 +1932,27 @@ address StubGenerator::generate_fill(StubId stub_id) {
default:
ShouldNotReachHere();
}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ GrowableArray extras;
+ bool add_handlers = ((t == T_BYTE) && !aligned);
+ int handlers_count = (add_handlers ? 1 : 0);
+ int expected_extras_count = (handlers_count * UnsafeMemoryAccess::COLUMN_COUNT); // 0/1 x UMAM {start,end,handler}
+ GrowableArray* extras_ptr = (add_handlers ? &extras : nullptr);
+ address start = load_archive_data(stub_id, nullptr, extras_ptr);
+ if (start != nullptr) {
+ assert(extras.length() == expected_extras_count,
+ "unexpected handler addresses count %d", extras.length());
+ if (add_handlers) {
+ // restore 1 x UMAM {start,end,handler} addresses from extras
+ register_unsafe_access_handlers(extras, 0, 1);
+ }
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
BLOCK_COMMENT("Entry:");
@@ -1753,7 +1965,7 @@ address StubGenerator::generate_fill(StubId stub_id) {
{
// Add set memory mark to protect against unsafe accesses faulting
- UnsafeMemoryAccessMark umam(this, ((t == T_BYTE) && !aligned), true);
+ UnsafeMemoryAccessMark umam(this, add_handlers, true);
__ generate_fill(t, aligned, to, value, r11, rax, xmm0);
}
@@ -1761,6 +1973,15 @@ address StubGenerator::generate_fill(StubId stub_id) {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ address end = __ pc();
+ if (add_handlers) {
+ retrieve_unsafe_access_handlers(start, end, extras);
+ }
+ assert(extras.length() == expected_extras_count,
+ "unexpected handler addresses count %d", extras.length());
+ // record the stub entry and end
+ store_archive_data(stub_id, start, end, nullptr, extras_ptr);
+
return start;
}
@@ -1788,10 +2009,29 @@ address StubGenerator::generate_conjoint_short_copy(address nooverlap_target, ad
return generate_conjoint_copy_avx3_masked(stub_id, entry, nooverlap_target);
}
#endif
-
+ GrowableArray entries;
+ GrowableArray extras;
+ int expected_entry_count = (entry != nullptr ? 2 : 1);
+ int expected_handler_count = (2 * UnsafeMemoryAccess::COLUMN_COUNT); // 2 x UMAM {start,end,handler}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == expected_entry_count, "sanity check");
+ GrowableArray* entries_ptr = (entry_count == 1 ? nullptr : &entries);
+ address start = load_archive_data(stub_id, entries_ptr, &extras);
+ if (start != nullptr) {
+ assert(entries.length() == expected_entry_count - 1,
+ "unexpected entry count %d", entries.length());
+ assert(extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+ if (entry != nullptr) {
+ *entry = entries.at(0);
+ }
+ // restore 2 UMAM {start,end,handler} addresses from extras
+ register_unsafe_access_handlers(extras, 0, 2);
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
DecoratorSet decorators = IN_HEAP | IS_ARRAY;
Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes;
@@ -1806,6 +2046,7 @@ address StubGenerator::generate_conjoint_short_copy(address nooverlap_target, ad
if (entry != nullptr) {
*entry = __ pc();
+ entries.append(*entry);
// caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
BLOCK_COMMENT("Entry:");
}
@@ -1864,6 +2105,16 @@ address StubGenerator::generate_conjoint_short_copy(address nooverlap_target, ad
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // retrieve the registered handler addresses
+ address end = __ pc();
+ retrieve_unsafe_access_handlers(start, end, extras);
+ assert(extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+
+ // record the stub entry and end plus the no_push entry and any
+ // extra handler addresses
+ store_archive_data(stub_id, start, end, entries_ptr, &extras);
+
return start;
}
@@ -1916,10 +2167,42 @@ address StubGenerator::generate_disjoint_int_oop_copy(StubId stub_id, address* e
return generate_disjoint_copy_avx3_masked(stub_id, entry);
}
#endif
+ GrowableArray entries;
+ GrowableArray extras;
+ bool add_handlers = !is_oop && !aligned;
+ bool add_relocs = UseZGC && is_oop;
+ bool add_extras = add_handlers || add_relocs;
+ int expected_entry_count = (entry != nullptr ? 2 : 1);
+ int expected_handler_count = (add_handlers ? 2 : 0) * UnsafeMemoryAccess::COLUMN_COUNT; // 0/2 x UMAM {start,end,handler}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == expected_entry_count, "sanity check");
+ GrowableArray* entries_ptr = (entry_count == 1 ? nullptr : &entries);
+ GrowableArray* extras_ptr = (add_extras ? &extras : nullptr);
+ address start = load_archive_data(stub_id, entries_ptr, extras_ptr);
+ if (start != nullptr) {
+ assert(entries.length() == expected_entry_count - 1,
+ "unexpected entry count %d", entries.length());
+ assert(!add_handlers || extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+ if (entry != nullptr) {
+ *entry = entries.at(0);
+ }
+ if (add_handlers) {
+ // restore 2 UMAM {start,end,handler} addresses from extras
+ register_unsafe_access_handlers(extras, 0, 2);
+ }
+#if INCLUDE_ZGC
+ // register addresses at which ZGC does colour patching
+ if (add_relocs) {
+ register_reloc_addresses(extras, 0, extras.length());
+ }
+#endif // INCLUDE_ZGC
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_exit;
const Register from = rdi; // source array address
@@ -1937,6 +2220,7 @@ address StubGenerator::generate_disjoint_int_oop_copy(StubId stub_id, address* e
if (entry != nullptr) {
*entry = __ pc();
+ entries.append(*entry);
// caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
BLOCK_COMMENT("Entry:");
}
@@ -1957,7 +2241,7 @@ address StubGenerator::generate_disjoint_int_oop_copy(StubId stub_id, address* e
{
// UnsafeMemoryAccess page error: continue after unsafe access
- UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true);
+ UnsafeMemoryAccessMark umam(this, add_handlers, true);
// 'from', 'to' and 'count' are now valid
__ movptr(dword_count, count);
__ shrptr(count, 1); // count => qword_count
@@ -1969,20 +2253,20 @@ address StubGenerator::generate_disjoint_int_oop_copy(StubId stub_id, address* e
__ jmp(L_copy_bytes);
// Copy trailing qwords
- __ BIND(L_copy_8_bytes);
+ __ BIND(L_copy_8_bytes);
__ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
__ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
__ increment(qword_count);
__ jcc(Assembler::notZero, L_copy_8_bytes);
// Check for and copy trailing dword
- __ BIND(L_copy_4_bytes);
+ __ BIND(L_copy_4_bytes);
__ testl(dword_count, 1); // Only byte test since the value is 0 or 1
__ jccb(Assembler::zero, L_exit);
__ movl(rax, Address(end_from, 8));
__ movl(Address(end_to, 8), rax);
}
-__ BIND(L_exit);
+ __ BIND(L_exit);
address ucme_exit_pc = __ pc();
bs->arraycopy_epilogue(_masm, decorators, type, from, to, dword_count);
restore_arg_regs_using_thread();
@@ -1993,12 +2277,30 @@ __ BIND(L_exit);
__ ret(0);
{
- UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, false, ucme_exit_pc);
+ UnsafeMemoryAccessMark umam(this, add_handlers, false, ucme_exit_pc);
// Copy in multi-bytes chunks
copy_bytes_forward(end_from, end_to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, is_oop ? T_OBJECT : T_INT);
__ jmp(L_copy_4_bytes);
}
+ // retrieve the registered handler addresses
+ address end = __ pc();
+ if (add_handlers) {
+ retrieve_unsafe_access_handlers(start, end, extras);
+ }
+ assert(extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+#if INCLUDE_ZGC
+ // retrieve addresses at which ZGC does colour patching
+ if (add_relocs) {
+ retrieve_reloc_addresses(start, end, extras);
+ }
+#endif // INCLUDE_ZGC
+
+ // record the stub entry and end plus the no_push entry and any
+ // extra handler addresses
+ store_archive_data(stub_id, start, end, entries_ptr, extras_ptr);
+
return start;
}
@@ -2047,10 +2349,42 @@ address StubGenerator::generate_conjoint_int_oop_copy(StubId stub_id, address no
return generate_conjoint_copy_avx3_masked(stub_id, entry, nooverlap_target);
}
#endif
+ bool add_handlers = !is_oop && !aligned;
+ bool add_relocs = UseZGC && is_oop;
+ bool add_extras = add_handlers || add_relocs;
+ GrowableArray entries;
+ GrowableArray extras;
+ int expected_entry_count = (entry != nullptr ? 2 : 1);
+ int expected_handler_count = (add_handlers ? 2 : 0) * UnsafeMemoryAccess::COLUMN_COUNT; // 0/2 x UMAM {start,end,handler}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == expected_entry_count, "sanity check");
+ GrowableArray* entries_ptr = (entry_count == 1 ? nullptr : &entries);
+ GrowableArray* extras_ptr = (add_extras ? &extras : nullptr);
+ address start = load_archive_data(stub_id, entries_ptr, extras_ptr);
+ if (start != nullptr) {
+ assert(entries.length() == expected_entry_count - 1,
+ "unexpected entry count %d", entries.length());
+ assert(!add_handlers || extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+ if (entry != nullptr) {
+ *entry = entries.at(0);
+ }
+ if (add_handlers) {
+ // restore 2 UMAM {start,end,handler} addresses from extras
+ register_unsafe_access_handlers(extras, 0, 2);
+ }
+#if INCLUDE_ZGC
+ // register addresses at which ZGC does colour patching
+ if (add_relocs) {
+ register_reloc_addresses(extras, 6, extras.length());
+ }
+#endif // INCLUDE_ZGC
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_copy_bytes, L_copy_8_bytes, L_exit;
const Register from = rdi; // source array address
@@ -2064,7 +2398,8 @@ address StubGenerator::generate_conjoint_int_oop_copy(StubId stub_id, address no
if (entry != nullptr) {
*entry = __ pc();
- // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
+ entries.append(*entry);
+ // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
BLOCK_COMMENT("Entry:");
}
@@ -2087,7 +2422,7 @@ address StubGenerator::generate_conjoint_int_oop_copy(StubId stub_id, address no
assert_clean_int(count, rax); // Make sure 'count' is clean int.
{
// UnsafeMemoryAccess page error: continue after unsafe access
- UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true);
+ UnsafeMemoryAccessMark umam(this, add_handlers, true);
// 'from', 'to' and 'count' are now valid
__ movptr(dword_count, count);
__ shrptr(count, 1); // count => qword_count
@@ -2102,7 +2437,7 @@ address StubGenerator::generate_conjoint_int_oop_copy(StubId stub_id, address no
__ jmp(L_copy_bytes);
// Copy trailing qwords
- __ BIND(L_copy_8_bytes);
+ __ BIND(L_copy_8_bytes);
__ movq(rax, Address(from, qword_count, Address::times_8, -8));
__ movq(Address(to, qword_count, Address::times_8, -8), rax);
__ decrement(qword_count);
@@ -2120,12 +2455,12 @@ address StubGenerator::generate_conjoint_int_oop_copy(StubId stub_id, address no
{
// UnsafeMemoryAccess page error: continue after unsafe access
- UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true);
+ UnsafeMemoryAccessMark umam(this, add_handlers, true);
// Copy in multi-bytes chunks
copy_bytes_backward(from, to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, is_oop ? T_OBJECT : T_INT);
}
-__ BIND(L_exit);
+ __ BIND(L_exit);
bs->arraycopy_epilogue(_masm, decorators, type, from, to, dword_count);
restore_arg_regs_using_thread();
INC_COUNTER_NP(SharedRuntime::_jint_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free
@@ -2134,6 +2469,23 @@ __ BIND(L_exit);
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // retrieve the registered handler addresses
+ address end = __ pc();
+ if (add_handlers) {
+ retrieve_unsafe_access_handlers(start, end, extras);
+ }
+ assert(extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+#if INCLUDE_ZGC
+ // retrieve addresses at which ZGC does colour patching
+ if (add_relocs) {
+ retrieve_reloc_addresses(start, end, extras);
+ }
+#endif // INCLUDE_ZGC
+ // record the stub entry and end plus the no_push entry and any
+ // extra handler addresses
+ store_archive_data(stub_id, start, end, entries_ptr, extras_ptr);
+
return start;
}
@@ -2180,10 +2532,42 @@ address StubGenerator::generate_disjoint_long_oop_copy(StubId stub_id, address *
return generate_disjoint_copy_avx3_masked(stub_id, entry);
}
#endif
+ bool add_handlers = !is_oop && !aligned;
+ bool add_relocs = UseZGC && is_oop;
+ bool add_extras = add_handlers || add_relocs;
+ GrowableArray entries;
+ GrowableArray extras;
+ int expected_entry_count = (entry != nullptr ? 2 : 1);
+ int expected_handler_count = (add_handlers ? 2 : 0) * UnsafeMemoryAccess::COLUMN_COUNT; // 0/2 x UMAM {start,end,handler}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == expected_entry_count, "sanity check");
+ GrowableArray* entries_ptr = (entry_count == 1 ? nullptr : &entries);
+ GrowableArray* extras_ptr = (add_extras ? &extras : nullptr);
+ address start = load_archive_data(stub_id, entries_ptr, extras_ptr);
+ if (start != nullptr) {
+ assert(entries.length() == expected_entry_count - 1,
+ "unexpected entry count %d", entries.length());
+ assert(!add_handlers || extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+ if (entry != nullptr) {
+ *entry = entries.at(0);
+ }
+ if (add_handlers) {
+ // restore 2 UMAM {start,end,handler} addresses from extras
+ register_unsafe_access_handlers(extras, 0, 2);
+ }
+#if INCLUDE_ZGC
+ // register addresses at which ZGC does colour patching
+ if (add_relocs) {
+ register_reloc_addresses(extras, 0, extras.length());
+ }
+#endif // INCLUDE_ZGC
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_copy_bytes, L_copy_8_bytes, L_exit;
const Register from = rdi; // source array address
@@ -2201,6 +2585,7 @@ address StubGenerator::generate_disjoint_long_oop_copy(StubId stub_id, address *
if (entry != nullptr) {
*entry = __ pc();
+ entries.append(*entry);
// caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
BLOCK_COMMENT("Entry:");
}
@@ -2221,7 +2606,7 @@ address StubGenerator::generate_disjoint_long_oop_copy(StubId stub_id, address *
bs->arraycopy_prologue(_masm, decorators, type, from, to, qword_count);
{
// UnsafeMemoryAccess page error: continue after unsafe access
- UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true);
+ UnsafeMemoryAccessMark umam(this, add_handlers, true);
// Copy from low to high addresses. Use 'to' as scratch.
__ lea(end_from, Address(from, qword_count, Address::times_8, -8));
@@ -2253,7 +2638,7 @@ address StubGenerator::generate_disjoint_long_oop_copy(StubId stub_id, address *
{
// UnsafeMemoryAccess page error: continue after unsafe access
- UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true);
+ UnsafeMemoryAccessMark umam(this, add_handlers, true);
// Copy in multi-bytes chunks
copy_bytes_forward(end_from, end_to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, is_oop ? T_OBJECT : T_LONG);
}
@@ -2269,6 +2654,23 @@ address StubGenerator::generate_disjoint_long_oop_copy(StubId stub_id, address *
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // retrieve the registered handler addresses
+ address end = __ pc();
+ if (add_handlers) {
+ retrieve_unsafe_access_handlers(start, end, extras);
+ }
+ assert(extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+#if INCLUDE_ZGC
+ // retrieve addresses at which ZGC does colour patching
+ if (add_relocs) {
+ retrieve_reloc_addresses(start, end, extras);
+ }
+#endif // INCLUDE_ZGC
+ // record the stub entry and end plus the no_push entry and any
+ // extra handler addresses
+ store_archive_data(stub_id, start, end, entries_ptr, extras_ptr);
+
return start;
}
@@ -2313,10 +2715,42 @@ address StubGenerator::generate_conjoint_long_oop_copy(StubId stub_id, address n
return generate_conjoint_copy_avx3_masked(stub_id, entry, nooverlap_target);
}
#endif
+ bool add_handlers = !is_oop && !aligned;
+ bool add_relocs = UseZGC && is_oop;
+ bool add_extras = add_handlers || add_relocs;
+ GrowableArray entries;
+ GrowableArray extras;
+ int expected_entry_count = (entry != nullptr ? 2 : 1);
+ int expected_handler_count = (add_handlers ? 2 : 0) * UnsafeMemoryAccess::COLUMN_COUNT; // 0/2 x UMAM {start,end,handler}
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == expected_entry_count, "sanity check");
+ GrowableArray* entries_ptr = (entry_count == 1 ? nullptr : &entries);
+ GrowableArray* extras_ptr = (add_extras ? &extras : nullptr);
+ address start = load_archive_data(stub_id, entries_ptr, extras_ptr);
+ if (start != nullptr) {
+ assert(entries.length() == expected_entry_count - 1,
+ "unexpected entry count %d", entries.length());
+ assert(!add_handlers || extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+ if (entry != nullptr) {
+ *entry = entries.at(0);
+ }
+ if (add_handlers) {
+ // restore 2 UMAM {start,end,handler} addresses from extras
+ register_unsafe_access_handlers(extras, 0, 2);
+ }
+#if INCLUDE_ZGC
+ // register addresses at which ZGC does colour patching
+ if (add_relocs) {
+ register_reloc_addresses(extras, 0, extras.length());
+ }
+#endif // INCLUDE_ZGC
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_copy_bytes, L_copy_8_bytes, L_exit;
const Register from = rdi; // source array address
@@ -2329,6 +2763,7 @@ address StubGenerator::generate_conjoint_long_oop_copy(StubId stub_id, address n
if (entry != nullptr) {
*entry = __ pc();
+ entries.append(*entry);
// caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
BLOCK_COMMENT("Entry:");
}
@@ -2350,7 +2785,7 @@ address StubGenerator::generate_conjoint_long_oop_copy(StubId stub_id, address n
bs->arraycopy_prologue(_masm, decorators, type, from, to, qword_count);
{
// UnsafeMemoryAccess page error: continue after unsafe access
- UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true);
+ UnsafeMemoryAccessMark umam(this, add_handlers, true);
__ jmp(L_copy_bytes);
@@ -2377,7 +2812,7 @@ address StubGenerator::generate_conjoint_long_oop_copy(StubId stub_id, address n
}
{
// UnsafeMemoryAccess page error: continue after unsafe access
- UnsafeMemoryAccessMark umam(this, !is_oop && !aligned, true);
+ UnsafeMemoryAccessMark umam(this, add_handlers, true);
// Copy in multi-bytes chunks
copy_bytes_backward(from, to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, is_oop ? T_OBJECT : T_LONG);
@@ -2393,6 +2828,24 @@ address StubGenerator::generate_conjoint_long_oop_copy(StubId stub_id, address n
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+
+ // retrieve the registered handler addresses
+ address end = __ pc();
+ if (add_handlers) {
+ retrieve_unsafe_access_handlers(start, end, extras);
+ }
+ assert(extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+#if INCLUDE_ZGC
+ // retrieve addresses at which ZGC does colour patching
+ if ((UseZGC && is_oop)) {
+ retrieve_reloc_addresses(start, end, extras);
+ }
+#endif // INCLUDE_ZGC
+ // record the stub entry and end plus the no_push entry and any
+ // extra handler addresses
+ store_archive_data(stub_id, start, end, entries_ptr, extras_ptr);
+
return start;
}
@@ -2448,6 +2901,28 @@ address StubGenerator::generate_checkcast_copy(StubId stub_id, address *entry) {
ShouldNotReachHere();
}
+ GrowableArray entries;
+ GrowableArray extras;
+ int expected_entry_count = (entry != nullptr ? 2 : 1);
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == expected_entry_count, "sanity check");
+ GrowableArray* entries_ptr = (entry_count == 1 ? nullptr : &entries);
+ GrowableArray* extras_ptr = (UseZGC ? &extras : nullptr);
+ address start = load_archive_data(stub_id, entries_ptr, extras_ptr);
+ if (start != nullptr) {
+ assert(entries.length() == expected_entry_count - 1,
+ "unexpected addresses count %d", entries.length());
+ if (entry != nullptr) {
+ *entry = entries.at(0);
+ }
+#if INCLUDE_ZGC
+ if (UseZGC) {
+ register_reloc_addresses(extras, 0, extras.length());
+ }
+#endif // INCLUDE_ZGC
+ return start;
+ }
+
Label L_load_element, L_store_element, L_do_card_marks, L_done;
// Input registers (after setup_arg_regs)
@@ -2477,7 +2952,7 @@ address StubGenerator::generate_checkcast_copy(StubId stub_id, address *entry) {
__ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter(); // required for proper stackwalking of RuntimeStub frame
@@ -2502,6 +2977,7 @@ address StubGenerator::generate_checkcast_copy(StubId stub_id, address *entry) {
// Caller of this entry point must set up the argument registers.
if (entry != nullptr) {
*entry = __ pc();
+ entries.append(*entry);
BLOCK_COMMENT("Entry:");
}
@@ -2636,6 +3112,16 @@ address StubGenerator::generate_checkcast_copy(StubId stub_id, address *entry) {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ address end = __ pc();
+#if INCLUDE_ZGC
+ // retrieve addresses at which ZGC does colour patching
+ if (UseZGC) {
+ retrieve_reloc_addresses(start, end, extras);
+ }
+#endif // INCLUDE_ZGC
+ // record the stub entry and end plus the no_push entry
+ store_archive_data(stub_id, start, end, entries_ptr, extras_ptr);
+
return start;
}
@@ -2655,6 +3141,14 @@ address StubGenerator::generate_checkcast_copy(StubId stub_id, address *entry) {
address StubGenerator::generate_unsafe_copy(address byte_copy_entry, address short_copy_entry,
address int_copy_entry, address long_copy_entry) {
+ StubId stub_id = StubId::stubgen_unsafe_arraycopy_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+
Label L_long_aligned, L_int_aligned, L_short_aligned;
// Input registers (before setup_arg_regs)
@@ -2666,9 +3160,8 @@ address StubGenerator::generate_unsafe_copy(address byte_copy_entry, address sho
const Register bits = rax; // test copy of low bits
__ align(CodeEntryAlignment);
- StubId stub_id = StubId::stubgen_unsafe_arraycopy_id;
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter(); // required for proper stackwalking of RuntimeStub frame
@@ -2700,6 +3193,9 @@ address StubGenerator::generate_unsafe_copy(address byte_copy_entry, address sho
__ shrptr(size, LogBytesPerLong); // size => qword_count
__ jump(RuntimeAddress(long_copy_entry));
+ // record the stub entry and end plus
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -2801,10 +3297,23 @@ static void do_setmemory_atomic_loop(USM_TYPE type, Register dest,
// to an int, short, or byte fill loop.
//
address StubGenerator::generate_unsafe_setmemory(address unsafe_byte_fill) {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_unsafe_setmemory_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ // we expect three set of extra unsafememory access handler entries
+ GrowableArray extras;
+ int expected_handler_count = 3 * UnsafeMemoryAccess::COLUMN_COUNT;
+ address start = load_archive_data(stub_id, nullptr, &extras);
+ if (start != nullptr) {
+ assert(extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+ register_unsafe_access_handlers(extras, 0, 3);
+ return start;
+ }
+
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter(); // required for proper stackwalking of RuntimeStub frame
assert(unsafe_byte_fill != nullptr, "Invalid call");
@@ -2894,6 +3403,16 @@ address StubGenerator::generate_unsafe_setmemory(address unsafe_byte_fill) {
__ jump(RuntimeAddress(unsafe_byte_fill));
}
+ // retrieve the registered handler addresses
+ address end = __ pc();
+ retrieve_unsafe_access_handlers(start, end, extras);
+ assert(extras.length() == expected_handler_count,
+ "unexpected handler addresses count %d", extras.length());
+
+ // record the stub entry and end plus the no_push entry and any
+ // extra handler addresses
+ store_archive_data(stub_id, start, end, nullptr, &extras);
+
return start;
}
@@ -2950,7 +3469,15 @@ address StubGenerator::generate_generic_copy(address byte_copy_entry, address sh
address int_copy_entry, address oop_copy_entry,
address long_copy_entry, address checkcast_copy_entry) {
- Label L_failed, L_failed_0, L_objArray;
+ StubId stub_id = StubId::stubgen_generic_arraycopy_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+
+ Label L_failed, L_failed_0, L_skip_failed_0, L_objArray;
Label L_copy_shorts, L_copy_ints, L_copy_longs;
// Input registers
@@ -2966,22 +3493,9 @@ address StubGenerator::generate_generic_copy(address byte_copy_entry, address sh
const Register rklass_tmp = rdi; // load_klass
#endif
- { int modulus = CodeEntryAlignment;
- int target = modulus - 5; // 5 = sizeof jmp(L_failed)
- int advance = target - (__ offset() % modulus);
- if (advance < 0) advance += modulus;
- if (advance > 0) __ nop(advance);
- }
- StubId stub_id = StubId::stubgen_generic_arraycopy_id;
StubCodeMark mark(this, stub_id);
-
- // Short-hop target to L_failed. Makes for denser prologue code.
- __ BIND(L_failed_0);
- __ jmp(L_failed);
- assert(__ offset() % CodeEntryAlignment == 0, "no further alignment needed");
-
__ align(CodeEntryAlignment);
- address start = __ pc();
+ start = __ pc();
__ enter(); // required for proper stackwalking of RuntimeStub frame
@@ -3022,7 +3536,8 @@ address StubGenerator::generate_generic_copy(address byte_copy_entry, address sh
// if (dst_pos < 0) return -1;
__ testl(dst_pos, dst_pos); // dst_pos (32-bits)
size_t j4off = __ offset();
- __ jccb(Assembler::negative, L_failed_0);
+ // skip over the failure trampoline
+ __ jccb(Assembler::positive, L_skip_failed_0);
// The first four tests are very dense code,
// but not quite dense enough to put four
@@ -3032,6 +3547,13 @@ address StubGenerator::generate_generic_copy(address byte_copy_entry, address sh
// Make sure of this.
guarantee(((j1off ^ j4off) & ~15) != 0, "I$ line of 1st & 4th jumps");
+ // Short-hop target to L_failed. Makes for denser prologue code.
+ __ BIND(L_failed_0);
+ __ jmp(L_failed);
+
+ // continue here if first 4 checks pass
+ __ bind(L_skip_failed_0);
+
// registers used as temp
const Register r11_length = r11; // elements count to copy
const Register r10_src_klass = r10; // array klass
@@ -3254,6 +3776,9 @@ __ BIND(L_failed);
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_cbrt.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_cbrt.cpp
index 73330dedc0f..4c647b7d9dc 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_cbrt.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_cbrt.cpp
@@ -191,8 +191,14 @@ ATTRIBUTE_ALIGNED(4) static const juint _D_table[] =
address StubGenerator::generate_libmCbrt() {
StubId stub_id = StubId::stubgen_dcbrt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_2TAG_PACKET_0_0_1, L_2TAG_PACKET_1_0_1, L_2TAG_PACKET_2_0_1;
Label B1_1, B1_2, B1_4;
@@ -335,7 +341,34 @@ address StubGenerator::generate_libmCbrt() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
#undef __
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_cbrt(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr))
+ ADD(_ABS_MASK);
+ ADD(_SIG_MASK);
+ ADD(_EXP_MASK);
+ ADD(_EXP_MSK2);
+ ADD(_EXP_MSK3);
+ ADD(_SCALE63);
+ ADD(_ZERON);
+ ADD(_INF);
+ ADD(_NEG_INF);
+ address coeff_table = (address)_coeff_table;
+ ADD(coeff_table);
+ ADD(coeff_table + 16);
+ ADD(coeff_table + 32);
+ ADD(coeff_table + 48);
+ ADD(_rcp_table);
+ ADD(_cbrt_table);
+ ADD(_D_table);
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_chacha.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_chacha.cpp
index 7afaf34e031..1fa51cd2f18 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_chacha.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_chacha.cpp
@@ -111,10 +111,16 @@ void StubGenerator::generate_chacha_stubs() {
/* The 2-block AVX/AVX2-enabled ChaCha20 block function implementation */
address StubGenerator::generate_chacha20Block_avx() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_chacha20Block_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_twoRounds;
const Register state = c_rarg0;
@@ -295,15 +301,25 @@ address StubGenerator::generate_chacha20Block_avx() {
}
__ leave();
__ ret(0);
+
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
/* The 4-block AVX512-enabled ChaCha20 block function implementation */
address StubGenerator::generate_chacha20Block_avx512() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_chacha20Block_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_twoRounds;
const Register state = c_rarg0;
@@ -466,6 +482,10 @@ address StubGenerator::generate_chacha20Block_avx512() {
__ vzeroupper();
__ leave();
__ ret(0);
+
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -584,3 +604,13 @@ bVec,
}
#undef __
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_chacha(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr))
+ ADD(CC20_COUNTER_ADD_AVX);
+ ADD(CC20_COUNTER_ADD_AVX512);
+ ADD(CC20_LROT_CONSTS);
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_constants.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_constants.cpp
index 93fa7e650db..19e1ca680b3 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_constants.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_constants.cpp
@@ -233,3 +233,30 @@ ATTRIBUTE_ALIGNED(16) static const juint _Ctable[] = {
};
address StubGenerator::Ctable = (address)_Ctable;
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_constants(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr))
+ ADD(_ONE);
+ ADD(_ONEHALF);
+ ADD(_SIGN_MASK);
+ ADD(_TWO_POW_55);
+ ADD(_TWO_POW_M55);
+ ADD(_SHIFTER);
+ ADD(_ZERO);
+ ADD(_SC_1);
+ ADD(_SC_2);
+ ADD(_SC_3);
+ ADD(_SC_4);
+ // Use value which was already cast to (address): StubGenerator::PI_4;
+ ADD(PI_4);
+ ADD(PI_4 + 8);
+ ADD(_PI32INV);
+ ADD(_NEG_ZERO);
+ ADD(_P_1);
+ ADD(_P_2);
+ ADD(_P_3);
+ ADD(_PI_INV_TABLE);
+ ADD(_Ctable);
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_cos.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_cos.cpp
index 8cb6ead21fd..8dedd50cd97 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_cos.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_cos.cpp
@@ -174,8 +174,14 @@
address StubGenerator::generate_libmCos() {
StubId stub_id = StubId::stubgen_dcos_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_2TAG_PACKET_0_0_1, L_2TAG_PACKET_1_0_1, L_2TAG_PACKET_2_0_1, L_2TAG_PACKET_3_0_1;
Label L_2TAG_PACKET_4_0_1, L_2TAG_PACKET_5_0_1, L_2TAG_PACKET_6_0_1, L_2TAG_PACKET_7_0_1;
@@ -619,6 +625,9 @@ address StubGenerator::generate_libmCos() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_dilithium.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_dilithium.cpp
index b9590939468..de8f52a3c05 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_dilithium.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_dilithium.cpp
@@ -401,10 +401,16 @@ static void storeXmms(Register destination, int offset, const XMMRegister xmmReg
//
static address generate_dilithiumAlmostNtt_avx(StubGenerator *stubgen,
int vector_len, MacroAssembler *_masm) {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_dilithiumAlmostNtt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = stubgen->load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(stubgen, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register coeffs = c_rarg0;
@@ -646,6 +652,9 @@ static address generate_dilithiumAlmostNtt_avx(StubGenerator *stubgen,
__ mov64(rax, 0); // return 0
__ ret(0);
+ // record the stub entry and end
+ stubgen->store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -657,10 +666,16 @@ static address generate_dilithiumAlmostNtt_avx(StubGenerator *stubgen,
// zetas (int[128*8]) = c_rarg1
static address generate_dilithiumAlmostInverseNtt_avx(StubGenerator *stubgen,
int vector_len, MacroAssembler *_masm) {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_dilithiumAlmostInverseNtt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = stubgen->load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(stubgen, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register coeffs = c_rarg0;
@@ -886,6 +901,9 @@ static address generate_dilithiumAlmostInverseNtt_avx(StubGenerator *stubgen,
__ mov64(rax, 0); // return 0
__ ret(0);
+ // record the stub entry and end
+ stubgen->store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -900,10 +918,16 @@ static address generate_dilithiumAlmostInverseNtt_avx(StubGenerator *stubgen,
static address generate_dilithiumNttMult_avx(StubGenerator *stubgen,
int vector_len, MacroAssembler *_masm) {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_dilithiumNttMult_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = stubgen->load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(stubgen, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
Label L_loop;
@@ -972,6 +996,9 @@ static address generate_dilithiumNttMult_avx(StubGenerator *stubgen,
__ mov64(rax, 0); // return 0
__ ret(0);
+ // record the stub entry and end
+ stubgen->store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -984,10 +1011,16 @@ static address generate_dilithiumNttMult_avx(StubGenerator *stubgen,
static address generate_dilithiumMontMulByConstant_avx(StubGenerator *stubgen,
int vector_len, MacroAssembler *_masm) {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_dilithiumMontMulByConstant_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = stubgen->load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(stubgen, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
Label L_loop;
@@ -1059,6 +1092,9 @@ static address generate_dilithiumMontMulByConstant_avx(StubGenerator *stubgen,
__ mov64(rax, 0); // return 0
__ ret(0);
+ // record the stub entry and end
+ stubgen->store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -1073,10 +1109,16 @@ static address generate_dilithiumMontMulByConstant_avx(StubGenerator *stubgen,
// multiplier (int) = c_rarg4
static address generate_dilithiumDecomposePoly_avx(StubGenerator *stubgen,
int vector_len, MacroAssembler *_masm) {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_dilithiumDecomposePoly_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = stubgen->load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(stubgen, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
Label L_loop;
@@ -1318,6 +1360,9 @@ static address generate_dilithiumDecomposePoly_avx(StubGenerator *stubgen,
__ mov64(rax, 0); // return 0
__ ret(0);
+ // record the stub entry and end
+ stubgen->store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -1340,3 +1385,21 @@ void StubGenerator::generate_dilithium_stubs() {
generate_dilithiumDecomposePoly_avx(this, vector_len, _masm);
}
}
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_dilithium(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr))
+ // use accessors to correctly identify the relevant addresses
+ ADD(unshufflePermsAddr(0));
+ ADD(unshufflePermsAddr(1));
+ ADD(unshufflePermsAddr(2));
+ ADD(unshufflePermsAddr(3));
+ ADD(unshufflePermsAddr(4));
+ ADD(unshufflePermsAddr(5));
+ ADD(dilithiumAvx512ConstsAddr(montQInvModRIdx));
+ ADD(dilithiumAvx512ConstsAddr(dilithium_qIdx));
+ ADD(dilithiumAvx512ConstsAddr(montRSquareModQIdx));
+ ADD(dilithiumAvx512ConstsAddr(barrettAddendIdx));
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_exp.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_exp.cpp
index 5130fd2c9d2..3c8babcbecf 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_exp.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_exp.cpp
@@ -166,8 +166,14 @@ ATTRIBUTE_ALIGNED(4) static const juint _INF[] =
address StubGenerator::generate_libmExp() {
StubId stub_id = StubId::stubgen_dexp_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_2TAG_PACKET_0_0_2, L_2TAG_PACKET_1_0_2, L_2TAG_PACKET_2_0_2, L_2TAG_PACKET_3_0_2;
Label L_2TAG_PACKET_4_0_2, L_2TAG_PACKET_5_0_2, L_2TAG_PACKET_6_0_2, L_2TAG_PACKET_7_0_2;
@@ -381,7 +387,32 @@ address StubGenerator::generate_libmExp() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
#undef __
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_exp(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr));
+ address cv = (address)_cv;
+ ADD(cv);
+ ADD(cv + 16);
+ ADD(cv + 32);
+ ADD(cv + 48);
+ ADD(cv + 64);
+ ADD(cv + 80);
+ ADD(_mmask);
+ ADD(_bias);
+ ADD(_Tbl_addr);
+ ADD(_ALLONES);
+ ADD(_ebias);
+ ADD(_XMAX);
+ ADD(_XMIN);
+ ADD(_INF);
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_fmod.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_fmod.cpp
index b1eaa4b8031..f53985a13b7 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_fmod.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_fmod.cpp
@@ -72,13 +72,19 @@ ATTRIBUTE_ALIGNED(32) static const uint64_t CONST_e307[] = {
};
address StubGenerator::generate_libmFmod() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_fmod_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter(); // required for proper stackwalking of RuntimeStub frame
- if (VM_Version::supports_avx512vlbwdq()) { // AVX512 version
+ if (VM_Version::supports_avx512vlbwdq() && VM_Version::supports_fma()) { // AVX512 version
// Source used to generate the AVX512 fmod assembly below:
//
@@ -521,7 +527,22 @@ address StubGenerator::generate_libmFmod() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
#undef __
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_fmod(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr));
+ ADD(CONST_NaN);
+ ADD(CONST_1p260);
+ ADD(CONST_MAX);
+ ADD(CONST_INF);
+ ADD(CONST_e307);
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_ghash.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_ghash.cpp
index 6f05b1ab5e6..9ebab07589e 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_ghash.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_ghash.cpp
@@ -80,11 +80,17 @@ void StubGenerator::generate_ghash_stubs() {
// Single and multi-block ghash operations.
address StubGenerator::generate_ghash_processBlocks() {
- __ align(CodeEntryAlignment);
- Label L_ghash_loop, L_exit;
StubId stub_id = StubId::stubgen_ghash_processBlocks_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ Label L_ghash_loop, L_exit;
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register state = c_rarg0;
const Register subkeyH = c_rarg1;
@@ -211,17 +217,25 @@ address StubGenerator::generate_ghash_processBlocks() {
__ leave();
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
// Ghash single and multi block operations using AVX instructions
address StubGenerator::generate_avx_ghash_processBlocks() {
- __ align(CodeEntryAlignment);
-
StubId stub_id = StubId::stubgen_ghash_processBlocks_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
// arguments
const Register state = c_rarg0;
@@ -237,6 +251,9 @@ address StubGenerator::generate_avx_ghash_processBlocks() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -538,3 +555,14 @@ void StubGenerator::generateHtbl_eight_blocks(Register htbl) {
}
#undef __
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_ghash(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr));
+ ADD(GHASH_SHUFFLE_MASK);
+ ADD(GHASH_LONG_SWAP_MASK);
+ ADD(GHASH_BYTE_SWAP_MASK);
+ ADD(GHASH_POLYNOMIAL);
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_kyber.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_kyber.cpp
index 7d5dee6a5df..347a9b936a8 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_kyber.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_kyber.cpp
@@ -400,10 +400,16 @@ static int xmm29_29[] = {29, 29, 29, 29};
// ntt_zetas (short[256]) = c_rarg1
address generate_kyberNtt_avx512(StubGenerator *stubgen,
MacroAssembler *_masm) {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_kyberNtt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = stubgen->load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(stubgen, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register coeffs = c_rarg0;
@@ -487,6 +493,9 @@ address generate_kyberNtt_avx512(StubGenerator *stubgen,
__ mov64(rax, 0); // return 0
__ ret(0);
+ // record the stub entry and end
+ stubgen->store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -496,11 +505,16 @@ address generate_kyberNtt_avx512(StubGenerator *stubgen,
// ntt_zetas (short[256]) = c_rarg1
address generate_kyberInverseNtt_avx512(StubGenerator *stubgen,
MacroAssembler *_masm) {
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_kyberInverseNtt_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = stubgen->load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(stubgen, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register coeffs = c_rarg0;
@@ -610,6 +624,9 @@ address generate_kyberInverseNtt_avx512(StubGenerator *stubgen,
__ mov64(rax, 0); // return 0
__ ret(0);
+ // record the stub entry and end
+ stubgen->store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -621,11 +638,16 @@ address generate_kyberInverseNtt_avx512(StubGenerator *stubgen,
// zetas (short[128]) = c_rarg3
address generate_kyberNttMult_avx512(StubGenerator *stubgen,
MacroAssembler *_masm) {
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_kyberNttMult_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = stubgen->load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(stubgen, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register result = c_rarg0;
@@ -731,6 +753,9 @@ address generate_kyberNttMult_avx512(StubGenerator *stubgen,
__ mov64(rax, 0); // return 0
__ ret(0);
+ // record the stub entry and end
+ stubgen->store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -741,11 +766,16 @@ address generate_kyberNttMult_avx512(StubGenerator *stubgen,
// b (short[256]) = c_rarg2
address generate_kyberAddPoly_2_avx512(StubGenerator *stubgen,
MacroAssembler *_masm) {
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_kyberAddPoly_2_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = stubgen->load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(stubgen, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register result = c_rarg0;
@@ -776,6 +806,9 @@ address generate_kyberAddPoly_2_avx512(StubGenerator *stubgen,
__ mov64(rax, 0); // return 0
__ ret(0);
+ // record the stub entry and end
+ stubgen->store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -787,11 +820,16 @@ address generate_kyberAddPoly_2_avx512(StubGenerator *stubgen,
// c (short[256]) = c_rarg3
address generate_kyberAddPoly_3_avx512(StubGenerator *stubgen,
MacroAssembler *_masm) {
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_kyberAddPoly_3_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = stubgen->load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(stubgen, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register result = c_rarg0;
@@ -830,6 +868,9 @@ address generate_kyberAddPoly_3_avx512(StubGenerator *stubgen,
__ mov64(rax, 0); // return 0
__ ret(0);
+ // record the stub entry and end
+ stubgen->store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -841,11 +882,16 @@ address generate_kyberAddPoly_3_avx512(StubGenerator *stubgen,
// parsedLength (int) = c_rarg3
address generate_kyber12To16_avx512(StubGenerator *stubgen,
MacroAssembler *_masm) {
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_kyber12To16_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = stubgen->load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(stubgen, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register condensed = c_rarg0;
@@ -984,6 +1030,9 @@ address generate_kyber12To16_avx512(StubGenerator *stubgen,
__ mov64(rax, 0); // return 0
__ ret(0);
+ // record the stub entry and end
+ stubgen->store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -993,11 +1042,16 @@ address generate_kyber12To16_avx512(StubGenerator *stubgen,
// coeffs (short[256]) = c_rarg0
address generate_kyberBarrettReduce_avx512(StubGenerator *stubgen,
MacroAssembler *_masm) {
-
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_kyberBarrettReduce_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = stubgen->load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(stubgen, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
const Register coeffs = c_rarg0;
@@ -1021,6 +1075,9 @@ address generate_kyberBarrettReduce_avx512(StubGenerator *stubgen,
__ mov64(rax, 0); // return 0
__ ret(0);
+ // record the stub entry and end
+ stubgen->store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -1038,3 +1095,24 @@ void StubGenerator::generate_kyber_stubs() {
}
}
}
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_kyber(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr))
+ // use accessors to correctly identify the relevant addresses
+ ADD(kyberAvx512NttPermsAddr());
+ ADD(kyberAvx512InverseNttPermsAddr());
+ ADD(kyberAvx512_nttMultPermsAddr());
+ ADD(kyberAvx512_12To16PermsAddr());
+ ADD(kyberAvx512_12To16DupAddr());
+ ADD(kyberAvx512_12To16ShiftAddr());
+ ADD(kyberAvx512_12To16AndAddr());
+ ADD(kyberAvx512ConstsAddr(qOffset));
+ ADD(kyberAvx512ConstsAddr(qInvModROffset));
+ ADD(kyberAvx512ConstsAddr(dimHalfInverseOffset));
+ ADD(kyberAvx512ConstsAddr(barretMultiplierOffset));
+ ADD(kyberAvx512ConstsAddr(montRSquareModqOffset));
+ ADD(kyberAvx512ConstsAddr(f00Offset));
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_log.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_log.cpp
index 6b5b4d704e3..07683a51e3d 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_log.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_log.cpp
@@ -177,8 +177,14 @@ ATTRIBUTE_ALIGNED(16) static const juint _coeff[] =
address StubGenerator::generate_libmLog() {
StubId stub_id = StubId::stubgen_dlog_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_2TAG_PACKET_0_0_2, L_2TAG_PACKET_1_0_2, L_2TAG_PACKET_2_0_2, L_2TAG_PACKET_3_0_2;
Label L_2TAG_PACKET_4_0_2, L_2TAG_PACKET_5_0_2, L_2TAG_PACKET_6_0_2, L_2TAG_PACKET_7_0_2;
@@ -359,6 +365,9 @@ address StubGenerator::generate_libmLog() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -516,8 +525,14 @@ ATTRIBUTE_ALIGNED(16) static const juint _coeff_log10[] =
address StubGenerator::generate_libmLog10() {
StubId stub_id = StubId::stubgen_dlog10_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_2TAG_PACKET_0_0_2, L_2TAG_PACKET_1_0_2, L_2TAG_PACKET_2_0_2, L_2TAG_PACKET_3_0_2;
Label L_2TAG_PACKET_4_0_2, L_2TAG_PACKET_5_0_2, L_2TAG_PACKET_6_0_2, L_2TAG_PACKET_7_0_2;
@@ -704,7 +719,38 @@ address StubGenerator::generate_libmLog10() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
#undef __
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_log(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr));
+ address log2 = (address)_log2;
+ address coeff = (address)_coeff;
+ address LOG10_E = (address)_LOG10_E;
+ address log2_log10 = (address)_log2_log10;
+ address coeff_log10 = (address)_coeff_log10;
+
+ ADD(_L_tbl);
+ ADD(log2);
+ ADD(log2 + 8);
+ ADD(coeff);
+ ADD(coeff + 16);
+ ADD(coeff + 32);
+ ADD(_HIGHSIGMASK_log10);
+ ADD(LOG10_E);
+ ADD(LOG10_E + 8);
+ ADD(_L_tbl_log10);
+ ADD(log2_log10);
+ ADD(log2_log10 + 8);
+ ADD(coeff_log10);
+ ADD(coeff_log10 + 16);
+ ADD(coeff_log10 + 32);
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_poly1305.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_poly1305.cpp
index c80b2d16181..ea7e6d64254 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_poly1305.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_poly1305.cpp
@@ -909,10 +909,16 @@ void StubGenerator::poly1305_process_blocks_avx512(
// After execution, input and length will point at remaining (unprocessed) data
// and accumulator will point to the current accumulator value
address StubGenerator::generate_poly1305_processBlocks() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_poly1305_processBlocks_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
// Save all 'SOE' registers
@@ -1028,6 +1034,10 @@ address StubGenerator::generate_poly1305_processBlocks() {
__ leave();
__ ret(0);
+
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -1695,3 +1705,14 @@ void StubGenerator::poly1305_msg_mul_reduce_vec4_avx2(
__ vpaddq(A1, A1, YTMP2, Assembler::AVX_256bit); //Add medium 42-bit bits from new blocks to accumulator
__ vpaddq(A1, A1, YTMP5, Assembler::AVX_256bit);
}
+#undef __
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_poly1305(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr));
+ ADD(POLY1305_PAD_MSG);
+ ADD(POLY1305_MASK42);
+ ADD(POLY1305_MASK44);
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_poly_mont.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_poly_mont.cpp
index c439e0b370f..308a8042993 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_poly_mont.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_poly_mont.cpp
@@ -558,10 +558,16 @@ void montgomeryMultiplyAVX2(const Register aLimbs, const Register bLimbs, const
}
address StubGenerator::generate_intpoly_montgomeryMult_P256() {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_intpoly_montgomeryMult_P256_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
if (VM_Version::supports_avx512ifma() && VM_Version::supports_avx512vlbw()) {
@@ -620,6 +626,10 @@ address StubGenerator::generate_intpoly_montgomeryMult_P256() {
__ leave();
__ ret(0);
+
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -680,10 +690,16 @@ address StubGenerator::generate_intpoly_assign() {
// P521OrderField: 19 = 8 + 8 + 2 + 1
// Special Cases 5, 10, 14, 16, 19
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_intpoly_assign_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
__ enter();
// Inputs
@@ -762,5 +778,24 @@ address StubGenerator::generate_intpoly_assign() {
__ bind(L_Done);
__ leave();
__ ret(0);
+
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
+#undef __
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_poly_mont(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr));
+ // use accessors to retrieve all correct addresses
+ ADD(shift_1L());
+ ADD(shift_1R());
+ ADD(p256_mask52());
+ ADD(mask_limb5());
+ ADD(modulus_p256());
+ ADD(modulus_p256(1));
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_pow.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_pow.cpp
index 3c3df7e6ac4..a9a6dc10da4 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_pow.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_pow.cpp
@@ -760,8 +760,14 @@ ATTRIBUTE_ALIGNED(8) static const juint _DOUBLE0DOT5[] = {
address StubGenerator::generate_libmPow() {
StubId stub_id = StubId::stubgen_dpow_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_2TAG_PACKET_0_0_2, L_2TAG_PACKET_1_0_2, L_2TAG_PACKET_2_0_2, L_2TAG_PACKET_3_0_2;
Label L_2TAG_PACKET_4_0_2, L_2TAG_PACKET_5_0_2, L_2TAG_PACKET_6_0_2, L_2TAG_PACKET_7_0_2;
@@ -1859,7 +1865,45 @@ address StubGenerator::generate_libmPow() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
#undef __
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_pow(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr));
+ address HIGHMASK_Y = (address)_HIGHMASK_Y;
+ address e_coeff = (address)_e_coeff;
+ address coeff_h = (address)_coeff_h;
+ address coeff_pow = (address)_coeff_pow;
+
+ ADD(_HIGHSIGMASK);
+ ADD(_LOG2_E);
+ ADD(HIGHMASK_Y);
+ ADD(HIGHMASK_Y + 8);
+ ADD(_T_exp);
+ ADD(e_coeff);
+ ADD(e_coeff + 16);
+ ADD(e_coeff + 32);
+ ADD(coeff_h);
+ ADD(coeff_h + 8);
+ ADD(_HIGHMASK_LOG_X);
+ ADD(_HALFMASK);
+ ADD(coeff_pow);
+ ADD(coeff_pow + 16);
+ ADD(coeff_pow + 32);
+ ADD(coeff_pow + 48);
+ ADD(coeff_pow + 64);
+ ADD(coeff_pow + 80);
+ ADD(_L_tbl_pow);
+ ADD(_log2_pow);
+ ADD(_DOUBLE2);
+ ADD(_DOUBLE0);
+ ADD(_DOUBLE0DOT5);
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_sha3.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_sha3.cpp
index f9d876f34f3..58f81652a0c 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_sha3.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_sha3.cpp
@@ -104,10 +104,15 @@ static address generate_sha3_implCompress(StubId stub_id,
default:
ShouldNotReachHere();
}
-
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = stubgen->load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
__ align(CodeEntryAlignment);
StubCodeMark mark(stubgen, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register buf = c_rarg0;
const Register state = c_rarg1;
@@ -316,6 +321,9 @@ static address generate_sha3_implCompress(StubId stub_id,
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ stubgen->store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -326,10 +334,16 @@ static address generate_sha3_implCompress(StubId stub_id,
// Performs two keccak() computations in parallel. The steps of the
// two computations are executed interleaved.
static address generate_double_keccak(StubGenerator *stubgen, MacroAssembler *_masm) {
- __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_double_keccak_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = stubgen->load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
+ __ align(CodeEntryAlignment);
StubCodeMark mark(stubgen, stub_id);
- address start = __ pc();
+ start = __ pc();
const Register state0 = c_rarg0;
const Register state1 = c_rarg1;
@@ -495,6 +509,9 @@ static address generate_double_keccak(StubGenerator *stubgen, MacroAssembler *_m
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ stubgen->store_archive_data(stub_id, start, __ pc());
+
return start;
}
@@ -508,3 +525,14 @@ void StubGenerator::generate_sha3_stubs() {
generate_sha3_implCompress(StubId::stubgen_sha3_implCompressMB_id, this, _masm);
}
}
+
+#undef __
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_sha3(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr));
+ ADD(round_constsAddr());
+ ADD(permsAndRotsAddr());
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_sin.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_sin.cpp
index 5290e737581..00c759a369b 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_sin.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_sin.cpp
@@ -181,8 +181,14 @@ ATTRIBUTE_ALIGNED(8) static const juint _ALL_ONES[] =
address StubGenerator::generate_libmSin() {
StubId stub_id = StubId::stubgen_dsin_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_2TAG_PACKET_0_0_1, L_2TAG_PACKET_1_0_1, L_2TAG_PACKET_2_0_1, L_2TAG_PACKET_3_0_1;
Label L_2TAG_PACKET_4_0_1, L_2TAG_PACKET_5_0_1, L_2TAG_PACKET_6_0_1, L_2TAG_PACKET_7_0_1;
@@ -645,7 +651,18 @@ address StubGenerator::generate_libmSin() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
#undef __
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_sin(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr));
+ ADD(_ALL_ONES);
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_sinh.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_sinh.cpp
index 86e4ac20176..9969866cfc7 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_sinh.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_sinh.cpp
@@ -290,8 +290,14 @@ ATTRIBUTE_ALIGNED(16) static const juint _T2_neg_f[] =
address StubGenerator::generate_libmSinh() {
StubId stub_id = StubId::stubgen_dsinh_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_2TAG_PACKET_0_0_2, L_2TAG_PACKET_1_0_2, L_2TAG_PACKET_3_0_2, L_2TAG_PACKET_4_0_2;
Label L_2TAG_PACKET_5_0_2, L_2TAG_PACKET_6_0_2;
@@ -519,7 +525,36 @@ address StubGenerator::generate_libmSinh() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
#undef __
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_sinh(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr));
+ address L2E = (address)_L2E;
+ address cv = (address)_cv;
+ address pv = (address)_pv;
+
+ ADD(L2E);
+ ADD(L2E + 8);
+ ADD(_HALFMASK);
+ ADD(_Shifter);
+ ADD(cv);
+ ADD(cv + 16);
+ ADD(cv + 32);
+ ADD(cv + 48);
+ ADD(cv + 64);
+ ADD(_T2f);
+ ADD(_T2_neg_f);
+ ADD(pv);
+ ADD(pv + 16);
+ ADD(pv + 32);
+ ADD(_MASK3);
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_tan.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_tan.cpp
index 4f14414652c..9f91b9e8f84 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_tan.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_tan.cpp
@@ -456,8 +456,14 @@ ATTRIBUTE_ALIGNED(8) static const juint _QQ_2_tan[] =
address StubGenerator::generate_libmTan() {
StubId stub_id = StubId::stubgen_dtan_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_2TAG_PACKET_0_0_1, L_2TAG_PACKET_1_0_1, L_2TAG_PACKET_2_0_1, L_2TAG_PACKET_3_0_1;
Label L_2TAG_PACKET_4_0_1, L_2TAG_PACKET_5_0_1, L_2TAG_PACKET_6_0_1, L_2TAG_PACKET_7_0_1;
@@ -1025,7 +1031,35 @@ address StubGenerator::generate_libmTan() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
#undef __
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_tan(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr));
+ address PI_4_tan = (address)_PI_4_tan;
+
+ ADD(_MUL16);
+ ADD(_sign_mask_tan);
+ ADD(_PI32INV_tan);
+ ADD(_P_1_tan);
+ ADD(_P_2_tan);
+ ADD(_P_3_tan);
+ ADD(_Ctable_tan);
+ ADD(_MASK_35_tan);
+ ADD(_Q_11_tan);
+ ADD(_Q_9_tan);
+ ADD(_Q_7_tan);
+ ADD(_Q_5_tan);
+ ADD(_Q_3_tan);
+ ADD(PI_4_tan);
+ ADD(PI_4_tan + 8);
+ ADD(_QQ_2_tan);
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_tanh.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_tanh.cpp
index dce4fbfc455..4f2fe8a460b 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_tanh.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_tanh.cpp
@@ -303,8 +303,14 @@ ATTRIBUTE_ALIGNED(16) static const juint _T2_neg_f[] =
address StubGenerator::generate_libmTanh() {
StubId stub_id = StubId::stubgen_dtanh_id;
+ int entry_count = StubInfo::entry_count(stub_id);
+ assert(entry_count == 1, "sanity check");
+ address start = load_archive_data(stub_id);
+ if (start != nullptr) {
+ return start;
+ }
StubCodeMark mark(this, stub_id);
- address start = __ pc();
+ start = __ pc();
Label L_2TAG_PACKET_0_0_1, L_2TAG_PACKET_1_0_1, L_2TAG_PACKET_2_0_1, L_2TAG_PACKET_3_0_1;
Label L_2TAG_PACKET_4_0_1, L_2TAG_PACKET_5_0_1;
@@ -495,7 +501,36 @@ address StubGenerator::generate_libmTanh() {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
+ // record the stub entry and end
+ store_archive_data(stub_id, start, __ pc());
+
return start;
}
#undef __
+
+#if INCLUDE_CDS
+void StubGenerator::init_AOTAddressTable_tanh(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr));
+ address L2E = (address)_L2E;
+ address cv = (address)_cv;
+ address pv = (address)_pv;
+
+ ADD(L2E);
+ ADD(L2E + 8);
+ ADD(_HALFMASK);
+ ADD(_ONEMASK);
+ ADD(_TWOMASK);
+ ADD(_Shifter);
+ ADD(cv);
+ ADD(cv + 16);
+ ADD(cv + 32);
+ ADD(_T2_neg_f);
+ ADD(pv);
+ ADD(pv + 16);
+ ADD(pv + 32);
+ ADD(_MASK3);
+ ADD(_RMASK);
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubRoutines_x86.cpp b/src/hotspot/cpu/x86/stubRoutines_x86.cpp
index ee9cea08e64..ce11925dde2 100644
--- a/src/hotspot/cpu/x86/stubRoutines_x86.cpp
+++ b/src/hotspot/cpu/x86/stubRoutines_x86.cpp
@@ -28,6 +28,10 @@
#include "runtime/stubRoutines.hpp"
#include "utilities/globalDefinitions.hpp"
#include "crc32c.h"
+#include "stubGenerator_x86_64.hpp"
+#ifdef COMPILER1
+#include "c1/c1_LIRAssembler.hpp"
+#endif
// Implementation of the platform-specific part of StubRoutines - for
// a description of how to extend it, see the stubRoutines.hpp file.
@@ -40,8 +44,12 @@
#define DEFINE_ARCH_ENTRY_INIT(arch, blob_name, stub_name, field_name, getter_name, init_function) \
address StubRoutines:: arch :: STUB_FIELD_NAME(field_name) = CAST_FROM_FN_PTR(address, init_function);
-STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY, DEFINE_ARCH_ENTRY_INIT)
+#define DEFINE_ARCH_ENTRY_ARRAY(arch, blob_name, stub_name, field_name, getter_name, count) \
+ address StubRoutines:: arch :: STUB_FIELD_NAME(field_name) [count];
+STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY, DEFINE_ARCH_ENTRY_INIT, DEFINE_ARCH_ENTRY_ARRAY)
+
+#undef DEFINE_ARCH_ENTRY_ARRAY
#undef DEFINE_ARCH_ENTRY_INIT
#undef DEFINE_ARCH_ENTRY
@@ -411,3 +419,46 @@ ATTRIBUTE_ALIGNED(64) const julong StubRoutines::x86::_k512_W[] =
0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL,
0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL,
};
+
+#if INCLUDE_CDS
+
+void StubRoutines::init_AOTAddressTable() {
+ ResourceMark rm;
+ GrowableArray external_addresses;
+ // publish static addresses referred to by main x86 generator and
+ // auxiliary x86 generators
+ StubGenerator::init_AOTAddressTable(external_addresses);
+ // publish external data addresses defined in nested x86 class
+ StubRoutines::x86::init_AOTAddressTable(external_addresses);
+#ifdef COMPILER1
+ LIR_Assembler::init_AOTAddressTable(external_addresses);
+#endif
+ AOTCodeCache::publish_external_addresses(external_addresses);
+}
+
+// publish addresses of external data defined in this file which may
+// be referenced from stub or code
+void StubRoutines::x86::init_AOTAddressTable(GrowableArray& external_addresses) {
+#define ADD(addr) external_addresses.append((address)(addr));
+ ADD(&_mxcsr_std);
+ ADD(&_mxcsr_rz);
+ ADD(crc_by128_masks_addr());
+ ADD(crc_by128_masks_addr() + 16);
+ ADD(crc_by128_masks_addr() + 32);
+ // this is added in generic code
+ // ADD(_crc_table);
+ ADD(crc_by128_masks_avx512_addr());
+ ADD(crc_by128_masks_avx512_addr() + 16);
+ ADD(crc_by128_masks_avx512_addr() + 32);
+ ADD(_crc_table_avx512);
+ ADD(_crc32c_table_avx512);
+ ADD(_shuf_table_crc32_avx512);
+ // n.b. call accessor for this one to ensure the table is generated
+ ADD(crc32c_table_addr());
+ ADD(_arrays_hashcode_powers_of_31);
+ ADD(_k256);
+ ADD(_k256_W);
+ ADD(_k512_W);
+#undef ADD
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/cpu/x86/stubRoutines_x86.hpp b/src/hotspot/cpu/x86/stubRoutines_x86.hpp
index 3654b644131..7283798888b 100644
--- a/src/hotspot/cpu/x86/stubRoutines_x86.hpp
+++ b/src/hotspot/cpu/x86/stubRoutines_x86.hpp
@@ -55,9 +55,13 @@ class x86 {
#define DECLARE_ARCH_ENTRY_INIT(arch, blob_name, stub_name, field_name, getter_name, init_function) \
DECLARE_ARCH_ENTRY(arch, blob_name, stub_name, field_name, getter_name)
-private:
- STUBGEN_ARCH_ENTRIES_DO(DECLARE_ARCH_ENTRY, DECLARE_ARCH_ENTRY_INIT)
+#define DECLARE_ARCH_ENTRY_ARRAY(arch, blob_name, stub_name, field_name, getter_name, count) \
+ static address STUB_FIELD_NAME(field_name) [count] ;
+private:
+ STUBGEN_ARCH_ENTRIES_DO(DECLARE_ARCH_ENTRY, DECLARE_ARCH_ENTRY_INIT, DECLARE_ARCH_ENTRY_ARRAY)
+
+#undef DECLARE_ARCH_ENTRY_ARRAY
#undef DECLARE_ARCH_ENTRY_INIT
#undef DECLARE_ARCH_ENTRY
@@ -70,9 +74,13 @@ private:
#define DEFINE_ARCH_ENTRY_GETTER_INIT(arch, blob_name, stub_name, field_name, getter_name, init_function) \
DEFINE_ARCH_ENTRY_GETTER(arch, blob_name, stub_name, field_name, getter_name)
-public:
- STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY_GETTER, DEFINE_ARCH_ENTRY_GETTER_INIT)
+#define DEFINE_ARCH_ENTRY_GETTER_ARRAY(arch, blob_name, stub_name, field_name, getter_name, count) \
+ static address getter_name(int idx) { return STUB_FIELD_NAME(field_name) [idx]; }
+public:
+ STUBGEN_ARCH_ENTRIES_DO(DEFINE_ARCH_ENTRY_GETTER, DEFINE_ARCH_ENTRY_GETTER_INIT, DEFINE_ARCH_ENTRY_GETTER_ARRAY)
+
+#undef DEFINE_ARCH_ENTRY_GETTER_ARRAY
#undef DEFINE_ARCH_ENTRY_GETTER_INIT
#undef DEFINE_ARCH_GETTER_ENTRY
@@ -112,6 +120,8 @@ public:
static address arrays_hashcode_powers_of_31() { return (address)_arrays_hashcode_powers_of_31; }
static void generate_CRC32C_table(bool is_pclmulqdq_supported);
+
+ static void init_AOTAddressTable(GrowableArray& external_addresses);
};
#endif // CPU_X86_STUBROUTINES_X86_HPP
diff --git a/src/hotspot/cpu/x86/vm_version_x86.cpp b/src/hotspot/cpu/x86/vm_version_x86.cpp
index 4301bd328d6..7d9ceb9d446 100644
--- a/src/hotspot/cpu/x86/vm_version_x86.cpp
+++ b/src/hotspot/cpu/x86/vm_version_x86.cpp
@@ -1094,15 +1094,36 @@ void VM_Version::get_processor_features() {
}
}
- // Currently APX support is only enabled for targets supporting AVX512VL feature.
- bool apx_supported = os_supports_apx_egprs() && supports_apx_f() && supports_avx512vl();
- if (UseAPX && !apx_supported) {
- warning("UseAPX is not supported on this CPU, setting it to false");
+ // Currently APX support is only enabled for targets supporting AVX512VL feature.
+ if (supports_apx_f() && os_supports_apx_egprs() && supports_avx512vl()) {
+ if (FLAG_IS_DEFAULT(UseAPX)) {
+ UseAPX = false; // by default UseAPX is false
+ _features.clear_feature(CPU_APX_F);
+ } else if (!UseAPX) {
+ _features.clear_feature(CPU_APX_F);
+ }
+ } else if (UseAPX) {
+ if (!FLAG_IS_DEFAULT(UseAPX)) {
+ warning("APX is not supported on this CPU, setting it to false)");
+ }
FLAG_SET_DEFAULT(UseAPX, false);
}
- if (!UseAPX) {
- _features.clear_feature(CPU_APX_F);
+ CHECK_CPU_FEATURE(supports_clmul, CLMUL);
+ CHECK_CPU_FEATURE(supports_aes, AES);
+ CHECK_CPU_FEATURE(supports_fma, FMA);
+
+ if (supports_sha() || (supports_avx2() && supports_bmi2())) {
+ if (FLAG_IS_DEFAULT(UseSHA)) {
+ UseSHA = true;
+ } else if (!UseSHA) {
+ _features.clear_feature(CPU_SHA);
+ }
+ } else if (UseSHA) {
+ if (!FLAG_IS_DEFAULT(UseSHA)) {
+ warning("SHA instructions are not available on this CPU");
+ }
+ FLAG_SET_DEFAULT(UseSHA, false);
}
if (FLAG_IS_DEFAULT(IntelJccErratumMitigation)) {
@@ -1152,10 +1173,50 @@ void VM_Version::get_processor_features() {
// Use AES instructions if available.
if (supports_aes()) {
- if (FLAG_IS_DEFAULT(UseAES)) {
- FLAG_SET_DEFAULT(UseAES, true);
+ if (supports_sse3()) {
+ if (FLAG_IS_DEFAULT(UseAESIntrinsics)) {
+ FLAG_SET_DEFAULT(UseAESIntrinsics, true);
+ }
+ } else if (UseAESIntrinsics) {
+ // The AES intrinsic stubs require AES instruction support (of course)
+ // but also require sse3 mode or higher for instructions it use.
+ if (!FLAG_IS_DEFAULT(UseAESIntrinsics)) {
+ warning("X86 AES intrinsics require SSE3 instructions or higher. Intrinsics will be disabled.");
+ }
+ FLAG_SET_DEFAULT(UseAESIntrinsics, false);
}
- if (!UseAES) {
+ if (!UseAESIntrinsics) {
+ if (UseAESCTRIntrinsics) {
+ if (!FLAG_IS_DEFAULT(UseAESCTRIntrinsics)) {
+ warning("AES-CTR intrinsics require UseAESIntrinsics flag to be enabled. Intrinsics will be disabled.");
+ }
+ FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
+ }
+ } else {
+ if (supports_sse4_1()) {
+ if (FLAG_IS_DEFAULT(UseAESCTRIntrinsics)) {
+ FLAG_SET_DEFAULT(UseAESCTRIntrinsics, true);
+ }
+ } else if (UseAESCTRIntrinsics) {
+ // The AES-CTR intrinsic stubs require AES instruction support (of course)
+ // but also require sse4.1 mode or higher for instructions it use.
+ if (!FLAG_IS_DEFAULT(UseAESCTRIntrinsics)) {
+ warning("X86 AES-CTR intrinsics require SSE4.1 instructions or higher. Intrinsics will be disabled.");
+ }
+ FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
+ }
+ }
+ } else {
+ if (!cpu_supports_aes()) {
+ if (UseAESIntrinsics && !FLAG_IS_DEFAULT(UseAESIntrinsics)) {
+ warning("AES intrinsics are not available on this CPU");
+ }
+ FLAG_SET_DEFAULT(UseAESIntrinsics, false);
+ if (UseAESCTRIntrinsics && !FLAG_IS_DEFAULT(UseAESCTRIntrinsics)) {
+ warning("AES-CTR intrinsics are not available on this CPU");
+ }
+ FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
+ } else if (!UseAES) {
if (UseAESIntrinsics && !FLAG_IS_DEFAULT(UseAESIntrinsics)) {
warning("AES intrinsics require UseAES flag to be enabled. Intrinsics will be disabled.");
}
@@ -1164,66 +1225,7 @@ void VM_Version::get_processor_features() {
warning("AES_CTR intrinsics require UseAES flag to be enabled. AES_CTR intrinsics will be disabled.");
}
FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
- } else {
- if (UseSSE > 2) {
- if (FLAG_IS_DEFAULT(UseAESIntrinsics)) {
- FLAG_SET_DEFAULT(UseAESIntrinsics, true);
- }
- } else {
- // The AES intrinsic stubs require AES instruction support (of course)
- // but also require sse3 mode or higher for instructions it use.
- if (UseAESIntrinsics && !FLAG_IS_DEFAULT(UseAESIntrinsics)) {
- warning("X86 AES intrinsics require SSE3 instructions or higher. Intrinsics will be disabled.");
- }
- FLAG_SET_DEFAULT(UseAESIntrinsics, false);
- }
-
- // --AES-CTR begins--
- if (!UseAESIntrinsics) {
- if (UseAESCTRIntrinsics && !FLAG_IS_DEFAULT(UseAESCTRIntrinsics)) {
- warning("AES-CTR intrinsics require UseAESIntrinsics flag to be enabled. Intrinsics will be disabled.");
- }
- FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
- } else {
- if (supports_sse4_1()) {
- if (FLAG_IS_DEFAULT(UseAESCTRIntrinsics)) {
- FLAG_SET_DEFAULT(UseAESCTRIntrinsics, true);
- }
- } else {
- // The AES-CTR intrinsic stubs require AES instruction support (of course)
- // but also require sse4.1 mode or higher for instructions it use.
- if (UseAESCTRIntrinsics && !FLAG_IS_DEFAULT(UseAESCTRIntrinsics)) {
- warning("X86 AES-CTR intrinsics require SSE4.1 instructions or higher. Intrinsics will be disabled.");
- }
- FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
- }
- }
- // --AES-CTR ends--
}
- } else if (UseAES || UseAESIntrinsics || UseAESCTRIntrinsics) {
- if (UseAES && !FLAG_IS_DEFAULT(UseAES)) {
- warning("AES instructions are not available on this CPU");
- }
- FLAG_SET_DEFAULT(UseAES, false);
- if (UseAESIntrinsics && !FLAG_IS_DEFAULT(UseAESIntrinsics)) {
- warning("AES intrinsics are not available on this CPU");
- }
- FLAG_SET_DEFAULT(UseAESIntrinsics, false);
- if (UseAESCTRIntrinsics && !FLAG_IS_DEFAULT(UseAESCTRIntrinsics)) {
- warning("AES-CTR intrinsics are not available on this CPU");
- }
- FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
- }
-
- // Use CLMUL instructions if available.
- if (supports_clmul()) {
- if (FLAG_IS_DEFAULT(UseCLMUL)) {
- UseCLMUL = true;
- }
- } else if (UseCLMUL) {
- if (!FLAG_IS_DEFAULT(UseCLMUL))
- warning("CLMUL instructions not available on this CPU (AVX may also be required)");
- FLAG_SET_DEFAULT(UseCLMUL, false);
}
if (UseCLMUL && (UseSSE > 2)) {
@@ -1264,8 +1266,9 @@ void VM_Version::get_processor_features() {
UseGHASHIntrinsics = true;
}
} else if (UseGHASHIntrinsics) {
- if (!FLAG_IS_DEFAULT(UseGHASHIntrinsics))
+ if (!FLAG_IS_DEFAULT(UseGHASHIntrinsics)) {
warning("GHASH intrinsic requires CLMUL and SSE2 instructions on this CPU");
+ }
FLAG_SET_DEFAULT(UseGHASHIntrinsics, false);
}
@@ -1275,26 +1278,27 @@ void VM_Version::get_processor_features() {
// based on the VM capabilities whether to use an AVX2 or AVX512-enabled
// version.
if (UseAVX >= 1) {
- if (FLAG_IS_DEFAULT(UseChaCha20Intrinsics)) {
- UseChaCha20Intrinsics = true;
- }
+ if (FLAG_IS_DEFAULT(UseChaCha20Intrinsics)) {
+ UseChaCha20Intrinsics = true;
+ }
} else if (UseChaCha20Intrinsics) {
- if (!FLAG_IS_DEFAULT(UseChaCha20Intrinsics)) {
- warning("ChaCha20 intrinsic requires AVX instructions");
- }
- FLAG_SET_DEFAULT(UseChaCha20Intrinsics, false);
+ if (!FLAG_IS_DEFAULT(UseChaCha20Intrinsics)) {
+ warning("ChaCha20 intrinsic requires AVX instructions");
+ }
+ FLAG_SET_DEFAULT(UseChaCha20Intrinsics, false);
}
// Kyber Intrinsics
// Currently we only have them for AVX512
if (supports_evex() && supports_avx512bw()) {
- if (FLAG_IS_DEFAULT(UseKyberIntrinsics)) {
- UseKyberIntrinsics = true;
- }
- } else
- if (UseKyberIntrinsics) {
- warning("Intrinsics for ML-KEM are not available on this CPU.");
- FLAG_SET_DEFAULT(UseKyberIntrinsics, false);
+ if (FLAG_IS_DEFAULT(UseKyberIntrinsics)) {
+ UseKyberIntrinsics = true;
+ }
+ } else if (UseKyberIntrinsics) {
+ if (!FLAG_IS_DEFAULT(UseKyberIntrinsics)) {
+ warning("Intrinsics for ML-KEM are not available on this CPU.");
+ }
+ FLAG_SET_DEFAULT(UseKyberIntrinsics, false);
}
// Dilithium Intrinsics
@@ -1303,8 +1307,10 @@ void VM_Version::get_processor_features() {
UseDilithiumIntrinsics = true;
}
} else if (UseDilithiumIntrinsics) {
+ if (!FLAG_IS_DEFAULT(UseDilithiumIntrinsics)) {
warning("Intrinsics for ML-DSA are not available on this CPU.");
- FLAG_SET_DEFAULT(UseDilithiumIntrinsics, false);
+ }
+ FLAG_SET_DEFAULT(UseDilithiumIntrinsics, false);
}
// Base64 Intrinsics (Check the condition for which the intrinsic will be active)
@@ -1313,39 +1319,24 @@ void VM_Version::get_processor_features() {
UseBASE64Intrinsics = true;
}
} else if (UseBASE64Intrinsics) {
- if (!FLAG_IS_DEFAULT(UseBASE64Intrinsics))
+ if (!FLAG_IS_DEFAULT(UseBASE64Intrinsics)) {
warning("Base64 intrinsic requires EVEX instructions on this CPU");
- FLAG_SET_DEFAULT(UseBASE64Intrinsics, false);
- }
-
- if (supports_fma()) {
- if (FLAG_IS_DEFAULT(UseFMA)) {
- UseFMA = true;
}
- } else if (UseFMA) {
- warning("FMA instructions are not available on this CPU");
- FLAG_SET_DEFAULT(UseFMA, false);
+ FLAG_SET_DEFAULT(UseBASE64Intrinsics, false);
}
if (FLAG_IS_DEFAULT(UseMD5Intrinsics)) {
UseMD5Intrinsics = true;
}
- if (supports_sha() || (supports_avx2() && supports_bmi2())) {
- if (FLAG_IS_DEFAULT(UseSHA)) {
- UseSHA = true;
- }
- } else if (UseSHA) {
- warning("SHA instructions are not available on this CPU");
- FLAG_SET_DEFAULT(UseSHA, false);
- }
-
if (supports_sha() && supports_sse4_1() && UseSHA) {
if (FLAG_IS_DEFAULT(UseSHA1Intrinsics)) {
FLAG_SET_DEFAULT(UseSHA1Intrinsics, true);
}
} else if (UseSHA1Intrinsics) {
- warning("Intrinsics for SHA-1 crypto hash functions not available on this CPU.");
+ if (!FLAG_IS_DEFAULT(UseSHA1Intrinsics)) {
+ warning("Intrinsics for SHA-1 crypto hash functions not available on this CPU.");
+ }
FLAG_SET_DEFAULT(UseSHA1Intrinsics, false);
}
@@ -1354,7 +1345,9 @@ void VM_Version::get_processor_features() {
FLAG_SET_DEFAULT(UseSHA256Intrinsics, true);
}
} else if (UseSHA256Intrinsics) {
- warning("Intrinsics for SHA-224 and SHA-256 crypto hash functions not available on this CPU.");
+ if (!FLAG_IS_DEFAULT(UseSHA256Intrinsics)) {
+ warning("Intrinsics for SHA-224 and SHA-256 crypto hash functions not available on this CPU.");
+ }
FLAG_SET_DEFAULT(UseSHA256Intrinsics, false);
}
@@ -1363,7 +1356,9 @@ void VM_Version::get_processor_features() {
FLAG_SET_DEFAULT(UseSHA512Intrinsics, true);
}
} else if (UseSHA512Intrinsics) {
- warning("Intrinsics for SHA-384 and SHA-512 crypto hash functions not available on this CPU.");
+ if (!FLAG_IS_DEFAULT(UseSHA512Intrinsics)) {
+ warning("Intrinsics for SHA-384 and SHA-512 crypto hash functions not available on this CPU.");
+ }
FLAG_SET_DEFAULT(UseSHA512Intrinsics, false);
}
@@ -1372,14 +1367,12 @@ void VM_Version::get_processor_features() {
FLAG_SET_DEFAULT(UseSHA3Intrinsics, true);
}
} else if (UseSHA3Intrinsics) {
- warning("Intrinsics for SHA3-224, SHA3-256, SHA3-384 and SHA3-512 crypto hash functions not available on this CPU.");
+ if (!FLAG_IS_DEFAULT(UseSHA3Intrinsics)) {
+ warning("Intrinsics for SHA3-224, SHA3-256, SHA3-384 and SHA3-512 crypto hash functions not available on this CPU.");
+ }
FLAG_SET_DEFAULT(UseSHA3Intrinsics, false);
}
- if (!(UseSHA1Intrinsics || UseSHA256Intrinsics || UseSHA512Intrinsics || UseSHA3Intrinsics)) {
- FLAG_SET_DEFAULT(UseSHA, false);
- }
-
#if COMPILER2_OR_JVMCI
int max_vector_size = 0;
if (UseAVX == 0 || !os_supports_avx_vectors()) {
@@ -1435,7 +1428,9 @@ void VM_Version::get_processor_features() {
FLAG_SET_DEFAULT(UsePoly1305Intrinsics, true);
}
} else if (UsePoly1305Intrinsics) {
- warning("Intrinsics for Poly1305 crypto hash functions not available on this CPU.");
+ if (!FLAG_IS_DEFAULT(UsePoly1305Intrinsics)) {
+ warning("Intrinsics for Poly1305 crypto hash functions not available on this CPU.");
+ }
FLAG_SET_DEFAULT(UsePoly1305Intrinsics, false);
}
@@ -1444,7 +1439,9 @@ void VM_Version::get_processor_features() {
FLAG_SET_DEFAULT(UseIntPolyIntrinsics, true);
}
} else if (UseIntPolyIntrinsics) {
- warning("Intrinsics for Polynomial crypto functions not available on this CPU.");
+ if (!FLAG_IS_DEFAULT(UseIntPolyIntrinsics)) {
+ warning("Intrinsics for Polynomial crypto functions not available on this CPU.");
+ }
FLAG_SET_DEFAULT(UseIntPolyIntrinsics, false);
}
@@ -1626,7 +1623,7 @@ void VM_Version::get_processor_features() {
}
#endif // COMPILER2
- if ((supports_sse4_2() && supports_ht()) || supports_avx()) { // Newest Intel cpus
+ if (is_intel_modern_cpu()) { // Newest Intel cpus
if (FLAG_IS_DEFAULT(UseUnalignedLoadStores)) {
UseUnalignedLoadStores = true; // use movdqu on newest Intel cpus
}
@@ -1694,8 +1691,8 @@ void VM_Version::get_processor_features() {
if (FLAG_IS_DEFAULT(UseSSE42Intrinsics)) {
FLAG_SET_DEFAULT(UseSSE42Intrinsics, true);
}
- } else {
- if (UseSSE42Intrinsics && !FLAG_IS_DEFAULT(UseSSE42Intrinsics)) {
+ } else if (UseSSE42Intrinsics) {
+ if (!FLAG_IS_DEFAULT(UseSSE42Intrinsics)) {
warning("SSE4.2 intrinsics require SSE4.2 instructions or higher. Intrinsics will be disabled.");
}
FLAG_SET_DEFAULT(UseSSE42Intrinsics, false);
@@ -1705,15 +1702,17 @@ void VM_Version::get_processor_features() {
UseVectorizedMismatchIntrinsic = true;
}
} else if (UseVectorizedMismatchIntrinsic) {
- if (!FLAG_IS_DEFAULT(UseVectorizedMismatchIntrinsic))
+ if (!FLAG_IS_DEFAULT(UseVectorizedMismatchIntrinsic)) {
warning("vectorizedMismatch intrinsics are not available on this CPU");
+ }
FLAG_SET_DEFAULT(UseVectorizedMismatchIntrinsic, false);
}
if (UseAVX >= 2) {
FLAG_SET_DEFAULT(UseVectorizedHashCodeIntrinsic, true);
} else if (UseVectorizedHashCodeIntrinsic) {
- if (!FLAG_IS_DEFAULT(UseVectorizedHashCodeIntrinsic))
+ if (!FLAG_IS_DEFAULT(UseVectorizedHashCodeIntrinsic)) {
warning("vectorizedHashCode intrinsics are not available on this CPU");
+ }
FLAG_SET_DEFAULT(UseVectorizedHashCodeIntrinsic, false);
}
@@ -1723,7 +1722,9 @@ void VM_Version::get_processor_features() {
UseCountLeadingZerosInstruction = true;
}
} else if (UseCountLeadingZerosInstruction) {
- warning("lzcnt instruction is not available on this CPU");
+ if (!FLAG_IS_DEFAULT(UseCountLeadingZerosInstruction)) {
+ warning("lzcnt instruction is not available on this CPU");
+ }
FLAG_SET_DEFAULT(UseCountLeadingZerosInstruction, false);
}
@@ -1739,7 +1740,9 @@ void VM_Version::get_processor_features() {
}
}
} else if (UseCountTrailingZerosInstruction) {
- warning("tzcnt instruction is not available on this CPU");
+ if (!FLAG_IS_DEFAULT(UseCountTrailingZerosInstruction)) {
+ warning("tzcnt instruction is not available on this CPU");
+ }
FLAG_SET_DEFAULT(UseCountTrailingZerosInstruction, false);
}
@@ -1750,7 +1753,9 @@ void VM_Version::get_processor_features() {
UseBMI1Instructions = true;
}
} else if (UseBMI1Instructions) {
- warning("BMI1 instructions are not available on this CPU (AVX is also required)");
+ if (!FLAG_IS_DEFAULT(UseBMI1Instructions)) {
+ warning("BMI1 instructions are not available on this CPU (AVX is also required)");
+ }
FLAG_SET_DEFAULT(UseBMI1Instructions, false);
}
@@ -1759,7 +1764,9 @@ void VM_Version::get_processor_features() {
UseBMI2Instructions = true;
}
} else if (UseBMI2Instructions) {
- warning("BMI2 instructions are not available on this CPU (AVX is also required)");
+ if (!FLAG_IS_DEFAULT(UseBMI2Instructions)) {
+ warning("BMI2 instructions are not available on this CPU (AVX is also required)");
+ }
FLAG_SET_DEFAULT(UseBMI2Instructions, false);
}
@@ -1769,7 +1776,9 @@ void VM_Version::get_processor_features() {
UsePopCountInstruction = true;
}
} else if (UsePopCountInstruction) {
- warning("POPCNT instruction is not available on this CPU");
+ if (!FLAG_IS_DEFAULT(UsePopCountInstruction)) {
+ warning("POPCNT instruction is not available on this CPU");
+ }
FLAG_SET_DEFAULT(UsePopCountInstruction, false);
}
@@ -1779,7 +1788,9 @@ void VM_Version::get_processor_features() {
UseFastStosb = true;
}
} else if (UseFastStosb) {
- warning("fast-string operations are not available on this CPU");
+ if (!FLAG_IS_DEFAULT(UseFastStosb)) {
+ warning("fast-string operations are not available on this CPU");
+ }
FLAG_SET_DEFAULT(UseFastStosb, false);
}
@@ -1805,7 +1816,9 @@ void VM_Version::get_processor_features() {
UseXMMForObjInit = true;
}
} else if (UseXMMForObjInit) {
- warning("UseXMMForObjInit requires SSE2 and unaligned load/stores. Feature is switched off.");
+ if (!FLAG_IS_DEFAULT(UseXMMForObjInit)) {
+ warning("UseXMMForObjInit requires SSE2 and unaligned load/stores. Feature is switched off.");
+ }
FLAG_SET_DEFAULT(UseXMMForObjInit, false);
}
@@ -1846,7 +1859,7 @@ void VM_Version::get_processor_features() {
if (is_intel() && is_intel_server_family() && supports_sse3()) {
if (FLAG_IS_DEFAULT(AllocatePrefetchLines) &&
- supports_sse4_2() && supports_ht()) { // Nehalem based cpus
+ is_intel_modern_cpu()) { // Nehalem based cpus
FLAG_SET_DEFAULT(AllocatePrefetchLines, 4);
}
#ifdef COMPILER2
@@ -1885,7 +1898,7 @@ void VM_Version::get_processor_features() {
if (FLAG_IS_DEFAULT(ContendedPaddingWidth) &&
(cache_line_size > ContendedPaddingWidth))
- ContendedPaddingWidth = cache_line_size;
+ ContendedPaddingWidth = cache_line_size;
// This machine allows unaligned memory accesses
if (FLAG_IS_DEFAULT(UseUnalignedAccesses)) {
@@ -2514,7 +2527,7 @@ const char* VM_Version::cpu_brand_string(void) {
}
int ret_val = cpu_extended_brand_string(_cpu_brand_string, CPU_EBS_MAX_LENGTH);
if (ret_val != OS_OK) {
- FREE_C_HEAP_ARRAY(char, _cpu_brand_string);
+ FREE_C_HEAP_ARRAY(_cpu_brand_string);
_cpu_brand_string = nullptr;
}
}
@@ -3279,7 +3292,7 @@ int VM_Version::allocate_prefetch_distance(bool use_watermark_prefetch) {
}
} else { // Intel
if (supports_sse3() && is_intel_server_family()) {
- if (supports_sse4_2() && supports_ht()) { // Nehalem based cpus
+ if (is_intel_modern_cpu()) { // Nehalem based cpus
return 192;
} else if (use_watermark_prefetch) { // watermark prefetching on Core
return 384;
diff --git a/src/hotspot/cpu/x86/vm_version_x86.hpp b/src/hotspot/cpu/x86/vm_version_x86.hpp
index a42558a8023..f721635a02e 100644
--- a/src/hotspot/cpu/x86/vm_version_x86.hpp
+++ b/src/hotspot/cpu/x86/vm_version_x86.hpp
@@ -535,6 +535,10 @@ protected:
static const char* _features_names[];
+ static void clear_feature(Feature_Flag feature) {
+ _features.clear_feature(feature);
+ }
+
static void clear_cpu_features() {
_features = VM_Features();
_cpu_features = VM_Features();
@@ -930,6 +934,7 @@ public:
// Feature identification not affected by VM flags
//
static bool cpu_supports_evex() { return _cpu_features.supports_feature(CPU_AVX512F); }
+ static bool cpu_supports_aes() { return _cpu_features.supports_feature(CPU_AES); }
static bool supports_avx512_simd_sort() {
if (supports_avx512dq()) {
@@ -958,6 +963,12 @@ public:
static bool is_intel_darkmont();
+ static bool is_intel_modern_cpu() {
+ precond(is_intel()); // should be called only for intel CPU
+ // Efficient cores in hybrid CPU may not support hyper-threads.
+ return (supports_avx() || (supports_sse4_2() && (supports_ht() || supports_hybrid())));
+ }
+
static bool is_intel_tsc_synched_at_init();
static void insert_features_names(VM_Version::VM_Features features, stringStream& ss);
diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad
index d3a25786b5f..0467f47a9f5 100644
--- a/src/hotspot/cpu/x86/x86.ad
+++ b/src/hotspot/cpu/x86/x86.ad
@@ -1708,84 +1708,99 @@ static void emit_cmpfp3(MacroAssembler* masm, Register dst) {
__ bind(done);
}
-// Math.min() # Math.max()
-// --------------------------
-// ucomis[s/d] #
-// ja -> b # a
-// jp -> NaN # NaN
-// jb -> a # b
-// je #
-// |-jz -> a | b # a & b
-// | -> a #
+enum FP_PREC {
+ fp_prec_hlf,
+ fp_prec_flt,
+ fp_prec_dbl
+};
+
+static inline void emit_fp_ucom(MacroAssembler* masm, enum FP_PREC pt,
+ XMMRegister p, XMMRegister q) {
+ if (pt == fp_prec_hlf) {
+ __ evucomish(p, q);
+ } else if (pt == fp_prec_flt) {
+ __ ucomiss(p, q);
+ } else {
+ __ ucomisd(p, q);
+ }
+}
+
+static inline void movfp(MacroAssembler* masm, enum FP_PREC pt,
+ XMMRegister dst, XMMRegister src, Register scratch) {
+ if (pt == fp_prec_hlf) {
+ __ movhlf(dst, src, scratch);
+ } else if (pt == fp_prec_flt) {
+ __ movflt(dst, src);
+ } else {
+ __ movdbl(dst, src);
+ }
+}
+
+// Math.min() # Math.max()
+// -----------------------------
+// (v)ucomis[h/s/d] #
+// ja -> b # a
+// jp -> NaN # NaN
+// jb -> a # b
+// je #
+// |-jz -> a | b # a & b
+// | -> a #
static void emit_fp_min_max(MacroAssembler* masm, XMMRegister dst,
XMMRegister a, XMMRegister b,
XMMRegister xmmt, Register rt,
- bool min, bool single) {
+ bool min, enum FP_PREC pt) {
Label nan, zero, below, above, done;
- if (single)
- __ ucomiss(a, b);
- else
- __ ucomisd(a, b);
+ emit_fp_ucom(masm, pt, a, b);
- if (dst->encoding() != (min ? b : a)->encoding())
+ if (dst->encoding() != (min ? b : a)->encoding()) {
__ jccb(Assembler::above, above); // CF=0 & ZF=0
- else
+ } else {
__ jccb(Assembler::above, done);
+ }
__ jccb(Assembler::parity, nan); // PF=1
__ jccb(Assembler::below, below); // CF=1
// equal
__ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
- if (single) {
- __ ucomiss(a, xmmt);
- __ jccb(Assembler::equal, zero);
+ emit_fp_ucom(masm, pt, a, xmmt);
- __ movflt(dst, a);
- __ jmp(done);
- }
- else {
- __ ucomisd(a, xmmt);
- __ jccb(Assembler::equal, zero);
+ __ jccb(Assembler::equal, zero);
+ movfp(masm, pt, dst, a, rt);
- __ movdbl(dst, a);
- __ jmp(done);
- }
+ __ jmp(done);
__ bind(zero);
- if (min)
+ if (min) {
__ vpor(dst, a, b, Assembler::AVX_128bit);
- else
+ } else {
__ vpand(dst, a, b, Assembler::AVX_128bit);
+ }
__ jmp(done);
__ bind(above);
- if (single)
- __ movflt(dst, min ? b : a);
- else
- __ movdbl(dst, min ? b : a);
+ movfp(masm, pt, dst, min ? b : a, rt);
__ jmp(done);
__ bind(nan);
- if (single) {
+ if (pt == fp_prec_hlf) {
+ __ movl(rt, 0x00007e00); // Float16.NaN
+ __ evmovw(dst, rt);
+ } else if (pt == fp_prec_flt) {
__ movl(rt, 0x7fc00000); // Float.NaN
__ movdl(dst, rt);
- }
- else {
+ } else {
__ mov64(rt, 0x7ff8000000000000L); // Double.NaN
__ movdq(dst, rt);
}
__ jmp(done);
__ bind(below);
- if (single)
- __ movflt(dst, min ? a : b);
- else
- __ movdbl(dst, min ? a : b);
+ movfp(masm, pt, dst, min ? a : b, rt);
__ bind(done);
}
@@ -2605,13 +2620,8 @@ uint BoxLockNode::size(PhaseRegAlloc *ra_) const
#ifndef PRODUCT
void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
{
- if (UseCompressedClassPointers) {
- st->print_cr("movl rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
- st->print_cr("\tcmpl rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
- } else {
- st->print_cr("movq rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
- st->print_cr("\tcmpq rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
- }
+ st->print_cr("movl rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
+ st->print_cr("\tcmpl rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
st->print_cr("\tjne SharedRuntime::_ic_miss_stub");
}
#endif
@@ -7350,146 +7360,140 @@ instruct loadAOTRCAddress(rRegP dst, immAOTRuntimeConstantsAddress con)
ins_pipe(ialu_reg_fat);
%}
+// min = java.lang.Math.min(float a, float b)
// max = java.lang.Math.max(float a, float b)
-instruct maxF_reg_avx10_2(regF dst, regF a, regF b) %{
- predicate(VM_Version::supports_avx10_2());
+instruct minmaxF_reg_avx10_2(regF dst, regF a, regF b)
+%{
+ predicate(VM_Version::supports_avx10_2() && !VLoopReductions::is_reduction(n));
match(Set dst (MaxF a b));
- format %{ "maxF $dst, $a, $b" %}
- ins_encode %{
- __ eminmaxss($dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, AVX10_2_MINMAX_MAX_COMPARE_SIGN);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// max = java.lang.Math.max(float a, float b)
-instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
- predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
- match(Set dst (MaxF a b));
- effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
- format %{ "maxF $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
- ins_encode %{
- __ vminmax_fp(Op_MaxV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xtmp, rRegI rtmp, rFlagsReg cr) %{
- predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
- match(Set dst (MaxF a b));
- effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
-
- format %{ "maxF_reduction $dst, $a, $b \t!using $xtmp and $rtmp as TEMP" %}
- ins_encode %{
- emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
- false /*min*/, true /*single*/);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// max = java.lang.Math.max(double a, double b)
-instruct maxD_reg_avx10_2(regD dst, regD a, regD b) %{
- predicate(VM_Version::supports_avx10_2());
- match(Set dst (MaxD a b));
- format %{ "maxD $dst, $a, $b" %}
- ins_encode %{
- __ eminmaxsd($dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, AVX10_2_MINMAX_MAX_COMPARE_SIGN);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// max = java.lang.Math.max(double a, double b)
-instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
- predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
- match(Set dst (MaxD a b));
- effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
- format %{ "maxD $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
- ins_encode %{
- __ vminmax_fp(Op_MaxV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xtmp, rRegL rtmp, rFlagsReg cr) %{
- predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
- match(Set dst (MaxD a b));
- effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
-
- format %{ "maxD_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
- ins_encode %{
- emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
- false /*min*/, false /*single*/);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// max = java.lang.Math.min(float a, float b)
-instruct minF_reg_avx10_2(regF dst, regF a, regF b) %{
- predicate(VM_Version::supports_avx10_2());
match(Set dst (MinF a b));
- format %{ "minF $dst, $a, $b" %}
+
+ format %{ "minmaxF $dst, $a, $b" %}
ins_encode %{
- __ eminmaxss($dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, AVX10_2_MINMAX_MIN_COMPARE_SIGN);
+ int opcode = this->ideal_Opcode();
+ __ sminmax_fp_avx10_2(opcode, T_FLOAT, $dst$$XMMRegister, k0, $a$$XMMRegister, $b$$XMMRegister);
+ %}
+ ins_pipe( pipe_slow );
+%}
+
+instruct minmaxF_reduction_reg_avx10_2(regF dst, regF a, regF b, regF xtmp, rRegI rtmp, rFlagsReg cr)
+%{
+ predicate(VM_Version::supports_avx10_2() && VLoopReductions::is_reduction(n));
+ match(Set dst (MaxF a b));
+ match(Set dst (MinF a b));
+ effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
+
+ format %{ "minmaxF_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
+ ins_encode %{
+ int opcode = this->ideal_Opcode();
+ bool min = (opcode == Op_MinF) ? true : false;
+ emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
+ min, fp_prec_flt /*pt*/);
%}
ins_pipe( pipe_slow );
%}
// min = java.lang.Math.min(float a, float b)
-instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
+// max = java.lang.Math.max(float a, float b)
+instruct minmaxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp)
+%{
predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
+ match(Set dst (MaxF a b));
match(Set dst (MinF a b));
effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
- format %{ "minF $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
+
+ format %{ "minmaxF $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
ins_encode %{
- __ vminmax_fp(Op_MinV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
+ int opcode = this->ideal_Opcode();
+ int param_opcode = (opcode == Op_MinF) ? Op_MinV : Op_MaxV;
+ __ vminmax_fp(param_opcode, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister,
+ $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
%}
ins_pipe( pipe_slow );
%}
-instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xtmp, rRegI rtmp, rFlagsReg cr) %{
+instruct minmaxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xtmp, rRegI rtmp, rFlagsReg cr)
+%{
predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
+ match(Set dst (MaxF a b));
match(Set dst (MinF a b));
effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
- format %{ "minF_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
+ format %{ "minmaxF_reduction $dst, $a, $b \t!using $xtmp and $rtmp as TEMP" %}
ins_encode %{
+ int opcode = this->ideal_Opcode();
+ bool min = (opcode == Op_MinF) ? true : false;
emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
- true /*min*/, true /*single*/);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// max = java.lang.Math.min(double a, double b)
-instruct minD_reg_avx10_2(regD dst, regD a, regD b) %{
- predicate(VM_Version::supports_avx10_2());
- match(Set dst (MinD a b));
- format %{ "minD $dst, $a, $b" %}
- ins_encode %{
- __ eminmaxsd($dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, AVX10_2_MINMAX_MIN_COMPARE_SIGN);
+ min, fp_prec_flt /*pt*/);
%}
ins_pipe( pipe_slow );
%}
// min = java.lang.Math.min(double a, double b)
-instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
- predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
+// max = java.lang.Math.max(double a, double b)
+instruct minmaxD_reg_avx10_2(regD dst, regD a, regD b)
+%{
+ predicate(VM_Version::supports_avx10_2() && !VLoopReductions::is_reduction(n));
+ match(Set dst (MaxD a b));
match(Set dst (MinD a b));
- effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
- format %{ "minD $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
+
+ format %{ "minmaxD $dst, $a, $b" %}
ins_encode %{
- __ vminmax_fp(Op_MinV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
+ int opcode = this->ideal_Opcode();
+ __ sminmax_fp_avx10_2(opcode, T_DOUBLE, $dst$$XMMRegister, k0, $a$$XMMRegister, $b$$XMMRegister);
%}
ins_pipe( pipe_slow );
%}
-instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xtmp, rRegL rtmp, rFlagsReg cr) %{
- predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
+instruct minmaxD_reduction_reg_avx10_2(regD dst, regD a, regD b, regD xtmp, rRegI rtmp, rFlagsReg cr)
+%{
+ predicate(VM_Version::supports_avx10_2() && VLoopReductions::is_reduction(n));
+ match(Set dst (MaxD a b));
match(Set dst (MinD a b));
effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
- format %{ "maxD_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
+ format %{ "minmaxD_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
ins_encode %{
+ int opcode = this->ideal_Opcode();
+ bool min = (opcode == Op_MinD) ? true : false;
emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
- true /*min*/, false /*single*/);
+ min, fp_prec_dbl /*pt*/);
+ %}
+ ins_pipe( pipe_slow );
+%}
+
+// min = java.lang.Math.min(double a, double b)
+// max = java.lang.Math.max(double a, double b)
+instruct minmaxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp)
+%{
+ predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
+ match(Set dst (MaxD a b));
+ match(Set dst (MinD a b));
+ effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
+
+ format %{ "minmaxD $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
+ ins_encode %{
+ int opcode = this->ideal_Opcode();
+ int param_opcode = (opcode == Op_MinD) ? Op_MinV : Op_MaxV;
+ __ vminmax_fp(param_opcode, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister,
+ $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
+ %}
+ ins_pipe( pipe_slow );
+%}
+
+instruct minmaxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xtmp, rRegL rtmp, rFlagsReg cr)
+%{
+ predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
+ match(Set dst (MaxD a b));
+ match(Set dst (MinD a b));
+ effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
+
+ format %{ "minmaxD_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
+ ins_encode %{
+ int opcode = this->ideal_Opcode();
+ bool min = (opcode == Op_MinD) ? true : false;
+ emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
+ min, fp_prec_dbl /*pt*/);
%}
ins_pipe( pipe_slow );
%}
@@ -14399,9 +14403,9 @@ instruct cmpF_cc_regCFE(rFlagsRegUCFE cr, regF src1, regF src2) %{
match(Set cr (CmpF src1 src2));
ins_cost(100);
- format %{ "vucomxss $src1, $src2" %}
+ format %{ "evucomxss $src1, $src2" %}
ins_encode %{
- __ vucomxss($src1$$XMMRegister, $src2$$XMMRegister);
+ __ evucomxss($src1$$XMMRegister, $src2$$XMMRegister);
%}
ins_pipe(pipe_slow);
%}
@@ -14421,9 +14425,9 @@ instruct cmpF_cc_memCFE(rFlagsRegUCFE cr, regF src1, memory src2) %{
match(Set cr (CmpF src1 (LoadF src2)));
ins_cost(100);
- format %{ "vucomxss $src1, $src2" %}
+ format %{ "evucomxss $src1, $src2" %}
ins_encode %{
- __ vucomxss($src1$$XMMRegister, $src2$$Address);
+ __ evucomxss($src1$$XMMRegister, $src2$$Address);
%}
ins_pipe(pipe_slow);
%}
@@ -14443,9 +14447,9 @@ instruct cmpF_cc_immCFE(rFlagsRegUCFE cr, regF src, immF con) %{
match(Set cr (CmpF src con));
ins_cost(100);
- format %{ "vucomxss $src, [$constantaddress]\t# load from constant table: float=$con" %}
+ format %{ "evucomxss $src, [$constantaddress]\t# load from constant table: float=$con" %}
ins_encode %{
- __ vucomxss($src$$XMMRegister, $constantaddress($con));
+ __ evucomxss($src$$XMMRegister, $constantaddress($con));
%}
ins_pipe(pipe_slow);
%}
@@ -14484,9 +14488,9 @@ instruct cmpD_cc_regCFE(rFlagsRegUCFE cr, regD src1, regD src2) %{
match(Set cr (CmpD src1 src2));
ins_cost(100);
- format %{ "vucomxsd $src1, $src2 test" %}
+ format %{ "evucomxsd $src1, $src2 test" %}
ins_encode %{
- __ vucomxsd($src1$$XMMRegister, $src2$$XMMRegister);
+ __ evucomxsd($src1$$XMMRegister, $src2$$XMMRegister);
%}
ins_pipe(pipe_slow);
%}
@@ -14506,9 +14510,9 @@ instruct cmpD_cc_memCFE(rFlagsRegUCFE cr, regD src1, memory src2) %{
match(Set cr (CmpD src1 (LoadD src2)));
ins_cost(100);
- format %{ "vucomxsd $src1, $src2" %}
+ format %{ "evucomxsd $src1, $src2" %}
ins_encode %{
- __ vucomxsd($src1$$XMMRegister, $src2$$Address);
+ __ evucomxsd($src1$$XMMRegister, $src2$$Address);
%}
ins_pipe(pipe_slow);
%}
@@ -14527,9 +14531,9 @@ instruct cmpD_cc_immCFE(rFlagsRegUCFE cr, regD src, immD con) %{
match(Set cr (CmpD src con));
ins_cost(100);
- format %{ "vucomxsd $src, [$constantaddress]\t# load from constant table: double=$con" %}
+ format %{ "evucomxsd $src, [$constantaddress]\t# load from constant table: double=$con" %}
ins_encode %{
- __ vucomxsd($src$$XMMRegister, $constantaddress($con));
+ __ evucomxsd($src$$XMMRegister, $constantaddress($con));
%}
ins_pipe(pipe_slow);
%}
@@ -18837,7 +18841,7 @@ instruct ReplHF_reg(vec dst, regF src, rRegI rtmp) %{
format %{ "replicateHF $dst, $src \t! using $rtmp as TEMP" %}
ins_encode %{
int vlen_enc = vector_length_encoding(this);
- __ vmovw($rtmp$$Register, $src$$XMMRegister);
+ __ evmovw($rtmp$$Register, $src$$XMMRegister);
__ evpbroadcastw($dst$$XMMRegister, $rtmp$$Register, vlen_enc);
%}
ins_pipe( pipe_slow );
@@ -20952,7 +20956,7 @@ instruct minmaxFP_reg_avx10_2(vec dst, vec a, vec b) %{
int vlen_enc = vector_length_encoding(this);
int opcode = this->ideal_Opcode();
BasicType elem_bt = Matcher::vector_element_basic_type(this);
- __ vminmax_fp(opcode, elem_bt, $dst$$XMMRegister, k0, $a$$XMMRegister, $b$$XMMRegister, vlen_enc);
+ __ vminmax_fp_avx10_2(opcode, elem_bt, $dst$$XMMRegister, k0, $a$$XMMRegister, $b$$XMMRegister, vlen_enc);
%}
ins_pipe( pipe_slow );
%}
@@ -23968,8 +23972,12 @@ instruct vmask_gen_imm(kReg dst, immL len, rRegL temp) %{
format %{ "vector_mask_gen $len \t! vector mask generator" %}
effect(TEMP temp);
ins_encode %{
- __ mov64($temp$$Register, (0xFFFFFFFFFFFFFFFFUL >> (64 -$len$$constant)));
- __ kmovql($dst$$KRegister, $temp$$Register);
+ if ($len$$constant > 0) {
+ __ mov64($temp$$Register, right_n_bits($len$$constant));
+ __ kmovql($dst$$KRegister, $temp$$Register);
+ } else {
+ __ kxorql($dst$$KRegister, $dst$$KRegister, $dst$$KRegister);
+ }
%}
ins_pipe( pipe_slow );
%}
@@ -25296,9 +25304,9 @@ instruct vector_selectfrom_twovectors_reg_evex(vec index, vec src1, vec src2)
instruct reinterpretS2HF(regF dst, rRegI src)
%{
match(Set dst (ReinterpretS2HF src));
- format %{ "vmovw $dst, $src" %}
+ format %{ "evmovw $dst, $src" %}
ins_encode %{
- __ vmovw($dst$$XMMRegister, $src$$Register);
+ __ evmovw($dst$$XMMRegister, $src$$Register);
%}
ins_pipe(pipe_slow);
%}
@@ -25306,9 +25314,9 @@ instruct reinterpretS2HF(regF dst, rRegI src)
instruct reinterpretHF2S(rRegI dst, regF src)
%{
match(Set dst (ReinterpretHF2S src));
- format %{ "vmovw $dst, $src" %}
+ format %{ "evmovw $dst, $src" %}
ins_encode %{
- __ vmovw($dst$$Register, $src$$XMMRegister);
+ __ evmovw($dst$$Register, $src$$XMMRegister);
%}
ins_pipe(pipe_slow);
%}
@@ -25362,10 +25370,11 @@ instruct scalar_minmax_HF_reg_avx10_2(regF dst, regF src1, regF src2)
predicate(VM_Version::supports_avx10_2());
match(Set dst (MaxHF src1 src2));
match(Set dst (MinHF src1 src2));
+
format %{ "scalar_min_max_fp16 $dst, $src1, $src2" %}
ins_encode %{
- int function = this->ideal_Opcode() == Op_MinHF ? AVX10_2_MINMAX_MIN_COMPARE_SIGN : AVX10_2_MINMAX_MAX_COMPARE_SIGN;
- __ eminmaxsh($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, function);
+ int opcode = this->ideal_Opcode();
+ __ sminmax_fp16_avx10_2(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, k0);
%}
ins_pipe( pipe_slow );
%}
@@ -25376,11 +25385,12 @@ instruct scalar_minmax_HF_reg(regF dst, regF src1, regF src2, kReg ktmp, regF xt
match(Set dst (MaxHF src1 src2));
match(Set dst (MinHF src1 src2));
effect(TEMP_DEF dst, TEMP ktmp, TEMP xtmp1, TEMP xtmp2);
+
format %{ "scalar_min_max_fp16 $dst, $src1, $src2\t using $ktmp, $xtmp1 and $xtmp2 as TEMP" %}
ins_encode %{
int opcode = this->ideal_Opcode();
- __ scalar_max_min_fp16(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $ktmp$$KRegister,
- $xtmp1$$XMMRegister, $xtmp2$$XMMRegister);
+ __ sminmax_fp16(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $ktmp$$KRegister,
+ $xtmp1$$XMMRegister, $xtmp2$$XMMRegister);
%}
ins_pipe( pipe_slow );
%}
@@ -25480,8 +25490,9 @@ instruct vector_minmax_HF_mem_avx10_2(vec dst, vec src1, memory src2)
format %{ "vector_min_max_fp16_mem $dst, $src1, $src2" %}
ins_encode %{
int vlen_enc = vector_length_encoding(this);
- int function = this->ideal_Opcode() == Op_MinVHF ? AVX10_2_MINMAX_MIN_COMPARE_SIGN : AVX10_2_MINMAX_MAX_COMPARE_SIGN;
- __ evminmaxph($dst$$XMMRegister, k0, $src1$$XMMRegister, $src2$$Address, true, function, vlen_enc);
+ int opcode = this->ideal_Opcode();
+ __ vminmax_fp16_avx10_2(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address,
+ k0, vlen_enc);
%}
ins_pipe( pipe_slow );
%}
@@ -25494,8 +25505,9 @@ instruct vector_minmax_HF_reg_avx10_2(vec dst, vec src1, vec src2)
format %{ "vector_min_max_fp16 $dst, $src1, $src2" %}
ins_encode %{
int vlen_enc = vector_length_encoding(this);
- int function = this->ideal_Opcode() == Op_MinVHF ? AVX10_2_MINMAX_MIN_COMPARE_SIGN : AVX10_2_MINMAX_MAX_COMPARE_SIGN;
- __ evminmaxph($dst$$XMMRegister, k0, $src1$$XMMRegister, $src2$$XMMRegister, true, function, vlen_enc);
+ int opcode = this->ideal_Opcode();
+ __ vminmax_fp16_avx10_2(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister,
+ k0, vlen_enc);
%}
ins_pipe( pipe_slow );
%}
@@ -25510,8 +25522,8 @@ instruct vector_minmax_HF_reg(vec dst, vec src1, vec src2, kReg ktmp, vec xtmp1,
ins_encode %{
int vlen_enc = vector_length_encoding(this);
int opcode = this->ideal_Opcode();
- __ vector_max_min_fp16(opcode, $dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, $ktmp$$KRegister,
- $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
+ __ vminmax_fp16(opcode, $dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, $ktmp$$KRegister,
+ $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
%}
ins_pipe( pipe_slow );
%}
diff --git a/src/hotspot/cpu/zero/stubDeclarations_zero.hpp b/src/hotspot/cpu/zero/stubDeclarations_zero.hpp
index 2357bbb5169..9abe313b3a7 100644
--- a/src/hotspot/cpu/zero/stubDeclarations_zero.hpp
+++ b/src/hotspot/cpu/zero/stubDeclarations_zero.hpp
@@ -29,35 +29,40 @@
#define STUBGEN_PREUNIVERSE_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(preuniverse, 0) \
#define STUBGEN_INITIAL_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(initial, 0) \
#define STUBGEN_CONTINUATION_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(continuation, 0) \
#define STUBGEN_COMPILER_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(compiler, 0) \
#define STUBGEN_FINAL_BLOBS_ARCH_DO(do_stub, \
do_arch_blob, \
do_arch_entry, \
- do_arch_entry_init) \
+ do_arch_entry_init, \
+ do_arch_entry_array) \
do_arch_blob(final, 0) \
diff --git a/src/hotspot/cpu/zero/stubGenerator_zero.cpp b/src/hotspot/cpu/zero/stubGenerator_zero.cpp
index 08cb173b507..569a2fa8ca9 100644
--- a/src/hotspot/cpu/zero/stubGenerator_zero.cpp
+++ b/src/hotspot/cpu/zero/stubGenerator_zero.cpp
@@ -213,7 +213,7 @@ class StubGenerator: public StubCodeGenerator {
}
public:
- StubGenerator(CodeBuffer* code, BlobId blob_id) : StubCodeGenerator(code, blob_id) {
+ StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData *stub_data) : StubCodeGenerator(code, blob_id, stub_data) {
switch(blob_id) {
case BlobId::stubgen_preuniverse_id:
generate_preuniverse_stubs();
@@ -237,8 +237,8 @@ class StubGenerator: public StubCodeGenerator {
}
};
-void StubGenerator_generate(CodeBuffer* code, BlobId blob_id) {
- StubGenerator g(code, blob_id);
+void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData *stub_data) {
+ StubGenerator g(code, blob_id, stub_data);
}
EntryFrame *EntryFrame::build(const intptr_t* parameters,
diff --git a/src/hotspot/cpu/zero/stubRoutines_zero.cpp b/src/hotspot/cpu/zero/stubRoutines_zero.cpp
index 9b53f09be5d..196907b061f 100644
--- a/src/hotspot/cpu/zero/stubRoutines_zero.cpp
+++ b/src/hotspot/cpu/zero/stubRoutines_zero.cpp
@@ -30,3 +30,9 @@
address StubRoutines::crc_table_addr() { ShouldNotCallThis(); return nullptr; }
address StubRoutines::crc32c_table_addr() { ShouldNotCallThis(); return nullptr; }
+
+#if INCLUDE_CDS
+// nothing to do for zero
+void StubRoutines::init_AOTAddressTable() {
+}
+#endif // INCLUDE_CDS
diff --git a/src/hotspot/os/aix/os_aix.cpp b/src/hotspot/os/aix/os_aix.cpp
index 3cad24d388c..32d845b2b6d 100644
--- a/src/hotspot/os/aix/os_aix.cpp
+++ b/src/hotspot/os/aix/os_aix.cpp
@@ -578,13 +578,13 @@ void os::init_system_properties_values() {
char *ld_library_path = NEW_C_HEAP_ARRAY(char, pathsize, mtInternal);
os::snprintf_checked(ld_library_path, pathsize, "%s%s" DEFAULT_LIBPATH, v, v_colon);
Arguments::set_library_path(ld_library_path);
- FREE_C_HEAP_ARRAY(char, ld_library_path);
+ FREE_C_HEAP_ARRAY(ld_library_path);
// Extensions directories.
os::snprintf_checked(buf, bufsize, "%s" EXTENSIONS_DIR, Arguments::get_java_home());
Arguments::set_ext_dirs(buf);
- FREE_C_HEAP_ARRAY(char, buf);
+ FREE_C_HEAP_ARRAY(buf);
#undef DEFAULT_LIBPATH
#undef EXTENSIONS_DIR
diff --git a/src/hotspot/os/aix/os_perf_aix.cpp b/src/hotspot/os/aix/os_perf_aix.cpp
index cbf78083483..3668ac6ba3f 100644
--- a/src/hotspot/os/aix/os_perf_aix.cpp
+++ b/src/hotspot/os/aix/os_perf_aix.cpp
@@ -258,10 +258,10 @@ bool CPUPerformanceInterface::CPUPerformance::initialize() {
CPUPerformanceInterface::CPUPerformance::~CPUPerformance() {
if (_lcpu_names) {
- FREE_C_HEAP_ARRAY(perfstat_id_t, _lcpu_names);
+ FREE_C_HEAP_ARRAY(_lcpu_names);
}
if (_prev_ticks) {
- FREE_C_HEAP_ARRAY(cpu_tick_store_t, _prev_ticks);
+ FREE_C_HEAP_ARRAY(_prev_ticks);
}
}
@@ -511,12 +511,12 @@ CPUInformationInterface::~CPUInformationInterface() {
if (_cpu_info != nullptr) {
if (_cpu_info->cpu_name() != nullptr) {
const char* cpu_name = _cpu_info->cpu_name();
- FREE_C_HEAP_ARRAY(char, cpu_name);
+ FREE_C_HEAP_ARRAY(cpu_name);
_cpu_info->set_cpu_name(nullptr);
}
if (_cpu_info->cpu_description() != nullptr) {
const char* cpu_desc = _cpu_info->cpu_description();
- FREE_C_HEAP_ARRAY(char, cpu_desc);
+ FREE_C_HEAP_ARRAY(cpu_desc);
_cpu_info->set_cpu_description(nullptr);
}
delete _cpu_info;
@@ -576,7 +576,7 @@ int NetworkPerformanceInterface::NetworkPerformance::network_utilization(Network
// check for error
if (n_records < 0) {
- FREE_C_HEAP_ARRAY(perfstat_netinterface_t, net_stats);
+ FREE_C_HEAP_ARRAY(net_stats);
return OS_ERR;
}
@@ -593,7 +593,7 @@ int NetworkPerformanceInterface::NetworkPerformance::network_utilization(Network
*network_interfaces = new_interface;
}
- FREE_C_HEAP_ARRAY(perfstat_netinterface_t, net_stats);
+ FREE_C_HEAP_ARRAY(net_stats);
return OS_OK;
}
diff --git a/src/hotspot/os/aix/porting_aix.cpp b/src/hotspot/os/aix/porting_aix.cpp
index b3f878fbfdd..f0527136d90 100644
--- a/src/hotspot/os/aix/porting_aix.cpp
+++ b/src/hotspot/os/aix/porting_aix.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2012, 2024 SAP SE. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
* Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -426,6 +426,10 @@ int dladdr(void* addr, Dl_info* info) {
}
+int JVM_dladdr(void* addr, Dl_info* info) {
+ return dladdr(addr, info);
+}
+
/////////////////////////////////////////////////////////////////////////////
// Native callstack dumping
diff --git a/src/hotspot/os/bsd/os_bsd.cpp b/src/hotspot/os/bsd/os_bsd.cpp
index a4d9a2197a5..4c77b619718 100644
--- a/src/hotspot/os/bsd/os_bsd.cpp
+++ b/src/hotspot/os/bsd/os_bsd.cpp
@@ -444,14 +444,14 @@ void os::init_system_properties_values() {
char *ld_library_path = NEW_C_HEAP_ARRAY(char, ld_library_path_size, mtInternal);
os::snprintf_checked(ld_library_path, ld_library_path_size, "%s%s" SYS_EXT_DIR "/lib/%s:" DEFAULT_LIBPATH, v, v_colon, cpu_arch);
Arguments::set_library_path(ld_library_path);
- FREE_C_HEAP_ARRAY(char, ld_library_path);
+ FREE_C_HEAP_ARRAY(ld_library_path);
}
// Extensions directories.
os::snprintf_checked(buf, bufsize, "%s" EXTENSIONS_DIR ":" SYS_EXT_DIR EXTENSIONS_DIR, Arguments::get_java_home());
Arguments::set_ext_dirs(buf);
- FREE_C_HEAP_ARRAY(char, buf);
+ FREE_C_HEAP_ARRAY(buf);
#else // __APPLE__
@@ -538,7 +538,7 @@ void os::init_system_properties_values() {
os::snprintf_checked(ld_library_path, ld_library_path_size, "%s%s%s%s%s" SYS_EXTENSIONS_DIR ":" SYS_EXTENSIONS_DIRS ":.",
v, v_colon, l, l_colon, user_home_dir);
Arguments::set_library_path(ld_library_path);
- FREE_C_HEAP_ARRAY(char, ld_library_path);
+ FREE_C_HEAP_ARRAY(ld_library_path);
}
// Extensions directories.
@@ -550,7 +550,7 @@ void os::init_system_properties_values() {
user_home_dir, Arguments::get_java_home());
Arguments::set_ext_dirs(buf);
- FREE_C_HEAP_ARRAY(char, buf);
+ FREE_C_HEAP_ARRAY(buf);
#undef SYS_EXTENSIONS_DIR
#undef SYS_EXTENSIONS_DIRS
diff --git a/src/hotspot/os/bsd/os_perf_bsd.cpp b/src/hotspot/os/bsd/os_perf_bsd.cpp
index 78d9519c3a7..47fe3a0d7e9 100644
--- a/src/hotspot/os/bsd/os_perf_bsd.cpp
+++ b/src/hotspot/os/bsd/os_perf_bsd.cpp
@@ -301,7 +301,7 @@ int SystemProcessInterface::SystemProcesses::system_processes(SystemProcess** sy
pids_bytes = proc_listpids(PROC_ALL_PIDS, 0, pids, pids_bytes);
if (pids_bytes <= 0) {
// couldn't fit buffer, retry.
- FREE_RESOURCE_ARRAY(pid_t, pids, pid_count);
+ FREE_RESOURCE_ARRAY(pids, pid_count);
pids = nullptr;
try_count++;
if (try_count > 3) {
@@ -381,12 +381,12 @@ CPUInformationInterface::~CPUInformationInterface() {
if (_cpu_info != nullptr) {
if (_cpu_info->cpu_name() != nullptr) {
const char* cpu_name = _cpu_info->cpu_name();
- FREE_C_HEAP_ARRAY(char, cpu_name);
+ FREE_C_HEAP_ARRAY(cpu_name);
_cpu_info->set_cpu_name(nullptr);
}
if (_cpu_info->cpu_description() != nullptr) {
const char* cpu_desc = _cpu_info->cpu_description();
- FREE_C_HEAP_ARRAY(char, cpu_desc);
+ FREE_C_HEAP_ARRAY(cpu_desc);
_cpu_info->set_cpu_description(nullptr);
}
delete _cpu_info;
diff --git a/src/hotspot/os/linux/cgroupSubsystem_linux.cpp b/src/hotspot/os/linux/cgroupSubsystem_linux.cpp
index 13a005591fb..4a2d75ecdf3 100644
--- a/src/hotspot/os/linux/cgroupSubsystem_linux.cpp
+++ b/src/hotspot/os/linux/cgroupSubsystem_linux.cpp
@@ -40,6 +40,8 @@
// Inlined from for portability.
#ifndef CGROUP2_SUPER_MAGIC
# define CGROUP2_SUPER_MAGIC 0x63677270
+#else
+ STATIC_ASSERT(CGROUP2_SUPER_MAGIC == 0x63677270);
#endif
// controller names have to match the *_IDX indices
diff --git a/src/hotspot/os/linux/hugepages.cpp b/src/hotspot/os/linux/hugepages.cpp
index 1340c470dff..b065f7b1496 100644
--- a/src/hotspot/os/linux/hugepages.cpp
+++ b/src/hotspot/os/linux/hugepages.cpp
@@ -35,7 +35,7 @@
#include
ExplicitHugePageSupport::ExplicitHugePageSupport() :
- _initialized{false}, _os_supported{}, _pre_allocated{}, _default_hugepage_size{SIZE_MAX}, _inconsistent{false} {}
+ _initialized{false}, _os_supported{}, _pre_allocated{}, _default_hugepage_size{0}, _inconsistent{false} {}
os::PageSizes ExplicitHugePageSupport::os_supported() const {
assert(_initialized, "Not initialized");
@@ -68,7 +68,7 @@ static size_t scan_default_hugepagesize() {
// format has been changed), we'll set largest page size to 0
FILE *fp = os::fopen("/proc/meminfo", "r");
- if (fp) {
+ if (fp != nullptr) {
while (!feof(fp)) {
int x = 0;
char buf[16];
@@ -81,7 +81,7 @@ static size_t scan_default_hugepagesize() {
// skip to next line
for (;;) {
int ch = fgetc(fp);
- if (ch == EOF || ch == (int)'\n') break;
+ if (ch == EOF || ch == '\n') break;
}
}
}
@@ -187,7 +187,7 @@ void ExplicitHugePageSupport::scan_os() {
}
THPSupport::THPSupport() :
- _initialized(false), _mode(THPMode::never), _pagesize(SIZE_MAX) {}
+ _initialized{false}, _mode{THPMode::never}, _pagesize{0} {}
THPMode THPSupport::mode() const {
@@ -221,7 +221,6 @@ void THPSupport::scan_os() {
}
// Scan large page size for THP from hpage_pmd_size
- _pagesize = 0;
if (read_number_file("/sys/kernel/mm/transparent_hugepage/hpage_pmd_size", &_pagesize)) {
assert(_pagesize > 0, "Expected");
}
diff --git a/src/hotspot/os/linux/os_linux.cpp b/src/hotspot/os/linux/os_linux.cpp
index c79b0ab9fb5..6927f5108ac 100644
--- a/src/hotspot/os/linux/os_linux.cpp
+++ b/src/hotspot/os/linux/os_linux.cpp
@@ -710,14 +710,14 @@ void os::init_system_properties_values() {
char *ld_library_path = NEW_C_HEAP_ARRAY(char, pathsize, mtInternal);
os::snprintf_checked(ld_library_path, pathsize, "%s%s" SYS_EXT_DIR "/lib:" DEFAULT_LIBPATH, v, v_colon);
Arguments::set_library_path(ld_library_path);
- FREE_C_HEAP_ARRAY(char, ld_library_path);
+ FREE_C_HEAP_ARRAY(ld_library_path);
}
// Extensions directories.
os::snprintf_checked(buf, bufsize, "%s" EXTENSIONS_DIR ":" SYS_EXT_DIR EXTENSIONS_DIR, Arguments::get_java_home());
Arguments::set_ext_dirs(buf);
- FREE_C_HEAP_ARRAY(char, buf);
+ FREE_C_HEAP_ARRAY(buf);
#undef DEFAULT_LIBPATH
#undef SYS_EXT_DIR
@@ -1313,7 +1313,7 @@ bool os::is_primordial_thread(void) {
// Find the virtual memory area that contains addr
static bool find_vma(address addr, address* vma_low, address* vma_high) {
FILE *fp = os::fopen("/proc/self/maps", "r");
- if (fp) {
+ if (fp != nullptr) {
address low, high;
while (!feof(fp)) {
if (fscanf(fp, "%p-%p", &low, &high) == 2) {
@@ -1326,7 +1326,7 @@ static bool find_vma(address addr, address* vma_low, address* vma_high) {
}
for (;;) {
int ch = fgetc(fp);
- if (ch == EOF || ch == (int)'\n') break;
+ if (ch == EOF || ch == '\n') break;
}
}
fclose(fp);
@@ -2163,8 +2163,6 @@ void os::print_os_info(outputStream* st) {
os::Posix::print_rlimit_info(st);
- os::print_open_file_descriptors(st);
-
os::Posix::print_load_average(st);
st->cr();
@@ -3437,7 +3435,7 @@ void os::Linux::rebuild_cpu_to_node_map() {
}
}
}
- FREE_C_HEAP_ARRAY(unsigned long, cpu_map);
+ FREE_C_HEAP_ARRAY(cpu_map);
}
int os::Linux::numa_node_to_cpus(int node, unsigned long *buffer, int bufferlen) {
@@ -4384,7 +4382,7 @@ int os::Linux::get_namespace_pid(int vmid) {
os::snprintf_checked(fname, sizeof(fname), "/proc/%d/status", vmid);
FILE *fp = os::fopen(fname, "r");
- if (fp) {
+ if (fp != nullptr) {
int pid, nspid;
int ret;
while (!feof(fp) && !ferror(fp)) {
@@ -4398,7 +4396,7 @@ int os::Linux::get_namespace_pid(int vmid) {
}
for (;;) {
int ch = fgetc(fp);
- if (ch == EOF || ch == (int)'\n') break;
+ if (ch == EOF || ch == '\n') break;
}
}
fclose(fp);
diff --git a/src/hotspot/os/linux/os_perf_linux.cpp b/src/hotspot/os/linux/os_perf_linux.cpp
index 9f91f3b4c0d..c0e863ed2a2 100644
--- a/src/hotspot/os/linux/os_perf_linux.cpp
+++ b/src/hotspot/os/linux/os_perf_linux.cpp
@@ -545,7 +545,7 @@ bool CPUPerformanceInterface::CPUPerformance::initialize() {
CPUPerformanceInterface::CPUPerformance::~CPUPerformance() {
if (_counters.cpus != nullptr) {
- FREE_C_HEAP_ARRAY(char, _counters.cpus);
+ FREE_C_HEAP_ARRAY(_counters.cpus);
}
}
@@ -811,7 +811,7 @@ int SystemProcessInterface::SystemProcesses::ProcessIterator::current(SystemProc
cmdline = get_cmdline();
if (cmdline != nullptr) {
process_info->set_command_line(allocate_string(cmdline));
- FREE_C_HEAP_ARRAY(char, cmdline);
+ FREE_C_HEAP_ARRAY(cmdline);
}
return OS_OK;
@@ -937,12 +937,12 @@ CPUInformationInterface::~CPUInformationInterface() {
if (_cpu_info != nullptr) {
if (_cpu_info->cpu_name() != nullptr) {
const char* cpu_name = _cpu_info->cpu_name();
- FREE_C_HEAP_ARRAY(char, cpu_name);
+ FREE_C_HEAP_ARRAY(cpu_name);
_cpu_info->set_cpu_name(nullptr);
}
if (_cpu_info->cpu_description() != nullptr) {
const char* cpu_desc = _cpu_info->cpu_description();
- FREE_C_HEAP_ARRAY(char, cpu_desc);
+ FREE_C_HEAP_ARRAY(cpu_desc);
_cpu_info->set_cpu_description(nullptr);
}
delete _cpu_info;
diff --git a/src/hotspot/os/linux/procMapsParser.cpp b/src/hotspot/os/linux/procMapsParser.cpp
index 0663cae61f3..00675683e34 100644
--- a/src/hotspot/os/linux/procMapsParser.cpp
+++ b/src/hotspot/os/linux/procMapsParser.cpp
@@ -45,7 +45,7 @@ ProcSmapsParser::ProcSmapsParser(FILE* f) :
}
ProcSmapsParser::~ProcSmapsParser() {
- FREE_C_HEAP_ARRAY(char, _line);
+ FREE_C_HEAP_ARRAY(_line);
}
bool ProcSmapsParser::read_line() {
diff --git a/src/hotspot/os/posix/include/jvm_md.h b/src/hotspot/os/posix/include/jvm_md.h
index eb8e1f0d7e9..061ef17aaae 100644
--- a/src/hotspot/os/posix/include/jvm_md.h
+++ b/src/hotspot/os/posix/include/jvm_md.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -62,6 +62,19 @@
#define JVM_X_OK X_OK
#define JVM_F_OK F_OK
+#if defined(AIX)
+#include "jni_md.h"
+#include "dl_info.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+JNIEXPORT int JVM_dladdr(void* addr, Dl_info* info);
+#ifdef __cplusplus
+}
+#endif
+#endif
+
/*
* File I/O
*/
diff --git a/src/hotspot/os/posix/perfMemory_posix.cpp b/src/hotspot/os/posix/perfMemory_posix.cpp
index b8be77c5e05..300c86ffc47 100644
--- a/src/hotspot/os/posix/perfMemory_posix.cpp
+++ b/src/hotspot/os/posix/perfMemory_posix.cpp
@@ -118,7 +118,7 @@ static void save_memory_to_file(char* addr, size_t size) {
}
}
}
- FREE_C_HEAP_ARRAY(char, destfile);
+ FREE_C_HEAP_ARRAY(destfile);
}
@@ -483,14 +483,14 @@ static char* get_user_name(uid_t uid) {
p->pw_name == nullptr ? "pw_name = null" : "pw_name zero length");
}
}
- FREE_C_HEAP_ARRAY(char, pwbuf);
+ FREE_C_HEAP_ARRAY(pwbuf);
return nullptr;
}
char* user_name = NEW_C_HEAP_ARRAY(char, strlen(p->pw_name) + 1, mtInternal);
strcpy(user_name, p->pw_name);
- FREE_C_HEAP_ARRAY(char, pwbuf);
+ FREE_C_HEAP_ARRAY(pwbuf);
return user_name;
}
@@ -572,7 +572,7 @@ static char* get_user_name_slow(int vmid, int nspid, TRAPS) {
DIR* subdirp = open_directory_secure(usrdir_name);
if (subdirp == nullptr) {
- FREE_C_HEAP_ARRAY(char, usrdir_name);
+ FREE_C_HEAP_ARRAY(usrdir_name);
continue;
}
@@ -583,7 +583,7 @@ static char* get_user_name_slow(int vmid, int nspid, TRAPS) {
// symlink can be exploited.
//
if (!is_directory_secure(usrdir_name)) {
- FREE_C_HEAP_ARRAY(char, usrdir_name);
+ FREE_C_HEAP_ARRAY(usrdir_name);
os::closedir(subdirp);
continue;
}
@@ -607,13 +607,13 @@ static char* get_user_name_slow(int vmid, int nspid, TRAPS) {
// don't follow symbolic links for the file
RESTARTABLE(::lstat(filename, &statbuf), result);
if (result == OS_ERR) {
- FREE_C_HEAP_ARRAY(char, filename);
+ FREE_C_HEAP_ARRAY(filename);
continue;
}
// skip over files that are not regular files.
if (!S_ISREG(statbuf.st_mode)) {
- FREE_C_HEAP_ARRAY(char, filename);
+ FREE_C_HEAP_ARRAY(filename);
continue;
}
@@ -623,7 +623,7 @@ static char* get_user_name_slow(int vmid, int nspid, TRAPS) {
if (statbuf.st_ctime > oldest_ctime) {
char* user = strchr(dentry->d_name, '_') + 1;
- FREE_C_HEAP_ARRAY(char, oldest_user);
+ FREE_C_HEAP_ARRAY(oldest_user);
oldest_user = NEW_C_HEAP_ARRAY(char, strlen(user)+1, mtInternal);
strcpy(oldest_user, user);
@@ -631,11 +631,11 @@ static char* get_user_name_slow(int vmid, int nspid, TRAPS) {
}
}
- FREE_C_HEAP_ARRAY(char, filename);
+ FREE_C_HEAP_ARRAY(filename);
}
}
os::closedir(subdirp);
- FREE_C_HEAP_ARRAY(char, usrdir_name);
+ FREE_C_HEAP_ARRAY(usrdir_name);
}
os::closedir(tmpdirp);
@@ -701,6 +701,39 @@ static void remove_file(const char* path) {
}
}
+// Files newer than this threshold are considered to belong to a JVM that may
+// still be starting up and are therefore not candidates for stale-file
+// cleanup. This avoids racing a concurrent JVM startup while scanning the
+// hsperfdata directory.
+static const time_t cleanup_grace_period_seconds = 5;
+
+static bool is_cleanup_candidate(const char* filename, const char* dirname) {
+ struct stat statbuf;
+ int result;
+
+ RESTARTABLE(::lstat(filename, &statbuf), result);
+ if (result == OS_ERR) {
+ log_debug(perf, memops)("lstat failed for %s/%s: %s", dirname, filename, os::strerror(errno));
+ return false;
+ }
+
+ if (!S_ISREG(statbuf.st_mode)) {
+ return false;
+ }
+
+ const time_t now = time(nullptr);
+ if (now == (time_t)-1) {
+ return false;
+ }
+
+ if (statbuf.st_mtime >= now - cleanup_grace_period_seconds) {
+ log_debug(perf, memops)("Skip cleanup of fresh file %s/%s", dirname, filename);
+ return false;
+ }
+
+ return true;
+}
+
// cleanup stale shared memory files
//
// This method attempts to remove all stale shared memory files in
@@ -744,6 +777,11 @@ static void cleanup_sharedmem_files(const char* dirname) {
continue;
}
+ if (!is_cleanup_candidate(filename, dirname)) {
+ errno = 0;
+ continue;
+ }
+
#if defined(LINUX)
// Special case on Linux, if multiple containers share the
// same /tmp directory:
@@ -872,16 +910,56 @@ static int create_sharedmem_file(const char* dirname, const char* filename, size
return -1;
}
- // Open the filename in the current directory.
- // Cannot use O_TRUNC here; truncation of an existing file has to happen
- // after the is_file_secure() check below.
- int fd;
- RESTARTABLE(os::open(filename, O_RDWR|O_CREAT|O_NOFOLLOW, S_IRUSR|S_IWUSR), fd);
+ int fd = OS_ERR;
+ static const int create_sharedmem_file_retry_count = LINUX_ONLY(3) NOT_LINUX(1);
+ for (int attempt = 0; attempt < create_sharedmem_file_retry_count; attempt++) {
+ // Open the filename in the current directory.
+ // Use O_EXCL so that startup never reuses an existing pid file unless it
+ // has first been proven stale and removed in `cleanup_sharedmem_files`.
+ RESTARTABLE(os::open(filename, O_RDWR|O_CREAT|O_EXCL|O_NOFOLLOW, S_IRUSR|S_IWUSR), fd);
+ if (fd == OS_ERR) {
+ break;
+ }
+
+#if defined(LINUX)
+ // On Linux, different containerized processes that share the same /tmp
+ // directory (e.g., with "docker --volume ...") may have the same pid and
+ // try to use the same file. To avoid conflicts among such processes, we
+ // allow only one of them (the winner of the flock() call) to write to the
+ // file. If we lose the race, assume we may have collided with a concurrent
+ // scavenger briefly holding the lock on a fresh file and retry a few times
+ // before giving up.
+ int n;
+ RESTARTABLE(::flock(fd, LOCK_EX|LOCK_NB), n);
+ if (n == 0) {
+ break;
+ }
+
+ const int flock_errno = errno;
+ ::close(fd);
+ fd = OS_ERR;
+
+ if (attempt + 1 == create_sharedmem_file_retry_count || flock_errno != EWOULDBLOCK) {
+ log_warning(perf, memops)("Cannot use file %s/%s because %s (errno = %d)", dirname, filename,
+ (flock_errno == EWOULDBLOCK) ?
+ "it is locked by another process" :
+ "flock() failed", flock_errno);
+ errno = flock_errno;
+ break;
+ }
+
+ // Short sleep to allow the lock to free up.
+ os::naked_short_sleep(1);
+#endif
+ }
+
if (fd == OS_ERR) {
if (log_is_enabled(Debug, perf)) {
LogStreamHandle(Debug, perf) log;
if (errno == ELOOP) {
log.print_cr("file %s is a symlink and is not secure", filename);
+ } else if (errno == EEXIST) {
+ log.print_cr("could not create file %s: existing file is not provably stale", filename);
} else {
log.print_cr("could not create file %s: %s", filename, os::strerror(errno));
}
@@ -901,27 +979,7 @@ static int create_sharedmem_file(const char* dirname, const char* filename, size
}
#if defined(LINUX)
- // On Linux, different containerized processes that share the same /tmp
- // directory (e.g., with "docker --volume ...") may have the same pid and
- // try to use the same file. To avoid conflicts among such
- // processes, we allow only one of them (the winner of the flock() call)
- // to write to the file. All the other processes will give up and will
- // have perfdata disabled.
- //
- // Note that the flock will be automatically given up when the winner
- // process exits.
- //
- // The locking protocol works only with other JVMs that have the JDK-8286030
- // fix. If you are sharing the /tmp difrectory among different containers,
- // do not use older JVMs that don't have this fix, or the behavior is undefined.
- int n;
- RESTARTABLE(::flock(fd, LOCK_EX|LOCK_NB), n);
- if (n != 0) {
- log_warning(perf, memops)("Cannot use file %s/%s because %s (errno = %d)", dirname, filename,
- (errno == EWOULDBLOCK) ?
- "it is locked by another process" :
- "flock() failed", errno);
- ::close(fd);
+ if (fd == OS_ERR) {
return -1;
}
#endif
@@ -1047,11 +1105,11 @@ static char* mmap_create_shared(size_t size) {
log_info(perf, memops)("Trying to open %s/%s", dirname, short_filename);
fd = create_sharedmem_file(dirname, short_filename, size);
- FREE_C_HEAP_ARRAY(char, user_name);
- FREE_C_HEAP_ARRAY(char, dirname);
+ FREE_C_HEAP_ARRAY(user_name);
+ FREE_C_HEAP_ARRAY(dirname);
if (fd == -1) {
- FREE_C_HEAP_ARRAY(char, filename);
+ FREE_C_HEAP_ARRAY(filename);
return nullptr;
}
@@ -1063,7 +1121,7 @@ static char* mmap_create_shared(size_t size) {
if (mapAddress == MAP_FAILED) {
log_debug(perf)("mmap failed - %s", os::strerror(errno));
remove_file(filename);
- FREE_C_HEAP_ARRAY(char, filename);
+ FREE_C_HEAP_ARRAY(filename);
return nullptr;
}
@@ -1113,7 +1171,7 @@ static void delete_shared_memory(char* addr, size_t size) {
remove_file(backing_store_file_name);
// Don't.. Free heap memory could deadlock os::abort() if it is called
// from signal handler. OS will reclaim the heap memory.
- // FREE_C_HEAP_ARRAY(char, backing_store_file_name);
+ // FREE_C_HEAP_ARRAY(backing_store_file_name);
backing_store_file_name = nullptr;
}
}
@@ -1165,8 +1223,8 @@ static void mmap_attach_shared(int vmid, char** addr, size_t* sizep, TRAPS) {
// store file, we don't follow them when attaching either.
//
if (!is_directory_secure(dirname)) {
- FREE_C_HEAP_ARRAY(char, dirname);
- FREE_C_HEAP_ARRAY(char, luser);
+ FREE_C_HEAP_ARRAY(dirname);
+ FREE_C_HEAP_ARRAY(luser);
THROW_MSG(vmSymbols::java_lang_IllegalArgumentException(),
"Process not found");
}
@@ -1178,9 +1236,9 @@ static void mmap_attach_shared(int vmid, char** addr, size_t* sizep, TRAPS) {
int fd = open_sharedmem_file(filename, file_flags, THREAD);
// free the c heap resources that are no longer needed
- FREE_C_HEAP_ARRAY(char, luser);
- FREE_C_HEAP_ARRAY(char, dirname);
- FREE_C_HEAP_ARRAY(char, filename);
+ FREE_C_HEAP_ARRAY(luser);
+ FREE_C_HEAP_ARRAY(dirname);
+ FREE_C_HEAP_ARRAY(filename);
if (HAS_PENDING_EXCEPTION) {
assert(fd == OS_ERR, "open_sharedmem_file always return OS_ERR on exceptions");
diff --git a/src/hotspot/os/windows/os_perf_windows.cpp b/src/hotspot/os/windows/os_perf_windows.cpp
index 9d04ae65954..59ea83b9148 100644
--- a/src/hotspot/os/windows/os_perf_windows.cpp
+++ b/src/hotspot/os/windows/os_perf_windows.cpp
@@ -178,9 +178,9 @@ static void destroy(MultiCounterQueryP query) {
for (int i = 0; i < query->noOfCounters; ++i) {
close_query(nullptr, &query->counters[i]);
}
- FREE_C_HEAP_ARRAY(char, query->counters);
+ FREE_C_HEAP_ARRAY(query->counters);
close_query(&query->query.pdh_query_handle, nullptr);
- FREE_C_HEAP_ARRAY(MultiCounterQueryS, query);
+ FREE_C_HEAP_ARRAY(query);
}
}
@@ -189,15 +189,15 @@ static void destroy_query_set(MultiCounterQuerySetP query_set) {
for (int j = 0; j < query_set->queries[i].noOfCounters; ++j) {
close_query(nullptr, &query_set->queries[i].counters[j]);
}
- FREE_C_HEAP_ARRAY(char, query_set->queries[i].counters);
+ FREE_C_HEAP_ARRAY(query_set->queries[i].counters);
close_query(&query_set->queries[i].query.pdh_query_handle, nullptr);
}
- FREE_C_HEAP_ARRAY(MultiCounterQueryS, query_set->queries);
+ FREE_C_HEAP_ARRAY(query_set->queries);
}
static void destroy(MultiCounterQuerySetP query) {
destroy_query_set(query);
- FREE_C_HEAP_ARRAY(MultiCounterQuerySetS, query);
+ FREE_C_HEAP_ARRAY(query);
}
static void destroy(ProcessQueryP query) {
@@ -229,7 +229,7 @@ static void allocate_counters(ProcessQueryP query, size_t nofCounters) {
}
static void deallocate_counters(MultiCounterQueryP query) {
- FREE_C_HEAP_ARRAY(char, query->counters);
+ FREE_C_HEAP_ARRAY(query->counters);
query->counters = nullptr;
query->noOfCounters = 0;
}
@@ -710,11 +710,11 @@ static const char* pdh_process_image_name() {
}
static void deallocate_pdh_constants() {
- FREE_C_HEAP_ARRAY(char, process_image_name);
+ FREE_C_HEAP_ARRAY(process_image_name);
process_image_name = nullptr;
- FREE_C_HEAP_ARRAY(char, pdh_process_instance_IDProcess_counter_fmt);
+ FREE_C_HEAP_ARRAY(pdh_process_instance_IDProcess_counter_fmt);
pdh_process_instance_IDProcess_counter_fmt = nullptr;
- FREE_C_HEAP_ARRAY(char, pdh_process_instance_wildcard_IDProcess_counter);
+ FREE_C_HEAP_ARRAY(pdh_process_instance_wildcard_IDProcess_counter);
pdh_process_instance_wildcard_IDProcess_counter = nullptr;
}
@@ -1445,9 +1445,9 @@ bool CPUInformationInterface::initialize() {
CPUInformationInterface::~CPUInformationInterface() {
if (_cpu_info != nullptr) {
- FREE_C_HEAP_ARRAY(char, _cpu_info->cpu_name());
+ FREE_C_HEAP_ARRAY(_cpu_info->cpu_name());
_cpu_info->set_cpu_name(nullptr);
- FREE_C_HEAP_ARRAY(char, _cpu_info->cpu_description());
+ FREE_C_HEAP_ARRAY(_cpu_info->cpu_description());
_cpu_info->set_cpu_description(nullptr);
delete _cpu_info;
}
diff --git a/src/hotspot/os/windows/os_windows.cpp b/src/hotspot/os/windows/os_windows.cpp
index 18d047348cb..9a987bf3762 100644
--- a/src/hotspot/os/windows/os_windows.cpp
+++ b/src/hotspot/os/windows/os_windows.cpp
@@ -334,14 +334,14 @@ void os::init_system_properties_values() {
home_path = NEW_C_HEAP_ARRAY(char, strlen(home_dir) + 1, mtInternal);
strcpy(home_path, home_dir);
Arguments::set_java_home(home_path);
- FREE_C_HEAP_ARRAY(char, home_path);
+ FREE_C_HEAP_ARRAY(home_path);
dll_path = NEW_C_HEAP_ARRAY(char, strlen(home_dir) + strlen(bin) + 1,
mtInternal);
strcpy(dll_path, home_dir);
strcat(dll_path, bin);
Arguments::set_dll_dir(dll_path);
- FREE_C_HEAP_ARRAY(char, dll_path);
+ FREE_C_HEAP_ARRAY(dll_path);
if (!set_boot_path('\\', ';')) {
vm_exit_during_initialization("Failed setting boot class path.", nullptr);
@@ -396,7 +396,7 @@ void os::init_system_properties_values() {
strcat(library_path, ";.");
Arguments::set_library_path(library_path);
- FREE_C_HEAP_ARRAY(char, library_path);
+ FREE_C_HEAP_ARRAY(library_path);
}
// Default extensions directory
@@ -1079,7 +1079,7 @@ void os::set_native_thread_name(const char *name) {
HRESULT hr = _SetThreadDescription(current, unicode_name);
if (FAILED(hr)) {
log_debug(os, thread)("set_native_thread_name: SetThreadDescription failed - falling back to debugger method");
- FREE_C_HEAP_ARRAY(WCHAR, unicode_name);
+ FREE_C_HEAP_ARRAY(unicode_name);
} else {
log_trace(os, thread)("set_native_thread_name: SetThreadDescription succeeded - new name: %s", name);
@@ -1102,7 +1102,7 @@ void os::set_native_thread_name(const char *name) {
LocalFree(thread_name);
}
#endif
- FREE_C_HEAP_ARRAY(WCHAR, unicode_name);
+ FREE_C_HEAP_ARRAY(unicode_name);
return;
}
} else {
@@ -2528,12 +2528,6 @@ LONG Handle_Exception(struct _EXCEPTION_POINTERS* exceptionInfo,
return EXCEPTION_CONTINUE_EXECUTION;
}
-
-// Used for PostMortemDump
-extern "C" void safepoints();
-extern "C" void find(int x);
-extern "C" void events();
-
// According to Windows API documentation, an illegal instruction sequence should generate
// the 0xC000001C exception code. However, real world experience shows that occasionnaly
// the execution of an illegal instruction can generate the exception code 0xC000001E. This
@@ -2903,7 +2897,7 @@ class NUMANodeListHolder {
int _numa_used_node_count;
void free_node_list() {
- FREE_C_HEAP_ARRAY(int, _numa_used_node_list);
+ FREE_C_HEAP_ARRAY(_numa_used_node_list);
}
public:
@@ -4750,7 +4744,7 @@ static wchar_t* wide_abs_unc_path(char const* path, errno_t & err, int additiona
LPWSTR unicode_path = nullptr;
err = convert_to_unicode(buf, &unicode_path);
- FREE_C_HEAP_ARRAY(char, buf);
+ FREE_C_HEAP_ARRAY(buf);
if (err != ERROR_SUCCESS) {
return nullptr;
}
@@ -4778,9 +4772,9 @@ static wchar_t* wide_abs_unc_path(char const* path, errno_t & err, int additiona
}
if (converted_path != unicode_path) {
- FREE_C_HEAP_ARRAY(WCHAR, converted_path);
+ FREE_C_HEAP_ARRAY(converted_path);
}
- FREE_C_HEAP_ARRAY(WCHAR, unicode_path);
+ FREE_C_HEAP_ARRAY(unicode_path);
return static_cast(result); // LPWSTR and wchat_t* are the same type on Windows.
}
@@ -5833,7 +5827,7 @@ int os::fork_and_exec(const char* cmd) {
exit_code = -1;
}
- FREE_C_HEAP_ARRAY(char, cmd_string);
+ FREE_C_HEAP_ARRAY(cmd_string);
return (int)exit_code;
}
diff --git a/src/hotspot/os/windows/perfMemory_windows.cpp b/src/hotspot/os/windows/perfMemory_windows.cpp
index dad2804f18a..8e698c53d28 100644
--- a/src/hotspot/os/windows/perfMemory_windows.cpp
+++ b/src/hotspot/os/windows/perfMemory_windows.cpp
@@ -113,7 +113,7 @@ static void save_memory_to_file(char* addr, size_t size) {
}
}
- FREE_C_HEAP_ARRAY(char, destfile);
+ FREE_C_HEAP_ARRAY(destfile);
}
// Shared Memory Implementation Details
@@ -319,7 +319,7 @@ static char* get_user_name_slow(int vmid) {
DIR* subdirp = os::opendir(usrdir_name);
if (subdirp == nullptr) {
- FREE_C_HEAP_ARRAY(char, usrdir_name);
+ FREE_C_HEAP_ARRAY(usrdir_name);
continue;
}
@@ -330,7 +330,7 @@ static char* get_user_name_slow(int vmid) {
// symlink can be exploited.
//
if (!is_directory_secure(usrdir_name)) {
- FREE_C_HEAP_ARRAY(char, usrdir_name);
+ FREE_C_HEAP_ARRAY(usrdir_name);
os::closedir(subdirp);
continue;
}
@@ -350,13 +350,13 @@ static char* get_user_name_slow(int vmid) {
strcat(filename, udentry->d_name);
if (::stat(filename, &statbuf) == OS_ERR) {
- FREE_C_HEAP_ARRAY(char, filename);
+ FREE_C_HEAP_ARRAY(filename);
continue;
}
// skip over files that are not regular files.
if ((statbuf.st_mode & S_IFMT) != S_IFREG) {
- FREE_C_HEAP_ARRAY(char, filename);
+ FREE_C_HEAP_ARRAY(filename);
continue;
}
@@ -378,18 +378,18 @@ static char* get_user_name_slow(int vmid) {
if (statbuf.st_ctime > latest_ctime) {
char* user = strchr(dentry->d_name, '_') + 1;
- FREE_C_HEAP_ARRAY(char, latest_user);
+ FREE_C_HEAP_ARRAY(latest_user);
latest_user = NEW_C_HEAP_ARRAY(char, strlen(user)+1, mtInternal);
strcpy(latest_user, user);
latest_ctime = statbuf.st_ctime;
}
- FREE_C_HEAP_ARRAY(char, filename);
+ FREE_C_HEAP_ARRAY(filename);
}
}
os::closedir(subdirp);
- FREE_C_HEAP_ARRAY(char, usrdir_name);
+ FREE_C_HEAP_ARRAY(usrdir_name);
}
os::closedir(tmpdirp);
@@ -481,7 +481,7 @@ static void remove_file(const char* dirname, const char* filename) {
}
}
- FREE_C_HEAP_ARRAY(char, path);
+ FREE_C_HEAP_ARRAY(path);
}
// returns true if the process represented by pid is alive, otherwise
@@ -708,11 +708,11 @@ static void free_security_desc(PSECURITY_DESCRIPTOR pSD) {
// be an ACL we enlisted. free the resources.
//
if (success && exists && pACL != nullptr && !isdefault) {
- FREE_C_HEAP_ARRAY(char, pACL);
+ FREE_C_HEAP_ARRAY(pACL);
}
// free the security descriptor
- FREE_C_HEAP_ARRAY(char, pSD);
+ FREE_C_HEAP_ARRAY(pSD);
}
}
@@ -768,7 +768,7 @@ static PSID get_user_sid(HANDLE hProcess) {
if (!GetTokenInformation(hAccessToken, TokenUser, token_buf, rsize, &rsize)) {
log_debug(perf)("GetTokenInformation failure: lasterror = %d, rsize = %d",
GetLastError(), rsize);
- FREE_C_HEAP_ARRAY(char, token_buf);
+ FREE_C_HEAP_ARRAY(token_buf);
CloseHandle(hAccessToken);
return nullptr;
}
@@ -779,15 +779,15 @@ static PSID get_user_sid(HANDLE hProcess) {
if (!CopySid(nbytes, pSID, token_buf->User.Sid)) {
log_debug(perf)("GetTokenInformation failure: lasterror = %d, rsize = %d",
GetLastError(), rsize);
- FREE_C_HEAP_ARRAY(char, token_buf);
- FREE_C_HEAP_ARRAY(char, pSID);
+ FREE_C_HEAP_ARRAY(token_buf);
+ FREE_C_HEAP_ARRAY(pSID);
CloseHandle(hAccessToken);
return nullptr;
}
// close the access token.
CloseHandle(hAccessToken);
- FREE_C_HEAP_ARRAY(char, token_buf);
+ FREE_C_HEAP_ARRAY(token_buf);
return pSID;
}
@@ -865,7 +865,7 @@ static bool add_allow_aces(PSECURITY_DESCRIPTOR pSD,
if (!InitializeAcl(newACL, newACLsize, ACL_REVISION)) {
log_debug(perf)("InitializeAcl failure: lasterror = %d", GetLastError());
- FREE_C_HEAP_ARRAY(char, newACL);
+ FREE_C_HEAP_ARRAY(newACL);
return false;
}
@@ -876,7 +876,7 @@ static bool add_allow_aces(PSECURITY_DESCRIPTOR pSD,
LPVOID ace;
if (!GetAce(oldACL, ace_index, &ace)) {
log_debug(perf)("InitializeAcl failure: lasterror = %d", GetLastError());
- FREE_C_HEAP_ARRAY(char, newACL);
+ FREE_C_HEAP_ARRAY(newACL);
return false;
}
if (((ACCESS_ALLOWED_ACE *)ace)->Header.AceFlags && INHERITED_ACE) {
@@ -901,7 +901,7 @@ static bool add_allow_aces(PSECURITY_DESCRIPTOR pSD,
if (!AddAce(newACL, ACL_REVISION, MAXDWORD, ace,
((PACE_HEADER)ace)->AceSize)) {
log_debug(perf)("AddAce failure: lasterror = %d", GetLastError());
- FREE_C_HEAP_ARRAY(char, newACL);
+ FREE_C_HEAP_ARRAY(newACL);
return false;
}
}
@@ -915,7 +915,7 @@ static bool add_allow_aces(PSECURITY_DESCRIPTOR pSD,
aces[i].mask, aces[i].pSid)) {
log_debug(perf)("AddAccessAllowedAce failure: lasterror = %d",
GetLastError());
- FREE_C_HEAP_ARRAY(char, newACL);
+ FREE_C_HEAP_ARRAY(newACL);
return false;
}
}
@@ -928,13 +928,13 @@ static bool add_allow_aces(PSECURITY_DESCRIPTOR pSD,
LPVOID ace;
if (!GetAce(oldACL, ace_index, &ace)) {
log_debug(perf)("InitializeAcl failure: lasterror = %d", GetLastError());
- FREE_C_HEAP_ARRAY(char, newACL);
+ FREE_C_HEAP_ARRAY(newACL);
return false;
}
if (!AddAce(newACL, ACL_REVISION, MAXDWORD, ace,
((PACE_HEADER)ace)->AceSize)) {
log_debug(perf)("AddAce failure: lasterror = %d", GetLastError());
- FREE_C_HEAP_ARRAY(char, newACL);
+ FREE_C_HEAP_ARRAY(newACL);
return false;
}
ace_index++;
@@ -944,7 +944,7 @@ static bool add_allow_aces(PSECURITY_DESCRIPTOR pSD,
// add the new ACL to the security descriptor.
if (!SetSecurityDescriptorDacl(pSD, TRUE, newACL, FALSE)) {
log_debug(perf)("SetSecurityDescriptorDacl failure: lasterror = %d", GetLastError());
- FREE_C_HEAP_ARRAY(char, newACL);
+ FREE_C_HEAP_ARRAY(newACL);
return false;
}
@@ -952,7 +952,7 @@ static bool add_allow_aces(PSECURITY_DESCRIPTOR pSD,
// protected prevents that.
if (!SetSecurityDescriptorControl(pSD, SE_DACL_PROTECTED, SE_DACL_PROTECTED)) {
log_debug(perf)("SetSecurityDescriptorControl failure: lasterror = %d", GetLastError());
- FREE_C_HEAP_ARRAY(char, newACL);
+ FREE_C_HEAP_ARRAY(newACL);
return false;
}
@@ -1057,7 +1057,7 @@ static LPSECURITY_ATTRIBUTES make_user_everybody_admin_security_attr(
// create a security attributes structure with access control
// entries as initialized above.
LPSECURITY_ATTRIBUTES lpSA = make_security_attr(aces, 3);
- FREE_C_HEAP_ARRAY(char, aces[0].pSid);
+ FREE_C_HEAP_ARRAY(aces[0].pSid);
FreeSid(everybodySid);
FreeSid(administratorsSid);
return(lpSA);
@@ -1341,8 +1341,8 @@ static char* mapping_create_shared(size_t size) {
// check that the file system is secure - i.e. it supports ACLs.
if (!is_filesystem_secure(dirname)) {
- FREE_C_HEAP_ARRAY(char, dirname);
- FREE_C_HEAP_ARRAY(char, user);
+ FREE_C_HEAP_ARRAY(dirname);
+ FREE_C_HEAP_ARRAY(user);
return nullptr;
}
@@ -1358,15 +1358,15 @@ static char* mapping_create_shared(size_t size) {
assert(((size != 0) && (size % os::vm_page_size() == 0)),
"unexpected PerfMemry region size");
- FREE_C_HEAP_ARRAY(char, user);
+ FREE_C_HEAP_ARRAY(user);
// create the shared memory resources
sharedmem_fileMapHandle =
create_sharedmem_resources(dirname, filename, objectname, size);
- FREE_C_HEAP_ARRAY(char, filename);
- FREE_C_HEAP_ARRAY(char, objectname);
- FREE_C_HEAP_ARRAY(char, dirname);
+ FREE_C_HEAP_ARRAY(filename);
+ FREE_C_HEAP_ARRAY(objectname);
+ FREE_C_HEAP_ARRAY(dirname);
if (sharedmem_fileMapHandle == nullptr) {
return nullptr;
@@ -1480,8 +1480,8 @@ static void open_file_mapping(int vmid, char** addrp, size_t* sizep, TRAPS) {
// store file, we also don't following them when attaching
//
if (!is_directory_secure(dirname)) {
- FREE_C_HEAP_ARRAY(char, dirname);
- FREE_C_HEAP_ARRAY(char, luser);
+ FREE_C_HEAP_ARRAY(dirname);
+ FREE_C_HEAP_ARRAY(luser);
THROW_MSG(vmSymbols::java_lang_IllegalArgumentException(),
"Process not found");
}
@@ -1498,10 +1498,10 @@ static void open_file_mapping(int vmid, char** addrp, size_t* sizep, TRAPS) {
char* robjectname = ResourceArea::strdup(THREAD, objectname);
// free the c heap resources that are no longer needed
- FREE_C_HEAP_ARRAY(char, luser);
- FREE_C_HEAP_ARRAY(char, dirname);
- FREE_C_HEAP_ARRAY(char, filename);
- FREE_C_HEAP_ARRAY(char, objectname);
+ FREE_C_HEAP_ARRAY(luser);
+ FREE_C_HEAP_ARRAY(dirname);
+ FREE_C_HEAP_ARRAY(filename);
+ FREE_C_HEAP_ARRAY(objectname);
size_t size;
if (*sizep == 0) {
diff --git a/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp b/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp
index 36599594842..49d879731ff 100644
--- a/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp
+++ b/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp
@@ -620,6 +620,8 @@ extern "C" {
assert(VM_Version::supports_sb(), "current CPU does not support SB instruction");
asm volatile(".inst 0xd50330ff" : : : "memory");
break;
+ case SpinWait::WFET:
+ ShouldNotReachHere();
#ifdef ASSERT
default:
ShouldNotReachHere();
diff --git a/src/hotspot/os_cpu/linux_aarch64/ic_ivau_probe_linux_aarch64.S b/src/hotspot/os_cpu/linux_aarch64/ic_ivau_probe_linux_aarch64.S
new file mode 100644
index 00000000000..b82053d37b9
--- /dev/null
+++ b/src/hotspot/os_cpu/linux_aarch64/ic_ivau_probe_linux_aarch64.S
@@ -0,0 +1,69 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "defs.S.inc"
+
+ # Probe whether IC IVAU is trapped.
+ #
+ # Returns 1 if IC IVAU is trapped (did not fault), 0 if not trapped
+ # (faulted on VA 0x0, signal handler redirected to continuation).
+ #
+ # int ic_ivau_probe(void);
+DECLARE_FUNC(ic_ivau_probe):
+DECLARE_FUNC(_ic_ivau_probe_fault):
+ ic ivau, xzr
+ mov x0, #1
+ ret
+DECLARE_FUNC(_ic_ivau_probe_continuation):
+ mov x0, #0
+ ret
+
+/* Emit .note.gnu.property section in case of PAC or BTI being enabled. */
+#ifdef __ARM_FEATURE_BTI_DEFAULT
+ #ifdef __ARM_FEATURE_PAC_DEFAULT
+ #define GNU_PROPERTY_AARCH64_FEATURE 3
+ #else
+ #define GNU_PROPERTY_AARCH64_FEATURE 1
+ #endif
+#else
+ #ifdef __ARM_FEATURE_PAC_DEFAULT
+ #define GNU_PROPERTY_AARCH64_FEATURE 2
+ #else
+ #define GNU_PROPERTY_AARCH64_FEATURE 0
+ #endif
+#endif
+
+#if (GNU_PROPERTY_AARCH64_FEATURE != 0)
+ .pushsection .note.gnu.property, "a"
+ .align 3
+ .long 4 /* name length */
+ .long 0x10 /* data length */
+ .long 5 /* note type: NT_GNU_PROPERTY_TYPE_0 */
+ .string "GNU" /* vendor name */
+ .long 0xc0000000 /* GNU_PROPERTY_AARCH64_FEATURE_1_AND */
+ .long 4 /* pr_datasze */
+ .long GNU_PROPERTY_AARCH64_FEATURE
+ .long 0
+ .popsection
+#endif
diff --git a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp
new file mode 100644
index 00000000000..11911a48e06
--- /dev/null
+++ b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.cpp
@@ -0,0 +1,28 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "runtime/icache.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+DEBUG_ONLY(THREAD_LOCAL AArch64ICacheInvalidationContext* AArch64ICacheInvalidationContext::_current_context = nullptr;)
diff --git a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp
index 8fbaa7a6b6e..5121a875701 100644
--- a/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp
+++ b/src/hotspot/os_cpu/linux_aarch64/icache_linux_aarch64.hpp
@@ -26,6 +26,11 @@
#ifndef OS_CPU_LINUX_AARCH64_ICACHE_AARCH64_HPP
#define OS_CPU_LINUX_AARCH64_ICACHE_AARCH64_HPP
+#include "memory/allocation.hpp"
+#include "runtime/vm_version.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "vm_version_aarch64.hpp"
+
// Interface for updating the instruction cache. Whenever the VM
// modifies code, part of the processor instruction cache potentially
// has to be flushed.
@@ -37,8 +42,105 @@ class ICache : public AbstractICache {
__builtin___clear_cache((char *)addr, (char *)(addr + 4));
}
static void invalidate_range(address start, int nbytes) {
- __builtin___clear_cache((char *)start, (char *)(start + nbytes));
+ if (NeoverseN1ICacheErratumMitigation) {
+ assert(VM_Version::is_cache_idc_enabled(),
+ "Expect CTR_EL0.IDC to be enabled for Neoverse N1 with erratum "
+ "1542419");
+ assert(!VM_Version::is_cache_dic_enabled(),
+ "Expect CTR_EL0.DIC to be disabled for Neoverse N1 with erratum "
+ "1542419");
+ assert(VM_Version::is_ic_ivau_trapped(), "Expect 'ic ivau, xzr' to be trapped");
+ asm volatile("dsb ish \n"
+ "ic ivau, xzr \n"
+ "dsb ish \n"
+ "isb \n"
+ : : : "memory");
+ } else {
+ __builtin___clear_cache((char *)start, (char *)(start + nbytes));
+ }
}
};
+class AArch64ICacheInvalidationContext : StackObj {
+ private:
+
+#ifdef ASSERT
+ static THREAD_LOCAL AArch64ICacheInvalidationContext* _current_context;
+#endif
+
+ bool _has_modified_code;
+
+ public:
+ NONCOPYABLE(AArch64ICacheInvalidationContext);
+
+ AArch64ICacheInvalidationContext()
+ : _has_modified_code(false) {
+ assert(_current_context == nullptr, "nested ICacheInvalidationContext not supported");
+#ifdef ASSERT
+ _current_context = this;
+#endif
+ }
+
+ ~AArch64ICacheInvalidationContext() {
+ DEBUG_ONLY(_current_context = nullptr);
+
+ if (!_has_modified_code || !UseSingleICacheInvalidation) {
+ return;
+ }
+
+ assert(VM_Version::is_cache_idc_enabled(), "Expect CTR_EL0.IDC to be enabled");
+
+ asm volatile("dsb ish" : : : "memory");
+
+ if (NeoverseN1ICacheErratumMitigation) {
+ assert(!VM_Version::is_cache_dic_enabled(),
+ "Expect CTR_EL0.DIC to be disabled for Neoverse N1 with erratum "
+ "1542419");
+ assert(VM_Version::is_ic_ivau_trapped(), "Expect 'ic ivau, xzr' to be trapped");
+
+ // Errata 1542419: Neoverse N1 cores with the 'COHERENT_ICACHE' feature
+ // may fetch stale instructions when software depends on
+ // prefetch-speculation-protection instead of explicit synchronization.
+ //
+ // Neoverse-N1 implementation mitigates the errata 1542419 with a
+ // workaround:
+ // - Disable coherent icache.
+ // - Trap IC IVAU instructions.
+ // - Execute:
+ // - tlbi vae3is, xzr
+ // - dsb sy
+ // - Ignore trapped IC IVAU instructions.
+ //
+ // `tlbi vae3is, xzr` invalidates all translation entries (all VAs, all
+ // possible levels). It waits for all memory accesses using in-scope old
+ // translation information to complete before it is considered complete.
+ //
+ // As this workaround has significant overhead, Arm Neoverse N1 (MP050)
+ // Software Developer Errata Notice version 29.0 suggests:
+ //
+ // "Since one TLB inner-shareable invalidation is enough to avoid this
+ // erratum, the number of injected TLB invalidations should be minimized
+ // in the trap handler to mitigate the performance impact due to this
+ // workaround."
+ // As the address for icache invalidation is not relevant and
+ // IC IVAU instruction is ignored, we use XZR in it.
+ asm volatile(
+ "ic ivau, xzr \n"
+ "dsb ish \n"
+ :
+ :
+ : "memory");
+ } else {
+ assert(VM_Version::is_cache_dic_enabled(), "Expect CTR_EL0.DIC to be enabled");
+ }
+ asm volatile("isb" : : : "memory");
+ }
+
+ void set_has_modified_code() {
+ _has_modified_code = true;
+ }
+};
+
+#define PD_ICACHE_INVALIDATION_CONTEXT AArch64ICacheInvalidationContext
+
#endif // OS_CPU_LINUX_AARCH64_ICACHE_AARCH64_HPP
diff --git a/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp
index da9e7e159f1..67e0569bf31 100644
--- a/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp
+++ b/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp
@@ -77,6 +77,11 @@
#define REG_LR 30
#define REG_BCP 22
+// IC IVAU trap probe.
+// Defined in ic_ivau_probe_linux_aarch64.S.
+extern "C" char _ic_ivau_probe_fault[] __attribute__ ((visibility ("hidden")));
+extern "C" char _ic_ivau_probe_continuation[] __attribute__ ((visibility ("hidden")));
+
NOINLINE address os::current_stack_pointer() {
return (address)__builtin_frame_address(0);
}
@@ -228,6 +233,12 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info,
}
}
+ // IC IVAU trap probe during VM_Version initialization.
+ // If IC IVAU is not trapped, it faults on unmapped VA 0x0.
+ if (sig == SIGSEGV && pc == (address)_ic_ivau_probe_fault) {
+ stub = (address)_ic_ivau_probe_continuation;
+ }
+
if (thread->thread_state() == _thread_in_Java) {
// Java thread running in Java code => find exception handler if any
// a fault inside compiled code, the interpreter, or a stub
diff --git a/src/hotspot/os_cpu/linux_aarch64/vm_version_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/vm_version_linux_aarch64.cpp
index 1fe06dc640d..ee2d3013c4c 100644
--- a/src/hotspot/os_cpu/linux_aarch64/vm_version_linux_aarch64.cpp
+++ b/src/hotspot/os_cpu/linux_aarch64/vm_version_linux_aarch64.cpp
@@ -31,6 +31,10 @@
#include
#include
+// IC IVAU trap probe.
+// Defined in ic_ivau_probe_linux_aarch64.S.
+extern "C" int ic_ivau_probe(void);
+
#ifndef HWCAP_AES
#define HWCAP_AES (1<<3)
#endif
@@ -95,6 +99,13 @@
#define HWCAP2_SVEBITPERM (1 << 4)
#endif
+#ifndef HWCAP2_ECV
+#define HWCAP2_ECV (1 << 19)
+#endif
+
+#ifndef HWCAP2_WFXT
+#define HWCAP2_WFXT (1u << 31)
+#endif
#ifndef PR_SVE_GET_VL
// For old toolchains which do not have SVE related macros defined.
#define PR_SVE_SET_VL 50
@@ -158,6 +169,12 @@ void VM_Version::get_os_cpu_info() {
if (auxv2 & HWCAP2_SVEBITPERM) {
set_feature(CPU_SVEBITPERM);
}
+ if (auxv2 & HWCAP2_ECV) {
+ set_feature(CPU_ECV);
+ }
+ if (auxv2 & HWCAP2_WFXT) {
+ set_feature(CPU_WFXT);
+ }
uint64_t ctr_el0;
uint64_t dczid_el0;
@@ -169,6 +186,12 @@ void VM_Version::get_os_cpu_info() {
_icache_line_size = (1 << (ctr_el0 & 0x0f)) * 4;
_dcache_line_size = (1 << ((ctr_el0 >> 16) & 0x0f)) * 4;
+ _cache_idc_enabled = ((ctr_el0 >> 28) & 0x1) != 0;
+ _cache_dic_enabled = ((ctr_el0 >> 29) & 0x1) != 0;
+
+ // Probe whether IC IVAU is trapped.
+ // Must run before VM_Version::initialize() sets NeoverseN1ICacheErratumMitigation.
+ _ic_ivau_trapped = (ic_ivau_probe() == 1);
if (!(dczid_el0 & 0x10)) {
_zva_length = 4 << (dczid_el0 & 0xf);
diff --git a/src/hotspot/os_cpu/windows_aarch64/prefetch_windows_aarch64.inline.hpp b/src/hotspot/os_cpu/windows_aarch64/prefetch_windows_aarch64.inline.hpp
index a360ee342be..a2c8f0c685c 100644
--- a/src/hotspot/os_cpu/windows_aarch64/prefetch_windows_aarch64.inline.hpp
+++ b/src/hotspot/os_cpu/windows_aarch64/prefetch_windows_aarch64.inline.hpp
@@ -27,10 +27,24 @@
// Included in runtime/prefetch.inline.hpp
+#include
+
+// __prefetch2(addr, prfop) emits a PRFM instruction.
+// The prfop encoding is:
+// type: PLD = 00, PLI = 01, PST = 10
+// target: L1 = 00, L2 = 01, L3 = 10
+// policy: KEEP = 0, STRM = 1
+
inline void Prefetch::read (const void *loc, intx interval) {
+ if (interval >= 0) {
+ __prefetch2((const char*) loc + interval, /* PLD + L1 + KEEP */ 0);
+ }
}
inline void Prefetch::write(void *loc, intx interval) {
+ if (interval >= 0) {
+ __prefetch2((char*) loc + interval, /* PST + L1 + KEEP */ 16);
+ }
}
#endif // OS_CPU_WINDOWS_AARCH64_PREFETCH_WINDOWS_AARCH64_INLINE_HPP
diff --git a/src/hotspot/os_cpu/windows_aarch64/sve_windows_aarch64.S b/src/hotspot/os_cpu/windows_aarch64/sve_windows_aarch64.S
new file mode 100644
index 00000000000..e0c85830bd4
--- /dev/null
+++ b/src/hotspot/os_cpu/windows_aarch64/sve_windows_aarch64.S
@@ -0,0 +1,42 @@
+;
+; Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
+; DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+;
+; This code is free software; you can redistribute it and/or modify it
+; under the terms of the GNU General Public License version 2 only, as
+; published by the Free Software Foundation.
+;
+; This code is distributed in the hope that it will be useful, but WITHOUT
+; ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+; FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+; version 2 for more details (a copy is included in the LICENSE file that
+; accompanied this code).
+;
+; You should have received a copy of the GNU General Public License version
+; 2 along with this work; if not, write to the Free Software Foundation,
+; Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+;
+; Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+; or visit www.oracle.com if you need additional information or have any
+; questions.
+;
+
+ ; Support for int get_sve_vector_length();
+ ;
+ ; Returns the current SVE vector length in bytes.
+ ; This function uses the INCB instruction which increments a register
+ ; by the number of bytes in an SVE vector register.
+ ;
+ ; Note: This function will fault if SVE is not available or enabled.
+ ; The caller must ensure SVE support is detected before calling.
+
+ ALIGN 4
+ EXPORT get_sve_vector_length
+ AREA sve_text, CODE
+
+get_sve_vector_length
+ mov x0, #0
+ incb x0
+ ret
+
+ END
diff --git a/src/hotspot/os_cpu/windows_aarch64/vm_version_windows_aarch64.cpp b/src/hotspot/os_cpu/windows_aarch64/vm_version_windows_aarch64.cpp
index 93beb549366..e78a37b4178 100644
--- a/src/hotspot/os_cpu/windows_aarch64/vm_version_windows_aarch64.cpp
+++ b/src/hotspot/os_cpu/windows_aarch64/vm_version_windows_aarch64.cpp
@@ -26,16 +26,19 @@
#include "runtime/os.hpp"
#include "runtime/vm_version.hpp"
+// Assembly function to get SVE vector length using INCB instruction
+extern "C" int get_sve_vector_length();
+
int VM_Version::get_current_sve_vector_length() {
assert(VM_Version::supports_sve(), "should not call this");
- ShouldNotReachHere();
- return 0;
+ // Use assembly instruction to get the actual SVE vector length
+ return VM_Version::supports_sve() ? get_sve_vector_length() : 0; // This value is in bytes
}
int VM_Version::set_and_get_current_sve_vector_length(int length) {
assert(VM_Version::supports_sve(), "should not call this");
- ShouldNotReachHere();
- return 0;
+ // Use assembly instruction to get the SVE vector length
+ return VM_Version::supports_sve() ? get_sve_vector_length() : 0; // This value is in bytes
}
void VM_Version::get_os_cpu_info() {
@@ -47,11 +50,29 @@ void VM_Version::get_os_cpu_info() {
set_feature(CPU_AES);
set_feature(CPU_SHA1);
set_feature(CPU_SHA2);
+ set_feature(CPU_PMULL);
}
if (IsProcessorFeaturePresent(PF_ARM_VFP_32_REGISTERS_AVAILABLE)) {
set_feature(CPU_ASIMD);
}
- // No check for CPU_PMULL, CPU_SVE, CPU_SVE2
+ if (IsProcessorFeaturePresent(PF_ARM_V81_ATOMIC_INSTRUCTIONS_AVAILABLE)) {
+ set_feature(CPU_LSE);
+ }
+ if (IsProcessorFeaturePresent(PF_ARM_SVE_INSTRUCTIONS_AVAILABLE)) {
+ set_feature(CPU_SVE);
+ }
+ if (IsProcessorFeaturePresent(PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE)) {
+ set_feature(CPU_SVE2);
+ }
+ if (IsProcessorFeaturePresent(PF_ARM_SVE_BITPERM_INSTRUCTIONS_AVAILABLE)) {
+ set_feature(CPU_SVEBITPERM);
+ }
+ if (IsProcessorFeaturePresent(PF_ARM_SHA3_INSTRUCTIONS_AVAILABLE)) {
+ set_feature(CPU_SHA3);
+ }
+ if (IsProcessorFeaturePresent(PF_ARM_SHA512_INSTRUCTIONS_AVAILABLE)) {
+ set_feature(CPU_SHA512);
+ }
__int64 dczid_el0 = _ReadStatusReg(0x5807 /* ARM64_DCZID_EL0 */);
@@ -102,8 +123,8 @@ void VM_Version::get_os_cpu_info() {
SYSTEM_INFO si;
GetSystemInfo(&si);
_model = si.wProcessorLevel;
- _variant = si.wProcessorRevision / 0xFF;
- _revision = si.wProcessorRevision & 0xFF;
+ _variant = (si.wProcessorRevision >> 8) & 0xFF; // Variant is the upper byte of wProcessorRevision
+ _revision = si.wProcessorRevision & 0xFF; // Revision is the lower byte of wProcessorRevision
}
}
}
diff --git a/src/hotspot/os_cpu/windows_x86/prefetch_windows_x86.inline.hpp b/src/hotspot/os_cpu/windows_x86/prefetch_windows_x86.inline.hpp
index 645fbe99a22..575eabc97dd 100644
--- a/src/hotspot/os_cpu/windows_x86/prefetch_windows_x86.inline.hpp
+++ b/src/hotspot/os_cpu/windows_x86/prefetch_windows_x86.inline.hpp
@@ -27,7 +27,18 @@
// Included in runtime/prefetch.inline.hpp
-inline void Prefetch::read (const void *loc, intx interval) {}
-inline void Prefetch::write(void *loc, intx interval) {}
+#include
+
+inline void Prefetch::read (const void *loc, intx interval) {
+ if (interval >= 0) {
+ _mm_prefetch((const char*) loc + interval, _MM_HINT_T0);
+ }
+}
+
+inline void Prefetch::write(void *loc, intx interval) {
+ if (interval >= 0) {
+ _mm_prefetch((const char*) loc + interval, _MM_HINT_T0);
+ }
+}
#endif // OS_CPU_WINDOWS_X86_PREFETCH_WINDOWS_X86_INLINE_HPP
diff --git a/src/hotspot/share/adlc/formssel.cpp b/src/hotspot/share/adlc/formssel.cpp
index 4dd2bff7c89..5802217c1c1 100644
--- a/src/hotspot/share/adlc/formssel.cpp
+++ b/src/hotspot/share/adlc/formssel.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -4233,11 +4233,13 @@ int MatchRule::is_expensive() const {
strcmp(opType,"PopulateIndex")==0 ||
strcmp(opType,"AddReductionVI")==0 ||
strcmp(opType,"AddReductionVL")==0 ||
+ strcmp(opType,"AddReductionVHF")==0 ||
strcmp(opType,"AddReductionVF")==0 ||
strcmp(opType,"AddReductionVD")==0 ||
strcmp(opType,"MulReductionVI")==0 ||
strcmp(opType,"MulReductionVL")==0 ||
strcmp(opType,"MulReductionVF")==0 ||
+ strcmp(opType,"MulReductionVHF")==0 ||
strcmp(opType,"MulReductionVD")==0 ||
strcmp(opType,"MinReductionV")==0 ||
strcmp(opType,"MaxReductionV")==0 ||
@@ -4348,9 +4350,9 @@ bool MatchRule::is_vector() const {
"MaxV", "MinV", "MinVHF", "MaxVHF", "UMinV", "UMaxV",
"CompressV", "ExpandV", "CompressM", "CompressBitsV", "ExpandBitsV",
"AddReductionVI", "AddReductionVL",
- "AddReductionVF", "AddReductionVD",
+ "AddReductionVHF", "AddReductionVF", "AddReductionVD",
"MulReductionVI", "MulReductionVL",
- "MulReductionVF", "MulReductionVD",
+ "MulReductionVHF", "MulReductionVF", "MulReductionVD",
"MaxReductionV", "MinReductionV",
"AndReductionV", "OrReductionV", "XorReductionV",
"MulAddVS2VI", "MacroLogicV",
diff --git a/src/hotspot/share/asm/codeBuffer.cpp b/src/hotspot/share/asm/codeBuffer.cpp
index ba525588f32..c6475050592 100644
--- a/src/hotspot/share/asm/codeBuffer.cpp
+++ b/src/hotspot/share/asm/codeBuffer.cpp
@@ -32,7 +32,6 @@
#include "oops/methodCounters.hpp"
#include "oops/methodData.hpp"
#include "oops/oop.inline.hpp"
-#include "runtime/icache.hpp"
#include "runtime/safepointVerifiers.hpp"
#include "utilities/align.hpp"
#include "utilities/copy.hpp"
@@ -418,7 +417,7 @@ void CodeSection::expand_locs(int new_capacity) {
new_capacity = old_capacity * 2;
relocInfo* locs_start;
if (_locs_own) {
- locs_start = REALLOC_RESOURCE_ARRAY(relocInfo, _locs_start, old_capacity, new_capacity);
+ locs_start = REALLOC_RESOURCE_ARRAY(_locs_start, old_capacity, new_capacity);
} else {
locs_start = NEW_RESOURCE_ARRAY(relocInfo, new_capacity);
Copy::conjoint_jbytes(_locs_start, locs_start, old_capacity * sizeof(relocInfo));
@@ -745,9 +744,6 @@ void CodeBuffer::copy_code_to(CodeBlob* dest_blob) {
// Done moving code bytes; were they the right size?
assert((int)align_up(dest.total_content_size(), oopSize) == dest_blob->content_size(), "sanity");
-
- // Flush generated code
- ICache::invalidate_range(dest_blob->code_begin(), dest_blob->code_size());
}
// Move all my code into another code buffer. Consult applicable
@@ -862,6 +858,13 @@ csize_t CodeBuffer::figure_expanded_capacities(CodeSection* which_cs,
}
void CodeBuffer::expand(CodeSection* which_cs, csize_t amount) {
+#ifdef ASSERT
+ // The code below copies contents across temp buffers. The following
+ // sizes relate to buffer contents, and should not be changed by buffer
+ // expansion.
+ int old_total_skipped = total_skipped_instructions_size();
+#endif
+
#ifndef PRODUCT
if (PrintNMethods && (WizardMode || Verbose)) {
tty->print("expanding CodeBuffer:");
@@ -920,6 +923,7 @@ void CodeBuffer::expand(CodeSection* which_cs, csize_t amount) {
assert(cb_sect->capacity() >= new_capacity[n], "big enough");
address cb_start = cb_sect->start();
cb_sect->set_end(cb_start + this_sect->size());
+ cb_sect->register_skipped(this_sect->_skipped_instructions_size);
if (this_sect->mark() == nullptr) {
cb_sect->clear_mark();
} else {
@@ -956,6 +960,9 @@ void CodeBuffer::expand(CodeSection* which_cs, csize_t amount) {
this->print_on(tty);
}
#endif //PRODUCT
+
+ assert(old_total_skipped == total_skipped_instructions_size(),
+ "Should match: %d == %d", old_total_skipped, total_skipped_instructions_size());
}
void CodeBuffer::adjust_internal_address(address from, address to) {
@@ -1140,7 +1147,7 @@ void AsmRemarks::clear() {
uint AsmRemarks::print(uint offset, outputStream* strm) const {
uint count = 0;
const char* prefix = " ;; ";
- const char* remstr = _remarks->lookup(offset);
+ const char* remstr = (_remarks ? _remarks->lookup(offset) : nullptr);
while (remstr != nullptr) {
strm->bol();
strm->print("%s", prefix);
diff --git a/src/hotspot/share/c1/c1_LIRGenerator.hpp b/src/hotspot/share/c1/c1_LIRGenerator.hpp
index ec0ea5dc047..8e30d05af6d 100644
--- a/src/hotspot/share/c1/c1_LIRGenerator.hpp
+++ b/src/hotspot/share/c1/c1_LIRGenerator.hpp
@@ -330,8 +330,9 @@ class LIRGenerator: public InstructionVisitor, public BlockClosure {
// volatile field operations are never patchable because a klass
// must be loaded to know it's volatile which means that the offset
- // it always known as well.
+ // is always known as well.
void volatile_field_store(LIR_Opr value, LIR_Address* address, CodeEmitInfo* info);
+ // volatile_field_load provides trailing membar semantics
void volatile_field_load(LIR_Address* address, LIR_Opr result, CodeEmitInfo* info);
void put_Object_unsafe(LIR_Opr src, LIR_Opr offset, LIR_Opr data, BasicType type, bool is_volatile);
diff --git a/src/hotspot/share/c1/c1_Runtime1.cpp b/src/hotspot/share/c1/c1_Runtime1.cpp
index 63764dd113a..38f563935e0 100644
--- a/src/hotspot/share/c1/c1_Runtime1.cpp
+++ b/src/hotspot/share/c1/c1_Runtime1.cpp
@@ -278,11 +278,9 @@ bool Runtime1::initialize(BufferBlob* blob) {
if (!generate_blob_for(blob, id)) {
return false;
}
- if (id == StubId::c1_forward_exception_id) {
- // publish early c1 stubs at this point so later stubs can refer to them
- AOTCodeCache::init_early_c1_table();
- }
}
+ // disallow any further c1 stub generation
+ AOTCodeCache::set_c1_stubs_complete();
// printing
#ifndef PRODUCT
if (PrintSimpleStubs) {
diff --git a/src/hotspot/share/cds/aotArtifactFinder.cpp b/src/hotspot/share/cds/aotArtifactFinder.cpp
index f85f1e46520..bd69b18a1aa 100644
--- a/src/hotspot/share/cds/aotArtifactFinder.cpp
+++ b/src/hotspot/share/cds/aotArtifactFinder.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -32,6 +32,7 @@
#include "cds/lambdaProxyClassDictionary.hpp"
#include "cds/regeneratedClasses.hpp"
#include "classfile/systemDictionaryShared.hpp"
+#include "classfile/vmClasses.hpp"
#include "logging/log.hpp"
#include "memory/metaspaceClosure.hpp"
#include "oops/instanceKlass.hpp"
@@ -169,6 +170,7 @@ void AOTArtifactFinder::find_artifacts() {
end_scanning_for_oops();
TrainingData::cleanup_training_data();
+ check_critical_classes();
}
void AOTArtifactFinder::start_scanning_for_oops() {
@@ -233,6 +235,7 @@ void AOTArtifactFinder::add_cached_instance_class(InstanceKlass* ik) {
bool created;
_seen_classes->put_if_absent(ik, &created);
if (created) {
+ check_critical_class(ik);
append_to_all_cached_classes(ik);
// All super types must be added.
@@ -310,3 +313,25 @@ void AOTArtifactFinder::all_cached_classes_do(MetaspaceClosure* it) {
it->push(_all_cached_classes->adr_at(i));
}
}
+
+void AOTArtifactFinder::check_critical_classes() {
+ if (CDSConfig::is_dumping_static_archive()) {
+ // vmClasses are store in the AOT cache (or AOT config file, or static archive).
+ // If any of the vmClasses is excluded, (usually due to incompatible JVMTI agent),
+ // the resulting cache/config/archive is unusable.
+ for (auto id : EnumRange{}) {
+ check_critical_class(vmClasses::klass_at(id));
+ }
+ }
+}
+
+void AOTArtifactFinder::check_critical_class(InstanceKlass* ik) {
+ if (SystemDictionaryShared::is_excluded_class(ik)) {
+ ResourceMark rm;
+ const char* msg = err_msg("Critical class %s has been excluded. %s cannot be written.",
+ ik->external_name(),
+ CDSConfig::type_of_archive_being_written());
+ AOTMetaspace::unrecoverable_writing_error(msg);
+ }
+}
+
diff --git a/src/hotspot/share/cds/aotArtifactFinder.hpp b/src/hotspot/share/cds/aotArtifactFinder.hpp
index 05bcde6b0ac..50057b6caee 100644
--- a/src/hotspot/share/cds/aotArtifactFinder.hpp
+++ b/src/hotspot/share/cds/aotArtifactFinder.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -81,12 +81,14 @@ class AOTArtifactFinder : AllStatic {
static void add_cached_type_array_class(TypeArrayKlass* tak);
static void add_cached_instance_class(InstanceKlass* ik);
static void append_to_all_cached_classes(Klass* k);
+ static void check_critical_class(InstanceKlass* ik);
public:
static void initialize();
static void find_artifacts();
static void add_cached_class(Klass* k);
static void add_aot_inited_class(InstanceKlass* ik);
static void all_cached_classes_do(MetaspaceClosure* it);
+ static void check_critical_classes();
static void dispose();
};
diff --git a/src/hotspot/share/cds/aotClassInitializer.cpp b/src/hotspot/share/cds/aotClassInitializer.cpp
index 06fc3af6f30..9ef96282aeb 100644
--- a/src/hotspot/share/cds/aotClassInitializer.cpp
+++ b/src/hotspot/share/cds/aotClassInitializer.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -59,6 +59,39 @@ bool AOTClassInitializer::can_archive_initialized_mirror(InstanceKlass* ik) {
return false;
}
+#ifdef ASSERT
+ // If code in ik is executed, then ik must be in the state of being_initialized or
+ // fully_initialized.
+ //
+ // Check that no user code is executed during the assembly phase. Otherwise the user
+ // code may introduce undesirable environment dependencies into the heap image.
+ // If any of these two flags are set, we allow user code to be executed
+ // in the assembly phase. Note that these flags are strictly for the purpose
+ // of testing HotSpot and are not available in product builds.
+ if (AOTInitTestClass == nullptr && ArchiveHeapTestClass == nullptr) {
+ if (ik->defined_by_boot_loader()) {
+ // We allow boot classes to be AOT-initialized, except for classes from
+ // -Xbootclasspath (cp index >= 1) be AOT-initialized, as such classes may be
+ // provided by the user application.
+ assert(ik->shared_classpath_index() <= 0,
+ "only boot classed loaded from the modules image can be AOT-initialized");
+ } else {
+ assert(ik->defined_by_platform_loader() || ik->defined_by_app_loader(),
+ "cannot AOT-initialized classed loaded by other loaders");
+
+ // Hidden classes from platform/app loaders need to be AOT-initialized to
+ // support AOT-linking of lambdas. These hidden classes are generated by the
+ // VM and do not contain user code.
+ if (!ik->is_hidden()) {
+ // OK: ik is an interface used by a lambda. When AOT-linking lambdas, we only
+ // support interfaces that are not interface_needs_clinit_execution_as_super().
+ // See AOTConstantPoolResolver::check_lambda_metafactory_signature().
+ assert(ik->is_interface() && !ik->interface_needs_clinit_execution_as_super(), "cannot execute Java code in assembly phase");
+ }
+ }
+ }
+#endif // ASSERT
+
// About "static field that may hold a different value" errors:
//
// Automatic selection for aot-inited classes
@@ -234,7 +267,8 @@ bool AOTClassInitializer::can_archive_initialized_mirror(InstanceKlass* ik) {
}
void AOTClassInitializer::call_runtime_setup(JavaThread* current, InstanceKlass* ik) {
- assert(ik->has_aot_initialized_mirror(), "sanity");
+ precond(ik->has_aot_initialized_mirror());
+ precond(!AOTLinkedClassBulkLoader::is_initializing_classes_early());
if (ik->is_runtime_setup_required()) {
if (log_is_enabled(Info, aot, init)) {
ResourceMark rm;
diff --git a/src/hotspot/share/cds/aotGrowableArray.hpp b/src/hotspot/share/cds/aotGrowableArray.hpp
deleted file mode 100644
index 0a0c137ed07..00000000000
--- a/src/hotspot/share/cds/aotGrowableArray.hpp
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef SHARE_AOT_AOTGROWABLEARRAY_HPP
-#define SHARE_AOT_AOTGROWABLEARRAY_HPP
-
-#include
-#include
-
-class AOTGrowableArrayHelper {
-public:
- static void deallocate(void* mem);
-};
-
-// An AOTGrowableArray provides the same functionality as a GrowableArray that
-// uses the C heap allocator. In addition, AOTGrowableArray can be iterated with
-// MetaspaceClosure. This type should be used for growable arrays that need to be
-// stored in the AOT cache. See ModuleEntry::_reads for an example.
-template
-class AOTGrowableArray : public GrowableArrayWithAllocator> {
- friend class VMStructs;
- friend class GrowableArrayWithAllocator;
-
- static E* allocate(int max, MemTag mem_tag) {
- return (E*)GrowableArrayCHeapAllocator::allocate(max, sizeof(E), mem_tag);
- }
-
- E* allocate() {
- return allocate(this->_capacity, mtClass);
- }
-
- void deallocate(E* mem) {
-#if INCLUDE_CDS
- AOTGrowableArrayHelper::deallocate(mem);
-#else
- GrowableArrayCHeapAllocator::deallocate(mem);
-#endif
- }
-
-public:
- AOTGrowableArray(int initial_capacity, MemTag mem_tag) :
- GrowableArrayWithAllocator(
- allocate(initial_capacity, mem_tag),
- initial_capacity) {}
-
- AOTGrowableArray() : AOTGrowableArray(0, mtClassShared) {}
-
- // methods required by MetaspaceClosure
- void metaspace_pointers_do(MetaspaceClosure* it);
- int size_in_heapwords() const { return (int)heap_word_size(sizeof(*this)); }
- MetaspaceClosureType type() const { return MetaspaceClosureType::GrowableArrayType; }
- static bool is_read_only_by_default() { return false; }
-};
-
-#endif // SHARE_AOT_AOTGROWABLEARRAY_HPP
diff --git a/src/hotspot/share/cds/aotLinkedClassBulkLoader.cpp b/src/hotspot/share/cds/aotLinkedClassBulkLoader.cpp
index 3653f9d518c..8129e6a5a81 100644
--- a/src/hotspot/share/cds/aotLinkedClassBulkLoader.cpp
+++ b/src/hotspot/share/cds/aotLinkedClassBulkLoader.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -116,11 +116,24 @@ void AOTLinkedClassBulkLoader::preload_classes_in_table(Array* c
}
}
+#ifdef ASSERT
+// true iff we are inside AOTLinkedClassBulkLoader::link_classes(), when
+// we are moving classes into the fully_initialized state before the
+// JVM is able to execute any bytecodes.
+static bool _is_initializing_classes_early = false;
+bool AOTLinkedClassBulkLoader::is_initializing_classes_early() {
+ return _is_initializing_classes_early;
+}
+#endif
+
// Some cached heap objects may hold references to methods in aot-linked
// classes (via MemberName). We need to make sure all classes are
// linked before executing any bytecode.
void AOTLinkedClassBulkLoader::link_classes(JavaThread* current) {
+ DEBUG_ONLY(_is_initializing_classes_early = true);
link_classes_impl(current);
+ DEBUG_ONLY(_is_initializing_classes_early = false);
+
if (current->has_pending_exception()) {
exit_on_exception(current);
}
@@ -135,6 +148,13 @@ void AOTLinkedClassBulkLoader::link_classes_impl(TRAPS) {
link_classes_in_table(table->boot2(), CHECK);
link_classes_in_table(table->platform(), CHECK);
link_classes_in_table(table->app(), CHECK);
+
+ init_classes_for_loader(Handle(), AOTLinkedClassTable::get()->boot1(), /*early_only=*/true, CHECK);
+ init_classes_for_loader(Handle(), AOTLinkedClassTable::get()->boot2(), /*early_only=*/true, CHECK);
+ init_classes_for_loader(Handle(), AOTLinkedClassTable::get()->platform(), /*early_only=*/true, CHECK);
+ init_classes_for_loader(Handle(), AOTLinkedClassTable::get()->app(), /*early_only=*/true, CHECK);
+
+ log_info(aot, init)("------ finished early class init");
}
void AOTLinkedClassBulkLoader::link_classes_in_table(Array* classes, TRAPS) {
@@ -216,7 +236,7 @@ void AOTLinkedClassBulkLoader::validate_module(Klass* k, const char* category_na
#endif
void AOTLinkedClassBulkLoader::init_javabase_classes(JavaThread* current) {
- init_classes_for_loader(Handle(), AOTLinkedClassTable::get()->boot1(), current);
+ init_classes_for_loader(Handle(), AOTLinkedClassTable::get()->boot1(), /*early_only=*/false, current);
if (current->has_pending_exception()) {
exit_on_exception(current);
}
@@ -246,9 +266,9 @@ void AOTLinkedClassBulkLoader::init_non_javabase_classes_impl(TRAPS) {
assert(h_system_loader() != nullptr, "must be");
AOTLinkedClassTable* table = AOTLinkedClassTable::get();
- init_classes_for_loader(Handle(), table->boot2(), CHECK);
- init_classes_for_loader(h_platform_loader, table->platform(), CHECK);
- init_classes_for_loader(h_system_loader, table->app(), CHECK);
+ init_classes_for_loader(Handle(), table->boot2(), /*early_only=*/false, CHECK);
+ init_classes_for_loader(h_platform_loader, table->platform(), /*early_only=*/false, CHECK);
+ init_classes_for_loader(h_system_loader, table->app(), /*early_only=*/false, CHECK);
if (Universe::is_fully_initialized() && VerifyDuringStartup) {
// Make sure we're still in a clean state.
@@ -260,6 +280,10 @@ void AOTLinkedClassBulkLoader::init_non_javabase_classes_impl(TRAPS) {
tty->print_cr("==================== archived_training_data ** after all classes preloaded ====================");
TrainingData::print_archived_training_data_on(tty);
}
+ LogStreamHandle(Info, aot, training, data) log;
+ if (log.is_enabled()) {
+ TrainingData::print_archived_training_data_on(&log);
+ }
}
// For the AOT cache to function properly, all classes in the AOTLinkedClassTable
@@ -324,22 +348,80 @@ void AOTLinkedClassBulkLoader::initiate_loading(JavaThread* current, const char*
}
}
-// Some AOT-linked classes for must be initialized early. This includes
-// - classes that were AOT-initialized by AOTClassInitializer
-// - the classes of all objects that are reachable from the archived mirrors of
-// the AOT-linked classes for .
-void AOTLinkedClassBulkLoader::init_classes_for_loader(Handle class_loader, Array* classes, TRAPS) {
+// Can we move ik into fully_initialized state before the JVM is able to execute
+// bytecodes?
+static bool is_early_init_possible(InstanceKlass* ik) {
+ if (ik->is_runtime_setup_required()) {
+ // Bytecodes need to be executed in order to initialize this class.
+ if (log_is_enabled(Debug, aot, init)) {
+ ResourceMark rm;
+ log_debug(aot, init)("No early init %s: needs runtimeSetup()",
+ ik->external_name());
+ }
+ return false;
+ }
+
+ if (ik->super() != nullptr && !ik->super()->is_initialized()) {
+ // is_runtime_setup_required() == true for a super type
+ if (log_is_enabled(Debug, aot, init)) {
+ ResourceMark rm;
+ log_debug(aot, init)("No early init %s: super type %s not initialized",
+ ik->external_name(), ik->super()->external_name());
+ }
+ return false;
+ }
+
+ Array* interfaces = ik->local_interfaces();
+ int num_interfaces = interfaces->length();
+ for (int i = 0; i < num_interfaces; i++) {
+ InstanceKlass* intf = interfaces->at(i);
+ if (!intf->is_initialized() && intf->interface_needs_clinit_execution_as_super(/*also_check_supers*/false)) {
+ // is_runtime_setup_required() == true for this interface
+ if (log_is_enabled(Debug, aot, init)) {
+ ResourceMark rm;
+ log_debug(aot, init)("No early init %s: interface type %s not initialized",
+ ik->external_name(), intf->external_name());
+ }
+ return false;
+ }
+ }
+
+ return true;
+}
+
+// Normally, classes are initialized on demand. However, some AOT-linked classes
+// for the class_loader must be proactively intialized, including:
+// - Classes that have an AOT-initialized mirror (they were AOT-initialized by
+// AOTClassInitializer during the assembly phase).
+// - The classes of all objects that are reachable from the archived mirrors of
+// the AOT-linked classes for the class_loader. These are recorded in the special
+// subgraph.
+//
+// (early_only == true) means that this function is called before the JVM
+// is capable of executing Java bytecodes.
+void AOTLinkedClassBulkLoader::init_classes_for_loader(Handle class_loader, Array* classes,
+ bool early_only, TRAPS) {
if (classes != nullptr) {
for (int i = 0; i < classes->length(); i++) {
InstanceKlass* ik = classes->at(i);
assert(ik->class_loader_data() != nullptr, "must be");
- if (ik->has_aot_initialized_mirror()) {
- ik->initialize_with_aot_initialized_mirror(CHECK);
+
+ bool do_init = ik->has_aot_initialized_mirror();
+ if (do_init && early_only && !is_early_init_possible(ik)) {
+ // ik will be proactively initialized later when init_classes_for_loader()
+ // is called again with (early_only == false).
+ do_init = false;
+ }
+
+ if (do_init) {
+ ik->initialize_with_aot_initialized_mirror(early_only, CHECK);
}
}
}
- HeapShared::init_classes_for_special_subgraph(class_loader, CHECK);
+ if (!early_only) {
+ HeapShared::init_classes_for_special_subgraph(class_loader, CHECK);
+ }
}
void AOTLinkedClassBulkLoader::replay_training_at_init(Array* classes, TRAPS) {
diff --git a/src/hotspot/share/cds/aotLinkedClassBulkLoader.hpp b/src/hotspot/share/cds/aotLinkedClassBulkLoader.hpp
index 31fdac386fe..24ff61cea1e 100644
--- a/src/hotspot/share/cds/aotLinkedClassBulkLoader.hpp
+++ b/src/hotspot/share/cds/aotLinkedClassBulkLoader.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -56,7 +56,7 @@ class AOTLinkedClassBulkLoader : AllStatic {
static void link_classes_impl(TRAPS);
static void link_classes_in_table(Array* classes, TRAPS);
static void init_non_javabase_classes_impl(TRAPS);
- static void init_classes_for_loader(Handle class_loader, Array* classes, TRAPS);
+ static void init_classes_for_loader(Handle class_loader, Array* classes, bool early_only, TRAPS);
static void replay_training_at_init(Array* classes, TRAPS) NOT_CDS_RETURN;
#ifdef ASSERT
@@ -73,8 +73,9 @@ public:
static void init_javabase_classes(JavaThread* current) NOT_CDS_RETURN;
static void init_non_javabase_classes(JavaThread* current) NOT_CDS_RETURN;
static void exit_on_exception(JavaThread* current);
-
static void replay_training_at_init_for_preloaded_classes(TRAPS) NOT_CDS_RETURN;
+
+ static bool is_initializing_classes_early() NOT_DEBUG({return false;});
};
#endif // SHARE_CDS_AOTLINKEDCLASSBULKLOADER_HPP
diff --git a/src/hotspot/share/cds/aotMapLogger.cpp b/src/hotspot/share/cds/aotMapLogger.cpp
index 98336ff9b1f..9f338826fd6 100644
--- a/src/hotspot/share/cds/aotMapLogger.cpp
+++ b/src/hotspot/share/cds/aotMapLogger.cpp
@@ -589,7 +589,6 @@ public:
}
Klass* real_klass() {
- assert(UseCompressedClassPointers, "heap archiving requires UseCompressedClassPointers");
return _data._klass;
}
diff --git a/src/hotspot/share/cds/aotMappedHeapLoader.hpp b/src/hotspot/share/cds/aotMappedHeapLoader.hpp
index 7c5ca1b1f9e..10f5ce3124f 100644
--- a/src/hotspot/share/cds/aotMappedHeapLoader.hpp
+++ b/src/hotspot/share/cds/aotMappedHeapLoader.hpp
@@ -54,7 +54,7 @@ public:
// Can this VM map archived heap region? Currently only G1+compressed{oops,cp}
static bool can_map() {
- CDS_JAVA_HEAP_ONLY(return (UseG1GC && UseCompressedClassPointers);)
+ CDS_JAVA_HEAP_ONLY(return UseG1GC;)
NOT_CDS_JAVA_HEAP(return false;)
}
diff --git a/src/hotspot/share/cds/aotMappedHeapWriter.cpp b/src/hotspot/share/cds/aotMappedHeapWriter.cpp
index 3456c845938..8f810ef5244 100644
--- a/src/hotspot/share/cds/aotMappedHeapWriter.cpp
+++ b/src/hotspot/share/cds/aotMappedHeapWriter.cpp
@@ -450,7 +450,6 @@ int AOTMappedHeapWriter::filler_array_length(size_t fill_bytes) {
}
HeapWord* AOTMappedHeapWriter::init_filler_array_at_buffer_top(int array_length, size_t fill_bytes) {
- assert(UseCompressedClassPointers, "Archived heap only supported for compressed klasses");
Klass* oak = Universe::objectArrayKlass(); // already relocated to point to archived klass
HeapWord* mem = offset_to_buffered_address(_buffer_used);
memset(mem, 0, fill_bytes);
@@ -724,7 +723,6 @@ template void AOTMappedHeapWriter::mark_oop_pointer(T* buffered_add
}
void AOTMappedHeapWriter::update_header_for_requested_obj(oop requested_obj, oop src_obj, Klass* src_klass) {
- assert(UseCompressedClassPointers, "Archived heap only supported for compressed klasses");
narrowKlass nk = ArchiveBuilder::current()->get_requested_narrow_klass(src_klass);
address buffered_addr = requested_addr_to_buffered_addr(cast_from_oop(requested_obj));
diff --git a/src/hotspot/share/cds/aotMetaspace.cpp b/src/hotspot/share/cds/aotMetaspace.cpp
index 76c37194882..4c23ede9cb8 100644
--- a/src/hotspot/share/cds/aotMetaspace.cpp
+++ b/src/hotspot/share/cds/aotMetaspace.cpp
@@ -250,9 +250,9 @@ static bool shared_base_too_high(char* specified_base, char* aligned_base, size_
static char* compute_shared_base(size_t cds_max) {
char* specified_base = (char*)SharedBaseAddress;
size_t alignment = AOTMetaspace::core_region_alignment();
- if (UseCompressedClassPointers && CompressedKlassPointers::needs_class_space()) {
- alignment = MAX2(alignment, Metaspace::reserve_alignment());
- }
+#if INCLUDE_CLASS_SPACE
+ alignment = MAX2(alignment, Metaspace::reserve_alignment());
+#endif
if (SharedBaseAddress == 0) {
// Special meaning of -XX:SharedBaseAddress=0 -> Always map archive at os-selected address.
@@ -949,11 +949,18 @@ void AOTMetaspace::dump_static_archive(TRAPS) {
ResourceMark rm(THREAD);
HandleMark hm(THREAD);
- if (CDSConfig::is_dumping_final_static_archive() && AOTPrintTrainingInfo) {
- tty->print_cr("==================== archived_training_data ** before dumping ====================");
- TrainingData::print_archived_training_data_on(tty);
+ if (CDSConfig::is_dumping_final_static_archive()) {
+ if (AOTPrintTrainingInfo) {
+ tty->print_cr("==================== archived_training_data ** before dumping ====================");
+ TrainingData::print_archived_training_data_on(tty);
+ }
+ LogStreamHandle(Info, aot, training, data) log;
+ if (log.is_enabled()) {
+ TrainingData::print_archived_training_data_on(&log);
+ }
}
+
StaticArchiveBuilder builder;
dump_static_archive_impl(builder, THREAD);
if (HAS_PENDING_EXCEPTION) {
@@ -1637,32 +1644,29 @@ MapArchiveResult AOTMetaspace::map_archives(FileMapInfo* static_mapinfo, FileMap
aot_log_debug(aot)("Failed to reserve spaces (use_requested_addr=%u)", (unsigned)use_requested_addr);
} else {
- if (Metaspace::using_class_space()) {
- prot_zone_size = protection_zone_size();
- }
+ CLASS_SPACE_ONLY(prot_zone_size = protection_zone_size();)
-#ifdef ASSERT
// Some sanity checks after reserving address spaces for archives
// and class space.
assert(archive_space_rs.is_reserved(), "Sanity");
- if (Metaspace::using_class_space()) {
- assert(archive_space_rs.base() == mapped_base_address &&
- archive_space_rs.size() > protection_zone_size(),
- "Archive space must lead and include the protection zone");
- // Class space must closely follow the archive space. Both spaces
- // must be aligned correctly.
- assert(class_space_rs.is_reserved() && class_space_rs.size() > 0,
- "A class space should have been reserved");
- assert(class_space_rs.base() >= archive_space_rs.end(),
- "class space should follow the cds archive space");
- assert(is_aligned(archive_space_rs.base(),
- core_region_alignment()),
- "Archive space misaligned");
- assert(is_aligned(class_space_rs.base(),
- Metaspace::reserve_alignment()),
- "class space misaligned");
- }
-#endif // ASSERT
+
+#if INCLUDE_CLASS_SPACE
+ assert(archive_space_rs.base() == mapped_base_address &&
+ archive_space_rs.size() > protection_zone_size(),
+ "Archive space must lead and include the protection zone");
+ // Class space must closely follow the archive space. Both spaces
+ // must be aligned correctly.
+ assert(class_space_rs.is_reserved() && class_space_rs.size() > 0,
+ "A class space should have been reserved");
+ assert(class_space_rs.base() >= archive_space_rs.end(),
+ "class space should follow the cds archive space");
+ assert(is_aligned(archive_space_rs.base(),
+ core_region_alignment()),
+ "Archive space misaligned");
+ assert(is_aligned(class_space_rs.base(),
+ Metaspace::reserve_alignment()),
+ "class space misaligned");
+#endif // INCLUDE_CLASS_SPACE
aot_log_info(aot)("Reserved archive_space_rs [" INTPTR_FORMAT " - " INTPTR_FORMAT "] (%zu) bytes%s",
p2i(archive_space_rs.base()), p2i(archive_space_rs.end()), archive_space_rs.size(),
@@ -1764,62 +1768,60 @@ MapArchiveResult AOTMetaspace::map_archives(FileMapInfo* static_mapinfo, FileMap
if (result == MAP_ARCHIVE_SUCCESS) {
SharedBaseAddress = (size_t)mapped_base_address;
-#ifdef _LP64
- if (Metaspace::using_class_space()) {
- assert(prot_zone_size > 0 &&
- *(mapped_base_address) == 'P' &&
- *(mapped_base_address + prot_zone_size - 1) == 'P',
- "Protection zone was overwritten?");
- // Set up ccs in metaspace.
- Metaspace::initialize_class_space(class_space_rs);
+#if INCLUDE_CLASS_SPACE
+ assert(prot_zone_size > 0 &&
+ *(mapped_base_address) == 'P' &&
+ *(mapped_base_address + prot_zone_size - 1) == 'P',
+ "Protection zone was overwritten?");
+ // Set up ccs in metaspace.
+ Metaspace::initialize_class_space(class_space_rs);
- // Set up compressed Klass pointer encoding: the encoding range must
- // cover both archive and class space.
- const address klass_range_start = (address)mapped_base_address;
- const size_t klass_range_size = (address)class_space_rs.end() - klass_range_start;
- if (INCLUDE_CDS_JAVA_HEAP || UseCompactObjectHeaders) {
- // The CDS archive may contain narrow Klass IDs that were precomputed at archive generation time:
- // - every archived java object header (only if INCLUDE_CDS_JAVA_HEAP)
- // - every archived Klass' prototype (only if +UseCompactObjectHeaders)
- //
- // In order for those IDs to still be valid, we need to dictate base and shift: base should be the
- // mapping start (including protection zone), shift should be the shift used at archive generation time.
- CompressedKlassPointers::initialize_for_given_encoding(
- klass_range_start, klass_range_size,
- klass_range_start, ArchiveBuilder::precomputed_narrow_klass_shift() // precomputed encoding, see ArchiveBuilder
- );
- assert(CompressedKlassPointers::base() == klass_range_start, "must be");
- } else {
- // Let JVM freely choose encoding base and shift
- CompressedKlassPointers::initialize(klass_range_start, klass_range_size);
- assert(CompressedKlassPointers::base() == nullptr ||
- CompressedKlassPointers::base() == klass_range_start, "must be");
- }
- // Establish protection zone, but only if we need one
- if (CompressedKlassPointers::base() == klass_range_start) {
- CompressedKlassPointers::establish_protection_zone(klass_range_start, prot_zone_size);
- }
+ // Set up compressed Klass pointer encoding: the encoding range must
+ // cover both archive and class space.
+ const address klass_range_start = (address)mapped_base_address;
+ const size_t klass_range_size = (address)class_space_rs.end() - klass_range_start;
+ if (INCLUDE_CDS_JAVA_HEAP || UseCompactObjectHeaders) {
+ // The CDS archive may contain narrow Klass IDs that were precomputed at archive generation time:
+ // - every archived java object header (only if INCLUDE_CDS_JAVA_HEAP)
+ // - every archived Klass' prototype (only if +UseCompactObjectHeaders)
+ //
+ // In order for those IDs to still be valid, we need to dictate base and shift: base should be the
+ // mapping start (including protection zone), shift should be the shift used at archive generation time.
+ CompressedKlassPointers::initialize_for_given_encoding(
+ klass_range_start, klass_range_size,
+ klass_range_start, ArchiveBuilder::precomputed_narrow_klass_shift() // precomputed encoding, see ArchiveBuilder
+ );
+ assert(CompressedKlassPointers::base() == klass_range_start, "must be");
+ } else {
+ // Let JVM freely choose encoding base and shift
+ CompressedKlassPointers::initialize(klass_range_start, klass_range_size);
+ assert(CompressedKlassPointers::base() == nullptr ||
+ CompressedKlassPointers::base() == klass_range_start, "must be");
+ }
+ // Establish protection zone, but only if we need one
+ if (CompressedKlassPointers::base() == klass_range_start) {
+ CompressedKlassPointers::establish_protection_zone(klass_range_start, prot_zone_size);
+ }
- if (static_mapinfo->can_use_heap_region()) {
- if (static_mapinfo->object_streaming_mode()) {
- HeapShared::initialize_loading_mode(HeapArchiveMode::_streaming);
- } else {
- // map_or_load_heap_region() compares the current narrow oop and klass encodings
- // with the archived ones, so it must be done after all encodings are determined.
- static_mapinfo->map_or_load_heap_region();
- HeapShared::initialize_loading_mode(HeapArchiveMode::_mapping);
- }
+ if (static_mapinfo->can_use_heap_region()) {
+ if (static_mapinfo->object_streaming_mode()) {
+ HeapShared::initialize_loading_mode(HeapArchiveMode::_streaming);
} else {
- FileMapRegion* r = static_mapinfo->region_at(AOTMetaspace::hp);
- if (r->used() > 0) {
- AOTMetaspace::report_loading_error("Cannot use CDS heap data.");
- }
- if (!CDSConfig::is_dumping_static_archive()) {
- CDSConfig::stop_using_full_module_graph("No CDS heap data");
- }
+ // map_or_load_heap_region() compares the current narrow oop and klass encodings
+ // with the archived ones, so it must be done after all encodings are determined.
+ static_mapinfo->map_or_load_heap_region();
+ HeapShared::initialize_loading_mode(HeapArchiveMode::_mapping);
+ }
+ } else {
+ FileMapRegion* r = static_mapinfo->region_at(AOTMetaspace::hp);
+ if (r->used() > 0) {
+ AOTMetaspace::report_loading_error("Cannot use CDS heap data.");
+ }
+ if (!CDSConfig::is_dumping_static_archive()) {
+ CDSConfig::stop_using_full_module_graph("No CDS heap data");
}
}
-#endif // _LP64
+#endif // INCLUDE_CLASS_SPACE
log_info(aot)("initial optimized module handling: %s", CDSConfig::is_using_optimized_module_handling() ? "enabled" : "disabled");
log_info(aot)("initial full module graph: %s", CDSConfig::is_using_full_module_graph() ? "enabled" : "disabled");
} else {
@@ -1852,8 +1854,13 @@ MapArchiveResult AOTMetaspace::map_archives(FileMapInfo* static_mapinfo, FileMap
// (The gap may result from different alignment requirements between metaspace
// and CDS)
//
-// If UseCompressedClassPointers is disabled, only one address space will be
-// reserved:
+// The range encompassing both spaces will be suitable to en/decode narrow Klass
+// pointers: the base will be valid for encoding the range [Base, End) and not
+// surpass the max. range for that encoding.
+//
+// On 32-bit, a "narrow" Klass is just the pointer itself, and the Klass encoding
+// range encompasses the whole address range. Consequently, we can "decode" and
+// "encode" any pointer anywhere, and so are free to place the CDS archive anywhere:
//
// +-- Base address End
// | |
@@ -1867,27 +1874,21 @@ MapArchiveResult AOTMetaspace::map_archives(FileMapInfo* static_mapinfo, FileMap
// use_archive_base_addr address is false, this base address is determined
// by the platform.
//
-// If UseCompressedClassPointers=1, the range encompassing both spaces will be
-// suitable to en/decode narrow Klass pointers: the base will be valid for
-// encoding, the range [Base, End) and not surpass the max. range for that encoding.
-//
// Return:
//
// - On success:
// - total_space_rs will be reserved as whole for archive_space_rs and
-// class_space_rs if UseCompressedClassPointers is true.
+// class_space_rs on 64-bit.
// On Windows, try reserve archive_space_rs and class_space_rs
// separately first if use_archive_base_addr is true.
// - archive_space_rs will be reserved and large enough to host static and
// if needed dynamic archive: [Base, A).
// archive_space_rs.base and size will be aligned to CDS reserve
// granularity.
-// - class_space_rs: If UseCompressedClassPointers=1, class_space_rs will
-// be reserved. Its start address will be aligned to metaspace reserve
-// alignment, which may differ from CDS alignment. It will follow the cds
-// archive space, close enough such that narrow class pointer encoding
-// covers both spaces.
-// If UseCompressedClassPointers=0, class_space_rs remains unreserved.
+// - class_space_rs: On 64-bit, class_space_rs will be reserved. Its start
+// address will be aligned to metaspace reserve alignment, which may differ
+// from CDS alignment. It will follow the cds archive space, close enough
+// such that narrow class pointer encoding covers both spaces.
// - On error: null is returned and the spaces remain unreserved.
char* AOTMetaspace::reserve_address_space_for_archives(FileMapInfo* static_mapinfo,
FileMapInfo* dynamic_mapinfo,
@@ -1903,32 +1904,34 @@ char* AOTMetaspace::reserve_address_space_for_archives(FileMapInfo* static_mapin
size_t archive_end_offset = (dynamic_mapinfo == nullptr) ? static_mapinfo->mapping_end_offset() : dynamic_mapinfo->mapping_end_offset();
size_t archive_space_size = align_up(archive_end_offset, archive_space_alignment);
- if (!Metaspace::using_class_space()) {
- // Get the simple case out of the way first:
- // no compressed class space, simple allocation.
+#if !INCLUDE_CLASS_SPACE
- // When running without class space, requested archive base should be aligned to cds core alignment.
- assert(is_aligned(base_address, archive_space_alignment),
- "Archive base address unaligned: " PTR_FORMAT ", needs alignment: %zu.",
- p2i(base_address), archive_space_alignment);
+ // Get the simple case out of the way first:
+ // no compressed class space, simple allocation.
- archive_space_rs = MemoryReserver::reserve((char*)base_address,
- archive_space_size,
- archive_space_alignment,
- os::vm_page_size(),
- mtNone);
- if (archive_space_rs.is_reserved()) {
- assert(base_address == nullptr ||
- (address)archive_space_rs.base() == base_address, "Sanity");
- // Register archive space with NMT.
- MemTracker::record_virtual_memory_tag(archive_space_rs, mtClassShared);
- return archive_space_rs.base();
- }
- return nullptr;
+ // When running without class space, requested archive base should be aligned to cds core alignment.
+ assert(is_aligned(base_address, archive_space_alignment),
+ "Archive base address unaligned: " PTR_FORMAT ", needs alignment: %zu.",
+ p2i(base_address), archive_space_alignment);
+
+ archive_space_rs = MemoryReserver::reserve((char*)base_address,
+ archive_space_size,
+ archive_space_alignment,
+ os::vm_page_size(),
+ mtNone);
+ if (archive_space_rs.is_reserved()) {
+ assert(base_address == nullptr ||
+ (address)archive_space_rs.base() == base_address, "Sanity");
+ // Register archive space with NMT.
+ MemTracker::record_virtual_memory_tag(archive_space_rs, mtClassShared);
+ return archive_space_rs.base();
}
-#ifdef _LP64
+ return nullptr;
+#else
+
+ // INCLUDE_CLASS_SPACE=1
// Complex case: two spaces adjacent to each other, both to be addressable
// with narrow class pointers.
// We reserve the whole range spanning both spaces, then split that range up.
@@ -2040,11 +2043,7 @@ char* AOTMetaspace::reserve_address_space_for_archives(FileMapInfo* static_mapin
return archive_space_rs.base();
-#else
- ShouldNotReachHere();
- return nullptr;
-#endif
-
+#endif // INCLUDE_CLASS_SPACE
}
void AOTMetaspace::release_reserved_spaces(ReservedSpace& total_space_rs,
diff --git a/src/hotspot/share/cds/aotStreamedHeapLoader.cpp b/src/hotspot/share/cds/aotStreamedHeapLoader.cpp
index 39f735543cd..7f9f8cf0628 100644
--- a/src/hotspot/share/cds/aotStreamedHeapLoader.cpp
+++ b/src/hotspot/share/cds/aotStreamedHeapLoader.cpp
@@ -797,7 +797,7 @@ void AOTStreamedHeapLoader::cleanup() {
Universe::vm_global()->release(&handles[num_null_handles], num_handles - num_null_handles);
}
- FREE_C_HEAP_ARRAY(void*, _object_index_to_heap_object_table);
+ FREE_C_HEAP_ARRAY(_object_index_to_heap_object_table);
// Unmap regions
FileMapInfo::current_info()->unmap_region(AOTMetaspace::hp);
diff --git a/src/hotspot/share/cds/aotStreamedHeapWriter.cpp b/src/hotspot/share/cds/aotStreamedHeapWriter.cpp
index ad363f21fdb..25bef10a673 100644
--- a/src/hotspot/share/cds/aotStreamedHeapWriter.cpp
+++ b/src/hotspot/share/cds/aotStreamedHeapWriter.cpp
@@ -369,7 +369,6 @@ template void AOTStreamedHeapWriter::map_oop_field_in_buffer(oop ob
}
void AOTStreamedHeapWriter::update_header_for_buffered_addr(address buffered_addr, oop src_obj, Klass* src_klass) {
- assert(UseCompressedClassPointers, "Archived heap only supported for compressed klasses");
narrowKlass nk = ArchiveBuilder::current()->get_requested_narrow_klass(src_klass);
markWord mw = markWord::prototype();
diff --git a/src/hotspot/share/cds/archiveBuilder.cpp b/src/hotspot/share/cds/archiveBuilder.cpp
index b2d6600e44f..cf51897c2f1 100644
--- a/src/hotspot/share/cds/archiveBuilder.cpp
+++ b/src/hotspot/share/cds/archiveBuilder.cpp
@@ -1092,20 +1092,17 @@ class RelocateBufferToRequested : public BitMapClosure {
}
};
-#ifdef _LP64
int ArchiveBuilder::precomputed_narrow_klass_shift() {
- // Legacy Mode:
- // We use 32 bits for narrowKlass, which should cover the full 4G Klass range. Shift can be 0.
+ // Standard Mode:
+ // We use 32 bits for narrowKlass, which should cover a full 4G Klass range. Shift can be 0.
// CompactObjectHeader Mode:
// narrowKlass is much smaller, and we use the highest possible shift value to later get the maximum
// Klass encoding range.
//
// Note that all of this may change in the future, if we decide to correct the pre-calculated
// narrow Klass IDs at archive load time.
- assert(UseCompressedClassPointers, "Only needed for compressed class pointers");
return UseCompactObjectHeaders ? CompressedKlassPointers::max_shift() : 0;
}
-#endif // _LP64
void ArchiveBuilder::relocate_to_requested() {
if (!ro_region()->is_packed()) {
@@ -1174,7 +1171,7 @@ void ArchiveBuilder::write_archive(FileMapInfo* mapinfo, AOTMappedHeapInfo* mapp
AOTMapLogger::dumptime_log(this, mapinfo, mapped_heap_info, streamed_heap_info, bitmap, bitmap_size_in_bytes);
}
CDS_JAVA_HEAP_ONLY(HeapShared::destroy_archived_object_cache());
- FREE_C_HEAP_ARRAY(char, bitmap);
+ FREE_C_HEAP_ARRAY(bitmap);
}
void ArchiveBuilder::write_region(FileMapInfo* mapinfo, int region_idx, DumpRegion* dump_region, bool read_only, bool allow_exec) {
diff --git a/src/hotspot/share/cds/archiveBuilder.hpp b/src/hotspot/share/cds/archiveBuilder.hpp
index b3667ea11b4..6a9df87092b 100644
--- a/src/hotspot/share/cds/archiveBuilder.hpp
+++ b/src/hotspot/share/cds/archiveBuilder.hpp
@@ -484,7 +484,6 @@ public:
void print_stats();
void report_out_of_space(const char* name, size_t needed_bytes);
-#ifdef _LP64
// The CDS archive contains pre-computed narrow Klass IDs. It carries them in the headers of
// archived heap objects. With +UseCompactObjectHeaders, it also carries them in prototypes
// in Klass.
@@ -504,7 +503,6 @@ public:
// TinyClassPointer Mode:
// We use the highest possible shift value to maximize the encoding range size.
static int precomputed_narrow_klass_shift();
-#endif // _LP64
};
diff --git a/src/hotspot/share/cds/archiveUtils.cpp b/src/hotspot/share/cds/archiveUtils.cpp
index f79b1e134e8..6e0608e196b 100644
--- a/src/hotspot/share/cds/archiveUtils.cpp
+++ b/src/hotspot/share/cds/archiveUtils.cpp
@@ -360,8 +360,8 @@ char* DumpRegion::allocate_metaspace_obj(size_t num_bytes, address src, Metaspac
bool is_instance_class = is_class && ((Klass*)src)->is_instance_klass();
#ifdef _LP64
- // More strict alignments needed for UseCompressedClassPointers
- if (is_class && UseCompressedClassPointers) {
+ // More strict alignments needed for Klass objects
+ if (is_class) {
size_t klass_alignment = checked_cast(nth_bit(ArchiveBuilder::precomputed_narrow_klass_shift()));
alignment = MAX2(alignment, klass_alignment);
precond(is_aligned(alignment, SharedSpaceObjectAlignment));
diff --git a/src/hotspot/share/cds/archiveUtils.hpp b/src/hotspot/share/cds/archiveUtils.hpp
index 84ad8e6bdf3..42455adedd0 100644
--- a/src/hotspot/share/cds/archiveUtils.hpp
+++ b/src/hotspot/share/cds/archiveUtils.hpp
@@ -36,6 +36,8 @@
#include "runtime/semaphore.hpp"
#include "utilities/bitMap.hpp"
#include "utilities/exceptions.hpp"
+#include "utilities/growableArray.hpp"
+#include "utilities/hashTable.hpp"
#include "utilities/macros.hpp"
class BootstrapInfo;
@@ -44,7 +46,6 @@ class ReservedSpace;
class VirtualSpace;
template class Array;
-template class GrowableArray;
// ArchivePtrMarker is used to mark the location of pointers embedded in a CDS archive. E.g., when an
// InstanceKlass k is dumped, we mark the location of the k->_name pointer by effectively calling
@@ -401,4 +402,39 @@ public:
void run_task(ArchiveWorkerTask* task);
};
+// A utility class for writing an array of unique items into the
+// AOT cache. For determinism, the order of the array is the same
+// as calls to add(). I.e., if items are added in the order
+// of A, B, A, C, B, D, then the array will be written as {A, B, C, D}
+template
+class ArchivableTable : public AnyObj {
+ using Table = HashTable;
+ Table* _seen_items;
+ GrowableArray* _ordered_array;
+public:
+ ArchivableTable() {
+ _seen_items = new (mtClassShared)Table();
+ _ordered_array = new (mtClassShared)GrowableArray(128, mtClassShared);
+ }
+
+ ~ArchivableTable() {
+ delete _seen_items;
+ delete _ordered_array;
+ }
+
+ void add(T t) {
+ bool created;
+ _seen_items->put_if_absent(t, &created);
+ if (created) {
+ _ordered_array->append(t);
+ }
+ }
+
+ Array* write_ordered_array() {
+ return ArchiveUtils::archive_array(_ordered_array);
+ }
+};
+
+using ArchivableKlassTable = ArchivableTable;
+
#endif // SHARE_CDS_ARCHIVEUTILS_HPP
diff --git a/src/hotspot/share/cds/cdsConfig.cpp b/src/hotspot/share/cds/cdsConfig.cpp
index 9d191bf0121..21066f76932 100644
--- a/src/hotspot/share/cds/cdsConfig.cpp
+++ b/src/hotspot/share/cds/cdsConfig.cpp
@@ -520,7 +520,7 @@ static void substitute_aot_filename(JVMFlagsEnum flag_enum) {
JVMFlag::Error err = JVMFlagAccess::set_ccstr(flag, &new_filename, JVMFlagOrigin::ERGONOMIC);
assert(err == JVMFlag::SUCCESS, "must never fail");
}
- FREE_C_HEAP_ARRAY(char, new_filename);
+ FREE_C_HEAP_ARRAY(new_filename);
}
void CDSConfig::check_aotmode_record() {
@@ -891,10 +891,6 @@ const char* CDSConfig::type_of_archive_being_written() {
// If an incompatible VM options is found, return a text message that explains why
static const char* check_options_incompatible_with_dumping_heap() {
#if INCLUDE_CDS_JAVA_HEAP
- if (!UseCompressedClassPointers) {
- return "UseCompressedClassPointers must be true";
- }
-
return nullptr;
#else
return "JVM not configured for writing Java heap objects";
diff --git a/src/hotspot/share/cds/classListWriter.cpp b/src/hotspot/share/cds/classListWriter.cpp
index 8e1f298e8e3..c90e233df73 100644
--- a/src/hotspot/share/cds/classListWriter.cpp
+++ b/src/hotspot/share/cds/classListWriter.cpp
@@ -49,7 +49,7 @@ void ClassListWriter::init() {
_classlist_file->print_cr("# This file is generated via the -XX:DumpLoadedClassList= option");
_classlist_file->print_cr("# and is used at CDS archive dump time (see -Xshare:dump).");
_classlist_file->print_cr("#");
- FREE_C_HEAP_ARRAY(char, list_name);
+ FREE_C_HEAP_ARRAY(list_name);
}
}
diff --git a/src/hotspot/share/cds/cppVtables.cpp b/src/hotspot/share/cds/cppVtables.cpp
index dc5a777d7b1..57da12dee48 100644
--- a/src/hotspot/share/cds/cppVtables.cpp
+++ b/src/hotspot/share/cds/cppVtables.cpp
@@ -22,7 +22,6 @@
*
*/
-#include "cds/aotGrowableArray.hpp"
#include "cds/aotMetaspace.hpp"
#include "cds/archiveBuilder.hpp"
#include "cds/archiveUtils.hpp"
@@ -41,6 +40,7 @@
#include "oops/typeArrayKlass.hpp"
#include "runtime/arguments.hpp"
#include "utilities/globalDefinitions.hpp"
+#include "utilities/growableArray.hpp"
// Objects of the Metadata types (such as Klass and ConstantPool) have C++ vtables.
// (In GCC this is the field ::_vptr, i.e., first word in the object.)
@@ -58,10 +58,10 @@
#ifndef PRODUCT
-// AOTGrowableArray has a vtable only when in non-product builds (due to
+// GrowableArray has a vtable only when in non-product builds (due to
// the virtual printing functions in AnyObj).
-using GrowableArray_ModuleEntry_ptr = AOTGrowableArray;
+using GrowableArray_ModuleEntry_ptr = GrowableArray;
#define DEBUG_CPP_VTABLE_TYPES_DO(f) \
f(GrowableArray_ModuleEntry_ptr) \
diff --git a/src/hotspot/share/cds/filemap.cpp b/src/hotspot/share/cds/filemap.cpp
index a07f13cefca..186dca4fa13 100644
--- a/src/hotspot/share/cds/filemap.cpp
+++ b/src/hotspot/share/cds/filemap.cpp
@@ -225,15 +225,9 @@ void FileMapHeader::populate(FileMapInfo *info, size_t core_region_alignment,
}
#endif
_compressed_oops = UseCompressedOops;
- _compressed_class_ptrs = UseCompressedClassPointers;
- if (UseCompressedClassPointers) {
-#ifdef _LP64
- _narrow_klass_pointer_bits = CompressedKlassPointers::narrow_klass_pointer_bits();
- _narrow_klass_shift = ArchiveBuilder::precomputed_narrow_klass_shift();
-#endif
- } else {
- _narrow_klass_pointer_bits = _narrow_klass_shift = -1;
- }
+ _narrow_klass_pointer_bits = CompressedKlassPointers::narrow_klass_pointer_bits();
+ _narrow_klass_shift = ArchiveBuilder::precomputed_narrow_klass_shift();
+
// Which JIT compier is used
_compiler_type = (u1)CompilerConfig::compiler_type();
_type_profile_level = TypeProfileLevel;
@@ -295,7 +289,6 @@ void FileMapHeader::print(outputStream* st) {
st->print_cr("- max_heap_size: %zu", _max_heap_size);
st->print_cr("- narrow_oop_mode: %d", _narrow_oop_mode);
st->print_cr("- compressed_oops: %d", _compressed_oops);
- st->print_cr("- compressed_class_ptrs: %d", _compressed_class_ptrs);
st->print_cr("- narrow_klass_pointer_bits: %d", _narrow_klass_pointer_bits);
st->print_cr("- narrow_klass_shift: %d", _narrow_klass_shift);
st->print_cr("- cloned_vtables: %u", cast_to_u4(_cloned_vtables));
@@ -409,7 +402,7 @@ public:
~FileHeaderHelper() {
if (_header != nullptr) {
- FREE_C_HEAP_ARRAY(char, _header);
+ FREE_C_HEAP_ARRAY(_header);
}
if (_fd != -1) {
::close(_fd);
@@ -1926,11 +1919,12 @@ bool FileMapHeader::validate() {
_has_platform_or_app_classes = false;
}
- aot_log_info(aot)("The %s was created with UseCompressedOops = %d, UseCompressedClassPointers = %d, UseCompactObjectHeaders = %d",
- file_type, compressed_oops(), compressed_class_pointers(), compact_headers());
- if (compressed_oops() != UseCompressedOops || compressed_class_pointers() != UseCompressedClassPointers) {
- aot_log_warning(aot)("Unable to use %s.\nThe saved state of UseCompressedOops and UseCompressedClassPointers is "
- "different from runtime, CDS will be disabled.", file_type);
+ aot_log_info(aot)("The %s was created with UseCompressedOops = %d, UseCompactObjectHeaders = %d",
+ file_type, compressed_oops(), compact_headers());
+ if (compressed_oops() != UseCompressedOops) {
+ aot_log_warning(aot)("Unable to use %s.\nThe saved state of UseCompressedOops (%d) is "
+ "different from runtime (%d), CDS will be disabled.", file_type,
+ compressed_oops(), UseCompressedOops);
return false;
}
diff --git a/src/hotspot/share/cds/filemap.hpp b/src/hotspot/share/cds/filemap.hpp
index 56b88df378a..bae08bd5bc7 100644
--- a/src/hotspot/share/cds/filemap.hpp
+++ b/src/hotspot/share/cds/filemap.hpp
@@ -120,7 +120,6 @@ private:
CompressedOops::Mode _narrow_oop_mode; // compressed oop encoding mode
bool _object_streaming_mode; // dump was created for object streaming
bool _compressed_oops; // save the flag UseCompressedOops
- bool _compressed_class_ptrs; // save the flag UseCompressedClassPointers
int _narrow_klass_pointer_bits; // save number of bits in narrowKlass
int _narrow_klass_shift; // save shift width used to pre-compute narrowKlass IDs in archived heap objects
narrowPtr _cloned_vtables; // The address of the first cloned vtable
@@ -200,7 +199,6 @@ public:
bool has_platform_or_app_classes() const { return _has_platform_or_app_classes; }
bool has_aot_linked_classes() const { return _has_aot_linked_classes; }
bool compressed_oops() const { return _compressed_oops; }
- bool compressed_class_pointers() const { return _compressed_class_ptrs; }
int narrow_klass_pointer_bits() const { return _narrow_klass_pointer_bits; }
int narrow_klass_shift() const { return _narrow_klass_shift; }
bool has_full_module_graph() const { return _has_full_module_graph; }
diff --git a/src/hotspot/share/cds/heapShared.cpp b/src/hotspot/share/cds/heapShared.cpp
index 708a19e9a7d..d75816656b0 100644
--- a/src/hotspot/share/cds/heapShared.cpp
+++ b/src/hotspot/share/cds/heapShared.cpp
@@ -112,6 +112,11 @@ static Klass* _test_class = nullptr;
static const ArchivedKlassSubGraphInfoRecord* _test_class_record = nullptr;
#endif
+#ifdef ASSERT
+// All classes that have at least one instance in the cached heap.
+static ArchivableKlassTable* _dumptime_classes_with_cached_oops = nullptr;
+static Array* _runtime_classes_with_cached_oops = nullptr;
+#endif
//
// If you add new entries to the following tables, you should know what you're doing!
@@ -391,6 +396,21 @@ void HeapShared::initialize_streaming() {
}
void HeapShared::enable_gc() {
+#ifdef ASSERT
+ // At this point, a GC may start and will be able to see some or all
+ // of the cached oops. The class of each oop seen by the GC must have
+ // already been loaded. One function with such a requirement is
+ // ClaimMetadataVisitingOopIterateClosure::do_klass().
+ if (is_archived_heap_in_use()) {
+ Array* klasses = _runtime_classes_with_cached_oops;
+
+ for (int i = 0; i < klasses->length(); i++) {
+ assert(klasses->at(i)->class_loader_data() != nullptr,
+ "class of cached oop must have been loaded");
+ }
+ }
+#endif
+
if (AOTStreamedHeapLoader::is_in_use()) {
AOTStreamedHeapLoader::enable_gc();
}
@@ -564,8 +584,10 @@ bool HeapShared::archive_object(oop obj, oop referrer, KlassSubGraphInfo* subgra
return false;
}
+ AOTArtifactFinder::add_cached_class(obj->klass());
AOTOopChecker::check(obj); // Make sure contents of this oop are safe.
count_allocation(obj->size());
+ DEBUG_ONLY(_dumptime_classes_with_cached_oops->add(obj->klass()));
if (HeapShared::is_writing_streaming_mode()) {
AOTStreamedHeapWriter::add_source_obj(obj);
@@ -667,11 +689,6 @@ public:
};
void HeapShared::add_scratch_resolved_references(ConstantPool* src, objArrayOop dest) {
- if (CDSConfig::is_dumping_preimage_static_archive() && scratch_resolved_references(src) != nullptr) {
- // We are in AOT training run. The class has been redefined and we are giving it a new resolved_reference.
- // Ignore it, as this class will be excluded from the AOT config.
- return;
- }
if (SystemDictionaryShared::is_builtin_loader(src->pool_holder()->class_loader_data())) {
_scratch_objects_table->set_oop(src, dest);
}
@@ -681,10 +698,17 @@ objArrayOop HeapShared::scratch_resolved_references(ConstantPool* src) {
return (objArrayOop)_scratch_objects_table->get_oop(src);
}
+void HeapShared::remove_scratch_resolved_references(ConstantPool* src) {
+ if (CDSConfig::is_dumping_heap()) {
+ _scratch_objects_table->remove_oop(src);
+ }
+}
+
void HeapShared::init_dumping() {
_scratch_objects_table = new (mtClass)MetaspaceObjToOopHandleTable();
_pending_roots = new GrowableArrayCHeap(500);
_pending_roots->append(nullptr); // root index 0 represents a null oop
+ DEBUG_ONLY(_dumptime_classes_with_cached_oops = new (mtClassShared)ArchivableKlassTable());
}
void HeapShared::init_scratch_objects_for_basic_type_mirrors(TRAPS) {
@@ -966,6 +990,8 @@ void HeapShared::write_heap(AOTMappedHeapInfo* mapped_heap_info, AOTStreamedHeap
ArchiveBuilder::OtherROAllocMark mark;
write_subgraph_info_table();
+ DEBUG_ONLY(_runtime_classes_with_cached_oops = _dumptime_classes_with_cached_oops->write_ordered_array());
+
delete _pending_roots;
_pending_roots = nullptr;
@@ -1277,6 +1303,7 @@ void HeapShared::serialize_tables(SerializeClosure* soc) {
_run_time_subgraph_info_table.serialize_header(soc);
soc->do_ptr(&_run_time_special_subgraph);
+ DEBUG_ONLY(soc->do_ptr(&_runtime_classes_with_cached_oops));
}
static void verify_the_heap(Klass* k, const char* which) {
diff --git a/src/hotspot/share/cds/heapShared.hpp b/src/hotspot/share/cds/heapShared.hpp
index c3ad1f666b1..10ea35ab56e 100644
--- a/src/hotspot/share/cds/heapShared.hpp
+++ b/src/hotspot/share/cds/heapShared.hpp
@@ -451,6 +451,7 @@ private:
static void write_heap(AOTMappedHeapInfo* mapped_heap_info, AOTStreamedHeapInfo* streamed_heap_info) NOT_CDS_JAVA_HEAP_RETURN;
static objArrayOop scratch_resolved_references(ConstantPool* src);
static void add_scratch_resolved_references(ConstantPool* src, objArrayOop dest) NOT_CDS_JAVA_HEAP_RETURN;
+ static void remove_scratch_resolved_references(ConstantPool* src) NOT_CDS_JAVA_HEAP_RETURN;
static void init_dumping() NOT_CDS_JAVA_HEAP_RETURN;
static void init_scratch_objects_for_basic_type_mirrors(TRAPS) NOT_CDS_JAVA_HEAP_RETURN;
static void init_box_classes(TRAPS) NOT_CDS_JAVA_HEAP_RETURN;
diff --git a/src/hotspot/share/cds/lambdaProxyClassDictionary.hpp b/src/hotspot/share/cds/lambdaProxyClassDictionary.hpp
index dfb75532917..b20e998bba6 100644
--- a/src/hotspot/share/cds/lambdaProxyClassDictionary.hpp
+++ b/src/hotspot/share/cds/lambdaProxyClassDictionary.hpp
@@ -22,8 +22,8 @@
*
*/
-#ifndef SHARE_CDS_LAMBDAPROXYCLASSINFO_HPP
-#define SHARE_CDS_LAMBDAPROXYCLASSINFO_HPP
+#ifndef SHARE_CDS_LAMBDAPROXYCLASSDICTIONARY_HPP
+#define SHARE_CDS_LAMBDAPROXYCLASSDICTIONARY_HPP
#include "cds/aotCompressedPointers.hpp"
#include "cds/aotMetaspace.hpp"
@@ -331,4 +331,4 @@ public:
static void print_statistics(outputStream* st, bool is_static_archive);
};
-#endif // SHARE_CDS_LAMBDAPROXYCLASSINFO_HPP
+#endif // SHARE_CDS_LAMBDAPROXYCLASSDICTIONARY_HPP
diff --git a/src/hotspot/share/ci/ciMethodData.cpp b/src/hotspot/share/ci/ciMethodData.cpp
index 533e8659968..5e623e2b965 100644
--- a/src/hotspot/share/ci/ciMethodData.cpp
+++ b/src/hotspot/share/ci/ciMethodData.cpp
@@ -537,8 +537,8 @@ void ciMethodData::clear_escape_info() {
if (mdo != nullptr) {
mdo->clear_escape_info();
ArgInfoData *aid = arg_info();
- int arg_count = (aid == nullptr) ? 0 : aid->number_of_args();
- for (int i = 0; i < arg_count; i++) {
+ int arg_size = (aid == nullptr) ? 0 : aid->size_of_args();
+ for (int i = 0; i < arg_size; i++) {
set_arg_modified(i, 0);
}
}
@@ -554,8 +554,8 @@ void ciMethodData::update_escape_info() {
mdo->set_arg_local(_arg_local);
mdo->set_arg_stack(_arg_stack);
mdo->set_arg_returned(_arg_returned);
- int arg_count = mdo->method()->size_of_parameters();
- for (int i = 0; i < arg_count; i++) {
+ int arg_size = mdo->method()->size_of_parameters();
+ for (int i = 0; i < arg_size; i++) {
mdo->set_arg_modified(i, arg_modified(i));
}
}
@@ -652,7 +652,7 @@ void ciMethodData::set_arg_modified(int arg, uint val) {
ArgInfoData *aid = arg_info();
if (aid == nullptr)
return;
- assert(arg >= 0 && arg < aid->number_of_args(), "valid argument number");
+ assert(arg >= 0 && arg < aid->size_of_args(), "valid argument number");
aid->set_arg_modified(arg, val);
}
@@ -672,7 +672,7 @@ uint ciMethodData::arg_modified(int arg) const {
ArgInfoData *aid = arg_info();
if (aid == nullptr)
return 0;
- assert(arg >= 0 && arg < aid->number_of_args(), "valid argument number");
+ assert(arg >= 0 && arg < aid->size_of_args(), "valid argument number");
return aid->arg_modified(arg);
}
diff --git a/src/hotspot/share/ci/ciReplay.cpp b/src/hotspot/share/ci/ciReplay.cpp
index 6266c024260..35522e75877 100644
--- a/src/hotspot/share/ci/ciReplay.cpp
+++ b/src/hotspot/share/ci/ciReplay.cpp
@@ -600,7 +600,7 @@ class CompileReplay : public StackObj {
_nesting.check(); // Check if a reallocation in the resource arena is safe
int new_length = _buffer_length * 2;
// Next call will throw error in case of OOM.
- _buffer = REALLOC_RESOURCE_ARRAY(char, _buffer, _buffer_length, new_length);
+ _buffer = REALLOC_RESOURCE_ARRAY(_buffer, _buffer_length, new_length);
_buffer_length = new_length;
}
if (c == '\n') {
diff --git a/src/hotspot/share/classfile/classFileParser.cpp b/src/hotspot/share/classfile/classFileParser.cpp
index c1f00cbe536..d5ee16fec32 100644
--- a/src/hotspot/share/classfile/classFileParser.cpp
+++ b/src/hotspot/share/classfile/classFileParser.cpp
@@ -34,6 +34,7 @@
#include "classfile/packageEntry.hpp"
#include "classfile/symbolTable.hpp"
#include "classfile/systemDictionary.hpp"
+#include "classfile/systemDictionaryShared.hpp"
#include "classfile/verificationType.hpp"
#include "classfile/verifier.hpp"
#include "classfile/vmClasses.hpp"
@@ -86,9 +87,6 @@
#include "utilities/macros.hpp"
#include "utilities/ostream.hpp"
#include "utilities/utf8.hpp"
-#if INCLUDE_CDS
-#include "classfile/systemDictionaryShared.hpp"
-#endif
// We generally try to create the oops directly when parsing, rather than
// allocating temporary data structures and copying the bytes twice. A
@@ -194,7 +192,7 @@ void ClassFileParser::parse_constant_pool_entries(const ClassFileStream* const s
// so we don't need bounds-check for reading tag.
const u1 tag = cfs->get_u1_fast();
switch (tag) {
- case JVM_CONSTANT_Class : {
+ case JVM_CONSTANT_Class: {
cfs->guarantee_more(3, CHECK); // name_index, tag/access_flags
const u2 name_index = cfs->get_u2_fast();
cp->klass_index_at_put(index, name_index);
@@ -2365,8 +2363,8 @@ Method* ClassFileParser::parse_method(const ClassFileStream* const cfs,
}
if (lvt_cnt == max_lvt_cnt) {
max_lvt_cnt <<= 1;
- localvariable_table_length = REALLOC_RESOURCE_ARRAY(u2, localvariable_table_length, lvt_cnt, max_lvt_cnt);
- localvariable_table_start = REALLOC_RESOURCE_ARRAY(const unsafe_u2*, localvariable_table_start, lvt_cnt, max_lvt_cnt);
+ localvariable_table_length = REALLOC_RESOURCE_ARRAY(localvariable_table_length, lvt_cnt, max_lvt_cnt);
+ localvariable_table_start = REALLOC_RESOURCE_ARRAY(localvariable_table_start, lvt_cnt, max_lvt_cnt);
}
localvariable_table_start[lvt_cnt] =
parse_localvariable_table(cfs,
@@ -2395,8 +2393,8 @@ Method* ClassFileParser::parse_method(const ClassFileStream* const cfs,
// Parse local variable type table
if (lvtt_cnt == max_lvtt_cnt) {
max_lvtt_cnt <<= 1;
- localvariable_type_table_length = REALLOC_RESOURCE_ARRAY(u2, localvariable_type_table_length, lvtt_cnt, max_lvtt_cnt);
- localvariable_type_table_start = REALLOC_RESOURCE_ARRAY(const unsafe_u2*, localvariable_type_table_start, lvtt_cnt, max_lvtt_cnt);
+ localvariable_type_table_length = REALLOC_RESOURCE_ARRAY(localvariable_type_table_length, lvtt_cnt, max_lvtt_cnt);
+ localvariable_type_table_start = REALLOC_RESOURCE_ARRAY(localvariable_type_table_start, lvtt_cnt, max_lvtt_cnt);
}
localvariable_type_table_start[lvtt_cnt] =
parse_localvariable_table(cfs,
@@ -4403,14 +4401,14 @@ void ClassFileParser::verify_legal_field_modifiers(jint flags,
TRAPS) const {
if (!_need_verify) { return; }
- const bool is_public = (flags & JVM_ACC_PUBLIC) != 0;
- const bool is_protected = (flags & JVM_ACC_PROTECTED) != 0;
- const bool is_private = (flags & JVM_ACC_PRIVATE) != 0;
- const bool is_static = (flags & JVM_ACC_STATIC) != 0;
- const bool is_final = (flags & JVM_ACC_FINAL) != 0;
- const bool is_volatile = (flags & JVM_ACC_VOLATILE) != 0;
- const bool is_transient = (flags & JVM_ACC_TRANSIENT) != 0;
- const bool is_enum = (flags & JVM_ACC_ENUM) != 0;
+ const bool is_public = (flags & JVM_ACC_PUBLIC) != 0;
+ const bool is_protected = (flags & JVM_ACC_PROTECTED) != 0;
+ const bool is_private = (flags & JVM_ACC_PRIVATE) != 0;
+ const bool is_static = (flags & JVM_ACC_STATIC) != 0;
+ const bool is_final = (flags & JVM_ACC_FINAL) != 0;
+ const bool is_volatile = (flags & JVM_ACC_VOLATILE) != 0;
+ const bool is_transient = (flags & JVM_ACC_TRANSIENT) != 0;
+ const bool is_enum = (flags & JVM_ACC_ENUM) != 0;
const bool major_gte_1_5 = _major_version >= JAVA_1_5_VERSION;
bool is_illegal = false;
@@ -5256,6 +5254,9 @@ void ClassFileParser::fill_instance_klass(InstanceKlass* ik,
if (!is_internal()) {
ik->print_class_load_logging(_loader_data, module_entry, _stream);
+ if (CDSConfig::is_dumping_archive()) {
+ SystemDictionaryShared::check_code_source(ik, _stream);
+ }
if (ik->minor_version() == JAVA_PREVIEW_MINOR_VERSION &&
ik->major_version() == JVM_CLASSFILE_MAJOR_VERSION &&
diff --git a/src/hotspot/share/classfile/classLoader.cpp b/src/hotspot/share/classfile/classLoader.cpp
index eced83577cb..c19a1770aef 100644
--- a/src/hotspot/share/classfile/classLoader.cpp
+++ b/src/hotspot/share/classfile/classLoader.cpp
@@ -251,7 +251,7 @@ const char* ClassPathEntry::copy_path(const char* path) {
}
ClassPathDirEntry::~ClassPathDirEntry() {
- FREE_C_HEAP_ARRAY(char, _dir);
+ FREE_C_HEAP_ARRAY(_dir);
}
ClassFileStream* ClassPathDirEntry::open_stream(JavaThread* current, const char* name) {
@@ -280,7 +280,7 @@ ClassFileStream* ClassPathDirEntry::open_stream(JavaThread* current, const char*
#ifdef ASSERT
// Freeing path is a no-op here as buffer prevents it from being reclaimed. But we keep it for
// debug builds so that we guard against use-after-free bugs.
- FREE_RESOURCE_ARRAY_IN_THREAD(current, char, path, path_len);
+ FREE_RESOURCE_ARRAY_IN_THREAD(current, path, path_len);
#endif
// We don't verify the length of the classfile stream fits in an int, but this is the
// bootloader so we have control of this.
@@ -291,7 +291,7 @@ ClassFileStream* ClassPathDirEntry::open_stream(JavaThread* current, const char*
}
}
}
- FREE_RESOURCE_ARRAY_IN_THREAD(current, char, path, path_len);
+ FREE_RESOURCE_ARRAY_IN_THREAD(current, path, path_len);
return nullptr;
}
@@ -302,7 +302,7 @@ ClassPathZipEntry::ClassPathZipEntry(jzfile* zip, const char* zip_name) : ClassP
ClassPathZipEntry::~ClassPathZipEntry() {
ZipLibrary::close(_zip);
- FREE_C_HEAP_ARRAY(char, _zip_name);
+ FREE_C_HEAP_ARRAY(_zip_name);
}
bool ClassPathZipEntry::has_entry(JavaThread* current, const char* name) {
@@ -1438,7 +1438,7 @@ bool ClassLoader::is_module_observable(const char* module_name) {
struct stat st;
const char *path = get_exploded_module_path(module_name, true);
bool res = os::stat(path, &st) == 0;
- FREE_C_HEAP_ARRAY(char, path);
+ FREE_C_HEAP_ARRAY(path);
return res;
}
jlong size;
diff --git a/src/hotspot/share/classfile/classLoaderData.cpp b/src/hotspot/share/classfile/classLoaderData.cpp
index dfc3b74db96..d1ea9c09d4c 100644
--- a/src/hotspot/share/classfile/classLoaderData.cpp
+++ b/src/hotspot/share/classfile/classLoaderData.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2012, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -46,6 +46,7 @@
// The bootstrap loader (represented by null) also has a ClassLoaderData,
// the singleton class the_null_class_loader_data().
+#include "cds/heapShared.hpp"
#include "classfile/classLoaderData.inline.hpp"
#include "classfile/classLoaderDataGraph.inline.hpp"
#include "classfile/dictionary.hpp"
@@ -899,6 +900,7 @@ void ClassLoaderData::free_deallocate_list() {
if (m->is_method()) {
MetadataFactory::free_metadata(this, (Method*)m);
} else if (m->is_constantPool()) {
+ HeapShared::remove_scratch_resolved_references((ConstantPool*)m);
MetadataFactory::free_metadata(this, (ConstantPool*)m);
} else if (m->is_klass()) {
MetadataFactory::free_metadata(this, (InstanceKlass*)m);
diff --git a/src/hotspot/share/classfile/compactHashtable.cpp b/src/hotspot/share/classfile/compactHashtable.cpp
index de67971c403..f06f1986d8b 100644
--- a/src/hotspot/share/classfile/compactHashtable.cpp
+++ b/src/hotspot/share/classfile/compactHashtable.cpp
@@ -69,7 +69,7 @@ CompactHashtableWriter::~CompactHashtableWriter() {
delete bucket;
}
- FREE_C_HEAP_ARRAY(GrowableArray*, _buckets);
+ FREE_C_HEAP_ARRAY(_buckets);
}
// Add an entry to the temporary hash table
diff --git a/src/hotspot/share/classfile/compactHashtable.hpp b/src/hotspot/share/classfile/compactHashtable.hpp
index 81f2951289d..1711c5f8cd3 100644
--- a/src/hotspot/share/classfile/compactHashtable.hpp
+++ b/src/hotspot/share/classfile/compactHashtable.hpp
@@ -307,14 +307,9 @@ public:
template
inline void iterate(ITER* iter) const { iterate([&](V v) { iter->do_value(v); }); }
- template
- inline void iterate(const Function& function) const { // lambda enabled API
- iterate(const_cast(function));
- }
-
// Iterate through the values in the table, stopping when the lambda returns false.
template
- inline void iterate(Function& function) const { // lambda enabled API
+ inline void iterate(Function function) const { // lambda enabled API
for (u4 i = 0; i < _bucket_count; i++) {
u4 bucket_info = _buckets[i];
u4 bucket_offset = BUCKET_OFFSET(bucket_info);
diff --git a/src/hotspot/share/classfile/fieldLayoutBuilder.cpp b/src/hotspot/share/classfile/fieldLayoutBuilder.cpp
index a87e12edc96..adf4e1e63fa 100644
--- a/src/hotspot/share/classfile/fieldLayoutBuilder.cpp
+++ b/src/hotspot/share/classfile/fieldLayoutBuilder.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -37,7 +37,7 @@
LayoutRawBlock::LayoutRawBlock(Kind kind, int size) :
_next_block(nullptr),
_prev_block(nullptr),
- _kind(kind),
+ _block_kind(kind),
_offset(-1),
_alignment(1),
_size(size),
@@ -52,7 +52,7 @@ LayoutRawBlock::LayoutRawBlock(Kind kind, int size) :
LayoutRawBlock::LayoutRawBlock(int index, Kind kind, int size, int alignment, bool is_reference) :
_next_block(nullptr),
_prev_block(nullptr),
- _kind(kind),
+ _block_kind(kind),
_offset(-1),
_alignment(alignment),
_size(size),
@@ -148,8 +148,8 @@ void FieldLayout::initialize_instance_layout(const InstanceKlass* super_klass, b
LayoutRawBlock* FieldLayout::first_field_block() {
LayoutRawBlock* block = _start;
- while (block->kind() != LayoutRawBlock::INHERITED && block->kind() != LayoutRawBlock::REGULAR
- && block->kind() != LayoutRawBlock::FLATTENED && block->kind() != LayoutRawBlock::PADDING) {
+ while (block->block_kind() != LayoutRawBlock::INHERITED && block->block_kind() != LayoutRawBlock::REGULAR
+ && block->block_kind() != LayoutRawBlock::FLATTENED && block->block_kind() != LayoutRawBlock::PADDING) {
block = block->next_block();
}
return block;
@@ -190,7 +190,7 @@ void FieldLayout::add(GrowableArray* list, LayoutRawBlock* star
assert(cursor != nullptr, "Sanity check");
last_search_success = true;
while (cursor != start) {
- if (cursor->kind() == LayoutRawBlock::EMPTY && cursor->fit(b->size(), b->alignment())) {
+ if (cursor->block_kind() == LayoutRawBlock::EMPTY && cursor->fit(b->size(), b->alignment())) {
if (candidate == nullptr || cursor->size() < candidate->size()) {
candidate = cursor;
}
@@ -202,7 +202,7 @@ void FieldLayout::add(GrowableArray* list, LayoutRawBlock* star
last_search_success = false;
}
assert(candidate != nullptr, "Candidate must not be null");
- assert(candidate->kind() == LayoutRawBlock::EMPTY, "Candidate must be an empty block");
+ assert(candidate->block_kind() == LayoutRawBlock::EMPTY, "Candidate must be an empty block");
assert(candidate->fit(b->size(), b->alignment()), "Candidate must be able to store the block");
}
@@ -221,7 +221,7 @@ void FieldLayout::add_field_at_offset(LayoutRawBlock* block, int offset, LayoutR
while (slot != nullptr) {
if ((slot->offset() <= block->offset() && (slot->offset() + slot->size()) > block->offset()) ||
slot == _last){
- assert(slot->kind() == LayoutRawBlock::EMPTY, "Matching slot must be an empty slot");
+ assert(slot->block_kind() == LayoutRawBlock::EMPTY, "Matching slot must be an empty slot");
assert(slot->size() >= block->offset() + block->size() ,"Matching slot must be big enough");
if (slot->offset() < block->offset()) {
int adjustment = block->offset() - slot->offset();
@@ -261,7 +261,7 @@ void FieldLayout::add_contiguously(GrowableArray* list, LayoutR
} else {
LayoutRawBlock* first = list->at(0);
candidate = last_block()->prev_block();
- while (candidate->kind() != LayoutRawBlock::EMPTY || !candidate->fit(size, first->alignment())) {
+ while (candidate->block_kind() != LayoutRawBlock::EMPTY || !candidate->fit(size, first->alignment())) {
if (candidate == start) {
candidate = last_block();
break;
@@ -269,7 +269,7 @@ void FieldLayout::add_contiguously(GrowableArray* list, LayoutR
candidate = candidate->prev_block();
}
assert(candidate != nullptr, "Candidate must not be null");
- assert(candidate->kind() == LayoutRawBlock::EMPTY, "Candidate must be an empty block");
+ assert(candidate->block_kind() == LayoutRawBlock::EMPTY, "Candidate must be an empty block");
assert(candidate->fit(size, first->alignment()), "Candidate must be able to store the whole contiguous block");
}
@@ -281,7 +281,7 @@ void FieldLayout::add_contiguously(GrowableArray* list, LayoutR
}
LayoutRawBlock* FieldLayout::insert_field_block(LayoutRawBlock* slot, LayoutRawBlock* block) {
- assert(slot->kind() == LayoutRawBlock::EMPTY, "Blocks can only be inserted in empty blocks");
+ assert(slot->block_kind() == LayoutRawBlock::EMPTY, "Blocks can only be inserted in empty blocks");
if (slot->offset() % block->alignment() != 0) {
int adjustment = block->alignment() - (slot->offset() % block->alignment());
LayoutRawBlock* adj = new LayoutRawBlock(LayoutRawBlock::EMPTY, adjustment);
@@ -362,7 +362,7 @@ void FieldLayout::fill_holes(const InstanceKlass* super_klass) {
b = b->next_block();
}
assert(b->next_block() == nullptr, "Invariant at this point");
- assert(b->kind() != LayoutRawBlock::EMPTY, "Sanity check");
+ assert(b->block_kind() != LayoutRawBlock::EMPTY, "Sanity check");
// If the super class has @Contended annotation, a padding block is
// inserted at the end to ensure that fields from the subclasses won't share
@@ -384,7 +384,7 @@ void FieldLayout::fill_holes(const InstanceKlass* super_klass) {
}
LayoutRawBlock* FieldLayout::insert(LayoutRawBlock* slot, LayoutRawBlock* block) {
- assert(slot->kind() == LayoutRawBlock::EMPTY, "Blocks can only be inserted in empty blocks");
+ assert(slot->block_kind() == LayoutRawBlock::EMPTY, "Blocks can only be inserted in empty blocks");
assert(slot->offset() % block->alignment() == 0, "Incompatible alignment");
block->set_offset(slot->offset());
slot->set_offset(slot->offset() + block->size());
@@ -425,7 +425,7 @@ void FieldLayout::print(outputStream* output, bool is_static, const InstanceKlas
ResourceMark rm;
LayoutRawBlock* b = _blocks;
while(b != _last) {
- switch(b->kind()) {
+ switch(b->block_kind()) {
case LayoutRawBlock::REGULAR: {
FieldInfo* fi = _field_info->adr_at(b->field_index());
output->print_cr(" @%d \"%s\" %s %d/%d %s",
@@ -596,11 +596,13 @@ void FieldLayoutBuilder::regular_field_sorting() {
}
}
-void FieldLayoutBuilder::insert_contended_padding(LayoutRawBlock* slot) {
+LayoutRawBlock* FieldLayoutBuilder::insert_contended_padding(LayoutRawBlock* slot) {
+ LayoutRawBlock* padding = nullptr;
if (ContendedPaddingWidth > 0) {
- LayoutRawBlock* padding = new LayoutRawBlock(LayoutRawBlock::PADDING, ContendedPaddingWidth);
+ padding = new LayoutRawBlock(LayoutRawBlock::PADDING, ContendedPaddingWidth);
_layout->insert(slot, padding);
}
+ return padding;
}
// Computation of regular classes layout is an evolution of the previous default layout
@@ -620,10 +622,14 @@ void FieldLayoutBuilder::compute_regular_layout() {
regular_field_sorting();
if (_is_contended) {
- _layout->set_start(_layout->last_block());
// insertion is currently easy because the current strategy doesn't try to fill holes
// in super classes layouts => the _start block is by consequence the _last_block
- insert_contended_padding(_layout->start());
+ _layout->set_start(_layout->last_block());
+ LayoutRawBlock* padding = insert_contended_padding(_layout->start());
+ if (padding != nullptr) {
+ // Setting the padding block as start ensures we do not insert past it.
+ _layout->set_start(padding);
+ }
need_tail_padding = true;
}
@@ -639,7 +645,13 @@ void FieldLayoutBuilder::compute_regular_layout() {
for (int i = 0; i < _contended_groups.length(); i++) {
FieldGroup* cg = _contended_groups.at(i);
LayoutRawBlock* start = _layout->last_block();
- insert_contended_padding(start);
+ LayoutRawBlock* padding = insert_contended_padding(start);
+
+ // Do not insert fields past the padding block.
+ if (padding != nullptr) {
+ start = padding;
+ }
+
_layout->add(cg->primitive_fields(), start);
_layout->add(cg->oop_fields(), start);
need_tail_padding = true;
diff --git a/src/hotspot/share/classfile/fieldLayoutBuilder.hpp b/src/hotspot/share/classfile/fieldLayoutBuilder.hpp
index 82bbaefc623..a45131ec9a3 100644
--- a/src/hotspot/share/classfile/fieldLayoutBuilder.hpp
+++ b/src/hotspot/share/classfile/fieldLayoutBuilder.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -65,7 +65,7 @@ class LayoutRawBlock : public ResourceObj {
private:
LayoutRawBlock* _next_block;
LayoutRawBlock* _prev_block;
- Kind _kind;
+ Kind _block_kind;
int _offset;
int _alignment;
int _size;
@@ -79,7 +79,7 @@ class LayoutRawBlock : public ResourceObj {
void set_next_block(LayoutRawBlock* next) { _next_block = next; }
LayoutRawBlock* prev_block() const { return _prev_block; }
void set_prev_block(LayoutRawBlock* prev) { _prev_block = prev; }
- Kind kind() const { return _kind; }
+ Kind block_kind() const { return _block_kind; }
int offset() const {
assert(_offset >= 0, "Must be initialized");
return _offset;
@@ -173,7 +173,7 @@ class FieldLayout : public ResourceObj {
LayoutRawBlock* first_empty_block() {
LayoutRawBlock* block = _start;
- while (block->kind() != LayoutRawBlock::EMPTY) {
+ while (block->block_kind() != LayoutRawBlock::EMPTY) {
block = block->next_block();
}
return block;
@@ -250,7 +250,7 @@ class FieldLayoutBuilder : public ResourceObj {
void build_layout();
void compute_regular_layout();
- void insert_contended_padding(LayoutRawBlock* slot);
+ LayoutRawBlock* insert_contended_padding(LayoutRawBlock* slot);
private:
void prologue();
diff --git a/src/hotspot/share/classfile/moduleEntry.cpp b/src/hotspot/share/classfile/moduleEntry.cpp
index b5b8aa4ef55..c7fadeaea9b 100644
--- a/src/hotspot/share/classfile/moduleEntry.cpp
+++ b/src/hotspot/share/classfile/moduleEntry.cpp
@@ -23,7 +23,6 @@
*/
#include "cds/aotClassLocation.hpp"
-#include "cds/aotGrowableArray.inline.hpp"
#include "cds/archiveBuilder.hpp"
#include "cds/archiveUtils.hpp"
#include "cds/cdsConfig.hpp"
@@ -168,7 +167,7 @@ void ModuleEntry::add_read(ModuleEntry* m) {
} else {
if (reads() == nullptr) {
// Lazily create a module's reads list
- AOTGrowableArray* new_reads = new (mtModule) AOTGrowableArray(MODULE_READS_SIZE, mtModule);
+ GrowableArray* new_reads = new (mtModule) GrowableArray(MODULE_READS_SIZE, mtModule);
set_reads(new_reads);
}
diff --git a/src/hotspot/share/classfile/moduleEntry.hpp b/src/hotspot/share/classfile/moduleEntry.hpp
index 1a0251a2c2a..10dec73e9fa 100644
--- a/src/hotspot/share/classfile/moduleEntry.hpp
+++ b/src/hotspot/share/classfile/moduleEntry.hpp
@@ -25,7 +25,6 @@
#ifndef SHARE_CLASSFILE_MODULEENTRY_HPP
#define SHARE_CLASSFILE_MODULEENTRY_HPP
-#include "cds/aotGrowableArray.hpp"
#include "jni.h"
#include "memory/metaspaceClosureType.hpp"
#include "oops/oopHandle.hpp"
@@ -70,7 +69,7 @@ private:
// for shared classes from this module
Symbol* _name; // name of this module
ClassLoaderData* _loader_data;
- AOTGrowableArray* _reads; // list of modules that are readable by this module
+ GrowableArray* _reads; // list of modules that are readable by this module
Symbol* _version; // module version number
Symbol* _location; // module location
@@ -118,10 +117,10 @@ public:
bool can_read(ModuleEntry* m) const;
bool has_reads_list() const;
- AOTGrowableArray* reads() const {
+ GrowableArray* reads() const {
return _reads;
}
- void set_reads(AOTGrowableArray* r) {
+ void set_reads(GrowableArray* r) {
_reads = r;
}
void pack_reads() {
diff --git a/src/hotspot/share/classfile/packageEntry.cpp b/src/hotspot/share/classfile/packageEntry.cpp
index 3e61f2e3a3e..3eb50fcb5a7 100644
--- a/src/hotspot/share/classfile/packageEntry.cpp
+++ b/src/hotspot/share/classfile/packageEntry.cpp
@@ -22,7 +22,6 @@
*
*/
-#include "cds/aotGrowableArray.inline.hpp"
#include "cds/aotMetaspace.hpp"
#include "cds/archiveBuilder.hpp"
#include "cds/archiveUtils.hpp"
@@ -83,7 +82,7 @@ void PackageEntry::add_qexport(ModuleEntry* m) {
if (!has_qual_exports_list()) {
// Lazily create a package's qualified exports list.
// Initial size is small, do not anticipate export lists to be large.
- _qualified_exports = new (mtModule) AOTGrowableArray(QUAL_EXP_SIZE, mtModule);
+ _qualified_exports = new (mtModule) GrowableArray(QUAL_EXP_SIZE, mtModule);
}
// Determine, based on this newly established export to module m,
diff --git a/src/hotspot/share/classfile/packageEntry.hpp b/src/hotspot/share/classfile/packageEntry.hpp
index 7b174a92287..e064e53b263 100644
--- a/src/hotspot/share/classfile/packageEntry.hpp
+++ b/src/hotspot/share/classfile/packageEntry.hpp
@@ -25,7 +25,6 @@
#ifndef SHARE_CLASSFILE_PACKAGEENTRY_HPP
#define SHARE_CLASSFILE_PACKAGEENTRY_HPP
-#include "cds/aotGrowableArray.hpp"
#include "classfile/moduleEntry.hpp"
#include "memory/metaspaceClosureType.hpp"
#include "oops/symbol.hpp"
@@ -116,7 +115,7 @@ private:
bool _must_walk_exports;
// Contains list of modules this package is qualifiedly exported to. Access
// to this list is protected by the Module_lock.
- AOTGrowableArray* _qualified_exports;
+ GrowableArray* _qualified_exports;
JFR_ONLY(DEFINE_TRACE_ID_FIELD;)
// Initial size of a package entry's list of qualified exports.
diff --git a/src/hotspot/share/classfile/resolutionErrors.cpp b/src/hotspot/share/classfile/resolutionErrors.cpp
index c41d5d2f052..958d4812cbb 100644
--- a/src/hotspot/share/classfile/resolutionErrors.cpp
+++ b/src/hotspot/share/classfile/resolutionErrors.cpp
@@ -114,15 +114,15 @@ ResolutionErrorEntry::~ResolutionErrorEntry() {
Symbol::maybe_decrement_refcount(_cause);
if (_message != nullptr) {
- FREE_C_HEAP_ARRAY(char, _message);
+ FREE_C_HEAP_ARRAY(_message);
}
if (_cause_msg != nullptr) {
- FREE_C_HEAP_ARRAY(char, _cause_msg);
+ FREE_C_HEAP_ARRAY(_cause_msg);
}
if (nest_host_error() != nullptr) {
- FREE_C_HEAP_ARRAY(char, nest_host_error());
+ FREE_C_HEAP_ARRAY(nest_host_error());
}
}
diff --git a/src/hotspot/share/classfile/stackMapTableFormat.hpp b/src/hotspot/share/classfile/stackMapTableFormat.hpp
index 2b89c53278a..4906f4b9d80 100644
--- a/src/hotspot/share/classfile/stackMapTableFormat.hpp
+++ b/src/hotspot/share/classfile/stackMapTableFormat.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2010, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2010, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -221,9 +221,11 @@ class stack_map_frame {
class same_frame : public stack_map_frame {
private:
static int frame_type_to_offset_delta(u1 frame_type) {
- return frame_type + 1; }
+ return frame_type + 1;
+ }
static u1 offset_delta_to_frame_type(int offset_delta) {
- return checked_cast(offset_delta - 1); }
+ return checked_cast(offset_delta - 1);
+ }
public:
@@ -327,9 +329,11 @@ class same_locals_1_stack_item_frame : public stack_map_frame {
address type_addr() const { return frame_type_addr() + sizeof(u1); }
static int frame_type_to_offset_delta(u1 frame_type) {
- return frame_type - 63; }
+ return frame_type - 63;
+ }
static u1 offset_delta_to_frame_type(int offset_delta) {
- return (u1)(offset_delta + 63); }
+ return (u1)(offset_delta + 63);
+ }
public:
static bool is_frame_type(u1 tag) {
@@ -657,9 +661,11 @@ class full_frame : public stack_map_frame {
address num_locals_addr() const { return offset_delta_addr() + sizeof(u2); }
address locals_addr() const { return num_locals_addr() + sizeof(u2); }
address stack_slots_addr(address end_of_locals) const {
- return end_of_locals; }
+ return end_of_locals;
+ }
address stack_addr(address end_of_locals) const {
- return stack_slots_addr(end_of_locals) + sizeof(u2); }
+ return stack_slots_addr(end_of_locals) + sizeof(u2);
+ }
enum { _frame_id = 255 };
@@ -930,11 +936,14 @@ class stack_map_table {
class stack_map_table_attribute {
private:
address name_index_addr() const {
- return (address)this; }
+ return (address)this;
+ }
address attribute_length_addr() const {
- return name_index_addr() + sizeof(u2); }
+ return name_index_addr() + sizeof(u2);
+ }
address stack_map_table_addr() const {
- return attribute_length_addr() + sizeof(u4); }
+ return attribute_length_addr() + sizeof(u4);
+ }
NONCOPYABLE(stack_map_table_attribute);
protected:
@@ -948,9 +957,11 @@ class stack_map_table_attribute {
}
u2 name_index() const {
- return Bytes::get_Java_u2(name_index_addr()); }
+ return Bytes::get_Java_u2(name_index_addr());
+ }
u4 attribute_length() const {
- return Bytes::get_Java_u4(attribute_length_addr()); }
+ return Bytes::get_Java_u4(attribute_length_addr());
+ }
stack_map_table* table() const {
return stack_map_table::at(stack_map_table_addr());
}
diff --git a/src/hotspot/share/classfile/systemDictionary.cpp b/src/hotspot/share/classfile/systemDictionary.cpp
index 0b47c749df8..8483551fd4f 100644
--- a/src/hotspot/share/classfile/systemDictionary.cpp
+++ b/src/hotspot/share/classfile/systemDictionary.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -416,7 +416,7 @@ static inline void log_circularity_error(Symbol* name, PlaceholderEntry* probe)
//
// resolve_with_circularity_detection adds a DETECT_CIRCULARITY placeholder to the placeholder table before calling
// resolve_instance_class_or_null. ClassCircularityError is detected when a DETECT_CIRCULARITY or LOAD_INSTANCE
-// placeholder for the same thread, class, classloader is found.
+// placeholder for the same thread, class, and classloader is found.
// This can be seen with logging option: -Xlog:class+load+placeholders=debug.
//
InstanceKlass* SystemDictionary::resolve_with_circularity_detection(Symbol* class_name,
diff --git a/src/hotspot/share/classfile/systemDictionaryShared.cpp b/src/hotspot/share/classfile/systemDictionaryShared.cpp
index 5947050e31a..fd30fc6766f 100644
--- a/src/hotspot/share/classfile/systemDictionaryShared.cpp
+++ b/src/hotspot/share/classfile/systemDictionaryShared.cpp
@@ -89,11 +89,9 @@ DEBUG_ONLY(bool SystemDictionaryShared::_class_loading_may_happen = true;)
#ifdef ASSERT
static void check_klass_after_loading(const Klass* k) {
-#ifdef _LP64
- if (k != nullptr && UseCompressedClassPointers) {
+ if (k != nullptr) {
CompressedKlassPointers::check_encodable(k);
}
-#endif
}
#endif
@@ -204,6 +202,20 @@ DumpTimeClassInfo* SystemDictionaryShared::get_info_locked(InstanceKlass* k) {
return info;
}
+void SystemDictionaryShared::check_code_source(InstanceKlass* ik, const ClassFileStream* cfs) {
+ if (CDSConfig::is_dumping_preimage_static_archive() && !is_builtin_loader(ik->class_loader_data())) {
+ if (cfs == nullptr || cfs->source() == nullptr || strncmp(cfs->source(), "file:", 5) != 0) {
+ // AOT cache filtering:
+ // For non-built-in loaders, cache only the classes that have a file: code source, so
+ // we can avoid caching dynamically generated classes that are likely to change from
+ // run to run. This is similar to the filtering in ClassListWriter::write_to_stream()
+ // for the classic CDS static archive.
+ SystemDictionaryShared::log_exclusion(ik, "Not loaded from \"file:\" code source");
+ SystemDictionaryShared::set_excluded(ik);
+ }
+ }
+}
+
bool SystemDictionaryShared::should_be_excluded_impl(InstanceKlass* k, DumpTimeClassInfo* info) {
assert_lock_strong(DumpTimeTable_lock);
diff --git a/src/hotspot/share/classfile/systemDictionaryShared.hpp b/src/hotspot/share/classfile/systemDictionaryShared.hpp
index 33b245e26fc..c837a386344 100644
--- a/src/hotspot/share/classfile/systemDictionaryShared.hpp
+++ b/src/hotspot/share/classfile/systemDictionaryShared.hpp
@@ -235,6 +235,7 @@ public:
static void update_shared_entry(InstanceKlass* klass, int id);
static void set_shared_class_misc_info(InstanceKlass* k, ClassFileStream* cfs);
+ static void check_code_source(InstanceKlass* ik, const ClassFileStream* cfs) NOT_CDS_RETURN;
static InstanceKlass* lookup_from_stream(Symbol* class_name,
Handle class_loader,
Handle protection_domain,
diff --git a/src/hotspot/share/classfile/vmIntrinsics.hpp b/src/hotspot/share/classfile/vmIntrinsics.hpp
index e84acd62284..3f85fd16b61 100644
--- a/src/hotspot/share/classfile/vmIntrinsics.hpp
+++ b/src/hotspot/share/classfile/vmIntrinsics.hpp
@@ -469,6 +469,9 @@ class methodHandle;
do_intrinsic(_Reference_clear0, java_lang_ref_Reference, clear0_name, void_method_signature, F_RN) \
do_intrinsic(_PhantomReference_clear0, java_lang_ref_PhantomReference, clear0_name, void_method_signature, F_RN) \
\
+ do_intrinsic(_Reference_reachabilityFence, java_lang_ref_Reference, reachabilityFence_name, object_void_signature, F_S) \
+ do_name(reachabilityFence_name, "reachabilityFence") \
+ \
/* support for com.sun.crypto.provider.AES_Crypt and some of its callers */ \
do_class(com_sun_crypto_provider_aescrypt, "com/sun/crypto/provider/AES_Crypt") \
do_intrinsic(_aescrypt_encryptBlock, com_sun_crypto_provider_aescrypt, encryptBlock_name, byteArray_int_byteArray_int_signature, F_R) \
diff --git a/src/hotspot/share/classfile/vmSymbols.hpp b/src/hotspot/share/classfile/vmSymbols.hpp
index 2ae42bebcfd..33d00b93365 100644
--- a/src/hotspot/share/classfile/vmSymbols.hpp
+++ b/src/hotspot/share/classfile/vmSymbols.hpp
@@ -702,6 +702,7 @@ class SerializeClosure;
template(appendToClassPathForInstrumentation_name, "appendToClassPathForInstrumentation") \
do_alias(appendToClassPathForInstrumentation_signature, string_void_signature) \
template(serializePropertiesToByteArray_name, "serializePropertiesToByteArray") \
+ template(serializeSecurityPropertiesToByteArray_name, "serializeSecurityPropertiesToByteArray") \
template(serializeAgentPropertiesToByteArray_name, "serializeAgentPropertiesToByteArray") \
template(encodeThrowable_name, "encodeThrowable") \
template(encodeThrowable_signature, "(Ljava/lang/Throwable;JI)I") \
diff --git a/src/hotspot/share/code/aotCodeCache.cpp b/src/hotspot/share/code/aotCodeCache.cpp
index b29cf906736..89b7d16c31e 100644
--- a/src/hotspot/share/code/aotCodeCache.cpp
+++ b/src/hotspot/share/code/aotCodeCache.cpp
@@ -33,13 +33,18 @@
#include "classfile/javaAssertions.hpp"
#include "code/aotCodeCache.hpp"
#include "code/codeCache.hpp"
+#include "gc/shared/barrierSetAssembler.hpp"
+#include "gc/shared/barrierSetNMethod.hpp"
#include "gc/shared/cardTableBarrierSet.hpp"
#include "gc/shared/gcConfig.hpp"
#include "logging/logStream.hpp"
#include "memory/memoryReserver.hpp"
+#include "prims/jvmtiThreadState.hpp"
+#include "prims/upcallLinker.hpp"
#include "runtime/deoptimization.hpp"
#include "runtime/flags/flagSetting.hpp"
#include "runtime/globals_extension.hpp"
+#include "runtime/icache.hpp"
#include "runtime/java.hpp"
#include "runtime/mutexLocker.hpp"
#include "runtime/os.inline.hpp"
@@ -73,11 +78,23 @@ const char* aot_code_entry_kind_name[] = {
#undef DECL_KIND_STRING
};
+// Stream to printing AOTCodeCache loading failure.
+// Print to error channel when -XX:AOTMode is set to "on"
+static LogStream& load_failure_log() {
+ static LogStream err_stream(LogLevel::Error, LogTagSetMapping::tagset());
+ static LogStream dbg_stream(LogLevel::Debug, LogTagSetMapping::tagset());
+ if (RequireSharedSpaces) {
+ return err_stream;
+ } else {
+ return dbg_stream;
+ }
+}
+
static void report_load_failure() {
if (AbortVMOnAOTCodeFailure) {
vm_exit_during_initialization("Unable to use AOT Code Cache.", nullptr);
}
- log_info(aot, codecache, init)("Unable to use AOT Code Cache.");
+ load_failure_log().print_cr("Unable to use AOT Code Cache.");
AOTCodeCache::disable_caching();
}
@@ -86,7 +103,7 @@ static void report_store_failure() {
tty->print_cr("Unable to create AOT Code Cache.");
vm_abort(false);
}
- log_info(aot, codecache, exit)("Unable to create AOT Code Cache.");
+ log_error(aot, codecache, exit)("Unable to create AOT Code Cache.");
AOTCodeCache::disable_caching();
}
@@ -156,10 +173,13 @@ static uint32_t encode_id(AOTCodeEntry::Kind kind, int id) {
} else if (kind == AOTCodeEntry::C1Blob) {
assert(StubInfo::is_c1(static_cast(id)), "not a c1 blob id %d", id);
return id;
- } else {
- // kind must be AOTCodeEntry::C2Blob
+ } else if (kind == AOTCodeEntry::C2Blob) {
assert(StubInfo::is_c2(static_cast(id)), "not a c2 blob id %d", id);
return id;
+ } else {
+ // kind must be AOTCodeEntry::StubGenBlob
+ assert(StubInfo::is_stubgen(static_cast(id)), "not a stubgen blob id %d", id);
+ return id;
}
}
@@ -184,9 +204,6 @@ void AOTCodeCache::initialize() {
return; // AOTCache must be specified to dump and use AOT code
}
- // Disable stubs caching until JDK-8357398 is fixed.
- FLAG_SET_ERGO(AOTStubCaching, false);
-
if (VerifyOops) {
// Disable AOT stubs caching when VerifyOops flag is on.
// Verify oops code generated a lot of C strings which overflow
@@ -284,6 +301,19 @@ bool AOTCodeCache::open_cache(bool is_dumping, bool is_using) {
return true;
}
+// Called after continuations_init() when continuation stub callouts
+// have been initialized
+void AOTCodeCache::init3() {
+ if (opened_cache == nullptr) {
+ return;
+ }
+ // initialize external routines for continuations so we can save
+ // generated continuation blob that references them
+ AOTCodeAddressTable* table = opened_cache->_table;
+ assert(table != nullptr, "should be initialized already");
+ table->init_extrs2();
+}
+
void AOTCodeCache::dump() {
if (is_on()) {
assert(is_on_for_dump(), "should be called only when dumping AOT code");
@@ -342,6 +372,7 @@ AOTCodeCache::AOTCodeCache(bool is_dumping, bool is_using) :
log_info (aot, codecache, init)("Loaded %u AOT code entries from AOT Code Cache", _load_header->entries_count());
log_debug(aot, codecache, init)(" Adapters: total=%u", _load_header->adapters_count());
log_debug(aot, codecache, init)(" Shared Blobs: total=%u", _load_header->shared_blobs_count());
+ log_debug(aot, codecache, init)(" StubGen Blobs: total=%d", _load_header->stubgen_blobs_count());
log_debug(aot, codecache, init)(" C1 Blobs: total=%u", _load_header->C1_blobs_count());
log_debug(aot, codecache, init)(" C2 Blobs: total=%u", _load_header->C2_blobs_count());
log_debug(aot, codecache, init)(" AOT code cache size: %u bytes", _load_header->cache_size());
@@ -359,58 +390,80 @@ AOTCodeCache::AOTCodeCache(bool is_dumping, bool is_using) :
_table = new AOTCodeAddressTable();
}
-void AOTCodeCache::init_early_stubs_table() {
- AOTCodeAddressTable* table = addr_table();
- if (table != nullptr) {
- table->init_early_stubs();
+void AOTCodeCache::add_stub_entries(StubId stub_id, address start, GrowableArray *entries, int begin_idx) {
+ EntryId entry_id = StubInfo::entry_base(stub_id);
+ add_stub_entry(entry_id, start);
+ // skip past first entry
+ entry_id = StubInfo::next_in_stub(stub_id, entry_id);
+ // now check for any more entries
+ int count = StubInfo::entry_count(stub_id) - 1;
+ assert(start != nullptr, "invalid start address for stub %s", StubInfo::name(stub_id));
+ assert(entries == nullptr || begin_idx + count <= entries->length(), "sanity");
+ // write any extra entries
+ for (int i = 0; i < count; i++) {
+ assert(entry_id != EntryId::NO_ENTRYID, "not enough entries for stub %s", StubInfo::name(stub_id));
+ address a = entries->at(begin_idx + i);
+ add_stub_entry(entry_id, a);
+ entry_id = StubInfo::next_in_stub(stub_id, entry_id);
+ }
+ assert(entry_id == EntryId::NO_ENTRYID, "too many entries for stub %s", StubInfo::name(stub_id));
+}
+
+void AOTCodeCache::add_stub_entry(EntryId entry_id, address a) {
+ if (a != nullptr) {
+ if (_table != nullptr) {
+ log_trace(aot, codecache, stubs)("Publishing stub entry %s at address " INTPTR_FORMAT, StubInfo::name(entry_id), p2i(a));
+ return _table->add_stub_entry(entry_id, a);
+ }
}
}
-void AOTCodeCache::init_shared_blobs_table() {
+void AOTCodeCache::set_shared_stubs_complete() {
AOTCodeAddressTable* table = addr_table();
if (table != nullptr) {
- table->init_shared_blobs();
+ table->set_shared_stubs_complete();
}
}
-void AOTCodeCache::init_early_c1_table() {
+void AOTCodeCache::set_c1_stubs_complete() {
AOTCodeAddressTable* table = addr_table();
if (table != nullptr) {
- table->init_early_c1();
+ table->set_c1_stubs_complete();
+ }
+}
+
+void AOTCodeCache::set_c2_stubs_complete() {
+ AOTCodeAddressTable* table = addr_table();
+ if (table != nullptr) {
+ table->set_c2_stubs_complete();
+ }
+}
+
+void AOTCodeCache::set_stubgen_stubs_complete() {
+ AOTCodeAddressTable* table = addr_table();
+ if (table != nullptr) {
+ table->set_stubgen_stubs_complete();
}
}
void AOTCodeCache::Config::record(uint cpu_features_offset) {
- _flags = 0;
-#ifdef ASSERT
- _flags |= debugVM;
-#endif
- if (UseCompressedOops) {
- _flags |= compressedOops;
- }
- if (UseCompressedClassPointers) {
- _flags |= compressedClassPointers;
- }
- if (UseTLAB) {
- _flags |= useTLAB;
- }
- if (JavaAssertions::systemClassDefault()) {
- _flags |= systemClassAssertions;
- }
- if (JavaAssertions::userClassDefault()) {
- _flags |= userClassAssertions;
- }
- if (EnableContended) {
- _flags |= enableContendedPadding;
- }
- if (RestrictContended) {
- _flags |= restrictContendedPadding;
- }
- _compressedOopShift = CompressedOops::shift();
+
+#define AOTCODECACHE_SAVE_VAR(type, name) _saved_ ## name = name;
+#define AOTCODECACHE_SAVE_FUN(type, name, fun) _saved_ ## name = fun;
+
+ AOTCODECACHE_CONFIGS_DO(AOTCODECACHE_SAVE_VAR, AOTCODECACHE_SAVE_FUN);
+
+ // Special configs that cannot be checked with macros
_compressedOopBase = CompressedOops::base();
- _compressedKlassShift = CompressedKlassPointers::shift();
- _contendedPaddingWidth = ContendedPaddingWidth;
- _gc = (uint)Universe::heap()->kind();
+
+#if defined(X86) && !defined(ZERO)
+ _useUnalignedLoadStores = UseUnalignedLoadStores;
+#endif
+
+#if defined(AARCH64) && !defined(ZERO)
+ _avoidUnalignedAccesses = AvoidUnalignedAccesses;
+#endif
+
_cpu_features_offset = cpu_features_offset;
}
@@ -441,78 +494,114 @@ bool AOTCodeCache::Config::verify_cpu_features(AOTCodeCache* cache) const {
}
}
} else {
- if (log.is_enabled()) {
+ if (load_failure_log().is_enabled()) {
ResourceMark rm; // required for stringStream::as_string()
stringStream ss;
char* runtime_cpu_features = NEW_RESOURCE_ARRAY(char, VM_Version::cpu_features_size());
VM_Version::store_cpu_features(runtime_cpu_features);
VM_Version::get_missing_features_name(cached_cpu_features_buffer, runtime_cpu_features, ss);
- log.print_cr("AOT Code Cache disabled: required cpu features are missing: %s", ss.as_string());
+ load_failure_log().print_cr("AOT Code Cache disabled: required cpu features are missing: %s", ss.as_string());
}
return false;
}
return true;
}
-bool AOTCodeCache::Config::verify(AOTCodeCache* cache) const {
- // First checks affect all cached AOT code
-#ifdef ASSERT
- if ((_flags & debugVM) == 0) {
- log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created by product VM, it can't be used by debug VM");
- return false;
- }
-#else
- if ((_flags & debugVM) != 0) {
- log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created by debug VM, it can't be used by product VM");
- return false;
- }
+#define AOTCODECACHE_DISABLED_MSG "AOT Code Cache disabled: it was created with %s = "
+
+// Special case, print "GC = ..." to be more understandable.
+inline void log_config_mismatch(CollectedHeap::Name saved, CollectedHeap::Name current, const char* name/*unused*/) {
+ load_failure_log().print_cr("AOT Code Cache disabled: it was created with GC = \"%s\" vs current \"%s\"",
+ GCConfig::hs_err_name(saved), GCConfig::hs_err_name(current));
+}
+
+inline void log_config_mismatch(bool saved, bool current, const char* name) {
+ load_failure_log().print_cr(AOTCODECACHE_DISABLED_MSG "%s vs current %s", name,
+ saved ? "true" : "false", current ? "true" : "false");
+}
+
+inline void log_config_mismatch(int saved, int current, const char* name) {
+ load_failure_log().print_cr(AOTCODECACHE_DISABLED_MSG "%d vs current %d", name, saved, current);
+}
+
+inline void log_config_mismatch(uint saved, uint current, const char* name) {
+ load_failure_log().print_cr(AOTCODECACHE_DISABLED_MSG "%u vs current %u", name, saved, current);
+}
+
+#ifdef _LP64
+inline void log_config_mismatch(intx saved, intx current, const char* name) {
+ load_failure_log().print_cr(AOTCODECACHE_DISABLED_MSG "%zd vs current %zd", name, saved, current);
+}
+
+inline void log_config_mismatch(uintx saved, uintx current, const char* name) {
+ load_failure_log().print_cr(AOTCODECACHE_DISABLED_MSG "%zu vs current %zu", name, saved, current);
+}
#endif
- CollectedHeap::Name aot_gc = (CollectedHeap::Name)_gc;
- if (aot_gc != Universe::heap()->kind()) {
- log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with different GC: %s vs current %s", GCConfig::hs_err_name(aot_gc), GCConfig::hs_err_name());
+template
+bool check_config(T saved, T current, const char* name) {
+ if (saved != current) {
+ log_config_mismatch(saved, current, name);
return false;
+ } else {
+ return true;
}
+}
- if (((_flags & compressedClassPointers) != 0) != UseCompressedClassPointers) {
- log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with UseCompressedClassPointers = %s", UseCompressedClassPointers ? "false" : "true");
- return false;
- }
- if (_compressedKlassShift != (uint)CompressedKlassPointers::shift()) {
- log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with CompressedKlassPointers::shift() = %d vs current %d", _compressedKlassShift, CompressedKlassPointers::shift());
- return false;
- }
-
- // The following checks do not affect AOT adapters caching
-
- if (((_flags & compressedOops) != 0) != UseCompressedOops) {
- log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with UseCompressedOops = %s", UseCompressedOops ? "false" : "true");
- AOTStubCaching = false;
- }
- if (_compressedOopShift != (uint)CompressedOops::shift()) {
- log_debug(aot, codecache, init)("AOT Code Cache disabled: it was created with different CompressedOops::shift(): %d vs current %d", _compressedOopShift, CompressedOops::shift());
- AOTStubCaching = false;
- }
-
- // This should be the last check as it only disables AOTStubCaching
- if ((_compressedOopBase == nullptr || CompressedOops::base() == nullptr) && (_compressedOopBase != CompressedOops::base())) {
- log_debug(aot, codecache, init)("AOTStubCaching is disabled: incompatible CompressedOops::base(): %p vs current %p", _compressedOopBase, CompressedOops::base());
- AOTStubCaching = false;
- }
-
+bool AOTCodeCache::Config::verify(AOTCodeCache* cache) const {
+ // check CPU features before checking flags that may be
+ // auto-configured in response to them
if (!verify_cpu_features(cache)) {
return false;
}
+
+ // Tests for config options which might affect validity of adapters,
+ // stubs or nmethods. Currently we take a pessemistic stand and
+ // drop the whole cache if any of these are changed.
+
+#define AOTCODECACHE_CHECK_VAR(type, name) \
+ if (!check_config(_saved_ ## name, name, #name)) { return false; }
+#define AOTCODECACHE_CHECK_FUN(type, name, fun) \
+ if (!check_config(_saved_ ## name, fun, #fun)) { return false; }
+
+ AOTCODECACHE_CONFIGS_DO(AOTCODECACHE_CHECK_VAR, AOTCODECACHE_CHECK_FUN);
+
+ // Special configs that cannot be checked with macros
+
+ if ((_compressedOopBase == nullptr || CompressedOops::base() == nullptr) && (_compressedOopBase != CompressedOops::base())) {
+ load_failure_log().print_cr("AOT Code Cache disabled: incompatible CompressedOops::base(): %p vs current %p",
+ _compressedOopBase, CompressedOops::base());
+ return false;
+ }
+
+#if defined(X86) && !defined(ZERO)
+ // switching off UseUnalignedLoadStores can affect validity of fill
+ // stubs
+ if (_useUnalignedLoadStores && !UseUnalignedLoadStores) {
+ log_config_mismatch(_useUnalignedLoadStores, UseUnalignedLoadStores, "UseUnalignedLoadStores");
+ return false;
+ }
+#endif // defined(X86) && !defined(ZERO)
+
+#if defined(AARCH64) && !defined(ZERO)
+ // switching on AvoidUnalignedAccesses may affect validity of array
+ // copy stubs and nmethods
+ if (!_avoidUnalignedAccesses && AvoidUnalignedAccesses) {
+ log_config_mismatch(_avoidUnalignedAccesses, AvoidUnalignedAccesses, "AvoidUnalignedAccesses");
+ return false;
+ }
+#endif // defined(AARCH64) && !defined(ZERO)
+
return true;
}
bool AOTCodeCache::Header::verify(uint load_size) const {
if (_version != AOT_CODE_VERSION) {
- log_debug(aot, codecache, init)("AOT Code Cache disabled: different AOT Code version %d vs %d recorded in AOT Code header", AOT_CODE_VERSION, _version);
+ load_failure_log().print_cr("AOT Code Cache disabled: different AOT Code version %d vs %d recorded in AOT Code header", AOT_CODE_VERSION, _version);
return false;
}
if (load_size < _cache_size) {
- log_debug(aot, codecache, init)("AOT Code Cache disabled: AOT Code Cache size %d < %d recorded in AOT Code header", load_size, _cache_size);
+ load_failure_log().print_cr("AOT Code Cache disabled: AOT Code Cache size %d < %d recorded in AOT Code header", load_size, _cache_size);
return false;
}
return true;
@@ -546,6 +635,13 @@ AOTCodeReader::AOTCodeReader(AOTCodeCache* cache, AOTCodeEntry* entry) {
_load_buffer = cache->cache_buffer();
_read_position = 0;
_lookup_failed = false;
+ _name = nullptr;
+ _reloc_data = nullptr;
+ _reloc_count = 0;
+ _oop_maps = nullptr;
+ _entry_kind = AOTCodeEntry::None;
+ _stub_data = nullptr;
+ _id = -1;
}
void AOTCodeReader::set_read_position(uint pos) {
@@ -763,6 +859,7 @@ bool AOTCodeCache::finish_write() {
AOTCodeEntry* entries_address = _store_entries; // Pointer to latest entry
uint adapters_count = 0;
uint shared_blobs_count = 0;
+ uint stubgen_blobs_count = 0;
uint C1_blobs_count = 0;
uint C2_blobs_count = 0;
uint max_size = 0;
@@ -779,7 +876,7 @@ bool AOTCodeCache::finish_write() {
current += size;
uint n = write_bytes(&(entries_address[i]), sizeof(AOTCodeEntry));
if (n != sizeof(AOTCodeEntry)) {
- FREE_C_HEAP_ARRAY(uint, search);
+ FREE_C_HEAP_ARRAY(search);
return false;
}
search[entries_count*2 + 0] = entries_address[i].id();
@@ -790,6 +887,8 @@ bool AOTCodeCache::finish_write() {
adapters_count++;
} else if (kind == AOTCodeEntry::SharedBlob) {
shared_blobs_count++;
+ } else if (kind == AOTCodeEntry::StubGenBlob) {
+ stubgen_blobs_count++;
} else if (kind == AOTCodeEntry::C1Blob) {
C1_blobs_count++;
} else if (kind == AOTCodeEntry::C2Blob) {
@@ -798,7 +897,7 @@ bool AOTCodeCache::finish_write() {
}
if (entries_count == 0) {
log_info(aot, codecache, exit)("AOT Code Cache was not created: no entires");
- FREE_C_HEAP_ARRAY(uint, search);
+ FREE_C_HEAP_ARRAY(search);
return true; // Nothing to write
}
assert(entries_count <= store_count, "%d > %d", entries_count, store_count);
@@ -814,7 +913,7 @@ bool AOTCodeCache::finish_write() {
qsort(search, entries_count, 2*sizeof(uint), uint_cmp);
search_size = 2 * entries_count * sizeof(uint);
copy_bytes((const char*)search, (address)current, search_size);
- FREE_C_HEAP_ARRAY(uint, search);
+ FREE_C_HEAP_ARRAY(search);
current += search_size;
// Write entries
@@ -826,6 +925,7 @@ bool AOTCodeCache::finish_write() {
log_debug(aot, codecache, exit)(" Adapters: total=%u", adapters_count);
log_debug(aot, codecache, exit)(" Shared Blobs: total=%d", shared_blobs_count);
+ log_debug(aot, codecache, exit)(" StubGen Blobs: total=%d", stubgen_blobs_count);
log_debug(aot, codecache, exit)(" C1 Blobs: total=%d", C1_blobs_count);
log_debug(aot, codecache, exit)(" C2 Blobs: total=%d", C2_blobs_count);
log_debug(aot, codecache, exit)(" AOT code cache size: %u bytes, max entry's size: %u bytes", size, max_size);
@@ -835,7 +935,8 @@ bool AOTCodeCache::finish_write() {
header->init(size, (uint)strings_count, strings_offset,
entries_count, new_entries_offset,
adapters_count, shared_blobs_count,
- C1_blobs_count, C2_blobs_count, cpu_features_offset);
+ stubgen_blobs_count, C1_blobs_count,
+ C2_blobs_count, cpu_features_offset);
log_info(aot, codecache, exit)("Wrote %d AOT code entries to AOT Code Cache", entries_count);
}
@@ -844,19 +945,53 @@ bool AOTCodeCache::finish_write() {
//------------------Store/Load AOT code ----------------------
-bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, uint id, const char* name) {
+bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, uint id, const char* name, AOTStubData* stub_data, CodeBuffer* code_buffer) {
+ assert(AOTCodeEntry::is_valid_entry_kind(entry_kind), "invalid entry_kind %d", entry_kind);
+
+ // we only expect stub data and a code buffer for a multi stub blob
+ assert(AOTCodeEntry::is_multi_stub_blob(entry_kind) == (stub_data != nullptr),
+ "entry_kind %d does not match stub_data pointer %p",
+ entry_kind, stub_data);
+
+ assert((stub_data == nullptr) == (code_buffer == nullptr),
+ "stub data and code buffer must both be null or both non null");
+
+ // If this is a stub and the cache is on for either load or dump we
+ // need to insert the stub entries into the AOTCacheAddressTable so
+ // that relocs which refer to entries defined by this blob get
+ // translated correctly.
+ //
+ // Entry insertion needs to be be done up front before writing the
+ // blob because some blobs rely on internal daisy-chain references
+ // from one entry to another.
+ //
+ // Entry insertion also needs to be done even if the cache is open
+ // for use but not for dump. This may be needed when an archived
+ // blob omits some entries -- either because of a config change or a
+ // load failure -- with the result that the entries end up being
+ // generated. These generated entry addresses may be needed to
+ // resolve references from subsequently loaded blobs (for either
+ // stubs or nmethods).
+
+ if (is_on() && AOTCodeEntry::is_blob(entry_kind)) {
+ publish_stub_addresses(blob, (BlobId)id, stub_data);
+ }
+
AOTCodeCache* cache = open_for_dump();
if (cache == nullptr) {
return false;
}
- assert(AOTCodeEntry::is_valid_entry_kind(entry_kind), "invalid entry_kind %d", entry_kind);
-
if (AOTCodeEntry::is_adapter(entry_kind) && !is_dumping_adapter()) {
return false;
}
if (AOTCodeEntry::is_blob(entry_kind) && !is_dumping_stub()) {
return false;
}
+ // we do not currently store C2 stubs because we are seeing weird
+ // memory errors when loading them -- see JDK-8357593
+ if (entry_kind == AOTCodeEntry::C2Blob) {
+ return false;
+ }
log_debug(aot, codecache, stubs)("Writing blob '%s' (id=%u, kind=%s) to AOT Code Cache", name, id, aot_code_entry_kind_name[entry_kind]);
#ifdef ASSERT
@@ -896,8 +1031,44 @@ bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind
}
CodeBlob::archive_blob(&blob, archive_buffer);
- uint reloc_data_size = blob.relocation_size();
- n = cache->write_bytes((address)blob.relocation_begin(), reloc_data_size);
+ // For a relocatable code blob its relocations are linked from the
+ // blob. However, for a non-relocatable (stubgen) blob we only have
+ // transient relocations attached to the code buffer that are added
+ // in order to support AOT-load time patching. in either case, we
+ // need to explicitly save these relocs when storing the blob to the
+ // archive so we can then reload them and reattach them to either
+ // the blob or to a code buffer when we reload the blob into a
+ // production JVM.
+ //
+ // Either way we are then in a position to iterate over the relocs
+ // and AOT patch the ones that refer to code that may move between
+ // assembly and production time. We also need to save and restore
+ // AOT address table indexes for the target addresses of affected
+ // relocs. That happens below.
+
+ int reloc_count;
+ address reloc_data;
+ if (AOTCodeEntry::is_multi_stub_blob(entry_kind)) {
+ CodeSection* cs = code_buffer->code_section(CodeBuffer::SECT_INSTS);
+ reloc_count = (cs->has_locs() ? cs->locs_count() : 0);
+ reloc_data = (reloc_count > 0 ? (address)cs->locs_start() : nullptr);
+ } else {
+ reloc_count = blob.relocation_size() / sizeof(relocInfo);
+ reloc_data = (address)blob.relocation_begin();
+ }
+ n = cache->write_bytes(&reloc_count, sizeof(int));
+ if (n != sizeof(int)) {
+ return false;
+ }
+ if (AOTCodeEntry::is_multi_stub_blob(entry_kind)) {
+ // align to heap word size before writing the relocs so we can
+ // install them into a code buffer when they get restored
+ if (!cache->align_write()) {
+ return false;
+ }
+ }
+ uint reloc_data_size = (uint)(reloc_count * sizeof(relocInfo));
+ n = cache->write_bytes(reloc_data, reloc_data_size);
if (n != reloc_data_size) {
return false;
}
@@ -910,8 +1081,40 @@ bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind
has_oop_maps = true;
}
+ // In the case of a multi-stub blob we need to write start, end,
+ // secondary entries and extras. For any other blob entry addresses
+ // beyond the blob start will be stored in the blob as offsets.
+ if (stub_data != nullptr) {
+ if (!cache->write_stub_data(blob, stub_data)) {
+ return false;
+ }
+ }
+
+ // now we have added all the other data we can write details of any
+ // extra the AOT relocations
+
+ bool write_ok = true;
+ if (AOTCodeEntry::is_multi_stub_blob(entry_kind)) {
+ if (reloc_count > 0) {
+ CodeSection* cs = code_buffer->code_section(CodeBuffer::SECT_INSTS);
+ RelocIterator iter(cs);
+ write_ok = cache->write_relocations(blob, iter);
+ }
+ } else {
+ RelocIterator iter(&blob);
+ write_ok = cache->write_relocations(blob, iter);
+ }
+
+ if (!write_ok) {
+ if (!cache->failed()) {
+ // We may miss an address in AOT table - skip this code blob.
+ cache->set_write_position(entry_position);
+ }
+ return false;
+ }
+
#ifndef PRODUCT
- // Write asm remarks
+ // Write asm remarks after relocation info
if (!cache->write_asm_remarks(blob)) {
return false;
}
@@ -920,15 +1123,8 @@ bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind
}
#endif /* PRODUCT */
- if (!cache->write_relocations(blob)) {
- if (!cache->failed()) {
- // We may miss an address in AOT table - skip this code blob.
- cache->set_write_position(entry_position);
- }
- return false;
- }
-
uint entry_size = cache->_write_position - entry_position;
+
AOTCodeEntry* entry = new(cache) AOTCodeEntry(entry_kind, encode_id(entry_kind, id),
entry_position, entry_size, name_offset, name_size,
blob_offset, has_oop_maps, blob.content_begin());
@@ -936,25 +1132,141 @@ bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind
return true;
}
-bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, BlobId id) {
- assert(AOTCodeEntry::is_blob(entry_kind),
- "wrong entry kind for blob id %s", StubInfo::name(id));
- return store_code_blob(blob, entry_kind, (uint)id, StubInfo::name(id));
+bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, uint id, const char* name) {
+ assert(!AOTCodeEntry::is_blob(entry_kind),
+ "wrong entry kind for numeric id %d", id);
+ return store_code_blob(blob, entry_kind, (uint)id, name, nullptr, nullptr);
}
-CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, uint id, const char* name) {
+bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, BlobId id) {
+ assert(AOTCodeEntry::is_single_stub_blob(entry_kind),
+ "wrong entry kind for blob id %s", StubInfo::name(id));
+ return store_code_blob(blob, entry_kind, (uint)id, StubInfo::name(id), nullptr, nullptr);
+}
+
+bool AOTCodeCache::store_code_blob(CodeBlob& blob, AOTCodeEntry::Kind entry_kind, BlobId id, AOTStubData* stub_data, CodeBuffer* code_buffer) {
+ assert(AOTCodeEntry::is_multi_stub_blob(entry_kind),
+ "wrong entry kind for multi stub blob id %s", StubInfo::name(id));
+ return store_code_blob(blob, entry_kind, (uint)id, StubInfo::name(id), stub_data, code_buffer);
+}
+
+bool AOTCodeCache::write_stub_data(CodeBlob &blob, AOTStubData *stub_data) {
+ BlobId blob_id = stub_data->blob_id();
+ StubId stub_id = StubInfo::stub_base(blob_id);
+ address blob_base = blob.code_begin();
+ int stub_cnt = StubInfo::stub_count(blob_id);
+ int n;
+
+ LogStreamHandle(Trace, aot, codecache, stubs) log;
+
+ if (log.is_enabled()) {
+ log.print_cr("======== Stub data starts at offset %d", _write_position);
+ }
+
+ for (int i = 0; i < stub_cnt; i++, stub_id = StubInfo::next_in_blob(blob_id, stub_id)) {
+ // for each stub we find in the ranges list we write an int
+ // sequence where
+ //
+ // - start_pos is the stub start address encoded as a code section offset
+ //
+ // - end is the stub end address encoded as an offset from start
+ //
+ // - N counts the number of stub-local entries/extras
+ //
+ // - offseti is a stub-local entry/extra address encoded as len for
+ // a null address otherwise as an offset in range [1,len-1]
+
+ StubAddrRange& range = stub_data->get_range(i);
+ GrowableArray& addresses = stub_data->address_array();
+ int base = range.start_index();
+ if (base >= 0) {
+ n = write_bytes(&stub_id, sizeof(StubId));
+ if (n != sizeof(StubId)) {
+ return false;
+ }
+ address start = addresses.at(base);
+ assert (blob_base <= start, "sanity");
+ uint offset = (uint)(start - blob_base);
+ n = write_bytes(&offset, sizeof(uint));
+ if (n != sizeof(int)) {
+ return false;
+ }
+ address end = addresses.at(base + 1);
+ assert (start < end, "sanity");
+ offset = (uint)(end - start);
+ n = write_bytes(&offset, sizeof(uint));
+ if (n != sizeof(int)) {
+ return false;
+ }
+ // write number of secondary and extra entries
+ int count = range.count() - 2;
+ n = write_bytes(&count, sizeof(int));
+ if (n != sizeof(int)) {
+ return false;
+ }
+ for (int j = 0; j < count; j++) {
+ address next = addresses.at(base + 2 + j);
+ if (next != nullptr) {
+ // n.b. This maps next == end to the stub length which
+ // means we will reconstitute the address as nullptr. That
+ // happens when we have a handler range covers the end of
+ // a stub and needs to be handled specially by the client
+ // that restores the extras.
+ assert(start <= next && next <= end, "sanity");
+ offset = (uint)(next - start);
+ } else {
+ // this can happen when a stub is not generated or an
+ // extra is the common handler target
+ offset = NULL_ADDRESS_MARKER;
+ }
+ n = write_bytes(&offset, sizeof(uint));
+ if (n != sizeof(int)) {
+ return false;
+ }
+ }
+ if (log.is_enabled()) {
+ log.print_cr("======== wrote stub %s and %d addresses up to offset %d",
+ StubInfo::name(stub_id), range.count(), _write_position);
+ }
+ }
+ }
+ // we should have exhausted all stub ids in the blob
+ assert(stub_id == StubId::NO_STUBID, "sanity");
+ // write NO_STUBID as an end marker
+ n = write_bytes(&stub_id, sizeof(StubId));
+ if (n != sizeof(StubId)) {
+ return false;
+ }
+
+ if (log.is_enabled()) {
+ log.print_cr("======== Stub data ends at offset %d", _write_position);
+ }
+
+ return true;
+}
+
+CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, uint id, const char* name, AOTStubData* stub_data) {
AOTCodeCache* cache = open_for_use();
if (cache == nullptr) {
return nullptr;
}
assert(AOTCodeEntry::is_valid_entry_kind(entry_kind), "invalid entry_kind %d", entry_kind);
+ assert(AOTCodeEntry::is_multi_stub_blob(entry_kind) == (stub_data != nullptr),
+ "entry_kind %d does not match stub_data pointer %p",
+ entry_kind, stub_data);
+
if (AOTCodeEntry::is_adapter(entry_kind) && !is_using_adapter()) {
return nullptr;
}
if (AOTCodeEntry::is_blob(entry_kind) && !is_using_stub()) {
return nullptr;
}
+ // we do not currently load C2 stubs because we are seeing weird
+ // memory errors when loading them -- see JDK-8357593
+ if (entry_kind == AOTCodeEntry::C2Blob) {
+ return nullptr;
+ }
log_debug(aot, codecache, stubs)("Reading blob '%s' (id=%u, kind=%s) from AOT Code Cache", name, id, aot_code_entry_kind_name[entry_kind]);
AOTCodeEntry* entry = cache->find_entry(entry_kind, encode_id(entry_kind, id));
@@ -962,20 +1274,32 @@ CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, uint id, c
return nullptr;
}
AOTCodeReader reader(cache, entry);
- CodeBlob* blob = reader.compile_code_blob(name);
+ CodeBlob* blob = reader.compile_code_blob(name, entry_kind, id, stub_data);
log_debug(aot, codecache, stubs)("%sRead blob '%s' (id=%u, kind=%s) from AOT Code Cache",
(blob == nullptr? "Failed to " : ""), name, id, aot_code_entry_kind_name[entry_kind]);
return blob;
}
-CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, BlobId id) {
- assert(AOTCodeEntry::is_blob(entry_kind),
- "wrong entry kind for blob id %s", StubInfo::name(id));
- return load_code_blob(entry_kind, (uint)id, StubInfo::name(id));
+CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, uint id, const char* name) {
+ assert(!AOTCodeEntry::is_blob(entry_kind),
+ "wrong entry kind for numeric id %d", id);
+ return load_code_blob(entry_kind, (uint)id, name, nullptr);
}
-CodeBlob* AOTCodeReader::compile_code_blob(const char* name) {
+CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, BlobId id) {
+ assert(AOTCodeEntry::is_single_stub_blob(entry_kind),
+ "wrong entry kind for blob id %s", StubInfo::name(id));
+ return load_code_blob(entry_kind, (uint)id, StubInfo::name(id), nullptr);
+}
+
+CodeBlob* AOTCodeCache::load_code_blob(AOTCodeEntry::Kind entry_kind, BlobId id, AOTStubData* stub_data) {
+ assert(AOTCodeEntry::is_multi_stub_blob(entry_kind),
+ "wrong entry kind for blob id %s", StubInfo::name(id));
+ return load_code_blob(entry_kind, (uint)id, StubInfo::name(id), stub_data);
+}
+
+CodeBlob* AOTCodeReader::compile_code_blob(const char* name, AOTCodeEntry::Kind entry_kind, int id, AOTStubData* stub_data) {
uint entry_position = _entry->offset();
// Read name
@@ -989,39 +1313,40 @@ CodeBlob* AOTCodeReader::compile_code_blob(const char* name) {
set_lookup_failed(); // Skip this blob
return nullptr;
}
+ _name = stored_name;
- // Read archived code blob
+ // Read archived code blob and related info
uint offset = entry_position + _entry->blob_offset();
CodeBlob* archived_blob = (CodeBlob*)addr(offset);
offset += archived_blob->size();
- address reloc_data = (address)addr(offset);
- offset += archived_blob->relocation_size();
+ _reloc_count = *(int*)addr(offset); offset += sizeof(int);
+ if (AOTCodeEntry::is_multi_stub_blob(entry_kind)) {
+ // position of relocs will have been aligned to heap word size so
+ // we can install them into a code buffer
+ offset = align_up(offset, DATA_ALIGNMENT);
+ }
+ _reloc_data = (address)addr(offset);
+ offset += _reloc_count * sizeof(relocInfo);
set_read_position(offset);
- ImmutableOopMapSet* oop_maps = nullptr;
if (_entry->has_oop_maps()) {
- oop_maps = read_oop_map_set();
+ _oop_maps = read_oop_map_set();
}
- CodeBlob* code_blob = CodeBlob::create(archived_blob,
- stored_name,
- reloc_data,
- oop_maps
- );
+ // record current context for use by that callback
+ _stub_data = stub_data;
+ _entry_kind = entry_kind;
+ _id = id;
+
+ // CodeBlob::restore() calls AOTCodeReader::restore()
+
+ CodeBlob* code_blob = CodeBlob::create(archived_blob, this);
+
if (code_blob == nullptr) { // no space left in CodeCache
return nullptr;
}
-#ifndef PRODUCT
- code_blob->asm_remarks().init();
- read_asm_remarks(code_blob->asm_remarks());
- code_blob->dbg_strings().init();
- read_dbg_strings(code_blob->dbg_strings());
-#endif // PRODUCT
-
- fix_relocations(code_blob);
-
#ifdef ASSERT
LogStreamHandle(Trace, aot, codecache, stubs) log;
if (log.is_enabled()) {
@@ -1032,15 +1357,221 @@ CodeBlob* AOTCodeReader::compile_code_blob(const char* name) {
return code_blob;
}
+void AOTCodeReader::restore(CodeBlob* code_blob) {
+ precond(AOTCodeCache::is_on_for_use());
+ precond(_name != nullptr);
+ precond(_reloc_data != nullptr);
+
+ code_blob->set_name(_name);
+ // Saved relocations need restoring except for the case of a
+ // multi-stub blob which has no runtime relocations. However, we may
+ // still have saved some (re-)load time relocs that were attached to
+ // the generator's code buffer. We don't attach them to the blob but
+ // they get processed below by fix_relocations.
+ if (!AOTCodeEntry::is_multi_stub_blob(_entry_kind)) {
+ code_blob->restore_mutable_data(_reloc_data);
+ }
+ code_blob->set_oop_maps(_oop_maps);
+
+ // if this is a multi stub blob load its entries
+ if (AOTCodeEntry::is_blob(_entry_kind)) {
+ BlobId blob_id = static_cast(_id);
+ if (StubInfo::is_stubgen(blob_id)) {
+ assert(_stub_data != nullptr, "sanity");
+ read_stub_data(code_blob, _stub_data);
+ }
+ // publish entries found either in stub_data or as offsets in blob
+ AOTCodeCache::publish_stub_addresses(*code_blob, blob_id, _stub_data);
+ }
+
+ // Now that all the entry points are in the address table we can
+ // read all the extra reloc info and fix up any addresses that need
+ // patching to adjust for a new location in a new JVM. We can be
+ // sure to correctly update all runtime references, including
+ // cross-linked stubs that are internally daisy-chained. If
+ // relocation fails and we have to re-generate any of the stubs then
+ // the entry points for newly generated stubs will get updated,
+ // ensuring that any other stubs or nmethods we need to relocate
+ // will use the correct address.
+
+ // if we have a relocatable code blob then the relocs are already
+ // attached to the blob and we can iterate over it to find the ones
+ // we need to patch. With a non-relocatable code blob we need to
+ // wrap it with a CodeBuffer and then reattach the relocs to the
+ // code buffer.
+
+ if (AOTCodeEntry::is_multi_stub_blob(_entry_kind)) {
+ // the blob doesn't have any proper runtime relocs but we can
+ // reinstate the AOT-load time relocs we saved from the code
+ // buffer that generated this blob in a new code buffer and use
+ // the latter to iterate over them
+ if (_reloc_count > 0) {
+ CodeBuffer code_buffer(code_blob);
+ relocInfo* locs = (relocInfo*)_reloc_data;
+ code_buffer.insts()->initialize_shared_locs(locs, _reloc_count);
+ code_buffer.insts()->set_locs_end(locs + _reloc_count);
+ CodeSection *cs = code_buffer.code_section(CodeBuffer::SECT_INSTS);
+ RelocIterator reloc_iter(cs);
+ fix_relocations(code_blob, reloc_iter);
+ }
+ } else {
+ // the AOT-load time relocs will be in the blob's restored relocs
+ RelocIterator reloc_iter(code_blob);
+ fix_relocations(code_blob, reloc_iter);
+ }
+
+#ifndef PRODUCT
+ code_blob->asm_remarks().init();
+ read_asm_remarks(code_blob->asm_remarks());
+ code_blob->dbg_strings().init();
+ read_dbg_strings(code_blob->dbg_strings());
+#endif // PRODUCT
+}
+
+void AOTCodeReader::read_stub_data(CodeBlob* code_blob, AOTStubData* stub_data) {
+ GrowableArray& addresses = stub_data->address_array();
+ // Read the list of stub ids and associated start, end, secondary
+ // and extra addresses and install them in the stub data.
+ //
+ // Also insert all start and secondary addresses into the AOTCache
+ // address table so we correctly relocate this blob and any followng
+ // blobs/nmethods.
+ //
+ // n.b. if an error occurs and we need to regenerate any of these
+ // stubs the address table will be updated as a side-effect of
+ // regeneration.
+
+ address blob_base = code_blob->code_begin();
+ uint blob_size = (uint)(code_blob->code_end() - blob_base);
+ int offset = read_position();
+ LogStreamHandle(Trace, aot, codecache, stubs) log;
+ if (log.is_enabled()) {
+ log.print_cr("======== Stub data starts at offset %d", offset);
+ }
+ // read stub and entries until we see NO_STUBID
+ StubId stub_id = *(StubId*)addr(offset); offset += sizeof(StubId);
+ // we ought to have at least one saved stub in the blob
+ assert(stub_id != StubId::NO_STUBID, "blob %s contains no stubs!", StubInfo::name(stub_data->blob_id()));
+ while (stub_id != StubId::NO_STUBID) {
+ assert(StubInfo::blob(stub_id) == stub_data->blob_id(), "sanity");
+ int idx = StubInfo::stubgen_offset_in_blob(stub_data->blob_id(), stub_id);
+ StubAddrRange& range = stub_data->get_range(idx);
+ // we should only see a stub once
+ assert(range.start_index() < 0, "repeated entry for stub %s", StubInfo::name(stub_id));
+ int address_base = addresses.length();
+ // start is an offset from the blob base
+ uint start = *(uint*)addr(offset); offset += sizeof(uint);
+ assert(start < blob_size, "stub %s start offset %d exceeds buffer length %d", StubInfo::name(stub_id), start, blob_size);
+ address stub_start = blob_base + start;
+ addresses.append(stub_start);
+ // end is an offset from the stub start
+ uint end = *(uint*)addr(offset); offset += sizeof(uint);
+ assert(start + end <= blob_size, "stub %s end offset %d exceeds remaining buffer length %d", StubInfo::name(stub_id), end, blob_size - start);
+ addresses.append(stub_start + end);
+ // read count of secondary entries plus extras
+ int entries_count = *(int*)addr(offset); offset += sizeof(int);
+ assert(entries_count >= (StubInfo::entry_count(stub_id) - 1), "not enough entries for %s", StubInfo::name(stub_id));
+ for (int i = 0; i < entries_count; i++) {
+ // entry offset is an offset from the stub start less than or
+ // equal to end
+ uint entry = *(uint*)addr(offset); offset += sizeof(uint);
+ if (entry <= end) {
+ // entry addresses may not address end but extras can
+ assert(entry < end || i >= StubInfo::entry_count(stub_id),
+ "entry offset 0x%x exceeds stub length 0x%x for stub %s",
+ entry, end, StubInfo::name(stub_id));
+ addresses.append(stub_start + entry);
+ } else {
+ // special case: entry encodes a nullptr
+ assert(entry == AOTCodeCache::NULL_ADDRESS_MARKER, "stub %s entry offset %d lies beyond stub end %d and does not equal NULL_ADDRESS_MARKER", StubInfo::name(stub_id), entry, end);
+ addresses.append(nullptr);
+ }
+ }
+ if (log.is_enabled()) {
+ log.print_cr("======== read stub %s and %d addresses up to offset %d",
+ StubInfo::name(stub_id), 2 + entries_count, offset);
+ }
+ range.init_entry(address_base, 2 + entries_count);
+ // move on to next stub or NO_STUBID
+ stub_id = *(StubId*)addr(offset); offset += sizeof(StubId);
+ }
+ if (log.is_enabled()) {
+ log.print_cr("======== Stub data ends at offset %d", offset);
+ }
+
+ set_read_position(offset);
+}
+
+void AOTCodeCache::publish_external_addresses(GrowableArray& addresses) {
+ DEBUG_ONLY( _passed_init2 = true; )
+ if (opened_cache == nullptr) {
+ return;
+ }
+
+ cache()->_table->add_external_addresses(addresses);
+}
+
+void AOTCodeCache::publish_stub_addresses(CodeBlob &code_blob, BlobId blob_id, AOTStubData *stub_data) {
+ if (stub_data != nullptr) {
+ // register all entries in stub
+ assert(StubInfo::stub_count(blob_id) > 1,
+ "multiple stub data provided for single stub blob %s",
+ StubInfo::name(blob_id));
+ assert(blob_id == stub_data->blob_id(),
+ "blob id %s does not match id in stub data %s",
+ StubInfo::name(blob_id),
+ StubInfo::name(stub_data->blob_id()));
+ // iterate over all stubs in the blob
+ StubId stub_id = StubInfo::stub_base(blob_id);
+ int stub_cnt = StubInfo::stub_count(blob_id);
+ GrowableArray& addresses = stub_data->address_array();
+ for (int i = 0; i < stub_cnt; i++) {
+ assert(stub_id != StubId::NO_STUBID, "sanity");
+ StubAddrRange& range = stub_data->get_range(i);
+ int base = range.start_index();
+ if (base >= 0) {
+ cache()->add_stub_entries(stub_id, addresses.at(base), &addresses, base + 2);
+ }
+ stub_id = StubInfo::next_in_blob(blob_id, stub_id);
+ }
+ // we should have exhausted all stub ids in the blob
+ assert(stub_id == StubId::NO_STUBID, "sanity");
+ } else {
+ // register entry or entries for a single stub blob
+ StubId stub_id = StubInfo::stub_base(blob_id);
+ assert(StubInfo::stub_count(blob_id) == 1,
+ "multiple stub blob %s provided without stub data",
+ StubInfo::name(blob_id));
+ address start = code_blob.code_begin();
+ if (StubInfo::entry_count(stub_id) == 1) {
+ assert(!code_blob.is_deoptimization_stub(), "expecting multiple entries for stub %s", StubInfo::name(stub_id));
+ // register the blob base address as the only entry
+ cache()->add_stub_entries(stub_id, start);
+ } else {
+ assert(code_blob.is_deoptimization_stub(), "only expecting one entry for stub %s", StubInfo::name(stub_id));
+ DeoptimizationBlob *deopt_blob = code_blob.as_deoptimization_blob();
+ assert(deopt_blob->unpack() == start, "unexpected offset 0x%x for deopt stub entry", (int)(deopt_blob->unpack() - start));
+ GrowableArray addresses;
+ addresses.append(deopt_blob->unpack_with_exception());
+ addresses.append(deopt_blob->unpack_with_reexecution());
+ addresses.append(deopt_blob->unpack_with_exception_in_tls());
+#if INCLUDE_JVMCI
+ addresses.append(deopt_blob->uncommon_trap());
+ addresses.append(deopt_blob->implicit_exception_uncommon_trap());
+#endif // INCLUDE_JVMCI
+ cache()->add_stub_entries(stub_id, start, &addresses, 0);
+ }
+ }
+}
+
// ------------ process code and data --------------
// Can't use -1. It is valid value for jump to iteself destination
// used by static call stub: see NativeJump::jump_destination().
#define BAD_ADDRESS_ID -2
-bool AOTCodeCache::write_relocations(CodeBlob& code_blob) {
+bool AOTCodeCache::write_relocations(CodeBlob& code_blob, RelocIterator& iter) {
GrowableArray