diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 0d8663fab1a..4d1e8a8be3d 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -327,8 +327,8 @@ jobs:
uses: ./.github/workflows/build-macos.yml
with:
platform: macos-x64
- runs-on: 'macos-13'
- xcode-toolset-version: '14.3.1'
+ runs-on: 'macos-15-intel'
+ xcode-toolset-version: '16.4'
configure-arguments: ${{ github.event.inputs.configure-arguments }}
make-arguments: ${{ github.event.inputs.make-arguments }}
dry-run: ${{ needs.prepare.outputs.dry-run == 'true' }}
@@ -340,8 +340,8 @@ jobs:
uses: ./.github/workflows/build-macos.yml
with:
platform: macos-aarch64
- runs-on: 'macos-14'
- xcode-toolset-version: '15.4'
+ runs-on: 'macos-15'
+ xcode-toolset-version: '16.4'
configure-arguments: ${{ github.event.inputs.configure-arguments }}
make-arguments: ${{ github.event.inputs.make-arguments }}
dry-run: ${{ needs.prepare.outputs.dry-run == 'true' }}
@@ -432,9 +432,9 @@ jobs:
with:
platform: macos-aarch64
bootjdk-platform: macos-aarch64
- runs-on: macos-14
+ runs-on: macos-15
dry-run: ${{ needs.prepare.outputs.dry-run == 'true' }}
- xcode-toolset-version: '15.4'
+ xcode-toolset-version: '16.4'
debug-suffix: -debug
test-windows-x64:
diff --git a/doc/hotspot-style.html b/doc/hotspot-style.html
index fb4cffc9d43..f1c25dab7f4 100644
--- a/doc/hotspot-style.html
+++ b/doc/hotspot-style.html
@@ -86,8 +86,9 @@ values
-Trailing return type syntax for functions (n2541)
-Variable templates (n3651, p0127r2)
Member initializers and aggregates (n3653)
Rvalue references and move semantics
diff --git a/doc/hotspot-style.md b/doc/hotspot-style.md
index 3fd5468d531..e49f49ec1c9 100644
--- a/doc/hotspot-style.md
+++ b/doc/hotspot-style.md
@@ -856,14 +856,19 @@ ordering, which may differ from (may be stronger than) sequentially
consistent. There are algorithms in HotSpot that are believed to rely
on that ordering.
-### Inline Variables
+### Variable Templates and Inline Variables
-Variables with static storage duration may be declared `inline`
-([p0386r2](https://wg21.link/p0386r2)). This has similar effects as for
-declaring a function inline: it can be defined, identically, in multiple
-translation units, must be defined in every translation unit in which it is
-[ODR used][ODR], and the behavior of the program is as if there is exactly one
-variable.
+The use of variable templates (including static data member templates)
+([N3651](https://wg21.link/N3651)) is permitted. They provide parameterized
+variables and constants in a simple and direct form, instead of requiring the
+use of various workarounds.
+
+Variables with static storage duration and variable templates may be declared
+`inline` ([p0386r2](https://wg21.link/p0386r2)), and this usage is
+permitted. This has similar effects as for declaring a function inline: it can
+be defined, identically, in multiple translation units, must be defined in
+every translation unit in which it is [ODR used][ODR], and the behavior of the
+program is as if there is exactly one variable.
Declaring a variable inline allows the complete definition to be in a header
file, rather than having a declaration in a header and the definition in a
@@ -874,13 +879,15 @@ make initialization order problems worse. The few ordering constraints
that exist for non-inline variables don't apply, as there isn't a single
program-designated translation unit containing the definition.
-A `constexpr` static data member is implicitly `inline`. As a consequence, an
-[ODR use][ODR] of such a variable doesn't require a definition in some .cpp
+A `constexpr` static data member or static data member template
+is implicitly `inline`. As a consequence, an
+[ODR use][ODR] of such a member doesn't require a definition in some .cpp
file. (This is a change from pre-C++17. Beginning with C++17, such a
definition is considered a duplicate definition, and is deprecated.)
-Declaring a `thread_local` variable `inline` is forbidden for HotSpot code.
-[The use of `thread_local`](#thread_local) is already heavily restricted.
+Declaring a `thread_local` variable template or `inline` variable is forbidden
+in HotSpot code. [The use of `thread_local`](#thread_local) is already
+heavily restricted.
### Initializing variables with static storage duration
@@ -1846,13 +1853,6 @@ See Object Lifetime: C++17 6.8/8, C++20 6.7.3/8
### Additional Undecided Features
-* Trailing return type syntax for functions
-([n2541](http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2008/n2541.htm))
-
-* Variable templates
-([n3651](https://isocpp.org/files/papers/N3651.pdf),
-[p0127r2](http://wg21.link/p0127r2))
-
* Member initializers and aggregates
([n3653](http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2013/n3653.html))
diff --git a/doc/testing.html b/doc/testing.html
index 89a9b1b23b7..b9838735e4f 100644
--- a/doc/testing.html
+++ b/doc/testing.html
@@ -450,7 +450,7 @@ itself (-timeoutFactor). Also, some test cases that
programmatically wait a certain amount of time will apply this factor.
If we run in forced compilation mode (-Xcomp), the build
system will automatically adjust this factor to compensate for less
-performance. Defaults to 1.
+performance. Defaults to 4.
FAILURE_HANDLER_TIMEOUT
Sets the argument -timeoutHandlerTimeout for JTReg. The
default value is 0. This is only valid if the failure handler is
diff --git a/doc/testing.md b/doc/testing.md
index 324f9645c27..0144610a5bf 100644
--- a/doc/testing.md
+++ b/doc/testing.md
@@ -387,7 +387,7 @@ The `TIMEOUT_FACTOR` is forwarded to JTReg framework itself
(`-timeoutFactor`). Also, some test cases that programmatically wait a
certain amount of time will apply this factor. If we run in forced
compilation mode (`-Xcomp`), the build system will automatically
-adjust this factor to compensate for less performance. Defaults to 1.
+adjust this factor to compensate for less performance. Defaults to 4.
#### FAILURE_HANDLER_TIMEOUT
diff --git a/make/Hsdis.gmk b/make/Hsdis.gmk
index a0fc031be1b..469cc488f16 100644
--- a/make/Hsdis.gmk
+++ b/make/Hsdis.gmk
@@ -114,7 +114,6 @@ ifeq ($(HSDIS_BACKEND), binutils)
TOOLCHAIN_TYPE := gcc
OPENJDK_TARGET_OS := linux
OPENJDK_TARGET_OS_TYPE := unix
- CC_OUT_OPTION := -o$(SPACE)
GENDEPS_FLAGS := -MMD -MF
CFLAGS_DEBUG_SYMBOLS := -g
DISABLED_WARNINGS :=
diff --git a/make/RunTests.gmk b/make/RunTests.gmk
index 7b05a0ba12f..947389f64f9 100644
--- a/make/RunTests.gmk
+++ b/make/RunTests.gmk
@@ -946,8 +946,8 @@ define SetupRunJtregTestBody
JTREG_ALL_OPTIONS := $$(JTREG_JAVA_OPTIONS) $$(JTREG_VM_OPTIONS)
JTREG_AUTO_PROBLEM_LISTS :=
- # Please reach consensus before changing this. It was not easy changing it to a `1`.
- JTREG_AUTO_TIMEOUT_FACTOR := 1
+ # Please reach consensus before changing this.
+ JTREG_AUTO_TIMEOUT_FACTOR := 4
ifneq ($$(findstring -Xcomp, $$(JTREG_ALL_OPTIONS)), )
JTREG_AUTO_PROBLEM_LISTS += ProblemList-Xcomp.txt
diff --git a/make/StaticLibs.gmk b/make/StaticLibs.gmk
index 80d6b4538c8..490180eb649 100644
--- a/make/StaticLibs.gmk
+++ b/make/StaticLibs.gmk
@@ -116,6 +116,9 @@ else ifeq ($(call isTargetOs, aix), true)
$(eval STATIC_LIB_EXPORT_FILES += $(lib).exp) \
)
STATIC_LIBS := -Wl,-bexpfull $(STATIC_LIB_FILES) $(addprefix -Wl$(COMMA)-bE:, $(STATIC_LIB_EXPORT_FILES))
+ ifeq ($(DEBUG_LEVEL), slowdebug)
+ STATIC_LIBS += -Wl,-bbigtoc
+ endif
else
$(error Unsupported platform)
endif
diff --git a/make/autoconf/basic_tools.m4 b/make/autoconf/basic_tools.m4
index 5367db46679..8e42f9205a4 100644
--- a/make/autoconf/basic_tools.m4
+++ b/make/autoconf/basic_tools.m4
@@ -363,7 +363,7 @@ AC_DEFUN_ONCE([BASIC_SETUP_COMPLEX_TOOLS],
# Check if it's a GNU date compatible version
AC_MSG_CHECKING([if date is a GNU compatible version])
- check_date=`$DATE --version 2>&1 | $GREP "GNU\|BusyBox"`
+ check_date=`$DATE --version 2>&1 | $GREP "GNU\|BusyBox\|uutils"`
if test "x$check_date" != x; then
AC_MSG_RESULT([yes])
IS_GNU_DATE=yes
diff --git a/make/autoconf/boot-jdk.m4 b/make/autoconf/boot-jdk.m4
index 4ba1f9f2089..b3dbc292919 100644
--- a/make/autoconf/boot-jdk.m4
+++ b/make/autoconf/boot-jdk.m4
@@ -408,27 +408,6 @@ AC_DEFUN_ONCE([BOOTJDK_SETUP_BOOT_JDK],
AC_MSG_CHECKING([if Boot JDK is 32 or 64 bits])
AC_MSG_RESULT([$BOOT_JDK_BITS])
- # Try to enable CDS
- AC_MSG_CHECKING([for local Boot JDK Class Data Sharing (CDS)])
- BOOT_JDK_CDS_ARCHIVE=$CONFIGURESUPPORT_OUTPUTDIR/classes.jsa
- UTIL_ADD_JVM_ARG_IF_OK([-XX:+UnlockDiagnosticVMOptions -XX:-VerifySharedSpaces -XX:SharedArchiveFile=$BOOT_JDK_CDS_ARCHIVE],boot_jdk_cds_args,[$JAVA])
-
- if test "x$boot_jdk_cds_args" != x; then
- # Try creating a CDS archive
- $JAVA $boot_jdk_cds_args -Xshare:dump > /dev/null 2>&1
- if test $? -eq 0; then
- BOOTJDK_USE_LOCAL_CDS=true
- AC_MSG_RESULT([yes, created])
- else
- # Generation failed, don't use CDS.
- BOOTJDK_USE_LOCAL_CDS=false
- AC_MSG_RESULT([no, creation failed])
- fi
- else
- BOOTJDK_USE_LOCAL_CDS=false
- AC_MSG_RESULT([no, -XX:SharedArchiveFile not supported])
- fi
-
BOOTJDK_SETUP_CLASSPATH
])
@@ -444,13 +423,8 @@ AC_DEFUN_ONCE([BOOTJDK_SETUP_BOOT_JDK_ARGUMENTS],
# Force en-US environment
UTIL_ADD_JVM_ARG_IF_OK([-Duser.language=en -Duser.country=US],boot_jdk_jvmargs,[$JAVA])
- if test "x$BOOTJDK_USE_LOCAL_CDS" = xtrue; then
- # Use our own CDS archive
- UTIL_ADD_JVM_ARG_IF_OK([$boot_jdk_cds_args -Xshare:auto],boot_jdk_jvmargs,[$JAVA])
- else
- # Otherwise optimistically use the system-wide one, if one is present
- UTIL_ADD_JVM_ARG_IF_OK([-Xshare:auto],boot_jdk_jvmargs,[$JAVA])
- fi
+ UTIL_ADD_JVM_ARG_IF_OK([-Xlog:all=off:stdout],boot_jdk_jvmargs,[$JAVA])
+ UTIL_ADD_JVM_ARG_IF_OK([-Xlog:all=warning:stderr],boot_jdk_jvmargs,[$JAVA])
# Finally append user provided options to allow them to override.
UTIL_ADD_JVM_ARG_IF_OK([$USER_BOOT_JDK_OPTIONS],boot_jdk_jvmargs,[$JAVA])
@@ -597,10 +571,9 @@ AC_DEFUN([BOOTJDK_SETUP_BUILD_JDK],
AC_ARG_WITH(build-jdk, [AS_HELP_STRING([--with-build-jdk],
[path to JDK of same version as is being built@<:@the newly built JDK@:>@])])
- CREATE_BUILDJDK=false
- EXTERNAL_BUILDJDK=false
- BUILD_JDK_FOUND="no"
+ EXTERNAL_BUILDJDK_PATH=""
if test "x$with_build_jdk" != "x"; then
+ BUILD_JDK_FOUND=no
BOOTJDK_CHECK_BUILD_JDK([
if test "x$with_build_jdk" != x; then
BUILD_JDK=$with_build_jdk
@@ -608,40 +581,15 @@ AC_DEFUN([BOOTJDK_SETUP_BUILD_JDK],
AC_MSG_NOTICE([Found potential Build JDK using configure arguments])
fi
])
- EXTERNAL_BUILDJDK=true
- else
- if test "x$COMPILE_TYPE" = "xcross"; then
- BUILD_JDK="\$(BUILDJDK_OUTPUTDIR)/jdk"
- BUILD_JDK_FOUND=yes
- CREATE_BUILDJDK=true
+ if test "x$BUILD_JDK_FOUND" != "xyes"; then
AC_MSG_CHECKING([for Build JDK])
- AC_MSG_RESULT([yes, will build it for the host platform])
- else
- BUILD_JDK="\$(JDK_OUTPUTDIR)"
- BUILD_JDK_FOUND=yes
- AC_MSG_CHECKING([for Build JDK])
- AC_MSG_RESULT([yes, will use output dir])
+ AC_MSG_RESULT([no])
+ AC_MSG_ERROR([Could not find a suitable Build JDK])
fi
+ EXTERNAL_BUILDJDK_PATH="$BUILD_JDK"
fi
- # Since these tools do not yet exist, we cannot use UTIL_FIXUP_EXECUTABLE to
- # detect the need of fixpath
- JMOD="$BUILD_JDK/bin/jmod"
- UTIL_ADD_FIXPATH(JMOD)
- JLINK="$BUILD_JDK/bin/jlink"
- UTIL_ADD_FIXPATH(JLINK)
- AC_SUBST(JMOD)
- AC_SUBST(JLINK)
-
- if test "x$BUILD_JDK_FOUND" != "xyes"; then
- AC_MSG_CHECKING([for Build JDK])
- AC_MSG_RESULT([no])
- AC_MSG_ERROR([Could not find a suitable Build JDK])
- fi
-
- AC_SUBST(CREATE_BUILDJDK)
- AC_SUBST(BUILD_JDK)
- AC_SUBST(EXTERNAL_BUILDJDK)
+ AC_SUBST(EXTERNAL_BUILDJDK_PATH)
])
# The docs-reference JDK is used to run javadoc for the docs-reference targets.
diff --git a/make/autoconf/bootcycle-spec.gmk.template b/make/autoconf/bootcycle-spec.gmk.template
index 8b6035606a5..d17a95e190a 100644
--- a/make/autoconf/bootcycle-spec.gmk.template
+++ b/make/autoconf/bootcycle-spec.gmk.template
@@ -44,7 +44,3 @@ JAVAC_CMD := $(FIXPATH) $(BOOT_JDK)/bin/javac
JAR_CMD := $(FIXPATH) $(BOOT_JDK)/bin/jar
# The bootcycle JVM arguments may differ from the original boot jdk.
JAVA_FLAGS_BIG := @BOOTCYCLE_JVM_ARGS_BIG@
-# Any CDS settings generated for the bootjdk are invalid in the bootcycle build.
-# By filtering out those JVM args, the bootcycle JVM will use its default
-# settings for CDS.
-JAVA_FLAGS := $(filter-out -XX:SharedArchiveFile% -Xshare%, $(JAVA_FLAGS))
diff --git a/make/autoconf/flags-cflags.m4 b/make/autoconf/flags-cflags.m4
index 6072cbc74dd..9d58a280998 100644
--- a/make/autoconf/flags-cflags.m4
+++ b/make/autoconf/flags-cflags.m4
@@ -37,56 +37,25 @@ AC_DEFUN([FLAGS_SETUP_SHARED_LIBS],
if test "x$TOOLCHAIN_TYPE" = xgcc; then
# Default works for linux, might work on other platforms as well.
SHARED_LIBRARY_FLAGS='-shared'
- # --disable-new-dtags forces use of RPATH instead of RUNPATH for rpaths.
- # This protects internal library dependencies within the JDK from being
- # overridden using LD_LIBRARY_PATH. See JDK-8326891 for more information.
- SET_EXECUTABLE_ORIGIN='-Wl,-rpath,\$$ORIGIN[$]1 -Wl,--disable-new-dtags'
- SET_SHARED_LIBRARY_ORIGIN="-Wl,-z,origin $SET_EXECUTABLE_ORIGIN"
- SET_SHARED_LIBRARY_NAME='-Wl,-soname=[$]1'
elif test "x$TOOLCHAIN_TYPE" = xclang; then
if test "x$OPENJDK_TARGET_OS" = xmacosx; then
# Linking is different on MacOSX
SHARED_LIBRARY_FLAGS="-dynamiclib -compatibility_version 1.0.0 -current_version 1.0.0"
- SET_EXECUTABLE_ORIGIN='-Wl,-rpath,@loader_path$(or [$]1,/.)'
- SET_SHARED_LIBRARY_ORIGIN="$SET_EXECUTABLE_ORIGIN"
- SET_SHARED_LIBRARY_NAME='-Wl,-install_name,@rpath/[$]1'
elif test "x$OPENJDK_TARGET_OS" = xaix; then
# Linking is different on aix
SHARED_LIBRARY_FLAGS="-shared -Wl,-bM:SRE -Wl,-bnoentry"
- SET_EXECUTABLE_ORIGIN=""
- SET_SHARED_LIBRARY_ORIGIN=''
- SET_SHARED_LIBRARY_NAME=''
else
# Default works for linux, might work on other platforms as well.
SHARED_LIBRARY_FLAGS='-shared'
- SET_EXECUTABLE_ORIGIN='-Wl,-rpath,\$$ORIGIN[$]1'
- if test "x$OPENJDK_TARGET_OS" = xlinux; then
- SET_EXECUTABLE_ORIGIN="$SET_EXECUTABLE_ORIGIN -Wl,--disable-new-dtags"
- fi
- SET_SHARED_LIBRARY_NAME='-Wl,-soname=[$]1'
-
- # arm specific settings
- if test "x$OPENJDK_TARGET_CPU" = "xarm"; then
- # '-Wl,-z,origin' isn't used on arm.
- SET_SHARED_LIBRARY_ORIGIN='-Wl,-rpath,\$$$$ORIGIN[$]1'
- else
- SET_SHARED_LIBRARY_ORIGIN="-Wl,-z,origin $SET_EXECUTABLE_ORIGIN"
- fi
fi
elif test "x$TOOLCHAIN_TYPE" = xmicrosoft; then
SHARED_LIBRARY_FLAGS="-dll"
- SET_EXECUTABLE_ORIGIN=''
- SET_SHARED_LIBRARY_ORIGIN=''
- SET_SHARED_LIBRARY_NAME=''
fi
- AC_SUBST(SET_EXECUTABLE_ORIGIN)
- AC_SUBST(SET_SHARED_LIBRARY_ORIGIN)
- AC_SUBST(SET_SHARED_LIBRARY_NAME)
AC_SUBST(SHARED_LIBRARY_FLAGS)
])
@@ -934,48 +903,6 @@ AC_DEFUN([FLAGS_SETUP_CFLAGS_CPU_DEP],
IF_FALSE: [$2FDLIBM_CFLAGS=""])
fi
AC_SUBST($2FDLIBM_CFLAGS)
-
- # Check whether the compiler supports the Arm C Language Extensions (ACLE)
- # for SVE. Set SVE_CFLAGS to -march=armv8-a+sve if it does.
- # ACLE and this flag are required to build the aarch64 SVE related functions in
- # libvectormath. Apple Silicon does not support SVE; use macOS as a proxy for
- # that check.
- if test "x$OPENJDK_TARGET_CPU" = "xaarch64" && test "x$OPENJDK_TARGET_OS" = "xlinux"; then
- if test "x$TOOLCHAIN_TYPE" = xgcc || test "x$TOOLCHAIN_TYPE" = xclang; then
- AC_LANG_PUSH(C)
- OLD_CFLAGS="$CFLAGS"
- CFLAGS="$CFLAGS -march=armv8-a+sve"
- AC_MSG_CHECKING([if Arm SVE ACLE is supported])
- AC_COMPILE_IFELSE([AC_LANG_PROGRAM([#include ],
- [
- svint32_t r = svdup_n_s32(1);
- return 0;
- ])],
- [
- AC_MSG_RESULT([yes])
- $2SVE_CFLAGS="-march=armv8-a+sve"
- # Switching the initialization mode with gcc from 'pattern' to 'zero'
- # avoids the use of unsupported `__builtin_clear_padding` for variable
- # length aggregates
- if test "x$DEBUG_LEVEL" != xrelease && test "x$TOOLCHAIN_TYPE" = xgcc ; then
- INIT_ZERO_FLAG="-ftrivial-auto-var-init=zero"
- FLAGS_COMPILER_CHECK_ARGUMENTS(ARGUMENT: [$INIT_ZERO_FLAG],
- IF_TRUE: [
- $2SVE_CFLAGS="${$2SVE_CFLAGS} $INIT_ZERO_FLAG"
- ]
- )
- fi
- ],
- [
- AC_MSG_RESULT([no])
- $2SVE_CFLAGS=""
- ]
- )
- CFLAGS="$OLD_CFLAGS"
- AC_LANG_POP(C)
- fi
- fi
- AC_SUBST($2SVE_CFLAGS)
])
AC_DEFUN_ONCE([FLAGS_SETUP_BRANCH_PROTECTION],
diff --git a/make/autoconf/flags-ldflags.m4 b/make/autoconf/flags-ldflags.m4
index a12a6e7f9a6..66f8904db89 100644
--- a/make/autoconf/flags-ldflags.m4
+++ b/make/autoconf/flags-ldflags.m4
@@ -79,7 +79,7 @@ AC_DEFUN([FLAGS_SETUP_LDFLAGS_HELPER],
fi
if test "x$OPENJDK_TARGET_OS" = xaix; then
BASIC_LDFLAGS="-Wl,-b64 -Wl,-brtl -Wl,-bnorwexec -Wl,-blibpath:/usr/lib:lib -Wl,-bnoexpall \
- -Wl,-bernotok -Wl,-bdatapsize:64k -Wl,-btextpsize:64k -Wl,-bstackpsize:64k"
+ -Wl,-bernotok -Wl,-bcdtors:mbr::s -Wl,-bdatapsize:64k -Wl,-btextpsize:64k -Wl,-bstackpsize:64k"
BASIC_LDFLAGS_JVM_ONLY="$BASIC_LDFLAGS_JVM_ONLY -Wl,-lC_r -Wl,-bbigtoc"
fi
@@ -98,7 +98,7 @@ AC_DEFUN([FLAGS_SETUP_LDFLAGS_HELPER],
# Setup OS-dependent LDFLAGS
if test "x$OPENJDK_TARGET_OS" = xmacosx && test "x$TOOLCHAIN_TYPE" = xclang; then
- # FIXME: We should really generalize SET_SHARED_LIBRARY_ORIGIN instead.
+ # FIXME: We should really generalize SetSharedLibraryOrigin instead.
OS_LDFLAGS_JVM_ONLY="-Wl,-rpath,@loader_path/. -Wl,-rpath,@loader_path/.."
OS_LDFLAGS="-mmacosx-version-min=$MACOSX_VERSION_MIN -Wl,-reproducible"
fi
diff --git a/make/autoconf/flags-other.m4 b/make/autoconf/flags-other.m4
index 9d41cf04791..4570f6ede78 100644
--- a/make/autoconf/flags-other.m4
+++ b/make/autoconf/flags-other.m4
@@ -107,6 +107,62 @@ AC_DEFUN([FLAGS_SETUP_NMFLAGS],
AC_SUBST(NMFLAGS)
])
+# Check whether the compiler supports the Arm C Language Extensions (ACLE)
+# for SVE. Set SVE_CFLAGS to -march=armv8-a+sve if it does.
+# ACLE and this flag are required to build the aarch64 SVE related functions
+# in libvectormath.
+AC_DEFUN([FLAGS_SETUP_SVE],
+[
+ AARCH64_SVE_AVAILABLE=false
+ # Apple Silicon does not support SVE; use macOS as a proxy for that check.
+ if test "x$OPENJDK_TARGET_CPU" = "xaarch64" && test "x$OPENJDK_TARGET_OS" = "xlinux"; then
+ if test "x$TOOLCHAIN_TYPE" = xgcc || test "x$TOOLCHAIN_TYPE" = xclang; then
+ # check the compiler and binutils support sve or not
+ AC_MSG_CHECKING([if Arm SVE ACLE is supported])
+ AC_LANG_PUSH([C])
+ saved_cflags="$CFLAGS"
+ CFLAGS="$CFLAGS -march=armv8-a+sve $CFLAGS_WARNINGS_ARE_ERRORS ARG_ARGUMENT"
+ AC_COMPILE_IFELSE([AC_LANG_PROGRAM(
+ [
+ #include
+ svfloat64_t a() {}
+ ],
+ [
+ svint32_t r = svdup_n_s32(1)
+ ])],
+ [
+ AARCH64_SVE_AVAILABLE=true
+ ]
+ )
+ CFLAGS="$saved_cflags"
+ AC_LANG_POP([C])
+ AC_MSG_RESULT([$AARCH64_SVE_AVAILABLE])
+ fi
+ fi
+
+ UTIL_ARG_ENABLE(NAME: aarch64-sve, DEFAULT: auto,
+ RESULT: AARCH64_SVE_ENABLED,
+ DESC: [Use SVE when compiling libsleef],
+ AVAILABLE: $AARCH64_SVE_AVAILABLE)
+ SVE_CFLAGS=""
+ if test "x$AARCH64_SVE_ENABLED" = xtrue; then
+ SVE_CFLAGS="-march=armv8-a+sve"
+ # Switching the initialization mode with gcc from 'pattern' to 'zero'
+ # avoids the use of unsupported `__builtin_clear_padding` for variable
+ # length aggregates
+ if test "x$DEBUG_LEVEL" != xrelease && test "x$TOOLCHAIN_TYPE" = xgcc ; then
+ AC_MSG_CHECKING([Switching the initialization mode with gcc from pattern to zero])
+ INIT_ZERO_FLAG="-ftrivial-auto-var-init=zero"
+ FLAGS_COMPILER_CHECK_ARGUMENTS(ARGUMENT: [$INIT_ZERO_FLAG],
+ IF_TRUE: [
+ SVE_CFLAGS="${SVE_CFLAGS} $INIT_ZERO_FLAG"
+ ]
+ )
+ fi
+ fi
+ AC_SUBST(SVE_CFLAGS)
+])
+
################################################################################
# platform independent
AC_DEFUN([FLAGS_SETUP_ASFLAGS],
diff --git a/make/autoconf/flags.m4 b/make/autoconf/flags.m4
index c810d15ebbc..c47947154c2 100644
--- a/make/autoconf/flags.m4
+++ b/make/autoconf/flags.m4
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -319,16 +319,10 @@ AC_DEFUN_ONCE([FLAGS_PRE_TOOLCHAIN],
AC_DEFUN([FLAGS_SETUP_TOOLCHAIN_CONTROL],
[
if test "x$TOOLCHAIN_TYPE" = xmicrosoft; then
- CC_OUT_OPTION=-Fo
if test "x$OPENJDK_TARGET_CPU" != xaarch64; then
AS_NON_ASM_EXTENSION_OPTION=-Ta
fi
- else
- # The option used to specify the target .o,.a or .so file.
- # When compiling, how to specify the to be created object file.
- CC_OUT_OPTION='-o$(SPACE)'
fi
- AC_SUBST(CC_OUT_OPTION)
AC_SUBST(AS_NON_ASM_EXTENSION_OPTION)
# Generate make dependency files
@@ -374,6 +368,7 @@ AC_DEFUN([FLAGS_SETUP_FLAGS],
FLAGS_SETUP_RCFLAGS
FLAGS_SETUP_NMFLAGS
+ FLAGS_SETUP_SVE
FLAGS_SETUP_ASFLAGS
FLAGS_SETUP_ASFLAGS_CPU_DEP([TARGET])
FLAGS_SETUP_ASFLAGS_CPU_DEP([BUILD], [OPENJDK_BUILD_])
diff --git a/make/autoconf/jdk-options.m4 b/make/autoconf/jdk-options.m4
index d4299078abf..bb188778001 100644
--- a/make/autoconf/jdk-options.m4
+++ b/make/autoconf/jdk-options.m4
@@ -565,9 +565,14 @@ AC_DEFUN_ONCE([JDKOPT_SETUP_UNDEFINED_BEHAVIOR_SANITIZER],
# with an additional define LLVM_SYMBOLIZER, which we set here.
# To calculate the correct llvm_symbolizer path we can use the location of the compiler, because
# their relation is fixed.
+ # In the ubsan case we have to link every binary with the C++-compiler as linker, because inherently
+ # the C-Compiler and the C++-compiler used as linker provide a different set of ubsan exports.
+ # Linking an executable with the C-compiler and one of its shared libraries with the C++-compiler
+ # leeds to unresolved symbols.
if test "x$TOOLCHAIN_TYPE" = "xclang" && test "x$OPENJDK_TARGET_OS" = "xaix"; then
- UBSAN_CFLAGS="$UBSAN_CFLAGS -fno-sanitize=function,vptr -DLLVM_SYMBOLIZER=$(dirname $(dirname $CC))/tools/ibm-llvm-symbolizer"
- UBSAN_LDFLAGS="$UBSAN_LDFLAGS -fno-sanitize=function,vptr -Wl,-bbigtoc"
+ UBSAN_CFLAGS="$UBSAN_CFLAGS -DLLVM_SYMBOLIZER=$(dirname $(dirname $CC))/tools/ibm-llvm-symbolizer"
+ UBSAN_LDFLAGS="$UBSAN_LDFLAGS -Wl,-bbigtoc"
+ LD="$LDCXX"
fi
UTIL_ARG_ENABLE(NAME: ubsan, DEFAULT: false, RESULT: UBSAN_ENABLED,
DESC: [enable UndefinedBehaviorSanitizer],
diff --git a/make/autoconf/lib-tests.m4 b/make/autoconf/lib-tests.m4
index 9eb5ee5a046..31d48055984 100644
--- a/make/autoconf/lib-tests.m4
+++ b/make/autoconf/lib-tests.m4
@@ -28,7 +28,7 @@
################################################################################
# Minimum supported versions
-JTREG_MINIMUM_VERSION=7.5.2
+JTREG_MINIMUM_VERSION=8.1
GTEST_MINIMUM_VERSION=1.14.0
################################################################################
diff --git a/make/autoconf/libraries.m4 b/make/autoconf/libraries.m4
index bf697928f1b..8dc3d55ed0c 100644
--- a/make/autoconf/libraries.m4
+++ b/make/autoconf/libraries.m4
@@ -136,12 +136,8 @@ AC_DEFUN_ONCE([LIB_SETUP_LIBRARIES],
BASIC_JVM_LIBS="$BASIC_JVM_LIBS $LIBPTHREAD"
fi
- # librt for legacy clock_gettime
+ # librt - for timers (timer_* functions)
if test "x$OPENJDK_TARGET_OS" = xlinux; then
- # Hotspot needs to link librt to get the clock_* functions.
- # But once our supported minimum build and runtime platform
- # has glibc 2.17, this can be removed as the functions are
- # in libc.
BASIC_JVM_LIBS="$BASIC_JVM_LIBS -lrt"
fi
diff --git a/make/autoconf/spec.gmk.template b/make/autoconf/spec.gmk.template
index ab6bb51c27e..0b336721d65 100644
--- a/make/autoconf/spec.gmk.template
+++ b/make/autoconf/spec.gmk.template
@@ -386,9 +386,22 @@ CAPSTONE_ARCH_AARCH64_NAME := @CAPSTONE_ARCH_AARCH64_NAME@
# it in sync.
BOOT_JDK := @BOOT_JDK@
-BUILD_JDK := @BUILD_JDK@
-CREATE_BUILDJDK := @CREATE_BUILDJDK@
-EXTERNAL_BUILDJDK := @EXTERNAL_BUILDJDK@
+EXTERNAL_BUILDJDK_PATH := @EXTERNAL_BUILDJDK_PATH@
+
+ifneq ($(EXTERNAL_BUILDJDK_PATH), )
+ EXTERNAL_BUILDJDK := true
+ CREATE_BUILDJDK := false
+ BUILD_JDK := $(EXTERNAL_BUILDJDK_PATH)
+else
+ EXTERNAL_BUILDJDK := false
+ ifeq ($(COMPILE_TYPE), cross)
+ CREATE_BUILDJDK := true
+ BUILD_JDK := $(BUILDJDK_OUTPUTDIR)/jdk
+ else
+ CREATE_BUILDJDK := false
+ BUILD_JDK := $(JDK_OUTPUTDIR)
+ endif
+endif
# Whether the boot jdk jar supports --date=TIMESTAMP
BOOT_JDK_JAR_SUPPORTS_DATE := @BOOT_JDK_JAR_SUPPORTS_DATE@
@@ -491,7 +504,6 @@ CXX_VERSION_NUMBER := @CXX_VERSION_NUMBER@
# Legacy support
HOTSPOT_TOOLCHAIN_TYPE := @HOTSPOT_TOOLCHAIN_TYPE@
-CC_OUT_OPTION := @CC_OUT_OPTION@
AS_NON_ASM_EXTENSION_OPTION := @AS_NON_ASM_EXTENSION_OPTION@
# Flags used for overriding the default opt setting for a C/C++ source file.
@@ -624,17 +636,8 @@ ASFLAGS_DEBUG_SYMBOLS := @ASFLAGS_DEBUG_SYMBOLS@
# Compress (or not) jars
COMPRESS_JARS := @COMPRESS_JARS@
-# Options to linker to specify the library name.
-# (Note absence of := assignment, because we do not want to evaluate the macro body here)
-SET_SHARED_LIBRARY_NAME = @SET_SHARED_LIBRARY_NAME@
-
SHARED_LIBRARY_FLAGS := @SHARED_LIBRARY_FLAGS@
-# Set origin using the linker, ie use the relative path to the dependent library to find the dependencies.
-# (Note absence of := assignment, because we do not want to evaluate the macro body here)
-SET_SHARED_LIBRARY_ORIGIN = @SET_SHARED_LIBRARY_ORIGIN@
-SET_EXECUTABLE_ORIGIN = @SET_EXECUTABLE_ORIGIN@
-
LIBRARY_PREFIX := @LIBRARY_PREFIX@
SHARED_LIBRARY_SUFFIX := @SHARED_LIBRARY_SUFFIX@
STATIC_LIBRARY_SUFFIX := @STATIC_LIBRARY_SUFFIX@
@@ -657,8 +660,8 @@ JAVA_CMD := @JAVA@
JAVAC_CMD := @JAVAC@
JAVADOC_CMD := @JAVADOC@
JAR_CMD := @JAR@
-JLINK_CMD := @JLINK@
-JMOD_CMD := @JMOD@
+JLINK_CMD := @FIXPATH@ $(BUILD_JDK)/bin/jlink
+JMOD_CMD := @FIXPATH@ $(BUILD_JDK)/bin/jmod
# These variables are meant to be used. They are defined with = instead of := to make
# it possible to override only the *_CMD variables.
JAVA = $(JAVA_CMD) $(JAVA_FLAGS_BIG) $(JAVA_FLAGS)
diff --git a/make/common/Execute.gmk b/make/common/Execute.gmk
index 4199e8f13b7..c195510151f 100644
--- a/make/common/Execute.gmk
+++ b/make/common/Execute.gmk
@@ -148,9 +148,12 @@ define SetupExecuteBody
$1_INFO := Running commands for $1
endif
+ $1_VARDEPS := $$($1_COMMAND) $$($1_PRE_COMMAND) $$($1_POST_COMMAND)
+ $1_VARDEPS_FILE := $$(call DependOnVariable, $1_VARDEPS, $$($1_BASE)_exec.vardeps)
+
ifneq ($$($1_PRE_COMMAND), )
- $$($1_PRE_MARKER): $$($1_DEPS)
+ $$($1_PRE_MARKER): $$($1_DEPS) $$($1_VARDEPS_FILE)
ifneq ($$($1_WARN), )
$$(call LogWarn, $$($1_WARN))
endif
@@ -176,7 +179,7 @@ define SetupExecuteBody
$1 := $$($1_PRE_MARKER) $$($1_EXEC_RESULT)
else
- $$($1_EXEC_RESULT): $$($1_DEPS)
+ $$($1_EXEC_RESULT): $$($1_DEPS) $$($1_VARDEPS_FILE)
ifneq ($$($1_WARN), )
$$(call LogWarn, $$($1_WARN))
endif
diff --git a/make/common/JdkNativeCompilation.gmk b/make/common/JdkNativeCompilation.gmk
index 0285669ffd8..29001e09bd0 100644
--- a/make/common/JdkNativeCompilation.gmk
+++ b/make/common/JdkNativeCompilation.gmk
@@ -30,6 +30,47 @@ ifeq ($(INCLUDE), true)
include NativeCompilation.gmk
+ifeq ($(call isCompiler, gcc), true)
+ # --disable-new-dtags forces use of RPATH instead of RUNPATH for rpaths.
+ # This protects internal library dependencies within the JDK from being
+ # overridden using LD_LIBRARY_PATH. See JDK-8326891 for more information.
+ SetExecutableOrigin = \
+ -Wl,-rpath,\$(DOLLAR)ORIGIN$1 -Wl,--disable-new-dtags
+ SetSharedLibraryOrigin = \
+ -Wl,-z,origin -Wl,-rpath,\$(DOLLAR)ORIGIN$1 -Wl,--disable-new-dtags
+else ifeq ($(call isCompiler, clang), true)
+ ifeq ($(call isTargetOs, macosx), true)
+ SetExecutableOrigin = \
+ -Wl,-rpath,@loader_path$(or $1,/.)
+ SetSharedLibraryOrigin = \
+ -Wl,-rpath,@loader_path$(or $1,/.)
+ else ifeq ($(call isTargetOs, aix), true)
+ SetExecutableOrigin =
+ SetSharedLibraryOrigin =
+ else
+ ifeq ($(call isTargetOs, linux), true)
+ SetExecutableOrigin = \
+ -Wl,-rpath,\$(DOLLAR)ORIGIN$1 -Wl,--disable-new-dtags
+ else
+ SetExecutableOrigin = \
+ -Wl,-rpath,\$(DOLLAR)ORIGIN$1
+ endif
+
+ ifeq ($(call isTargetOs, arm), true)
+ SetSharedLibraryOrigin = \
+ -Wl,-rpath,\$(DOLLAR)ORIGIN$1
+ else
+ SetSharedLibraryOrigin = \
+ -Wl,-z,origin -Wl,-rpath,\$(DOLLAR)ORIGIN$1
+ endif
+ endif
+else ifeq ($(call isCompiler, microsoft), true)
+ SetExecutableOrigin =
+ SetSharedLibraryOrigin =
+else
+ $(error Unknown toolchain)
+endif
+
FindSrcDirsForComponent += \
$(call uniq, $(wildcard \
$(TOPDIR)/src/$(strip $1)/$(OPENJDK_TARGET_OS)/native/$(strip $2) \
@@ -444,9 +485,9 @@ define SetupJdkNativeCompilationBody
ifneq ($$($1_LD_SET_ORIGIN), false)
ifeq ($$($1_TYPE), EXECUTABLE)
- $1_LDFLAGS += $$(call SET_EXECUTABLE_ORIGIN)
+ $1_LDFLAGS += $$(call SetExecutableOrigin)
else
- $1_LDFLAGS += $$(call SET_SHARED_LIBRARY_ORIGIN)
+ $1_LDFLAGS += $$(call SetSharedLibraryOrigin)
endif
endif
# APPEND_LDFLAGS, if it exists, must be set after the origin flags
diff --git a/make/common/modules/LauncherCommon.gmk b/make/common/modules/LauncherCommon.gmk
index 700c0de74d5..0b420df5684 100644
--- a/make/common/modules/LauncherCommon.gmk
+++ b/make/common/modules/LauncherCommon.gmk
@@ -156,8 +156,8 @@ define SetupBuildLauncherBody
DISABLED_WARNINGS_gcc := unused-function unused-variable, \
DISABLED_WARNINGS_clang := unused-function, \
LDFLAGS := $$($1_LDFLAGS), \
- LDFLAGS_linux := $$(call SET_EXECUTABLE_ORIGIN,/../lib), \
- LDFLAGS_macosx := $$(call SET_EXECUTABLE_ORIGIN,/../lib), \
+ LDFLAGS_linux := $$(call SetExecutableOrigin,/../lib), \
+ LDFLAGS_macosx := $$(call SetExecutableOrigin,/../lib), \
LDFLAGS_FILTER_OUT := $$($1_LDFLAGS_FILTER_OUT), \
JDK_LIBS := $$($1_JDK_LIBS), \
JDK_LIBS_windows := $$($1_JDK_LIBS_windows), \
diff --git a/make/common/native/CompileFile.gmk b/make/common/native/CompileFile.gmk
index a8d46949788..10326d3066c 100644
--- a/make/common/native/CompileFile.gmk
+++ b/make/common/native/CompileFile.gmk
@@ -93,6 +93,14 @@ DEPENDENCY_TARGET_SED_PATTERN := \
-e 's/$$$$/ :/' \
#
+################################################################################
+# Setup compiler-specific argument to specify output file
+ifeq ($(call isCompiler, microsoft), true)
+ CC_OUT_OPTION := -Fo
+else
+ CC_OUT_OPTION := -o$(SPACE)
+endif
+
################################################################################
# Create the recipe needed to compile a single native source file.
#
@@ -334,7 +342,7 @@ define CreateWindowsResourceFile
$$(call LogInfo, Compiling resource $$(notdir $$($1_VERSIONINFO_RESOURCE)) (for $$($1_BASENAME)))
$$(call MakeDir, $$(@D) $$($1_OBJECT_DIR))
$$(call ExecuteWithLog, $$@, $$(call MakeCommandRelative, \
- $$($1_RC) $$($1_RCFLAGS) $$($1_SYSROOT_CFLAGS) $(CC_OUT_OPTION)$$@ \
+ $$($1_RC) $$($1_RCFLAGS) $$($1_SYSROOT_CFLAGS) -Fo$$@ \
$$($1_VERSIONINFO_RESOURCE) 2>&1 ))
# Windows RC compiler does not support -showIncludes, so we mis-use CL
# for this. Filter out RC specific arguments that are unknown to CL.
@@ -344,7 +352,7 @@ define CreateWindowsResourceFile
$$(call ExecuteWithLog, $$($1_RES_DEPS_FILE)$(OBJ_SUFFIX), \
$$($1_CC) $$(filter-out -l%, $$($1_RCFLAGS)) \
$$($1_SYSROOT_CFLAGS) -showIncludes -nologo -TC \
- $(CC_OUT_OPTION)$$($1_RES_DEPS_FILE)$(OBJ_SUFFIX) -P -Fi$$($1_RES_DEPS_FILE).pp \
+ -Fo$$($1_RES_DEPS_FILE)$(OBJ_SUFFIX) -P -Fi$$($1_RES_DEPS_FILE).pp \
$$($1_VERSIONINFO_RESOURCE)) 2>&1 \
| $(TR) -d '\r' | $(GREP) -v -e "^Note: including file:" \
-e "^$$(notdir $$($1_VERSIONINFO_RESOURCE))$$$$" || test "$$$$?" = "1" ; \
diff --git a/make/common/native/Link.gmk b/make/common/native/Link.gmk
index e888edfcc4c..855e50bddfb 100644
--- a/make/common/native/Link.gmk
+++ b/make/common/native/Link.gmk
@@ -50,6 +50,26 @@ GetEntitlementsFile = \
$(if $(wildcard $f), $f, $(DEFAULT_ENTITLEMENTS_FILE)) \
)
+ifeq ($(call isCompiler, gcc), true)
+ SetSharedLibraryName = \
+ -Wl,-soname=$1
+else ifeq ($(call isCompiler, clang), true)
+ ifeq ($(call isTargetOs, macosx), true)
+ SetSharedLibraryName = \
+ -Wl,-install_name,@rpath/$1
+ else ifeq ($(call isTargetOs, aix), true)
+ SetSharedLibraryName =
+ else
+ # Default works for linux, might work on other platforms as well.
+ SetSharedLibraryName = \
+ -Wl,-soname=$1
+ endif
+else ifeq ($(call isCompiler, microsoft), true)
+ SetSharedLibraryName =
+else
+ $(error Unknown toolchain)
+endif
+
################################################################################
define SetupLinking
# Unless specifically set, stripping should only happen if symbols are also
@@ -131,7 +151,7 @@ define CreateDynamicLibraryOrExecutable
# A shared dynamic library or an executable binary has been specified
ifeq ($$($1_TYPE), LIBRARY)
# Generating a dynamic library.
- $1_EXTRA_LDFLAGS += $$(call SET_SHARED_LIBRARY_NAME,$$($1_BASENAME))
+ $1_EXTRA_LDFLAGS += $$(call SetSharedLibraryName,$$($1_BASENAME))
endif
ifeq ($(MACOSX_CODESIGN_MODE), hardened)
diff --git a/make/conf/github-actions.conf b/make/conf/github-actions.conf
index d2b6cd23128..bd73e909062 100644
--- a/make/conf/github-actions.conf
+++ b/make/conf/github-actions.conf
@@ -26,24 +26,24 @@
# Versions and download locations for dependencies used by GitHub Actions (GHA)
GTEST_VERSION=1.14.0
-JTREG_VERSION=7.5.2+1
+JTREG_VERSION=8.1+1
LINUX_X64_BOOT_JDK_EXT=tar.gz
-LINUX_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk24/1f9ff9062db4449d8ca828c504ffae90/36/GPL/openjdk-24_linux-x64_bin.tar.gz
-LINUX_X64_BOOT_JDK_SHA256=88b090fa80c6c1d084ec9a755233967458788e2c0777ae2e172230c5c692d7ef
+LINUX_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk25/bd75d5f9689641da8e1daabeccb5528b/36/GPL/openjdk-25_linux-x64_bin.tar.gz
+LINUX_X64_BOOT_JDK_SHA256=59cdcaf255add4721de38eb411d4ecfe779356b61fb671aee63c7dec78054c2b
ALPINE_LINUX_X64_BOOT_JDK_EXT=tar.gz
-ALPINE_LINUX_X64_BOOT_JDK_URL=https://github.com/adoptium/temurin24-binaries/releases/download/jdk-24%2B36/OpenJDK24U-jdk_x64_alpine-linux_hotspot_24_36.tar.gz
-ALPINE_LINUX_X64_BOOT_JDK_SHA256=a642608f0da78344ee6812fb1490b8bc1d7ad5a18064c70994d6f330568c51cb
+ALPINE_LINUX_X64_BOOT_JDK_URL=https://github.com/adoptium/temurin25-binaries/releases/download/jdk-25%2B36/OpenJDK25U-jdk_x64_alpine-linux_hotspot_25_36.tar.gz
+ALPINE_LINUX_X64_BOOT_JDK_SHA256=637e47474d411ed86134f413af7d5fef4180ddb0bf556347b7e74a88cf8904c8
MACOS_AARCH64_BOOT_JDK_EXT=tar.gz
-MACOS_AARCH64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk24/1f9ff9062db4449d8ca828c504ffae90/36/GPL/openjdk-24_macos-aarch64_bin.tar.gz
-MACOS_AARCH64_BOOT_JDK_SHA256=f7133238a12714a62c5ad2bd4da6741130be1a82512065da9ca23dee26b2d3d3
+MACOS_AARCH64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk25/bd75d5f9689641da8e1daabeccb5528b/36/GPL/openjdk-25_macos-aarch64_bin.tar.gz
+MACOS_AARCH64_BOOT_JDK_SHA256=2006337bf326fdfdf6117081751ba38c1c8706d63419ecac7ff102ff7c776876
MACOS_X64_BOOT_JDK_EXT=tar.gz
-MACOS_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk24/1f9ff9062db4449d8ca828c504ffae90/36/GPL/openjdk-24_macos-x64_bin.tar.gz
-MACOS_X64_BOOT_JDK_SHA256=6bbfb1d01741cbe55ab90299cb91464b695de9a3ace85c15131aa2f50292f321
+MACOS_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk25/bd75d5f9689641da8e1daabeccb5528b/36/GPL/openjdk-25_macos-x64_bin.tar.gz
+MACOS_X64_BOOT_JDK_SHA256=47482ad9888991ecac9b2bcc131e2b53ff78aff275104cef85f66252308e8a09
WINDOWS_X64_BOOT_JDK_EXT=zip
-WINDOWS_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk24/1f9ff9062db4449d8ca828c504ffae90/36/GPL/openjdk-24_windows-x64_bin.zip
-WINDOWS_X64_BOOT_JDK_SHA256=11d1d9f6ac272d5361c8a0bef01894364081c7fb1a6914c2ad2fc312ae83d63b
+WINDOWS_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk25/bd75d5f9689641da8e1daabeccb5528b/36/GPL/openjdk-25_windows-x64_bin.zip
+WINDOWS_X64_BOOT_JDK_SHA256=85bcc178461e2cb3c549ab9ca9dfa73afd54c09a175d6510d0884071867137d3
diff --git a/make/conf/jib-profiles.js b/make/conf/jib-profiles.js
index d4877604a90..795335d7c3c 100644
--- a/make/conf/jib-profiles.js
+++ b/make/conf/jib-profiles.js
@@ -387,8 +387,8 @@ var getJibProfilesCommon = function (input, data) {
};
};
- common.boot_jdk_version = "24";
- common.boot_jdk_build_number = "36";
+ common.boot_jdk_version = "25";
+ common.boot_jdk_build_number = "37";
common.boot_jdk_home = input.get("boot_jdk", "install_path") + "/jdk-"
+ common.boot_jdk_version
+ (input.build_os == "macosx" ? ".jdk/Contents/Home" : "");
@@ -1174,9 +1174,9 @@ var getJibProfilesDependencies = function (input, common) {
jtreg: {
server: "jpg",
product: "jtreg",
- version: "7.5.2",
+ version: "8.1",
build_number: "1",
- file: "bundles/jtreg-7.5.2+1.zip",
+ file: "bundles/jtreg-8.1+1.zip",
environment_name: "JT_HOME",
environment_path: input.get("jtreg", "home_path") + "/bin",
configure_args: "--with-jtreg=" + input.get("jtreg", "home_path"),
diff --git a/make/conf/version-numbers.conf b/make/conf/version-numbers.conf
index 38d6e42dff9..977809535ba 100644
--- a/make/conf/version-numbers.conf
+++ b/make/conf/version-numbers.conf
@@ -37,6 +37,6 @@ DEFAULT_VERSION_DATE=2026-03-17
DEFAULT_VERSION_CLASSFILE_MAJOR=70 # "`$EXPR $DEFAULT_VERSION_FEATURE + 44`"
DEFAULT_VERSION_CLASSFILE_MINOR=0
DEFAULT_VERSION_DOCS_API_SINCE=11
-DEFAULT_ACCEPTABLE_BOOT_VERSIONS="24 25 26"
+DEFAULT_ACCEPTABLE_BOOT_VERSIONS="25 26"
DEFAULT_JDK_SOURCE_TARGET_VERSION=26
DEFAULT_PROMOTED_VERSION_PRE=ea
diff --git a/make/data/charsetmapping/IBM930.c2b b/make/data/charsetmapping/IBM930.c2b
index 88754763fe3..72107424104 100644
--- a/make/data/charsetmapping/IBM930.c2b
+++ b/make/data/charsetmapping/IBM930.c2b
@@ -32,11 +32,6 @@
547d 92ca
53da 9b7e
446e f86f
-#
-# we should use this one instead of the 4260<-ff0d
-#4260 2212
-4260 ff0d
-#
426A 00A6
43A1 301C
444A 2014
diff --git a/make/data/charsetmapping/IBM930.map b/make/data/charsetmapping/IBM930.map
index 4b9dad9526b..7939e795bdf 100644
--- a/make/data/charsetmapping/IBM930.map
+++ b/make/data/charsetmapping/IBM930.map
@@ -25,13 +25,6 @@
# 4260 <--> 2212
# 426A <--> 00A6
#
-# Warning:
-# "our old" implementation seems agree with above "new" mappings
-# except the entries 4260 <-> 2212. To keep the "compatbility"
-# with the "old" implementation, I changed the entries "temporarily"
-# 4260 <-> 2212
-# 4260 <- ff0d
-#
00 0000
01 0001
02 0002
@@ -407,8 +400,7 @@ FF 009F
425D FF09
425E FF1B
425F FFE2
-#4260 FF0D
-4260 2212
+4260 FF0D
4261 FF0F
426A FFE4
426B FF0C
diff --git a/make/devkit/Makefile b/make/devkit/Makefile
index d2167bf33fa..30e0dce0839 100644
--- a/make/devkit/Makefile
+++ b/make/devkit/Makefile
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2013, 2024, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2013, 2025, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -57,61 +57,61 @@
COMMA := ,
-os := $(shell uname -o)
-cpu := $(shell uname -p)
+OS := $(shell uname -o)
+CPU := $(shell uname -m)
# Figure out what platform this is building on.
-me := $(cpu)-$(if $(findstring Linux,$(os)),linux-gnu)
+ME := $(CPU)-$(if $(findstring Linux,$(OS)),linux-gnu)
-$(info Building on platform $(me))
+$(info Building on platform $(ME))
#
# By default just build for the current platform, which is assumed to be Linux
#
ifeq ($(TARGETS), )
- platforms := $(me)
- host_platforms := $(platforms)
+ PLATFORMS := $(ME)
+ HOST_PLATFORMS := $(PLATFORMS)
else
- platforms := $(subst $(COMMA), , $(TARGETS))
- host_platforms := $(me)
+ PLATFORMS := $(subst $(COMMA), , $(TARGETS))
+ HOST_PLATFORMS := $(ME)
endif
-target_platforms := $(platforms)
-$(info host_platforms $(host_platforms))
-$(info target_platforms $(target_platforms))
+TARGET_PLATFORMS := $(PLATFORMS)
+$(info HOST_PLATFORMS $(HOST_PLATFORMS))
+$(info TARGET_PLATFORMS $(TARGET_PLATFORMS))
-all compile : $(platforms)
+all compile : $(PLATFORMS)
ifeq ($(SKIP_ME), )
- $(foreach p,$(filter-out $(me),$(platforms)),$(eval $(p) : $$(me)))
+ $(foreach p,$(filter-out $(ME),$(PLATFORMS)),$(eval $(p) : $$(ME)))
endif
OUTPUT_ROOT = $(abspath ../../build/devkit)
RESULT = $(OUTPUT_ROOT)/result
-submakevars = HOST=$@ BUILD=$(me) RESULT=$(RESULT) OUTPUT_ROOT=$(OUTPUT_ROOT)
+SUBMAKEVARS = HOST=$@ BUILD=$(ME) RESULT=$(RESULT) OUTPUT_ROOT=$(OUTPUT_ROOT)
-$(host_platforms) :
+$(HOST_PLATFORMS) :
@echo 'Building compilers for $@'
- @echo 'Targets: $(target_platforms)'
- for p in $(filter $@, $(target_platforms)) $(filter-out $@, $(target_platforms)); do \
- $(MAKE) -f Tools.gmk download-rpms $(submakevars) \
+ @echo 'Targets: $(TARGET_PLATFORMS)'
+ for p in $(filter $@, $(TARGET_PLATFORMS)) $(filter-out $@, $(TARGET_PLATFORMS)); do \
+ $(MAKE) -f Tools.gmk download-rpms $(SUBMAKEVARS) \
TARGET=$$p PREFIX=$(RESULT)/$@-to-$$p && \
- $(MAKE) -f Tools.gmk all $(submakevars) \
+ $(MAKE) -f Tools.gmk all $(SUBMAKEVARS) \
TARGET=$$p PREFIX=$(RESULT)/$@-to-$$p && \
- $(MAKE) -f Tools.gmk ccache $(submakevars) \
+ $(MAKE) -f Tools.gmk ccache $(SUBMAKEVARS) \
TARGET=$@ PREFIX=$(RESULT)/$@-to-$$p || exit 1 ; \
done
@echo 'All done"'
-today := $(shell date +%Y%m%d)
+TODAY := $(shell date +%Y%m%d)
define Mktar
- $(1)-to-$(2)_tar = $$(RESULT)/sdk-$(1)-to-$(2)-$$(today).tar.gz
+ $(1)-to-$(2)_tar = $$(RESULT)/sdk-$(1)-to-$(2)-$$(TODAY).tar.gz
$$($(1)-to-$(2)_tar) : PLATFORM = $(1)-to-$(2)
TARFILES += $$($(1)-to-$(2)_tar)
endef
-$(foreach p,$(host_platforms),$(foreach t,$(target_platforms),$(eval $(call Mktar,$(p),$(t)))))
+$(foreach p,$(HOST_PLATFORMS),$(foreach t,$(TARGET_PLATFORMS),$(eval $(call Mktar,$(p),$(t)))))
tars : all $(TARFILES)
onlytars : $(TARFILES)
@@ -119,9 +119,9 @@ onlytars : $(TARFILES)
$(MAKE) -r -f Tars.gmk SRC_DIR=$(RESULT)/$(PLATFORM) TAR_FILE=$@
clean :
- rm -rf $(addprefix ../../build/devkit/, result $(host_platforms))
+ rm -rf $(addprefix ../../build/devkit/, result $(HOST_PLATFORMS))
dist-clean: clean
rm -rf $(addprefix ../../build/devkit/, src download)
FORCE :
-.PHONY : all compile tars $(configs) $(host_platforms) clean dist-clean
+.PHONY : all compile tars $(HOST_PLATFORMS) clean dist-clean
diff --git a/make/devkit/Tools.gmk b/make/devkit/Tools.gmk
index f27d47b822c..db77f2f3f74 100644
--- a/make/devkit/Tools.gmk
+++ b/make/devkit/Tools.gmk
@@ -39,7 +39,7 @@
# Fix this...
#
-uppercase = $(shell echo $1 | tr a-z A-Z)
+lowercase = $(shell echo $1 | tr A-Z a-z)
$(info TARGET=$(TARGET))
$(info HOST=$(HOST))
@@ -104,26 +104,48 @@ endif
################################################################################
# Define external dependencies
-gcc_ver_only := 14.2.0
-binutils_ver_only := 2.43
-ccache_ver_only := 4.10.2
+GNU_BASE_URL := https://ftp.gnu.org/pub/gnu
+
+BINUTILS_VER_ONLY := 2.43
+BINUTILS_BASE_URL := $(GNU_BASE_URL)/binutils
+BINUTILS_SHA512 := 93e063163e54d6a6ee2bd48dc754270bf757a3635b49a702ed6b310e929e94063958512d191e66beaf44275f7ea60865dbde138b624626739679fcc306b133bb
+
+CCACHE_VER_ONLY := 4.10.2
+CCACHE_BASE_URL := https://github.com/ccache/ccache/releases/download
CCACHE_CMAKE_BASED := 1
-mpfr_ver_only := 4.2.1
-gmp_ver_only := 6.3.0
-mpc_ver_only := 1.3.1
-gdb_ver_only := 15.2
+CCACHE_SHA512 := 3815c71d7266c32839acb306763268018acc58b3bbbd9ec79fc101e4217c1720d2ad2f01645bf69168c1c61d27700b6f3bb755cfa82689cca69824f015653f3c
-dependencies := gcc binutils ccache mpfr gmp mpc gdb
+GCC_VER_ONLY := 14.2.0
+GCC_BASE_URL := $(GNU_BASE_URL)/gcc
+GCC_SHA512 := 932bdef0cda94bacedf452ab17f103c0cb511ff2cec55e9112fc0328cbf1d803b42595728ea7b200e0a057c03e85626f937012e49a7515bc5dd256b2bf4bc396
-$(foreach dep,$(dependencies),$(eval $(dep)_ver := $(dep)-$($(dep)_ver_only)))
+GDB_VER_ONLY := 15.2
+GDB_BASE_URL := $(GNU_BASE_URL)/gdb
+GDB_SHA512 := 624007deceb5b15ba89c0725883d1a699fa46714ef30887f3d0165e17c5d65d634671740a135aa69e437d916218abb08cfa2a38ed309ff19d48f51da56b2a8ba
-GCC := http://ftp.gnu.org/pub/gnu/gcc/$(gcc_ver)/$(gcc_ver).tar.xz
-BINUTILS := http://ftp.gnu.org/pub/gnu/binutils/$(binutils_ver).tar.gz
-CCACHE := https://github.com/ccache/ccache/releases/download/v$(ccache_ver_only)/$(ccache_ver).tar.xz
-MPFR := https://www.mpfr.org/$(mpfr_ver)/$(mpfr_ver).tar.bz2
-GMP := http://ftp.gnu.org/pub/gnu/gmp/$(gmp_ver).tar.bz2
-MPC := http://ftp.gnu.org/pub/gnu/mpc/$(mpc_ver).tar.gz
-GDB := http://ftp.gnu.org/gnu/gdb/$(gdb_ver).tar.xz
+GMP_VER_ONLY := 6.3.0
+GMP_BASE_URL := $(GNU_BASE_URL)/gmp
+GMP_SHA512 := e85a0dab5195889948a3462189f0e0598d331d3457612e2d3350799dba2e244316d256f8161df5219538eb003e4b5343f989aaa00f96321559063ed8c8f29fd2
+
+MPC_VER_ONLY := 1.3.1
+MPC_BASE_URL := $(GNU_BASE_URL)/mpc
+MPC_SHA512 := 4bab4ef6076f8c5dfdc99d810b51108ced61ea2942ba0c1c932d624360a5473df20d32b300fc76f2ba4aa2a97e1f275c9fd494a1ba9f07c4cb2ad7ceaeb1ae97
+
+MPFR_VER_ONLY := 4.2.1
+MPFR_BASE_URL := https://www.mpfr.org
+MPFR_SHA512 := bc68c0d755d5446403644833ecbb07e37360beca45f474297b5d5c40926df1efc3e2067eecffdf253f946288bcca39ca89b0613f545d46a9e767d1d4cf358475
+
+DEPENDENCIES := BINUTILS CCACHE GCC GDB GMP MPC MPFR
+
+$(foreach dep,$(DEPENDENCIES),$(eval $(dep)_VER := $(call lowercase,$(dep)-$($(dep)_VER_ONLY))))
+
+BINUTILS_URL := $(BINUTILS_BASE_URL)/$(BINUTILS_VER).tar.xz
+CCACHE_URL := $(CCACHE_BASE_URL)/v$(CCACHE_VER_ONLY)/$(CCACHE_VER).tar.xz
+GCC_URL := $(GCC_BASE_URL)/$(GCC_VER)/$(GCC_VER).tar.xz
+GDB_URL := $(GDB_BASE_URL)/$(GDB_VER).tar.xz
+GMP_URL := $(GMP_BASE_URL)/$(GMP_VER).tar.xz
+MPC_URL := $(MPC_BASE_URL)/$(MPC_VER).tar.gz
+MPFR_URL := $(MPFR_BASE_URL)/$(MPFR_VER)/$(MPFR_VER).tar.xz
REQUIRED_MIN_MAKE_MAJOR_VERSION := 4
ifneq ($(REQUIRED_MIN_MAKE_MAJOR_VERSION),)
@@ -180,10 +202,10 @@ DOWNLOAD_RPMS := $(DOWNLOAD)/rpms/$(TARGET)-$(LINUX_VERSION)
SRCDIR := $(OUTPUT_ROOT)/src
# Marker file for unpacking rpms
-rpms := $(SYSROOT)/rpms_unpacked
+RPMS := $(SYSROOT)/rpms_unpacked
# Need to patch libs that are linker scripts to use non-absolute paths
-libs := $(SYSROOT)/libs_patched
+LIBS := $(SYSROOT)/libs_patched
################################################################################
# Download RPMs
@@ -198,10 +220,10 @@ download-rpms:
################################################################################
# Unpack source packages
-# Generate downloading + unpacking of sources.
-define Download
+# Generate downloading + checksum verification of sources.
+define DownloadVerify
# Allow override
- $(1)_DIRNAME ?= $(basename $(basename $(notdir $($(1)))))
+ $(1)_DIRNAME ?= $(basename $(basename $(notdir $($(1)_URL))))
$(1)_DIR = $(abspath $(SRCDIR)/$$($(1)_DIRNAME))
ifeq ($$($(1)_CMAKE_BASED),)
$(1)_CFG = $$($(1)_DIR)/configure
@@ -212,7 +234,7 @@ define Download
$(1)_SRC_MARKER = $$($(1)_DIR)/CMakeLists.txt
$(1)_CONFIG = $$(CMAKE_CONFIG) $$($(1)_DIR)
endif
- $(1)_FILE = $(DOWNLOAD)/$(notdir $($(1)))
+ $(1)_FILE = $(DOWNLOAD)/$(notdir $($(1)_URL))
$$($(1)_SRC_MARKER) : $$($(1)_FILE)
mkdir -p $$(SRCDIR)
@@ -224,11 +246,20 @@ define Download
touch $$@
$$($(1)_FILE) :
- wget -P $(DOWNLOAD) $$($(1))
+ mkdir -p $$(@D)
+ wget -O - $$($(1)_URL) > $$@.tmp
+ sha512_actual="$$$$(sha512sum $$@.tmp | awk '{ print $$$$1; }')"; \
+ if [ x"$$$${sha512_actual}" != x"$$($(1)_SHA512)" ]; then \
+ echo "Checksum mismatch for $$@.tmp"; \
+ echo " Expected: $$($(1)_SHA512)"; \
+ echo " Actual: $$$${sha512_actual}"; \
+ exit 1; \
+ fi
+ mv $$@.tmp $$@
endef
# Download and unpack all source packages
-$(foreach dep,$(dependencies),$(eval $(call Download,$(call uppercase,$(dep)))))
+$(foreach dep,$(DEPENDENCIES),$(eval $(call DownloadVerify,$(dep))))
################################################################################
# Unpack RPMS
@@ -250,7 +281,7 @@ RPM_FILE_LIST := $(sort $(foreach a, $(RPM_ARCHS), \
# Note. For building linux you should install rpm2cpio.
define unrpm
$(SYSROOT)/$(notdir $(1)).unpacked : $(1)
- $$(rpms) : $(SYSROOT)/$(notdir $(1)).unpacked
+ $$(RPMS) : $(SYSROOT)/$(notdir $(1)).unpacked
endef
%.unpacked :
@@ -277,7 +308,7 @@ $(foreach p,$(RPM_FILE_LIST),$(eval $(call unrpm,$(p))))
# have it anyway, but just to make sure...
# Patch libc.so and libpthread.so to force linking against libraries in sysroot
# and not the ones installed on the build machine.
-$(libs) : $(rpms)
+$(LIBS) : $(RPMS)
@echo Patching libc and pthreads
@(for f in `find $(SYSROOT) -name libc.so -o -name libpthread.so`; do \
(cat $$f | sed -e 's|/usr/lib64/||g' \
@@ -293,10 +324,10 @@ $(libs) : $(rpms)
# Create links for ffi header files so that they become visible by default when using the
# devkit.
ifeq ($(ARCH), x86_64)
- $(SYSROOT)/usr/include/ffi.h: $(rpms)
+ $(SYSROOT)/usr/include/ffi.h: $(RPMS)
cd $(@D) && rm -f $(@F) && ln -s ../lib/libffi-*/include/$(@F) .
- $(SYSROOT)/usr/include/ffitarget.h: $(rpms)
+ $(SYSROOT)/usr/include/ffitarget.h: $(RPMS)
cd $(@D) && rm -f $(@F) && ln -s ../lib/libffi-*/include/$(@F) .
SYSROOT_LINKS += $(SYSROOT)/usr/include/ffi.h $(SYSROOT)/usr/include/ffitarget.h
@@ -305,7 +336,7 @@ endif
################################################################################
# Define marker files for each source package to be compiled
-$(foreach dep,$(dependencies),$(eval $(dep) = $(TARGETDIR)/$($(dep)_ver).done))
+$(foreach dep,$(DEPENDENCIES),$(eval $(dep) = $(TARGETDIR)/$($(dep)_VER).done))
################################################################################
@@ -345,48 +376,48 @@ TOOLS ?= $(call declare_tools,_FOR_TARGET,$(TARGET)-)
# CFLAG_ to most likely -m32.
define mk_bfd
$$(info Libs for $(1))
- $$(BUILDDIR)/$$(binutils_ver)-$(subst /,-,$(1))/Makefile \
+ $$(BUILDDIR)/$$(BINUTILS_VER)-$(subst /,-,$(1))/Makefile \
: CFLAGS += $$(CFLAGS_$(1))
- $$(BUILDDIR)/$$(binutils_ver)-$(subst /,-,$(1))/Makefile \
+ $$(BUILDDIR)/$$(BINUTILS_VER)-$(subst /,-,$(1))/Makefile \
: LIBDIRS = --libdir=$(TARGETDIR)/$(1)
- bfdlib += $$(TARGETDIR)/$$(binutils_ver)-$(subst /,-,$(1)).done
- bfdmakes += $$(BUILDDIR)/$$(binutils_ver)-$(subst /,-,$(1))/Makefile
+ BFDLIB += $$(TARGETDIR)/$$(BINUTILS_VER)-$(subst /,-,$(1)).done
+ BFDMAKES += $$(BUILDDIR)/$$(BINUTILS_VER)-$(subst /,-,$(1))/Makefile
endef
# Create one set of bfds etc for each multilib arch
$(foreach l,$(LIBDIRS),$(eval $(call mk_bfd,$(l))))
# Only build these two libs.
-$(bfdlib) : MAKECMD = all-libiberty all-bfd
-$(bfdlib) : INSTALLCMD = install-libiberty install-bfd
+$(BFDLIB) : MAKECMD = all-libiberty all-bfd
+$(BFDLIB) : INSTALLCMD = install-libiberty install-bfd
# Building targets libbfd + libiberty. HOST==TARGET, i.e not
# for a cross env.
-$(bfdmakes) : CONFIG = --target=$(TARGET) \
+$(BFDMAKES) : CONFIG = --target=$(TARGET) \
--host=$(TARGET) --build=$(BUILD) \
--prefix=$(TARGETDIR) \
--with-sysroot=$(SYSROOT) \
$(LIBDIRS)
-$(bfdmakes) : TOOLS = $(call declare_tools,_FOR_TARGET,$(TARGET)-) $(call declare_tools,,$(TARGET)-)
+$(BFDMAKES) : TOOLS = $(call declare_tools,_FOR_TARGET,$(TARGET)-) $(call declare_tools,,$(TARGET)-)
################################################################################
-$(gcc) \
- $(binutils) \
- $(gmp) \
- $(mpfr) \
- $(mpc) \
- $(bfdmakes) \
- $(ccache) : ENVS += $(TOOLS)
+$(GCC) \
+ $(BINUTILS) \
+ $(GMP) \
+ $(MPFR) \
+ $(MPC) \
+ $(BFDMAKES) \
+ $(CCACHE) : ENVS += $(TOOLS)
# libdir to work around hateful bfd stuff installing into wrong dirs...
# ensure we have 64 bit bfd support in the HOST library. I.e our
# compiler on i686 will know 64 bit symbols, BUT later
# we build just the libs again for TARGET, then with whatever the arch
# wants.
-$(BUILDDIR)/$(binutils_ver)/Makefile : CONFIG += --enable-64-bit-bfd --libdir=$(PREFIX)/$(word 1,$(LIBDIRS))
+$(BUILDDIR)/$(BINUTILS_VER)/Makefile : CONFIG += --enable-64-bit-bfd --libdir=$(PREFIX)/$(word 1,$(LIBDIRS))
ifeq ($(filter $(ARCH), s390x riscv64 ppc64le), )
# gold compiles but cannot link properly on s390x @ gcc 13.2 and Fedore 41
@@ -397,8 +428,8 @@ endif
# Makefile creation. Simply run configure in build dir.
# Setting CFLAGS to -O2 generates a much faster ld.
-$(bfdmakes) \
-$(BUILDDIR)/$(binutils_ver)/Makefile \
+$(BFDMAKES) \
+$(BUILDDIR)/$(BINUTILS_VER)/Makefile \
: $(BINUTILS_CFG)
$(info Configuring $@. Log in $(@D)/log.config)
@mkdir -p $(@D)
@@ -417,7 +448,7 @@ $(BUILDDIR)/$(binutils_ver)/Makefile \
) > $(@D)/log.config 2>&1
@echo 'done'
-$(BUILDDIR)/$(mpfr_ver)/Makefile \
+$(BUILDDIR)/$(MPFR_VER)/Makefile \
: $(MPFR_CFG)
$(info Configuring $@. Log in $(@D)/log.config)
@mkdir -p $(@D)
@@ -432,7 +463,7 @@ $(BUILDDIR)/$(mpfr_ver)/Makefile \
) > $(@D)/log.config 2>&1
@echo 'done'
-$(BUILDDIR)/$(gmp_ver)/Makefile \
+$(BUILDDIR)/$(GMP_VER)/Makefile \
: $(GMP_CFG)
$(info Configuring $@. Log in $(@D)/log.config)
@mkdir -p $(@D)
@@ -449,7 +480,7 @@ $(BUILDDIR)/$(gmp_ver)/Makefile \
) > $(@D)/log.config 2>&1
@echo 'done'
-$(BUILDDIR)/$(mpc_ver)/Makefile \
+$(BUILDDIR)/$(MPC_VER)/Makefile \
: $(MPC_CFG)
$(info Configuring $@. Log in $(@D)/log.config)
@mkdir -p $(@D)
@@ -468,11 +499,11 @@ $(BUILDDIR)/$(mpc_ver)/Makefile \
# Only valid if glibc target -> linux
# proper destructor handling for c++
ifneq (,$(findstring linux,$(TARGET)))
- $(BUILDDIR)/$(gcc_ver)/Makefile : CONFIG += --enable-__cxa_atexit
+ $(BUILDDIR)/$(GCC_VER)/Makefile : CONFIG += --enable-__cxa_atexit
endif
ifeq ($(ARCH), armhfp)
- $(BUILDDIR)/$(gcc_ver)/Makefile : CONFIG += --with-float=hard
+ $(BUILDDIR)/$(GCC_VER)/Makefile : CONFIG += --with-float=hard
endif
ifneq ($(filter riscv64 ppc64le s390x, $(ARCH)), )
@@ -487,7 +518,7 @@ endif
# skip native language.
# and link and assemble with the binutils we created
# earlier, so --with-gnu*
-$(BUILDDIR)/$(gcc_ver)/Makefile \
+$(BUILDDIR)/$(GCC_VER)/Makefile \
: $(GCC_CFG)
$(info Configuring $@. Log in $(@D)/log.config)
mkdir -p $(@D)
@@ -509,17 +540,17 @@ $(BUILDDIR)/$(gcc_ver)/Makefile \
@echo 'done'
# need binutils for gcc
-$(gcc) : $(binutils)
+$(GCC) : $(BINUTILS)
# as of 4.3 or so need these for doing config
-$(BUILDDIR)/$(gcc_ver)/Makefile : $(gmp) $(mpfr) $(mpc)
-$(mpfr) : $(gmp)
-$(mpc) : $(gmp) $(mpfr)
+$(BUILDDIR)/$(GCC_VER)/Makefile : $(GMP) $(MPFR) $(MPC)
+$(MPFR) : $(GMP)
+$(MPC) : $(GMP) $(MPFR)
################################################################################
# Build gdb but only where host and target match
ifeq ($(HOST), $(TARGET))
- $(BUILDDIR)/$(gdb_ver)/Makefile: $(GDB_CFG)
+ $(BUILDDIR)/$(GDB_VER)/Makefile: $(GDB_CFG)
$(info Configuring $@. Log in $(@D)/log.config)
mkdir -p $(@D)
( \
@@ -532,9 +563,9 @@ ifeq ($(HOST), $(TARGET))
) > $(@D)/log.config 2>&1
@echo 'done'
- $(gdb): $(gcc)
+ $(GDB): $(GCC)
else
- $(BUILDDIR)/$(gdb_ver)/Makefile:
+ $(BUILDDIR)/$(GDB_VER)/Makefile:
$(info Faking $@, not used when cross-compiling)
mkdir -p $(@D)
echo "install:" > $@
@@ -543,7 +574,7 @@ endif
################################################################################
# very straightforward. just build a ccache. it is only for host.
-$(BUILDDIR)/$(ccache_ver)/Makefile \
+$(BUILDDIR)/$(CCACHE_VER)/Makefile \
: $(CCACHE_SRC_MARKER)
$(info Configuring $@. Log in $(@D)/log.config)
@mkdir -p $(@D)
@@ -554,12 +585,12 @@ $(BUILDDIR)/$(ccache_ver)/Makefile \
) > $(@D)/log.config 2>&1
@echo 'done'
-gccpatch = $(TARGETDIR)/gcc-patched
+GCC_PATCHED = $(TARGETDIR)/gcc-patched
################################################################################
# For some reason cpp is not created as a target-compiler
ifeq ($(HOST),$(TARGET))
- $(gccpatch) : $(gcc) link_libs
+ $(GCC_PATCHED) : $(GCC) link_libs
@echo -n 'Creating compiler symlinks...'
@for f in cpp; do \
if [ ! -e $(PREFIX)/bin/$(TARGET)-$$f ]; \
@@ -587,7 +618,7 @@ ifeq ($(HOST),$(TARGET))
done;)
@echo 'done'
else
- $(gccpatch) :
+ $(GCC_PATCHED) :
@echo 'done'
endif
@@ -615,7 +646,7 @@ $(PREFIX)/devkit.info:
echo '# This file describes to configure how to interpret the contents of this' >> $@
echo '# devkit' >> $@
echo '' >> $@
- echo 'DEVKIT_NAME="$(gcc_ver) - $(LINUX_VERSION)"' >> $@
+ echo 'DEVKIT_NAME="$(GCC_VER) - $(LINUX_VERSION)"' >> $@
echo 'DEVKIT_TOOLCHAIN_PATH="$$DEVKIT_ROOT/bin"' >> $@
echo 'DEVKIT_SYSROOT="$$DEVKIT_ROOT/$(TARGET)/sysroot"' >> $@
echo 'DEVKIT_EXTRA_PATH="$$DEVKIT_ROOT/bin"' >> $@
@@ -651,32 +682,32 @@ ifeq ($(TARGET), $(HOST))
@echo 'Creating missing $* soft link'
ln -s $(TARGET)-$* $@
- missing-links := $(addprefix $(PREFIX)/bin/, \
- addr2line ar as c++ c++filt dwp elfedit g++ gcc gcc-$(gcc_ver_only) gprof ld ld.bfd \
+ MISSING_LINKS := $(addprefix $(PREFIX)/bin/, \
+ addr2line ar as c++ c++filt dwp elfedit g++ gcc gcc-$(GCC_VER_ONLY) gprof ld ld.bfd \
ld.gold nm objcopy objdump ranlib readelf size strings strip)
endif
# Add link to work around "plugin needed to handle lto object" (JDK-8344272)
-$(PREFIX)/lib/bfd-plugins/liblto_plugin.so: $(PREFIX)/libexec/gcc/$(TARGET)/$(gcc_ver_only)/liblto_plugin.so
+$(PREFIX)/lib/bfd-plugins/liblto_plugin.so: $(PREFIX)/libexec/gcc/$(TARGET)/$(GCC_VER_ONLY)/liblto_plugin.so
@echo 'Creating missing $(@F) soft link'
@mkdir -p $(@D)
ln -s $$(realpath -s --relative-to=$(@D) $<) $@
-missing-links += $(PREFIX)/lib/bfd-plugins/liblto_plugin.so
+MISSING_LINKS += $(PREFIX)/lib/bfd-plugins/liblto_plugin.so
################################################################################
-bfdlib : $(bfdlib)
-binutils : $(binutils)
-rpms : $(rpms)
-libs : $(libs)
+bfdlib : $(BFDLIB)
+binutils : $(BINUTILS)
+rpms : $(RPMS)
+libs : $(LIBS)
sysroot : rpms libs
-gcc : sysroot $(gcc) $(gccpatch)
-gdb : $(gdb)
-all : binutils gcc bfdlib $(PREFIX)/devkit.info $(missing-links) $(SYSROOT_LINKS) \
+gcc : sysroot $(GCC) $(GCC_PATCHED)
+gdb : $(GDB)
+all : binutils gcc bfdlib $(PREFIX)/devkit.info $(MISSING_LINKS) $(SYSROOT_LINKS) \
$(THESE_MAKEFILES) gdb
# this is only built for host. so separate.
-ccache : $(ccache)
+ccache : $(CCACHE)
.PHONY : gcc all binutils bfdlib link_libs rpms libs sysroot
diff --git a/make/devkit/createAutoconfBundle.sh b/make/devkit/createAutoconfBundle.sh
index ebe9c427f76..4697e4eb1e3 100644
--- a/make/devkit/createAutoconfBundle.sh
+++ b/make/devkit/createAutoconfBundle.sh
@@ -93,7 +93,7 @@ elif test "x$TARGET_PLATFORM" = xlinux_x64; then
rpm2cpio $OUTPUT_ROOT/m4-$M4_VERSION.el6.x86_64.rpm | cpio -d -i
elif test "x$TARGET_PLATFORM" = xlinux_x86; then
M4_VERSION=1.4.13-5
- wget http://yum.oracle.com/repo/OracleLinux/OL6/latest/i386/getPackage/m4-$M4_VERSION.el6.i686.rpm
+ wget https://yum.oracle.com/repo/OracleLinux/OL6/latest/i386/getPackage/m4-$M4_VERSION.el6.i686.rpm
cd $IMAGE_DIR
rpm2cpio $OUTPUT_ROOT/m4-$M4_VERSION.el6.i686.rpm | cpio -d -i
else
diff --git a/make/hotspot/lib/CompileGtest.gmk b/make/hotspot/lib/CompileGtest.gmk
index 30d3e3b524c..d615e254f5a 100644
--- a/make/hotspot/lib/CompileGtest.gmk
+++ b/make/hotspot/lib/CompileGtest.gmk
@@ -152,7 +152,7 @@ $(eval $(call SetupJdkExecutable, BUILD_GTEST_LAUNCHER, \
-I$(GTEST_FRAMEWORK_SRC)/googlemock \
-I$(GTEST_FRAMEWORK_SRC)/googlemock/include, \
LD_SET_ORIGIN := false, \
- LDFLAGS_unix := $(call SET_SHARED_LIBRARY_ORIGIN), \
+ LDFLAGS_unix := $(call SetSharedLibraryOrigin), \
JDK_LIBS := gtest:libjvm, \
COPY_DEBUG_SYMBOLS := $(GTEST_COPY_DEBUG_SYMBOLS), \
ZIP_EXTERNAL_DEBUG_SYMBOLS := false, \
diff --git a/make/hotspot/lib/CompileJvm.gmk b/make/hotspot/lib/CompileJvm.gmk
index bf92c50576a..b5a8b4ef2a1 100644
--- a/make/hotspot/lib/CompileJvm.gmk
+++ b/make/hotspot/lib/CompileJvm.gmk
@@ -189,7 +189,6 @@ $(eval $(call SetupJdkLibrary, BUILD_LIBJVM, \
DISABLED_WARNINGS_gcc_c1_Runtime1_aarch64.cpp := unused-const-variable, \
DISABLED_WARNINGS_gcc_cgroupV1Subsystem_linux.cpp := address, \
DISABLED_WARNINGS_gcc_cgroupV2Subsystem_linux.cpp := address, \
- DISABLED_WARNINGS_gcc_g1FreeIdSet.cpp := unused-const-variable, \
DISABLED_WARNINGS_gcc_handshake.cpp := stringop-overflow, \
DISABLED_WARNINGS_gcc_interp_masm_x86.cpp := uninitialized, \
DISABLED_WARNINGS_gcc_javaClasses.cpp := unused-const-variable, \
diff --git a/make/hotspot/lib/JvmFeatures.gmk b/make/hotspot/lib/JvmFeatures.gmk
index 0fd1c752174..79bbd6a4106 100644
--- a/make/hotspot/lib/JvmFeatures.gmk
+++ b/make/hotspot/lib/JvmFeatures.gmk
@@ -57,7 +57,7 @@ ifeq ($(call check-jvm-feature, zero), true)
-DZERO_LIBARCH='"$(OPENJDK_TARGET_CPU_LEGACY_LIB)"' $(LIBFFI_CFLAGS)
JVM_LIBS_FEATURES += $(LIBFFI_LIBS)
ifeq ($(ENABLE_LIBFFI_BUNDLING), true)
- JVM_LDFLAGS_FEATURES += $(call SET_EXECUTABLE_ORIGIN,/..)
+ JVM_LDFLAGS_FEATURES += $(call SetExecutableOrigin,/..)
endif
else
JVM_EXCLUDE_PATTERNS += /zero/
diff --git a/make/ide/vscode/hotspot/template-workspace.jsonc b/make/ide/vscode/hotspot/template-workspace.jsonc
index c582c48047d..8a349b232ce 100644
--- a/make/ide/vscode/hotspot/template-workspace.jsonc
+++ b/make/ide/vscode/hotspot/template-workspace.jsonc
@@ -22,6 +22,7 @@
// Java extension
"jdk.project.jdkhome": "{{OUTPUTDIR}}/jdk",
"jdk.java.onSave.organizeImports": false, // prevents unnecessary changes
+ "jdk.serverVmOptions": ["-Xmx2G"], // prevent out of memory
// Additional conventions
"files.associations": {
diff --git a/make/modules/java.base/Gensrc.gmk b/make/modules/java.base/Gensrc.gmk
index 2750a6c8791..79db438934e 100644
--- a/make/modules/java.base/Gensrc.gmk
+++ b/make/modules/java.base/Gensrc.gmk
@@ -33,7 +33,6 @@ include gensrc/GensrcBuffer.gmk
include gensrc/GensrcCharacterData.gmk
include gensrc/GensrcCharsetCoder.gmk
include gensrc/GensrcCharsetMapping.gmk
-include gensrc/GensrcExceptions.gmk
include gensrc/GensrcMisc.gmk
include gensrc/GensrcModuleLoaderMap.gmk
include gensrc/GensrcRegex.gmk
diff --git a/make/modules/java.base/Lib.gmk b/make/modules/java.base/Lib.gmk
index 51d323a0344..41da22f8cb2 100644
--- a/make/modules/java.base/Lib.gmk
+++ b/make/modules/java.base/Lib.gmk
@@ -154,6 +154,8 @@ endif
################################################################################
## Build libsyslookup
+## The LIBDL dependency on Linux is needed to dynamically access libdl symbols,
+## which may be needed as part of resolving some standard symbols
################################################################################
$(eval $(call SetupJdkLibrary, BUILD_LIBSYSLOOKUP, \
@@ -196,7 +198,7 @@ ifeq ($(call isTargetOs, linux)+$(call isTargetCpu, x86_64)+$(INCLUDE_COMPILER2)
OPTIMIZATION := HIGH, \
CXXFLAGS := -std=c++17, \
DISABLED_WARNINGS_gcc := unused-variable, \
- LIBS_linux := $(LIBDL) $(LIBM), \
+ LIBS_linux := $(LIBM), \
))
TARGETS += $(BUILD_LIBSIMD_SORT)
diff --git a/make/modules/java.base/gensrc/GensrcExceptions.gmk b/make/modules/java.base/gensrc/GensrcExceptions.gmk
deleted file mode 100644
index baa61596d6b..00000000000
--- a/make/modules/java.base/gensrc/GensrcExceptions.gmk
+++ /dev/null
@@ -1,57 +0,0 @@
-#
-# Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved.
-# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-#
-# This code is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License version 2 only, as
-# published by the Free Software Foundation. Oracle designates this
-# particular file as subject to the "Classpath" exception as provided
-# by Oracle in the LICENSE file that accompanied this code.
-#
-# This code is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# version 2 for more details (a copy is included in the LICENSE file that
-# accompanied this code).
-#
-# You should have received a copy of the GNU General Public License version
-# 2 along with this work; if not, write to the Free Software Foundation,
-# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
-# or visit www.oracle.com if you need additional information or have any
-# questions.
-#
-
-include MakeIncludeStart.gmk
-ifeq ($(INCLUDE), true)
-
-################################################################################
-
-GENSRC_EXCEPTIONS :=
-
-GENSRC_EXCEPTIONS_DST := $(SUPPORT_OUTPUTDIR)/gensrc/java.base/java/nio
-
-GENSRC_EXCEPTIONS_SRC := $(MODULE_SRC)/share/classes/java/nio
-GENSRC_EXCEPTIONS_CMD := $(TOPDIR)/make/scripts/genExceptions.sh
-
-GENSRC_EXCEPTIONS_SRC_DIRS := . charset channels
-
-$(GENSRC_EXCEPTIONS_DST)/_the.%.marker: $(GENSRC_EXCEPTIONS_SRC)/%/exceptions \
- $(GENSRC_EXCEPTIONS_CMD)
- $(call LogInfo, Generating exceptions java.nio $*)
- $(call MakeDir, $(@D)/$*)
- SCRIPTS="$(TOPDIR)/make/scripts" AWK="$(AWK)" SH="$(SH)" $(SH) \
- $(GENSRC_EXCEPTIONS_CMD) $< $(@D)/$* $(LOG_DEBUG)
- $(TOUCH) $@
-
-GENSRC_EXCEPTIONS += $(foreach D, $(GENSRC_EXCEPTIONS_SRC_DIRS), $(GENSRC_EXCEPTIONS_DST)/_the.$(D).marker)
-
-$(GENSRC_EXCEPTIONS): $(BUILD_TOOLS_JDK)
-
-TARGETS += $(GENSRC_EXCEPTIONS)
-
-################################################################################
-
-endif # include guard
-include MakeIncludeEnd.gmk
diff --git a/make/modules/java.desktop/lib/ClientLibraries.gmk b/make/modules/java.desktop/lib/ClientLibraries.gmk
index a69b65180d7..2c29092cdd6 100644
--- a/make/modules/java.desktop/lib/ClientLibraries.gmk
+++ b/make/modules/java.desktop/lib/ClientLibraries.gmk
@@ -51,7 +51,7 @@ $(eval $(call SetupJdkLibrary, BUILD_LIBMLIB_IMAGE, \
$(LIBMLIB_IMAGE_CFLAGS), \
DISABLED_WARNINGS_gcc := unused-function, \
DISABLED_WARNINGS_clang_mlib_ImageCreate.c := unused-function, \
- LIBS_unix := $(LIBDL) $(LIBM), \
+ LIBS_unix := $(LIBM), \
))
TARGETS += $(BUILD_LIBMLIB_IMAGE)
@@ -264,7 +264,7 @@ ifeq ($(ENABLE_HEADLESS_ONLY), false)
JDK_LIBS_macosx := libosxapp, \
LIBS := $(GIFLIB_LIBS) $(LIBJPEG_LIBS) $(LIBZ_LIBS) $(PNG_LIBS) $(ICONV_LIBS), \
LIBS_unix := $(LIBM) $(LIBPTHREAD), \
- LIBS_linux := $(LIBDL) $(X_LIBS) -lX11 -lXext, \
+ LIBS_linux := $(X_LIBS) -lX11 -lXext, \
LIBS_macosx := \
-framework ApplicationServices \
-framework Cocoa \
diff --git a/make/modules/java.instrument/Lib.gmk b/make/modules/java.instrument/Lib.gmk
index 609814c86ed..4181cdf81c9 100644
--- a/make/modules/java.instrument/Lib.gmk
+++ b/make/modules/java.instrument/Lib.gmk
@@ -43,8 +43,6 @@ $(eval $(call SetupJdkLibrary, BUILD_LIBINSTRUMENT, \
JDK_LIBS := java.base:libjava java.base:libjli java.base:libjvm, \
LIBS := $(ICONV_LIBS), \
LIBS_unix := $(LIBZ_LIBS), \
- LIBS_linux := $(LIBDL), \
- LIBS_aix := $(LIBDL), \
LIBS_macosx := \
-framework ApplicationServices \
-framework Cocoa \
diff --git a/make/modules/java.scripting/Java.gmk b/make/modules/java.scripting/Java.gmk
index 1909ec1a2df..6e5ea0e2c73 100644
--- a/make/modules/java.scripting/Java.gmk
+++ b/make/modules/java.scripting/Java.gmk
@@ -27,7 +27,5 @@
DOCLINT += -Xdoclint:all/protected \
'-Xdoclint/package:java.*,javax.*'
-COPY += .js
-CLEAN += .properties
################################################################################
diff --git a/make/modules/java.scripting/Launcher.gmk b/make/modules/java.scripting/Launcher.gmk
deleted file mode 100644
index ee6b93fbf2c..00000000000
--- a/make/modules/java.scripting/Launcher.gmk
+++ /dev/null
@@ -1,39 +0,0 @@
-#
-# Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved.
-# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-#
-# This code is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License version 2 only, as
-# published by the Free Software Foundation. Oracle designates this
-# particular file as subject to the "Classpath" exception as provided
-# by Oracle in the LICENSE file that accompanied this code.
-#
-# This code is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# version 2 for more details (a copy is included in the LICENSE file that
-# accompanied this code).
-#
-# You should have received a copy of the GNU General Public License version
-# 2 along with this work; if not, write to the Free Software Foundation,
-# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
-# or visit www.oracle.com if you need additional information or have any
-# questions.
-#
-
-################################################################################
-
-include LauncherCommon.gmk
-
-################################################################################
-## Build jrunscript
-################################################################################
-
-$(eval $(call SetupBuildLauncher, jrunscript, \
- MAIN_CLASS := com.sun.tools.script.shell.Main, \
- JAVA_ARGS := --add-modules ALL-DEFAULT, \
-))
-
-################################################################################
diff --git a/make/modules/jdk.sctp/Lib.gmk b/make/modules/jdk.sctp/Lib.gmk
index 327f4fa6104..9a70996d9cb 100644
--- a/make/modules/jdk.sctp/Lib.gmk
+++ b/make/modules/jdk.sctp/Lib.gmk
@@ -41,7 +41,7 @@ ifeq ($(call isTargetOs, linux), true)
java.base:libnio \
java.base:libnio/ch, \
JDK_LIBS := java.base:libjava java.base:libnet, \
- LIBS_linux := $(LIBDL) $(LIBPTHREAD), \
+ LIBS_linux := $(LIBDL), \
))
TARGETS += $(BUILD_LIBSCTP)
diff --git a/make/scripts/addNotices.sh b/make/scripts/addNotices.sh
deleted file mode 100644
index d9864818a14..00000000000
--- a/make/scripts/addNotices.sh
+++ /dev/null
@@ -1,45 +0,0 @@
-#! /bin/sh
-#
-# Copyright (c) 2007, 2020, Oracle and/or its affiliates. All rights reserved.
-# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-#
-# This code is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License version 2 only, as
-# published by the Free Software Foundation. Oracle designates this
-# particular file as subject to the "Classpath" exception as provided
-# by Oracle in the LICENSE file that accompanied this code.
-#
-# This code is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# version 2 for more details (a copy is included in the LICENSE file that
-# accompanied this code).
-#
-# You should have received a copy of the GNU General Public License version
-# 2 along with this work; if not, write to the Free Software Foundation,
-# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
-# or visit www.oracle.com if you need additional information or have any
-# questions.
-#
-
-# Parse the first contiguous comment block in this script and generate
-# a java comment block. If this script is invoked with a copyright
-# year/year range, the java comment block will contain a Sun copyright.
-
-COPYRIGHT_YEARS="$1"
-
-cat <<__END__
-/*
-__END__
-
-if [ "x$COPYRIGHT_YEARS" != x ]; then
- cat <<__END__
- * Copyright (c) $COPYRIGHT_YEARS Oracle and/or its affiliates. All rights reserved.
-__END__
-fi
-
-$AWK ' /^#.*Copyright.*Oracle/ { next }
- /^#([^!]|$)/ { sub(/^#/, " *"); print }
- /^$/ { print " */"; exit } ' $0
diff --git a/make/scripts/genExceptions.sh b/make/scripts/genExceptions.sh
deleted file mode 100644
index 7c191189827..00000000000
--- a/make/scripts/genExceptions.sh
+++ /dev/null
@@ -1,116 +0,0 @@
-#! /bin/sh
-#
-# Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
-# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-#
-# This code is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License version 2 only, as
-# published by the Free Software Foundation. Oracle designates this
-# particular file as subject to the "Classpath" exception as provided
-# by Oracle in the LICENSE file that accompanied this code.
-#
-# This code is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# version 2 for more details (a copy is included in the LICENSE file that
-# accompanied this code).
-#
-# You should have received a copy of the GNU General Public License version
-# 2 along with this work; if not, write to the Free Software Foundation,
-# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
-# or visit www.oracle.com if you need additional information or have any
-# questions.
-#
-
-# Generate exception classes
-
-SPEC=$1
-DST=$2
-
-gen() {
- ID=$1
- WHAT=$2
- SVUID=$3
- ARG_TYPE=$4
- ARG_ID=$5
- ARG_PROP=$6
- ARG_PHRASE=$7
- ARG_PARAM="$ARG_TYPE$ $ARG_ID"
- echo '-->' $DST/$ID.java
- out=$DST/${ID}.java
-
- $SH ${SCRIPTS}/addNotices.sh "$COPYRIGHT_YEARS" > $out
-
- cat >>$out <<__END__
-
-// -- This file was mechanically generated: Do not edit! -- //
-
-package $PACKAGE;
-
-
-/**$WHAT
- *
- * @since $SINCE
- */
-
-public `if [ ${ABSTRACT:-0} = 1 ];
- then echo 'abstract '; fi`class $ID
- extends ${SUPER}
-{
-
- @java.io.Serial
- private static final long serialVersionUID = $SVUID;
-__END__
-
- if [ $ARG_ID ]; then
-
- cat >>$out <<__END__
-
- /**
- * The $ARG_PHRASE.
- *
- * @serial
- */
- private $ARG_TYPE $ARG_ID;
-
- /**
- * Constructs an instance of this class.
- *
- * @param $ARG_ID
- * The $ARG_PHRASE
- */
- public $ID($ARG_TYPE $ARG_ID) {
- super(String.valueOf($ARG_ID));
- this.$ARG_ID = $ARG_ID;
- }
-
- /**
- * Retrieves the $ARG_PHRASE.
- *
- * @return The $ARG_PHRASE
- */
- public $ARG_TYPE get$ARG_PROP() {
- return $ARG_ID;
- }
-
-}
-__END__
-
- else
-
- cat >>$out <<__END__
-
- /**
- * Constructs an instance of this class.
- */
- public $ID() { }
-
-}
-__END__
-
- fi
-}
-
-. $SPEC
diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad
index e0459716122..51cdf8c71df 100644
--- a/src/hotspot/cpu/aarch64/aarch64.ad
+++ b/src/hotspot/cpu/aarch64/aarch64.ad
@@ -2568,10 +2568,6 @@ RegMask Matcher::modL_proj_mask() {
return RegMask();
}
-const RegMask Matcher::method_handle_invoke_SP_save_mask() {
- return FP_REG_mask();
-}
-
bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
Node* u = addp->fast_out(i);
diff --git a/src/hotspot/cpu/aarch64/aarch64_vector.ad b/src/hotspot/cpu/aarch64/aarch64_vector.ad
index 67c4dad27a7..ef35b66003d 100644
--- a/src/hotspot/cpu/aarch64/aarch64_vector.ad
+++ b/src/hotspot/cpu/aarch64/aarch64_vector.ad
@@ -216,11 +216,6 @@ source %{
return false;
}
break;
- case Op_ExpandV:
- if (UseSVE < 2 || is_subword_type(bt)) {
- return false;
- }
- break;
case Op_VectorMaskToLong:
if (UseSVE > 0 && vlen > 64) {
return false;
@@ -7113,10 +7108,39 @@ instruct vcompressS(vReg dst, vReg src, pReg pg,
ins_pipe(pipe_slow);
%}
-instruct vexpand(vReg dst, vReg src, pRegGov pg) %{
+instruct vexpand_neon(vReg dst, vReg src, vReg mask, vReg tmp1, vReg tmp2) %{
+ predicate(UseSVE == 0);
+ match(Set dst (ExpandV src mask));
+ effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2);
+ format %{ "vexpand_neon $dst, $src, $mask\t# KILL $tmp1, $tmp2" %}
+ ins_encode %{
+ BasicType bt = Matcher::vector_element_basic_type(this);
+ int length_in_bytes = (int) Matcher::vector_length_in_bytes(this);
+ __ vector_expand_neon($dst$$FloatRegister, $src$$FloatRegister, $mask$$FloatRegister,
+ $tmp1$$FloatRegister, $tmp2$$FloatRegister, bt, length_in_bytes);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct vexpand_sve(vReg dst, vReg src, pRegGov pg, vReg tmp1, vReg tmp2) %{
+ predicate(UseSVE == 1 || (UseSVE == 2 && type2aelembytes(Matcher::vector_element_basic_type(n)) < 4));
+ match(Set dst (ExpandV src pg));
+ effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2);
+ format %{ "vexpand_sve $dst, $src, $pg\t# KILL $tmp1, $tmp2" %}
+ ins_encode %{
+ BasicType bt = Matcher::vector_element_basic_type(this);
+ int length_in_bytes = (int) Matcher::vector_length_in_bytes(this);
+ __ vector_expand_sve($dst$$FloatRegister, $src$$FloatRegister, $pg$$PRegister,
+ $tmp1$$FloatRegister, $tmp2$$FloatRegister, bt, length_in_bytes);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct vexpand_sve2_SD(vReg dst, vReg src, pRegGov pg) %{
+ predicate(UseSVE == 2 && type2aelembytes(Matcher::vector_element_basic_type(n)) >= 4);
match(Set dst (ExpandV src pg));
effect(TEMP_DEF dst);
- format %{ "vexpand $dst, $pg, $src" %}
+ format %{ "vexpand_sve2_SD $dst, $src, $pg" %}
ins_encode %{
// Example input: src = 1 2 3 4 5 6 7 8
// pg = 1 0 0 1 1 0 1 1
@@ -7127,7 +7151,6 @@ instruct vexpand(vReg dst, vReg src, pRegGov pg) %{
// for TBL whose value is used to select the indexed element from src vector.
BasicType bt = Matcher::vector_element_basic_type(this);
- assert(UseSVE == 2 && !is_subword_type(bt), "unsupported");
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
// dst = 0 0 0 0 0 0 0 0
__ sve_dup($dst$$FloatRegister, size, 0);
diff --git a/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 b/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4
index 28f91204ec3..012de7e46d8 100644
--- a/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4
+++ b/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4
@@ -206,11 +206,6 @@ source %{
return false;
}
break;
- case Op_ExpandV:
- if (UseSVE < 2 || is_subword_type(bt)) {
- return false;
- }
- break;
case Op_VectorMaskToLong:
if (UseSVE > 0 && vlen > 64) {
return false;
@@ -5101,10 +5096,39 @@ instruct vcompressS(vReg dst, vReg src, pReg pg,
ins_pipe(pipe_slow);
%}
-instruct vexpand(vReg dst, vReg src, pRegGov pg) %{
+instruct vexpand_neon(vReg dst, vReg src, vReg mask, vReg tmp1, vReg tmp2) %{
+ predicate(UseSVE == 0);
+ match(Set dst (ExpandV src mask));
+ effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2);
+ format %{ "vexpand_neon $dst, $src, $mask\t# KILL $tmp1, $tmp2" %}
+ ins_encode %{
+ BasicType bt = Matcher::vector_element_basic_type(this);
+ int length_in_bytes = (int) Matcher::vector_length_in_bytes(this);
+ __ vector_expand_neon($dst$$FloatRegister, $src$$FloatRegister, $mask$$FloatRegister,
+ $tmp1$$FloatRegister, $tmp2$$FloatRegister, bt, length_in_bytes);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct vexpand_sve(vReg dst, vReg src, pRegGov pg, vReg tmp1, vReg tmp2) %{
+ predicate(UseSVE == 1 || (UseSVE == 2 && type2aelembytes(Matcher::vector_element_basic_type(n)) < 4));
+ match(Set dst (ExpandV src pg));
+ effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2);
+ format %{ "vexpand_sve $dst, $src, $pg\t# KILL $tmp1, $tmp2" %}
+ ins_encode %{
+ BasicType bt = Matcher::vector_element_basic_type(this);
+ int length_in_bytes = (int) Matcher::vector_length_in_bytes(this);
+ __ vector_expand_sve($dst$$FloatRegister, $src$$FloatRegister, $pg$$PRegister,
+ $tmp1$$FloatRegister, $tmp2$$FloatRegister, bt, length_in_bytes);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct vexpand_sve2_SD(vReg dst, vReg src, pRegGov pg) %{
+ predicate(UseSVE == 2 && type2aelembytes(Matcher::vector_element_basic_type(n)) >= 4);
match(Set dst (ExpandV src pg));
effect(TEMP_DEF dst);
- format %{ "vexpand $dst, $pg, $src" %}
+ format %{ "vexpand_sve2_SD $dst, $src, $pg" %}
ins_encode %{
// Example input: src = 1 2 3 4 5 6 7 8
// pg = 1 0 0 1 1 0 1 1
@@ -5115,7 +5139,6 @@ instruct vexpand(vReg dst, vReg src, pRegGov pg) %{
// for TBL whose value is used to select the indexed element from src vector.
BasicType bt = Matcher::vector_element_basic_type(this);
- assert(UseSVE == 2 && !is_subword_type(bt), "unsupported");
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
// dst = 0 0 0 0 0 0 0 0
__ sve_dup($dst$$FloatRegister, size, 0);
diff --git a/src/hotspot/cpu/aarch64/assembler_aarch64.hpp b/src/hotspot/cpu/aarch64/assembler_aarch64.hpp
index a5d2cbfac98..4c4251fbe9f 100644
--- a/src/hotspot/cpu/aarch64/assembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/assembler_aarch64.hpp
@@ -4068,6 +4068,13 @@ public:
INSN(sve_brkb, 0b10); // Break before first true condition
#undef INSN
+ // SVE move prefix (unpredicated)
+ void sve_movprfx(FloatRegister Zd, FloatRegister Zn) {
+ starti;
+ f(0b00000100, 31, 24), f(0b00, 23, 22), f(0b1, 21), f(0b00000, 20, 16);
+ f(0b101111, 15, 10), rf(Zn, 5), rf(Zd, 0);
+ }
+
// Element count and increment scalar (SVE)
#define INSN(NAME, TYPE) \
void NAME(Register Xdn, unsigned imm4 = 1, int pattern = 0b11111) { \
diff --git a/src/hotspot/cpu/aarch64/assembler_aarch64.inline.hpp b/src/hotspot/cpu/aarch64/assembler_aarch64.inline.hpp
index e7efe472b82..fb14d588f04 100644
--- a/src/hotspot/cpu/aarch64/assembler_aarch64.inline.hpp
+++ b/src/hotspot/cpu/aarch64/assembler_aarch64.inline.hpp
@@ -26,7 +26,7 @@
#ifndef CPU_AARCH64_ASSEMBLER_AARCH64_INLINE_HPP
#define CPU_AARCH64_ASSEMBLER_AARCH64_INLINE_HPP
-#include "asm/assembler.inline.hpp"
+#include "asm/assembler.hpp"
#include "asm/codeBuffer.hpp"
#include "code/codeCache.hpp"
diff --git a/src/hotspot/cpu/aarch64/c1_CodeStubs_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_CodeStubs_aarch64.cpp
index 954e4abee14..9bf46678535 100644
--- a/src/hotspot/cpu/aarch64/c1_CodeStubs_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c1_CodeStubs_aarch64.cpp
@@ -216,10 +216,10 @@ void MonitorEnterStub::emit_code(LIR_Assembler* ce) {
void MonitorExitStub::emit_code(LIR_Assembler* ce) {
__ bind(_entry);
- if (_compute_lock) {
- // lock_reg was destroyed by fast unlocking attempt => recompute it
- ce->monitor_address(_monitor_ix, _lock_reg);
- }
+
+ // lock_reg was destroyed by fast unlocking attempt => recompute it
+ ce->monitor_address(_monitor_ix, _lock_reg);
+
ce->store_parameter(_lock_reg->as_register(), 0);
// note: non-blocking leaf routine => no call info needed
StubId exit_id;
diff --git a/src/hotspot/cpu/aarch64/c1_FrameMap_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_FrameMap_aarch64.cpp
index 9d30092b45a..83d0952dcb4 100644
--- a/src/hotspot/cpu/aarch64/c1_FrameMap_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c1_FrameMap_aarch64.cpp
@@ -383,13 +383,6 @@ LIR_Opr FrameMap::stack_pointer() {
return FrameMap::sp_opr;
}
-
-// JSR 292
-LIR_Opr FrameMap::method_handle_invoke_SP_save_opr() {
- return LIR_OprFact::illegalOpr; // Not needed on aarch64
-}
-
-
bool FrameMap::validate_frame() {
return true;
}
diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
index d788c0c201a..9ab463125fe 100644
--- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
@@ -409,7 +409,7 @@ int LIR_Assembler::emit_unwind_handler() {
MonitorExitStub* stub = nullptr;
if (method()->is_synchronized()) {
monitor_address(0, FrameMap::r0_opr);
- stub = new MonitorExitStub(FrameMap::r0_opr, true, 0);
+ stub = new MonitorExitStub(FrameMap::r0_opr, 0);
__ unlock_object(r5, r4, r0, r6, *stub->entry());
__ bind(*stub->continuation());
}
@@ -2481,7 +2481,6 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
Register lock = op->lock_opr()->as_register();
Register temp = op->scratch_opr()->as_register();
if (op->code() == lir_lock) {
- assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
// add debug info for NullPointerException only if one is possible
int null_check_offset = __ lock_object(hdr, obj, lock, temp, *op->stub()->entry());
if (op->info() != nullptr) {
@@ -2489,7 +2488,6 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
}
// done
} else if (op->code() == lir_unlock) {
- assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
__ unlock_object(hdr, obj, lock, temp, *op->stub()->entry());
} else {
Unimplemented();
diff --git a/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp
index 8a79274b2ff..31c36e749c5 100644
--- a/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp
@@ -59,28 +59,28 @@ void C1_MacroAssembler::float_cmp(bool is_float, int unordered_result,
}
}
-int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Register temp, Label& slow_case) {
- assert_different_registers(hdr, obj, disp_hdr, temp, rscratch2);
+int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register basic_lock, Register temp, Label& slow_case) {
+ assert_different_registers(hdr, obj, basic_lock, temp, rscratch2);
int null_check_offset = -1;
verify_oop(obj);
// save object being locked into the BasicObjectLock
- str(obj, Address(disp_hdr, BasicObjectLock::obj_offset()));
+ str(obj, Address(basic_lock, BasicObjectLock::obj_offset()));
null_check_offset = offset();
- lightweight_lock(disp_hdr, obj, hdr, temp, rscratch2, slow_case);
+ lightweight_lock(basic_lock, obj, hdr, temp, rscratch2, slow_case);
return null_check_offset;
}
-void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_hdr, Register temp, Label& slow_case) {
- assert_different_registers(hdr, obj, disp_hdr, temp, rscratch2);
+void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register basic_lock, Register temp, Label& slow_case) {
+ assert_different_registers(hdr, obj, basic_lock, temp, rscratch2);
// load object
- ldr(obj, Address(disp_hdr, BasicObjectLock::obj_offset()));
+ ldr(obj, Address(basic_lock, BasicObjectLock::obj_offset()));
verify_oop(obj);
lightweight_unlock(obj, hdr, temp, rscratch2, slow_case);
diff --git a/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.hpp
index fc8e83d706b..7b181b104c1 100644
--- a/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -55,19 +55,19 @@ using MacroAssembler::null_check;
Register result);
// locking
- // hdr : must be r0, contents destroyed
- // obj : must point to the object to lock, contents preserved
- // disp_hdr: must point to the displaced header location, contents preserved
- // temp : temporary register, must not be rscratch1 or rscratch2
+ // hdr : must be r0, contents destroyed
+ // obj : must point to the object to lock, contents preserved
+ // basic_lock: must point to the basic lock, contents preserved
+ // temp : temporary register, must not be rscratch1 or rscratch2
// returns code offset at which to add null check debug information
- int lock_object (Register swap, Register obj, Register disp_hdr, Register temp, Label& slow_case);
+ int lock_object (Register swap, Register obj, Register basic_lock, Register temp, Label& slow_case);
// unlocking
- // hdr : contents destroyed
- // obj : must point to the object to lock, contents preserved
- // disp_hdr: must be r0 & must point to the displaced header location, contents destroyed
- // temp : temporary register, must not be rscratch1 or rscratch2
- void unlock_object(Register swap, Register obj, Register lock, Register temp, Label& slow_case);
+ // hdr : contents destroyed
+ // obj : must point to the object to lock, contents preserved
+ // basic_lock: must be r0 & must point to the basic lock, contents destroyed
+ // temp : temporary register, must not be rscratch1 or rscratch2
+ void unlock_object(Register swap, Register obj, Register basic_lock, Register temp, Label& slow_case);
void initialize_object(
Register obj, // result: pointer to object after successful allocation
diff --git a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
index b1562c54f4e..b61a0e4e378 100644
--- a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
@@ -2771,3 +2771,90 @@ void C2_MacroAssembler::select_from_two_vectors(FloatRegister dst, FloatRegister
select_from_two_vectors_neon(dst, src1, src2, dst, tmp, vector_length_in_bytes);
}
}
+
+// Vector expand implementation. Elements from the src vector are expanded into
+// the dst vector under the control of the vector mask.
+// Since there are no native instructions directly corresponding to expand before
+// SVE2p2, the following implementations mainly leverages the TBL instruction to
+// implement expand. To compute the index input for TBL, the prefix sum algorithm
+// (https://en.wikipedia.org/wiki/Prefix_sum) is used. The same algorithm is used
+// for NEON and SVE, but with different instructions where appropriate.
+
+// Vector expand implementation for NEON.
+//
+// An example of 128-bit Byte vector:
+// Data direction: high <== low
+// Input:
+// src = g f e d c b a 9 8 7 6 5 4 3 2 1
+// mask = 0 0 -1 -1 0 0 -1 -1 0 0 -1 -1 0 0 -1 -1
+// Expected result:
+// dst = 0 0 8 7 0 0 6 5 0 0 4 3 0 0 2 1
+void C2_MacroAssembler::vector_expand_neon(FloatRegister dst, FloatRegister src, FloatRegister mask,
+ FloatRegister tmp1, FloatRegister tmp2, BasicType bt,
+ int vector_length_in_bytes) {
+ assert(vector_length_in_bytes <= 16, "the vector length in bytes for NEON must be <= 16");
+ assert_different_registers(dst, src, mask, tmp1, tmp2);
+ // Since the TBL instruction only supports byte table, we need to
+ // compute indices in byte type for all types.
+ SIMD_Arrangement size = vector_length_in_bytes == 16 ? T16B : T8B;
+ // tmp1 = 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
+ dup(tmp1, size, zr);
+ // dst = 0 0 1 1 0 0 1 1 0 0 1 1 0 0 1 1
+ negr(dst, size, mask);
+ // Calculate vector index for TBL with prefix sum algorithm.
+ // dst = 8 8 8 7 6 6 6 5 4 4 4 3 2 2 2 1
+ for (int i = 1; i < vector_length_in_bytes; i <<= 1) {
+ ext(tmp2, size, tmp1, dst, vector_length_in_bytes - i);
+ addv(dst, size, tmp2, dst);
+ }
+ // tmp2 = 0 0 -1 -1 0 0 -1 -1 0 0 -1 -1 0 0 -1 -1
+ orr(tmp2, size, mask, mask);
+ // tmp2 = 0 0 8 7 0 0 6 5 0 0 4 3 0 0 2 1
+ bsl(tmp2, size, dst, tmp1);
+ // tmp1 = 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1
+ movi(tmp1, size, 1);
+ // dst = -1 -1 7 6 -1 -1 5 4 -1 -1 3 2 -1 -1 1 0
+ subv(dst, size, tmp2, tmp1);
+ // dst = 0 0 8 7 0 0 6 5 0 0 4 3 0 0 2 1
+ tbl(dst, size, src, 1, dst);
+}
+
+// Vector expand implementation for SVE.
+//
+// An example of 128-bit Short vector:
+// Data direction: high <== low
+// Input:
+// src = gf ed cb a9 87 65 43 21
+// pg = 00 01 00 01 00 01 00 01
+// Expected result:
+// dst = 00 87 00 65 00 43 00 21
+void C2_MacroAssembler::vector_expand_sve(FloatRegister dst, FloatRegister src, PRegister pg,
+ FloatRegister tmp1, FloatRegister tmp2, BasicType bt,
+ int vector_length_in_bytes) {
+ assert(UseSVE > 0, "expand implementation only for SVE");
+ assert_different_registers(dst, src, tmp1, tmp2);
+ SIMD_RegVariant size = elemType_to_regVariant(bt);
+
+ // tmp1 = 00 00 00 00 00 00 00 00
+ sve_dup(tmp1, size, 0);
+ sve_movprfx(tmp2, tmp1);
+ // tmp2 = 00 01 00 01 00 01 00 01
+ sve_cpy(tmp2, size, pg, 1, true);
+ // Calculate vector index for TBL with prefix sum algorithm.
+ // tmp2 = 04 04 03 03 02 02 01 01
+ for (int i = type2aelembytes(bt); i < vector_length_in_bytes; i <<= 1) {
+ sve_movprfx(dst, tmp1);
+ // The EXT instruction operates on the full-width sve register. The correct
+ // index calculation method is:
+ // vector_length_in_bytes - i + MaxVectorSize - vector_length_in_bytes =>
+ // MaxVectorSize - i.
+ sve_ext(dst, tmp2, MaxVectorSize - i);
+ sve_add(tmp2, size, dst, tmp2);
+ }
+ // dst = 00 04 00 03 00 02 00 01
+ sve_sel(dst, size, pg, tmp2, tmp1);
+ // dst = -1 03 -1 02 -1 01 -1 00
+ sve_sub(dst, size, 1);
+ // dst = 00 87 00 65 00 43 00 21
+ sve_tbl(dst, size, src, dst);
+}
\ No newline at end of file
diff --git a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp
index 0403a27910f..cb8ded142f4 100644
--- a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp
@@ -204,4 +204,10 @@
FloatRegister index, FloatRegister tmp, BasicType bt,
unsigned vector_length_in_bytes);
+ void vector_expand_neon(FloatRegister dst, FloatRegister src, FloatRegister mask,
+ FloatRegister tmp1, FloatRegister tmp2, BasicType bt,
+ int vector_length_in_bytes);
+ void vector_expand_sve(FloatRegister dst, FloatRegister src, PRegister pg,
+ FloatRegister tmp1, FloatRegister tmp2, BasicType bt,
+ int vector_length_in_bytes);
#endif // CPU_AARCH64_C2_MACROASSEMBLER_AARCH64_HPP
diff --git a/src/hotspot/cpu/aarch64/frame_aarch64.cpp b/src/hotspot/cpu/aarch64/frame_aarch64.cpp
index aff50b9cf2f..bdbef53bfdb 100644
--- a/src/hotspot/cpu/aarch64/frame_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/frame_aarch64.cpp
@@ -228,8 +228,7 @@ bool frame::safe_for_sender(JavaThread *thread) {
nmethod* nm = sender_blob->as_nmethod_or_null();
if (nm != nullptr) {
- if (nm->is_deopt_mh_entry(sender_pc) || nm->is_deopt_entry(sender_pc) ||
- nm->method()->is_method_handle_intrinsic()) {
+ if (nm->is_deopt_entry(sender_pc) || nm->method()->is_method_handle_intrinsic()) {
return false;
}
}
@@ -454,48 +453,6 @@ JavaThread** frame::saved_thread_address(const frame& f) {
return thread_addr;
}
-//------------------------------------------------------------------------------
-// frame::verify_deopt_original_pc
-//
-// Verifies the calculated original PC of a deoptimization PC for the
-// given unextended SP.
-#ifdef ASSERT
-void frame::verify_deopt_original_pc(nmethod* nm, intptr_t* unextended_sp) {
- frame fr;
-
- // This is ugly but it's better than to change {get,set}_original_pc
- // to take an SP value as argument. And it's only a debugging
- // method anyway.
- fr._unextended_sp = unextended_sp;
-
- address original_pc = nm->get_original_pc(&fr);
- assert(nm->insts_contains_inclusive(original_pc),
- "original PC must be in the main code section of the compiled method (or must be immediately following it)");
-}
-#endif
-
-//------------------------------------------------------------------------------
-// frame::adjust_unextended_sp
-#ifdef ASSERT
-void frame::adjust_unextended_sp() {
- // On aarch64, sites calling method handle intrinsics and lambda forms are treated
- // as any other call site. Therefore, no special action is needed when we are
- // returning to any of these call sites.
-
- if (_cb != nullptr) {
- nmethod* sender_nm = _cb->as_nmethod_or_null();
- if (sender_nm != nullptr) {
- // If the sender PC is a deoptimization point, get the original PC.
- if (sender_nm->is_deopt_entry(_pc) ||
- sender_nm->is_deopt_mh_entry(_pc)) {
- verify_deopt_original_pc(sender_nm, _unextended_sp);
- }
- }
- }
-}
-#endif
-
-
//------------------------------------------------------------------------------
// frame::sender_for_interpreter_frame
frame frame::sender_for_interpreter_frame(RegisterMap* map) const {
diff --git a/src/hotspot/cpu/aarch64/frame_aarch64.hpp b/src/hotspot/cpu/aarch64/frame_aarch64.hpp
index da020b4234d..231710df7d7 100644
--- a/src/hotspot/cpu/aarch64/frame_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/frame_aarch64.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -141,8 +141,6 @@
int _offset_unextended_sp; // for use in stack-chunk frames
};
- void adjust_unextended_sp() NOT_DEBUG_RETURN;
-
// true means _sp value is correct and we can use it to get the sender's sp
// of the compiled frame, otherwise, _sp value may be invalid and we can use
// _fp to get the sender's sp if PreserveFramePointer is enabled.
@@ -152,11 +150,6 @@
return (intptr_t*) addr_at(offset);
}
-#ifdef ASSERT
- // Used in frame::sender_for_{interpreter,compiled}_frame
- static void verify_deopt_original_pc(nmethod* nm, intptr_t* unextended_sp);
-#endif
-
public:
// Constructors
diff --git a/src/hotspot/cpu/aarch64/frame_aarch64.inline.hpp b/src/hotspot/cpu/aarch64/frame_aarch64.inline.hpp
index 47ae93a4932..cb53d8663ad 100644
--- a/src/hotspot/cpu/aarch64/frame_aarch64.inline.hpp
+++ b/src/hotspot/cpu/aarch64/frame_aarch64.inline.hpp
@@ -116,8 +116,6 @@ inline void frame::init(intptr_t* sp, intptr_t* fp, address pc) {
}
inline void frame::setup(address pc) {
- adjust_unextended_sp();
-
address original_pc = get_deopt_original_pc();
if (original_pc != nullptr) {
_pc = original_pc;
@@ -223,7 +221,6 @@ inline frame::frame(intptr_t* sp, intptr_t* fp) {
// assert(_pc != nullptr, "no pc?");
_cb = CodeCache::find_blob(_pc);
- adjust_unextended_sp();
address original_pc = get_deopt_original_pc();
if (original_pc != nullptr) {
diff --git a/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp
index 42f3c4a015a..974527f30e9 100644
--- a/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp
@@ -86,15 +86,48 @@ void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm
}
}
-void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
- Register start, Register count, Register scratch, RegSet saved_regs) {
- __ push(saved_regs, sp);
- assert_different_registers(start, count, scratch);
- assert_different_registers(c_rarg0, count);
- __ mov(c_rarg0, start);
- __ mov(c_rarg1, count);
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_post_entry), 2);
- __ pop(saved_regs, sp);
+void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm,
+ DecoratorSet decorators,
+ Register start,
+ Register count,
+ Register scratch,
+ RegSet saved_regs) {
+
+ Label done;
+ Label loop;
+ Label next;
+
+ __ cbz(count, done);
+
+ // Calculate the number of card marks to set. Since the object might start and
+ // end within a card, we need to calculate this via the card table indexes of
+ // the actual start and last addresses covered by the object.
+ // Temporarily use the count register for the last element address.
+ __ lea(count, Address(start, count, Address::lsl(LogBytesPerHeapOop))); // end = start + count << LogBytesPerHeapOop
+ __ sub(count, count, BytesPerHeapOop); // Use last element address for end.
+
+ __ lsr(start, start, CardTable::card_shift());
+ __ lsr(count, count, CardTable::card_shift());
+ __ sub(count, count, start); // Number of bytes to mark - 1.
+
+ // Add card table base offset to start.
+ __ ldr(scratch, Address(rthread, in_bytes(G1ThreadLocalData::card_table_base_offset())));
+ __ add(start, start, scratch);
+
+ __ bind(loop);
+ if (UseCondCardMark) {
+ __ ldrb(scratch, Address(start, count));
+ // Instead of loading clean_card_val and comparing, we exploit the fact that
+ // the LSB of non-clean cards is always 0, and the LSB of clean cards 1.
+ __ tbz(scratch, 0, next);
+ }
+ static_assert(G1CardTable::dirty_card_val() == 0, "must be to use zr");
+ __ strb(zr, Address(start, count));
+ __ bind(next);
+ __ subs(count, count, 1);
+ __ br(Assembler::GE, loop);
+
+ __ bind(done);
}
static void generate_queue_test_and_insertion(MacroAssembler* masm, ByteSize index_offset, ByteSize buffer_offset, Label& runtime,
@@ -199,13 +232,17 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
}
-static void generate_post_barrier_fast_path(MacroAssembler* masm,
- const Register store_addr,
- const Register new_val,
- const Register tmp1,
- const Register tmp2,
- Label& done,
- bool new_val_may_be_null) {
+static void generate_post_barrier(MacroAssembler* masm,
+ const Register store_addr,
+ const Register new_val,
+ const Register thread,
+ const Register tmp1,
+ const Register tmp2,
+ Label& done,
+ bool new_val_may_be_null) {
+ assert(thread == rthread, "must be");
+ assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, noreg, rscratch1);
+
// Does store cross heap regions?
__ eor(tmp1, store_addr, new_val); // tmp1 := store address ^ new value
__ lsr(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes); // tmp1 := ((store address ^ new value) >> LogOfHRGrainBytes)
@@ -214,33 +251,19 @@ static void generate_post_barrier_fast_path(MacroAssembler* masm,
if (new_val_may_be_null) {
__ cbz(new_val, done);
}
- // Storing region crossing non-null, is card young?
+ // Storing region crossing non-null.
__ lsr(tmp1, store_addr, CardTable::card_shift()); // tmp1 := card address relative to card table base
- __ load_byte_map_base(tmp2); // tmp2 := card table base address
- __ add(tmp1, tmp1, tmp2); // tmp1 := card address
- __ ldrb(tmp2, Address(tmp1)); // tmp2 := card
- __ cmpw(tmp2, (int)G1CardTable::g1_young_card_val()); // tmp2 := card == young_card_val?
-}
-static void generate_post_barrier_slow_path(MacroAssembler* masm,
- const Register thread,
- const Register tmp1,
- const Register tmp2,
- Label& done,
- Label& runtime) {
- __ membar(Assembler::StoreLoad); // StoreLoad membar
- __ ldrb(tmp2, Address(tmp1)); // tmp2 := card
- __ cbzw(tmp2, done);
- // Storing a region crossing, non-null oop, card is clean.
- // Dirty card and log.
- STATIC_ASSERT(CardTable::dirty_card_val() == 0);
- __ strb(zr, Address(tmp1)); // *(card address) := dirty_card_val
- generate_queue_test_and_insertion(masm,
- G1ThreadLocalData::dirty_card_queue_index_offset(),
- G1ThreadLocalData::dirty_card_queue_buffer_offset(),
- runtime,
- thread, tmp1, tmp2, rscratch1);
- __ b(done);
+ Address card_table_addr(thread, in_bytes(G1ThreadLocalData::card_table_base_offset()));
+ __ ldr(tmp2, card_table_addr); // tmp2 := card table base address
+ if (UseCondCardMark) {
+ __ ldrb(rscratch1, Address(tmp1, tmp2)); // rscratch1 := card
+ // Instead of loading clean_card_val and comparing, we exploit the fact that
+ // the LSB of non-clean cards is always 0, and the LSB of clean cards 1.
+ __ tbz(rscratch1, 0, done);
+ }
+ static_assert(G1CardTable::dirty_card_val() == 0, "must be to use zr");
+ __ strb(zr, Address(tmp1, tmp2)); // *(card address) := dirty_card_val
}
void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
@@ -249,27 +272,8 @@ void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
Register thread,
Register tmp1,
Register tmp2) {
- assert(thread == rthread, "must be");
- assert_different_registers(store_addr, new_val, thread, tmp1, tmp2,
- rscratch1);
- assert(store_addr != noreg && new_val != noreg && tmp1 != noreg
- && tmp2 != noreg, "expecting a register");
-
Label done;
- Label runtime;
-
- generate_post_barrier_fast_path(masm, store_addr, new_val, tmp1, tmp2, done, true /* new_val_may_be_null */);
- // If card is young, jump to done
- __ br(Assembler::EQ, done);
- generate_post_barrier_slow_path(masm, thread, tmp1, tmp2, done, runtime);
-
- __ bind(runtime);
- // save the live input values
- RegSet saved = RegSet::of(store_addr);
- __ push(saved, sp);
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), tmp1, thread);
- __ pop(saved, sp);
-
+ generate_post_barrier(masm, store_addr, new_val, thread, tmp1, tmp2, done, false /* new_val_may_be_null */);
__ bind(done);
}
@@ -329,38 +333,10 @@ void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm,
Register thread,
Register tmp1,
Register tmp2,
- G1PostBarrierStubC2* stub) {
- assert(thread == rthread, "must be");
- assert_different_registers(store_addr, new_val, thread, tmp1, tmp2,
- rscratch1);
- assert(store_addr != noreg && new_val != noreg && tmp1 != noreg
- && tmp2 != noreg, "expecting a register");
-
- stub->initialize_registers(thread, tmp1, tmp2);
-
- bool new_val_may_be_null = (stub->barrier_data() & G1C2BarrierPostNotNull) == 0;
- generate_post_barrier_fast_path(masm, store_addr, new_val, tmp1, tmp2, *stub->continuation(), new_val_may_be_null);
- // If card is not young, jump to stub (slow path)
- __ br(Assembler::NE, *stub->entry());
-
- __ bind(*stub->continuation());
-}
-
-void G1BarrierSetAssembler::generate_c2_post_barrier_stub(MacroAssembler* masm,
- G1PostBarrierStubC2* stub) const {
- Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
- Label runtime;
- Register thread = stub->thread();
- Register tmp1 = stub->tmp1(); // tmp1 holds the card address.
- Register tmp2 = stub->tmp2();
- assert(stub->tmp3() == noreg, "not needed in this platform");
-
- __ bind(*stub->entry());
- generate_post_barrier_slow_path(masm, thread, tmp1, tmp2, *stub->continuation(), runtime);
-
- __ bind(runtime);
- generate_c2_barrier_runtime_call(masm, stub, tmp1, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry));
- __ b(*stub->continuation());
+ bool new_val_may_be_null) {
+ Label done;
+ generate_post_barrier(masm, store_addr, new_val, thread, tmp1, tmp2, done, new_val_may_be_null);
+ __ bind(done);
}
#endif // COMPILER2
@@ -456,20 +432,19 @@ void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrier
__ b(*stub->continuation());
}
-void G1BarrierSetAssembler::gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) {
- G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
- __ bind(*stub->entry());
- assert(stub->addr()->is_register(), "Precondition.");
- assert(stub->new_val()->is_register(), "Precondition.");
- Register new_val_reg = stub->new_val()->as_register();
- __ cbz(new_val_reg, *stub->continuation());
- ce->store_parameter(stub->addr()->as_pointer_register(), 0);
- __ far_call(RuntimeAddress(bs->post_barrier_c1_runtime_code_blob()->code_begin()));
- __ b(*stub->continuation());
-}
-
#undef __
+void G1BarrierSetAssembler::g1_write_barrier_post_c1(MacroAssembler* masm,
+ Register store_addr,
+ Register new_val,
+ Register thread,
+ Register tmp1,
+ Register tmp2) {
+ Label done;
+ generate_post_barrier(masm, store_addr, new_val, thread, tmp1, tmp2, done, true /* new_val_may_be_null */);
+ masm->bind(done);
+}
+
#define __ sasm->
void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
@@ -521,74 +496,6 @@ void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler*
__ epilogue();
}
-void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) {
- __ prologue("g1_post_barrier", false);
-
- // arg0: store_address
- Address store_addr(rfp, 2*BytesPerWord);
-
- BarrierSet* bs = BarrierSet::barrier_set();
- CardTableBarrierSet* ctbs = barrier_set_cast(bs);
- CardTable* ct = ctbs->card_table();
-
- Label done;
- Label runtime;
-
- // At this point we know new_value is non-null and the new_value crosses regions.
- // Must check to see if card is already dirty
-
- const Register thread = rthread;
-
- Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
- Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
-
- const Register card_offset = rscratch2;
- // LR is free here, so we can use it to hold the byte_map_base.
- const Register byte_map_base = lr;
-
- assert_different_registers(card_offset, byte_map_base, rscratch1);
-
- __ load_parameter(0, card_offset);
- __ lsr(card_offset, card_offset, CardTable::card_shift());
- __ load_byte_map_base(byte_map_base);
- __ ldrb(rscratch1, Address(byte_map_base, card_offset));
- __ cmpw(rscratch1, (int)G1CardTable::g1_young_card_val());
- __ br(Assembler::EQ, done);
-
- assert((int)CardTable::dirty_card_val() == 0, "must be 0");
-
- __ membar(Assembler::StoreLoad);
- __ ldrb(rscratch1, Address(byte_map_base, card_offset));
- __ cbzw(rscratch1, done);
-
- // storing region crossing non-null, card is clean.
- // dirty card and log.
- __ strb(zr, Address(byte_map_base, card_offset));
-
- // Convert card offset into an address in card_addr
- Register card_addr = card_offset;
- __ add(card_addr, byte_map_base, card_addr);
-
- __ ldr(rscratch1, queue_index);
- __ cbz(rscratch1, runtime);
- __ sub(rscratch1, rscratch1, wordSize);
- __ str(rscratch1, queue_index);
-
- // Reuse LR to hold buffer_addr
- const Register buffer_addr = lr;
-
- __ ldr(buffer_addr, buffer);
- __ str(card_addr, Address(buffer_addr, rscratch1));
- __ b(done);
-
- __ bind(runtime);
- __ push_call_clobbered_registers();
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
- __ pop_call_clobbered_registers();
- __ bind(done);
- __ epilogue();
-}
-
#undef __
#endif // COMPILER1
diff --git a/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.hpp
index 04ac2096096..72040cd7ad2 100644
--- a/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -32,9 +32,7 @@
class LIR_Assembler;
class StubAssembler;
class G1PreBarrierStub;
-class G1PostBarrierStub;
class G1PreBarrierStubC2;
-class G1PostBarrierStubC2;
class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
protected:
@@ -65,10 +63,15 @@ protected:
public:
#ifdef COMPILER1
void gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub);
- void gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub);
void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm);
- void generate_c1_post_barrier_runtime_stub(StubAssembler* sasm);
+
+ void g1_write_barrier_post_c1(MacroAssembler* masm,
+ Register store_addr,
+ Register new_val,
+ Register thread,
+ Register tmp1,
+ Register tmp2);
#endif
#ifdef COMPILER2
@@ -87,9 +90,7 @@ public:
Register thread,
Register tmp1,
Register tmp2,
- G1PostBarrierStubC2* c2_stub);
- void generate_c2_post_barrier_stub(MacroAssembler* masm,
- G1PostBarrierStubC2* stub) const;
+ bool new_val_may_be_null);
#endif
void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
diff --git a/src/hotspot/cpu/aarch64/gc/g1/g1_aarch64.ad b/src/hotspot/cpu/aarch64/gc/g1/g1_aarch64.ad
index 081a67d6880..18fc27a4af4 100644
--- a/src/hotspot/cpu/aarch64/gc/g1/g1_aarch64.ad
+++ b/src/hotspot/cpu/aarch64/gc/g1/g1_aarch64.ad
@@ -1,5 +1,5 @@
//
-// Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
@@ -62,13 +62,13 @@ static void write_barrier_post(MacroAssembler* masm,
Register new_val,
Register tmp1,
Register tmp2) {
- if (!G1PostBarrierStubC2::needs_barrier(node)) {
+ if (!G1BarrierStubC2::needs_post_barrier(node)) {
return;
}
Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
G1BarrierSetAssembler* g1_asm = static_cast(BarrierSet::barrier_set()->barrier_set_assembler());
- G1PostBarrierStubC2* const stub = G1PostBarrierStubC2::create(node);
- g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, rthread, tmp1, tmp2, stub);
+ bool new_val_may_be_null = G1BarrierStubC2::post_new_val_may_be_null(node);
+ g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, rthread, tmp1, tmp2, new_val_may_be_null);
}
%}
diff --git a/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.cpp
index 302701e1cad..021af3e5698 100644
--- a/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.cpp
@@ -275,7 +275,7 @@ address BarrierSetAssembler::patching_epoch_addr() {
}
void BarrierSetAssembler::increment_patching_epoch() {
- Atomic::inc(&_patching_epoch);
+ AtomicAccess::inc(&_patching_epoch);
}
void BarrierSetAssembler::clear_patching_epoch() {
@@ -398,14 +398,18 @@ void BarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register
OptoReg::Name BarrierSetAssembler::encode_float_vector_register_size(const Node* node, OptoReg::Name opto_reg) {
switch (node->ideal_reg()) {
case Op_RegF:
+ case Op_RegI: // RA may place scalar values (Op_RegI/N/L/P) in FP registers when UseFPUForSpilling is enabled
+ case Op_RegN:
// No need to refine. The original encoding is already fine to distinguish.
- assert(opto_reg % 4 == 0, "Float register should only occupy a single slot");
+ assert(opto_reg % 4 == 0, "32-bit register should only occupy a single slot");
break;
// Use different encoding values of the same fp/vector register to help distinguish different sizes.
// Such as V16. The OptoReg::name and its corresponding slot value are
// "V16": 64, "V16_H": 65, "V16_J": 66, "V16_K": 67.
case Op_RegD:
case Op_VecD:
+ case Op_RegL:
+ case Op_RegP:
opto_reg &= ~3;
opto_reg |= 1;
break;
diff --git a/src/hotspot/cpu/aarch64/gc/shared/barrierSetNMethod_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shared/barrierSetNMethod_aarch64.cpp
index 3a4ba913a8f..4d5ca01b6b4 100644
--- a/src/hotspot/cpu/aarch64/gc/shared/barrierSetNMethod_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/gc/shared/barrierSetNMethod_aarch64.cpp
@@ -112,22 +112,22 @@ public:
}
int get_value() {
- return Atomic::load_acquire(guard_addr());
+ return AtomicAccess::load_acquire(guard_addr());
}
void set_value(int value, int bit_mask) {
if (bit_mask == ~0) {
- Atomic::release_store(guard_addr(), value);
+ AtomicAccess::release_store(guard_addr(), value);
return;
}
assert((value & ~bit_mask) == 0, "trying to set bits outside the mask");
value &= bit_mask;
- int old_value = Atomic::load(guard_addr());
+ int old_value = AtomicAccess::load(guard_addr());
while (true) {
// Only bits in the mask are changed
int new_value = value | (old_value & ~bit_mask);
if (new_value == old_value) break;
- int v = Atomic::cmpxchg(guard_addr(), old_value, new_value, memory_order_release);
+ int v = AtomicAccess::cmpxchg(guard_addr(), old_value, new_value, memory_order_release);
if (v == old_value) break;
old_value = v;
}
diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp
index ad3a171c103..ae2819e78ca 100644
--- a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp
@@ -275,7 +275,6 @@ public:
Label* entry();
};
-
class ZStoreBarrierStubC2Aarch64 : public ZStoreBarrierStubC2 {
private:
bool _deferred_emit;
diff --git a/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad b/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad
index 78dc5d56bbd..8149c7e4c97 100644
--- a/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad
+++ b/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad
@@ -239,7 +239,6 @@ instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP ne
ins_pipe(pipe_slow);
%}
-
instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, rFlagsReg cr) %{
match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
diff --git a/src/hotspot/cpu/aarch64/globalDefinitions_aarch64.hpp b/src/hotspot/cpu/aarch64/globalDefinitions_aarch64.hpp
index 948ba97aa22..1e788590b64 100644
--- a/src/hotspot/cpu/aarch64/globalDefinitions_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/globalDefinitions_aarch64.hpp
@@ -35,8 +35,6 @@ const bool CCallingConventionRequiresIntsAsLongs = false;
#define SUPPORTS_NATIVE_CX8
-#define SUPPORT_MONITOR_COUNT
-
// Aarch64 was not originally defined to be multi-copy-atomic, but now
// is. See: "Simplifying ARM Concurrency: Multicopy-atomic Axiomatic
// and Operational Models for ARMv8"
diff --git a/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp b/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp
index 607912e6e49..6f8795494a2 100644
--- a/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp
@@ -1704,3 +1704,14 @@ void InterpreterMacroAssembler::load_method_entry(Register cache, Register index
add(cache, cache, Array::base_offset_in_bytes());
lea(cache, Address(cache, index));
}
+
+#ifdef ASSERT
+void InterpreterMacroAssembler::verify_field_offset(Register reg) {
+ // Verify the field offset is not in the header, implicitly checks for 0
+ Label L;
+ subs(zr, reg, oopDesc::base_offset_in_bytes());
+ br(Assembler::GE, L);
+ stop("bad field offset");
+ bind(L);
+}
+#endif
diff --git a/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp b/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp
index e896a2a9430..e07e6e49f53 100644
--- a/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp
@@ -319,6 +319,8 @@ class InterpreterMacroAssembler: public MacroAssembler {
void load_resolved_indy_entry(Register cache, Register index);
void load_field_entry(Register cache, Register index, int bcp_offset = 1);
void load_method_entry(Register cache, Register index, int bcp_offset = 1);
+
+ void verify_field_offset(Register reg) NOT_DEBUG_RETURN;
};
#endif // CPU_AARCH64_INTERP_MASM_AARCH64_HPP
diff --git a/src/hotspot/cpu/aarch64/javaFrameAnchor_aarch64.hpp b/src/hotspot/cpu/aarch64/javaFrameAnchor_aarch64.hpp
index 8d125d3c027..a6e7d74f528 100644
--- a/src/hotspot/cpu/aarch64/javaFrameAnchor_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/javaFrameAnchor_aarch64.hpp
@@ -39,24 +39,22 @@ public:
// 3 - restoring an old state (javaCalls)
void clear(void) {
+ // No hardware barriers are necessary. All members are volatile and the profiler
+ // is run from a signal handler and only observers the thread its running on.
+
// clearing _last_Java_sp must be first
_last_Java_sp = nullptr;
- OrderAccess::release();
_last_Java_fp = nullptr;
_last_Java_pc = nullptr;
}
void copy(JavaFrameAnchor* src) {
- // In order to make sure the transition state is valid for "this"
+ // No hardware barriers are necessary. All members are volatile and the profiler
+ // is run from a signal handler and only observers the thread its running on.
+
// We must clear _last_Java_sp before copying the rest of the new data
- //
- // Hack Alert: Temporary bugfix for 4717480/4721647
- // To act like previous version (pd_cache_state) don't null _last_Java_sp
- // unless the value is changing
- //
if (_last_Java_sp != src->_last_Java_sp) {
_last_Java_sp = nullptr;
- OrderAccess::release();
}
_last_Java_fp = src->_last_Java_fp;
_last_Java_pc = src->_last_Java_pc;
diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
index 3999beeec2b..a37edab8578 100644
--- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
@@ -634,12 +634,13 @@ void MacroAssembler::set_last_Java_frame(Register last_java_sp,
last_java_sp = esp;
}
- str(last_java_sp, Address(rthread, JavaThread::last_Java_sp_offset()));
-
// last_java_fp is optional
if (last_java_fp->is_valid()) {
str(last_java_fp, Address(rthread, JavaThread::last_Java_fp_offset()));
}
+
+ // We must set sp last.
+ str(last_java_sp, Address(rthread, JavaThread::last_Java_sp_offset()));
}
void MacroAssembler::set_last_Java_frame(Register last_java_sp,
@@ -5630,38 +5631,6 @@ void MacroAssembler::tlab_allocate(Register obj,
bs->tlab_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
}
-void MacroAssembler::inc_held_monitor_count(Register tmp) {
- Address dst(rthread, JavaThread::held_monitor_count_offset());
-#ifdef ASSERT
- ldr(tmp, dst);
- increment(tmp);
- str(tmp, dst);
- Label ok;
- tbz(tmp, 63, ok);
- STOP("assert(held monitor count underflow)");
- should_not_reach_here();
- bind(ok);
-#else
- increment(dst);
-#endif
-}
-
-void MacroAssembler::dec_held_monitor_count(Register tmp) {
- Address dst(rthread, JavaThread::held_monitor_count_offset());
-#ifdef ASSERT
- ldr(tmp, dst);
- decrement(tmp);
- str(tmp, dst);
- Label ok;
- tbz(tmp, 63, ok);
- STOP("assert(held monitor count underflow)");
- should_not_reach_here();
- bind(ok);
-#else
- decrement(dst);
-#endif
-}
-
void MacroAssembler::verify_tlab() {
#ifdef ASSERT
if (UseTLAB && VerifyOops) {
diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
index fe2440fd3fd..705bd19093c 100644
--- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
@@ -983,9 +983,6 @@ public:
void push_cont_fastpath(Register java_thread = rthread);
void pop_cont_fastpath(Register java_thread = rthread);
- void inc_held_monitor_count(Register tmp);
- void dec_held_monitor_count(Register tmp);
-
// Round up to a power of two
void round_to(Register reg, int modulus);
@@ -1623,7 +1620,7 @@ public:
FloatRegister p, FloatRegister z, FloatRegister t1);
void ghash_reduce_wide(int index, FloatRegister result, FloatRegister lo, FloatRegister hi,
FloatRegister p, FloatRegister z, FloatRegister t1);
- void ghash_processBlocks_wide(address p, Register state, Register subkeyH,
+ void ghash_processBlocks_wide(Label& p, Register state, Register subkeyH,
Register data, Register blocks, int unrolls);
diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64_aes.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64_aes.cpp
index 84b85b7b445..25ec9cf9bdd 100644
--- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64_aes.cpp
+++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64_aes.cpp
@@ -507,7 +507,7 @@ void MacroAssembler::ghash_modmul(FloatRegister result,
//
// Clobbers all vector registers.
//
-void MacroAssembler::ghash_processBlocks_wide(address field_polynomial, Register state,
+void MacroAssembler::ghash_processBlocks_wide(Label& field_polynomial, Register state,
Register subkeyH,
Register data, Register blocks, int unrolls) {
int register_stride = 7;
@@ -531,7 +531,10 @@ void MacroAssembler::ghash_processBlocks_wide(address field_polynomial, Register
FloatRegister p = v31;
eor(vzr, T16B, vzr, vzr); // zero register
- ldrq(p, field_polynomial); // The field polynomial
+ // load polynomial via label which must identify local data in the
+ // same code stub
+ adr(rscratch1, field_polynomial);
+ ldrq(p, rscratch1); // The field polynomial
ldrq(v0, Address(state));
ldrq(Hprime, Address(subkeyH));
diff --git a/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp b/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp
index f5d7d9e4387..94694b58d2f 100644
--- a/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp
@@ -90,7 +90,6 @@ void Relocation::pd_set_call_destination(address x) {
void trampoline_stub_Relocation::pd_fix_owner_after_move() {
NativeCall* call = nativeCall_at(owner());
- assert(call->raw_destination() == owner(), "destination should be empty");
address trampoline = addr();
address dest = nativeCallTrampolineStub_at(trampoline)->destination();
if (!Assembler::reachable_from_branch_at(owner(), dest)) {
diff --git a/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp b/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp
index 748c3e8fb11..39609cbe0ac 100644
--- a/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp
@@ -683,12 +683,12 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
int comp_args_on_stack,
const BasicType *sig_bt,
const VMRegPair *regs,
- AdapterHandlerEntry* handler) {
- address i2c_entry = __ pc();
+ address entry_address[AdapterBlob::ENTRY_COUNT]) {
+ entry_address[AdapterBlob::I2C] = __ pc();
gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs);
- address c2i_unverified_entry = __ pc();
+ entry_address[AdapterBlob::C2I_Unverified] = __ pc();
Label skip_fixup;
Register data = rscratch2;
@@ -718,10 +718,10 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
__ block_comment("} c2i_unverified_entry");
}
- address c2i_entry = __ pc();
+ entry_address[AdapterBlob::C2I] = __ pc();
// Class initialization barrier for static methods
- address c2i_no_clinit_check_entry = nullptr;
+ entry_address[AdapterBlob::C2I_No_Clinit_Check] = nullptr;
if (VM_Version::supports_fast_class_init_checks()) {
Label L_skip_barrier;
@@ -736,15 +736,13 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
__ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
__ bind(L_skip_barrier);
- c2i_no_clinit_check_entry = __ pc();
+ entry_address[AdapterBlob::C2I_No_Clinit_Check] = __ pc();
}
BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
bs->c2i_entry_barrier(masm);
gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup);
-
- handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry);
return;
}
@@ -987,11 +985,8 @@ static void fill_continuation_entry(MacroAssembler* masm) {
__ ldr(rscratch1, Address(rthread, JavaThread::cont_fastpath_offset()));
__ str(rscratch1, Address(sp, ContinuationEntry::parent_cont_fastpath_offset()));
- __ ldr(rscratch1, Address(rthread, JavaThread::held_monitor_count_offset()));
- __ str(rscratch1, Address(sp, ContinuationEntry::parent_held_monitor_count_offset()));
__ str(zr, Address(rthread, JavaThread::cont_fastpath_offset()));
- __ str(zr, Address(rthread, JavaThread::held_monitor_count_offset()));
}
// on entry, sp points to the ContinuationEntry
@@ -1007,50 +1002,6 @@ static void continuation_enter_cleanup(MacroAssembler* masm) {
#endif
__ ldr(rscratch1, Address(sp, ContinuationEntry::parent_cont_fastpath_offset()));
__ str(rscratch1, Address(rthread, JavaThread::cont_fastpath_offset()));
-
- if (CheckJNICalls) {
- // Check if this is a virtual thread continuation
- Label L_skip_vthread_code;
- __ ldrw(rscratch1, Address(sp, ContinuationEntry::flags_offset()));
- __ cbzw(rscratch1, L_skip_vthread_code);
-
- // If the held monitor count is > 0 and this vthread is terminating then
- // it failed to release a JNI monitor. So we issue the same log message
- // that JavaThread::exit does.
- __ ldr(rscratch1, Address(rthread, JavaThread::jni_monitor_count_offset()));
- __ cbz(rscratch1, L_skip_vthread_code);
-
- // Save return value potentially containing the exception oop in callee-saved R19.
- __ mov(r19, r0);
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::log_jni_monitor_still_held));
- // Restore potential return value.
- __ mov(r0, r19);
-
- // For vthreads we have to explicitly zero the JNI monitor count of the carrier
- // on termination. The held count is implicitly zeroed below when we restore from
- // the parent held count (which has to be zero).
- __ str(zr, Address(rthread, JavaThread::jni_monitor_count_offset()));
-
- __ bind(L_skip_vthread_code);
- }
-#ifdef ASSERT
- else {
- // Check if this is a virtual thread continuation
- Label L_skip_vthread_code;
- __ ldrw(rscratch1, Address(sp, ContinuationEntry::flags_offset()));
- __ cbzw(rscratch1, L_skip_vthread_code);
-
- // See comment just above. If not checking JNI calls the JNI count is only
- // needed for assertion checking.
- __ str(zr, Address(rthread, JavaThread::jni_monitor_count_offset()));
-
- __ bind(L_skip_vthread_code);
- }
-#endif
-
- __ ldr(rscratch1, Address(sp, ContinuationEntry::parent_held_monitor_count_offset()));
- __ str(rscratch1, Address(rthread, JavaThread::held_monitor_count_offset()));
-
__ ldr(rscratch2, Address(sp, ContinuationEntry::parent_offset()));
__ str(rscratch2, Address(rthread, JavaThread::cont_entry_offset()));
__ add(rfp, sp, (int)ContinuationEntry::size());
@@ -1763,9 +1714,6 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
Label lock_done;
if (method->is_synchronized()) {
- Label count;
- const int mark_word_offset = BasicLock::displaced_header_offset_in_bytes();
-
// Get the handle (the 2nd argument)
__ mov(oop_handle_reg, c_rarg1);
diff --git a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp
index fa7329f4942..ffe5afd93cb 100644
--- a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp
@@ -42,7 +42,7 @@
#include "prims/methodHandles.hpp"
#include "prims/upcallLinker.hpp"
#include "runtime/arguments.hpp"
-#include "runtime/atomic.hpp"
+#include "runtime/atomicAccess.hpp"
#include "runtime/continuation.hpp"
#include "runtime/continuationEntry.inline.hpp"
#include "runtime/frame.inline.hpp"
@@ -802,7 +802,7 @@ class StubGenerator: public StubCodeGenerator {
//
// s and d are adjusted to point to the remaining words to copy
//
- void generate_copy_longs(StubId stub_id, DecoratorSet decorators, Label &start, Register s, Register d, Register count) {
+ address generate_copy_longs(StubId stub_id, DecoratorSet decorators, Register s, Register d, Register count) {
BasicType type;
copy_direction direction;
@@ -854,7 +854,7 @@ class StubGenerator: public StubCodeGenerator {
StubCodeMark mark(this, stub_id);
- __ bind(start);
+ address start = __ pc();
Label unaligned_copy_long;
if (AvoidUnalignedAccesses) {
@@ -894,9 +894,9 @@ class StubGenerator: public StubCodeGenerator {
int prefetch = PrefetchCopyIntervalInBytes;
bool use_stride = false;
if (direction == copy_backwards) {
- use_stride = prefetch > 256;
- prefetch = -prefetch;
- if (use_stride) __ mov(stride, prefetch);
+ use_stride = prefetch > 256;
+ prefetch = -prefetch;
+ if (use_stride) __ mov(stride, prefetch);
}
__ bind(again);
@@ -1026,9 +1026,9 @@ class StubGenerator: public StubCodeGenerator {
int prefetch = PrefetchCopyIntervalInBytes;
bool use_stride = false;
if (direction == copy_backwards) {
- use_stride = prefetch > 256;
- prefetch = -prefetch;
- if (use_stride) __ mov(stride, prefetch);
+ use_stride = prefetch > 256;
+ prefetch = -prefetch;
+ if (use_stride) __ mov(stride, prefetch);
}
__ bind(again);
@@ -1037,15 +1037,15 @@ class StubGenerator: public StubCodeGenerator {
__ prfm(use_stride ? Address(s, stride) : Address(s, prefetch), PLDL1KEEP);
if (direction == copy_forwards) {
- // allowing for the offset of -8 the store instructions place
- // registers into the target 64 bit block at the following
- // offsets
- //
- // t0 at offset 0
- // t1 at offset 8, t2 at offset 16
- // t3 at offset 24, t4 at offset 32
- // t5 at offset 40, t6 at offset 48
- // t7 at offset 56
+ // allowing for the offset of -8 the store instructions place
+ // registers into the target 64 bit block at the following
+ // offsets
+ //
+ // t0 at offset 0
+ // t1 at offset 8, t2 at offset 16
+ // t3 at offset 24, t4 at offset 32
+ // t5 at offset 40, t6 at offset 48
+ // t7 at offset 56
bs.copy_store_at_8(Address(d, 1 * unit), t0);
bs.copy_store_at_16(Address(d, 2 * unit), t1, t2);
@@ -1057,18 +1057,18 @@ class StubGenerator: public StubCodeGenerator {
bs.copy_store_at_8(Address(__ pre(d, 8 * unit)), t7);
bs.copy_load_at_16(t6, t7, Address(__ pre(s, 8 * unit)));
} else {
- // d was not offset when we started so the registers are
- // written into the 64 bit block preceding d with the following
- // offsets
- //
- // t1 at offset -8
- // t3 at offset -24, t0 at offset -16
- // t5 at offset -48, t2 at offset -32
- // t7 at offset -56, t4 at offset -48
- // t6 at offset -64
- //
- // note that this matches the offsets previously noted for the
- // loads
+ // d was not offset when we started so the registers are
+ // written into the 64 bit block preceding d with the following
+ // offsets
+ //
+ // t1 at offset -8
+ // t3 at offset -24, t0 at offset -16
+ // t5 at offset -48, t2 at offset -32
+ // t7 at offset -56, t4 at offset -48
+ // t6 at offset -64
+ //
+ // note that this matches the offsets previously noted for the
+ // loads
bs.copy_store_at_8(Address(d, 1 * unit), t1);
bs.copy_store_at_16(Address(d, 3 * unit), t3, t0);
@@ -1109,10 +1109,10 @@ class StubGenerator: public StubCodeGenerator {
{
Label L1, L2;
__ tbz(count, exact_log2(4), L1);
- // this is the same as above but copying only 4 longs hence
- // with only one intervening stp between the str instructions
- // but note that the offsets and registers still follow the
- // same pattern
+ // this is the same as above but copying only 4 longs hence
+ // with only one intervening stp between the str instructions
+ // but note that the offsets and registers still follow the
+ // same pattern
bs.copy_load_at_16(t0, t1, Address(s, 2 * unit));
bs.copy_load_at_16(t2, t3, Address(__ pre(s, 4 * unit)));
if (direction == copy_forwards) {
@@ -1127,10 +1127,10 @@ class StubGenerator: public StubCodeGenerator {
__ bind(L1);
__ tbz(count, 1, L2);
- // this is the same as above but copying only 2 longs hence
- // there is no intervening stp between the str instructions
- // but note that the offset and register patterns are still
- // the same
+ // this is the same as above but copying only 2 longs hence
+ // there is no intervening stp between the str instructions
+ // but note that the offset and register patterns are still
+ // the same
bs.copy_load_at_16(t0, t1, Address(__ pre(s, 2 * unit)));
if (direction == copy_forwards) {
bs.copy_store_at_8(Address(d, 1 * unit), t0);
@@ -1141,18 +1141,20 @@ class StubGenerator: public StubCodeGenerator {
}
__ bind(L2);
- // for forwards copy we need to re-adjust the offsets we
- // applied so that s and d are follow the last words written
+ // for forwards copy we need to re-adjust the offsets we
+ // applied so that s and d are follow the last words written
- if (direction == copy_forwards) {
- __ add(s, s, 16);
- __ add(d, d, 8);
- }
+ if (direction == copy_forwards) {
+ __ add(s, s, 16);
+ __ add(d, d, 8);
+ }
}
__ ret(lr);
- }
+ }
+
+ return start;
}
// Small copy: less than 16 bytes.
@@ -1206,10 +1208,6 @@ class StubGenerator: public StubCodeGenerator {
}
}
- Label copy_f, copy_b;
- Label copy_obj_f, copy_obj_b;
- Label copy_obj_uninit_f, copy_obj_uninit_b;
-
// All-singing all-dancing memory copy.
//
// Copy count units of memory from s to d. The size of a unit is
@@ -1447,19 +1445,19 @@ class StubGenerator: public StubCodeGenerator {
}
if (direction == copy_forwards) {
if (type != T_OBJECT) {
- __ bl(copy_f);
+ __ bl(StubRoutines::aarch64::copy_byte_f());
} else if ((decorators & IS_DEST_UNINITIALIZED) != 0) {
- __ bl(copy_obj_uninit_f);
+ __ bl(StubRoutines::aarch64::copy_oop_uninit_f());
} else {
- __ bl(copy_obj_f);
+ __ bl(StubRoutines::aarch64::copy_oop_f());
}
} else {
if (type != T_OBJECT) {
- __ bl(copy_b);
+ __ bl(StubRoutines::aarch64::copy_byte_b());
} else if ((decorators & IS_DEST_UNINITIALIZED) != 0) {
- __ bl(copy_obj_uninit_b);
+ __ bl(StubRoutines::aarch64::copy_oop_uninit_b());
} else {
- __ bl(copy_obj_b);
+ __ bl(StubRoutines::aarch64::copy_oop_b());
}
}
@@ -1522,11 +1520,11 @@ class StubGenerator: public StubCodeGenerator {
// the hardware handle it. The two dwords within qwords that span
// cache line boundaries will still be loaded and stored atomically.
//
- // Side Effects: entry is set to the (post push) entry point so it
- // can be used by the corresponding conjoint copy
- // method
+ // Side Effects: nopush_entry is set to the (post push) entry point
+ // so it can be used by the corresponding conjoint
+ // copy method
//
- address generate_disjoint_copy(StubId stub_id, address *entry) {
+ address generate_disjoint_copy(StubId stub_id, address *nopush_entry) {
Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
RegSet saved_reg = RegSet::of(s, d, count);
int size;
@@ -1615,8 +1613,8 @@ class StubGenerator: public StubCodeGenerator {
address start = __ pc();
__ enter();
- if (entry != nullptr) {
- *entry = __ pc();
+ if (nopush_entry != nullptr) {
+ *nopush_entry = __ pc();
// caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
BLOCK_COMMENT("Entry:");
}
@@ -1679,10 +1677,10 @@ class StubGenerator: public StubCodeGenerator {
// cache line boundaries will still be loaded and stored atomically.
//
// Side Effects:
- // entry is set to the no-overlap entry point so it can be used by
- // some other conjoint copy method
+ // nopush_entry is set to the no-overlap entry point so it can be
+ // used by some other conjoint copy method
//
- address generate_conjoint_copy(StubId stub_id, address nooverlap_target, address *entry) {
+ address generate_conjoint_copy(StubId stub_id, address nooverlap_target, address *nopush_entry) {
Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
RegSet saved_regs = RegSet::of(s, d, count);
int size;
@@ -1769,16 +1767,19 @@ class StubGenerator: public StubCodeGenerator {
address start = __ pc();
__ enter();
- if (entry != nullptr) {
- *entry = __ pc();
+ if (nopush_entry != nullptr) {
+ *nopush_entry = __ pc();
// caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
BLOCK_COMMENT("Entry:");
}
// use fwd copy when (d-s) above_equal (count*size)
+ Label L_overlapping;
__ sub(rscratch1, d, s);
__ cmp(rscratch1, count, Assembler::LSL, exact_log2(size));
- __ br(Assembler::HS, nooverlap_target);
+ __ br(Assembler::LO, L_overlapping);
+ __ b(RuntimeAddress(nooverlap_target));
+ __ bind(L_overlapping);
DecoratorSet decorators = IN_HEAP | IS_ARRAY;
if (dest_uninitialized) {
@@ -1850,7 +1851,7 @@ class StubGenerator: public StubCodeGenerator {
// r0 == 0 - success
// r0 == -1^K - failure, where K is partial transfer count
//
- address generate_checkcast_copy(StubId stub_id, address *entry) {
+ address generate_checkcast_copy(StubId stub_id, address *nopush_entry) {
bool dest_uninitialized;
switch (stub_id) {
case StubId::stubgen_checkcast_arraycopy_id:
@@ -1911,8 +1912,8 @@ class StubGenerator: public StubCodeGenerator {
#endif //ASSERT
// Caller of this entry point must set up the argument registers.
- if (entry != nullptr) {
- *entry = __ pc();
+ if (nopush_entry != nullptr) {
+ *nopush_entry = __ pc();
BLOCK_COMMENT("Entry:");
}
@@ -2724,13 +2725,21 @@ class StubGenerator: public StubCodeGenerator {
}
void generate_arraycopy_stubs() {
- address entry;
- address entry_jbyte_arraycopy;
- address entry_jshort_arraycopy;
- address entry_jint_arraycopy;
- address entry_oop_arraycopy;
- address entry_jlong_arraycopy;
- address entry_checkcast_arraycopy;
+ // Some copy stubs publish a normal entry and then a 2nd 'fallback'
+ // entry immediately following their stack push. This can be used
+ // as a post-push branch target for compatible stubs when they
+ // identify a special case that can be handled by the fallback
+ // stub e.g a disjoint copy stub may be use as a special case
+ // fallback for its compatible conjoint copy stub.
+ //
+ // A no push entry is always returned in the following local and
+ // then published by assigning to the appropriate entry field in
+ // class StubRoutines. The entry value is then passed to the
+ // generator for the compatible stub. That means the entry must be
+ // listed when saving to/restoring from the AOT cache, ensuring
+ // that the inter-stub jumps are noted at AOT-cache save and
+ // relocated at AOT cache load.
+ address nopush_entry;
// generate the common exit first so later stubs can rely on it if
// they want an UnsafeMemoryAccess exit non-local to the stub
@@ -2738,83 +2747,123 @@ class StubGenerator: public StubCodeGenerator {
// register the stub as the default exit with class UnsafeMemoryAccess
UnsafeMemoryAccess::set_common_exit_stub_pc(StubRoutines::_unsafecopy_common_exit);
- generate_copy_longs(StubId::stubgen_copy_byte_f_id, IN_HEAP | IS_ARRAY, copy_f, r0, r1, r15);
- generate_copy_longs(StubId::stubgen_copy_byte_b_id, IN_HEAP | IS_ARRAY, copy_b, r0, r1, r15);
+ // generate and publish arch64-specific bulk copy routines first
+ // so we can call them from other copy stubs
+ StubRoutines::aarch64::_copy_byte_f = generate_copy_longs(StubId::stubgen_copy_byte_f_id, IN_HEAP | IS_ARRAY, r0, r1, r15);
+ StubRoutines::aarch64::_copy_byte_b = generate_copy_longs(StubId::stubgen_copy_byte_b_id, IN_HEAP | IS_ARRAY, r0, r1, r15);
- generate_copy_longs(StubId::stubgen_copy_oop_f_id, IN_HEAP | IS_ARRAY, copy_obj_f, r0, r1, r15);
- generate_copy_longs(StubId::stubgen_copy_oop_b_id, IN_HEAP | IS_ARRAY, copy_obj_b, r0, r1, r15);
+ StubRoutines::aarch64::_copy_oop_f = generate_copy_longs(StubId::stubgen_copy_oop_f_id, IN_HEAP | IS_ARRAY, r0, r1, r15);
+ StubRoutines::aarch64::_copy_oop_b = generate_copy_longs(StubId::stubgen_copy_oop_b_id, IN_HEAP | IS_ARRAY, r0, r1, r15);
- generate_copy_longs(StubId::stubgen_copy_oop_uninit_f_id, IN_HEAP | IS_ARRAY | IS_DEST_UNINITIALIZED, copy_obj_uninit_f, r0, r1, r15);
- generate_copy_longs(StubId::stubgen_copy_oop_uninit_b_id, IN_HEAP | IS_ARRAY | IS_DEST_UNINITIALIZED, copy_obj_uninit_b, r0, r1, r15);
+ StubRoutines::aarch64::_copy_oop_uninit_f = generate_copy_longs(StubId::stubgen_copy_oop_uninit_f_id, IN_HEAP | IS_ARRAY | IS_DEST_UNINITIALIZED, r0, r1, r15);
+ StubRoutines::aarch64::_copy_oop_uninit_b = generate_copy_longs(StubId::stubgen_copy_oop_uninit_b_id, IN_HEAP | IS_ARRAY | IS_DEST_UNINITIALIZED, r0, r1, r15);
StubRoutines::aarch64::_zero_blocks = generate_zero_blocks();
//*** jbyte
// Always need aligned and unaligned versions
- StubRoutines::_jbyte_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_jbyte_disjoint_arraycopy_id, &entry);
- StubRoutines::_jbyte_arraycopy = generate_conjoint_copy(StubId::stubgen_jbyte_arraycopy_id, entry, &entry_jbyte_arraycopy);
- StubRoutines::_arrayof_jbyte_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jbyte_disjoint_arraycopy_id, &entry);
- StubRoutines::_arrayof_jbyte_arraycopy = generate_conjoint_copy(StubId::stubgen_arrayof_jbyte_arraycopy_id, entry, nullptr);
+ StubRoutines::_jbyte_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_jbyte_disjoint_arraycopy_id, &nopush_entry);
+ // disjoint nopush entry is needed by conjoint copy
+ StubRoutines::_jbyte_disjoint_arraycopy_nopush = nopush_entry;
+ StubRoutines::_jbyte_arraycopy = generate_conjoint_copy(StubId::stubgen_jbyte_arraycopy_id, StubRoutines::_jbyte_disjoint_arraycopy_nopush, &nopush_entry);
+ // conjoint nopush entry is needed by generic/unsafe copy
+ StubRoutines::_jbyte_arraycopy_nopush = nopush_entry;
+ StubRoutines::_arrayof_jbyte_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jbyte_disjoint_arraycopy_id, &nopush_entry);
+ // disjoint arrayof nopush entry is needed by conjoint copy
+ StubRoutines::_arrayof_jbyte_disjoint_arraycopy_nopush = nopush_entry;
+ StubRoutines::_arrayof_jbyte_arraycopy = generate_conjoint_copy(StubId::stubgen_arrayof_jbyte_arraycopy_id, StubRoutines::_arrayof_jbyte_disjoint_arraycopy_nopush, nullptr);
//*** jshort
// Always need aligned and unaligned versions
- StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_jshort_disjoint_arraycopy_id, &entry);
- StubRoutines::_jshort_arraycopy = generate_conjoint_copy(StubId::stubgen_jshort_arraycopy_id, entry, &entry_jshort_arraycopy);
- StubRoutines::_arrayof_jshort_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jshort_disjoint_arraycopy_id, &entry);
- StubRoutines::_arrayof_jshort_arraycopy = generate_conjoint_copy(StubId::stubgen_arrayof_jshort_arraycopy_id, entry, nullptr);
+ StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_jshort_disjoint_arraycopy_id, &nopush_entry);
+ // disjoint nopush entry is needed by conjoint copy
+ StubRoutines::_jshort_disjoint_arraycopy_nopush = nopush_entry;
+ StubRoutines::_jshort_arraycopy = generate_conjoint_copy(StubId::stubgen_jshort_arraycopy_id, StubRoutines::_jshort_disjoint_arraycopy_nopush, &nopush_entry);
+ // conjoint nopush entry is used by generic/unsafe copy
+ StubRoutines::_jshort_arraycopy_nopush = nopush_entry;
+ StubRoutines::_arrayof_jshort_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jshort_disjoint_arraycopy_id, &nopush_entry);
+ // disjoint arrayof nopush entry is needed by conjoint copy
+ StubRoutines::_arrayof_jshort_disjoint_arraycopy_nopush = nopush_entry;
+ StubRoutines::_arrayof_jshort_arraycopy = generate_conjoint_copy(StubId::stubgen_arrayof_jshort_arraycopy_id, StubRoutines::_arrayof_jshort_disjoint_arraycopy_nopush, nullptr);
//*** jint
// Aligned versions
- StubRoutines::_arrayof_jint_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jint_disjoint_arraycopy_id, &entry);
- StubRoutines::_arrayof_jint_arraycopy = generate_conjoint_copy(StubId::stubgen_arrayof_jint_arraycopy_id, entry, &entry_jint_arraycopy);
+ StubRoutines::_arrayof_jint_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jint_disjoint_arraycopy_id, &nopush_entry);
+ // disjoint arrayof nopush entry is needed by conjoint copy
+ StubRoutines::_arrayof_jint_disjoint_arraycopy_nopush = nopush_entry;
+ StubRoutines::_arrayof_jint_arraycopy = generate_conjoint_copy(StubId::stubgen_arrayof_jint_arraycopy_id, StubRoutines::_arrayof_jint_disjoint_arraycopy_nopush, nullptr);
// In 64 bit we need both aligned and unaligned versions of jint arraycopy.
- // entry_jint_arraycopy always points to the unaligned version
- StubRoutines::_jint_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_jint_disjoint_arraycopy_id, &entry);
- StubRoutines::_jint_arraycopy = generate_conjoint_copy(StubId::stubgen_jint_arraycopy_id, entry, &entry_jint_arraycopy);
+ // jint_arraycopy_nopush always points to the unaligned version
+ StubRoutines::_jint_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_jint_disjoint_arraycopy_id, &nopush_entry);
+ // disjoint nopush entry is needed by conjoint copy
+ StubRoutines::_jint_disjoint_arraycopy_nopush = nopush_entry;
+ StubRoutines::_jint_arraycopy = generate_conjoint_copy(StubId::stubgen_jint_arraycopy_id, StubRoutines::_jint_disjoint_arraycopy_nopush, &nopush_entry);
+ // conjoint nopush entry is needed by generic/unsafe copy
+ StubRoutines::_jint_arraycopy_nopush = nopush_entry;
//*** jlong
// It is always aligned
- StubRoutines::_arrayof_jlong_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jlong_disjoint_arraycopy_id, &entry);
- StubRoutines::_arrayof_jlong_arraycopy = generate_conjoint_copy(StubId::stubgen_arrayof_jlong_arraycopy_id, entry, &entry_jlong_arraycopy);
+ StubRoutines::_arrayof_jlong_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jlong_disjoint_arraycopy_id, &nopush_entry);
+ // disjoint arrayof nopush entry is needed by conjoint copy
+ StubRoutines::_arrayof_jlong_disjoint_arraycopy_nopush = nopush_entry;
+ StubRoutines::_arrayof_jlong_arraycopy = generate_conjoint_copy(StubId::stubgen_arrayof_jlong_arraycopy_id, StubRoutines::_arrayof_jlong_disjoint_arraycopy_nopush, &nopush_entry);
+ // conjoint nopush entry is needed by generic/unsafe copy
+ StubRoutines::_jlong_arraycopy_nopush = nopush_entry;
+ // disjoint normal/nopush and conjoint normal entries are not
+ // generated since the arrayof versions are the same
StubRoutines::_jlong_disjoint_arraycopy = StubRoutines::_arrayof_jlong_disjoint_arraycopy;
+ StubRoutines::_jlong_disjoint_arraycopy_nopush = StubRoutines::_arrayof_jlong_disjoint_arraycopy_nopush;
StubRoutines::_jlong_arraycopy = StubRoutines::_arrayof_jlong_arraycopy;
//*** oops
{
- // With compressed oops we need unaligned versions; notice that
- // we overwrite entry_oop_arraycopy.
- bool aligned = !UseCompressedOops;
-
StubRoutines::_arrayof_oop_disjoint_arraycopy
- = generate_disjoint_copy(StubId::stubgen_arrayof_oop_disjoint_arraycopy_id, &entry);
+ = generate_disjoint_copy(StubId::stubgen_arrayof_oop_disjoint_arraycopy_id, &nopush_entry);
+ // disjoint arrayof nopush entry is needed by conjoint copy
+ StubRoutines::_arrayof_oop_disjoint_arraycopy_nopush = nopush_entry;
StubRoutines::_arrayof_oop_arraycopy
- = generate_conjoint_copy(StubId::stubgen_arrayof_oop_arraycopy_id, entry, &entry_oop_arraycopy);
+ = generate_conjoint_copy(StubId::stubgen_arrayof_oop_arraycopy_id, StubRoutines::_arrayof_oop_disjoint_arraycopy_nopush, &nopush_entry);
+ // conjoint arrayof nopush entry is needed by generic/unsafe copy
+ StubRoutines::_oop_arraycopy_nopush = nopush_entry;
// Aligned versions without pre-barriers
StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit
- = generate_disjoint_copy(StubId::stubgen_arrayof_oop_disjoint_arraycopy_uninit_id, &entry);
+ = generate_disjoint_copy(StubId::stubgen_arrayof_oop_disjoint_arraycopy_uninit_id, &nopush_entry);
+ // disjoint arrayof+uninit nopush entry is needed by conjoint copy
+ StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit_nopush = nopush_entry;
+ // note that we don't need a returned nopush entry because the
+ // generic/unsafe copy does not cater for uninit arrays.
StubRoutines::_arrayof_oop_arraycopy_uninit
- = generate_conjoint_copy(StubId::stubgen_arrayof_oop_arraycopy_uninit_id, entry, nullptr);
+ = generate_conjoint_copy(StubId::stubgen_arrayof_oop_arraycopy_uninit_id, StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit_nopush, nullptr);
}
+ // for oop copies reuse arrayof entries for non-arrayof cases
StubRoutines::_oop_disjoint_arraycopy = StubRoutines::_arrayof_oop_disjoint_arraycopy;
+ StubRoutines::_oop_disjoint_arraycopy_nopush = StubRoutines::_arrayof_oop_disjoint_arraycopy_nopush;
StubRoutines::_oop_arraycopy = StubRoutines::_arrayof_oop_arraycopy;
StubRoutines::_oop_disjoint_arraycopy_uninit = StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit;
+ StubRoutines::_oop_disjoint_arraycopy_uninit_nopush = StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit_nopush;
StubRoutines::_oop_arraycopy_uninit = StubRoutines::_arrayof_oop_arraycopy_uninit;
- StubRoutines::_checkcast_arraycopy = generate_checkcast_copy(StubId::stubgen_checkcast_arraycopy_id, &entry_checkcast_arraycopy);
+ StubRoutines::_checkcast_arraycopy = generate_checkcast_copy(StubId::stubgen_checkcast_arraycopy_id, &nopush_entry);
+ // checkcast nopush entry is needed by generic copy
+ StubRoutines::_checkcast_arraycopy_nopush = nopush_entry;
+ // note that we don't need a returned nopush entry because the
+ // generic copy does not cater for uninit arrays.
StubRoutines::_checkcast_arraycopy_uninit = generate_checkcast_copy(StubId::stubgen_checkcast_arraycopy_uninit_id, nullptr);
- StubRoutines::_unsafe_arraycopy = generate_unsafe_copy(entry_jbyte_arraycopy,
- entry_jshort_arraycopy,
- entry_jint_arraycopy,
- entry_jlong_arraycopy);
+ // unsafe arraycopy may fallback on conjoint stubs
+ StubRoutines::_unsafe_arraycopy = generate_unsafe_copy(StubRoutines::_jbyte_arraycopy_nopush,
+ StubRoutines::_jshort_arraycopy_nopush,
+ StubRoutines::_jint_arraycopy_nopush,
+ StubRoutines::_jlong_arraycopy_nopush);
- StubRoutines::_generic_arraycopy = generate_generic_copy(entry_jbyte_arraycopy,
- entry_jshort_arraycopy,
- entry_jint_arraycopy,
- entry_oop_arraycopy,
- entry_jlong_arraycopy,
- entry_checkcast_arraycopy);
+ // generic arraycopy may fallback on conjoint stubs
+ StubRoutines::_generic_arraycopy = generate_generic_copy(StubRoutines::_jbyte_arraycopy_nopush,
+ StubRoutines::_jshort_arraycopy_nopush,
+ StubRoutines::_jint_arraycopy_nopush,
+ StubRoutines::_oop_arraycopy_nopush,
+ StubRoutines::_jlong_arraycopy_nopush,
+ StubRoutines::_checkcast_arraycopy_nopush);
StubRoutines::_jbyte_fill = generate_fill(StubId::stubgen_jbyte_fill_id);
StubRoutines::_jshort_fill = generate_fill(StubId::stubgen_jshort_fill_id);
@@ -3402,14 +3451,9 @@ class StubGenerator: public StubCodeGenerator {
// counter = c_rarg7 - 16 bytes of CTR
// return - number of processed bytes
address generate_galoisCounterMode_AESCrypt() {
- address ghash_polynomial = __ pc();
- __ emit_int64(0x87); // The low-order bits of the field
- // polynomial (i.e. p = z^7+z^2+z+1)
- // repeated in the low and high parts of a
- // 128-bit vector
- __ emit_int64(0x87);
+ Label ghash_polynomial; // local data generated after code
- __ align(CodeEntryAlignment);
+ __ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_galoisCounterMode_AESCrypt_id;
StubCodeMark mark(this, stub_id);
address start = __ pc();
@@ -3514,7 +3558,17 @@ class StubGenerator: public StubCodeGenerator {
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(lr);
- return start;
+
+ // bind label and generate polynomial data
+ __ align(wordSize * 2);
+ __ bind(ghash_polynomial);
+ __ emit_int64(0x87); // The low-order bits of the field
+ // polynomial (i.e. p = z^7+z^2+z+1)
+ // repeated in the low and high parts of a
+ // 128-bit vector
+ __ emit_int64(0x87);
+
+ return start;
}
class Cached64Bytes {
@@ -4559,16 +4613,6 @@ class StubGenerator: public StubCodeGenerator {
// by the second lane from all vectors and so on.
address generate_chacha20Block_blockpar() {
Label L_twoRounds, L_cc20_const;
- // The constant data is broken into two 128-bit segments to be loaded
- // onto FloatRegisters. The first 128 bits are a counter add overlay
- // that adds +0/+1/+2/+3 to the vector holding replicated state[12].
- // The second 128-bits is a table constant used for 8-bit left rotations.
- __ BIND(L_cc20_const);
- __ emit_int64(0x0000000100000000UL);
- __ emit_int64(0x0000000300000002UL);
- __ emit_int64(0x0605040702010003UL);
- __ emit_int64(0x0E0D0C0F0A09080BUL);
-
__ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_chacha20Block_id;
StubCodeMark mark(this, stub_id);
@@ -4716,6 +4760,17 @@ class StubGenerator: public StubCodeGenerator {
__ leave();
__ ret(lr);
+ // bind label and generate local constant data used by this stub
+ // The constant data is broken into two 128-bit segments to be loaded
+ // onto FloatRegisters. The first 128 bits are a counter add overlay
+ // that adds +0/+1/+2/+3 to the vector holding replicated state[12].
+ // The second 128-bits is a table constant used for 8-bit left rotations.
+ __ BIND(L_cc20_const);
+ __ emit_int64(0x0000000100000000UL);
+ __ emit_int64(0x0000000300000002UL);
+ __ emit_int64(0x0605040702010003UL);
+ __ emit_int64(0x0E0D0C0F0A09080BUL);
+
return start;
}
@@ -6036,10 +6091,6 @@ class StubGenerator: public StubCodeGenerator {
address generate_kyber12To16() {
Label L_F00, L_loop, L_end;
- __ BIND(L_F00);
- __ emit_int64(0x0f000f000f000f00);
- __ emit_int64(0x0f000f000f000f00);
-
__ align(CodeEntryAlignment);
StubId stub_id = StubId::stubgen_kyber12To16_id;
StubCodeMark mark(this, stub_id);
@@ -6233,6 +6284,11 @@ class StubGenerator: public StubCodeGenerator {
__ mov(r0, zr); // return 0
__ ret(lr);
+ // bind label and generate constant data used by this stub
+ __ BIND(L_F00);
+ __ emit_int64(0x0f000f000f000f00);
+ __ emit_int64(0x0f000f000f000f00);
+
return start;
}
@@ -9642,14 +9698,7 @@ class StubGenerator: public StubCodeGenerator {
StubId stub_id = StubId::stubgen_ghash_processBlocks_id;
StubCodeMark mark(this, stub_id);
- __ align(wordSize * 2);
- address p = __ pc();
- __ emit_int64(0x87); // The low-order bits of the field
- // polynomial (i.e. p = z^7+z^2+z+1)
- // repeated in the low and high parts of a
- // 128-bit vector
- __ emit_int64(0x87);
-
+ Label polynomial; // local data generated at end of stub
__ align(CodeEntryAlignment);
address start = __ pc();
@@ -9661,7 +9710,8 @@ class StubGenerator: public StubCodeGenerator {
FloatRegister vzr = v30;
__ eor(vzr, __ T16B, vzr, vzr); // zero register
- __ ldrq(v24, p); // The field polynomial
+ __ adr(rscratch1, polynomial);
+ __ ldrq(v24, rscratch1); // The field polynomial
__ ldrq(v0, Address(state));
__ ldrq(v1, Address(subkeyH));
@@ -9701,6 +9751,15 @@ class StubGenerator: public StubCodeGenerator {
__ st1(v0, __ T16B, state);
__ ret(lr);
+ // bind label and generate local polynomial data
+ __ align(wordSize * 2);
+ __ bind(polynomial);
+ __ emit_int64(0x87); // The low-order bits of the field
+ // polynomial (i.e. p = z^7+z^2+z+1)
+ // repeated in the low and high parts of a
+ // 128-bit vector
+ __ emit_int64(0x87);
+
return start;
}
@@ -9709,14 +9768,7 @@ class StubGenerator: public StubCodeGenerator {
StubId stub_id = StubId::stubgen_ghash_processBlocks_wide_id;
StubCodeMark mark(this, stub_id);
- __ align(wordSize * 2);
- address p = __ pc();
- __ emit_int64(0x87); // The low-order bits of the field
- // polynomial (i.e. p = z^7+z^2+z+1)
- // repeated in the low and high parts of a
- // 128-bit vector
- __ emit_int64(0x87);
-
+ Label polynomial; // local data generated after stub
__ align(CodeEntryAlignment);
address start = __ pc();
@@ -9738,7 +9790,7 @@ class StubGenerator: public StubCodeGenerator {
__ st1(v8, v9, v10, v11, __ T16B, Address(sp));
}
- __ ghash_processBlocks_wide(p, state, subkeyH, data, blocks, unroll);
+ __ ghash_processBlocks_wide(polynomial, state, subkeyH, data, blocks, unroll);
if (unroll > 1) {
// And restore state
@@ -9751,7 +9803,17 @@ class StubGenerator: public StubCodeGenerator {
__ ret(lr);
+ // bind label and generate polynomial data
+ __ align(wordSize * 2);
+ __ bind(polynomial);
+ __ emit_int64(0x87); // The low-order bits of the field
+ // polynomial (i.e. p = z^7+z^2+z+1)
+ // repeated in the low and high parts of a
+ // 128-bit vector
+ __ emit_int64(0x87);
+
return start;
+
}
void generate_base64_encode_simdround(Register src, Register dst,
@@ -10265,7 +10327,7 @@ class StubGenerator: public StubCodeGenerator {
#if defined (LINUX) && !defined (__ARM_FEATURE_ATOMICS)
- // ARMv8.1 LSE versions of the atomic stubs used by Atomic::PlatformXX.
+ // ARMv8.1 LSE versions of the atomic stubs used by AtomicAccess::PlatformXX.
//
// If LSE is in use, generate LSE versions of all the stubs. The
// non-LSE versions are in atomic_aarch64.S.
diff --git a/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp b/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp
index 2ccde98d98d..f4774f31bbd 100644
--- a/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp
@@ -168,6 +168,7 @@ void TemplateTable::patch_bytecode(Bytecodes::Code bc, Register bc_reg,
Register temp_reg, bool load_bc_into_bc_reg/*=true*/,
int byte_no)
{
+ assert_different_registers(bc_reg, temp_reg);
if (!RewriteBytecodes) return;
Label L_patch_done;
@@ -231,9 +232,12 @@ void TemplateTable::patch_bytecode(Bytecodes::Code bc, Register bc_reg,
__ stop("patching the wrong bytecode");
__ bind(L_okay);
#endif
-
- // patch bytecode
- __ strb(bc_reg, at_bcp(0));
+ // Patch bytecode with release store to coordinate with ResolvedFieldEntry loads
+ // in fast bytecode codelets. load_field_entry has a memory barrier that gains
+ // the needed ordering, together with control dependency on entering the fast codelet
+ // itself.
+ __ lea(temp_reg, at_bcp(0));
+ __ stlrb(bc_reg, temp_reg);
__ bind(L_patch_done);
}
@@ -2269,7 +2273,7 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no,
assert_different_registers(Rcache, index, temp);
assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
- Label resolved, clinit_barrier_slow;
+ Label L_clinit_barrier_slow, L_done;
Bytecodes::Code code = bytecode();
__ load_method_entry(Rcache, index);
@@ -2284,11 +2288,20 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no,
// Load-acquire the bytecode to match store-release in InterpreterRuntime
__ ldarb(temp, temp);
__ subs(zr, temp, (int) code); // have we resolved this bytecode?
- __ br(Assembler::EQ, resolved);
+
+ // Class initialization barrier for static methods
+ if (VM_Version::supports_fast_class_init_checks() && bytecode() == Bytecodes::_invokestatic) {
+ __ br(Assembler::NE, L_clinit_barrier_slow);
+ __ ldr(temp, Address(Rcache, in_bytes(ResolvedMethodEntry::method_offset())));
+ __ load_method_holder(temp, temp);
+ __ clinit_barrier(temp, rscratch1, &L_done, /*L_slow_path*/ nullptr);
+ __ bind(L_clinit_barrier_slow);
+ } else {
+ __ br(Assembler::EQ, L_done);
+ }
// resolve first time through
// Class initialization barrier slow path lands here as well.
- __ bind(clinit_barrier_slow);
address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
__ mov(temp, (int) code);
__ call_VM(noreg, entry, temp);
@@ -2297,14 +2310,7 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no,
__ load_method_entry(Rcache, index);
// n.b. unlike x86 Rcache is now rcpool plus the indexed offset
// so all clients ofthis method must be modified accordingly
- __ bind(resolved);
-
- // Class initialization barrier for static methods
- if (VM_Version::supports_fast_class_init_checks() && bytecode() == Bytecodes::_invokestatic) {
- __ ldr(temp, Address(Rcache, in_bytes(ResolvedMethodEntry::method_offset())));
- __ load_method_holder(temp, temp);
- __ clinit_barrier(temp, rscratch1, nullptr, &clinit_barrier_slow);
- }
+ __ bind(L_done);
}
void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
@@ -2313,7 +2319,7 @@ void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
const Register temp = r19;
assert_different_registers(Rcache, index, temp);
- Label resolved;
+ Label L_clinit_barrier_slow, L_done;
Bytecodes::Code code = bytecode();
switch (code) {
@@ -2332,16 +2338,29 @@ void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
// Load-acquire the bytecode to match store-release in ResolvedFieldEntry::fill_in()
__ ldarb(temp, temp);
__ subs(zr, temp, (int) code); // have we resolved this bytecode?
- __ br(Assembler::EQ, resolved);
+
+ // Class initialization barrier for static fields
+ if (VM_Version::supports_fast_class_init_checks() &&
+ (bytecode() == Bytecodes::_getstatic || bytecode() == Bytecodes::_putstatic)) {
+ const Register field_holder = temp;
+
+ __ br(Assembler::NE, L_clinit_barrier_slow);
+ __ ldr(field_holder, Address(Rcache, in_bytes(ResolvedFieldEntry::field_holder_offset())));
+ __ clinit_barrier(field_holder, rscratch1, &L_done, /*L_slow_path*/ nullptr);
+ __ bind(L_clinit_barrier_slow);
+ } else {
+ __ br(Assembler::EQ, L_done);
+ }
// resolve first time through
+ // Class initialization barrier slow path lands here as well.
address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
__ mov(temp, (int) code);
__ call_VM(noreg, entry, temp);
// Update registers with resolved info
__ load_field_entry(Rcache, index);
- __ bind(resolved);
+ __ bind(L_done);
}
void TemplateTable::load_resolved_field_entry(Register obj,
@@ -3079,6 +3098,7 @@ void TemplateTable::fast_storefield(TosState state)
// R1: field offset, R2: field holder, R5: flags
load_resolved_field_entry(r2, r2, noreg, r1, r5);
+ __ verify_field_offset(r1);
{
Label notVolatile;
@@ -3168,6 +3188,8 @@ void TemplateTable::fast_accessfield(TosState state)
__ load_field_entry(r2, r1);
__ load_sized_value(r1, Address(r2, in_bytes(ResolvedFieldEntry::field_offset_offset())), sizeof(int), true /*is_signed*/);
+ __ verify_field_offset(r1);
+
__ load_unsigned_byte(r3, Address(r2, in_bytes(ResolvedFieldEntry::flags_offset())));
// r0: object
@@ -3234,7 +3256,9 @@ void TemplateTable::fast_xaccess(TosState state)
__ ldr(r0, aaddress(0));
// access constant pool cache
__ load_field_entry(r2, r3, 2);
+
__ load_sized_value(r1, Address(r2, in_bytes(ResolvedFieldEntry::field_offset_offset())), sizeof(int), true /*is_signed*/);
+ __ verify_field_offset(r1);
// 8179954: We need to make sure that the code generated for
// volatile accesses forms a sequentially-consistent set of
diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
index 308deeaf5e2..a04e9defa4b 100644
--- a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
@@ -1,6 +1,7 @@
/*
* Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2015, 2020, Red Hat Inc. All rights reserved.
+ * Copyright 2025 Arm Limited and/or its affiliates.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -222,10 +223,13 @@ void VM_Version::initialize() {
// Neoverse
// N1: 0xd0c
// N2: 0xd49
+ // N3: 0xd8e
// V1: 0xd40
// V2: 0xd4f
+ // V3: 0xd84
if (_cpu == CPU_ARM && (model_is(0xd0c) || model_is(0xd49) ||
- model_is(0xd40) || model_is(0xd4f))) {
+ model_is(0xd40) || model_is(0xd4f) ||
+ model_is(0xd8e) || model_is(0xd84))) {
if (FLAG_IS_DEFAULT(UseSIMDForMemoryOps)) {
FLAG_SET_DEFAULT(UseSIMDForMemoryOps, true);
}
@@ -260,7 +264,9 @@ void VM_Version::initialize() {
// Neoverse
// V1: 0xd40
// V2: 0xd4f
- if (_cpu == CPU_ARM && (model_is(0xd40) || model_is(0xd4f))) {
+ // V3: 0xd84
+ if (_cpu == CPU_ARM &&
+ (model_is(0xd40) || model_is(0xd4f) || model_is(0xd84))) {
if (FLAG_IS_DEFAULT(UseCryptoPmullForCRC32)) {
FLAG_SET_DEFAULT(UseCryptoPmullForCRC32, true);
}
diff --git a/src/hotspot/cpu/arm/arm.ad b/src/hotspot/cpu/arm/arm.ad
index 2835a256153..68fece5263d 100644
--- a/src/hotspot/cpu/arm/arm.ad
+++ b/src/hotspot/cpu/arm/arm.ad
@@ -1154,10 +1154,6 @@ RegMask Matcher::modL_proj_mask() {
return RegMask();
}
-const RegMask Matcher::method_handle_invoke_SP_save_mask() {
- return FP_REGP_mask();
-}
-
bool maybe_far_call(const CallNode *n) {
return !MacroAssembler::_reachable_from_cache(n->as_Call()->entry_point());
}
@@ -1248,23 +1244,6 @@ encode %{
__ set_inst_mark(mark);
%}
- enc_class preserve_SP %{
- // preserve mark
- address mark = __ inst_mark();
- DEBUG_ONLY(int off0 = __ offset());
- // FP is preserved across all calls, even compiled calls.
- // Use it to preserve SP in places where the callee might change the SP.
- __ mov(Rmh_SP_save, SP);
- DEBUG_ONLY(int off1 = __ offset());
- assert(off1 - off0 == 4, "correct size prediction");
- // restore mark
- __ set_inst_mark(mark);
- %}
-
- enc_class restore_SP %{
- __ mov(SP, Rmh_SP_save);
- %}
-
enc_class Java_Dynamic_Call (method meth) %{
Register R8_ic_reg = reg_to_register_object(Matcher::inline_cache_reg_encode());
assert(R8_ic_reg == Ricklass, "should be");
@@ -8799,7 +8778,6 @@ instruct safePoint_poll(iRegP poll, R12RegI tmp, flagsReg icc) %{
// Call Java Static Instruction
instruct CallStaticJavaDirect( method meth ) %{
match(CallStaticJava);
- predicate(! ((CallStaticJavaNode*)n)->is_method_handle_invoke());
effect(USE meth);
ins_cost(CALL_COST);
@@ -8808,20 +8786,6 @@ instruct CallStaticJavaDirect( method meth ) %{
ins_pipe(simple_call);
%}
-// Call Java Static Instruction (method handle version)
-instruct CallStaticJavaHandle( method meth ) %{
- match(CallStaticJava);
- predicate(((CallStaticJavaNode*)n)->is_method_handle_invoke());
- effect(USE meth);
- // FP is saved by all callees (for interpreter stack correction).
- // We use it here for a similar purpose, in {preserve,restore}_FP.
-
- ins_cost(CALL_COST);
- format %{ "CALL,static/MethodHandle ==> " %}
- ins_encode( SetInstMark, preserve_SP, Java_Static_Call( meth ), restore_SP, call_epilog, ClearInstMark );
- ins_pipe(simple_call);
-%}
-
// Call Java Dynamic Instruction
instruct CallDynamicJavaDirect( method meth ) %{
match(CallDynamicJava);
diff --git a/src/hotspot/cpu/arm/arm_32.ad b/src/hotspot/cpu/arm/arm_32.ad
index 1c15d55fbc3..00bf3bd61e4 100644
--- a/src/hotspot/cpu/arm/arm_32.ad
+++ b/src/hotspot/cpu/arm/arm_32.ad
@@ -1,5 +1,5 @@
//
-// Copyright (c) 2008, 2024, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2008, 2025, Oracle and/or its affiliates. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
@@ -432,8 +432,7 @@ OptoRegPair c2::return_value(int ideal_reg) {
int MachCallStaticJavaNode::ret_addr_offset() {
bool far = (_method == nullptr) ? maybe_far_call(this) : !cache_reachable();
- return ((far ? 3 : 1) + (_method_handle_invoke ? 1 : 0)) *
- NativeInstruction::instruction_size;
+ return (far ? 3 : 1) * NativeInstruction::instruction_size;
}
int MachCallDynamicJavaNode::ret_addr_offset() {
diff --git a/src/hotspot/cpu/arm/c1_CodeStubs_arm.cpp b/src/hotspot/cpu/arm/c1_CodeStubs_arm.cpp
index efdc190f09a..8e49cfcbcaa 100644
--- a/src/hotspot/cpu/arm/c1_CodeStubs_arm.cpp
+++ b/src/hotspot/cpu/arm/c1_CodeStubs_arm.cpp
@@ -200,9 +200,10 @@ void MonitorEnterStub::emit_code(LIR_Assembler* ce) {
void MonitorExitStub::emit_code(LIR_Assembler* ce) {
__ bind(_entry);
- if (_compute_lock) {
- ce->monitor_address(_monitor_ix, _lock_reg);
- }
+
+ // lock_reg was destroyed by fast unlocking attempt => recompute it
+ ce->monitor_address(_monitor_ix, _lock_reg);
+
const Register lock_reg = _lock_reg->as_pointer_register();
ce->verify_reserved_argument_area_size(1);
diff --git a/src/hotspot/cpu/arm/c1_FrameMap_arm.cpp b/src/hotspot/cpu/arm/c1_FrameMap_arm.cpp
index 0fd113c8ceb..a820da4d283 100644
--- a/src/hotspot/cpu/arm/c1_FrameMap_arm.cpp
+++ b/src/hotspot/cpu/arm/c1_FrameMap_arm.cpp
@@ -174,11 +174,6 @@ LIR_Opr FrameMap::stack_pointer() {
return FrameMap::SP_opr;
}
-LIR_Opr FrameMap::method_handle_invoke_SP_save_opr() {
- assert(Rmh_SP_save == FP, "Fix register used for saving SP for MethodHandle calls");
- return FP_opr;
-}
-
bool FrameMap::validate_frame() {
int max_offset = in_bytes(framesize_in_bytes());
int java_index = 0;
diff --git a/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp b/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp
index d6ed82dcdb2..219c49d1f14 100644
--- a/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp
+++ b/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp
@@ -245,7 +245,7 @@ int LIR_Assembler::emit_unwind_handler() {
MonitorExitStub* stub = nullptr;
if (method()->is_synchronized()) {
monitor_address(0, FrameMap::R0_opr);
- stub = new MonitorExitStub(FrameMap::R0_opr, true, 0);
+ stub = new MonitorExitStub(FrameMap::R0_opr, 0);
__ unlock_object(R2, R1, R0, *stub->entry());
__ bind(*stub->continuation());
}
@@ -2427,7 +2427,6 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
Register lock = op->lock_opr()->as_pointer_register();
if (op->code() == lir_lock) {
- assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
int null_check_offset = __ lock_object(hdr, obj, lock, *op->stub()->entry());
if (op->info() != nullptr) {
add_debug_info_for_null_check(null_check_offset, op->info());
diff --git a/src/hotspot/cpu/arm/c1_MacroAssembler_arm.cpp b/src/hotspot/cpu/arm/c1_MacroAssembler_arm.cpp
index f2b08269750..ca7711353d2 100644
--- a/src/hotspot/cpu/arm/c1_MacroAssembler_arm.cpp
+++ b/src/hotspot/cpu/arm/c1_MacroAssembler_arm.cpp
@@ -176,17 +176,17 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len,
initialize_object(obj, tmp1, klass, len, tmp2, tmp3, header_size_in_bytes, -1, /* is_tlab_allocated */ UseTLAB);
}
-int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) {
+int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register basic_lock, Label& slow_case) {
int null_check_offset = 0;
const Register tmp2 = Rtemp; // Rtemp should be free at c1 LIR level
- assert_different_registers(hdr, obj, disp_hdr, tmp2);
+ assert_different_registers(hdr, obj, basic_lock, tmp2);
assert(BasicObjectLock::lock_offset() == 0, "adjust this code");
assert(oopDesc::mark_offset_in_bytes() == 0, "Required by atomic instructions");
// save object being locked into the BasicObjectLock
- str(obj, Address(disp_hdr, BasicObjectLock::obj_offset()));
+ str(obj, Address(basic_lock, BasicObjectLock::obj_offset()));
null_check_offset = offset();
@@ -197,26 +197,26 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr
b(slow_case, ne);
}
- Register t1 = disp_hdr; // Needs saving, probably
- Register t2 = hdr; // blow
- Register t3 = Rtemp; // blow
+ Register t1 = basic_lock; // Needs saving, probably
+ Register t2 = hdr; // blow
+ Register t3 = Rtemp; // blow
lightweight_lock(obj, t1, t2, t3, 1 /* savemask - save t1 */, slow_case);
// Success: fall through
return null_check_offset;
}
-void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) {
- assert_different_registers(hdr, obj, disp_hdr, Rtemp);
+void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register basic_lock, Label& slow_case) {
+ assert_different_registers(hdr, obj, basic_lock, Rtemp);
assert(BasicObjectLock::lock_offset() == 0, "adjust this code");
assert(oopDesc::mark_offset_in_bytes() == 0, "Required by atomic instructions");
- ldr(obj, Address(disp_hdr, BasicObjectLock::obj_offset()));
+ ldr(obj, Address(basic_lock, BasicObjectLock::obj_offset()));
- Register t1 = disp_hdr; // Needs saving, probably
- Register t2 = hdr; // blow
- Register t3 = Rtemp; // blow
+ Register t1 = basic_lock; // Needs saving, probably
+ Register t2 = hdr; // blow
+ Register t3 = Rtemp; // blow
lightweight_unlock(obj, t1, t2, t3, 1 /* savemask - save t1 */, slow_case);
// Success: fall through
diff --git a/src/hotspot/cpu/arm/c1_MacroAssembler_arm.hpp b/src/hotspot/cpu/arm/c1_MacroAssembler_arm.hpp
index 0a626822a9b..fd88b6c4fe9 100644
--- a/src/hotspot/cpu/arm/c1_MacroAssembler_arm.hpp
+++ b/src/hotspot/cpu/arm/c1_MacroAssembler_arm.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2008, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -59,9 +59,9 @@
max_array_allocation_length = 0x01000000
};
- int lock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case);
+ int lock_object(Register hdr, Register obj, Register basic_lock, Label& slow_case);
- void unlock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case);
+ void unlock_object(Register hdr, Register obj, Register basic_lock, Label& slow_case);
// This platform only uses signal-based null checks. The Label is not needed.
void null_check(Register r, Label *Lnull = nullptr) { MacroAssembler::null_check(r); }
diff --git a/src/hotspot/cpu/arm/c1_Runtime1_arm.cpp b/src/hotspot/cpu/arm/c1_Runtime1_arm.cpp
index 32da2c24d26..9983f2ce7a2 100644
--- a/src/hotspot/cpu/arm/c1_Runtime1_arm.cpp
+++ b/src/hotspot/cpu/arm/c1_Runtime1_arm.cpp
@@ -275,14 +275,6 @@ OopMapSet* Runtime1::generate_exception_throw(StubAssembler* sasm, address targe
}
-static void restore_sp_for_method_handle(StubAssembler* sasm) {
- // Restore SP from its saved reg (FP) if the exception PC is a MethodHandle call site.
- __ ldr_s32(Rtemp, Address(Rthread, JavaThread::is_method_handle_return_offset()));
- __ cmp(Rtemp, 0);
- __ mov(SP, Rmh_SP_save, ne);
-}
-
-
OopMapSet* Runtime1::generate_handle_exception(StubId id, StubAssembler* sasm) {
__ block_comment("generate_handle_exception");
@@ -339,7 +331,6 @@ OopMapSet* Runtime1::generate_handle_exception(StubId id, StubAssembler* sasm) {
break;
case StubId::c1_handle_exception_from_callee_id:
restore_live_registers_without_return(sasm); // must not jump immediately to handler
- restore_sp_for_method_handle(sasm);
__ ret();
break;
default: ShouldNotReachHere();
@@ -372,9 +363,6 @@ void Runtime1::generate_unwind_exception(StubAssembler* sasm) {
// Jump to handler
__ verify_not_null_oop(Rexception_obj);
- // JSR292 extension
- restore_sp_for_method_handle(sasm);
-
__ jump(R0);
}
diff --git a/src/hotspot/cpu/arm/frame_arm.cpp b/src/hotspot/cpu/arm/frame_arm.cpp
index 2722f93edec..7a23296a3d4 100644
--- a/src/hotspot/cpu/arm/frame_arm.cpp
+++ b/src/hotspot/cpu/arm/frame_arm.cpp
@@ -329,56 +329,6 @@ JavaThread** frame::saved_thread_address(const frame& f) {
return nullptr;
}
-//------------------------------------------------------------------------------
-// frame::verify_deopt_original_pc
-//
-// Verifies the calculated original PC of a deoptimization PC for the
-// given unextended SP. The unextended SP might also be the saved SP
-// for MethodHandle call sites.
-#ifdef ASSERT
-void frame::verify_deopt_original_pc(nmethod* nm, intptr_t* unextended_sp, bool is_method_handle_return) {
- frame fr;
-
- // This is ugly but it's better than to change {get,set}_original_pc
- // to take an SP value as argument. And it's only a debugging
- // method anyway.
- fr._unextended_sp = unextended_sp;
-
- address original_pc = nm->get_original_pc(&fr);
- assert(nm->insts_contains_inclusive(original_pc),
- "original PC must be in the main code section of the compiled method (or must be immediately following it)");
- assert(nm->is_method_handle_return(original_pc) == is_method_handle_return, "must be");
-}
-#endif
-
-//------------------------------------------------------------------------------
-// frame::adjust_unextended_sp
-void frame::adjust_unextended_sp() {
- // same as on x86
-
- // If we are returning to a compiled MethodHandle call site, the
- // saved_fp will in fact be a saved value of the unextended SP. The
- // simplest way to tell whether we are returning to such a call site
- // is as follows:
-
- nmethod* sender_nm = (_cb == nullptr) ? nullptr : _cb->as_nmethod_or_null();
- if (sender_nm != nullptr) {
- // If the sender PC is a deoptimization point, get the original
- // PC. For MethodHandle call site the unextended_sp is stored in
- // saved_fp.
- if (sender_nm->is_deopt_mh_entry(_pc)) {
- DEBUG_ONLY(verify_deopt_mh_original_pc(sender_nm, _fp));
- _unextended_sp = _fp;
- }
- else if (sender_nm->is_deopt_entry(_pc)) {
- DEBUG_ONLY(verify_deopt_original_pc(sender_nm, _unextended_sp));
- }
- else if (sender_nm->is_method_handle_return(_pc)) {
- _unextended_sp = _fp;
- }
- }
-}
-
//------------------------------------------------------------------------------
// frame::update_map_with_saved_link
void frame::update_map_with_saved_link(RegisterMap* map, intptr_t** link_addr) {
diff --git a/src/hotspot/cpu/arm/frame_arm.hpp b/src/hotspot/cpu/arm/frame_arm.hpp
index dec27554a47..6d4ac042831 100644
--- a/src/hotspot/cpu/arm/frame_arm.hpp
+++ b/src/hotspot/cpu/arm/frame_arm.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2008, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -85,20 +85,11 @@
// original sp.
intptr_t* _unextended_sp;
- void adjust_unextended_sp();
intptr_t* ptr_at_addr(int offset) const {
return (intptr_t*) addr_at(offset);
}
-#ifdef ASSERT
- // Used in frame::sender_for_{interpreter,compiled}_frame
- static void verify_deopt_original_pc(nmethod* nm, intptr_t* unextended_sp, bool is_method_handle_return = false);
- static void verify_deopt_mh_original_pc(nmethod* nm, intptr_t* unextended_sp) {
- verify_deopt_original_pc(nm, unextended_sp, true);
- }
-#endif
-
public:
// Constructors
diff --git a/src/hotspot/cpu/arm/frame_arm.inline.hpp b/src/hotspot/cpu/arm/frame_arm.inline.hpp
index 4be190f0504..42e034cfdc7 100644
--- a/src/hotspot/cpu/arm/frame_arm.inline.hpp
+++ b/src/hotspot/cpu/arm/frame_arm.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2008, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -112,8 +112,6 @@ inline void frame::init(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, add
}
inline void frame::setup(address pc) {
- adjust_unextended_sp();
-
address original_pc = get_deopt_original_pc();
if (original_pc != nullptr) {
_pc = original_pc;
diff --git a/src/hotspot/cpu/arm/gc/g1/g1BarrierSetAssembler_arm.cpp b/src/hotspot/cpu/arm/gc/g1/g1BarrierSetAssembler_arm.cpp
index 049477cda76..74a9f273a43 100644
--- a/src/hotspot/cpu/arm/gc/g1/g1BarrierSetAssembler_arm.cpp
+++ b/src/hotspot/cpu/arm/gc/g1/g1BarrierSetAssembler_arm.cpp
@@ -198,15 +198,18 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
__ bind(done);
}
-static void generate_post_barrier_fast_path(MacroAssembler* masm,
- const Register store_addr,
- const Register new_val,
- const Register tmp1,
- const Register tmp2,
- Label& done,
- bool new_val_may_be_null) {
- // Does store cross heap regions?
+static void generate_post_barrier(MacroAssembler* masm,
+ const Register store_addr,
+ const Register new_val,
+ const Register thread,
+ const Register tmp1,
+ const Register tmp2,
+ Label& done,
+ bool new_val_may_be_null) {
+ assert(thread == Rthread, "must be");
+ assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, noreg);
+ // Does store cross heap regions?
__ eor(tmp1, store_addr, new_val);
__ movs(tmp1, AsmOperand(tmp1, lsr, G1HeapRegion::LogOfHRGrainBytes));
__ b(done, eq);
@@ -215,76 +218,34 @@ static void generate_post_barrier_fast_path(MacroAssembler* masm,
if (new_val_may_be_null) {
__ cbz(new_val, done);
}
- // storing region crossing non-null, is card already dirty?
- const Register card_addr = tmp1;
- CardTableBarrierSet* ct = barrier_set_cast(BarrierSet::barrier_set());
- __ mov_address(tmp2, (address)ct->card_table()->byte_map_base());
- __ add(card_addr, tmp2, AsmOperand(store_addr, lsr, CardTable::card_shift()));
+ // storing region crossing non-null, is card already non-clean?
+ Address card_table_addr(thread, in_bytes(G1ThreadLocalData::card_table_base_offset()));
+ __ ldr(tmp2, card_table_addr);
+ __ add(tmp1, tmp2, AsmOperand(store_addr, lsr, CardTable::card_shift()));
- __ ldrb(tmp2, Address(card_addr));
- __ cmp(tmp2, (int)G1CardTable::g1_young_card_val());
+ if (UseCondCardMark) {
+ __ ldrb(tmp2, Address(tmp1));
+ // Instead of loading clean_card_val and comparing, we exploit the fact that
+ // the LSB of non-clean cards is always 0, and the LSB of clean cards 1.
+ __ tbz(tmp2, 0, done);
+ }
+
+ static_assert(G1CardTable::dirty_card_val() == 0, "must be to use zero_register()");
+ __ zero_register(tmp2);
+ __ strb(tmp2, Address(tmp1)); // *(card address) := dirty_card_val
}
-static void generate_post_barrier_slow_path(MacroAssembler* masm,
- const Register thread,
- const Register tmp1,
- const Register tmp2,
- const Register tmp3,
- Label& done,
- Label& runtime) {
- __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad), tmp2);
- assert(CardTable::dirty_card_val() == 0, "adjust this code");
- // card_addr is loaded by generate_post_barrier_fast_path
- const Register card_addr = tmp1;
- __ ldrb(tmp2, Address(card_addr));
- __ cbz(tmp2, done);
-
- // storing a region crossing, non-null oop, card is clean.
- // dirty card and log.
-
- __ strb(__ zero_register(tmp2), Address(card_addr));
- generate_queue_test_and_insertion(masm,
- G1ThreadLocalData::dirty_card_queue_index_offset(),
- G1ThreadLocalData::dirty_card_queue_buffer_offset(),
- runtime,
- thread, card_addr, tmp2, tmp3);
- __ b(done);
-}
-
-
// G1 post-barrier.
// Blows all volatile registers R0-R3, LR).
void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
- Register store_addr,
- Register new_val,
- Register tmp1,
- Register tmp2,
- Register tmp3) {
+ Register store_addr,
+ Register new_val,
+ Register tmp1,
+ Register tmp2,
+ Register tmp3) {
Label done;
- Label runtime;
-
- generate_post_barrier_fast_path(masm, store_addr, new_val, tmp1, tmp2, done, true /* new_val_may_be_null */);
- // If card is young, jump to done
- // card_addr and card are loaded by generate_post_barrier_fast_path
- const Register card = tmp2;
- const Register card_addr = tmp1;
- __ b(done, eq);
- generate_post_barrier_slow_path(masm, Rthread, card_addr, tmp2, tmp3, done, runtime);
-
- __ bind(runtime);
-
- RegisterSet set = RegisterSet(store_addr) | RegisterSet(R0, R3) | RegisterSet(R12);
- __ push(set);
-
- if (card_addr != R0) {
- __ mov(R0, card_addr);
- }
- __ mov(R1, Rthread);
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), R0, R1);
-
- __ pop(set);
-
+ generate_post_barrier(masm, store_addr, new_val, Rthread, tmp1, tmp2, done, true /* new_val_may_be_null */);
__ bind(done);
}
@@ -344,35 +305,10 @@ void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm,
Register tmp1,
Register tmp2,
Register tmp3,
- G1PostBarrierStubC2* stub) {
- assert(thread == Rthread, "must be");
- assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, noreg);
-
- stub->initialize_registers(thread, tmp1, tmp2, tmp3);
-
- bool new_val_may_be_null = (stub->barrier_data() & G1C2BarrierPostNotNull) == 0;
- generate_post_barrier_fast_path(masm, store_addr, new_val, tmp1, tmp2, *stub->continuation(), new_val_may_be_null);
- // If card is not young, jump to stub (slow path)
- __ b(*stub->entry(), ne);
-
- __ bind(*stub->continuation());
-}
-
-void G1BarrierSetAssembler::generate_c2_post_barrier_stub(MacroAssembler* masm,
- G1PostBarrierStubC2* stub) const {
- Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
- Label runtime;
- Register thread = stub->thread();
- Register tmp1 = stub->tmp1(); // tmp1 holds the card address.
- Register tmp2 = stub->tmp2();
- Register tmp3 = stub->tmp3();
-
- __ bind(*stub->entry());
- generate_post_barrier_slow_path(masm, thread, tmp1, tmp2, tmp3, *stub->continuation(), runtime);
-
- __ bind(runtime);
- generate_c2_barrier_runtime_call(masm, stub, tmp1, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), tmp2);
- __ b(*stub->continuation());
+ bool new_val_may_be_null) {
+ Label done;
+ generate_post_barrier(masm, store_addr, new_val, thread, tmp1, tmp2, done, new_val_may_be_null);
+ __ bind(done);
}
#endif // COMPILER2
@@ -463,20 +399,19 @@ void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrier
__ b(*stub->continuation());
}
-void G1BarrierSetAssembler::gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) {
- G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
- __ bind(*stub->entry());
- assert(stub->addr()->is_register(), "Precondition.");
- assert(stub->new_val()->is_register(), "Precondition.");
- Register new_val_reg = stub->new_val()->as_register();
- __ cbz(new_val_reg, *stub->continuation());
- ce->verify_reserved_argument_area_size(1);
- __ str(stub->addr()->as_pointer_register(), Address(SP));
- __ call(bs->post_barrier_c1_runtime_code_blob()->code_begin(), relocInfo::runtime_call_type);
- __ b(*stub->continuation());
+#undef __
+
+void G1BarrierSetAssembler::g1_write_barrier_post_c1(MacroAssembler* masm,
+ Register store_addr,
+ Register new_val,
+ Register thread,
+ Register tmp1,
+ Register tmp2) {
+ Label done;
+ generate_post_barrier(masm, store_addr, new_val, thread, tmp1, tmp2, done, true /* new_val_may_be_null */);
+ masm->bind(done);
}
-#undef __
#define __ sasm->
void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
@@ -536,102 +471,6 @@ void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler*
__ b(done);
}
-void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) {
- // Input:
- // - store_addr, pushed on the stack
-
- __ set_info("g1_post_barrier_slow_id", false);
-
- Label done;
- Label recheck;
- Label runtime;
-
- Address queue_index(Rthread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
- Address buffer(Rthread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
-
- AddressLiteral cardtable(ci_card_table_address_as(), relocInfo::none);
-
- // save at least the registers that need saving if the runtime is called
- const RegisterSet saved_regs = RegisterSet(R0,R3) | RegisterSet(R12) | RegisterSet(LR);
- const int nb_saved_regs = 6;
- assert(nb_saved_regs == saved_regs.size(), "fix nb_saved_regs");
- __ push(saved_regs);
-
- const Register r_card_addr_0 = R0; // must be R0 for the slow case
- const Register r_obj_0 = R0;
- const Register r_card_base_1 = R1;
- const Register r_tmp2 = R2;
- const Register r_index_2 = R2;
- const Register r_buffer_3 = R3;
- const Register tmp1 = Rtemp;
-
- __ ldr(r_obj_0, Address(SP, nb_saved_regs*wordSize));
- // Note: there is a comment in x86 code about not using
- // ExternalAddress / lea, due to relocation not working
- // properly for that address. Should be OK for arm, where we
- // explicitly specify that 'cardtable' has a relocInfo::none
- // type.
- __ lea(r_card_base_1, cardtable);
- __ add(r_card_addr_0, r_card_base_1, AsmOperand(r_obj_0, lsr, CardTable::card_shift()));
-
- // first quick check without barrier
- __ ldrb(r_tmp2, Address(r_card_addr_0));
-
- __ cmp(r_tmp2, (int)G1CardTable::g1_young_card_val());
- __ b(recheck, ne);
-
- __ bind(done);
-
- __ pop(saved_regs);
-
- __ ret();
-
- __ bind(recheck);
-
- __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad), tmp1);
-
- // reload card state after the barrier that ensures the stored oop was visible
- __ ldrb(r_tmp2, Address(r_card_addr_0));
-
- assert(CardTable::dirty_card_val() == 0, "adjust this code");
- __ cbz(r_tmp2, done);
-
- // storing region crossing non-null, card is clean.
- // dirty card and log.
-
- assert(0 == (int)CardTable::dirty_card_val(), "adjust this code");
- if ((ci_card_table_address_as() & 0xff) == 0) {
- // Card table is aligned so the lowest byte of the table address base is zero.
- __ strb(r_card_base_1, Address(r_card_addr_0));
- } else {
- __ strb(__ zero_register(r_tmp2), Address(r_card_addr_0));
- }
-
- __ ldr(r_index_2, queue_index);
- __ ldr(r_buffer_3, buffer);
-
- __ subs(r_index_2, r_index_2, wordSize);
- __ b(runtime, lt); // go to runtime if now negative
-
- __ str(r_index_2, queue_index);
-
- __ str(r_card_addr_0, Address(r_buffer_3, r_index_2));
-
- __ b(done);
-
- __ bind(runtime);
-
- __ save_live_registers();
-
- assert(r_card_addr_0 == c_rarg0, "card_addr should be in R0");
- __ mov(c_rarg1, Rthread);
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), c_rarg0, c_rarg1);
-
- __ restore_live_registers_without_return();
-
- __ b(done);
-}
-
#undef __
#endif // COMPILER1
diff --git a/src/hotspot/cpu/arm/gc/g1/g1BarrierSetAssembler_arm.hpp b/src/hotspot/cpu/arm/gc/g1/g1BarrierSetAssembler_arm.hpp
index 4e49e655e3e..9e0eff4601b 100644
--- a/src/hotspot/cpu/arm/gc/g1/g1BarrierSetAssembler_arm.hpp
+++ b/src/hotspot/cpu/arm/gc/g1/g1BarrierSetAssembler_arm.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -32,9 +32,7 @@
class LIR_Assembler;
class StubAssembler;
class G1PreBarrierStub;
-class G1PostBarrierStub;
class G1PreBarrierStubC2;
-class G1PostBarrierStubC2;
class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
protected:
@@ -66,10 +64,15 @@ public:
#ifdef COMPILER1
public:
void gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub);
- void gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub);
void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm);
- void generate_c1_post_barrier_runtime_stub(StubAssembler* sasm);
+
+ void g1_write_barrier_post_c1(MacroAssembler* masm,
+ Register store_addr,
+ Register new_val,
+ Register thread,
+ Register tmp1,
+ Register tmp2);
#endif
#ifdef COMPILER2
@@ -89,9 +92,7 @@ public:
Register tmp1,
Register tmp2,
Register tmp3,
- G1PostBarrierStubC2* c2_stub);
- void generate_c2_post_barrier_stub(MacroAssembler* masm,
- G1PostBarrierStubC2* stub) const;
+ bool new_val_may_be_null);
#endif
};
diff --git a/src/hotspot/cpu/arm/gc/g1/g1_arm.ad b/src/hotspot/cpu/arm/gc/g1/g1_arm.ad
index 8a0a9e1aa53..e905ba9ff67 100644
--- a/src/hotspot/cpu/arm/gc/g1/g1_arm.ad
+++ b/src/hotspot/cpu/arm/gc/g1/g1_arm.ad
@@ -1,5 +1,5 @@
//
-// Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
@@ -63,13 +63,13 @@ static void write_barrier_post(MacroAssembler* masm,
Register tmp1,
Register tmp2,
Register tmp3) {
- if (!G1PostBarrierStubC2::needs_barrier(node)) {
+ if (!G1BarrierStubC2::needs_post_barrier(node)) {
return;
}
Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
G1BarrierSetAssembler* g1_asm = static_cast(BarrierSet::barrier_set()->barrier_set_assembler());
- G1PostBarrierStubC2* const stub = G1PostBarrierStubC2::create(node);
- g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, Rthread, tmp1, tmp2, tmp3, stub);
+ bool new_val_may_be_null = G1BarrierStubC2::post_new_val_may_be_null(node);
+ g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, Rthread, tmp1, tmp2, tmp3, new_val_may_be_null);
}
%}
diff --git a/src/hotspot/cpu/arm/gc/shared/barrierSetNMethod_arm.cpp b/src/hotspot/cpu/arm/gc/shared/barrierSetNMethod_arm.cpp
index 81b14f28c35..2c93b5ac549 100644
--- a/src/hotspot/cpu/arm/gc/shared/barrierSetNMethod_arm.cpp
+++ b/src/hotspot/cpu/arm/gc/shared/barrierSetNMethod_arm.cpp
@@ -48,22 +48,22 @@ class NativeNMethodBarrier: public NativeInstruction {
public:
int get_value() {
- return Atomic::load_acquire(guard_addr());
+ return AtomicAccess::load_acquire(guard_addr());
}
void set_value(int value, int bit_mask) {
if (bit_mask == ~0) {
- Atomic::release_store(guard_addr(), value);
+ AtomicAccess::release_store(guard_addr(), value);
return;
}
assert((value & ~bit_mask) == 0, "trying to set bits outside the mask");
value &= bit_mask;
- int old_value = Atomic::load(guard_addr());
+ int old_value = AtomicAccess::load(guard_addr());
while (true) {
// Only bits in the mask are changed
int new_value = value | (old_value & ~bit_mask);
if (new_value == old_value) break;
- int v = Atomic::cmpxchg(guard_addr(), old_value, new_value, memory_order_release);
+ int v = AtomicAccess::cmpxchg(guard_addr(), old_value, new_value, memory_order_release);
if (v == old_value) break;
old_value = v;
}
diff --git a/src/hotspot/cpu/arm/register_arm.hpp b/src/hotspot/cpu/arm/register_arm.hpp
index fca23d07fee..e0688af0d36 100644
--- a/src/hotspot/cpu/arm/register_arm.hpp
+++ b/src/hotspot/cpu/arm/register_arm.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2008, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -364,7 +364,6 @@ class VFPSystemRegisterImpl : public AbstractRegisterImpl {
// This does not seem to conflict with Rexception_pc
// In case of issues, R3 might be OK but adapters calling the runtime would have to save it
#define R5_mh R5 // MethodHandle register, used during the call setup
-#define Rmh_SP_save FP // for C1
/*
* C++ Interpreter Register Defines
diff --git a/src/hotspot/cpu/arm/runtime_arm.cpp b/src/hotspot/cpu/arm/runtime_arm.cpp
index ea4c9a76381..8d48de5795a 100644
--- a/src/hotspot/cpu/arm/runtime_arm.cpp
+++ b/src/hotspot/cpu/arm/runtime_arm.cpp
@@ -264,11 +264,6 @@ ExceptionBlob* OptoRuntime::generate_exception_blob() {
__ raw_pop(FP, LR);
- // Restore SP from its saved reg (FP) if the exception PC is a MethodHandle call site.
- __ ldr(Rtemp, Address(Rthread, JavaThread::is_method_handle_return_offset()));
- __ cmp(Rtemp, 0);
- __ mov(SP, Rmh_SP_save, ne);
-
// R0 contains handler address
// Since this may be the deopt blob we must set R5 to look like we returned
// from the original pc that threw the exception
diff --git a/src/hotspot/cpu/arm/sharedRuntime_arm.cpp b/src/hotspot/cpu/arm/sharedRuntime_arm.cpp
index dcf631525ab..99d8773368d 100644
--- a/src/hotspot/cpu/arm/sharedRuntime_arm.cpp
+++ b/src/hotspot/cpu/arm/sharedRuntime_arm.cpp
@@ -617,11 +617,11 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
int comp_args_on_stack,
const BasicType *sig_bt,
const VMRegPair *regs,
- AdapterHandlerEntry* handler) {
- address i2c_entry = __ pc();
+ address entry_address[AdapterBlob::ENTRY_COUNT]) {
+ entry_address[AdapterBlob::I2C] = __ pc();
gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs);
- address c2i_unverified_entry = __ pc();
+ entry_address[AdapterBlob::C2I_Unverified] = __ pc();
Label skip_fixup;
const Register receiver = R0;
const Register holder_klass = Rtemp; // XXX should be OK for C2 but not 100% sure
@@ -634,10 +634,9 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
__ b(skip_fixup, eq);
__ jump(SharedRuntime::get_ic_miss_stub(), relocInfo::runtime_call_type, noreg, ne);
- address c2i_entry = __ pc();
+ entry_address[AdapterBlob::C2I] = __ pc();
+ entry_address[AdapterBlob::C2I_No_Clinit_Check] = nullptr;
gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup);
-
- handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, nullptr);
return;
}
@@ -1129,7 +1128,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
const Register sync_handle = R5;
const Register sync_obj = R6;
- const Register disp_hdr = altFP_7_11;
+ const Register basic_lock = altFP_7_11;
const Register tmp = R8;
Label slow_lock, lock_done, fast_lock;
@@ -1140,7 +1139,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ mov(sync_handle, R1);
log_trace(fastlock)("SharedRuntime lock fast");
- __ lightweight_lock(sync_obj /* object */, disp_hdr /* t1 */, tmp /* t2 */, Rtemp /* t3 */,
+ __ lightweight_lock(sync_obj /* object */, basic_lock /* t1 */, tmp /* t2 */, Rtemp /* t3 */,
0x7 /* savemask */, slow_lock);
// Fall through to lock_done
__ bind(lock_done);
@@ -1255,7 +1254,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
// last_Java_frame is already set, so do call_VM manually; no exception can occur
__ mov(R0, sync_obj);
- __ mov(R1, disp_hdr);
+ __ mov(R1, basic_lock);
__ mov(R2, Rthread);
__ call(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_locking_C));
@@ -1270,12 +1269,12 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
// Clear pending exception before reentering VM.
// Can store the oop in register since it is a leaf call.
- assert_different_registers(Rtmp_save1, sync_obj, disp_hdr);
+ assert_different_registers(Rtmp_save1, sync_obj, basic_lock);
__ ldr(Rtmp_save1, Address(Rthread, Thread::pending_exception_offset()));
Register zero = __ zero_register(Rtemp);
__ str(zero, Address(Rthread, Thread::pending_exception_offset()));
__ mov(R0, sync_obj);
- __ mov(R1, disp_hdr);
+ __ mov(R1, basic_lock);
__ mov(R2, Rthread);
__ call(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_unlocking_C));
__ str(Rtmp_save1, Address(Rthread, Thread::pending_exception_offset()));
diff --git a/src/hotspot/cpu/arm/stubGenerator_arm.cpp b/src/hotspot/cpu/arm/stubGenerator_arm.cpp
index b81400ae877..a36ad3a0c47 100644
--- a/src/hotspot/cpu/arm/stubGenerator_arm.cpp
+++ b/src/hotspot/cpu/arm/stubGenerator_arm.cpp
@@ -421,7 +421,8 @@ class StubGenerator: public StubCodeGenerator {
}
- // As per atomic.hpp the Atomic read-modify-write operations must be logically implemented as:
+ // As per atomicAccess.hpp the atomic read-modify-write operations must be
+ // logically implemented as:
// ; ;
// But for load-linked/store-conditional based systems a fence here simply means
// no load/store can be reordered with respect to the initial load-linked, so we have:
@@ -440,7 +441,7 @@ class StubGenerator: public StubCodeGenerator {
// be removed in the future.
// Implementation of atomic_add(jint add_value, volatile jint* dest)
- // used by Atomic::add(volatile jint* dest, jint add_value)
+ // used by AtomicAccess::add(volatile jint* dest, jint add_value)
//
// Arguments :
//
@@ -492,7 +493,7 @@ class StubGenerator: public StubCodeGenerator {
}
// Implementation of jint atomic_xchg(jint exchange_value, volatile jint* dest)
- // used by Atomic::add(volatile jint* dest, jint exchange_value)
+ // used by AtomicAccess::add(volatile jint* dest, jint exchange_value)
//
// Arguments :
//
@@ -542,7 +543,7 @@ class StubGenerator: public StubCodeGenerator {
}
// Implementation of jint atomic_cmpxchg(jint exchange_value, volatile jint *dest, jint compare_value)
- // used by Atomic::cmpxchg(volatile jint *dest, jint compare_value, jint exchange_value)
+ // used by AtomicAccess::cmpxchg(volatile jint *dest, jint compare_value, jint exchange_value)
//
// Arguments :
//
@@ -582,7 +583,7 @@ class StubGenerator: public StubCodeGenerator {
return start;
}
- // Support for jlong Atomic::cmpxchg(jlong exchange_value, volatile jlong *dest, jlong compare_value)
+ // Support for jlong AtomicAccess::cmpxchg(jlong exchange_value, volatile jlong *dest, jlong compare_value)
// reordered before by a wrapper to (jlong compare_value, jlong exchange_value, volatile jlong *dest)
//
// Arguments :
@@ -3010,6 +3011,10 @@ class StubGenerator: public StubCodeGenerator {
// Note: the disjoint stubs must be generated first, some of
// the conjoint stubs use them.
+ // Note: chaining of stubs does not rely on branching to an
+ // auxiliary post-push entry because none of the stubs
+ // push/pop a frame.
+
// these need always status in case they are called from generic_arraycopy
StubRoutines::_jbyte_disjoint_arraycopy = generate_primitive_copy(StubId::stubgen_jbyte_disjoint_arraycopy_id);
StubRoutines::_jshort_disjoint_arraycopy = generate_primitive_copy(StubId::stubgen_jshort_disjoint_arraycopy_id);
@@ -3023,6 +3028,7 @@ class StubGenerator: public StubCodeGenerator {
StubRoutines::_arrayof_jlong_disjoint_arraycopy = generate_primitive_copy(StubId::stubgen_arrayof_jlong_disjoint_arraycopy_id);
StubRoutines::_arrayof_oop_disjoint_arraycopy = generate_oop_copy (StubId::stubgen_arrayof_oop_disjoint_arraycopy_id);
+ // disjoint copy entry is needed by conjoint copy
// these need always status in case they are called from generic_arraycopy
StubRoutines::_jbyte_arraycopy = generate_primitive_copy(StubId::stubgen_jbyte_arraycopy_id, StubRoutines::_jbyte_disjoint_arraycopy);
StubRoutines::_jshort_arraycopy = generate_primitive_copy(StubId::stubgen_jshort_arraycopy_id, StubRoutines::_jshort_disjoint_arraycopy);
diff --git a/src/hotspot/cpu/ppc/assembler_ppc.hpp b/src/hotspot/cpu/ppc/assembler_ppc.hpp
index aa330bfa1a7..15e38411482 100644
--- a/src/hotspot/cpu/ppc/assembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/assembler_ppc.hpp
@@ -2059,12 +2059,12 @@ class Assembler : public AbstractAssembler {
protected:
inline void tdi_unchecked(int tobits, Register a, int si16);
inline void twi_unchecked(int tobits, Register a, int si16);
+ public:
inline void tdi( int tobits, Register a, int si16); // asserts UseSIGTRAP
inline void twi( int tobits, Register a, int si16); // asserts UseSIGTRAP
inline void td( int tobits, Register a, Register b); // asserts UseSIGTRAP
inline void tw( int tobits, Register a, Register b); // asserts UseSIGTRAP
- public:
static bool is_tdi(int x, int tobits, int ra, int si16) {
return (TDI_OPCODE == (x & TDI_OPCODE_MASK))
&& (tobits == inv_to_field(x))
diff --git a/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp b/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp
index 24601c5d3b0..7e49ec7455d 100644
--- a/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp
+++ b/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp
@@ -26,7 +26,7 @@
#ifndef CPU_PPC_ASSEMBLER_PPC_INLINE_HPP
#define CPU_PPC_ASSEMBLER_PPC_INLINE_HPP
-#include "asm/assembler.inline.hpp"
+#include "asm/assembler.hpp"
#include "asm/codeBuffer.hpp"
#include "code/codeCache.hpp"
#include "runtime/vm_version.hpp"
diff --git a/src/hotspot/cpu/ppc/atomicAccess_ppc.hpp b/src/hotspot/cpu/ppc/atomicAccess_ppc.hpp
new file mode 100644
index 00000000000..a0ff19e6171
--- /dev/null
+++ b/src/hotspot/cpu/ppc/atomicAccess_ppc.hpp
@@ -0,0 +1,649 @@
+/*
+ * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef CPU_PPC_ATOMICACCESS_PPC_HPP
+#define CPU_PPC_ATOMICACCESS_PPC_HPP
+
+#ifndef PPC64
+#error "Atomic currently only implemented for PPC64"
+#endif
+
+#include "orderAccess_ppc.hpp"
+#include "utilities/debug.hpp"
+
+// Implementation of class AtomicAccess
+
+//
+// machine barrier instructions:
+//
+// - sync two-way memory barrier, aka fence
+// - lwsync orders Store|Store,
+// Load|Store,
+// Load|Load,
+// but not Store|Load
+// - eieio orders memory accesses for device memory (only)
+// - isync invalidates speculatively executed instructions
+// From the POWER ISA 2.06 documentation:
+// "[...] an isync instruction prevents the execution of
+// instructions following the isync until instructions
+// preceding the isync have completed, [...]"
+// From IBM's AIX assembler reference:
+// "The isync [...] instructions causes the processor to
+// refetch any instructions that might have been fetched
+// prior to the isync instruction. The instruction isync
+// causes the processor to wait for all previous instructions
+// to complete. Then any instructions already fetched are
+// discarded and instruction processing continues in the
+// environment established by the previous instructions."
+//
+// semantic barrier instructions:
+// (as defined in orderAccess.hpp)
+//
+// - release orders Store|Store, (maps to lwsync)
+// Load|Store
+// - acquire orders Load|Store, (maps to lwsync)
+// Load|Load
+// - fence orders Store|Store, (maps to sync)
+// Load|Store,
+// Load|Load,
+// Store|Load
+//
+
+inline void pre_membar(atomic_memory_order order) {
+ switch (order) {
+ case memory_order_relaxed:
+ case memory_order_acquire: break;
+ case memory_order_release:
+ case memory_order_acq_rel: __asm__ __volatile__ ("lwsync" : : : "memory"); break;
+ default /*conservative*/ : __asm__ __volatile__ ("sync" : : : "memory"); break;
+ }
+}
+
+inline void post_membar(atomic_memory_order order) {
+ switch (order) {
+ case memory_order_relaxed:
+ case memory_order_release: break;
+ case memory_order_acquire:
+ case memory_order_acq_rel: __asm__ __volatile__ ("isync" : : : "memory"); break;
+ default /*conservative*/ : __asm__ __volatile__ ("sync" : : : "memory"); break;
+ }
+}
+
+
+
+template
+struct AtomicAccess::PlatformAdd {
+ template
+ D add_then_fetch(D volatile* dest, I add_value, atomic_memory_order order) const;
+
+ template
+ D fetch_then_add(D volatile* dest, I add_value, atomic_memory_order order) const {
+ return add_then_fetch(dest, add_value, order) - add_value;
+ }
+};
+
+template<>
+template
+inline D AtomicAccess::PlatformAdd<4>::add_then_fetch(D volatile* dest, I add_value,
+ atomic_memory_order order) const {
+ STATIC_ASSERT(4 == sizeof(I));
+ STATIC_ASSERT(4 == sizeof(D));
+
+ D result;
+
+ pre_membar(order);
+
+ __asm__ __volatile__ (
+ "1: lwarx %[result], 0, %[dest] \n"
+ " add %[result], %[result], %[add_value] \n"
+ " stwcx. %[result], 0, %[dest] \n"
+ " bne- 1b \n"
+ : [result] "=&r" (result)
+ : [add_value] "r" (add_value),
+ [dest] "b" (dest)
+ : "cc", "memory" );
+
+ post_membar(order);
+
+ return result;
+}
+
+
+template<>
+template
+inline D AtomicAccess::PlatformAdd<8>::add_then_fetch(D volatile* dest, I add_value,
+ atomic_memory_order order) const {
+ STATIC_ASSERT(8 == sizeof(I));
+ STATIC_ASSERT(8 == sizeof(D));
+
+ D result;
+
+ pre_membar(order);
+
+ __asm__ __volatile__ (
+ "1: ldarx %[result], 0, %[dest] \n"
+ " add %[result], %[result], %[add_value] \n"
+ " stdcx. %[result], 0, %[dest] \n"
+ " bne- 1b \n"
+ : [result] "=&r" (result)
+ : [add_value] "r" (add_value),
+ [dest] "b" (dest)
+ : "cc", "memory" );
+
+ post_membar(order);
+
+ return result;
+}
+
+template<>
+template
+inline T AtomicAccess::PlatformXchg<4>::operator()(T volatile* dest,
+ T exchange_value,
+ atomic_memory_order order) const {
+ STATIC_ASSERT(4 == sizeof(T));
+ // Note that xchg doesn't necessarily do an acquire
+ // (see synchronizer.cpp).
+
+ T old_value;
+
+ pre_membar(order);
+
+ __asm__ __volatile__ (
+ /* atomic loop */
+ "1: \n"
+ " lwarx %[old_value], 0, %[dest] \n"
+ " stwcx. %[exchange_value], 0, %[dest] \n"
+ " bne- 1b \n"
+ /* exit */
+ "2: \n"
+ /* out */
+ : [old_value] "=&r" (old_value)
+ /* in */
+ : [dest] "b" (dest),
+ [exchange_value] "r" (exchange_value)
+ /* clobber */
+ : "cc",
+ "memory"
+ );
+
+ post_membar(order);
+
+ return old_value;
+}
+
+template<>
+template
+inline T AtomicAccess::PlatformXchg<8>::operator()(T volatile* dest,
+ T exchange_value,
+ atomic_memory_order order) const {
+ STATIC_ASSERT(8 == sizeof(T));
+ // Note that xchg doesn't necessarily do an acquire
+ // (see synchronizer.cpp).
+
+ T old_value;
+
+ pre_membar(order);
+
+ __asm__ __volatile__ (
+ /* atomic loop */
+ "1: \n"
+ " ldarx %[old_value], 0, %[dest] \n"
+ " stdcx. %[exchange_value], 0, %[dest] \n"
+ " bne- 1b \n"
+ /* exit */
+ "2: \n"
+ /* out */
+ : [old_value] "=&r" (old_value)
+ /* in */
+ : [dest] "b" (dest),
+ [exchange_value] "r" (exchange_value)
+ /* clobber */
+ : "cc",
+ "memory"
+ );
+
+ post_membar(order);
+
+ return old_value;
+}
+
+template<>
+template
+inline T AtomicAccess::PlatformCmpxchg<1>::operator()(T volatile* dest,
+ T compare_value,
+ T exchange_value,
+ atomic_memory_order order) const {
+ STATIC_ASSERT(1 == sizeof(T));
+
+ // Note that cmpxchg guarantees a two-way memory barrier across
+ // the cmpxchg, so it's really a 'fence_cmpxchg_fence' if not
+ // specified otherwise (see atomicAccess.hpp).
+
+ const unsigned int masked_compare_val = (unsigned int)(unsigned char)compare_value;
+
+ unsigned int old_value;
+
+ pre_membar(order);
+
+ __asm__ __volatile__ (
+ /* simple guard */
+ " lbz %[old_value], 0(%[dest]) \n"
+ " cmpw %[masked_compare_val], %[old_value] \n"
+ " bne- 2f \n"
+ /* atomic loop */
+ "1: \n"
+ " lbarx %[old_value], 0, %[dest] \n"
+ " cmpw %[masked_compare_val], %[old_value] \n"
+ " bne- 2f \n"
+ " stbcx. %[exchange_value], 0, %[dest] \n"
+ " bne- 1b \n"
+ /* exit */
+ "2: \n"
+ /* out */
+ : [old_value] "=&r" (old_value)
+ /* in */
+ : [dest] "b" (dest),
+ [masked_compare_val] "r" (masked_compare_val),
+ [exchange_value] "r" (exchange_value)
+ /* clobber */
+ : "cc",
+ "memory"
+ );
+
+ post_membar(order);
+
+ return PrimitiveConversions::cast((unsigned char)old_value);
+}
+
+template<>
+template
+inline T AtomicAccess::PlatformCmpxchg<4>::operator()(T volatile* dest,
+ T compare_value,
+ T exchange_value,
+ atomic_memory_order order) const {
+ STATIC_ASSERT(4 == sizeof(T));
+
+ // Note that cmpxchg guarantees a two-way memory barrier across
+ // the cmpxchg, so it's really a 'fence_cmpxchg_fence' if not
+ // specified otherwise (see atomicAccess.hpp).
+
+ T old_value;
+
+ pre_membar(order);
+
+ __asm__ __volatile__ (
+ /* simple guard */
+ " lwz %[old_value], 0(%[dest]) \n"
+ " cmpw %[compare_value], %[old_value] \n"
+ " bne- 2f \n"
+ /* atomic loop */
+ "1: \n"
+ " lwarx %[old_value], 0, %[dest] \n"
+ " cmpw %[compare_value], %[old_value] \n"
+ " bne- 2f \n"
+ " stwcx. %[exchange_value], 0, %[dest] \n"
+ " bne- 1b \n"
+ /* exit */
+ "2: \n"
+ /* out */
+ : [old_value] "=&r" (old_value)
+ /* in */
+ : [dest] "b" (dest),
+ [compare_value] "r" (compare_value),
+ [exchange_value] "r" (exchange_value)
+ /* clobber */
+ : "cc",
+ "memory"
+ );
+
+ post_membar(order);
+
+ return old_value;
+}
+
+template<>
+template
+inline T AtomicAccess::PlatformCmpxchg<8>::operator()(T volatile* dest,
+ T compare_value,
+ T exchange_value,
+ atomic_memory_order order) const {
+ STATIC_ASSERT(8 == sizeof(T));
+
+ // Note that cmpxchg guarantees a two-way memory barrier across
+ // the cmpxchg, so it's really a 'fence_cmpxchg_fence' if not
+ // specified otherwise (see atomicAccess.hpp).
+
+ T old_value;
+
+ pre_membar(order);
+
+ __asm__ __volatile__ (
+ /* simple guard */
+ " ld %[old_value], 0(%[dest]) \n"
+ " cmpd %[compare_value], %[old_value] \n"
+ " bne- 2f \n"
+ /* atomic loop */
+ "1: \n"
+ " ldarx %[old_value], 0, %[dest] \n"
+ " cmpd %[compare_value], %[old_value] \n"
+ " bne- 2f \n"
+ " stdcx. %[exchange_value], 0, %[dest] \n"
+ " bne- 1b \n"
+ /* exit */
+ "2: \n"
+ /* out */
+ : [old_value] "=&r" (old_value)
+ /* in */
+ : [dest] "b" (dest),
+ [compare_value] "r" (compare_value),
+ [exchange_value] "r" (exchange_value)
+ /* clobber */
+ : "cc",
+ "memory"
+ );
+
+ post_membar(order);
+
+ return old_value;
+}
+
+template
+struct AtomicAccess::PlatformOrderedLoad
+{
+ template
+ T operator()(const volatile T* p) const {
+ T t = AtomicAccess::load(p);
+ // Use twi-isync for load_acquire (faster than lwsync).
+ __asm__ __volatile__ ("twi 0,%0,0\n isync\n" : : "r" (t) : "memory");
+ return t;
+ }
+};
+
+template<>
+class AtomicAccess::PlatformBitops<4, true> {
+public:
+ template
+ T fetch_then_and(T volatile* dest, T bits, atomic_memory_order order) const {
+ STATIC_ASSERT(4 == sizeof(T));
+ T old_value, result;
+
+ pre_membar(order);
+
+ __asm__ __volatile__ (
+ "1: lwarx %[old_value], 0, %[dest] \n"
+ " and %[result], %[old_value], %[bits] \n"
+ " stwcx. %[result], 0, %[dest] \n"
+ " bne- 1b \n"
+ : [old_value] "=&r" (old_value),
+ [result] "=&r" (result)
+ : [dest] "b" (dest),
+ [bits] "r" (bits)
+ : "cc", "memory" );
+
+ post_membar(order);
+ return old_value;
+ }
+
+ template
+ T fetch_then_or(T volatile* dest, T bits, atomic_memory_order order) const {
+ STATIC_ASSERT(4 == sizeof(T));
+ T old_value, result;
+
+ pre_membar(order);
+
+ __asm__ __volatile__ (
+ "1: lwarx %[old_value], 0, %[dest] \n"
+ " or %[result], %[old_value], %[bits] \n"
+ " stwcx. %[result], 0, %[dest] \n"
+ " bne- 1b \n"
+ : [old_value] "=&r" (old_value),
+ [result] "=&r" (result)
+ : [dest] "b" (dest),
+ [bits] "r" (bits)
+ : "cc", "memory" );
+
+ post_membar(order);
+ return old_value;
+ }
+
+ template
+ T fetch_then_xor(T volatile* dest, T bits, atomic_memory_order order) const {
+ STATIC_ASSERT(4 == sizeof(T));
+ T old_value, result;
+
+ pre_membar(order);
+
+ __asm__ __volatile__ (
+ "1: lwarx %[old_value], 0, %[dest] \n"
+ " xor %[result], %[old_value], %[bits] \n"
+ " stwcx. %[result], 0, %[dest] \n"
+ " bne- 1b \n"
+ : [old_value] "=&r" (old_value),
+ [result] "=&r" (result)
+ : [dest] "b" (dest),
+ [bits] "r" (bits)
+ : "cc", "memory" );
+
+ post_membar(order);
+ return old_value;
+ }
+
+ template
+ T and_then_fetch(T volatile* dest, T bits, atomic_memory_order order) const {
+ STATIC_ASSERT(4 == sizeof(T));
+ T result;
+
+ pre_membar(order);
+
+ __asm__ __volatile__ (
+ "1: lwarx %[result], 0, %[dest] \n"
+ " and %[result], %[result], %[bits] \n"
+ " stwcx. %[result], 0, %[dest] \n"
+ " bne- 1b \n"
+ : [result] "=&r" (result)
+ : [dest] "b" (dest),
+ [bits] "r" (bits)
+ : "cc", "memory" );
+
+ post_membar(order);
+ return result;
+ }
+
+ template
+ T or_then_fetch(T volatile* dest, T bits, atomic_memory_order order) const {
+ STATIC_ASSERT(4 == sizeof(T));
+ T result;
+
+ pre_membar(order);
+
+ __asm__ __volatile__ (
+ "1: lwarx %[result], 0, %[dest] \n"
+ " or %[result], %[result], %[bits] \n"
+ " stwcx. %[result], 0, %[dest] \n"
+ " bne- 1b \n"
+ : [result] "=&r" (result)
+ : [dest] "b" (dest),
+ [bits] "r" (bits)
+ : "cc", "memory" );
+
+ post_membar(order);
+ return result;
+ }
+
+ template
+ T xor_then_fetch(T volatile* dest, T bits, atomic_memory_order order) const {
+ STATIC_ASSERT(4 == sizeof(T));
+ T result;
+
+ pre_membar(order);
+
+ __asm__ __volatile__ (
+ "1: lwarx %[result], 0, %[dest] \n"
+ " xor %[result], %[result], %[bits] \n"
+ " stwcx. %[result], 0, %[dest] \n"
+ " bne- 1b \n"
+ : [result] "=&r" (result)
+ : [dest] "b" (dest),
+ [bits] "r" (bits)
+ : "cc", "memory" );
+
+ post_membar(order);
+ return result;
+ }
+};
+
+template<>
+class AtomicAccess::PlatformBitops<8, true> {
+public:
+ template
+ T fetch_then_and(T volatile* dest, T bits, atomic_memory_order order) const {
+ STATIC_ASSERT(8 == sizeof(T));
+ T old_value, result;
+
+ pre_membar(order);
+
+ __asm__ __volatile__ (
+ "1: ldarx %[old_value], 0, %[dest] \n"
+ " and %[result], %[old_value], %[bits] \n"
+ " stdcx. %[result], 0, %[dest] \n"
+ " bne- 1b \n"
+ : [old_value] "=&r" (old_value),
+ [result] "=&r" (result)
+ : [dest] "b" (dest),
+ [bits] "r" (bits)
+ : "cc", "memory" );
+
+ post_membar(order);
+ return old_value;
+ }
+
+ template
+ T fetch_then_or(T volatile* dest, T bits, atomic_memory_order order) const {
+ STATIC_ASSERT(8 == sizeof(T));
+ T old_value, result;
+
+ pre_membar(order);
+
+ __asm__ __volatile__ (
+ "1: ldarx %[old_value], 0, %[dest] \n"
+ " or %[result], %[old_value], %[bits] \n"
+ " stdcx. %[result], 0, %[dest] \n"
+ " bne- 1b \n"
+ : [old_value] "=&r" (old_value),
+ [result] "=&r" (result)
+ : [dest] "b" (dest),
+ [bits] "r" (bits)
+ : "cc", "memory" );
+
+ post_membar(order);
+ return old_value;
+ }
+
+ template
+ T fetch_then_xor(T volatile* dest, T bits, atomic_memory_order order) const {
+ STATIC_ASSERT(8 == sizeof(T));
+ T old_value, result;
+
+ pre_membar(order);
+
+ __asm__ __volatile__ (
+ "1: ldarx %[old_value], 0, %[dest] \n"
+ " xor %[result], %[old_value], %[bits] \n"
+ " stdcx. %[result], 0, %[dest] \n"
+ " bne- 1b \n"
+ : [old_value] "=&r" (old_value),
+ [result] "=&r" (result)
+ : [dest] "b" (dest),
+ [bits] "r" (bits)
+ : "cc", "memory" );
+
+ post_membar(order);
+ return old_value;
+ }
+
+ template
+ T and_then_fetch(T volatile* dest, T bits, atomic_memory_order order) const {
+ STATIC_ASSERT(8 == sizeof(T));
+ T result;
+
+ pre_membar(order);
+
+ __asm__ __volatile__ (
+ "1: ldarx %[result], 0, %[dest] \n"
+ " and %[result], %[result], %[bits] \n"
+ " stdcx. %[result], 0, %[dest] \n"
+ " bne- 1b \n"
+ : [result] "=&r" (result)
+ : [dest] "b" (dest),
+ [bits] "r" (bits)
+ : "cc", "memory" );
+
+ post_membar(order);
+ return result;
+ }
+
+ template
+ T or_then_fetch(T volatile* dest, T bits, atomic_memory_order order) const {
+ STATIC_ASSERT(8 == sizeof(T));
+ T result;
+
+ pre_membar(order);
+
+ __asm__ __volatile__ (
+ "1: ldarx %[result], 0, %[dest] \n"
+ " or %[result], %[result], %[bits] \n"
+ " stdcx. %[result], 0, %[dest] \n"
+ " bne- 1b \n"
+ : [result] "=&r" (result)
+ : [dest] "b" (dest),
+ [bits] "r" (bits)
+ : "cc", "memory" );
+
+ post_membar(order);
+ return result;
+ }
+
+ template
+ T xor_then_fetch(T volatile* dest, T bits, atomic_memory_order order) const {
+ STATIC_ASSERT(8 == sizeof(T));
+ T result;
+
+ pre_membar(order);
+
+ __asm__ __volatile__ (
+ "1: ldarx %[result], 0, %[dest] \n"
+ " xor %[result], %[result], %[bits] \n"
+ " stdcx. %[result], 0, %[dest] \n"
+ " bne- 1b \n"
+ : [result] "=&r" (result)
+ : [dest] "b" (dest),
+ [bits] "r" (bits)
+ : "cc", "memory" );
+
+ post_membar(order);
+ return result;
+ }
+};
+#endif // CPU_PPC_ATOMICACCESS_PPC_HPP
diff --git a/src/hotspot/cpu/ppc/c1_CodeStubs_ppc.cpp b/src/hotspot/cpu/ppc/c1_CodeStubs_ppc.cpp
index 438521b0a9b..61780a73969 100644
--- a/src/hotspot/cpu/ppc/c1_CodeStubs_ppc.cpp
+++ b/src/hotspot/cpu/ppc/c1_CodeStubs_ppc.cpp
@@ -268,9 +268,10 @@ void MonitorEnterStub::emit_code(LIR_Assembler* ce) {
void MonitorExitStub::emit_code(LIR_Assembler* ce) {
__ bind(_entry);
- if (_compute_lock) {
- ce->monitor_address(_monitor_ix, _lock_reg);
- }
+
+ // lock_reg was destroyed by fast unlocking attempt => recompute it
+ ce->monitor_address(_monitor_ix, _lock_reg);
+
address stub = Runtime1::entry_for(ce->compilation()->has_fpu_code() ? StubId::c1_monitorexit_id : StubId::c1_monitorexit_nofpu_id);
//__ load_const_optimized(R0, stub);
__ add_const_optimized(R0, R29_TOC, MacroAssembler::offset_to_global_toc(stub));
diff --git a/src/hotspot/cpu/ppc/c1_FrameMap_ppc.cpp b/src/hotspot/cpu/ppc/c1_FrameMap_ppc.cpp
index 8ce324a570b..cb9368f2ce9 100644
--- a/src/hotspot/cpu/ppc/c1_FrameMap_ppc.cpp
+++ b/src/hotspot/cpu/ppc/c1_FrameMap_ppc.cpp
@@ -374,15 +374,6 @@ LIR_Opr FrameMap::stack_pointer() {
return SP_opr;
}
-
-// JSR 292
-// On PPC64, there is no need to save the SP, because neither
-// method handle intrinsics, nor compiled lambda forms modify it.
-LIR_Opr FrameMap::method_handle_invoke_SP_save_opr() {
- return LIR_OprFact::illegalOpr;
-}
-
-
bool FrameMap::validate_frame() {
int max_offset = in_bytes(framesize_in_bytes());
int java_index = 0;
diff --git a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp
index 3ca75305eca..108da2039f6 100644
--- a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp
@@ -227,7 +227,7 @@ int LIR_Assembler::emit_unwind_handler() {
MonitorExitStub* stub = nullptr;
if (method()->is_synchronized()) {
monitor_address(0, FrameMap::R4_opr);
- stub = new MonitorExitStub(FrameMap::R4_opr, true, 0);
+ stub = new MonitorExitStub(FrameMap::R4_opr, 0);
__ unlock_object(R5, R6, R4, *stub->entry());
__ bind(*stub->continuation());
}
@@ -2614,7 +2614,6 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
// Obj may not be an oop.
if (op->code() == lir_lock) {
MonitorEnterStub* stub = (MonitorEnterStub*)op->stub();
- assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
// Add debug info for NullPointerException only if one is possible.
if (op->info() != nullptr) {
if (!os::zero_page_read_protected() || !ImplicitNullChecks) {
@@ -2626,7 +2625,6 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
__ lock_object(hdr, obj, lock, op->scratch_opr()->as_register(), *op->stub()->entry());
} else {
assert (op->code() == lir_unlock, "Invalid code, expected lir_unlock");
- assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
__ unlock_object(hdr, obj, lock, *op->stub()->entry());
}
__ bind(*op->stub()->continuation());
diff --git a/src/hotspot/cpu/ppc/gc/g1/g1BarrierSetAssembler_ppc.cpp b/src/hotspot/cpu/ppc/gc/g1/g1BarrierSetAssembler_ppc.cpp
index 4fb13422f59..65bfd683abd 100644
--- a/src/hotspot/cpu/ppc/gc/g1/g1BarrierSetAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/gc/g1/g1BarrierSetAssembler_ppc.cpp
@@ -28,7 +28,6 @@
#include "gc/g1/g1BarrierSetAssembler.hpp"
#include "gc/g1/g1BarrierSetRuntime.hpp"
#include "gc/g1/g1CardTable.hpp"
-#include "gc/g1/g1DirtyCardQueue.hpp"
#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1SATBMarkQueueSet.hpp"
#include "gc/g1/g1ThreadLocalData.hpp"
@@ -230,78 +229,52 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, Decorator
__ bind(filtered);
}
-static void generate_region_crossing_test(MacroAssembler* masm, const Register store_addr, const Register new_val) {
- __ xorr(R0, store_addr, new_val); // tmp1 := store address ^ new value
- __ srdi_(R0, R0, G1HeapRegion::LogOfHRGrainBytes); // tmp1 := ((store address ^ new value) >> LogOfHRGrainBytes)
-}
+static void generate_post_barrier(MacroAssembler* masm,
+ const Register store_addr,
+ const Register new_val,
+ const Register thread,
+ const Register tmp1,
+ const Register tmp2,
+ Label& done,
+ bool new_val_may_be_null) {
+ assert_different_registers(store_addr, new_val, tmp1, R0);
+ assert_different_registers(store_addr, tmp1, tmp2, R0);
-static Address generate_card_young_test(MacroAssembler* masm, const Register store_addr, const Register tmp1, const Register tmp2) {
- CardTableBarrierSet* ct = barrier_set_cast(BarrierSet::barrier_set());
- __ load_const_optimized(tmp1, (address)(ct->card_table()->byte_map_base()), tmp2);
- __ srdi(tmp2, store_addr, CardTable::card_shift()); // tmp1 := card address relative to card table base
- __ lbzx(R0, tmp1, tmp2); // tmp1 := card address
- __ cmpwi(CR0, R0, (int)G1CardTable::g1_young_card_val());
- return Address(tmp1, tmp2); // return card address
-}
+ __ xorr(R0, store_addr, new_val); // R0 := store address ^ new value
+ __ srdi_(R0, R0, G1HeapRegion::LogOfHRGrainBytes); // R0 := ((store address ^ new value) >> LogOfHRGrainBytes)
+ __ beq(CR0, done);
-static void generate_card_dirty_test(MacroAssembler* masm, Address card_addr) {
- __ membar(Assembler::StoreLoad); // Must reload after StoreLoad membar due to concurrent refinement
- __ lbzx(R0, card_addr.base(), card_addr.index()); // tmp2 := card
- __ cmpwi(CR0, R0, (int)G1CardTable::dirty_card_val()); // tmp2 := card == dirty_card_val?
+ // Crosses regions, storing null?
+ if (!new_val_may_be_null) {
+#ifdef ASSERT
+ __ cmpdi(CR0, new_val, 0);
+ __ asm_assert_ne("null oop not allowed (G1 post)"); // Checked by caller.
+#endif
+ } else {
+ __ cmpdi(CR0, new_val, 0);
+ __ beq(CR0, done);
+ }
+
+ __ ld(tmp1, G1ThreadLocalData::card_table_base_offset(), thread);
+ __ srdi(tmp2, store_addr, CardTable::card_shift()); // tmp2 := card address relative to card table base
+ if (UseCondCardMark) {
+ __ lbzx(R0, tmp1, tmp2);
+ __ cmpwi(CR0, R0, (int)G1CardTable::clean_card_val());
+ __ bne(CR0, done);
+ }
+
+ __ li(R0, G1CardTable::dirty_card_val());
+ __ stbx(R0, tmp1, tmp2);
}
void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm, DecoratorSet decorators,
Register store_addr, Register new_val,
- Register tmp1, Register tmp2, Register tmp3,
- MacroAssembler::PreservationLevel preservation_level) {
+ Register tmp1, Register tmp2) {
bool not_null = (decorators & IS_NOT_NULL) != 0;
- Label runtime, filtered;
- assert_different_registers(store_addr, new_val, tmp1, tmp2);
-
- CardTableBarrierSet* ct = barrier_set_cast(BarrierSet::barrier_set());
-
- generate_region_crossing_test(masm, store_addr, new_val);
- __ beq(CR0, filtered);
-
- // Crosses regions, storing null?
- if (not_null) {
-#ifdef ASSERT
- __ cmpdi(CR0, new_val, 0);
- __ asm_assert_ne("null oop not allowed (G1 post)"); // Checked by caller.
-#endif
- } else {
- __ cmpdi(CR0, new_val, 0);
- __ beq(CR0, filtered);
- }
-
- Address card_addr = generate_card_young_test(masm, store_addr, tmp1, tmp2);
- __ beq(CR0, filtered);
-
- generate_card_dirty_test(masm, card_addr);
- __ beq(CR0, filtered);
-
- __ li(R0, (int)G1CardTable::dirty_card_val());
- __ stbx(R0, card_addr.base(), card_addr.index()); // *(card address) := dirty_card_val
-
- Register Rcard_addr = tmp3;
- __ add(Rcard_addr, card_addr.base(), card_addr.index()); // This is the address which needs to get enqueued.
-
- generate_queue_insertion(masm,
- G1ThreadLocalData::dirty_card_queue_index_offset(),
- G1ThreadLocalData::dirty_card_queue_buffer_offset(),
- runtime, Rcard_addr, tmp1);
- __ b(filtered);
-
- __ bind(runtime);
-
- assert(preservation_level == MacroAssembler::PRESERVATION_NONE,
- "g1_write_barrier_post doesn't support preservation levels higher than PRESERVATION_NONE");
-
- // Save the live input values.
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), Rcard_addr, R16_thread);
-
- __ bind(filtered);
+ Label done;
+ generate_post_barrier(masm, store_addr, new_val, R16_thread, tmp1, tmp2, done, !not_null);
+ __ bind(done);
}
void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
@@ -333,8 +306,7 @@ void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet deco
}
g1_write_barrier_post(masm, decorators,
base, val,
- tmp1, tmp2, tmp3,
- preservation_level);
+ tmp1, tmp2);
}
}
@@ -457,70 +429,29 @@ void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm,
Register new_val,
Register tmp1,
Register tmp2,
- G1PostBarrierStubC2* stub,
+ bool new_val_may_be_null,
bool decode_new_val) {
assert_different_registers(store_addr, new_val, tmp1, R0);
assert_different_registers(store_addr, tmp1, tmp2, R0);
- stub->initialize_registers(R16_thread, tmp1, tmp2);
+ Label done;
- bool null_check_required = (stub->barrier_data() & G1C2BarrierPostNotNull) == 0;
Register new_val_decoded = new_val;
if (decode_new_val) {
assert(UseCompressedOops, "or should not be here");
- if (null_check_required && CompressedOops::base() != nullptr) {
+ if (new_val_may_be_null && CompressedOops::base() != nullptr) {
// We prefer doing the null check after the region crossing check.
// Only compressed oop modes with base != null require a null check here.
__ cmpwi(CR0, new_val, 0);
- __ beq(CR0, *stub->continuation());
- null_check_required = false;
+ __ beq(CR0, done);
+ new_val_may_be_null = false;
}
new_val_decoded = __ decode_heap_oop_not_null(tmp2, new_val);
}
- generate_region_crossing_test(masm, store_addr, new_val_decoded);
- __ beq(CR0, *stub->continuation());
-
- // crosses regions, storing null?
- if (null_check_required) {
- __ cmpdi(CR0, new_val_decoded, 0);
- __ beq(CR0, *stub->continuation());
- }
-
- Address card_addr = generate_card_young_test(masm, store_addr, tmp1, tmp2);
- assert(card_addr.base() == tmp1 && card_addr.index() == tmp2, "needed by post barrier stub");
- __ bc_far_optimized(Assembler::bcondCRbiIs0, __ bi0(CR0, Assembler::equal), *stub->entry());
-
- __ bind(*stub->continuation());
-}
-
-void G1BarrierSetAssembler::generate_c2_post_barrier_stub(MacroAssembler* masm,
- G1PostBarrierStubC2* stub) const {
- Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
- Label runtime;
- Address card_addr(stub->tmp1(), stub->tmp2()); // See above.
-
- __ bind(*stub->entry());
-
- generate_card_dirty_test(masm, card_addr);
- __ bc_far_optimized(Assembler::bcondCRbiIs1, __ bi0(CR0, Assembler::equal), *stub->continuation());
-
- __ li(R0, (int)G1CardTable::dirty_card_val());
- __ stbx(R0, card_addr.base(), card_addr.index()); // *(card address) := dirty_card_val
-
- Register Rcard_addr = stub->tmp1();
- __ add(Rcard_addr, card_addr.base(), card_addr.index()); // This is the address which needs to get enqueued.
-
- generate_queue_insertion(masm,
- G1ThreadLocalData::dirty_card_queue_index_offset(),
- G1ThreadLocalData::dirty_card_queue_buffer_offset(),
- runtime, Rcard_addr, stub->tmp2());
- __ b(*stub->continuation());
-
- __ bind(runtime);
- generate_c2_barrier_runtime_call(masm, stub, Rcard_addr, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry));
- __ b(*stub->continuation());
+ generate_post_barrier(masm, store_addr, new_val_decoded, R16_thread, tmp1, tmp2, done, new_val_may_be_null);
+ __ bind(done);
}
#endif // COMPILER2
@@ -558,28 +489,19 @@ void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrier
__ b(*stub->continuation());
}
-void G1BarrierSetAssembler::gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) {
- G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
- __ bind(*stub->entry());
+#undef __
- assert(stub->addr()->is_register(), "Precondition.");
- assert(stub->new_val()->is_register(), "Precondition.");
- Register addr_reg = stub->addr()->as_pointer_register();
- Register new_val_reg = stub->new_val()->as_register();
-
- __ cmpdi(CR0, new_val_reg, 0);
- __ bc_far_optimized(Assembler::bcondCRbiIs1, __ bi0(CR0, Assembler::equal), *stub->continuation());
-
- address c_code = bs->post_barrier_c1_runtime_code_blob()->code_begin();
- //__ load_const_optimized(R0, c_code);
- __ add_const_optimized(R0, R29_TOC, MacroAssembler::offset_to_global_toc(c_code));
- __ mtctr(R0);
- __ mr(R0, addr_reg); // Pass addr in R0.
- __ bctrl();
- __ b(*stub->continuation());
+void G1BarrierSetAssembler::g1_write_barrier_post_c1(MacroAssembler* masm,
+ Register store_addr,
+ Register new_val,
+ Register thread,
+ Register tmp1,
+ Register tmp2) {
+ Label done;
+ generate_post_barrier(masm, store_addr, new_val, thread, tmp1, tmp2, done, true /* new_val_may_be_null */);
+ masm->bind(done);
}
-#undef __
#define __ sasm->
void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
@@ -642,86 +564,6 @@ void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler*
__ b(restart);
}
-void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) {
- G1BarrierSet* bs = barrier_set_cast(BarrierSet::barrier_set());
-
- __ set_info("g1_post_barrier_slow_id", false);
-
- // Using stack slots: spill addr, spill tmp2
- const int stack_slots = 2;
- Register tmp = R0;
- Register addr = R14;
- Register tmp2 = R15;
- CardTable::CardValue* byte_map_base = bs->card_table()->byte_map_base();
-
- Label restart, refill, ret;
-
- // Spill
- __ std(addr, -8, R1_SP);
- __ std(tmp2, -16, R1_SP);
-
- __ srdi(addr, R0, CardTable::card_shift()); // Addr is passed in R0.
- __ load_const_optimized(/*cardtable*/ tmp2, byte_map_base, tmp);
- __ add(addr, tmp2, addr);
- __ lbz(tmp, 0, addr); // tmp := [addr + cardtable]
-
- // Return if young card.
- __ cmpwi(CR0, tmp, G1CardTable::g1_young_card_val());
- __ beq(CR0, ret);
-
- // Return if sequential consistent value is already dirty.
- __ membar(Assembler::StoreLoad);
- __ lbz(tmp, 0, addr); // tmp := [addr + cardtable]
-
- __ cmpwi(CR0, tmp, G1CardTable::dirty_card_val());
- __ beq(CR0, ret);
-
- // Not dirty.
-
- // First, dirty it.
- __ li(tmp, G1CardTable::dirty_card_val());
- __ stb(tmp, 0, addr);
-
- int dirty_card_q_index_byte_offset = in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset());
- int dirty_card_q_buf_byte_offset = in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset());
-
- __ bind(restart);
-
- // Get the index into the update buffer. G1DirtyCardQueue::_index is
- // a size_t so ld_ptr is appropriate here.
- __ ld(tmp2, dirty_card_q_index_byte_offset, R16_thread);
-
- // index == 0?
- __ cmpdi(CR0, tmp2, 0);
- __ beq(CR0, refill);
-
- __ ld(tmp, dirty_card_q_buf_byte_offset, R16_thread);
- __ addi(tmp2, tmp2, -oopSize);
-
- __ std(tmp2, dirty_card_q_index_byte_offset, R16_thread);
- __ add(tmp2, tmp, tmp2);
- __ std(addr, 0, tmp2); // [_buf + index] :=
-
- // Restore temp registers and return-from-leaf.
- __ bind(ret);
- __ ld(tmp2, -16, R1_SP);
- __ ld(addr, -8, R1_SP);
- __ blr();
-
- __ bind(refill);
- const int nbytes_save = (MacroAssembler::num_volatile_regs + stack_slots) * BytesPerWord;
- __ save_volatile_gprs(R1_SP, -nbytes_save); // except R0
- __ mflr(R0);
- __ std(R0, _abi0(lr), R1_SP);
- __ push_frame_reg_args(nbytes_save, R0); // dummy frame for C call
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1DirtyCardQueueSet::handle_zero_index_for_thread), R16_thread);
- __ pop_frame();
- __ ld(R0, _abi0(lr), R1_SP);
- __ mtlr(R0);
- __ restore_volatile_gprs(R1_SP, -nbytes_save); // except R0
- __ b(restart);
-}
-
#undef __
#endif // COMPILER1
diff --git a/src/hotspot/cpu/ppc/gc/g1/g1BarrierSetAssembler_ppc.hpp b/src/hotspot/cpu/ppc/gc/g1/g1BarrierSetAssembler_ppc.hpp
index 33cb89dacc6..e059cc661af 100644
--- a/src/hotspot/cpu/ppc/gc/g1/g1BarrierSetAssembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/gc/g1/g1BarrierSetAssembler_ppc.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, 2021 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -37,9 +37,7 @@
class LIR_Assembler;
class StubAssembler;
class G1PreBarrierStub;
-class G1PostBarrierStub;
class G1PreBarrierStubC2;
-class G1PostBarrierStubC2;
class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
protected:
@@ -56,8 +54,7 @@ protected:
MacroAssembler::PreservationLevel preservation_level);
void g1_write_barrier_post(MacroAssembler* masm, DecoratorSet decorators,
Register store_addr, Register new_val,
- Register tmp1, Register tmp2, Register tmp3,
- MacroAssembler::PreservationLevel preservation_level);
+ Register tmp1, Register tmp2);
virtual void oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Register base, RegisterOrConstant ind_or_offs, Register val,
@@ -79,17 +76,21 @@ public:
Register new_val,
Register tmp1,
Register tmp2,
- G1PostBarrierStubC2* c2_stub,
+ bool new_val_may_be_null,
bool decode_new_val);
- void generate_c2_post_barrier_stub(MacroAssembler* masm,
- G1PostBarrierStubC2* stub) const;
#endif
#ifdef COMPILER1
void gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub);
- void gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub);
void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm);
- void generate_c1_post_barrier_runtime_stub(StubAssembler* sasm);
+
+ void g1_write_barrier_post_c1(MacroAssembler* masm,
+ Register store_addr,
+ Register new_val,
+ Register thread,
+ Register tmp1,
+ Register tmp2);
+
#endif
virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
diff --git a/src/hotspot/cpu/ppc/gc/g1/g1_ppc.ad b/src/hotspot/cpu/ppc/gc/g1/g1_ppc.ad
index 4f24efe872b..0a4a9442855 100644
--- a/src/hotspot/cpu/ppc/gc/g1/g1_ppc.ad
+++ b/src/hotspot/cpu/ppc/gc/g1/g1_ppc.ad
@@ -1,5 +1,5 @@
//
-// Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
// Copyright (c) 2025 SAP SE. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
@@ -64,13 +64,13 @@ static void post_write_barrier(MacroAssembler* masm,
Register tmp1,
Register tmp2,
bool decode_new_val = false) {
- if (!G1PostBarrierStubC2::needs_barrier(node)) {
+ if (!G1BarrierStubC2::needs_post_barrier(node)) {
return;
}
Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
G1BarrierSetAssembler* g1_asm = static_cast(BarrierSet::barrier_set()->barrier_set_assembler());
- G1PostBarrierStubC2* const stub = G1PostBarrierStubC2::create(node);
- g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, tmp1, tmp2, stub, decode_new_val);
+ bool new_val_may_be_null = G1BarrierStubC2::post_new_val_may_be_null(node);
+ g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, tmp1, tmp2, new_val_may_be_null, decode_new_val);
}
%}
diff --git a/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.cpp b/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.cpp
index b2e830bcdb8..8712c75711d 100644
--- a/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.cpp
@@ -193,14 +193,19 @@ void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm, Register t
// Low order half of 64 bit value is currently used.
__ ld(R0, in_bytes(bs_nm->thread_disarmed_guard_value_offset()), R16_thread);
- __ cmpw(CR0, R0, tmp);
- // Load stub address using toc (fixed instruction size, unlike load_const_optimized)
- __ calculate_address_from_global_toc(tmp, StubRoutines::method_entry_barrier(),
- true, true, false); // 2 instructions
- __ mtctr(tmp);
+ if (TrapBasedNMethodEntryBarriers) {
+ __ tw(Assembler::traptoLessThanUnsigned | Assembler::traptoGreaterThanUnsigned, R0, tmp);
+ } else {
+ __ cmpw(CR0, R0, tmp);
- __ bnectrl(CR0);
+ // Load stub address using toc (fixed instruction size, unlike load_const_optimized)
+ __ calculate_address_from_global_toc(tmp, StubRoutines::method_entry_barrier(),
+ true, true, false); // 2 instructions
+ __ mtctr(tmp);
+
+ __ bnectrl(CR0);
+ }
// Oops may have been changed. Make those updates observable.
// "isync" can serve both, data and instruction patching.
diff --git a/src/hotspot/cpu/ppc/gc/shared/barrierSetNMethod_ppc.cpp b/src/hotspot/cpu/ppc/gc/shared/barrierSetNMethod_ppc.cpp
index 02423e13308..96fa03df519 100644
--- a/src/hotspot/cpu/ppc/gc/shared/barrierSetNMethod_ppc.cpp
+++ b/src/hotspot/cpu/ppc/gc/shared/barrierSetNMethod_ppc.cpp
@@ -73,7 +73,7 @@ public:
u_char buf[NativeMovRegMem::instruction_size];
uint64_t u64;
} new_mov_instr, old_mov_instr;
- new_mov_instr.u64 = old_mov_instr.u64 = Atomic::load(instr);
+ new_mov_instr.u64 = old_mov_instr.u64 = AtomicAccess::load(instr);
while (true) {
// Only bits in the mask are changed
int old_value = nativeMovRegMem_at(old_mov_instr.buf)->offset();
@@ -81,7 +81,7 @@ public:
if (new_value == old_value) return; // skip icache flush if nothing changed
nativeMovRegMem_at(new_mov_instr.buf)->set_offset(new_value, false /* no icache flush */);
// Swap in the new value
- uint64_t v = Atomic::cmpxchg(instr, old_mov_instr.u64, new_mov_instr.u64, memory_order_relaxed);
+ uint64_t v = AtomicAccess::cmpxchg(instr, old_mov_instr.u64, new_mov_instr.u64, memory_order_relaxed);
if (v == old_mov_instr.u64) break;
old_mov_instr.u64 = v;
}
@@ -100,17 +100,21 @@ public:
verify_op_code(current_instruction, Assembler::LD_OPCODE);
- // cmpw (mnemonic)
- verify_op_code(current_instruction, Assembler::CMP_OPCODE);
+ if (TrapBasedNMethodEntryBarriers) {
+ verify_op_code(current_instruction, Assembler::TW_OPCODE);
+ } else {
+ // cmpw (mnemonic)
+ verify_op_code(current_instruction, Assembler::CMP_OPCODE);
- // calculate_address_from_global_toc (compound instruction)
- verify_op_code_manually(current_instruction, MacroAssembler::is_addis(*current_instruction));
- verify_op_code_manually(current_instruction, MacroAssembler::is_addi(*current_instruction));
+ // calculate_address_from_global_toc (compound instruction)
+ verify_op_code_manually(current_instruction, MacroAssembler::is_addis(*current_instruction));
+ verify_op_code_manually(current_instruction, MacroAssembler::is_addi(*current_instruction));
- verify_op_code_manually(current_instruction, MacroAssembler::is_mtctr(*current_instruction));
+ verify_op_code_manually(current_instruction, MacroAssembler::is_mtctr(*current_instruction));
- // bnectrl (mnemonic) (weak check; not checking the exact type)
- verify_op_code(current_instruction, Assembler::BCCTR_OPCODE);
+ // bnectrl (mnemonic) (weak check; not checking the exact type)
+ verify_op_code(current_instruction, Assembler::BCCTR_OPCODE);
+ }
// isync is optional
}
@@ -131,9 +135,10 @@ private:
static NativeNMethodBarrier* get_nmethod_barrier(nmethod* nm) {
BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
- address barrier_address = nm->code_begin() + nm->frame_complete_offset() + (-8 * 4);
+ address barrier_address = nm->code_begin() + nm->frame_complete_offset() -
+ (TrapBasedNMethodEntryBarriers ? 4 : 8) * BytesPerInstWord;
if (bs_asm->nmethod_patching_type() != NMethodPatchingType::stw_instruction_and_data_patch) {
- barrier_address -= 4; // isync (see nmethod_entry_barrier)
+ barrier_address -= BytesPerInstWord; // isync (see nmethod_entry_barrier)
}
auto barrier = reinterpret_cast(barrier_address);
diff --git a/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.cpp b/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.cpp
index e30e8c82d23..0aa5858c8e6 100644
--- a/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.cpp
@@ -114,7 +114,6 @@ public:
}
};
-
void ZBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Register base, RegisterOrConstant ind_or_offs, Register dst,
Register tmp1, Register tmp2,
@@ -561,7 +560,6 @@ void ZBarrierSetAssembler::generate_conjoint_oop_copy(MacroAssembler* masm, bool
copy_store_at_slow(masm, R4_ARG2, tmp, store_bad, store_good, dest_uninitialized);
}
-
// Verify a colored pointer.
void ZBarrierSetAssembler::check_oop(MacroAssembler *masm, Register obj, const char* msg) {
if (!VerifyOops) {
@@ -583,7 +581,6 @@ void ZBarrierSetAssembler::check_oop(MacroAssembler *masm, Register obj, const c
__ bind(done);
}
-
void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register dst, Register jni_env,
Register obj, Register tmp, Label& slowpath) {
__ block_comment("try_resolve_jobject_in_native (zgc) {");
diff --git a/src/hotspot/cpu/ppc/globalDefinitions_ppc.hpp b/src/hotspot/cpu/ppc/globalDefinitions_ppc.hpp
index f8f15741301..6c41e56b20b 100644
--- a/src/hotspot/cpu/ppc/globalDefinitions_ppc.hpp
+++ b/src/hotspot/cpu/ppc/globalDefinitions_ppc.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2016 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -43,8 +43,6 @@ const bool CCallingConventionRequiresIntsAsLongs = true;
#define SUPPORTS_NATIVE_CX8
-#define SUPPORT_MONITOR_COUNT
-
// PPC64 is not specified as multi-copy-atomic
// So we must not #define CPU_MULTI_COPY_ATOMIC
diff --git a/src/hotspot/cpu/ppc/globals_ppc.hpp b/src/hotspot/cpu/ppc/globals_ppc.hpp
index 65334bf0389..41a8e821ada 100644
--- a/src/hotspot/cpu/ppc/globals_ppc.hpp
+++ b/src/hotspot/cpu/ppc/globals_ppc.hpp
@@ -145,6 +145,8 @@ define_pd_global(intx, InitArrayShortSize, 9*BytesPerLong);
"switch off all optimizations requiring SIGTRAP.") \
product(bool, TrapBasedICMissChecks, true, DIAGNOSTIC, \
"Raise and handle SIGTRAP if inline cache miss detected.") \
+ product(bool, TrapBasedNMethodEntryBarriers, true, DIAGNOSTIC, \
+ "Raise and handle SIGTRAP if nmethod entry barrier armed.") \
\
product(bool, TraceTraps, false, DIAGNOSTIC, \
"Trace all traps the signal handler handles.") \
diff --git a/src/hotspot/cpu/ppc/javaFrameAnchor_ppc.hpp b/src/hotspot/cpu/ppc/javaFrameAnchor_ppc.hpp
index 00a6b4cbf95..431166e9d66 100644
--- a/src/hotspot/cpu/ppc/javaFrameAnchor_ppc.hpp
+++ b/src/hotspot/cpu/ppc/javaFrameAnchor_ppc.hpp
@@ -33,17 +33,17 @@ public:
// 2 - saving a current state (javaCalls)
// 3 - restoring an old state (javaCalls)
+ // No hardware barriers are necessary. All members are volatile and the profiler
+ // is run from a signal handler and only observers the thread its running on.
+
inline void clear(void) {
// clearing _last_Java_sp must be first
_last_Java_sp = nullptr;
- // fence?
- OrderAccess::release();
_last_Java_pc = nullptr;
}
inline void set(intptr_t* sp, address pc) {
_last_Java_pc = pc;
- OrderAccess::release();
_last_Java_sp = sp;
}
@@ -56,11 +56,9 @@ public:
// unless the value is changing.
if (_last_Java_sp != src->_last_Java_sp) {
_last_Java_sp = nullptr;
- OrderAccess::release();
}
_last_Java_pc = src->_last_Java_pc;
// Must be last so profiler will always see valid frame if has_last_frame() is true.
- OrderAccess::release();
_last_Java_sp = src->_last_Java_sp;
}
@@ -75,6 +73,6 @@ public:
intptr_t* last_Java_fp() const { return *(intptr_t**)_last_Java_sp; }
- void set_last_Java_sp(intptr_t* sp) { OrderAccess::release(); _last_Java_sp = sp; }
+ void set_last_Java_sp(intptr_t* sp) { _last_Java_sp = sp; }
#endif // CPU_PPC_JAVAFRAMEANCHOR_PPC_HPP
diff --git a/src/hotspot/cpu/ppc/nativeInst_ppc.cpp b/src/hotspot/cpu/ppc/nativeInst_ppc.cpp
index ca492329729..ebdfd9a57d6 100644
--- a/src/hotspot/cpu/ppc/nativeInst_ppc.cpp
+++ b/src/hotspot/cpu/ppc/nativeInst_ppc.cpp
@@ -347,7 +347,7 @@ void NativeGeneralJump::replace_mt_safe(address instr_addr, address code_buffer)
// Finally patch out the jump.
volatile juint *jump_addr = (volatile juint*)instr_addr;
// Release not needed because caller uses invalidate_range after copying the remaining bytes.
- //Atomic::release_store(jump_addr, *((juint*)code_buffer));
+ //AtomicAccess::release_store(jump_addr, *((juint*)code_buffer));
*jump_addr = *((juint*)code_buffer); // atomically store code over branch instruction
ICache::ppc64_flush_icache_bytes(instr_addr, NativeGeneralJump::instruction_size);
}
@@ -402,7 +402,7 @@ void NativePostCallNop::make_deopt() {
bool NativePostCallNop::patch(int32_t oopmap_slot, int32_t cb_offset) {
int32_t i2, i1;
assert(is_aligned(cb_offset, 4), "cb offset alignment does not match instruction alignment");
- assert(!decode(i1, i2), "already patched");
+ assert(!decode(i1, i2) || NMethodRelocation, "already patched");
cb_offset = cb_offset >> 2;
if (((oopmap_slot & ppc_oopmap_slot_mask) != oopmap_slot) || ((cb_offset & ppc_cb_offset_mask) != cb_offset)) {
diff --git a/src/hotspot/cpu/ppc/nativeInst_ppc.hpp b/src/hotspot/cpu/ppc/nativeInst_ppc.hpp
index d5dec3f4b1f..dcb5c2bb3cb 100644
--- a/src/hotspot/cpu/ppc/nativeInst_ppc.hpp
+++ b/src/hotspot/cpu/ppc/nativeInst_ppc.hpp
@@ -77,6 +77,12 @@ class NativeInstruction {
}
#endif
+ bool is_sigtrap_nmethod_entry_barrier() {
+ assert(UseSIGTRAP && TrapBasedNMethodEntryBarriers, "precondition");
+ return Assembler::is_tw(long_at(0), Assembler::traptoLessThanUnsigned | Assembler::traptoGreaterThanUnsigned,
+ 0, -1);
+ }
+
bool is_safepoint_poll() {
// The current arguments of the instruction are not checked!
if (USE_POLL_BIT_ONLY) {
diff --git a/src/hotspot/cpu/ppc/orderAccess_ppc.hpp b/src/hotspot/cpu/ppc/orderAccess_ppc.hpp
new file mode 100644
index 00000000000..f5fa8da755f
--- /dev/null
+++ b/src/hotspot/cpu/ppc/orderAccess_ppc.hpp
@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef CPU_PPC_ORDERACCESS_PPC_HPP
+#define CPU_PPC_ORDERACCESS_PPC_HPP
+
+#ifndef PPC64
+#error "OrderAccess currently only implemented for PPC64"
+#endif
+
+// Compiler version last used for testing: gcc 4.1.2
+// Please update this information when this file changes
+
+// Implementation of class OrderAccess.
+
+//
+// Machine barrier instructions:
+//
+// - sync Two-way memory barrier, aka fence.
+// - lwsync orders Store|Store,
+// Load|Store,
+// Load|Load,
+// but not Store|Load
+// - eieio orders Store|Store
+// - isync Invalidates speculatively executed instructions,
+// but isync may complete before storage accesses
+// associated with instructions preceding isync have
+// been performed.
+//
+// Semantic barrier instructions:
+// (as defined in orderAccess.hpp)
+//
+// - release orders Store|Store, (maps to lwsync)
+// Load|Store
+// - acquire orders Load|Store, (maps to lwsync)
+// Load|Load
+// - fence orders Store|Store, (maps to sync)
+// Load|Store,
+// Load|Load,
+// Store|Load
+//
+
+#define inlasm_sync() __asm__ __volatile__ ("sync" : : : "memory");
+#define inlasm_lwsync() __asm__ __volatile__ ("lwsync" : : : "memory");
+#define inlasm_eieio() __asm__ __volatile__ ("eieio" : : : "memory");
+#define inlasm_isync() __asm__ __volatile__ ("isync" : : : "memory");
+
+inline void OrderAccess::loadload() { inlasm_lwsync(); }
+inline void OrderAccess::storestore() { inlasm_lwsync(); }
+inline void OrderAccess::loadstore() { inlasm_lwsync(); }
+inline void OrderAccess::storeload() { inlasm_sync(); }
+
+inline void OrderAccess::acquire() { inlasm_lwsync(); }
+inline void OrderAccess::release() { inlasm_lwsync(); }
+inline void OrderAccess::fence() { inlasm_sync(); }
+inline void OrderAccess::cross_modify_fence_impl()
+ { inlasm_isync(); }
+
+#undef inlasm_sync
+#undef inlasm_lwsync
+#undef inlasm_eieio
+#undef inlasm_isync
+
+#endif // CPU_PPC_ORDERACCESS_PPC_HPP
diff --git a/src/hotspot/cpu/ppc/ppc.ad b/src/hotspot/cpu/ppc/ppc.ad
index 290369360fc..2c83b2d5765 100644
--- a/src/hotspot/cpu/ppc/ppc.ad
+++ b/src/hotspot/cpu/ppc/ppc.ad
@@ -2473,10 +2473,6 @@ RegMask Matcher::modL_proj_mask() {
return RegMask();
}
-const RegMask Matcher::method_handle_invoke_SP_save_mask() {
- return RegMask();
-}
-
%}
//----------ENCODING BLOCK-----------------------------------------------------
@@ -3434,7 +3430,6 @@ encode %{
// Create the call node.
CallDynamicJavaDirectSchedNode *call = new CallDynamicJavaDirectSchedNode();
- call->_method_handle_invoke = _method_handle_invoke;
call->_vtable_index = _vtable_index;
call->_method = _method;
call->_optimized_virtual = _optimized_virtual;
diff --git a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
index 2a9bad059ba..9fe7e1f22ff 100644
--- a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
+++ b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
@@ -1199,16 +1199,11 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
int comp_args_on_stack,
const BasicType *sig_bt,
const VMRegPair *regs,
- AdapterHandlerEntry* handler) {
- address i2c_entry;
- address c2i_unverified_entry;
- address c2i_entry;
-
-
+ address entry_address[AdapterBlob::ENTRY_COUNT]) {
// entry: i2c
__ align(CodeEntryAlignment);
- i2c_entry = __ pc();
+ entry_address[AdapterBlob::I2C] = __ pc();
gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs);
@@ -1216,7 +1211,7 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
__ align(CodeEntryAlignment);
BLOCK_COMMENT("c2i unverified entry");
- c2i_unverified_entry = __ pc();
+ entry_address[AdapterBlob::C2I_Unverified] = __ pc();
// inline_cache contains a CompiledICData
const Register ic = R19_inline_cache_reg;
@@ -1244,10 +1239,10 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
// entry: c2i
- c2i_entry = __ pc();
+ entry_address[AdapterBlob::C2I] = __ pc();
// Class initialization barrier for static methods
- address c2i_no_clinit_check_entry = nullptr;
+ entry_address[AdapterBlob::C2I_No_Clinit_Check] = nullptr;
if (VM_Version::supports_fast_class_init_checks()) {
Label L_skip_barrier;
@@ -1266,15 +1261,13 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
__ bctr();
__ bind(L_skip_barrier);
- c2i_no_clinit_check_entry = __ pc();
+ entry_address[AdapterBlob::C2I_No_Clinit_Check] = __ pc();
}
BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
bs->c2i_entry_barrier(masm, /* tmp register*/ ic_klass, /* tmp register*/ receiver_klass, /* tmp register*/ code);
gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, call_interpreter, ientry);
-
- handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry);
return;
}
@@ -1646,7 +1639,6 @@ static void fill_continuation_entry(MacroAssembler* masm, Register reg_cont_obj,
assert_different_registers(reg_cont_obj, reg_flags);
Register zero = R8_ARG6;
Register tmp2 = R9_ARG7;
- Register tmp3 = R10_ARG8;
DEBUG_ONLY(__ block_comment("fill {"));
#ifdef ASSERT
@@ -1662,12 +1654,9 @@ static void fill_continuation_entry(MacroAssembler* masm, Register reg_cont_obj,
__ stw(zero, in_bytes(ContinuationEntry::pin_count_offset()), R1_SP);
__ ld_ptr(tmp2, JavaThread::cont_fastpath_offset(), R16_thread);
- __ ld(tmp3, in_bytes(JavaThread::held_monitor_count_offset()), R16_thread);
__ st_ptr(tmp2, ContinuationEntry::parent_cont_fastpath_offset(), R1_SP);
- __ std(tmp3, in_bytes(ContinuationEntry::parent_held_monitor_count_offset()), R1_SP);
__ st_ptr(zero, JavaThread::cont_fastpath_offset(), R16_thread);
- __ std(zero, in_bytes(JavaThread::held_monitor_count_offset()), R16_thread);
DEBUG_ONLY(__ block_comment("} fill"));
}
@@ -1688,7 +1677,6 @@ static void fill_continuation_entry(MacroAssembler* masm, Register reg_cont_obj,
static void continuation_enter_cleanup(MacroAssembler* masm) {
Register tmp1 = R8_ARG6;
Register tmp2 = R9_ARG7;
- Register tmp3 = R10_ARG8;
#ifdef ASSERT
__ block_comment("clean {");
@@ -1699,57 +1687,8 @@ static void continuation_enter_cleanup(MacroAssembler* masm) {
__ ld_ptr(tmp1, ContinuationEntry::parent_cont_fastpath_offset(), R1_SP);
__ st_ptr(tmp1, JavaThread::cont_fastpath_offset(), R16_thread);
-
- if (CheckJNICalls) {
- // Check if this is a virtual thread continuation
- Label L_skip_vthread_code;
- __ lwz(R0, in_bytes(ContinuationEntry::flags_offset()), R1_SP);
- __ cmpwi(CR0, R0, 0);
- __ beq(CR0, L_skip_vthread_code);
-
- // If the held monitor count is > 0 and this vthread is terminating then
- // it failed to release a JNI monitor. So we issue the same log message
- // that JavaThread::exit does.
- __ ld(R0, in_bytes(JavaThread::jni_monitor_count_offset()), R16_thread);
- __ cmpdi(CR0, R0, 0);
- __ beq(CR0, L_skip_vthread_code);
-
- // Save return value potentially containing the exception oop
- Register ex_oop = R15_esp; // nonvolatile register
- __ mr(ex_oop, R3_RET);
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::log_jni_monitor_still_held));
- // Restore potental return value
- __ mr(R3_RET, ex_oop);
-
- // For vthreads we have to explicitly zero the JNI monitor count of the carrier
- // on termination. The held count is implicitly zeroed below when we restore from
- // the parent held count (which has to be zero).
- __ li(tmp1, 0);
- __ std(tmp1, in_bytes(JavaThread::jni_monitor_count_offset()), R16_thread);
-
- __ bind(L_skip_vthread_code);
- }
-#ifdef ASSERT
- else {
- // Check if this is a virtual thread continuation
- Label L_skip_vthread_code;
- __ lwz(R0, in_bytes(ContinuationEntry::flags_offset()), R1_SP);
- __ cmpwi(CR0, R0, 0);
- __ beq(CR0, L_skip_vthread_code);
-
- // See comment just above. If not checking JNI calls the JNI count is only
- // needed for assertion checking.
- __ li(tmp1, 0);
- __ std(tmp1, in_bytes(JavaThread::jni_monitor_count_offset()), R16_thread);
-
- __ bind(L_skip_vthread_code);
- }
-#endif
-
- __ ld(tmp2, in_bytes(ContinuationEntry::parent_held_monitor_count_offset()), R1_SP);
- __ ld_ptr(tmp3, ContinuationEntry::parent_offset(), R1_SP);
- __ std(tmp2, in_bytes(JavaThread::held_monitor_count_offset()), R16_thread);
- __ st_ptr(tmp3, JavaThread::cont_entry_offset(), R16_thread);
+ __ ld_ptr(tmp2, ContinuationEntry::parent_offset(), R1_SP);
+ __ st_ptr(tmp2, JavaThread::cont_entry_offset(), R16_thread);
DEBUG_ONLY(__ block_comment("} clean"));
}
diff --git a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp
index f9f43ade501..948092bbb9a 100644
--- a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp
+++ b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp
@@ -3277,8 +3277,12 @@ class StubGenerator: public StubCodeGenerator {
// register the stub as the default exit with class UnsafeMemoryAccess
UnsafeMemoryAccess::set_common_exit_stub_pc(StubRoutines::_unsafecopy_common_exit);
- // Note: the disjoint stubs must be generated first, some of
- // the conjoint stubs use them.
+ // Note: the disjoint stubs must be generated first, some of the
+ // conjoint stubs use them.
+
+ // Note: chaining of stubs does not rely on branching to an
+ // auxiliary post-push entry because none of the stubs
+ // push/pop a frame.
// non-aligned disjoint versions
StubRoutines::_jbyte_disjoint_arraycopy = generate_disjoint_byte_copy(StubId::stubgen_jbyte_disjoint_arraycopy_id);
diff --git a/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp b/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp
index 7431f77aeff..41fbe66647e 100644
--- a/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp
+++ b/src/hotspot/cpu/ppc/templateTable_ppc_64.cpp
@@ -2179,17 +2179,11 @@ void TemplateTable::_return(TosState state) {
// - Rscratch
void TemplateTable::resolve_cache_and_index_for_method(int byte_no, Register Rcache, Register Rscratch) {
assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
- Label Lresolved, Ldone, L_clinit_barrier_slow;
+
+ Label L_clinit_barrier_slow, L_done;
Register Rindex = Rscratch;
Bytecodes::Code code = bytecode();
- switch (code) {
- case Bytecodes::_nofast_getfield: code = Bytecodes::_getfield; break;
- case Bytecodes::_nofast_putfield: code = Bytecodes::_putfield; break;
- default:
- break;
- }
-
const int bytecode_offset = (byte_no == f1_byte) ? in_bytes(ResolvedMethodEntry::bytecode1_offset())
: in_bytes(ResolvedMethodEntry::bytecode2_offset());
__ load_method_entry(Rcache, Rindex);
@@ -2197,20 +2191,8 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no, Register Rca
__ lbz(Rscratch, bytecode_offset, Rcache);
// Acquire by cmp-br-isync (see below).
__ cmpdi(CR0, Rscratch, (int)code);
- __ beq(CR0, Lresolved);
+ __ bne(CR0, L_clinit_barrier_slow);
- // Class initialization barrier slow path lands here as well.
- __ bind(L_clinit_barrier_slow);
-
- address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
- __ li(R4_ARG2, code);
- __ call_VM(noreg, entry, R4_ARG2, true);
-
- // Update registers with resolved info.
- __ load_method_entry(Rcache, Rindex);
- __ b(Ldone);
-
- __ bind(Lresolved);
__ isync(); // Order load wrt. succeeding loads.
// Class initialization barrier for static methods
@@ -2220,18 +2202,26 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no, Register Rca
__ ld(method, in_bytes(ResolvedMethodEntry::method_offset()), Rcache);
__ load_method_holder(klass, method);
- __ clinit_barrier(klass, R16_thread, nullptr /*L_fast_path*/, &L_clinit_barrier_slow);
+ __ clinit_barrier(klass, R16_thread, &L_done, /*L_slow_path*/ nullptr);
+ } else {
+ __ b(L_done);
}
- __ bind(Ldone);
+ // Class initialization barrier slow path lands here as well.
+ __ bind(L_clinit_barrier_slow);
+ address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
+ __ li(R4_ARG2, code);
+ __ call_VM(noreg, entry, R4_ARG2);
+
+ // Update registers with resolved info.
+ __ load_method_entry(Rcache, Rindex);
+ __ bind(L_done);
}
-void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
- Register Rcache,
- Register index) {
+void TemplateTable::resolve_cache_and_index_for_field(int byte_no, Register Rcache, Register index) {
assert_different_registers(Rcache, index);
- Label resolved;
+ Label L_clinit_barrier_slow, L_done;
Bytecodes::Code code = bytecode();
switch (code) {
@@ -2246,19 +2236,34 @@ void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
: in_bytes(ResolvedFieldEntry::put_code_offset());
__ lbz(R0, code_offset, Rcache);
__ cmpwi(CR0, R0, (int)code); // have we resolved this bytecode?
- __ beq(CR0, resolved);
+ __ bne(CR0, L_clinit_barrier_slow);
+
+ __ isync(); // Order load wrt. succeeding loads.
+
+ // Class initialization barrier for static fields
+ if (VM_Version::supports_fast_class_init_checks() &&
+ (bytecode() == Bytecodes::_getstatic || bytecode() == Bytecodes::_putstatic)) {
+ const Register field_holder = R4_ARG2;
+
+ // InterpreterRuntime::resolve_get_put sets field_holder and finally release-stores put_code.
+ // We have seen the released put_code above and will read the corresponding field_holder and init_state
+ // (ordered by compare-branch-isync).
+ __ ld(field_holder, ResolvedFieldEntry::field_holder_offset(), Rcache);
+ __ clinit_barrier(field_holder, R16_thread, &L_done, /*L_slow_path*/ nullptr);
+ } else {
+ __ b(L_done);
+ }
// resolve first time through
+ // Class initialization barrier slow path lands here as well.
+ __ bind(L_clinit_barrier_slow);
address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
- __ li(R4_ARG2, (int)code);
+ __ li(R4_ARG2, code);
__ call_VM(noreg, entry, R4_ARG2);
// Update registers with resolved info
__ load_field_entry(Rcache, index);
- __ bind(resolved);
-
- // Use acquire semantics for the bytecode (see ResolvedFieldEntry::fill_in()).
- __ isync(); // Order load wrt. succeeding loads.
+ __ bind(L_done);
}
void TemplateTable::load_resolved_field_entry(Register obj,
diff --git a/src/hotspot/cpu/ppc/vm_version_ppc.cpp b/src/hotspot/cpu/ppc/vm_version_ppc.cpp
index 1d2b8d3ca04..8b1de754650 100644
--- a/src/hotspot/cpu/ppc/vm_version_ppc.cpp
+++ b/src/hotspot/cpu/ppc/vm_version_ppc.cpp
@@ -87,8 +87,10 @@ void VM_Version::initialize() {
if (!UseSIGTRAP) {
MSG(TrapBasedICMissChecks);
MSG(TrapBasedNullChecks);
- FLAG_SET_ERGO(TrapBasedNullChecks, false);
- FLAG_SET_ERGO(TrapBasedICMissChecks, false);
+ MSG(TrapBasedNMethodEntryBarriers);
+ FLAG_SET_ERGO(TrapBasedNullChecks, false);
+ FLAG_SET_ERGO(TrapBasedICMissChecks, false);
+ FLAG_SET_ERGO(TrapBasedNMethodEntryBarriers, false);
}
#ifdef COMPILER2
@@ -97,6 +99,10 @@ void VM_Version::initialize() {
FLAG_SET_ERGO(TrapBasedRangeChecks, false);
}
+ if (FLAG_IS_DEFAULT(UsePopCountInstruction)) {
+ FLAG_SET_ERGO(UsePopCountInstruction, true);
+ }
+
if (PowerArchitecturePPC64 >= 9) {
// Performance is good since Power9.
if (FLAG_IS_DEFAULT(SuperwordUseVSX)) {
diff --git a/src/hotspot/cpu/riscv/assembler_riscv.hpp b/src/hotspot/cpu/riscv/assembler_riscv.hpp
index c267cb669ac..0712b60fb2a 100644
--- a/src/hotspot/cpu/riscv/assembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/assembler_riscv.hpp
@@ -912,6 +912,43 @@ protected:
emit(insn);
}
+ public:
+
+ static uint32_t encode_csrrw(Register Rd, const uint32_t csr, Register Rs1) {
+ guarantee(is_uimm12(csr), "csr is invalid");
+ uint32_t insn = 0;
+ patch((address)&insn, 6, 0, 0b1110011);
+ patch((address)&insn, 14, 12, 0b001);
+ patch_reg((address)&insn, 7, Rd);
+ patch_reg((address)&insn, 15, Rs1);
+ patch((address)&insn, 31, 20, csr);
+ return insn;
+ }
+
+ static uint32_t encode_jal(Register Rd, const int32_t offset) {
+ guarantee(is_simm21(offset) && ((offset % 2) == 0), "offset is invalid.");
+ uint32_t insn = 0;
+ patch((address)&insn, 6, 0, 0b1101111);
+ patch_reg((address)&insn, 7, Rd);
+ patch((address)&insn, 19, 12, (uint32_t)((offset >> 12) & 0xff));
+ patch((address)&insn, 20, (uint32_t)((offset >> 11) & 0x1));
+ patch((address)&insn, 30, 21, (uint32_t)((offset >> 1) & 0x3ff));
+ patch((address)&insn, 31, (uint32_t)((offset >> 20) & 0x1));
+ return insn;
+ }
+
+ static uint32_t encode_jalr(Register Rd, Register Rs, const int32_t offset) {
+ guarantee(is_simm12(offset), "offset is invalid.");
+ uint32_t insn = 0;
+ patch((address)&insn, 6, 0, 0b1100111);
+ patch_reg((address)&insn, 7, Rd);
+ patch((address)&insn, 14, 12, 0b000);
+ patch_reg((address)&insn, 15, Rs);
+ int32_t val = offset & 0xfff;
+ patch((address)&insn, 31, 20, val);
+ return insn;
+ }
+
protected:
enum barrier {
@@ -3667,19 +3704,15 @@ public:
// --------------------------
// Upper Immediate Instruction
// --------------------------
-#define INSN(NAME) \
- void NAME(Register Rd, int32_t imm) { \
- /* lui -> c.lui */ \
- if (do_compress() && (Rd != x0 && Rd != x2 && imm != 0 && is_simm18(imm))) { \
- c_lui(Rd, imm); \
- return; \
- } \
- _lui(Rd, imm); \
+ void lui(Register Rd, int32_t imm) {
+ /* lui -> c.lui */
+ if (do_compress() && (Rd != x0 && Rd != x2 && imm != 0 && is_simm18(imm))) {
+ c_lui(Rd, imm);
+ return;
+ }
+ _lui(Rd, imm);
}
- INSN(lui);
-
-#undef INSN
// Cache Management Operations
// These instruction may be turned off for user space.
diff --git a/src/hotspot/cpu/riscv/assembler_riscv.inline.hpp b/src/hotspot/cpu/riscv/assembler_riscv.inline.hpp
index 1f9e6df2172..e85b64bd6ba 100644
--- a/src/hotspot/cpu/riscv/assembler_riscv.inline.hpp
+++ b/src/hotspot/cpu/riscv/assembler_riscv.inline.hpp
@@ -27,7 +27,7 @@
#ifndef CPU_RISCV_ASSEMBLER_RISCV_INLINE_HPP
#define CPU_RISCV_ASSEMBLER_RISCV_INLINE_HPP
-#include "asm/assembler.inline.hpp"
+#include "asm/assembler.hpp"
#include "asm/codeBuffer.hpp"
#include "code/codeCache.hpp"
diff --git a/src/hotspot/cpu/riscv/c1_CodeStubs_riscv.cpp b/src/hotspot/cpu/riscv/c1_CodeStubs_riscv.cpp
index 0c16e632e5a..a8a21342248 100644
--- a/src/hotspot/cpu/riscv/c1_CodeStubs_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c1_CodeStubs_riscv.cpp
@@ -204,10 +204,10 @@ void MonitorEnterStub::emit_code(LIR_Assembler* ce) {
void MonitorExitStub::emit_code(LIR_Assembler* ce) {
__ bind(_entry);
- if (_compute_lock) {
- // lock_reg was destroyed by fast unlocking attempt => recompute it
- ce->monitor_address(_monitor_ix, _lock_reg);
- }
+
+ // lock_reg was destroyed by fast unlocking attempt => recompute it
+ ce->monitor_address(_monitor_ix, _lock_reg);
+
ce->store_parameter(_lock_reg->as_register(), 0);
// note: non-blocking leaf routine => no call info needed
StubId exit_id;
diff --git a/src/hotspot/cpu/riscv/c1_FrameMap_riscv.cpp b/src/hotspot/cpu/riscv/c1_FrameMap_riscv.cpp
index d3ccd46048b..b3a0b261f54 100644
--- a/src/hotspot/cpu/riscv/c1_FrameMap_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c1_FrameMap_riscv.cpp
@@ -377,11 +377,6 @@ LIR_Opr FrameMap::stack_pointer() {
return FrameMap::sp_opr;
}
-// JSR 292
-LIR_Opr FrameMap::method_handle_invoke_SP_save_opr() {
- return LIR_OprFact::illegalOpr; // Not needed on riscv
-}
-
bool FrameMap::validate_frame() {
return true;
}
diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp
index c3fe72870cf..9d8ae770ccf 100644
--- a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp
@@ -338,7 +338,7 @@ int LIR_Assembler::emit_unwind_handler() {
MonitorExitStub* stub = nullptr;
if (method()->is_synchronized()) {
monitor_address(0, FrameMap::r10_opr);
- stub = new MonitorExitStub(FrameMap::r10_opr, true, 0);
+ stub = new MonitorExitStub(FrameMap::r10_opr, 0);
__ unlock_object(x15, x14, x10, x16, *stub->entry());
__ bind(*stub->continuation());
}
@@ -1350,6 +1350,7 @@ void LIR_Assembler::align_call(LIR_Code code) {
}
void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
+ Assembler::IncompressibleScope scope(_masm);
address call = __ reloc_call(Address(op->addr(), rtype));
if (call == nullptr) {
bailout("reloc call address stub overflow");
@@ -1360,6 +1361,7 @@ void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
}
void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
+ Assembler::IncompressibleScope scope(_masm);
address call = __ ic_call(op->addr());
if (call == nullptr) {
bailout("reloc call address stub overflow");
@@ -1494,14 +1496,12 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
Register lock = op->lock_opr()->as_register();
Register temp = op->scratch_opr()->as_register();
if (op->code() == lir_lock) {
- assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
// add debug info for NullPointerException only if one is possible
int null_check_offset = __ lock_object(hdr, obj, lock, temp, *op->stub()->entry());
if (op->info() != nullptr) {
add_debug_info_for_null_check(null_check_offset, op->info());
}
} else if (op->code() == lir_unlock) {
- assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
__ unlock_object(hdr, obj, lock, temp, *op->stub()->entry());
} else {
Unimplemented();
@@ -1844,6 +1844,10 @@ void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest, LIR_PatchCode patch_code, C
void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
assert(!tmp->is_valid(), "don't need temporary");
+ Assembler::IncompressibleScope scope(_masm);
+ // Post call nops must be natural aligned due to cmodx rules.
+ align_call(lir_rtcall);
+
__ rt_call(dest);
if (info != nullptr) {
diff --git a/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp
index 8198192f506..8e989de2665 100644
--- a/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp
@@ -48,27 +48,27 @@ void C1_MacroAssembler::float_cmp(bool is_float, int unordered_result,
}
}
-int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Register temp, Label& slow_case) {
- assert_different_registers(hdr, obj, disp_hdr, temp, t0, t1);
+int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register basic_lock, Register temp, Label& slow_case) {
+ assert_different_registers(hdr, obj, basic_lock, temp, t0, t1);
int null_check_offset = -1;
verify_oop(obj);
// save object being locked into the BasicObjectLock
- sd(obj, Address(disp_hdr, BasicObjectLock::obj_offset()));
+ sd(obj, Address(basic_lock, BasicObjectLock::obj_offset()));
null_check_offset = offset();
- lightweight_lock(disp_hdr, obj, hdr, temp, t1, slow_case);
+ lightweight_lock(basic_lock, obj, hdr, temp, t1, slow_case);
return null_check_offset;
}
-void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_hdr, Register temp, Label& slow_case) {
- assert_different_registers(hdr, obj, disp_hdr, temp, t0, t1);
+void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register basic_lock, Register temp, Label& slow_case) {
+ assert_different_registers(hdr, obj, basic_lock, temp, t0, t1);
// load object
- ld(obj, Address(disp_hdr, BasicObjectLock::obj_offset()));
+ ld(obj, Address(basic_lock, BasicObjectLock::obj_offset()));
verify_oop(obj);
lightweight_unlock(obj, hdr, temp, t1, slow_case);
diff --git a/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.hpp b/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.hpp
index 561053045ec..16e76884049 100644
--- a/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2015, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -56,19 +56,19 @@ using MacroAssembler::null_check;
Register result);
// locking
- // hdr : must be x10, contents destroyed
- // obj : must point to the object to lock, contents preserved
- // disp_hdr: must point to the displaced header location, contents preserved
- // temp : temporary register, must not be scratch register t0 or t1
+ // hdr : must be x10, contents destroyed
+ // obj : must point to the object to lock, contents preserved
+ // basic_lock: must point to the basic_lock, contents preserved
+ // temp : temporary register, must not be scratch register t0 or t1
// returns code offset at which to add null check debug information
- int lock_object(Register swap, Register obj, Register disp_hdr, Register temp, Label& slow_case);
+ int lock_object(Register swap, Register obj, Register basic_lock, Register temp, Label& slow_case);
// unlocking
- // hdr : contents destroyed
- // obj : must point to the object to lock, contents preserved
- // disp_hdr: must be x10 & must point to the displaced header location, contents destroyed
- // temp : temporary register, must not be scratch register t0 or t1
- void unlock_object(Register swap, Register obj, Register lock, Register temp, Label& slow_case);
+ // hdr : contents destroyed
+ // obj : must point to the object to lock, contents preserved
+ // basic_lock: must be x10 & must point to the basic lock, contents destroyed
+ // temp : temporary register, must not be scratch register t0 or t1
+ void unlock_object(Register swap, Register obj, Register basic_lock, Register temp, Label& slow_case);
void initialize_object(
Register obj, // result: pointer to object after successful allocation
diff --git a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp
index 1bdb7bc2f7c..154b62db47f 100644
--- a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp
@@ -1687,6 +1687,7 @@ void C2_MacroAssembler::arrays_hashcode(Register ary, Register cnt, Register res
Register tmp4, Register tmp5, Register tmp6,
BasicType eltype)
{
+ assert(!UseRVV, "sanity");
assert_different_registers(ary, cnt, result, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, t0, t1);
const int elsize = arrays_hashcode_elsize(eltype);
@@ -1759,29 +1760,143 @@ void C2_MacroAssembler::arrays_hashcode(Register ary, Register cnt, Register res
BLOCK_COMMENT("} // arrays_hashcode");
}
+void C2_MacroAssembler::arrays_hashcode_v(Register ary, Register cnt, Register result,
+ Register tmp1, Register tmp2, Register tmp3,
+ BasicType eltype)
+{
+ assert(UseRVV, "sanity");
+ assert(StubRoutines::riscv::arrays_hashcode_powers_of_31() != nullptr, "sanity");
+ assert_different_registers(ary, cnt, result, tmp1, tmp2, tmp3, t0, t1);
+
+ // The MaxVectorSize should have been set by detecting RVV max vector register
+ // size when check UseRVV (i.e. MaxVectorSize == VM_Version::_initial_vector_length).
+ // Let's use T_INT as all hashCode calculations eventually deal with ints.
+ const int lmul = 2;
+ const int stride = MaxVectorSize / sizeof(jint) * lmul;
+
+ const int elsize_bytes = arrays_hashcode_elsize(eltype);
+ const int elsize_shift = exact_log2(elsize_bytes);
+
+ switch (eltype) {
+ case T_BOOLEAN: BLOCK_COMMENT("arrays_hashcode_v(unsigned byte) {"); break;
+ case T_CHAR: BLOCK_COMMENT("arrays_hashcode_v(char) {"); break;
+ case T_BYTE: BLOCK_COMMENT("arrays_hashcode_v(byte) {"); break;
+ case T_SHORT: BLOCK_COMMENT("arrays_hashcode_v(short) {"); break;
+ case T_INT: BLOCK_COMMENT("arrays_hashcode_v(int) {"); break;
+ default:
+ ShouldNotReachHere();
+ }
+
+ const Register pow31_highest = tmp1;
+ const Register ary_end = tmp2;
+ const Register consumed = tmp3;
+
+ const VectorRegister v_sum = v2;
+ const VectorRegister v_src = v4;
+ const VectorRegister v_coeffs = v6;
+ const VectorRegister v_tmp = v8;
+
+ const address adr_pows31 = StubRoutines::riscv::arrays_hashcode_powers_of_31()
+ + sizeof(jint);
+ Label VEC_LOOP, DONE, SCALAR_TAIL, SCALAR_TAIL_LOOP;
+
+ // NB: at this point (a) 'result' already has some value,
+ // (b) 'cnt' is not 0 or 1, see java code for details.
+
+ andi(t0, cnt, ~(stride - 1));
+ beqz(t0, SCALAR_TAIL);
+
+ la(t1, ExternalAddress(adr_pows31));
+ lw(pow31_highest, Address(t1, -1 * sizeof(jint)));
+
+ vsetvli(consumed, cnt, Assembler::e32, Assembler::m2);
+ vle32_v(v_coeffs, t1); // 31^^(stride - 1) ... 31^^0
+ vmv_v_x(v_sum, x0);
+
+ bind(VEC_LOOP);
+ arrays_hashcode_elload_v(v_src, v_tmp, ary, eltype);
+ vmul_vv(v_src, v_src, v_coeffs);
+ vmadd_vx(v_sum, pow31_highest, v_src);
+ mulw(result, result, pow31_highest);
+ shadd(ary, consumed, ary, t0, elsize_shift);
+ subw(cnt, cnt, consumed);
+ andi(t1, cnt, ~(stride - 1));
+ bnez(t1, VEC_LOOP);
+
+ vmv_s_x(v_tmp, x0);
+ vredsum_vs(v_sum, v_sum, v_tmp);
+ vmv_x_s(t0, v_sum);
+ addw(result, result, t0);
+ beqz(cnt, DONE);
+
+ bind(SCALAR_TAIL);
+ shadd(ary_end, cnt, ary, t0, elsize_shift);
+
+ bind(SCALAR_TAIL_LOOP);
+ arrays_hashcode_elload(t0, Address(ary), eltype);
+ slli(t1, result, 5); // optimize 31 * result
+ subw(result, t1, result); // with result<<5 - result
+ addw(result, result, t0);
+ addi(ary, ary, elsize_bytes);
+ bne(ary, ary_end, SCALAR_TAIL_LOOP);
+
+ bind(DONE);
+ BLOCK_COMMENT("} // arrays_hashcode_v");
+}
+
int C2_MacroAssembler::arrays_hashcode_elsize(BasicType eltype) {
switch (eltype) {
- case T_BOOLEAN: return sizeof(jboolean);
- case T_BYTE: return sizeof(jbyte);
- case T_SHORT: return sizeof(jshort);
- case T_CHAR: return sizeof(jchar);
- case T_INT: return sizeof(jint);
- default:
- ShouldNotReachHere();
- return -1;
+ case T_BOOLEAN: return sizeof(jboolean);
+ case T_BYTE: return sizeof(jbyte);
+ case T_SHORT: return sizeof(jshort);
+ case T_CHAR: return sizeof(jchar);
+ case T_INT: return sizeof(jint);
+ default:
+ ShouldNotReachHere();
+ return -1;
}
}
void C2_MacroAssembler::arrays_hashcode_elload(Register dst, Address src, BasicType eltype) {
switch (eltype) {
- // T_BOOLEAN used as surrogate for unsigned byte
- case T_BOOLEAN: lbu(dst, src); break;
- case T_BYTE: lb(dst, src); break;
- case T_SHORT: lh(dst, src); break;
- case T_CHAR: lhu(dst, src); break;
- case T_INT: lw(dst, src); break;
- default:
- ShouldNotReachHere();
+ // T_BOOLEAN used as surrogate for unsigned byte
+ case T_BOOLEAN: lbu(dst, src); break;
+ case T_BYTE: lb(dst, src); break;
+ case T_SHORT: lh(dst, src); break;
+ case T_CHAR: lhu(dst, src); break;
+ case T_INT: lw(dst, src); break;
+ default:
+ ShouldNotReachHere();
+ }
+}
+
+void C2_MacroAssembler::arrays_hashcode_elload_v(VectorRegister vdst,
+ VectorRegister vtmp,
+ Register src,
+ BasicType eltype) {
+ assert_different_registers(vdst, vtmp);
+ switch (eltype) {
+ case T_BOOLEAN:
+ vle8_v(vtmp, src);
+ vzext_vf4(vdst, vtmp);
+ break;
+ case T_BYTE:
+ vle8_v(vtmp, src);
+ vsext_vf4(vdst, vtmp);
+ break;
+ case T_CHAR:
+ vle16_v(vtmp, src);
+ vzext_vf2(vdst, vtmp);
+ break;
+ case T_SHORT:
+ vle16_v(vtmp, src);
+ vsext_vf2(vdst, vtmp);
+ break;
+ case T_INT:
+ vle32_v(vdst, src);
+ break;
+ default:
+ ShouldNotReachHere();
}
}
diff --git a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp
index 309ef8d9d5e..2d5339dc153 100644
--- a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp
@@ -92,11 +92,15 @@
Register tmp3, Register tmp4,
Register tmp5, Register tmp6,
BasicType eltype);
-
- // helper function for arrays_hashcode
int arrays_hashcode_elsize(BasicType eltype);
void arrays_hashcode_elload(Register dst, Address src, BasicType eltype);
+ void arrays_hashcode_v(Register ary, Register cnt, Register result,
+ Register tmp1, Register tmp2, Register tmp3,
+ BasicType eltype);
+ void arrays_hashcode_elload_v(VectorRegister vdst, VectorRegister vtmp,
+ Register src, BasicType eltype);
+
void string_equals(Register r1, Register r2,
Register result, Register cnt1);
diff --git a/src/hotspot/cpu/riscv/frame_riscv.cpp b/src/hotspot/cpu/riscv/frame_riscv.cpp
index b677c980c78..19dbdd6aeae 100644
--- a/src/hotspot/cpu/riscv/frame_riscv.cpp
+++ b/src/hotspot/cpu/riscv/frame_riscv.cpp
@@ -217,8 +217,7 @@ bool frame::safe_for_sender(JavaThread *thread) {
nmethod* nm = sender_blob->as_nmethod_or_null();
if (nm != nullptr) {
- if (nm->is_deopt_mh_entry(sender_pc) || nm->is_deopt_entry(sender_pc) ||
- nm->method()->is_method_handle_intrinsic()) {
+ if (nm->is_deopt_entry(sender_pc) || nm->method()->is_method_handle_intrinsic()) {
return false;
}
}
@@ -427,49 +426,6 @@ JavaThread** frame::saved_thread_address(const frame& f) {
return thread_addr;
}
-//------------------------------------------------------------------------------
-// frame::verify_deopt_original_pc
-//
-// Verifies the calculated original PC of a deoptimization PC for the
-// given unextended SP.
-#ifdef ASSERT
-void frame::verify_deopt_original_pc(nmethod* nm, intptr_t* unextended_sp) {
- frame fr;
-
- // This is ugly but it's better than to change {get,set}_original_pc
- // to take an SP value as argument. And it's only a debugging
- // method anyway.
- fr._unextended_sp = unextended_sp;
-
- assert_cond(nm != nullptr);
- address original_pc = nm->get_original_pc(&fr);
- assert(nm->insts_contains_inclusive(original_pc),
- "original PC must be in the main code section of the compiled method (or must be immediately following it)");
-}
-#endif
-
-//------------------------------------------------------------------------------
-// frame::adjust_unextended_sp
-#ifdef ASSERT
-void frame::adjust_unextended_sp() {
- // On riscv, sites calling method handle intrinsics and lambda forms are treated
- // as any other call site. Therefore, no special action is needed when we are
- // returning to any of these call sites.
-
- if (_cb != nullptr) {
- nmethod* sender_nm = _cb->as_nmethod_or_null();
- if (sender_nm != nullptr) {
- // If the sender PC is a deoptimization point, get the original PC.
- if (sender_nm->is_deopt_entry(_pc) ||
- sender_nm->is_deopt_mh_entry(_pc)) {
- verify_deopt_original_pc(sender_nm, _unextended_sp);
- }
- }
- }
-}
-#endif
-
-
//------------------------------------------------------------------------------
// frame::sender_for_interpreter_frame
frame frame::sender_for_interpreter_frame(RegisterMap* map) const {
diff --git a/src/hotspot/cpu/riscv/frame_riscv.hpp b/src/hotspot/cpu/riscv/frame_riscv.hpp
index b4540c45ab8..ce5a8dde230 100644
--- a/src/hotspot/cpu/riscv/frame_riscv.hpp
+++ b/src/hotspot/cpu/riscv/frame_riscv.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -179,17 +179,10 @@
int _offset_unextended_sp; // for use in stack-chunk frames
};
- void adjust_unextended_sp() NOT_DEBUG_RETURN;
-
intptr_t* ptr_at_addr(int offset) const {
return (intptr_t*) addr_at(offset);
}
-#ifdef ASSERT
- // Used in frame::sender_for_{interpreter,compiled}_frame
- static void verify_deopt_original_pc(nmethod* nm, intptr_t* unextended_sp);
-#endif
-
public:
// Constructors
diff --git a/src/hotspot/cpu/riscv/frame_riscv.inline.hpp b/src/hotspot/cpu/riscv/frame_riscv.inline.hpp
index fb31760e20b..51a203c548c 100644
--- a/src/hotspot/cpu/riscv/frame_riscv.inline.hpp
+++ b/src/hotspot/cpu/riscv/frame_riscv.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -114,8 +114,6 @@ inline void frame::init(intptr_t* ptr_sp, intptr_t* ptr_fp, address pc) {
}
inline void frame::setup(address pc) {
- adjust_unextended_sp();
-
address original_pc = get_deopt_original_pc();
if (original_pc != nullptr) {
_pc = original_pc;
@@ -215,7 +213,6 @@ inline frame::frame(intptr_t* ptr_sp, intptr_t* ptr_fp) {
// value.
_cb = CodeCache::find_blob(_pc);
- adjust_unextended_sp();
address original_pc = get_deopt_original_pc();
if (original_pc != nullptr) {
diff --git a/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.cpp b/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.cpp
index ef5dcdd8074..64e9cbe1f47 100644
--- a/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.cpp
@@ -87,15 +87,54 @@ void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm
}
}
-void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
- Register start, Register count, Register tmp, RegSet saved_regs) {
- __ push_reg(saved_regs, sp);
+void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm,
+ DecoratorSet decorators,
+ Register start,
+ Register count,
+ Register tmp,
+ RegSet saved_regs) {
assert_different_registers(start, count, tmp);
- assert_different_registers(c_rarg0, count);
- __ mv(c_rarg0, start);
- __ mv(c_rarg1, count);
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_post_entry), 2);
- __ pop_reg(saved_regs, sp);
+
+ Label loop, next, done;
+
+ // Zero count? Nothing to do.
+ __ beqz(count, done);
+
+ // Calculate the number of card marks to set. Since the object might start and
+ // end within a card, we need to calculate this via the card table indexes of
+ // the actual start and last addresses covered by the object.
+ // Temporarily use the count register for the last element address.
+ __ shadd(count, count, start, tmp, LogBytesPerHeapOop); // end = start + count << LogBytesPerHeapOop
+ __ subi(count, count, BytesPerHeapOop); // Use last element address for end.
+
+ __ srli(start, start, CardTable::card_shift());
+ __ srli(count, count, CardTable::card_shift());
+ __ sub(count, count, start); // Number of bytes to mark - 1.
+
+ // Add card table base offset to start.
+ Address card_table_address(xthread, G1ThreadLocalData::card_table_base_offset());
+ __ ld(tmp, card_table_address);
+ __ add(start, start, tmp);
+
+ __ bind(loop);
+ if (UseCondCardMark) {
+ __ add(tmp, start, count);
+ __ lbu(tmp, Address(tmp, 0));
+ static_assert((uint)G1CardTable::clean_card_val() == 0xff, "must be");
+ __ subi(tmp, tmp, G1CardTable::clean_card_val()); // Convert to clean_card_value() to a comparison
+ // against zero to avoid use of an extra temp.
+ __ bnez(tmp, next);
+ }
+
+ __ add(tmp, start, count);
+ static_assert(G1CardTable::dirty_card_val() == 0, "must be to use zr");
+ __ sb(zr, Address(tmp, 0));
+
+ __ bind(next);
+ __ subi(count, count, 1);
+ __ bgez(count, loop);
+
+ __ bind(done);
}
static void generate_queue_test_and_insertion(MacroAssembler* masm, ByteSize index_offset, ByteSize buffer_offset, Label& runtime,
@@ -189,47 +228,40 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
}
-static void generate_post_barrier_fast_path(MacroAssembler* masm,
- const Register store_addr,
- const Register new_val,
- const Register tmp1,
- const Register tmp2,
- Label& done,
- bool new_val_may_be_null) {
+static void generate_post_barrier(MacroAssembler* masm,
+ const Register store_addr,
+ const Register new_val,
+ const Register thread,
+ const Register tmp1,
+ const Register tmp2,
+ Label& done,
+ bool new_val_may_be_null) {
+ assert(thread == xthread, "must be");
+ assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, noreg);
// Does store cross heap regions?
__ xorr(tmp1, store_addr, new_val); // tmp1 := store address ^ new value
__ srli(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes); // tmp1 := ((store address ^ new value) >> LogOfHRGrainBytes)
__ beqz(tmp1, done);
+
// Crosses regions, storing null?
if (new_val_may_be_null) {
__ beqz(new_val, done);
}
- // Storing region crossing non-null, is card young?
+ // Storing region crossing non-null, is card clean?
__ srli(tmp1, store_addr, CardTable::card_shift()); // tmp1 := card address relative to card table base
- __ load_byte_map_base(tmp2); // tmp2 := card table base address
- __ add(tmp1, tmp1, tmp2); // tmp1 := card address
- __ lbu(tmp2, Address(tmp1)); // tmp2 := card
-}
-static void generate_post_barrier_slow_path(MacroAssembler* masm,
- const Register thread,
- const Register tmp1,
- const Register tmp2,
- Label& done,
- Label& runtime) {
- __ membar(MacroAssembler::StoreLoad); // StoreLoad membar
- __ lbu(tmp2, Address(tmp1)); // tmp2 := card
- __ beqz(tmp2, done, true);
- // Storing a region crossing, non-null oop, card is clean.
- // Dirty card and log.
- STATIC_ASSERT(CardTable::dirty_card_val() == 0);
- __ sb(zr, Address(tmp1)); // *(card address) := dirty_card_val
- generate_queue_test_and_insertion(masm,
- G1ThreadLocalData::dirty_card_queue_index_offset(),
- G1ThreadLocalData::dirty_card_queue_buffer_offset(),
- runtime,
- thread, tmp1, tmp2, t0);
- __ j(done);
+ Address card_table_address(xthread, G1ThreadLocalData::card_table_base_offset());
+ __ ld(tmp2, card_table_address); // tmp2 := card table base address
+ __ add(tmp1, tmp1, tmp2); // tmp1 := card address
+ if (UseCondCardMark) {
+ static_assert((uint)G1CardTable::clean_card_val() == 0xff, "must be");
+ __ lbu(tmp2, Address(tmp1, 0)); // tmp2 := card
+ __ subi(tmp2, tmp2, G1CardTable::clean_card_val()); // Convert to clean_card_value() to a comparison
+ // against zero to avoid use of an extra temp.
+ __ bnez(tmp2, done);
+ }
+ static_assert((uint)G1CardTable::dirty_card_val() == 0, "must be to use zr");
+ __ sb(zr, Address(tmp1, 0));
}
void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
@@ -238,27 +270,8 @@ void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
Register thread,
Register tmp1,
Register tmp2) {
- assert(thread == xthread, "must be");
- assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, t0);
- assert(store_addr != noreg && new_val != noreg && tmp1 != noreg && tmp2 != noreg,
- "expecting a register");
-
Label done;
- Label runtime;
-
- generate_post_barrier_fast_path(masm, store_addr, new_val, tmp1, tmp2, done, true /* new_val_may_be_null */);
- // If card is young, jump to done (tmp2 holds the card value)
- __ mv(t0, (int)G1CardTable::g1_young_card_val());
- __ beq(tmp2, t0, done); // card == young_card_val?
- generate_post_barrier_slow_path(masm, thread, tmp1, tmp2, done, runtime);
-
- __ bind(runtime);
- // save the live input values
- RegSet saved = RegSet::of(store_addr);
- __ push_reg(saved, sp);
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), tmp1, thread);
- __ pop_reg(saved, sp);
-
+ generate_post_barrier(masm, store_addr, new_val, thread, tmp1, tmp2, done, true /* new_val_may_be_null */);
__ bind(done);
}
@@ -318,37 +331,10 @@ void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm,
Register thread,
Register tmp1,
Register tmp2,
- G1PostBarrierStubC2* stub) {
- assert(thread == xthread, "must be");
- assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, t0);
- assert(store_addr != noreg && new_val != noreg && tmp1 != noreg && tmp2 != noreg,
- "expecting a register");
-
- stub->initialize_registers(thread, tmp1, tmp2);
-
- bool new_val_may_be_null = (stub->barrier_data() & G1C2BarrierPostNotNull) == 0;
- generate_post_barrier_fast_path(masm, store_addr, new_val, tmp1, tmp2, *stub->continuation(), new_val_may_be_null);
- // If card is not young, jump to stub (slow path) (tmp2 holds the card value)
- __ mv(t0, (int)G1CardTable::g1_young_card_val());
- __ bne(tmp2, t0, *stub->entry(), true);
-
- __ bind(*stub->continuation());
-}
-
-void G1BarrierSetAssembler::generate_c2_post_barrier_stub(MacroAssembler* masm,
- G1PostBarrierStubC2* stub) const {
- Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
- Label runtime;
- Register thread = stub->thread();
- Register tmp1 = stub->tmp1(); // tmp1 holds the card address.
- Register tmp2 = stub->tmp2();
-
- __ bind(*stub->entry());
- generate_post_barrier_slow_path(masm, thread, tmp1, tmp2, *stub->continuation(), runtime);
-
- __ bind(runtime);
- generate_c2_barrier_runtime_call(masm, stub, tmp1, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry));
- __ j(*stub->continuation());
+ bool new_val_may_be_null) {
+ Label done;
+ generate_post_barrier(masm, store_addr, new_val, thread, tmp1, tmp2, done, new_val_may_be_null);
+ __ bind(done);
}
#endif // COMPILER2
@@ -443,20 +429,19 @@ void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrier
__ j(*stub->continuation());
}
-void G1BarrierSetAssembler::gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) {
- G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
- __ bind(*stub->entry());
- assert(stub->addr()->is_register(), "Precondition");
- assert(stub->new_val()->is_register(), "Precondition");
- Register new_val_reg = stub->new_val()->as_register();
- __ beqz(new_val_reg, *stub->continuation(), /* is_far */ true);
- ce->store_parameter(stub->addr()->as_pointer_register(), 0);
- __ far_call(RuntimeAddress(bs->post_barrier_c1_runtime_code_blob()->code_begin()));
- __ j(*stub->continuation());
-}
-
#undef __
+void G1BarrierSetAssembler::g1_write_barrier_post_c1(MacroAssembler* masm,
+ Register store_addr,
+ Register new_val,
+ Register thread,
+ Register tmp1,
+ Register tmp2) {
+ Label done;
+ generate_post_barrier(masm, store_addr, new_val, thread, tmp1, tmp2, done, true /* new_val_may_be_null */);
+ masm->bind(done);
+}
+
#define __ sasm->
void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
@@ -507,74 +492,6 @@ void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler*
__ epilogue();
}
-void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) {
- __ prologue("g1_post_barrier", false);
-
- // arg0 : store_address
- Address store_addr(fp, 2 * BytesPerWord); // 2 BytesPerWord from fp
-
- BarrierSet* bs = BarrierSet::barrier_set();
- CardTableBarrierSet* ctbs = barrier_set_cast(bs);
-
- Label done;
- Label runtime;
-
- // At this point we know new_value is non-null and the new_value crosses regions.
- // Must check to see if card is already dirty
- const Register thread = xthread;
-
- Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
- Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
-
- const Register card_offset = t1;
- // RA is free here, so we can use it to hold the byte_map_base.
- const Register byte_map_base = ra;
-
- assert_different_registers(card_offset, byte_map_base, t0);
-
- __ load_parameter(0, card_offset);
- __ srli(card_offset, card_offset, CardTable::card_shift());
- __ load_byte_map_base(byte_map_base);
-
- // Convert card offset into an address in card_addr
- Register card_addr = card_offset;
- __ add(card_addr, byte_map_base, card_addr);
-
- __ lbu(t0, Address(card_addr, 0));
- __ sub(t0, t0, (int)G1CardTable::g1_young_card_val());
- __ beqz(t0, done);
-
- assert((int)CardTable::dirty_card_val() == 0, "must be 0");
-
- __ membar(MacroAssembler::StoreLoad);
- __ lbu(t0, Address(card_addr, 0));
- __ beqz(t0, done);
-
- // storing region crossing non-null, card is clean.
- // dirty card and log.
- __ sb(zr, Address(card_addr, 0));
-
- __ ld(t0, queue_index);
- __ beqz(t0, runtime);
- __ subi(t0, t0, wordSize);
- __ sd(t0, queue_index);
-
- // Reuse RA to hold buffer_addr
- const Register buffer_addr = ra;
-
- __ ld(buffer_addr, buffer);
- __ add(t0, buffer_addr, t0);
- __ sd(card_addr, Address(t0, 0));
- __ j(done);
-
- __ bind(runtime);
- __ push_call_clobbered_registers();
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
- __ pop_call_clobbered_registers();
- __ bind(done);
- __ epilogue();
-}
-
#undef __
#endif // COMPILER1
diff --git a/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.hpp b/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.hpp
index 26310231362..654ba934242 100644
--- a/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, 2024, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -35,9 +35,7 @@ class LIR_Assembler;
#endif
class StubAssembler;
class G1PreBarrierStub;
-class G1PostBarrierStub;
class G1PreBarrierStubC2;
-class G1PostBarrierStubC2;
class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
protected:
@@ -68,10 +66,16 @@ protected:
public:
#ifdef COMPILER1
void gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub);
- void gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub);
void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm);
- void generate_c1_post_barrier_runtime_stub(StubAssembler* sasm);
+
+ void g1_write_barrier_post_c1(MacroAssembler* masm,
+ Register store_addr,
+ Register new_val,
+ Register thread,
+ Register tmp1,
+ Register tmp2);
+
#endif
#ifdef COMPILER2
@@ -90,9 +94,7 @@ public:
Register thread,
Register tmp1,
Register tmp2,
- G1PostBarrierStubC2* c2_stub);
- void generate_c2_post_barrier_stub(MacroAssembler* masm,
- G1PostBarrierStubC2* stub) const;
+ bool new_val_may_be_null);
#endif
void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
diff --git a/src/hotspot/cpu/riscv/gc/g1/g1_riscv.ad b/src/hotspot/cpu/riscv/gc/g1/g1_riscv.ad
index 7a525323021..8461a36e68c 100644
--- a/src/hotspot/cpu/riscv/gc/g1/g1_riscv.ad
+++ b/src/hotspot/cpu/riscv/gc/g1/g1_riscv.ad
@@ -1,5 +1,5 @@
//
-// Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
// Copyright (c) 2024, Huawei Technologies Co., Ltd. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
@@ -63,13 +63,13 @@ static void write_barrier_post(MacroAssembler* masm,
Register new_val,
Register tmp1,
Register tmp2) {
- if (!G1PostBarrierStubC2::needs_barrier(node)) {
+ if (!G1BarrierStubC2::needs_post_barrier(node)) {
return;
}
Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
G1BarrierSetAssembler* g1_asm = static_cast(BarrierSet::barrier_set()->barrier_set_assembler());
- G1PostBarrierStubC2* const stub = G1PostBarrierStubC2::create(node);
- g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, xthread, tmp1, tmp2, stub);
+ bool new_val_may_be_null = G1BarrierStubC2::post_new_val_may_be_null(node);
+ g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, xthread, tmp1, tmp2, new_val_may_be_null);
}
%}
diff --git a/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.cpp b/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.cpp
index 387db778c1f..f5916000890 100644
--- a/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.cpp
@@ -217,7 +217,7 @@ address BarrierSetAssembler::patching_epoch_addr() {
}
void BarrierSetAssembler::increment_patching_epoch() {
- Atomic::inc(&_patching_epoch);
+ AtomicAccess::inc(&_patching_epoch);
}
void BarrierSetAssembler::clear_patching_epoch() {
diff --git a/src/hotspot/cpu/riscv/gc/shared/barrierSetNMethod_riscv.cpp b/src/hotspot/cpu/riscv/gc/shared/barrierSetNMethod_riscv.cpp
index 4fa9b4b04fb..5003b9584a3 100644
--- a/src/hotspot/cpu/riscv/gc/shared/barrierSetNMethod_riscv.cpp
+++ b/src/hotspot/cpu/riscv/gc/shared/barrierSetNMethod_riscv.cpp
@@ -106,22 +106,22 @@ public:
}
int get_value() {
- return Atomic::load_acquire(guard_addr());
+ return AtomicAccess::load_acquire(guard_addr());
}
void set_value(int value, int bit_mask) {
if (bit_mask == ~0) {
- Atomic::release_store(guard_addr(), value);
+ AtomicAccess::release_store(guard_addr(), value);
return;
}
assert((value & ~bit_mask) == 0, "trying to set bits outside the mask");
value &= bit_mask;
- int old_value = Atomic::load(guard_addr());
+ int old_value = AtomicAccess::load(guard_addr());
while (true) {
// Only bits in the mask are changed
int new_value = value | (old_value & ~bit_mask);
if (new_value == old_value) break;
- int v = Atomic::cmpxchg(guard_addr(), old_value, new_value, memory_order_release);
+ int v = AtomicAccess::cmpxchg(guard_addr(), old_value, new_value, memory_order_release);
if (v == old_value) break;
old_value = v;
}
diff --git a/src/hotspot/cpu/riscv/globalDefinitions_riscv.hpp b/src/hotspot/cpu/riscv/globalDefinitions_riscv.hpp
index 407017ee1c0..57223cf4390 100644
--- a/src/hotspot/cpu/riscv/globalDefinitions_riscv.hpp
+++ b/src/hotspot/cpu/riscv/globalDefinitions_riscv.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2015, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -44,8 +44,6 @@ const bool CCallingConventionRequiresIntsAsLongs = false;
#define SUPPORTS_NATIVE_CX8
-#define SUPPORT_MONITOR_COUNT
-
#define SUPPORT_RESERVED_STACK_AREA
#define USE_POINTERS_TO_REGISTER_IMPL_ARRAY
diff --git a/src/hotspot/cpu/riscv/javaFrameAnchor_riscv.hpp b/src/hotspot/cpu/riscv/javaFrameAnchor_riscv.hpp
index 1293fae0c0b..6bf1230914a 100644
--- a/src/hotspot/cpu/riscv/javaFrameAnchor_riscv.hpp
+++ b/src/hotspot/cpu/riscv/javaFrameAnchor_riscv.hpp
@@ -39,25 +39,23 @@ public:
// 3 - restoring an old state (javaCalls)
void clear(void) {
+ // No hardware barriers are necessary. All members are volatile and the profiler
+ // is run from a signal handler and the only observer is the thread its running on.
+
// clearing _last_Java_sp must be first
_last_Java_sp = nullptr;
- OrderAccess::release();
_last_Java_fp = nullptr;
_last_Java_pc = nullptr;
}
void copy(JavaFrameAnchor* src) {
- // In order to make sure the transition state is valid for "this"
+ // No hardware barriers are necessary. All members are volatile and the profiler
+ // is run from a signal handler and the only observer is the thread its running on.
+
// We must clear _last_Java_sp before copying the rest of the new data
- //
- // Hack Alert: Temporary bugfix for 4717480/4721647
- // To act like previous version (pd_cache_state) don't null _last_Java_sp
- // unless the value is changing
- //
assert(src != nullptr, "Src should not be null.");
if (_last_Java_sp != src->_last_Java_sp) {
_last_Java_sp = nullptr;
- OrderAccess::release();
}
_last_Java_fp = src->_last_Java_fp;
_last_Java_pc = src->_last_Java_pc;
diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
index 8f136135a89..700e42e6194 100644
--- a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
@@ -225,36 +225,6 @@ void MacroAssembler::pop_cont_fastpath(Register java_thread) {
bind(done);
}
-void MacroAssembler::inc_held_monitor_count(Register tmp) {
- Address dst(xthread, JavaThread::held_monitor_count_offset());
- ld(tmp, dst);
- addi(tmp, tmp, 1);
- sd(tmp, dst);
-#ifdef ASSERT
- Label ok;
- test_bit(tmp, tmp, 63);
- beqz(tmp, ok);
- STOP("assert(held monitor count overflow)");
- should_not_reach_here();
- bind(ok);
-#endif
-}
-
-void MacroAssembler::dec_held_monitor_count(Register tmp) {
- Address dst(xthread, JavaThread::held_monitor_count_offset());
- ld(tmp, dst);
- subi(tmp, tmp, 1);
- sd(tmp, dst);
-#ifdef ASSERT
- Label ok;
- test_bit(tmp, tmp, 63);
- beqz(tmp, ok);
- STOP("assert(held monitor count underflow)");
- should_not_reach_here();
- bind(ok);
-#endif
-}
-
int MacroAssembler::align(int modulus, int extra_offset) {
CompressibleScope scope(this);
intptr_t before = offset();
@@ -355,14 +325,15 @@ void MacroAssembler::call_VM(Register oop_result,
}
void MacroAssembler::post_call_nop() {
+ assert(!in_compressible_scope(), "Must be");
+ assert_alignment(pc());
if (!Continuations::enabled()) {
return;
}
- relocate(post_call_nop_Relocation::spec(), [&] {
- InlineSkippedInstructionsCounter skipCounter(this);
- nop();
- li32(zr, 0);
- });
+ relocate(post_call_nop_Relocation::spec());
+ InlineSkippedInstructionsCounter skipCounter(this);
+ nop();
+ li32(zr, 0);
}
// these are no-ops overridden by InterpreterMacroAssembler
@@ -389,12 +360,14 @@ void MacroAssembler::set_last_Java_frame(Register last_java_sp,
last_java_sp = esp;
}
- sd(last_java_sp, Address(xthread, JavaThread::last_Java_sp_offset()));
-
// last_java_fp is optional
if (last_java_fp->is_valid()) {
sd(last_java_fp, Address(xthread, JavaThread::last_Java_fp_offset()));
}
+
+ // We must set sp last.
+ sd(last_java_sp, Address(xthread, JavaThread::last_Java_sp_offset()));
+
}
void MacroAssembler::set_last_Java_frame(Register last_java_sp,
@@ -5013,7 +4986,7 @@ address MacroAssembler::reloc_call(Address entry, Register tmp) {
address MacroAssembler::ic_call(address entry, jint method_index) {
RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index);
- IncompressibleScope scope(this); // relocations
+ assert(!in_compressible_scope(), "Must be");
movptr(t0, (address)Universe::non_oop_word(), t1);
assert_cond(entry != nullptr);
return reloc_call(Address(entry, rh));
@@ -5871,13 +5844,14 @@ void MacroAssembler::fill_words(Register base, Register cnt, Register value) {
// in cnt.
//
// NOTE: This is intended to be used in the zero_blocks() stub. If
-// you want to use it elsewhere, note that cnt must be >= CacheLineSize.
+// you want to use it elsewhere, note that cnt must be >= zicboz_block_size.
void MacroAssembler::zero_dcache_blocks(Register base, Register cnt, Register tmp1, Register tmp2) {
+ int zicboz_block_size = VM_Version::zicboz_block_size.value();
Label initial_table_end, loop;
// Align base with cache line size.
neg(tmp1, base);
- andi(tmp1, tmp1, CacheLineSize - 1);
+ andi(tmp1, tmp1, zicboz_block_size - 1);
// tmp1: the number of bytes to be filled to align the base with cache line size.
add(base, base, tmp1);
@@ -5887,16 +5861,16 @@ void MacroAssembler::zero_dcache_blocks(Register base, Register cnt, Register tm
la(tmp1, initial_table_end);
sub(tmp2, tmp1, tmp2);
jr(tmp2);
- for (int i = -CacheLineSize + wordSize; i < 0; i += wordSize) {
+ for (int i = -zicboz_block_size + wordSize; i < 0; i += wordSize) {
sd(zr, Address(base, i));
}
bind(initial_table_end);
- mv(tmp1, CacheLineSize / wordSize);
+ mv(tmp1, zicboz_block_size / wordSize);
bind(loop);
cbo_zero(base);
sub(cnt, cnt, tmp1);
- addi(base, base, CacheLineSize);
+ addi(base, base, zicboz_block_size);
bge(cnt, tmp1, loop);
}
diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
index 13b70d5dbd7..9e713e90270 100644
--- a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2024, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -849,9 +849,6 @@ public:
void push_cont_fastpath(Register java_thread = xthread);
void pop_cont_fastpath(Register java_thread = xthread);
- void inc_held_monitor_count(Register tmp);
- void dec_held_monitor_count(Register tmp);
-
// if heap base register is used - reinit it with the correct value
void reinit_heapbase();
diff --git a/src/hotspot/cpu/riscv/nativeInst_riscv.cpp b/src/hotspot/cpu/riscv/nativeInst_riscv.cpp
index eb8ef6349b0..5d1cac72ade 100644
--- a/src/hotspot/cpu/riscv/nativeInst_riscv.cpp
+++ b/src/hotspot/cpu/riscv/nativeInst_riscv.cpp
@@ -28,11 +28,13 @@
#include "code/compiledIC.hpp"
#include "nativeInst_riscv.hpp"
#include "oops/oop.inline.hpp"
+#include "runtime/atomicAccess.hpp"
#include "runtime/handles.hpp"
#include "runtime/orderAccess.hpp"
#include "runtime/safepoint.hpp"
#include "runtime/sharedRuntime.hpp"
#include "runtime/stubRoutines.hpp"
+#include "utilities/align.hpp"
#include "utilities/ostream.hpp"
#ifdef COMPILER1
#include "c1/c1_Runtime1.hpp"
@@ -52,15 +54,15 @@ address NativeCall::destination() const {
address addr = instruction_address();
assert(NativeCall::is_at(addr), "unexpected code at call site");
- address destination = MacroAssembler::target_addr_for_insn(addr);
+ address stub_addr = MacroAssembler::target_addr_for_insn(addr);
CodeBlob* cb = CodeCache::find_blob(addr);
assert(cb != nullptr && cb->is_nmethod(), "nmethod expected");
nmethod *nm = (nmethod *)cb;
- assert(nm != nullptr, "Sanity");
- assert(nm->stub_contains(destination), "Sanity");
- assert(destination != nullptr, "Sanity");
- return stub_address_destination_at(destination);
+ assert(nm->stub_contains(stub_addr), "Sanity");
+ assert(stub_addr != nullptr, "Sanity");
+
+ return stub_address_destination_at(stub_addr);
}
address NativeCall::reloc_destination() {
@@ -89,6 +91,30 @@ void NativeCall::print() {
tty->print_cr(PTR_FORMAT ": auipc,ld,jalr x1, offset/reg, ", p2i(instruction_address()));
}
+void NativeCall::optimize_call(address dest, bool mt_safe) {
+ // Skip over auipc + ld
+ address jmp_ins_pc = instruction_address() + 2 * NativeInstruction::instruction_size;
+ // Rutime calls may be unaligned, but they are never changed after relocation.
+ assert(!mt_safe || is_aligned(jmp_ins_pc, NativeInstruction::instruction_size), "Must be naturally aligned: %p", jmp_ins_pc);
+ // If reachable use JAL
+ if (Assembler::reachable_from_branch_at(jmp_ins_pc, dest)) {
+ int64_t distance = dest - jmp_ins_pc;
+ uint32_t new_jal = Assembler::encode_jal(ra, distance);
+ AtomicAccess::store((uint32_t *)jmp_ins_pc, new_jal);
+ } else if (!MacroAssembler::is_jalr_at(jmp_ins_pc)) { // The jalr is always identical: jalr ra, 0(t1)
+ uint32_t new_jalr = Assembler::encode_jalr(ra, t1, 0);
+ AtomicAccess::store((uint32_t *)jmp_ins_pc, new_jalr);
+ } else {
+ // No change to instruction stream
+ return;
+ }
+ // We changed instruction stream
+ if (mt_safe) {
+ // IC invalidate provides a leading full fence, it thus happens after we changed the instruction stream.
+ ICache::invalidate_range(jmp_ins_pc, NativeInstruction::instruction_size);
+ }
+}
+
bool NativeCall::set_destination_mt_safe(address dest) {
assert(NativeCall::is_at(instruction_address()), "unexpected code at call site");
assert((CodeCache_lock->is_locked() || SafepointSynchronize::is_at_safepoint()) ||
@@ -96,15 +122,17 @@ bool NativeCall::set_destination_mt_safe(address dest) {
"concurrent code patching");
address stub_addr = stub_address();
- if (stub_addr != nullptr) {
- set_stub_address_destination_at(stub_addr, dest);
- return true;
- }
+ assert(stub_addr != nullptr, "No stub?");
+ set_stub_address_destination_at(stub_addr, dest); // release
+ // optimize_call happens after we stored new address in addr stub.
+ // patches jalr -> jal/jal -> jalr depending on dest
+ optimize_call(dest, true);
- return false;
+ return true;
}
-bool NativeCall::reloc_set_destination(address dest) {
+// The argument passed in is the address to the stub containing the destination
+bool NativeCall::reloc_set_destination(address stub_addr) {
address call_addr = instruction_address();
assert(NativeCall::is_at(call_addr), "unexpected code at call site");
@@ -113,10 +141,12 @@ bool NativeCall::reloc_set_destination(address dest) {
if (code->is_nmethod()) {
// TODO: Need to revisit this when porting the AOT features.
- assert(dest != nullptr, "Sanity");
- assert(dest == trampoline_stub_Relocation::get_trampoline_for(call_addr,
- code->as_nmethod()), "Sanity");
- MacroAssembler::pd_patch_instruction_size(call_addr, dest);
+ assert(stub_addr != nullptr, "Sanity");
+ assert(stub_addr == trampoline_stub_Relocation::get_trampoline_for(call_addr, code->as_nmethod()), "Sanity");
+ MacroAssembler::pd_patch_instruction_size(call_addr, stub_addr); // patches auipc + ld to stub_addr
+
+ address dest = stub_address_destination_at(stub_addr);
+ optimize_call(dest, false); // patches jalr -> jal/jal -> jalr depending on dest
}
return true;
@@ -142,9 +172,9 @@ address NativeCall::stub_address() {
CodeBlob *code = CodeCache::find_blob(call_addr);
assert(code != nullptr, "Could not find the containing code blob");
- address dest = MacroAssembler::target_addr_for_insn(call_addr);
- assert(code->contains(dest), "Sanity");
- return dest;
+ address stub_addr = MacroAssembler::target_addr_for_insn(call_addr);
+ assert(code->contains(stub_addr), "Sanity");
+ return stub_addr;
}
bool NativeCall::is_at(address addr) {
@@ -160,6 +190,15 @@ bool NativeCall::is_at(address addr) {
(MacroAssembler::extract_rd(addr + 2 * instr_size) == x1)) {
return true;
}
+ if (MacroAssembler::is_auipc_at(addr) &&
+ MacroAssembler::is_ld_at(addr + instr_size) &&
+ MacroAssembler::is_jal_at(addr + 2 * instr_size) &&
+ (MacroAssembler::extract_rd(addr) == x6) &&
+ (MacroAssembler::extract_rd(addr + instr_size) == x6) &&
+ (MacroAssembler::extract_rs1(addr + instr_size) == x6) &&
+ (MacroAssembler::extract_rd(addr + 2 * instr_size) == x1)) {
+ return true;
+ }
return false;
}
@@ -292,13 +331,10 @@ bool NativeInstruction::is_safepoint_poll() {
return MacroAssembler::is_lwu_to_zr(address(this));
}
-void NativeIllegalInstruction::insert(address code_pos) {
- assert_cond(code_pos != nullptr);
- Assembler::sd_instr(code_pos, 0xffffffff); // all bits ones is permanently reserved as an illegal instruction
-}
-
bool NativeInstruction::is_stop() {
- return uint_at(0) == 0xc0101073; // an illegal instruction, 'csrrw x0, time, x0'
+ // an illegal instruction, 'csrrw x0, time, x0'
+ uint32_t encoded = Assembler::encode_csrrw(x0, Assembler::time, x0);
+ return uint_at(0) == encoded;
}
//-------------------------------------------------------------------
@@ -308,6 +344,8 @@ void NativeGeneralJump::insert_unconditional(address code_pos, address entry) {
MacroAssembler a(&cb);
Assembler::IncompressibleScope scope(&a); // Fixed length: see NativeGeneralJump::get_instruction_size()
+ MacroAssembler::assert_alignment(code_pos);
+
int32_t offset = 0;
a.movptr(t1, entry, offset, t0); // lui, lui, slli, add
a.jr(t1, offset); // jalr
@@ -339,6 +377,7 @@ bool NativePostCallNop::decode(int32_t& oopmap_slot, int32_t& cb_offset) const {
}
bool NativePostCallNop::patch(int32_t oopmap_slot, int32_t cb_offset) {
+ MacroAssembler::assert_alignment(addr_at(4));
if (((oopmap_slot & 0xff) != oopmap_slot) || ((cb_offset & 0xffffff) != cb_offset)) {
return false; // cannot encode
}
@@ -350,14 +389,17 @@ bool NativePostCallNop::patch(int32_t oopmap_slot, int32_t cb_offset) {
return true; // successfully encoded
}
-void NativeDeoptInstruction::verify() {
+bool NativeDeoptInstruction::is_deopt_at(address instr) {
+ assert(instr != nullptr, "Must be");
+ uint32_t value = Assembler::ld_instr(instr);
+ uint32_t encoded = Assembler::encode_csrrw(x0, Assembler::instret, x0);
+ return value == encoded;
}
// Inserts an undefined instruction at a given pc
void NativeDeoptInstruction::insert(address code_pos) {
- // 0xc0201073 encodes CSRRW x0, instret, x0
- uint32_t insn = 0xc0201073;
- uint32_t *pos = (uint32_t *) code_pos;
- *pos = insn;
+ MacroAssembler::assert_alignment(code_pos);
+ uint32_t encoded = Assembler::encode_csrrw(x0, Assembler::instret, x0);
+ Assembler::sd_instr(code_pos, encoded);
ICache::invalidate_range(code_pos, 4);
}
diff --git a/src/hotspot/cpu/riscv/nativeInst_riscv.hpp b/src/hotspot/cpu/riscv/nativeInst_riscv.hpp
index 8e6632419e2..d990cfbc50d 100644
--- a/src/hotspot/cpu/riscv/nativeInst_riscv.hpp
+++ b/src/hotspot/cpu/riscv/nativeInst_riscv.hpp
@@ -156,6 +156,10 @@ class NativeCall: private NativeInstruction {
static void set_stub_address_destination_at(address dest, address value);
// return target address at stub
static address stub_address_destination_at(address src);
+ // We either have a jalr or jal depending on distance to old destination.
+ // This method emits a new jal if new destination is within jal reach.
+ // Otherwise restores the jalr which can reach any destination.
+ void optimize_call(address dest, bool mt_safe = true);
};
// An interface for accessing/manipulating native mov reg, imm instructions.
@@ -290,12 +294,6 @@ inline NativeGeneralJump* nativeGeneralJump_at(address addr) {
return jump;
}
-class NativeIllegalInstruction: public NativeInstruction {
- public:
- // Insert illegal opcode as specific address
- static void insert(address code_pos);
-};
-
inline bool NativeInstruction::is_nop() const {
uint32_t insn = Assembler::ld_instr(addr_at(0));
return insn == 0x13;
@@ -349,14 +347,7 @@ class NativeDeoptInstruction: public NativeInstruction {
address instruction_address() const { return addr_at(instruction_offset); }
address next_instruction_address() const { return addr_at(instruction_size); }
- void verify();
-
- static bool is_deopt_at(address instr) {
- assert(instr != nullptr, "");
- uint32_t value = Assembler::ld_instr(instr);
- // 0xc0201073 encodes CSRRW x0, instret, x0
- return value == 0xc0201073;
- }
+ static bool is_deopt_at(address instr);
// MT-safe patching
static void insert(address code_pos);
diff --git a/src/hotspot/cpu/riscv/riscv.ad b/src/hotspot/cpu/riscv/riscv.ad
index 739a525c9a4..009acd628a0 100644
--- a/src/hotspot/cpu/riscv/riscv.ad
+++ b/src/hotspot/cpu/riscv/riscv.ad
@@ -1269,6 +1269,26 @@ int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
return align_up(current_offset, alignment_required()) - current_offset;
}
+int CallRuntimeDirectNode::compute_padding(int current_offset) const
+{
+ return align_up(current_offset, alignment_required()) - current_offset;
+}
+
+int CallLeafDirectNode::compute_padding(int current_offset) const
+{
+ return align_up(current_offset, alignment_required()) - current_offset;
+}
+
+int CallLeafDirectVectorNode::compute_padding(int current_offset) const
+{
+ return align_up(current_offset, alignment_required()) - current_offset;
+}
+
+int CallLeafNoFPDirectNode::compute_padding(int current_offset) const
+{
+ return align_up(current_offset, alignment_required()) - current_offset;
+}
+
//=============================================================================
#ifndef PRODUCT
@@ -2132,10 +2152,6 @@ RegMask Matcher::modL_proj_mask() {
return RegMask();
}
-const RegMask Matcher::method_handle_invoke_SP_save_mask() {
- return FP_REG_mask();
-}
-
bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
assert_cond(addp != nullptr);
for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
@@ -8175,7 +8191,7 @@ instruct unnecessary_membar_volatile_rvtso() %{
ins_cost(0);
size(0);
-
+
format %{ "#@unnecessary_membar_volatile_rvtso (unnecessary so empty encoding)" %}
ins_encode %{
__ block_comment("unnecessary_membar_volatile_rvtso");
@@ -10509,6 +10525,7 @@ instruct CallRuntimeDirect(method meth)
ins_encode(riscv_enc_java_to_runtime(meth));
ins_pipe(pipe_class_call);
+ ins_alignment(4);
%}
// Call Runtime Instruction
@@ -10526,6 +10543,7 @@ instruct CallLeafDirect(method meth)
ins_encode(riscv_enc_java_to_runtime(meth));
ins_pipe(pipe_class_call);
+ ins_alignment(4);
%}
// Call Runtime Instruction without safepoint and with vector arguments
@@ -10543,6 +10561,7 @@ instruct CallLeafDirectVector(method meth)
ins_encode(riscv_enc_java_to_runtime(meth));
ins_pipe(pipe_class_call);
+ ins_alignment(4);
%}
// Call Runtime Instruction
@@ -10560,6 +10579,7 @@ instruct CallLeafNoFPDirect(method meth)
ins_encode(riscv_enc_java_to_runtime(meth));
ins_pipe(pipe_class_call);
+ ins_alignment(4);
%}
// ============================================================================
@@ -10971,6 +10991,7 @@ instruct arrays_hashcode(iRegP_R11 ary, iRegI_R12 cnt, iRegI_R10 result, immI ba
iRegLNoSp tmp3, iRegLNoSp tmp4,
iRegLNoSp tmp5, iRegLNoSp tmp6, rFlagsReg cr)
%{
+ predicate(!UseRVV);
match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
diff --git a/src/hotspot/cpu/riscv/riscv_v.ad b/src/hotspot/cpu/riscv/riscv_v.ad
index f2845ee2a6c..fe323474d60 100644
--- a/src/hotspot/cpu/riscv/riscv_v.ad
+++ b/src/hotspot/cpu/riscv/riscv_v.ad
@@ -4080,6 +4080,28 @@ instruct varray_equalsC(iRegP_R11 ary1, iRegP_R12 ary2, iRegI_R10 result,
ins_pipe(pipe_class_memory);
%}
+// fast ArraysSupport.vectorizedHashCode
+instruct varrays_hashcode(iRegP_R11 ary, iRegI_R12 cnt, iRegI_R10 result, immI basic_type,
+ vReg_V2 v2, vReg_V3 v3, vReg_V4 v4, vReg_V5 v5,
+ vReg_V6 v6, vReg_V7 v7, vReg_V8 v8, vReg_V9 v9,
+ iRegLNoSp tmp1, iRegLNoSp tmp2, iRegLNoSp tmp3,
+ rFlagsReg cr)
+%{
+ predicate(UseRVV);
+ match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
+ effect(USE_KILL ary, USE_KILL cnt, USE basic_type,
+ TEMP v2, TEMP v3, TEMP v4, TEMP v5, TEMP v6, TEMP v7, TEMP v8, TEMP v9,
+ TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
+
+ format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
+ ins_encode %{
+ __ arrays_hashcode_v($ary$$Register, $cnt$$Register, $result$$Register,
+ $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
+ (BasicType)$basic_type$$constant);
+ %}
+ ins_pipe(pipe_class_memory);
+%}
+
instruct vstring_compareU_128b(iRegP_R11 str1, iRegI_R12 cnt1, iRegP_R13 str2, iRegI_R14 cnt2,
iRegI_R10 result, vReg_V4 v4, vReg_V5 v5, vReg_V6 v6, vReg_V7 v7,
vReg_V8 v8, vReg_V9 v9, vReg_V10 v10, vReg_V11 v11,
diff --git a/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp b/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp
index 2cc1fb8eeff..b303178a666 100644
--- a/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp
+++ b/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp
@@ -602,11 +602,11 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
int comp_args_on_stack,
const BasicType *sig_bt,
const VMRegPair *regs,
- AdapterHandlerEntry* handler) {
- address i2c_entry = __ pc();
+ address entry_address[AdapterBlob::ENTRY_COUNT]) {
+ entry_address[AdapterBlob::I2C] = __ pc();
gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs);
- address c2i_unverified_entry = __ pc();
+ entry_address[AdapterBlob::C2I_Unverified] = __ pc();
Label skip_fixup;
const Register receiver = j_rarg0;
@@ -633,10 +633,10 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
__ block_comment("} c2i_unverified_entry");
}
- address c2i_entry = __ pc();
+ entry_address[AdapterBlob::C2I] = __ pc();
// Class initialization barrier for static methods
- address c2i_no_clinit_check_entry = nullptr;
+ entry_address[AdapterBlob::C2I_No_Clinit_Check] = nullptr;
if (VM_Version::supports_fast_class_init_checks()) {
Label L_skip_barrier;
@@ -651,15 +651,13 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
__ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
__ bind(L_skip_barrier);
- c2i_no_clinit_check_entry = __ pc();
+ entry_address[AdapterBlob::C2I_No_Clinit_Check] = __ pc();
}
BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
bs->c2i_entry_barrier(masm);
gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup);
-
- handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry);
return;
}
@@ -887,11 +885,8 @@ static void fill_continuation_entry(MacroAssembler* masm) {
__ ld(t0, Address(xthread, JavaThread::cont_fastpath_offset()));
__ sd(t0, Address(sp, ContinuationEntry::parent_cont_fastpath_offset()));
- __ ld(t0, Address(xthread, JavaThread::held_monitor_count_offset()));
- __ sd(t0, Address(sp, ContinuationEntry::parent_held_monitor_count_offset()));
__ sd(zr, Address(xthread, JavaThread::cont_fastpath_offset()));
- __ sd(zr, Address(xthread, JavaThread::held_monitor_count_offset()));
}
// on entry, sp points to the ContinuationEntry
@@ -907,50 +902,6 @@ static void continuation_enter_cleanup(MacroAssembler* masm) {
__ ld(t0, Address(sp, ContinuationEntry::parent_cont_fastpath_offset()));
__ sd(t0, Address(xthread, JavaThread::cont_fastpath_offset()));
-
- if (CheckJNICalls) {
- // Check if this is a virtual thread continuation
- Label L_skip_vthread_code;
- __ lwu(t0, Address(sp, ContinuationEntry::flags_offset()));
- __ beqz(t0, L_skip_vthread_code);
-
- // If the held monitor count is > 0 and this vthread is terminating then
- // it failed to release a JNI monitor. So we issue the same log message
- // that JavaThread::exit does.
- __ ld(t0, Address(xthread, JavaThread::jni_monitor_count_offset()));
- __ beqz(t0, L_skip_vthread_code);
-
- // Save return value potentially containing the exception oop in callee-saved x9
- __ mv(x9, x10);
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::log_jni_monitor_still_held));
- // Restore potential return value
- __ mv(x10, x9);
-
- // For vthreads we have to explicitly zero the JNI monitor count of the carrier
- // on termination. The held count is implicitly zeroed below when we restore from
- // the parent held count (which has to be zero).
- __ sd(zr, Address(xthread, JavaThread::jni_monitor_count_offset()));
-
- __ bind(L_skip_vthread_code);
- }
-#ifdef ASSERT
- else {
- // Check if this is a virtual thread continuation
- Label L_skip_vthread_code;
- __ lwu(t0, Address(sp, ContinuationEntry::flags_offset()));
- __ beqz(t0, L_skip_vthread_code);
-
- // See comment just above. If not checking JNI calls the JNI count is only
- // needed for assertion checking.
- __ sd(zr, Address(xthread, JavaThread::jni_monitor_count_offset()));
-
- __ bind(L_skip_vthread_code);
- }
-#endif
-
- __ ld(t0, Address(sp, ContinuationEntry::parent_held_monitor_count_offset()));
- __ sd(t0, Address(xthread, JavaThread::held_monitor_count_offset()));
-
__ ld(t0, Address(sp, ContinuationEntry::parent_offset()));
__ sd(t0, Address(xthread, JavaThread::cont_entry_offset()));
__ add(fp, sp, (int)ContinuationEntry::size() + 2 * wordSize /* 2 extra words to match up with leave() */);
@@ -1004,20 +955,23 @@ static void gen_continuation_enter(MacroAssembler* masm,
__ bnez(c_rarg2, call_thaw);
- // Make sure the call is patchable
- __ align(NativeInstruction::instruction_size);
+ address call_pc;
+ {
+ Assembler::IncompressibleScope scope(masm);
+ // Make sure the call is patchable
+ __ align(NativeInstruction::instruction_size);
- const address tr_call = __ reloc_call(resolve);
- if (tr_call == nullptr) {
- fatal("CodeCache is full at gen_continuation_enter");
+ call_pc = __ reloc_call(resolve);
+ if (call_pc == nullptr) {
+ fatal("CodeCache is full at gen_continuation_enter");
+ }
+
+ oop_maps->add_gc_map(__ pc() - start, map);
+ __ post_call_nop();
}
-
- oop_maps->add_gc_map(__ pc() - start, map);
- __ post_call_nop();
-
__ j(exit);
- address stub = CompiledDirectCall::emit_to_interp_stub(masm, tr_call);
+ address stub = CompiledDirectCall::emit_to_interp_stub(masm, call_pc);
if (stub == nullptr) {
fatal("CodeCache is full at gen_continuation_enter");
}
@@ -1036,26 +990,36 @@ static void gen_continuation_enter(MacroAssembler* masm,
__ bnez(c_rarg2, call_thaw);
- // Make sure the call is patchable
- __ align(NativeInstruction::instruction_size);
+ address call_pc;
+ {
+ Assembler::IncompressibleScope scope(masm);
+ // Make sure the call is patchable
+ __ align(NativeInstruction::instruction_size);
- const address tr_call = __ reloc_call(resolve);
- if (tr_call == nullptr) {
- fatal("CodeCache is full at gen_continuation_enter");
+ call_pc = __ reloc_call(resolve);
+ if (call_pc == nullptr) {
+ fatal("CodeCache is full at gen_continuation_enter");
+ }
+
+ oop_maps->add_gc_map(__ pc() - start, map);
+ __ post_call_nop();
}
- oop_maps->add_gc_map(__ pc() - start, map);
- __ post_call_nop();
-
__ j(exit);
__ bind(call_thaw);
- ContinuationEntry::_thaw_call_pc_offset = __ pc() - start;
- __ rt_call(CAST_FROM_FN_PTR(address, StubRoutines::cont_thaw()));
- oop_maps->add_gc_map(__ pc() - start, map->deep_copy());
- ContinuationEntry::_return_pc_offset = __ pc() - start;
- __ post_call_nop();
+ // Post call nops must be natural aligned due to cmodx rules.
+ {
+ Assembler::IncompressibleScope scope(masm);
+ __ align(NativeInstruction::instruction_size);
+
+ ContinuationEntry::_thaw_call_pc_offset = __ pc() - start;
+ __ rt_call(CAST_FROM_FN_PTR(address, StubRoutines::cont_thaw()));
+ oop_maps->add_gc_map(__ pc() - start, map->deep_copy());
+ ContinuationEntry::_return_pc_offset = __ pc() - start;
+ __ post_call_nop();
+ }
__ bind(exit);
ContinuationEntry::_cleanup_offset = __ pc() - start;
@@ -1084,7 +1048,7 @@ static void gen_continuation_enter(MacroAssembler* masm,
__ jr(x11); // the exception handler
}
- address stub = CompiledDirectCall::emit_to_interp_stub(masm, tr_call);
+ address stub = CompiledDirectCall::emit_to_interp_stub(masm, call_pc);
if (stub == nullptr) {
fatal("CodeCache is full at gen_continuation_enter");
}
@@ -1117,10 +1081,16 @@ static void gen_continuation_yield(MacroAssembler* masm,
__ mv(c_rarg1, sp);
+ // Post call nops must be natural aligned due to cmodx rules.
+ __ align(NativeInstruction::instruction_size);
+
frame_complete = __ pc() - start;
address the_pc = __ pc();
- __ post_call_nop(); // this must be exactly after the pc value that is pushed into the frame info, we use this nop for fast CodeBlob lookup
+ {
+ Assembler::IncompressibleScope scope(masm);
+ __ post_call_nop(); // this must be exactly after the pc value that is pushed into the frame info, we use this nop for fast CodeBlob lookup
+ }
__ mv(c_rarg0, xthread);
__ set_last_Java_frame(sp, fp, the_pc, t0);
@@ -1679,8 +1649,6 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
Label lock_done;
if (method->is_synchronized()) {
- const int mark_word_offset = BasicLock::displaced_header_offset_in_bytes();
-
// Get the handle (the 2nd argument)
__ mv(oop_handle_reg, c_rarg1);
diff --git a/src/hotspot/cpu/riscv/stubDeclarations_riscv.hpp b/src/hotspot/cpu/riscv/stubDeclarations_riscv.hpp
index fe7f52884fa..f977d759d20 100644
--- a/src/hotspot/cpu/riscv/stubDeclarations_riscv.hpp
+++ b/src/hotspot/cpu/riscv/stubDeclarations_riscv.hpp
@@ -73,6 +73,9 @@
do_stub(compiler, string_indexof_linear_ul) \
do_arch_entry(riscv, compiler, string_indexof_linear_ul, \
string_indexof_linear_ul, string_indexof_linear_ul) \
+ do_stub(compiler, arrays_hashcode_powers_of_31) \
+ do_arch_entry(riscv, compiler, arrays_hashcode_powers_of_31, \
+ arrays_hashcode_powers_of_31, arrays_hashcode_powers_of_31) \
#define STUBGEN_FINAL_BLOBS_ARCH_DO(do_stub, \
diff --git a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp
index 7db426327ee..ec268d9bb65 100644
--- a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp
+++ b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp
@@ -683,10 +683,11 @@ class StubGenerator: public StubCodeGenerator {
address start = __ pc();
if (UseBlockZeroing) {
- // Ensure count >= 2*CacheLineSize so that it still deserves a cbo.zero
- // after alignment.
+ int zicboz_block_size = VM_Version::zicboz_block_size.value();
+ // Ensure count >= 2 * zicboz_block_size so that it still deserves
+ // a cbo.zero after alignment.
Label small;
- int low_limit = MAX2(2 * CacheLineSize, BlockZeroingLowLimit) / wordSize;
+ int low_limit = MAX2(2 * zicboz_block_size, (int)BlockZeroingLowLimit) / wordSize;
__ mv(tmp1, low_limit);
__ blt(cnt, tmp1, small);
__ zero_dcache_blocks(base, cnt, tmp1, tmp2);
@@ -731,8 +732,7 @@ class StubGenerator: public StubCodeGenerator {
//
// s and d are adjusted to point to the remaining words to copy
//
- void generate_copy_longs(StubId stub_id, Label &start,
- Register s, Register d, Register count) {
+ address generate_copy_longs(StubId stub_id, Register s, Register d, Register count) {
BasicType type;
copy_direction direction;
switch (stub_id) {
@@ -762,7 +762,7 @@ class StubGenerator: public StubCodeGenerator {
Label again, drain;
StubCodeMark mark(this, stub_id);
__ align(CodeEntryAlignment);
- __ bind(start);
+ address start = __ pc();
if (direction == copy_forwards) {
__ sub(s, s, bias);
@@ -878,9 +878,9 @@ class StubGenerator: public StubCodeGenerator {
}
__ ret();
- }
- Label copy_f, copy_b;
+ return start;
+ }
typedef void (MacroAssembler::*copy_insn)(Register Rd, const Address &adr, Register temp);
@@ -1098,8 +1098,8 @@ class StubGenerator: public StubCodeGenerator {
// stub_id - is used to name the stub and identify all details of
// how to perform the copy.
//
- // entry - is assigned to the stub's post push entry point unless
- // it is null
+ // nopush_entry - is assigned to the stub's post push entry point
+ // unless it is null
//
// Inputs:
// c_rarg0 - source array address
@@ -1110,11 +1110,11 @@ class StubGenerator: public StubCodeGenerator {
// the hardware handle it. The two dwords within qwords that span
// cache line boundaries will still be loaded and stored atomically.
//
- // Side Effects: entry is set to the (post push) entry point so it
- // can be used by the corresponding conjoint copy
- // method
+ // Side Effects: nopush_entry is set to the (post push) entry point
+ // so it can be used by the corresponding conjoint
+ // copy method
//
- address generate_disjoint_copy(StubId stub_id, address* entry) {
+ address generate_disjoint_copy(StubId stub_id, address* nopush_entry) {
size_t size;
bool aligned;
bool is_oop;
@@ -1203,8 +1203,8 @@ class StubGenerator: public StubCodeGenerator {
address start = __ pc();
__ enter();
- if (entry != nullptr) {
- *entry = __ pc();
+ if (nopush_entry != nullptr) {
+ *nopush_entry = __ pc();
// caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
BLOCK_COMMENT("Entry:");
}
@@ -1255,8 +1255,8 @@ class StubGenerator: public StubCodeGenerator {
// corresponding disjoint copy routine which can be
// jumped to if the ranges do not actually overlap
//
- // entry - is assigned to the stub's post push entry point unless
- // it is null
+ // nopush_entry - is assigned to the stub's post push entry point
+ // unless it is null
//
// Inputs:
// c_rarg0 - source array address
@@ -1268,10 +1268,10 @@ class StubGenerator: public StubCodeGenerator {
// cache line boundaries will still be loaded and stored atomically.
//
// Side Effects:
- // entry is set to the no-overlap entry point so it can be used by
- // some other conjoint copy method
+ // nopush_entry is set to the no-overlap entry point so it can be
+ // used by some other conjoint copy method
//
- address generate_conjoint_copy(StubId stub_id, address nooverlap_target, address *entry) {
+ address generate_conjoint_copy(StubId stub_id, address nooverlap_target, address *nopush_entry) {
const Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
RegSet saved_regs = RegSet::of(s, d, count);
int size;
@@ -1358,8 +1358,8 @@ class StubGenerator: public StubCodeGenerator {
address start = __ pc();
__ enter();
- if (entry != nullptr) {
- *entry = __ pc();
+ if (nopush_entry != nullptr) {
+ *nopush_entry = __ pc();
// caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
BLOCK_COMMENT("Entry:");
}
@@ -1369,7 +1369,7 @@ class StubGenerator: public StubCodeGenerator {
__ slli(t1, count, exact_log2(size));
Label L_continue;
__ bltu(t0, t1, L_continue);
- __ j(nooverlap_target);
+ __ j(RuntimeAddress(nooverlap_target));
__ bind(L_continue);
DecoratorSet decorators = IN_HEAP | IS_ARRAY;
@@ -1444,7 +1444,7 @@ class StubGenerator: public StubCodeGenerator {
// x10 == 0 - success
// x10 == -1^K - failure, where K is partial transfer count
//
- address generate_checkcast_copy(StubId stub_id, address* entry) {
+ address generate_checkcast_copy(StubId stub_id, address* nopush_entry) {
bool dest_uninitialized;
switch (stub_id) {
case StubId::stubgen_checkcast_arraycopy_id:
@@ -1495,8 +1495,8 @@ class StubGenerator: public StubCodeGenerator {
__ enter(); // required for proper stackwalking of RuntimeStub frame
// Caller of this entry point must set up the argument registers.
- if (entry != nullptr) {
- *entry = __ pc();
+ if (nopush_entry != nullptr) {
+ *nopush_entry = __ pc();
BLOCK_COMMENT("Entry:");
}
@@ -2293,13 +2293,21 @@ class StubGenerator: public StubCodeGenerator {
}
void generate_arraycopy_stubs() {
- address entry = nullptr;
- address entry_jbyte_arraycopy = nullptr;
- address entry_jshort_arraycopy = nullptr;
- address entry_jint_arraycopy = nullptr;
- address entry_oop_arraycopy = nullptr;
- address entry_jlong_arraycopy = nullptr;
- address entry_checkcast_arraycopy = nullptr;
+ // Some copy stubs publish a normal entry and then a 2nd 'fallback'
+ // entry immediately following their stack push. This can be used
+ // as a post-push branch target for compatible stubs when they
+ // identify a special case that can be handled by the fallback
+ // stub e.g a disjoint copy stub may be use as a special case
+ // fallback for its compatible conjoint copy stub.
+ //
+ // A no push entry is always returned in the following local and
+ // then published by assigning to the appropriate entry field in
+ // class StubRoutines. The entry value is then passed to the
+ // generator for the compatible stub. That means the entry must be
+ // listed when saving to/restoring from the AOT cache, ensuring
+ // that the inter-stub jumps are noted at AOT-cache save and
+ // relocated at AOT cache load.
+ address nopush_entry = nullptr;
// generate the common exit first so later stubs can rely on it if
// they want an UnsafeMemoryAccess exit non-local to the stub
@@ -2307,72 +2315,117 @@ class StubGenerator: public StubCodeGenerator {
// register the stub as the default exit with class UnsafeMemoryAccess
UnsafeMemoryAccess::set_common_exit_stub_pc(StubRoutines::_unsafecopy_common_exit);
- generate_copy_longs(StubId::stubgen_copy_byte_f_id, copy_f, c_rarg0, c_rarg1, t1);
- generate_copy_longs(StubId::stubgen_copy_byte_b_id, copy_b, c_rarg0, c_rarg1, t1);
+ // generate and publish riscv-specific bulk copy routines first
+ // so we can call them from other copy stubs
+ StubRoutines::riscv::_copy_byte_f = generate_copy_longs(StubId::stubgen_copy_byte_f_id, c_rarg0, c_rarg1, t1);
+ StubRoutines::riscv::_copy_byte_b = generate_copy_longs(StubId::stubgen_copy_byte_b_id, c_rarg0, c_rarg1, t1);
StubRoutines::riscv::_zero_blocks = generate_zero_blocks();
//*** jbyte
// Always need aligned and unaligned versions
- StubRoutines::_jbyte_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_jbyte_disjoint_arraycopy_id, &entry);
- StubRoutines::_jbyte_arraycopy = generate_conjoint_copy(StubId::stubgen_jbyte_arraycopy_id, entry, &entry_jbyte_arraycopy);
- StubRoutines::_arrayof_jbyte_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jbyte_disjoint_arraycopy_id, &entry);
- StubRoutines::_arrayof_jbyte_arraycopy = generate_conjoint_copy(StubId::stubgen_arrayof_jbyte_arraycopy_id, entry, nullptr);
+ StubRoutines::_jbyte_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_jbyte_disjoint_arraycopy_id, &nopush_entry);
+ // disjoint nopush entry is needed by conjoint copy
+ StubRoutines::_jbyte_disjoint_arraycopy_nopush = nopush_entry;
+ StubRoutines::_jbyte_arraycopy = generate_conjoint_copy(StubId::stubgen_jbyte_arraycopy_id, StubRoutines::_jbyte_disjoint_arraycopy_nopush, &nopush_entry);
+ // conjoint nopush entry is needed by generic/unsafe copy
+ StubRoutines::_jbyte_arraycopy_nopush = nopush_entry;
+ StubRoutines::_arrayof_jbyte_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jbyte_disjoint_arraycopy_id, &nopush_entry);
+ // disjoint arrayof nopush entry is needed by conjoint copy
+ StubRoutines::_arrayof_jbyte_disjoint_arraycopy_nopush = nopush_entry;
+ StubRoutines::_arrayof_jbyte_arraycopy = generate_conjoint_copy(StubId::stubgen_arrayof_jbyte_arraycopy_id, StubRoutines::_arrayof_jbyte_disjoint_arraycopy_nopush, nullptr);
//*** jshort
// Always need aligned and unaligned versions
- StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_jshort_disjoint_arraycopy_id, &entry);
- StubRoutines::_jshort_arraycopy = generate_conjoint_copy(StubId::stubgen_jshort_arraycopy_id, entry, &entry_jshort_arraycopy);
- StubRoutines::_arrayof_jshort_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jshort_disjoint_arraycopy_id, &entry);
- StubRoutines::_arrayof_jshort_arraycopy = generate_conjoint_copy(StubId::stubgen_arrayof_jshort_arraycopy_id, entry, nullptr);
+ StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_jshort_disjoint_arraycopy_id, &nopush_entry);
+ // disjoint nopush entry is needed by conjoint copy
+ StubRoutines::_jshort_disjoint_arraycopy_nopush = nopush_entry;
+ StubRoutines::_jshort_arraycopy = generate_conjoint_copy(StubId::stubgen_jshort_arraycopy_id, StubRoutines::_jshort_disjoint_arraycopy_nopush, &nopush_entry);
+ // conjoint nopush entry is used by generic/unsafe copy
+ StubRoutines::_jshort_arraycopy_nopush = nopush_entry;
+ StubRoutines::_arrayof_jshort_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jshort_disjoint_arraycopy_id, &nopush_entry);
+ // disjoint arrayof nopush entry is needed by conjoint copy
+ StubRoutines::_arrayof_jshort_disjoint_arraycopy_nopush = nopush_entry;
+ StubRoutines::_arrayof_jshort_arraycopy = generate_conjoint_copy(StubId::stubgen_arrayof_jshort_arraycopy_id, StubRoutines::_arrayof_jshort_disjoint_arraycopy_nopush, nullptr);
//*** jint
// Aligned versions
- StubRoutines::_arrayof_jint_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jint_disjoint_arraycopy_id, &entry);
- StubRoutines::_arrayof_jint_arraycopy = generate_conjoint_copy(StubId::stubgen_arrayof_jint_arraycopy_id, entry, &entry_jint_arraycopy);
+ StubRoutines::_arrayof_jint_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jint_disjoint_arraycopy_id, &nopush_entry);
+ // disjoint arrayof nopush entry is needed by conjoint copy
+ StubRoutines::_arrayof_jint_disjoint_arraycopy_nopush = nopush_entry;
+ StubRoutines::_arrayof_jint_arraycopy = generate_conjoint_copy(StubId::stubgen_arrayof_jint_arraycopy_id, StubRoutines::_arrayof_jint_disjoint_arraycopy_nopush, nullptr);
// In 64 bit we need both aligned and unaligned versions of jint arraycopy.
// entry_jint_arraycopy always points to the unaligned version
- StubRoutines::_jint_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_jint_disjoint_arraycopy_id, &entry);
- StubRoutines::_jint_arraycopy = generate_conjoint_copy(StubId::stubgen_jint_arraycopy_id, entry, &entry_jint_arraycopy);
+ StubRoutines::_jint_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_jint_disjoint_arraycopy_id, &nopush_entry);
+ // disjoint nopush entry is needed by conjoint copy
+ StubRoutines::_jint_disjoint_arraycopy_nopush = nopush_entry;
+ StubRoutines::_jint_arraycopy = generate_conjoint_copy(StubId::stubgen_jint_arraycopy_id, StubRoutines::_jint_disjoint_arraycopy_nopush, &nopush_entry);
+ // conjoint nopush entry is needed by generic/unsafe copy
+ StubRoutines::_jint_arraycopy_nopush = nopush_entry;
//*** jlong
// It is always aligned
- StubRoutines::_arrayof_jlong_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jlong_disjoint_arraycopy_id, &entry);
- StubRoutines::_arrayof_jlong_arraycopy = generate_conjoint_copy(StubId::stubgen_arrayof_jlong_arraycopy_id, entry, &entry_jlong_arraycopy);
+ StubRoutines::_arrayof_jlong_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jlong_disjoint_arraycopy_id, &nopush_entry);
+ // disjoint arrayof nopush entry is needed by conjoint copy
+ StubRoutines::_arrayof_jlong_disjoint_arraycopy_nopush = nopush_entry;
+ StubRoutines::_arrayof_jlong_arraycopy = generate_conjoint_copy(StubId::stubgen_arrayof_jlong_arraycopy_id, StubRoutines::_arrayof_jlong_disjoint_arraycopy_nopush, &nopush_entry);
+ // conjoint nopush entry is needed by generic/unsafe copy
+ StubRoutines::_jlong_arraycopy_nopush = nopush_entry;
+ // disjoint normal/nopush and conjoint normal entries are not
+ // generated since the arrayof versions are the same
StubRoutines::_jlong_disjoint_arraycopy = StubRoutines::_arrayof_jlong_disjoint_arraycopy;
+ StubRoutines::_jlong_disjoint_arraycopy_nopush = StubRoutines::_arrayof_jlong_disjoint_arraycopy_nopush;
StubRoutines::_jlong_arraycopy = StubRoutines::_arrayof_jlong_arraycopy;
//*** oops
StubRoutines::_arrayof_oop_disjoint_arraycopy
- = generate_disjoint_copy(StubId::stubgen_arrayof_oop_disjoint_arraycopy_id, &entry);
+ = generate_disjoint_copy(StubId::stubgen_arrayof_oop_disjoint_arraycopy_id, &nopush_entry);
+ // disjoint arrayof nopush entry is needed by conjoint copy
+ StubRoutines::_arrayof_oop_disjoint_arraycopy_nopush = nopush_entry;
StubRoutines::_arrayof_oop_arraycopy
- = generate_conjoint_copy(StubId::stubgen_arrayof_oop_arraycopy_id, entry, &entry_oop_arraycopy);
+ = generate_conjoint_copy(StubId::stubgen_arrayof_oop_arraycopy_id, StubRoutines::_arrayof_oop_disjoint_arraycopy_nopush, &nopush_entry);
+ // conjoint arrayof nopush entry is needed by generic/unsafe copy
+ StubRoutines::_oop_arraycopy_nopush = nopush_entry;
// Aligned versions without pre-barriers
StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit
- = generate_disjoint_copy(StubId::stubgen_arrayof_oop_disjoint_arraycopy_uninit_id, &entry);
- StubRoutines::_arrayof_oop_arraycopy_uninit
- = generate_conjoint_copy(StubId::stubgen_arrayof_oop_arraycopy_uninit_id, entry, nullptr);
+ = generate_disjoint_copy(StubId::stubgen_arrayof_oop_disjoint_arraycopy_uninit_id, &nopush_entry);
+ // disjoint arrayof+uninit nopush entry is needed by conjoint copy
+ StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit_nopush = nopush_entry;
+ // note that we don't need a returned nopush entry because the
+ // generic/unsafe copy does not cater for uninit arrays.
+ StubRoutines::_arrayof_oop_arraycopy_uninit
+ = generate_conjoint_copy(StubId::stubgen_arrayof_oop_arraycopy_uninit_id, StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit_nopush, nullptr);
+
+ // for oop copies reuse arrayof entries for non-arrayof cases
StubRoutines::_oop_disjoint_arraycopy = StubRoutines::_arrayof_oop_disjoint_arraycopy;
+ StubRoutines::_oop_disjoint_arraycopy_nopush = StubRoutines::_arrayof_oop_disjoint_arraycopy_nopush;
StubRoutines::_oop_arraycopy = StubRoutines::_arrayof_oop_arraycopy;
StubRoutines::_oop_disjoint_arraycopy_uninit = StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit;
+ StubRoutines::_oop_disjoint_arraycopy_uninit_nopush = StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit_nopush;
StubRoutines::_oop_arraycopy_uninit = StubRoutines::_arrayof_oop_arraycopy_uninit;
- StubRoutines::_checkcast_arraycopy = generate_checkcast_copy(StubId::stubgen_checkcast_arraycopy_id, &entry_checkcast_arraycopy);
+ StubRoutines::_checkcast_arraycopy = generate_checkcast_copy(StubId::stubgen_checkcast_arraycopy_id, &nopush_entry);
+ // checkcast nopush entry is needed by generic copy
+ StubRoutines::_checkcast_arraycopy_nopush = nopush_entry;
+ // note that we don't need a returned nopush entry because the
+ // generic copy does not cater for uninit arrays.
StubRoutines::_checkcast_arraycopy_uninit = generate_checkcast_copy(StubId::stubgen_checkcast_arraycopy_uninit_id, nullptr);
- StubRoutines::_unsafe_arraycopy = generate_unsafe_copy(entry_jbyte_arraycopy,
- entry_jshort_arraycopy,
- entry_jint_arraycopy,
- entry_jlong_arraycopy);
+ // unsafe arraycopy may fallback on conjoint stubs
+ StubRoutines::_unsafe_arraycopy = generate_unsafe_copy(StubRoutines::_jbyte_arraycopy_nopush,
+ StubRoutines::_jshort_arraycopy_nopush,
+ StubRoutines::_jint_arraycopy_nopush,
+ StubRoutines::_jlong_arraycopy_nopush);
- StubRoutines::_generic_arraycopy = generate_generic_copy(entry_jbyte_arraycopy,
- entry_jshort_arraycopy,
- entry_jint_arraycopy,
- entry_oop_arraycopy,
- entry_jlong_arraycopy,
- entry_checkcast_arraycopy);
+ // generic arraycopy may fallback on conjoint stubs
+ StubRoutines::_generic_arraycopy = generate_generic_copy(StubRoutines::_jbyte_arraycopy_nopush,
+ StubRoutines::_jshort_arraycopy_nopush,
+ StubRoutines::_jint_arraycopy_nopush,
+ StubRoutines::_oop_arraycopy_nopush,
+ StubRoutines::_jlong_arraycopy_nopush,
+ StubRoutines::_checkcast_arraycopy_nopush);
StubRoutines::_jbyte_fill = generate_fill(StubId::stubgen_jbyte_fill_id);
StubRoutines::_jshort_fill = generate_fill(StubId::stubgen_jshort_fill_id);
@@ -6571,6 +6624,24 @@ static const int64_t right_3_bits = right_n_bits(3);
return start;
}
+ address generate_arrays_hashcode_powers_of_31() {
+ assert(UseRVV, "sanity");
+ const int lmul = 2;
+ const int stride = MaxVectorSize / sizeof(jint) * lmul;
+ __ align(CodeEntryAlignment);
+ StubCodeMark mark(this, "StubRoutines", "arrays_hashcode_powers_of_31");
+ address start = __ pc();
+ for (int i = stride; i >= 0; i--) {
+ jint power_of_31 = 1;
+ for (int j = i; j > 0; j--) {
+ power_of_31 = java_multiply(power_of_31, 31);
+ }
+ __ emit_int32(power_of_31);
+ }
+
+ return start;
+ }
+
#endif // COMPILER2
/**
@@ -6765,6 +6836,10 @@ static const int64_t right_3_bits = right_n_bits(3);
StubRoutines::_bigIntegerRightShiftWorker = generate_bigIntegerRightShift();
}
+ if (UseVectorizedHashCodeIntrinsic && UseRVV) {
+ StubRoutines::riscv::_arrays_hashcode_powers_of_31 = generate_arrays_hashcode_powers_of_31();
+ }
+
if (UseSHA256Intrinsics) {
Sha2Generator sha2(_masm, this);
StubRoutines::_sha256_implCompress = sha2.generate_sha256_implCompress(StubId::stubgen_sha256_implCompress_id);
diff --git a/src/hotspot/cpu/riscv/templateTable_riscv.cpp b/src/hotspot/cpu/riscv/templateTable_riscv.cpp
index 1011b8f1e7b..2697b3e46dc 100644
--- a/src/hotspot/cpu/riscv/templateTable_riscv.cpp
+++ b/src/hotspot/cpu/riscv/templateTable_riscv.cpp
@@ -2168,7 +2168,7 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no,
assert_different_registers(Rcache, index, temp);
assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
- Label resolved, clinit_barrier_slow;
+ Label L_clinit_barrier_slow, L_done;
Bytecodes::Code code = bytecode();
__ load_method_entry(Rcache, index);
@@ -2185,12 +2185,20 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no,
__ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
__ mv(t0, (int) code);
- __ beq(temp, t0, resolved); // have we resolved this bytecode?
+
+ // Class initialization barrier for static methods
+ if (VM_Version::supports_fast_class_init_checks() && bytecode() == Bytecodes::_invokestatic) {
+ __ bne(temp, t0, L_clinit_barrier_slow); // have we resolved this bytecode?
+ __ ld(temp, Address(Rcache, in_bytes(ResolvedMethodEntry::method_offset())));
+ __ load_method_holder(temp, temp);
+ __ clinit_barrier(temp, t0, &L_done, /*L_slow_path*/ nullptr);
+ __ bind(L_clinit_barrier_slow);
+ } else {
+ __ beq(temp, t0, L_done); // have we resolved this bytecode?
+ }
// resolve first time through
// Class initialization barrier slow path lands here as well.
- __ bind(clinit_barrier_slow);
-
address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
__ mv(temp, (int) code);
__ call_VM(noreg, entry, temp);
@@ -2199,14 +2207,7 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no,
__ load_method_entry(Rcache, index);
// n.b. unlike x86 Rcache is now rcpool plus the indexed offset
// so all clients ofthis method must be modified accordingly
- __ bind(resolved);
-
- // Class initialization barrier for static methods
- if (VM_Version::supports_fast_class_init_checks() && bytecode() == Bytecodes::_invokestatic) {
- __ ld(temp, Address(Rcache, in_bytes(ResolvedMethodEntry::method_offset())));
- __ load_method_holder(temp, temp);
- __ clinit_barrier(temp, t0, nullptr, &clinit_barrier_slow);
- }
+ __ bind(L_done);
}
void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
@@ -2215,13 +2216,13 @@ void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
const Register temp = x9;
assert_different_registers(Rcache, index, temp);
- Label resolved;
+ Label L_clinit_barrier_slow, L_done;
Bytecodes::Code code = bytecode();
switch (code) {
- case Bytecodes::_nofast_getfield: code = Bytecodes::_getfield; break;
- case Bytecodes::_nofast_putfield: code = Bytecodes::_putfield; break;
- default: break;
+ case Bytecodes::_nofast_getfield: code = Bytecodes::_getfield; break;
+ case Bytecodes::_nofast_putfield: code = Bytecodes::_putfield; break;
+ default: break;
}
assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
@@ -2235,16 +2236,29 @@ void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
__ lbu(temp, Address(temp, 0));
__ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
__ mv(t0, (int) code); // have we resolved this bytecode?
- __ beq(temp, t0, resolved);
+
+ // Class initialization barrier for static fields
+ if (VM_Version::supports_fast_class_init_checks() &&
+ (bytecode() == Bytecodes::_getstatic || bytecode() == Bytecodes::_putstatic)) {
+ const Register field_holder = temp;
+
+ __ bne(temp, t0, L_clinit_barrier_slow);
+ __ ld(field_holder, Address(Rcache, in_bytes(ResolvedFieldEntry::field_holder_offset())));
+ __ clinit_barrier(field_holder, t0, &L_done, /*L_slow_path*/ nullptr);
+ __ bind(L_clinit_barrier_slow);
+ } else {
+ __ beq(temp, t0, L_done);
+ }
// resolve first time through
+ // Class initialization barrier slow path lands here as well.
address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
__ mv(temp, (int) code);
__ call_VM(noreg, entry, temp);
// Update registers with resolved info
__ load_field_entry(Rcache, index);
- __ bind(resolved);
+ __ bind(L_done);
}
void TemplateTable::load_resolved_field_entry(Register obj,
diff --git a/src/hotspot/cpu/riscv/vm_version_riscv.cpp b/src/hotspot/cpu/riscv/vm_version_riscv.cpp
index 4b437896dcd..6f4babc872f 100644
--- a/src/hotspot/cpu/riscv/vm_version_riscv.cpp
+++ b/src/hotspot/cpu/riscv/vm_version_riscv.cpp
@@ -35,15 +35,28 @@
uint32_t VM_Version::_initial_vector_length = 0;
-#define DEF_RV_FEATURE(NAME, PRETTY, BIT, FSTRING, FLAGF) \
-VM_Version::NAME##RVFeatureValue VM_Version::NAME(PRETTY, BIT, FSTRING);
-RV_FEATURE_FLAGS(DEF_RV_FEATURE)
+#define DEF_RV_EXT_FEATURE(PRETTY, LINUX_BIT, FSTRING, FLAGF) \
+VM_Version::ext_##PRETTY##RVExtFeatureValue VM_Version::ext_##PRETTY;
+RV_EXT_FEATURE_FLAGS(DEF_RV_EXT_FEATURE)
+#undef DEF_RV_EXT_FEATURE
-#define ADD_RV_FEATURE_IN_LIST(NAME, PRETTY, BIT, FSTRING, FLAGF) \
- &VM_Version::NAME,
-VM_Version::RVFeatureValue* VM_Version::_feature_list[] = {
-RV_FEATURE_FLAGS(ADD_RV_FEATURE_IN_LIST)
+#define DEF_RV_NON_EXT_FEATURE(PRETTY, LINUX_BIT, FSTRING, FLAGF) \
+VM_Version::PRETTY##RVNonExtFeatureValue VM_Version::PRETTY;
+RV_NON_EXT_FEATURE_FLAGS(DEF_RV_NON_EXT_FEATURE)
+#undef DEF_RV_NON_EXT_FEATURE
+
+#define ADD_RV_EXT_FEATURE_IN_LIST(PRETTY, LINUX_BIT, FSTRING, FLAGF) \
+ &VM_Version::ext_##PRETTY,
+#define ADD_RV_NON_EXT_FEATURE_IN_LIST(PRETTY, LINUX_BIT, FSTRING, FLAGF) \
+ &VM_Version::PRETTY,
+ VM_Version::RVFeatureValue* VM_Version::_feature_list[] = {
+ RV_EXT_FEATURE_FLAGS(ADD_RV_EXT_FEATURE_IN_LIST)
+ RV_NON_EXT_FEATURE_FLAGS(ADD_RV_NON_EXT_FEATURE_IN_LIST)
nullptr};
+#undef ADD_RV_NON_EXT_FEATURE_IN_LIST
+#undef ADD_RV_EXT_FEATURE_IN_LIST
+
+VM_Version::RVExtFeatures* VM_Version::_rv_ext_features = new VM_Version::RVExtFeatures();
void VM_Version::useRVA20U64Profile() {
RV_USE_RVA20U64;
@@ -90,17 +103,6 @@ void VM_Version::common_initialize() {
useRVA23U64Profile();
}
- // Enable vendor specific features
-
- if (mvendorid.enabled()) {
- // Rivos
- if (mvendorid.value() == RIVOS) {
- if (FLAG_IS_DEFAULT(UseConservativeFence)) {
- FLAG_SET_DEFAULT(UseConservativeFence, false);
- }
- }
- }
-
if (UseZic64b) {
if (CacheLineSize != 64) {
assert(!FLAG_IS_DEFAULT(CacheLineSize), "default cache line size should be 64 bytes");
@@ -135,7 +137,7 @@ void VM_Version::common_initialize() {
FLAG_SET_DEFAULT(UseSignumIntrinsic, true);
}
- if (UseRVC && !ext_C.enabled()) {
+ if (UseRVC && !ext_c.enabled()) {
warning("RVC is not supported on this CPU");
FLAG_SET_DEFAULT(UseRVC, false);
@@ -147,7 +149,7 @@ void VM_Version::common_initialize() {
if (FLAG_IS_DEFAULT(AvoidUnalignedAccesses)) {
FLAG_SET_DEFAULT(AvoidUnalignedAccesses,
- unaligned_access.value() != MISALIGNED_FAST);
+ unaligned_scalar.value() != MISALIGNED_SCALAR_FAST);
}
if (!AvoidUnalignedAccesses) {
@@ -162,7 +164,12 @@ void VM_Version::common_initialize() {
// This machine has fast unaligned memory accesses
if (FLAG_IS_DEFAULT(UseUnalignedAccesses)) {
FLAG_SET_DEFAULT(UseUnalignedAccesses,
- unaligned_access.value() == MISALIGNED_FAST);
+ (unaligned_scalar.value() == MISALIGNED_SCALAR_FAST));
+ }
+
+ if (FLAG_IS_DEFAULT(AlignVector)) {
+ FLAG_SET_DEFAULT(AlignVector,
+ unaligned_vector.value() != MISALIGNED_VECTOR_FAST);
}
#ifdef __riscv_ztso
@@ -181,12 +188,13 @@ void VM_Version::common_initialize() {
FLAG_SET_DEFAULT(UsePopCountInstruction, false);
}
- if (UseZicboz) {
+ if (UseZicboz && zicboz_block_size.value() > 0) {
+ assert(is_power_of_2(zicboz_block_size.value()), "Sanity");
if (FLAG_IS_DEFAULT(UseBlockZeroing)) {
FLAG_SET_DEFAULT(UseBlockZeroing, true);
}
if (FLAG_IS_DEFAULT(BlockZeroingLowLimit)) {
- FLAG_SET_DEFAULT(BlockZeroingLowLimit, 2 * CacheLineSize);
+ FLAG_SET_DEFAULT(BlockZeroingLowLimit, 4 * zicboz_block_size.value());
}
} else if (UseBlockZeroing) {
warning("Block zeroing is not available");
@@ -194,13 +202,8 @@ void VM_Version::common_initialize() {
}
if (UseRVV) {
- if (!ext_V.enabled() && FLAG_IS_DEFAULT(UseRVV)) {
- warning("RVV is not supported on this CPU");
- FLAG_SET_DEFAULT(UseRVV, false);
- } else {
- // read vector length from vector CSR vlenb
- _initial_vector_length = cpu_vector_length();
- }
+ // read vector length from vector CSR vlenb
+ _initial_vector_length = cpu_vector_length();
}
// Misc Intrinsics that could depend on RVV.
@@ -220,36 +223,6 @@ void VM_Version::common_initialize() {
warning("CRC32C intrinsics are not available on this CPU.");
FLAG_SET_DEFAULT(UseCRC32CIntrinsics, false);
}
-
- // UseZvbb (depends on RVV).
- if (UseZvbb && !UseRVV) {
- warning("Cannot enable UseZvbb on cpu without RVV support.");
- FLAG_SET_DEFAULT(UseZvbb, false);
- }
-
- // UseZvbc (depends on RVV).
- if (UseZvbc && !UseRVV) {
- warning("Cannot enable UseZvbc on cpu without RVV support.");
- FLAG_SET_DEFAULT(UseZvbc, false);
- }
-
- // UseZvkn (depends on RVV).
- if (UseZvkn && !UseRVV) {
- warning("Cannot enable UseZvkn on cpu without RVV support.");
- FLAG_SET_DEFAULT(UseZvkn, false);
- }
-
- // UseZvfh (depends on RVV)
- if (UseZvfh) {
- if (!UseRVV) {
- warning("Cannot enable UseZvfh on cpu without RVV support.");
- FLAG_SET_DEFAULT(UseZvfh, false);
- }
- if (!UseZfh) {
- warning("Cannot enable UseZvfh on cpu without Zfh support.");
- FLAG_SET_DEFAULT(UseZvfh, false);
- }
- }
}
#ifdef COMPILER2
@@ -476,10 +449,6 @@ void VM_Version::c2_initialize() {
warning("AES/CTR intrinsics are not available on this CPU");
FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
}
-
- if (FLAG_IS_DEFAULT(AlignVector)) {
- FLAG_SET_DEFAULT(AlignVector, AvoidUnalignedAccesses);
- }
}
#endif // COMPILER2
diff --git a/src/hotspot/cpu/riscv/vm_version_riscv.hpp b/src/hotspot/cpu/riscv/vm_version_riscv.hpp
index 188f7e0b1c3..f74992cbc37 100644
--- a/src/hotspot/cpu/riscv/vm_version_riscv.hpp
+++ b/src/hotspot/cpu/riscv/vm_version_riscv.hpp
@@ -46,31 +46,25 @@ class VM_Version : public Abstract_VM_Version {
RIVOS = 0x6cf, // JEDEC: 0x4f, Bank: 14
};
+ class RVExtFeatures;
+
class RVFeatureValue {
const char* const _pretty;
const bool _feature_string;
- const uint64_t _feature_bit;
- bool _enabled;
- int64_t _value;
+ const uint64_t _linux_feature_bit;
+
public:
- RVFeatureValue(const char* pretty, int bit_num, bool fstring) :
- _pretty(pretty), _feature_string(fstring), _feature_bit(nth_bit(bit_num)),
- _enabled(false), _value(-1) {
- }
- void enable_feature(int64_t value = 0) {
- _enabled = true;
- _value = value;
- }
- void disable_feature() {
- _enabled = false;
- _value = -1;
+ RVFeatureValue(const char* pretty, int linux_bit_num, bool fstring) :
+ _pretty(pretty), _feature_string(fstring), _linux_feature_bit(nth_bit(linux_bit_num)) {
}
+ virtual void enable_feature(int64_t value = 0) = 0;
+ virtual void disable_feature() = 0;
const char* pretty() { return _pretty; }
- uint64_t feature_bit() { return _feature_bit; }
+ uint64_t feature_bit() { return _linux_feature_bit; }
bool feature_string() { return _feature_string; }
- bool enabled() { return _enabled; }
- int64_t value() { return _value; }
+ virtual bool enabled() = 0;
virtual void update_flag() = 0;
+ virtual void log_enabled() = 0;
};
#define UPDATE_DEFAULT(flag) \
@@ -86,132 +80,316 @@ class VM_Version : public Abstract_VM_Version {
} \
} \
- #define UPDATE_DEFAULT_DEP(flag, dep) \
- void update_flag() { \
- assert(enabled(), "Must be."); \
- /* dep must be declared before */ \
- assert((uintptr_t)(this) > \
- (uintptr_t)(&dep), "Invalid"); \
- if (FLAG_IS_DEFAULT(flag)) { \
- if (dep.enabled()) { \
- FLAG_SET_DEFAULT(flag, true); \
- } else { \
- FLAG_SET_DEFAULT(flag, false); \
- /* Sync CPU features with flags */ \
- disable_feature(); \
- } \
- } else { \
- /* Sync CPU features with flags */ \
- if (!flag) { \
- disable_feature(); \
- } \
- } \
- } \
+ #define UPDATE_DEFAULT_DEP(flag, dep0, ...) \
+ void update_flag() { \
+ assert(enabled(), "Must be."); \
+ DEBUG_ONLY(verify_deps(dep0, ##__VA_ARGS__)); \
+ if (FLAG_IS_DEFAULT(flag)) { \
+ if (deps_all_enabled(dep0, ##__VA_ARGS__)) { \
+ FLAG_SET_DEFAULT(flag, true); \
+ } else { \
+ FLAG_SET_DEFAULT(flag, false); \
+ stringStream ss; \
+ deps_string(ss, dep0, ##__VA_ARGS__); \
+ warning("Cannot enable " #flag ", it's missing dependent extension(s) %s", ss.as_string(true)); \
+ /* Sync CPU features with flags */ \
+ disable_feature(); \
+ } \
+ } else { \
+ /* Sync CPU features with flags */ \
+ if (!flag) { \
+ disable_feature(); \
+ } else if (!deps_all_enabled(dep0, ##__VA_ARGS__)) { \
+ FLAG_SET_DEFAULT(flag, false); \
+ stringStream ss; \
+ deps_string(ss, dep0, ##__VA_ARGS__); \
+ warning("Cannot enable " #flag ", it's missing dependent extension(s) %s", ss.as_string(true)); \
+ /* Sync CPU features with flags */ \
+ disable_feature(); \
+ } \
+ } \
+ } \
#define NO_UPDATE_DEFAULT \
void update_flag() {} \
- // Frozen standard extensions
- // I RV64I
- // M Integer Multiplication and Division
- // A Atomic Instructions
- // F Single-Precision Floating-Point
- // D Single-Precision Floating-Point
- // (G = M + A + F + D)
- // Q Quad-Precision Floating-Point
- // C Compressed Instructions
- // H Hypervisor
- //
- // Others, open and non-standard
- // V Vector
- //
- // Cache Management Operations
- // Zicbom Cache Block Management Operations
- // Zicboz Cache Block Zero Operations
- // Zicbop Cache Block Prefetch Operations
- //
- // Bit-manipulation
- // Zba Address generation instructions
- // Zbb Basic bit-manipulation
- // Zbc Carry-less multiplication
- // Zbs Single-bit instructions
- //
- // Zfh Half-Precision Floating-Point instructions
- // Zfhmin Minimal Half-Precision Floating-Point instructions
- //
- // Zicond Conditional operations
- //
- // Zicsr Control and Status Register (CSR) Instructions
- // Zifencei Instruction-Fetch Fence
- // Zic64b Cache blocks must be 64 bytes in size, naturally aligned in the address space.
- // Zihintpause Pause instruction HINT
- //
- // Zc Code Size Reduction - Additional compressed instructions.
- // Zcb Simple code-size saving instructions
- //
- // Other features and settings
- // mvendorid Manufactory JEDEC id encoded, ISA vol 2 3.1.2..
- // marchid Id for microarch. Mvendorid plus marchid uniquely identify the microarch.
- // mimpid A unique encoding of the version of the processor implementation.
- // unaligned_access Unaligned memory accesses (unknown, unspported, emulated, slow, firmware, fast)
- // satp mode SATP bits (number of virtual addr bits) mbare, sv39, sv48, sv57, sv64
+
+ class RVExtFeatureValue : public RVFeatureValue {
+ const uint32_t _cpu_feature_index;
+
+ public:
+ RVExtFeatureValue(const char* pretty, int linux_bit_num, uint32_t cpu_feature_index, bool fstring) :
+ RVFeatureValue(pretty, linux_bit_num, fstring),
+ _cpu_feature_index(cpu_feature_index) {
+ }
+ int cpu_feature_index() {
+ // Can be used to check, for example, v is declared before Zvfh in RV_EXT_FEATURE_FLAGS.
+ return _cpu_feature_index;
+ }
+ bool enabled() {
+ return RVExtFeatures::current()->support_feature(_cpu_feature_index);
+ }
+ void enable_feature(int64_t value = 0) {
+ RVExtFeatures::current()->set_feature(_cpu_feature_index);
+ }
+ void disable_feature() {
+ RVExtFeatures::current()->clear_feature(_cpu_feature_index);
+ }
+ void log_enabled();
+
+ protected:
+ bool deps_all_enabled(RVExtFeatureValue* dep0, ...) {
+ assert(dep0 != nullptr, "must not");
+
+ va_list va;
+ va_start(va, dep0);
+ RVExtFeatureValue* next = dep0;
+ bool enabled = true;
+ while (next != nullptr && enabled) {
+ enabled = next->enabled();
+ next = va_arg(va, RVExtFeatureValue*);
+ }
+ va_end(va);
+ return enabled;
+ }
+
+ void deps_string(stringStream& ss, RVExtFeatureValue* dep0, ...) {
+ assert(dep0 != nullptr, "must not");
+ ss.print("%s (%s)", dep0->pretty(), dep0->enabled() ? "enabled" : "disabled");
+
+ va_list va;
+ va_start(va, dep0);
+ RVExtFeatureValue* next = nullptr;
+ while ((next = va_arg(va, RVExtFeatureValue*)) != nullptr) {
+ ss.print(", %s (%s)", next->pretty(), next->enabled() ? "enabled" : "disabled");
+ }
+ va_end(va);
+ }
+
+#ifdef ASSERT
+ void verify_deps(RVExtFeatureValue* dep0, ...) {
+ assert(dep0 != nullptr, "must not");
+ assert(cpu_feature_index() >= 0, "must");
+
+ va_list va;
+ va_start(va, dep0);
+ RVExtFeatureValue* next = dep0;
+ while (next != nullptr) {
+ assert(next->cpu_feature_index() >= 0, "must");
+ // We only need to check depenency relationship for extension flags.
+ // The dependant ones must be declared before this, for example, v must be declared
+ // before Zvfh in RV_EXT_FEATURE_FLAGS. The reason is in setup_cpu_available_features
+ // we need to make sure v is `update_flag`ed before Zvfh, so Zvfh is `update_flag`ed
+ // based on v.
+ assert(cpu_feature_index() > next->cpu_feature_index(), "Invalid");
+ next = va_arg(va, RVExtFeatureValue*);
+ }
+ va_end(va);
+ }
+#endif // ASSERT
+ };
+
+ class RVNonExtFeatureValue : public RVFeatureValue {
+ static const int64_t DEFAULT_VALUE = -1;
+ int64_t _value;
+
+ public:
+ RVNonExtFeatureValue(const char* pretty, int linux_bit_num, bool fstring) :
+ RVFeatureValue(pretty, linux_bit_num, fstring),
+ _value(DEFAULT_VALUE) {
+ }
+ bool enabled() { return _value != DEFAULT_VALUE; }
+ void enable_feature(int64_t value) {
+ assert(value != DEFAULT_VALUE, "Sanity");
+ _value = value;
+ }
+ void disable_feature() { _value = DEFAULT_VALUE; }
+ int64_t value() { return _value; }
+ void log_enabled();
+ };
public:
#define RV_NO_FLAG_BIT (BitsPerWord+1) // nth_bit will return 0 on values larger than BitsPerWord
// Note: the order matters, depender should be after their dependee. E.g. ext_V before ext_Zvbb.
- // declaration name , extension name, bit pos ,in str, mapped flag)
- #define RV_FEATURE_FLAGS(decl) \
- decl(ext_I , "i" , ('I' - 'A'), true , NO_UPDATE_DEFAULT) \
- decl(ext_M , "m" , ('M' - 'A'), true , NO_UPDATE_DEFAULT) \
- decl(ext_A , "a" , ('A' - 'A'), true , NO_UPDATE_DEFAULT) \
- decl(ext_F , "f" , ('F' - 'A'), true , NO_UPDATE_DEFAULT) \
- decl(ext_D , "d" , ('D' - 'A'), true , NO_UPDATE_DEFAULT) \
- decl(ext_C , "c" , ('C' - 'A'), true , UPDATE_DEFAULT(UseRVC)) \
- decl(ext_Q , "q" , ('Q' - 'A'), true , NO_UPDATE_DEFAULT) \
- decl(ext_H , "h" , ('H' - 'A'), true , NO_UPDATE_DEFAULT) \
- decl(ext_V , "v" , ('V' - 'A'), true , UPDATE_DEFAULT(UseRVV)) \
- decl(ext_Zicbom , "Zicbom" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZicbom)) \
- decl(ext_Zicboz , "Zicboz" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZicboz)) \
- decl(ext_Zicbop , "Zicbop" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZicbop)) \
- decl(ext_Zba , "Zba" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZba)) \
- decl(ext_Zbb , "Zbb" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZbb)) \
- decl(ext_Zbc , "Zbc" , RV_NO_FLAG_BIT, true , NO_UPDATE_DEFAULT) \
- decl(ext_Zbs , "Zbs" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZbs)) \
- decl(ext_Zbkb , "Zbkb" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZbkb)) \
- decl(ext_Zcb , "Zcb" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZcb)) \
- decl(ext_Zfa , "Zfa" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZfa)) \
- decl(ext_Zfh , "Zfh" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZfh)) \
- decl(ext_Zfhmin , "Zfhmin" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZfhmin)) \
- decl(ext_Zicsr , "Zicsr" , RV_NO_FLAG_BIT, true , NO_UPDATE_DEFAULT) \
- decl(ext_Zicntr , "Zicntr" , RV_NO_FLAG_BIT, true , NO_UPDATE_DEFAULT) \
- decl(ext_Zifencei , "Zifencei" , RV_NO_FLAG_BIT, true , NO_UPDATE_DEFAULT) \
- decl(ext_Zic64b , "Zic64b" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZic64b)) \
- decl(ext_Ztso , "Ztso" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZtso)) \
- decl(ext_Zihintpause , "Zihintpause" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZihintpause)) \
- decl(ext_Zacas , "Zacas" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZacas)) \
- decl(ext_Zvbb , "Zvbb" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT_DEP(UseZvbb, ext_V)) \
- decl(ext_Zvbc , "Zvbc" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT_DEP(UseZvbc, ext_V)) \
- decl(ext_Zvfh , "Zvfh" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT_DEP(UseZvfh, ext_V)) \
- decl(ext_Zvkn , "Zvkn" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT_DEP(UseZvkn, ext_V)) \
- decl(ext_Zicond , "Zicond" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZicond)) \
- decl(mvendorid , "VendorId" , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT) \
- decl(marchid , "ArchId" , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT) \
- decl(mimpid , "ImpId" , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT) \
- decl(unaligned_access, "Unaligned" , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT) \
- decl(satp_mode , "SATP" , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT) \
+ //
+ // Fields description in `decl`:
+ // declaration name, extension name, bit value from linux, feature string?, mapped flag)
+ #define RV_EXT_FEATURE_FLAGS(decl) \
+ /* A Atomic Instructions */ \
+ decl(a , ('A' - 'A'), true , NO_UPDATE_DEFAULT) \
+ /* C Compressed Instructions */ \
+ decl(c , ('C' - 'A'), true , UPDATE_DEFAULT(UseRVC)) \
+ /* D Single-Precision Floating-Point */ \
+ decl(d , ('D' - 'A'), true , NO_UPDATE_DEFAULT) \
+ /* F Single-Precision Floating-Point */ \
+ decl(f , ('F' - 'A'), true , NO_UPDATE_DEFAULT) \
+ /* H Hypervisor */ \
+ decl(h , ('H' - 'A'), true , NO_UPDATE_DEFAULT) \
+ /* I RV64I */ \
+ decl(i , ('I' - 'A'), true , NO_UPDATE_DEFAULT) \
+ /* M Integer Multiplication and Division */ \
+ decl(m , ('M' - 'A'), true , NO_UPDATE_DEFAULT) \
+ /* Q Quad-Precision Floating-Point */ \
+ decl(q , ('Q' - 'A'), true , NO_UPDATE_DEFAULT) \
+ /* V Vector */ \
+ decl(v , ('V' - 'A'), true , UPDATE_DEFAULT(UseRVV)) \
+ \
+ /* ----------------------- Other extensions ----------------------- */ \
+ \
+ /* Atomic compare-and-swap (CAS) instructions */ \
+ decl(Zacas , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZacas)) \
+ /* Zba Address generation instructions */ \
+ decl(Zba , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZba)) \
+ /* Zbb Basic bit-manipulation */ \
+ decl(Zbb , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZbb)) \
+ /* Zbc Carry-less multiplication */ \
+ decl(Zbc , RV_NO_FLAG_BIT, true , NO_UPDATE_DEFAULT) \
+ /* Bitmanip instructions for Cryptography */ \
+ decl(Zbkb , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZbkb)) \
+ /* Zbs Single-bit instructions */ \
+ decl(Zbs , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZbs)) \
+ /* Zcb Simple code-size saving instructions */ \
+ decl(Zcb , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZcb)) \
+ /* Additional Floating-Point instructions */ \
+ decl(Zfa , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZfa)) \
+ /* Zfh Half-Precision Floating-Point instructions */ \
+ decl(Zfh , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZfh)) \
+ /* Zfhmin Minimal Half-Precision Floating-Point instructions */ \
+ decl(Zfhmin , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZfhmin)) \
+ /* Zicbom Cache Block Management Operations */ \
+ decl(Zicbom , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZicbom)) \
+ /* Zicbop Cache Block Prefetch Operations */ \
+ decl(Zicbop , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZicbop)) \
+ /* Zicboz Cache Block Zero Operations */ \
+ decl(Zicboz , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZicboz)) \
+ /* Base Counters and Timers */ \
+ decl(Zicntr , RV_NO_FLAG_BIT, true , NO_UPDATE_DEFAULT) \
+ /* Zicond Conditional operations */ \
+ decl(Zicond , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZicond)) \
+ /* Zicsr Control and Status Register (CSR) Instructions */ \
+ decl(Zicsr , RV_NO_FLAG_BIT, true , NO_UPDATE_DEFAULT) \
+ /* Zic64b Cache blocks must be 64 bytes in size, naturally aligned in the address space. */ \
+ decl(Zic64b , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZic64b)) \
+ /* Zifencei Instruction-Fetch Fence */ \
+ decl(Zifencei , RV_NO_FLAG_BIT, true , NO_UPDATE_DEFAULT) \
+ /* Zihintpause Pause instruction HINT */ \
+ decl(Zihintpause , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZihintpause)) \
+ /* Total Store Ordering */ \
+ decl(Ztso , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZtso)) \
+ /* Vector Basic Bit-manipulation */ \
+ decl(Zvbb , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT_DEP(UseZvbb, &ext_v, nullptr)) \
+ /* Vector Carryless Multiplication */ \
+ decl(Zvbc , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT_DEP(UseZvbc, &ext_v, nullptr)) \
+ /* Vector Extension for Half-Precision Floating-Point */ \
+ decl(Zvfh , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT_DEP(UseZvfh, &ext_v, &ext_Zfh, nullptr)) \
+ /* Shorthand for Zvkned + Zvknhb + Zvkb + Zvkt */ \
+ decl(Zvkn , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT_DEP(UseZvkn, &ext_v, nullptr)) \
- #define DECLARE_RV_FEATURE(NAME, PRETTY, BIT, FSTRING, FLAGF) \
- struct NAME##RVFeatureValue : public RVFeatureValue { \
- NAME##RVFeatureValue(const char* pretty, int bit, bool fstring) : \
- RVFeatureValue(pretty, bit, fstring) {} \
- FLAGF; \
- }; \
- static NAME##RVFeatureValue NAME; \
+ #define DECLARE_RV_EXT_FEATURE(PRETTY, LINUX_BIT, FSTRING, FLAGF) \
+ struct ext_##PRETTY##RVExtFeatureValue : public RVExtFeatureValue { \
+ ext_##PRETTY##RVExtFeatureValue() : \
+ RVExtFeatureValue(#PRETTY, LINUX_BIT, RVExtFeatures::CPU_##ext_##PRETTY, FSTRING) {} \
+ FLAGF; \
+ }; \
+ static ext_##PRETTY##RVExtFeatureValue ext_##PRETTY; \
- RV_FEATURE_FLAGS(DECLARE_RV_FEATURE)
- #undef DECLARE_RV_FEATURE
+ RV_EXT_FEATURE_FLAGS(DECLARE_RV_EXT_FEATURE)
+ #undef DECLARE_RV_EXT_FEATURE
+
+ // Non-extension features
+ //
+ #define RV_NON_EXT_FEATURE_FLAGS(decl) \
+ /* Id for microarch. Mvendorid plus marchid uniquely identify the microarch. */ \
+ decl(marchid , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT) \
+ /* A unique encoding of the version of the processor implementation. */ \
+ decl(mimpid , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT) \
+ /* Manufactory JEDEC id encoded, ISA vol 2 3.1.2.. */ \
+ decl(mvendorid , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT) \
+ /* SATP bits (number of virtual addr bits) mbare, sv39, sv48, sv57, sv64 */ \
+ decl(satp_mode , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT) \
+ /* Performance of misaligned scalar accesses (unknown, emulated, slow, fast, unsupported) */ \
+ decl(unaligned_scalar , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT) \
+ /* Performance of misaligned vector accesses (unknown, unspported, slow, fast) */ \
+ decl(unaligned_vector , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT) \
+ decl(zicboz_block_size , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT) \
+
+ #define DECLARE_RV_NON_EXT_FEATURE(PRETTY, LINUX_BIT, FSTRING, FLAGF) \
+ struct PRETTY##RVNonExtFeatureValue : public RVNonExtFeatureValue { \
+ PRETTY##RVNonExtFeatureValue() : \
+ RVNonExtFeatureValue(#PRETTY, LINUX_BIT, FSTRING) {} \
+ FLAGF; \
+ }; \
+ static PRETTY##RVNonExtFeatureValue PRETTY; \
+
+ RV_NON_EXT_FEATURE_FLAGS(DECLARE_RV_NON_EXT_FEATURE)
+ #undef DECLARE_RV_NON_EXT_FEATURE
+
+private:
+ // Utility for AOT CPU feature store/check.
+ class RVExtFeatures : public CHeapObj {
+ public:
+ enum RVFeatureIndex {
+ #define DECLARE_RV_FEATURE_ENUM(PRETTY, LINUX_BIT, FSTRING, FLAGF) CPU_##ext_##PRETTY,
+
+ RV_EXT_FEATURE_FLAGS(DECLARE_RV_FEATURE_ENUM)
+ MAX_CPU_FEATURE_INDEX
+ #undef DECLARE_RV_FEATURE_ENUM
+ };
+ private:
+ uint64_t _features_bitmap[(MAX_CPU_FEATURE_INDEX / BitsPerLong) + 1];
+ STATIC_ASSERT(sizeof(_features_bitmap) * BitsPerByte >= MAX_CPU_FEATURE_INDEX);
+
+ // Number of 8-byte elements in _features_bitmap.
+ constexpr static int element_count() {
+ return sizeof(_features_bitmap) / sizeof(uint64_t);
+ }
+
+ static int element_index(RVFeatureIndex feature) {
+ int idx = feature / BitsPerLong;
+ assert(idx < element_count(), "Features array index out of bounds");
+ return idx;
+ }
+
+ static uint64_t feature_bit(RVFeatureIndex feature) {
+ return (1ULL << (feature % BitsPerLong));
+ }
+
+ static RVFeatureIndex convert(uint32_t index) {
+ assert(index < MAX_CPU_FEATURE_INDEX, "must");
+ return (RVFeatureIndex)index;
+ }
+
+ public:
+ static RVExtFeatures* current() {
+ return _rv_ext_features;
+ }
+
+ RVExtFeatures() {
+ for (int i = 0; i < element_count(); i++) {
+ _features_bitmap[i] = 0;
+ }
+ }
+
+ void set_feature(uint32_t feature) {
+ RVFeatureIndex f = convert(feature);
+ int idx = element_index(f);
+ _features_bitmap[idx] |= feature_bit(f);
+ }
+
+ void clear_feature(uint32_t feature) {
+ RVFeatureIndex f = convert(feature);
+ int idx = element_index(f);
+ _features_bitmap[idx] &= ~feature_bit(f);
+ }
+
+ bool support_feature(uint32_t feature) {
+ RVFeatureIndex f = convert(feature);
+ int idx = element_index(f);
+ return (_features_bitmap[idx] & feature_bit(f)) != 0;
+ }
+ };
// enable extensions based on profile, current supported profiles:
// RVA20U64
@@ -275,16 +453,24 @@ class VM_Version : public Abstract_VM_Version {
static VM_MODE parse_satp_mode(const char* vm_mode);
// Values from riscv_hwprobe()
- enum UNALIGNED_ACCESS : int {
- MISALIGNED_UNKNOWN = 0,
- MISALIGNED_EMULATED = 1,
- MISALIGNED_SLOW = 2,
- MISALIGNED_FAST = 3,
- MISALIGNED_UNSUPPORTED = 4
+ enum UNALIGNED_SCALAR_ACCESS : int {
+ MISALIGNED_SCALAR_UNKNOWN = 0,
+ MISALIGNED_SCALAR_EMULATED = 1,
+ MISALIGNED_SCALAR_SLOW = 2,
+ MISALIGNED_SCALAR_FAST = 3,
+ MISALIGNED_SCALAR_UNSUPPORTED = 4
+ };
+
+ enum UNALIGNED_VECTOR_ACCESS : int {
+ MISALIGNED_VECTOR_UNKNOWN = 0,
+ MISALIGNED_VECTOR_SLOW = 2,
+ MISALIGNED_VECTOR_FAST = 3,
+ MISALIGNED_VECTOR_UNSUPPORTED = 4
};
// Null terminated list
static RVFeatureValue* _feature_list[];
+ static RVExtFeatures* _rv_ext_features;
// Enables features in _feature_list
static void setup_cpu_available_features();
diff --git a/src/hotspot/cpu/s390/assembler_s390.inline.hpp b/src/hotspot/cpu/s390/assembler_s390.inline.hpp
index 567f3d75a62..3bab60f0bb6 100644
--- a/src/hotspot/cpu/s390/assembler_s390.inline.hpp
+++ b/src/hotspot/cpu/s390/assembler_s390.inline.hpp
@@ -26,7 +26,7 @@
#ifndef CPU_S390_ASSEMBLER_S390_INLINE_HPP
#define CPU_S390_ASSEMBLER_S390_INLINE_HPP
-#include "asm/assembler.inline.hpp"
+#include "asm/assembler.hpp"
#include "asm/codeBuffer.hpp"
#include "code/codeCache.hpp"
diff --git a/src/hotspot/cpu/s390/c1_CodeStubs_s390.cpp b/src/hotspot/cpu/s390/c1_CodeStubs_s390.cpp
index 68e7114b3b6..f1272ee1a22 100644
--- a/src/hotspot/cpu/s390/c1_CodeStubs_s390.cpp
+++ b/src/hotspot/cpu/s390/c1_CodeStubs_s390.cpp
@@ -234,12 +234,10 @@ void MonitorEnterStub::emit_code(LIR_Assembler* ce) {
void MonitorExitStub::emit_code(LIR_Assembler* ce) {
__ bind(_entry);
// Move address of the BasicObjectLock into Z_R1_scratch.
- if (_compute_lock) {
- // Lock_reg was destroyed by fast unlocking attempt => recompute it.
- ce->monitor_address(_monitor_ix, FrameMap::as_opr(Z_R1_scratch));
- } else {
- __ lgr_if_needed(Z_R1_scratch, _lock_reg->as_register());
- }
+
+ // Lock_reg was destroyed by fast unlocking attempt => recompute it.
+ ce->monitor_address(_monitor_ix, FrameMap::as_opr(Z_R1_scratch));
+
// Note: non-blocking leaf routine => no call info needed.
StubId exit_id;
if (ce->compilation()->has_fpu_code()) {
diff --git a/src/hotspot/cpu/s390/c1_FrameMap_s390.cpp b/src/hotspot/cpu/s390/c1_FrameMap_s390.cpp
index ddba445154a..e219e9bbb40 100644
--- a/src/hotspot/cpu/s390/c1_FrameMap_s390.cpp
+++ b/src/hotspot/cpu/s390/c1_FrameMap_s390.cpp
@@ -282,13 +282,6 @@ LIR_Opr FrameMap::stack_pointer() {
return Z_SP_opr;
}
-// JSR 292
-// On ZARCH_64, there is no need to save the SP, because neither
-// method handle intrinsics nor compiled lambda forms modify it.
-LIR_Opr FrameMap::method_handle_invoke_SP_save_opr() {
- return LIR_OprFact::illegalOpr;
-}
-
bool FrameMap::validate_frame() {
return true;
}
diff --git a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp
index b875eeca9ad..298234156c3 100644
--- a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp
@@ -228,7 +228,7 @@ int LIR_Assembler::emit_unwind_handler() {
// StubId::c1_monitorexit_id expects lock address in Z_R1_scratch.
LIR_Opr lock = FrameMap::as_opr(Z_R1_scratch);
monitor_address(0, lock);
- stub = new MonitorExitStub(lock, true, 0);
+ stub = new MonitorExitStub(lock, 0);
__ unlock_object(Rtmp1, Rtmp2, lock->as_register(), *stub->entry());
__ bind(*stub->continuation());
}
@@ -2711,7 +2711,6 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
Register hdr = op->hdr_opr()->as_register();
Register lock = op->lock_opr()->as_register();
if (op->code() == lir_lock) {
- assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
// Add debug info for NullPointerException only if one is possible.
if (op->info() != nullptr) {
add_debug_info_for_null_check_here(op->info());
@@ -2719,7 +2718,6 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
__ lock_object(hdr, obj, lock, *op->stub()->entry());
// done
} else if (op->code() == lir_unlock) {
- assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
__ unlock_object(hdr, obj, lock, *op->stub()->entry());
} else {
ShouldNotReachHere();
diff --git a/src/hotspot/cpu/s390/c2_globals_s390.hpp b/src/hotspot/cpu/s390/c2_globals_s390.hpp
index 94190c25f5b..f1e757889b0 100644
--- a/src/hotspot/cpu/s390/c2_globals_s390.hpp
+++ b/src/hotspot/cpu/s390/c2_globals_s390.hpp
@@ -45,8 +45,7 @@ define_pd_global(intx, CompileThreshold, 10000);
define_pd_global(intx, OnStackReplacePercentage, 140);
define_pd_global(intx, ConditionalMoveLimit, 4);
define_pd_global(intx, FreqInlineSize, 175);
-// 10 prevents spill-split-recycle sanity check in JVM2008.xml.transform.
-define_pd_global(intx, InteriorEntryAlignment, 2);
+define_pd_global(intx, InteriorEntryAlignment, 4);
define_pd_global(size_t, NewSizeThreadIncrease, ScaleForWordSize(4*K));
define_pd_global(intx, RegisterCostAreaRatio, 12000);
define_pd_global(intx, LoopUnrollLimit, 60);
diff --git a/src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.cpp b/src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.cpp
index dea3317270e..e4fe690663a 100644
--- a/src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.cpp
@@ -28,7 +28,6 @@
#include "gc/g1/g1BarrierSetAssembler.hpp"
#include "gc/g1/g1BarrierSetRuntime.hpp"
#include "gc/g1/g1CardTable.hpp"
-#include "gc/g1/g1DirtyCardQueue.hpp"
#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1SATBMarkQueueSet.hpp"
#include "gc/g1/g1ThreadLocalData.hpp"
@@ -205,104 +204,71 @@ void G1BarrierSetAssembler::generate_c2_pre_barrier_stub(MacroAssembler* masm,
BLOCK_COMMENT("} generate_c2_pre_barrier_stub");
}
+static void generate_post_barrier(MacroAssembler* masm,
+ const Register store_addr,
+ const Register new_val,
+ const Register thread,
+ const Register tmp1,
+ const Register tmp2,
+ Label& done,
+ bool new_val_may_be_null) {
+
+ __ block_comment("generate_post_barrier {");
+
+ assert(thread == Z_thread, "must be");
+ assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, noreg);
+
+ // Does store cross heap regions?
+ if (VM_Version::has_DistinctOpnds()) {
+ __ z_xgrk(tmp1, store_addr, new_val); // tmp1 := store address ^ new value
+ } else {
+ __ z_lgr(tmp1, store_addr);
+ __ z_xgr(tmp1, new_val);
+ }
+ __ z_srag(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes); // tmp1 := ((store address ^ new value) >> LogOfHRGrainBytes)
+ __ branch_optimized(Assembler::bcondEqual, done);
+
+ // Crosses regions, storing null?
+ if (new_val_may_be_null) {
+ __ z_ltgr(new_val, new_val);
+ __ z_bre(done);
+ } else {
+#ifdef ASSERT
+ __ z_ltgr(new_val, new_val);
+ __ asm_assert(Assembler::bcondNotZero, "null oop not allowed (G1 post)", 0x322); // Checked by caller.
+#endif
+ }
+
+ __ z_srag(tmp1, store_addr, CardTable::card_shift());
+
+ Address card_table_addr(thread, in_bytes(G1ThreadLocalData::card_table_base_offset()));
+ __ z_alg(tmp1, card_table_addr); // tmp1 := card address
+
+ if(UseCondCardMark) {
+ __ z_cli(0, tmp1, G1CardTable::clean_card_val());
+ __ branch_optimized(Assembler::bcondNotEqual, done);
+ }
+
+ static_assert(G1CardTable::dirty_card_val() == 0, "must be to use z_mvi");
+ __ z_mvi(0, tmp1, G1CardTable::dirty_card_val()); // *(card address) := dirty_card_val
+
+ __ block_comment("} generate_post_barrier");
+}
+
void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm,
Register store_addr,
Register new_val,
Register thread,
Register tmp1,
Register tmp2,
- G1PostBarrierStubC2* stub) {
+ bool new_val_may_be_null) {
BLOCK_COMMENT("g1_write_barrier_post_c2 {");
-
- assert(thread == Z_thread, "must be");
- assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, Z_R1_scratch);
-
- assert(store_addr != noreg && new_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register");
-
- stub->initialize_registers(thread, tmp1, tmp2);
-
- BLOCK_COMMENT("generate_region_crossing_test {");
- if (VM_Version::has_DistinctOpnds()) {
- __ z_xgrk(tmp1, store_addr, new_val);
- } else {
- __ z_lgr(tmp1, store_addr);
- __ z_xgr(tmp1, new_val);
- }
- __ z_srag(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes);
- __ branch_optimized(Assembler::bcondEqual, *stub->continuation());
- BLOCK_COMMENT("} generate_region_crossing_test");
-
- // crosses regions, storing null?
- if ((stub->barrier_data() & G1C2BarrierPostNotNull) == 0) {
- __ z_ltgr(new_val, new_val);
- __ branch_optimized(Assembler::bcondEqual, *stub->continuation());
- }
-
- BLOCK_COMMENT("generate_card_young_test {");
- CardTableBarrierSet* ct = barrier_set_cast(BarrierSet::barrier_set());
- // calculate address of card
- __ load_const_optimized(tmp2, (address)ct->card_table()->byte_map_base()); // Card table base.
- __ z_srlg(tmp1, store_addr, CardTable::card_shift()); // Index into card table.
- __ z_algr(tmp1, tmp2); // Explicit calculation needed for cli.
-
- // Filter young.
- __ z_cli(0, tmp1, G1CardTable::g1_young_card_val());
-
- BLOCK_COMMENT("} generate_card_young_test");
-
- // From here on, tmp1 holds the card address.
- __ branch_optimized(Assembler::bcondNotEqual, *stub->entry());
-
- __ bind(*stub->continuation());
-
+ Label done;
+ generate_post_barrier(masm, store_addr, new_val, thread, tmp1, tmp2, done, new_val_may_be_null);
+ __ bind(done);
BLOCK_COMMENT("} g1_write_barrier_post_c2");
}
-void G1BarrierSetAssembler::generate_c2_post_barrier_stub(MacroAssembler* masm,
- G1PostBarrierStubC2* stub) const {
-
- BLOCK_COMMENT("generate_c2_post_barrier_stub {");
-
- Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
- Label runtime;
-
- Register thread = stub->thread();
- Register tmp1 = stub->tmp1(); // tmp1 holds the card address.
- Register tmp2 = stub->tmp2();
- Register Rcard_addr = tmp1;
-
- __ bind(*stub->entry());
-
- BLOCK_COMMENT("generate_card_clean_test {");
- __ z_sync(); // Required to support concurrent cleaning.
- __ z_cli(0, Rcard_addr, 0); // Reload after membar.
- __ branch_optimized(Assembler::bcondEqual, *stub->continuation());
- BLOCK_COMMENT("} generate_card_clean_test");
-
- BLOCK_COMMENT("generate_dirty_card {");
- // Storing a region crossing, non-null oop, card is clean.
- // Dirty card and log.
- STATIC_ASSERT(CardTable::dirty_card_val() == 0);
- __ z_mvi(0, Rcard_addr, CardTable::dirty_card_val());
- BLOCK_COMMENT("} generate_dirty_card");
-
- generate_queue_test_and_insertion(masm,
- G1ThreadLocalData::dirty_card_queue_index_offset(),
- G1ThreadLocalData::dirty_card_queue_buffer_offset(),
- runtime,
- Z_thread, tmp1, tmp2);
-
- __ branch_optimized(Assembler::bcondAlways, *stub->continuation());
-
- __ bind(runtime);
-
- generate_c2_barrier_runtime_call(masm, stub, tmp1, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry));
-
- __ branch_optimized(Assembler::bcondAlways, *stub->continuation());
-
- BLOCK_COMMENT("} generate_c2_post_barrier_stub");
-}
-
#endif //COMPILER2
void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
@@ -451,99 +417,9 @@ void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm, Decorato
Register Rtmp1, Register Rtmp2, Register Rtmp3) {
bool not_null = (decorators & IS_NOT_NULL) != 0;
- assert_different_registers(Rstore_addr, Rnew_val, Rtmp1, Rtmp2); // Most probably, Rnew_val == Rtmp3.
-
- Label callRuntime, filtered;
-
- CardTableBarrierSet* ct = barrier_set_cast(BarrierSet::barrier_set());
-
- BLOCK_COMMENT("g1_write_barrier_post {");
-
- // Does store cross heap regions?
- // It does if the two addresses specify different grain addresses.
- if (VM_Version::has_DistinctOpnds()) {
- __ z_xgrk(Rtmp1, Rstore_addr, Rnew_val);
- } else {
- __ z_lgr(Rtmp1, Rstore_addr);
- __ z_xgr(Rtmp1, Rnew_val);
- }
- __ z_srag(Rtmp1, Rtmp1, G1HeapRegion::LogOfHRGrainBytes);
- __ z_bre(filtered);
-
- // Crosses regions, storing null?
- if (not_null) {
-#ifdef ASSERT
- __ z_ltgr(Rnew_val, Rnew_val);
- __ asm_assert(Assembler::bcondNotZero, "null oop not allowed (G1 post)", 0x322); // Checked by caller.
-#endif
- } else {
- __ z_ltgr(Rnew_val, Rnew_val);
- __ z_bre(filtered);
- }
-
- Rnew_val = noreg; // end of lifetime
-
- // Storing region crossing non-null, is card already dirty?
- assert_different_registers(Rtmp1, Rtmp2, Rtmp3);
- // Make sure not to use Z_R0 for any of these registers.
- Register Rcard_addr = (Rtmp1 != Z_R0_scratch) ? Rtmp1 : Rtmp3;
- Register Rbase = (Rtmp2 != Z_R0_scratch) ? Rtmp2 : Rtmp3;
-
- // calculate address of card
- __ load_const_optimized(Rbase, (address)ct->card_table()->byte_map_base()); // Card table base.
- __ z_srlg(Rcard_addr, Rstore_addr, CardTable::card_shift()); // Index into card table.
- __ z_algr(Rcard_addr, Rbase); // Explicit calculation needed for cli.
- Rbase = noreg; // end of lifetime
-
- // Filter young.
- __ z_cli(0, Rcard_addr, G1CardTable::g1_young_card_val());
- __ z_bre(filtered);
-
- // Check the card value. If dirty, we're done.
- // This also avoids false sharing of the (already dirty) card.
- __ z_sync(); // Required to support concurrent cleaning.
- __ z_cli(0, Rcard_addr, G1CardTable::dirty_card_val()); // Reload after membar.
- __ z_bre(filtered);
-
- // Storing a region crossing, non-null oop, card is clean.
- // Dirty card and log.
- __ z_mvi(0, Rcard_addr, G1CardTable::dirty_card_val());
-
- Register Rcard_addr_x = Rcard_addr;
- Register Rqueue_index = (Rtmp2 != Z_R0_scratch) ? Rtmp2 : Rtmp1;
- if (Rcard_addr == Rqueue_index) {
- Rcard_addr_x = Z_R0_scratch; // Register shortage. We have to use Z_R0.
- }
- __ lgr_if_needed(Rcard_addr_x, Rcard_addr);
-
- generate_queue_test_and_insertion(masm,
- G1ThreadLocalData::dirty_card_queue_index_offset(),
- G1ThreadLocalData::dirty_card_queue_buffer_offset(),
- callRuntime,
- Z_thread, Rcard_addr_x, Rqueue_index);
- __ z_bru(filtered);
-
- __ bind(callRuntime);
-
- // TODO: do we need a frame? Introduced to be on the safe side.
- bool needs_frame = true;
- __ lgr_if_needed(Rcard_addr, Rcard_addr_x); // copy back asap. push_frame will destroy Z_R0_scratch!
-
- // VM call need frame to access(write) O register.
- if (needs_frame) {
- __ save_return_pc();
- __ push_frame_abi160(0); // Will use Z_R0 as tmp on old CPUs.
- }
-
- // Save the live input values.
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), Rcard_addr, Z_thread);
-
- if (needs_frame) {
- __ pop_frame();
- __ restore_return_pc();
- }
-
- __ bind(filtered);
+ Label done;
+ generate_post_barrier(masm, Rstore_addr, Rnew_val, Z_thread, Rtmp1, Rtmp2, done, !not_null);
+ __ bind(done);
BLOCK_COMMENT("} g1_write_barrier_post");
}
@@ -615,22 +491,19 @@ void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrier
__ branch_optimized(Assembler::bcondAlways, *stub->continuation());
}
-void G1BarrierSetAssembler::gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) {
- G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
- __ bind(*stub->entry());
- ce->check_reserved_argument_area(16); // RT stub needs 2 spill slots.
- assert(stub->addr()->is_register(), "Precondition.");
- assert(stub->new_val()->is_register(), "Precondition.");
- Register new_val_reg = stub->new_val()->as_register();
- __ z_ltgr(new_val_reg, new_val_reg);
- __ branch_optimized(Assembler::bcondZero, *stub->continuation());
- __ z_lgr(Z_R1_scratch, stub->addr()->as_pointer_register());
- ce->emit_call_c(bs->post_barrier_c1_runtime_code_blob()->code_begin());
- __ branch_optimized(Assembler::bcondAlways, *stub->continuation());
-}
-
#undef __
+void G1BarrierSetAssembler::g1_write_barrier_post_c1(MacroAssembler* masm,
+ Register store_addr,
+ Register new_val,
+ Register thread,
+ Register tmp1,
+ Register tmp2) {
+ Label done;
+ generate_post_barrier(masm, store_addr, new_val, thread, tmp1, tmp2, done, true /* new_val_may_be_null */);
+ masm->bind(done);
+}
+
#define __ sasm->
static OopMap* save_volatile_registers(StubAssembler* sasm, Register return_pc = Z_R14) {
@@ -705,92 +578,6 @@ void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler*
__ z_bru(restart);
}
-void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) {
- // Z_R1_scratch: oop address, address of updated memory slot
-
- BarrierSet* bs = BarrierSet::barrier_set();
- __ set_info("g1_post_barrier_slow_id", false);
-
- Register addr_oop = Z_R1_scratch;
- Register addr_card = Z_R1_scratch;
- Register r1 = Z_R6; // Must be saved/restored.
- Register r2 = Z_R7; // Must be saved/restored.
- Register cardtable = r1; // Must be non-volatile, because it is used to save addr_card.
- CardTableBarrierSet* ctbs = barrier_set_cast(bs);
- CardTable* ct = ctbs->card_table();
- CardTable::CardValue* byte_map_base = ct->byte_map_base();
-
- // Save registers used below (see assertion in G1PreBarrierStub::emit_code()).
- __ z_stg(r1, 0*BytesPerWord + FrameMap::first_available_sp_in_frame, Z_SP);
-
- Label not_already_dirty, restart, refill, young_card;
-
- // Calculate address of card corresponding to the updated oop slot.
- AddressLiteral rs(byte_map_base);
- __ z_srlg(addr_card, addr_oop, CardTable::card_shift());
- addr_oop = noreg; // dead now
- __ load_const_optimized(cardtable, rs); // cardtable :=
- __ z_agr(addr_card, cardtable); // addr_card := addr_oop>>card_shift + cardtable
-
- __ z_cli(0, addr_card, (int)G1CardTable::g1_young_card_val());
- __ z_bre(young_card);
-
- __ z_sync(); // Required to support concurrent cleaning.
-
- __ z_cli(0, addr_card, (int)CardTable::dirty_card_val());
- __ z_brne(not_already_dirty);
-
- __ bind(young_card);
- // We didn't take the branch, so we're already dirty: restore
- // used registers and return.
- __ z_lg(r1, 0*BytesPerWord + FrameMap::first_available_sp_in_frame, Z_SP);
- __ z_br(Z_R14);
-
- // Not dirty.
- __ bind(not_already_dirty);
-
- // First, dirty it: [addr_card] := 0
- __ z_mvi(0, addr_card, CardTable::dirty_card_val());
-
- Register idx = cardtable; // Must be non-volatile, because it is used to save addr_card.
- Register buf = r2;
- cardtable = noreg; // now dead
-
- // Save registers used below (see assertion in G1PreBarrierStub::emit_code()).
- __ z_stg(r2, 1*BytesPerWord + FrameMap::first_available_sp_in_frame, Z_SP);
-
- ByteSize dirty_card_q_index_byte_offset = G1ThreadLocalData::dirty_card_queue_index_offset();
- ByteSize dirty_card_q_buf_byte_offset = G1ThreadLocalData::dirty_card_queue_buffer_offset();
-
- __ bind(restart);
-
- // Get the index into the update buffer. G1DirtyCardQueue::_index is
- // a size_t so z_ltg is appropriate here.
- __ z_ltg(idx, Address(Z_thread, dirty_card_q_index_byte_offset));
-
- // index == 0?
- __ z_brz(refill);
-
- __ z_lg(buf, Address(Z_thread, dirty_card_q_buf_byte_offset));
- __ add2reg(idx, -oopSize);
-
- __ z_stg(addr_card, 0, idx, buf); // [_buf + index] :=
- __ z_stg(idx, Address(Z_thread, dirty_card_q_index_byte_offset));
- // Restore killed registers and return.
- __ z_lg(r1, 0*BytesPerWord + FrameMap::first_available_sp_in_frame, Z_SP);
- __ z_lg(r2, 1*BytesPerWord + FrameMap::first_available_sp_in_frame, Z_SP);
- __ z_br(Z_R14);
-
- __ bind(refill);
- save_volatile_registers(sasm);
- __ z_lgr(idx, addr_card); // Save addr_card, tmp3 must be non-volatile.
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1DirtyCardQueueSet::handle_zero_index_for_thread),
- Z_thread);
- __ z_lgr(addr_card, idx);
- restore_volatile_registers(sasm); // Restore addr_card.
- __ z_bru(restart);
-}
-
#undef __
#endif // COMPILER1
diff --git a/src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.hpp b/src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.hpp
index 0f0bdd8b83c..fdec751c43b 100644
--- a/src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.hpp
+++ b/src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -33,9 +33,7 @@
class LIR_Assembler;
class StubAssembler;
class G1PreBarrierStub;
-class G1PostBarrierStub;
class G1PreBarrierStubC2;
-class G1PostBarrierStubC2;
class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
protected:
@@ -60,10 +58,16 @@ class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
public:
#ifdef COMPILER1
void gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub);
- void gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub);
void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm);
- void generate_c1_post_barrier_runtime_stub(StubAssembler* sasm);
+
+ void g1_write_barrier_post_c1(MacroAssembler* masm,
+ Register store_addr,
+ Register new_val,
+ Register thread,
+ Register tmp1,
+ Register tmp2);
+
#endif // COMPILER1
#ifdef COMPILER2
@@ -81,9 +85,7 @@ class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
Register thread,
Register tmp1,
Register tmp2,
- G1PostBarrierStubC2* c2_stub);
- void generate_c2_post_barrier_stub(MacroAssembler* masm,
- G1PostBarrierStubC2* stub) const;
+ bool new_val_may_be_null);
#endif // COMPILER2
virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
diff --git a/src/hotspot/cpu/s390/gc/g1/g1_s390.ad b/src/hotspot/cpu/s390/gc/g1/g1_s390.ad
index 31f60c4aeff..7aed374fdae 100644
--- a/src/hotspot/cpu/s390/gc/g1/g1_s390.ad
+++ b/src/hotspot/cpu/s390/gc/g1/g1_s390.ad
@@ -1,5 +1,5 @@
//
-// Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
// Copyright 2024 IBM Corporation. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
@@ -62,13 +62,13 @@ static void write_barrier_post(MacroAssembler* masm,
Register new_val,
Register tmp1,
Register tmp2) {
- if (!G1PostBarrierStubC2::needs_barrier(node)) {
+ if (!G1BarrierStubC2::needs_post_barrier(node)) {
return;
}
Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
G1BarrierSetAssembler* g1_asm = static_cast(BarrierSet::barrier_set()->barrier_set_assembler());
- G1PostBarrierStubC2* const stub = G1PostBarrierStubC2::create(node);
- g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, Z_thread, tmp1, tmp2, stub);
+ bool new_val_may_be_null = G1BarrierStubC2::post_new_val_may_be_null(node);
+ g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, Z_thread, tmp1, tmp2, new_val_may_be_null);
}
%} // source
diff --git a/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.cpp b/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.cpp
index 2d663061aec..c6f5a4e119c 100644
--- a/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.cpp
@@ -171,6 +171,7 @@ void BarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Re
void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm) {
BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
+ __ align(4, __ offset() + OFFSET_TO_PATCHABLE_DATA); // must align the following block which requires atomic updates
__ block_comment("nmethod_entry_barrier (nmethod_entry_barrier) {");
// Load jump addr:
diff --git a/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.hpp b/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.hpp
index acc0d3b4988..3e0b2be4873 100644
--- a/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.hpp
+++ b/src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.hpp
@@ -66,6 +66,14 @@ public:
OptoReg::Name refine_register(const Node* node,
OptoReg::Name opto_reg) const;
#endif // COMPILER2
+
+ static const int OFFSET_TO_PATCHABLE_DATA_INSTRUCTION = 6 + 6 + 6; // iihf(6) + iilf(6) + lg(6)
+ static const int BARRIER_TOTAL_LENGTH = OFFSET_TO_PATCHABLE_DATA_INSTRUCTION + 6 + 6 + 2; // cfi(6) + larl(6) + bcr(2)
+
+ // first 2 bytes are for cfi instruction opcode and next 4 bytes will be the value/data to be patched,
+ // so we are skipping first 2 bytes and returning the address of value/data field
+ static const int OFFSET_TO_PATCHABLE_DATA = 6 + 6 + 6 + 2; // iihf(6) + iilf(6) + lg(6) + CFI_OPCODE(2)
+
};
#ifdef COMPILER2
diff --git a/src/hotspot/cpu/s390/gc/shared/barrierSetNMethod_s390.cpp b/src/hotspot/cpu/s390/gc/shared/barrierSetNMethod_s390.cpp
index 1a609ad8d45..f6bf137da1c 100644
--- a/src/hotspot/cpu/s390/gc/shared/barrierSetNMethod_s390.cpp
+++ b/src/hotspot/cpu/s390/gc/shared/barrierSetNMethod_s390.cpp
@@ -26,26 +26,32 @@
#include "code/codeBlob.hpp"
#include "code/nativeInst.hpp"
#include "code/nmethod.hpp"
+#include "gc/shared/barrierSetAssembler.hpp"
#include "gc/shared/barrierSetNMethod.hpp"
#include "utilities/debug.hpp"
class NativeMethodBarrier: public NativeInstruction {
private:
- static const int PATCHABLE_INSTRUCTION_OFFSET = 3*6; // bytes
address get_barrier_start_address() const {
return NativeInstruction::addr_at(0);
}
address get_patchable_data_address() const {
- address inst_addr = get_barrier_start_address() + PATCHABLE_INSTRUCTION_OFFSET;
+ address start_address = get_barrier_start_address();
+#ifdef ASSERT
+ address inst_addr = start_address + BarrierSetAssembler::OFFSET_TO_PATCHABLE_DATA_INSTRUCTION;
- DEBUG_ONLY(Assembler::is_z_cfi(*((long*)inst_addr)));
- return inst_addr + 2;
+ unsigned long instr = 0;
+ Assembler::get_instruction(inst_addr, &instr);
+ assert(Assembler::is_z_cfi(instr), "sanity check");
+#endif // ASSERT
+
+ return start_address + BarrierSetAssembler::OFFSET_TO_PATCHABLE_DATA;
}
public:
- static const int BARRIER_TOTAL_LENGTH = PATCHABLE_INSTRUCTION_OFFSET + 2*6 + 2; // bytes
+ static const int BARRIER_TOTAL_LENGTH = BarrierSetAssembler::BARRIER_TOTAL_LENGTH;
int get_guard_value() const {
address data_addr = get_patchable_data_address();
@@ -64,12 +70,12 @@ class NativeMethodBarrier: public NativeInstruction {
assert((value & ~bit_mask) == 0, "trying to set bits outside the mask");
value &= bit_mask;
int32_t* data_addr = (int32_t*)get_patchable_data_address();
- int old_value = Atomic::load(data_addr);
+ int old_value = AtomicAccess::load(data_addr);
while (true) {
// Only bits in the mask are changed
int new_value = value | (old_value & ~bit_mask);
if (new_value == old_value) break;
- int v = Atomic::cmpxchg(data_addr, old_value, new_value, memory_order_release);
+ int v = AtomicAccess::cmpxchg(data_addr, old_value, new_value, memory_order_release);
if (v == old_value) break;
old_value = v;
}
@@ -77,23 +83,30 @@ class NativeMethodBarrier: public NativeInstruction {
#ifdef ASSERT
void verify() const {
+ unsigned long instr = 0;
int offset = 0; // bytes
const address start = get_barrier_start_address();
- MacroAssembler::is_load_const(/* address */ start + offset); // two instructions
+ assert(MacroAssembler::is_load_const(/* address */ start + offset), "sanity check"); // two instructions
offset += Assembler::instr_len(&start[offset]);
offset += Assembler::instr_len(&start[offset]);
- Assembler::is_z_lg(*((long*)(start + offset)));
+ Assembler::get_instruction(start + offset, &instr);
+ assert(Assembler::is_z_lg(instr), "sanity check");
offset += Assembler::instr_len(&start[offset]);
- Assembler::is_z_cfi(*((long*)(start + offset)));
+ // it will be assignment operation, So it doesn't matter what value is already present in instr
+ // hence, no need to 0 it out.
+ Assembler::get_instruction(start + offset, &instr);
+ assert(Assembler::is_z_cfi(instr), "sanity check");
offset += Assembler::instr_len(&start[offset]);
- Assembler::is_z_larl(*((long*)(start + offset)));
+ Assembler::get_instruction(start + offset, &instr);
+ assert(Assembler::is_z_larl(instr), "sanity check");
offset += Assembler::instr_len(&start[offset]);
- Assembler::is_z_bcr(*((long*)(start + offset)));
+ Assembler::get_instruction(start + offset, &instr);
+ assert(Assembler::is_z_bcr(instr), "sanity check");
offset += Assembler::instr_len(&start[offset]);
assert(offset == BARRIER_TOTAL_LENGTH, "check offset == barrier length constant");
diff --git a/src/hotspot/cpu/s390/s390.ad b/src/hotspot/cpu/s390/s390.ad
index 4c6b0d96e6f..cfc8b19534b 100644
--- a/src/hotspot/cpu/s390/s390.ad
+++ b/src/hotspot/cpu/s390/s390.ad
@@ -1947,6 +1947,7 @@ bool Matcher::is_spillable_arg(int reg) {
uint Matcher::int_pressure_limit()
{
// Medium size register set, 6 special purpose regs, 3 SOE regs.
+ // 10 prevents spill-split-recycle sanity check in JVM2008.xml.transform.
return (INTPRESSURE == -1) ? 10 : INTPRESSURE;
}
@@ -1979,11 +1980,6 @@ RegMask Matcher::modL_proj_mask() {
return _Z_RARG3_LONG_REG_mask;
}
-// Copied from sparc.
-const RegMask Matcher::method_handle_invoke_SP_save_mask() {
- return RegMask();
-}
-
// Should the matcher clone input 'm' of node 'n'?
bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
if (is_encode_and_store_pattern(n, m)) {
diff --git a/src/hotspot/cpu/s390/sharedRuntime_s390.cpp b/src/hotspot/cpu/s390/sharedRuntime_s390.cpp
index 3b1d06cf560..db1ee8351ac 100644
--- a/src/hotspot/cpu/s390/sharedRuntime_s390.cpp
+++ b/src/hotspot/cpu/s390/sharedRuntime_s390.cpp
@@ -2347,13 +2347,11 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
int comp_args_on_stack,
const BasicType *sig_bt,
const VMRegPair *regs,
- AdapterHandlerEntry* handler) {
+ address entry_address[AdapterBlob::ENTRY_COUNT]) {
__ align(CodeEntryAlignment);
- address i2c_entry = __ pc();
+ entry_address[AdapterBlob::I2C] = __ pc();
gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs);
- address c2i_unverified_entry;
-
Label skip_fixup;
{
Label ic_miss;
@@ -2363,7 +2361,7 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
// Unverified Entry Point UEP
__ align(CodeEntryAlignment);
- c2i_unverified_entry = __ pc();
+ entry_address[AdapterBlob::C2I_Unverified] = __ pc();
__ ic_check(2);
__ z_lg(Z_method, Address(Z_inline_cache, CompiledICData::speculated_method_offset()));
@@ -2376,10 +2374,10 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
// Fallthru to VEP. Duplicate LTG, but saved taken branch.
}
- address c2i_entry = __ pc();
+ entry_address[AdapterBlob::C2I] = __ pc();
// Class initialization barrier for static methods
- address c2i_no_clinit_check_entry = nullptr;
+ entry_address[AdapterBlob::C2I_No_Clinit_Check] = nullptr;
if (VM_Version::supports_fast_class_init_checks()) {
Label L_skip_barrier;
@@ -2396,12 +2394,10 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
__ z_br(klass);
__ bind(L_skip_barrier);
- c2i_no_clinit_check_entry = __ pc();
+ entry_address[AdapterBlob::C2I_No_Clinit_Check] = __ pc();
}
gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup);
-
- handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry);
return;
}
diff --git a/src/hotspot/cpu/s390/templateTable_s390.cpp b/src/hotspot/cpu/s390/templateTable_s390.cpp
index 2b39cc8318c..4e8fdf275e4 100644
--- a/src/hotspot/cpu/s390/templateTable_s390.cpp
+++ b/src/hotspot/cpu/s390/templateTable_s390.cpp
@@ -2360,7 +2360,7 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no,
assert_different_registers(Rcache, index);
assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
- Label resolved, clinit_barrier_slow;
+ Label L_clinit_barrier_slow, L_done;
Bytecodes::Code code = bytecode();
switch (code) {
@@ -2375,27 +2375,30 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no,
__ load_method_entry(Rcache, index);
__ z_cli(Address(Rcache, bc_offset), code);
- __ z_bre(resolved);
+
+ // Class initialization barrier for static methods
+ if (VM_Version::supports_fast_class_init_checks() && bytecode() == Bytecodes::_invokestatic) {
+ const Register method = Z_R1_scratch;
+ const Register klass = Z_R1_scratch;
+ __ z_brne(L_clinit_barrier_slow);
+ __ z_lg(method, Address(Rcache, in_bytes(ResolvedMethodEntry::method_offset())));
+ __ load_method_holder(klass, method);
+ __ clinit_barrier(klass, Z_thread, &L_done, /*L_slow_path*/ nullptr);
+ __ bind(L_clinit_barrier_slow);
+ } else {
+ __ z_bre(L_done);
+ }
// Resolve first time through
// Class initialization barrier slow path lands here as well.
- __ bind(clinit_barrier_slow);
address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
__ load_const_optimized(Z_ARG2, (int)code);
__ call_VM(noreg, entry, Z_ARG2);
// Update registers with resolved info.
__ load_method_entry(Rcache, index);
- __ bind(resolved);
- // Class initialization barrier for static methods
- if (VM_Version::supports_fast_class_init_checks() && bytecode() == Bytecodes::_invokestatic) {
- const Register method = Z_R1_scratch;
- const Register klass = Z_R1_scratch;
- __ z_lg(method, Address(Rcache, in_bytes(ResolvedMethodEntry::method_offset())));
- __ load_method_holder(klass, method);
- __ clinit_barrier(klass, Z_thread, nullptr /*L_fast_path*/, &clinit_barrier_slow);
- }
+ __ bind(L_done);
BLOCK_COMMENT("} resolve_cache_and_index_for_method");
}
@@ -2408,7 +2411,7 @@ void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
assert_different_registers(cache, index);
assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
- NearLabel resolved;
+ Label L_clinit_barrier_slow, L_done;
Bytecodes::Code code = bytecode();
switch (code) {
@@ -2422,9 +2425,22 @@ void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
in_bytes(ResolvedFieldEntry::put_code_offset()) ;
__ z_cli(Address(cache, code_offset), code);
- __ z_bre(resolved);
+
+ // Class initialization barrier for static fields
+ if (VM_Version::supports_fast_class_init_checks() &&
+ (bytecode() == Bytecodes::_getstatic || bytecode() == Bytecodes::_putstatic)) {
+ const Register field_holder = index;
+
+ __ z_brne(L_clinit_barrier_slow);
+ __ load_sized_value(field_holder, Address(cache, ResolvedFieldEntry::field_holder_offset()), sizeof(void*), false);
+ __ clinit_barrier(field_holder, Z_thread, &L_done, /*L_slow_path*/ nullptr);
+ __ bind(L_clinit_barrier_slow);
+ } else {
+ __ z_bre(L_done);
+ }
// resolve first time through
+ // Class initialization barrier slow path lands here as well.
address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
__ load_const_optimized(Z_ARG2, (int)code);
__ call_VM(noreg, entry, Z_ARG2);
@@ -2432,7 +2448,7 @@ void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
// Update registers with resolved info.
__ load_field_entry(cache, index);
- __ bind(resolved);
+ __ bind(L_done);
BLOCK_COMMENT("} resolve_cache_and_index_for_field");
}
diff --git a/src/hotspot/cpu/x86/assembler_x86.cpp b/src/hotspot/cpu/x86/assembler_x86.cpp
index d1b6897f287..fd62e9358bf 100644
--- a/src/hotspot/cpu/x86/assembler_x86.cpp
+++ b/src/hotspot/cpu/x86/assembler_x86.cpp
@@ -1398,11 +1398,7 @@ void Assembler::addl(Address dst, Register src) {
void Assembler::eaddl(Register dst, Address src1, Register src2, bool no_flags) {
InstructionMark im(this);
- InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
- eevex_prefix_ndd(src1, dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
- emit_int8(0x01);
- emit_operand(src2, src1, 0);
+ emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x01, no_flags, false /* is_map1 */, true /* is_commutative */);
}
void Assembler::addl(Register dst, int32_t imm32) {
@@ -1432,11 +1428,7 @@ void Assembler::addl(Register dst, Register src) {
}
void Assembler::eaddl(Register dst, Register src1, Register src2, bool no_flags) {
- InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- // NDD shares its encoding bits with NDS bits for regular EVEX instruction.
- // Therefore, DST is passed as the second argument to minimize changes in the leaf level routine.
- (void)emit_eevex_prefix_or_demote_ndd(src1->encoding(), dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
- emit_arith(0x03, 0xC0, src1, src2);
+ emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x03, 0xC0, no_flags, true /* is_commutative */);
}
void Assembler::addr_nop_4() {
@@ -1657,17 +1649,18 @@ void Assembler::eandl(Register dst, Register src1, Address src2, bool no_flags)
emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x23, no_flags);
}
+void Assembler::eandl(Register dst, Address src1, Register src2, bool no_flags) {
+ InstructionMark im(this);
+ emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x21, no_flags, false /* is_map1 */, true /* is_commutative */);
+}
+
void Assembler::andl(Register dst, Register src) {
(void) prefix_and_encode(dst->encoding(), src->encoding());
emit_arith(0x23, 0xC0, dst, src);
}
void Assembler::eandl(Register dst, Register src1, Register src2, bool no_flags) {
- InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- // NDD shares its encoding bits with NDS bits for regular EVEX instruction.
- // Therefore, DST is passed as the second argument to minimize changes in the leaf level routine.
- (void) emit_eevex_prefix_or_demote_ndd(src1->encoding(), dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
- emit_arith(0x23, 0xC0, src1, src2);
+ emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x23, 0xC0, no_flags, true /* is_commutative */);
}
void Assembler::andnl(Register dst, Register src1, Register src2) {
@@ -2232,6 +2225,44 @@ void Assembler::cvttss2sil(Register dst, XMMRegister src) {
emit_int16(0x2C, (0xC0 | encode));
}
+void Assembler::evcvttss2sisl(Register dst, XMMRegister src) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ attributes.set_is_evex_instruction();
+ int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
+ emit_int16(0x6D, (0xC0 | encode));
+}
+
+void Assembler::evcvttss2sisl(Register dst, Address src) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
+ attributes.set_is_evex_instruction();
+ vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
+ emit_int8((unsigned char)0x6D);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::evcvttss2sisq(Register dst, XMMRegister src) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ attributes.set_is_evex_instruction();
+ int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
+ emit_int16(0x6D, (0xC0 | encode));
+}
+
+void Assembler::evcvttss2sisq(Register dst, Address src) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
+ attributes.set_is_evex_instruction();
+ vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_MAP5, &attributes);
+ emit_int8((unsigned char)0x6D);
+ emit_operand(dst, src, 0);
+}
+
void Assembler::cvttpd2dq(XMMRegister dst, XMMRegister src) {
int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
@@ -2317,6 +2348,25 @@ void Assembler::vcvttps2dq(XMMRegister dst, XMMRegister src, int vector_len) {
emit_int16(0x5B, (0xC0 | encode));
}
+void Assembler::evcvttps2dqs(XMMRegister dst, XMMRegister src, int vector_len) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes);
+ emit_int16(0x6D, (0xC0 | encode));
+}
+
+void Assembler::evcvttps2dqs(XMMRegister dst, Address src, int vector_len) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
+ attributes.set_is_evex_instruction();
+ vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes);
+ emit_int8((unsigned char)0x6D);
+ emit_operand(dst, src, 0);
+}
+
void Assembler::vcvttpd2dq(XMMRegister dst, XMMRegister src, int vector_len) {
assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
@@ -2324,6 +2374,25 @@ void Assembler::vcvttpd2dq(XMMRegister dst, XMMRegister src, int vector_len) {
emit_int16((unsigned char)0xE6, (0xC0 | encode));
}
+void Assembler::evcvttpd2dqs(XMMRegister dst, XMMRegister src, int vector_len) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes);
+ emit_int16(0x6D, (0xC0 | encode));
+}
+
+void Assembler::evcvttpd2dqs(XMMRegister dst, Address src, int vector_len) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
+ attributes.set_is_evex_instruction();
+ vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_MAP5, &attributes);
+ emit_int8((unsigned char)0x6D);
+ emit_operand(dst, src, 0);
+}
+
void Assembler::vcvtps2dq(XMMRegister dst, XMMRegister src, int vector_len) {
assert(vector_len <= AVX_256bit ? VM_Version::supports_avx() : VM_Version::supports_evex(), "");
InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
@@ -2339,6 +2408,25 @@ void Assembler::evcvttps2qq(XMMRegister dst, XMMRegister src, int vector_len) {
emit_int16(0x7A, (0xC0 | encode));
}
+void Assembler::evcvttps2qqs(XMMRegister dst, XMMRegister src, int vector_len) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_MAP5, &attributes);
+ emit_int16(0x6D, (0xC0 | encode));
+}
+
+void Assembler::evcvttps2qqs(XMMRegister dst, Address src, int vector_len) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_HV, /* input_size_in_bits */ EVEX_32bit);
+ attributes.set_is_evex_instruction();
+ vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_MAP5, &attributes);
+ emit_int8((unsigned char)0x6D);
+ emit_operand(dst, src, 0);
+}
+
void Assembler::evcvtpd2qq(XMMRegister dst, XMMRegister src, int vector_len) {
assert(VM_Version::supports_avx512dq(), "");
InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
@@ -2363,6 +2451,25 @@ void Assembler::evcvttpd2qq(XMMRegister dst, XMMRegister src, int vector_len) {
emit_int16(0x7A, (0xC0 | encode));
}
+void Assembler::evcvttpd2qqs(XMMRegister dst, XMMRegister src, int vector_len) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_is_evex_instruction();
+ int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_MAP5, &attributes);
+ emit_int16(0x6D, (0xC0 | encode));
+}
+
+void Assembler::evcvttpd2qqs(XMMRegister dst, Address src, int vector_len) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
+ attributes.set_is_evex_instruction();
+ vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_MAP5, &attributes);
+ emit_int8((unsigned char)0x6D);
+ emit_operand(dst, src, 0);
+}
+
void Assembler::evcvtqq2pd(XMMRegister dst, XMMRegister src, int vector_len) {
assert(VM_Version::supports_avx512dq(), "");
InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
@@ -2519,7 +2626,7 @@ void Assembler::imull(Register dst, Register src) {
}
void Assembler::eimull(Register dst, Register src1, Register src2, bool no_flags) {
- emit_eevex_or_demote(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0xAF, no_flags, true /* is_map1 */, true /* swap */);
+ emit_eevex_or_demote(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0xAF, no_flags, true /* is_map1 */, true /* swap */, true /* is_commutative */);
}
void Assembler::imull(Register dst, Address src, int32_t value) {
@@ -4419,11 +4526,7 @@ void Assembler::enotl(Register dst, Register src) {
}
void Assembler::eorw(Register dst, Register src1, Register src2, bool no_flags) {
- InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- // NDD shares its encoding bits with NDS bits for regular EVEX instruction.
- // Therefore, DST is passed as the second argument to minimize changes in the leaf level routine.
- (void) emit_eevex_prefix_or_demote_ndd(src1->encoding(), dst->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
- emit_arith(0x0B, 0xC0, src1, src2);
+ emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_66, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_16bit, 0x0B, 0xC0, no_flags, true /* is_commutative */);
}
void Assembler::orl(Address dst, int32_t imm32) {
@@ -4467,11 +4570,7 @@ void Assembler::orl(Register dst, Register src) {
}
void Assembler::eorl(Register dst, Register src1, Register src2, bool no_flags) {
- InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- // NDD shares its encoding bits with NDS bits for regular EVEX instruction.
- // Therefore, DST is passed as the second argument to minimize changes in the leaf level routine.
- (void) emit_eevex_prefix_or_demote_ndd(src1->encoding(), dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
- emit_arith(0x0B, 0xC0, src1, src2);
+ emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x0B, 0xC0, no_flags, true /* is_commutative */);
}
void Assembler::orl(Address dst, Register src) {
@@ -4483,11 +4582,7 @@ void Assembler::orl(Address dst, Register src) {
void Assembler::eorl(Register dst, Address src1, Register src2, bool no_flags) {
InstructionMark im(this);
- InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
- eevex_prefix_ndd(src1, dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
- emit_int8(0x09);
- emit_operand(src2, src1, 0);
+ emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x09, no_flags, false /* is_map1 */, true /* is_commutative */);
}
void Assembler::orb(Address dst, int imm8) {
@@ -4517,11 +4612,7 @@ void Assembler::orb(Address dst, Register src) {
void Assembler::eorb(Register dst, Address src1, Register src2, bool no_flags) {
InstructionMark im(this);
- InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_8bit);
- eevex_prefix_ndd(src1, dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
- emit_int8(0x08);
- emit_operand(src2, src1, 0);
+ emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_8bit, 0x08, no_flags, false /* is_map1 */, true /* is_commutative */);
}
void Assembler::packsswb(XMMRegister dst, XMMRegister src) {
@@ -7323,11 +7414,7 @@ void Assembler::xorl(Register dst, Register src) {
}
void Assembler::exorl(Register dst, Register src1, Register src2, bool no_flags) {
- InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- // NDD shares its encoding bits with NDS bits for regular EVEX instruction.
- // Therefore, DST is passed as the second argument to minimize changes in the leaf level routine.
- (void) emit_eevex_prefix_or_demote_ndd(src1->encoding(), dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
- emit_arith(0x33, 0xC0, src1, src2);
+ emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x33, 0xC0, no_flags, true /* is_commutative */);
}
void Assembler::xorl(Address dst, Register src) {
@@ -7339,11 +7426,7 @@ void Assembler::xorl(Address dst, Register src) {
void Assembler::exorl(Register dst, Address src1, Register src2, bool no_flags) {
InstructionMark im(this);
- InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
- eevex_prefix_ndd(src1, dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
- emit_int8(0x31);
- emit_operand(src2, src1, 0);
+ emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x31, no_flags, false /* is_map1 */, true /* is_commutative */);
}
void Assembler::xorb(Register dst, Address src) {
@@ -7367,11 +7450,7 @@ void Assembler::xorb(Address dst, Register src) {
void Assembler::exorb(Register dst, Address src1, Register src2, bool no_flags) {
InstructionMark im(this);
- InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_8bit);
- eevex_prefix_ndd(src1, dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
- emit_int8(0x30);
- emit_operand(src2, src1, 0);
+ emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_8bit, 0x30, no_flags, false /* is_map1 */, true /* is_commutative */);
}
void Assembler::xorw(Register dst, Address src) {
@@ -12955,6 +13034,31 @@ void Assembler::eevex_prefix_ndd(Address adr, int ndd_enc, int xreg_enc, VexSimd
vex_prefix(adr, ndd_enc, xreg_enc, pre, opc, attributes, /* nds_is_ndd */ true, no_flags);
}
+void Assembler::emit_eevex_or_demote(Register dst, Address src1, Register src2, VexSimdPrefix pre, VexOpcode opc,
+ int size, int opcode_byte, bool no_flags, bool is_map1, bool is_commutative) {
+ if (is_commutative && is_demotable(no_flags, dst->encoding(), src2->encoding())) {
+ // Opcode byte adjustment due to mismatch between NDD and equivalent demotable variant
+ opcode_byte += 2;
+ if (size == EVEX_64bit) {
+ emit_prefix_and_int8(get_prefixq(src1, dst, is_map1), opcode_byte);
+ } else {
+ // For 32-bit, 16-bit and 8-bit
+ if (size == EVEX_16bit) {
+ emit_int8(0x66);
+ }
+ prefix(src1, dst, false, is_map1);
+ emit_int8(opcode_byte);
+ }
+ } else {
+ bool vex_w = (size == EVEX_64bit) ? true : false;
+ InstructionAttr attributes(AVX_128bit, vex_w, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, size);
+ eevex_prefix_ndd(src1, dst->encoding(), src2->encoding(), pre, opc, &attributes, no_flags);
+ emit_int8(opcode_byte);
+ }
+ emit_operand(src2, src1, 0);
+}
+
void Assembler::emit_eevex_or_demote(Register dst, Register src1, Address src2, VexSimdPrefix pre, VexOpcode opc,
int size, int opcode_byte, bool no_flags, bool is_map1) {
if (is_demotable(no_flags, dst->encoding(), src1->encoding())) {
@@ -13055,18 +13159,20 @@ void Assembler::emit_eevex_or_demote(int dst_enc, int nds_enc, int src_enc, int8
}
void Assembler::emit_eevex_or_demote(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc,
- int size, int opcode_byte, bool no_flags, bool is_map1, bool swap) {
+ int size, int opcode_byte, bool no_flags, bool is_map1, bool swap, bool is_commutative) {
int encode;
bool is_prefixq = (size == EVEX_64bit) ? true : false;
- if (is_demotable(no_flags, dst_enc, nds_enc)) {
+ bool first_operand_demotable = is_demotable(no_flags, dst_enc, nds_enc);
+ bool second_operand_demotable = is_commutative && is_demotable(no_flags, dst_enc, src_enc);
+ if (first_operand_demotable || second_operand_demotable) {
if (size == EVEX_16bit) {
emit_int8(0x66);
}
-
+ int src = first_operand_demotable ? src_enc : nds_enc;
if (swap) {
- encode = is_prefixq ? prefixq_and_encode(dst_enc, src_enc, is_map1) : prefix_and_encode(dst_enc, src_enc, is_map1);
+ encode = is_prefixq ? prefixq_and_encode(dst_enc, src, is_map1) : prefix_and_encode(dst_enc, src, is_map1);
} else {
- encode = is_prefixq ? prefixq_and_encode(src_enc, dst_enc, is_map1) : prefix_and_encode(src_enc, dst_enc, is_map1);
+ encode = is_prefixq ? prefixq_and_encode(src, dst_enc, is_map1) : prefix_and_encode(src, dst_enc, is_map1);
}
emit_opcode_prefix_and_encoding((unsigned char)opcode_byte, 0xC0, encode);
} else {
@@ -13114,6 +13220,26 @@ int Assembler::eevex_prefix_and_encode_nf(int dst_enc, int nds_enc, int src_enc,
return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes, /* src_is_gpr */ true, /* nds_is_ndd */ false, no_flags);
}
+void Assembler::emit_eevex_prefix_or_demote_arith_ndd(Register dst, Register src1, Register src2, VexSimdPrefix pre, VexOpcode opc,
+ int size, int op1, int op2, bool no_flags, bool is_commutative) {
+ bool demotable = is_demotable(no_flags, dst->encoding(), src1->encoding());
+ if (!demotable && is_commutative) {
+ if (is_demotable(no_flags, dst->encoding(), src2->encoding())) {
+ // swap src1 and src2
+ Register tmp = src1;
+ src1 = src2;
+ src2 = tmp;
+ }
+ }
+ bool vex_w = (size == EVEX_64bit) ? true : false;
+ bool use_prefixq = vex_w;
+ InstructionAttr attributes(AVX_128bit, vex_w, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ // NDD shares its encoding bits with NDS bits for regular EVEX instruction.
+ // Therefore, DST is passed as the second argument to minimize changes in the leaf level routine.
+ (void)emit_eevex_prefix_or_demote_ndd(src1->encoding(), dst->encoding(), src2->encoding(), pre, opc, &attributes, no_flags, use_prefixq);
+ emit_arith(op1, op2, src1, src2);
+}
+
void Assembler::emit_eevex_prefix_or_demote_arith_ndd(Register dst, Register nds, int32_t imm32, VexSimdPrefix pre, VexOpcode opc,
int size, int op1, int op2, bool no_flags) {
int dst_enc = dst->encoding();
@@ -13124,7 +13250,6 @@ void Assembler::emit_eevex_prefix_or_demote_arith_ndd(Register dst, Register nds
} else {
bool vex_w = (size == EVEX_64bit) ? true : false;
InstructionAttr attributes(AVX_128bit, vex_w, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- //attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, size);
attributes.set_is_evex_instruction();
vex_prefix_and_encode(0, dst_enc, nds_enc, pre, opc, &attributes, /* src_is_gpr */ true, /* nds_is_ndd */ true, no_flags);
@@ -13769,7 +13894,7 @@ void Assembler::pdepq(Register dst, Register src1, Address src2) {
void Assembler::sarxl(Register dst, Register src1, Register src2) {
assert(VM_Version::supports_bmi2(), "");
- InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes, true);
emit_int16((unsigned char)0xF7, (0xC0 | encode));
}
@@ -13777,7 +13902,7 @@ void Assembler::sarxl(Register dst, Register src1, Register src2) {
void Assembler::sarxl(Register dst, Address src1, Register src2) {
assert(VM_Version::supports_bmi2(), "");
InstructionMark im(this);
- InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
emit_int8((unsigned char)0xF7);
@@ -13786,7 +13911,7 @@ void Assembler::sarxl(Register dst, Address src1, Register src2) {
void Assembler::sarxq(Register dst, Register src1, Register src2) {
assert(VM_Version::supports_bmi2(), "");
- InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes, true);
emit_int16((unsigned char)0xF7, (0xC0 | encode));
}
@@ -13794,7 +13919,7 @@ void Assembler::sarxq(Register dst, Register src1, Register src2) {
void Assembler::sarxq(Register dst, Address src1, Register src2) {
assert(VM_Version::supports_bmi2(), "");
InstructionMark im(this);
- InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
emit_int8((unsigned char)0xF7);
@@ -13803,7 +13928,7 @@ void Assembler::sarxq(Register dst, Address src1, Register src2) {
void Assembler::shlxl(Register dst, Register src1, Register src2) {
assert(VM_Version::supports_bmi2(), "");
- InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes, true);
emit_int16((unsigned char)0xF7, (0xC0 | encode));
}
@@ -13811,7 +13936,7 @@ void Assembler::shlxl(Register dst, Register src1, Register src2) {
void Assembler::shlxl(Register dst, Address src1, Register src2) {
assert(VM_Version::supports_bmi2(), "");
InstructionMark im(this);
- InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
emit_int8((unsigned char)0xF7);
@@ -13820,7 +13945,7 @@ void Assembler::shlxl(Register dst, Address src1, Register src2) {
void Assembler::shlxq(Register dst, Register src1, Register src2) {
assert(VM_Version::supports_bmi2(), "");
- InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes, true);
emit_int16((unsigned char)0xF7, (0xC0 | encode));
}
@@ -13828,7 +13953,7 @@ void Assembler::shlxq(Register dst, Register src1, Register src2) {
void Assembler::shlxq(Register dst, Address src1, Register src2) {
assert(VM_Version::supports_bmi2(), "");
InstructionMark im(this);
- InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
emit_int8((unsigned char)0xF7);
@@ -13837,7 +13962,7 @@ void Assembler::shlxq(Register dst, Address src1, Register src2) {
void Assembler::shrxl(Register dst, Register src1, Register src2) {
assert(VM_Version::supports_bmi2(), "");
- InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes, true);
emit_int16((unsigned char)0xF7, (0xC0 | encode));
}
@@ -13845,7 +13970,7 @@ void Assembler::shrxl(Register dst, Register src1, Register src2) {
void Assembler::shrxl(Register dst, Address src1, Register src2) {
assert(VM_Version::supports_bmi2(), "");
InstructionMark im(this);
- InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
emit_int8((unsigned char)0xF7);
@@ -13854,7 +13979,7 @@ void Assembler::shrxl(Register dst, Address src1, Register src2) {
void Assembler::shrxq(Register dst, Register src1, Register src2) {
assert(VM_Version::supports_bmi2(), "");
- InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes, true);
emit_int16((unsigned char)0xF7, (0xC0 | encode));
}
@@ -13862,7 +13987,7 @@ void Assembler::shrxq(Register dst, Register src1, Register src2) {
void Assembler::shrxq(Register dst, Address src1, Register src2) {
assert(VM_Version::supports_bmi2(), "");
InstructionMark im(this);
- InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+ InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
emit_int8((unsigned char)0xF7);
@@ -14623,11 +14748,7 @@ void Assembler::addq(Address dst, Register src) {
void Assembler::eaddq(Register dst, Address src1, Register src2, bool no_flags) {
InstructionMark im(this);
- InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
- eevex_prefix_ndd(src1, dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
- emit_int8(0x01);
- emit_operand(src2, src1, 0);
+ emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x01, no_flags, false /* is_map1 */, true /* is_commutative */);
}
void Assembler::addq(Register dst, int32_t imm32) {
@@ -14656,11 +14777,7 @@ void Assembler::addq(Register dst, Register src) {
}
void Assembler::eaddq(Register dst, Register src1, Register src2, bool no_flags) {
- InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- // NDD shares its encoding bits with NDS bits for regular EVEX instruction.
- // Therefore, DST is passed as the second argument to minimize changes in the leaf level routine.
- (void) emit_eevex_prefix_or_demote_ndd(src1->encoding(), dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */);
- emit_arith(0x03, 0xC0, src1, src2);
+ emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x03, 0xC0, no_flags, true /* is_commutative */);
}
void Assembler::adcxq(Register dst, Register src) {
@@ -14753,11 +14870,7 @@ void Assembler::andq(Register dst, Register src) {
}
void Assembler::eandq(Register dst, Register src1, Register src2, bool no_flags) {
- InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- // NDD shares its encoding bits with NDS bits for regular EVEX instruction.
- // Therefore, DST is passed as the second argument to minimize changes in the leaf level routine.
- (void) emit_eevex_prefix_or_demote_ndd(src1->encoding(), dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */);
- emit_arith(0x23, 0xC0, src1, src2);
+ emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x23, 0xC0, no_flags, true /* is_commutative */);
}
void Assembler::andq(Address dst, Register src) {
@@ -14768,11 +14881,7 @@ void Assembler::andq(Address dst, Register src) {
void Assembler::eandq(Register dst, Address src1, Register src2, bool no_flags) {
InstructionMark im(this);
- InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
- eevex_prefix_ndd(src1, dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
- emit_int8(0x21);
- emit_operand(src2, src1, 0);
+ emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x21, no_flags, false /* is_map1 */, true /* is_commutative */);
}
void Assembler::andnq(Register dst, Register src1, Register src2) {
@@ -14993,6 +15102,44 @@ void Assembler::cvttsd2siq(Register dst, Address src) {
emit_operand(dst, src, 0);
}
+void Assembler::evcvttsd2sisl(Register dst, XMMRegister src) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ attributes.set_is_evex_instruction();
+ int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_MAP5, &attributes);
+ emit_int16(0x6D, (0xC0 | encode));
+}
+
+void Assembler::evcvttsd2sisl(Register dst, Address src) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
+ attributes.set_is_evex_instruction();
+ vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_MAP5, &attributes);
+ emit_int8((unsigned char)0x6D);
+ emit_operand(dst, src, 0);
+}
+
+void Assembler::evcvttsd2sisq(Register dst, XMMRegister src) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ attributes.set_is_evex_instruction();
+ int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_MAP5, &attributes);
+ emit_int16(0x6D, (0xC0 | encode));
+}
+
+void Assembler::evcvttsd2sisq(Register dst, Address src) {
+ assert(VM_Version::supports_avx10_2(), "");
+ InstructionMark im(this);
+ InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+ attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
+ attributes.set_is_evex_instruction();
+ vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_MAP5, &attributes);
+ emit_int8((unsigned char)0x6D);
+ emit_operand(dst, src, 0);
+}
+
void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
@@ -15118,7 +15265,7 @@ void Assembler::eimulq(Register dst, Register src, bool no_flags) {
}
void Assembler::eimulq(Register dst, Register src1, Register src2, bool no_flags) {
- emit_eevex_or_demote(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0xAF, no_flags, true /* is_map1 */, true /* swap */);
+ emit_eevex_or_demote(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0xAF, no_flags, true /* is_map1 */, true /* swap */, true /* is_commutative */);
}
void Assembler::imulq(Register src) {
@@ -15580,11 +15727,7 @@ void Assembler::orq(Address dst, Register src) {
void Assembler::eorq(Register dst, Address src1, Register src2, bool no_flags) {
InstructionMark im(this);
- InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
- eevex_prefix_ndd(src1, dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
- emit_int8(0x09);
- emit_operand(src2, src1, 0);
+ emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x09, no_flags, false /* is_map1 */, true /* is_commutative */);
}
void Assembler::orq(Register dst, int32_t imm32) {
@@ -15624,13 +15767,8 @@ void Assembler::orq(Register dst, Register src) {
}
void Assembler::eorq(Register dst, Register src1, Register src2, bool no_flags) {
- InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- // NDD shares its encoding bits with NDS bits for regular EVEX instruction.
- // Therefore, DST is passed as the second argument to minimize changes in the leaf level routine.
- (void) emit_eevex_prefix_or_demote_ndd(src1->encoding(), dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */);
- emit_arith(0x0B, 0xC0, src1, src2);
+ emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x0B, 0xC0, no_flags, true /* is_commutative */);
}
-
void Assembler::popcntq(Register dst, Address src) {
assert(VM_Version::supports_popcnt(), "must support");
InstructionMark im(this);
@@ -16372,11 +16510,7 @@ void Assembler::xorq(Register dst, Register src) {
}
void Assembler::exorq(Register dst, Register src1, Register src2, bool no_flags) {
- InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- // NDD shares its encoding bits with NDS bits for regular EVEX instruction.
- // Therefore, DST is passed as the second argument to minimize changes in the leaf level routine.
- (void) emit_eevex_prefix_or_demote_ndd(src1->encoding(), dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */);
- emit_arith(0x33, 0xC0, src1, src2);
+ emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x33, 0xC0, no_flags, true /* is_commutative */);
}
void Assembler::xorq(Register dst, Address src) {
@@ -16430,11 +16564,7 @@ void Assembler::esetzucc(Condition cc, Register dst) {
void Assembler::exorq(Register dst, Address src1, Register src2, bool no_flags) {
InstructionMark im(this);
- InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
- attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
- eevex_prefix_ndd(src1, dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
- emit_int8(0x31);
- emit_operand(src2, src1, 0);
+ emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x31, no_flags, false /* is_map1 */, true /* is_commutative */);
}
void InstructionAttr::set_address_attributes(int tuple_type, int input_size_in_bits) {
diff --git a/src/hotspot/cpu/x86/assembler_x86.hpp b/src/hotspot/cpu/x86/assembler_x86.hpp
index 45c24f8c832..c863191df4c 100644
--- a/src/hotspot/cpu/x86/assembler_x86.hpp
+++ b/src/hotspot/cpu/x86/assembler_x86.hpp
@@ -807,14 +807,20 @@ private:
int emit_eevex_prefix_or_demote_ndd(int dst_enc, int nds_enc, VexSimdPrefix pre, VexOpcode opc,
InstructionAttr *attributes, bool no_flags = false, bool use_prefixq = false);
+ void emit_eevex_prefix_or_demote_arith_ndd(Register dst, Register src1, Register src2, VexSimdPrefix pre, VexOpcode opc,
+ int size, int op1, int op2, bool no_flags = false, bool is_commutative = false);
+
void emit_eevex_prefix_or_demote_arith_ndd(Register dst, Register nds, int32_t imm32, VexSimdPrefix pre, VexOpcode opc,
int size, int op1, int op2, bool no_flags);
void emit_eevex_or_demote(Register dst, Register src1, Address src2, VexSimdPrefix pre, VexOpcode opc,
int size, int opcode_byte, bool no_flags = false, bool is_map1 = false);
+ void emit_eevex_or_demote(Register dst, Address src1, Register src2, VexSimdPrefix pre, VexOpcode opc,
+ int size, int opcode_byte, bool no_flags = false, bool is_map1 = false, bool is_commutative = false);
+
void emit_eevex_or_demote(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc,
- int size, int opcode_byte, bool no_flags, bool is_map1 = false, bool swap = false);
+ int size, int opcode_byte, bool no_flags, bool is_map1 = false, bool swap = false, bool is_commutative = false);
void emit_eevex_or_demote(int dst_enc, int nds_enc, int src_enc, int8_t imm8, VexSimdPrefix pre, VexOpcode opc,
int size, int opcode_byte, bool no_flags, bool is_map1 = false);
@@ -1149,6 +1155,7 @@ private:
void eandl(Register dst, Register src, int32_t imm32, bool no_flags);
void andl(Register dst, Address src);
void eandl(Register dst, Register src1, Address src2, bool no_flags);
+ void eandl(Register dst, Address src1, Register src2, bool no_flags);
void andl(Register dst, Register src);
void eandl(Register dst, Register src1, Register src2, bool no_flags);
void andl(Address dst, Register src);
@@ -1309,11 +1316,19 @@ private:
void cvttsd2sil(Register dst, XMMRegister src);
void cvttsd2siq(Register dst, Address src);
void cvttsd2siq(Register dst, XMMRegister src);
+ void evcvttsd2sisl(Register dst, XMMRegister src);
+ void evcvttsd2sisl(Register dst, Address src);
+ void evcvttsd2sisq(Register dst, XMMRegister src);
+ void evcvttsd2sisq(Register dst, Address src);
// Convert with Truncation Scalar Single-Precision Floating-Point Value to Doubleword Integer
void cvttss2sil(Register dst, XMMRegister src);
void cvttss2siq(Register dst, XMMRegister src);
void cvtss2sil(Register dst, XMMRegister src);
+ void evcvttss2sisl(Register dst, XMMRegister src);
+ void evcvttss2sisl(Register dst, Address src);
+ void evcvttss2sisq(Register dst, XMMRegister src);
+ void evcvttss2sisq(Register dst, Address src);
// Convert vector double to int
void cvttpd2dq(XMMRegister dst, XMMRegister src);
@@ -1325,7 +1340,11 @@ private:
// Convert vector float to int/long
void vcvtps2dq(XMMRegister dst, XMMRegister src, int vector_len);
void vcvttps2dq(XMMRegister dst, XMMRegister src, int vector_len);
+ void evcvttps2dqs(XMMRegister dst, XMMRegister src, int vector_len);
+ void evcvttps2dqs(XMMRegister dst, Address src, int vector_len);
void evcvttps2qq(XMMRegister dst, XMMRegister src, int vector_len);
+ void evcvttps2qqs(XMMRegister dst, XMMRegister src, int vector_len);
+ void evcvttps2qqs(XMMRegister dst, Address src, int vector_len);
// Convert vector long to vector FP
void evcvtqq2ps(XMMRegister dst, XMMRegister src, int vector_len);
@@ -1334,9 +1353,13 @@ private:
// Convert vector double to long
void evcvtpd2qq(XMMRegister dst, XMMRegister src, int vector_len);
void evcvttpd2qq(XMMRegister dst, XMMRegister src, int vector_len);
+ void evcvttpd2qqs(XMMRegister dst, XMMRegister src, int vector_len);
+ void evcvttpd2qqs(XMMRegister dst, Address src, int vector_len);
// Convert vector double to int
void vcvttpd2dq(XMMRegister dst, XMMRegister src, int vector_len);
+ void evcvttpd2dqs(XMMRegister dst, XMMRegister src, int vector_len);
+ void evcvttpd2dqs(XMMRegister dst, Address src, int vector_len);
// Evex casts with truncation
void evpmovwb(XMMRegister dst, XMMRegister src, int vector_len);
diff --git a/src/hotspot/cpu/x86/c1_CodeStubs_x86.cpp b/src/hotspot/cpu/x86/c1_CodeStubs_x86.cpp
index 2fd067a7749..95ce48f34db 100644
--- a/src/hotspot/cpu/x86/c1_CodeStubs_x86.cpp
+++ b/src/hotspot/cpu/x86/c1_CodeStubs_x86.cpp
@@ -207,10 +207,10 @@ void MonitorEnterStub::emit_code(LIR_Assembler* ce) {
void MonitorExitStub::emit_code(LIR_Assembler* ce) {
__ bind(_entry);
- if (_compute_lock) {
- // lock_reg was destroyed by fast unlocking attempt => recompute it
- ce->monitor_address(_monitor_ix, _lock_reg);
- }
+
+ // lock_reg was destroyed by fast unlocking attempt => recompute it
+ ce->monitor_address(_monitor_ix, _lock_reg);
+
ce->store_parameter(_lock_reg->as_register(), 0);
// note: non-blocking leaf routine => no call info needed
StubId exit_id;
diff --git a/src/hotspot/cpu/x86/c1_FrameMap_x86.cpp b/src/hotspot/cpu/x86/c1_FrameMap_x86.cpp
index bdbab432180..68c9814fd20 100644
--- a/src/hotspot/cpu/x86/c1_FrameMap_x86.cpp
+++ b/src/hotspot/cpu/x86/c1_FrameMap_x86.cpp
@@ -326,13 +326,6 @@ LIR_Opr FrameMap::stack_pointer() {
return FrameMap::rsp_opr;
}
-// JSR 292
-// On x86, there is no need to save the SP, because neither
-// method handle intrinsics, nor compiled lambda forms modify it.
-LIR_Opr FrameMap::method_handle_invoke_SP_save_opr() {
- return LIR_OprFact::illegalOpr;
-}
-
bool FrameMap::validate_frame() {
return true;
}
diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp
index 98759295bb1..edeb0baea0e 100644
--- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp
@@ -412,7 +412,7 @@ int LIR_Assembler::emit_unwind_handler() {
MonitorExitStub* stub = nullptr;
if (method()->is_synchronized()) {
monitor_address(0, FrameMap::rax_opr);
- stub = new MonitorExitStub(FrameMap::rax_opr, true, 0);
+ stub = new MonitorExitStub(FrameMap::rax_opr, 0);
__ unlock_object(rdi, rsi, rax, *stub->entry());
__ bind(*stub->continuation());
}
@@ -2730,7 +2730,6 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
Register hdr = op->hdr_opr()->as_register();
Register lock = op->lock_opr()->as_register();
if (op->code() == lir_lock) {
- assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
Register tmp = op->scratch_opr()->as_register();
// add debug info for NullPointerException only if one is possible
int null_check_offset = __ lock_object(hdr, obj, lock, tmp, *op->stub()->entry());
@@ -2739,7 +2738,6 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) {
}
// done
} else if (op->code() == lir_unlock) {
- assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
__ unlock_object(hdr, obj, lock, *op->stub()->entry());
} else {
Unimplemented();
diff --git a/src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp
index 36efeafa940..c3d45f9d15d 100644
--- a/src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp
@@ -41,32 +41,32 @@
#include "utilities/checkedCast.hpp"
#include "utilities/globalDefinitions.hpp"
-int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Register tmp, Label& slow_case) {
+int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register basic_lock, Register tmp, Label& slow_case) {
assert(hdr == rax, "hdr must be rax, for the cmpxchg instruction");
- assert_different_registers(hdr, obj, disp_hdr, tmp);
+ assert_different_registers(hdr, obj, basic_lock, tmp);
int null_check_offset = -1;
verify_oop(obj);
// save object being locked into the BasicObjectLock
- movptr(Address(disp_hdr, BasicObjectLock::obj_offset()), obj);
+ movptr(Address(basic_lock, BasicObjectLock::obj_offset()), obj);
null_check_offset = offset();
- lightweight_lock(disp_hdr, obj, hdr, tmp, slow_case);
+ lightweight_lock(basic_lock, obj, hdr, tmp, slow_case);
return null_check_offset;
}
-void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) {
- assert(disp_hdr == rax, "disp_hdr must be rax, for the cmpxchg instruction");
- assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
+void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register basic_lock, Label& slow_case) {
+ assert(basic_lock == rax, "basic_lock must be rax, for the cmpxchg instruction");
+ assert(hdr != obj && hdr != basic_lock && obj != basic_lock, "registers must be different");
// load object
- movptr(obj, Address(disp_hdr, BasicObjectLock::obj_offset()));
+ movptr(obj, Address(basic_lock, BasicObjectLock::obj_offset()));
verify_oop(obj);
- lightweight_unlock(obj, disp_hdr, hdr, slow_case);
+ lightweight_unlock(obj, rax, hdr, slow_case);
}
diff --git a/src/hotspot/cpu/x86/c1_MacroAssembler_x86.hpp b/src/hotspot/cpu/x86/c1_MacroAssembler_x86.hpp
index 6344a7b6ef1..f33e47aadb3 100644
--- a/src/hotspot/cpu/x86/c1_MacroAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/c1_MacroAssembler_x86.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -46,17 +46,17 @@
void initialize_body(Register obj, Register len_in_bytes, int hdr_size_in_bytes, Register t1);
// locking
- // hdr : must be rax, contents destroyed
- // obj : must point to the object to lock, contents preserved
- // disp_hdr: must point to the displaced header location, contents preserved
+ // hdr : must be rax, contents destroyed
+ // obj : must point to the object to lock, contents preserved
+ // basic_lock: must point to the basic lock, contents preserved
// returns code offset at which to add null check debug information
- int lock_object (Register swap, Register obj, Register disp_hdr, Register tmp, Label& slow_case);
+ int lock_object (Register swap, Register obj, Register basic_lock, Register tmp, Label& slow_case);
// unlocking
- // hdr : contents destroyed
- // obj : must point to the object to lock, contents preserved
- // disp_hdr: must be eax & must point to the displaced header location, contents destroyed
- void unlock_object(Register swap, Register obj, Register lock, Label& slow_case);
+ // hdr : contents destroyed
+ // obj : must point to the object to lock, contents preserved
+ // basic_lock: must be eax & must point to the basic lock, contents destroyed
+ void unlock_object(Register swap, Register obj, Register basic_lock, Label& slow_case);
void initialize_object(
Register obj, // result: pointer to object after successful allocation
diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
index 8c3f33e0aca..8386e57c389 100644
--- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
@@ -5053,12 +5053,12 @@ void C2_MacroAssembler::vector_cast_int_to_subword(BasicType to_elem_bt, XMMRegi
}
vpackuswb(dst, dst, zero, vec_enc);
break;
- default: assert(false, "%s", type2name(to_elem_bt));
+ default: assert(false, "Unexpected basic type for target of vector cast int to subword: %s", type2name(to_elem_bt));
}
}
/*
- * Algorithm for vector D2L and F2I conversions:-
+ * Algorithm for vector D2L and F2I conversions (AVX 10.2 unsupported):-
* a) Perform vector D2L/F2I cast.
* b) Choose fast path if none of the result vector lane contains 0x80000000 value.
* It signifies that source value could be any of the special floating point
@@ -5096,7 +5096,7 @@ void C2_MacroAssembler::vector_castF2X_evex(BasicType to_elem_bt, XMMRegister ds
case T_BYTE:
evpmovdb(dst, dst, vec_enc);
break;
- default: assert(false, "%s", type2name(to_elem_bt));
+ default: assert(false, "Unexpected basic type for target of vector castF2X EVEX: %s", type2name(to_elem_bt));
}
}
@@ -5143,7 +5143,7 @@ void C2_MacroAssembler::vector_castD2X_evex(BasicType to_elem_bt, XMMRegister ds
evpmovsqd(dst, dst, vec_enc);
evpmovdb(dst, dst, vec_enc);
break;
- default: assert(false, "%s", type2name(to_elem_bt));
+ default: assert(false, "Unexpected basic type for target of vector castD2X AVX512DQ EVEX: %s", type2name(to_elem_bt));
}
} else {
assert(type2aelembytes(to_elem_bt) <= 4, "");
@@ -5158,11 +5158,91 @@ void C2_MacroAssembler::vector_castD2X_evex(BasicType to_elem_bt, XMMRegister ds
case T_BYTE:
evpmovdb(dst, dst, vec_enc);
break;
- default: assert(false, "%s", type2name(to_elem_bt));
+ default: assert(false, "Unexpected basic type for target of vector castD2X EVEX: %s", type2name(to_elem_bt));
}
}
}
+void C2_MacroAssembler::vector_castF2X_avx10(BasicType to_elem_bt, XMMRegister dst, XMMRegister src, int vec_enc) {
+ switch(to_elem_bt) {
+ case T_LONG:
+ evcvttps2qqs(dst, src, vec_enc);
+ break;
+ case T_INT:
+ evcvttps2dqs(dst, src, vec_enc);
+ break;
+ case T_SHORT:
+ evcvttps2dqs(dst, src, vec_enc);
+ evpmovdw(dst, dst, vec_enc);
+ break;
+ case T_BYTE:
+ evcvttps2dqs(dst, src, vec_enc);
+ evpmovdb(dst, dst, vec_enc);
+ break;
+ default: assert(false, "Unexpected basic type for target of vector castF2X AVX10 (reg src): %s", type2name(to_elem_bt));
+ }
+}
+
+void C2_MacroAssembler::vector_castF2X_avx10(BasicType to_elem_bt, XMMRegister dst, Address src, int vec_enc) {
+ switch(to_elem_bt) {
+ case T_LONG:
+ evcvttps2qqs(dst, src, vec_enc);
+ break;
+ case T_INT:
+ evcvttps2dqs(dst, src, vec_enc);
+ break;
+ case T_SHORT:
+ evcvttps2dqs(dst, src, vec_enc);
+ evpmovdw(dst, dst, vec_enc);
+ break;
+ case T_BYTE:
+ evcvttps2dqs(dst, src, vec_enc);
+ evpmovdb(dst, dst, vec_enc);
+ break;
+ default: assert(false, "Unexpected basic type for target of vector castF2X AVX10 (mem src): %s", type2name(to_elem_bt));
+ }
+}
+
+void C2_MacroAssembler::vector_castD2X_avx10(BasicType to_elem_bt, XMMRegister dst, XMMRegister src, int vec_enc) {
+ switch(to_elem_bt) {
+ case T_LONG:
+ evcvttpd2qqs(dst, src, vec_enc);
+ break;
+ case T_INT:
+ evcvttpd2dqs(dst, src, vec_enc);
+ break;
+ case T_SHORT:
+ evcvttpd2dqs(dst, src, vec_enc);
+ evpmovdw(dst, dst, vec_enc);
+ break;
+ case T_BYTE:
+ evcvttpd2dqs(dst, src, vec_enc);
+ evpmovdb(dst, dst, vec_enc);
+ break;
+ default: assert(false, "Unexpected basic type for target of vector castD2X AVX10 (reg src): %s", type2name(to_elem_bt));
+ }
+}
+
+void C2_MacroAssembler::vector_castD2X_avx10(BasicType to_elem_bt, XMMRegister dst, Address src, int vec_enc) {
+ switch(to_elem_bt) {
+ case T_LONG:
+ evcvttpd2qqs(dst, src, vec_enc);
+ break;
+ case T_INT:
+ evcvttpd2dqs(dst, src, vec_enc);
+ break;
+ case T_SHORT:
+ evcvttpd2dqs(dst, src, vec_enc);
+ evpmovdw(dst, dst, vec_enc);
+ break;
+ case T_BYTE:
+ evcvttpd2dqs(dst, src, vec_enc);
+ evpmovdb(dst, dst, vec_enc);
+ break;
+ default: assert(false, "Unexpected basic type for target of vector castD2X AVX10 (mem src): %s", type2name(to_elem_bt));
+ }
+}
+
void C2_MacroAssembler::vector_round_double_evex(XMMRegister dst, XMMRegister src,
AddressLiteral double_sign_flip, AddressLiteral new_mxcsr, int vec_enc,
Register tmp, XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2) {
diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp
index 950fcb75290..aaee25f440a 100644
--- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp
@@ -347,6 +347,13 @@ public:
XMMRegister xtmp2, XMMRegister xtmp3, XMMRegister xtmp4, XMMRegister xtmp5,
AddressLiteral float_sign_flip, Register rscratch, int vec_enc);
+ void vector_castF2X_avx10(BasicType to_elem_bt, XMMRegister dst, XMMRegister src, int vec_enc);
+
+ void vector_castF2X_avx10(BasicType to_elem_bt, XMMRegister dst, Address src, int vec_enc);
+
+ void vector_castD2X_avx10(BasicType to_elem_bt, XMMRegister dst, XMMRegister src, int vec_enc);
+
+ void vector_castD2X_avx10(BasicType to_elem_bt, XMMRegister dst, Address src, int vec_enc);
void vector_cast_double_to_int_special_cases_avx(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
XMMRegister xtmp3, XMMRegister xtmp4, XMMRegister xtmp5, Register rscratch,
diff --git a/src/hotspot/cpu/x86/frame_x86.cpp b/src/hotspot/cpu/x86/frame_x86.cpp
index 46ffda93699..5f52f2fabf2 100644
--- a/src/hotspot/cpu/x86/frame_x86.cpp
+++ b/src/hotspot/cpu/x86/frame_x86.cpp
@@ -219,8 +219,7 @@ bool frame::safe_for_sender(JavaThread *thread) {
nmethod* nm = sender_blob->as_nmethod_or_null();
if (nm != nullptr) {
- if (nm->is_deopt_mh_entry(sender_pc) || nm->is_deopt_entry(sender_pc) ||
- nm->method()->is_method_handle_intrinsic()) {
+ if (nm->is_deopt_entry(sender_pc) || nm->method()->is_method_handle_intrinsic()) {
return false;
}
}
@@ -443,47 +442,6 @@ JavaThread** frame::saved_thread_address(const frame& f) {
return thread_addr;
}
-//------------------------------------------------------------------------------
-// frame::verify_deopt_original_pc
-//
-// Verifies the calculated original PC of a deoptimization PC for the
-// given unextended SP.
-#ifdef ASSERT
-void frame::verify_deopt_original_pc(nmethod* nm, intptr_t* unextended_sp) {
- frame fr;
-
- // This is ugly but it's better than to change {get,set}_original_pc
- // to take an SP value as argument. And it's only a debugging
- // method anyway.
- fr._unextended_sp = unextended_sp;
-
- address original_pc = nm->get_original_pc(&fr);
- assert(nm->insts_contains_inclusive(original_pc),
- "original PC must be in the main code section of the compiled method (or must be immediately following it) original_pc: " INTPTR_FORMAT " unextended_sp: " INTPTR_FORMAT " name: %s", p2i(original_pc), p2i(unextended_sp), nm->name());
-}
-#endif
-
-//------------------------------------------------------------------------------
-// frame::adjust_unextended_sp
-#ifdef ASSERT
-void frame::adjust_unextended_sp() {
- // On x86, sites calling method handle intrinsics and lambda forms are treated
- // as any other call site. Therefore, no special action is needed when we are
- // returning to any of these call sites.
-
- if (_cb != nullptr) {
- nmethod* sender_nm = _cb->as_nmethod_or_null();
- if (sender_nm != nullptr) {
- // If the sender PC is a deoptimization point, get the original PC.
- if (sender_nm->is_deopt_entry(_pc) ||
- sender_nm->is_deopt_mh_entry(_pc)) {
- verify_deopt_original_pc(sender_nm, _unextended_sp);
- }
- }
- }
-}
-#endif
-
//------------------------------------------------------------------------------
// frame::sender_for_interpreter_frame
frame frame::sender_for_interpreter_frame(RegisterMap* map) const {
diff --git a/src/hotspot/cpu/x86/frame_x86.hpp b/src/hotspot/cpu/x86/frame_x86.hpp
index f3034ee9263..19f37c42cf4 100644
--- a/src/hotspot/cpu/x86/frame_x86.hpp
+++ b/src/hotspot/cpu/x86/frame_x86.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -138,17 +138,10 @@
int _offset_unextended_sp; // for use in stack-chunk frames
};
- void adjust_unextended_sp() NOT_DEBUG_RETURN;
-
intptr_t* ptr_at_addr(int offset) const {
return (intptr_t*) addr_at(offset);
}
-#ifdef ASSERT
- // Used in frame::sender_for_{interpreter,compiled}_frame
- static void verify_deopt_original_pc(nmethod* nm, intptr_t* unextended_sp);
-#endif
-
public:
// Constructors
diff --git a/src/hotspot/cpu/x86/frame_x86.inline.hpp b/src/hotspot/cpu/x86/frame_x86.inline.hpp
index afc4ab8767b..ca51fe66786 100644
--- a/src/hotspot/cpu/x86/frame_x86.inline.hpp
+++ b/src/hotspot/cpu/x86/frame_x86.inline.hpp
@@ -111,8 +111,6 @@ inline void frame::init(intptr_t* sp, intptr_t* fp, address pc) {
}
inline void frame::setup(address pc) {
- adjust_unextended_sp();
-
address original_pc = get_deopt_original_pc();
if (original_pc != nullptr) {
_pc = original_pc;
@@ -209,7 +207,6 @@ inline frame::frame(intptr_t* sp, intptr_t* fp) {
// assert(_pc != nullptr, "no pc?");
_cb = CodeCache::find_blob(_pc);
- adjust_unextended_sp();
address original_pc = get_deopt_original_pc();
if (original_pc != nullptr) {
diff --git a/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp
index c1920b52837..a846289d91b 100644
--- a/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp
@@ -89,19 +89,53 @@ void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm
void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
Register addr, Register count, Register tmp) {
- __ push_call_clobbered_registers(false /* save_fpu */);
- if (c_rarg0 == count) { // On win64 c_rarg0 == rcx
- assert_different_registers(c_rarg1, addr);
- __ mov(c_rarg1, count);
- __ mov(c_rarg0, addr);
- } else {
- assert_different_registers(c_rarg0, count);
- __ mov(c_rarg0, addr);
- __ mov(c_rarg1, count);
- }
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_post_entry), 2);
- __ pop_call_clobbered_registers(false /* save_fpu */);
+ Label done;
+ __ testptr(count, count);
+ __ jcc(Assembler::zero, done);
+
+ // Calculate end address in "count".
+ Address::ScaleFactor scale = UseCompressedOops ? Address::times_4 : Address::times_8;
+ __ leaq(count, Address(addr, count, scale));
+
+ // Calculate start card address in "addr".
+ __ shrptr(addr, CardTable::card_shift());
+
+ Register thread = r15_thread;
+
+ __ movptr(tmp, Address(thread, in_bytes(G1ThreadLocalData::card_table_base_offset())));
+ __ addptr(addr, tmp);
+
+ // Calculate address of card of last word in the array.
+ __ subptr(count, 1);
+ __ shrptr(count, CardTable::card_shift());
+ __ addptr(count, tmp);
+
+ Label loop;
+ // Iterate from start card to end card (inclusive).
+ __ bind(loop);
+
+ Label is_clean_card;
+ if (UseCondCardMark) {
+ __ cmpb(Address(addr, 0), G1CardTable::clean_card_val());
+ __ jcc(Assembler::equal, is_clean_card);
+ } else {
+ __ movb(Address(addr, 0), G1CardTable::dirty_card_val());
+ }
+
+ Label next_card;
+ __ bind(next_card);
+ __ addptr(addr, sizeof(CardTable::CardValue));
+ __ cmpptr(addr, count);
+ __ jcc(Assembler::belowEqual, loop);
+ __ jmp(done);
+
+ __ bind(is_clean_card);
+ // Card was clean. Dirty card and go to next..
+ __ movb(Address(addr, 0), G1CardTable::dirty_card_val());
+ __ jmp(next_card);
+
+ __ bind(done);
}
void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
@@ -182,7 +216,6 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
// If expand_call is true then we expand the call_VM_leaf macro
// directly to skip generating the check by
// InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
-
const Register thread = r15_thread;
Label done;
@@ -235,76 +268,49 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
__ bind(done);
}
-static void generate_post_barrier_fast_path(MacroAssembler* masm,
- const Register store_addr,
- const Register new_val,
- const Register tmp,
- const Register tmp2,
- Label& done,
- bool new_val_may_be_null) {
- CardTableBarrierSet* ct = barrier_set_cast(BarrierSet::barrier_set());
+static void generate_post_barrier(MacroAssembler* masm,
+ const Register store_addr,
+ const Register new_val,
+ const Register tmp1,
+ Label& done,
+ bool new_val_may_be_null) {
+
+ assert_different_registers(store_addr, new_val, tmp1, noreg);
+
+ Register thread = r15_thread;
+
// Does store cross heap regions?
- __ movptr(tmp, store_addr); // tmp := store address
- __ xorptr(tmp, new_val); // tmp := store address ^ new value
- __ shrptr(tmp, G1HeapRegion::LogOfHRGrainBytes); // ((store address ^ new value) >> LogOfHRGrainBytes) == 0?
+ __ movptr(tmp1, store_addr); // tmp1 := store address
+ __ xorptr(tmp1, new_val); // tmp1 := store address ^ new value
+ __ shrptr(tmp1, G1HeapRegion::LogOfHRGrainBytes); // ((store address ^ new value) >> LogOfHRGrainBytes) == 0?
__ jcc(Assembler::equal, done);
+
// Crosses regions, storing null?
if (new_val_may_be_null) {
- __ cmpptr(new_val, NULL_WORD); // new value == null?
+ __ cmpptr(new_val, NULL_WORD); // new value == null?
__ jcc(Assembler::equal, done);
}
- // Storing region crossing non-null, is card young?
- __ movptr(tmp, store_addr); // tmp := store address
- __ shrptr(tmp, CardTable::card_shift()); // tmp := card address relative to card table base
- // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT
- // a valid address and therefore is not properly handled by the relocation code.
- __ movptr(tmp2, (intptr_t)ct->card_table()->byte_map_base()); // tmp2 := card table base address
- __ addptr(tmp, tmp2); // tmp := card address
- __ cmpb(Address(tmp, 0), G1CardTable::g1_young_card_val()); // *(card address) == young_card_val?
-}
-static void generate_post_barrier_slow_path(MacroAssembler* masm,
- const Register thread,
- const Register tmp,
- const Register tmp2,
- Label& done,
- Label& runtime) {
- __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad)); // StoreLoad membar
- __ cmpb(Address(tmp, 0), G1CardTable::dirty_card_val()); // *(card address) == dirty_card_val?
- __ jcc(Assembler::equal, done);
+ __ movptr(tmp1, store_addr); // tmp1 := store address
+ __ shrptr(tmp1, CardTable::card_shift()); // tmp1 := card address relative to card table base
+
+ Address card_table_addr(thread, in_bytes(G1ThreadLocalData::card_table_base_offset()));
+ __ addptr(tmp1, card_table_addr); // tmp1 := card address
+ if (UseCondCardMark) {
+ __ cmpb(Address(tmp1, 0), G1CardTable::clean_card_val()); // *(card address) == clean_card_val?
+ __ jcc(Assembler::notEqual, done);
+ }
// Storing a region crossing, non-null oop, card is clean.
- // Dirty card and log.
- __ movb(Address(tmp, 0), G1CardTable::dirty_card_val()); // *(card address) := dirty_card_val
- generate_queue_insertion(masm,
- G1ThreadLocalData::dirty_card_queue_index_offset(),
- G1ThreadLocalData::dirty_card_queue_buffer_offset(),
- runtime,
- thread, tmp, tmp2);
- __ jmp(done);
+ // Dirty card.
+ __ movb(Address(tmp1, 0), G1CardTable::dirty_card_val()); // *(card address) := dirty_card_val
}
void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
Register store_addr,
Register new_val,
- Register tmp,
- Register tmp2) {
- const Register thread = r15_thread;
-
+ Register tmp) {
Label done;
- Label runtime;
-
- generate_post_barrier_fast_path(masm, store_addr, new_val, tmp, tmp2, done, true /* new_val_may_be_null */);
- // If card is young, jump to done
- __ jcc(Assembler::equal, done);
- generate_post_barrier_slow_path(masm, thread, tmp, tmp2, done, runtime);
-
- __ bind(runtime);
- // save the live input values
- RegSet saved = RegSet::of(store_addr);
- __ push_set(saved);
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), tmp, thread);
- __ pop_set(saved);
-
+ generate_post_barrier(masm, store_addr, new_val, tmp, done, true /* new_val_may_be_null */);
__ bind(done);
}
@@ -367,34 +373,10 @@ void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm,
Register store_addr,
Register new_val,
Register tmp,
- Register tmp2,
- G1PostBarrierStubC2* stub) {
- const Register thread = r15_thread;
- stub->initialize_registers(thread, tmp, tmp2);
-
- bool new_val_may_be_null = (stub->barrier_data() & G1C2BarrierPostNotNull) == 0;
- generate_post_barrier_fast_path(masm, store_addr, new_val, tmp, tmp2, *stub->continuation(), new_val_may_be_null);
- // If card is not young, jump to stub (slow path)
- __ jcc(Assembler::notEqual, *stub->entry());
-
- __ bind(*stub->continuation());
-}
-
-void G1BarrierSetAssembler::generate_c2_post_barrier_stub(MacroAssembler* masm,
- G1PostBarrierStubC2* stub) const {
- Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
- Label runtime;
- Register thread = stub->thread();
- Register tmp = stub->tmp1(); // tmp holds the card address.
- Register tmp2 = stub->tmp2();
- assert(stub->tmp3() == noreg, "not needed in this platform");
-
- __ bind(*stub->entry());
- generate_post_barrier_slow_path(masm, thread, tmp, tmp2, *stub->continuation(), runtime);
-
- __ bind(runtime);
- generate_c2_barrier_runtime_call(masm, stub, tmp, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry));
- __ jmp(*stub->continuation());
+ bool new_val_may_be_null) {
+ Label done;
+ generate_post_barrier(masm, store_addr, new_val, tmp, done, new_val_may_be_null);
+ __ bind(done);
}
#endif // COMPILER2
@@ -441,8 +423,7 @@ void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet deco
g1_write_barrier_post(masm /*masm*/,
tmp1 /* store_adr */,
new_val /* new_val */,
- tmp3 /* tmp */,
- tmp2 /* tmp2 */);
+ tmp3 /* tmp */);
}
}
}
@@ -476,21 +457,19 @@ void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrier
}
-void G1BarrierSetAssembler::gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) {
- G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
- __ bind(*stub->entry());
- assert(stub->addr()->is_register(), "Precondition.");
- assert(stub->new_val()->is_register(), "Precondition.");
- Register new_val_reg = stub->new_val()->as_register();
- __ cmpptr(new_val_reg, NULL_WORD);
- __ jcc(Assembler::equal, *stub->continuation());
- ce->store_parameter(stub->addr()->as_pointer_register(), 0);
- __ call(RuntimeAddress(bs->post_barrier_c1_runtime_code_blob()->code_begin()));
- __ jmp(*stub->continuation());
-}
-
#undef __
+void G1BarrierSetAssembler::g1_write_barrier_post_c1(MacroAssembler* masm,
+ Register store_addr,
+ Register new_val,
+ Register thread,
+ Register tmp1,
+ Register tmp2 /* unused on x86 */) {
+ Label done;
+ generate_post_barrier(masm, store_addr, new_val, tmp1, done, true /* new_val_may_be_null */);
+ masm->bind(done);
+}
+
#define __ sasm->
void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
@@ -555,78 +534,6 @@ void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler*
__ epilogue();
}
-void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) {
- __ prologue("g1_post_barrier", false);
-
- CardTableBarrierSet* ct =
- barrier_set_cast(BarrierSet::barrier_set());
-
- Label done;
- Label enqueued;
- Label runtime;
-
- // At this point we know new_value is non-null and the new_value crosses regions.
- // Must check to see if card is already dirty
-
- const Register thread = r15_thread;
-
- Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
- Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
-
- __ push_ppx(rax);
- __ push_ppx(rcx);
-
- const Register cardtable = rax;
- const Register card_addr = rcx;
-
- __ load_parameter(0, card_addr);
- __ shrptr(card_addr, CardTable::card_shift());
- // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT
- // a valid address and therefore is not properly handled by the relocation code.
- __ movptr(cardtable, (intptr_t)ct->card_table()->byte_map_base());
- __ addptr(card_addr, cardtable);
-
- __ cmpb(Address(card_addr, 0), G1CardTable::g1_young_card_val());
- __ jcc(Assembler::equal, done);
-
- __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
- __ cmpb(Address(card_addr, 0), CardTable::dirty_card_val());
- __ jcc(Assembler::equal, done);
-
- // storing region crossing non-null, card is clean.
- // dirty card and log.
-
- __ movb(Address(card_addr, 0), CardTable::dirty_card_val());
-
- const Register tmp = rdx;
- __ push_ppx(rdx);
-
- __ movptr(tmp, queue_index);
- __ testptr(tmp, tmp);
- __ jcc(Assembler::zero, runtime);
- __ subptr(tmp, wordSize);
- __ movptr(queue_index, tmp);
- __ addptr(tmp, buffer);
- __ movptr(Address(tmp, 0), card_addr);
- __ jmp(enqueued);
-
- __ bind(runtime);
- __ push_call_clobbered_registers();
-
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
-
- __ pop_call_clobbered_registers();
-
- __ bind(enqueued);
- __ pop_ppx(rdx);
-
- __ bind(done);
- __ pop_ppx(rcx);
- __ pop_ppx(rax);
-
- __ epilogue();
-}
-
#undef __
#endif // COMPILER1
diff --git a/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.hpp
index 774e87b916c..4b2de41de69 100644
--- a/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -31,10 +31,8 @@
class LIR_Assembler;
class StubAssembler;
class G1PreBarrierStub;
-class G1PostBarrierStub;
class G1BarrierStubC2;
class G1PreBarrierStubC2;
-class G1PostBarrierStubC2;
class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
protected:
@@ -51,22 +49,28 @@ class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
void g1_write_barrier_post(MacroAssembler* masm,
Register store_addr,
Register new_val,
- Register tmp,
- Register tmp2);
+ Register tmp);
virtual void oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Address dst, Register val, Register tmp1, Register tmp2, Register tmp3);
public:
- void gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub);
- void gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub);
-
- void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm);
- void generate_c1_post_barrier_runtime_stub(StubAssembler* sasm);
-
virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Register dst, Address src, Register tmp1);
+#ifdef COMPILER1
+ void gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub);
+
+ void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm);
+
+ void g1_write_barrier_post_c1(MacroAssembler* masm,
+ Register store_addr,
+ Register new_val,
+ Register thread,
+ Register tmp1,
+ Register tmp2);
+#endif
+
#ifdef COMPILER2
void g1_write_barrier_pre_c2(MacroAssembler* masm,
Register obj,
@@ -79,10 +83,7 @@ class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
Register store_addr,
Register new_val,
Register tmp,
- Register tmp2,
- G1PostBarrierStubC2* c2_stub);
- void generate_c2_post_barrier_stub(MacroAssembler* masm,
- G1PostBarrierStubC2* stub) const;
+ bool new_val_may_be_null);
#endif // COMPILER2
};
diff --git a/src/hotspot/cpu/x86/gc/g1/g1_x86_64.ad b/src/hotspot/cpu/x86/gc/g1/g1_x86_64.ad
index 819cd97696c..94607cd6796 100644
--- a/src/hotspot/cpu/x86/gc/g1/g1_x86_64.ad
+++ b/src/hotspot/cpu/x86/gc/g1/g1_x86_64.ad
@@ -1,5 +1,5 @@
//
-// Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
@@ -59,15 +59,14 @@ static void write_barrier_post(MacroAssembler* masm,
const MachNode* node,
Register store_addr,
Register new_val,
- Register tmp1,
- Register tmp2) {
- if (!G1PostBarrierStubC2::needs_barrier(node)) {
+ Register tmp1) {
+ if (!G1BarrierStubC2::needs_post_barrier(node)) {
return;
}
Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
G1BarrierSetAssembler* g1_asm = static_cast(BarrierSet::barrier_set()->barrier_set_assembler());
- G1PostBarrierStubC2* const stub = G1PostBarrierStubC2::create(node);
- g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, tmp1, tmp2, stub);
+ bool new_val_may_be_null = G1BarrierStubC2::post_new_val_may_be_null(node);
+ g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, tmp1, new_val_may_be_null);
}
%}
@@ -95,8 +94,7 @@ instruct g1StoreP(memory mem, any_RegP src, rRegP tmp1, rRegP tmp2, rRegP tmp3,
write_barrier_post(masm, this,
$tmp1$$Register /* store_addr */,
$src$$Register /* new_val */,
- $tmp3$$Register /* tmp1 */,
- $tmp2$$Register /* tmp2 */);
+ $tmp3$$Register /* tmp1 */);
%}
ins_pipe(ialu_mem_reg);
%}
@@ -127,8 +125,7 @@ instruct g1StoreN(memory mem, rRegN src, rRegP tmp1, rRegP tmp2, rRegP tmp3, rFl
write_barrier_post(masm, this,
$tmp1$$Register /* store_addr */,
$tmp2$$Register /* new_val */,
- $tmp3$$Register /* tmp1 */,
- $tmp2$$Register /* tmp2 */);
+ $tmp3$$Register /* tmp1 */);
%}
ins_pipe(ialu_mem_reg);
%}
@@ -158,8 +155,7 @@ instruct g1EncodePAndStoreN(memory mem, any_RegP src, rRegP tmp1, rRegP tmp2, rR
write_barrier_post(masm, this,
$tmp1$$Register /* store_addr */,
$src$$Register /* new_val */,
- $tmp3$$Register /* tmp1 */,
- $tmp2$$Register /* tmp2 */);
+ $tmp3$$Register /* tmp1 */);
%}
ins_pipe(ialu_mem_reg);
%}
@@ -187,8 +183,7 @@ instruct g1CompareAndExchangeP(indirect mem, rRegP newval, rRegP tmp1, rRegP tmp
write_barrier_post(masm, this,
$mem$$Register /* store_addr */,
$tmp1$$Register /* new_val */,
- $tmp2$$Register /* tmp1 */,
- $tmp3$$Register /* tmp2 */);
+ $tmp2$$Register /* tmp1 */);
%}
ins_pipe(pipe_cmpxchg);
%}
@@ -214,8 +209,7 @@ instruct g1CompareAndExchangeN(indirect mem, rRegN newval, rRegP tmp1, rRegP tmp
write_barrier_post(masm, this,
$mem$$Register /* store_addr */,
$tmp1$$Register /* new_val */,
- $tmp2$$Register /* tmp1 */,
- $tmp3$$Register /* tmp2 */);
+ $tmp2$$Register /* tmp1 */);
%}
ins_pipe(pipe_cmpxchg);
%}
@@ -246,8 +240,7 @@ instruct g1CompareAndSwapP(rRegI res, indirect mem, rRegP newval, rRegP tmp1, rR
write_barrier_post(masm, this,
$mem$$Register /* store_addr */,
$tmp1$$Register /* new_val */,
- $tmp2$$Register /* tmp1 */,
- $tmp3$$Register /* tmp2 */);
+ $tmp2$$Register /* tmp1 */);
%}
ins_pipe(pipe_cmpxchg);
%}
@@ -279,8 +272,7 @@ instruct g1CompareAndSwapN(rRegI res, indirect mem, rRegN newval, rRegP tmp1, rR
write_barrier_post(masm, this,
$mem$$Register /* store_addr */,
$tmp1$$Register /* new_val */,
- $tmp2$$Register /* tmp1 */,
- $tmp3$$Register /* tmp2 */);
+ $tmp2$$Register /* tmp1 */);
%}
ins_pipe(pipe_cmpxchg);
%}
@@ -303,8 +295,7 @@ instruct g1GetAndSetP(indirect mem, rRegP newval, rRegP tmp1, rRegP tmp2, rRegP
write_barrier_post(masm, this,
$mem$$Register /* store_addr */,
$tmp1$$Register /* new_val */,
- $tmp2$$Register /* tmp1 */,
- $tmp3$$Register /* tmp2 */);
+ $tmp2$$Register /* tmp1 */);
%}
ins_pipe(pipe_cmpxchg);
%}
@@ -328,8 +319,7 @@ instruct g1GetAndSetN(indirect mem, rRegN newval, rRegP tmp1, rRegP tmp2, rRegP
write_barrier_post(masm, this,
$mem$$Register /* store_addr */,
$tmp1$$Register /* new_val */,
- $tmp2$$Register /* tmp1 */,
- $tmp3$$Register /* tmp2 */);
+ $tmp2$$Register /* tmp1 */);
%}
ins_pipe(pipe_cmpxchg);
%}
diff --git a/src/hotspot/cpu/x86/gc/shared/barrierSetNMethod_x86.cpp b/src/hotspot/cpu/x86/gc/shared/barrierSetNMethod_x86.cpp
index 124daef4fa7..40311f746ea 100644
--- a/src/hotspot/cpu/x86/gc/shared/barrierSetNMethod_x86.cpp
+++ b/src/hotspot/cpu/x86/gc/shared/barrierSetNMethod_x86.cpp
@@ -65,12 +65,12 @@ public:
assert(align_up(immediate_address(), sizeof(jint)) ==
align_down(immediate_address(), sizeof(jint)), "immediate not aligned");
jint* data_addr = (jint*)immediate_address();
- jint old_value = Atomic::load(data_addr);
+ jint old_value = AtomicAccess::load(data_addr);
while (true) {
// Only bits in the mask are changed
jint new_value = imm | (old_value & ~bit_mask);
if (new_value == old_value) break;
- jint v = Atomic::cmpxchg(data_addr, old_value, new_value, memory_order_release);
+ jint v = AtomicAccess::cmpxchg(data_addr, old_value, new_value, memory_order_release);
if (v == old_value) break;
old_value = v;
}
diff --git a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp
index 0b19180ea06..ae93cca8c19 100644
--- a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp
@@ -1410,11 +1410,9 @@ void ZBarrierSetAssembler::patch_barriers() {
}
}
-
#undef __
#define __ masm->
-
void ZBarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) {
// C1 calls verfy_oop in the middle of barriers, before they have been uncolored
// and after being colored. Therefore, we must deal with colored oops as well.
diff --git a/src/hotspot/cpu/x86/globalDefinitions_x86.hpp b/src/hotspot/cpu/x86/globalDefinitions_x86.hpp
index 3c1474ae861..abbeb66a1ca 100644
--- a/src/hotspot/cpu/x86/globalDefinitions_x86.hpp
+++ b/src/hotspot/cpu/x86/globalDefinitions_x86.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -34,8 +34,6 @@ const bool CCallingConventionRequiresIntsAsLongs = false;
#define SUPPORTS_NATIVE_CX8
-#define SUPPORT_MONITOR_COUNT
-
#define CPU_MULTI_COPY_ATOMIC
// The expected size in bytes of a cache line.
diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp
index 0b41d594194..4f19b30b832 100644
--- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp
@@ -2431,14 +2431,6 @@ void MacroAssembler::pop_cont_fastpath() {
bind(L_done);
}
-void MacroAssembler::inc_held_monitor_count() {
- incrementq(Address(r15_thread, JavaThread::held_monitor_count_offset()));
-}
-
-void MacroAssembler::dec_held_monitor_count() {
- decrementq(Address(r15_thread, JavaThread::held_monitor_count_offset()));
-}
-
#ifdef ASSERT
void MacroAssembler::stop_if_in_cont(Register cont, const char* name) {
Label no_cont;
@@ -2897,7 +2889,8 @@ void MacroAssembler::vbroadcastss(XMMRegister dst, AddressLiteral src, int vecto
// vblendvps(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len, bool compute_mask = true, XMMRegister scratch = xnoreg)
void MacroAssembler::vblendvps(XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister mask, int vector_len, bool compute_mask, XMMRegister scratch) {
// WARN: Allow dst == (src1|src2), mask == scratch
- bool blend_emulation = EnableX86ECoreOpts && UseAVX > 1;
+ bool blend_emulation = EnableX86ECoreOpts && UseAVX > 1 &&
+ !(VM_Version::is_intel_darkmont() && (dst == src1)); // partially fixed on Darkmont
bool scratch_available = scratch != xnoreg && scratch != src1 && scratch != src2 && scratch != dst;
bool dst_available = dst != mask && (dst != src1 || dst != src2);
if (blend_emulation && scratch_available && dst_available) {
@@ -2921,7 +2914,8 @@ void MacroAssembler::vblendvps(XMMRegister dst, XMMRegister src1, XMMRegister sr
// vblendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len, bool compute_mask = true, XMMRegister scratch = xnoreg)
void MacroAssembler::vblendvpd(XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister mask, int vector_len, bool compute_mask, XMMRegister scratch) {
// WARN: Allow dst == (src1|src2), mask == scratch
- bool blend_emulation = EnableX86ECoreOpts && UseAVX > 1;
+ bool blend_emulation = EnableX86ECoreOpts && UseAVX > 1 &&
+ !(VM_Version::is_intel_darkmont() && (dst == src1)); // partially fixed on Darkmont
bool scratch_available = scratch != xnoreg && scratch != src1 && scratch != src2 && scratch != dst && (!compute_mask || scratch != mask);
bool dst_available = dst != mask && (dst != src1 || dst != src2);
if (blend_emulation && scratch_available && dst_available) {
@@ -5845,7 +5839,7 @@ void MacroAssembler::generate_fill(BasicType t, bool aligned,
orl(value, rtmp);
}
- cmpptr(count, 2<sleep(FT_SLEEP_MILLISECS);
-
- end = os::elapsed_counter();
- OrderAccess::fence();
- fend = os::rdtsc();
-
- time_base += end - start;
- time_fast += fend - fstart;
-
- // basis for calculating the os tick start
- // to fast time tick start offset
- time_base_elapsed += end;
- time_fast_elapsed += (fend - _epoch);
- }
-
- time_base /= loopcount;
- time_fast /= loopcount;
- time_base_elapsed /= loopcount;
- time_fast_elapsed /= loopcount;
-}
-
-static jlong initialize_frequency() {
- assert(0 == tsc_frequency, "invariant");
+jlong Rdtsc::initialize_frequency() {
+ assert(0 == _tsc_frequency, "invariant");
assert(0 == _epoch, "invariant");
const jlong initial_counter = set_epoch();
if (initial_counter == 0) {
@@ -102,29 +59,6 @@ static jlong initialize_frequency() {
// for invariant tsc platforms, take the maximum qualified cpu frequency
tsc_freq = (double)VM_Version::maximum_qualified_cpu_frequency();
os_to_tsc_conv_factor = tsc_freq / os_freq;
- } else {
- // use measurements to estimate
- // a conversion factor and the tsc frequency
-
- volatile jlong time_base = 0;
- volatile jlong time_fast = 0;
- volatile jlong time_base_elapsed = 0;
- volatile jlong time_fast_elapsed = 0;
-
- // do measurements to get base data
- // on os timer and fast ticks tsc time relation.
- do_time_measurements(time_base, time_fast, time_base_elapsed, time_fast_elapsed);
-
- // if invalid measurements, cannot proceed
- if (time_fast == 0 || time_base == 0) {
- return 0;
- }
-
- os_to_tsc_conv_factor = (double)time_fast / (double)time_base;
- if (os_to_tsc_conv_factor > 1) {
- // estimate on tsc counter frequency
- tsc_freq = os_to_tsc_conv_factor * os_freq;
- }
}
if ((tsc_freq < 0) || (tsc_freq > 0 && tsc_freq <= os_freq) || (os_to_tsc_conv_factor <= 1)) {
@@ -136,47 +70,52 @@ static jlong initialize_frequency() {
return (jlong)tsc_freq;
}
-static bool initialize_elapsed_counter() {
- tsc_frequency = initialize_frequency();
- return tsc_frequency != 0 && _epoch != 0;
+bool Rdtsc::initialize_elapsed_counter() {
+ _tsc_frequency = initialize_frequency();
+ return _tsc_frequency != 0 && _epoch != 0;
}
static bool ergonomics() {
- const bool invtsc_support = Rdtsc::is_supported();
- if (FLAG_IS_DEFAULT(UseFastUnorderedTimeStamps) && invtsc_support) {
- FLAG_SET_ERGO(UseFastUnorderedTimeStamps, true);
- }
+ if (Rdtsc::is_supported()) {
+ // Use rdtsc when it is supported by default
+ FLAG_SET_ERGO_IF_DEFAULT(UseFastUnorderedTimeStamps, true);
+ } else if (UseFastUnorderedTimeStamps) {
+ assert(!FLAG_IS_DEFAULT(UseFastUnorderedTimeStamps), "Unexpected default value");
- bool ft_enabled = UseFastUnorderedTimeStamps && invtsc_support;
-
- if (!ft_enabled) {
- if (UseFastUnorderedTimeStamps && VM_Version::supports_tsc()) {
- warning("\nThe hardware does not support invariant tsc (INVTSC) register and/or cannot guarantee tsc synchronization between sockets at startup.\n"\
- "Values returned via rdtsc() are not guaranteed to be accurate, esp. when comparing values from cross sockets reads. Enabling UseFastUnorderedTimeStamps on non-invariant tsc hardware should be considered experimental.\n");
- ft_enabled = true;
- }
- }
-
- if (!ft_enabled) {
- // Warn if unable to support command-line flag
- if (UseFastUnorderedTimeStamps && !VM_Version::supports_tsc()) {
+ if (VM_Version::supports_tsc()) {
+ warning("Ignoring UseFastUnorderedTimeStamps, the hardware does not support invariant tsc (INVTSC) register and/or cannot guarantee tsc synchronization between sockets at startup.\n"
+ "Values returned via rdtsc() are not guaranteed to be accurate, esp. when comparing values from cross sockets reads.");
+ } else {
warning("Ignoring UseFastUnorderedTimeStamps, hardware does not support normal tsc");
}
+
+ // We do not support non invariant rdtsc
+ FLAG_SET_ERGO(UseFastUnorderedTimeStamps, false);
}
- return ft_enabled;
+ return UseFastUnorderedTimeStamps;
+}
+
+bool Rdtsc::initialize() {
+ precond(AtomicAccess::xchg(&_initialized, 1) == 0);
+ assert(0 == _tsc_frequency, "invariant");
+ assert(0 == _epoch, "invariant");
+
+ if (!ergonomics()) {
+ // We decided to ergonomically not support rdtsc.
+ return false;
+ }
+
+ // Try to initialize the elapsed counter
+ return initialize_elapsed_counter();
}
bool Rdtsc::is_supported() {
return VM_Version::supports_tscinv_ext();
}
-bool Rdtsc::is_elapsed_counter_enabled() {
- return rdtsc_elapsed_counter_enabled;
-}
-
jlong Rdtsc::frequency() {
- return tsc_frequency;
+ return _tsc_frequency;
}
jlong Rdtsc::elapsed_counter() {
@@ -191,19 +130,7 @@ jlong Rdtsc::raw() {
return os::rdtsc();
}
-bool Rdtsc::initialize() {
- static bool initialized = false;
- if (!initialized) {
- assert(!rdtsc_elapsed_counter_enabled, "invariant");
- VM_Version::initialize_tsc();
- assert(0 == tsc_frequency, "invariant");
- assert(0 == _epoch, "invariant");
- bool result = initialize_elapsed_counter(); // init hw
- if (result) {
- result = ergonomics(); // check logical state
- }
- rdtsc_elapsed_counter_enabled = result;
- initialized = true;
- }
- return rdtsc_elapsed_counter_enabled;
+bool Rdtsc::enabled() {
+ static bool enabled = initialize();
+ return enabled;
}
diff --git a/src/hotspot/cpu/x86/rdtsc_x86.hpp b/src/hotspot/cpu/x86/rdtsc_x86.hpp
index d9e77a0ae6b..a6519c439d3 100644
--- a/src/hotspot/cpu/x86/rdtsc_x86.hpp
+++ b/src/hotspot/cpu/x86/rdtsc_x86.hpp
@@ -38,14 +38,24 @@
// INVTSC is a minimal requirement for auto-enablement.
class Rdtsc : AllStatic {
+ private:
+ DEBUG_ONLY(static volatile int _initialized;)
+ static jlong _epoch;
+ static jlong _tsc_frequency;
+
+ static jlong set_epoch();
+
+ static jlong initialize_frequency();
+ static bool initialize_elapsed_counter();
+ static bool initialize();
+
public:
static jlong elapsed_counter(); // provides quick time stamps
static jlong frequency(); // tsc register
static bool is_supported(); // InvariantTSC
static jlong raw(); // direct rdtsc() access
- static bool is_elapsed_counter_enabled(); // turn off with -XX:-UseFastUnorderedTimeStamps
static jlong epoch();
- static bool initialize();
+ static bool enabled();
};
#endif // CPU_X86_RDTSC_X86_HPP
diff --git a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp
index d60f535fefc..e702b587edd 100644
--- a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp
+++ b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp
@@ -1007,8 +1007,8 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
int comp_args_on_stack,
const BasicType *sig_bt,
const VMRegPair *regs,
- AdapterHandlerEntry* handler) {
- address i2c_entry = __ pc();
+ address entry_address[AdapterBlob::ENTRY_COUNT]) {
+ entry_address[AdapterBlob::I2C] = __ pc();
gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs);
@@ -1021,7 +1021,7 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
// On exit from the interpreter, the interpreter will restore our SP (lest the
// compiled code, which relies solely on SP and not RBP, get sick).
- address c2i_unverified_entry = __ pc();
+ entry_address[AdapterBlob::C2I_Unverified] = __ pc();
Label skip_fixup;
Register data = rax;
@@ -1039,10 +1039,10 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
__ jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
}
- address c2i_entry = __ pc();
+ entry_address[AdapterBlob::C2I] = __ pc();
// Class initialization barrier for static methods
- address c2i_no_clinit_check_entry = nullptr;
+ entry_address[AdapterBlob::C2I_No_Clinit_Check] = nullptr;
if (VM_Version::supports_fast_class_init_checks()) {
Label L_skip_barrier;
Register method = rbx;
@@ -1061,15 +1061,13 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
__ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
__ bind(L_skip_barrier);
- c2i_no_clinit_check_entry = __ pc();
+ entry_address[AdapterBlob::C2I_No_Clinit_Check] = __ pc();
}
BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
bs->c2i_entry_barrier(masm);
gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup);
-
- handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry);
return;
}
@@ -1354,11 +1352,8 @@ static void fill_continuation_entry(MacroAssembler* masm, Register reg_cont_obj,
__ movptr(rax, Address(r15_thread, JavaThread::cont_fastpath_offset()));
__ movptr(Address(rsp, ContinuationEntry::parent_cont_fastpath_offset()), rax);
- __ movq(rax, Address(r15_thread, JavaThread::held_monitor_count_offset()));
- __ movq(Address(rsp, ContinuationEntry::parent_held_monitor_count_offset()), rax);
__ movptr(Address(r15_thread, JavaThread::cont_fastpath_offset()), 0);
- __ movq(Address(r15_thread, JavaThread::held_monitor_count_offset()), 0);
}
//---------------------------- continuation_enter_cleanup ---------------------------
@@ -1382,49 +1377,6 @@ static void continuation_enter_cleanup(MacroAssembler* masm) {
#endif
__ movptr(rbx, Address(rsp, ContinuationEntry::parent_cont_fastpath_offset()));
__ movptr(Address(r15_thread, JavaThread::cont_fastpath_offset()), rbx);
-
- if (CheckJNICalls) {
- // Check if this is a virtual thread continuation
- Label L_skip_vthread_code;
- __ cmpl(Address(rsp, ContinuationEntry::flags_offset()), 0);
- __ jcc(Assembler::equal, L_skip_vthread_code);
-
- // If the held monitor count is > 0 and this vthread is terminating then
- // it failed to release a JNI monitor. So we issue the same log message
- // that JavaThread::exit does.
- __ cmpptr(Address(r15_thread, JavaThread::jni_monitor_count_offset()), 0);
- __ jcc(Assembler::equal, L_skip_vthread_code);
-
- // rax may hold an exception oop, save it before the call
- __ push(rax);
- __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::log_jni_monitor_still_held));
- __ pop(rax);
-
- // For vthreads we have to explicitly zero the JNI monitor count of the carrier
- // on termination. The held count is implicitly zeroed below when we restore from
- // the parent held count (which has to be zero).
- __ movq(Address(r15_thread, JavaThread::jni_monitor_count_offset()), 0);
-
- __ bind(L_skip_vthread_code);
- }
-#ifdef ASSERT
- else {
- // Check if this is a virtual thread continuation
- Label L_skip_vthread_code;
- __ cmpl(Address(rsp, ContinuationEntry::flags_offset()), 0);
- __ jcc(Assembler::equal, L_skip_vthread_code);
-
- // See comment just above. If not checking JNI calls the JNI count is only
- // needed for assertion checking.
- __ movq(Address(r15_thread, JavaThread::jni_monitor_count_offset()), 0);
-
- __ bind(L_skip_vthread_code);
- }
-#endif
-
- __ movq(rbx, Address(rsp, ContinuationEntry::parent_held_monitor_count_offset()));
- __ movq(Address(r15_thread, JavaThread::held_monitor_count_offset()), rbx);
-
__ movptr(rbx, Address(rsp, ContinuationEntry::parent_offset()));
__ movptr(Address(r15_thread, JavaThread::cont_entry_offset()), rbx);
__ addptr(rsp, checked_cast(ContinuationEntry::size()));
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp
index 743457f87af..d53fafafdb4 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp
@@ -76,50 +76,95 @@ static uint& get_profile_ctr(int shift) {
#endif // !PRODUCT
void StubGenerator::generate_arraycopy_stubs() {
- address entry;
- address entry_jbyte_arraycopy;
- address entry_jshort_arraycopy;
- address entry_jint_arraycopy;
- address entry_oop_arraycopy;
- address entry_jlong_arraycopy;
- address entry_checkcast_arraycopy;
+ // Some copy stubs publish a normal entry and then a 2nd 'fallback'
+ // entry immediately following their stack push. This can be used
+ // as a post-push branch target for compatible stubs when they
+ // identify a special case that can be handled by the fallback
+ // stub e.g a disjoint copy stub may be use as a special case
+ // fallback for its compatible conjoint copy stub.
+ //
+ // A no push entry is always returned in the following local and
+ // then published by assigning to the appropriate entry field in
+ // class StubRoutines. The entry value is then passed to the
+ // generator for the compatible stub. That means the entry must be
+ // listed when saving to/restoring from the AOT cache, ensuring
+ // that the inter-stub jumps are noted at AOT-cache save and
+ // relocated at AOT cache load.
+ address nopush_entry;
- StubRoutines::_jbyte_disjoint_arraycopy = generate_disjoint_byte_copy(&entry);
- StubRoutines::_jbyte_arraycopy = generate_conjoint_byte_copy(entry, &entry_jbyte_arraycopy);
+ StubRoutines::_jbyte_disjoint_arraycopy = generate_disjoint_byte_copy(&nopush_entry);
+ // disjoint nopush entry is needed by conjoint copy
+ StubRoutines::_jbyte_disjoint_arraycopy_nopush = nopush_entry;
+ StubRoutines::_jbyte_arraycopy = generate_conjoint_byte_copy(StubRoutines::_jbyte_disjoint_arraycopy_nopush, &nopush_entry);
+ // conjoint nopush entry is needed by generic/unsafe copy
+ StubRoutines::_jbyte_arraycopy_nopush = nopush_entry;
- StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_short_copy(&entry);
- StubRoutines::_jshort_arraycopy = generate_conjoint_short_copy(entry, &entry_jshort_arraycopy);
+ StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_short_copy(&nopush_entry);
+ // disjoint nopush entry is needed by conjoint copy
+ StubRoutines::_jshort_disjoint_arraycopy_nopush = nopush_entry;
+ StubRoutines::_jshort_arraycopy = generate_conjoint_short_copy(StubRoutines::_jshort_disjoint_arraycopy_nopush, &nopush_entry);
+ // conjoint nopush entry is needed by generic/unsafe copy
+ StubRoutines::_jshort_arraycopy_nopush = nopush_entry;
- StubRoutines::_jint_disjoint_arraycopy = generate_disjoint_int_oop_copy(StubId::stubgen_jint_disjoint_arraycopy_id, &entry);
- StubRoutines::_jint_arraycopy = generate_conjoint_int_oop_copy(StubId::stubgen_jint_arraycopy_id, entry, &entry_jint_arraycopy);
+ StubRoutines::_jint_disjoint_arraycopy = generate_disjoint_int_oop_copy(StubId::stubgen_jint_disjoint_arraycopy_id, &nopush_entry);
+ // disjoint nopush entry is needed by conjoint copy
+ StubRoutines::_jint_disjoint_arraycopy_nopush = nopush_entry;
+ StubRoutines::_jint_arraycopy = generate_conjoint_int_oop_copy(StubId::stubgen_jint_arraycopy_id, StubRoutines::_jint_disjoint_arraycopy_nopush, &nopush_entry);
+ // conjoint nopush entry is needed by generic/unsafe copy
+ StubRoutines::_jint_arraycopy_nopush = nopush_entry;
+
+ StubRoutines::_jlong_disjoint_arraycopy = generate_disjoint_long_oop_copy(StubId::stubgen_jlong_disjoint_arraycopy_id, &nopush_entry);
+ // disjoint nopush entry is needed by conjoint copy
+ StubRoutines::_jlong_disjoint_arraycopy_nopush = nopush_entry;
+ StubRoutines::_jlong_arraycopy = generate_conjoint_long_oop_copy(StubId::stubgen_jlong_arraycopy_id, StubRoutines::_jlong_disjoint_arraycopy_nopush, &nopush_entry);
+ // conjoint nopush entry is needed by generic/unsafe copy
+ StubRoutines::_jlong_arraycopy_nopush = nopush_entry;
- StubRoutines::_jlong_disjoint_arraycopy = generate_disjoint_long_oop_copy(StubId::stubgen_jlong_disjoint_arraycopy_id, &entry);
- StubRoutines::_jlong_arraycopy = generate_conjoint_long_oop_copy(StubId::stubgen_jlong_arraycopy_id, entry, &entry_jlong_arraycopy);
if (UseCompressedOops) {
- StubRoutines::_oop_disjoint_arraycopy = generate_disjoint_int_oop_copy(StubId::stubgen_oop_disjoint_arraycopy_id, &entry);
- StubRoutines::_oop_arraycopy = generate_conjoint_int_oop_copy(StubId::stubgen_oop_arraycopy_id, entry, &entry_oop_arraycopy);
- StubRoutines::_oop_disjoint_arraycopy_uninit = generate_disjoint_int_oop_copy(StubId::stubgen_oop_disjoint_arraycopy_uninit_id, &entry);
- StubRoutines::_oop_arraycopy_uninit = generate_conjoint_int_oop_copy(StubId::stubgen_oop_arraycopy_uninit_id, entry, nullptr);
+ StubRoutines::_oop_disjoint_arraycopy = generate_disjoint_int_oop_copy(StubId::stubgen_oop_disjoint_arraycopy_id, &nopush_entry);
+ // disjoint nopush entry is needed by conjoint copy
+ StubRoutines::_oop_disjoint_arraycopy_nopush = nopush_entry;
+ StubRoutines::_oop_arraycopy = generate_conjoint_int_oop_copy(StubId::stubgen_oop_arraycopy_id, StubRoutines::_oop_disjoint_arraycopy_nopush, &nopush_entry);
+ // conjoint nopush entry is needed by generic/unsafe copy
+ StubRoutines::_oop_arraycopy_nopush = nopush_entry;
+ StubRoutines::_oop_disjoint_arraycopy_uninit = generate_disjoint_int_oop_copy(StubId::stubgen_oop_disjoint_arraycopy_uninit_id, &nopush_entry);
+ // disjoint nopush entry is needed by conjoint copy
+ StubRoutines::_oop_disjoint_arraycopy_uninit_nopush = nopush_entry;
+ // note that we don't need a returned nopush entry because the
+ // generic/unsafe copy does not cater for uninit arrays.
+ StubRoutines::_oop_arraycopy_uninit = generate_conjoint_int_oop_copy(StubId::stubgen_oop_arraycopy_uninit_id, StubRoutines::_oop_disjoint_arraycopy_uninit_nopush, nullptr);
} else {
- StubRoutines::_oop_disjoint_arraycopy = generate_disjoint_long_oop_copy(StubId::stubgen_oop_disjoint_arraycopy_id, &entry);
- StubRoutines::_oop_arraycopy = generate_conjoint_long_oop_copy(StubId::stubgen_oop_arraycopy_id, entry, &entry_oop_arraycopy);
- StubRoutines::_oop_disjoint_arraycopy_uninit = generate_disjoint_long_oop_copy(StubId::stubgen_oop_disjoint_arraycopy_uninit_id, &entry);
- StubRoutines::_oop_arraycopy_uninit = generate_conjoint_long_oop_copy(StubId::stubgen_oop_arraycopy_uninit_id, entry, nullptr);
+ StubRoutines::_oop_disjoint_arraycopy = generate_disjoint_long_oop_copy(StubId::stubgen_oop_disjoint_arraycopy_id, &nopush_entry);
+ // disjoint nopush entry is needed by conjoint copy
+ StubRoutines::_oop_disjoint_arraycopy_nopush = nopush_entry;
+ StubRoutines::_oop_arraycopy = generate_conjoint_long_oop_copy(StubId::stubgen_oop_arraycopy_id, StubRoutines::_oop_disjoint_arraycopy_nopush, &nopush_entry);
+ // conjoint nopush entry is needed by generic/unsafe copy
+ StubRoutines::_oop_arraycopy_nopush = nopush_entry;
+ StubRoutines::_oop_disjoint_arraycopy_uninit = generate_disjoint_long_oop_copy(StubId::stubgen_oop_disjoint_arraycopy_uninit_id, &nopush_entry);
+ // disjoint nopush entry is needed by conjoint copy
+ StubRoutines::_oop_disjoint_arraycopy_uninit_nopush = nopush_entry;
+ // note that we don't need a returned nopush entry because the
+ // generic/unsafe copy does not cater for uninit arrays.
+ StubRoutines::_oop_arraycopy_uninit = generate_conjoint_long_oop_copy(StubId::stubgen_oop_arraycopy_uninit_id, StubRoutines::_oop_disjoint_arraycopy_uninit_nopush, nullptr);
}
- StubRoutines::_checkcast_arraycopy = generate_checkcast_copy(StubId::stubgen_checkcast_arraycopy_id, &entry_checkcast_arraycopy);
+ StubRoutines::_checkcast_arraycopy = generate_checkcast_copy(StubId::stubgen_checkcast_arraycopy_id, &nopush_entry);
+ // checkcast nopush entry is needed by generic copy
+ StubRoutines::_checkcast_arraycopy_nopush = nopush_entry;
+ // note that we don't need a returned nopush entry because the
+ // generic copy does not cater for uninit arrays.
StubRoutines::_checkcast_arraycopy_uninit = generate_checkcast_copy(StubId::stubgen_checkcast_arraycopy_uninit_id, nullptr);
- StubRoutines::_unsafe_arraycopy = generate_unsafe_copy(entry_jbyte_arraycopy,
- entry_jshort_arraycopy,
- entry_jint_arraycopy,
- entry_jlong_arraycopy);
- StubRoutines::_generic_arraycopy = generate_generic_copy(entry_jbyte_arraycopy,
- entry_jshort_arraycopy,
- entry_jint_arraycopy,
- entry_oop_arraycopy,
- entry_jlong_arraycopy,
- entry_checkcast_arraycopy);
+ StubRoutines::_unsafe_arraycopy = generate_unsafe_copy(StubRoutines::_jbyte_arraycopy_nopush,
+ StubRoutines::_jshort_arraycopy_nopush,
+ StubRoutines::_jint_arraycopy_nopush,
+ StubRoutines::_jlong_arraycopy_nopush);
+ StubRoutines::_generic_arraycopy = generate_generic_copy(StubRoutines::_jbyte_arraycopy_nopush,
+ StubRoutines::_jshort_arraycopy_nopush,
+ StubRoutines::_jint_arraycopy_nopush,
+ StubRoutines::_oop_arraycopy_nopush,
+ StubRoutines::_jlong_arraycopy_nopush,
+ StubRoutines::_checkcast_arraycopy_nopush);
StubRoutines::_jbyte_fill = generate_fill(StubId::stubgen_jbyte_fill_id);
StubRoutines::_jshort_fill = generate_fill(StubId::stubgen_jshort_fill_id);
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_cbrt.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_cbrt.cpp
index c35f1b9f65e..73330dedc0f 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_cbrt.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_cbrt.cpp
@@ -49,142 +49,142 @@
/* Represents 0x7FFFFFFFFFFFFFFF double precision in lower 64 bits*/
ATTRIBUTE_ALIGNED(16) static const juint _ABS_MASK[] =
{
- 4294967295, 2147483647, 0, 0
+ 0xFFFFFFFFUL, 0x7FFFFFFFUL, 0x00000000UL, 0x00000000UL
};
ATTRIBUTE_ALIGNED(4) static const juint _SIG_MASK[] =
{
- 0, 1032192
+ 0x00000000UL, 0x000FC000UL
};
ATTRIBUTE_ALIGNED(4) static const juint _EXP_MASK[] =
{
- 0, 3220176896
+ 0x00000000UL, 0xBFF00000UL
};
ATTRIBUTE_ALIGNED(4) static const juint _EXP_MSK2[] =
{
- 0, 3220193280
+ 0x00000000UL, 0xBFF04000UL
};
ATTRIBUTE_ALIGNED(4) static const juint _EXP_MSK3[] =
{
- 4294967295, 1048575
+ 0xFFFFFFFFUL, 0x000FFFFFUL
};
ATTRIBUTE_ALIGNED(4) static const juint _SCALE63[] =
{
- 0, 1138753536
+ 0x00000000UL, 0x43E00000UL
};
ATTRIBUTE_ALIGNED(4) static const juint _ZERON[] =
{
- 0, 2147483648
+ 0x00000000UL, 0x80000000UL
};
ATTRIBUTE_ALIGNED(4) static const juint _INF[] =
{
- 0, 2146435072
+ 0x00000000UL, 0x7FF00000UL
};
ATTRIBUTE_ALIGNED(4) static const juint _NEG_INF[] =
{
- 0, 4293918720
+ 0x00000000UL, 0xFFF00000UL
};
ATTRIBUTE_ALIGNED(16) static const juint _coeff_table[] =
{
- 1553778919, 3213899486, 3534952507, 3215266280, 1646371399,
- 3214412045, 477218588, 3216798151, 3582521621, 1066628362,
- 1007461464, 1068473053, 889629714, 1067378449, 1431655765,
- 1070945621
+ 0x5C9CC8E7UL, 0xBF9036DEUL, 0xD2B3183BUL, 0xBFA511E8UL, 0x6221A247UL,
+ 0xBF98090DUL, 0x1C71C71CUL, 0xBFBC71C7UL, 0xD588F115UL, 0x3F93750AUL,
+ 0x3C0CA458UL, 0x3FAF9ADDUL, 0x3506AC12UL, 0x3F9EE711UL, 0x55555555UL,
+ 0x3FD55555UL
};
ATTRIBUTE_ALIGNED(4) static const juint _rcp_table[] =
{
- 528611360, 3220144632, 2884679527, 3220082993, 1991868891, 3220024928,
- 2298714891, 3219970134, 58835168, 3219918343, 3035110223, 3219869313,
- 1617585086, 3219822831, 2500867033, 3219778702, 4241943008, 3219736752,
- 258732970, 3219696825, 404232216, 3219658776, 2172167368, 3219622476,
- 1544257904, 3219587808, 377579543, 3219554664, 1616385542, 3219522945,
- 813783277, 3219492562, 3940743189, 3219463431, 2689777499, 3219435478,
- 1700977147, 3219408632, 3169102082, 3219382828, 327235604, 3219358008,
- 1244336319, 3219334115, 1300311200, 3219311099, 3095471925, 3219288912,
- 2166487928, 3219267511, 2913108253, 3219246854, 293672978, 3219226904,
- 288737297, 3219207624, 1810275472, 3219188981, 174592167, 3219170945,
- 3539053052, 3219153485, 2164392968, 3219136576
+ 0x1F81F820UL, 0xBFEF81F8UL, 0xABF0B767UL, 0xBFEE9131UL, 0x76B981DBUL, 0xBFEDAE60UL,
+ 0x89039B0BUL, 0xBFECD856UL, 0x0381C0E0UL, 0xBFEC0E07UL, 0xB4E81B4FUL, 0xBFEB4E81UL,
+ 0x606A63BEUL, 0xBFEA98EFUL, 0x951033D9UL, 0xBFE9EC8EUL, 0xFCD6E9E0UL, 0xBFE948B0UL,
+ 0x0F6BF3AAUL, 0xBFE8ACB9UL, 0x18181818UL, 0xBFE81818UL, 0x8178A4C8UL, 0xBFE78A4CUL,
+ 0x5C0B8170UL, 0xBFE702E0UL, 0x16816817UL, 0xBFE68168UL, 0x60581606UL, 0xBFE60581UL,
+ 0x308158EDUL, 0xBFE58ED2UL, 0xEAE2F815UL, 0xBFE51D07UL, 0xA052BF5BUL, 0xBFE4AFD6UL,
+ 0x6562D9FBUL, 0xBFE446F8UL, 0xBCE4A902UL, 0xBFE3E22CUL, 0x13813814UL, 0xBFE38138UL,
+ 0x4A2B10BFUL, 0xBFE323E3UL, 0x4D812CA0UL, 0xBFE2C9FBUL, 0xB8812735UL, 0xBFE27350UL,
+ 0x8121FB78UL, 0xBFE21FB7UL, 0xADA2811DUL, 0xBFE1CF06UL, 0x11811812UL, 0xBFE18118UL,
+ 0x1135C811UL, 0xBFE135C8UL, 0x6BE69C90UL, 0xBFE0ECF5UL, 0x0A6810A7UL, 0xBFE0A681UL,
+ 0xD2F1A9FCUL, 0xBFE0624DUL, 0x81020408UL, 0xBFE02040UL
};
ATTRIBUTE_ALIGNED(4) static const juint _cbrt_table[] =
{
- 572345495, 1072698681, 1998204467, 1072709382, 3861501553, 1072719872,
- 2268192434, 1072730162, 2981979308, 1072740260, 270859143, 1072750176,
- 2958651392, 1072759916, 313113243, 1072769490, 919449400, 1072778903,
- 2809328903, 1072788162, 2222981587, 1072797274, 2352530781, 1072806244,
- 594152517, 1072815078, 1555767199, 1072823780, 4282421314, 1072832355,
- 2355578597, 1072840809, 1162590619, 1072849145, 797864051, 1072857367,
- 431273680, 1072865479, 2669831148, 1072873484, 733477752, 1072881387,
- 4280220604, 1072889189, 801961634, 1072896896, 2915370760, 1072904508,
- 1159613482, 1072912030, 2689944798, 1072919463, 1248687822, 1072926811,
- 2967951030, 1072934075, 630170432, 1072941259, 3760898254, 1072948363,
- 0, 1072955392, 2370273294, 1072962345, 1261754802, 1072972640,
- 546334065, 1072986123, 1054893830, 1072999340, 1571187597, 1073012304,
- 1107975175, 1073025027, 3606909377, 1073037519, 1113616747, 1073049792,
- 4154744632, 1073061853, 3358931423, 1073073713, 4060702372, 1073085379,
- 747576176, 1073096860, 3023138255, 1073108161, 1419988548, 1073119291,
- 1914185305, 1073130255, 294389948, 1073141060, 3761802570, 1073151710,
- 978281566, 1073162213, 823148820, 1073172572, 2420954441, 1073182792,
- 3815449908, 1073192878, 2046058587, 1073202835, 1807524753, 1073212666,
- 2628681401, 1073222375, 3225667357, 1073231966, 1555307421, 1073241443,
- 3454043099, 1073250808, 1208137896, 1073260066, 3659916772, 1073269218,
- 1886261264, 1073278269, 3593647839, 1073287220, 3086012205, 1073296075,
- 2769796922, 1073304836, 888716057, 1073317807, 2201465623, 1073334794,
- 164369365, 1073351447, 3462666733, 1073367780, 2773905457, 1073383810,
- 1342879088, 1073399550, 2543933975, 1073415012, 1684477781, 1073430209,
- 3532178543, 1073445151, 1147747300, 1073459850, 1928031793, 1073474314,
- 2079717015, 1073488553, 4016765315, 1073502575, 3670431139, 1073516389,
- 3549227225, 1073530002, 11637607, 1073543422, 588220169, 1073556654,
- 2635407503, 1073569705, 2042029317, 1073582582, 1925128962, 1073595290,
- 4136375664, 1073607834, 759964600, 1073620221, 4257606771, 1073632453,
- 297278907, 1073644538, 3655053093, 1073656477, 2442253172, 1073668277,
- 1111876799, 1073679941, 3330973139, 1073691472, 3438879452, 1073702875,
- 3671565478, 1073714153, 1317849547, 1073725310, 1642364115, 1073736348
+ 0x221D4C97UL, 0x3FF01539UL, 0x771A2E33UL, 0x3FF03F06UL, 0xE629D671UL, 0x3FF06800UL,
+ 0x8731DEB2UL, 0x3FF09032UL, 0xB1BD64ACUL, 0x3FF0B7A4UL, 0x1024FB87UL, 0x3FF0DE60UL,
+ 0xB0597000UL, 0x3FF1046CUL, 0x12A9BA9BUL, 0x3FF129D2UL, 0x36CDAF38UL, 0x3FF14E97UL,
+ 0xA772F507UL, 0x3FF172C2UL, 0x848001D3UL, 0x3FF1965AUL, 0x8C38C55DUL, 0x3FF1B964UL,
+ 0x236A0C45UL, 0x3FF1DBE6UL, 0x5CBB1F9FUL, 0x3FF1FDE4UL, 0xFF409042UL, 0x3FF21F63UL,
+ 0x8C6746E5UL, 0x3FF24069UL, 0x454BB99BUL, 0x3FF260F9UL, 0x2F8E7073UL, 0x3FF28117UL,
+ 0x19B4B6D0UL, 0x3FF2A0C7UL, 0x9F2263ECUL, 0x3FF2C00CUL, 0x2BB7FB78UL, 0x3FF2DEEBUL,
+ 0xFF1EFBBCUL, 0x3FF2FD65UL, 0x2FCCF6A2UL, 0x3FF31B80UL, 0xADC50708UL, 0x3FF3393CUL,
+ 0x451E4C2AUL, 0x3FF3569EUL, 0xA0554CDEUL, 0x3FF373A7UL, 0x4A6D76CEUL, 0x3FF3905BUL,
+ 0xB0E756B6UL, 0x3FF3ACBBUL, 0x258FA340UL, 0x3FF3C8CBUL, 0xE02AC0CEUL, 0x3FF3E48BUL,
+ 0x00000000UL, 0x3FF40000UL, 0x8D47800EUL, 0x3FF41B29UL, 0x4B34D9B2UL, 0x3FF44360UL,
+ 0x20906571UL, 0x3FF4780BUL, 0x3EE06706UL, 0x3FF4ABACUL, 0x5DA66B8DUL, 0x3FF4DE50UL,
+ 0x420A5C07UL, 0x3FF51003UL, 0xD6FD11C1UL, 0x3FF540CFUL, 0x4260716BUL, 0x3FF570C0UL,
+ 0xF7A45F38UL, 0x3FF59FDDUL, 0xC83539DFUL, 0x3FF5CE31UL, 0xF20966A4UL, 0x3FF5FBC3UL,
+ 0x2C8F1B70UL, 0x3FF6289CUL, 0xB4316DCFUL, 0x3FF654C1UL, 0x54A34E44UL, 0x3FF6803BUL,
+ 0x72182659UL, 0x3FF6AB0FUL, 0x118C08BCUL, 0x3FF6D544UL, 0xE0388D4AUL, 0x3FF6FEDEUL,
+ 0x3A4F645EUL, 0x3FF727E5UL, 0x31104114UL, 0x3FF7505CUL, 0x904CD549UL, 0x3FF77848UL,
+ 0xE36B2534UL, 0x3FF79FAEUL, 0x79F4605BUL, 0x3FF7C693UL, 0x6BBCA391UL, 0x3FF7ECFAUL,
+ 0x9CAE7EB9UL, 0x3FF812E7UL, 0xC043C71DUL, 0x3FF8385EUL, 0x5CB41B9DUL, 0x3FF85D63UL,
+ 0xCDE083DBUL, 0x3FF881F8UL, 0x4802B8A8UL, 0x3FF8A622UL, 0xDA25E5E4UL, 0x3FF8C9E2UL,
+ 0x706E1010UL, 0x3FF8ED3DUL, 0xD632B6DFUL, 0x3FF91034UL, 0xB7F0CF2DUL, 0x3FF932CBUL,
+ 0xA517BF3AUL, 0x3FF95504UL, 0x34F8BB19UL, 0x3FF987AFUL, 0x8337B317UL, 0x3FF9CA0AUL,
+ 0x09CC13D5UL, 0x3FFA0B17UL, 0xCE6419EDUL, 0x3FFA4AE4UL, 0xA5567031UL, 0x3FFA8982UL,
+ 0x500AB570UL, 0x3FFAC6FEUL, 0x97A15A17UL, 0x3FFB0364UL, 0x64671755UL, 0x3FFB3EC1UL,
+ 0xD288C46FUL, 0x3FFB791FUL, 0x44693BE4UL, 0x3FFBB28AUL, 0x72EB6E31UL, 0x3FFBEB0AUL,
+ 0x7BF5F697UL, 0x3FFC22A9UL, 0xEF6AF983UL, 0x3FFC596FUL, 0xDAC655A3UL, 0x3FFC8F65UL,
+ 0xD38CE8D9UL, 0x3FFCC492UL, 0x00B19367UL, 0x3FFCF8FEUL, 0x230F8709UL, 0x3FFD2CAEUL,
+ 0x9D15208FUL, 0x3FFD5FA9UL, 0x79B6E505UL, 0x3FFD91F6UL, 0x72BF2302UL, 0x3FFDC39AUL,
+ 0xF68C1570UL, 0x3FFDF49AUL, 0x2D4C23B8UL, 0x3FFE24FDUL, 0xFDC5EC73UL, 0x3FFE54C5UL,
+ 0x11B81DBBUL, 0x3FFE83FAUL, 0xD9DBAF25UL, 0x3FFEB29DUL, 0x9191D374UL, 0x3FFEE0B5UL,
+ 0x4245E4BFUL, 0x3FFF0E45UL, 0xC68A9DD3UL, 0x3FFF3B50UL, 0xCCF922DCUL, 0x3FFF67DBUL,
+ 0xDAD7A4A6UL, 0x3FFF93E9UL, 0x4E8CC9CBUL, 0x3FFFBF7EUL, 0x61E47CD3UL, 0x3FFFEA9CUL
};
ATTRIBUTE_ALIGNED(4) static const juint _D_table[] =
{
- 4050900474, 1014427190, 1157977860, 1016444461, 1374568199, 1017271387,
- 2809163288, 1016882676, 3742377377, 1013168191, 3101606597, 1017541672,
- 65224358, 1017217597, 2691591250, 1017266643, 4020758549, 1017689313,
- 1316310992, 1018030788, 1031537856, 1014090882, 3261395239, 1016413641,
- 886424999, 1016313335, 3114776834, 1014195875, 1681120620, 1017825416,
- 1329600273, 1016625740, 465474623, 1017097119, 4251633980, 1017169077,
- 1986990133, 1017710645, 752958613, 1017159641, 2216216792, 1018020163,
- 4282860129, 1015924861, 1557627859, 1016039538, 3889219754, 1018086237,
- 3684996408, 1017353275, 723532103, 1017717141, 2951149676, 1012528470,
- 831890937, 1017830553, 1031212645, 1017387331, 2741737450, 1017604974,
- 2863311531, 1003776682, 4276736099, 1013153088, 4111778382, 1015673686,
- 1728065769, 1016413986, 2708718031, 1018078833, 1069335005, 1015291224,
- 700037144, 1016482032, 2904566452, 1017226861, 4074156649, 1017622651,
- 25019565, 1015245366, 3601952608, 1015771755, 3267129373, 1017904664,
- 503203103, 1014921629, 2122011730, 1018027866, 3927295461, 1014189456,
- 2790625147, 1016024251, 1330460186, 1016940346, 4033568463, 1015538390,
- 3695818227, 1017509621, 257573361, 1017208868, 3227697852, 1017337964,
- 234118548, 1017169577, 4009025803, 1017278524, 1948343394, 1017749310,
- 678398162, 1018144239, 3083864863, 1016669086, 2415453452, 1017890370,
- 175467344, 1017330033, 3197359580, 1010339928, 2071276951, 1015941358,
- 268372543, 1016737773, 938132959, 1017389108, 1816750559, 1017337448,
- 4119203749, 1017152174, 2578653878, 1013108497, 2470331096, 1014678606,
- 123855735, 1016553320, 1265650889, 1014782687, 3414398172, 1017182638,
- 1040773369, 1016158401, 3483628886, 1016886550, 4140499405, 1016191425,
- 3893477850, 1016964495, 3935319771, 1009634717, 2978982660, 1015027112,
- 2452709923, 1017990229, 3190365712, 1015835149, 4237588139, 1015832925,
- 2610678389, 1017962711, 2127316774, 1017405770, 824267502, 1017959463,
- 2165924042, 1017912225, 2774007076, 1013257418, 4123916326, 1017582284,
- 1976417958, 1016959909, 4092806412, 1017711279, 119251817, 1015363631,
- 3475418768, 1017675415, 1972580503, 1015470684, 815541017, 1017517969,
- 2429917451, 1017397776, 4062888482, 1016749897, 68284153, 1017925678,
- 2207779246, 1016320298, 1183466520, 1017408657, 143326427, 1017060403
+ 0xF173D5FAUL, 0x3C76EE36UL, 0x45055704UL, 0x3C95B62DUL, 0x51EE3F07UL, 0x3CA2545BUL,
+ 0xA7706E18UL, 0x3C9C65F4UL, 0xDF1025A1UL, 0x3C63B83FUL, 0xB8DEC2C5UL, 0x3CA67428UL,
+ 0x03E33EA6UL, 0x3CA1823DUL, 0xA06E6C52UL, 0x3CA241D3UL, 0xEFA7E815UL, 0x3CA8B4E1UL,
+ 0x4E754FD0UL, 0x3CADEAC4UL, 0x3D7C04C0UL, 0x3C71CC82UL, 0xC264F127UL, 0x3C953DC9UL,
+ 0x34D5C5A7UL, 0x3C93B5F7UL, 0xB9A7B902UL, 0x3C7366A3UL, 0x6433DD6CUL, 0x3CAAC888UL,
+ 0x4F401711UL, 0x3C987A4CUL, 0x1BBE943FUL, 0x3C9FAB9FUL, 0xFD6AC93CUL, 0x3CA0C4B5UL,
+ 0x766F1035UL, 0x3CA90835UL, 0x2CE13C95UL, 0x3CA09FD9UL, 0x8418C8D8UL, 0x3CADC143UL,
+ 0xFF474261UL, 0x3C8DC87DUL, 0x5CD783D3UL, 0x3C8F8872UL, 0xE7D0C8AAUL, 0x3CAEC35DUL,
+ 0xDBA49538UL, 0x3CA3943BUL, 0x2B203947UL, 0x3CA92195UL, 0xAFE6F86CUL, 0x3C59F556UL,
+ 0x3195A5F9UL, 0x3CAADC99UL, 0x3D770E65UL, 0x3CA41943UL, 0xA36B97EAUL, 0x3CA76B6EUL,
+ 0xAAAAAAABUL, 0x3BD46AAAUL, 0xFEE9D063UL, 0x3C637D40UL, 0xF514C24EUL, 0x3C89F356UL,
+ 0x670030E9UL, 0x3C953F22UL, 0xA173C1CFUL, 0x3CAEA671UL, 0x3FBCC1DDUL, 0x3C841D58UL,
+ 0x29B9B818UL, 0x3C9648F0UL, 0xAD202AB4UL, 0x3CA1A66DUL, 0xF2D6B269UL, 0x3CA7B07BUL,
+ 0x017DC4ADUL, 0x3C836A36UL, 0xD6B16F60UL, 0x3C8B726BUL, 0xC2BC701DUL, 0x3CABFE18UL,
+ 0x1DFE451FUL, 0x3C7E799DUL, 0x7E7B5452UL, 0x3CADDF5AUL, 0xEA15C5E5UL, 0x3C734D90UL,
+ 0xA6558F7BUL, 0x3C8F4CBBUL, 0x4F4D361AUL, 0x3C9D473AUL, 0xF06B5ECFUL, 0x3C87E2D6UL,
+ 0xDC49B5F3UL, 0x3CA5F6F5UL, 0x0F5A41F1UL, 0x3CA16024UL, 0xC062C2BCUL, 0x3CA3586CUL,
+ 0x0DF45D94UL, 0x3CA0C6A9UL, 0xEEF4E10BUL, 0x3CA2703CUL, 0x74215C62UL, 0x3CA99F3EUL,
+ 0x286F88D2UL, 0x3CAFA5EFUL, 0xB7D00B1FUL, 0x3C99239EUL, 0x8FF8E50CUL, 0x3CABC642UL,
+ 0x0A756B50UL, 0x3CA33971UL, 0xBE93D5DCUL, 0x3C389058UL, 0x7B752D97UL, 0x3C8E08EEUL,
+ 0x0FFF0A3FUL, 0x3C9A2FEDUL, 0x37EAC5DFUL, 0x3CA42034UL, 0x6C4969DFUL, 0x3CA35668UL,
+ 0xF5860FA5UL, 0x3CA082AEUL, 0x99B322B6UL, 0x3C62CF11UL, 0x933E42D8UL, 0x3C7AC44EUL,
+ 0x0761E377UL, 0x3C975F68UL, 0x4B704CC9UL, 0x3C7C5ADFUL, 0xCB8394DCUL, 0x3CA0F9AEUL,
+ 0x3E08F0F9UL, 0x3C9158C1UL, 0xCFA3F556UL, 0x3C9C7516UL, 0xF6CB01CDUL, 0x3C91D9C1UL,
+ 0xE811C1DAUL, 0x3C9DA58FUL, 0xEA9036DBUL, 0x3C2DCD9DUL, 0xB18FAB04UL, 0x3C8015A8UL,
+ 0x92316223UL, 0x3CAD4C55UL, 0xBE291E10UL, 0x3C8C6A0DUL, 0xFC9476ABUL, 0x3C8C615DUL,
+ 0x9B9BCA75UL, 0x3CACE0D7UL, 0x7ECC4726UL, 0x3CA4614AUL, 0x312152EEUL, 0x3CACD427UL,
+ 0x811960CAUL, 0x3CAC1BA1UL, 0xA557FD24UL, 0x3C6514CAUL, 0xF5CDF826UL, 0x3CA712CCUL,
+ 0x75CDBEA6UL, 0x3C9D93A5UL, 0xF3F3450CUL, 0x3CA90AAFUL, 0x071BA369UL, 0x3C85382FUL,
+ 0xCF26AE90UL, 0x3CA87E97UL, 0x75933097UL, 0x3C86DA5CUL, 0x309C2B19UL, 0x3CA61791UL,
+ 0x90D5990BUL, 0x3CA44210UL, 0xF22AC222UL, 0x3C9A5F49UL, 0x0411EEF9UL, 0x3CAC502EUL,
+ 0x839809AEUL, 0x3C93D12AUL, 0x468A4418UL, 0x3CA46C91UL, 0x088AFCDBUL, 0x3C9F1C33UL
};
#define __ _masm->
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_tanh.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_tanh.cpp
index 372c4898f37..dce4fbfc455 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64_tanh.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_tanh.cpp
@@ -77,226 +77,226 @@
ATTRIBUTE_ALIGNED(4) static const juint _HALFMASK[] =
{
- 4160749568, 2147483647
+ 0xF8000000UL, 0x7FFFFFFFUL
};
ATTRIBUTE_ALIGNED(4) static const juint _ONEMASK[] =
{
- 0, 1072693248
+ 0x00000000UL, 0x3FF00000UL
};
ATTRIBUTE_ALIGNED(4) static const juint _TWOMASK[] =
{
- 0, 1073741824
+ 0x00000000UL, 0x40000000UL
};
ATTRIBUTE_ALIGNED(16) static const juint _MASK3[] =
{
- 0, 4294967280, 0, 4294967280
+ 0x00000000UL, 0xFFFFFFF0UL, 0x00000000UL, 0xFFFFFFF0UL
};
ATTRIBUTE_ALIGNED(16) static const juint _RMASK[] =
{
- 4294705152, 4294967295, 4294705152, 4294967295
+ 0xFFFC0000UL, 0xFFFFFFFFUL, 0xFFFC0000UL, 0xFFFFFFFFUL
};
ATTRIBUTE_ALIGNED(16) static const juint _L2E[] =
{
- 1610612736, 1082594631, 4166901572, 1055174155
+ 0x60000000UL, 0x40871547UL, 0xF85DDF44UL, 0x3EE4AE0BUL
};
ATTRIBUTE_ALIGNED(16) static const juint _Shifter[] =
{
- 0, 1127743488, 0, 3275227136
+ 0x00000000UL, 0x43380000UL, 0x00000000UL, 0xC3380000UL
};
ATTRIBUTE_ALIGNED(16) static const juint _cv[] =
{
- 3884607281, 3168131199, 3607404735, 3190582024, 1874480759,
- 1032041131, 4286760334, 1053736893, 4277811695, 3211144770,
- 0, 0
+ 0xE78A6731UL, 0xBCD5D87FUL, 0xD704A0BFUL, 0xBE2C6B08UL, 0x6FBA4E77UL,
+ 0x3D83B2ABUL, 0xFF82C58EUL, 0x3ECEBFBDUL, 0xFEFA39EFUL, 0xBF662E42UL,
+ 0x00000000UL, 0x00000000UL
};
ATTRIBUTE_ALIGNED(4) static const juint _pv[] =
{
- 236289503, 1064135997, 463583772, 3215696314, 1441186365,
- 3212977891, 286331153, 1069617425, 2284589306, 1066820852,
- 1431655765, 3218429269
+ 0x0E157DDFUL, 0x3F6D6D3DUL, 0x1BA1BA1CUL, 0xBFABA1BAUL, 0x55E6C23DUL,
+ 0xBF8226E3UL, 0x11111111UL, 0x3FC11111UL, 0x882C10FAUL, 0x3F9664F4UL,
+ 0x55555555UL, 0xBFD55555UL
};
ATTRIBUTE_ALIGNED(16) static const juint _T2_neg_f[] =
{
- 0, 1072693248, 0, 0, 1797923801, 1072687577,
- 1950547427, 1013229059, 730821105, 1072681922, 2523232743, 1012067188,
- 915592468, 1072676282, 352947894, 3161024371, 2174652632, 1072670657,
- 4087714590, 1014450259, 35929225, 1072665048, 2809788041, 3159436968,
- 2912730644, 1072659453, 3490067722, 3163405074, 2038973688, 1072653874,
- 892941374, 1016046459, 1533953344, 1072648310, 769171851, 1015665633,
- 1222472308, 1072642761, 1054357470, 3161021018, 929806999, 1072637227,
- 3205336643, 1015259557, 481706282, 1072631708, 1696079173, 3162710528,
- 3999357479, 1072626203, 2258941616, 1015924724, 2719515920, 1072620714,
- 2760332941, 1015137933, 764307441, 1072615240, 3021057420, 3163329523,
- 2256325230, 1072609780, 580117746, 1015317295, 2728693978, 1072604335,
- 396109971, 3163462691, 2009970496, 1072598905, 2159039665, 3162572948,
- 4224142467, 1072593489, 3389820386, 1015207202, 610758006, 1072588089,
- 1965209397, 3161866232, 3884662774, 1072582702, 2158611599, 1014210185,
- 991358482, 1072577331, 838715019, 3163157668, 351641897, 1072571974,
- 2172261526, 3163010599, 1796832535, 1072566631, 3176955716, 3160585513,
- 863738719, 1072561303, 1326992220, 3162613197, 1679558232, 1072555989,
- 2390342287, 3163333970, 4076975200, 1072550689, 2029000899, 1015208535,
- 3594158869, 1072545404, 2456521700, 3163256561, 64696965, 1072540134,
- 1768797490, 1015816960, 1912561781, 1072534877, 3147495102, 1015678253,
- 382305176, 1072529635, 2347622376, 3162578625, 3898795731, 1072524406,
- 1249994144, 1011869818, 3707479175, 1072519192, 3613079303, 1014164738,
- 3939148246, 1072513992, 3210352148, 1015274323, 135105010, 1072508807,
- 1906148728, 3163375739, 721996136, 1072503635, 563754734, 1015371318,
- 1242007932, 1072498477, 1132034716, 3163339831, 1532734324, 1072493333,
- 3094216535, 3163162857, 1432208378, 1072488203, 1401068914, 3162363963,
- 778901109, 1072483087, 2248183955, 3161268751, 3706687593, 1072477984,
- 3521726940, 1013253067, 1464976603, 1072472896, 3507292405, 3161977534,
- 2483480501, 1072467821, 1216371780, 1013034172, 2307442995, 1072462760,
- 3190117721, 3162404539, 777507147, 1072457713, 4282924205, 1015187533,
- 2029714210, 1072452679, 613660079, 1015099143, 1610600570, 1072447659,
- 3766732298, 1015760183, 3657065772, 1072442652, 399025623, 3162957078,
- 3716502172, 1072437659, 2303740125, 1014042725, 1631695677, 1072432680,
- 2717633076, 3162344026, 1540824585, 1072427714, 1064017011, 3163487690,
- 3287523847, 1072422761, 1625971539, 3157009955, 2420883922, 1072417822,
- 2049810052, 1014119888, 3080351519, 1072412896, 3379126788, 3157218001,
- 815859274, 1072407984, 240396590, 3163487443, 4062661092, 1072403084,
- 1422616006, 3163255318, 4076559943, 1072398198, 2119478331, 3160758351,
- 703710506, 1072393326, 1384660846, 1015195891, 2380618042, 1072388466,
- 3149557219, 3163320799, 364333489, 1072383620, 3923737744, 3161421373,
- 3092190715, 1072378786, 814012168, 3159523422, 1822067026, 1072373966,
- 1241994956, 1015340290, 697153126, 1072369159, 1283515429, 3163283189,
- 3861050111, 1072364364, 254893773, 3162813180, 2572866477, 1072359583,
- 878562433, 1015521741, 977020788, 1072354815, 3065100517, 1015541563,
- 3218338682, 1072350059, 3404164304, 3162477108, 557149882, 1072345317,
- 3672720709, 1014537265, 1434058175, 1072340587, 251133233, 1015085769,
- 1405169241, 1072335870, 2998539689, 3162830951, 321958744, 1072331166,
- 3401933767, 1015794558, 2331271250, 1072326474, 812057446, 1012207446,
- 2990417245, 1072321795, 3683467745, 3163369326, 2152073944, 1072317129,
- 1486860576, 3163203456, 3964284211, 1072312475, 2111583915, 1015427164,
- 3985553595, 1072307834, 4002146062, 1015834136, 2069751141, 1072303206,
- 1562170675, 3162724681, 2366108318, 1072298590, 2867985102, 3161762254,
- 434316067, 1072293987, 2028358766, 1013458122, 424392917, 1072289396,
- 2749202995, 3162838718, 2191782032, 1072284817, 2960257726, 1013742662,
- 1297350157, 1072280251, 1308022040, 3163412558, 1892288442, 1072275697,
- 2446255666, 3162600381, 3833209506, 1072271155, 2722920684, 1013754842,
- 2682146384, 1072266626, 2082178513, 3163363419, 2591453363, 1072262109,
- 2132396182, 3159074198, 3418903055, 1072257604, 2527457337, 3160820604,
- 727685349, 1072253112, 2038246809, 3162358742, 2966275557, 1072248631,
- 2176155324, 3159842759, 1403662306, 1072244163, 2788809599, 3161671007,
- 194117574, 1072239707, 777528612, 3163412089, 3492293770, 1072235262,
- 2248032210, 1015386826, 2568320822, 1072230830, 2732824428, 1014352915,
- 1577608921, 1072226410, 1875489510, 3162968394, 380978316, 1072222002,
- 854188970, 3160462686, 3134592888, 1072217605, 4232266862, 1015991134,
- 1110089947, 1072213221, 1451641639, 1015474673, 2759350287, 1072208848,
- 1148526634, 1015894933, 3649726105, 1072204487, 4085036346, 1015649474,
- 3643909174, 1072200138, 3537586109, 1014354647, 2604962541, 1072195801,
- 2614425274, 3163539192, 396319521, 1072191476, 4172420816, 3159074632,
- 1176749997, 1072187162, 2738998779, 3162035844, 515457527, 1072182860,
- 836709333, 1015651226, 2571947539, 1072178569, 3558159064, 3163376669,
- 2916157145, 1072174290, 219487565, 1015309367, 1413356050, 1072170023,
- 1651349291, 3162668166, 2224145553, 1072165767, 3482522030, 3161489169,
- 919555682, 1072161523, 3121969534, 1012948226, 1660913392, 1072157290,
- 4218599604, 1015135707, 19972402, 1072153069, 3507899862, 1016009292,
- 158781403, 1072148859, 2221464712, 3163286453, 1944781191, 1072144660,
- 3993278767, 3161724279, 950803702, 1072140473, 1655364926, 1015237032,
- 1339972927, 1072136297, 167908909, 1015572152, 2980802057, 1072132132,
- 378619896, 1015773303, 1447192521, 1072127979, 1462857171, 3162514521,
- 903334909, 1072123837, 1636462108, 1015039997, 1218806132, 1072119706,
- 1818613052, 3162548441, 2263535754, 1072115586, 752233586, 3162639008,
- 3907805044, 1072111477, 2257091225, 3161550407, 1727278727, 1072107380,
- 3562710623, 1011471940, 4182873220, 1072103293, 629542646, 3161996303,
- 2555984613, 1072099218, 2652555442, 3162552692, 1013258799, 1072095154,
- 1748797611, 3160129082, 3721688645, 1072091100, 3069276937, 1015839401,
- 1963711167, 1072087058, 1744767757, 3160574294, 4201977662, 1072083026,
- 748330254, 1013594357, 1719614413, 1072079006, 330458198, 3163282740,
- 2979960120, 1072074996, 2599109725, 1014498493, 3561793907, 1072070997,
- 1157054053, 1011890350, 3339203574, 1072067009, 1483497780, 3162408754,
- 2186617381, 1072063032, 2270764084, 3163272713, 4273770423, 1072059065,
- 3383180809, 3163218901, 885834528, 1072055110, 1973258547, 3162261564,
- 488188413, 1072051165, 3199821029, 1015564048, 2956612997, 1072047230,
- 2118169751, 3162735553, 3872257780, 1072043306, 1253592103, 1015958334,
- 3111574537, 1072039393, 2606161479, 3162759746, 551349105, 1072035491,
- 3821916050, 3162106589, 363667784, 1072031599, 813753950, 1015785209,
- 2425981843, 1072027717, 2830390851, 3163346599, 2321106615, 1072023846,
- 2171176610, 1009535771, 4222122499, 1072019985, 1277378074, 3163256737,
- 3712504873, 1072016135, 88491949, 1015427660, 671025100, 1072012296,
- 3832014351, 3163022030, 3566716925, 1072008466, 1536826856, 1014142433,
- 3689071823, 1072004647, 2321004996, 3162552716, 917841882, 1072000839,
- 18715565, 1015659308, 3723038930, 1071997040, 378465264, 3162569582,
- 3395129871, 1071993252, 4025345435, 3162335388, 4109806887, 1071989474,
- 422403966, 1014469229, 1453150082, 1071985707, 498154669, 3161488062,
- 3896463087, 1071981949, 1139797873, 3161233805, 2731501122, 1071978202,
- 1774031855, 3162470021, 2135241198, 1071974465, 1236747871, 1013589147,
- 1990012071, 1071970738, 3529070563, 3162813193, 2178460671, 1071967021,
- 777878098, 3162842493, 2583551245, 1071963314, 3161094195, 1015606491,
- 3088564500, 1071959617, 1762311517, 1015045673, 3577096743, 1071955930,
- 2951496418, 1013793687, 3933059031, 1071952253, 2133366768, 3161531832,
- 4040676318, 1071948586, 4090609238, 1015663458, 3784486610, 1071944929,
- 1581883040, 3161698953, 3049340112, 1071941282, 3062915824, 1013170595,
- 1720398391, 1071937645, 3980678963, 3163300080, 3978100823, 1071934017,
- 3513027190, 1015845963, 1118294578, 1071930400, 2197495694, 3159909401,
- 1617004845, 1071926792, 82804944, 1010342778, 1065662932, 1071923194,
- 2533670915, 1014530238, 3645941911, 1071919605, 3814685081, 3161573341,
- 654919306, 1071916027, 3232961757, 3163047469, 569847338, 1071912458,
- 472945272, 3159290729, 3278348324, 1071908898, 3069497416, 1014750712,
- 78413852, 1071905349, 4183226867, 3163017251, 3743175029, 1071901808,
- 2072812490, 3162175075, 1276261410, 1071898278, 300981948, 1014684169,
- 1156440435, 1071894757, 2351451249, 1013967056, 3272845541, 1071891245,
- 928852419, 3163488248, 3219942644, 1071887743, 3798990616, 1015368806,
- 887463927, 1071884251, 3596744163, 3160794166, 460407023, 1071880768,
- 4237175092, 3163138469, 1829099622, 1071877294, 1016661181, 3163461005,
- 589198666, 1071873830, 2664346172, 3163157962, 926591435, 1071870375,
- 3208833762, 3162913514, 2732492859, 1071866929, 2691479646, 3162255684,
- 1603444721, 1071863493, 1548633640, 3162201326, 1726216749, 1071860066,
- 2466808228, 3161676405, 2992903935, 1071856648, 2218154406, 1015228193,
- 1000925746, 1071853240, 1018491672, 3163309544, 4232894513, 1071849840,
- 2383938684, 1014668519, 3991843581, 1071846450, 4092853457, 1014585763,
- 171030293, 1071843070, 3526460132, 1014428778, 1253935211, 1071839698,
- 1395382931, 3159702613, 2839424854, 1071836335, 1171596163, 1013041679,
- 526652809, 1071832982, 4223459736, 1015879375, 2799960843, 1071829637,
- 1423655381, 1015022151, 964107055, 1071826302, 2800439588, 3162833221,
- 3504003472, 1071822975, 3594001060, 3157330652, 1724976915, 1071819658,
- 420909223, 3163117379, 4112506593, 1071816349, 2947355221, 1014371048,
- 1972484976, 1071813050, 675290301, 3161640050, 3790955393, 1071809759,
- 2352942462, 3163180090, 874372905, 1071806478, 100263788, 1015940732,
- 1709341917, 1071803205, 2571168217, 1014152499, 1897844341, 1071799941,
- 1254300460, 1015275938, 1337108031, 1071796686, 3203724452, 1014677845,
- 4219606026, 1071793439, 2434574742, 1014681548, 1853186616, 1071790202,
- 3066496371, 1015656574, 2725843665, 1071786973, 1433917087, 1014838523,
- 2440944790, 1071783753, 2492769774, 1014147454, 897099801, 1071780542,
- 754756297, 1015241005, 2288159958, 1071777339, 2169144469, 1014876021,
- 2218315341, 1071774145, 2694295388, 3163288868, 586995997, 1071770960,
- 41662348, 3162627992, 1588871207, 1071767783, 143439582, 3162963416,
- 828946858, 1071764615, 10642492, 1015939438, 2502433899, 1071761455,
- 2148595913, 1015023991, 2214878420, 1071758304, 892270087, 3163116422,
- 4162030108, 1071755161, 2763428480, 1015529349, 3949972341, 1071752027,
- 2068408548, 1014913868, 1480023343, 1071748902, 2247196168, 1015327453,
- 948735466, 1071745785, 3516338028, 3162574883, 2257959872, 1071742676,
- 3802946148, 1012964927, 1014845819, 1071739576, 3117910646, 3161559105,
- 1416741826, 1071736484, 2196380210, 1011413563, 3366293073, 1071733400,
- 3119426314, 1014120554, 2471440686, 1071730325, 968836267, 3162214888,
- 2930322912, 1071727258, 2599499422, 3162714047, 351405227, 1071724200,
- 3125337328, 3159822479, 3228316108, 1071721149, 3010241991, 3158422804,
- 2875075254, 1071718107, 4144233330, 3163333716, 3490863953, 1071715073,
- 960797498, 3162948880, 685187902, 1071712048, 378731989, 1014843115,
- 2952712987, 1071709030, 3293494651, 3160120301, 1608493509, 1071706021,
- 3159622171, 3162807737, 852742562, 1071703020, 667253586, 1009793559,
- 590962156, 1071700027, 3829346666, 3163275597, 728909815, 1071697042,
- 383930225, 1015029468, 1172597893, 1071694065, 114433263, 1015347593,
- 1828292879, 1071691096, 1255956747, 1015588398, 2602514713, 1071688135,
- 2268929336, 1014354284, 3402036099, 1071685182, 405889334, 1015105656,
- 4133881824, 1071682237, 2148155345, 3162931299, 410360776, 1071679301,
- 1269990655, 1011975870, 728934454, 1071676372, 1413842688, 1014178612,
- 702412510, 1071673451, 3803266087, 3162280415, 238821257, 1071670538,
- 1469694871, 3162884987, 3541402996, 1071667632, 2759177317, 1014854626,
- 1928746161, 1071664735, 983617676, 1014285177, 3899555717, 1071661845,
- 427280750, 3162546972, 772914124, 1071658964, 4004372762, 1012230161,
- 1048019041, 1071656090, 1398474845, 3160510595, 339411585, 1071653224,
- 264588982, 3161636657, 2851812149, 1071650365, 2595802551, 1015767337,
- 4200250559, 1071647514, 2808127345, 3161781938
+ 0x00000000UL, 0x3FF00000UL, 0x00000000UL, 0x00000000UL, 0x6B2A23D9UL, 0x3FEFE9D9UL,
+ 0x7442FDE3UL, 0x3C64A603UL, 0x2B8F71F1UL, 0x3FEFD3C2UL, 0x966579E7UL, 0x3C52EB74UL,
+ 0x3692D514UL, 0x3FEFBDBAUL, 0x15098EB6UL, 0xBC696773UL, 0x819E90D8UL, 0x3FEFA7C1UL,
+ 0xF3A5931EUL, 0x3C774853UL, 0x02243C89UL, 0x3FEF91D8UL, 0xA779F689UL, 0xBC512EA8UL,
+ 0xAD9CBE14UL, 0x3FEF7BFDUL, 0xD006350AUL, 0xBC8DBB12UL, 0x798844F8UL, 0x3FEF6632UL,
+ 0x3539343EUL, 0x3C8FA37BUL, 0x5B6E4540UL, 0x3FEF5076UL, 0x2DD8A18BUL, 0x3C89D3E1UL,
+ 0x48DD7274UL, 0x3FEF3AC9UL, 0x3ED837DEUL, 0xBC695A5AUL, 0x376BBA97UL, 0x3FEF252BUL,
+ 0xBF0D8E43UL, 0x3C83A1A5UL, 0x1CB6412AUL, 0x3FEF0F9CUL, 0x65181D45UL, 0xBC832200UL,
+ 0xEE615A27UL, 0x3FEEFA1BUL, 0x86A4B6B0UL, 0x3C8DC7F4UL, 0xA2188510UL, 0x3FEEE4AAUL,
+ 0xA487568DUL, 0x3C81C68DUL, 0x2D8E67F1UL, 0x3FEECF48UL, 0xB411AD8CUL, 0xBC8C93F3UL,
+ 0x867CCA6EUL, 0x3FEEB9F4UL, 0x2293E4F2UL, 0x3C84832FUL, 0xA2A490DAUL, 0x3FEEA4AFUL,
+ 0x179C2893UL, 0xBC8E9C23UL, 0x77CDB740UL, 0x3FEE8F79UL, 0x80B054B1UL, 0xBC810894UL,
+ 0xFBC74C83UL, 0x3FEE7A51UL, 0xCA0C8DE2UL, 0x3C82D522UL, 0x24676D76UL, 0x3FEE6539UL,
+ 0x7522B735UL, 0xBC763FF8UL, 0xE78B3FF6UL, 0x3FEE502EUL, 0x80A9CC8FUL, 0x3C739E89UL,
+ 0x3B16EE12UL, 0x3FEE3B33UL, 0x31FDC68BUL, 0xBC89F4A4UL, 0x14F5A129UL, 0x3FEE2646UL,
+ 0x817A1496UL, 0xBC87B627UL, 0x6B197D17UL, 0x3FEE1167UL, 0xBD5C7F44UL, 0xBC62B529UL,
+ 0x337B9B5FUL, 0x3FEDFC97UL, 0x4F184B5CUL, 0xBC81A5CDUL, 0x641C0658UL, 0x3FEDE7D5UL,
+ 0x8E79BA8FUL, 0xBC8CA552UL, 0xF301B460UL, 0x3FEDD321UL, 0x78F018C3UL, 0x3C82DA57UL,
+ 0xD63A8315UL, 0x3FEDBE7CUL, 0x926B8BE4UL, 0xBC8B76F1UL, 0x03DB3285UL, 0x3FEDA9E6UL,
+ 0x696DB532UL, 0x3C8C2300UL, 0x71FF6075UL, 0x3FED955DUL, 0xBB9AF6BEUL, 0x3C8A052DUL,
+ 0x16C98398UL, 0x3FED80E3UL, 0x8BEDDFE8UL, 0xBC811EC1UL, 0xE862E6D3UL, 0x3FED6C76UL,
+ 0x4A8165A0UL, 0x3C4FE87AUL, 0xDCFBA487UL, 0x3FED5818UL, 0xD75B3707UL, 0x3C72ED02UL,
+ 0xEACAA1D6UL, 0x3FED43C8UL, 0xBF5A1614UL, 0x3C83DB53UL, 0x080D89F2UL, 0x3FED2F87UL,
+ 0x719D8578UL, 0xBC8D487BUL, 0x2B08C968UL, 0x3FED1B53UL, 0x219A36EEUL, 0x3C855636UL,
+ 0x4A07897CUL, 0x3FED072DUL, 0x43797A9CUL, 0xBC8CBC37UL, 0x5B5BAB74UL, 0x3FECF315UL,
+ 0xB86DFF57UL, 0xBC8A08E9UL, 0x555DC3FAUL, 0x3FECDF0BUL, 0x53829D72UL, 0xBC7DD83BUL,
+ 0x2E6D1675UL, 0x3FECCB0FUL, 0x86009093UL, 0xBC6D220FUL, 0xDCEF9069UL, 0x3FECB720UL,
+ 0xD1E949DCUL, 0x3C6503CBUL, 0x5751C4DBUL, 0x3FECA340UL, 0xD10D08F5UL, 0xBC77F2BEUL,
+ 0x9406E7B5UL, 0x3FEC8F6DUL, 0x48805C44UL, 0x3C61ACBCUL, 0x8988C933UL, 0x3FEC7BA8UL,
+ 0xBE255559UL, 0xBC7E76BBUL, 0x2E57D14BUL, 0x3FEC67F1UL, 0xFF483CADUL, 0x3C82884DUL,
+ 0x78FAFB22UL, 0x3FEC5447UL, 0x2493B5AFUL, 0x3C812F07UL, 0x5FFFD07AUL, 0x3FEC40ABUL,
+ 0xE083C60AUL, 0x3C8B4537UL, 0xD9FA652CUL, 0x3FEC2D1CUL, 0x17C8A5D7UL, 0xBC86E516UL,
+ 0xDD85529CUL, 0x3FEC199BUL, 0x895048DDUL, 0x3C711065UL, 0x6141B33DUL, 0x3FEC0628UL,
+ 0xA1FBCA34UL, 0xBC7D8A5AUL, 0x5BD71E09UL, 0x3FEBF2C2UL, 0x3F6B9C73UL, 0xBC8EFDCAUL,
+ 0xC3F3A207UL, 0x3FEBDF69UL, 0x60EA5B53UL, 0xBC2C2623UL, 0x904BC1D2UL, 0x3FEBCC1EUL,
+ 0x7A2D9E84UL, 0x3C723DD0UL, 0xB79A6F1FUL, 0x3FEBB8E0UL, 0xC9696204UL, 0xBC2F52D1UL,
+ 0x30A1064AUL, 0x3FEBA5B0UL, 0x0E54292EUL, 0xBC8EFCD3UL, 0xF22749E4UL, 0x3FEB928CUL,
+ 0x54CB65C6UL, 0xBC8B7216UL, 0xF2FB5E47UL, 0x3FEB7F76UL, 0x7E54AC3BUL, 0xBC65584FUL,
+ 0x29F1C52AUL, 0x3FEB6C6EUL, 0x52883F6EUL, 0x3C82A8F3UL, 0x8DE5593AUL, 0x3FEB5972UL,
+ 0xBBBA6DE3UL, 0xBC8C71DFUL, 0x15B749B1UL, 0x3FEB4684UL, 0xE9DF7C90UL, 0xBC6F763DUL,
+ 0xB84F15FBUL, 0x3FEB33A2UL, 0x3084D708UL, 0xBC52805EUL, 0x6C9A8952UL, 0x3FEB20CEUL,
+ 0x4A0756CCUL, 0x3C84DD02UL, 0x298DB666UL, 0x3FEB0E07UL, 0x4C80E425UL, 0xBC8BDEF5UL,
+ 0xE622F2FFUL, 0x3FEAFB4CUL, 0x0F315ECDUL, 0xBC84B2FCUL, 0x995AD3ADUL, 0x3FEAE89FUL,
+ 0x345DCC81UL, 0x3C87A1CDUL, 0x3A3C2774UL, 0x3FEAD5FFUL, 0xB6B1B8E5UL, 0x3C87EF3BUL,
+ 0xBFD3F37AUL, 0x3FEAC36BUL, 0xCAE76CD0UL, 0xBC7F9234UL, 0x21356EBAUL, 0x3FEAB0E5UL,
+ 0xDAE94545UL, 0x3C789C31UL, 0x5579FDBFUL, 0x3FEA9E6BUL, 0x0EF7FD31UL, 0x3C80FAC9UL,
+ 0x53C12E59UL, 0x3FEA8BFEUL, 0xB2BA15A9UL, 0xBC84F867UL, 0x1330B358UL, 0x3FEA799EUL,
+ 0xCAC563C7UL, 0x3C8BCB7EUL, 0x8AF46052UL, 0x3FEA674AUL, 0x30670366UL, 0x3C550F56UL,
+ 0xB23E255DUL, 0x3FEA5503UL, 0xDB8D41E1UL, 0xBC8D2F6EUL, 0x80460AD8UL, 0x3FEA42C9UL,
+ 0x589FB120UL, 0xBC8AA780UL, 0xEC4A2D33UL, 0x3FEA309BUL, 0x7DDC36ABUL, 0x3C86305CUL,
+ 0xED8EB8BBUL, 0x3FEA1E7AUL, 0xEE8BE70EUL, 0x3C8C6618UL, 0x7B5DE565UL, 0x3FEA0C66UL,
+ 0x5D1CD533UL, 0xBC835949UL, 0x8D07F29EUL, 0x3FE9FA5EUL, 0xAAF1FACEUL, 0xBC74A9CEUL,
+ 0x19E32323UL, 0x3FE9E863UL, 0x78E64C6EUL, 0x3C6824CAUL, 0x194BB8D5UL, 0x3FE9D674UL,
+ 0xA3DD8233UL, 0xBC8516BEUL, 0x82A3F090UL, 0x3FE9C491UL, 0xB071F2BEUL, 0x3C6C7C46UL,
+ 0x4D53FE0DUL, 0x3FE9B2BBUL, 0x4DF6D518UL, 0xBC8DD84EUL, 0x70CA07BAUL, 0x3FE9A0F1UL,
+ 0x91CEE632UL, 0xBC8173BDUL, 0xE47A22A2UL, 0x3FE98F33UL, 0xA24C78ECUL, 0x3C6CABDAUL,
+ 0x9FDE4E50UL, 0x3FE97D82UL, 0x7C1B85D1UL, 0xBC8D185BUL, 0x9A7670B3UL, 0x3FE96BDDUL,
+ 0x7F19C896UL, 0xBC4BA596UL, 0xCBC8520FUL, 0x3FE95A44UL, 0x96A5F039UL, 0xBC664B7CUL,
+ 0x2B5F98E5UL, 0x3FE948B8UL, 0x797D2D99UL, 0xBC7DC3D6UL, 0xB0CDC5E5UL, 0x3FE93737UL,
+ 0x81B57EBCUL, 0xBC575FC7UL, 0x53AA2FE2UL, 0x3FE925C3UL, 0xA639DB7FUL, 0xBC73455FUL,
+ 0x0B91FFC6UL, 0x3FE9145BUL, 0x2E582524UL, 0xBC8DD679UL, 0xD0282C8AUL, 0x3FE902FEUL,
+ 0x85FE3FD2UL, 0x3C8592CAUL, 0x99157736UL, 0x3FE8F1AEUL, 0xA2E3976CUL, 0x3C75CC13UL,
+ 0x5E0866D9UL, 0x3FE8E06AUL, 0x6FC9B2E6UL, 0xBC87114AUL, 0x16B5448CUL, 0x3FE8CF32UL,
+ 0x32E9E3AAUL, 0xBC60D55EUL, 0xBAD61778UL, 0x3FE8BE05UL, 0xFC43446EUL, 0x3C8ECB5EUL,
+ 0x422AA0DBUL, 0x3FE8ACE5UL, 0x56864B27UL, 0x3C86E9F1UL, 0xA478580FUL, 0x3FE89BD0UL,
+ 0x4475202AUL, 0x3C8D5395UL, 0xD98A6699UL, 0x3FE88AC7UL, 0xF37CB53AUL, 0x3C8994C2UL,
+ 0xD931A436UL, 0x3FE879CAUL, 0xD2DB47BDUL, 0x3C75D2D7UL, 0x9B4492EDUL, 0x3FE868D9UL,
+ 0x9BD4F6BAUL, 0xBC8FC6F8UL, 0x179F5B21UL, 0x3FE857F4UL, 0xF8B216D0UL, 0xBC4BA748UL,
+ 0x4623C7ADUL, 0x3FE8471AUL, 0xA341CDFBUL, 0xBC78D684UL, 0x1EB941F7UL, 0x3FE8364CUL,
+ 0x31DF2BD5UL, 0x3C899B9AUL, 0x994CCE13UL, 0x3FE82589UL, 0xD41532D8UL, 0xBC8D4C1DUL,
+ 0xADD106D9UL, 0x3FE814D2UL, 0x0D151D4DUL, 0x3C846437UL, 0x543E1A12UL, 0x3FE80427UL,
+ 0x626D972BUL, 0xBC827C86UL, 0x8491C491UL, 0x3FE7F387UL, 0xCF9311AEUL, 0xBC707F11UL,
+ 0x36CF4E62UL, 0x3FE7E2F3UL, 0xBA15797EUL, 0x3C605D02UL, 0x62FF86F0UL, 0x3FE7D26AUL,
+ 0xFB72B8B4UL, 0x3C81BDDBUL, 0x0130C132UL, 0x3FE7C1EDUL, 0xD1164DD6UL, 0x3C8F124CUL,
+ 0x0976CFDBUL, 0x3FE7B17BUL, 0x8468DC88UL, 0xBC8BEBB5UL, 0x73EB0187UL, 0x3FE7A114UL,
+ 0xEE04992FUL, 0xBC741577UL, 0x38AC1CF6UL, 0x3FE790B9UL, 0x62AADD3EUL, 0x3C8349A8UL,
+ 0x4FDE5D3FUL, 0x3FE78069UL, 0x0A02162DUL, 0x3C8866B8UL, 0xB1AB6E09UL, 0x3FE77024UL,
+ 0x169147F8UL, 0x3C8B7877UL, 0x564267C9UL, 0x3FE75FEBUL, 0x57316DD3UL, 0xBC802459UL,
+ 0x35D7CBFDUL, 0x3FE74FBDUL, 0x618A6E1CUL, 0x3C8047FDUL, 0x48A58174UL, 0x3FE73F9AUL,
+ 0x6C65D53CUL, 0xBC80A8D9UL, 0x86EAD08AUL, 0x3FE72F82UL, 0x2CD62C72UL, 0xBC820AA0UL,
+ 0xE8EC5F74UL, 0x3FE71F75UL, 0x86887A99UL, 0xBC716E47UL, 0x66F42E87UL, 0x3FE70F74UL,
+ 0xD45AA65FUL, 0x3C49D644UL, 0xF9519484UL, 0x3FE6FF7DUL, 0x25860EF6UL, 0xBC783C0FUL,
+ 0x98593AE5UL, 0x3FE6EF92UL, 0x9E1AC8B2UL, 0xBC80B974UL, 0x3C651A2FUL, 0x3FE6DFB2UL,
+ 0x683C88ABUL, 0xBC5BBE3AUL, 0xDDD47645UL, 0x3FE6CFDCUL, 0xB6F17309UL, 0x3C8C7AA9UL,
+ 0x750BDABFUL, 0x3FE6C012UL, 0x67FF0B0DUL, 0xBC628956UL, 0xFA75173EUL, 0x3FE6B052UL,
+ 0x2C9A9D0EUL, 0x3C6A38F5UL, 0x667F3BCDUL, 0x3FE6A09EUL, 0x13B26456UL, 0xBC8BDD34UL,
+ 0xB19E9538UL, 0x3FE690F4UL, 0x9AEB445DUL, 0x3C7804BDUL, 0xD44CA973UL, 0x3FE68155UL,
+ 0x44F73E65UL, 0x3C5038AEUL, 0xC70833F6UL, 0x3FE671C1UL, 0x586C6134UL, 0xBC7E8732UL,
+ 0x82552225UL, 0x3FE66238UL, 0x87591C34UL, 0xBC8BB609UL, 0xFEBC8FB7UL, 0x3FE652B9UL,
+ 0xC9A73E09UL, 0xBC8AE3D5UL, 0x34CCC320UL, 0x3FE64346UL, 0x759D8933UL, 0xBC7C483CUL,
+ 0x1D1929FDUL, 0x3FE633DDUL, 0xBEB964E5UL, 0x3C884710UL, 0xB03A5585UL, 0x3FE6247EUL,
+ 0x7E40B497UL, 0xBC8383C1UL, 0xE6CDF6F4UL, 0x3FE6152AUL, 0x4AB84C27UL, 0x3C8E4B3EUL,
+ 0xB976DC09UL, 0x3FE605E1UL, 0x9B56DE47UL, 0xBC83E242UL, 0x20DCEB71UL, 0x3FE5F6A3UL,
+ 0xE3CDCF92UL, 0xBC79EADDUL, 0x15AD2148UL, 0x3FE5E76FUL, 0x3080E65EUL, 0x3C8BA6F9UL,
+ 0x90998B93UL, 0x3FE5D845UL, 0xA8B45643UL, 0xBC8CD6A7UL, 0x8A5946B7UL, 0x3FE5C926UL,
+ 0x816986A2UL, 0x3C2C4B1BUL, 0xFBA87A03UL, 0x3FE5BA11UL, 0x4C233E1AUL, 0xBC8B77A1UL,
+ 0xDD485429UL, 0x3FE5AB07UL, 0x054647ADUL, 0x3C86324CUL, 0x27FF07CCUL, 0x3FE59C08UL,
+ 0xE467E60FUL, 0xBC87E2CEUL, 0xD497C7FDUL, 0x3FE58D12UL, 0x5B9A1DE8UL, 0x3C7295E1UL,
+ 0xDBE2C4CFUL, 0x3FE57E27UL, 0x8A57B9C4UL, 0xBC80B98CUL, 0x36B527DAUL, 0x3FE56F47UL,
+ 0x011D93ADUL, 0x3C89BB2CUL, 0xDDE910D2UL, 0x3FE56070UL, 0x168EEBF0UL, 0xBC80FB6EUL,
+ 0xCA5D920FUL, 0x3FE551A4UL, 0xEFEDE59BUL, 0xBC7D689CUL, 0xF4F6AD27UL, 0x3FE542E2UL,
+ 0x192D5F7EUL, 0x3C77926DUL, 0x569D4F82UL, 0x3FE5342BUL, 0x1DB13CADUL, 0xBC707ABEUL,
+ 0xE83F4EEFUL, 0x3FE5257DUL, 0x43EFEF71UL, 0xBC6C998DUL, 0xA2CF6642UL, 0x3FE516DAUL,
+ 0x69BD93EFUL, 0xBC7F7685UL, 0x7F4531EEUL, 0x3FE50841UL, 0x49B7465FUL, 0x3C6A249BUL,
+ 0x769D2CA7UL, 0x3FE4F9B2UL, 0xD25957E3UL, 0xBC84B309UL, 0x81D8ABFFUL, 0x3FE4EB2DUL,
+ 0x2E5D7A52UL, 0xBC85257DUL, 0x99FDDD0DUL, 0x3FE4DCB2UL, 0xBC6A7833UL, 0x3C88ECDBUL,
+ 0xB817C114UL, 0x3FE4CE41UL, 0x690ABD5DUL, 0x3C805E29UL, 0xD5362A27UL, 0x3FE4BFDAUL,
+ 0xAFEC42E2UL, 0x3C6D4397UL, 0xEA6DB7D7UL, 0x3FE4B17DUL, 0x7F2897F0UL, 0xBC7125B8UL,
+ 0xF0D7D3DEUL, 0x3FE4A32AUL, 0xF3D1BE56UL, 0x3C89CB62UL, 0xE192AED2UL, 0x3FE494E1UL,
+ 0x5E499EA0UL, 0xBC73B289UL, 0xB5C13CD0UL, 0x3FE486A2UL, 0xB69062F0UL, 0x3C63C1A3UL,
+ 0x668B3237UL, 0x3FE4786DUL, 0xED445733UL, 0xBC8C20F0UL, 0xED1D0057UL, 0x3FE46A41UL,
+ 0xD1648A76UL, 0x3C8C944BUL, 0x42A7D232UL, 0x3FE45C20UL, 0x82FB1F8EUL, 0xBC586419UL,
+ 0x6061892DUL, 0x3FE44E08UL, 0x04EF80D0UL, 0x3C389B7AUL, 0x3F84B9D4UL, 0x3FE43FFAUL,
+ 0x9704C003UL, 0x3C7880BEUL, 0xD950A897UL, 0x3FE431F5UL, 0xE35F7999UL, 0xBC71C7DDUL,
+ 0x2709468AUL, 0x3FE423FBUL, 0xC0B314DDUL, 0xBC88462DUL, 0x21F72E2AUL, 0x3FE4160AUL,
+ 0x1C309278UL, 0xBC4EF369UL, 0xC367A024UL, 0x3FE40822UL, 0xB6F4D048UL, 0x3C7BDDF8UL,
+ 0x04AC801CUL, 0x3FE3FA45UL, 0xF956F9F3UL, 0xBC87D023UL, 0xDF1C5175UL, 0x3FE3EC70UL,
+ 0x7B8C9BCAUL, 0xBC7AF663UL, 0x4C123422UL, 0x3FE3DEA6UL, 0x11F09EBCUL, 0x3C7ADA09UL,
+ 0x44EDE173UL, 0x3FE3D0E5UL, 0x8C284C71UL, 0x3C6FE8D0UL, 0xC313A8E5UL, 0x3FE3C32DUL,
+ 0x375D29C3UL, 0xBC8EFFF8UL, 0xBFEC6CF4UL, 0x3FE3B57FUL, 0xE26FFF18UL, 0x3C854C66UL,
+ 0x34E59FF7UL, 0x3FE3A7DBUL, 0xD661F5E3UL, 0xBC65E436UL, 0x1B7140EFUL, 0x3FE39A40UL,
+ 0xFC8E2934UL, 0xBC89A9A5UL, 0x6D05D866UL, 0x3FE38CAEUL, 0x3C9904BDUL, 0xBC8E958DUL,
+ 0x231E754AUL, 0x3FE37F26UL, 0x9ECEB23CUL, 0xBC89F5CAUL, 0x373AA9CBUL, 0x3FE371A7UL,
+ 0xBF42EAE2UL, 0xBC863AEAUL, 0xA2DE883BUL, 0x3FE36431UL, 0xA06CB85EUL, 0xBC7C3144UL,
+ 0x5F929FF1UL, 0x3FE356C5UL, 0x5C4E4628UL, 0xBC7B5CEEUL, 0x66E3FA2DUL, 0x3FE34962UL,
+ 0x930881A4UL, 0xBC735A75UL, 0xB26416FFUL, 0x3FE33C08UL, 0x843659A6UL, 0x3C832721UL,
+ 0x3BA8EA32UL, 0x3FE32EB8UL, 0x3CB4F318UL, 0xBC8C45E8UL, 0xFC4CD831UL, 0x3FE32170UL,
+ 0x8E18047CUL, 0x3C7A9CE7UL, 0xEDEEB2FDUL, 0x3FE31432UL, 0xF3F3FCD1UL, 0x3C7959A3UL,
+ 0x0A31B715UL, 0x3FE306FEUL, 0xD23182E4UL, 0x3C76F46AUL, 0x4ABD886BUL, 0x3FE2F9D2UL,
+ 0x532BDA93UL, 0xBC553C55UL, 0xA93E2F56UL, 0x3FE2ECAFUL, 0x45D52383UL, 0x3C61CA0FUL,
+ 0x1F641589UL, 0x3FE2DF96UL, 0xFBBCE198UL, 0x3C8D16CFUL, 0xA6E4030BUL, 0x3FE2D285UL,
+ 0x54DB41D5UL, 0x3C800247UL, 0x39771B2FUL, 0x3FE2C57EUL, 0xA6EB5124UL, 0xBC850145UL,
+ 0xD0DAD990UL, 0x3FE2B87FUL, 0xD6381AA4UL, 0xBC310ADCUL, 0x66D10F13UL, 0x3FE2AB8AUL,
+ 0x191690A7UL, 0xBC895743UL, 0xF51FDEE1UL, 0x3FE29E9DUL, 0xAFAD1255UL, 0x3C7612E8UL,
+ 0x7591BB70UL, 0x3FE291BAUL, 0x28401CBDUL, 0xBC72CC72UL, 0xE1F56381UL, 0x3FE284DFUL,
+ 0x8C3F0D7EUL, 0xBC8A4C3AUL, 0x341DDF29UL, 0x3FE2780EUL, 0x05F9E76CUL, 0x3C8E067CUL,
+ 0x65E27CDDUL, 0x3FE26B45UL, 0x9940E9D9UL, 0x3C72BD33UL, 0x711ECE75UL, 0x3FE25E85UL,
+ 0x4AC31B2CUL, 0x3C83E1A2UL, 0x4FB2A63FUL, 0x3FE251CEUL, 0xBEF4F4A4UL, 0x3C7AC155UL,
+ 0xFB82140AUL, 0x3FE2451FUL, 0x911CA996UL, 0x3C7ACFCCUL, 0x6E756238UL, 0x3FE2387AUL,
+ 0xB6C70573UL, 0x3C89B07EUL, 0xA27912D1UL, 0x3FE22BDDUL, 0x5577D69FUL, 0x3C7D34FBUL,
+ 0x917DDC96UL, 0x3FE21F49UL, 0x9494A5EEUL, 0x3C72A97EUL, 0x3578A819UL, 0x3FE212BEUL,
+ 0x2CFCAAC9UL, 0x3C83592DUL, 0x88628CD6UL, 0x3FE2063BUL, 0x814A8495UL, 0x3C7DC775UL,
+ 0x8438CE4DUL, 0x3FE1F9C1UL, 0xA097AF5CUL, 0xBC8BF524UL, 0x22FCD91DUL, 0x3FE1ED50UL,
+ 0x027BB78CUL, 0xBC81DF98UL, 0x5EB44027UL, 0x3FE1E0E7UL, 0x088CB6DEUL, 0xBC86FDD8UL,
+ 0x3168B9AAUL, 0x3FE1D487UL, 0x00A2643CUL, 0x3C8E016EUL, 0x95281C6BUL, 0x3FE1C82FUL,
+ 0x8010F8C9UL, 0x3C800977UL, 0x84045CD4UL, 0x3FE1BBE0UL, 0x352EF607UL, 0xBC895386UL,
+ 0xF8138A1CUL, 0x3FE1AF99UL, 0xA4B69280UL, 0x3C87BF85UL, 0xEB6FCB75UL, 0x3FE1A35BUL,
+ 0x7B4968E4UL, 0x3C7E5B4CUL, 0x58375D2FUL, 0x3FE19726UL, 0x85F17E08UL, 0x3C84AADDUL,
+ 0x388C8DEAUL, 0x3FE18AF9UL, 0xD1970F6CUL, 0xBC811023UL, 0x8695BBC0UL, 0x3FE17ED4UL,
+ 0xE2AC5A64UL, 0x3C609E3FUL, 0x3C7D517BUL, 0x3FE172B8UL, 0xB9D78A76UL, 0xBC719041UL,
+ 0x5471C3C2UL, 0x3FE166A4UL, 0x82EA1A32UL, 0x3C48F23BUL, 0xC8A58E51UL, 0x3FE15A98UL,
+ 0xB9EEAB0AUL, 0x3C72406AUL, 0x934F312EUL, 0x3FE14E95UL, 0x39BF44ABUL, 0xBC7B91E8UL,
+ 0xAEA92DE0UL, 0x3FE1429AUL, 0x9AF1369EUL, 0xBC832FBFUL, 0x14F204ABUL, 0x3FE136A8UL,
+ 0xBA48DCF0UL, 0xBC57108FUL, 0xC06C31CCUL, 0x3FE12ABDUL, 0xB36CA5C7UL, 0xBC41B514UL,
+ 0xAB5E2AB6UL, 0x3FE11EDBUL, 0xF703FB72UL, 0xBC8CA454UL, 0xD0125B51UL, 0x3FE11301UL,
+ 0x39449B3AUL, 0xBC86C510UL, 0x28D7233EUL, 0x3FE10730UL, 0x1692FDD5UL, 0x3C7D46EBUL,
+ 0xAFFED31BUL, 0x3FE0FB66UL, 0xC44EBD7BUL, 0xBC5B9BEDUL, 0x5FDFA9C5UL, 0x3FE0EFA5UL,
+ 0xBC54021BUL, 0xBC849DB9UL, 0x32D3D1A2UL, 0x3FE0E3ECUL, 0x27C57B52UL, 0x3C303A17UL,
+ 0x23395DECUL, 0x3FE0D83BUL, 0xE43F316AUL, 0xBC8BC14DUL, 0x2B7247F7UL, 0x3FE0CC92UL,
+ 0x16E24F71UL, 0x3C801EDCUL, 0x45E46C85UL, 0x3FE0C0F1UL, 0x06D21CEFUL, 0x3C84F989UL,
+ 0x6CF9890FUL, 0x3FE0B558UL, 0x4ADC610BUL, 0x3C88A62EUL, 0x9B1F3919UL, 0x3FE0A9C7UL,
+ 0x873D1D38UL, 0x3C75D16CUL, 0xCAC6F383UL, 0x3FE09E3EUL, 0x18316136UL, 0x3C814878UL,
+ 0xF66607E0UL, 0x3FE092BDUL, 0x800A3FD1UL, 0xBC868063UL, 0x18759BC8UL, 0x3FE08745UL,
+ 0x4BB284FFUL, 0x3C5186BEUL, 0x2B72A836UL, 0x3FE07BD4UL, 0x54458700UL, 0x3C732334UL,
+ 0x29DDF6DEUL, 0x3FE0706BUL, 0xE2B13C27UL, 0xBC7C91DFUL, 0x0E3C1F89UL, 0x3FE0650AUL,
+ 0x5799C397UL, 0xBC85CB7BUL, 0xD3158574UL, 0x3FE059B0UL, 0xA475B465UL, 0x3C7D73E2UL,
+ 0x72F654B1UL, 0x3FE04E5FUL, 0x3AA0D08CUL, 0x3C74C379UL, 0xE86E7F85UL, 0x3FE04315UL,
+ 0x1977C96EUL, 0xBC80A31CUL, 0x2E11BBCCUL, 0x3FE037D4UL, 0xEEADE11AUL, 0x3C556811UL,
+ 0x3E778061UL, 0x3FE02C9AUL, 0x535B085DUL, 0xBC619083UL, 0x143B0281UL, 0x3FE02168UL,
+ 0x0FC54EB6UL, 0xBC72BF31UL, 0xA9FB3335UL, 0x3FE0163DUL, 0x9AB8CDB7UL, 0x3C8B6129UL,
+ 0xFA5ABCBFUL, 0x3FE00B1AUL, 0xA7609F71UL, 0xBC74F6B2UL
};
#define __ _masm->
diff --git a/src/hotspot/cpu/x86/templateTable_x86.cpp b/src/hotspot/cpu/x86/templateTable_x86.cpp
index 82ca18d8a1f..c8dafb2d023 100644
--- a/src/hotspot/cpu/x86/templateTable_x86.cpp
+++ b/src/hotspot/cpu/x86/templateTable_x86.cpp
@@ -2196,8 +2196,7 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no,
const Register temp = rbx;
assert_different_registers(cache, index, temp);
- Label L_clinit_barrier_slow;
- Label resolved;
+ Label L_clinit_barrier_slow, L_done;
Bytecodes::Code code = bytecode();
@@ -2215,37 +2214,38 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no,
ShouldNotReachHere();
}
__ cmpl(temp, code); // have we resolved this bytecode?
- __ jcc(Assembler::equal, resolved);
-
- // resolve first time through
- // Class initialization barrier slow path lands here as well.
- __ bind(L_clinit_barrier_slow);
- address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
- __ movl(temp, code);
- __ call_VM(noreg, entry, temp);
- // Update registers with resolved info
- __ load_method_entry(cache, index);
-
- __ bind(resolved);
// Class initialization barrier for static methods
if (VM_Version::supports_fast_class_init_checks() && bytecode() == Bytecodes::_invokestatic) {
const Register method = temp;
const Register klass = temp;
+ __ jcc(Assembler::notEqual, L_clinit_barrier_slow);
__ movptr(method, Address(cache, in_bytes(ResolvedMethodEntry::method_offset())));
__ load_method_holder(klass, method);
- __ clinit_barrier(klass, nullptr /*L_fast_path*/, &L_clinit_barrier_slow);
+ __ clinit_barrier(klass, &L_done, /*L_slow_path*/ nullptr);
+ __ bind(L_clinit_barrier_slow);
+ } else {
+ __ jcc(Assembler::equal, L_done);
}
+
+ // resolve first time through
+ // Class initialization barrier slow path lands here as well.
+ address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
+ __ movl(temp, code);
+ __ call_VM(noreg, entry, temp);
+ // Update registers with resolved info
+ __ load_method_entry(cache, index);
+ __ bind(L_done);
}
void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
- Register cache,
- Register index) {
+ Register cache,
+ Register index) {
const Register temp = rbx;
assert_different_registers(cache, index, temp);
- Label resolved;
+ Label L_clinit_barrier_slow, L_done;
Bytecodes::Code code = bytecode();
switch (code) {
@@ -2262,16 +2262,28 @@ void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
__ load_unsigned_byte(temp, Address(cache, in_bytes(ResolvedFieldEntry::put_code_offset())));
}
__ cmpl(temp, code); // have we resolved this bytecode?
- __ jcc(Assembler::equal, resolved);
+
+ // Class initialization barrier for static fields
+ if (VM_Version::supports_fast_class_init_checks() &&
+ (bytecode() == Bytecodes::_getstatic || bytecode() == Bytecodes::_putstatic)) {
+ const Register field_holder = temp;
+
+ __ jcc(Assembler::notEqual, L_clinit_barrier_slow);
+ __ movptr(field_holder, Address(cache, in_bytes(ResolvedFieldEntry::field_holder_offset())));
+ __ clinit_barrier(field_holder, &L_done, /*L_slow_path*/ nullptr);
+ __ bind(L_clinit_barrier_slow);
+ } else {
+ __ jcc(Assembler::equal, L_done);
+ }
// resolve first time through
+ // Class initialization barrier slow path lands here as well.
address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
__ movl(temp, code);
__ call_VM(noreg, entry, temp);
// Update registers with resolved info
__ load_field_entry(cache, index);
-
- __ bind(resolved);
+ __ bind(L_done);
}
void TemplateTable::load_resolved_field_entry(Register obj,
diff --git a/src/hotspot/cpu/x86/vm_version_x86.cpp b/src/hotspot/cpu/x86/vm_version_x86.cpp
index 094ab370190..213e988a581 100644
--- a/src/hotspot/cpu/x86/vm_version_x86.cpp
+++ b/src/hotspot/cpu/x86/vm_version_x86.cpp
@@ -62,7 +62,7 @@ address VM_Version::_cpuinfo_segv_addr_apx = nullptr;
address VM_Version::_cpuinfo_cont_addr_apx = nullptr;
static BufferBlob* stub_blob;
-static const int stub_size = 2000;
+static const int stub_size = 2550;
int VM_Version::VM_Features::_features_bitmap_size = sizeof(VM_Version::VM_Features::_features_bitmap) / BytesPerLong;
@@ -73,10 +73,12 @@ extern "C" {
typedef void (*get_cpu_info_stub_t)(void*);
typedef void (*detect_virt_stub_t)(uint32_t, uint32_t*);
typedef void (*clear_apx_test_state_t)(void);
+ typedef void (*getCPUIDBrandString_stub_t)(void*);
}
static get_cpu_info_stub_t get_cpu_info_stub = nullptr;
static detect_virt_stub_t detect_virt_stub = nullptr;
static clear_apx_test_state_t clear_apx_test_state_stub = nullptr;
+static getCPUIDBrandString_stub_t getCPUIDBrandString_stub = nullptr;
bool VM_Version::supports_clflush() {
// clflush should always be available on x86_64
@@ -139,7 +141,7 @@ class VM_Version_StubGenerator: public StubCodeGenerator {
const uint32_t CPU_FAMILY_486 = (4 << CPU_FAMILY_SHIFT);
bool use_evex = FLAG_IS_DEFAULT(UseAVX) || (UseAVX > 2);
- Label detect_486, cpu486, detect_586, std_cpuid1, std_cpuid4, std_cpuid24;
+ Label detect_486, cpu486, detect_586, std_cpuid1, std_cpuid4, std_cpuid24, std_cpuid29;
Label sef_cpuid, sefsl1_cpuid, ext_cpuid, ext_cpuid1, ext_cpuid5, ext_cpuid7;
Label ext_cpuid8, done, wrapup, vector_save_restore, apx_save_restore_warning;
Label legacy_setup, save_restore_except, legacy_save_restore, start_simd_check;
@@ -338,6 +340,16 @@ class VM_Version_StubGenerator: public StubCodeGenerator {
__ movl(Address(rsi, 0), rax);
__ movl(Address(rsi, 4), rdx);
+ //
+ // cpuid(0x29) APX NCI NDD NF (EAX = 29H, ECX = 0).
+ //
+ __ bind(std_cpuid29);
+ __ movl(rax, 0x29);
+ __ movl(rcx, 0);
+ __ cpuid();
+ __ lea(rsi, Address(rbp, in_bytes(VM_Version::std_cpuid29_offset())));
+ __ movl(Address(rsi, 0), rbx);
+
//
// cpuid(0x24) Converged Vector ISA Main Leaf (EAX = 24H, ECX = 0).
//
@@ -1016,16 +1028,6 @@ void VM_Version::get_processor_features() {
_features.clear_feature(CPU_AVX10_2);
}
- // Currently APX support is only enabled for targets supporting AVX512VL feature.
- bool apx_supported = os_supports_apx_egprs() && supports_apx_f() && supports_avx512vl();
- if (UseAPX && !apx_supported) {
- warning("UseAPX is not supported on this CPU, setting it to false");
- FLAG_SET_DEFAULT(UseAPX, false);
- }
-
- if (!UseAPX) {
- _features.clear_feature(CPU_APX_F);
- }
if (UseAVX < 2) {
_features.clear_feature(CPU_AVX2);
@@ -1049,6 +1051,7 @@ void VM_Version::get_processor_features() {
_features.clear_feature(CPU_VZEROUPPER);
_features.clear_feature(CPU_AVX512BW);
_features.clear_feature(CPU_AVX512VL);
+ _features.clear_feature(CPU_APX_F);
_features.clear_feature(CPU_AVX512DQ);
_features.clear_feature(CPU_AVX512_VNNI);
_features.clear_feature(CPU_AVX512_VAES);
@@ -1068,6 +1071,17 @@ void VM_Version::get_processor_features() {
}
}
+ // Currently APX support is only enabled for targets supporting AVX512VL feature.
+ bool apx_supported = os_supports_apx_egprs() && supports_apx_f() && supports_avx512vl();
+ if (UseAPX && !apx_supported) {
+ warning("UseAPX is not supported on this CPU, setting it to false");
+ FLAG_SET_DEFAULT(UseAPX, false);
+ }
+
+ if (!UseAPX) {
+ _features.clear_feature(CPU_APX_F);
+ }
+
if (FLAG_IS_DEFAULT(IntelJccErratumMitigation)) {
_has_intel_jcc_erratum = compute_has_intel_jcc_erratum();
FLAG_SET_ERGO(IntelJccErratumMitigation, _has_intel_jcc_erratum);
@@ -2081,6 +2095,10 @@ bool VM_Version::is_intel_cascade_lake() {
return is_intel_skylake() && _stepping >= 5;
}
+bool VM_Version::is_intel_darkmont() {
+ return is_intel() && is_intel_server_family() && (_model == 0xCC || _model == 0xDD);
+}
+
// avx3_threshold() sets the threshold at which 64-byte instructions are used
// for implementing the array copy and clear operations.
// The Intel platforms that supports the serialize instruction
@@ -2115,6 +2133,8 @@ void VM_Version::initialize() {
g.generate_detect_virt());
clear_apx_test_state_stub = CAST_TO_FN_PTR(clear_apx_test_state_t,
g.clear_apx_test_state());
+ getCPUIDBrandString_stub = CAST_TO_FN_PTR(getCPUIDBrandString_stub_t,
+ g.generate_getCPUIDBrandString());
get_processor_features();
Assembler::precompute_instructions();
@@ -2171,15 +2191,6 @@ typedef enum {
TM_FLAG = 0x20000000
} FeatureEdxFlag;
-static BufferBlob* cpuid_brand_string_stub_blob;
-static const int cpuid_brand_string_stub_size = 550;
-
-extern "C" {
- typedef void (*getCPUIDBrandString_stub_t)(void*);
-}
-
-static getCPUIDBrandString_stub_t getCPUIDBrandString_stub = nullptr;
-
// VM_Version statics
enum {
ExtendedFamilyIdLength_INTEL = 16,
@@ -2472,19 +2483,6 @@ const char* const _feature_extended_ecx_id[] = {
""
};
-void VM_Version::initialize_tsc(void) {
- ResourceMark rm;
-
- cpuid_brand_string_stub_blob = BufferBlob::create("getCPUIDBrandString_stub", cpuid_brand_string_stub_size);
- if (cpuid_brand_string_stub_blob == nullptr) {
- vm_exit_during_initialization("Unable to allocate getCPUIDBrandString_stub");
- }
- CodeBuffer c(cpuid_brand_string_stub_blob);
- VM_Version_StubGenerator g(&c);
- getCPUIDBrandString_stub = CAST_TO_FN_PTR(getCPUIDBrandString_stub_t,
- g.generate_getCPUIDBrandString());
-}
-
const char* VM_Version::cpu_model_description(void) {
uint32_t cpu_family = extended_cpu_family();
uint32_t cpu_model = extended_cpu_model();
@@ -2573,7 +2571,12 @@ void VM_Version::resolve_cpu_information_details(void) {
_no_of_threads = os::processor_count();
// find out number of threads per cpu package
- int threads_per_package = threads_per_core() * cores_per_cpu();
+ int threads_per_package = _cpuid_info.tpl_cpuidB1_ebx.bits.logical_cpus;
+ if (threads_per_package == 0) {
+ // Fallback code to avoid div by zero in subsequent code.
+ // CPUID 0Bh (ECX = 1) might return 0 on older AMD processor (EPYC 7763 at least)
+ threads_per_package = threads_per_core() * cores_per_cpu();
+ }
// use amount of threads visible to the process in order to guess number of sockets
_no_of_sockets = _no_of_threads / threads_per_package;
@@ -2727,6 +2730,10 @@ size_t VM_Version::cpu_write_support_string(char* const buf, size_t buf_len) {
WRITE_TO_BUF("Invariant TSC");
}
+ if (supports_hybrid()) {
+ WRITE_TO_BUF("Hybrid Architecture");
+ }
+
return written;
}
@@ -2912,7 +2919,8 @@ VM_Version::VM_Features VM_Version::CpuidInfo::feature_flags() const {
if (std_cpuid1_ecx.bits.popcnt != 0)
vm_features.set_feature(CPU_POPCNT);
if (sefsl1_cpuid7_edx.bits.apx_f != 0 &&
- xem_xcr0_eax.bits.apx_f != 0) {
+ xem_xcr0_eax.bits.apx_f != 0 &&
+ std_cpuid29_ebx.bits.apx_nci_ndd_nf != 0) {
vm_features.set_feature(CPU_APX_F);
}
if (std_cpuid1_ecx.bits.avx != 0 &&
diff --git a/src/hotspot/cpu/x86/vm_version_x86.hpp b/src/hotspot/cpu/x86/vm_version_x86.hpp
index 54b3a93d64b..aa9a527e0b7 100644
--- a/src/hotspot/cpu/x86/vm_version_x86.hpp
+++ b/src/hotspot/cpu/x86/vm_version_x86.hpp
@@ -306,6 +306,14 @@ class VM_Version : public Abstract_VM_Version {
} bits;
};
+ union StdCpuidEax29Ecx0 {
+ uint32_t value;
+ struct {
+ uint32_t apx_nci_ndd_nf : 1,
+ : 31;
+ } bits;
+ };
+
union StdCpuid24MainLeafEax {
uint32_t value;
struct {
@@ -591,6 +599,10 @@ protected:
StdCpuid24MainLeafEax std_cpuid24_eax;
StdCpuid24MainLeafEbx std_cpuid24_ebx;
+ // cpuid function 0x29 APX Advanced Performance Extensions Leaf
+ // eax = 0x29, ecx = 0
+ StdCpuidEax29Ecx0 std_cpuid29_ebx;
+
// cpuid function 0xB (processor topology)
// ecx = 0
uint32_t tpl_cpuidB0_eax;
@@ -711,6 +723,7 @@ public:
static ByteSize std_cpuid0_offset() { return byte_offset_of(CpuidInfo, std_max_function); }
static ByteSize std_cpuid1_offset() { return byte_offset_of(CpuidInfo, std_cpuid1_eax); }
static ByteSize std_cpuid24_offset() { return byte_offset_of(CpuidInfo, std_cpuid24_eax); }
+ static ByteSize std_cpuid29_offset() { return byte_offset_of(CpuidInfo, std_cpuid29_ebx); }
static ByteSize dcp_cpuid4_offset() { return byte_offset_of(CpuidInfo, dcp_cpuid4_eax); }
static ByteSize sef_cpuid7_offset() { return byte_offset_of(CpuidInfo, sef_cpuid7_eax); }
static ByteSize sefsl1_cpuid7_offset() { return byte_offset_of(CpuidInfo, sefsl1_cpuid7_eax); }
@@ -760,7 +773,9 @@ public:
_features.set_feature(CPU_SSE2);
_features.set_feature(CPU_VZEROUPPER);
}
- static void set_apx_cpuFeatures() { _features.set_feature(CPU_APX_F); }
+ static void set_apx_cpuFeatures() {
+ _features.set_feature(CPU_APX_F);
+ }
static void set_bmi_cpuFeatures() {
_features.set_feature(CPU_BMI1);
_features.set_feature(CPU_BMI2);
@@ -923,6 +938,8 @@ public:
static bool is_intel_cascade_lake();
+ static bool is_intel_darkmont();
+
static int avx3_threshold();
static bool is_intel_tsc_synched_at_init();
@@ -1076,7 +1093,6 @@ public:
static bool supports_tscinv_ext(void);
- static void initialize_tsc();
static void initialize_cpu_information(void);
};
diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad
index 2eb748e350c..efe0482e095 100644
--- a/src/hotspot/cpu/x86/x86.ad
+++ b/src/hotspot/cpu/x86/x86.ad
@@ -7664,8 +7664,11 @@ instruct vcastFtoD_reg(vec dst, vec src) %{
instruct castFtoX_reg_avx(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, vec xtmp4, rFlagsReg cr) %{
- predicate(!VM_Version::supports_avx512vl() && Matcher::vector_length_in_bytes(n->in(1)) < 64 &&
- type2aelembytes(Matcher::vector_element_basic_type(n)) <= 4);
+ predicate(!VM_Version::supports_avx10_2() &&
+ !VM_Version::supports_avx512vl() &&
+ Matcher::vector_length_in_bytes(n->in(1)) < 64 &&
+ type2aelembytes(Matcher::vector_element_basic_type(n)) <= 4 &&
+ is_integral_type(Matcher::vector_element_basic_type(n)));
match(Set dst (VectorCastF2X src));
effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP xtmp4, KILL cr);
format %{ "vector_cast_f2x $dst,$src\t! using $xtmp1, $xtmp2, $xtmp3 and $xtmp4 as TEMP" %}
@@ -7687,7 +7690,8 @@ instruct castFtoX_reg_avx(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, vec
%}
instruct castFtoX_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
- predicate((VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n->in(1)) == 64) &&
+ predicate(!VM_Version::supports_avx10_2() &&
+ (VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n->in(1)) == 64) &&
is_integral_type(Matcher::vector_element_basic_type(n)));
match(Set dst (VectorCastF2X src));
effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2, KILL cr);
@@ -7709,6 +7713,33 @@ instruct castFtoX_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2, kReg ktmp1, k
ins_pipe( pipe_slow );
%}
+instruct castFtoX_reg_avx10(vec dst, vec src) %{
+ predicate(VM_Version::supports_avx10_2() &&
+ is_integral_type(Matcher::vector_element_basic_type(n)));
+ match(Set dst (VectorCastF2X src));
+ format %{ "vector_cast_f2x_avx10 $dst, $src\t!" %}
+ ins_encode %{
+ BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
+ int vlen_enc = (to_elem_bt == T_LONG) ? vector_length_encoding(this) : vector_length_encoding(this, $src);
+ __ vector_castF2X_avx10(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
+ %}
+ ins_pipe( pipe_slow );
+%}
+
+instruct castFtoX_mem_avx10(vec dst, memory src) %{
+ predicate(VM_Version::supports_avx10_2() &&
+ is_integral_type(Matcher::vector_element_basic_type(n)));
+ match(Set dst (VectorCastF2X (LoadVector src)));
+ format %{ "vector_cast_f2x_avx10 $dst, $src\t!" %}
+ ins_encode %{
+ int vlen = Matcher::vector_length(this);
+ BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
+ int vlen_enc = (to_elem_bt == T_LONG) ? vector_length_encoding(this) : vector_length_encoding(vlen * sizeof(jfloat));
+ __ vector_castF2X_avx10(to_elem_bt, $dst$$XMMRegister, $src$$Address, vlen_enc);
+ %}
+ ins_pipe( pipe_slow );
+%}
+
instruct vcastDtoF_reg(vec dst, vec src) %{
predicate(Matcher::vector_element_basic_type(n) == T_FLOAT);
match(Set dst (VectorCastD2X src));
@@ -7721,7 +7752,9 @@ instruct vcastDtoF_reg(vec dst, vec src) %{
%}
instruct castDtoX_reg_avx(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, vec xtmp4, vec xtmp5, rFlagsReg cr) %{
- predicate(!VM_Version::supports_avx512vl() && Matcher::vector_length_in_bytes(n->in(1)) < 64 &&
+ predicate(!VM_Version::supports_avx10_2() &&
+ !VM_Version::supports_avx512vl() &&
+ Matcher::vector_length_in_bytes(n->in(1)) < 64 &&
is_integral_type(Matcher::vector_element_basic_type(n)));
match(Set dst (VectorCastD2X src));
effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP xtmp4, TEMP xtmp5, KILL cr);
@@ -7737,7 +7770,8 @@ instruct castDtoX_reg_avx(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, vec
%}
instruct castDtoX_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
- predicate((VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n->in(1)) == 64) &&
+ predicate(!VM_Version::supports_avx10_2() &&
+ (VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n->in(1)) == 64) &&
is_integral_type(Matcher::vector_element_basic_type(n)));
match(Set dst (VectorCastD2X src));
effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2, KILL cr);
@@ -7753,6 +7787,33 @@ instruct castDtoX_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2, kReg ktmp1, k
ins_pipe( pipe_slow );
%}
+instruct castDtoX_reg_avx10(vec dst, vec src) %{
+ predicate(VM_Version::supports_avx10_2() &&
+ is_integral_type(Matcher::vector_element_basic_type(n)));
+ match(Set dst (VectorCastD2X src));
+ format %{ "vector_cast_d2x_avx10 $dst, $src\t!" %}
+ ins_encode %{
+ int vlen_enc = vector_length_encoding(this, $src);
+ BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
+ __ vector_castD2X_avx10(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
+ %}
+ ins_pipe( pipe_slow );
+%}
+
+instruct castDtoX_mem_avx10(vec dst, memory src) %{
+ predicate(VM_Version::supports_avx10_2() &&
+ is_integral_type(Matcher::vector_element_basic_type(n)));
+ match(Set dst (VectorCastD2X (LoadVector src)));
+ format %{ "vector_cast_d2x_avx10 $dst, $src\t!" %}
+ ins_encode %{
+ int vlen = Matcher::vector_length(this);
+ int vlen_enc = vector_length_encoding(vlen * sizeof(jdouble));
+ BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
+ __ vector_castD2X_avx10(to_elem_bt, $dst$$XMMRegister, $src$$Address, vlen_enc);
+ %}
+ ins_pipe( pipe_slow );
+%}
+
instruct vucast(vec dst, vec src) %{
match(Set dst (VectorUCastB2X src));
match(Set dst (VectorUCastS2X src));
diff --git a/src/hotspot/cpu/x86/x86_64.ad b/src/hotspot/cpu/x86/x86_64.ad
index 0914bea82a1..b40f9e2924a 100644
--- a/src/hotspot/cpu/x86/x86_64.ad
+++ b/src/hotspot/cpu/x86/x86_64.ad
@@ -1697,11 +1697,6 @@ RegMask Matcher::modL_proj_mask() {
return LONG_RDX_REG_mask();
}
-// Register for saving SP into on method handle invokes. Not used on x86_64.
-const RegMask Matcher::method_handle_invoke_SP_save_mask() {
- return NO_REG_mask();
-}
-
%}
//----------ENCODING BLOCK-----------------------------------------------------
@@ -11712,6 +11707,7 @@ instruct convD2F_reg_mem(regF dst, memory src)
// XXX do mem variants
instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
%{
+ predicate(!VM_Version::supports_avx10_2());
match(Set dst (ConvF2I src));
effect(KILL cr);
format %{ "convert_f2i $dst, $src" %}
@@ -11721,8 +11717,31 @@ instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
ins_pipe(pipe_slow);
%}
+instruct convF2I_reg_reg_avx10(rRegI dst, regF src)
+%{
+ predicate(VM_Version::supports_avx10_2());
+ match(Set dst (ConvF2I src));
+ format %{ "evcvttss2sisl $dst, $src" %}
+ ins_encode %{
+ __ evcvttss2sisl($dst$$Register, $src$$XMMRegister);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct convF2I_reg_mem_avx10(rRegI dst, memory src)
+%{
+ predicate(VM_Version::supports_avx10_2());
+ match(Set dst (ConvF2I (LoadF src)));
+ format %{ "evcvttss2sisl $dst, $src" %}
+ ins_encode %{
+ __ evcvttss2sisl($dst$$Register, $src$$Address);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
%{
+ predicate(!VM_Version::supports_avx10_2());
match(Set dst (ConvF2L src));
effect(KILL cr);
format %{ "convert_f2l $dst, $src"%}
@@ -11732,8 +11751,31 @@ instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
ins_pipe(pipe_slow);
%}
+instruct convF2L_reg_reg_avx10(rRegL dst, regF src)
+%{
+ predicate(VM_Version::supports_avx10_2());
+ match(Set dst (ConvF2L src));
+ format %{ "evcvttss2sisq $dst, $src" %}
+ ins_encode %{
+ __ evcvttss2sisq($dst$$Register, $src$$XMMRegister);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct convF2L_reg_mem_avx10(rRegL dst, memory src)
+%{
+ predicate(VM_Version::supports_avx10_2());
+ match(Set dst (ConvF2L (LoadF src)));
+ format %{ "evcvttss2sisq $dst, $src" %}
+ ins_encode %{
+ __ evcvttss2sisq($dst$$Register, $src$$Address);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
%{
+ predicate(!VM_Version::supports_avx10_2());
match(Set dst (ConvD2I src));
effect(KILL cr);
format %{ "convert_d2i $dst, $src"%}
@@ -11743,8 +11785,31 @@ instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
ins_pipe(pipe_slow);
%}
+instruct convD2I_reg_reg_avx10(rRegI dst, regD src)
+%{
+ predicate(VM_Version::supports_avx10_2());
+ match(Set dst (ConvD2I src));
+ format %{ "evcvttsd2sisl $dst, $src" %}
+ ins_encode %{
+ __ evcvttsd2sisl($dst$$Register, $src$$XMMRegister);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct convD2I_reg_mem_avx10(rRegI dst, memory src)
+%{
+ predicate(VM_Version::supports_avx10_2());
+ match(Set dst (ConvD2I (LoadD src)));
+ format %{ "evcvttsd2sisl $dst, $src" %}
+ ins_encode %{
+ __ evcvttsd2sisl($dst$$Register, $src$$Address);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
%{
+ predicate(!VM_Version::supports_avx10_2());
match(Set dst (ConvD2L src));
effect(KILL cr);
format %{ "convert_d2l $dst, $src"%}
@@ -11754,6 +11819,28 @@ instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
ins_pipe(pipe_slow);
%}
+instruct convD2L_reg_reg_avx10(rRegL dst, regD src)
+%{
+ predicate(VM_Version::supports_avx10_2());
+ match(Set dst (ConvD2L src));
+ format %{ "evcvttsd2sisq $dst, $src" %}
+ ins_encode %{
+ __ evcvttsd2sisq($dst$$Register, $src$$XMMRegister);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct convD2L_reg_mem_avx10(rRegL dst, memory src)
+%{
+ predicate(VM_Version::supports_avx10_2());
+ match(Set dst (ConvD2L (LoadD src)));
+ format %{ "evcvttsd2sisq $dst, $src" %}
+ ins_encode %{
+ __ evcvttsd2sisq($dst$$Register, $src$$Address);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
instruct round_double_reg(rRegL dst, regD src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
%{
match(Set dst (RoundD src));
diff --git a/src/hotspot/cpu/zero/assembler_zero.inline.hpp b/src/hotspot/cpu/zero/assembler_zero.inline.hpp
index 0a3f4fc25fa..d78eb39c973 100644
--- a/src/hotspot/cpu/zero/assembler_zero.inline.hpp
+++ b/src/hotspot/cpu/zero/assembler_zero.inline.hpp
@@ -26,7 +26,7 @@
#ifndef CPU_ZERO_ASSEMBLER_ZERO_INLINE_HPP
#define CPU_ZERO_ASSEMBLER_ZERO_INLINE_HPP
-#include "asm/assembler.inline.hpp"
+#include "asm/assembler.hpp"
#include "asm/codeBuffer.hpp"
#include "code/codeCache.hpp"
#include "runtime/handles.inline.hpp"
diff --git a/src/hotspot/cpu/zero/sharedRuntime_zero.cpp b/src/hotspot/cpu/zero/sharedRuntime_zero.cpp
index b2e406c205b..6e18097c992 100644
--- a/src/hotspot/cpu/zero/sharedRuntime_zero.cpp
+++ b/src/hotspot/cpu/zero/sharedRuntime_zero.cpp
@@ -38,12 +38,6 @@
#include "opto/runtime.hpp"
#endif
-
-static address zero_null_code_stub() {
- address start = ShouldNotCallThisStub();
- return start;
-}
-
int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
VMRegPair *regs,
int total_args_passed) {
@@ -55,12 +49,9 @@ void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
int comp_args_on_stack,
const BasicType *sig_bt,
const VMRegPair *regs,
- AdapterHandlerEntry* handler) {
- // foil any attempt to call the i2c, c2i or unverified c2i entries
- handler->set_entry_points(CAST_FROM_FN_PTR(address,zero_null_code_stub),
- CAST_FROM_FN_PTR(address,zero_null_code_stub),
- CAST_FROM_FN_PTR(address,zero_null_code_stub),
- nullptr);
+ address entry_address[AdapterBlob::ENTRY_COUNT]) {
+ ShouldNotCallThis();
+ return;
}
nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
diff --git a/src/hotspot/os/aix/os_aix.cpp b/src/hotspot/os/aix/os_aix.cpp
index 2dd60b51119..0c0c2808fa1 100644
--- a/src/hotspot/os/aix/os_aix.cpp
+++ b/src/hotspot/os/aix/os_aix.cpp
@@ -43,7 +43,7 @@
#include "prims/jniFastGetField.hpp"
#include "prims/jvm_misc.hpp"
#include "runtime/arguments.hpp"
-#include "runtime/atomic.hpp"
+#include "runtime/atomicAccess.hpp"
#include "runtime/globals.hpp"
#include "runtime/globals_extension.hpp"
#include "runtime/interfaceSupport.inline.hpp"
@@ -169,7 +169,7 @@ static void vmembk_print_on(outputStream* os);
////////////////////////////////////////////////////////////////////////////////
// global variables (for a description see os_aix.hpp)
-size_t os::Aix::_physical_memory = 0;
+physical_memory_size_type os::Aix::_physical_memory = 0;
pthread_t os::Aix::_main_thread = ((pthread_t)0);
@@ -254,43 +254,43 @@ static bool is_close_to_brk(address a) {
return false;
}
-bool os::free_memory(size_t& value) {
+bool os::free_memory(physical_memory_size_type& value) {
return Aix::available_memory(value);
}
-bool os::available_memory(size_t& value) {
+bool os::available_memory(physical_memory_size_type& value) {
return Aix::available_memory(value);
}
-bool os::Aix::available_memory(size_t& value) {
+bool os::Aix::available_memory(physical_memory_size_type& value) {
os::Aix::meminfo_t mi;
if (os::Aix::get_meminfo(&mi)) {
- value = static_cast(mi.real_free);
+ value = static_cast(mi.real_free);
return true;
} else {
return false;
}
}
-bool os::total_swap_space(size_t& value) {
+bool os::total_swap_space(physical_memory_size_type& value) {
perfstat_memory_total_t memory_info;
if (libperfstat::perfstat_memory_total(nullptr, &memory_info, sizeof(perfstat_memory_total_t), 1) == -1) {
return false;
}
- value = static_cast(memory_info.pgsp_total * 4 * K);
+ value = static_cast(memory_info.pgsp_total * 4 * K);
return true;
}
-bool os::free_swap_space(size_t& value) {
+bool os::free_swap_space(physical_memory_size_type& value) {
perfstat_memory_total_t memory_info;
if (libperfstat::perfstat_memory_total(nullptr, &memory_info, sizeof(perfstat_memory_total_t), 1) == -1) {
return false;
}
- value = static_cast(memory_info.pgsp_free * 4 * K);
+ value = static_cast(memory_info.pgsp_free * 4 * K);
return true;
}
-size_t os::physical_memory() {
+physical_memory_size_type os::physical_memory() {
return Aix::physical_memory();
}
@@ -329,7 +329,7 @@ void os::Aix::initialize_system_info() {
if (!os::Aix::get_meminfo(&mi)) {
assert(false, "os::Aix::get_meminfo failed.");
}
- _physical_memory = static_cast(mi.real_total);
+ _physical_memory = static_cast(mi.real_total);
}
// Helper function for tracing page sizes.
@@ -2192,7 +2192,7 @@ jint os::init_2(void) {
os::Posix::init_2();
trcVerbose("processor count: %d", os::_processor_count);
- trcVerbose("physical memory: %zu", Aix::_physical_memory);
+ trcVerbose("physical memory: " PHYS_MEM_TYPE_FORMAT, Aix::_physical_memory);
// Initially build up the loaded dll map.
LoadedLibraries::reload();
diff --git a/src/hotspot/os/aix/os_aix.hpp b/src/hotspot/os/aix/os_aix.hpp
index 1530f2adb76..a7bac40e79b 100644
--- a/src/hotspot/os/aix/os_aix.hpp
+++ b/src/hotspot/os/aix/os_aix.hpp
@@ -35,7 +35,7 @@ class os::Aix {
private:
- static size_t _physical_memory;
+ static physical_memory_size_type _physical_memory;
static pthread_t _main_thread;
// 0 = uninitialized, otherwise 16 bit number:
@@ -54,9 +54,9 @@ class os::Aix {
// 1 - EXTSHM=ON
static int _extshm;
- static bool available_memory(size_t& value);
- static bool free_memory(size_t& value);
- static size_t physical_memory() { return _physical_memory; }
+ static bool available_memory(physical_memory_size_type& value);
+ static bool free_memory(physical_memory_size_type& value);
+ static physical_memory_size_type physical_memory() { return _physical_memory; }
static void initialize_system_info();
// OS recognitions (AIX OS level) call this before calling Aix::os_version().
diff --git a/src/hotspot/os/bsd/os_bsd.cpp b/src/hotspot/os/bsd/os_bsd.cpp
index 8b75c0dcdd8..8c5bbd58a84 100644
--- a/src/hotspot/os/bsd/os_bsd.cpp
+++ b/src/hotspot/os/bsd/os_bsd.cpp
@@ -39,7 +39,7 @@
#include "prims/jniFastGetField.hpp"
#include "prims/jvm_misc.hpp"
#include "runtime/arguments.hpp"
-#include "runtime/atomic.hpp"
+#include "runtime/atomicAccess.hpp"
#include "runtime/globals.hpp"
#include "runtime/globals_extension.hpp"
#include "runtime/interfaceSupport.inline.hpp"
@@ -114,7 +114,7 @@
////////////////////////////////////////////////////////////////////////////////
// global variables
-size_t os::Bsd::_physical_memory = 0;
+physical_memory_size_type os::Bsd::_physical_memory = 0;
#ifdef __APPLE__
mach_timebase_info_data_t os::Bsd::_timebase_info = {0, 0};
@@ -133,19 +133,19 @@ static volatile int processor_id_next = 0;
////////////////////////////////////////////////////////////////////////////////
// utility functions
-bool os::available_memory(size_t& value) {
+bool os::available_memory(physical_memory_size_type& value) {
return Bsd::available_memory(value);
}
-bool os::free_memory(size_t& value) {
+bool os::free_memory(physical_memory_size_type& value) {
return Bsd::available_memory(value);
}
// Available here means free. Note that this number is of no much use. As an estimate
// for future memory pressure it is far too conservative, since MacOS will use a lot
// of unused memory for caches, and return it willingly in case of needs.
-bool os::Bsd::available_memory(size_t& value) {
- uint64_t available = static_cast(physical_memory() >> 2);
+bool os::Bsd::available_memory(physical_memory_size_type& value) {
+ physical_memory_size_type available = physical_memory() >> 2;
#ifdef __APPLE__
mach_msg_type_number_t count = HOST_VM_INFO64_COUNT;
vm_statistics64_data_t vmstat;
@@ -160,7 +160,7 @@ bool os::Bsd::available_memory(size_t& value) {
return false;
}
#endif
- value = static_cast(available);
+ value = available;
return true;
}
@@ -180,35 +180,35 @@ void os::Bsd::print_uptime_info(outputStream* st) {
}
}
-bool os::total_swap_space(size_t& value) {
+bool os::total_swap_space(physical_memory_size_type& value) {
#if defined(__APPLE__)
struct xsw_usage vmusage;
size_t size = sizeof(vmusage);
if (sysctlbyname("vm.swapusage", &vmusage, &size, nullptr, 0) != 0) {
return false;
}
- value = static_cast(vmusage.xsu_total);
+ value = static_cast(vmusage.xsu_total);
return true;
#else
return false;
#endif
}
-bool os::free_swap_space(size_t& value) {
+bool os::free_swap_space(physical_memory_size_type& value) {
#if defined(__APPLE__)
struct xsw_usage vmusage;
size_t size = sizeof(vmusage);
if (sysctlbyname("vm.swapusage", &vmusage, &size, nullptr, 0) != 0) {
return false;
}
- value = static_cast(vmusage.xsu_avail);
+ value = static_cast(vmusage.xsu_avail);
return true;
#else
return false;
#endif
}
-size_t os::physical_memory() {
+physical_memory_size_type os::physical_memory() {
return Bsd::physical_memory();
}
@@ -286,7 +286,7 @@ void os::Bsd::initialize_system_info() {
len = sizeof(mem_val);
if (sysctl(mib, 2, &mem_val, &len, nullptr, 0) != -1) {
assert(len == sizeof(mem_val), "unexpected data size");
- _physical_memory = static_cast(mem_val);
+ _physical_memory = static_cast(mem_val);
} else {
_physical_memory = 256 * 1024 * 1024; // fallback (XXXBSD?)
}
@@ -297,7 +297,7 @@ void os::Bsd::initialize_system_info() {
// datasize rlimit restricts us anyway.
struct rlimit limits;
getrlimit(RLIMIT_DATA, &limits);
- _physical_memory = MIN2(_physical_memory, static_cast(limits.rlim_cur));
+ _physical_memory = MIN2(_physical_memory, static_cast(limits.rlim_cur));
}
#endif
}
@@ -809,7 +809,7 @@ jlong os::javaTimeNanos() {
if (now <= prev) {
return prev; // same or retrograde time;
}
- const uint64_t obsv = Atomic::cmpxchg(&Bsd::_max_abstime, prev, now);
+ const uint64_t obsv = AtomicAccess::cmpxchg(&Bsd::_max_abstime, prev, now);
assert(obsv >= prev, "invariant"); // Monotonicity
// If the CAS succeeded then we're done and return "now".
// If the CAS failed and the observed value "obsv" is >= now then
@@ -1469,12 +1469,12 @@ void os::print_memory_info(outputStream* st) {
st->print("Memory:");
st->print(" %zuk page", os::vm_page_size()>>10);
- size_t phys_mem = os::physical_memory();
- st->print(", physical %zuk",
+ physical_memory_size_type phys_mem = os::physical_memory();
+ st->print(", physical " PHYS_MEM_TYPE_FORMAT "k",
phys_mem >> 10);
- size_t avail_mem = 0;
+ physical_memory_size_type avail_mem = 0;
(void)os::available_memory(avail_mem);
- st->print("(%zuk free)",
+ st->print("(" PHYS_MEM_TYPE_FORMAT "k free)",
avail_mem >> 10);
if((sysctlbyname("vm.swapusage", &swap_usage, &size, nullptr, 0) == 0) || (errno == ENOMEM)) {
@@ -2133,14 +2133,14 @@ uint os::processor_id() {
__asm__ ("cpuid\n\t" : "+a" (eax), "+b" (ebx), "+c" (ecx), "+d" (edx) : );
uint apic_id = (ebx >> 24) & (processor_id_map_size - 1);
- int processor_id = Atomic::load(&processor_id_map[apic_id]);
+ int processor_id = AtomicAccess::load(&processor_id_map[apic_id]);
while (processor_id < 0) {
// Assign processor id to APIC id
- processor_id = Atomic::cmpxchg(&processor_id_map[apic_id], processor_id_unassigned, processor_id_assigning);
+ processor_id = AtomicAccess::cmpxchg(&processor_id_map[apic_id], processor_id_unassigned, processor_id_assigning);
if (processor_id == processor_id_unassigned) {
- processor_id = Atomic::fetch_then_add(&processor_id_next, 1) % os::processor_count();
- Atomic::store(&processor_id_map[apic_id], processor_id);
+ processor_id = AtomicAccess::fetch_then_add(&processor_id_next, 1) % os::processor_count();
+ AtomicAccess::store(&processor_id_map[apic_id], processor_id);
}
}
diff --git a/src/hotspot/os/bsd/os_bsd.hpp b/src/hotspot/os/bsd/os_bsd.hpp
index 173cc5a40ad..82002917f39 100644
--- a/src/hotspot/os/bsd/os_bsd.hpp
+++ b/src/hotspot/os/bsd/os_bsd.hpp
@@ -42,12 +42,12 @@ class os::Bsd {
protected:
- static size_t _physical_memory;
+ static physical_memory_size_type _physical_memory;
static pthread_t _main_thread;
- static bool available_memory(size_t& value);
- static bool free_memory(size_t& value);
- static size_t physical_memory() { return _physical_memory; }
+ static bool available_memory(physical_memory_size_type& value);
+ static bool free_memory(physical_memory_size_type& value);
+ static physical_memory_size_type physical_memory() { return _physical_memory; }
static void initialize_system_info();
static void rebuild_cpu_to_node_map();
diff --git a/src/hotspot/os/linux/cgroupSubsystem_linux.cpp b/src/hotspot/os/linux/cgroupSubsystem_linux.cpp
index f935e2cbb9c..f5c4abeb4ca 100644
--- a/src/hotspot/os/linux/cgroupSubsystem_linux.cpp
+++ b/src/hotspot/os/linux/cgroupSubsystem_linux.cpp
@@ -665,15 +665,13 @@ int CgroupSubsystem::active_processor_count() {
* -1 for unlimited
* OSCONTAINER_ERROR for not supported
*/
-jlong CgroupSubsystem::memory_limit_in_bytes() {
+jlong CgroupSubsystem::memory_limit_in_bytes(julong upper_bound) {
CachingCgroupController* contrl = memory_controller();
CachedMetric* memory_limit = contrl->metrics_cache();
if (!memory_limit->should_check_metric()) {
return memory_limit->value();
}
- julong phys_mem = static_cast(os::Linux::physical_memory());
- log_trace(os, container)("total physical memory: " JULONG_FORMAT, phys_mem);
- jlong mem_limit = contrl->controller()->read_memory_limit_in_bytes(phys_mem);
+ jlong mem_limit = contrl->controller()->read_memory_limit_in_bytes(upper_bound);
// Update cached metric to avoid re-reading container settings too often
memory_limit->set_value(mem_limit, OSCONTAINER_CACHE_TIMEOUT);
return mem_limit;
@@ -841,21 +839,16 @@ jlong CgroupController::limit_from_str(char* limit_str) {
// CgroupSubsystem implementations
-jlong CgroupSubsystem::memory_and_swap_limit_in_bytes() {
- julong phys_mem = static_cast(os::Linux::physical_memory());
- julong host_swap = os::Linux::host_swap();
- return memory_controller()->controller()->memory_and_swap_limit_in_bytes(phys_mem, host_swap);
+jlong CgroupSubsystem::memory_and_swap_limit_in_bytes(julong upper_mem_bound, julong upper_swap_bound) {
+ return memory_controller()->controller()->memory_and_swap_limit_in_bytes(upper_mem_bound, upper_swap_bound);
}
-jlong CgroupSubsystem::memory_and_swap_usage_in_bytes() {
- julong phys_mem = static_cast(os::Linux::physical_memory());
- julong host_swap = os::Linux::host_swap();
- return memory_controller()->controller()->memory_and_swap_usage_in_bytes(phys_mem, host_swap);
+jlong CgroupSubsystem::memory_and_swap_usage_in_bytes(julong upper_mem_bound, julong upper_swap_bound) {
+ return memory_controller()->controller()->memory_and_swap_usage_in_bytes(upper_mem_bound, upper_swap_bound);
}
-jlong CgroupSubsystem::memory_soft_limit_in_bytes() {
- julong phys_mem = static_cast(os::Linux::physical_memory());
- return memory_controller()->controller()->memory_soft_limit_in_bytes(phys_mem);
+jlong CgroupSubsystem::memory_soft_limit_in_bytes(julong upper_bound) {
+ return memory_controller()->controller()->memory_soft_limit_in_bytes(upper_bound);
}
jlong CgroupSubsystem::memory_throttle_limit_in_bytes() {
@@ -894,7 +887,6 @@ jlong CgroupSubsystem::cpu_usage_in_micros() {
return cpuacct_controller()->cpu_usage_in_micros();
}
-void CgroupSubsystem::print_version_specific_info(outputStream* st) {
- julong phys_mem = static_cast(os::Linux::physical_memory());
- memory_controller()->controller()->print_version_specific_info(st, phys_mem);
+void CgroupSubsystem::print_version_specific_info(outputStream* st, julong upper_mem_bound) {
+ memory_controller()->controller()->print_version_specific_info(st, upper_mem_bound);
}
diff --git a/src/hotspot/os/linux/cgroupSubsystem_linux.hpp b/src/hotspot/os/linux/cgroupSubsystem_linux.hpp
index 22e57d56c93..62a61432665 100644
--- a/src/hotspot/os/linux/cgroupSubsystem_linux.hpp
+++ b/src/hotspot/os/linux/cgroupSubsystem_linux.hpp
@@ -233,14 +233,14 @@ class CgroupMemoryController: public CHeapObj {
public:
virtual jlong read_memory_limit_in_bytes(julong upper_bound) = 0;
virtual jlong memory_usage_in_bytes() = 0;
- virtual jlong memory_and_swap_limit_in_bytes(julong host_mem, julong host_swap) = 0;
- virtual jlong memory_and_swap_usage_in_bytes(julong host_mem, julong host_swap) = 0;
+ virtual jlong memory_and_swap_limit_in_bytes(julong upper_mem_bound, julong upper_swap_bound) = 0;
+ virtual jlong memory_and_swap_usage_in_bytes(julong upper_mem_bound, julong upper_swap_bound) = 0;
virtual jlong memory_soft_limit_in_bytes(julong upper_bound) = 0;
virtual jlong memory_throttle_limit_in_bytes() = 0;
virtual jlong memory_max_usage_in_bytes() = 0;
virtual jlong rss_usage_in_bytes() = 0;
virtual jlong cache_usage_in_bytes() = 0;
- virtual void print_version_specific_info(outputStream* st, julong host_mem) = 0;
+ virtual void print_version_specific_info(outputStream* st, julong upper_mem_bound) = 0;
virtual bool needs_hierarchy_adjustment() = 0;
virtual bool is_read_only() = 0;
virtual const char* subsystem_path() = 0;
@@ -251,7 +251,7 @@ class CgroupMemoryController: public CHeapObj {
class CgroupSubsystem: public CHeapObj {
public:
- jlong memory_limit_in_bytes();
+ jlong memory_limit_in_bytes(julong upper_bound);
int active_processor_count();
virtual jlong pids_max() = 0;
@@ -272,14 +272,14 @@ class CgroupSubsystem: public CHeapObj {
jlong cpu_usage_in_micros();
jlong memory_usage_in_bytes();
- jlong memory_and_swap_limit_in_bytes();
- jlong memory_and_swap_usage_in_bytes();
- jlong memory_soft_limit_in_bytes();
+ jlong memory_and_swap_limit_in_bytes(julong upper_mem_bound, julong upper_swap_bound);
+ jlong memory_and_swap_usage_in_bytes(julong upper_mem_bound, julong upper_swap_bound);
+ jlong memory_soft_limit_in_bytes(julong upper_bound);
jlong memory_throttle_limit_in_bytes();
jlong memory_max_usage_in_bytes();
jlong rss_usage_in_bytes();
jlong cache_usage_in_bytes();
- void print_version_specific_info(outputStream* st);
+ void print_version_specific_info(outputStream* st, julong upper_mem_bound);
};
// Utility class for storing info retrieved from /proc/cgroups,
diff --git a/src/hotspot/os/linux/cgroupV1Subsystem_linux.cpp b/src/hotspot/os/linux/cgroupV1Subsystem_linux.cpp
index 64cb53eda28..90f01565b84 100644
--- a/src/hotspot/os/linux/cgroupV1Subsystem_linux.cpp
+++ b/src/hotspot/os/linux/cgroupV1Subsystem_linux.cpp
@@ -136,35 +136,35 @@ bool CgroupV1Controller::needs_hierarchy_adjustment() {
}
static inline
-void verbose_log(julong read_mem_limit, julong host_mem) {
+void verbose_log(julong read_mem_limit, julong upper_mem_bound) {
if (log_is_enabled(Debug, os, container)) {
jlong mem_limit = (jlong)read_mem_limit; // account for negative values
- if (mem_limit < 0 || read_mem_limit >= host_mem) {
+ if (mem_limit < 0 || read_mem_limit >= upper_mem_bound) {
const char *reason;
if (mem_limit == OSCONTAINER_ERROR) {
reason = "failed";
} else if (mem_limit == -1) {
reason = "unlimited";
} else {
- assert(read_mem_limit >= host_mem, "Expected read value exceeding host_mem");
+ assert(read_mem_limit >= upper_mem_bound, "Expected read value exceeding upper memory bound");
// Exceeding physical memory is treated as unlimited. This implementation
// caps it at host_mem since Cg v1 has no value to represent 'max'.
reason = "ignored";
}
- log_debug(os, container)("container memory limit %s: " JLONG_FORMAT ", using host value " JLONG_FORMAT,
- reason, mem_limit, host_mem);
+ log_debug(os, container)("container memory limit %s: " JLONG_FORMAT ", upper bound is " JLONG_FORMAT,
+ reason, mem_limit, upper_mem_bound);
}
}
}
-jlong CgroupV1MemoryController::read_memory_limit_in_bytes(julong phys_mem) {
+jlong CgroupV1MemoryController::read_memory_limit_in_bytes(julong upper_bound) {
julong memlimit;
CONTAINER_READ_NUMBER_CHECKED(reader(), "/memory.limit_in_bytes", "Memory Limit", memlimit);
- if (memlimit >= phys_mem) {
- verbose_log(memlimit, phys_mem);
+ if (memlimit >= upper_bound) {
+ verbose_log(memlimit, upper_bound);
return (jlong)-1;
} else {
- verbose_log(memlimit, phys_mem);
+ verbose_log(memlimit, upper_bound);
return (jlong)memlimit;
}
}
@@ -181,10 +181,10 @@ jlong CgroupV1MemoryController::read_memory_limit_in_bytes(julong phys_mem) {
* * -1 if there isn't any limit in place (note: includes values which exceed a physical
* upper bound)
*/
-jlong CgroupV1MemoryController::read_mem_swap(julong host_total_memsw) {
+jlong CgroupV1MemoryController::read_mem_swap(julong upper_memsw_bound) {
julong memswlimit;
CONTAINER_READ_NUMBER_CHECKED(reader(), "/memory.memsw.limit_in_bytes", "Memory and Swap Limit", memswlimit);
- if (memswlimit >= host_total_memsw) {
+ if (memswlimit >= upper_memsw_bound) {
log_trace(os, container)("Memory and Swap Limit is: Unlimited");
return (jlong)-1;
} else {
@@ -192,8 +192,8 @@ jlong CgroupV1MemoryController::read_mem_swap(julong host_total_memsw) {
}
}
-jlong CgroupV1MemoryController::memory_and_swap_limit_in_bytes(julong host_mem, julong host_swap) {
- jlong memory_swap = read_mem_swap(host_mem + host_swap);
+jlong CgroupV1MemoryController::memory_and_swap_limit_in_bytes(julong upper_mem_bound, julong upper_swap_bound) {
+ jlong memory_swap = read_mem_swap(upper_mem_bound + upper_swap_bound);
if (memory_swap == -1) {
return memory_swap;
}
@@ -202,7 +202,7 @@ jlong CgroupV1MemoryController::memory_and_swap_limit_in_bytes(julong host_mem,
// supported.
jlong swappiness = read_mem_swappiness();
if (swappiness == 0 || memory_swap == OSCONTAINER_ERROR) {
- jlong memlimit = read_memory_limit_in_bytes(host_mem);
+ jlong memlimit = read_memory_limit_in_bytes(upper_mem_bound);
if (memory_swap == OSCONTAINER_ERROR) {
log_trace(os, container)("Memory and Swap Limit has been reset to " JLONG_FORMAT " because swap is not supported", memlimit);
} else {
@@ -220,9 +220,9 @@ jlong memory_swap_usage_impl(CgroupController* ctrl) {
return (jlong)memory_swap_usage;
}
-jlong CgroupV1MemoryController::memory_and_swap_usage_in_bytes(julong phys_mem, julong host_swap) {
- jlong memory_sw_limit = memory_and_swap_limit_in_bytes(phys_mem, host_swap);
- jlong memory_limit = read_memory_limit_in_bytes(phys_mem);
+jlong CgroupV1MemoryController::memory_and_swap_usage_in_bytes(julong upper_mem_bound, julong upper_swap_bound) {
+ jlong memory_sw_limit = memory_and_swap_limit_in_bytes(upper_mem_bound, upper_swap_bound);
+ jlong memory_limit = read_memory_limit_in_bytes(upper_mem_bound);
if (memory_sw_limit > 0 && memory_limit > 0) {
jlong delta_swap = memory_sw_limit - memory_limit;
if (delta_swap > 0) {
@@ -238,10 +238,10 @@ jlong CgroupV1MemoryController::read_mem_swappiness() {
return (jlong)swappiness;
}
-jlong CgroupV1MemoryController::memory_soft_limit_in_bytes(julong phys_mem) {
+jlong CgroupV1MemoryController::memory_soft_limit_in_bytes(julong upper_bound) {
julong memsoftlimit;
CONTAINER_READ_NUMBER_CHECKED(reader(), "/memory.soft_limit_in_bytes", "Memory Soft Limit", memsoftlimit);
- if (memsoftlimit >= phys_mem) {
+ if (memsoftlimit >= upper_bound) {
log_trace(os, container)("Memory Soft Limit is: Unlimited");
return (jlong)-1;
} else {
@@ -336,10 +336,10 @@ jlong CgroupV1MemoryController::kernel_memory_usage_in_bytes() {
return (jlong)kmem_usage;
}
-jlong CgroupV1MemoryController::kernel_memory_limit_in_bytes(julong phys_mem) {
+jlong CgroupV1MemoryController::kernel_memory_limit_in_bytes(julong upper_bound) {
julong kmem_limit;
CONTAINER_READ_NUMBER_CHECKED(reader(), "/memory.kmem.limit_in_bytes", "Kernel Memory Limit", kmem_limit);
- if (kmem_limit >= phys_mem) {
+ if (kmem_limit >= upper_bound) {
return (jlong)-1;
}
return (jlong)kmem_limit;
@@ -351,9 +351,9 @@ jlong CgroupV1MemoryController::kernel_memory_max_usage_in_bytes() {
return (jlong)kmem_max_usage;
}
-void CgroupV1MemoryController::print_version_specific_info(outputStream* st, julong phys_mem) {
+void CgroupV1MemoryController::print_version_specific_info(outputStream* st, julong mem_bound) {
jlong kmem_usage = kernel_memory_usage_in_bytes();
- jlong kmem_limit = kernel_memory_limit_in_bytes(phys_mem);
+ jlong kmem_limit = kernel_memory_limit_in_bytes(mem_bound);
jlong kmem_max_usage = kernel_memory_max_usage_in_bytes();
OSContainer::print_container_helper(st, kmem_limit, "kernel_memory_limit_in_bytes");
diff --git a/src/hotspot/os/linux/cgroupV1Subsystem_linux.hpp b/src/hotspot/os/linux/cgroupV1Subsystem_linux.hpp
index 33bf2ed6e55..02b2c6a9fce 100644
--- a/src/hotspot/os/linux/cgroupV1Subsystem_linux.hpp
+++ b/src/hotspot/os/linux/cgroupV1Subsystem_linux.hpp
@@ -79,17 +79,17 @@ class CgroupV1MemoryController final : public CgroupMemoryController {
}
jlong read_memory_limit_in_bytes(julong upper_bound) override;
jlong memory_usage_in_bytes() override;
- jlong memory_and_swap_limit_in_bytes(julong host_mem, julong host_swap) override;
- jlong memory_and_swap_usage_in_bytes(julong host_mem, julong host_swap) override;
+ jlong memory_and_swap_limit_in_bytes(julong upper_mem_bound, julong upper_swap_bound) override;
+ jlong memory_and_swap_usage_in_bytes(julong upper_mem_bound, julong upper_swap_bound) override;
jlong memory_soft_limit_in_bytes(julong upper_bound) override;
jlong memory_throttle_limit_in_bytes() override;
jlong memory_max_usage_in_bytes() override;
jlong rss_usage_in_bytes() override;
jlong cache_usage_in_bytes() override;
jlong kernel_memory_usage_in_bytes();
- jlong kernel_memory_limit_in_bytes(julong host_mem);
+ jlong kernel_memory_limit_in_bytes(julong upper_bound);
jlong kernel_memory_max_usage_in_bytes();
- void print_version_specific_info(outputStream* st, julong host_mem) override;
+ void print_version_specific_info(outputStream* st, julong upper_mem_bound) override;
bool needs_hierarchy_adjustment() override {
return reader()->needs_hierarchy_adjustment();
}
@@ -101,7 +101,7 @@ class CgroupV1MemoryController final : public CgroupMemoryController {
const char* cgroup_path() override { return reader()->cgroup_path(); }
private:
jlong read_mem_swappiness();
- jlong read_mem_swap(julong host_total_memsw);
+ jlong read_mem_swap(julong upper_memsw_bound);
public:
CgroupV1MemoryController(const CgroupV1Controller& reader)
diff --git a/src/hotspot/os/linux/cgroupV2Subsystem_linux.cpp b/src/hotspot/os/linux/cgroupV2Subsystem_linux.cpp
index 661f7e909b5..38258a1f049 100644
--- a/src/hotspot/os/linux/cgroupV2Subsystem_linux.cpp
+++ b/src/hotspot/os/linux/cgroupV2Subsystem_linux.cpp
@@ -181,7 +181,7 @@ jlong CgroupV2MemoryController::memory_usage_in_bytes() {
return (jlong)memusage;
}
-jlong CgroupV2MemoryController::memory_soft_limit_in_bytes(julong phys_mem) {
+jlong CgroupV2MemoryController::memory_soft_limit_in_bytes(julong upper_bound) {
jlong mem_soft_limit;
CONTAINER_READ_NUMBER_CHECKED_MAX(reader(), "/memory.low", "Memory Soft Limit", mem_soft_limit);
return mem_soft_limit;
@@ -224,19 +224,19 @@ jlong CgroupV2MemoryController::cache_usage_in_bytes() {
// respectively. In order to properly report a cgroup v1 like
// compound value we need to sum the two values. Setting a swap limit
// without also setting a memory limit is not allowed.
-jlong CgroupV2MemoryController::memory_and_swap_limit_in_bytes(julong phys_mem,
- julong host_swap /* unused in cg v2 */) {
+jlong CgroupV2MemoryController::memory_and_swap_limit_in_bytes(julong upper_mem_bound,
+ julong upper_swap_bound /* unused in cg v2 */) {
jlong swap_limit;
bool is_ok = reader()->read_number_handle_max("/memory.swap.max", &swap_limit);
if (!is_ok) {
// Some container tests rely on this trace logging to happen.
log_trace(os, container)("Swap Limit failed: %d", OSCONTAINER_ERROR);
// swap disabled at kernel level, treat it as no swap
- return read_memory_limit_in_bytes(phys_mem);
+ return read_memory_limit_in_bytes(upper_mem_bound);
}
log_trace(os, container)("Swap Limit is: " JLONG_FORMAT, swap_limit);
if (swap_limit >= 0) {
- jlong memory_limit = read_memory_limit_in_bytes(phys_mem);
+ jlong memory_limit = read_memory_limit_in_bytes(upper_mem_bound);
assert(memory_limit >= 0, "swap limit without memory limit?");
return memory_limit + swap_limit;
}
@@ -252,7 +252,7 @@ jlong memory_swap_current_value(CgroupV2Controller* ctrl) {
return (jlong)swap_current;
}
-jlong CgroupV2MemoryController::memory_and_swap_usage_in_bytes(julong host_mem, julong host_swap) {
+jlong CgroupV2MemoryController::memory_and_swap_usage_in_bytes(julong upper_mem_bound, julong upper_swap_bound) {
jlong memory_usage = memory_usage_in_bytes();
if (memory_usage >= 0) {
jlong swap_current = memory_swap_current_value(reader());
@@ -276,7 +276,7 @@ jlong memory_limit_value(CgroupV2Controller* ctrl) {
* memory limit in bytes or
* -1 for unlimited, OSCONTAINER_ERROR for an error
*/
-jlong CgroupV2MemoryController::read_memory_limit_in_bytes(julong phys_mem) {
+jlong CgroupV2MemoryController::read_memory_limit_in_bytes(julong upper_bound) {
jlong limit = memory_limit_value(reader());
if (log_is_enabled(Trace, os, container)) {
if (limit == -1) {
@@ -287,18 +287,18 @@ jlong CgroupV2MemoryController::read_memory_limit_in_bytes(julong phys_mem) {
}
if (log_is_enabled(Debug, os, container)) {
julong read_limit = (julong)limit; // avoid signed/unsigned compare
- if (limit < 0 || read_limit >= phys_mem) {
+ if (limit < 0 || read_limit >= upper_bound) {
const char* reason;
if (limit == -1) {
reason = "unlimited";
} else if (limit == OSCONTAINER_ERROR) {
reason = "failed";
} else {
- assert(read_limit >= phys_mem, "Expected mem limit to exceed host memory");
+ assert(read_limit >= upper_bound, "Expected mem limit to exceed upper memory bound");
reason = "ignored";
}
- log_debug(os, container)("container memory limit %s: " JLONG_FORMAT ", using host value " JLONG_FORMAT,
- reason, limit, phys_mem);
+ log_debug(os, container)("container memory limit %s: " JLONG_FORMAT ", upper bound is " JLONG_FORMAT,
+ reason, limit, upper_bound);
}
}
return limit;
@@ -327,7 +327,7 @@ bool CgroupV2Controller::needs_hierarchy_adjustment() {
return strcmp(_cgroup_path, "/") != 0;
}
-void CgroupV2MemoryController::print_version_specific_info(outputStream* st, julong phys_mem) {
+void CgroupV2MemoryController::print_version_specific_info(outputStream* st, julong upper_mem_bound) {
jlong swap_current = memory_swap_current_value(reader());
jlong swap_limit = memory_swap_limit_value(reader());
diff --git a/src/hotspot/os/linux/cgroupV2Subsystem_linux.hpp b/src/hotspot/os/linux/cgroupV2Subsystem_linux.hpp
index e26f37925ca..07db126ce90 100644
--- a/src/hotspot/os/linux/cgroupV2Subsystem_linux.hpp
+++ b/src/hotspot/os/linux/cgroupV2Subsystem_linux.hpp
@@ -115,15 +115,15 @@ class CgroupV2MemoryController final: public CgroupMemoryController {
}
jlong read_memory_limit_in_bytes(julong upper_bound) override;
- jlong memory_and_swap_limit_in_bytes(julong host_mem, julong host_swp) override;
- jlong memory_and_swap_usage_in_bytes(julong host_mem, julong host_swp) override;
+ jlong memory_and_swap_limit_in_bytes(julong upper_mem_bound, julong upper_swap_bound) override;
+ jlong memory_and_swap_usage_in_bytes(julong upper_mem_bound, julong upper_swap_bound) override;
jlong memory_soft_limit_in_bytes(julong upper_bound) override;
jlong memory_throttle_limit_in_bytes() override;
jlong memory_usage_in_bytes() override;
jlong memory_max_usage_in_bytes() override;
jlong rss_usage_in_bytes() override;
jlong cache_usage_in_bytes() override;
- void print_version_specific_info(outputStream* st, julong host_mem) override;
+ void print_version_specific_info(outputStream* st, julong upper_mem_bound) override;
bool is_read_only() override {
return reader()->is_read_only();
}
diff --git a/src/hotspot/os/linux/compilerThreadTimeout_linux.hpp b/src/hotspot/os/linux/compilerThreadTimeout_linux.hpp
index 2dc6fa7b9c9..7c27080eb5e 100644
--- a/src/hotspot/os/linux/compilerThreadTimeout_linux.hpp
+++ b/src/hotspot/os/linux/compilerThreadTimeout_linux.hpp
@@ -46,6 +46,10 @@ class CompilerThreadTimeoutLinux : public CHeapObj {
bool init_timeout();
void arm();
void disarm();
+ void reset() {
+ disarm();
+ arm();
+ };
};
#endif //LINUX_COMPILER_THREAD_TIMEOUT_LINUX_HPP
diff --git a/src/hotspot/os/linux/memMapPrinter_linux.cpp b/src/hotspot/os/linux/memMapPrinter_linux.cpp
index 91fd552eec5..228a823dcb0 100644
--- a/src/hotspot/os/linux/memMapPrinter_linux.cpp
+++ b/src/hotspot/os/linux/memMapPrinter_linux.cpp
@@ -106,6 +106,7 @@ public:
PRINTIF(info.swap > 0, "swap");
PRINTIF(info.ht, "huge");
PRINTIF(info.anonhugepages > 0, "thp");
+ PRINTIF(info.thpeligible, "thpel");
PRINTIF(info.hg, "thpad");
PRINTIF(info.nh, "nothp");
if (num_printed == 0) {
@@ -135,6 +136,7 @@ public:
st->print_cr(" com: mapping committed (swap space reserved)");
st->print_cr(" swap: mapping partly or completely swapped out");
st->print_cr(" thp: mapping uses THP");
+ st->print_cr(" thpel: mapping is THP-eligible");
st->print_cr(" thpad: mapping is THP-madvised");
st->print_cr(" nothp: mapping is forbidden to use THP");
st->print_cr(" huge: mapping uses hugetlb pages");
diff --git a/src/hotspot/os/linux/osContainer_linux.cpp b/src/hotspot/os/linux/osContainer_linux.cpp
index 899e7535fde..561f2d4926c 100644
--- a/src/hotspot/os/linux/osContainer_linux.cpp
+++ b/src/hotspot/os/linux/osContainer_linux.cpp
@@ -84,8 +84,8 @@ void OSContainer::init() {
// We can be in one of two cases:
// 1.) On a physical Linux system without any limit
// 2.) On a physical Linux system with a limit enforced by other means (like systemd slice)
- any_mem_cpu_limit_present = cgroup_subsystem->memory_limit_in_bytes() > 0 ||
- os::Linux::active_processor_count() != cgroup_subsystem->active_processor_count();
+ any_mem_cpu_limit_present = memory_limit_in_bytes() > 0 ||
+ os::Linux::active_processor_count() != active_processor_count();
if (any_mem_cpu_limit_present) {
reason = " because either a cpu or a memory limit is present";
} else {
@@ -103,24 +103,47 @@ const char * OSContainer::container_type() {
return cgroup_subsystem->container_type();
}
+bool OSContainer::available_memory_in_container(julong& value) {
+ jlong mem_limit = memory_limit_in_bytes();
+ jlong mem_usage = memory_usage_in_bytes();
+
+ if (mem_limit > 0 && mem_usage <= 0) {
+ log_debug(os, container)("container memory usage failed: " JLONG_FORMAT, mem_usage);
+ }
+
+ if (mem_limit <= 0 || mem_usage <= 0) {
+ return false;
+ }
+
+ value = mem_limit > mem_usage ? static_cast(mem_limit - mem_usage) : 0;
+
+ return true;
+}
+
jlong OSContainer::memory_limit_in_bytes() {
assert(cgroup_subsystem != nullptr, "cgroup subsystem not available");
- return cgroup_subsystem->memory_limit_in_bytes();
+ julong phys_mem = static_cast(os::Linux::physical_memory());
+ return cgroup_subsystem->memory_limit_in_bytes(phys_mem);
}
jlong OSContainer::memory_and_swap_limit_in_bytes() {
assert(cgroup_subsystem != nullptr, "cgroup subsystem not available");
- return cgroup_subsystem->memory_and_swap_limit_in_bytes();
+ julong phys_mem = static_cast(os::Linux::physical_memory());
+ julong host_swap = os::Linux::host_swap();
+ return cgroup_subsystem->memory_and_swap_limit_in_bytes(phys_mem, host_swap);
}
jlong OSContainer::memory_and_swap_usage_in_bytes() {
assert(cgroup_subsystem != nullptr, "cgroup subsystem not available");
- return cgroup_subsystem->memory_and_swap_usage_in_bytes();
+ julong phys_mem = static_cast(os::Linux::physical_memory());
+ julong host_swap = os::Linux::host_swap();
+ return cgroup_subsystem->memory_and_swap_usage_in_bytes(phys_mem, host_swap);
}
jlong OSContainer::memory_soft_limit_in_bytes() {
assert(cgroup_subsystem != nullptr, "cgroup subsystem not available");
- return cgroup_subsystem->memory_soft_limit_in_bytes();
+ julong phys_mem = static_cast(os::Linux::physical_memory());
+ return cgroup_subsystem->memory_soft_limit_in_bytes(phys_mem);
}
jlong OSContainer::memory_throttle_limit_in_bytes() {
@@ -150,7 +173,8 @@ jlong OSContainer::cache_usage_in_bytes() {
void OSContainer::print_version_specific_info(outputStream* st) {
assert(cgroup_subsystem != nullptr, "cgroup subsystem not available");
- cgroup_subsystem->print_version_specific_info(st);
+ julong phys_mem = static_cast(os::Linux::physical_memory());
+ cgroup_subsystem->print_version_specific_info(st, phys_mem);
}
char * OSContainer::cpu_cpuset_cpus() {
diff --git a/src/hotspot/os/linux/osContainer_linux.hpp b/src/hotspot/os/linux/osContainer_linux.hpp
index d9b024dbbb5..6258714c48b 100644
--- a/src/hotspot/os/linux/osContainer_linux.hpp
+++ b/src/hotspot/os/linux/osContainer_linux.hpp
@@ -50,6 +50,7 @@ class OSContainer: AllStatic {
static inline bool is_containerized();
static const char * container_type();
+ static bool available_memory_in_container(julong& value);
static jlong memory_limit_in_bytes();
static jlong memory_and_swap_limit_in_bytes();
static jlong memory_and_swap_usage_in_bytes();
diff --git a/src/hotspot/os/linux/os_linux.cpp b/src/hotspot/os/linux/os_linux.cpp
index 9f896d62d4d..772b170d11c 100644
--- a/src/hotspot/os/linux/os_linux.cpp
+++ b/src/hotspot/os/linux/os_linux.cpp
@@ -42,7 +42,7 @@
#include "prims/jniFastGetField.hpp"
#include "prims/jvm_misc.hpp"
#include "runtime/arguments.hpp"
-#include "runtime/atomic.hpp"
+#include "runtime/atomicAccess.hpp"
#include "runtime/globals.hpp"
#include "runtime/globals_extension.hpp"
#include "runtime/init.hpp"
@@ -154,13 +154,12 @@ enum CoredumpFilterBit {
////////////////////////////////////////////////////////////////////////////////
// global variables
-size_t os::Linux::_physical_memory = 0;
+physical_memory_size_type os::Linux::_physical_memory = 0;
address os::Linux::_initial_thread_stack_bottom = nullptr;
uintptr_t os::Linux::_initial_thread_stack_size = 0;
int (*os::Linux::_pthread_getcpuclockid)(pthread_t, clockid_t *) = nullptr;
-int (*os::Linux::_pthread_setname_np)(pthread_t, const char*) = nullptr;
pthread_t os::Linux::_main_thread;
bool os::Linux::_supports_fast_thread_cpu_time = false;
const char * os::Linux::_libc_version = nullptr;
@@ -214,32 +213,19 @@ static bool suppress_primordial_thread_resolution = false;
// utility functions
-julong os::Linux::available_memory_in_container() {
- julong avail_mem = static_cast(-1L);
- if (OSContainer::is_containerized()) {
- jlong mem_limit = OSContainer::memory_limit_in_bytes();
- jlong mem_usage;
- if (mem_limit > 0 && (mem_usage = OSContainer::memory_usage_in_bytes()) < 1) {
- log_debug(os, container)("container memory usage failed: " JLONG_FORMAT ", using host value", mem_usage);
- }
- if (mem_limit > 0 && mem_usage > 0) {
- avail_mem = mem_limit > mem_usage ? (julong)mem_limit - (julong)mem_usage : 0;
- }
+bool os::available_memory(physical_memory_size_type& value) {
+ julong avail_mem = 0;
+ if (OSContainer::is_containerized() && OSContainer::available_memory_in_container(avail_mem)) {
+ log_trace(os)("available container memory: " JULONG_FORMAT, avail_mem);
+ value = static_cast(avail_mem);
+ return true;
}
- return avail_mem;
-}
-bool os::available_memory(size_t& value) {
return Linux::available_memory(value);
}
-bool os::Linux::available_memory(size_t& value) {
- julong avail_mem = available_memory_in_container();
- if (avail_mem != static_cast(-1L)) {
- log_trace(os)("available container memory: " JULONG_FORMAT, avail_mem);
- value = static_cast(avail_mem);
- return true;
- }
+bool os::Linux::available_memory(physical_memory_size_type& value) {
+ julong avail_mem = static_cast(-1L);
FILE *fp = os::fopen("/proc/meminfo", "r");
if (fp != nullptr) {
@@ -253,47 +239,48 @@ bool os::Linux::available_memory(size_t& value) {
fclose(fp);
}
if (avail_mem == static_cast(-1L)) {
- size_t free_mem = 0;
+ physical_memory_size_type free_mem = 0;
if (!free_memory(free_mem)) {
return false;
}
avail_mem = static_cast(free_mem);
}
log_trace(os)("available memory: " JULONG_FORMAT, avail_mem);
- value = static_cast(avail_mem);
+ value = static_cast(avail_mem);
return true;
}
-bool os::free_memory(size_t& value) {
+bool os::free_memory(physical_memory_size_type& value) {
+ julong free_mem = 0;
+ if (OSContainer::is_containerized() && OSContainer::available_memory_in_container(free_mem)) {
+ log_trace(os)("free container memory: " JULONG_FORMAT, free_mem);
+ value = static_cast(free_mem);
+ return true;
+ }
+
return Linux::free_memory(value);
}
-bool os::Linux::free_memory(size_t& value) {
+bool os::Linux::free_memory(physical_memory_size_type& value) {
// values in struct sysinfo are "unsigned long"
struct sysinfo si;
- julong free_mem = available_memory_in_container();
- if (free_mem != static_cast(-1L)) {
- log_trace(os)("free container memory: " JULONG_FORMAT, free_mem);
- value = static_cast(free_mem);
- return true;
- }
int ret = sysinfo(&si);
if (ret != 0) {
return false;
}
- free_mem = (julong)si.freeram * si.mem_unit;
+ julong free_mem = (julong)si.freeram * si.mem_unit;
log_trace(os)("free memory: " JULONG_FORMAT, free_mem);
- value = static_cast(free_mem);
+ value = static_cast(free_mem);
return true;
}
-bool os::total_swap_space(size_t& value) {
+bool os::total_swap_space(physical_memory_size_type& value) {
if (OSContainer::is_containerized()) {
jlong memory_and_swap_limit_in_bytes = OSContainer::memory_and_swap_limit_in_bytes();
jlong memory_limit_in_bytes = OSContainer::memory_limit_in_bytes();
if (memory_limit_in_bytes > 0 && memory_and_swap_limit_in_bytes > 0) {
- value = static_cast(memory_and_swap_limit_in_bytes - memory_limit_in_bytes);
+ value = static_cast(memory_and_swap_limit_in_bytes - memory_limit_in_bytes);
return true;
}
} // fallback to the host swap space if the container did return the unbound value of -1
@@ -303,30 +290,30 @@ bool os::total_swap_space(size_t& value) {
assert(false, "sysinfo failed in total_swap_space(): %s", os::strerror(errno));
return false;
}
- value = static_cast(si.totalswap * si.mem_unit);
+ value = static_cast(si.totalswap) * si.mem_unit;
return true;
}
-static bool host_free_swap_f(size_t& value) {
+static bool host_free_swap_f(physical_memory_size_type& value) {
struct sysinfo si;
int ret = sysinfo(&si);
if (ret != 0) {
assert(false, "sysinfo failed in host_free_swap_f(): %s", os::strerror(errno));
return false;
}
- value = static_cast(si.freeswap * si.mem_unit);
+ value = static_cast(si.freeswap) * si.mem_unit;
return true;
}
-bool os::free_swap_space(size_t& value) {
+bool os::free_swap_space(physical_memory_size_type& value) {
// os::total_swap_space() might return the containerized limit which might be
// less than host_free_swap(). The upper bound of free swap needs to be the lower of the two.
- size_t total_swap_space = 0;
- size_t host_free_swap = 0;
+ physical_memory_size_type total_swap_space = 0;
+ physical_memory_size_type host_free_swap = 0;
if (!os::total_swap_space(total_swap_space) || !host_free_swap_f(host_free_swap)) {
return false;
}
- size_t host_free_swap_val = MIN2(total_swap_space, host_free_swap);
+ physical_memory_size_type host_free_swap_val = MIN2(total_swap_space, host_free_swap);
if (OSContainer::is_containerized()) {
jlong mem_swap_limit = OSContainer::memory_and_swap_limit_in_bytes();
jlong mem_limit = OSContainer::memory_limit_in_bytes();
@@ -342,39 +329,48 @@ bool os::free_swap_space(size_t& value) {
jlong delta_usage = mem_swap_usage - mem_usage;
if (delta_usage >= 0) {
jlong free_swap = delta_limit - delta_usage;
- value = free_swap >= 0 ? static_cast(free_swap) : 0;
+ value = free_swap >= 0 ? static_cast(free_swap) : 0;
return true;
}
}
}
// unlimited or not supported. Fall through to return host value
log_trace(os,container)("os::free_swap_space: container_swap_limit=" JLONG_FORMAT
- " container_mem_limit=" JLONG_FORMAT " returning host value: %zu",
+ " container_mem_limit=" JLONG_FORMAT " returning host value: " PHYS_MEM_TYPE_FORMAT,
mem_swap_limit, mem_limit, host_free_swap_val);
}
value = host_free_swap_val;
return true;
}
-size_t os::physical_memory() {
+physical_memory_size_type os::physical_memory() {
if (OSContainer::is_containerized()) {
jlong mem_limit;
if ((mem_limit = OSContainer::memory_limit_in_bytes()) > 0) {
log_trace(os)("total container memory: " JLONG_FORMAT, mem_limit);
- return static_cast(mem_limit);
+ return static_cast(mem_limit);
}
}
- size_t phys_mem = Linux::physical_memory();
- log_trace(os)("total system memory: %zu", phys_mem);
+ physical_memory_size_type phys_mem = Linux::physical_memory();
+ log_trace(os)("total system memory: " PHYS_MEM_TYPE_FORMAT, phys_mem);
return phys_mem;
}
+// Returns the resident set size (RSS) of the process.
+// Falls back to using VmRSS from /proc/self/status if /proc/self/smaps_rollup is unavailable.
+// Note: On kernels with memory cgroups or shared memory, VmRSS may underreport RSS.
+// Users requiring accurate RSS values should be aware of this limitation.
size_t os::rss() {
size_t size = 0;
- os::Linux::meminfo_t info;
- if (os::Linux::query_process_memory_info(&info)) {
- size = info.vmrss * K;
+ os::Linux::accurate_meminfo_t accurate_info;
+ if (os::Linux::query_accurate_process_memory_info(&accurate_info) && accurate_info.rss != -1) {
+ size = accurate_info.rss * K;
+ } else {
+ os::Linux::meminfo_t info;
+ if (os::Linux::query_process_memory_info(&info)) {
+ size = info.vmrss * K;
+ }
}
return size;
}
@@ -541,7 +537,7 @@ void os::Linux::initialize_system_info() {
fclose(fp);
}
}
- _physical_memory = static_cast(sysconf(_SC_PHYS_PAGES)) * static_cast(sysconf(_SC_PAGESIZE));
+ _physical_memory = static_cast(sysconf(_SC_PHYS_PAGES)) * static_cast(sysconf(_SC_PAGESIZE));
assert(processor_count() > 0, "linux error");
}
@@ -851,11 +847,9 @@ static void *thread_native_entry(Thread *thread) {
osthread->set_thread_id(checked_cast(os::current_thread_id()));
if (UseNUMA) {
- int lgrp_id = os::numa_get_group_id();
- if (lgrp_id != -1) {
- thread->set_lgrp_id(lgrp_id);
- }
+ thread->update_lgrp_id();
}
+
// initialize signal mask for this thread
PosixSignals::hotspot_sigmask(thread);
@@ -1182,10 +1176,7 @@ bool os::create_attached_thread(JavaThread* thread) {
thread->set_osthread(osthread);
if (UseNUMA) {
- int lgrp_id = os::numa_get_group_id();
- if (lgrp_id != -1) {
- thread->set_lgrp_id(lgrp_id);
- }
+ thread->update_lgrp_id();
}
if (os::is_primordial_thread()) {
@@ -2362,6 +2353,37 @@ bool os::Linux::query_process_memory_info(os::Linux::meminfo_t* info) {
return false;
}
+// Accurate memory information need Linux 4.14 or newer
+bool os::Linux::query_accurate_process_memory_info(os::Linux::accurate_meminfo_t* info) {
+ FILE* f = os::fopen("/proc/self/smaps_rollup", "r");
+ if (f == nullptr) {
+ return false;
+ }
+
+ const size_t num_values = sizeof(os::Linux::accurate_meminfo_t) / sizeof(size_t);
+ size_t num_found = 0;
+ char buf[256];
+ info->rss = info->pss = info->pssdirty = info->pssanon =
+ info->pssfile = info->pssshmem = info->swap = info->swappss = -1;
+
+ while (::fgets(buf, sizeof(buf), f) != nullptr && num_found < num_values) {
+ if ( (info->rss == -1 && sscanf(buf, "Rss: %zd kB", &info->rss) == 1) ||
+ (info->pss == -1 && sscanf(buf, "Pss: %zd kB", &info->pss) == 1) ||
+ (info->pssdirty == -1 && sscanf(buf, "Pss_Dirty: %zd kB", &info->pssdirty) == 1) ||
+ (info->pssanon == -1 && sscanf(buf, "Pss_Anon: %zd kB", &info->pssanon) == 1) ||
+ (info->pssfile == -1 && sscanf(buf, "Pss_File: %zd kB", &info->pssfile) == 1) ||
+ (info->pssshmem == -1 && sscanf(buf, "Pss_Shmem: %zd kB", &info->pssshmem) == 1) ||
+ (info->swap == -1 && sscanf(buf, "Swap: %zd kB", &info->swap) == 1) ||
+ (info->swappss == -1 && sscanf(buf, "SwapPss: %zd kB", &info->swappss) == 1)
+ )
+ {
+ num_found ++;
+ }
+ }
+ fclose(f);
+ return true;
+}
+
#ifdef __GLIBC__
// For Glibc, print a one-liner with the malloc tunables.
// Most important and popular is MALLOC_ARENA_MAX, but we are
@@ -2569,12 +2591,12 @@ void os::print_memory_info(outputStream* st) {
// values in struct sysinfo are "unsigned long"
struct sysinfo si;
sysinfo(&si);
- size_t phys_mem = physical_memory();
- st->print(", physical %zuk",
+ physical_memory_size_type phys_mem = physical_memory();
+ st->print(", physical " PHYS_MEM_TYPE_FORMAT "k",
phys_mem >> 10);
- size_t avail_mem = 0;
+ physical_memory_size_type avail_mem = 0;
(void)os::available_memory(avail_mem);
- st->print("(%zuk free)",
+ st->print("(" PHYS_MEM_TYPE_FORMAT "k free)",
avail_mem >> 10);
st->print(", swap " UINT64_FORMAT "k",
((jlong)si.totalswap * si.mem_unit) >> 10);
@@ -4336,10 +4358,6 @@ void os::init(void) {
// _main_thread points to the thread that created/loaded the JVM.
Linux::_main_thread = pthread_self();
- // retrieve entry point for pthread_setname_np
- Linux::_pthread_setname_np =
- (int(*)(pthread_t, const char*))dlsym(RTLD_DEFAULT, "pthread_setname_np");
-
check_pax();
// Check the availability of MADV_POPULATE_WRITE.
@@ -4781,8 +4799,8 @@ static bool should_warn_invalid_processor_id() {
static volatile int warn_once = 1;
- if (Atomic::load(&warn_once) == 0 ||
- Atomic::xchg(&warn_once, 0) == 0) {
+ if (AtomicAccess::load(&warn_once) == 0 ||
+ AtomicAccess::xchg(&warn_once, 0) == 0) {
// Don't warn more than once
return false;
}
@@ -4816,14 +4834,24 @@ uint os::processor_id() {
}
void os::set_native_thread_name(const char *name) {
- if (Linux::_pthread_setname_np) {
- char buf [16]; // according to glibc manpage, 16 chars incl. '/0'
- (void) os::snprintf(buf, sizeof(buf), "%s", name);
- buf[sizeof(buf) - 1] = '\0';
- const int rc = Linux::_pthread_setname_np(pthread_self(), buf);
- // ERANGE should not happen; all other errors should just be ignored.
- assert(rc != ERANGE, "pthread_setname_np failed");
+ char buf[16]; // according to glibc manpage, 16 chars incl. '/0'
+ // We may need to truncate the thread name. Since a common pattern
+ // for thread names is to be both longer than 15 chars and have a
+ // trailing number ("DispatcherWorkerThread21", "C2 CompilerThread#54" etc),
+ // we preserve the end of the thread name by truncating the middle
+ // (e.g. "Dispatc..read21").
+ const size_t len = strlen(name);
+ if (len < sizeof(buf)) {
+ strcpy(buf, name);
+ } else {
+ (void) os::snprintf(buf, sizeof(buf), "%.7s..%.6s", name, name + len - 6);
}
+ // Note: we use the system call here instead of calling pthread_setname_np
+ // since this is the only way to make ASAN aware of our thread names. Even
+ // though ASAN intercepts both prctl and pthread_setname_np, it only processes
+ // the thread name given to the former.
+ int rc = prctl(PR_SET_NAME, buf);
+ assert(rc == 0, "prctl(PR_SET_NAME) failed");
}
////////////////////////////////////////////////////////////////////////////////
diff --git a/src/hotspot/os/linux/os_linux.hpp b/src/hotspot/os/linux/os_linux.hpp
index d3e0d6c5668..b77cd9f3c81 100644
--- a/src/hotspot/os/linux/os_linux.hpp
+++ b/src/hotspot/os/linux/os_linux.hpp
@@ -33,7 +33,6 @@ class os::Linux {
friend class os;
static int (*_pthread_getcpuclockid)(pthread_t, clockid_t *);
- static int (*_pthread_setname_np)(pthread_t, const char*);
static address _initial_thread_stack_bottom;
static uintptr_t _initial_thread_stack_size;
@@ -50,11 +49,11 @@ class os::Linux {
protected:
- static size_t _physical_memory;
+ static physical_memory_size_type _physical_memory;
static pthread_t _main_thread;
- static bool available_memory(size_t& value);
- static bool free_memory(size_t& value);
+ static bool available_memory(physical_memory_size_type& value);
+ static bool free_memory(physical_memory_size_type& value);
static void initialize_system_info();
@@ -117,7 +116,7 @@ class os::Linux {
static address initial_thread_stack_bottom(void) { return _initial_thread_stack_bottom; }
static uintptr_t initial_thread_stack_size(void) { return _initial_thread_stack_size; }
- static size_t physical_memory() { return _physical_memory; }
+ static physical_memory_size_type physical_memory() { return _physical_memory; }
static julong host_swap();
static intptr_t* ucontext_get_sp(const ucontext_t* uc);
@@ -181,6 +180,23 @@ class os::Linux {
// fields will contain -1.
static bool query_process_memory_info(meminfo_t* info);
+ // Output structure for query_accurate_process_memory_info() (all values in KB)
+ struct accurate_meminfo_t {
+ ssize_t rss; // current resident set size
+ ssize_t pss; // current proportional set size
+ ssize_t pssdirty; // proportional set size (dirty)
+ ssize_t pssanon; // proportional set size (anonymous mappings)
+ ssize_t pssfile; // proportional set size (file mappings)
+ ssize_t pssshmem; // proportional set size (shared mappings)
+ ssize_t swap; // swapped out
+ ssize_t swappss; // proportional set size (swapped out)
+ };
+
+ // Attempts to query accurate memory information from /proc/self/smaps_rollup and return it in the output structure.
+ // May fail (returns false) or succeed (returns true) but not all output fields are available; unavailable
+ // fields will contain -1.
+ static bool query_accurate_process_memory_info(accurate_meminfo_t* info);
+
// Tells if the user asked for transparent huge pages.
static bool _thp_requested;
diff --git a/src/hotspot/os/linux/procMapsParser.cpp b/src/hotspot/os/linux/procMapsParser.cpp
index 71b828bcefb..47c5c6cc594 100644
--- a/src/hotspot/os/linux/procMapsParser.cpp
+++ b/src/hotspot/os/linux/procMapsParser.cpp
@@ -76,8 +76,16 @@ void ProcSmapsParser::scan_additional_line(ProcSmapsInfo& out) {
SCAN("Private_Hugetlb", out.private_hugetlb);
SCAN("Shared_Hugetlb", out.shared_hugetlb);
SCAN("Swap", out.swap);
- int i = 0;
#undef SCAN
+
+ // scan THPeligible into a bool
+ int thpel = 0;
+ if (::sscanf(_line, "THPeligible: %d", &thpel) == 1) {
+ assert(thpel == 1 || thpel == 0, "Unexpected value %d", thpel);
+ out.thpeligible = (thpel == 1);
+ return;
+ }
+
// scan some flags too
if (strncmp(_line, "VmFlags:", 8) == 0) {
#define SCAN(flag) { out.flag = (::strstr(_line + 8, " " #flag) != nullptr); }
diff --git a/src/hotspot/os/linux/procMapsParser.hpp b/src/hotspot/os/linux/procMapsParser.hpp
index 0971c4fb084..06035333b2f 100644
--- a/src/hotspot/os/linux/procMapsParser.hpp
+++ b/src/hotspot/os/linux/procMapsParser.hpp
@@ -49,6 +49,7 @@ struct ProcSmapsInfo {
size_t shared_hugetlb;
size_t anonhugepages;
size_t swap;
+ bool thpeligible;
bool rd, wr, ex;
bool sh; // shared
bool nr; // no reserve
@@ -64,7 +65,7 @@ struct ProcSmapsInfo {
from = to = nullptr;
prot[0] = filename[0] = '\0';
kernelpagesize = rss = private_hugetlb = shared_hugetlb = anonhugepages = swap = 0;
- rd = wr = ex = sh = nr = hg = ht = nh = false;
+ thpeligible = rd = wr = ex = sh = nr = hg = ht = nh = false;
}
};
diff --git a/src/hotspot/os/posix/forbiddenFunctions_posix.hpp b/src/hotspot/os/posix/forbiddenFunctions_posix.hpp
index 9f9881202c7..529cf22078e 100644
--- a/src/hotspot/os/posix/forbiddenFunctions_posix.hpp
+++ b/src/hotspot/os/posix/forbiddenFunctions_posix.hpp
@@ -37,28 +37,28 @@
#endif
// If needed, add os::strndup and use that instead.
-FORBID_C_FUNCTION(char* strndup(const char*, size_t), "don't use");
+FORBID_C_FUNCTION(char* strndup(const char*, size_t), noexcept, "don't use");
// These are unimplementable for Windows, and they aren't useful for a
// POSIX implementation of NMT either.
// https://stackoverflow.com/questions/62962839/stdaligned-alloc-missing-from-visual-studio-2019
-FORBID_C_FUNCTION(int posix_memalign(void**, size_t, size_t), "don't use");
-FORBID_C_FUNCTION(void* aligned_alloc(size_t, size_t), "don't use");
+FORBID_C_FUNCTION(int posix_memalign(void**, size_t, size_t), noexcept, "don't use");
+FORBID_C_FUNCTION(void* aligned_alloc(size_t, size_t), noexcept, "don't use");
// realpath with a null second argument mallocs a string for the result.
// With a non-null second argument, there is a risk of buffer overrun.
PRAGMA_DIAG_PUSH
FORBIDDEN_FUNCTION_IGNORE_CLANG_FORTIFY_WARNING
-FORBID_C_FUNCTION(char* realpath(const char*, char*), "use os::realpath");
+FORBID_C_FUNCTION(char* realpath(const char*, char*), noexcept, "use os::realpath");
PRAGMA_DIAG_POP
// Returns a malloc'ed string.
-FORBID_C_FUNCTION(char* get_current_dir_name(), "use os::get_current_directory");
+FORBID_C_FUNCTION(char* get_current_dir_name(), noexcept, "use os::get_current_directory");
// Problematic API that should never be used.
-FORBID_C_FUNCTION(char* getwd(char*), "use os::get_current_directory");
+FORBID_C_FUNCTION(char* getwd(char*), noexcept, "use os::get_current_directory");
// BSD utility that is subtly different from realloc.
-FORBID_C_FUNCTION(void* reallocf(void*, size_t), "use os::realloc");
+FORBID_C_FUNCTION(void* reallocf(void*, size_t), /* not noexcept */, "use os::realloc");
#endif // OS_POSIX_FORBIDDENFUNCTIONS_POSIX_HPP
diff --git a/src/hotspot/os/posix/os_posix.cpp b/src/hotspot/os/posix/os_posix.cpp
index 8f188ef002f..1a04cbba0de 100644
--- a/src/hotspot/os/posix/os_posix.cpp
+++ b/src/hotspot/os/posix/os_posix.cpp
@@ -31,7 +31,7 @@
#include "nmt/memTracker.hpp"
#include "os_posix.inline.hpp"
#include "runtime/arguments.hpp"
-#include "runtime/atomic.hpp"
+#include "runtime/atomicAccess.hpp"
#include "runtime/frame.inline.hpp"
#include "runtime/globals_extension.hpp"
#include "runtime/interfaceSupport.inline.hpp"
@@ -1691,7 +1691,7 @@ void PlatformEvent::park() { // AKA "down()"
// atomically decrement _event
for (;;) {
v = _event;
- if (Atomic::cmpxchg(&_event, v, v - 1) == v) break;
+ if (AtomicAccess::cmpxchg(&_event, v, v - 1) == v) break;
}
guarantee(v >= 0, "invariant");
@@ -1738,7 +1738,7 @@ int PlatformEvent::park_nanos(jlong nanos) {
// atomically decrement _event
for (;;) {
v = _event;
- if (Atomic::cmpxchg(&_event, v, v - 1) == v) break;
+ if (AtomicAccess::cmpxchg(&_event, v, v - 1) == v) break;
}
guarantee(v >= 0, "invariant");
@@ -1794,7 +1794,7 @@ void PlatformEvent::unpark() {
// but only in the correctly written condition checking loops of ObjectMonitor,
// Mutex/Monitor, and JavaThread::sleep
- if (Atomic::xchg(&_event, 1) >= 0) return;
+ if (AtomicAccess::xchg(&_event, 1) >= 0) return;
int status = pthread_mutex_lock(_mutex);
assert_status(status == 0, status, "mutex_lock");
@@ -1847,9 +1847,9 @@ void Parker::park(bool isAbsolute, jlong time) {
// Optional fast-path check:
// Return immediately if a permit is available.
- // We depend on Atomic::xchg() having full barrier semantics
+ // We depend on AtomicAccess::xchg() having full barrier semantics
// since we are doing a lock-free update to _counter.
- if (Atomic::xchg(&_counter, 0) > 0) return;
+ if (AtomicAccess::xchg(&_counter, 0) > 0) return;
JavaThread *jt = JavaThread::current();
diff --git a/src/hotspot/os/posix/os_posix.inline.hpp b/src/hotspot/os/posix/os_posix.inline.hpp
index 67be82e4d63..f1ec9411f2a 100644
--- a/src/hotspot/os/posix/os_posix.inline.hpp
+++ b/src/hotspot/os/posix/os_posix.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -34,9 +34,6 @@
#include
#include
-// Aix does not have NUMA support but need these for compilation.
-inline bool os::numa_has_group_homing() { AIX_ONLY(ShouldNotReachHere();) return false; }
-
// Platform Mutex/Monitor implementation
inline void PlatformMutex::lock() {
diff --git a/src/hotspot/os/posix/safefetch_sigjmp.cpp b/src/hotspot/os/posix/safefetch_sigjmp.cpp
index 9f6ef34070b..572805acb7b 100644
--- a/src/hotspot/os/posix/safefetch_sigjmp.cpp
+++ b/src/hotspot/os/posix/safefetch_sigjmp.cpp
@@ -67,6 +67,14 @@ ATTRIBUTE_NO_ASAN static bool _SafeFetchXX_internal(const T *adr, T* result) {
T n = 0;
+#ifdef AIX
+ // AIX allows reading from nullptr without signalling
+ if (adr == nullptr) {
+ *result = 0;
+ return false;
+ }
+#endif
+
// Set up a jump buffer. Anchor its pointer in TLS. Then read from the unsafe address.
// If that address was invalid, we fault, and in the signal handler we will jump back
// to the jump point.
diff --git a/src/hotspot/os/posix/signals_posix.cpp b/src/hotspot/os/posix/signals_posix.cpp
index 0157d354f40..714eac12d22 100644
--- a/src/hotspot/os/posix/signals_posix.cpp
+++ b/src/hotspot/os/posix/signals_posix.cpp
@@ -28,7 +28,7 @@
#include "jvm.h"
#include "logging/log.hpp"
#include "os_posix.hpp"
-#include "runtime/atomic.hpp"
+#include "runtime/atomicAccess.hpp"
#include "runtime/globals.hpp"
#include "runtime/interfaceSupport.inline.hpp"
#include "runtime/java.hpp"
@@ -356,7 +356,7 @@ static void jdk_misc_signal_init() {
void os::signal_notify(int sig) {
if (sig_semaphore != nullptr) {
- Atomic::inc(&pending_signals[sig]);
+ AtomicAccess::inc(&pending_signals[sig]);
sig_semaphore->signal();
} else {
// Signal thread is not created with ReduceSignalUsage and jdk_misc_signal_init
@@ -369,7 +369,7 @@ static int check_pending_signals() {
for (;;) {
for (int i = 0; i < NSIG + 1; i++) {
jint n = pending_signals[i];
- if (n > 0 && n == Atomic::cmpxchg(&pending_signals[i], n, n - 1)) {
+ if (n > 0 && n == AtomicAccess::cmpxchg(&pending_signals[i], n, n - 1)) {
return i;
}
}
@@ -621,9 +621,7 @@ int JVM_HANDLE_XXX_SIGNAL(int sig, siginfo_t* info,
if (cb != nullptr && cb->is_nmethod()) {
nmethod* nm = cb->as_nmethod();
assert(nm->insts_contains_inclusive(pc), "");
- address deopt = nm->is_method_handle_return(pc) ?
- nm->deopt_mh_handler_begin() :
- nm->deopt_handler_begin();
+ address deopt = nm->deopt_handler_begin();
assert(deopt != nullptr, "");
frame fr = os::fetch_frame_from_context(uc);
diff --git a/src/hotspot/os/posix/suspendResume_posix.cpp b/src/hotspot/os/posix/suspendResume_posix.cpp
index dbd0e791d77..01f67a981a8 100644
--- a/src/hotspot/os/posix/suspendResume_posix.cpp
+++ b/src/hotspot/os/posix/suspendResume_posix.cpp
@@ -22,7 +22,7 @@
*
*/
-#include "runtime/atomic.hpp"
+#include "runtime/atomicAccess.hpp"
#include "suspendResume_posix.hpp"
/* try to switch state from state "from" to state "to"
@@ -31,7 +31,7 @@
SuspendResume::State SuspendResume::switch_state(SuspendResume::State from,
SuspendResume::State to)
{
- SuspendResume::State result = Atomic::cmpxchg(&_state, from, to);
+ SuspendResume::State result = AtomicAccess::cmpxchg(&_state, from, to);
if (result == from) {
// success
return to;
diff --git a/src/hotspot/os/windows/forbiddenFunctions_windows.hpp b/src/hotspot/os/windows/forbiddenFunctions_windows.hpp
index 81599522ed4..6e2800aa916 100644
--- a/src/hotspot/os/windows/forbiddenFunctions_windows.hpp
+++ b/src/hotspot/os/windows/forbiddenFunctions_windows.hpp
@@ -29,10 +29,12 @@
#include // for size_t
+// NOTE: The Windows C standard library doesn't declare functions "noexcept".
+
// _fullpath with a null first argument mallocs a string for the result.
-FORBID_IMPORTED_C_FUNCTION(char* _fullpath(char*, const char*, size_t), "use os::realpath");
+FORBID_IMPORTED_C_FUNCTION(char* _fullpath(char*, const char*, size_t), /* not noexcept */, "use os::realpath");
// _snprintf does NOT null terminate if the output would exceed the buffer size.
-FORBID_C_FUNCTION(int _snprintf(char*, size_t, const char*, ...), "use os::snprintf");
+FORBID_C_FUNCTION(int _snprintf(char*, size_t, const char*, ...), /* not noexcept */, "use os::snprintf");
#endif // OS_WINDOWS_FORBIDDENFUNCTIONS_WINDOWS_HPP
diff --git a/src/hotspot/os/windows/os_windows.cpp b/src/hotspot/os/windows/os_windows.cpp
index 4061ccf9dac..ba05d390c9f 100644
--- a/src/hotspot/os/windows/os_windows.cpp
+++ b/src/hotspot/os/windows/os_windows.cpp
@@ -42,7 +42,7 @@
#include "prims/jniFastGetField.hpp"
#include "prims/jvm_misc.hpp"
#include "runtime/arguments.hpp"
-#include "runtime/atomic.hpp"
+#include "runtime/atomicAccess.hpp"
#include "runtime/globals.hpp"
#include "runtime/globals_extension.hpp"
#include "runtime/interfaceSupport.inline.hpp"
@@ -534,13 +534,6 @@ static unsigned thread_native_entry(void* t) {
OSThread* osthr = thread->osthread();
assert(osthr->get_state() == RUNNABLE, "invalid os thread state");
- if (UseNUMA) {
- int lgrp_id = os::numa_get_group_id();
- if (lgrp_id != -1) {
- thread->set_lgrp_id(lgrp_id);
- }
- }
-
// Diagnostic code to investigate JDK-6573254
int res = 30115; // non-java thread
if (thread->is_Java_thread()) {
@@ -598,13 +591,6 @@ static OSThread* create_os_thread(Thread* thread, HANDLE thread_handle,
osthread->set_thread_handle(thread_handle);
osthread->set_thread_id(thread_id);
- if (UseNUMA) {
- int lgrp_id = os::numa_get_group_id();
- if (lgrp_id != -1) {
- thread->set_lgrp_id(lgrp_id);
- }
- }
-
// Initial thread state is INITIALIZED, not SUSPENDED
osthread->set_state(INITIALIZED);
@@ -848,22 +834,22 @@ jlong os::elapsed_frequency() {
}
-bool os::available_memory(size_t& value) {
+bool os::available_memory(physical_memory_size_type& value) {
return win32::available_memory(value);
}
-bool os::free_memory(size_t& value) {
+bool os::free_memory(physical_memory_size_type& value) {
return win32::available_memory(value);
}
-bool os::win32::available_memory(size_t& value) {
+bool os::win32::available_memory(physical_memory_size_type& value) {
// Use GlobalMemoryStatusEx() because GlobalMemoryStatus() may return incorrect
// value if total memory is larger than 4GB
MEMORYSTATUSEX ms;
ms.dwLength = sizeof(ms);
BOOL res = GlobalMemoryStatusEx(&ms);
if (res == TRUE) {
- value = static_cast(ms.ullAvailPhys);
+ value = static_cast(ms.ullAvailPhys);
return true;
} else {
assert(false, "GlobalMemoryStatusEx failed in os::win32::available_memory(): %lu", ::GetLastError());
@@ -871,12 +857,12 @@ bool os::win32::available_memory(size_t& value) {
}
}
-bool os::total_swap_space(size_t& value) {
+bool os::total_swap_space(physical_memory_size_type& value) {
MEMORYSTATUSEX ms;
ms.dwLength = sizeof(ms);
BOOL res = GlobalMemoryStatusEx(&ms);
if (res == TRUE) {
- value = static_cast(ms.ullTotalPageFile);
+ value = static_cast(ms.ullTotalPageFile);
return true;
} else {
assert(false, "GlobalMemoryStatusEx failed in os::total_swap_space(): %lu", ::GetLastError());
@@ -884,12 +870,12 @@ bool os::total_swap_space(size_t& value) {
}
}
-bool os::free_swap_space(size_t& value) {
+bool os::free_swap_space(physical_memory_size_type& value) {
MEMORYSTATUSEX ms;
ms.dwLength = sizeof(ms);
BOOL res = GlobalMemoryStatusEx(&ms);
if (res == TRUE) {
- value = static_cast(ms.ullAvailPageFile);
+ value = static_cast(ms.ullAvailPageFile);
return true;
} else {
assert(false, "GlobalMemoryStatusEx failed in os::free_swap_space(): %lu", ::GetLastError());
@@ -897,7 +883,7 @@ bool os::free_swap_space(size_t& value) {
}
}
-size_t os::physical_memory() {
+physical_memory_size_type os::physical_memory() {
return win32::physical_memory();
}
@@ -2462,7 +2448,7 @@ static void jdk_misc_signal_init() {
void os::signal_notify(int sig) {
if (sig_sem != nullptr) {
- Atomic::inc(&pending_signals[sig]);
+ AtomicAccess::inc(&pending_signals[sig]);
sig_sem->signal();
} else {
// Signal thread is not created with ReduceSignalUsage and jdk_misc_signal_init
@@ -2475,7 +2461,7 @@ static int check_pending_signals() {
while (true) {
for (int i = 0; i < NSIG + 1; i++) {
jint n = pending_signals[i];
- if (n > 0 && n == Atomic::cmpxchg(&pending_signals[i], n, n - 1)) {
+ if (n > 0 && n == AtomicAccess::cmpxchg(&pending_signals[i], n, n - 1)) {
return i;
}
}
@@ -2644,14 +2630,13 @@ LONG WINAPI topLevelExceptionFilter(struct _EXCEPTION_POINTERS* exceptionInfo) {
DWORD exception_code = exception_record->ExceptionCode;
#if defined(_M_ARM64)
address pc = (address) exceptionInfo->ContextRecord->Pc;
+
+ if (handle_safefetch(exception_code, pc, (void*)exceptionInfo->ContextRecord)) {
+ return EXCEPTION_CONTINUE_EXECUTION;
+ }
#elif defined(_M_AMD64)
address pc = (address) exceptionInfo->ContextRecord->Rip;
-#else
- #error unknown architecture
-#endif
- Thread* t = Thread::current_or_null_safe();
-#if defined(_M_AMD64)
if ((exception_code == EXCEPTION_ACCESS_VIOLATION) &&
VM_Version::is_cpuinfo_segv_addr(pc)) {
// Verify that OS save/restore AVX registers.
@@ -2664,6 +2649,8 @@ LONG WINAPI topLevelExceptionFilter(struct _EXCEPTION_POINTERS* exceptionInfo) {
VM_Version::clear_apx_test_state();
return Handle_Exception(exceptionInfo, VM_Version::cpuinfo_cont_addr_apx());
}
+#else
+ #error unknown architecture
#endif
#ifdef CAN_SHOW_REGISTERS_ON_ASSERT
@@ -2674,6 +2661,7 @@ LONG WINAPI topLevelExceptionFilter(struct _EXCEPTION_POINTERS* exceptionInfo) {
}
#endif
+ Thread* t = Thread::current_or_null_safe();
if (t != nullptr && t->is_Java_thread()) {
JavaThread* thread = JavaThread::cast(t);
bool in_java = thread->thread_state() == _thread_in_Java;
@@ -2704,10 +2692,8 @@ LONG WINAPI topLevelExceptionFilter(struct _EXCEPTION_POINTERS* exceptionInfo) {
// Fatal red zone violation.
overflow_state->disable_stack_red_zone();
tty->print_raw_cr("An unrecoverable stack overflow has occurred.");
-#if !defined(USE_VECTORED_EXCEPTION_HANDLING)
report_error(t, exception_code, pc, exception_record,
exceptionInfo->ContextRecord);
-#endif
return EXCEPTION_CONTINUE_SEARCH;
}
} else if (exception_code == EXCEPTION_ACCESS_VIOLATION) {
@@ -2759,10 +2745,8 @@ LONG WINAPI topLevelExceptionFilter(struct _EXCEPTION_POINTERS* exceptionInfo) {
}
// Stack overflow or null pointer exception in native code.
-#if !defined(USE_VECTORED_EXCEPTION_HANDLING)
report_error(t, exception_code, pc, exception_record,
exceptionInfo->ContextRecord);
-#endif
return EXCEPTION_CONTINUE_SEARCH;
} // /EXCEPTION_ACCESS_VIOLATION
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
@@ -2811,9 +2795,7 @@ LONG WINAPI topLevelExceptionFilter(struct _EXCEPTION_POINTERS* exceptionInfo) {
if (cb != nullptr && cb->is_nmethod()) {
nmethod* nm = cb->as_nmethod();
frame fr = os::fetch_frame_from_context((void*)exceptionInfo->ContextRecord);
- address deopt = nm->is_method_handle_return(pc) ?
- nm->deopt_mh_handler_begin() :
- nm->deopt_handler_begin();
+ address deopt = nm->deopt_handler_begin();
assert(nm->insts_contains_inclusive(pc), "");
nm->set_original_pc(&fr, pc);
// Set pc to handler
@@ -2824,41 +2806,21 @@ LONG WINAPI topLevelExceptionFilter(struct _EXCEPTION_POINTERS* exceptionInfo) {
}
}
-#if !defined(USE_VECTORED_EXCEPTION_HANDLING)
- if (exception_code != EXCEPTION_BREAKPOINT) {
+ bool should_report_error = (exception_code != EXCEPTION_BREAKPOINT);
+
+#if defined(_M_ARM64)
+ should_report_error = should_report_error &&
+ FAILED(exception_code) &&
+ (exception_code != EXCEPTION_UNCAUGHT_CXX_EXCEPTION);
+#endif
+
+ if (should_report_error) {
report_error(t, exception_code, pc, exception_record,
exceptionInfo->ContextRecord);
}
-#endif
- return EXCEPTION_CONTINUE_SEARCH;
-}
-
-#if defined(USE_VECTORED_EXCEPTION_HANDLING)
-LONG WINAPI topLevelVectoredExceptionFilter(struct _EXCEPTION_POINTERS* exceptionInfo) {
- PEXCEPTION_RECORD exceptionRecord = exceptionInfo->ExceptionRecord;
-#if defined(_M_ARM64)
- address pc = (address) exceptionInfo->ContextRecord->Pc;
-#elif defined(_M_AMD64)
- address pc = (address) exceptionInfo->ContextRecord->Rip;
-#else
- #error unknown architecture
-#endif
-
- // Fast path for code part of the code cache
- if (CodeCache::low_bound() <= pc && pc < CodeCache::high_bound()) {
- return topLevelExceptionFilter(exceptionInfo);
- }
-
- // If the exception occurred in the codeCache, pass control
- // to our normal exception handler.
- CodeBlob* cb = CodeCache::find_blob(pc);
- if (cb != nullptr) {
- return topLevelExceptionFilter(exceptionInfo);
- }
return EXCEPTION_CONTINUE_SEARCH;
}
-#endif
#if defined(USE_VECTORED_EXCEPTION_HANDLING)
LONG WINAPI topLevelUnhandledExceptionFilter(struct _EXCEPTION_POINTERS* exceptionInfo) {
@@ -3961,25 +3923,25 @@ int os::current_process_id() {
return (_initial_pid ? _initial_pid : _getpid());
}
-int os::win32::_processor_type = 0;
+int os::win32::_processor_type = 0;
// Processor level is not available on non-NT systems, use vm_version instead
-int os::win32::_processor_level = 0;
-size_t os::win32::_physical_memory = 0;
+int os::win32::_processor_level = 0;
+physical_memory_size_type os::win32::_physical_memory = 0;
-bool os::win32::_is_windows_server = false;
+bool os::win32::_is_windows_server = false;
// 6573254
// Currently, the bug is observed across all the supported Windows releases,
// including the latest one (as of this writing - Windows Server 2012 R2)
-bool os::win32::_has_exit_bug = true;
+bool os::win32::_has_exit_bug = true;
-int os::win32::_major_version = 0;
-int os::win32::_minor_version = 0;
-int os::win32::_build_number = 0;
-int os::win32::_build_minor = 0;
+int os::win32::_major_version = 0;
+int os::win32::_minor_version = 0;
+int os::win32::_build_number = 0;
+int os::win32::_build_minor = 0;
-bool os::win32::_processor_group_warning_displayed = false;
-bool os::win32::_job_object_processor_group_warning_displayed = false;
+bool os::win32::_processor_group_warning_displayed = false;
+bool os::win32::_job_object_processor_group_warning_displayed = false;
void getWindowsInstallationType(char* buffer, int bufferSize) {
HKEY hKey;
@@ -4198,7 +4160,7 @@ void os::win32::initialize_system_info() {
if (res != TRUE) {
assert(false, "GlobalMemoryStatusEx failed in os::win32::initialize_system_info(): %lu", ::GetLastError());
}
- _physical_memory = static_cast(ms.ullTotalPhys);
+ _physical_memory = static_cast(ms.ullTotalPhys);
if (FLAG_IS_DEFAULT(MaxRAM)) {
// Adjust MaxRAM according to the maximum virtual address space available.
@@ -4297,15 +4259,15 @@ static void exit_process_or_thread(Ept what, int exit_code) {
// The first thread that reached this point, initializes the critical section.
if (!InitOnceExecuteOnce(&init_once_crit_sect, init_crit_sect_call, &crit_sect, nullptr)) {
warning("crit_sect initialization failed in %s: %d\n", __FILE__, __LINE__);
- } else if (Atomic::load_acquire(&process_exiting) == 0) {
+ } else if (AtomicAccess::load_acquire(&process_exiting) == 0) {
if (what != EPT_THREAD) {
// Atomically set process_exiting before the critical section
// to increase the visibility between racing threads.
- Atomic::cmpxchg(&process_exiting, (DWORD)0, GetCurrentThreadId());
+ AtomicAccess::cmpxchg(&process_exiting, (DWORD)0, GetCurrentThreadId());
}
EnterCriticalSection(&crit_sect);
- if (what == EPT_THREAD && Atomic::load_acquire(&process_exiting) == 0) {
+ if (what == EPT_THREAD && AtomicAccess::load_acquire(&process_exiting) == 0) {
// Remove from the array those handles of the threads that have completed exiting.
for (i = 0, j = 0; i < handle_count; ++i) {
res = WaitForSingleObject(handles[i], 0 /* don't wait */);
@@ -4418,7 +4380,7 @@ static void exit_process_or_thread(Ept what, int exit_code) {
}
if (!registered &&
- Atomic::load_acquire(&process_exiting) != 0 &&
+ AtomicAccess::load_acquire(&process_exiting) != 0 &&
process_exiting != GetCurrentThreadId()) {
// Some other thread is about to call exit(), so we don't let
// the current unregistered thread proceed to exit() or _endthreadex()
@@ -4535,7 +4497,7 @@ jint os::init_2(void) {
// Setup Windows Exceptions
#if defined(USE_VECTORED_EXCEPTION_HANDLING)
- topLevelVectoredExceptionHandler = AddVectoredExceptionHandler(1, topLevelVectoredExceptionFilter);
+ topLevelVectoredExceptionHandler = AddVectoredExceptionHandler(1, topLevelExceptionFilter);
previousUnhandledExceptionFilter = SetUnhandledExceptionFilter(topLevelUnhandledExceptionFilter);
#endif
@@ -5584,7 +5546,7 @@ int PlatformEvent::park(jlong Millis) {
int v;
for (;;) {
v = _Event;
- if (Atomic::cmpxchg(&_Event, v, v-1) == v) break;
+ if (AtomicAccess::cmpxchg(&_Event, v, v-1) == v) break;
}
guarantee((v == 0) || (v == 1), "invariant");
if (v != 0) return OS_OK;
@@ -5647,7 +5609,7 @@ void PlatformEvent::park() {
int v;
for (;;) {
v = _Event;
- if (Atomic::cmpxchg(&_Event, v, v-1) == v) break;
+ if (AtomicAccess::cmpxchg(&_Event, v, v-1) == v) break;
}
guarantee((v == 0) || (v == 1), "invariant");
if (v != 0) return;
@@ -5694,7 +5656,7 @@ void PlatformEvent::unpark() {
// from the first park() call after an unpark() call which will help
// shake out uses of park() and unpark() without condition variables.
- if (Atomic::xchg(&_Event, 1) >= 0) return;
+ if (AtomicAccess::xchg(&_Event, 1) >= 0) return;
::SetEvent(_ParkHandle);
}
diff --git a/src/hotspot/os/windows/os_windows.hpp b/src/hotspot/os/windows/os_windows.hpp
index 1426dc8be93..f1153bbbfd3 100644
--- a/src/hotspot/os/windows/os_windows.hpp
+++ b/src/hotspot/os/windows/os_windows.hpp
@@ -38,18 +38,18 @@ class os::win32 {
friend class os;
protected:
- static int _processor_type;
- static int _processor_level;
- static size_t _physical_memory;
- static bool _is_windows_server;
- static bool _has_exit_bug;
- static bool _processor_group_warning_displayed;
- static bool _job_object_processor_group_warning_displayed;
+ static int _processor_type;
+ static int _processor_level;
+ static physical_memory_size_type _physical_memory;
+ static bool _is_windows_server;
+ static bool _has_exit_bug;
+ static bool _processor_group_warning_displayed;
+ static bool _job_object_processor_group_warning_displayed;
- static int _major_version;
- static int _minor_version;
- static int _build_number;
- static int _build_minor;
+ static int _major_version;
+ static int _minor_version;
+ static int _build_number;
+ static int _build_minor;
static void print_windows_version(outputStream* st);
static void print_uptime_info(outputStream* st);
@@ -102,9 +102,9 @@ class os::win32 {
static int processor_level() {
return _processor_level;
}
- static bool available_memory(size_t& value);
- static bool free_memory(size_t& value);
- static size_t physical_memory() { return _physical_memory; }
+ static bool available_memory(physical_memory_size_type& value);
+ static bool free_memory(physical_memory_size_type& value);
+ static physical_memory_size_type physical_memory() { return _physical_memory; }
// load dll from Windows system directory or Windows directory
static HINSTANCE load_Windows_dll(const char* name, char *ebuf, int ebuflen);
@@ -150,6 +150,8 @@ public:
// signal support
static void* install_signal_handler(int sig, signal_handler_t handler);
static void* user_handler();
+
+ static void context_set_pc(CONTEXT* uc, address pc);
};
#endif // OS_WINDOWS_OS_WINDOWS_HPP
diff --git a/src/hotspot/os/windows/os_windows.inline.hpp b/src/hotspot/os/windows/os_windows.inline.hpp
index 41471224b6c..9eebbee2d8b 100644
--- a/src/hotspot/os/windows/os_windows.inline.hpp
+++ b/src/hotspot/os/windows/os_windows.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -62,8 +62,6 @@ inline void os::map_stack_shadow_pages(address sp) {
state->set_shadow_zone_growth_watermark(original_sp);
}
-inline bool os::numa_has_group_homing() { return false; }
-
// Platform Mutex/Monitor implementation
inline void PlatformMutex::lock() {
diff --git a/src/hotspot/os/windows/safefetch_static_windows.cpp b/src/hotspot/os/windows/safefetch_static_windows.cpp
new file mode 100644
index 00000000000..3ea8b96b32d
--- /dev/null
+++ b/src/hotspot/os/windows/safefetch_static_windows.cpp
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2022 SAP SE. All rights reserved.
+ * Copyright (c) 2022, 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+
+#include "os_windows.hpp"
+#include "runtime/os.hpp"
+#include "runtime/safefetch.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+#ifdef SAFEFETCH_METHOD_STATIC_ASSEMBLY
+
+// SafeFetch handling, static assembly style:
+//
+// SafeFetch32 and SafeFetchN are implemented via static assembly
+// and live in os_cpu/xx_xx/safefetch_xx_xx.S
+
+extern "C" char _SafeFetch32_continuation[];
+extern "C" char _SafeFetch32_fault[];
+
+#ifdef _LP64
+extern "C" char _SafeFetchN_continuation[];
+extern "C" char _SafeFetchN_fault[];
+#endif // _LP64
+
+bool handle_safefetch(int exception_code, address pc, void* context) {
+ CONTEXT* ctx = (CONTEXT*)context;
+ if (exception_code == EXCEPTION_ACCESS_VIOLATION && ctx != nullptr) {
+ if (pc == (address)_SafeFetch32_fault) {
+ os::win32::context_set_pc(ctx, (address)_SafeFetch32_continuation);
+ return true;
+ }
+#ifdef _LP64
+ if (pc == (address)_SafeFetchN_fault) {
+ os::win32::context_set_pc(ctx, (address)_SafeFetchN_continuation);
+ return true;
+ }
+#endif
+ }
+ return false;
+}
+
+#endif // SAFEFETCH_METHOD_STATIC_ASSEMBLY
diff --git a/src/hotspot/cpu/aarch64/bytecodes_aarch64.cpp b/src/hotspot/os_cpu/aix_ppc/atomicAccess_aix_ppc.hpp
similarity index 80%
rename from src/hotspot/cpu/aarch64/bytecodes_aarch64.cpp
rename to src/hotspot/os_cpu/aix_ppc/atomicAccess_aix_ppc.hpp
index 119ad8baec3..9a314667fcb 100644
--- a/src/hotspot/cpu/aarch64/bytecodes_aarch64.cpp
+++ b/src/hotspot/os_cpu/aix_ppc/atomicAccess_aix_ppc.hpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2014, Red Hat Inc. All rights reserved.
+ * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -23,6 +23,5 @@
*
*/
-#include "interpreter/bytecodes.hpp"
-
-
+// Including inline assembler functions that are shared between multiple PPC64 platforms.
+#include "atomicAccess_ppc.hpp"
diff --git a/src/hotspot/os_cpu/aix_ppc/atomic_aix_ppc.hpp b/src/hotspot/os_cpu/aix_ppc/atomic_aix_ppc.hpp
deleted file mode 100644
index 722dffc150d..00000000000
--- a/src/hotspot/os_cpu/aix_ppc/atomic_aix_ppc.hpp
+++ /dev/null
@@ -1,417 +0,0 @@
-/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2019 SAP SE. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef OS_CPU_AIX_PPC_ATOMIC_AIX_PPC_HPP
-#define OS_CPU_AIX_PPC_ATOMIC_AIX_PPC_HPP
-
-#ifndef PPC64
-#error "Atomic currently only implemented for PPC64"
-#endif
-
-#include "orderAccess_aix_ppc.hpp"
-#include "utilities/debug.hpp"
-
-// Implementation of class atomic
-
-//
-// machine barrier instructions:
-//
-// - sync two-way memory barrier, aka fence
-// - lwsync orders Store|Store,
-// Load|Store,
-// Load|Load,
-// but not Store|Load
-// - eieio orders memory accesses for device memory (only)
-// - isync invalidates speculatively executed instructions
-// From the POWER ISA 2.06 documentation:
-// "[...] an isync instruction prevents the execution of
-// instructions following the isync until instructions
-// preceding the isync have completed, [...]"
-// From IBM's AIX assembler reference:
-// "The isync [...] instructions causes the processor to
-// refetch any instructions that might have been fetched
-// prior to the isync instruction. The instruction isync
-// causes the processor to wait for all previous instructions
-// to complete. Then any instructions already fetched are
-// discarded and instruction processing continues in the
-// environment established by the previous instructions."
-//
-// semantic barrier instructions:
-// (as defined in orderAccess.hpp)
-//
-// - release orders Store|Store, (maps to lwsync)
-// Load|Store
-// - acquire orders Load|Store, (maps to lwsync)
-// Load|Load
-// - fence orders Store|Store, (maps to sync)
-// Load|Store,
-// Load|Load,
-// Store|Load
-//
-
-inline void pre_membar(atomic_memory_order order) {
- switch (order) {
- case memory_order_relaxed:
- case memory_order_acquire: break;
- case memory_order_release:
- case memory_order_acq_rel: __asm__ __volatile__ ("lwsync" : : : "memory"); break;
- default /*conservative*/ : __asm__ __volatile__ ("sync" : : : "memory"); break;
- }
-}
-
-inline void post_membar(atomic_memory_order order) {
- switch (order) {
- case memory_order_relaxed:
- case memory_order_release: break;
- case memory_order_acquire:
- case memory_order_acq_rel: __asm__ __volatile__ ("isync" : : : "memory"); break;
- default /*conservative*/ : __asm__ __volatile__ ("sync" : : : "memory"); break;
- }
-}
-
-
-template
-struct Atomic::PlatformAdd {
- template
- D add_then_fetch(D volatile* dest, I add_value, atomic_memory_order order) const;
-
- template
- D fetch_then_add(D volatile* dest, I add_value, atomic_memory_order order) const {
- return add_then_fetch(dest, add_value, order) - add_value;
- }
-};
-
-template<>
-template
-inline D Atomic::PlatformAdd<4>::add_then_fetch(D volatile* dest, I add_value,
- atomic_memory_order order) const {
- STATIC_ASSERT(4 == sizeof(I));
- STATIC_ASSERT(4 == sizeof(D));
-
- D result;
-
- pre_membar(order);
-
- __asm__ __volatile__ (
- "1: lwarx %0, 0, %2 \n"
- " add %0, %0, %1 \n"
- " stwcx. %0, 0, %2 \n"
- " bne- 1b \n"
- : /*%0*/"=&r" (result)
- : /*%1*/"r" (add_value), /*%2*/"r" (dest)
- : "cc", "memory" );
-
- post_membar(order);
-
- return result;
-}
-
-
-template<>
-template
-inline D Atomic::PlatformAdd<8>::add_then_fetch(D volatile* dest, I add_value,
- atomic_memory_order order) const {
- STATIC_ASSERT(8 == sizeof(I));
- STATIC_ASSERT(8 == sizeof(D));
-
- D result;
-
- pre_membar(order);
-
- __asm__ __volatile__ (
- "1: ldarx %0, 0, %2 \n"
- " add %0, %0, %1 \n"
- " stdcx. %0, 0, %2 \n"
- " bne- 1b \n"
- : /*%0*/"=&r" (result)
- : /*%1*/"r" (add_value), /*%2*/"r" (dest)
- : "cc", "memory" );
-
- post_membar(order);
-
- return result;
-}
-
-template<>
-template
-inline T Atomic::PlatformXchg<4>::operator()(T volatile* dest,
- T exchange_value,
- atomic_memory_order order) const {
- // Note that xchg doesn't necessarily do an acquire
- // (see synchronizer.cpp).
-
- T old_value;
- const uint64_t zero = 0;
-
- pre_membar(order);
-
- __asm__ __volatile__ (
- /* atomic loop */
- "1: \n"
- " lwarx %[old_value], %[dest], %[zero] \n"
- " stwcx. %[exchange_value], %[dest], %[zero] \n"
- " bne- 1b \n"
- /* exit */
- "2: \n"
- /* out */
- : [old_value] "=&r" (old_value),
- "=m" (*dest)
- /* in */
- : [dest] "b" (dest),
- [zero] "r" (zero),
- [exchange_value] "r" (exchange_value),
- "m" (*dest)
- /* clobber */
- : "cc",
- "memory"
- );
-
- post_membar(order);
-
- return old_value;
-}
-
-template<>
-template
-inline T Atomic::PlatformXchg<8>::operator()(T volatile* dest,
- T exchange_value,
- atomic_memory_order order) const {
- STATIC_ASSERT(8 == sizeof(T));
- // Note that xchg doesn't necessarily do an acquire
- // (see synchronizer.cpp).
-
- T old_value;
- const uint64_t zero = 0;
-
- pre_membar(order);
-
- __asm__ __volatile__ (
- /* atomic loop */
- "1: \n"
- " ldarx %[old_value], %[dest], %[zero] \n"
- " stdcx. %[exchange_value], %[dest], %[zero] \n"
- " bne- 1b \n"
- /* exit */
- "2: \n"
- /* out */
- : [old_value] "=&r" (old_value),
- "=m" (*dest)
- /* in */
- : [dest] "b" (dest),
- [zero] "r" (zero),
- [exchange_value] "r" (exchange_value),
- "m" (*dest)
- /* clobber */
- : "cc",
- "memory"
- );
-
- post_membar(order);
-
- return old_value;
-}
-
-template<>
-template
-inline T Atomic::PlatformCmpxchg<1>::operator()(T volatile* dest,
- T compare_value,
- T exchange_value,
- atomic_memory_order order) const {
- STATIC_ASSERT(1 == sizeof(T));
-
- // Note that cmpxchg guarantees a two-way memory barrier across
- // the cmpxchg, so it's really a 'fence_cmpxchg_fence' if not
- // specified otherwise (see atomic.hpp).
-
- // Using 32 bit internally.
- volatile int *dest_base = (volatile int*)((uintptr_t)dest & ~3);
-
-#ifdef VM_LITTLE_ENDIAN
- const unsigned int shift_amount = ((uintptr_t)dest & 3) * 8;
-#else
- const unsigned int shift_amount = ((~(uintptr_t)dest) & 3) * 8;
-#endif
- const unsigned int masked_compare_val = ((unsigned int)(unsigned char)compare_value),
- masked_exchange_val = ((unsigned int)(unsigned char)exchange_value),
- xor_value = (masked_compare_val ^ masked_exchange_val) << shift_amount;
-
- unsigned int old_value, value32;
-
- pre_membar(order);
-
- __asm__ __volatile__ (
- /* simple guard */
- " lbz %[old_value], 0(%[dest]) \n"
- " cmpw %[masked_compare_val], %[old_value] \n"
- " bne- 2f \n"
- /* atomic loop */
- "1: \n"
- " lwarx %[value32], 0, %[dest_base] \n"
- /* extract byte and compare */
- " srd %[old_value], %[value32], %[shift_amount] \n"
- " clrldi %[old_value], %[old_value], 56 \n"
- " cmpw %[masked_compare_val], %[old_value] \n"
- " bne- 2f \n"
- /* replace byte and try to store */
- " xor %[value32], %[xor_value], %[value32] \n"
- " stwcx. %[value32], 0, %[dest_base] \n"
- " bne- 1b \n"
- /* exit */
- "2: \n"
- /* out */
- : [old_value] "=&r" (old_value),
- [value32] "=&r" (value32),
- "=m" (*dest),
- "=m" (*dest_base)
- /* in */
- : [dest] "b" (dest),
- [dest_base] "b" (dest_base),
- [shift_amount] "r" (shift_amount),
- [masked_compare_val] "r" (masked_compare_val),
- [xor_value] "r" (xor_value),
- "m" (*dest),
- "m" (*dest_base)
- /* clobber */
- : "cc",
- "memory"
- );
-
- post_membar(order);
-
- return PrimitiveConversions::cast((unsigned char)old_value);
-}
-
-template<>
-template
-inline T Atomic::PlatformCmpxchg<4>::operator()(T volatile* dest,
- T compare_value,
- T exchange_value,
- atomic_memory_order order) const {
- STATIC_ASSERT(4 == sizeof(T));
-
- // Note that cmpxchg guarantees a two-way memory barrier across
- // the cmpxchg, so it's really a 'fence_cmpxchg_fence' if not
- // specified otherwise (see atomic.hpp).
-
- T old_value;
- const uint64_t zero = 0;
-
- pre_membar(order);
-
- __asm__ __volatile__ (
- /* simple guard */
- " lwz %[old_value], 0(%[dest]) \n"
- " cmpw %[compare_value], %[old_value] \n"
- " bne- 2f \n"
- /* atomic loop */
- "1: \n"
- " lwarx %[old_value], %[dest], %[zero] \n"
- " cmpw %[compare_value], %[old_value] \n"
- " bne- 2f \n"
- " stwcx. %[exchange_value], %[dest], %[zero] \n"
- " bne- 1b \n"
- /* exit */
- "2: \n"
- /* out */
- : [old_value] "=&r" (old_value),
- "=m" (*dest)
- /* in */
- : [dest] "b" (dest),
- [zero] "r" (zero),
- [compare_value] "r" (compare_value),
- [exchange_value] "r" (exchange_value),
- "m" (*dest)
- /* clobber */
- : "cc",
- "memory"
- );
-
- post_membar(order);
-
- return old_value;
-}
-
-template<>
-template
-inline T Atomic::PlatformCmpxchg<8>::operator()(T volatile* dest,
- T compare_value,
- T exchange_value,
- atomic_memory_order order) const {
- STATIC_ASSERT(8 == sizeof(T));
-
- // Note that cmpxchg guarantees a two-way memory barrier across
- // the cmpxchg, so it's really a 'fence_cmpxchg_fence' if not
- // specified otherwise (see atomic.hpp).
-
- T old_value;
- const uint64_t zero = 0;
-
- pre_membar(order);
-
- __asm__ __volatile__ (
- /* simple guard */
- " ld %[old_value], 0(%[dest]) \n"
- " cmpd %[compare_value], %[old_value] \n"
- " bne- 2f \n"
- /* atomic loop */
- "1: \n"
- " ldarx %[old_value], %[dest], %[zero] \n"
- " cmpd %[compare_value], %[old_value] \n"
- " bne- 2f \n"
- " stdcx. %[exchange_value], %[dest], %[zero] \n"
- " bne- 1b \n"
- /* exit */
- "2: \n"
- /* out */
- : [old_value] "=&r" (old_value),
- "=m" (*dest)
- /* in */
- : [dest] "b" (dest),
- [zero] "r" (zero),
- [compare_value] "r" (compare_value),
- [exchange_value] "r" (exchange_value),
- "m" (*dest)
- /* clobber */
- : "cc",
- "memory"
- );
-
- post_membar(order);
-
- return old_value;
-}
-
-template
-struct Atomic::PlatformOrderedLoad {
- template
- T operator()(const volatile T* p) const {
- T t = Atomic::load(p);
- // Use twi-isync for load_acquire (faster than lwsync).
- __asm__ __volatile__ ("twi 0,%0,0\n isync\n" : : "r" (t) : "memory");
- return t;
- }
-};
-
-#endif // OS_CPU_AIX_PPC_ATOMIC_AIX_PPC_HPP
diff --git a/src/hotspot/os_cpu/aix_ppc/orderAccess_aix_ppc.hpp b/src/hotspot/os_cpu/aix_ppc/orderAccess_aix_ppc.hpp
index 9ca6c18d5bb..d395add6d69 100644
--- a/src/hotspot/os_cpu/aix_ppc/orderAccess_aix_ppc.hpp
+++ b/src/hotspot/os_cpu/aix_ppc/orderAccess_aix_ppc.hpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2019 SAP SE. All rights reserved.
+ * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -23,62 +23,5 @@
*
*/
-#ifndef OS_CPU_AIX_PPC_ORDERACCESS_AIX_PPC_HPP
-#define OS_CPU_AIX_PPC_ORDERACCESS_AIX_PPC_HPP
-
-// Included in orderAccess.hpp header file.
-
-// Compiler version last used for testing: xlc 12
-// Please update this information when this file changes
-
-// Implementation of class OrderAccess.
-
-//
-// Machine barrier instructions:
-//
-// - sync Two-way memory barrier, aka fence.
-// - lwsync orders Store|Store,
-// Load|Store,
-// Load|Load,
-// but not Store|Load
-// - eieio orders Store|Store
-// - isync Invalidates speculatively executed instructions,
-// but isync may complete before storage accesses
-// associated with instructions preceding isync have
-// been performed.
-//
-// Semantic barrier instructions:
-// (as defined in orderAccess.hpp)
-//
-// - release orders Store|Store, (maps to lwsync)
-// Load|Store
-// - acquire orders Load|Store, (maps to lwsync)
-// Load|Load
-// - fence orders Store|Store, (maps to sync)
-// Load|Store,
-// Load|Load,
-// Store|Load
-//
-
-#define inlasm_sync() __asm__ __volatile__ ("sync" : : : "memory");
-#define inlasm_lwsync() __asm__ __volatile__ ("lwsync" : : : "memory");
-#define inlasm_eieio() __asm__ __volatile__ ("eieio" : : : "memory");
-#define inlasm_isync() __asm__ __volatile__ ("isync" : : : "memory");
-
-inline void OrderAccess::loadload() { inlasm_lwsync(); }
-inline void OrderAccess::storestore() { inlasm_lwsync(); }
-inline void OrderAccess::loadstore() { inlasm_lwsync(); }
-inline void OrderAccess::storeload() { inlasm_sync(); }
-
-inline void OrderAccess::acquire() { inlasm_lwsync(); }
-inline void OrderAccess::release() { inlasm_lwsync(); }
-inline void OrderAccess::fence() { inlasm_sync(); }
-inline void OrderAccess::cross_modify_fence_impl()
- { inlasm_isync(); }
-
-#undef inlasm_sync
-#undef inlasm_lwsync
-#undef inlasm_eieio
-#undef inlasm_isync
-
-#endif // OS_CPU_AIX_PPC_ORDERACCESS_AIX_PPC_HPP
+// Including inline assembler functions that are shared between multiple PPC64 platforms.
+#include "orderAccess_ppc.hpp"
diff --git a/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp b/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp
index 98b17aaacfc..afef21b091a 100644
--- a/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp
+++ b/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp
@@ -1,6 +1,6 @@
/*
* Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2024 SAP SE. All rights reserved.
+ * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -254,6 +254,18 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info,
goto run_stub;
}
+ // SIGTRAP-based nmethod entry barriers.
+ else if (sig == SIGTRAP && TrapBasedNMethodEntryBarriers &&
+ nativeInstruction_at(pc)->is_sigtrap_nmethod_entry_barrier() &&
+ CodeCache::contains((void*) pc)) {
+ if (TraceTraps) {
+ tty->print_cr("trap: nmethod entry barrier at " INTPTR_FORMAT " (SIGTRAP)", p2i(pc));
+ }
+ stub = StubRoutines::method_entry_barrier();
+ uc->uc_mcontext.jmp_context.lr = (uintptr_t)(pc + BytesPerInstWord); // emulate call by setting LR
+ goto run_stub;
+ }
+
// SIGTRAP-based ic miss check in compiled code.
else if (sig == SIGTRAP && TrapBasedICMissChecks &&
nativeInstruction_at(pc)->is_sigtrap_ic_miss_check()) {
@@ -282,6 +294,7 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info,
tty->print_cr("trap: null_check at " INTPTR_FORMAT " (SIGSEGV)", p2i(pc));
}
stub = SharedRuntime::continuation_for_implicit_exception(thread, pc, SharedRuntime::IMPLICIT_NULL);
+ goto run_stub;
}
#ifdef COMPILER2
diff --git a/src/hotspot/os_cpu/bsd_aarch64/atomic_bsd_aarch64.hpp b/src/hotspot/os_cpu/bsd_aarch64/atomicAccess_bsd_aarch64.hpp
similarity index 78%
rename from src/hotspot/os_cpu/bsd_aarch64/atomic_bsd_aarch64.hpp
rename to src/hotspot/os_cpu/bsd_aarch64/atomicAccess_bsd_aarch64.hpp
index 14c093de8dd..3d2c632ace8 100644
--- a/src/hotspot/os_cpu/bsd_aarch64/atomic_bsd_aarch64.hpp
+++ b/src/hotspot/os_cpu/bsd_aarch64/atomicAccess_bsd_aarch64.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2019, Red Hat Inc. All rights reserved.
* Copyright (c) 2021, Azul Systems, Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -24,17 +24,17 @@
*
*/
-#ifndef OS_CPU_BSD_AARCH64_ATOMIC_BSD_AARCH64_HPP
-#define OS_CPU_BSD_AARCH64_ATOMIC_BSD_AARCH64_HPP
+#ifndef OS_CPU_BSD_AARCH64_ATOMICACCESS_BSD_AARCH64_HPP
+#define OS_CPU_BSD_AARCH64_ATOMICACCESS_BSD_AARCH64_HPP
#include "utilities/debug.hpp"
-// Implementation of class atomic
+// Implementation of class AtomicAccess
// Note that memory_order_conservative requires a full barrier after atomic stores.
// See https://patchwork.kernel.org/patch/3575821/
template
-struct Atomic::PlatformAdd {
+struct AtomicAccess::PlatformAdd {
template
D add_then_fetch(D volatile* dest, I add_value, atomic_memory_order order) const {
if (order == memory_order_relaxed) {
@@ -54,9 +54,9 @@ struct Atomic::PlatformAdd {
template
template
-inline T Atomic::PlatformXchg::operator()(T volatile* dest,
- T exchange_value,
- atomic_memory_order order) const {
+inline T AtomicAccess::PlatformXchg::operator()(T volatile* dest,
+ T exchange_value,
+ atomic_memory_order order) const {
STATIC_ASSERT(byte_size == sizeof(T));
T res = __atomic_exchange_n(dest, exchange_value, __ATOMIC_RELEASE);
FULL_MEM_BARRIER;
@@ -65,10 +65,10 @@ inline T Atomic::PlatformXchg::operator()(T volatile* dest,
template
template
-inline T Atomic::PlatformCmpxchg::operator()(T volatile* dest,
- T compare_value,
- T exchange_value,
- atomic_memory_order order) const {
+inline T AtomicAccess::PlatformCmpxchg::operator()(T volatile* dest,
+ T compare_value,
+ T exchange_value,
+ atomic_memory_order order) const {
STATIC_ASSERT(byte_size == sizeof(T));
if (order == memory_order_conservative) {
T value = compare_value;
@@ -109,25 +109,24 @@ inline T Atomic::PlatformCmpxchg::operator()(T volatile* dest,
}
template
-struct Atomic::PlatformOrderedLoad
+struct AtomicAccess::PlatformOrderedLoad
{
template
T operator()(const volatile T* p) const { T data; __atomic_load(const_cast(p), &data, __ATOMIC_ACQUIRE); return data; }
};
template
-struct Atomic::PlatformOrderedStore
+struct AtomicAccess::PlatformOrderedStore
{
template
void operator()(volatile T* p, T v) const { __atomic_store(const_cast(p), &v, __ATOMIC_RELEASE); }
};
template
-struct Atomic::PlatformOrderedStore
+struct AtomicAccess::PlatformOrderedStore
{
template
void operator()(volatile T* p, T v) const { release_store(p, v); OrderAccess::fence(); }
};
-
-#endif // OS_CPU_BSD_AARCH64_ATOMIC_BSD_AARCH64_HPP
+#endif // OS_CPU_BSD_AARCH64_ATOMICACCESS_BSD_AARCH64_HPP
diff --git a/src/hotspot/os_cpu/bsd_x86/atomic_bsd_x86.hpp b/src/hotspot/os_cpu/bsd_x86/atomicAccess_bsd_x86.hpp
similarity index 64%
rename from src/hotspot/os_cpu/bsd_x86/atomic_bsd_x86.hpp
rename to src/hotspot/os_cpu/bsd_x86/atomicAccess_bsd_x86.hpp
index 9ba246f553d..975580fbd71 100644
--- a/src/hotspot/os_cpu/bsd_x86/atomic_bsd_x86.hpp
+++ b/src/hotspot/os_cpu/bsd_x86/atomicAccess_bsd_x86.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -22,13 +22,13 @@
*
*/
-#ifndef OS_CPU_BSD_X86_ATOMIC_BSD_X86_HPP
-#define OS_CPU_BSD_X86_ATOMIC_BSD_X86_HPP
+#ifndef OS_CPU_BSD_X86_ATOMICACCESS_BSD_X86_HPP
+#define OS_CPU_BSD_X86_ATOMICACCESS_BSD_X86_HPP
-// Implementation of class atomic
+// Implementation of class AtomicAccess
template
-struct Atomic::PlatformAdd {
+struct AtomicAccess::PlatformAdd {
template
D fetch_then_add(D volatile* dest, I add_value, atomic_memory_order /* order */) const;
@@ -40,8 +40,8 @@ struct Atomic::PlatformAdd {
template<>
template
-inline D Atomic::PlatformAdd<4>::fetch_then_add(D volatile* dest, I add_value,
- atomic_memory_order /* order */) const {
+inline D AtomicAccess::PlatformAdd<4>::fetch_then_add(D volatile* dest, I add_value,
+ atomic_memory_order /* order */) const {
STATIC_ASSERT(4 == sizeof(I));
STATIC_ASSERT(4 == sizeof(D));
D old_value;
@@ -54,9 +54,9 @@ inline D Atomic::PlatformAdd<4>::fetch_then_add(D volatile* dest, I add_value,
template<>
template
-inline T Atomic::PlatformXchg<4>::operator()(T volatile* dest,
- T exchange_value,
- atomic_memory_order /* order */) const {
+inline T AtomicAccess::PlatformXchg<4>::operator()(T volatile* dest,
+ T exchange_value,
+ atomic_memory_order /* order */) const {
STATIC_ASSERT(4 == sizeof(T));
__asm__ volatile ( "xchgl (%2),%0"
: "=r" (exchange_value)
@@ -67,10 +67,10 @@ inline T Atomic::PlatformXchg<4>::operator()(T volatile* dest,
template<>
template
-inline T Atomic::PlatformCmpxchg<1>::operator()(T volatile* dest,
- T compare_value,
- T exchange_value,
- atomic_memory_order /* order */) const {
+inline T AtomicAccess::PlatformCmpxchg<1>::operator()(T volatile* dest,
+ T compare_value,
+ T exchange_value,
+ atomic_memory_order /* order */) const {
STATIC_ASSERT(1 == sizeof(T));
__asm__ volatile ( "lock cmpxchgb %1,(%3)"
: "=a" (exchange_value)
@@ -81,10 +81,10 @@ inline T Atomic::PlatformCmpxchg<1>::operator()(T volatile* dest,
template<>
template
-inline T Atomic::PlatformCmpxchg<4>::operator()(T volatile* dest,
- T compare_value,
- T exchange_value,
- atomic_memory_order /* order */) const {
+inline T AtomicAccess::PlatformCmpxchg<4>::operator()(T volatile* dest,
+ T compare_value,
+ T exchange_value,
+ atomic_memory_order /* order */) const {
STATIC_ASSERT(4 == sizeof(T));
__asm__ volatile ( "lock cmpxchgl %1,(%3)"
: "=a" (exchange_value)
@@ -96,8 +96,8 @@ inline T Atomic::PlatformCmpxchg<4>::operator()(T volatile* dest,
#ifdef AMD64
template<>
template
-inline D Atomic::PlatformAdd<8>::fetch_then_add(D volatile* dest, I add_value,
- atomic_memory_order /* order */) const {
+inline D AtomicAccess::PlatformAdd<8>::fetch_then_add(D volatile* dest, I add_value,
+ atomic_memory_order /* order */) const {
STATIC_ASSERT(8 == sizeof(I));
STATIC_ASSERT(8 == sizeof(D));
D old_value;
@@ -110,9 +110,9 @@ inline D Atomic::PlatformAdd<8>::fetch_then_add(D volatile* dest, I add_value,
template<>
template
-inline T Atomic::PlatformXchg<8>::operator()(T volatile* dest,
- T exchange_value,
- atomic_memory_order /* order */) const {
+inline T AtomicAccess::PlatformXchg<8>::operator()(T volatile* dest,
+ T exchange_value,
+ atomic_memory_order /* order */) const {
STATIC_ASSERT(8 == sizeof(T));
__asm__ __volatile__ ("xchgq (%2),%0"
: "=r" (exchange_value)
@@ -123,10 +123,10 @@ inline T Atomic::PlatformXchg<8>::operator()(T volatile* dest,
template<>
template
-inline T Atomic::PlatformCmpxchg<8>::operator()(T volatile* dest,
- T compare_value,
- T exchange_value,
- atomic_memory_order /* order */) const {
+inline T AtomicAccess::PlatformCmpxchg<8>::operator()(T volatile* dest,
+ T compare_value,
+ T exchange_value,
+ atomic_memory_order /* order */) const {
STATIC_ASSERT(8 == sizeof(T));
__asm__ __volatile__ ( "lock cmpxchgq %1,(%3)"
: "=a" (exchange_value)
@@ -145,25 +145,25 @@ extern "C" {
template<>
template
-inline T Atomic::PlatformCmpxchg<8>::operator()(T volatile* dest,
- T compare_value,
- T exchange_value,
- atomic_memory_order /* order */) const {
+inline T AtomicAccess::PlatformCmpxchg<8>::operator()(T volatile* dest,
+ T compare_value,
+ T exchange_value,
+ atomic_memory_order /* order */) const {
STATIC_ASSERT(8 == sizeof(T));
return cmpxchg_using_helper(_Atomic_cmpxchg_long, dest, compare_value, exchange_value);
}
// No direct support for 8-byte xchg; emulate using cmpxchg.
template<>
-struct Atomic::PlatformXchg<8> : Atomic::XchgUsingCmpxchg<8> {};
+struct AtomicAccess::PlatformXchg<8> : AtomicAccess::XchgUsingCmpxchg<8> {};
// No direct support for 8-byte add; emulate using cmpxchg.
template<>
-struct Atomic::PlatformAdd<8> : Atomic::AddUsingCmpxchg<8> {};
+struct AtomicAccess::PlatformAdd<8> : AtomicAccess::AddUsingCmpxchg<8> {};
template<>
template
-inline T Atomic::PlatformLoad<8>::operator()(T const volatile* src) const {
+inline T AtomicAccess::PlatformLoad<8>::operator()(T const volatile* src) const {
STATIC_ASSERT(8 == sizeof(T));
volatile int64_t dest;
_Atomic_move_long(reinterpret_cast(src), reinterpret_cast(&dest));
@@ -172,8 +172,8 @@ inline T Atomic::PlatformLoad<8>::operator()(T const volatile* src) const {
template<>
template
-inline void Atomic::PlatformStore<8>::operator()(T volatile* dest,
- T store_value) const {
+inline void AtomicAccess::PlatformStore<8>::operator()(T volatile* dest,
+ T store_value) const {
STATIC_ASSERT(8 == sizeof(T));
_Atomic_move_long(reinterpret_cast(&store_value), reinterpret_cast(dest));
}
@@ -181,7 +181,7 @@ inline void Atomic::PlatformStore<8>::operator()(T volatile* dest,
#endif // AMD64
template<>
-struct Atomic::PlatformOrderedStore<1, RELEASE_X_FENCE>
+struct AtomicAccess::PlatformOrderedStore<1, RELEASE_X_FENCE>
{
template
void operator()(volatile T* p, T v) const {
@@ -193,7 +193,7 @@ struct Atomic::PlatformOrderedStore<1, RELEASE_X_FENCE>
};
template<>
-struct Atomic::PlatformOrderedStore<2, RELEASE_X_FENCE>
+struct AtomicAccess::PlatformOrderedStore<2, RELEASE_X_FENCE>
{
template
void operator()(volatile T* p, T v) const {
@@ -205,7 +205,7 @@ struct Atomic::PlatformOrderedStore<2, RELEASE_X_FENCE>
};
template<>
-struct Atomic::PlatformOrderedStore<4, RELEASE_X_FENCE>
+struct AtomicAccess::PlatformOrderedStore<4, RELEASE_X_FENCE>
{
template
void operator()(volatile T* p, T v) const {
@@ -218,7 +218,7 @@ struct Atomic::PlatformOrderedStore<4, RELEASE_X_FENCE>
#ifdef AMD64
template<>
-struct Atomic::PlatformOrderedStore<8, RELEASE_X_FENCE>
+struct AtomicAccess::PlatformOrderedStore<8, RELEASE_X_FENCE>
{
template
void operator()(volatile T* p, T v) const {
@@ -230,4 +230,4 @@ struct Atomic::PlatformOrderedStore<8, RELEASE_X_FENCE>
};
#endif // AMD64
-#endif // OS_CPU_BSD_X86_ATOMIC_BSD_X86_HPP
+#endif // OS_CPU_BSD_X86_ATOMICACCESS_BSD_X86_HPP
diff --git a/src/hotspot/os_cpu/bsd_zero/atomic_bsd_zero.hpp b/src/hotspot/os_cpu/bsd_zero/atomicAccess_bsd_zero.hpp
similarity index 63%
rename from src/hotspot/os_cpu/bsd_zero/atomic_bsd_zero.hpp
rename to src/hotspot/os_cpu/bsd_zero/atomicAccess_bsd_zero.hpp
index 37cd93e765d..6a720dac54e 100644
--- a/src/hotspot/os_cpu/bsd_zero/atomic_bsd_zero.hpp
+++ b/src/hotspot/os_cpu/bsd_zero/atomicAccess_bsd_zero.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright 2007, 2008, 2011, 2015, Red Hat, Inc.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -23,16 +23,16 @@
*
*/
-#ifndef OS_CPU_BSD_ZERO_ATOMIC_BSD_ZERO_HPP
-#define OS_CPU_BSD_ZERO_ATOMIC_BSD_ZERO_HPP
+#ifndef OS_CPU_BSD_ZERO_ATOMICACCESS_BSD_ZERO_HPP
+#define OS_CPU_BSD_ZERO_ATOMICACCESS_BSD_ZERO_HPP
#include "orderAccess_bsd_zero.hpp"
#include "runtime/os.hpp"
-// Implementation of class atomic
+// Implementation of class AtomicAccess
template
-struct Atomic::PlatformAdd {
+struct AtomicAccess::PlatformAdd {
template
D add_then_fetch(D volatile* dest, I add_value, atomic_memory_order order) const;
@@ -44,8 +44,8 @@ struct Atomic::PlatformAdd {
template<>
template
-inline D Atomic::PlatformAdd<4>::add_then_fetch(D volatile* dest, I add_value,
- atomic_memory_order order) const {
+inline D AtomicAccess::PlatformAdd<4>::add_then_fetch(D volatile* dest, I add_value,
+ atomic_memory_order order) const {
STATIC_ASSERT(4 == sizeof(I));
STATIC_ASSERT(4 == sizeof(D));
@@ -56,8 +56,8 @@ inline D Atomic::PlatformAdd<4>::add_then_fetch(D volatile* dest, I add_value,
template<>
template
-inline D Atomic::PlatformAdd<8>::add_then_fetch(D volatile* dest, I add_value,
- atomic_memory_order order) const {
+inline D AtomicAccess::PlatformAdd<8>::add_then_fetch(D volatile* dest, I add_value,
+ atomic_memory_order order) const {
STATIC_ASSERT(8 == sizeof(I));
STATIC_ASSERT(8 == sizeof(D));
@@ -68,9 +68,9 @@ inline D Atomic::PlatformAdd<8>::add_then_fetch(D volatile* dest, I add_value,
template<>
template
-inline T Atomic::PlatformXchg<4>::operator()(T volatile* dest,
- T exchange_value,
- atomic_memory_order order) const {
+inline T AtomicAccess::PlatformXchg<4>::operator()(T volatile* dest,
+ T exchange_value,
+ atomic_memory_order order) const {
STATIC_ASSERT(4 == sizeof(T));
FULL_MEM_BARRIER;
T result = __atomic_exchange_n(dest, exchange_value, __ATOMIC_RELAXED);
@@ -80,9 +80,9 @@ inline T Atomic::PlatformXchg<4>::operator()(T volatile* dest,
template<>
template
-inline T Atomic::PlatformXchg<8>::operator()(T volatile* dest,
- T exchange_value,
- atomic_memory_order order) const {
+inline T AtomicAccess::PlatformXchg<8>::operator()(T volatile* dest,
+ T exchange_value,
+ atomic_memory_order order) const {
STATIC_ASSERT(8 == sizeof(T));
FULL_MEM_BARRIER;
T result = __atomic_exchange_n(dest, exchange_value, __ATOMIC_RELAXED);
@@ -92,14 +92,14 @@ inline T Atomic::PlatformXchg<8>::operator()(T volatile* dest,
// No direct support for cmpxchg of bytes; emulate using int.
template<>
-struct Atomic::PlatformCmpxchg<1> : Atomic::CmpxchgByteUsingInt {};
+struct AtomicAccess::PlatformCmpxchg<1> : AtomicAccess::CmpxchgByteUsingInt {};
template<>
template
-inline T Atomic::PlatformCmpxchg<4>::operator()(T volatile* dest,
- T compare_value,
- T exchange_value,
- atomic_memory_order order) const {
+inline T AtomicAccess::PlatformCmpxchg<4>::operator()(T volatile* dest,
+ T compare_value,
+ T exchange_value,
+ atomic_memory_order order) const {
STATIC_ASSERT(4 == sizeof(T));
T value = compare_value;
FULL_MEM_BARRIER;
@@ -111,10 +111,10 @@ inline T Atomic::PlatformCmpxchg<4>::operator()(T volatile* dest,
template<>
template
-inline T Atomic::PlatformCmpxchg<8>::operator()(T volatile* dest,
- T compare_value,
- T exchange_value,
- atomic_memory_order order) const {
+inline T AtomicAccess::PlatformCmpxchg<8>::operator()(T volatile* dest,
+ T compare_value,
+ T exchange_value,
+ atomic_memory_order order) const {
STATIC_ASSERT(8 == sizeof(T));
T value = compare_value;
@@ -134,7 +134,7 @@ inline void atomic_copy64(const volatile void *src, volatile void *dst) {
template<>
template
-inline T Atomic::PlatformLoad<8>::operator()(T const volatile* src) const {
+inline T AtomicAccess::PlatformLoad<8>::operator()(T const volatile* src) const {
STATIC_ASSERT(8 == sizeof(T));
T dest;
__atomic_load(const_cast(src), &dest, __ATOMIC_RELAXED);
@@ -143,10 +143,10 @@ inline T Atomic::PlatformLoad<8>::operator()(T const volatile* src) const {
template<>
template
-inline void Atomic::PlatformStore<8>::operator()(T volatile* dest,
- T store_value) const {
+inline void AtomicAccess::PlatformStore<8>::operator()(T volatile* dest,
+ T store_value) const {
STATIC_ASSERT(8 == sizeof(T));
__atomic_store(dest, &store_value, __ATOMIC_RELAXED);
}
-#endif // OS_CPU_BSD_ZERO_ATOMIC_BSD_ZERO_HPP
+#endif // OS_CPU_BSD_ZERO_ATOMICACCESS_BSD_ZERO_HPP
diff --git a/src/hotspot/os_cpu/bsd_zero/os_bsd_zero.cpp b/src/hotspot/os_cpu/bsd_zero/os_bsd_zero.cpp
index 3fefbdbe56c..facad184426 100644
--- a/src/hotspot/os_cpu/bsd_zero/os_bsd_zero.cpp
+++ b/src/hotspot/os_cpu/bsd_zero/os_bsd_zero.cpp
@@ -24,7 +24,6 @@
*/
#include "asm/assembler.inline.hpp"
-#include "atomic_bsd_zero.hpp"
#include "classfile/vmSymbols.hpp"
#include "code/vtableStubs.hpp"
#include "interpreter/interpreter.hpp"
@@ -36,6 +35,7 @@
#include "prims/jniFastGetField.hpp"
#include "prims/jvm_misc.hpp"
#include "runtime/arguments.hpp"
+#include "runtime/atomicAccess.hpp"
#include "runtime/frame.inline.hpp"
#include "runtime/interfaceSupport.inline.hpp"
#include "runtime/java.hpp"
diff --git a/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.hpp b/src/hotspot/os_cpu/linux_aarch64/atomicAccess_linux_aarch64.hpp
similarity index 74%
rename from src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.hpp
rename to src/hotspot/os_cpu/linux_aarch64/atomicAccess_linux_aarch64.hpp
index fa1ab952442..6e5f53edfa3 100644
--- a/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.hpp
+++ b/src/hotspot/os_cpu/linux_aarch64/atomicAccess_linux_aarch64.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -23,13 +23,13 @@
*
*/
-#ifndef OS_CPU_LINUX_AARCH64_ATOMIC_LINUX_AARCH64_HPP
-#define OS_CPU_LINUX_AARCH64_ATOMIC_LINUX_AARCH64_HPP
+#ifndef OS_CPU_LINUX_AARCH64_ATOMICACCESS_LINUX_AARCH64_HPP
+#define OS_CPU_LINUX_AARCH64_ATOMICACCESS_LINUX_AARCH64_HPP
#include "atomic_aarch64.hpp"
#include "runtime/vm_version.hpp"
-// Implementation of class atomic
+// Implementation of class AtomicAccess
// Note that memory_order_conservative requires a full barrier after atomic stores.
// See https://patchwork.kernel.org/patch/3575821/
@@ -70,7 +70,7 @@ inline D atomic_fastcall(F stub, volatile D *dest, T1 arg1, T2 arg2) {
}
template
-struct Atomic::PlatformAdd {
+struct AtomicAccess::PlatformAdd {
template
D fetch_then_add(D volatile* dest, I add_value, atomic_memory_order order) const;
@@ -83,8 +83,8 @@ struct Atomic::PlatformAdd {
template<>
template
-inline D Atomic::PlatformAdd<4>::fetch_then_add(D volatile* dest, I add_value,
- atomic_memory_order order) const {
+inline D AtomicAccess::PlatformAdd<4>::fetch_then_add(D volatile* dest, I add_value,
+ atomic_memory_order order) const {
STATIC_ASSERT(4 == sizeof(I));
STATIC_ASSERT(4 == sizeof(D));
aarch64_atomic_stub_t stub;
@@ -99,8 +99,8 @@ inline D Atomic::PlatformAdd<4>::fetch_then_add(D volatile* dest, I add_value,
template<>
template
-inline D Atomic::PlatformAdd<8>::fetch_then_add(D volatile* dest, I add_value,
- atomic_memory_order order) const {
+inline D AtomicAccess::PlatformAdd<8>::fetch_then_add(D volatile* dest, I add_value,
+ atomic_memory_order order) const {
STATIC_ASSERT(8 == sizeof(I));
STATIC_ASSERT(8 == sizeof(D));
aarch64_atomic_stub_t stub;
@@ -115,9 +115,9 @@ inline D Atomic::PlatformAdd<8>::fetch_then_add(D volatile* dest, I add_value,
template<>
template
-inline T Atomic::PlatformXchg<4>::operator()(T volatile* dest,
- T exchange_value,
- atomic_memory_order order) const {
+inline T AtomicAccess::PlatformXchg<4>::operator()(T volatile* dest,
+ T exchange_value,
+ atomic_memory_order order) const {
STATIC_ASSERT(4 == sizeof(T));
T old_value = atomic_fastcall(aarch64_atomic_xchg_4_impl, dest, exchange_value);
return old_value;
@@ -125,8 +125,8 @@ inline T Atomic::PlatformXchg<4>::operator()(T volatile* dest,
template<>
template
-inline T Atomic::PlatformXchg<8>::operator()(T volatile* dest, T exchange_value,
- atomic_memory_order order) const {
+inline T AtomicAccess::PlatformXchg<8>::operator()(T volatile* dest, T exchange_value,
+ atomic_memory_order order) const {
STATIC_ASSERT(8 == sizeof(T));
T old_value = atomic_fastcall(aarch64_atomic_xchg_8_impl, dest, exchange_value);
return old_value;
@@ -134,10 +134,10 @@ inline T Atomic::PlatformXchg<8>::operator()(T volatile* dest, T exchange_value,
template<>
template
-inline T Atomic::PlatformCmpxchg<1>::operator()(T volatile* dest,
- T compare_value,
- T exchange_value,
- atomic_memory_order order) const {
+inline T AtomicAccess::PlatformCmpxchg<1>::operator()(T volatile* dest,
+ T compare_value,
+ T exchange_value,
+ atomic_memory_order order) const {
STATIC_ASSERT(1 == sizeof(T));
aarch64_atomic_stub_t stub;
switch (order) {
@@ -152,10 +152,10 @@ inline T Atomic::PlatformCmpxchg<1>::operator()(T volatile* dest,
template<>
template
-inline T Atomic::PlatformCmpxchg<4>::operator()(T volatile* dest,
- T compare_value,
- T exchange_value,
- atomic_memory_order order) const {
+inline T AtomicAccess::PlatformCmpxchg<4>::operator()(T volatile* dest,
+ T compare_value,
+ T exchange_value,
+ atomic_memory_order order) const {
STATIC_ASSERT(4 == sizeof(T));
aarch64_atomic_stub_t stub;
switch (order) {
@@ -175,10 +175,10 @@ inline T Atomic::PlatformCmpxchg<4>::operator()(T volatile* dest,
template<>
template
-inline T Atomic::PlatformCmpxchg<8>::operator()(T volatile* dest,
- T compare_value,
- T exchange_value,
- atomic_memory_order order) const {
+inline T AtomicAccess::PlatformCmpxchg<8>::operator()(T volatile* dest,
+ T compare_value,
+ T exchange_value,
+ atomic_memory_order order) const {
STATIC_ASSERT(8 == sizeof(T));
aarch64_atomic_stub_t stub;
switch (order) {
@@ -197,24 +197,24 @@ inline T Atomic::PlatformCmpxchg<8>::operator()(T volatile* dest,
}
template
-struct Atomic::PlatformOrderedLoad
+struct AtomicAccess::PlatformOrderedLoad
{
template
T operator()(const volatile T* p) const { T data; __atomic_load(const_cast(p), &data, __ATOMIC_ACQUIRE); return data; }
};
template
-struct Atomic::PlatformOrderedStore
+struct AtomicAccess::PlatformOrderedStore
{
template
void operator()(volatile T* p, T v) const { __atomic_store(const_cast(p), &v, __ATOMIC_RELEASE); }
};
template
-struct Atomic::PlatformOrderedStore
+struct AtomicAccess::PlatformOrderedStore
{
template
void operator()(volatile T* p, T v) const { release_store(p, v); OrderAccess::fence(); }
};
-#endif // OS_CPU_LINUX_AARCH64_ATOMIC_LINUX_AARCH64_HPP
+#endif // OS_CPU_LINUX_AARCH64_ATOMICACCESS_LINUX_AARCH64_HPP
diff --git a/src/hotspot/os_cpu/linux_arm/atomic_linux_arm.hpp b/src/hotspot/os_cpu/linux_arm/atomicAccess_linux_arm.hpp
similarity index 75%
rename from src/hotspot/os_cpu/linux_arm/atomic_linux_arm.hpp
rename to src/hotspot/os_cpu/linux_arm/atomicAccess_linux_arm.hpp
index 4346920cd37..5b5f9da51a6 100644
--- a/src/hotspot/os_cpu/linux_arm/atomic_linux_arm.hpp
+++ b/src/hotspot/os_cpu/linux_arm/atomicAccess_linux_arm.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2008, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -22,14 +22,14 @@
*
*/
-#ifndef OS_CPU_LINUX_ARM_ATOMIC_LINUX_ARM_HPP
-#define OS_CPU_LINUX_ARM_ATOMIC_LINUX_ARM_HPP
+#ifndef OS_CPU_LINUX_ARM_ATOMICACCESS_LINUX_ARM_HPP
+#define OS_CPU_LINUX_ARM_ATOMICACCESS_LINUX_ARM_HPP
#include "memory/allStatic.hpp"
#include "runtime/os.hpp"
#include "runtime/vm_version.hpp"
-// Implementation of class atomic
+// Implementation of class AtomicAccess
class ARMAtomicFuncs : AllStatic {
public:
@@ -78,7 +78,7 @@ public:
template<>
template
-inline T Atomic::PlatformLoad<8>::operator()(T const volatile* src) const {
+inline T AtomicAccess::PlatformLoad<8>::operator()(T const volatile* src) const {
STATIC_ASSERT(8 == sizeof(T));
return PrimitiveConversions::cast(
(*ARMAtomicFuncs::_load_long_func)(reinterpret_cast(src)));
@@ -86,20 +86,20 @@ inline T Atomic::PlatformLoad<8>::operator()(T const volatile* src) const {
template<>
template
-inline void Atomic::PlatformStore<8>::operator()(T volatile* dest,
- T store_value) const {
+inline void AtomicAccess::PlatformStore<8>::operator()(T volatile* dest,
+ T store_value) const {
STATIC_ASSERT(8 == sizeof(T));
(*ARMAtomicFuncs::_store_long_func)(
PrimitiveConversions::cast(store_value), reinterpret_cast(dest));
}
-// As per atomic.hpp all read-modify-write operations have to provide two-way
+// As per atomicAccess.hpp all read-modify-write operations have to provide two-way
// barriers semantics.
//
// For ARMv7 we add explicit barriers in the stubs.
template
-struct Atomic::PlatformAdd {
+struct AtomicAccess::PlatformAdd {
template
D add_then_fetch(D volatile* dest, I add_value, atomic_memory_order order) const;
@@ -111,8 +111,8 @@ struct Atomic::PlatformAdd {
template<>
template
-inline D Atomic::PlatformAdd<4>::add_then_fetch(D volatile* dest, I add_value,
- atomic_memory_order order) const {
+inline D AtomicAccess::PlatformAdd<4>::add_then_fetch(D volatile* dest, I add_value,
+ atomic_memory_order order) const {
STATIC_ASSERT(4 == sizeof(I));
STATIC_ASSERT(4 == sizeof(D));
return add_using_helper(ARMAtomicFuncs::_add_func, dest, add_value);
@@ -121,26 +121,26 @@ inline D Atomic::PlatformAdd<4>::add_then_fetch(D volatile* dest, I add_value,
template<>
template
-inline T Atomic::PlatformXchg<4>::operator()(T volatile* dest,
- T exchange_value,
- atomic_memory_order order) const {
+inline T AtomicAccess::PlatformXchg<4>::operator()(T volatile* dest,
+ T exchange_value,
+ atomic_memory_order order) const {
STATIC_ASSERT(4 == sizeof(T));
return xchg_using_helper(ARMAtomicFuncs::_xchg_func, dest, exchange_value);
}
// No direct support for 8-byte xchg; emulate using cmpxchg.
template<>
-struct Atomic::PlatformXchg<8> : Atomic::XchgUsingCmpxchg<8> {};
+struct AtomicAccess::PlatformXchg<8> : AtomicAccess::XchgUsingCmpxchg<8> {};
// No direct support for 8-byte add; emulate using cmpxchg.
template<>
-struct Atomic::PlatformAdd<8> : Atomic::AddUsingCmpxchg<8> {};
+struct AtomicAccess::PlatformAdd<8> : AtomicAccess::AddUsingCmpxchg<8> {};
// The memory_order parameter is ignored - we always provide the strongest/most-conservative ordering
// No direct support for cmpxchg of bytes; emulate using int.
template<>
-struct Atomic::PlatformCmpxchg<1> : Atomic::CmpxchgByteUsingInt {};
+struct AtomicAccess::PlatformCmpxchg<1> : AtomicAccess::CmpxchgByteUsingInt {};
inline int32_t reorder_cmpxchg_func(int32_t exchange_value,
@@ -160,22 +160,22 @@ inline int64_t reorder_cmpxchg_long_func(int64_t exchange_value,
template<>
template
-inline T Atomic::PlatformCmpxchg<4>::operator()(T volatile* dest,
- T compare_value,
- T exchange_value,
- atomic_memory_order order) const {
+inline T AtomicAccess::PlatformCmpxchg<4>::operator()(T volatile* dest,
+ T compare_value,
+ T exchange_value,
+ atomic_memory_order order) const {
STATIC_ASSERT(4 == sizeof(T));
return cmpxchg_using_helper(reorder_cmpxchg_func, dest, compare_value, exchange_value);
}
template<>
template
-inline T Atomic::PlatformCmpxchg<8>::operator()(T volatile* dest,
- T compare_value,
- T exchange_value,
- atomic_memory_order order) const {
+inline T AtomicAccess::PlatformCmpxchg<8>::operator()(T volatile* dest,
+ T compare_value,
+ T exchange_value,
+ atomic_memory_order order) const {
STATIC_ASSERT(8 == sizeof(T));
return cmpxchg_using_helper(reorder_cmpxchg_long_func, dest, compare_value, exchange_value);
}
-#endif // OS_CPU_LINUX_ARM_ATOMIC_LINUX_ARM_HPP
+#endif // OS_CPU_LINUX_ARM_ATOMICACCESS_LINUX_ARM_HPP
diff --git a/test/jdk/sun/tools/jrunscript/Hello.java b/src/hotspot/os_cpu/linux_ppc/atomicAccess_linux_ppc.hpp
similarity index 79%
rename from test/jdk/sun/tools/jrunscript/Hello.java
rename to src/hotspot/os_cpu/linux_ppc/atomicAccess_linux_ppc.hpp
index 48b6c9298ab..9a314667fcb 100644
--- a/test/jdk/sun/tools/jrunscript/Hello.java
+++ b/src/hotspot/os_cpu/linux_ppc/atomicAccess_linux_ppc.hpp
@@ -1,5 +1,6 @@
/*
- * Copyright (c) 2005, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -19,16 +20,8 @@
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
+ *
*/
-/**
- * This is a test program used in the test jrunscript-cp.sh
- *
- *
- */
-public class Hello {
- public Hello() {}
- public String getString() {
- return "hello";
- }
-}
+// Including inline assembler functions that are shared between multiple PPC64 platforms.
+#include "atomicAccess_ppc.hpp"
diff --git a/src/hotspot/os_cpu/linux_ppc/atomic_linux_ppc.hpp b/src/hotspot/os_cpu/linux_ppc/atomic_linux_ppc.hpp
deleted file mode 100644
index 1e4eb37cdac..00000000000
--- a/src/hotspot/os_cpu/linux_ppc/atomic_linux_ppc.hpp
+++ /dev/null
@@ -1,395 +0,0 @@
-/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2019 SAP SE. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef OS_CPU_LINUX_PPC_ATOMIC_LINUX_PPC_HPP
-#define OS_CPU_LINUX_PPC_ATOMIC_LINUX_PPC_HPP
-
-#ifndef PPC64
-#error "Atomic currently only implemented for PPC64"
-#endif
-
-#include "orderAccess_linux_ppc.hpp"
-#include "utilities/debug.hpp"
-
-// Implementation of class atomic
-
-//
-// machine barrier instructions:
-//
-// - sync two-way memory barrier, aka fence
-// - lwsync orders Store|Store,
-// Load|Store,
-// Load|Load,
-// but not Store|Load
-// - eieio orders memory accesses for device memory (only)
-// - isync invalidates speculatively executed instructions
-// From the POWER ISA 2.06 documentation:
-// "[...] an isync instruction prevents the execution of
-// instructions following the isync until instructions
-// preceding the isync have completed, [...]"
-// From IBM's AIX assembler reference:
-// "The isync [...] instructions causes the processor to
-// refetch any instructions that might have been fetched
-// prior to the isync instruction. The instruction isync
-// causes the processor to wait for all previous instructions
-// to complete. Then any instructions already fetched are
-// discarded and instruction processing continues in the
-// environment established by the previous instructions."
-//
-// semantic barrier instructions:
-// (as defined in orderAccess.hpp)
-//
-// - release orders Store|Store, (maps to lwsync)
-// Load|Store
-// - acquire orders Load|Store, (maps to lwsync)
-// Load|Load
-// - fence orders Store|Store, (maps to sync)
-// Load|Store,
-// Load|Load,
-// Store|Load
-//
-
-inline void pre_membar(atomic_memory_order order) {
- switch (order) {
- case memory_order_relaxed:
- case memory_order_acquire: break;
- case memory_order_release:
- case memory_order_acq_rel: __asm__ __volatile__ ("lwsync" : : : "memory"); break;
- default /*conservative*/ : __asm__ __volatile__ ("sync" : : : "memory"); break;
- }
-}
-
-inline void post_membar(atomic_memory_order order) {
- switch (order) {
- case memory_order_relaxed:
- case memory_order_release: break;
- case memory_order_acquire:
- case memory_order_acq_rel: __asm__ __volatile__ ("isync" : : : "memory"); break;
- default /*conservative*/ : __asm__ __volatile__ ("sync" : : : "memory"); break;
- }
-}
-
-
-template
-struct Atomic::PlatformAdd {
- template
- D add_then_fetch(D volatile* dest, I add_value, atomic_memory_order order) const;
-
- template
- D fetch_then_add(D volatile* dest, I add_value, atomic_memory_order order) const {
- return add_then_fetch(dest, add_value, order) - add_value;
- }
-};
-
-template<>
-template
-inline D Atomic::PlatformAdd<4>::add_then_fetch(D volatile* dest, I add_value,
- atomic_memory_order order) const {
- STATIC_ASSERT(4 == sizeof(I));
- STATIC_ASSERT(4 == sizeof(D));
-
- D result;
-
- pre_membar(order);
-
- __asm__ __volatile__ (
- "1: lwarx %0, 0, %2 \n"
- " add %0, %0, %1 \n"
- " stwcx. %0, 0, %2 \n"
- " bne- 1b \n"
- : /*%0*/"=&r" (result)
- : /*%1*/"r" (add_value), /*%2*/"r" (dest)
- : "cc", "memory" );
-
- post_membar(order);
-
- return result;
-}
-
-
-template<>
-template
-inline D Atomic::PlatformAdd<8>::add_then_fetch(D volatile* dest, I add_value,
- atomic_memory_order order) const {
- STATIC_ASSERT(8 == sizeof(I));
- STATIC_ASSERT(8 == sizeof(D));
-
- D result;
-
- pre_membar(order);
-
- __asm__ __volatile__ (
- "1: ldarx %0, 0, %2 \n"
- " add %0, %0, %1 \n"
- " stdcx. %0, 0, %2 \n"
- " bne- 1b \n"
- : /*%0*/"=&r" (result)
- : /*%1*/"r" (add_value), /*%2*/"r" (dest)
- : "cc", "memory" );
-
- post_membar(order);
-
- return result;
-}
-
-template<>
-template
-inline T Atomic::PlatformXchg<4>::operator()(T volatile* dest,
- T exchange_value,
- atomic_memory_order order) const {
- // Note that xchg doesn't necessarily do an acquire
- // (see synchronizer.cpp).
-
- T old_value;
- const uint64_t zero = 0;
-
- pre_membar(order);
-
- __asm__ __volatile__ (
- /* atomic loop */
- "1: \n"
- " lwarx %[old_value], %[dest], %[zero] \n"
- " stwcx. %[exchange_value], %[dest], %[zero] \n"
- " bne- 1b \n"
- /* exit */
- "2: \n"
- /* out */
- : [old_value] "=&r" (old_value),
- "=m" (*dest)
- /* in */
- : [dest] "b" (dest),
- [zero] "r" (zero),
- [exchange_value] "r" (exchange_value),
- "m" (*dest)
- /* clobber */
- : "cc",
- "memory"
- );
-
- post_membar(order);
-
- return old_value;
-}
-
-template<>
-template
-inline T Atomic::PlatformXchg<8>::operator()(T volatile* dest,
- T exchange_value,
- atomic_memory_order order) const {
- STATIC_ASSERT(8 == sizeof(T));
- // Note that xchg doesn't necessarily do an acquire
- // (see synchronizer.cpp).
-
- T old_value;
- const uint64_t zero = 0;
-
- pre_membar(order);
-
- __asm__ __volatile__ (
- /* atomic loop */
- "1: \n"
- " ldarx %[old_value], %[dest], %[zero] \n"
- " stdcx. %[exchange_value], %[dest], %[zero] \n"
- " bne- 1b \n"
- /* exit */
- "2: \n"
- /* out */
- : [old_value] "=&r" (old_value),
- "=m" (*dest)
- /* in */
- : [dest] "b" (dest),
- [zero] "r" (zero),
- [exchange_value] "r" (exchange_value),
- "m" (*dest)
- /* clobber */
- : "cc",
- "memory"
- );
-
- post_membar(order);
-
- return old_value;
-}
-
-template<>
-template