infernap12/jdk
1
0
Fork 0
mirror of https://github.com/openjdk/jdk.git synced 2026-08-04 07:05:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'openjdk:master' into dwhite-simpler-apx-regression

Browse Source
This commit is contained in:
Derek White 2026-08-03 13:34:17 -04:00 committed by GitHub
commit dbd4fd3b59
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
152 changed files with 3370 additions and 833 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
make/modules/jdk.crypto.cryptoki/Lib.gmk
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved.
# Copyright (c) 2011, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@ -35,8 +35,6 @@ $(eval $(call SetupJdkLibrary, BUILD_LIBJ2PKCS11, \
NAME := j2pkcs11, \
OPTIMIZATION := LOW, \
EXTRA_HEADER_DIRS := java.base:libjava, \
DISABLED_WARNINGS_gcc_p11_md.c := unused-variable, \
DISABLED_WARNINGS_clang_p11_md.c := unused-variable, \
DISABLED_WARNINGS_clang_p11_util.c := format-nonliteral, \
LIBS_unix := $(LIBDL), \
))

4
src/hotspot/cpu/aarch64/aarch64.ad
View File

@ -14212,7 +14212,7 @@ instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Uni
instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
%{
predicate(((ClearArrayNode*)n)->word_copy_only());
predicate(((ClearArrayNode*)n)->requires_word_fill());
match(Set dummy (ClearArray (Binary cnt base) val));
effect(USE_KILL cnt, USE_KILL base, KILL cr);
@ -14230,7 +14230,7 @@ instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, immL0 zero
%{
predicate((uint64_t)n->in(2)->in(1)->get_long()
< (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
&& !((ClearArrayNode*)n)->word_copy_only());
&& !((ClearArrayNode*)n)->requires_word_fill());
match(Set dummy (ClearArray (Binary cnt base) zero));
effect(TEMP temp, USE_KILL base, KILL cr);

6
src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
View File

@ -7911,8 +7911,10 @@ void MacroAssembler::fast_lock(Register basic_lock, Register obj, Register t1, R
// Try to lock. Transition lock bits 0b01 => 0b00
assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
orr(mark, mark, markWord::unlocked_value);
// Mask inline_type bit such that we go to the slow path if object is an inline type
andr(mark, mark, ~((int) markWord::inline_type_bit_in_place));
if (Arguments::is_valhalla_enabled()) {
// Mask inline_type bit such that we go to the slow path if object is an inline type
andr(mark, mark, ~((int) markWord::inline_type_bit_in_place));
}
eor(t, mark, markWord::unlocked_value);
cmpxchg(/*addr*/ obj, /*expected*/ mark, /*new*/ t, Assembler::xword, memory_order_acquire);

5
src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp
View File

@ -2922,11 +2922,10 @@ RuntimeStub* SharedRuntime::generate_resolve_blob(StubId id, address destination
}
BufferedInlineTypeBlob* SharedRuntime::generate_buffered_inline_type_adapter(const InlineKlass* vk) {
BufferBlob* buf = BufferBlob::create("inline types pack/unpack", 16 * K);
if (buf == nullptr) {
CodeBuffer buffer("inline types pack/unpack", 16 * K, 0);
if (buffer.blob() == nullptr) {
return nullptr;
}
CodeBuffer buffer(buf);
short buffer_locs[20];
buffer.insts()->initialize_shared_locs((relocInfo*)buffer_locs,
sizeof(buffer_locs)/sizeof(relocInfo));

10
src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp
View File

@ -2623,11 +2623,13 @@ class StubGenerator: public StubCodeGenerator {
__ eor(rscratch2, rscratch2, scratch_src_klass);
__ cbnz(rscratch2, L_failed);
// Check for flat inline type array -> return -1
__ test_flat_array_oop(src, rscratch2, L_failed);
if (Arguments::is_valhalla_enabled()) {
// Check for flat inline type array -> return -1
__ test_flat_array_oop(src, rscratch2, L_failed);
// Check for null-free (non-flat) inline type array -> handle as object array
__ test_null_free_array_oop(src, rscratch2, L_objArray);
// Check for null-free (non-flat) inline type array -> handle as object array
__ test_null_free_array_oop(src, rscratch2, L_objArray);
}
// if (!src->is_Array()) return -1;
__ tbz(lh, 31, L_failed); // i.e. (lh >= 0)

4
src/hotspot/cpu/ppc/ppc.ad
View File

@ -11096,8 +11096,8 @@ instruct inlineCallClearArray(rarg1RegL cnt, rarg2RegP base, immL_0 zero, Univer
%}
// Clear-array with dynamic array length and non-zero value.
instruct inlineCallClearArrayWordCopy(rarg1RegL cnt, rarg2RegP base, iRegLdst val, Universe dummy, regCTR ctr) %{
predicate(((ClearArrayNode*)n)->word_copy_only());
instruct inlineCallClearArrayWordFill(rarg1RegL cnt, rarg2RegP base, iRegLdst val, Universe dummy, regCTR ctr) %{
predicate(((ClearArrayNode*)n)->requires_word_fill());
match(Set dummy (ClearArray (Binary cnt base) val));
effect(USE_KILL base, KILL ctr);
ins_cost(8 * MEMORY_REF_COST);

13
src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
View File

@ -2174,12 +2174,12 @@ void MacroAssembler::vector_update_crc32(Register crc, Register buf, Register le
mv(tmp5, 0xff);
if (MaxVectorSize == 16) {
vsetivli(zr, N, Assembler::e32, Assembler::m4, Assembler::ma, Assembler::ta);
vsetivli(zr, N, Assembler::e32, Assembler::m4, Assembler::mu, Assembler::tu);
} else if (MaxVectorSize == 32) {
vsetivli(zr, N, Assembler::e32, Assembler::m2, Assembler::ma, Assembler::ta);
vsetivli(zr, N, Assembler::e32, Assembler::m2, Assembler::mu, Assembler::tu);
} else {
assert(MaxVectorSize > 32, "sanity");
vsetivli(zr, N, Assembler::e32, Assembler::m1, Assembler::ma, Assembler::ta);
vsetivli(zr, N, Assembler::e32, Assembler::m1, Assembler::mu, Assembler::tu);
}
vmv_v_x(vcrc, zr);
@ -7064,8 +7064,11 @@ void MacroAssembler::fast_lock(Register basic_lock, Register obj, Register tmp1,
// Try to lock. Transition lock-bits 0b01 => 0b00
assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid a la");
ori(mark, mark, markWord::unlocked_value);
// Mask inline_type bit such that we go to the slow path if object is an inline type
andi(mark, mark, ~((int) markWord::inline_type_bit_in_place));
if (Arguments::is_valhalla_enabled()) {
// Mask inline_type bit such that we go to the slow path if object is an inline type
andi(mark, mark, ~((int) markWord::inline_type_bit_in_place));
}
xori(t, mark, markWord::unlocked_value);
cmpxchg(/*addr*/ obj, /*expected*/ mark, /*new*/ t, Assembler::int64,
/*acquire*/ Assembler::aq, /*release*/ Assembler::relaxed, /*result*/ t);

4
src/hotspot/cpu/riscv/riscv.ad
View File

@ -11376,7 +11376,7 @@ instruct clearArray_reg_reg(iRegL_R29 cnt, iRegP_R28 base, iRegL val,
Universe dummy)
%{
// temp registers must match the one used in StubGenerator::generate_zero_blocks()
predicate(((ClearArrayNode*)n)->word_copy_only());
predicate(((ClearArrayNode*)n)->requires_word_fill());
match(Set dummy (ClearArray (Binary cnt base) val));
effect(USE_KILL cnt, USE_KILL base, TEMP tmp1, TEMP tmp2, KILL cr);
@ -11395,7 +11395,7 @@ instruct clearArray_imm_reg(immL cnt, iRegP_R28 base, immL0 zero, Universe dummy
predicate(!UseRVV
&& (uint64_t)n->in(2)->in(1)->get_long()
< (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
&& !((ClearArrayNode*)n)->word_copy_only());
&& !((ClearArrayNode*)n)->requires_word_fill());
match(Set dummy (ClearArray (Binary cnt base) zero));
effect(USE_KILL base, KILL cr);

15
src/hotspot/cpu/riscv/stubGenerator_riscv.cpp
View File

@ -1925,15 +1925,16 @@ class StubGenerator: public StubCodeGenerator {
__ load_klass(t1, dst);
__ bne(t1, scratch_src_klass, L_failed);
// Check for flat inline type array -> return -1
__ test_flat_array_oop(src, t1, L_failed);
if (Arguments::is_valhalla_enabled()) {
// Check for flat inline type array -> return -1
__ test_flat_array_oop(src, t1, L_failed);
// Check for null-free (non-flat) inline type array -> handle as object array
__ test_null_free_array_oop(src, t1, L_objArray);
// Check for null-free (non-flat) inline type array -> handle as object array
__ test_null_free_array_oop(src, t1, L_objArray);
}
// if src->is_Array() isn't null then return -1
// i.e. (lh >= 0)
__ bgez(lh, L_failed);
// if (!src->is_Array()) return -1;
__ bgez(lh, L_failed); // i.e. (lh >= 0)
// At this point, it is known to be a typeArray (array_tag 0x3).
#ifdef ASSERT

1
src/hotspot/cpu/s390/continuationEntry_s390.inline.hpp
View File

@ -26,6 +26,7 @@
#ifndef CPU_S390_CONTINUATIONENTRY_S390_INLINE_HPP
#define CPU_S390_CONTINUATIONENTRY_S390_INLINE_HPP
#include "code/codeCache.hpp"
#include "oops/method.inline.hpp"
#include "runtime/frame.inline.hpp"
#include "runtime/registerMap.hpp"

34
src/hotspot/cpu/x86/macroAssembler_x86.cpp
View File

@ -6506,11 +6506,12 @@ void MacroAssembler::remove_frame(int initial_framesize, bool needs_stack_repair
#ifdef COMPILER2
// clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM/ZMM registers
void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register val, XMMRegister xtmp, KRegister mask) {
// Fill memory with 'val', for 'cnt' qwords starting at 'base', using XMM/YMM/ZMM registers.
void MacroAssembler::xmm_fill_mem(Register base, Register cnt, Register val, XMMRegister xtmp, KRegister mask) {
// cnt - number of qwords (8-byte words).
// base - start address, qword aligned.
Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end;
// val - qword pattern to fill.
Label L_fill_64_bytes, L_loop, L_sloop, L_tail, L_end;
bool use64byteVector = (MaxVectorSize == 64) && (CopyAVX3Threshold == 0) && VM_Version::supports_bmi2();
if (use64byteVector) {
evpbroadcastq(xtmp, val, AVX_512bit);
@ -6522,7 +6523,7 @@ void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register val, XM
movdq(xtmp, val);
punpcklqdq(xtmp, xtmp);
}
jmp(L_zero_64_bytes);
jmp(L_fill_64_bytes);
BIND(L_loop);
if (MaxVectorSize >= 32) {
@ -6535,11 +6536,11 @@ void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register val, XM
}
addptr(base, 64);
BIND(L_zero_64_bytes);
BIND(L_fill_64_bytes);
subptr(cnt, 8);
jccb(Assembler::greaterEqual, L_loop);
// Copy trailing 64 bytes
// Fill trailing 64 bytes.
if (use64byteVector) {
addptr(cnt, 8);
jccb(Assembler::equal, L_end);
@ -6665,10 +6666,12 @@ void MacroAssembler::clear_mem(Register base, int cnt, Register rtmp, XMMRegiste
}
void MacroAssembler::clear_mem(Register base, Register cnt, Register val, XMMRegister xtmp,
bool is_large, bool word_copy_only, KRegister mask) {
bool is_large, bool requires_word_fill, KRegister mask) {
// cnt - number of qwords (8-byte words).
// base - start address, qword aligned.
// is_large - if optimizers know cnt is larger than InitArrayShortSize
// requires_word_fill - if true, val contains the qword pattern to fill; if
// false, val is scratch and this method creates zero
assert(base==rdi, "base register must be edi for rep stos");
assert(val==rax, "val register must be eax for rep stos");
assert(cnt==rcx, "cnt register must be ecx for rep stos");
@ -6677,6 +6680,10 @@ void MacroAssembler::clear_mem(Register base, Register cnt, Register val, XMMReg
Label DONE;
if (!requires_word_fill) {
xorptr(val, val);
}
if (!is_large) {
Label LOOP, LONG;
cmpptr(cnt, InitArrayShortSize/BytesPerLong);
@ -6695,12 +6702,13 @@ void MacroAssembler::clear_mem(Register base, Register cnt, Register val, XMMReg
BIND(LONG);
}
// Use longer rep-prefixed ops for non-small counts:
if (UseFastStosb && !word_copy_only) {
// Use longer rep-prefixed ops for non-small counts. rep stosb is valid only
// for zeroing; an arbitrary qword pattern must be copied in full.
if (UseFastStosb && !requires_word_fill) {
shlptr(cnt, 3); // convert to number of bytes
rep_stosb();
} else if (UseXMMForObjInit) {
xmm_clear_mem(base, cnt, val, xtmp, mask);
xmm_fill_mem(base, cnt, val, xtmp, mask);
} else {
rep_stos();
}
@ -10617,8 +10625,10 @@ void MacroAssembler::fast_lock(Register basic_lock, Register obj, Register reg_r
movptr(tmp, reg_rax);
andptr(tmp, ~(int32_t)markWord::unlocked_value);
orptr(reg_rax, markWord::unlocked_value);
// Mask inline_type bit such that we go to the slow path if object is an inline type
andptr(reg_rax, ~((int) markWord::inline_type_bit_in_place));
if (Arguments::is_valhalla_enabled()) {
// Mask inline_type bit such that we go to the slow path if object is an inline type
andptr(reg_rax, ~((int) markWord::inline_type_bit_in_place));
}
lock(); cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
jcc(Assembler::notEqual, slow);

11
src/hotspot/cpu/x86/macroAssembler_x86.hpp
View File

@ -1970,15 +1970,16 @@ public:
// Inline type specific methods
#include "asm/macroAssembler_common.hpp"
// clear memory of size 'cnt' qwords, starting at 'base';
// if 'is_large' is set, do not try to produce short loop
void clear_mem(Register base, Register cnt, Register val, XMMRegister xtmp, bool is_large, bool word_copy_only, KRegister mask=knoreg);
// Clear or fill 'cnt' qwords starting at 'base'. If 'requires_word_fill' is
// set, use 'val' as the fill value; otherwise, create zero in 'val'. If
// 'is_large' is set, do not try to produce a short loop.
void clear_mem(Register base, Register cnt, Register val, XMMRegister xtmp, bool is_large, bool requires_word_fill, KRegister mask=knoreg);
// clear memory initialization sequence for constant size;
void clear_mem(Register base, int cnt, Register rtmp, XMMRegister xtmp, KRegister mask=knoreg);
// clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM registers
void xmm_clear_mem(Register base, Register cnt, Register rtmp, XMMRegister xtmp, KRegister mask=knoreg);
// Fill memory with 'val', for 'cnt' qwords starting at 'base', using XMM/YMM/ZMM registers.
void xmm_fill_mem(Register base, Register cnt, Register val, XMMRegister xtmp, KRegister mask=knoreg);
// Fill primitive arrays
void generate_fill(BasicType t, bool aligned,

5
src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp
View File

@ -3731,11 +3731,10 @@ void SharedRuntime::montgomery_square(jint *a_ints, jint *n_ints,
}
BufferedInlineTypeBlob* SharedRuntime::generate_buffered_inline_type_adapter(const InlineKlass* vk) {
BufferBlob* buf = BufferBlob::create("inline types pack/unpack", 16 * K);
if (buf == nullptr) {
CodeBuffer buffer("inline types pack/unpack", 16 * K, 0);
if (buffer.blob() == nullptr) {
return nullptr;
}
CodeBuffer buffer(buf);
short buffer_locs[20];
buffer.insts()->initialize_shared_locs((relocInfo*)buffer_locs,
sizeof(buffer_locs)/sizeof(relocInfo));

11
src/hotspot/cpu/x86/stubGenerator_x86_64_arraycopy.cpp
View File

@ -26,6 +26,7 @@
#include "gc/shared/barrierSet.hpp"
#include "gc/shared/barrierSetAssembler.hpp"
#include "oops/objArrayKlass.hpp"
#include "runtime/arguments.hpp"
#include "runtime/sharedRuntime.hpp"
#include "runtime/stubRoutines.hpp"
#include "stubGenerator_x86_64.hpp"
@ -3599,11 +3600,13 @@ address StubGenerator::generate_generic_copy(address byte_copy_entry, address sh
__ cmpq(r10_src_klass, rax);
__ jcc(Assembler::notEqual, L_failed);
// Check for flat inline type array -> return -1
__ test_flat_array_oop(src, rax, L_failed);
if (Arguments::is_valhalla_enabled()) {
// Check for flat inline type array -> return -1
__ test_flat_array_oop(src, rax, L_failed);
// Check for null-free (non-flat) inline type array -> handle as object array
__ test_null_free_array_oop(src, rax, L_objArray);
// Check for null-free (non-flat) inline type array -> handle as object array
__ test_null_free_array_oop(src, rax, L_objArray);
}
const Register rax_lh = rax; // layout helper
__ movl(rax_lh, Address(r10_src_klass, lh_offset));

265
src/hotspot/cpu/x86/x86.ad
View File

@ -14774,16 +14774,18 @@ instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
%}
// Fast clearing of an array
// Small non-constant lenght ClearArray for non-AVX512 targets.
instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
// Small zero fill for non-AVX512 targets.
instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, immL0 zero, rax_RegL val,
Universe dummy, rFlagsReg cr)
%{
predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
match(Set dummy (ClearArray (Binary cnt base) val));
effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
predicate((UseAVX <= 2) &&
!((ClearArrayNode*)n)->is_large() &&
((ClearArrayNode*)n)->is_zero_fill());
match(Set dummy (ClearArray (Binary cnt base) zero));
effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL val, KILL cr);
format %{ $$template
$$emit$$"xorq rax, rax\t# ClearArray:\n\t"
$$emit$$"cmp InitArrayShortSize,rcx\n\t"
$$emit$$"jg LARGE\n\t"
$$emit$$"dec rcx\n\t"
@ -14802,24 +14804,24 @@ instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
$$emit$$"vinserti128_high $tmp, $tmp\n\t"
$$emit$$"jmpq L_zero_64_bytes\n\t"
$$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
$$emit$$"vmovdqu $tmp,(rax)\n\t"
$$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
$$emit$$"add 0x40,rax\n\t"
$$emit$$"vmovdqu $tmp,($base)\n\t"
$$emit$$"vmovdqu $tmp,0x20($base)\n\t"
$$emit$$"add 0x40,$base\n\t"
$$emit$$"# L_zero_64_bytes:\n\t"
$$emit$$"sub 0x8,rcx\n\t"
$$emit$$"jge L_loop\n\t"
$$emit$$"add 0x4,rcx\n\t"
$$emit$$"jl L_tail\n\t"
$$emit$$"vmovdqu $tmp,(rax)\n\t"
$$emit$$"add 0x20,rax\n\t"
$$emit$$"vmovdqu $tmp,($base)\n\t"
$$emit$$"add 0x20,$base\n\t"
$$emit$$"sub 0x4,rcx\n\t"
$$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
$$emit$$"add 0x4,rcx\n\t"
$$emit$$"jle L_end\n\t"
$$emit$$"dec rcx\n\t"
$$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
$$emit$$"vmovq xmm0,(rax)\n\t"
$$emit$$"add 0x8,rax\n\t"
$$emit$$"vmovq $tmp,($base)\n\t"
$$emit$$"add 0x8,$base\n\t"
$$emit$$"dec rcx\n\t"
$$emit$$"jge L_sloop\n\t"
$$emit$$"# L_end:\n\t"
@ -14835,10 +14837,13 @@ instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
ins_pipe(pipe_slow);
%}
instruct rep_stos_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
// Small word fill for non-AVX512 targets.
instruct rep_stos_word_fill(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
Universe dummy, rFlagsReg cr)
%{
predicate(!((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
predicate((UseAVX <= 2) &&
!((ClearArrayNode*)n)->is_large() &&
((ClearArrayNode*)n)->requires_word_fill());
match(Set dummy (ClearArray (Binary cnt base) val));
effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
@ -14856,26 +14861,26 @@ instruct rep_stos_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
$$emit$$"movdq $tmp, $val\n\t"
$$emit$$"punpcklqdq $tmp, $tmp\n\t"
$$emit$$"vinserti128_high $tmp, $tmp\n\t"
$$emit$$"jmpq L_zero_64_bytes\n\t"
$$emit$$"jmpq L_fill_64_bytes\n\t"
$$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
$$emit$$"vmovdqu $tmp,(rax)\n\t"
$$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
$$emit$$"add 0x40,rax\n\t"
$$emit$$"# L_zero_64_bytes:\n\t"
$$emit$$"vmovdqu $tmp,($base)\n\t"
$$emit$$"vmovdqu $tmp,0x20($base)\n\t"
$$emit$$"add 0x40,$base\n\t"
$$emit$$"# L_fill_64_bytes:\n\t"
$$emit$$"sub 0x8,rcx\n\t"
$$emit$$"jge L_loop\n\t"
$$emit$$"add 0x4,rcx\n\t"
$$emit$$"jl L_tail\n\t"
$$emit$$"vmovdqu $tmp,(rax)\n\t"
$$emit$$"add 0x20,rax\n\t"
$$emit$$"vmovdqu $tmp,($base)\n\t"
$$emit$$"add 0x20,$base\n\t"
$$emit$$"sub 0x4,rcx\n\t"
$$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
$$emit$$"# L_tail:\t# Filling tail bytes\n\t"
$$emit$$"add 0x4,rcx\n\t"
$$emit$$"jle L_end\n\t"
$$emit$$"dec rcx\n\t"
$$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
$$emit$$"vmovq xmm0,(rax)\n\t"
$$emit$$"add 0x8,rax\n\t"
$$emit$$"vmovq $tmp,($base)\n\t"
$$emit$$"add 0x8,$base\n\t"
$$emit$$"dec rcx\n\t"
$$emit$$"jge L_sloop\n\t"
$$emit$$"# L_end:\n\t"
@ -14891,14 +14896,16 @@ instruct rep_stos_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
ins_pipe(pipe_slow);
%}
// Small non-constant length ClearArray for AVX512 targets.
instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
// Small zero fill for AVX512 targets.
instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, immL0 zero, rax_RegL val,
Universe dummy, rFlagsReg cr)
%{
predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
match(Set dummy (ClearArray (Binary cnt base) val));
predicate((UseAVX > 2) &&
!((ClearArrayNode*)n)->is_large() &&
((ClearArrayNode*)n)->is_zero_fill());
match(Set dummy (ClearArray (Binary cnt base) zero));
ins_cost(125);
effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL val, KILL cr);
format %{ $$template
$$emit$$"xorq rax, rax\t# ClearArray:\n\t"
@ -14915,28 +14922,29 @@ instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_
$$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
$$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--\n\t"
} else if (UseXMMForObjInit) {
$$emit$$"mov rdi,rax\n\t"
$$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
$$emit$$"movdq $tmp, $val\n\t"
$$emit$$"punpcklqdq $tmp, $tmp\n\t"
$$emit$$"vinserti128_high $tmp, $tmp\n\t"
$$emit$$"jmpq L_zero_64_bytes\n\t"
$$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
$$emit$$"vmovdqu ymm0,(rax)\n\t"
$$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
$$emit$$"add 0x40,rax\n\t"
$$emit$$"vmovdqu $tmp,($base)\n\t"
$$emit$$"vmovdqu $tmp,0x20($base)\n\t"
$$emit$$"add 0x40,$base\n\t"
$$emit$$"# L_zero_64_bytes:\n\t"
$$emit$$"sub 0x8,rcx\n\t"
$$emit$$"jge L_loop\n\t"
$$emit$$"add 0x4,rcx\n\t"
$$emit$$"jl L_tail\n\t"
$$emit$$"vmovdqu ymm0,(rax)\n\t"
$$emit$$"add 0x20,rax\n\t"
$$emit$$"vmovdqu $tmp,($base)\n\t"
$$emit$$"add 0x20,$base\n\t"
$$emit$$"sub 0x4,rcx\n\t"
$$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
$$emit$$"add 0x4,rcx\n\t"
$$emit$$"jle L_end\n\t"
$$emit$$"dec rcx\n\t"
$$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
$$emit$$"vmovq xmm0,(rax)\n\t"
$$emit$$"add 0x8,rax\n\t"
$$emit$$"vmovq $tmp,($base)\n\t"
$$emit$$"add 0x8,$base\n\t"
$$emit$$"dec rcx\n\t"
$$emit$$"jge L_sloop\n\t"
$$emit$$"# L_end:\n\t"
@ -14952,16 +14960,18 @@ instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_
ins_pipe(pipe_slow);
%}
instruct rep_stos_evex_word_copy(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
// Small word fill for AVX512 targets.
instruct rep_stos_evex_word_fill(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
Universe dummy, rFlagsReg cr)
%{
predicate(!((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
predicate((UseAVX > 2) &&
!((ClearArrayNode*)n)->is_large() &&
((ClearArrayNode*)n)->requires_word_fill());
match(Set dummy (ClearArray (Binary cnt base) val));
ins_cost(125);
effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
format %{ $$template
$$emit$$"xorq rax, rax\t# ClearArray:\n\t"
$$emit$$"cmp InitArrayShortSize,rcx\n\t"
$$emit$$"jg LARGE\n\t"
$$emit$$"dec rcx\n\t"
@ -14971,32 +14981,30 @@ instruct rep_stos_evex_word_copy(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg
$$emit$$"jge LOOP\n\t"
$$emit$$"jmp DONE\n\t"
$$emit$$"# LARGE:\n\t"
if (UseFastStosb) {
$$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
$$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--\n\t"
} else if (UseXMMForObjInit) {
$$emit$$"mov rdi,rax\n\t"
$$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
$$emit$$"jmpq L_zero_64_bytes\n\t"
if (UseXMMForObjInit) {
$$emit$$"movdq $tmp, $val\n\t"
$$emit$$"punpcklqdq $tmp, $tmp\n\t"
$$emit$$"vinserti128_high $tmp, $tmp\n\t"
$$emit$$"jmpq L_fill_64_bytes\n\t"
$$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
$$emit$$"vmovdqu ymm0,(rax)\n\t"
$$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
$$emit$$"add 0x40,rax\n\t"
$$emit$$"# L_zero_64_bytes:\n\t"
$$emit$$"vmovdqu $tmp,($base)\n\t"
$$emit$$"vmovdqu $tmp,0x20($base)\n\t"
$$emit$$"add 0x40,$base\n\t"
$$emit$$"# L_fill_64_bytes:\n\t"
$$emit$$"sub 0x8,rcx\n\t"
$$emit$$"jge L_loop\n\t"
$$emit$$"add 0x4,rcx\n\t"
$$emit$$"jl L_tail\n\t"
$$emit$$"vmovdqu ymm0,(rax)\n\t"
$$emit$$"add 0x20,rax\n\t"
$$emit$$"vmovdqu $tmp,($base)\n\t"
$$emit$$"add 0x20,$base\n\t"
$$emit$$"sub 0x4,rcx\n\t"
$$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
$$emit$$"# L_tail:\t# Filling tail bytes\n\t"
$$emit$$"add 0x4,rcx\n\t"
$$emit$$"jle L_end\n\t"
$$emit$$"dec rcx\n\t"
$$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
$$emit$$"vmovq xmm0,(rax)\n\t"
$$emit$$"add 0x8,rax\n\t"
$$emit$$"vmovq $tmp,($base)\n\t"
$$emit$$"add 0x8,$base\n\t"
$$emit$$"dec rcx\n\t"
$$emit$$"jge L_sloop\n\t"
$$emit$$"# L_end:\n\t"
@ -15012,15 +15020,18 @@ instruct rep_stos_evex_word_copy(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg
ins_pipe(pipe_slow);
%}
// Large non-constant length ClearArray for non-AVX512 targets.
instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
// Large zero fill for non-AVX512 targets.
instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, immL0 zero, rax_RegL val,
Universe dummy, rFlagsReg cr)
%{
predicate(((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
match(Set dummy (ClearArray (Binary cnt base) val));
effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
predicate((UseAVX <= 2) &&
((ClearArrayNode*)n)->is_large() &&
((ClearArrayNode*)n)->is_zero_fill());
match(Set dummy (ClearArray (Binary cnt base) zero));
effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL val, KILL cr);
format %{ $$template
$$emit$$"xorq rax, rax\t# ClearArray:\n\t"
if (UseFastStosb) {
$$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
$$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--"
@ -15030,24 +15041,24 @@ instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
$$emit$$"vinserti128_high $tmp, $tmp\n\t"
$$emit$$"jmpq L_zero_64_bytes\n\t"
$$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
$$emit$$"vmovdqu $tmp,(rax)\n\t"
$$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
$$emit$$"add 0x40,rax\n\t"
$$emit$$"vmovdqu $tmp,($base)\n\t"
$$emit$$"vmovdqu $tmp,0x20($base)\n\t"
$$emit$$"add 0x40,$base\n\t"
$$emit$$"# L_zero_64_bytes:\n\t"
$$emit$$"sub 0x8,rcx\n\t"
$$emit$$"jge L_loop\n\t"
$$emit$$"add 0x4,rcx\n\t"
$$emit$$"jl L_tail\n\t"
$$emit$$"vmovdqu $tmp,(rax)\n\t"
$$emit$$"add 0x20,rax\n\t"
$$emit$$"vmovdqu $tmp,($base)\n\t"
$$emit$$"add 0x20,$base\n\t"
$$emit$$"sub 0x4,rcx\n\t"
$$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
$$emit$$"add 0x4,rcx\n\t"
$$emit$$"jle L_end\n\t"
$$emit$$"dec rcx\n\t"
$$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
$$emit$$"vmovq xmm0,(rax)\n\t"
$$emit$$"add 0x8,rax\n\t"
$$emit$$"vmovq $tmp,($base)\n\t"
$$emit$$"add 0x8,$base\n\t"
$$emit$$"dec rcx\n\t"
$$emit$$"jge L_sloop\n\t"
$$emit$$"# L_end:\n\t"
@ -15062,10 +15073,13 @@ instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
ins_pipe(pipe_slow);
%}
instruct rep_stos_large_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
// Large word fill for non-AVX512 targets.
instruct rep_stos_large_word_fill(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
Universe dummy, rFlagsReg cr)
%{
predicate(((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
predicate((UseAVX <= 2) &&
((ClearArrayNode*)n)->is_large() &&
((ClearArrayNode*)n)->requires_word_fill());
match(Set dummy (ClearArray (Binary cnt base) val));
effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
@ -15074,26 +15088,26 @@ instruct rep_stos_large_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_Reg
$$emit$$"movdq $tmp, $val\n\t"
$$emit$$"punpcklqdq $tmp, $tmp\n\t"
$$emit$$"vinserti128_high $tmp, $tmp\n\t"
$$emit$$"jmpq L_zero_64_bytes\n\t"
$$emit$$"jmpq L_fill_64_bytes\n\t"
$$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
$$emit$$"vmovdqu $tmp,(rax)\n\t"
$$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
$$emit$$"add 0x40,rax\n\t"
$$emit$$"# L_zero_64_bytes:\n\t"
$$emit$$"vmovdqu $tmp,($base)\n\t"
$$emit$$"vmovdqu $tmp,0x20($base)\n\t"
$$emit$$"add 0x40,$base\n\t"
$$emit$$"# L_fill_64_bytes:\n\t"
$$emit$$"sub 0x8,rcx\n\t"
$$emit$$"jge L_loop\n\t"
$$emit$$"add 0x4,rcx\n\t"
$$emit$$"jl L_tail\n\t"
$$emit$$"vmovdqu $tmp,(rax)\n\t"
$$emit$$"add 0x20,rax\n\t"
$$emit$$"vmovdqu $tmp,($base)\n\t"
$$emit$$"add 0x20,$base\n\t"
$$emit$$"sub 0x4,rcx\n\t"
$$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
$$emit$$"# L_tail:\t# Filling tail bytes\n\t"
$$emit$$"add 0x4,rcx\n\t"
$$emit$$"jle L_end\n\t"
$$emit$$"dec rcx\n\t"
$$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
$$emit$$"vmovq xmm0,(rax)\n\t"
$$emit$$"add 0x8,rax\n\t"
$$emit$$"vmovq $tmp,($base)\n\t"
$$emit$$"add 0x8,$base\n\t"
$$emit$$"dec rcx\n\t"
$$emit$$"jge L_sloop\n\t"
$$emit$$"# L_end:\n\t"
@ -15108,47 +15122,49 @@ instruct rep_stos_large_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_Reg
ins_pipe(pipe_slow);
%}
// Large non-constant length ClearArray for AVX512 targets.
instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
// Large zero fill for AVX512 targets.
instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, immL0 zero, rax_RegL val,
Universe dummy, rFlagsReg cr)
%{
predicate(((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
match(Set dummy (ClearArray (Binary cnt base) val));
effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
predicate((UseAVX > 2) &&
((ClearArrayNode*)n)->is_large() &&
((ClearArrayNode*)n)->is_zero_fill());
match(Set dummy (ClearArray (Binary cnt base) zero));
effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL val, KILL cr);
format %{ $$template
$$emit$$"xorq $val, $val\t# ClearArray:\n\t"
if (UseFastStosb) {
$$emit$$"xorq rax, rax\t# ClearArray:\n\t"
$$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
$$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--"
} else if (UseXMMForObjInit) {
$$emit$$"mov rdi,rax\t# ClearArray:\n\t"
$$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
$$emit$$"movdq $tmp, $val\n\t"
$$emit$$"punpcklqdq $tmp, $tmp\n\t"
$$emit$$"vinserti128_high $tmp, $tmp\n\t"
$$emit$$"jmpq L_zero_64_bytes\n\t"
$$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
$$emit$$"vmovdqu ymm0,(rax)\n\t"
$$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
$$emit$$"add 0x40,rax\n\t"
$$emit$$"vmovdqu $tmp,($base)\n\t"
$$emit$$"vmovdqu $tmp,0x20($base)\n\t"
$$emit$$"add 0x40,$base\n\t"
$$emit$$"# L_zero_64_bytes:\n\t"
$$emit$$"sub 0x8,rcx\n\t"
$$emit$$"jge L_loop\n\t"
$$emit$$"add 0x4,rcx\n\t"
$$emit$$"jl L_tail\n\t"
$$emit$$"vmovdqu ymm0,(rax)\n\t"
$$emit$$"add 0x20,rax\n\t"
$$emit$$"vmovdqu $tmp,($base)\n\t"
$$emit$$"add 0x20,$base\n\t"
$$emit$$"sub 0x4,rcx\n\t"
$$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
$$emit$$"add 0x4,rcx\n\t"
$$emit$$"jle L_end\n\t"
$$emit$$"dec rcx\n\t"
$$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
$$emit$$"vmovq xmm0,(rax)\n\t"
$$emit$$"add 0x8,rax\n\t"
$$emit$$"vmovq $tmp,($base)\n\t"
$$emit$$"add 0x8,$base\n\t"
$$emit$$"dec rcx\n\t"
$$emit$$"jge L_sloop\n\t"
$$emit$$"# L_end:\n\t"
} else {
$$emit$$"xorq rax, rax\t# ClearArray:\n\t"
$$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--"
}
%}
@ -15159,46 +15175,45 @@ instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp
ins_pipe(pipe_slow);
%}
instruct rep_stos_large_evex_word_copy(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
// Large word fill for AVX512 targets.
instruct rep_stos_large_evex_word_fill(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
Universe dummy, rFlagsReg cr)
%{
predicate(((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
predicate((UseAVX > 2) &&
((ClearArrayNode*)n)->is_large() &&
((ClearArrayNode*)n)->requires_word_fill());
match(Set dummy (ClearArray (Binary cnt base) val));
effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
format %{ $$template
if (UseFastStosb) {
$$emit$$"xorq rax, rax\t# ClearArray:\n\t"
$$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
$$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--"
} else if (UseXMMForObjInit) {
$$emit$$"mov rdi,rax\t# ClearArray:\n\t"
$$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
$$emit$$"jmpq L_zero_64_bytes\n\t"
if (UseXMMForObjInit) {
$$emit$$"movdq $tmp, $val\t# ClearArray:\n\t"
$$emit$$"punpcklqdq $tmp, $tmp\n\t"
$$emit$$"vinserti128_high $tmp, $tmp\n\t"
$$emit$$"jmpq L_fill_64_bytes\n\t"
$$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
$$emit$$"vmovdqu ymm0,(rax)\n\t"
$$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
$$emit$$"add 0x40,rax\n\t"
$$emit$$"# L_zero_64_bytes:\n\t"
$$emit$$"vmovdqu $tmp,($base)\n\t"
$$emit$$"vmovdqu $tmp,0x20($base)\n\t"
$$emit$$"add 0x40,$base\n\t"
$$emit$$"# L_fill_64_bytes:\n\t"
$$emit$$"sub 0x8,rcx\n\t"
$$emit$$"jge L_loop\n\t"
$$emit$$"add 0x4,rcx\n\t"
$$emit$$"jl L_tail\n\t"
$$emit$$"vmovdqu ymm0,(rax)\n\t"
$$emit$$"add 0x20,rax\n\t"
$$emit$$"vmovdqu $tmp,($base)\n\t"
$$emit$$"add 0x20,$base\n\t"
$$emit$$"sub 0x4,rcx\n\t"
$$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
$$emit$$"# L_tail:\t# Filling tail bytes\n\t"
$$emit$$"add 0x4,rcx\n\t"
$$emit$$"jle L_end\n\t"
$$emit$$"dec rcx\n\t"
$$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
$$emit$$"vmovq xmm0,(rax)\n\t"
$$emit$$"add 0x8,rax\n\t"
$$emit$$"vmovq $tmp,($base)\n\t"
$$emit$$"add 0x8,$base\n\t"
$$emit$$"dec rcx\n\t"
$$emit$$"jge L_sloop\n\t"
$$emit$$"# L_end:\n\t"
} else {
$$emit$$"xorq rax, rax\t# ClearArray:\n\t"
$$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--"
}
%}
@ -15209,14 +15224,16 @@ instruct rep_stos_large_evex_word_copy(rcx_RegL cnt, rdi_RegP base, legRegD tmp,
ins_pipe(pipe_slow);
%}
// Small constant length ClearArray for AVX512 targets.
instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rax_RegL val, kReg ktmp, Universe dummy, rFlagsReg cr)
// Small constant-count zero fill for AVX512 targets.
instruct rep_stos_im(immL cnt, rRegP base, regD tmp, immL0 zero, rRegI val, kReg ktmp, Universe dummy, rFlagsReg cr)
%{
predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() &&
((MaxVectorSize >= 32) && VM_Version::supports_avx512vl()));
match(Set dummy (ClearArray (Binary cnt base) val));
predicate((MaxVectorSize >= 32) &&
VM_Version::supports_avx512vl() &&
!((ClearArrayNode*)n)->is_large() &&
((ClearArrayNode*)n)->is_zero_fill());
match(Set dummy (ClearArray (Binary cnt base) zero));
ins_cost(100);
effect(TEMP tmp, USE_KILL val, TEMP ktmp, KILL cr);
effect(TEMP tmp, TEMP val, TEMP ktmp, KILL cr);
format %{ "clear_mem_imm $base , $cnt \n\t" %}
ins_encode %{
__ clear_mem($base$$Register, $cnt$$constant, $val$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);

37
src/hotspot/share/c1/c1_GraphBuilder.cpp
View File

@ -1092,12 +1092,15 @@ void GraphBuilder::load_indexed(BasicType type) {
bool is_null_free = array_klass->is_elem_null_free();
bool will_link;
ciField* next_field = s.get_field(will_link);
bool next_needs_patching = !next_field->holder()->is_initialized() ||
ciInstanceKlass* next_holder = next_field->holder();
bool next_needs_patching = !next_holder->is_initialized() ||
!next_field->will_link(method(), Bytecodes::_getfield) ||
PatchALot;
bool needs_atomic_access = array_klass->is_elem_atomic();
// Offset adjustment for delayed reads requires a concrete inline holder
bool next_holder_is_inlinetype = next_holder->is_inlinetype();
can_delay_access = is_null_free && C1UseDelayedFlattenedFieldReads &&
!next_needs_patching && !needs_atomic_access;
!next_needs_patching && !needs_atomic_access && next_holder_is_inlinetype;
}
if (can_delay_access) {
// potentially optimizable array access, storing information for delayed decision
@ -1107,16 +1110,20 @@ void GraphBuilder::load_indexed(BasicType type) {
set_pending_load_indexed(dli);
return; // Nothing else to do for now
} else {
NewInstance* buffer = new NewInstance(elem_klass, state_before, false, true);
buffer->set_null_free(true);
_memory->new_instance(buffer);
result = append_split(buffer);
load_indexed = new LoadIndexed(array, index, length, type, state_before);
load_indexed->set_buffer(buffer);
// The LoadIndexed node will initialize this instance by copying from
// the flat field. Ensure these stores are visible before any
// subsequent store that publishes this reference.
need_membar = true;
// Deoptimize on non-null because buffering requires the value class to be initialized
bool assert_null = !array_klass->is_elem_null_free() && !elem_klass->is_initialized();
if (!assert_null) {
NewInstance* buffer = new NewInstance(elem_klass, state_before, false, true);
buffer->set_null_free(true);
_memory->new_instance(buffer);
result = append_split(buffer);
load_indexed->set_buffer(buffer);
// The LoadIndexed node will initialize this instance by copying from
// the flat field. Ensure these stores are visible before any
// subsequent store that publishes this reference.
need_membar = true;
}
}
} else {
load_indexed = new LoadIndexed(array, index, length, type, state_before);
@ -1998,12 +2005,16 @@ void GraphBuilder::access_field(Bytecodes::Code code) {
s.next();
if (s.cur_bc() == Bytecodes::_getfield && !needs_patching) {
ciField* next_field = s.get_field(will_link);
bool next_needs_patching = !next_field->holder()->is_loaded() ||
ciInstanceKlass* next_holder = next_field->holder();
bool next_needs_patching = !next_holder->is_loaded() ||
!next_field->will_link(method(), Bytecodes::_getfield) ||
PatchALot;
// We can't update the offset for atomic accesses
bool next_needs_atomic_access = next_field->is_flat() && next_field->is_atomic();
can_delay_access = C1UseDelayedFlattenedFieldReads && !next_needs_patching && !next_needs_atomic_access && next_field->is_null_free();
// Offset adjustment for delayed reads requires a concrete inline holder
bool next_holder_is_inlinetype = next_holder->is_inlinetype();
can_delay_access = C1UseDelayedFlattenedFieldReads && !next_needs_patching && !next_needs_atomic_access &&
next_field->is_null_free() && next_holder_is_inlinetype;
}
}

83
src/hotspot/share/c1/c1_LIRGenerator.cpp
View File

@ -772,9 +772,9 @@ void LIRGenerator::arraycopy_helper(Intrinsic* x, int* flagsp, ciArrayKlass** ex
if (expected_type == nullptr) expected_type = src_declared_type;
if (expected_type == nullptr) expected_type = dst_declared_type;
if (expected_type != nullptr && expected_type->is_obj_array_klass()) {
if (expected_type != nullptr && expected_type->is_obj_array_klass() && !expected_type->is_refined()) {
// For a direct pointer comparison, we need the refined array klass pointer
expected_type = ciObjArrayKlass::make(expected_type->as_array_klass()->element_klass());
expected_type = ciObjArrayKlass::make(expected_type->as_array_klass()->element_klass(), true /* refined_type */);
}
src_objarray = (src_exact_type && src_exact_type->is_obj_array_klass()) || (src_declared_type && src_declared_type->is_obj_array_klass());
@ -1489,7 +1489,20 @@ LIR_Opr LIRGenerator::load_constant(Constant* x) {
LIR_Opr LIRGenerator::load_constant(LIR_Const* c) {
BasicType t = c->type();
for (int i = 0; i < _constants.length() && !in_conditional_code(); i++) {
if (in_conditional_code()) {
// TODO 8353851: Control flow introduced by check_flat_array() is currently opaque to the register allocator.
// Do not use or update the constant -> register cache in such conditional code because the register allocator could
// spill a constant and only rematerialize it into a register in one branch of check_flat_array() but not the other.
// Since the control flow is opaque to the register allocator, it assumes the rematerialized constant in the register
// dominates all subsequent uses in the block and does not insert another rematerialization. When taking the
// non-rematerialized branch of check_flat_array() at runtime, the register contains garbage potentially causing
// a crash.
LIR_Opr result = new_register(t);
__ move(c, result);
return result;
}
for (int i = 0; i < _constants.length(); i++) {
LIR_Const* other = _constants.at(i);
if (t == other->type()) {
switch (t) {
@ -1513,11 +1526,9 @@ LIR_Opr LIRGenerator::load_constant(LIR_Const* c) {
}
LIR_Opr result = new_register(t);
__ move((LIR_Opr)c, result);
if (!in_conditional_code()) {
_constants.append(c);
_reg_for_constants.append(result);
}
__ move(c, result);
_constants.append(c);
_reg_for_constants.append(result);
return result;
}
@ -2156,21 +2167,41 @@ void LIRGenerator::do_LoadField(LoadField* x) {
ciInlineKlass* vk = field->type()->as_inline_klass();
#ifdef ASSERT
assert(field->is_atomic(), "No atomic access required");
assert(!is_volatile, "Flat fields cannot be volatile");
assert(x->state_before() != nullptr, "Needs state before");
#endif
// Allocate buffer (we can't easily do this conditionally on the null check below
// because branches added in the LIR are opaque to the register allocator).
NewInstance* buffer = new NewInstance(vk, x->state_before(), false, true);
do_NewInstance(buffer);
LIRItem dest(buffer, this);
NewInstance* buffer = nullptr;
bool assert_null = !field->is_null_free() && !vk->is_initialized();
if (!assert_null) {
// Allocate the buffer before loading the payload because allocation may safepoint
// and a payload may contain oops represented as raw bits and thus invisible to the GC.
// We can't easily allocate conditionally on the null check below because branches
// added in the LIR are opaque to the register allocator.
buffer = new NewInstance(vk, x->state_before(), false, true);
do_NewInstance(buffer);
}
// Copy the payload to the buffer
BasicType bt = vk->atomic_size_to_basic_type(field->is_null_free());
LIR_Opr payload = new_register((bt == T_LONG) ? bt : T_INT);
access_load_at(decorators, bt, object, LIR_OprFact::intConst(field->offset_in_bytes()), payload,
// Make sure to emit an implicit null check
info ? new CodeEmitInfo(info) : nullptr, info);
if (assert_null) {
// Deoptimize on non-null because buffering requires the value class to be initialized
CodeEmitInfo* null_assert_info = state_for(x, x->state_before());
__ logical_and(payload, null_marker_mask(bt, field), payload);
__ cmp(lir_cond_notEqual, payload, (bt == T_LONG) ? LIR_OprFact::longConst(0) : LIR_OprFact::intConst(0));
__ branch(lir_cond_notEqual, new DeoptimizeStub(null_assert_info, Deoptimization::Reason_null_assert,
Deoptimization::Action_make_not_entrant));
__ move(LIR_OprFact::oopConst(nullptr), rlock_result(x));
return;
}
// Copy the payload to the buffer
assert(buffer != nullptr, "buffer required");
LIRItem dest(buffer, this);
access_store_at(decorators, bt, dest, LIR_OprFact::intConst(vk->payload_offset()), payload);
if (field->is_null_free()) {
@ -2353,6 +2384,30 @@ void LIRGenerator::do_LoadIndexed(LoadIndexed* x) {
}
}
ciFlatArrayKlass* flat_array_klass = x->array()->is_loaded_flat_array() ?
x->array()->declared_type()->as_flat_array_klass() : nullptr;
bool assert_null = flat_array_klass != nullptr && !flat_array_klass->is_elem_null_free() &&
!flat_array_klass->element_klass()->as_inline_klass()->is_initialized();
if (assert_null) {
// Deoptimize on non-null because buffering requires the value class to be initialized
assert(x->buffer() == nullptr && x->delayed() == nullptr, "null assertion should not buffer");
assert(flat_array_klass->is_elem_atomic(), "nullable flat arrays must use an atomic layout");
ciInlineKlass* elem_klass = flat_array_klass->element_klass()->as_inline_klass();
CodeEmitInfo* null_assert_info = state_for(x, x->state_before());
BasicType bt = elem_klass->atomic_size_to_basic_type(false);
LIR_Opr elm_op = get_and_load_element_address(array, index);
ComputedAddressValue* elm_resolved_addr = new ComputedAddressValue(as_ValueType(bt), elm_op);
LIRItem elm_item(elm_resolved_addr, this);
LIR_Opr payload = new_register((bt == T_LONG) ? bt : T_INT);
access_load_at(IN_HEAP, bt, elm_item, LIR_OprFact::intConst(0), payload, nullptr, nullptr);
__ logical_and(payload, null_marker_mask(bt, elem_klass->null_marker_offset_in_payload()), payload);
__ cmp(lir_cond_notEqual, payload, (bt == T_LONG) ? LIR_OprFact::longConst(0) : LIR_OprFact::intConst(0));
__ branch(lir_cond_notEqual, new DeoptimizeStub(null_assert_info, Deoptimization::Reason_null_assert,
Deoptimization::Action_make_not_entrant));
__ move(LIR_OprFact::oopConst(nullptr), rlock_result(x));
return;
}
Value element = nullptr;
if (x->buffer() != nullptr) {
assert(x->array()->is_loaded_flat_array(), "must be");

10
src/hotspot/share/c1/c1_Runtime1.cpp
View File

@ -383,10 +383,10 @@ const char* Runtime1::name_for_address(address entry) {
return pd_name_for_address(entry);
}
static void allocate_instance(JavaThread* current, Klass* klass, TRAPS) {
JRT_ENTRY(void, Runtime1::new_instance(JavaThread* current, Klass* klass))
#ifndef PRODUCT
if (PrintC1Statistics) {
Runtime1::_new_instance_slowcase_cnt++;
_new_instance_slowcase_cnt++;
}
#endif
assert(klass->is_klass(), "not a class");
@ -400,10 +400,6 @@ static void allocate_instance(JavaThread* current, Klass* klass, TRAPS) {
current->set_vm_result_oop(obj);
JRT_END
JRT_ENTRY(void, Runtime1::new_instance(JavaThread* current, Klass* klass))
allocate_instance(current, klass, CHECK);
JRT_END
JRT_ENTRY(void, Runtime1::new_type_array(JavaThread* current, Klass* klass, jint length))
#ifndef PRODUCT
if (PrintC1Statistics) {
@ -1188,7 +1184,7 @@ JRT_ENTRY(void, Runtime1::patch_code(JavaThread* current, StubId stub_id ))
{ Bytecode_anewarray anew(caller_method(), caller_method->bcp_from(bci));
Klass* ek = caller_method->constants()->klass_at(anew.index(), CHECK);
k = ek->array_klass(CHECK);
if (!k->is_typeArray_klass() && !k->is_refArray_klass() && !k->is_flatArray_klass()) {
if (k->is_unrefined_objArray_klass()) {
k = ObjArrayKlass::cast(k)->klass_with_properties(ArrayProperties::Default(), THREAD);
}
if (k->is_flatArray_klass()) {

17
src/hotspot/share/cds/aotMetaspace.cpp
View File

@ -1221,8 +1221,8 @@ void AOTMetaspace::dump_static_archive_impl(StaticArchiveBuilder& builder, TRAPS
assert(!_output_mapinfo->is_open(), "Must be closed already");
_output_mapinfo = nullptr;
if (status && CDSConfig::is_dumping_preimage_static_archive()) {
tty->print_cr("%s AOTConfiguration recorded: %s",
CDSConfig::has_temp_aot_config_file() ? "Temporary" : "", AOTConfiguration);
tty->print_cr("%sAOTConfiguration recorded: %s",
CDSConfig::has_temp_aot_config_file() ? "Temporary " : "", AOTConfiguration);
if (CDSConfig::is_single_command_training()) {
fork_and_dump_final_static_archive(CHECK);
}
@ -1359,8 +1359,19 @@ void AOTMetaspace::fork_and_dump_final_static_archive(TRAPS) {
tty->print_cr("Launching child process %s to assemble AOT cache %s using configuration %s", cmd, AOTCacheOutput, AOTConfiguration);
int status = exec_jvm_with_java_tool_options(cmd, CHECK);
if (status != 0) {
// We do this in all cases when the child process is launched because:
// - the AOT training process is about to exit; or
// - jcmd or AOTCacheMXBean is used to end AOT training.
//
// The child process is just a convenient way to get a fresh JVM state to
// assemble the AOT cache. Logically, we consider the AOT assembly to be
// executed as part of the current JVM. If the child process has failed,
// we should exit the current JVM as well.
//
// To help debugging, if we have created a temporary AOT config file, do not
// delete it.
log_error(aot)("Child process failed; status = %d", status);
// We leave the temp config file for debugging
vm_exit(status);
} else if (CDSConfig::has_temp_aot_config_file()) {
const char* tmp_config = AOTConfiguration;
// On Windows, need WRITE permission to remove the file.

27
src/hotspot/share/ci/bcEscapeAnalyzer.cpp
View File

@ -189,14 +189,25 @@ void BCEscapeAnalyzer::set_global_escape(ArgumentMap vars, bool merge) {
}
void BCEscapeAnalyzer::set_modified(ArgumentMap vars, int offs, int size) {
for (int i = 0; i < _arg_size; i++) {
if (vars.contains(i)) {
set_arg_modified(i, offs, size);
}
}
if (vars.contains_unknown())
if (vars.contains_unknown()) {
_unknown_modified = true;
}
}
void BCEscapeAnalyzer::set_modified_any_offset(ArgumentMap vars) {
for (int i = 0; i < _arg_size; i++) {
if (vars.contains(i)) {
_arg_modified[i] = (uint)-1;
}
}
if (vars.contains_unknown()) {
_unknown_modified = true;
}
}
bool BCEscapeAnalyzer::is_recursive_call(ciMethod* callee) {
@ -227,7 +238,7 @@ bool BCEscapeAnalyzer::is_arg_modified(int arg, int offset, int size_in_bytes) {
void BCEscapeAnalyzer::set_arg_modified(int arg, int offset, int size_in_bytes) {
if (offset == OFFSET_ANY) {
_arg_modified[arg] = (uint) -1;
_arg_modified[arg] = (uint)-1;
return;
}
assert(arg >= 0 && arg < _arg_size, "must be an argument.");
@ -537,7 +548,7 @@ void BCEscapeAnalyzer::iterate_one_block(ciBlock *blk, StateInfo &state, Growabl
state.spop();
ArgumentMap arr = state.apop();
set_method_escape(arr);
set_modified(arr, OFFSET_ANY, type2size[T_INT]*HeapWordSize);
set_modified_any_offset(arr);
break;
}
case Bytecodes::_lastore:
@ -547,7 +558,7 @@ void BCEscapeAnalyzer::iterate_one_block(ciBlock *blk, StateInfo &state, Growabl
state.spop();
ArgumentMap arr = state.apop();
set_method_escape(arr);
set_modified(arr, OFFSET_ANY, type2size[T_LONG]*HeapWordSize);
set_modified_any_offset(arr);
break;
}
case Bytecodes::_aastore:
@ -555,10 +566,8 @@ void BCEscapeAnalyzer::iterate_one_block(ciBlock *blk, StateInfo &state, Growabl
set_global_escape(state.apop());
state.spop();
ArgumentMap arr = state.apop();
// If the array is a flat array, a larger part of it is modified than
// the size of a reference. However, if OFFSET_ANY is given as
// parameter to set_modified(), size is not taken into account.
set_modified(arr, OFFSET_ANY, type2size[T_OBJECT]*HeapWordSize);
// If the array is a flat array, a larger part of it is modified than the size of a reference.
set_modified_any_offset(arr);
break;
}
case Bytecodes::_pop:

1
src/hotspot/share/ci/bcEscapeAnalyzer.hpp
View File

@ -84,6 +84,7 @@ class BCEscapeAnalyzer : public ArenaObj {
void set_method_escape(ArgumentMap vars);
void set_global_escape(ArgumentMap vars, bool merge = false);
void set_modified(ArgumentMap vars, int offs, int size);
void set_modified_any_offset(ArgumentMap vars);
bool is_recursive_call(ciMethod* callee);
void invoke(StateInfo &state, Bytecodes::Code code, ciMethod* target, ciKlass* holder);

2
src/hotspot/share/ci/ciArrayKlass.cpp
View File

@ -60,7 +60,7 @@ ciType* ciArrayKlass::element_type() {
if (is_type_array_klass()) {
return ciType::make(as_type_array_klass()->element_type());
} else {
return element_klass()->as_klass();
return as_obj_array_klass()->element_klass()->as_klass();
}
}

2
src/hotspot/share/ci/ciClassList.hpp
View File

@ -117,7 +117,7 @@ friend class ciReplay; \
friend class ciTypeArray; \
friend class ciType; \
friend class ciReturnAddress; \
friend class ciWrapper; \
friend class ciWrapper; \
friend class ciKlass; \
friend class ciInstanceKlass; \
friend class ciInlineKlass; \

13
src/hotspot/share/ci/ciEnv.cpp
View File

@ -187,9 +187,9 @@ ciEnv::ciEnv(CompileTask* task)
// {
// RecordLocation fp(this, "field1");
// // location: "field1"
// { RecordLocation fp(this, " field2"); // location: "field1 field2" }
// { RecordLocation fp(this, "field2"); // location: "field1 field2" }
// // location: "field1"
// { RecordLocation fp(this, " field3"); // location: "field1 field3" }
// { RecordLocation fp(this, "field3"); // location: "field1 field3" }
// // location: "field1"
// }
// // location: ""
@ -225,10 +225,13 @@ public:
// append a new component
ATTRIBUTE_PRINTF(3, 4)
RecordLocation(ciEnv* ci, const char* fmt, ...) {
end = ci->_dyno_name + strlen(ci->_dyno_name);
size_t len = strlen(ci->_dyno_name);
end = ci->_dyno_name + len;
va_list args;
va_start(args, fmt);
push(ci, " ");
if (len > 0) {
push(ci, " ");
}
push_va(ci, fmt, args);
va_end(args);
}
@ -490,7 +493,7 @@ ciKlass* ciEnv::get_klass_by_name_impl(ciKlass* accessing_klass,
require_local);
if (elem_klass != nullptr && elem_klass->is_loaded()) {
// Now make an array for it
return ciArrayKlass::make(elem_klass);
return ciObjArrayKlass::make_impl(elem_klass);
}
}

1
src/hotspot/share/ci/ciFlatArray.hpp
View File

@ -31,6 +31,7 @@
// ciFlatArray
//
// This class represents a flatArrayOop in the HotSpot virtual machine.
// TODO 8388127: Sync ciArray class hierarchy with arrayOopDesc class hierarchy.
class ciFlatArray : public ciArray {
CI_PACKAGE_ACCESS

36
src/hotspot/share/ci/ciInstanceKlass.cpp
View File

@ -838,8 +838,8 @@ public:
StaticFieldPrinter(out), _obj(obj) {
}
void do_field(fieldDescriptor* fd) {
do_field_helper(fd, _obj, true);
_out->print(" ");
do_field_helper(fd, _obj, true);
}
};
@ -865,27 +865,48 @@ void StaticFieldPrinter::do_field_helper(fieldDescriptor* fd, oop mirror, bool i
case T_ARRAY: // fall-through
case T_OBJECT:
if (!fd->is_null_free_inline_type()) {
_out->print("%s ", fd->signature()->as_quoted_ascii());
_out->print("%s", fd->signature()->as_quoted_ascii());
oop value = mirror->obj_field_acquire(fd->offset());
if (value == nullptr) {
if (field_type == T_ARRAY) {
_out->print("%d", -1);
_out->print(" %d", -1);
}
_out->cr();
} else if (value->is_instance()) {
assert(field_type == T_OBJECT, "");
if (value->is_a(vmClasses::String_klass())) {
const char* ascii_value = java_lang_String::as_quoted_ascii(value);
_out->print("\"%s\"", (ascii_value != nullptr) ? ascii_value : "");
_out->print(" \"%s\"", (ascii_value != nullptr) ? ascii_value : "");
} else {
const char* klass_name = value->klass()->name()->as_quoted_ascii();
_out->print("%s", klass_name);
_out->print(" %s", klass_name);
}
} else if (value->is_array()) {
arrayOop a = (arrayOop)value;
_out->print("%d", a->length());
_out->print(" %d", a->length());
if (value->is_objArray()) {
objArrayOop oa = (objArrayOop)value;
if (value->is_flatArray()) {
FlatArrayKlass* klass = ((flatArrayOop)oa)->klass();
LayoutKind lk = klass->layout_kind();
_out->print(" flat");
if (LayoutKindHelper::is_nullable_flat(lk)) {
_out->print(" nullable");
} else {
_out->print(" null-free");
}
if (LayoutKindHelper::is_atomic_flat(lk)) {
_out->print(" atomic");
} else {
_out->print(" non-atomic");
}
} else {
_out->print(" ref");
if (oa->klass()->is_null_free_array_klass()) {
_out->print(" null-free");
} else {
_out->print(" nullable");
}
}
const char* klass_name = value->klass()->name()->as_quoted_ascii();
_out->print(" %s", klass_name);
}
@ -895,6 +916,7 @@ void StaticFieldPrinter::do_field_helper(fieldDescriptor* fd, oop mirror, bool i
break;
} else {
// handling of null free inline type
_out->print("%s", fd->signature()->as_quoted_ascii());
ResetNoHandleMark rnhm;
Thread* THREAD = Thread::current();
SignatureStream ss(fd->signature(), false);

5
src/hotspot/share/ci/ciMethod.cpp
View File

@ -1044,10 +1044,7 @@ bool ciMethod::is_compiled_lambda_form() const {
// ciMethod::is_object_constructor
//
bool ciMethod::is_object_constructor() const {
return (name() == ciSymbols::object_initializer_name()
&& signature()->return_type()->is_void());
// Note: We can't test is_static, because that would
// require the method to be loaded. Sometimes it isn't.
return name() == ciSymbols::object_initializer_name();
}
// ------------------------------------------------------------------

6
src/hotspot/share/ci/ciObjArrayKlass.cpp
View File

@ -190,12 +190,12 @@ ciObjArrayKlass* ciObjArrayKlass::make(ciKlass* element_klass, bool refined_type
GUARDED_VM_ENTRY(return make_impl(element_klass, refined_type, null_free, atomic);)
}
ciArrayKlass* ciObjArrayKlass::make(ciKlass* element_klass, int dims) {
ciObjArrayKlass* ciObjArrayKlass::make(ciKlass* element_klass, int dims) {
ciKlass* klass = element_klass;
for (int i = 0; i < dims; i++) {
klass = ciObjArrayKlass::make(klass, /* refined_type = */ false);
klass = make(klass, /* refined_type = */ false);
}
return klass->as_array_klass();
return klass->as_obj_array_klass();
}
ciKlass* ciObjArrayKlass::exact_klass() {

2
src/hotspot/share/ci/ciObjArrayKlass.hpp
View File

@ -70,7 +70,7 @@ public:
bool is_obj_array_klass() const { return true; }
static ciObjArrayKlass* make(ciKlass* element_klass, bool refined_type = true, bool null_free = false, bool atomic = true);
static ciArrayKlass* make(ciKlass* element_klass, int dims);
static ciObjArrayKlass* make(ciKlass* element_klass, int dims);
virtual ciKlass* exact_klass();

37
src/hotspot/share/ci/ciReplay.cpp
View File

@ -514,15 +514,14 @@ class CompileReplay : public StackObj {
return k;
}
obj = ciReplay::obj_field(obj, field);
// TODO 8350865 I think we need to handle null-free/flat arrays here
if (obj != nullptr && obj->is_refArray()) {
refArrayOop arr = oop_cast<refArrayOop>(obj);
if (obj != nullptr && obj->is_objArray()) {
objArrayOop arr = oop_cast<objArrayOop>(obj);
int index = parse_int("index");
if (index >= arr->length()) {
report_error("bad array index");
return nullptr;
}
obj = arr->obj_at(index);
obj = arr->obj_at(index, THREAD);
}
} while (obj != nullptr);
if (obj == nullptr) {
@ -825,7 +824,7 @@ class CompileReplay : public StackObj {
rec->_instructions_size = parse_int("instructions_size");
}
// ciMethodData <klass> <name> <signature> <state> <invocation_counter> orig <length> <byte>* data <length> <ptr>* oops <length> (<offset> <klass>)* methods <length> (<offset> <klass> <name> <signature>)*
// ciMethodData <klass> <name> <signature> <state> <invocation_counter> orig <length> <byte>* data <length> <ptr>* oops <length> (<offset> <klass> <array properties>?)* methods <length> (<offset> <klass> <name> <signature>)*
void process_ciMethodData(TRAPS) {
Method* method = parse_method(CHECK);
if (had_error()) return;
@ -1139,12 +1138,26 @@ class CompileReplay : public StackObj {
value = oopFactory::new_longArray(length, CHECK_(true));
} else if (field_signature[0] == JVM_SIGNATURE_ARRAY &&
field_signature[1] == JVM_SIGNATURE_CLASS) {
Klass* actual_array_klass = parse_klass(CHECK_(true));
// TODO 8350865 I think we need to handle null-free/flat arrays here
// This handling will change the array property argument passed to the
// factory below
Klass* kelem = ObjArrayKlass::cast(actual_array_klass)->element_klass();
value = oopFactory::new_objArray(kelem, length, CHECK_(true));
const char* flatness = parse_string();
if (strcmp(flatness, "ref") == 0) {
const char* nullability = parse_string();
bool null_restricted = (strcmp(nullability, "null-free") == 0);
Klass* actual_array_klass = parse_klass(CHECK_(true));
Klass* kelem = ObjArrayKlass::cast(actual_array_klass)->element_klass();
ArrayProperties props = ArrayProperties::Default().with_non_atomic(false).with_null_restricted(null_restricted);
value = oopFactory::new_refArray(kelem, length, props, CHECK_(true));
} else if (strcmp(flatness, "flat") == 0) {
const char* nullability = parse_string();
const char* atomicity = parse_string();
bool null_restricted = (strcmp(nullability, "null-free") == 0);
bool non_atomic = (strcmp(atomicity, "non-atomic") == 0);
Klass* actual_array_klass = parse_klass(CHECK_(true));
Klass* kelem = ObjArrayKlass::cast(actual_array_klass)->element_klass();
ArrayProperties props = ArrayProperties::Default().with_non_atomic(non_atomic).with_null_restricted(null_restricted);
value = oopFactory::new_flatArray(InlineKlass::cast(kelem), length, props, CHECK_(true));
} else {
report_error("unrecognized array kind");
}
} else {
report_error("unhandled array staticfield");
}
@ -1190,7 +1203,7 @@ class CompileReplay : public StackObj {
fieldDescriptor fd;
Symbol* name = SymbolTable::new_symbol(field_name);
Symbol* sig = SymbolTable::new_symbol(field_signature);
if (!k->find_local_field(name, sig, &fd) ||
if (!k->find_local_field(name, sig, &fd, _version >= 3) ||
!fd.is_static() ||
fd.has_initial_value()) {
report_error(field_name);

2
src/hotspot/share/ci/ciReplay.hpp
View File

@ -134,7 +134,7 @@ class ciReplay {
// 1: first instanceKlass sets protection domain (8275868)
// replace current_mileage with invocation_count (8276095)
// 2: incremental inlining support (8254108)
// 3: value class array support (8375548)
// 3: value class array support (8375548 & 8388709)
#define REPLAY_VERSION 3 // current version, bump up for incompatible changes
#endif // SHARE_CI_CIREPLAY_HPP

3
src/hotspot/share/ci/ciSignature.cpp
View File

@ -56,6 +56,9 @@ ciSignature::ciSignature(ciKlass* accessing_klass, const constantPoolHandle& cpo
} else {
type = ciType::make(ss.type());
}
assert(type == type->unwrap(), "signature type should not be wrapped");
if (ss.at_return_type()) {
// don't include return type in size calculation
_return_type = type;

6
src/hotspot/share/ci/ciSignature.hpp
View File

@ -57,7 +57,11 @@ public:
ciKlass* accessing_klass() const { return _accessing_klass; }
ciType* return_type() const { return _return_type; }
ciType* type_at(int index) const { return _types.at(index)->unwrap(); }
ciType* type_at(int index) const {
ciType* type = _types.at(index);
assert(type == type->unwrap(), "signature type should not be wrapped");
return _types.at(index)->unwrap();
}
int size() const { return _size; }
int count() const { return _types.length(); }

14
src/hotspot/share/ci/ciTypeFlow.cpp
View File

@ -576,11 +576,9 @@ void ciTypeFlow::StateVector::push_translate(ciType* type) {
}
}
// ------------------------------------------------------------------
// ciTypeFlow::StateVector::do_aload
void ciTypeFlow::StateVector::do_aload(ciBytecodeStream* str) {
void ciTypeFlow::StateVector::do_aaload(ciBytecodeStream* str) {
pop_int();
ciArrayKlass* array_klass = pop_objOrFlatArray();
ciObjArrayKlass* array_klass = pop_objArray();
if (array_klass == nullptr) {
// Did aload on a null reference; push a null and ignore the exception.
// This instruction will never continue normally. All we have to do
@ -955,13 +953,13 @@ bool ciTypeFlow::StateVector::apply_one_bytecode(ciBytecodeStream* str) {
}
switch(str->cur_bc()) {
case Bytecodes::_aaload: do_aload(str); break;
case Bytecodes::_aaload: do_aaload(str); break;
case Bytecodes::_aastore:
{
pop_object();
pop_int();
pop_objOrFlatArray();
pop_objArray();
break;
}
case Bytecodes::_aconst_null:
@ -983,7 +981,7 @@ bool ciTypeFlow::StateVector::apply_one_bytecode(ciBytecodeStream* str) {
if (!will_link) {
trap(str, element_klass, str->get_klass_index());
} else {
push_object(ciArrayKlass::make(element_klass));
push_object(ciObjArrayKlass::make(element_klass,/* refined_type = */ false));
}
break;
}
@ -3225,7 +3223,7 @@ void ciTypeFlow::record_failure(const char* reason) {
}
ciType* ciTypeFlow::mark_as_early_larval(ciType* type) {
// Wrap the type to carry the information that it is null-free
// Wrap the type to carry the information that it is "early larval"
return env()->make_early_larval_wrapper(type);
}

17
src/hotspot/share/ci/ciTypeFlow.hpp
View File

@ -334,14 +334,15 @@ public:
type_at_tos()->is_array_klass(), "must be array type");
pop();
}
// pop_objOrFlatArray and pop_typeArray narrow the tos to ciObjArrayKlass,
// ciFlatArrayKlass or ciTypeArrayKlass (resp.). In the rare case that an explicit
// null is popped from the stack, we return null. Caller beware.
ciArrayKlass* pop_objOrFlatArray() {
// pop_objArray and pop_typeArray narrow the tos to ciObjArrayKlass
// or ciTypeArrayKlass (resp.). In the rare case that an explicit
// null is popped from the stack, we return null. Caller beware.
ciObjArrayKlass* pop_objArray() {
ciType* array = pop_value();
if (array == null_type()) return nullptr;
assert(array->is_obj_array_klass(), "must be an object array type");
return array->as_array_klass();
if (array == null_type()) {
return nullptr;
}
return array->as_obj_array_klass();
}
ciTypeArrayKlass* pop_typeArray() {
ciType* array = pop_value();
@ -355,7 +356,7 @@ public:
void do_null_assert(ciKlass* unloaded_klass);
// Helper convenience routines.
void do_aload(ciBytecodeStream* str);
void do_aaload(ciBytecodeStream* str);
void do_checkcast(ciBytecodeStream* str);
void do_getfield(ciBytecodeStream* str);
void do_getstatic(ciBytecodeStream* str);

3
src/hotspot/share/code/scopeDesc.hpp
View File

@ -110,7 +110,8 @@ class ScopeDesc : public ResourceObj {
bool _has_ea_local_in_scope; // One or more NoEscape or ArgEscape objects exist in
// any of the scopes at compiled pc.
bool _arg_escape; // Compiled Java call in youngest scope passes ArgEscape
// Decoding offsets
// Decoding offsets
int _decode_offset;
int _sender_decode_offset;
int _locals_decode_offset;

1
src/hotspot/share/code/vtableStubs.cpp
View File

@ -253,6 +253,7 @@ inline uint VtableStubs::hash(bool is_vtable_stub, int vtable_index, bool caller
// Assumption: receiver_location < 4 in most cases.
int hash = ((vtable_index << 2) ^ VtableStub::receiver_location()->value()) + vtable_index;
if (caller_is_c1) {
// We have different vtable stubs for C1 and C2. We therefore make sure to get different hashes.
hash = 7 - hash;
}
return (is_vtable_stub ? ~hash : hash) & mask;

2
src/hotspot/share/compiler/methodMatcher.cpp
View File

@ -304,7 +304,7 @@ void MethodMatcher::parse_method_pattern(char*& line, const char*& error_msg, Me
(strchr(method_name, JVM_SIGNATURE_ENDSPECIAL) != nullptr)) {
if (!vmSymbols::object_initializer_name()->equals(method_name) &&
!vmSymbols::class_initializer_name()->equals(method_name)) {
error_msg = "Chars '<' and '>' only allowed in <init>, <clinit>";
error_msg = "Chars '<' and '>' only allowed in <init> and <clinit>";
return;
}
}

6
src/hotspot/share/gc/g1/g1ConcurrentMark.hpp
View File

@ -144,7 +144,7 @@ private:
Atomic<TaskQueueEntryChunk*>* _buckets;
char _pad0[DEFAULT_PADDING_SIZE];
Atomic<size_t> _size;
char _pad4[DEFAULT_PADDING_SIZE - sizeof(size_t)];
char _pad4[DEFAULT_PADDING_SIZE - sizeof(_size)];
size_t bucket_size(size_t bucket) {
return (bucket == 0) ?
@ -211,10 +211,10 @@ private:
char _pad0[DEFAULT_PADDING_SIZE];
Atomic<TaskQueueEntryChunk*> _free_list; // Linked list of free chunks that can be allocated by users.
char _pad1[DEFAULT_PADDING_SIZE - sizeof(TaskQueueEntryChunk*)];
char _pad1[DEFAULT_PADDING_SIZE - sizeof(_free_list)];
Atomic<TaskQueueEntryChunk*> _chunk_list; // List of chunks currently containing data.
Atomic<size_t> _chunks_in_chunk_list;
char _pad2[DEFAULT_PADDING_SIZE - sizeof(TaskQueueEntryChunk*) - sizeof(_chunks_in_chunk_list)];
char _pad2[DEFAULT_PADDING_SIZE - sizeof(_chunk_list) - sizeof(_chunks_in_chunk_list)];
// Atomically add the given chunk to the list.
void add_chunk_to_list(Atomic<TaskQueueEntryChunk*>* list, TaskQueueEntryChunk* elem);

4
src/hotspot/share/gc/g1/g1ConcurrentRefineThread.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -40,8 +40,6 @@ class G1ConcurrentRefineThread: public ConcurrentGCThread {
Monitor _notifier;
bool _requested_active;
uint _worker_id;
G1ConcurrentRefine* _cr;
NONCOPYABLE(G1ConcurrentRefineThread);

11
src/hotspot/share/gc/g1/g1FullGCAdjustTask.cpp
View File

@ -52,12 +52,11 @@ public:
class G1AdjustRegionClosure : public G1HeapRegionClosure {
G1FullCollector* _collector;
G1CMBitMap* _bitmap;
uint _worker_id;
public:
G1AdjustRegionClosure(G1FullCollector* collector, uint worker_id) :
public:
G1AdjustRegionClosure(G1FullCollector* collector) :
_collector(collector),
_bitmap(collector->mark_bitmap()),
_worker_id(worker_id) { }
_bitmap(collector->mark_bitmap()) { }
bool do_heap_region(G1HeapRegion* r) {
G1AdjustClosure cl(_collector);
@ -103,7 +102,7 @@ void G1FullGCAdjustTask::work(uint worker_id) {
_root_processor.process_all_roots(&_adjust, &adjust_cld, &adjust_code);
// Now adjust pointers region by region
G1AdjustRegionClosure blk(collector(), worker_id);
G1AdjustRegionClosure blk(collector());
G1CollectedHeap::heap()->heap_region_par_iterate_from_worker_offset(&blk, &_hrclaimer, worker_id);
log_task("Adjust task", worker_id, start);
}

2
src/hotspot/share/gc/g1/g1FullGCMarker.cpp
View File

@ -40,7 +40,7 @@ G1FullGCMarker::G1FullGCMarker(G1FullCollector* collector,
_bitmap(collector->mark_bitmap()),
_task_queue(),
_partial_array_splitter(collector->partial_array_state_manager(), collector->workers()),
_mark_closure(worker_id, this, ClassLoaderData::_claim_stw_fullgc_mark, G1CollectedHeap::heap()->ref_processor_stw()),
_mark_closure(this, ClassLoaderData::_claim_stw_fullgc_mark, G1CollectedHeap::heap()->ref_processor_stw()),
_stack_closure(this),
_cld_closure(mark_closure(), ClassLoaderData::_claim_stw_fullgc_mark),
_mark_stats_cache(mark_stats, G1RegionMarkStatsCache::RegionMarkStatsCacheSize) {

8
src/hotspot/share/gc/g1/g1FullGCOopClosures.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -60,13 +60,11 @@ public:
class G1MarkAndPushClosure : public ClaimMetadataVisitingOopIterateClosure {
G1FullGCMarker* _marker;
uint _worker_id;
public:
G1MarkAndPushClosure(uint worker_id, G1FullGCMarker* marker, int claim, ReferenceDiscoverer* ref) :
G1MarkAndPushClosure(G1FullGCMarker* marker, int claim, ReferenceDiscoverer* ref) :
ClaimMetadataVisitingOopIterateClosure(claim, ref),
_marker(marker),
_worker_id(worker_id) { }
_marker(marker) { }
template <class T> inline void do_oop_work(T* p);
virtual void do_oop(oop* p);

4
src/hotspot/share/gc/g1/g1GCParPhaseTimesTracker.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -49,7 +49,7 @@ class G1EvacPhaseTimesTracker : public G1GCParPhaseTimesTracker {
G1EvacPhaseWithTrimTimeTracker _trim_tracker;
public:
G1EvacPhaseTimesTracker(G1GCPhaseTimes* phase_times, G1ParScanThreadState* pss, G1GCPhaseTimes::GCParPhases phase, uint worker_id);
G1EvacPhaseTimesTracker(G1GCPhaseTimes* phase_times, G1ParScanThreadState* par_scan_state, G1GCPhaseTimes::GCParPhases phase);
virtual ~G1EvacPhaseTimesTracker();
};

17
src/hotspot/share/gc/g1/g1GCPhaseTimes.cpp
View File

@ -581,14 +581,14 @@ const char* G1GCPhaseTimes::phase_name(GCParPhases phase) {
return phase_times->_gc_par_phases[phase]->short_name();
}
G1EvacPhaseWithTrimTimeTracker::G1EvacPhaseWithTrimTimeTracker(G1ParScanThreadState* pss, Tickspan& total_time, Tickspan& trim_time) :
_pss(pss),
G1EvacPhaseWithTrimTimeTracker::G1EvacPhaseWithTrimTimeTracker(G1ParScanThreadState* par_scan_state, Tickspan& total_time, Tickspan& trim_time) :
_par_scan_state(par_scan_state),
_start(Ticks::now()),
_total_time(total_time),
_trim_time(trim_time),
_stopped(false) {
assert(_pss->trim_ticks().value() == 0, "Possibly remaining trim ticks left over from previous use");
assert(_par_scan_state->trim_ticks().value() == 0, "Possibly remaining trim ticks left over from previous use");
}
G1EvacPhaseWithTrimTimeTracker::~G1EvacPhaseWithTrimTimeTracker() {
@ -599,9 +599,9 @@ G1EvacPhaseWithTrimTimeTracker::~G1EvacPhaseWithTrimTimeTracker() {
void G1EvacPhaseWithTrimTimeTracker::stop() {
assert(!_stopped, "Should only be called once");
_total_time += (Ticks::now() - _start) - _pss->trim_ticks();
_trim_time += _pss->trim_ticks();
_pss->reset_trim_ticks();
_total_time += (Ticks::now() - _start) - _par_scan_state->trim_ticks();
_trim_time += _par_scan_state->trim_ticks();
_par_scan_state->reset_trim_ticks();
_stopped = true;
}
@ -625,9 +625,8 @@ G1GCParPhaseTimesTracker::~G1GCParPhaseTimesTracker() {
G1EvacPhaseTimesTracker::G1EvacPhaseTimesTracker(G1GCPhaseTimes* phase_times,
G1ParScanThreadState* pss,
G1GCPhaseTimes::GCParPhases phase,
uint worker_id) :
G1GCParPhaseTimesTracker(phase_times, phase, worker_id),
G1GCPhaseTimes::GCParPhases phase) :
G1GCParPhaseTimesTracker(phase_times, phase, pss->worker_id()),
_total_time(),
_trim_time(),
_trim_tracker(pss, _total_time, _trim_time) {

4
src/hotspot/share/gc/g1/g1GCPhaseTimes.hpp
View File

@ -413,7 +413,7 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
};
class G1EvacPhaseWithTrimTimeTracker : public StackObj {
G1ParScanThreadState* _pss;
G1ParScanThreadState* _par_scan_state;
Ticks _start;
Tickspan& _total_time;
@ -421,7 +421,7 @@ class G1EvacPhaseWithTrimTimeTracker : public StackObj {
bool _stopped;
public:
G1EvacPhaseWithTrimTimeTracker(G1ParScanThreadState* pss, Tickspan& total_time, Tickspan& trim_time);
G1EvacPhaseWithTrimTimeTracker(G1ParScanThreadState* par_scan_state, Tickspan& total_time, Tickspan& trim_time);
~G1EvacPhaseWithTrimTimeTracker();
void stop();

12
src/hotspot/share/gc/g1/g1NMethodClosure.cpp
View File

@ -28,11 +28,17 @@
#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "gc/g1/g1NMethodClosure.hpp"
#include "gc/g1/g1ParScanThreadState.inline.hpp"
#include "gc/shared/barrierSetNMethod.hpp"
#include "oops/access.inline.hpp"
#include "oops/compressedOops.inline.hpp"
#include "oops/oop.inline.hpp"
G1NMethodClosure::G1NMethodClosure(OopClosure* oc, bool strong, G1ParScanThreadState* par_scan_state) :
_oc(oc, par_scan_state),
_marking_oc(par_scan_state->worker_id()),
_strong(strong) { }
template <typename T>
void G1NMethodClosure::HeapRegionGatheringOopClosure::do_oop_work(T* p) {
T old_oop_or_narrowoop = RawAccess<>::oop_load(p);
@ -65,17 +71,17 @@ void G1NMethodClosure::HeapRegionGatheringOopClosure::do_oop_work(T* p) {
}
}
G1NMethodClosure::HeapRegionGatheringOopClosure::HeapRegionGatheringOopClosure(OopClosure* oc, G1ParScanThreadState* pss) :
G1NMethodClosure::HeapRegionGatheringOopClosure::HeapRegionGatheringOopClosure(OopClosure* oc, G1ParScanThreadState* par_scan_state) :
_g1h(G1CollectedHeap::heap()),
_work(oc),
_pss(pss),
_par_scan_state(par_scan_state),
_nm(nullptr),
_affected_regions(5) {
}
void G1NMethodClosure::HeapRegionGatheringOopClosure::add_to_remsets() {
while (!_affected_regions.is_empty()) {
_pss->remember_nmethod_into_region(_affected_regions.pop(), _nm);
_par_scan_state->remember_nmethod_into_region(_affected_regions.pop(), _nm);
}
}

7
src/hotspot/share/gc/g1/g1NMethodClosure.hpp
View File

@ -38,7 +38,7 @@ class G1NMethodClosure : public NMethodClosure {
class HeapRegionGatheringOopClosure : public OopClosure {
G1CollectedHeap* _g1h;
OopClosure* _work;
G1ParScanThreadState* _pss;
G1ParScanThreadState* _par_scan_state;
nmethod* _nm;
GrowableArrayCHeap<G1HeapRegion*, mtGC> _affected_regions;
@ -47,7 +47,7 @@ class G1NMethodClosure : public NMethodClosure {
void do_oop_work(T* p);
public:
HeapRegionGatheringOopClosure(OopClosure* oc, G1ParScanThreadState* pss);
HeapRegionGatheringOopClosure(OopClosure* oc, G1ParScanThreadState* par_scan_state);
~HeapRegionGatheringOopClosure() = default;
void do_oop(oop* o);
@ -81,8 +81,7 @@ class G1NMethodClosure : public NMethodClosure {
bool _strong;
public:
G1NMethodClosure(uint worker_id, OopClosure* oc, bool strong, G1ParScanThreadState* pss) :
_oc(oc, pss), _marking_oc(worker_id), _strong(strong) { }
G1NMethodClosure(OopClosure* oc, bool strong, G1ParScanThreadState* par_scan_state);
void do_evacuation_and_fixup(nmethod* nm);
void do_marking(nmethod* nm);

75
src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
View File

@ -55,20 +55,12 @@
// Explicit NOINLINE to block ATTRIBUTE_FLATTENing.
#define MAYBE_INLINE_EVACUATION NOT_DEBUG(inline) DEBUG_ONLY(NOINLINE)
// Good estimate for the initial table size.
static uint initial_nmethod_table_size(G1CollectedHeap* g1h) {
// The +1 is both to consider the retained old region likely to be added, and avoid zero-sized initial tables.
return MIN3(g1h->collection_set()->num_regions(), g1h->max_num_regions() / 2, g1h->num_available_regions()) + 1;
}
G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h,
G1ParScanThreadStateSet* per_thread_states,
uint worker_id,
uint num_workers,
G1CollectionSet* collection_set,
G1EvacFailureRegions* evac_failure_regions)
: _g1h(g1h),
_per_thread_states(per_thread_states),
_task_queue(g1h->task_queue(worker_id)),
_ct(g1h->refinement_table()),
_closures(nullptr),
@ -91,10 +83,7 @@ G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h,
_max_num_optional_regions(collection_set->num_optional_regions()),
_numa(g1h->numa()),
_obj_alloc_stat(nullptr),
// The initial size estimate is relatively conservative, assuming that all regions
// in the collection set get evacuated into the same amount of new regions.
_nmethods_to_add(initial_nmethod_table_size(g1h),
MAX2(initial_nmethod_table_size(g1h), _g1h->max_num_regions() / 2)),
_code_root_pairs(32),
ALLOCATION_FAILURE_INJECTOR_ONLY(_allocation_failure_inject_counter(0) COMMA)
_evacuation_failed_info(),
_evac_failure_regions(evac_failure_regions),
@ -141,12 +130,6 @@ size_t G1ParScanThreadState::flush_stats(size_t* surviving_young_words, uint num
}
G1ParScanThreadState::~G1ParScanThreadState() {
auto delete_all = [&] (uint region, G1NmethodSet* nmethods) -> bool {
delete nmethods;
return true;
};
_nmethods_to_add.iterate(delete_all);
delete _plab_allocator;
delete _closures;
FREE_C_HEAP_ARRAY(_surviving_young_words_base);
@ -597,7 +580,6 @@ G1ParScanThreadState* G1ParScanThreadStateSet::state_for_worker(uint worker_id)
if (_states[worker_id] == nullptr) {
_states[worker_id] =
new G1ParScanThreadState(_g1h,
this,
worker_id,
_num_workers,
_collection_set,
@ -649,51 +631,18 @@ void G1ParScanThreadStateSet::destroy_worker_states() {
}
}
void G1ParScanThreadStateSet::update_nmethod_regions_to_add(G1NmethodsToAdd* nmethods) {
if (nmethods->number_of_entries() == 0) {
return;
}
// Take the key set, look which are not yet in the global set, and update the necessary ones.
ResourceMark rm;
GrowableArray<uint> regions_to_add = GrowableArray<uint>(nmethods->table_size());
nmethods->iterate_all([&] (uint& region, void*) {
if (_has_nmethods_to_add.par_set_bit(region, memory_order_relaxed)) {
regions_to_add.push(region);
}
});
uint num_regions_to_add = (uint)regions_to_add.length();
if (num_regions_to_add == 0) {
return;
}
uint first_index = _num_nmethod_regions_to_add.fetch_then_add(num_regions_to_add, memory_order_relaxed);
guarantee(first_index + num_regions_to_add <= _g1h->max_num_regions(), "must be");
memcpy(&_nmethod_regions_to_add[first_index], regions_to_add.adr_at(0), num_regions_to_add * sizeof(uint));
}
void G1ParScanThreadStateSet::par_iterate_nmethod_regions_to_add(G1HeapRegionClosure* cl,
G1HeapRegionClaimer* claimer,
uint worker_id) {
_g1h->par_iterate_regions_array(cl, claimer, _nmethod_regions_to_add, num_nmethod_regions_to_add(), worker_id);
}
void G1ParScanThreadStateSet::record_unused_optional_region(G1HeapRegion* hr) {
for (uint worker_index = 0; worker_index < _num_workers; ++worker_index) {
G1ParScanThreadState* pss = _states[worker_index];
for (uint worker_id = 0; worker_id < _num_workers; ++worker_id) {
G1ParScanThreadState* pss = _states[worker_id];
assert(pss != nullptr, "must be initialized");
size_t used_memory = pss->oops_into_optional_region(hr)->used_memory();
_g1h->phase_times()->record_or_add_thread_work_item(G1GCPhaseTimes::OptScanHR, worker_index, used_memory, G1GCPhaseTimes::ScanHRUsedMemory);
_g1h->phase_times()->record_or_add_thread_work_item(G1GCPhaseTimes::OptScanHR, worker_id, used_memory, G1GCPhaseTimes::ScanHRUsedMemory);
}
}
void G1ParScanThreadState::record_evacuation_failed_region(G1HeapRegion* r, uint worker_id, bool cause_pinned) {
if (_evac_failure_regions->record(worker_id, r->hrm_index(), cause_pinned)) {
void G1ParScanThreadState::record_evacuation_failed_region(G1HeapRegion* r, bool cause_pinned) {
if (_evac_failure_regions->record(worker_id(), r->hrm_index(), cause_pinned)) {
G1HeapRegionPrinter::evac_failure(r);
}
}
@ -707,7 +656,7 @@ oop G1ParScanThreadState::handle_evacuation_failure_par(oop old, markWord m, Kla
// Forward-to-self succeeded. We are the "owner" of the object.
G1HeapRegion* r = _g1h->heap_region_containing(old);
record_evacuation_failed_region(r, _worker_id, cause_pinned);
record_evacuation_failed_region(r, cause_pinned);
// Mark the failing object in the marking bitmap and later use the bitmap to handle
// evacuation failure recovery.
@ -737,10 +686,6 @@ oop G1ParScanThreadState::handle_evacuation_failure_par(oop old, markWord m, Kla
}
}
void G1ParScanThreadState::update_nmethod_regions_to_add() {
_per_thread_states->update_nmethod_regions_to_add(&_nmethods_to_add);
}
void G1ParScanThreadState::initialize_numa_stats() {
if (_numa->is_enabled()) {
LogTarget(Info, gc, heap, numa) lt;
@ -785,10 +730,7 @@ G1ParScanThreadStateSet::G1ParScanThreadStateSet(G1CollectedHeap* g1h,
_surviving_young_words_total(NEW_C_HEAP_ARRAY(size_t, collection_set->num_young_regions() + 1, mtGC)),
_num_workers(num_workers),
_flushed(false),
_evac_failure_regions(evac_failure_regions),
_has_nmethods_to_add(g1h->max_num_regions(), mtGC),
_num_nmethod_regions_to_add(0),
_nmethod_regions_to_add(NEW_C_HEAP_ARRAY(uint, g1h->max_num_regions(), mtGC)) // Conservative length estimation.
_evac_failure_regions(evac_failure_regions)
{
for (uint i = 0; i < num_workers; ++i) {
_states[i] = nullptr;
@ -800,7 +742,6 @@ G1ParScanThreadStateSet::~G1ParScanThreadStateSet() {
for (uint i = 0; i < _num_workers; i++) {
assert(_states[i] == nullptr, "must be");
}
FREE_C_HEAP_ARRAY(_nmethod_regions_to_add);
FREE_C_HEAP_ARRAY(_states);
FREE_C_HEAP_ARRAY(_surviving_young_words_total);
}

36
src/hotspot/share/gc/g1/g1ParScanThreadState.hpp
View File

@ -37,9 +37,7 @@
#include "gc/shared/taskqueue.hpp"
#include "memory/allocation.hpp"
#include "oops/oop.hpp"
#include "runtime/atomic.hpp"
#include "utilities/growableArray.hpp"
#include "utilities/resizableHashTable.hpp"
#include "utilities/ticks.hpp"
class G1CardTable;
@ -52,11 +50,13 @@ class G1PLABAllocator;
class G1HeapRegion;
class outputStream;
typedef GrowableArrayCHeap<nmethod*, mtGC> G1NmethodSet;
typedef ResizeableHashTable<uint, G1NmethodSet*, AnyObj::C_HEAP, mtGC> G1NmethodsToAdd;
// A code root pair gathered during code root scanning.
struct G1CodeRootPair {
uint _region_idx;
nmethod* _nmethod;
};
class G1ParScanThreadState : public CHeapObj<mtGC> {
G1CollectedHeap* _g1h;
G1ParScanThreadStateSet* _per_thread_states;
G1ScannerTasksQueue* _task_queue;
G1CardTable* _ct;
G1EvacuationRootClosures* _closures;
@ -103,8 +103,8 @@ class G1ParScanThreadState : public CHeapObj<mtGC> {
// transferred when flushed.
size_t* _obj_alloc_stat;
// The nmethods that were found during code root scan that need to be redistributed.
G1NmethodsToAdd _nmethods_to_add;
// Code root pairs to add after evacuation.
GrowableArrayCHeap<G1CodeRootPair, mtGC> _code_root_pairs;
// Per-thread evacuation failure data structures.
ALLOCATION_FAILURE_INJECTOR_ONLY(size_t _allocation_failure_inject_counter;)
@ -124,7 +124,6 @@ class G1ParScanThreadState : public CHeapObj<mtGC> {
public:
G1ParScanThreadState(G1CollectedHeap* g1h,
G1ParScanThreadStateSet* per_thread_states,
uint worker_id,
uint num_workers,
G1CollectionSet* collection_set,
@ -251,19 +250,13 @@ public:
Tickspan trim_ticks() const;
void reset_trim_ticks();
void record_evacuation_failed_region(G1HeapRegion* r, uint worker_id, bool cause_pinned);
void record_evacuation_failed_region(G1HeapRegion* r, bool cause_pinned);
// An attempt to evacuate "obj" has failed; take necessary steps.
oop handle_evacuation_failure_par(oop obj, markWord m, Klass* klass, G1HeapRegionAttr attr, size_t word_sz, bool cause_pinned);
inline void remember_nmethod_into_region(G1HeapRegion* r, nmethod* nm);
// Updates the global set of regions that need updates to the code root set
// later with the ones gathered so far.
void update_nmethod_regions_to_add();
inline size_t num_nmethods(uint index) const;
// Iterate nmethods stored for the given region index.
template <typename Function>
inline void iterate_nmethods(uint index, Function fn);
const GrowableArrayCHeap<G1CodeRootPair, mtGC>& code_root_pairs() const { return _code_root_pairs; }
template <typename T>
inline void remember_root_into_optional_region(T* p);
@ -282,10 +275,6 @@ class G1ParScanThreadStateSet : public StackObj {
bool _flushed;
G1EvacFailureRegions* _evac_failure_regions;
CHeapBitMap _has_nmethods_to_add;
Atomic<uint> _num_nmethod_regions_to_add;
uint* _nmethod_regions_to_add;
public:
G1ParScanThreadStateSet(G1CollectedHeap* g1h,
uint num_workers,
@ -296,13 +285,6 @@ class G1ParScanThreadStateSet : public StackObj {
void flush_stats();
void destroy_worker_states();
// Updates the region set that has code root updates with the regions in the given set.
void update_nmethod_regions_to_add(G1NmethodsToAdd* nmethods);
void par_iterate_nmethod_regions_to_add(G1HeapRegionClosure* cl,
G1HeapRegionClaimer* claimer,
uint worker_id);
uint num_nmethod_regions_to_add() const { return _num_nmethod_regions_to_add.load_relaxed(); }
void record_unused_optional_region(G1HeapRegion* hr);
#if TASKQUEUE_STATS
void print_partial_array_task_stats();

29
src/hotspot/share/gc/g1/g1ParScanThreadState.inline.hpp
View File

@ -71,34 +71,7 @@ inline void G1ParScanThreadState::reset_trim_ticks() {
}
inline void G1ParScanThreadState::remember_nmethod_into_region(G1HeapRegion* r, nmethod* nm) {
uint index = r->hrm_index();
G1NmethodSet** nmethods = _nmethods_to_add.get(index);
if (nmethods != nullptr) {
(*nmethods)->push(nm);
} else {
G1NmethodSet* new_set = new G1NmethodSet(3);
new_set->push(nm);
bool put_result = _nmethods_to_add.put(index, new_set);
assert(put_result, "must be");
_nmethods_to_add.maybe_grow(3 /* load_factor */);
}
}
inline size_t G1ParScanThreadState::num_nmethods(uint region) const {
G1NmethodSet** nmethods = _nmethods_to_add.get(region);
return nmethods != nullptr ? (size_t)(*nmethods)->length() : 0;
}
template <typename Function>
inline void G1ParScanThreadState::iterate_nmethods(uint index, Function fn) {
G1NmethodSet** nmethods = _nmethods_to_add.get(index);
if (nmethods == nullptr) {
return;
}
for (nmethod* nm : **nmethods) {
fn(nm);
}
_code_root_pairs.push(G1CodeRootPair{r->hrm_index(), nm});
}
template <typename T>

46
src/hotspot/share/gc/g1/g1RemSet.cpp
View File

@ -398,13 +398,9 @@ class G1ScanHRForRegionClosure : public G1HeapRegionClosure {
G1CollectedHeap* _g1h;
G1CardTable* _ct;
G1ParScanThreadState* _pss;
G1RemSetScanState* _scan_state;
G1GCPhaseTimes::GCParPhases _phase;
uint _worker_id;
G1ParScanThreadState* _pss;
size_t _cards_pending;
size_t _cards_empty;
@ -493,15 +489,11 @@ class G1ScanHRForRegionClosure : public G1HeapRegionClosure {
public:
G1ScanHRForRegionClosure(G1RemSetScanState* scan_state,
G1ParScanThreadState* pss,
uint worker_id,
G1GCPhaseTimes::GCParPhases phase,
bool remember_already_scanned_cards) :
_g1h(G1CollectedHeap::heap()),
_ct(_g1h->card_table()),
_pss(pss),
_scan_state(scan_state),
_phase(phase),
_worker_id(worker_id),
_pss(pss),
_cards_pending(0),
_cards_empty(0),
_cards_scanned(0),
@ -540,12 +532,13 @@ public:
};
void G1RemSet::scan_heap_roots(G1ParScanThreadState* pss,
uint worker_id,
G1GCPhaseTimes::GCParPhases scan_phase,
G1GCPhaseTimes::GCParPhases objcopy_phase,
bool remember_already_scanned_cards) {
uint worker_id = pss->worker_id();
EventGCPhaseParallel event;
G1ScanHRForRegionClosure cl(_scan_state, pss, worker_id, scan_phase, remember_already_scanned_cards);
G1ScanHRForRegionClosure cl(_scan_state, pss, remember_already_scanned_cards);
_scan_state->iterate_dirty_regions_from(&cl, worker_id);
event.commit(GCId::current(), worker_id, G1GCPhaseTimes::phase_name(scan_phase));
@ -587,19 +580,12 @@ public:
// increment to fix up non-card related roots.
class G1ScanCodeRootsClosure : public G1HeapRegionClosure {
G1ParScanThreadState* _pss;
G1RemSetScanState* _scan_state;
uint _worker_id;
size_t _code_roots_scanned;
public:
G1ScanCodeRootsClosure(G1RemSetScanState* scan_state,
G1ParScanThreadState* pss,
uint worker_id) :
G1ScanCodeRootsClosure(G1ParScanThreadState* pss) :
_pss(pss),
_scan_state(scan_state),
_worker_id(worker_id),
_code_roots_scanned(0) { }
bool do_heap_region(G1HeapRegion* r) {
@ -614,7 +600,6 @@ public:
};
void G1RemSet::scan_collection_set_code_roots(G1ParScanThreadState* pss,
uint worker_id,
G1GCPhaseTimes::GCParPhases coderoots_phase,
G1GCPhaseTimes::GCParPhases objcopy_phase) {
EventGCPhaseParallel event;
@ -622,16 +607,15 @@ void G1RemSet::scan_collection_set_code_roots(G1ParScanThreadState* pss,
Tickspan code_root_trim_partially_time;
G1GCPhaseTimes* p = _g1h->phase_times();
uint worker_id = pss->worker_id();
{
G1EvacPhaseWithTrimTimeTracker timer(pss, code_root_scan_time, code_root_trim_partially_time);
G1ScanCodeRootsClosure cl(_scan_state, pss, worker_id);
G1ScanCodeRootsClosure cl(pss);
// Code roots work distribution occurs inside the iteration method. So scan all collection
// set regions for all threads.
_g1h->collection_set_iterate_increment_from(&cl, worker_id);
pss->update_nmethod_regions_to_add();
p->record_or_add_thread_work_item(coderoots_phase, worker_id, cl.code_roots_scanned(), G1GCPhaseTimes::CodeRootsScannedNMethods);
}
@ -644,10 +628,6 @@ void G1RemSet::scan_collection_set_code_roots(G1ParScanThreadState* pss,
class G1ScanOptionalRemSetRootsClosure : public G1HeapRegionClosure {
G1ParScanThreadState* _pss;
uint _worker_id;
G1GCPhaseTimes::GCParPhases _scan_phase;
size_t _opt_roots_scanned;
size_t _opt_refs_scanned;
@ -663,12 +643,8 @@ class G1ScanOptionalRemSetRootsClosure : public G1HeapRegionClosure {
}
public:
G1ScanOptionalRemSetRootsClosure(G1ParScanThreadState* pss,
uint worker_id,
G1GCPhaseTimes::GCParPhases scan_phase) :
G1ScanOptionalRemSetRootsClosure(G1ParScanThreadState* pss) :
_pss(pss),
_worker_id(worker_id),
_scan_phase(scan_phase),
_opt_roots_scanned(0),
_opt_refs_scanned(0),
_opt_refs_memory_used(0) { }
@ -686,7 +662,6 @@ public:
};
void G1RemSet::scan_collection_set_optional_roots(G1ParScanThreadState* pss,
uint worker_id,
G1GCPhaseTimes::GCParPhases scan_phase,
G1GCPhaseTimes::GCParPhases objcopy_phase) {
assert(scan_phase == G1GCPhaseTimes::OptScanHR, "must be");
@ -699,7 +674,8 @@ void G1RemSet::scan_collection_set_optional_roots(G1ParScanThreadState* pss,
G1GCPhaseTimes* p = _g1h->phase_times();
G1ScanOptionalRemSetRootsClosure cl(pss, worker_id, scan_phase);
G1ScanOptionalRemSetRootsClosure cl(pss);
uint worker_id = pss->worker_id();
// The individual references for the optional remembered set are per-worker, so every worker
// always need to scan all regions (no claimer).
_g1h->collection_set_iterate_increment_from(&cl, worker_id);

5
src/hotspot/share/gc/g1/g1RemSet.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -80,7 +80,6 @@ public:
// Scan all cards in the non-collection set regions that potentially contain
// references into the current whole collection set.
void scan_heap_roots(G1ParScanThreadState* pss,
uint worker_id,
G1GCPhaseTimes::GCParPhases scan_phase,
G1GCPhaseTimes::GCParPhases objcopy_phase,
bool remember_already_scanned_cards);
@ -109,12 +108,10 @@ public:
// Do work for regions in the current increment of the collection set, scanning
// non-card based (heap) roots.
void scan_collection_set_code_roots(G1ParScanThreadState* pss,
uint worker_id,
G1GCPhaseTimes::GCParPhases coderoots_phase,
G1GCPhaseTimes::GCParPhases objcopy_phase);
void scan_collection_set_optional_roots(G1ParScanThreadState* pss,
uint worker_id,
G1GCPhaseTimes::GCParPhases scan_phase,
G1GCPhaseTimes::GCParPhases objcopy_phase);

5
src/hotspot/share/gc/g1/g1RootProcessor.cpp
View File

@ -52,12 +52,13 @@ G1RootProcessor::G1RootProcessor(G1CollectedHeap* g1h, bool is_parallel) :
_threads_claim_token_scope(),
_is_parallel(is_parallel) {}
void G1RootProcessor::evacuate_roots(G1ParScanThreadState* pss, uint worker_id) {
void G1RootProcessor::evacuate_roots(G1ParScanThreadState* pss) {
G1GCPhaseTimes* phase_times = _g1h->phase_times();
G1EvacPhaseTimesTracker timer(phase_times, pss, G1GCPhaseTimes::ExtRootScan, worker_id);
G1EvacPhaseTimesTracker timer(phase_times, pss, G1GCPhaseTimes::ExtRootScan);
G1EvacuationRootClosures* closures = pss->closures();
uint worker_id = pss->worker_id();
process_java_roots(closures, phase_times, worker_id);
process_vm_roots(closures, phase_times, worker_id);

4
src/hotspot/share/gc/g1/g1RootProcessor.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2015, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -80,7 +80,7 @@ public:
// Apply correct closures from pss to the strongly and weakly reachable roots in the system
// in a single pass.
// Record and report timing measurements for sub phases using worker_id.
void evacuate_roots(G1ParScanThreadState* pss, uint worker_id);
void evacuate_roots(G1ParScanThreadState* pss);
// Apply oops, clds and blobs to all strongly reachable roots in the system
void process_strong_roots(OopClosure* oops,

2
src/hotspot/share/gc/g1/g1SharedClosures.hpp
View File

@ -55,7 +55,7 @@ public:
_oops_in_cld(g1h, pss),
_oops_in_nmethod(g1h, pss),
_clds(&_oops_in_cld, process_only_dirty),
_nmethods(pss->worker_id(), &_oops_in_nmethod, should_mark, pss) {}
_nmethods(&_oops_in_nmethod, should_mark, pss) {}
};
#endif // SHARE_GC_G1_G1SHAREDCLOSURES_HPP

73
src/hotspot/share/gc/g1/g1YoungCollector.cpp
View File

@ -580,7 +580,6 @@ class G1ParEvacuateFollowersClosure : public VoidClosure {
void start_term_time() { _term_attempts++; _start_term = os::elapsedTime(); }
void end_term_time() { _term_time += (os::elapsedTime() - _start_term); }
G1CollectedHeap* _g1h;
G1ParScanThreadState* _par_scan_state;
G1ScannerTasksQueueSet* _queues;
TaskTerminator* _terminator;
@ -592,22 +591,20 @@ class G1ParEvacuateFollowersClosure : public VoidClosure {
inline bool offer_termination() {
EventGCPhaseParallel event;
G1ParScanThreadState* const pss = par_scan_state();
start_term_time();
const bool res = (terminator() == nullptr) ? true : terminator()->offer_termination();
end_term_time();
event.commit(GCId::current(), pss->worker_id(), G1GCPhaseTimes::phase_name(G1GCPhaseTimes::Termination));
event.commit(GCId::current(), par_scan_state()->worker_id(), G1GCPhaseTimes::phase_name(G1GCPhaseTimes::Termination));
return res;
}
public:
G1ParEvacuateFollowersClosure(G1CollectedHeap* g1h,
G1ParScanThreadState* par_scan_state,
G1ParEvacuateFollowersClosure(G1ParScanThreadState* par_scan_state,
G1ScannerTasksQueueSet* queues,
TaskTerminator* terminator,
G1GCPhaseTimes::GCParPhases phase)
: _start_term(0.0), _term_time(0.0), _term_attempts(0),
_g1h(g1h), _par_scan_state(par_scan_state),
_par_scan_state(par_scan_state),
_queues(queues), _terminator(terminator), _phase(phase) {}
void do_void() {
@ -632,23 +629,22 @@ class G1EvacuateRegionsBaseTask : public WorkerTask {
// regions as there is no guarantee that there is a reference reachable by
// Java code (i.e. only by native code) that adds it to the evacuation failed
// regions.
void record_pinned_regions(G1ParScanThreadState* pss, uint worker_id) {
void record_pinned_regions(G1ParScanThreadState* pss) {
class RecordPinnedRegionClosure : public G1HeapRegionClosure {
G1ParScanThreadState* _pss;
uint _worker_id;
public:
RecordPinnedRegionClosure(G1ParScanThreadState* pss, uint worker_id) : _pss(pss), _worker_id(worker_id) { }
RecordPinnedRegionClosure(G1ParScanThreadState* pss) : _pss(pss) { }
bool do_heap_region(G1HeapRegion* r) {
if (r->has_pinned_objects()) {
_pss->record_evacuation_failed_region(r, _worker_id, true /* cause_pinned */);
_pss->record_evacuation_failed_region(r, true /* cause_pinned */);
}
return false;
}
} cl(pss, worker_id);
} cl(pss);
_g1h->collection_set_iterate_increment_from(&cl, worker_id);
_g1h->collection_set_iterate_increment_from(&cl, pss->worker_id());
}
protected:
@ -659,18 +655,18 @@ protected:
TaskTerminator _terminator;
void evacuate_live_objects(G1ParScanThreadState* pss,
uint worker_id,
G1GCPhaseTimes::GCParPhases objcopy_phase,
G1GCPhaseTimes::GCParPhases termination_phase) {
G1GCPhaseTimes* p = _g1h->phase_times();
Ticks start = Ticks::now();
G1ParEvacuateFollowersClosure cl(_g1h, pss, _task_queues, &_terminator, objcopy_phase);
G1ParEvacuateFollowersClosure cl(pss, _task_queues, &_terminator, objcopy_phase);
cl.do_void();
assert(pss->queue_is_empty(), "should be empty");
Tickspan evac_time = (Ticks::now() - start);
uint worker_id = pss->worker_id();
p->record_or_add_time_secs(objcopy_phase, worker_id, evac_time.seconds() - cl.term_time());
if (termination_phase == G1GCPhaseTimes::Termination) {
@ -689,9 +685,9 @@ protected:
virtual void end_work(uint worker_id) { }
virtual void scan_roots(G1ParScanThreadState* pss, uint worker_id) = 0;
virtual void scan_roots(G1ParScanThreadState* pss) = 0;
virtual void evacuate_live_objects(G1ParScanThreadState* pss, uint worker_id) = 0;
virtual void evacuate_live_objects(G1ParScanThreadState* pss) = 0;
private:
Atomic<bool> _pinned_regions_recorded;
@ -719,10 +715,10 @@ public:
pss->set_ref_discoverer(_g1h->ref_processor_stw());
if (_pinned_regions_recorded.compare_set(false, true)) {
record_pinned_regions(pss, worker_id);
record_pinned_regions(pss);
}
scan_roots(pss, worker_id);
evacuate_live_objects(pss, worker_id);
scan_roots(pss);
evacuate_live_objects(pss);
}
end_work(worker_id);
@ -733,29 +729,26 @@ class G1EvacuateRegionsTask : public G1EvacuateRegionsBaseTask {
G1RootProcessor* _root_processor;
bool _has_optional_evacuation_work;
void scan_roots(G1ParScanThreadState* pss, uint worker_id) {
_root_processor->evacuate_roots(pss, worker_id);
_g1h->rem_set()->scan_heap_roots(pss, worker_id, G1GCPhaseTimes::ScanHR, G1GCPhaseTimes::ObjCopy, _has_optional_evacuation_work);
_g1h->rem_set()->scan_collection_set_code_roots(pss, worker_id, G1GCPhaseTimes::CodeRoots, G1GCPhaseTimes::ObjCopy);
void scan_roots(G1ParScanThreadState* pss) {
_root_processor->evacuate_roots(pss);
_g1h->rem_set()->scan_heap_roots(pss, G1GCPhaseTimes::ScanHR, G1GCPhaseTimes::ObjCopy, _has_optional_evacuation_work);
_g1h->rem_set()->scan_collection_set_code_roots(pss, G1GCPhaseTimes::CodeRoots, G1GCPhaseTimes::ObjCopy);
// There are no optional roots to scan right now.
#ifdef ASSERT
class VerifyOptionalCollectionSetRootsEmptyClosure : public G1HeapRegionClosure {
G1ParScanThreadState* _pss;
public:
VerifyOptionalCollectionSetRootsEmptyClosure(G1ParScanThreadState* pss) : _pss(pss) { }
bool do_heap_region(G1HeapRegion* r) override {
assert(!r->has_index_in_opt_cset(), "must be");
return false;
}
} cl(pss);
_g1h->collection_set_iterate_increment_from(&cl, worker_id);
} cl;
_g1h->collection_set_iterate_increment_from(&cl, pss->worker_id());
#endif
}
void evacuate_live_objects(G1ParScanThreadState* pss, uint worker_id) {
G1EvacuateRegionsBaseTask::evacuate_live_objects(pss, worker_id, G1GCPhaseTimes::ObjCopy, G1GCPhaseTimes::Termination);
void evacuate_live_objects(G1ParScanThreadState* pss) {
G1EvacuateRegionsBaseTask::evacuate_live_objects(pss, G1GCPhaseTimes::ObjCopy, G1GCPhaseTimes::Termination);
}
void start_work(uint worker_id) {
@ -767,8 +760,7 @@ class G1EvacuateRegionsTask : public G1EvacuateRegionsBaseTask {
}
public:
G1EvacuateRegionsTask(G1CollectedHeap* g1h,
G1ParScanThreadStateSet* per_thread_states,
G1EvacuateRegionsTask(G1ParScanThreadStateSet* per_thread_states,
G1ScannerTasksQueueSet* task_queues,
G1RootProcessor* root_processor,
uint num_workers,
@ -791,8 +783,7 @@ void G1YoungCollector::evacuate_initial_collection_set(G1ParScanThreadStateSet*
Ticks start_processing = Ticks::now();
{
G1RootProcessor root_processor(_g1h, num_workers > 1 /* is_parallel */);
G1EvacuateRegionsTask g1_par_task(_g1h,
per_thread_states,
G1EvacuateRegionsTask g1_par_task(per_thread_states,
task_queues(),
&root_processor,
num_workers,
@ -814,14 +805,14 @@ void G1YoungCollector::evacuate_initial_collection_set(G1ParScanThreadStateSet*
class G1EvacuateOptionalRegionsTask : public G1EvacuateRegionsBaseTask {
void scan_roots(G1ParScanThreadState* pss, uint worker_id) {
_g1h->rem_set()->scan_heap_roots(pss, worker_id, G1GCPhaseTimes::OptScanHR, G1GCPhaseTimes::OptObjCopy, true /* remember_already_scanned_cards */);
_g1h->rem_set()->scan_collection_set_code_roots(pss, worker_id, G1GCPhaseTimes::OptCodeRoots, G1GCPhaseTimes::OptObjCopy);
_g1h->rem_set()->scan_collection_set_optional_roots(pss, worker_id, G1GCPhaseTimes::OptScanHR, G1GCPhaseTimes::ObjCopy);
void scan_roots(G1ParScanThreadState* pss) {
_g1h->rem_set()->scan_heap_roots(pss, G1GCPhaseTimes::OptScanHR, G1GCPhaseTimes::OptObjCopy, true /* remember_already_scanned_cards */);
_g1h->rem_set()->scan_collection_set_code_roots(pss, G1GCPhaseTimes::OptCodeRoots, G1GCPhaseTimes::OptObjCopy);
_g1h->rem_set()->scan_collection_set_optional_roots(pss, G1GCPhaseTimes::OptScanHR, G1GCPhaseTimes::ObjCopy);
}
void evacuate_live_objects(G1ParScanThreadState* pss, uint worker_id) {
G1EvacuateRegionsBaseTask::evacuate_live_objects(pss, worker_id, G1GCPhaseTimes::OptObjCopy, G1GCPhaseTimes::OptTermination);
void evacuate_live_objects(G1ParScanThreadState* pss) {
G1EvacuateRegionsBaseTask::evacuate_live_objects(pss, G1GCPhaseTimes::OptObjCopy, G1GCPhaseTimes::OptTermination);
}
public:
@ -982,7 +973,7 @@ public:
G1STWIsAliveClosure is_alive(&_g1h);
G1CopyingKeepAliveClosure keep_alive(&_g1h, pss);
G1EnqueueDiscoveredFieldClosure enqueue(&_g1h, pss);
G1ParEvacuateFollowersClosure complete_gc(&_g1h, pss, &_task_queues, _tm == RefProcThreadModel::Single ? nullptr : &_terminator, G1GCPhaseTimes::ObjCopy);
G1ParEvacuateFollowersClosure complete_gc(pss, &_task_queues, _tm == RefProcThreadModel::Single ? nullptr : &_terminator, G1GCPhaseTimes::ObjCopy);
_rp_task->rp_work(worker_id, &is_alive, &keep_alive, &enqueue, &complete_gc);
// We have completed copying any necessary live referent objects.

77
src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp
View File

@ -122,52 +122,49 @@ public:
class G1PostEvacuateCollectionSetCleanupTask1::UpdateCodeRootsTask
: public G1AbstractSubTask
{
class ProcessRegionClosure : public G1HeapRegionClosure {
G1ParScanThreadStateSet* _psss;
public:
ProcessRegionClosure(G1ParScanThreadStateSet* psss) : _psss(psss) { }
bool do_heap_region(G1HeapRegion* r) override {
uint index = r->hrm_index();
size_t num_nmethods = 0;
for (uint i = 0; i < _psss->num_workers(); i++) {
G1ParScanThreadState* pss = _psss->state_for_worker(i);
num_nmethods += pss->num_nmethods(index);
}
if (num_nmethods != 0) {
// Notify the code root sets that we are going to add code roots.
r->rem_set()->prepare_for_adding_code_roots(num_nmethods);
// Add roots.
for (uint i = 0; i < _psss->num_workers(); i++) {
G1ParScanThreadState* pss = _psss->state_for_worker(i);
pss->iterate_nmethods(index, [&] (nmethod* nm) { r->add_code_root(nm); });
}
}
return false;
}
};
G1ParScanThreadStateSet* _psss;
G1HeapRegionClaimer _claimer;
public:
UpdateCodeRootsTask(G1ParScanThreadStateSet* per_thread_states)
: G1AbstractSubTask(G1GCPhaseTimes::UpdateCodeRoots), _psss(per_thread_states), _claimer(0) { }
: G1AbstractSubTask(G1GCPhaseTimes::UpdateCodeRoots), _psss(per_thread_states) { }
double worker_cost() const override {
return _psss->num_nmethod_regions_to_add();
}
void set_max_workers(uint max_workers) override {
_claimer.set_n_workers(max_workers);
}
double worker_cost() const override { return 1.0; }
// Add code roots serially to avoid lock and resize contention.
void do_work(uint worker_id) override {
ProcessRegionClosure cl(_psss);
_psss->par_iterate_nmethod_regions_to_add(&cl, &_claimer, worker_id);
G1CollectedHeap* g1h = G1CollectedHeap::heap();
uint max_regions = g1h->max_num_regions();
uint* counts = NEW_C_HEAP_ARRAY(uint, max_regions, mtGC);
memset(counts, 0, max_regions * sizeof(uint));
// Pass 1: count the number of nmethods to add per region across all workers.
for (uint i = 0; i < _psss->num_workers(); i++) {
G1ParScanThreadState* pss = _psss->state_for_worker(i);
const GrowableArrayCHeap<G1CodeRootPair, mtGC>& pairs = pss->code_root_pairs();
for (const G1CodeRootPair& pair : pairs) {
counts[pair._region_idx]++;
}
}
// Pass 2: pre-size each region's code root set once, then add all nmethods.
for (uint i = 0; i < _psss->num_workers(); i++) {
G1ParScanThreadState* pss = _psss->state_for_worker(i);
const GrowableArrayCHeap<G1CodeRootPair, mtGC>& pairs = pss->code_root_pairs();
for (const G1CodeRootPair& pair : pairs) {
uint region_idx = pair._region_idx;
G1HeapRegion* region = g1h->region_at(region_idx);
if (counts[region_idx] > 0) {
// First occurrence of this region: pre-size its code root set to the
// final size so it never needs to grow under the (single-threaded) add.
region->rem_set()->prepare_for_adding_code_roots(counts[region_idx]);
counts[region_idx] = 0;
}
region->add_code_root(pair._nmethod);
}
}
FREE_C_HEAP_ARRAY(counts);
}
};
@ -384,7 +381,7 @@ G1PostEvacuateCollectionSetCleanupTask1::G1PostEvacuateCollectionSetCleanupTask1
if (SampleCollectionSetCandidatesTask::should_execute()) {
add_serial_task(new SampleCollectionSetCandidatesTask());
}
add_parallel_task(new UpdateCodeRootsTask(per_thread_states));
add_serial_task(new UpdateCodeRootsTask(per_thread_states));
add_parallel_task(G1CollectedHeap::heap()->rem_set()->create_cleanup_after_scan_heap_roots_task());
if (evac_failed) {

10
src/hotspot/share/gc/shared/c1/cardTableBarrierSetC1.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -22,6 +22,7 @@
*
*/
#include "c1/c1_IR.hpp"
#include "ci/ciInlineKlass.hpp"
#include "code/aotCodeCache.hpp"
#include "gc/shared/c1/cardTableBarrierSetC1.hpp"
@ -49,7 +50,12 @@ void CardTableBarrierSetC1::store_at_resolved(LIRAccess& access, LIR_Opr value)
ciField* field = vk->nonstatic_field_at(i);
if (!field->type()->is_primitive_type()) {
int off = access.offset().opr().as_jint() + field->offset_in_bytes() - vk->payload_offset();
LIRAccess inner_access(access.gen(), decorators, access.base(), LIR_OprFact::intConst(off), field->type()->basic_type(), access.patch_emit_info(), access.access_emit_info());
// Each pre-barrier needs its own CodeEmitInfo
CodeEmitInfo* info = access.patch_emit_info();
if (info != nullptr) {
info = new CodeEmitInfo(info);
}
LIRAccess inner_access(access.gen(), decorators, access.base(), LIR_OprFact::intConst(off), field->type()->basic_type(), info, access.access_emit_info());
pre_barrier(inner_access, resolve_address(inner_access, false),
LIR_OprFact::illegalOpr /* pre_val */, inner_access.patch_emit_info());
}

2
src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp
View File

@ -444,7 +444,7 @@ void ZBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* a
if (offset != arrayOopDesc::base_offset_in_bytes(T_OBJECT)) {
assert(UseCompactObjectHeaders, "should only happen with COH");
assert((arrayOopDesc::base_offset_in_bytes(T_OBJECT) - offset) == BytesPerLong, "unexpected offset");
length = phase->transform_later(new SubXNode(length, phase->longcon(1))); // Size is in longs
length = phase->transform_later(new SubXNode(length, phase->MakeConX(1))); // Size is in longs
src_offset = phase->longcon(arrayOopDesc::base_offset_in_bytes(T_OBJECT));
dest_offset = src_offset;
}

11
src/hotspot/share/memory/oopFactory.cpp
View File

@ -79,7 +79,8 @@ typeArrayOop oopFactory::new_longArray(int length, TRAPS) {
// create java.lang.Object[]
refArrayOop oopFactory::new_objectArray(int length, TRAPS) {
return Universe::objectArrayKlass()->allocate_instance(length, THREAD);
objArrayOop array = Universe::objectArrayKlass()->allocate_instance(length, CHECK_NULL);
return oop_cast<refArrayOop>(array);
}
typeArrayOop oopFactory::new_charArray(const char* utf8_str, TRAPS) {
@ -117,7 +118,7 @@ objArrayOop oopFactory::new_objArray(Klass* klass, int length, ArrayProperties p
}
objArrayOop oopFactory::new_objArray(Klass* klass, int length, TRAPS) {
return new_objArray(klass, length, ArrayProperties::Default(), THREAD);
return new_objArray(klass, length, ArrayProperties::Default(), THREAD);
}
refArrayOop oopFactory::new_refArray(Klass* klass, int length, ArrayProperties properties, TRAPS) {
@ -126,7 +127,8 @@ refArrayOop oopFactory::new_refArray(Klass* klass, int length, ArrayProperties p
ObjArrayKlass* oak = ObjArrayKlass::cast(ak)->klass_from_description(ad, CHECK_NULL);
// Cast below must pass because the array description required a RefArrayKlass
RefArrayKlass* rak = RefArrayKlass::cast(oak);
return rak->allocate_instance(length, CHECK_NULL);
objArrayOop array = rak->allocate_instance(length, CHECK_NULL);
return oop_cast<refArrayOop>(array);
}
refArrayOop oopFactory::new_refArray(Klass* klass, int length, TRAPS) {
@ -138,7 +140,8 @@ flatArrayOop oopFactory::new_flatArray(InlineKlass* ik, int length, ArrayPropert
ObjArrayKlass* oak = ObjArrayKlass::cast(ak)->klass_with_properties(props, CHECK_NULL);
FlatArrayKlass* fak = FlatArrayKlass::cast(oak);
return fak->allocate_instance(length, THREAD);
objArrayOop array = fak->allocate_instance(length, CHECK_NULL);
return oop_cast<flatArrayOop>(array);
}
refArrayHandle oopFactory::new_refArray_handle(Klass* klass, int length, TRAPS) {

4
src/hotspot/share/oops/flatArrayKlass.cpp
View File

@ -146,12 +146,12 @@ void FlatArrayKlass::metaspace_pointers_do(MetaspaceClosure* it) {
}
// Oops allocation...
flatArrayOop FlatArrayKlass::allocate_instance(int length, TRAPS) {
objArrayOop FlatArrayKlass::allocate_instance(int length, TRAPS) {
assert(UseArrayFlattening, "Must be enabled");
check_array_allocation_length(length, max_elements(), CHECK_NULL);
int size = flatArrayOopDesc::object_size(layout_helper(), length);
oop array = Universe::heap()->array_allocate(this, size, length, true, CHECK_NULL);
return oop_cast<flatArrayOop>(array);
return oop_cast<objArrayOop>(array);
}
oop FlatArrayKlass::multi_allocate(int rank, jint* last_size, TRAPS) {

2
src/hotspot/share/oops/flatArrayKlass.hpp
View File

@ -98,7 +98,7 @@ class FlatArrayKlass : public ObjArrayKlass {
size_t oop_size(oop obj) const override;
// Oop Allocation
flatArrayOop allocate_instance(int length, TRAPS);
objArrayOop allocate_instance(int length, TRAPS) override final;
oop multi_allocate(int rank, jint* sizes, TRAPS) override;

17
src/hotspot/share/oops/instanceKlass.cpp
View File

@ -1583,7 +1583,7 @@ void InstanceKlass::initialize_impl(TRAPS) {
call_class_initializer(THREAD);
}
if (has_strict_static_fields() && !HAS_PENDING_EXCEPTION) {
if (has_strict_static_fields() && !HAS_PENDING_EXCEPTION && !ReplayCompiles) {
// Step 9 also verifies that strict static fields have been initialized.
// Status bits were set in ClassFileParser::post_process_parsed_stream.
// After <clinit>, bits must all be clear, or else we must throw an error.
@ -2134,12 +2134,25 @@ bool InstanceKlass::find_local_field(Symbol* name, Symbol* sig, fieldDescriptor*
if (fs.lookup(name, sig)) {
assert(fs.name() == name, "name must match");
assert(fs.signature() == sig, "signature must match");
fd->reinitialize(const_cast<InstanceKlass*>(this), fs.to_FieldInfo());
fd->reinitialize(this, fs.to_FieldInfo());
return true;
}
return false;
}
bool InstanceKlass::find_local_field(Symbol* name, Symbol* sig, fieldDescriptor* fd, bool also_internal) const {
if (!also_internal) {
return find_local_field( name, sig, fd);
}
for (AllFieldStream fs(this); !fs.done(); fs.next()) {
if (fs.name() == name && fs.signature() == sig) {
fd->reinitialize(this, fs.to_FieldInfo());
return true;
}
}
return false;
}
Klass* InstanceKlass::find_interface_field(Symbol* name, Symbol* sig, fieldDescriptor* fd) const {
const int n = local_interfaces()->length();

2
src/hotspot/share/oops/instanceKlass.hpp
View File

@ -669,6 +669,8 @@ public:
// find local field, returns true if found
bool find_local_field(Symbol* name, Symbol* sig, fieldDescriptor* fd) const;
// find local field, returns true if found
bool find_local_field(Symbol* name, Symbol* sig, fieldDescriptor* fd, bool also_internal) const;
// find field in direct superinterfaces, returns the interface in which the field is defined
Klass* find_interface_field(Symbol* name, Symbol* sig, fieldDescriptor* fd) const;
// find field according to JVM spec 5.4.3.2, returns the klass in which the field is defined

14
src/hotspot/share/oops/objArrayKlass.cpp
View File

@ -237,18 +237,12 @@ ObjArrayKlass* ObjArrayKlass::allocate_klass_from_description(ArrayDescription a
}
objArrayOop ObjArrayKlass::allocate_instance(int length, ArrayProperties props, TRAPS) {
check_array_allocation_length(length, arrayOopDesc::max_array_length(T_OBJECT), CHECK_NULL);
ObjArrayKlass* ak = klass_with_properties(props, CHECK_NULL);
switch (ak->kind()) {
case Klass::RefArrayKlassKind:
return RefArrayKlass::cast(ak)->allocate_instance(length, CHECK_NULL);
return ak->allocate_instance(length, CHECK_NULL);
}
case Klass::FlatArrayKlassKind:
return FlatArrayKlass::cast(ak)->allocate_instance(length, CHECK_NULL);
default:
ShouldNotReachHere();
}
objArrayOop ObjArrayKlass::allocate_instance(int length, TRAPS) {
ShouldNotReachHere();
}
oop ObjArrayKlass::multi_allocate(int rank, jint* sizes, TRAPS) {

1
src/hotspot/share/oops/objArrayKlass.hpp
View File

@ -100,6 +100,7 @@ class ObjArrayKlass : public ArrayKlass {
int n, Klass* element_klass, TRAPS);
oop multi_allocate(int rank, jint* sizes, TRAPS) override;
virtual objArrayOop allocate_instance(int length, TRAPS);
// Copying
void copy_array(arrayOop s, int src_pos, arrayOop d, int dst_pos, int length, TRAPS) override;

4
src/hotspot/share/oops/refArrayKlass.cpp
View File

@ -111,12 +111,12 @@ size_t RefArrayKlass::oop_size(oop obj) const {
return refArrayOop(obj)->object_size();
}
refArrayOop RefArrayKlass::allocate_instance(int length, TRAPS) {
objArrayOop RefArrayKlass::allocate_instance(int length, TRAPS) {
check_array_allocation_length(length, arrayOopDesc::max_array_length(T_OBJECT), CHECK_NULL);
size_t size = refArrayOopDesc::object_size(length);
oop array = Universe::heap()->array_allocate(
this, size, length, /* do_zero */ true, CHECK_NULL);
return oop_cast<refArrayOop>(array);
return oop_cast<objArrayOop>(array);
}
static void throw_array_null_pointer_store_exception(arrayOop src, arrayOop dst, TRAPS) {

2
src/hotspot/share/oops/refArrayKlass.hpp
View File

@ -60,7 +60,7 @@ class RefArrayKlass : public ObjArrayKlass {
int n, Klass* element_klass,
ArrayProperties props, TRAPS);
refArrayOop allocate_instance(int length, TRAPS);
objArrayOop allocate_instance(int length, TRAPS) override final;
// Copying
void copy_array(arrayOop s, int src_pos, arrayOop d, int dst_pos, int length, TRAPS) override;

2
src/hotspot/share/oops/symbol.hpp
View File

@ -249,7 +249,7 @@ class Symbol : public MetaspaceObj {
int index_of_at(int i, const char* substr, int substr_len) const;
// Three-way compare for sorting; returns -1/0/1 if receiver is </==/> than arg
// note that the ordering is not alfabetical
// note that the ordering is not alphabetical
inline int fast_compare(const Symbol* other) const;
// Returns receiver converted to null-terminated UTF-8 string; string is

2
src/hotspot/share/opto/bytecodeInfo.cpp
View File

@ -89,7 +89,7 @@ static bool is_init_with_ea(ciMethod* callee_method,
if (callee_method->is_object_constructor()) {
return true; // constructor
}
if ((caller_method->is_object_constructor() || caller_method->is_class_initializer()) &&
if (caller_method->is_object_constructor() &&
caller_method != C->method() &&
caller_method->holder()->is_subclass_of(callee_method->holder())) {
return true; // super constructor is called from inlined constructor

38
src/hotspot/share/opto/callGenerator.cpp
View File

@ -27,6 +27,7 @@
#include "ci/ciMemberName.hpp"
#include "ci/ciMethodHandle.hpp"
#include "ci/ciObjArray.hpp"
#include "ci/ciStreams.hpp"
#include "classfile/javaClasses.hpp"
#include "compiler/compileLog.hpp"
#include "oops/accessDecorators.hpp"
@ -772,13 +773,19 @@ void CallGenerator::do_late_inline_helper() {
Node* buffer_oop = nullptr;
ciMethod* inline_method = inline_cg()->method();
ciType* return_type = inline_method->return_type();
if (!call->tf()->returns_inline_type_as_fields() &&
return_type->is_inlinetype() && return_type->as_inline_klass()->can_be_returned_as_fields()) {
assert(is_mh_late_inline(), "Unexpected return type");
// Allocate a buffer for the inline type returned as fields because the caller expects an oop return.
// Moving this after the call would require distinct JVM states: a next-BCI state with the result for
// deoptimization at an allocation safepoint and an invoke-BCI state for exceptions like OOME. The
// pre-call state can safely execute the call if allocation deoptimizes.
bool needs_return_buffer = !call->tf()->returns_inline_type_as_fields() &&
return_type->is_inlinetype() &&
return_type->as_inline_klass()->can_be_returned_as_fields();
// A non-null scalarized return would require a buffer. Since allocating that buffer could
// initialize the value class, speculate that the result is null and deoptimize otherwise.
bool assert_null_return = needs_return_buffer && !return_type->as_inline_klass()->is_initialized();
assert(!needs_return_buffer || is_mh_late_inline(), "Unexpected return type");
// Allocate a buffer for the inline type returned as fields because the caller expects an oop return.
// Do this before the method handle call in case the buffer allocation triggers deoptimization and
// we need to "re-execute" the call in the interpreter (to make sure the call is only executed once).
if (needs_return_buffer && !assert_null_return) {
GraphKit arg_kit(jvms, &gvn);
{
PreserveReexecuteState preexecs(&arg_kit);
@ -840,6 +847,20 @@ void CallGenerator::do_late_inline_helper() {
if (vt != nullptr) {
if (call->tf()->returns_inline_type_as_fields()) {
vt->replace_call_results(&kit, call, C);
} else if (assert_null_return && !vt->is_allocated(&kit.gvn())) {
// Deoptimize if the result is non-null.
// Put the trap at the next bytecode to avoid re-executing the method handle call.
ciBytecodeStream iter(kit.method());
iter.force_bci(kit.bci());
assert(Bytecodes::is_invoke(iter.cur_bc()), "unexpected bytecode: %s", Bytecodes::name(iter.cur_bc()));
int bci = kit.bci();
kit.push(vt);
kit.set_bci(iter.next_bci());
result = kit.null_assert(vt);
kit.set_bci(bci);
if (!kit.stopped()) {
result = kit.pop();
}
} else {
// Result might still be allocated (for example, if it has been stored to a non-flat field)
if (!vt->is_allocated(&kit.gvn())) {
@ -868,8 +889,8 @@ void CallGenerator::do_late_inline_helper() {
oop->init_req(2, buffer_oop);
mem->init_req(2, kit.merged_memory());
// Update oop input to buffer
kit.gvn().hash_delete(vt);
// Use cloned InlineTypeNode to propagate oop from now on
vt = vt->clone_if_required(&kit.gvn(), kit.map());
vt->set_oop(kit.gvn(), kit.gvn().transform(oop));
vt->set_is_buffered(kit.gvn());
vt = kit.gvn().transform(vt)->as_InlineType();
@ -899,6 +920,7 @@ void CallGenerator::do_late_inline_helper() {
}
}
C->set_do_cleanup(kit.stopped()); // path is dead; needs cleanup
kit.replace_call(call, result, true, do_asserts);
}
}

18
src/hotspot/share/opto/callnode.cpp
View File

@ -85,7 +85,7 @@ const RegMask &StartNode::in_RegMask(uint) const {
//------------------------------match------------------------------------------
// Construct projections for incoming parameters, and their RegMask info
Node *StartNode::match(const ProjNode *proj, const Matcher *match, const RegMask* mask) {
Node* StartNode::match(const ProjNode* proj, const Matcher* match) {
switch (proj->_con) {
case TypeFunc::Control:
case TypeFunc::I_O:
@ -782,15 +782,16 @@ void CallNode::calling_convention(BasicType* sig_bt, VMRegPair *parm_regs, uint
//------------------------------match------------------------------------------
// Construct projections for control, I/O, memory-fields, ..., and
// return result(s) along with their RegMask info
Node *CallNode::match(const ProjNode *proj, const Matcher *match, const RegMask* mask) {
Node* CallNode::match(const ProjNode* proj, const Matcher* match) {
uint con = proj->_con;
const TypeTuple* range_cc = tf()->range_cc();
const TypeTuple* range_cc = _tf->range_cc();
if (con >= TypeFunc::Parms) {
if (tf()->returns_inline_type_as_fields()) {
if (_tf->returns_inline_type_as_fields()) {
// The call returns multiple values (inline type fields): we
// create one projection per returned value.
assert(con <= TypeFunc::Parms+1 || InlineTypeReturnedAsFields, "only for multi value return");
uint ideal_reg = range_cc->field_at(con)->ideal_reg();
const RegMask* mask = match->return_values_mask(_tf);
return new MachProjNode(this, con, mask[con-TypeFunc::Parms], ideal_reg);
} else {
if (con == TypeFunc::Parms) {
@ -1403,7 +1404,7 @@ bool CallStaticJavaNode::remove_unknown_flat_array_load(PhaseIterGVN* igvn, Node
Node* CallStaticJavaNode::replace_is_substitutable(PhaseIterGVN* igvn) {
Node* left = in(TypeFunc::Parms);
Node* right = in(TypeFunc::Parms + 1);
if (!InlineTypeNode::can_emit_substitutability_check(left, right)) {
if (!InlineTypeNode::can_emit_substitutability_check(igvn, left, right)) {
return nullptr;
}
@ -1732,7 +1733,9 @@ Node *SafePointNode::Ideal(PhaseGVN *phase, bool can_reshape) {
for (uint i = jvms()->debug_start(); i < jvms()->debug_end(); i++) {
Node* n = in(i)->uncast();
if (n->is_InlineType()) {
n->as_InlineType()->make_scalar_in_safepoints(phase->is_IterGVN(), true, this);
if (!n->as_InlineType()->make_scalar_in_safepoints(phase->is_IterGVN(), true, this)) {
return nullptr;
}
}
}
}
@ -2052,8 +2055,7 @@ AllocateNode::AllocateNode(Compile* C, const TypeFunc *atype,
void AllocateNode::compute_MemBar_redundancy(ciMethod* initializer)
{
assert(initializer != nullptr &&
(initializer->is_object_constructor() || initializer->is_class_initializer()),
assert(initializer != nullptr && initializer->is_object_constructor(),
"unexpected initializer method");
BCEscapeAnalyzer* analyzer = initializer->get_bcea();
if (analyzer == nullptr) {

4
src/hotspot/share/opto/callnode.hpp
View File

@ -76,7 +76,7 @@ public:
virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
virtual void calling_convention( BasicType* sig_bt, VMRegPair *parm_reg, uint length ) const;
virtual const RegMask &in_RegMask(uint) const;
virtual Node *match(const ProjNode *proj, const Matcher *m, const RegMask* mask);
virtual Node* match(const ProjNode* proj, const Matcher* m);
virtual uint ideal_reg() const { return 0; }
#ifndef PRODUCT
virtual void dump_spec(outputStream *st) const;
@ -799,7 +799,7 @@ public:
virtual bool cmp(const Node &n) const;
virtual uint size_of() const = 0;
virtual void calling_convention(BasicType* sig_bt, VMRegPair* parm_regs, uint argcnt) const;
virtual Node* match(const ProjNode* proj, const Matcher* m, const RegMask* mask);
virtual Node* match(const ProjNode* proj, const Matcher* m);
virtual uint ideal_reg() const { return NotAMachineReg; }
// Are we guaranteed that this node is a safepoint? Not true for leaf calls and
// for some macro nodes whose expansion does not have a safepoint on the fast path.

5
src/hotspot/share/opto/cfgnode.cpp
View File

@ -3115,6 +3115,7 @@ private:
_clones.map(phi->_idx, vt);
Node_List casts;
for (uint i = 1; i < phi->req(); ++i) {
assert(casts.size() == 0, "must be cleared");
Node* n = phi->in(i);
if (n == nullptr) {
continue;
@ -3142,6 +3143,10 @@ private:
n->as_InlineType()->set_oop(*_phase, _phase->transform(cast));
n = _phase->transform(n);
if (n->is_top()) {
if (casts.size() > 0) {
// We could be skipping some unprocessed casts that are also dead. Clear the list for the next phi input.
casts.clear();
}
break;
}
}

23
src/hotspot/share/opto/compile.cpp
View File

@ -2130,7 +2130,10 @@ void Compile::process_inline_types(PhaseIterGVN &igvn, bool remove) {
set_scalarize_in_safepoints(true);
for (int i = _inline_type_nodes.length()-1; i >= 0; i--) {
InlineTypeNode* vt = _inline_type_nodes.at(i)->as_InlineType();
vt->make_scalar_in_safepoints(&igvn);
if (!vt->make_scalar_in_safepoints(&igvn)) {
record_failure("out of nodes during scalarization");
return;
}
igvn.record_for_igvn(vt);
}
if (remove) {
@ -3062,9 +3065,9 @@ void Compile::Optimize() {
if (failing()) return;
if (AlwaysIncrementalInline || StressIncrementalInlining) {
inline_incrementally(igvn);
}
// inline_boxing_calls() may introduce new late inline candidates
// in stress modes or w/ some compile directives.
inline_incrementally(igvn);
print_method(PHASE_INCREMENTAL_BOXING_INLINE, 2);
@ -3113,6 +3116,9 @@ void Compile::Optimize() {
// Process inline type nodes now that all inlining is over
process_inline_types(igvn);
if (failing()) {
return;
}
adjust_flat_array_access_aliases(igvn);
@ -3299,6 +3305,9 @@ void Compile::Optimize() {
// Process inline types before macro expansion. Otherwise, we will not be able to
// remove unused allocations because it cannot match the expanded allocation.
process_inline_types(igvn);
if (failing()) {
return;
}
{
TracePhase tp(_t_macroExpand);
@ -3332,6 +3341,9 @@ void Compile::Optimize() {
// Process inline type nodes again and remove them. From here
// on we don't need to keep track of field values anymore.
process_inline_types(igvn, /* remove= */ true);
if (failing()) {
return;
}
{
TracePhase tp(_t_barrierExpand);
@ -4926,6 +4938,7 @@ bool Compile::final_graph_reshaping() {
bool Compile::too_many_traps(ciMethod* method,
int bci,
Deoptimization::DeoptReason reason) {
assert(reason > Deoptimization::Reason_none && reason <= Deoptimization::Reason_LIMIT, "invalid reason");
ciMethodData* md = method->method_data();
if (md->is_empty()) {
// Assume the trap has not occurred, or that it occurred only
@ -4951,6 +4964,7 @@ bool Compile::too_many_traps(ciMethod* method,
// Less-accurate variant which does not require a method and bci.
bool Compile::too_many_traps(Deoptimization::DeoptReason reason,
ciMethodData* logmd) {
assert(reason > Deoptimization::Reason_none && reason <= Deoptimization::Reason_LIMIT, "invalid reason");
if (trap_count(reason) >= Deoptimization::per_method_trap_limit(reason)) {
// Too many traps globally.
// Note that we use cumulative trap_count, not just md->trap_count.
@ -4975,6 +4989,7 @@ bool Compile::too_many_traps(Deoptimization::DeoptReason reason,
bool Compile::too_many_recompiles(ciMethod* method,
int bci,
Deoptimization::DeoptReason reason) {
assert(reason > Deoptimization::Reason_none && reason <= Deoptimization::Reason_LIMIT, "invalid reason");
ciMethodData* md = method->method_data();
if (md->is_empty()) {
// Assume the trap has not occurred, or that it occurred only

8
src/hotspot/share/opto/divnode.cpp
View File

@ -1658,7 +1658,7 @@ DivModLNode* DivModLNode::make(Node* div_or_mod) {
//------------------------------match------------------------------------------
// return result(s) along with their RegMask info
Node *DivModINode::match(const ProjNode *proj, const Matcher *match, const RegMask* mask) {
Node* DivModINode::match(const ProjNode* proj, const Matcher* match) {
uint ideal_reg = proj->ideal_reg();
RegMask rm;
if (proj->_con == first_proj_num) {
@ -1673,7 +1673,7 @@ Node *DivModINode::match(const ProjNode *proj, const Matcher *match, const RegMa
//------------------------------match------------------------------------------
// return result(s) along with their RegMask info
Node *DivModLNode::match(const ProjNode *proj, const Matcher *match, const RegMask* mask) {
Node* DivModLNode::match(const ProjNode* proj, const Matcher* match) {
uint ideal_reg = proj->ideal_reg();
RegMask rm;
if (proj->_con == first_proj_num) {
@ -1711,7 +1711,7 @@ UDivModLNode* UDivModLNode::make(Node* div_or_mod) {
//------------------------------match------------------------------------------
// return result(s) along with their RegMask info
Node* UDivModINode::match(const ProjNode* proj, const Matcher* match, const RegMask* mask) {
Node* UDivModINode::match(const ProjNode* proj, const Matcher* match) {
uint ideal_reg = proj->ideal_reg();
RegMask rm;
if (proj->_con == first_proj_num) {
@ -1726,7 +1726,7 @@ Node* UDivModINode::match(const ProjNode* proj, const Matcher* match, const RegM
//------------------------------match------------------------------------------
// return result(s) along with their RegMask info
Node* UDivModLNode::match( const ProjNode* proj, const Matcher* match, const RegMask* mask) {
Node* UDivModLNode::match(const ProjNode* proj, const Matcher* match) {
uint ideal_reg = proj->ideal_reg();
RegMask rm;
if (proj->_con == first_proj_num) {

8
src/hotspot/share/opto/divnode.hpp
View File

@ -255,7 +255,7 @@ public:
DivModINode(Node* ctrl, Node* dividend, Node* divisor) : DivModNode(ctrl, dividend, divisor) {}
virtual int Opcode() const;
virtual const Type *bottom_type() const { return TypeTuple::INT_PAIR; }
virtual Node *match(const ProjNode *proj, const Matcher *m, const RegMask* mask);
virtual Node* match(const ProjNode* proj, const Matcher* m);
// Make a divmod and associated projections from a div or mod.
static DivModINode* make(Node* div_or_mod);
@ -268,7 +268,7 @@ public:
DivModLNode(Node* ctrl, Node* dividend, Node* divisor) : DivModNode(ctrl, dividend, divisor) {}
virtual int Opcode() const;
virtual const Type *bottom_type() const { return TypeTuple::LONG_PAIR; }
virtual Node *match(const ProjNode *proj, const Matcher *m, const RegMask* mask);
virtual Node *match(const ProjNode* proj, const Matcher* m);
// Make a divmod and associated projections from a div or mod.
static DivModLNode* make(Node* div_or_mod);
@ -282,7 +282,7 @@ public:
UDivModINode(Node* ctrl, Node* dividend, Node* divisor) : DivModNode(ctrl, dividend, divisor) {}
virtual int Opcode() const;
virtual const Type *bottom_type() const { return TypeTuple::INT_PAIR; }
virtual Node* match(const ProjNode* proj, const Matcher* m, const RegMask* mask);
virtual Node* match(const ProjNode* proj, const Matcher* m);
// Make a divmod and associated projections from a div or mod.
static UDivModINode* make(Node* div_or_mod);
@ -295,7 +295,7 @@ public:
UDivModLNode(Node* ctrl, Node* dividend, Node* divisor) : DivModNode(ctrl, dividend, divisor) {}
virtual int Opcode() const;
virtual const Type *bottom_type() const { return TypeTuple::LONG_PAIR; }
virtual Node* match(const ProjNode* proj, const Matcher* m, const RegMask* mask);
virtual Node* match(const ProjNode* proj, const Matcher* m);
// Make a divmod and associated projections from a div or mod.
static UDivModLNode* make(Node* div_or_mod);

5
src/hotspot/share/opto/escape.cpp
View File

@ -1374,7 +1374,10 @@ bool ConnectionGraph::reduce_phi_on_safepoints_helper(Node* ophi, Node* cast, No
const bool allow_oop = !merge_t->is_flat();
for (uint j = 0; j < value_worklist.size(); ++j) {
InlineTypeNode* vt = value_worklist.at(j)->as_InlineType();
vt->make_scalar_in_safepoints(_igvn, allow_oop);
if (!vt->make_scalar_in_safepoints(_igvn, allow_oop)) {
sfpt->restore_non_debug_edges(non_debug_edges_worklist);
return false;
}
}
}

2
src/hotspot/share/opto/graphKit.hpp
View File

@ -772,7 +772,7 @@ class GraphKit : public Phase {
}
bool too_many_traps_or_recompiles(Deoptimization::DeoptReason reason) {
return C->too_many_traps_or_recompiles(method(), bci(), reason);
return C->too_many_traps_or_recompiles(method(), bci(), reason);
}
// Returns the object (if any) which was created the moment before.

48
src/hotspot/share/opto/inlinetypenode.cpp
View File

@ -324,11 +324,11 @@ void InlineTypeNode::make_scalar_in_safepoint(PhaseIterGVN* igvn, Unique_Node_Li
}
}
void InlineTypeNode::make_scalar_in_safepoints(PhaseIterGVN* igvn, bool allow_oop) {
make_scalar_in_safepoints(igvn, allow_oop, nullptr);
bool InlineTypeNode::make_scalar_in_safepoints(PhaseIterGVN* igvn, bool allow_oop) {
return make_scalar_in_safepoints(igvn, allow_oop, nullptr);
}
void InlineTypeNode::make_scalar_in_safepoints(PhaseIterGVN* igvn, bool allow_oop, SafePointNode* safepoint) {
bool InlineTypeNode::make_scalar_in_safepoints(PhaseIterGVN* igvn, bool allow_oop, SafePointNode* safepoint) {
// If the inline type has a constant or loaded oop, use the oop instead of scalarization
// in the safepoint to avoid keeping field loads live just for the debug info.
Node* oop = get_oop();
@ -378,8 +378,13 @@ void InlineTypeNode::make_scalar_in_safepoints(PhaseIterGVN* igvn, bool allow_oo
safepoints.push(safepoint);
}
// Scalarize the inline type in all safepoint uses but first check if we
// have enough nodes left to create a new SafePointScalarObjectNode per use.
Compile* C = igvn->C;
if ((C->live_nodes() + safepoints.size() + NodeLimitFudgeFactor) > C->max_node_limit()) {
return false;
}
Unique_Node_List vt_worklist;
// Process all safepoint uses and scalarize inline type
while (safepoints.size() > 0) {
SafePointNode* sfpt = safepoints.pop()->as_SafePoint();
if (use_oop) {
@ -397,11 +402,14 @@ void InlineTypeNode::make_scalar_in_safepoints(PhaseIterGVN* igvn, bool allow_oo
// Now scalarize non-flat fields
for (uint i = 0; i < vt_worklist.size(); ++i) {
InlineTypeNode* vt = vt_worklist.at(i)->isa_InlineType();
vt->make_scalar_in_safepoints(igvn);
if (!vt->make_scalar_in_safepoints(igvn)) {
return false;
}
}
if (outcnt() == 0) {
igvn->record_for_igvn(this);
}
return true;
}
void InlineTypeNode::load(GraphKit* kit, Node* base, Node* ptr, bool immutable_memory, bool trust_null_free_oop, DecoratorSet decorators) {
@ -598,14 +606,32 @@ static bool check_cycle(ciInlineKlass* vk) {
return false;
}
// Check if 'lhs' and 'rhs' are the same oop, possibly wrapped in an InlineTypeNode.
static bool same_oop(PhaseGVN* phase, Node* lhs, Node* rhs) {
InlineTypeNode* lhs_inline = lhs->isa_InlineType();
if (lhs_inline != nullptr && lhs_inline->is_allocated(phase)) {
lhs = lhs_inline->get_oop();
}
InlineTypeNode* rhs_inline = rhs->isa_InlineType();
if (rhs_inline != nullptr && rhs_inline->is_allocated(phase)) {
rhs = rhs_inline->get_oop();
}
return lhs->eqv_uncast(rhs);
}
// Check if a substitutability check between 'lhs' and 'rhs' can be implemented in IR
bool InlineTypeNode::can_emit_substitutability_check(Node* lhs, Node* rhs) {
bool InlineTypeNode::can_emit_substitutability_check(PhaseGVN* phase, Node* lhs, Node* rhs) {
// We can't create new InlineTypeNodes after macro expansion
if (!phase->C->allow_macro_nodes()) {
return false;
}
if (!lhs->bottom_type()->isa_ptr() ||
(rhs != nullptr && !rhs->bottom_type()->isa_ptr())) {
return false;
}
if (rhs != nullptr && lhs->eqv_uncast(rhs)) {
if (rhs != nullptr && same_oop(phase, lhs, rhs)) {
return true;
}
@ -641,7 +667,7 @@ bool InlineTypeNode::can_emit_substitutability_check(Node* lhs, Node* rhs) {
Node* lhs_fv = lhs_inline->field_value(i);
Node* rhs_fv = rhs_inline != nullptr ? rhs_inline->field_value(i) : nullptr;
if (!can_emit_substitutability_check(lhs_fv, rhs_fv)) {
if (!can_emit_substitutability_check(phase, lhs_fv, rhs_fv)) {
return false;
}
}
@ -696,7 +722,7 @@ static Node* emit_substitutability_check_pointer(GraphKit* kit, PhiNode* result,
}
Node* cmp = nullptr;
if (lhs->eqv_uncast(rhs)) {
if (same_oop(&gvn, lhs, rhs)) {
cmp = kit->intcon(0);
} else if (!lhs_type->is_ptr()->can_be_inline_type() || !rhs_type->is_ptr()->can_be_inline_type()) {
// If one of the sides is not a value object, can only be substitutable if they are the same
@ -2287,10 +2313,12 @@ const Type* LoadFlatNode::Value(PhaseGVN* phase) const {
}
const Type* StoreFlatNode::Value(PhaseGVN* phase) const {
Node* val = in(TypeFunc::Parms + 2);
if (phase->type(in(TypeFunc::Control)) == Type::TOP || phase->type(in(TypeFunc::Memory)) == Type::TOP ||
phase->type(base()) == Type::TOP || phase->type(ptr()) == Type::TOP || phase->type(value()) == Type::TOP) {
phase->type(base()) == Type::TOP || phase->type(ptr()) == Type::TOP || phase->type(val) == Type::TOP) {
return Type::TOP;
}
assert(val->is_InlineType(), "must be InlineTypeNode: %s", val->Name());
return bottom_type();
}

6
src/hotspot/share/opto/inlinetypenode.hpp
View File

@ -122,9 +122,9 @@ public:
uint field_index(int offset) const;
// Replace InlineTypeNodes in debug info at safepoints with SafePointScalarObjectNodes
void make_scalar_in_safepoints(PhaseIterGVN* igvn, bool allow_oop = true);
[[nodiscard]] bool make_scalar_in_safepoints(PhaseIterGVN* igvn, bool allow_oop = true);
// Variant that allows to limit to a single safepoint. If nullptr is given, all safepoint uses will be considered.
void make_scalar_in_safepoints(PhaseIterGVN* igvn, bool allow_oop, SafePointNode* safepoint);
[[nodiscard]] bool make_scalar_in_safepoints(PhaseIterGVN* igvn, bool allow_oop, SafePointNode* safepoint);
// Store the inline type as a flat (headerless) representation
void store_flat(GraphKit* kit, Node* base, Node* ptr, bool atomic, bool immutable_memory, bool null_free, DecoratorSet decorators);
@ -132,7 +132,7 @@ public:
void store_flat_array(GraphKit* kit, Node* base, Node* idx);
// Implementation of the substitutability check for acmp
static bool can_emit_substitutability_check(Node* lhs, Node* rhs);
static bool can_emit_substitutability_check(PhaseGVN* phase, Node* lhs, Node* rhs);
static Node* emit_substitutability_check(GraphKit* kit, Node* lhs, Node* rhs);
// Allocates the inline type (if not yet allocated)

77
src/hotspot/share/opto/library_call.cpp
View File

@ -64,6 +64,7 @@
#include "prims/jvmtiExport.hpp"
#include "prims/jvmtiThreadState.hpp"
#include "prims/unsafe.hpp"
#include "runtime/arguments.hpp"
#include "runtime/globals.hpp"
#include "runtime/jniHandles.inline.hpp"
#include "runtime/mountUnmountDisabler.hpp"
@ -2782,13 +2783,17 @@ bool LibraryCallKit::inline_unsafe_flat_access(bool is_store, AccessKind kind) {
value = new_value;
}
assert(value_type->inline_klass() == value_klass, "value is of type %s while valueType is %s", value_type->inline_klass()->name()->as_utf8(), value_klass->name()->as_utf8());
assert(value_type == TypePtr::NULL_PTR || value_type->inline_klass() == value_klass,
"value is of type %s while value klass is %s", value_type->inline_klass()->name()->as_utf8(), value_klass->name()->as_utf8());
if (layout == LayoutKind::REFERENCE) {
const TypePtr* ptr_type = (decorators & C2_MISMATCHED) != 0 ? TypeRawPtr::BOTTOM : _gvn.type(ptr)->is_ptr();
access_store_at(base, ptr, ptr_type, value, value_type, T_OBJECT, decorators);
} else {
bool atomic = LayoutKindHelper::is_atomic_flat(layout);
bool null_free = !LayoutKindHelper::is_nullable_flat(layout);
if (null_free) {
null_check(value);
}
value->as_InlineType()->store_flat(this, base, ptr, atomic, immutable_memory, null_free, decorators);
}
@ -4686,6 +4691,7 @@ bool LibraryCallKit::inline_native_subtype_check() {
// {P,P} & superc!=subc => false
_prim_same_path, // {P,P} & superc==subc => true
_prim_1_path, // {N,P} => false
_ref_same_path, // {N,N} & superk==subk => true
_ref_subtype_path, // {N,N} & subtype check wins => true
_both_ref_path, // {N,N} & subtype check loses => false
PATH_LIMIT
@ -4733,6 +4739,16 @@ bool LibraryCallKit::inline_native_subtype_check() {
// now we have two reference types, in klasses[0..1]
Node* subk = klasses[1]; // the argument to isAssignableFrom
Node* superk = klasses[0]; // the receiver
// gen_subtype_check() refines exact array superklasses for comparison with
// (refined) klasses loaded from the header. Since both operands here are unrefined
// klasses, handle equality first. Unequal types then use the regular hierarchy check.
Node* cmp = _gvn.transform(new CmpPNode(subk, superk));
Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::eq));
IfNode* iff = create_and_xform_if(control(), bol, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
region->set_req(_ref_same_path, _gvn.transform(new IfTrueNode(iff)));
set_control(_gvn.transform(new IfFalseNode(iff)));
region->set_req(_both_ref_path, gen_subtype_check(subk, superk));
region->set_req(_ref_subtype_path, control());
}
@ -4757,6 +4773,7 @@ bool LibraryCallKit::inline_native_subtype_check() {
// these are the only paths that produce 'true':
phi->set_req(_prim_same_path, intcon(1));
phi->set_req(_ref_same_path, intcon(1));
phi->set_req(_ref_subtype_path, intcon(1));
// pull together the cases:
@ -5227,11 +5244,13 @@ bool LibraryCallKit::inline_array_copyOf(bool is_copyOfRange) {
// should be thrown
generate_negative_guard(length, bailout, &length);
// Handle inline type arrays
// TODO 8251971 This is too strong
generate_fair_guard(flat_array_test(load_object_klass(original)), bailout);
generate_fair_guard(flat_array_test(refined_klass_node), bailout);
generate_fair_guard(null_free_array_test(original), bailout);
if (Arguments::is_valhalla_enabled()) {
// Handle inline type arrays
// TODO 8251971 This is too strong
generate_fair_guard(flat_array_test(load_object_klass(original)), bailout);
generate_fair_guard(flat_array_test(refined_klass_node), bailout);
generate_fair_guard(null_free_array_test(original), bailout);
}
// Bail out if start is larger than the original length
Node* orig_tail = _gvn.transform(new SubINode(orig_length, start));
@ -6844,32 +6863,34 @@ bool LibraryCallKit::inline_arraycopy() {
slow_region->add_req(not_subtype_ctrl);
}
// TODO 8251971 Improve this. What about atomicity? Make sure this is always folded for type arrays.
// If destination is null-restricted, source must be null-restricted as well: src_null_restricted || !dst_null_restricted
Node* src_klass = load_object_klass(src);
Node* adr_prop_src = basic_plus_adr(top(), src_klass, in_bytes(ArrayKlass::properties_offset()));
Node* prop_src = _gvn.transform(LoadNode::make(_gvn, control(), immutable_memory(), adr_prop_src,
_gvn.type(adr_prop_src)->is_ptr(), TypeInt::INT, T_INT,
MemNode::unordered));
Node* adr_prop_dest = basic_plus_adr(top(), refined_dest_klass, in_bytes(ArrayKlass::properties_offset()));
Node* prop_dest = _gvn.transform(LoadNode::make(_gvn, control(), immutable_memory(), adr_prop_dest,
_gvn.type(adr_prop_dest)->is_ptr(), TypeInt::INT, T_INT,
MemNode::unordered));
if (Arguments::is_valhalla_enabled()) {
// TODO 8251971 Improve this. What about atomicity? Make sure this is always folded for type arrays.
// If destination is null-restricted, source must be null-restricted as well: src_null_restricted || !dst_null_restricted
Node* src_klass = load_object_klass(src);
Node* adr_prop_src = basic_plus_adr(top(), src_klass, in_bytes(ArrayKlass::properties_offset()));
Node* prop_src = _gvn.transform(LoadNode::make(_gvn, control(), immutable_memory(), adr_prop_src,
_gvn.type(adr_prop_src)->is_ptr(), TypeInt::INT, T_INT,
MemNode::unordered));
Node* adr_prop_dest = basic_plus_adr(top(), refined_dest_klass, in_bytes(ArrayKlass::properties_offset()));
Node* prop_dest = _gvn.transform(LoadNode::make(_gvn, control(), immutable_memory(), adr_prop_dest,
_gvn.type(adr_prop_dest)->is_ptr(), TypeInt::INT, T_INT,
MemNode::unordered));
const ArrayProperties props_null_restricted = ArrayProperties::Default().with_null_restricted();
jint props_value = (jint)props_null_restricted.value();
const ArrayProperties props_null_restricted = ArrayProperties::Default().with_null_restricted();
jint props_value = (jint)props_null_restricted.value();
prop_dest = _gvn.transform(new XorINode(prop_dest, intcon(props_value)));
prop_src = _gvn.transform(new OrINode(prop_dest, prop_src));
prop_src = _gvn.transform(new AndINode(prop_src, intcon(props_value)));
prop_dest = _gvn.transform(new XorINode(prop_dest, intcon(props_value)));
prop_src = _gvn.transform(new OrINode(prop_dest, prop_src));
prop_src = _gvn.transform(new AndINode(prop_src, intcon(props_value)));
Node* chk = _gvn.transform(new CmpINode(prop_src, intcon(props_value)));
Node* tst = _gvn.transform(new BoolNode(chk, BoolTest::ne));
generate_fair_guard(tst, slow_region);
Node* chk = _gvn.transform(new CmpINode(prop_src, intcon(props_value)));
Node* tst = _gvn.transform(new BoolNode(chk, BoolTest::ne));
generate_fair_guard(tst, slow_region);
// TODO 8251971 This is too strong
generate_fair_guard(flat_array_test(src), slow_region);
generate_fair_guard(flat_array_test(dest), slow_region);
// TODO 8251971 This is too strong
generate_fair_guard(flat_array_test(src), slow_region);
generate_fair_guard(flat_array_test(dest), slow_region);
}
{
PreserveJVMState pjvms(this);

4
src/hotspot/share/opto/loopnode.hpp
View File

@ -84,10 +84,10 @@ protected:
LoopNestInnerLoop = 1<<15,
LoopNestLongOuterLoop = 1<<16,
MultiversionFastLoop = 1<<17,
MultiversionSlowLoop = 2<<17,
MultiversionSlowLoop = 2<<17, // 1<<18
MultiversionDelayedSlowLoop = 3<<17,
MultiversionFlagsMask = 3<<17,
FlatArrays = 1<<18};
FlatArrays = 1<<19};
char _unswitch_count;
enum { _unswitch_max=3 };

2
src/hotspot/share/opto/loopopts.cpp
View File

@ -1133,7 +1133,7 @@ void PhaseIdealLoop::move_flat_array_check_out_of_loop(Node* n) {
return;
}
Node* mem = n->in(FlatArrayCheckNode::Memory);
Node* array = n->in(FlatArrayCheckNode::ArrayOrKlass)->uncast();
Node* array = n->in(FlatArrayCheckNode::ArrayOrKlass);
IdealLoopTree* check_loop = get_loop(get_ctrl(n));
IdealLoopTree* ary_loop = get_loop(get_ctrl(array));

8
src/hotspot/share/opto/macro.cpp
View File

@ -1316,7 +1316,10 @@ bool PhaseMacroExpand::scalar_replacement(AllocateNode* alloc, Unique_Node_List&
bool allow_oop = (res_type != nullptr) && !res_type->is_flat();
for (uint i = 0; i < value_worklist.size(); ++i) {
InlineTypeNode* vt = value_worklist.at(i)->as_InlineType();
vt->make_scalar_in_safepoints(&_igvn, allow_oop);
if (!vt->make_scalar_in_safepoints(&_igvn, allow_oop)) {
C->record_failure("out of nodes during scalarization");
return false;
}
}
return true;
}
@ -3294,6 +3297,9 @@ void PhaseMacroExpand::eliminate_macro_nodes(bool eliminate_locks) {
BarrierSet::barrier_set()->barrier_set_c2()->is_gc_barrier_node(n),
"unknown node type in macro list");
}
if (C->failing()) {
return;
}
assert(success == (C->macro_count() < old_macro_count), "elimination reduces macro count");
progress = progress || success;
if (success) {

3
src/hotspot/share/opto/macroArrayCopy.cpp
View File

@ -33,6 +33,7 @@
#include "opto/macro.hpp"
#include "opto/runtime.hpp"
#include "opto/vectornode.hpp"
#include "runtime/arguments.hpp"
#include "runtime/stubRoutines.hpp"
#include "utilities/align.hpp"
#include "utilities/powerOfTwo.hpp"
@ -1604,7 +1605,7 @@ void PhaseMacroExpand::expand_arraycopy_node(ArrayCopyNode *ac) {
// TODO 8251971 This is too strong
// We need to be careful here because 'adjust_for_flat_array' will adjust offsets/length etc. which then does not work anymore for the slow call to SharedRuntime::slow_arraycopy_C.
assert(top_src->is_flat() == top_dest->is_flat(), "must have bailed out before");
if (!flat_and_same_nullness) {
if (Arguments::is_valhalla_enabled() && !flat_and_same_nullness) {
generate_flat_array_guard(&ctrl, src, merge_mem, slow_region);
generate_flat_array_guard(&ctrl, dest, merge_mem, slow_region);
generate_null_free_array_guard(&ctrl, dest, merge_mem, slow_region);

8
src/hotspot/share/opto/matcher.cpp
View File

@ -168,7 +168,7 @@ void Matcher::verify_new_nodes_only(Node* xroot) {
// Array of RegMask, one per returned values (inline type instances can
// be returned as multiple return values, one per field)
RegMask* Matcher::return_values_mask(const TypeFunc* tf) {
RegMask* Matcher::return_values_mask(const TypeFunc* tf) const {
const TypeTuple* range = tf->range_cc();
uint cnt = range->cnt() - TypeFunc::Parms;
if (cnt == 0) {
@ -1089,11 +1089,7 @@ Node *Matcher::xform( Node *n, int max_stack ) {
}
if (m == nullptr) {
// Convert to machine-dependent projection
RegMask* mask = nullptr;
if (n->in(0)->is_Call() && n->in(0)->as_Call()->tf()->returns_inline_type_as_fields()) {
mask = return_values_mask(n->in(0)->as_Call()->tf());
}
m = n->in(0)->as_Multi()->match(n->as_Proj(), this, mask);
m = n->in(0)->as_Multi()->match(n->as_Proj(), this);
NOT_PRODUCT(record_new2old(m, n);)
}
if (m->in(0) != nullptr) // m might be top

2
src/hotspot/share/opto/matcher.hpp
View File

@ -264,7 +264,7 @@ public:
// Helper for match
OptoReg::Name warp_incoming_stk_arg( VMReg reg );
RegMask* return_values_mask(const TypeFunc* tf);
RegMask* return_values_mask(const TypeFunc* tf) const;
// Transform, then walk. Does implicit DCE while walking.
// Name changed from "transform" to avoid it being virtual.

112
src/hotspot/share/opto/memnode.cpp
View File

@ -2149,20 +2149,27 @@ AllocateNode* LoadNode::is_new_object_mark_load() const {
return nullptr;
}
//------------------------------Ideal------------------------------------------
// If the load is from Field memory and the pointer is non-null, it might be possible to
// zero out the control input.
// If the offset is constant and the base is an object allocation,
// try to hook me up to the exact initializing store.
Node *LoadNode::Ideal(PhaseGVN *phase, bool can_reshape) {
if (has_pinned_control_dependency()) {
return nullptr;
}
Node* p = MemNode::Ideal_common(phase, can_reshape);
if (p) return (p == NodeSentinel) ? nullptr : p;
Node* LoadNode::Ideal(PhaseGVN* phase, bool can_reshape) {
if (has_pinned_control_dependency()) { return nullptr; }
Node* p = Ideal_load_common(phase, can_reshape);
if (p == NodeSentinel) { return nullptr; }
Node* ctrl = in(MemNode::Control);
if (p == nullptr && !can_reshape) {
phase->record_for_igvn(this);
}
return p;
}
Node* LoadNode::Ideal_load_common(PhaseGVN* phase, bool can_reshape) {
Node* p = MemNode::Ideal_common(phase, can_reshape);
if (p != nullptr) { return p; }
Node* ctrl = in(MemNode::Control);
Node* address = in(MemNode::Address);
bool addr_mark = ((phase->type(address)->isa_oopptr() || phase->type(address)->isa_narrowoop()) &&
@ -2180,9 +2187,9 @@ Node *LoadNode::Ideal(PhaseGVN *phase, bool can_reshape) {
}
intptr_t ignore = 0;
Node* base = AddPNode::Ideal_base_and_offset(address, phase, ignore);
if (base != nullptr
&& phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw) {
Node* base = AddPNode::Ideal_base_and_offset(address, phase, ignore);
if (base != nullptr &&
phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw) {
// Check for useless control edge in some common special cases
if (in(MemNode::Control) != nullptr
// TODO 8350865 Can we re-enable this?
@ -2197,26 +2204,26 @@ Node *LoadNode::Ideal(PhaseGVN *phase, bool can_reshape) {
}
Node* mem = in(MemNode::Memory);
const TypePtr *addr_t = phase->type(address)->isa_ptr();
const TypePtr* addr_t = phase->type(address)->isa_ptr();
if (can_reshape && (addr_t != nullptr)) {
// try to optimize our memory input
Node* opt_mem = MemNode::optimize_memory_chain(mem, addr_t, this, phase);
if (opt_mem != mem) {
set_req_X(MemNode::Memory, opt_mem, phase);
if (phase->type( opt_mem ) == Type::TOP) return nullptr;
if (phase->type(opt_mem) == Type::TOP) { return NodeSentinel; }
return this;
}
const TypeOopPtr *t_oop = addr_t->isa_oopptr();
const TypeOopPtr* t_oop = addr_t->isa_oopptr();
if ((t_oop != nullptr) &&
(t_oop->is_known_instance_field() ||
t_oop->is_ptr_to_boxed_value())) {
PhaseIterGVN *igvn = phase->is_IterGVN();
PhaseIterGVN* igvn = phase->is_IterGVN();
assert(igvn != nullptr, "must be PhaseIterGVN when can_reshape is true");
if (igvn->_worklist.member(opt_mem)) {
// Delay this transformation until memory Phi is processed.
igvn->_worklist.push(this);
return nullptr;
return NodeSentinel;
}
// Split instance field load through Phi.
Node* result = split_through_phi(phase);
@ -2234,7 +2241,7 @@ Node *LoadNode::Ideal(PhaseGVN *phase, bool can_reshape) {
// barriers etc.) alone
if (in(0) != nullptr && !adr_type()->isa_rawptr() && can_reshape) {
for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) {
Node *use = mem->fast_out(i);
Node* use = mem->fast_out(i);
if (use != this &&
use->Opcode() == Opcode() &&
use->in(0) != nullptr &&
@ -2287,10 +2294,6 @@ Node *LoadNode::Ideal(PhaseGVN *phase, bool can_reshape) {
}
}
if (!can_reshape) {
phase->record_for_igvn(this);
}
return nullptr;
}
@ -2909,16 +2912,23 @@ Node* LoadKlassNode::Identity(PhaseGVN* phase) {
Node* LoadNode::klass_identity_common(PhaseGVN* phase) {
Node* x = LoadNode::Identity(phase);
if (x != this) return x;
if (x != this) { return x; }
Node* k = find_known_klass(phase);
return k == nullptr ? this : k;
}
// Find an existing Klass node from a recognized allocation or
// class-mirror pattern.
Node* LoadNode::find_known_klass(PhaseGVN* phase) const {
// Take apart the address into an oop and offset.
// Return 'this' if we cannot.
Node* adr = in(MemNode::Address);
// Return 'nullptr' if we cannot.
Node* adr = in(MemNode::Address);
intptr_t offset = 0;
Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
if (base == nullptr) return this;
Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
if (base == nullptr) { return nullptr; }
const TypeOopPtr* toop = phase->type(adr)->isa_oopptr();
if (toop == nullptr) return this;
if (toop == nullptr) { return nullptr; }
// Step over potential GC barrier for OopHandle resolve
BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
@ -2973,7 +2983,7 @@ Node* LoadNode::klass_identity_common(PhaseGVN* phase) {
}
}
return this;
return nullptr;
}
LoadNode* LoadNode::clone_pinned() const {
@ -2998,7 +3008,32 @@ LoadNode* LoadNode::pin_node_under_control_impl() const {
return nullptr;
}
//------------------------------Value------------------------------------------
Node* LoadNKlassNode::Ideal(PhaseGVN* phase, bool can_reshape) {
bool pinned = has_pinned_control_dependency();
if (!pinned) {
Node* p = Ideal_load_common(phase, can_reshape);
if (p == NodeSentinel) { return nullptr; }
if (p != nullptr) { return p; }
}
// To clean up reflective code, simplify k.java_mirror.as_klass to narrow k.
// Also feed through the klass in Allocate(...klass...)._klass.
Node* k = find_known_klass(phase);
if (k != nullptr) {
const Type* t = phase->type(k);
if (t != Type::TOP) {
assert(t->isa_klassptr(), "must be a klass pointer");
return new EncodePKlassNode(k, t->make_narrowklass());
}
}
if (!pinned && !can_reshape) {
phase->record_for_igvn(this);
}
return nullptr;
}
const Type* LoadNKlassNode::Value(PhaseGVN* phase) const {
const Type *t = klass_value_common(phase);
if (t == Type::TOP)
@ -3007,18 +3042,11 @@ const Type* LoadNKlassNode::Value(PhaseGVN* phase) const {
return t->make_narrowklass();
}
//------------------------------Identity---------------------------------------
// To clean up reflective code, simplify k.java_mirror.as_klass to narrow k.
// Also feed through the klass in Allocate(...klass...)._klass.
Node* LoadNKlassNode::Identity(PhaseGVN* phase) {
Node *x = klass_identity_common(phase);
const Type *t = phase->type( x );
if( t == Type::TOP ) return x;
if( t->isa_narrowklass()) return x;
assert (!t->isa_narrowoop(), "no narrow oop here");
return phase->transform(new EncodePKlassNode(x, t->make_narrowklass()));
Node* x = klass_identity_common(phase);
const Type* t = phase->type(x);
if (t == Type::TOP || t->isa_narrowklass()) { return x; }
return this;
}
//------------------------------Value-----------------------------------------
@ -4815,7 +4843,7 @@ const Type* MemBarNode::Value(PhaseGVN* phase) const {
//------------------------------match------------------------------------------
// Construct projections for memory.
Node *MemBarNode::match(const ProjNode *proj, const Matcher *m, const RegMask* mask) {
Node* MemBarNode::match(const ProjNode* proj, const Matcher* m) {
switch (proj->_con) {
case TypeFunc::Control:
case TypeFunc::Memory:

15
src/hotspot/share/opto/memnode.hpp
View File

@ -271,6 +271,7 @@ protected:
virtual Node* find_previous_arraycopy(PhaseValues* phase, Node* ld_alloc, Node*& mem, bool can_see_stored_value) const;
Node* can_see_stored_value_through_membars(Node* st, PhaseValues* phase) const;
Node* Ideal_load_common(PhaseGVN* phase, bool can_reshape);
public:
LoadNode(Node *c, Node *mem, Node *adr, const TypePtr* at, const Type *rt, MemOrd mo, ControlDependency control_dependency)
@ -322,6 +323,7 @@ public:
// Common methods for LoadKlass and LoadNKlass nodes.
const Type* klass_value_common(PhaseGVN* phase) const;
Node* klass_identity_common(PhaseGVN* phase);
Node* find_known_klass(PhaseGVN* phase) const;
virtual uint ideal_reg() const;
virtual const Type *bottom_type() const;
@ -618,6 +620,7 @@ public:
virtual const Type* Value(PhaseGVN* phase) const;
virtual Node* Identity(PhaseGVN* phase);
virtual Node* Ideal(PhaseGVN* phase, bool can_reshape);
};
//------------------------------StoreNode--------------------------------------
@ -1151,13 +1154,16 @@ public:
//------------------------------ClearArray-------------------------------------
class ClearArrayNode: public Node {
private:
// True if cnt is larger than InitArrayShortSize
bool _is_large;
bool _word_copy_only;
// True if the fill value is a non-constant or non-zero 64-bit value. Such a
// value must be copied as a complete word and cannot use byte-wise zeroing.
bool _requires_word_fill;
static Node* make_address(Node* dest, Node* offset, bool raw_base, PhaseGVN* phase);
public:
ClearArrayNode( Node *ctrl, Node *arymem, Node *word_cnt, Node *base, Node* val, bool is_large)
: Node(ctrl, arymem, word_cnt, base, val), _is_large(is_large),
_word_copy_only(val->bottom_type()->isa_long() && (!val->bottom_type()->is_long()->is_con() || val->bottom_type()->is_long()->get_con() != 0)) {
_requires_word_fill(val->bottom_type()->isa_long() && (!val->bottom_type()->is_long()->is_con() || val->bottom_type()->is_long()->get_con() != 0)) {
init_class_id(Class_ClearArray);
}
virtual int Opcode() const;
@ -1169,7 +1175,8 @@ public:
virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
virtual uint match_edge(uint idx) const;
bool is_large() const { return _is_large; }
bool word_copy_only() const { return _word_copy_only; }
bool is_zero_fill() const { return !_requires_word_fill; }
bool requires_word_fill() const { return _requires_word_fill; }
virtual uint size_of() const { return sizeof(ClearArrayNode); }
virtual uint hash() const { return Node::hash() + _is_large; }
virtual bool cmp(const Node& n) const {
@ -1251,7 +1258,7 @@ public:
virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
virtual uint match_edge(uint idx) const { return 0; }
virtual const Type *bottom_type() const { return TypeTuple::MEMBAR; }
virtual Node *match(const ProjNode *proj, const Matcher *m, const RegMask* mask);
virtual Node* match(const ProjNode* proj, const Matcher* m);
// Factory method. Builds a wide or narrow membar.
// Optional 'precedent' becomes an extra edge if not null.
static MemBarNode* make(Compile* C, int opcode,

2
src/hotspot/share/opto/mulnode.cpp
View File

@ -626,7 +626,7 @@ UMulHiLoLNode* UMulHiLoLNode::make(Node* umul_hi) {
return umul_hi_lo;
}
Node* MulHiLoLNode::match(const ProjNode* proj, const Matcher* match, const RegMask*) {
Node* MulHiLoLNode::match(const ProjNode* proj, const Matcher* match) {
uint ideal_reg = proj->ideal_reg();
RegMask rm;
if (proj->_con == first_proj_num) {

2
src/hotspot/share/opto/mulnode.hpp
View File

@ -217,7 +217,7 @@ public:
virtual int Opcode() const;
virtual const Type* bottom_type() const { return TypeTuple::LONG_PAIR; }
virtual Node* match(const ProjNode* proj, const Matcher* m, const RegMask* mask);
virtual Node* match(const ProjNode* proj, const Matcher* m);
static MulHiLoLNode* make(Node* mul_hi);
};

2
src/hotspot/share/opto/multnode.cpp
View File

@ -40,7 +40,7 @@ const RegMask &MultiNode::out_RegMask() const {
return RegMask::EMPTY;
}
Node *MultiNode::match(const ProjNode *proj, const Matcher *m, const RegMask* mask) { return proj->clone(); }
Node* MultiNode::match(const ProjNode* proj, const Matcher* m) { return proj->clone(); }
//------------------------------proj_out---------------------------------------
// Get a named projection or null if not found

2
src/hotspot/share/opto/multnode.hpp
View File

@ -43,7 +43,7 @@ public:
virtual bool is_CFG() const { return true; }
virtual uint hash() const { return NO_HASH; } // CFG nodes do not hash
virtual const RegMask &out_RegMask() const;
virtual Node *match(const ProjNode *proj, const Matcher *m, const RegMask* mask);
virtual Node* match(const ProjNode* proj, const Matcher* m);
virtual uint ideal_reg() const { return NotAMachineReg; }
ProjNode* proj_out(uint which_proj) const; // Get a named projection
ProjNode* proj_out_or_null(uint which_proj) const;

2
src/hotspot/share/opto/parse.hpp
View File

@ -490,7 +490,7 @@ class Parse : public GraphKit {
void do_one_bytecode();
// helper function to generate array store check
Node* array_store_check(Node*& adr, const Type*& elemtype);
Node* array_store_check(const Type*& elemtype);
// Helper function to generate array load
void array_load(BasicType etype);
Node* load_from_unknown_flat_array(Node* array, Node* array_index, const TypeOopPtr* element_ptr);

2
src/hotspot/share/opto/parse1.cpp
View File

@ -1105,7 +1105,7 @@ void Parse::do_exits() {
// such unusual early publications. But no barrier is needed on
// exceptional returns, since they cannot publish normally.
//
if ((method()->is_object_constructor() || method()->is_class_initializer()) &&
if (method()->is_object_constructor() &&
(wrote_non_strict_final() || wrote_stable() ||
(AlwaysSafeConstructors && wrote_fields()) ||
(support_IRIW_for_not_multiple_copy_atomic_cpu && wrote_volatile()))) {

4
src/hotspot/share/opto/parse2.cpp
View File

@ -204,7 +204,7 @@ void Parse::array_store(BasicType bt) {
Node* stored_value_casted = nullptr;
if (bt == T_OBJECT) {
stored_value_casted = array_store_check(adr, elemtype);
stored_value_casted = array_store_check(elemtype);
if (stopped()) {
return;
}
@ -581,7 +581,7 @@ Node* Parse::speculate_non_flat_array(Node* const array, const TypeAryPtr* const
!too_many_traps_or_recompiles(Deoptimization::Reason_speculate_class_check)) {
flat_array = false;
reason = Deoptimization::Reason_speculate_class_check;
} else if (UseArrayLoadStoreProfile && !too_many_traps_or_recompiles(reason)) {
} else if (UseArrayLoadStoreProfile && !too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) {
ciKlass* profiled_array_type = nullptr;
ciKlass* profiled_element_type = nullptr;
ProfilePtrKind element_ptr = ProfileMaybeNull;

9
src/hotspot/share/opto/parseHelper.cpp
View File

@ -140,10 +140,9 @@ void Parse::do_instanceof() {
//------------------------------array_store_check------------------------------
// pull array from stack and check that the store is valid
Node* Parse::array_store_check(Node*& adr, const Type*& elemtype) {
Node* Parse::array_store_check(const Type*& elemtype) {
// Shorthand access to array store elements without popping them.
Node *obj = peek(0);
Node *idx = peek(1);
Node *ary = peek(2);
if (_gvn.type(obj) == TypePtr::NULL_PTR) {
@ -233,11 +232,7 @@ Node* Parse::array_store_check(Node*& adr, const Type*& elemtype) {
Node* cast = _gvn.transform(new CheckCastPPNode(control(), ary, extak->as_exact_instance_type()));
replace_in_map(ary, cast);
ary = cast;
// Recompute element type and address
const TypeAryPtr* arytype = _gvn.type(ary)->is_aryptr();
elemtype = arytype->elem();
adr = array_element_address(ary, idx, T_OBJECT, arytype->size(), control());
elemtype = _gvn.type(ary)->is_aryptr()->elem();
CompileLog* log = C->log();
if (log != nullptr) {

4
src/hotspot/share/opto/runtime.cpp
View File

@ -363,8 +363,8 @@ JRT_BLOCK_ENTRY(void, OptoRuntime::new_array_C(Klass* array_type, int len, oopDe
result = oopFactory::new_typeArray(elem_type, len, THREAD);
} else {
Handle holder(current, array_type->klass_holder()); // keep the array klass alive
ObjArrayKlass* oak = ObjArrayKlass::cast(array_type);
result = oopFactory::new_objArray(oak->element_klass(), len, oak->properties(), THREAD);
result = ObjArrayKlass::cast(array_type)->allocate_instance(len, THREAD);
assert(HAS_PENDING_EXCEPTION || result->klass() == array_type, "array klass must be preserved");
if (!HAS_PENDING_EXCEPTION && array_type->is_null_free_array_klass() && !h_init_val.is_null()) {
// Null-free arrays need to be initialized
#ifdef ASSERT

Some files were not shown because too many files have changed in this diff Show More