diff --git a/src/hotspot/share/opto/arraycopynode.cpp b/src/hotspot/share/opto/arraycopynode.cpp index 4ee6107fe54..3b006ae6b11 100644 --- a/src/hotspot/share/opto/arraycopynode.cpp +++ b/src/hotspot/share/opto/arraycopynode.cpp @@ -426,7 +426,13 @@ Node* ArrayCopyNode::array_copy_forward(PhaseGVN *phase, for (int i = 1; i < count; i++) { Node* off = phase->MakeConX(type2aelembytes(copy_type) * i); Node* next_src = phase->transform(new AddPNode(base_src,adr_src,off)); + // We may have narrowed the type of next_src right before calling this method but because this runs with + // PhaseIterGVN::_delay_transform true, explicitly update the type of the AddP so it's consistent with its + // base and load() picks the right memory slice. + phase->set_type(next_src, next_src->Value(phase)); Node* next_dest = phase->transform(new AddPNode(base_dest,adr_dest,off)); + // Same as above + phase->set_type(next_dest, next_dest->Value(phase)); v = load(bs, phase, forward_ctl, mm, next_src, atp_src, value_type, copy_type); store(bs, phase, forward_ctl, mm, next_dest, atp_dest, v, value_type, copy_type); } @@ -464,7 +470,12 @@ Node* ArrayCopyNode::array_copy_backward(PhaseGVN *phase, for (int i = count-1; i >= 1; i--) { Node* off = phase->MakeConX(type2aelembytes(copy_type) * i); Node* next_src = phase->transform(new AddPNode(base_src,adr_src,off)); + // We may have narrowed the type of next_src right before calling this method but because this runs with + // PhaseIterGVN::_delay_transform true, explicitly update the type of the AddP so it's consistent with its + // base and store() picks the right memory slice. + phase->set_type(next_src, next_src->Value(phase)); Node* next_dest = phase->transform(new AddPNode(base_dest,adr_dest,off)); + phase->set_type(next_dest, next_dest->Value(phase)); Node* v = load(bs, phase, backward_ctl, mm, next_src, atp_src, value_type, copy_type); store(bs, phase, backward_ctl, mm, next_dest, atp_dest, v, value_type, copy_type); } @@ -592,6 +603,17 @@ Node *ArrayCopyNode::Ideal(PhaseGVN *phase, bool can_reshape) { const Type* value_type = nullptr; bool disjoint_bases = false; + Node* src = in(ArrayCopyNode::Src); + Node* dest = in(ArrayCopyNode::Dest); + // EA may have moved an input to a new slice. EA stores the new address types in the ArrayCopy node itself + // (_src_type/_dest_type). phase->type(src) and _src_type or phase->type(dest) and _dest_type may be different + // when this transformation runs if igvn hasn't had a chance to propagate the new types yet. Make sure the new + // types are taken into account so new Load/Store nodes are created on the right slice. + const TypePtr* atp_src = get_address_type(phase, _src_type, src); + const TypePtr* atp_dest = get_address_type(phase, _dest_type, dest); + phase->set_type(src, phase->type(src)->join_speculative(atp_src)); + phase->set_type(dest, phase->type(dest)->join_speculative(atp_dest)); + if (!prepare_array_copy(phase, can_reshape, adr_src, base_src, adr_dest, base_dest, copy_type, value_type, disjoint_bases)) { @@ -600,10 +622,6 @@ Node *ArrayCopyNode::Ideal(PhaseGVN *phase, bool can_reshape) { return nullptr; } - Node* src = in(ArrayCopyNode::Src); - Node* dest = in(ArrayCopyNode::Dest); - const TypePtr* atp_src = get_address_type(phase, _src_type, src); - const TypePtr* atp_dest = get_address_type(phase, _dest_type, dest); Node* in_mem = in(TypeFunc::Memory); if (can_reshape) { diff --git a/src/hotspot/share/opto/buildOopMap.cpp b/src/hotspot/share/opto/buildOopMap.cpp index 675113163e8..e3a94f78d9c 100644 --- a/src/hotspot/share/opto/buildOopMap.cpp +++ b/src/hotspot/share/opto/buildOopMap.cpp @@ -377,6 +377,9 @@ OopMap *OopFlow::build_oop_map( Node *n, int max_reg, PhaseRegAlloc *regalloc, i worklist.push(u); } } + if (m->is_SpillCopy()) { + worklist.push(m->in(1)); + } } } #endif diff --git a/src/hotspot/share/opto/callnode.cpp b/src/hotspot/share/opto/callnode.cpp index 9b3d7b38d15..d4c832359d6 100644 --- a/src/hotspot/share/opto/callnode.cpp +++ b/src/hotspot/share/opto/callnode.cpp @@ -844,6 +844,10 @@ bool CallNode::may_modify(const TypeOopPtr* t_oop, PhaseValues* phase) { } } guarantee(dest != nullptr, "Call had only one ptr in, broken IR!"); + if (phase->type(dest)->isa_rawptr()) { + // may happen for an arraycopy that initializes a newly allocated object. Conservatively return true; + return true; + } if (!dest->is_top() && may_modify_arraycopy_helper(phase->type(dest)->is_oopptr(), t_oop, phase)) { return true; } @@ -1742,7 +1746,7 @@ Node *AllocateNode::make_ideal_mark(PhaseGVN* phase, Node* control, Node* mem) { if (UseCompactObjectHeaders) { Node* klass_node = in(AllocateNode::KlassNode); Node* proto_adr = phase->transform(new AddPNode(phase->C->top(), klass_node, phase->MakeConX(in_bytes(Klass::prototype_header_offset())))); - mark_node = LoadNode::make(*phase, control, mem, proto_adr, TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(), MemNode::unordered); + mark_node = LoadNode::make(*phase, control, mem, proto_adr, phase->type(proto_adr)->is_ptr(), TypeX_X, TypeX_X->basic_type(), MemNode::unordered); } else { // For now only enable fast locking for non-array types mark_node = phase->MakeConX(markWord::prototype().value()); diff --git a/src/hotspot/share/opto/library_call.cpp b/src/hotspot/share/opto/library_call.cpp index 3627d06a87a..b3ee060d75f 100644 --- a/src/hotspot/share/opto/library_call.cpp +++ b/src/hotspot/share/opto/library_call.cpp @@ -3076,7 +3076,8 @@ bool LibraryCallKit::inline_native_vthread_start_transition(address funcAddr, co Node* global_disable_addr = makecon(TypeRawPtr::make((address)MountUnmountDisabler::global_vthread_transition_disable_count_address())); Node* global_disable = ideal.load(ideal.ctrl(), global_disable_addr, TypeInt::INT, T_INT, Compile::AliasIdxRaw, true /*require_atomic_access*/); Node* vt_disable_addr = basic_plus_adr(vt_oop, java_lang_Thread::vthread_transition_disable_count_offset()); - Node* vt_disable = ideal.load(ideal.ctrl(), vt_disable_addr, TypeInt::INT, T_INT, Compile::AliasIdxRaw, true /*require_atomic_access*/); + const TypePtr* vt_disable_addr_t = _gvn.type(vt_disable_addr)->is_ptr(); + Node* vt_disable = ideal.load(ideal.ctrl(), vt_disable_addr, TypeInt::INT, T_INT, C->get_alias_index(vt_disable_addr_t), true /*require_atomic_access*/); Node* disabled = _gvn.transform(new AddINode(global_disable, vt_disable)); ideal.if_then(disabled, BoolTest::ne, ideal.ConI(0)); { diff --git a/src/hotspot/share/opto/macro.cpp b/src/hotspot/share/opto/macro.cpp index 510531c95cd..b9045ddcf4c 100644 --- a/src/hotspot/share/opto/macro.cpp +++ b/src/hotspot/share/opto/macro.cpp @@ -1313,7 +1313,9 @@ void PhaseMacroExpand::expand_allocate_common( initial_slow_test = nullptr; } - bool allocation_has_use = (alloc->result_cast() != nullptr); + // ArrayCopyNode right after an allocation operates on the raw result projection for the Allocate node so it's not + // safe to remove such an allocation even if it has no result cast. + bool allocation_has_use = (alloc->result_cast() != nullptr) || (alloc->initialization() != nullptr && alloc->initialization()->is_complete_with_arraycopy()); if (!allocation_has_use) { InitializeNode* init = alloc->initialization(); if (init != nullptr) { diff --git a/src/hotspot/share/opto/macro.hpp b/src/hotspot/share/opto/macro.hpp index 0f8d8fb5172..070bfbe1478 100644 --- a/src/hotspot/share/opto/macro.hpp +++ b/src/hotspot/share/opto/macro.hpp @@ -40,14 +40,15 @@ private: public: // Helper methods roughly modeled after GraphKit: - Node* basic_plus_adr(Node* base, int offset) { - return (offset == 0)? base: basic_plus_adr(base, MakeConX(offset)); + Node* basic_plus_adr(Node* ptr, int offset, bool raw_base = false) { + return (offset == 0)? ptr: basic_plus_adr(ptr, MakeConX(offset), raw_base); } Node* basic_plus_adr(Node* base, Node* ptr, int offset) { return (offset == 0)? ptr: basic_plus_adr(base, ptr, MakeConX(offset)); } - Node* basic_plus_adr(Node* base, Node* offset) { - return basic_plus_adr(base, base, offset); + Node* basic_plus_adr(Node* ptr, Node* offset, bool raw_base = false) { + Node* base = raw_base ? top() : ptr; + return basic_plus_adr(base, ptr, offset); } Node* basic_plus_adr(Node* base, Node* ptr, Node* offset) { Node* adr = new AddPNode(base, ptr, offset); @@ -109,7 +110,7 @@ private: // More helper methods modeled after GraphKit for array copy void insert_mem_bar(Node** ctrl, Node** mem, int opcode, int alias_idx, Node* precedent = nullptr); - Node* array_element_address(Node* ary, Node* idx, BasicType elembt); + Node* array_element_address(Node* ary, Node* idx, BasicType elembt, bool raw_base); Node* ConvI2L(Node* offset); // helper methods modeled after LibraryCallKit for array copy diff --git a/src/hotspot/share/opto/macroArrayCopy.cpp b/src/hotspot/share/opto/macroArrayCopy.cpp index 0719ffc45a5..344561606b5 100644 --- a/src/hotspot/share/opto/macroArrayCopy.cpp +++ b/src/hotspot/share/opto/macroArrayCopy.cpp @@ -55,10 +55,10 @@ void PhaseMacroExpand::insert_mem_bar(Node** ctrl, Node** mem, int opcode, int a } } -Node* PhaseMacroExpand::array_element_address(Node* ary, Node* idx, BasicType elembt) { +Node* PhaseMacroExpand::array_element_address(Node* ary, Node* idx, BasicType elembt, bool raw_base) { uint shift = exact_log2(type2aelembytes(elembt)); uint header = arrayOopDesc::base_offset_in_bytes(elembt); - Node* base = basic_plus_adr(ary, header); + Node* base = basic_plus_adr(ary, header, raw_base); #ifdef _LP64 // see comment in GraphKit::array_element_address int index_max = max_jint - 1; // array size is max_jint, index is one less @@ -67,7 +67,7 @@ Node* PhaseMacroExpand::array_element_address(Node* ary, Node* idx, BasicType el #endif Node* scale = new LShiftXNode(idx, intcon(shift)); transform_later(scale); - return basic_plus_adr(ary, base, scale); + return basic_plus_adr(raw_base ? top() : ary, base, scale); } Node* PhaseMacroExpand::ConvI2L(Node* offset) { @@ -379,6 +379,7 @@ Node* PhaseMacroExpand::generate_arraycopy(ArrayCopyNode *ac, AllocateArrayNode* bool disjoint_bases, bool length_never_negative, RegionNode* slow_region) { + Node* orig_dest = dest; if (slow_region == nullptr) { slow_region = new RegionNode(1); transform_later(slow_region); @@ -411,6 +412,7 @@ Node* PhaseMacroExpand::generate_arraycopy(ArrayCopyNode *ac, AllocateArrayNode* assert(dest->is_CheckCastPP(), "sanity"); assert(dest->in(0)->in(0) == init, "dest pinned"); adr_type = TypeRawPtr::BOTTOM; // all initializations are into raw memory + dest = dest->in(1); // writing to raw memory requires a raw base // From this point on, every exit path is responsible for // initializing any non-copied parts of the object to zero. // Also, if this flag is set we make sure that arraycopy interacts properly @@ -771,7 +773,7 @@ Node* PhaseMacroExpand::generate_arraycopy(ArrayCopyNode *ac, AllocateArrayNode* local_mem = generate_slow_arraycopy(ac, &local_ctrl, local_mem, &local_io, adr_type, - src, src_offset, dest, dest_offset, + src, src_offset, orig_dest, dest_offset, copy_length, /*dest_uninitialized*/false); result_region->init_req(slow_call_path, local_ctrl); @@ -839,7 +841,7 @@ Node* PhaseMacroExpand::generate_arraycopy(ArrayCopyNode *ac, AllocateArrayNode* _igvn.replace_node(_callprojs.fallthrough_catchproj, *ctrl); #ifdef ASSERT - const TypeOopPtr* dest_t = _igvn.type(dest)->is_oopptr(); + const TypeOopPtr* dest_t = _igvn.type(orig_dest)->is_oopptr(); if (dest_t->is_known_instance()) { ArrayCopyNode* ac = nullptr; assert(ArrayCopyNode::may_modify(dest_t, (*ctrl)->in(0)->as_MemBar(), &_igvn, ac), "dependency on arraycopy lost"); @@ -915,12 +917,12 @@ void PhaseMacroExpand::generate_clear_array(Node* ctrl, MergeMemNode* merge_mem, if (start_con >= 0 && end_con >= 0) { // Constant start and end. Simple. mem = ClearArrayNode::clear_memory(ctrl, mem, dest, - start_con, end_con, false, &_igvn); + start_con, end_con, adr_type == TypeRawPtr::BOTTOM, &_igvn); } else if (start_con >= 0 && dest_size != top()) { // Constant start, pre-rounded end after the tail of the array. Node* end = dest_size; mem = ClearArrayNode::clear_memory(ctrl, mem, dest, - start_con, end, false, &_igvn); + start_con, end, adr_type == TypeRawPtr::BOTTOM, &_igvn); } else if (start_con >= 0 && slice_len != top()) { // Constant start, non-constant end. End needs rounding up. // End offset = round_up(abase + ((slice_idx_con + slice_len) << scale), 8) @@ -933,7 +935,7 @@ void PhaseMacroExpand::generate_clear_array(Node* ctrl, MergeMemNode* merge_mem, end = transform_later(new AddXNode(end, MakeConX(end_base)) ); end = transform_later(new AndXNode(end, MakeConX(~end_round)) ); mem = ClearArrayNode::clear_memory(ctrl, mem, dest, - start_con, end, false, &_igvn); + start_con, end, adr_type == TypeRawPtr::BOTTOM, &_igvn); } else if (start_con < 0 && dest_size != top()) { // Non-constant start, pre-rounded end after the tail of the array. // This is almost certainly a "round-to-end" operation. @@ -960,14 +962,14 @@ void PhaseMacroExpand::generate_clear_array(Node* ctrl, MergeMemNode* merge_mem, if (bump_bit != 0) { // Store a zero to the immediately preceding jint: Node* x1 = transform_later(new AddXNode(start, MakeConX(-bump_bit)) ); - Node* p1 = basic_plus_adr(dest, x1); + Node* p1 = basic_plus_adr(dest, x1, adr_type == TypeRawPtr::BOTTOM); mem = StoreNode::make(_igvn, ctrl, mem, p1, adr_type, intcon(0), T_INT, MemNode::unordered); mem = transform_later(mem); } } Node* end = dest_size; // pre-rounded mem = ClearArrayNode::clear_memory(ctrl, mem, dest, - start, end, false, &_igvn); + start, end, adr_type == TypeRawPtr::BOTTOM, &_igvn); } else { // Non-constant start, unrounded non-constant end. // (Nobody zeroes a random midsection of an array using this routine.) @@ -1009,7 +1011,7 @@ bool PhaseMacroExpand::generate_block_arraycopy(Node** ctrl, MergeMemNode** mem, if (((src_off | dest_off) & (BytesPerLong-1)) == BytesPerInt && ((src_off ^ dest_off) & (BytesPerLong-1)) == 0) { Node* sptr = basic_plus_adr(src, src_off); - Node* dptr = basic_plus_adr(dest, dest_off); + Node* dptr = basic_plus_adr(dest, dest_off, adr_type == TypeRawPtr::BOTTOM); const TypePtr* s_adr_type = _igvn.type(sptr)->is_ptr(); assert(s_adr_type->isa_aryptr(), "impossible slice"); uint s_alias_idx = C->get_alias_index(s_adr_type); @@ -1037,7 +1039,7 @@ bool PhaseMacroExpand::generate_block_arraycopy(Node** ctrl, MergeMemNode** mem, // Do this copy by giant steps. Node* sptr = basic_plus_adr(src, src_off); - Node* dptr = basic_plus_adr(dest, dest_off); + Node* dptr = basic_plus_adr(dest, dest_off, adr_type == TypeRawPtr::BOTTOM); Node* countx = dest_size; countx = transform_later(new SubXNode(countx, MakeConX(dest_off))); countx = transform_later(new URShiftXNode(countx, intcon(LogBytesPerLong))); @@ -1134,8 +1136,8 @@ Node* PhaseMacroExpand::generate_checkcast_arraycopy(Node** ctrl, MergeMemNode** Node* check_offset = ConvI2X(transform_later(n3)); Node* check_value = dest_elem_klass; - Node* src_start = array_element_address(src, src_offset, T_OBJECT); - Node* dest_start = array_element_address(dest, dest_offset, T_OBJECT); + Node* src_start = array_element_address(src, src_offset, T_OBJECT, false); + Node* dest_start = array_element_address(dest, dest_offset, T_OBJECT, adr_type == TypeRawPtr::BOTTOM); const TypeFunc* call_type = OptoRuntime::checkcast_arraycopy_Type(); Node* call = make_leaf_call(*ctrl, *mem, call_type, copyfunc_addr, "checkcast_arraycopy", adr_type, @@ -1190,8 +1192,8 @@ void PhaseMacroExpand::generate_unchecked_arraycopy(Node** ctrl, MergeMemNode** Node* src_start = src; Node* dest_start = dest; if (src_offset != nullptr || dest_offset != nullptr) { - src_start = array_element_address(src, src_offset, basic_elem_type); - dest_start = array_element_address(dest, dest_offset, basic_elem_type); + src_start = array_element_address(src, src_offset, basic_elem_type, false); + dest_start = array_element_address(dest, dest_offset, basic_elem_type, adr_type == TypeRawPtr::BOTTOM); } // Figure out which arraycopy runtime method to call. diff --git a/src/hotspot/share/opto/memnode.cpp b/src/hotspot/share/opto/memnode.cpp index f7da9a96d34..6cb14444f6b 100644 --- a/src/hotspot/share/opto/memnode.cpp +++ b/src/hotspot/share/opto/memnode.cpp @@ -1010,6 +1010,7 @@ bool LoadNode::is_immutable_value(Node* adr) { Node* LoadNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, const Type* rt, BasicType bt, MemOrd mo, ControlDependency control_dependency, bool require_atomic_access, bool unaligned, bool mismatched, bool unsafe, uint8_t barrier_data) { Compile* C = gvn.C; + assert(adr->is_top() || C->get_alias_index(gvn.type(adr)->is_ptr()) == C->get_alias_index(adr_type), "adr and adr_type must agree"); // sanity check the alias category against the created node type assert(!(adr_type->isa_oopptr() && @@ -1411,8 +1412,12 @@ Node* LoadNode::convert_to_unsigned_load(PhaseGVN& gvn) { assert(false, "no unsigned variant: %s", Name()); return nullptr; } + const Type* mem_t = gvn.type(in(MemNode::Address)); + if (mem_t == Type::TOP) { + return gvn.C->top(); + } return LoadNode::make(gvn, in(MemNode::Control), in(MemNode::Memory), in(MemNode::Address), - raw_adr_type(), rt, bt, _mo, _control_dependency, + mem_t->is_ptr(), rt, bt, _mo, _control_dependency, false /*require_atomic_access*/, is_unaligned_access(), is_mismatched_access()); } @@ -1431,8 +1436,12 @@ Node* LoadNode::convert_to_signed_load(PhaseGVN& gvn) { assert(false, "no signed variant: %s", Name()); return nullptr; } + const Type* mem_t = gvn.type(in(MemNode::Address)); + if (mem_t == Type::TOP) { + return gvn.C->top(); + } return LoadNode::make(gvn, in(MemNode::Control), in(MemNode::Memory), in(MemNode::Address), - raw_adr_type(), rt, bt, _mo, _control_dependency, + mem_t->is_ptr(), rt, bt, _mo, _control_dependency, false /*require_atomic_access*/, is_unaligned_access(), is_mismatched_access()); } @@ -1459,8 +1468,12 @@ Node* LoadNode::convert_to_reinterpret_load(PhaseGVN& gvn, const Type* rt) { const int op = Opcode(); bool require_atomic_access = (op == Op_LoadL && ((LoadLNode*)this)->require_atomic_access()) || (op == Op_LoadD && ((LoadDNode*)this)->require_atomic_access()); + const Type* mem_t = gvn.type(in(MemNode::Address)); + if (mem_t == Type::TOP) { + return gvn.C->top(); + } return LoadNode::make(gvn, in(MemNode::Control), in(MemNode::Memory), in(MemNode::Address), - raw_adr_type(), rt, bt, _mo, _control_dependency, + mem_t->is_ptr(), rt, bt, _mo, _control_dependency, require_atomic_access, is_unaligned_access(), is_mismatched); } @@ -1482,8 +1495,12 @@ Node* StoreNode::convert_to_reinterpret_store(PhaseGVN& gvn, Node* val, const Ty const int op = Opcode(); bool require_atomic_access = (op == Op_StoreL && ((StoreLNode*)this)->require_atomic_access()) || (op == Op_StoreD && ((StoreDNode*)this)->require_atomic_access()); + const Type* mem_t = gvn.type(in(MemNode::Address)); + if (mem_t == Type::TOP) { + return gvn.C->top(); + } StoreNode* st = StoreNode::make(gvn, in(MemNode::Control), in(MemNode::Memory), in(MemNode::Address), - raw_adr_type(), val, bt, _mo, require_atomic_access); + mem_t->is_ptr(), val, bt, _mo, require_atomic_access); bool is_mismatched = is_mismatched_access(); const TypeRawPtr* raw_type = gvn.type(in(MemNode::Memory))->isa_rawptr(); @@ -2778,6 +2795,7 @@ Node* LoadRangeNode::Identity(PhaseGVN* phase) { StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo, bool require_atomic_access) { assert((mo == unordered || mo == release), "unexpected"); Compile* C = gvn.C; + assert(adr_type == nullptr || adr->is_top() || C->get_alias_index(gvn.type(adr)->is_ptr()) == C->get_alias_index(adr_type), "adr and adr_type must agree"); assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw || ctl != nullptr, "raw memory operations should have control edge"); @@ -5070,7 +5088,7 @@ Node* InitializeNode::capture_store(StoreNode* st, intptr_t start, else ins_req(i, C->top()); // build a new edge } - Node* new_st = st->clone(); + Node* new_st = st->clone_with_adr_type(TypeRawPtr::BOTTOM); BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); new_st->set_req(MemNode::Control, in(Control)); new_st->set_req(MemNode::Memory, prev_mem); diff --git a/src/hotspot/share/opto/memnode.hpp b/src/hotspot/share/opto/memnode.hpp index 2f26c67f0a8..30d44e82016 100644 --- a/src/hotspot/share/opto/memnode.hpp +++ b/src/hotspot/share/opto/memnode.hpp @@ -173,6 +173,14 @@ public: static void dump_adr_type(const TypePtr* adr_type, outputStream* st); virtual void dump_spec(outputStream *st) const; #endif + + MemNode* clone_with_adr_type(const TypePtr* adr_type) const { + MemNode* new_node = clone()->as_Mem(); +#ifdef ASSERT + new_node->_adr_type = adr_type; +#endif + return new_node; + } }; // Analyze a MemNode to try to prove that it is independent from other memory accesses