From 3481252ced7c06c44154ceccc56b12cfd9a490c3 Mon Sep 17 00:00:00 2001 From: Aleksey Shipilev Date: Mon, 1 Dec 2025 08:41:18 +0000 Subject: [PATCH] 8372188: AArch64: Generate atomic match rules from M4 stencils Reviewed-by: aph, haosun --- make/hotspot/gensrc/GensrcAdlc.gmk | 1 + src/hotspot/cpu/aarch64/aarch64.ad | 1032 ------------------ src/hotspot/cpu/aarch64/aarch64_atomic.ad | 909 +++++++++++++++ src/hotspot/cpu/aarch64/aarch64_atomic_ad.m4 | 246 +++++ src/hotspot/cpu/aarch64/cas.m4 | 161 --- 5 files changed, 1156 insertions(+), 1193 deletions(-) create mode 100644 src/hotspot/cpu/aarch64/aarch64_atomic.ad create mode 100644 src/hotspot/cpu/aarch64/aarch64_atomic_ad.m4 delete mode 100644 src/hotspot/cpu/aarch64/cas.m4 diff --git a/make/hotspot/gensrc/GensrcAdlc.gmk b/make/hotspot/gensrc/GensrcAdlc.gmk index 4cecc3070e7..14117421d1b 100644 --- a/make/hotspot/gensrc/GensrcAdlc.gmk +++ b/make/hotspot/gensrc/GensrcAdlc.gmk @@ -170,6 +170,7 @@ ifeq ($(call check-jvm-feature, compiler2), true) ifeq ($(HOTSPOT_TARGET_CPU_ARCH), aarch64) AD_SRC_FILES += $(call uniq, $(wildcard $(foreach d, $(AD_SRC_ROOTS), \ $d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/$(HOTSPOT_TARGET_CPU_ARCH)_vector.ad \ + $d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/$(HOTSPOT_TARGET_CPU_ARCH)_atomic.ad \ ))) endif diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad index fc53c10311b..9c76ed24788 100644 --- a/src/hotspot/cpu/aarch64/aarch64.ad +++ b/src/hotspot/cpu/aarch64/aarch64.ad @@ -3342,73 +3342,6 @@ encode %{ __ cmpw(rscratch1, zr); %} - enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{ - guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); - __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register, - Assembler::xword, /*acquire*/ false, /*release*/ true, - /*weak*/ false, noreg); - %} - - enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{ - guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); - __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register, - Assembler::word, /*acquire*/ false, /*release*/ true, - /*weak*/ false, noreg); - %} - - enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{ - guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); - __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register, - Assembler::halfword, /*acquire*/ false, /*release*/ true, - /*weak*/ false, noreg); - %} - - enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{ - guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); - __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register, - Assembler::byte, /*acquire*/ false, /*release*/ true, - /*weak*/ false, noreg); - %} - - - // The only difference between aarch64_enc_cmpxchg and - // aarch64_enc_cmpxchg_acq is that we use load-acquire in the - // CompareAndSwap sequence to serve as a barrier on acquiring a - // lock. - enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegL oldval, iRegL newval) %{ - guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); - __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register, - Assembler::xword, /*acquire*/ true, /*release*/ true, - /*weak*/ false, noreg); - %} - - enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegI oldval, iRegI newval) %{ - guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); - __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register, - Assembler::word, /*acquire*/ true, /*release*/ true, - /*weak*/ false, noreg); - %} - - enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegI oldval, iRegI newval) %{ - guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); - __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register, - Assembler::halfword, /*acquire*/ true, /*release*/ true, - /*weak*/ false, noreg); - %} - - enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegI oldval, iRegI newval) %{ - guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); - __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register, - Assembler::byte, /*acquire*/ true, /*release*/ true, - /*weak*/ false, noreg); - %} - - // auxiliary used for CompareAndSwapX to set result register - enc_class aarch64_enc_cset_eq(iRegI res) %{ - Register res_reg = as_Register($res$$reg); - __ cset(res_reg, Assembler::EQ); - %} - // prefetch encodings enc_class aarch64_enc_prefetchw(memory mem) %{ @@ -8380,971 +8313,6 @@ instruct castVVMask(pRegGov dst) ins_pipe(pipe_class_empty); %} -// ============================================================================ -// Atomic operation instructions -// - -// standard CompareAndSwapX when we are using barriers -// these have higher priority than the rules selected by a predicate - -// XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher -// can't match them - -instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ - - match(Set res (CompareAndSwapB mem (Binary oldval newval))); - ins_cost(2 * VOLATILE_REF_COST); - - effect(KILL cr); - - format %{ - "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval" - "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)" - %} - - ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval), - aarch64_enc_cset_eq(res)); - - ins_pipe(pipe_slow); -%} - -instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ - - match(Set res (CompareAndSwapS mem (Binary oldval newval))); - ins_cost(2 * VOLATILE_REF_COST); - - effect(KILL cr); - - format %{ - "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval" - "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)" - %} - - ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval), - aarch64_enc_cset_eq(res)); - - ins_pipe(pipe_slow); -%} - -instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ - - match(Set res (CompareAndSwapI mem (Binary oldval newval))); - ins_cost(2 * VOLATILE_REF_COST); - - effect(KILL cr); - - format %{ - "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval" - "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)" - %} - - ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval), - aarch64_enc_cset_eq(res)); - - ins_pipe(pipe_slow); -%} - -instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{ - - match(Set res (CompareAndSwapL mem (Binary oldval newval))); - ins_cost(2 * VOLATILE_REF_COST); - - effect(KILL cr); - - format %{ - "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval" - "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)" - %} - - ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval), - aarch64_enc_cset_eq(res)); - - ins_pipe(pipe_slow); -%} - -instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ - - match(Set res (CompareAndSwapP mem (Binary oldval newval))); - predicate(n->as_LoadStore()->barrier_data() == 0); - ins_cost(2 * VOLATILE_REF_COST); - - effect(KILL cr); - - format %{ - "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval" - "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)" - %} - - ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval), - aarch64_enc_cset_eq(res)); - - ins_pipe(pipe_slow); -%} - -instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{ - - match(Set res (CompareAndSwapN mem (Binary oldval newval))); - predicate(n->as_LoadStore()->barrier_data() == 0); - ins_cost(2 * VOLATILE_REF_COST); - - effect(KILL cr); - - format %{ - "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval" - "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)" - %} - - ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval), - aarch64_enc_cset_eq(res)); - - ins_pipe(pipe_slow); -%} - -// alternative CompareAndSwapX when we are eliding barriers - -instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ - - predicate(needs_acquiring_load_exclusive(n)); - match(Set res (CompareAndSwapB mem (Binary oldval newval))); - ins_cost(VOLATILE_REF_COST); - - effect(KILL cr); - - format %{ - "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval" - "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)" - %} - - ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval), - aarch64_enc_cset_eq(res)); - - ins_pipe(pipe_slow); -%} - -instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ - - predicate(needs_acquiring_load_exclusive(n)); - match(Set res (CompareAndSwapS mem (Binary oldval newval))); - ins_cost(VOLATILE_REF_COST); - - effect(KILL cr); - - format %{ - "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval" - "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)" - %} - - ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval), - aarch64_enc_cset_eq(res)); - - ins_pipe(pipe_slow); -%} - -instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ - - predicate(needs_acquiring_load_exclusive(n)); - match(Set res (CompareAndSwapI mem (Binary oldval newval))); - ins_cost(VOLATILE_REF_COST); - - effect(KILL cr); - - format %{ - "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval" - "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)" - %} - - ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval), - aarch64_enc_cset_eq(res)); - - ins_pipe(pipe_slow); -%} - -instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{ - - predicate(needs_acquiring_load_exclusive(n)); - match(Set res (CompareAndSwapL mem (Binary oldval newval))); - ins_cost(VOLATILE_REF_COST); - - effect(KILL cr); - - format %{ - "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval" - "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)" - %} - - ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval), - aarch64_enc_cset_eq(res)); - - ins_pipe(pipe_slow); -%} - -instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ - - predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0)); - match(Set res (CompareAndSwapP mem (Binary oldval newval))); - ins_cost(VOLATILE_REF_COST); - - effect(KILL cr); - - format %{ - "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval" - "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)" - %} - - ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval), - aarch64_enc_cset_eq(res)); - - ins_pipe(pipe_slow); -%} - -instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{ - - predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0); - match(Set res (CompareAndSwapN mem (Binary oldval newval))); - ins_cost(VOLATILE_REF_COST); - - effect(KILL cr); - - format %{ - "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval" - "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)" - %} - - ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval), - aarch64_enc_cset_eq(res)); - - ins_pipe(pipe_slow); -%} - - -// --------------------------------------------------------------------- - -// BEGIN This section of the file is automatically generated. Do not edit -------------- - -// Sundry CAS operations. Note that release is always true, -// regardless of the memory ordering of the CAS. This is because we -// need the volatile case to be sequentially consistent but there is -// no trailing StoreLoad barrier emitted by C2. Unfortunately we -// can't check the type of memory ordering here, so we always emit a -// STLXR. - -// This section is generated from cas.m4 - - -// This pattern is generated automatically from cas.m4. -// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE -instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ - match(Set res (CompareAndExchangeB mem (Binary oldval newval))); - ins_cost(2 * VOLATILE_REF_COST); - effect(TEMP_DEF res, KILL cr); - format %{ - "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval" - %} - ins_encode %{ - __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, - Assembler::byte, /*acquire*/ false, /*release*/ true, - /*weak*/ false, $res$$Register); - __ sxtbw($res$$Register, $res$$Register); - %} - ins_pipe(pipe_slow); -%} - -// This pattern is generated automatically from cas.m4. -// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE -instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ - match(Set res (CompareAndExchangeS mem (Binary oldval newval))); - ins_cost(2 * VOLATILE_REF_COST); - effect(TEMP_DEF res, KILL cr); - format %{ - "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval" - %} - ins_encode %{ - __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, - Assembler::halfword, /*acquire*/ false, /*release*/ true, - /*weak*/ false, $res$$Register); - __ sxthw($res$$Register, $res$$Register); - %} - ins_pipe(pipe_slow); -%} - -// This pattern is generated automatically from cas.m4. -// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE -instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ - match(Set res (CompareAndExchangeI mem (Binary oldval newval))); - ins_cost(2 * VOLATILE_REF_COST); - effect(TEMP_DEF res, KILL cr); - format %{ - "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval" - %} - ins_encode %{ - __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, - Assembler::word, /*acquire*/ false, /*release*/ true, - /*weak*/ false, $res$$Register); - %} - ins_pipe(pipe_slow); -%} - -// This pattern is generated automatically from cas.m4. -// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE -instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{ - match(Set res (CompareAndExchangeL mem (Binary oldval newval))); - ins_cost(2 * VOLATILE_REF_COST); - effect(TEMP_DEF res, KILL cr); - format %{ - "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval" - %} - ins_encode %{ - __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, - Assembler::xword, /*acquire*/ false, /*release*/ true, - /*weak*/ false, $res$$Register); - %} - ins_pipe(pipe_slow); -%} - -// This pattern is generated automatically from cas.m4. -// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE -instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{ - predicate(n->as_LoadStore()->barrier_data() == 0); - match(Set res (CompareAndExchangeN mem (Binary oldval newval))); - ins_cost(2 * VOLATILE_REF_COST); - effect(TEMP_DEF res, KILL cr); - format %{ - "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval" - %} - ins_encode %{ - __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, - Assembler::word, /*acquire*/ false, /*release*/ true, - /*weak*/ false, $res$$Register); - %} - ins_pipe(pipe_slow); -%} - -// This pattern is generated automatically from cas.m4. -// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE -instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ - predicate(n->as_LoadStore()->barrier_data() == 0); - match(Set res (CompareAndExchangeP mem (Binary oldval newval))); - ins_cost(2 * VOLATILE_REF_COST); - effect(TEMP_DEF res, KILL cr); - format %{ - "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval" - %} - ins_encode %{ - __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, - Assembler::xword, /*acquire*/ false, /*release*/ true, - /*weak*/ false, $res$$Register); - %} - ins_pipe(pipe_slow); -%} - -// This pattern is generated automatically from cas.m4. -// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE -instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ - predicate(needs_acquiring_load_exclusive(n)); - match(Set res (CompareAndExchangeB mem (Binary oldval newval))); - ins_cost(VOLATILE_REF_COST); - effect(TEMP_DEF res, KILL cr); - format %{ - "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval" - %} - ins_encode %{ - __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, - Assembler::byte, /*acquire*/ true, /*release*/ true, - /*weak*/ false, $res$$Register); - __ sxtbw($res$$Register, $res$$Register); - %} - ins_pipe(pipe_slow); -%} - -// This pattern is generated automatically from cas.m4. -// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE -instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ - predicate(needs_acquiring_load_exclusive(n)); - match(Set res (CompareAndExchangeS mem (Binary oldval newval))); - ins_cost(VOLATILE_REF_COST); - effect(TEMP_DEF res, KILL cr); - format %{ - "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval" - %} - ins_encode %{ - __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, - Assembler::halfword, /*acquire*/ true, /*release*/ true, - /*weak*/ false, $res$$Register); - __ sxthw($res$$Register, $res$$Register); - %} - ins_pipe(pipe_slow); -%} - -// This pattern is generated automatically from cas.m4. -// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE -instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ - predicate(needs_acquiring_load_exclusive(n)); - match(Set res (CompareAndExchangeI mem (Binary oldval newval))); - ins_cost(VOLATILE_REF_COST); - effect(TEMP_DEF res, KILL cr); - format %{ - "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval" - %} - ins_encode %{ - __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, - Assembler::word, /*acquire*/ true, /*release*/ true, - /*weak*/ false, $res$$Register); - %} - ins_pipe(pipe_slow); -%} - -// This pattern is generated automatically from cas.m4. -// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE -instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{ - predicate(needs_acquiring_load_exclusive(n)); - match(Set res (CompareAndExchangeL mem (Binary oldval newval))); - ins_cost(VOLATILE_REF_COST); - effect(TEMP_DEF res, KILL cr); - format %{ - "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval" - %} - ins_encode %{ - __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, - Assembler::xword, /*acquire*/ true, /*release*/ true, - /*weak*/ false, $res$$Register); - %} - ins_pipe(pipe_slow); -%} - -// This pattern is generated automatically from cas.m4. -// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE -instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{ - predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0); - match(Set res (CompareAndExchangeN mem (Binary oldval newval))); - ins_cost(VOLATILE_REF_COST); - effect(TEMP_DEF res, KILL cr); - format %{ - "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval" - %} - ins_encode %{ - __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, - Assembler::word, /*acquire*/ true, /*release*/ true, - /*weak*/ false, $res$$Register); - %} - ins_pipe(pipe_slow); -%} - -// This pattern is generated automatically from cas.m4. -// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE -instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ - predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0)); - match(Set res (CompareAndExchangeP mem (Binary oldval newval))); - ins_cost(VOLATILE_REF_COST); - effect(TEMP_DEF res, KILL cr); - format %{ - "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval" - %} - ins_encode %{ - __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, - Assembler::xword, /*acquire*/ true, /*release*/ true, - /*weak*/ false, $res$$Register); - %} - ins_pipe(pipe_slow); -%} - -// This pattern is generated automatically from cas.m4. -// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE -instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ - match(Set res (WeakCompareAndSwapB mem (Binary oldval newval))); - ins_cost(2 * VOLATILE_REF_COST); - effect(KILL cr); - format %{ - "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval" - "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" - %} - ins_encode %{ - __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, - Assembler::byte, /*acquire*/ false, /*release*/ true, - /*weak*/ true, noreg); - __ csetw($res$$Register, Assembler::EQ); - %} - ins_pipe(pipe_slow); -%} - -// This pattern is generated automatically from cas.m4. -// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE -instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ - match(Set res (WeakCompareAndSwapS mem (Binary oldval newval))); - ins_cost(2 * VOLATILE_REF_COST); - effect(KILL cr); - format %{ - "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval" - "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" - %} - ins_encode %{ - __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, - Assembler::halfword, /*acquire*/ false, /*release*/ true, - /*weak*/ true, noreg); - __ csetw($res$$Register, Assembler::EQ); - %} - ins_pipe(pipe_slow); -%} - -// This pattern is generated automatically from cas.m4. -// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE -instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ - match(Set res (WeakCompareAndSwapI mem (Binary oldval newval))); - ins_cost(2 * VOLATILE_REF_COST); - effect(KILL cr); - format %{ - "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval" - "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" - %} - ins_encode %{ - __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, - Assembler::word, /*acquire*/ false, /*release*/ true, - /*weak*/ true, noreg); - __ csetw($res$$Register, Assembler::EQ); - %} - ins_pipe(pipe_slow); -%} - -// This pattern is generated automatically from cas.m4. -// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE -instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{ - match(Set res (WeakCompareAndSwapL mem (Binary oldval newval))); - ins_cost(2 * VOLATILE_REF_COST); - effect(KILL cr); - format %{ - "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval" - "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" - %} - ins_encode %{ - __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, - Assembler::xword, /*acquire*/ false, /*release*/ true, - /*weak*/ true, noreg); - __ csetw($res$$Register, Assembler::EQ); - %} - ins_pipe(pipe_slow); -%} - -// This pattern is generated automatically from cas.m4. -// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE -instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{ - predicate(n->as_LoadStore()->barrier_data() == 0); - match(Set res (WeakCompareAndSwapN mem (Binary oldval newval))); - ins_cost(2 * VOLATILE_REF_COST); - effect(KILL cr); - format %{ - "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval" - "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" - %} - ins_encode %{ - __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, - Assembler::word, /*acquire*/ false, /*release*/ true, - /*weak*/ true, noreg); - __ csetw($res$$Register, Assembler::EQ); - %} - ins_pipe(pipe_slow); -%} - -// This pattern is generated automatically from cas.m4. -// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE -instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ - predicate(n->as_LoadStore()->barrier_data() == 0); - match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); - ins_cost(2 * VOLATILE_REF_COST); - effect(KILL cr); - format %{ - "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval" - "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" - %} - ins_encode %{ - __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, - Assembler::xword, /*acquire*/ false, /*release*/ true, - /*weak*/ true, noreg); - __ csetw($res$$Register, Assembler::EQ); - %} - ins_pipe(pipe_slow); -%} - -// This pattern is generated automatically from cas.m4. -// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE -instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ - predicate(needs_acquiring_load_exclusive(n)); - match(Set res (WeakCompareAndSwapB mem (Binary oldval newval))); - ins_cost(VOLATILE_REF_COST); - effect(KILL cr); - format %{ - "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval" - "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" - %} - ins_encode %{ - __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, - Assembler::byte, /*acquire*/ true, /*release*/ true, - /*weak*/ true, noreg); - __ csetw($res$$Register, Assembler::EQ); - %} - ins_pipe(pipe_slow); -%} - -// This pattern is generated automatically from cas.m4. -// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE -instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ - predicate(needs_acquiring_load_exclusive(n)); - match(Set res (WeakCompareAndSwapS mem (Binary oldval newval))); - ins_cost(VOLATILE_REF_COST); - effect(KILL cr); - format %{ - "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval" - "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" - %} - ins_encode %{ - __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, - Assembler::halfword, /*acquire*/ true, /*release*/ true, - /*weak*/ true, noreg); - __ csetw($res$$Register, Assembler::EQ); - %} - ins_pipe(pipe_slow); -%} - -// This pattern is generated automatically from cas.m4. -// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE -instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ - predicate(needs_acquiring_load_exclusive(n)); - match(Set res (WeakCompareAndSwapI mem (Binary oldval newval))); - ins_cost(VOLATILE_REF_COST); - effect(KILL cr); - format %{ - "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval" - "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" - %} - ins_encode %{ - __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, - Assembler::word, /*acquire*/ true, /*release*/ true, - /*weak*/ true, noreg); - __ csetw($res$$Register, Assembler::EQ); - %} - ins_pipe(pipe_slow); -%} - -// This pattern is generated automatically from cas.m4. -// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE -instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{ - predicate(needs_acquiring_load_exclusive(n)); - match(Set res (WeakCompareAndSwapL mem (Binary oldval newval))); - ins_cost(VOLATILE_REF_COST); - effect(KILL cr); - format %{ - "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval" - "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" - %} - ins_encode %{ - __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, - Assembler::xword, /*acquire*/ true, /*release*/ true, - /*weak*/ true, noreg); - __ csetw($res$$Register, Assembler::EQ); - %} - ins_pipe(pipe_slow); -%} - -// This pattern is generated automatically from cas.m4. -// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE -instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{ - predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0); - match(Set res (WeakCompareAndSwapN mem (Binary oldval newval))); - ins_cost(VOLATILE_REF_COST); - effect(KILL cr); - format %{ - "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval" - "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" - %} - ins_encode %{ - __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, - Assembler::word, /*acquire*/ true, /*release*/ true, - /*weak*/ true, noreg); - __ csetw($res$$Register, Assembler::EQ); - %} - ins_pipe(pipe_slow); -%} - -// This pattern is generated automatically from cas.m4. -// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE -instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ - predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0)); - match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); - ins_cost(VOLATILE_REF_COST); - effect(KILL cr); - format %{ - "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval" - "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" - %} - ins_encode %{ - __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, - Assembler::xword, /*acquire*/ true, /*release*/ true, - /*weak*/ true, noreg); - __ csetw($res$$Register, Assembler::EQ); - %} - ins_pipe(pipe_slow); -%} - -// END This section of the file is automatically generated. Do not edit -------------- -// --------------------------------------------------------------------- - -instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{ - match(Set prev (GetAndSetI mem newv)); - ins_cost(2 * VOLATILE_REF_COST); - format %{ "atomic_xchgw $prev, $newv, [$mem]" %} - ins_encode %{ - __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base)); - %} - ins_pipe(pipe_serial); -%} - -instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{ - match(Set prev (GetAndSetL mem newv)); - ins_cost(2 * VOLATILE_REF_COST); - format %{ "atomic_xchg $prev, $newv, [$mem]" %} - ins_encode %{ - __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base)); - %} - ins_pipe(pipe_serial); -%} - -instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{ - predicate(n->as_LoadStore()->barrier_data() == 0); - match(Set prev (GetAndSetN mem newv)); - ins_cost(2 * VOLATILE_REF_COST); - format %{ "atomic_xchgw $prev, $newv, [$mem]" %} - ins_encode %{ - __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base)); - %} - ins_pipe(pipe_serial); -%} - -instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{ - predicate(n->as_LoadStore()->barrier_data() == 0); - match(Set prev (GetAndSetP mem newv)); - ins_cost(2 * VOLATILE_REF_COST); - format %{ "atomic_xchg $prev, $newv, [$mem]" %} - ins_encode %{ - __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base)); - %} - ins_pipe(pipe_serial); -%} - -instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{ - predicate(needs_acquiring_load_exclusive(n)); - match(Set prev (GetAndSetI mem newv)); - ins_cost(VOLATILE_REF_COST); - format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %} - ins_encode %{ - __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base)); - %} - ins_pipe(pipe_serial); -%} - -instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{ - predicate(needs_acquiring_load_exclusive(n)); - match(Set prev (GetAndSetL mem newv)); - ins_cost(VOLATILE_REF_COST); - format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %} - ins_encode %{ - __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base)); - %} - ins_pipe(pipe_serial); -%} - -instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{ - predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0); - match(Set prev (GetAndSetN mem newv)); - ins_cost(VOLATILE_REF_COST); - format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %} - ins_encode %{ - __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base)); - %} - ins_pipe(pipe_serial); -%} - -instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{ - predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0)); - match(Set prev (GetAndSetP mem newv)); - ins_cost(VOLATILE_REF_COST); - format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %} - ins_encode %{ - __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base)); - %} - ins_pipe(pipe_serial); -%} - - -instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{ - match(Set newval (GetAndAddL mem incr)); - ins_cost(2 * VOLATILE_REF_COST + 1); - format %{ "get_and_addL $newval, [$mem], $incr" %} - ins_encode %{ - __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base)); - %} - ins_pipe(pipe_serial); -%} - -instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{ - predicate(n->as_LoadStore()->result_not_used()); - match(Set dummy (GetAndAddL mem incr)); - ins_cost(2 * VOLATILE_REF_COST); - format %{ "get_and_addL [$mem], $incr" %} - ins_encode %{ - __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base)); - %} - ins_pipe(pipe_serial); -%} - -instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{ - match(Set newval (GetAndAddL mem incr)); - ins_cost(2 * VOLATILE_REF_COST + 1); - format %{ "get_and_addL $newval, [$mem], $incr" %} - ins_encode %{ - __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base)); - %} - ins_pipe(pipe_serial); -%} - -instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{ - predicate(n->as_LoadStore()->result_not_used()); - match(Set dummy (GetAndAddL mem incr)); - ins_cost(2 * VOLATILE_REF_COST); - format %{ "get_and_addL [$mem], $incr" %} - ins_encode %{ - __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base)); - %} - ins_pipe(pipe_serial); -%} - -instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{ - match(Set newval (GetAndAddI mem incr)); - ins_cost(2 * VOLATILE_REF_COST + 1); - format %{ "get_and_addI $newval, [$mem], $incr" %} - ins_encode %{ - __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base)); - %} - ins_pipe(pipe_serial); -%} - -instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{ - predicate(n->as_LoadStore()->result_not_used()); - match(Set dummy (GetAndAddI mem incr)); - ins_cost(2 * VOLATILE_REF_COST); - format %{ "get_and_addI [$mem], $incr" %} - ins_encode %{ - __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base)); - %} - ins_pipe(pipe_serial); -%} - -instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{ - match(Set newval (GetAndAddI mem incr)); - ins_cost(2 * VOLATILE_REF_COST + 1); - format %{ "get_and_addI $newval, [$mem], $incr" %} - ins_encode %{ - __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base)); - %} - ins_pipe(pipe_serial); -%} - -instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{ - predicate(n->as_LoadStore()->result_not_used()); - match(Set dummy (GetAndAddI mem incr)); - ins_cost(2 * VOLATILE_REF_COST); - format %{ "get_and_addI [$mem], $incr" %} - ins_encode %{ - __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base)); - %} - ins_pipe(pipe_serial); -%} - -instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{ - predicate(needs_acquiring_load_exclusive(n)); - match(Set newval (GetAndAddL mem incr)); - ins_cost(VOLATILE_REF_COST + 1); - format %{ "get_and_addL_acq $newval, [$mem], $incr" %} - ins_encode %{ - __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base)); - %} - ins_pipe(pipe_serial); -%} - -instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{ - predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n)); - match(Set dummy (GetAndAddL mem incr)); - ins_cost(VOLATILE_REF_COST); - format %{ "get_and_addL_acq [$mem], $incr" %} - ins_encode %{ - __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base)); - %} - ins_pipe(pipe_serial); -%} - -instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{ - predicate(needs_acquiring_load_exclusive(n)); - match(Set newval (GetAndAddL mem incr)); - ins_cost(VOLATILE_REF_COST + 1); - format %{ "get_and_addL_acq $newval, [$mem], $incr" %} - ins_encode %{ - __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base)); - %} - ins_pipe(pipe_serial); -%} - -instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{ - predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n)); - match(Set dummy (GetAndAddL mem incr)); - ins_cost(VOLATILE_REF_COST); - format %{ "get_and_addL_acq [$mem], $incr" %} - ins_encode %{ - __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base)); - %} - ins_pipe(pipe_serial); -%} - -instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{ - predicate(needs_acquiring_load_exclusive(n)); - match(Set newval (GetAndAddI mem incr)); - ins_cost(VOLATILE_REF_COST + 1); - format %{ "get_and_addI_acq $newval, [$mem], $incr" %} - ins_encode %{ - __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base)); - %} - ins_pipe(pipe_serial); -%} - -instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{ - predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n)); - match(Set dummy (GetAndAddI mem incr)); - ins_cost(VOLATILE_REF_COST); - format %{ "get_and_addI_acq [$mem], $incr" %} - ins_encode %{ - __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base)); - %} - ins_pipe(pipe_serial); -%} - -instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{ - predicate(needs_acquiring_load_exclusive(n)); - match(Set newval (GetAndAddI mem incr)); - ins_cost(VOLATILE_REF_COST + 1); - format %{ "get_and_addI_acq $newval, [$mem], $incr" %} - ins_encode %{ - __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base)); - %} - ins_pipe(pipe_serial); -%} - -instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{ - predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n)); - match(Set dummy (GetAndAddI mem incr)); - ins_cost(VOLATILE_REF_COST); - format %{ "get_and_addI_acq [$mem], $incr" %} - ins_encode %{ - __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base)); - %} - ins_pipe(pipe_serial); -%} - // Manifest a CmpU result in an integer register. // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0) instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags) diff --git a/src/hotspot/cpu/aarch64/aarch64_atomic.ad b/src/hotspot/cpu/aarch64/aarch64_atomic.ad new file mode 100644 index 00000000000..faac2a43110 --- /dev/null +++ b/src/hotspot/cpu/aarch64/aarch64_atomic.ad @@ -0,0 +1,909 @@ +// Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved. +// Copyright (c) 2016, 2021, Red Hat Inc. All rights reserved. +// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +// +// This code is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License version 2 only, as +// published by the Free Software Foundation. +// +// This code is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// version 2 for more details (a copy is included in the LICENSE file that +// accompanied this code). +// +// You should have received a copy of the GNU General Public License version +// 2 along with this work; if not, write to the Free Software Foundation, +// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +// +// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA +// or visit www.oracle.com if you need additional information or have any +// questions. +// +// + +// BEGIN This file is automatically generated. Do not edit -------------- + +// Sundry CAS operations. Note that release is always true, +// regardless of the memory ordering of the CAS. This is because we +// need the volatile case to be sequentially consistent but there is +// no trailing StoreLoad barrier emitted by C2. Unfortunately we +// can't check the type of memory ordering here, so we always emit a +// STLXR. + +// This section is generated from aarch64_atomic_ad.m4 + + +instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ + match(Set res (CompareAndExchangeB mem (Binary oldval newval))); + ins_cost(2*VOLATILE_REF_COST); + effect(TEMP_DEF res, KILL cr); + format %{ + "cmpxchgb $res = $mem, $oldval, $newval\t# (byte) if $mem == $oldval then $mem <-- $newval" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::byte, /*acquire*/ false, /*release*/ true, + /*weak*/ false, $res$$Register); + __ sxtbw($res$$Register, $res$$Register); + %} + ins_pipe(pipe_slow); +%} + +instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ + match(Set res (CompareAndExchangeS mem (Binary oldval newval))); + ins_cost(2*VOLATILE_REF_COST); + effect(TEMP_DEF res, KILL cr); + format %{ + "cmpxchgs $res = $mem, $oldval, $newval\t# (short) if $mem == $oldval then $mem <-- $newval" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::halfword, /*acquire*/ false, /*release*/ true, + /*weak*/ false, $res$$Register); + __ sxthw($res$$Register, $res$$Register); + %} + ins_pipe(pipe_slow); +%} + +instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ + match(Set res (CompareAndExchangeI mem (Binary oldval newval))); + ins_cost(2*VOLATILE_REF_COST); + effect(TEMP_DEF res, KILL cr); + format %{ + "cmpxchgw $res = $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::word, /*acquire*/ false, /*release*/ true, + /*weak*/ false, $res$$Register); + %} + ins_pipe(pipe_slow); +%} + +instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{ + match(Set res (CompareAndExchangeL mem (Binary oldval newval))); + ins_cost(2*VOLATILE_REF_COST); + effect(TEMP_DEF res, KILL cr); + format %{ + "cmpxchg $res = $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::xword, /*acquire*/ false, /*release*/ true, + /*weak*/ false, $res$$Register); + %} + ins_pipe(pipe_slow); +%} + +instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{ + predicate(n->as_LoadStore()->barrier_data() == 0); + match(Set res (CompareAndExchangeN mem (Binary oldval newval))); + ins_cost(2*VOLATILE_REF_COST); + effect(TEMP_DEF res, KILL cr); + format %{ + "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::word, /*acquire*/ false, /*release*/ true, + /*weak*/ false, $res$$Register); + %} + ins_pipe(pipe_slow); +%} + +instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ + predicate(n->as_LoadStore()->barrier_data() == 0); + match(Set res (CompareAndExchangeP mem (Binary oldval newval))); + ins_cost(2*VOLATILE_REF_COST); + effect(TEMP_DEF res, KILL cr); + format %{ + "cmpxchg $res = $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::xword, /*acquire*/ false, /*release*/ true, + /*weak*/ false, $res$$Register); + %} + ins_pipe(pipe_slow); +%} + +instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ + predicate(needs_acquiring_load_exclusive(n)); + match(Set res (CompareAndExchangeB mem (Binary oldval newval))); + ins_cost(VOLATILE_REF_COST); + effect(TEMP_DEF res, KILL cr); + format %{ + "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte) if $mem == $oldval then $mem <-- $newval" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::byte, /*acquire*/ true, /*release*/ true, + /*weak*/ false, $res$$Register); + __ sxtbw($res$$Register, $res$$Register); + %} + ins_pipe(pipe_slow); +%} + +instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ + predicate(needs_acquiring_load_exclusive(n)); + match(Set res (CompareAndExchangeS mem (Binary oldval newval))); + ins_cost(VOLATILE_REF_COST); + effect(TEMP_DEF res, KILL cr); + format %{ + "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short) if $mem == $oldval then $mem <-- $newval" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::halfword, /*acquire*/ true, /*release*/ true, + /*weak*/ false, $res$$Register); + __ sxthw($res$$Register, $res$$Register); + %} + ins_pipe(pipe_slow); +%} + +instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ + predicate(needs_acquiring_load_exclusive(n)); + match(Set res (CompareAndExchangeI mem (Binary oldval newval))); + ins_cost(VOLATILE_REF_COST); + effect(TEMP_DEF res, KILL cr); + format %{ + "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::word, /*acquire*/ true, /*release*/ true, + /*weak*/ false, $res$$Register); + %} + ins_pipe(pipe_slow); +%} + +instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{ + predicate(needs_acquiring_load_exclusive(n)); + match(Set res (CompareAndExchangeL mem (Binary oldval newval))); + ins_cost(VOLATILE_REF_COST); + effect(TEMP_DEF res, KILL cr); + format %{ + "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::xword, /*acquire*/ true, /*release*/ true, + /*weak*/ false, $res$$Register); + %} + ins_pipe(pipe_slow); +%} + +instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{ + predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0); + match(Set res (CompareAndExchangeN mem (Binary oldval newval))); + ins_cost(VOLATILE_REF_COST); + effect(TEMP_DEF res, KILL cr); + format %{ + "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::word, /*acquire*/ true, /*release*/ true, + /*weak*/ false, $res$$Register); + %} + ins_pipe(pipe_slow); +%} + +instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ + predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0)); + match(Set res (CompareAndExchangeP mem (Binary oldval newval))); + ins_cost(VOLATILE_REF_COST); + effect(TEMP_DEF res, KILL cr); + format %{ + "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::xword, /*acquire*/ true, /*release*/ true, + /*weak*/ false, $res$$Register); + %} + ins_pipe(pipe_slow); +%} + +instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ + match(Set res (CompareAndSwapB mem (Binary oldval newval))); + ins_cost(2*VOLATILE_REF_COST); + effect(KILL cr); + format %{ + "cmpxchgb $res = $mem, $oldval, $newval\t# (byte) if $mem == $oldval then $mem <-- $newval" + "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::byte, /*acquire*/ false, /*release*/ true, + /*weak*/ false, noreg); + __ csetw($res$$Register, Assembler::EQ); + %} + ins_pipe(pipe_slow); +%} + +instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ + match(Set res (CompareAndSwapS mem (Binary oldval newval))); + ins_cost(2*VOLATILE_REF_COST); + effect(KILL cr); + format %{ + "cmpxchgs $res = $mem, $oldval, $newval\t# (short) if $mem == $oldval then $mem <-- $newval" + "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::halfword, /*acquire*/ false, /*release*/ true, + /*weak*/ false, noreg); + __ csetw($res$$Register, Assembler::EQ); + %} + ins_pipe(pipe_slow); +%} + +instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ + match(Set res (CompareAndSwapI mem (Binary oldval newval))); + ins_cost(2*VOLATILE_REF_COST); + effect(KILL cr); + format %{ + "cmpxchgw $res = $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval" + "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::word, /*acquire*/ false, /*release*/ true, + /*weak*/ false, noreg); + __ csetw($res$$Register, Assembler::EQ); + %} + ins_pipe(pipe_slow); +%} + +instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{ + match(Set res (CompareAndSwapL mem (Binary oldval newval))); + ins_cost(2*VOLATILE_REF_COST); + effect(KILL cr); + format %{ + "cmpxchg $res = $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval" + "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::xword, /*acquire*/ false, /*release*/ true, + /*weak*/ false, noreg); + __ csetw($res$$Register, Assembler::EQ); + %} + ins_pipe(pipe_slow); +%} + +instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{ + predicate(n->as_LoadStore()->barrier_data() == 0); + match(Set res (CompareAndSwapN mem (Binary oldval newval))); + ins_cost(2*VOLATILE_REF_COST); + effect(KILL cr); + format %{ + "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval" + "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::word, /*acquire*/ false, /*release*/ true, + /*weak*/ false, noreg); + __ csetw($res$$Register, Assembler::EQ); + %} + ins_pipe(pipe_slow); +%} + +instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ + predicate(n->as_LoadStore()->barrier_data() == 0); + match(Set res (CompareAndSwapP mem (Binary oldval newval))); + ins_cost(2*VOLATILE_REF_COST); + effect(KILL cr); + format %{ + "cmpxchg $res = $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval" + "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::xword, /*acquire*/ false, /*release*/ true, + /*weak*/ false, noreg); + __ csetw($res$$Register, Assembler::EQ); + %} + ins_pipe(pipe_slow); +%} + +instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ + predicate(needs_acquiring_load_exclusive(n)); + match(Set res (CompareAndSwapB mem (Binary oldval newval))); + ins_cost(VOLATILE_REF_COST); + effect(KILL cr); + format %{ + "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte) if $mem == $oldval then $mem <-- $newval" + "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::byte, /*acquire*/ true, /*release*/ true, + /*weak*/ false, noreg); + __ csetw($res$$Register, Assembler::EQ); + %} + ins_pipe(pipe_slow); +%} + +instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ + predicate(needs_acquiring_load_exclusive(n)); + match(Set res (CompareAndSwapS mem (Binary oldval newval))); + ins_cost(VOLATILE_REF_COST); + effect(KILL cr); + format %{ + "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short) if $mem == $oldval then $mem <-- $newval" + "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::halfword, /*acquire*/ true, /*release*/ true, + /*weak*/ false, noreg); + __ csetw($res$$Register, Assembler::EQ); + %} + ins_pipe(pipe_slow); +%} + +instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ + predicate(needs_acquiring_load_exclusive(n)); + match(Set res (CompareAndSwapI mem (Binary oldval newval))); + ins_cost(VOLATILE_REF_COST); + effect(KILL cr); + format %{ + "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval" + "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::word, /*acquire*/ true, /*release*/ true, + /*weak*/ false, noreg); + __ csetw($res$$Register, Assembler::EQ); + %} + ins_pipe(pipe_slow); +%} + +instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{ + predicate(needs_acquiring_load_exclusive(n)); + match(Set res (CompareAndSwapL mem (Binary oldval newval))); + ins_cost(VOLATILE_REF_COST); + effect(KILL cr); + format %{ + "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval" + "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::xword, /*acquire*/ true, /*release*/ true, + /*weak*/ false, noreg); + __ csetw($res$$Register, Assembler::EQ); + %} + ins_pipe(pipe_slow); +%} + +instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{ + predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0); + match(Set res (CompareAndSwapN mem (Binary oldval newval))); + ins_cost(VOLATILE_REF_COST); + effect(KILL cr); + format %{ + "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval" + "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::word, /*acquire*/ true, /*release*/ true, + /*weak*/ false, noreg); + __ csetw($res$$Register, Assembler::EQ); + %} + ins_pipe(pipe_slow); +%} + +instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ + predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0)); + match(Set res (CompareAndSwapP mem (Binary oldval newval))); + ins_cost(VOLATILE_REF_COST); + effect(KILL cr); + format %{ + "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval" + "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::xword, /*acquire*/ true, /*release*/ true, + /*weak*/ false, noreg); + __ csetw($res$$Register, Assembler::EQ); + %} + ins_pipe(pipe_slow); +%} + +instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ + match(Set res (WeakCompareAndSwapB mem (Binary oldval newval))); + ins_cost(2*VOLATILE_REF_COST); + effect(KILL cr); + format %{ + "cmpxchgb_weak $res = $mem, $oldval, $newval\t# (byte) if $mem == $oldval then $mem <-- $newval" + "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::byte, /*acquire*/ false, /*release*/ true, + /*weak*/ true, noreg); + __ csetw($res$$Register, Assembler::EQ); + %} + ins_pipe(pipe_slow); +%} + +instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ + match(Set res (WeakCompareAndSwapS mem (Binary oldval newval))); + ins_cost(2*VOLATILE_REF_COST); + effect(KILL cr); + format %{ + "cmpxchgs_weak $res = $mem, $oldval, $newval\t# (short) if $mem == $oldval then $mem <-- $newval" + "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::halfword, /*acquire*/ false, /*release*/ true, + /*weak*/ true, noreg); + __ csetw($res$$Register, Assembler::EQ); + %} + ins_pipe(pipe_slow); +%} + +instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ + match(Set res (WeakCompareAndSwapI mem (Binary oldval newval))); + ins_cost(2*VOLATILE_REF_COST); + effect(KILL cr); + format %{ + "cmpxchgw_weak $res = $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval" + "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::word, /*acquire*/ false, /*release*/ true, + /*weak*/ true, noreg); + __ csetw($res$$Register, Assembler::EQ); + %} + ins_pipe(pipe_slow); +%} + +instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{ + match(Set res (WeakCompareAndSwapL mem (Binary oldval newval))); + ins_cost(2*VOLATILE_REF_COST); + effect(KILL cr); + format %{ + "cmpxchg_weak $res = $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval" + "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::xword, /*acquire*/ false, /*release*/ true, + /*weak*/ true, noreg); + __ csetw($res$$Register, Assembler::EQ); + %} + ins_pipe(pipe_slow); +%} + +instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{ + predicate(n->as_LoadStore()->barrier_data() == 0); + match(Set res (WeakCompareAndSwapN mem (Binary oldval newval))); + ins_cost(2*VOLATILE_REF_COST); + effect(KILL cr); + format %{ + "cmpxchgw_weak $res = $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval" + "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::word, /*acquire*/ false, /*release*/ true, + /*weak*/ true, noreg); + __ csetw($res$$Register, Assembler::EQ); + %} + ins_pipe(pipe_slow); +%} + +instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ + predicate(n->as_LoadStore()->barrier_data() == 0); + match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); + ins_cost(2*VOLATILE_REF_COST); + effect(KILL cr); + format %{ + "cmpxchg_weak $res = $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval" + "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::xword, /*acquire*/ false, /*release*/ true, + /*weak*/ true, noreg); + __ csetw($res$$Register, Assembler::EQ); + %} + ins_pipe(pipe_slow); +%} + +instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ + predicate(needs_acquiring_load_exclusive(n)); + match(Set res (WeakCompareAndSwapB mem (Binary oldval newval))); + ins_cost(VOLATILE_REF_COST); + effect(KILL cr); + format %{ + "cmpxchgb_acq_weak $res = $mem, $oldval, $newval\t# (byte) if $mem == $oldval then $mem <-- $newval" + "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::byte, /*acquire*/ true, /*release*/ true, + /*weak*/ true, noreg); + __ csetw($res$$Register, Assembler::EQ); + %} + ins_pipe(pipe_slow); +%} + +instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ + predicate(needs_acquiring_load_exclusive(n)); + match(Set res (WeakCompareAndSwapS mem (Binary oldval newval))); + ins_cost(VOLATILE_REF_COST); + effect(KILL cr); + format %{ + "cmpxchgs_acq_weak $res = $mem, $oldval, $newval\t# (short) if $mem == $oldval then $mem <-- $newval" + "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::halfword, /*acquire*/ true, /*release*/ true, + /*weak*/ true, noreg); + __ csetw($res$$Register, Assembler::EQ); + %} + ins_pipe(pipe_slow); +%} + +instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ + predicate(needs_acquiring_load_exclusive(n)); + match(Set res (WeakCompareAndSwapI mem (Binary oldval newval))); + ins_cost(VOLATILE_REF_COST); + effect(KILL cr); + format %{ + "cmpxchgw_acq_weak $res = $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval" + "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::word, /*acquire*/ true, /*release*/ true, + /*weak*/ true, noreg); + __ csetw($res$$Register, Assembler::EQ); + %} + ins_pipe(pipe_slow); +%} + +instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{ + predicate(needs_acquiring_load_exclusive(n)); + match(Set res (WeakCompareAndSwapL mem (Binary oldval newval))); + ins_cost(VOLATILE_REF_COST); + effect(KILL cr); + format %{ + "cmpxchg_acq_weak $res = $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval" + "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::xword, /*acquire*/ true, /*release*/ true, + /*weak*/ true, noreg); + __ csetw($res$$Register, Assembler::EQ); + %} + ins_pipe(pipe_slow); +%} + +instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{ + predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0); + match(Set res (WeakCompareAndSwapN mem (Binary oldval newval))); + ins_cost(VOLATILE_REF_COST); + effect(KILL cr); + format %{ + "cmpxchgw_acq_weak $res = $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval" + "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::word, /*acquire*/ true, /*release*/ true, + /*weak*/ true, noreg); + __ csetw($res$$Register, Assembler::EQ); + %} + ins_pipe(pipe_slow); +%} + +instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ + predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0)); + match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); + ins_cost(VOLATILE_REF_COST); + effect(KILL cr); + format %{ + "cmpxchg_acq_weak $res = $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval" + "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::xword, /*acquire*/ true, /*release*/ true, + /*weak*/ true, noreg); + __ csetw($res$$Register, Assembler::EQ); + %} + ins_pipe(pipe_slow); +%} + +instruct getAndSetI(indirect mem, iRegI newval, iRegINoSp oldval) %{ + match(Set oldval (GetAndSetI mem newval)); + ins_cost(2*VOLATILE_REF_COST); + format %{ "atomic_xchgw $oldval, $newval, [$mem]" %} + ins_encode %{ + __ atomic_xchgw($oldval$$Register, $newval$$Register, as_Register($mem$$base)); + %} + ins_pipe(pipe_serial); +%} + +instruct getAndSetL(indirect mem, iRegL newval, iRegLNoSp oldval) %{ + match(Set oldval (GetAndSetL mem newval)); + ins_cost(2*VOLATILE_REF_COST); + format %{ "atomic_xchg $oldval, $newval, [$mem]" %} + ins_encode %{ + __ atomic_xchg($oldval$$Register, $newval$$Register, as_Register($mem$$base)); + %} + ins_pipe(pipe_serial); +%} + +instruct getAndSetN(indirect mem, iRegN newval, iRegNNoSp oldval) %{ + predicate(n->as_LoadStore()->barrier_data() == 0); + match(Set oldval (GetAndSetN mem newval)); + ins_cost(2*VOLATILE_REF_COST); + format %{ "atomic_xchgw $oldval, $newval, [$mem]" %} + ins_encode %{ + __ atomic_xchgw($oldval$$Register, $newval$$Register, as_Register($mem$$base)); + %} + ins_pipe(pipe_serial); +%} + +instruct getAndSetP(indirect mem, iRegP newval, iRegPNoSp oldval) %{ + predicate(n->as_LoadStore()->barrier_data() == 0); + match(Set oldval (GetAndSetP mem newval)); + ins_cost(2*VOLATILE_REF_COST); + format %{ "atomic_xchg $oldval, $newval, [$mem]" %} + ins_encode %{ + __ atomic_xchg($oldval$$Register, $newval$$Register, as_Register($mem$$base)); + %} + ins_pipe(pipe_serial); +%} + +instruct getAndSetIAcq(indirect mem, iRegI newval, iRegINoSp oldval) %{ + predicate(needs_acquiring_load_exclusive(n)); + match(Set oldval (GetAndSetI mem newval)); + ins_cost(2*VOLATILE_REF_COST); + format %{ "atomic_xchgw_acq $oldval, $newval, [$mem]" %} + ins_encode %{ + __ atomic_xchgalw($oldval$$Register, $newval$$Register, as_Register($mem$$base)); + %} + ins_pipe(pipe_serial); +%} + +instruct getAndSetLAcq(indirect mem, iRegL newval, iRegLNoSp oldval) %{ + predicate(needs_acquiring_load_exclusive(n)); + match(Set oldval (GetAndSetL mem newval)); + ins_cost(2*VOLATILE_REF_COST); + format %{ "atomic_xchg_acq $oldval, $newval, [$mem]" %} + ins_encode %{ + __ atomic_xchgal($oldval$$Register, $newval$$Register, as_Register($mem$$base)); + %} + ins_pipe(pipe_serial); +%} + +instruct getAndSetNAcq(indirect mem, iRegN newval, iRegNNoSp oldval) %{ + predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0); + match(Set oldval (GetAndSetN mem newval)); + ins_cost(2*VOLATILE_REF_COST); + format %{ "atomic_xchgw_acq $oldval, $newval, [$mem]" %} + ins_encode %{ + __ atomic_xchgalw($oldval$$Register, $newval$$Register, as_Register($mem$$base)); + %} + ins_pipe(pipe_serial); +%} + +instruct getAndSetPAcq(indirect mem, iRegP newval, iRegPNoSp oldval) %{ + predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0)); + match(Set oldval (GetAndSetP mem newval)); + ins_cost(2*VOLATILE_REF_COST); + format %{ "atomic_xchg_acq $oldval, $newval, [$mem]" %} + ins_encode %{ + __ atomic_xchgal($oldval$$Register, $newval$$Register, as_Register($mem$$base)); + %} + ins_pipe(pipe_serial); +%} + +instruct getAndAddI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{ + match(Set newval (GetAndAddI mem incr)); + ins_cost(2*VOLATILE_REF_COST+1); + format %{ "get_and_addI $newval, [$mem], $incr" %} + ins_encode %{ + __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base)); + %} + ins_pipe(pipe_serial); +%} + +instruct getAndAddIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{ + predicate(needs_acquiring_load_exclusive(n)); + match(Set newval (GetAndAddI mem incr)); + ins_cost(VOLATILE_REF_COST+1); + format %{ "get_and_addI_acq $newval, [$mem], $incr" %} + ins_encode %{ + __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base)); + %} + ins_pipe(pipe_serial); +%} + +instruct getAndAddINoRes(indirect mem, Universe dummy, iRegIorL2I incr) %{ + predicate(n->as_LoadStore()->result_not_used()); + match(Set dummy (GetAndAddI mem incr)); + ins_cost(2*VOLATILE_REF_COST); + format %{ "get_and_addI noreg, [$mem], $incr" %} + ins_encode %{ + __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base)); + %} + ins_pipe(pipe_serial); +%} + +instruct getAndAddIAcqNoRes(indirect mem, Universe dummy, iRegIorL2I incr) %{ + predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n)); + match(Set dummy (GetAndAddI mem incr)); + ins_cost(VOLATILE_REF_COST); + format %{ "get_and_addI_acq noreg, [$mem], $incr" %} + ins_encode %{ + __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base)); + %} + ins_pipe(pipe_serial); +%} + +instruct getAndAddIConst(indirect mem, iRegINoSp newval, immIAddSub incr) %{ + match(Set newval (GetAndAddI mem incr)); + ins_cost(2*VOLATILE_REF_COST+1); + format %{ "get_and_addI $newval, [$mem], $incr" %} + ins_encode %{ + __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base)); + %} + ins_pipe(pipe_serial); +%} + +instruct getAndAddIAcqConst(indirect mem, iRegINoSp newval, immIAddSub incr) %{ + predicate(needs_acquiring_load_exclusive(n)); + match(Set newval (GetAndAddI mem incr)); + ins_cost(VOLATILE_REF_COST+1); + format %{ "get_and_addI_acq $newval, [$mem], $incr" %} + ins_encode %{ + __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base)); + %} + ins_pipe(pipe_serial); +%} + +instruct getAndAddINoResConst(indirect mem, Universe dummy, immIAddSub incr) %{ + predicate(n->as_LoadStore()->result_not_used()); + match(Set dummy (GetAndAddI mem incr)); + ins_cost(2*VOLATILE_REF_COST); + format %{ "get_and_addI noreg, [$mem], $incr" %} + ins_encode %{ + __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base)); + %} + ins_pipe(pipe_serial); +%} + +instruct getAndAddIAcqNoResConst(indirect mem, Universe dummy, immIAddSub incr) %{ + predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n)); + match(Set dummy (GetAndAddI mem incr)); + ins_cost(VOLATILE_REF_COST); + format %{ "get_and_addI_acq noreg, [$mem], $incr" %} + ins_encode %{ + __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base)); + %} + ins_pipe(pipe_serial); +%} + +instruct getAndAddL(indirect mem, iRegLNoSp newval, iRegL incr) %{ + match(Set newval (GetAndAddL mem incr)); + ins_cost(2*VOLATILE_REF_COST+1); + format %{ "get_and_addL $newval, [$mem], $incr" %} + ins_encode %{ + __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base)); + %} + ins_pipe(pipe_serial); +%} + +instruct getAndAddLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{ + predicate(needs_acquiring_load_exclusive(n)); + match(Set newval (GetAndAddL mem incr)); + ins_cost(VOLATILE_REF_COST+1); + format %{ "get_and_addL_acq $newval, [$mem], $incr" %} + ins_encode %{ + __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base)); + %} + ins_pipe(pipe_serial); +%} + +instruct getAndAddLNoRes(indirect mem, Universe dummy, iRegL incr) %{ + predicate(n->as_LoadStore()->result_not_used()); + match(Set dummy (GetAndAddL mem incr)); + ins_cost(2*VOLATILE_REF_COST); + format %{ "get_and_addL noreg, [$mem], $incr" %} + ins_encode %{ + __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base)); + %} + ins_pipe(pipe_serial); +%} + +instruct getAndAddLAcqNoRes(indirect mem, Universe dummy, iRegL incr) %{ + predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n)); + match(Set dummy (GetAndAddL mem incr)); + ins_cost(VOLATILE_REF_COST); + format %{ "get_and_addL_acq noreg, [$mem], $incr" %} + ins_encode %{ + __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base)); + %} + ins_pipe(pipe_serial); +%} + +instruct getAndAddLConst(indirect mem, iRegLNoSp newval, immLAddSub incr) %{ + match(Set newval (GetAndAddL mem incr)); + ins_cost(2*VOLATILE_REF_COST+1); + format %{ "get_and_addL $newval, [$mem], $incr" %} + ins_encode %{ + __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base)); + %} + ins_pipe(pipe_serial); +%} + +instruct getAndAddLAcqConst(indirect mem, iRegLNoSp newval, immLAddSub incr) %{ + predicate(needs_acquiring_load_exclusive(n)); + match(Set newval (GetAndAddL mem incr)); + ins_cost(VOLATILE_REF_COST+1); + format %{ "get_and_addL_acq $newval, [$mem], $incr" %} + ins_encode %{ + __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base)); + %} + ins_pipe(pipe_serial); +%} + +instruct getAndAddLNoResConst(indirect mem, Universe dummy, immLAddSub incr) %{ + predicate(n->as_LoadStore()->result_not_used()); + match(Set dummy (GetAndAddL mem incr)); + ins_cost(2*VOLATILE_REF_COST); + format %{ "get_and_addL noreg, [$mem], $incr" %} + ins_encode %{ + __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base)); + %} + ins_pipe(pipe_serial); +%} + +instruct getAndAddLAcqNoResConst(indirect mem, Universe dummy, immLAddSub incr) %{ + predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n)); + match(Set dummy (GetAndAddL mem incr)); + ins_cost(VOLATILE_REF_COST); + format %{ "get_and_addL_acq noreg, [$mem], $incr" %} + ins_encode %{ + __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base)); + %} + ins_pipe(pipe_serial); +%} diff --git a/src/hotspot/cpu/aarch64/aarch64_atomic_ad.m4 b/src/hotspot/cpu/aarch64/aarch64_atomic_ad.m4 new file mode 100644 index 00000000000..721b720873a --- /dev/null +++ b/src/hotspot/cpu/aarch64/aarch64_atomic_ad.m4 @@ -0,0 +1,246 @@ +// Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved. +// Copyright (c) 2016, 2021, Red Hat Inc. All rights reserved. +// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +// +// This code is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License version 2 only, as +// published by the Free Software Foundation. +// +// This code is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// version 2 for more details (a copy is included in the LICENSE file that +// accompanied this code). +// +// You should have received a copy of the GNU General Public License version +// 2 along with this work; if not, write to the Free Software Foundation, +// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +// +// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA +// or visit www.oracle.com if you need additional information or have any +// questions. +// +// + +// BEGIN This file is automatically generated. Do not edit -------------- + +// Sundry CAS operations. Note that release is always true, +// regardless of the memory ordering of the CAS. This is because we +// need the volatile case to be sequentially consistent but there is +// no trailing StoreLoad barrier emitted by C2. Unfortunately we +// can't check the type of memory ordering here, so we always emit a +// STLXR. + +// This section is generated from aarch64_atomic_ad.m4 + +dnl Return Arg1 with two spaces before it. We need this because m4 +dnl strips leading spaces from macro args. +define(`INDENT', ` $1')dnl +dnl +dnl +dnl +dnl ====================== CompareAndExchange* +dnl +define(`CAE_INSN1', +` +instruct compareAndExchange$1$7(iReg$2NoSp res, indirect mem, iReg$2 oldval, iReg$2 newval, rFlagsReg cr) %{ +ifelse($7,Acq,INDENT(predicate(needs_acquiring_load_exclusive(n));),`dnl') + match(Set res (CompareAndExchange$1 mem (Binary oldval newval))); + ins_cost(`'ifelse($7,Acq,,2*)VOLATILE_REF_COST); + effect(TEMP_DEF res, KILL cr); + format %{ + "cmpxchg$5`'ifelse($7,Acq,_acq,) $res = $mem, $oldval, $newval\t# ($3) if $mem == $oldval then $mem <-- $newval" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::$4, /*acquire*/ ifelse($7,Acq,true,false), /*release*/ true, + /*weak*/ false, $res$$Register); + __ $6($res$$Register, $res$$Register); + %} + ins_pipe(pipe_slow); +%}')dnl +define(`CAE_INSN2', +` +instruct compareAndExchange$1$6(iReg$2NoSp res, indirect mem, iReg$2 oldval, iReg$2 newval, rFlagsReg cr) %{ +ifelse($1$6,PAcq,INDENT(predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));), + $1$6,NAcq,INDENT(predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);), + $1,P,INDENT(predicate(n->as_LoadStore()->barrier_data() == 0);), + $1,N,INDENT(predicate(n->as_LoadStore()->barrier_data() == 0);), + $6,Acq,INDENT(predicate(needs_acquiring_load_exclusive(n));), + `dnl') + match(Set res (CompareAndExchange$1 mem (Binary oldval newval))); + ins_cost(`'ifelse($6,Acq,,2*)VOLATILE_REF_COST); + effect(TEMP_DEF res, KILL cr); + format %{ + "cmpxchg$5`'ifelse($6,Acq,_acq,) $res = $mem, $oldval, $newval\t# ($3) if $mem == $oldval then $mem <-- $newval" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::$4, /*acquire*/ ifelse($6,Acq,true,false), /*release*/ true, + /*weak*/ false, $res$$Register); + %} + ins_pipe(pipe_slow); +%}')dnl +dnl +CAE_INSN1(B, I, byte, byte, b, sxtbw, ) +CAE_INSN1(S, I, short, halfword, s, sxthw, ) +CAE_INSN2(I, I, int, word, w, , ) +CAE_INSN2(L, L, long, xword, , , ) +CAE_INSN2(N, N, narrow oop, word, w, , ) +CAE_INSN2(P, P, ptr, xword, , , ) +dnl +CAE_INSN1(B, I, byte, byte, b, sxtbw, Acq) +CAE_INSN1(S, I, short, halfword, s, sxthw, Acq) +CAE_INSN2(I, I, int, word, w, Acq) +CAE_INSN2(L, L, long, xword, , Acq) +CAE_INSN2(N, N, narrow oop, word, w, Acq) +CAE_INSN2(P, P, ptr, xword, , Acq) +dnl +dnl +dnl +dnl ====================== (Weak)CompareAndSwap* +dnl +define(`CAS_INSN1', +` +instruct ifelse($7,Weak,'weakCompare`,'compare`)AndSwap$1$6(iRegINoSp res, indirect mem, iReg$2 oldval, iReg$2 newval, rFlagsReg cr) %{ +ifelse($6,Acq,INDENT(predicate(needs_acquiring_load_exclusive(n));),`dnl') + match(Set res ($7CompareAndSwap$1 mem (Binary oldval newval))); + ins_cost(`'ifelse($6,Acq,,2*)VOLATILE_REF_COST); + effect(KILL cr); + format %{ + "cmpxchg$5`'ifelse($6,Acq,_acq,)`'ifelse($7,Weak,_weak) $res = $mem, $oldval, $newval\t# ($3) if $mem == $oldval then $mem <-- $newval" + "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::$4, /*acquire*/ ifelse($6,Acq,true,false), /*release*/ true, + /*weak*/ ifelse($7,Weak,true,false), noreg); + __ csetw($res$$Register, Assembler::EQ); + %} + ins_pipe(pipe_slow); +%}')dnl +dnl +define(`CAS_INSN2', +` +instruct ifelse($7,Weak,'weakCompare`,'compare`)AndSwap$1$6(iRegINoSp res, indirect mem, iReg$2 oldval, iReg$2 newval, rFlagsReg cr) %{ +ifelse($1$6,PAcq,INDENT(predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));), + $1$6,NAcq,INDENT(predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);), + $1,P,INDENT(predicate(n->as_LoadStore()->barrier_data() == 0);), + $1,N,INDENT(predicate(n->as_LoadStore()->barrier_data() == 0);), + $6,Acq,INDENT(predicate(needs_acquiring_load_exclusive(n));), + `dnl') + match(Set res ($7CompareAndSwap$1 mem (Binary oldval newval))); + ins_cost(`'ifelse($6,Acq,,2*)VOLATILE_REF_COST); + effect(KILL cr); + format %{ + "cmpxchg$5`'ifelse($6,Acq,_acq,)`'ifelse($7,Weak,_weak) $res = $mem, $oldval, $newval\t# ($3) if $mem == $oldval then $mem <-- $newval" + "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" + %} + ins_encode %{ + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, + Assembler::$4, /*acquire*/ ifelse($6,Acq,true,false), /*release*/ true, + /*weak*/ ifelse($7,Weak,true,false), noreg); + __ csetw($res$$Register, Assembler::EQ); + %} + ins_pipe(pipe_slow); +%}')dnl +dnl +CAS_INSN1(B, I, byte, byte, b, , ) +CAS_INSN1(S, I, short, halfword, s, , ) +CAS_INSN2(I, I, int, word, w, , ) +CAS_INSN2(L, L, long, xword, , , ) +CAS_INSN2(N, N, narrow oop, word, w, , ) +CAS_INSN2(P, P, ptr, xword, , , ) +dnl +CAS_INSN1(B, I, byte, byte, b, Acq, ) +CAS_INSN1(S, I, short, halfword, s, Acq, ) +CAS_INSN2(I, I, int, word, w, Acq, ) +CAS_INSN2(L, L, long, xword, , Acq, ) +CAS_INSN2(N, N, narrow oop, word, w, Acq, ) +CAS_INSN2(P, P, ptr, xword, , Acq, ) +dnl +CAS_INSN1(B, I, byte, byte, b, , Weak) +CAS_INSN1(S, I, short, halfword, s, , Weak) +CAS_INSN2(I, I, int, word, w, , Weak) +CAS_INSN2(L, L, long, xword, , , Weak) +CAS_INSN2(N, N, narrow oop, word, w, , Weak) +CAS_INSN2(P, P, ptr, xword, , , Weak) +dnl +CAS_INSN1(B, I, byte, byte, b, Acq, Weak) +CAS_INSN1(S, I, short, halfword, s, Acq, Weak) +CAS_INSN2(I, I, int, word, w, Acq, Weak) +CAS_INSN2(L, L, long, xword, , Acq, Weak) +CAS_INSN2(N, N, narrow oop, word, w, Acq, Weak) +CAS_INSN2(P, P, ptr, xword, , Acq, Weak) +dnl +dnl +dnl +dnl ====================== GetAndSet* +dnl +define(`GAS_INSN1', +` +instruct getAndSet$1$3(indirect mem, iReg$1 newval, iReg$1NoSp oldval) %{ +ifelse($1$3,PAcq,INDENT(predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));), + $1$3,NAcq,INDENT(predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);), + $1,P,INDENT(predicate(n->as_LoadStore()->barrier_data() == 0);), + $1,N,INDENT(predicate(n->as_LoadStore()->barrier_data() == 0);), + $3,Acq,INDENT(predicate(needs_acquiring_load_exclusive(n));), + `dnl') + match(Set oldval (GetAndSet$1 mem newval)); + ins_cost(`'ifelse($4,Acq,,2*)VOLATILE_REF_COST); + format %{ "atomic_xchg$2`'ifelse($3,Acq,_acq) $oldval, $newval, [$mem]" %} + ins_encode %{ + __ atomic_xchg`'ifelse($3,Acq,al)$2($oldval$$Register, $newval$$Register, as_Register($mem$$base)); + %} + ins_pipe(pipe_serial); +%}')dnl +dnl +GAS_INSN1(I, w, ) +GAS_INSN1(L, , ) +GAS_INSN1(N, w, ) +GAS_INSN1(P, , ) +dnl +GAS_INSN1(I, w, Acq) +GAS_INSN1(L, , Acq) +GAS_INSN1(N, w, Acq) +GAS_INSN1(P, , Acq) +dnl +dnl +dnl +dnl ====================== GetAndAdd* +dnl +define(`GAA_INSN1', +` +instruct getAndAdd$1$4$5$6(indirect mem, `'ifelse($5,NoRes,Universe dummy,iReg$1NoSp newval), `'ifelse($6,Const,imm$1AddSub incr,iReg$2 incr)) %{ +ifelse($4$5,AcqNoRes,INDENT(predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));), + $5,NoRes,INDENT(predicate(n->as_LoadStore()->result_not_used());), + $4,Acq,INDENT(predicate(needs_acquiring_load_exclusive(n));), + `dnl') + match(Set ifelse($5,NoRes,dummy,newval) (GetAndAdd$1 mem incr)); + ins_cost(`'ifelse($4,Acq,,2*)VOLATILE_REF_COST`'ifelse($5,NoRes,,+1)); + format %{ "get_and_add$1`'ifelse($4,Acq,_acq) `'ifelse($5,NoRes,noreg,$newval), [$mem], $incr" %} + ins_encode %{ + __ atomic_add`'ifelse($4,Acq,al)$3(`'ifelse($5,NoRes,noreg,$newval$$Register), `'ifelse($6,Const,$incr$$constant,$incr$$Register), as_Register($mem$$base)); + %} + ins_pipe(pipe_serial); +%}')dnl +dnl +dnl +GAA_INSN1(I, IorL2I, w, , , ) +GAA_INSN1(I, IorL2I, w, Acq, , ) +GAA_INSN1(I, IorL2I, w, , NoRes, ) +GAA_INSN1(I, IorL2I, w, Acq, NoRes, ) +GAA_INSN1(I, I, w, , , Const) +GAA_INSN1(I, I, w, Acq, , Const) +GAA_INSN1(I, I, w, , NoRes, Const) +GAA_INSN1(I, I, w, Acq, NoRes, Const) +dnl +GAA_INSN1(L, L, , , , ) +GAA_INSN1(L, L, , Acq, , ) +GAA_INSN1(L, L, , , NoRes, ) +GAA_INSN1(L, L, , Acq, NoRes, ) +GAA_INSN1(L, L, , , , Const) +GAA_INSN1(L, L, , Acq, , Const) +GAA_INSN1(L, L, , , NoRes, Const) +GAA_INSN1(L, L, , Acq, NoRes, Const) +dnl diff --git a/src/hotspot/cpu/aarch64/cas.m4 b/src/hotspot/cpu/aarch64/cas.m4 deleted file mode 100644 index 7e13e153db1..00000000000 --- a/src/hotspot/cpu/aarch64/cas.m4 +++ /dev/null @@ -1,161 +0,0 @@ -dnl Copyright (c) 2016, 2021, Red Hat Inc. All rights reserved. -dnl DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. -dnl -dnl This code is free software; you can redistribute it and/or modify it -dnl under the terms of the GNU General Public License version 2 only, as -dnl published by the Free Software Foundation. -dnl -dnl This code is distributed in the hope that it will be useful, but WITHOUT -dnl ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -dnl FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -dnl version 2 for more details (a copy is included in the LICENSE file that -dnl accompanied this code). -dnl -dnl You should have received a copy of the GNU General Public License version -dnl 2 along with this work; if not, write to the Free Software Foundation, -dnl Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. -dnl -dnl Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA -dnl or visit www.oracle.com if you need additional information or have any -dnl questions. -dnl -dnl -dnl Process this file with m4 cas.m4 to generate the CAE and wCAS -dnl instructions used in aarch64.ad. -dnl - -// BEGIN This section of the file is automatically generated. Do not edit -------------- - -// Sundry CAS operations. Note that release is always true, -// regardless of the memory ordering of the CAS. This is because we -// need the volatile case to be sequentially consistent but there is -// no trailing StoreLoad barrier emitted by C2. Unfortunately we -// can't check the type of memory ordering here, so we always emit a -// STLXR. - -// This section is generated from cas.m4 - -dnl Return Arg1 with two spaces before it. We need this because m4 -dnl strips leading spaces from macro args. -define(`INDENT', ` $1')dnl -dnl -define(`CAS_INSN', -` -// This pattern is generated automatically from cas.m4. -// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE -instruct compareAndExchange$1$6(iReg$2NoSp res, indirect mem, iReg$2 oldval, iReg$2 newval, rFlagsReg cr) %{ -ifelse($1$6,PAcq,INDENT(predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));), - $1$6,NAcq,INDENT(predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);), - $1,P,INDENT(predicate(n->as_LoadStore()->barrier_data() == 0);), - $1,N,INDENT(predicate(n->as_LoadStore()->barrier_data() == 0);), - $6,Acq,INDENT(predicate(needs_acquiring_load_exclusive(n));), - `dnl') - match(Set res (CompareAndExchange$1 mem (Binary oldval newval))); - ifelse($6,Acq,'ins_cost(VOLATILE_REF_COST);`,'ins_cost(2 * VOLATILE_REF_COST);`) - effect(TEMP_DEF res, KILL cr); - format %{ - "cmpxchg$5`'ifelse($6,Acq,_acq,) $res = $mem, $oldval, $newval\t# ($3, weak) if $mem == $oldval then $mem <-- $newval" - %} - ins_encode %{ - __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, - Assembler::$4, /*acquire*/ ifelse($6,Acq,true,false), /*release*/ true, - /*weak*/ false, $res$$Register); - %} - ins_pipe(pipe_slow); -%}')dnl -define(`CAS_INSN4', -` -// This pattern is generated automatically from cas.m4. -// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE -instruct compareAndExchange$1$7(iReg$2NoSp res, indirect mem, iReg$2 oldval, iReg$2 newval, rFlagsReg cr) %{ -ifelse($7,Acq,INDENT(predicate(needs_acquiring_load_exclusive(n));),`dnl') - match(Set res (CompareAndExchange$1 mem (Binary oldval newval))); - ifelse($7,Acq,'ins_cost(VOLATILE_REF_COST);`,'ins_cost(2 * VOLATILE_REF_COST);`) - effect(TEMP_DEF res, KILL cr); - format %{ - "cmpxchg$5`'ifelse($7,Acq,_acq,) $res = $mem, $oldval, $newval\t# ($3, weak) if $mem == $oldval then $mem <-- $newval" - %} - ins_encode %{ - __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, - Assembler::$4, /*acquire*/ ifelse($7,Acq,true,false), /*release*/ true, - /*weak*/ false, $res$$Register); - __ $6($res$$Register, $res$$Register); - %} - ins_pipe(pipe_slow); -%}')dnl -CAS_INSN4(B,I,byte,byte,b,sxtbw) -CAS_INSN4(S,I,short,halfword,s,sxthw) -CAS_INSN(I,I,int,word,w) -CAS_INSN(L,L,long,xword) -CAS_INSN(N,N,narrow oop,word,w) -CAS_INSN(P,P,ptr,xword) -dnl -CAS_INSN4(B,I,byte,byte,b,sxtbw,Acq) -CAS_INSN4(S,I,short,halfword,s,sxthw,Acq) -CAS_INSN(I,I,int,word,w,Acq) -CAS_INSN(L,L,long,xword,,Acq) -CAS_INSN(N,N,narrow oop,word,w,Acq) -CAS_INSN(P,P,ptr,xword,,Acq) -dnl -define(`CAS_INSN2', -` -// This pattern is generated automatically from cas.m4. -// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE -instruct weakCompareAndSwap$1$6(iRegINoSp res, indirect mem, iReg$2 oldval, iReg$2 newval, rFlagsReg cr) %{ -ifelse($6,Acq,INDENT(predicate(needs_acquiring_load_exclusive(n));),`dnl') - match(Set res (WeakCompareAndSwap$1 mem (Binary oldval newval))); - ifelse($6,Acq,'ins_cost(VOLATILE_REF_COST);`,'ins_cost(2 * VOLATILE_REF_COST);`) - effect(KILL cr); - format %{ - "cmpxchg$5`'ifelse($6,Acq,_acq,) $res = $mem, $oldval, $newval\t# ($3, weak) if $mem == $oldval then $mem <-- $newval" - "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" - %} - ins_encode %{ - __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, - Assembler::$4, /*acquire*/ ifelse($6,Acq,true,false), /*release*/ true, - /*weak*/ true, noreg); - __ csetw($res$$Register, Assembler::EQ); - %} - ins_pipe(pipe_slow); -%}')dnl -define(`CAS_INSN3', -` -// This pattern is generated automatically from cas.m4. -// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE -instruct weakCompareAndSwap$1$6(iRegINoSp res, indirect mem, iReg$2 oldval, iReg$2 newval, rFlagsReg cr) %{ -ifelse($1$6,PAcq,INDENT(predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));), - $1$6,NAcq,INDENT(predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);), - $1,P,INDENT(predicate(n->as_LoadStore()->barrier_data() == 0);), - $1,N,INDENT(predicate(n->as_LoadStore()->barrier_data() == 0);), - $6,Acq,INDENT(predicate(needs_acquiring_load_exclusive(n));), - `dnl') - match(Set res (WeakCompareAndSwap$1 mem (Binary oldval newval))); - ifelse($6,Acq,'ins_cost(VOLATILE_REF_COST);`,'ins_cost(2 * VOLATILE_REF_COST);`) - effect(KILL cr); - format %{ - "cmpxchg$5`'ifelse($6,Acq,_acq,) $res = $mem, $oldval, $newval\t# ($3, weak) if $mem == $oldval then $mem <-- $newval" - "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" - %} - ins_encode %{ - __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, - Assembler::$4, /*acquire*/ ifelse($6,Acq,true,false), /*release*/ true, - /*weak*/ true, noreg); - __ csetw($res$$Register, Assembler::EQ); - %} - ins_pipe(pipe_slow); -%}')dnl -CAS_INSN2(B,I,byte,byte,b) -CAS_INSN2(S,I,short,halfword,s) -CAS_INSN3(I,I,int,word,w) -CAS_INSN3(L,L,long,xword) -CAS_INSN3(N,N,narrow oop,word,w) -CAS_INSN3(P,P,ptr,xword) -CAS_INSN2(B,I,byte,byte,b,Acq) -CAS_INSN2(S,I,short,halfword,s,Acq) -CAS_INSN3(I,I,int,word,w,Acq) -CAS_INSN3(L,L,long,xword,,Acq) -CAS_INSN3(N,N,narrow oop,word,w,Acq) -CAS_INSN3(P,P,ptr,xword,,Acq) -dnl - -// END This section of the file is automatically generated. Do not edit --------------