910 // branches as most expensive. the first two have a low as well as a
911 // normal cost. huge cost appears to be a way of saying don't do
912 // something
913
914 definitions %{
915 // The default cost (of a register move instruction).
916 int_def INSN_COST ( 100, 100);
917 int_def BRANCH_COST ( 200, 2 * INSN_COST);
918 int_def CALL_COST ( 200, 2 * INSN_COST);
919 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
920 %}
921
922
923 //----------SOURCE BLOCK-------------------------------------------------------
924 // This is a block of C++ code which provides values, functions, and
925 // definitions necessary in the rest of the architecture description
926
927 source_hpp %{
928
929 #include "opto/addnode.hpp"
930
931 class CallStubImpl {
932
933 //--------------------------------------------------------------
934 //---< Used for optimization in Compile::shorten_branches >---
935 //--------------------------------------------------------------
936
937 public:
938 // Size of call trampoline stub.
939 static uint size_call_trampoline() {
940 return 0; // no call trampolines on this platform
941 }
942
943 // number of relocations needed by a call trampoline stub
944 static uint reloc_call_trampoline() {
945 return 0; // no call trampolines on this platform
946 }
947 };
948
949 class HandlerImpl {
2781 }
2782 }
2783 __ cmpw(rscratch1, zr);
2784 %}
2785
2786 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
2787 MacroAssembler _masm(&cbuf);
2788 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2789 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2790 Assembler::xword, /*acquire*/ false, /*release*/ true);
2791 %}
2792
2793 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
2794 MacroAssembler _masm(&cbuf);
2795 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2796 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2797 Assembler::word, /*acquire*/ false, /*release*/ true);
2798 %}
2799
2800
2801 // The only difference between aarch64_enc_cmpxchg and
2802 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
2803 // CompareAndSwap sequence to serve as a barrier on acquiring a
2804 // lock.
2805 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
2806 MacroAssembler _masm(&cbuf);
2807 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2808 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2809 Assembler::xword, /*acquire*/ true, /*release*/ true);
2810 %}
2811
2812 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
2813 MacroAssembler _masm(&cbuf);
2814 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2815 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2816 Assembler::word, /*acquire*/ true, /*release*/ true);
2817 %}
2818
2819 // auxiliary used for CompareAndSwapX to set result register
2820 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
2821 MacroAssembler _masm(&cbuf);
2822 Register res_reg = as_Register($res$$reg);
2823 __ cset(res_reg, Assembler::EQ);
2824 %}
2825
2826 // prefetch encodings
2827
2828 enc_class aarch64_enc_prefetchr(memory mem) %{
2829 MacroAssembler _masm(&cbuf);
2830 Register base = as_Register($mem$$base);
2831 int index = $mem$$index;
2832 int scale = $mem$$scale;
2833 int disp = $mem$$disp;
2834 if (index == -1) {
2835 __ prfm(Address(base, disp), PLDL1KEEP);
2836 } else {
2837 Register index_reg = as_Register(index);
2838 if (disp == 0) {
4197 op_cost(0);
4198 format %{ %}
4199 interface(CONST_INTER);
4200 %}
4201
4202 // Polling Page Pointer Immediate
4203 operand immPollPage()
4204 %{
4205 predicate((address)n->get_ptr() == os::get_polling_page());
4206 match(ConP);
4207
4208 op_cost(0);
4209 format %{ %}
4210 interface(CONST_INTER);
4211 %}
4212
4213 // Card Table Byte Map Base
4214 operand immByteMapBase()
4215 %{
4216 // Get base of card map
4217 predicate((jbyte*)n->get_ptr() ==
4218 ((CardTableModRefBS*)(Universe::heap()->barrier_set()))->byte_map_base);
4219 match(ConP);
4220
4221 op_cost(0);
4222 format %{ %}
4223 interface(CONST_INTER);
4224 %}
4225
4226 // Pointer Immediate Minus One
4227 // this is used when we want to write the current PC to the thread anchor
4228 operand immP_M1()
4229 %{
4230 predicate(n->get_ptr() == -1);
4231 match(ConP);
4232
4233 op_cost(0);
4234 format %{ %}
4235 interface(CONST_INTER);
4236 %}
4237
4238 // Pointer Immediate Minus Two
6922 match(Set mem (StoreL mem zero));
6923 predicate(!needs_releasing_store(n));
6924
6925 ins_cost(INSN_COST);
6926 format %{ "str zr, $mem\t# int" %}
6927
6928 ins_encode(aarch64_enc_str0(mem));
6929
6930 ins_pipe(istore_mem);
6931 %}
6932
6933 // Store Pointer
6934 instruct storeP(iRegP src, memory mem)
6935 %{
6936 match(Set mem (StoreP mem src));
6937 predicate(!needs_releasing_store(n));
6938
6939 ins_cost(INSN_COST);
6940 format %{ "str $src, $mem\t# ptr" %}
6941
6942 ins_encode(aarch64_enc_str(src, mem));
6943
6944 ins_pipe(istore_reg_mem);
6945 %}
6946
6947 // Store Pointer
6948 instruct storeimmP0(immP0 zero, memory mem)
6949 %{
6950 match(Set mem (StoreP mem zero));
6951 predicate(!needs_releasing_store(n));
6952
6953 ins_cost(INSN_COST);
6954 format %{ "str zr, $mem\t# ptr" %}
6955
6956 ins_encode(aarch64_enc_str0(mem));
6957
6958 ins_pipe(istore_mem);
6959 %}
6960
6961 // Store Compressed Pointer
6962 instruct storeN(iRegN src, memory mem)
8046 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8047
8048 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8049 ins_cost(2 * VOLATILE_REF_COST);
8050
8051 effect(KILL cr);
8052
8053 format %{
8054 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8055 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8056 %}
8057
8058 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8059 aarch64_enc_cset_eq(res));
8060
8061 ins_pipe(pipe_slow);
8062 %}
8063
8064 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8065
8066 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8067 ins_cost(2 * VOLATILE_REF_COST);
8068
8069 effect(KILL cr);
8070
8071 format %{
8072 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8073 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8074 %}
8075
8076 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8077 aarch64_enc_cset_eq(res));
8078
8079 ins_pipe(pipe_slow);
8080 %}
8081
8082 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8083
8084 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8085 ins_cost(2 * VOLATILE_REF_COST);
8086
8087 effect(KILL cr);
8088
8089 format %{
8090 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8091 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8092 %}
8093
8094 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8095 aarch64_enc_cset_eq(res));
8096
8097 ins_pipe(pipe_slow);
8098 %}
8099
8100
8101 // alternative CompareAndSwapX when we are eliding barriers
8102
8103 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8104
8105 predicate(needs_acquiring_load_exclusive(n));
8106 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8107 ins_cost(VOLATILE_REF_COST);
8108
8109 effect(KILL cr);
8110
8111 format %{
8112 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8113 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8114 %}
8115
8116 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8117 aarch64_enc_cset_eq(res));
8118
8119 ins_pipe(pipe_slow);
8123
8124 predicate(needs_acquiring_load_exclusive(n));
8125 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8126 ins_cost(VOLATILE_REF_COST);
8127
8128 effect(KILL cr);
8129
8130 format %{
8131 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8132 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8133 %}
8134
8135 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8136 aarch64_enc_cset_eq(res));
8137
8138 ins_pipe(pipe_slow);
8139 %}
8140
8141 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8142
8143 predicate(needs_acquiring_load_exclusive(n));
8144 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8145 ins_cost(VOLATILE_REF_COST);
8146
8147 effect(KILL cr);
8148
8149 format %{
8150 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8151 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8152 %}
8153
8154 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8155 aarch64_enc_cset_eq(res));
8156
8157 ins_pipe(pipe_slow);
8158 %}
8159
8160 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8161
8162 predicate(needs_acquiring_load_exclusive(n));
8163 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8164 ins_cost(VOLATILE_REF_COST);
8165
8166 effect(KILL cr);
8167
8168 format %{
8169 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8170 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8171 %}
8172
8173 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8174 aarch64_enc_cset_eq(res));
8175
8176 ins_pipe(pipe_slow);
8177 %}
8178
8179
8180 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
8181 match(Set prev (GetAndSetI mem newv));
8182 ins_cost(2 * VOLATILE_REF_COST);
8183 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8184 ins_encode %{
8185 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8186 %}
8187 ins_pipe(pipe_serial);
8188 %}
8189
8190 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
8191 match(Set prev (GetAndSetL mem newv));
8192 ins_cost(2 * VOLATILE_REF_COST);
8193 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
8194 ins_encode %{
8195 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8196 %}
8197 ins_pipe(pipe_serial);
8198 %}
|
910 // branches as most expensive. the first two have a low as well as a
911 // normal cost. huge cost appears to be a way of saying don't do
912 // something
913
914 definitions %{
915 // The default cost (of a register move instruction).
916 int_def INSN_COST ( 100, 100);
917 int_def BRANCH_COST ( 200, 2 * INSN_COST);
918 int_def CALL_COST ( 200, 2 * INSN_COST);
919 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
920 %}
921
922
923 //----------SOURCE BLOCK-------------------------------------------------------
924 // This is a block of C++ code which provides values, functions, and
925 // definitions necessary in the rest of the architecture description
926
927 source_hpp %{
928
929 #include "opto/addnode.hpp"
930 #if INCLUDE_ALL_GCS
931 #include "shenandoahBarrierSetAssembler_aarch64.hpp"
932 #endif
933
934 class CallStubImpl {
935
936 //--------------------------------------------------------------
937 //---< Used for optimization in Compile::shorten_branches >---
938 //--------------------------------------------------------------
939
940 public:
941 // Size of call trampoline stub.
942 static uint size_call_trampoline() {
943 return 0; // no call trampolines on this platform
944 }
945
946 // number of relocations needed by a call trampoline stub
947 static uint reloc_call_trampoline() {
948 return 0; // no call trampolines on this platform
949 }
950 };
951
952 class HandlerImpl {
2784 }
2785 }
2786 __ cmpw(rscratch1, zr);
2787 %}
2788
2789 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
2790 MacroAssembler _masm(&cbuf);
2791 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2792 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2793 Assembler::xword, /*acquire*/ false, /*release*/ true);
2794 %}
2795
2796 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
2797 MacroAssembler _masm(&cbuf);
2798 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2799 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2800 Assembler::word, /*acquire*/ false, /*release*/ true);
2801 %}
2802
2803
2804 enc_class aarch64_enc_cmpxchg_oop_shenandoah(memory mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, iRegINoSp res) %{
2805 MacroAssembler _masm(&cbuf);
2806 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2807 Register tmp = $tmp$$Register;
2808 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
2809 ShenandoahBarrierSetAssembler::bsasm()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
2810 /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ false, $res$$Register);
2811 %}
2812
2813 // The only difference between aarch64_enc_cmpxchg and
2814 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
2815 // CompareAndSwap sequence to serve as a barrier on acquiring a
2816 // lock.
2817 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
2818 MacroAssembler _masm(&cbuf);
2819 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2820 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2821 Assembler::xword, /*acquire*/ true, /*release*/ true);
2822 %}
2823
2824 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
2825 MacroAssembler _masm(&cbuf);
2826 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2827 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2828 Assembler::word, /*acquire*/ true, /*release*/ true);
2829 %}
2830
2831 enc_class aarch64_enc_cmpxchg_acq_oop_shenandoah(memory mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, iRegINoSp res) %{
2832 MacroAssembler _masm(&cbuf);
2833 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2834 Register tmp = $tmp$$Register;
2835 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
2836 ShenandoahBarrierSetAssembler::bsasm()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
2837 /*acquire*/ true, /*release*/ true, /*weak*/ false, /*is_cae*/ false,
2838 $res$$Register);
2839 %}
2840
2841 // auxiliary used for CompareAndSwapX to set result register
2842 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
2843 MacroAssembler _masm(&cbuf);
2844 Register res_reg = as_Register($res$$reg);
2845 __ cset(res_reg, Assembler::EQ);
2846 %}
2847
2848 // prefetch encodings
2849
2850 enc_class aarch64_enc_prefetchr(memory mem) %{
2851 MacroAssembler _masm(&cbuf);
2852 Register base = as_Register($mem$$base);
2853 int index = $mem$$index;
2854 int scale = $mem$$scale;
2855 int disp = $mem$$disp;
2856 if (index == -1) {
2857 __ prfm(Address(base, disp), PLDL1KEEP);
2858 } else {
2859 Register index_reg = as_Register(index);
2860 if (disp == 0) {
4219 op_cost(0);
4220 format %{ %}
4221 interface(CONST_INTER);
4222 %}
4223
4224 // Polling Page Pointer Immediate
4225 operand immPollPage()
4226 %{
4227 predicate((address)n->get_ptr() == os::get_polling_page());
4228 match(ConP);
4229
4230 op_cost(0);
4231 format %{ %}
4232 interface(CONST_INTER);
4233 %}
4234
4235 // Card Table Byte Map Base
4236 operand immByteMapBase()
4237 %{
4238 // Get base of card map
4239 predicate(!UseShenandoahGC && // TODO: Should really check for BS::is_a, see JDK-8193193
4240 (jbyte*)n->get_ptr() == ((CardTableModRefBS*)(Universe::heap()->barrier_set()))->byte_map_base);
4241 match(ConP);
4242
4243 op_cost(0);
4244 format %{ %}
4245 interface(CONST_INTER);
4246 %}
4247
4248 // Pointer Immediate Minus One
4249 // this is used when we want to write the current PC to the thread anchor
4250 operand immP_M1()
4251 %{
4252 predicate(n->get_ptr() == -1);
4253 match(ConP);
4254
4255 op_cost(0);
4256 format %{ %}
4257 interface(CONST_INTER);
4258 %}
4259
4260 // Pointer Immediate Minus Two
6944 match(Set mem (StoreL mem zero));
6945 predicate(!needs_releasing_store(n));
6946
6947 ins_cost(INSN_COST);
6948 format %{ "str zr, $mem\t# int" %}
6949
6950 ins_encode(aarch64_enc_str0(mem));
6951
6952 ins_pipe(istore_mem);
6953 %}
6954
6955 // Store Pointer
6956 instruct storeP(iRegP src, memory mem)
6957 %{
6958 match(Set mem (StoreP mem src));
6959 predicate(!needs_releasing_store(n));
6960
6961 ins_cost(INSN_COST);
6962 format %{ "str $src, $mem\t# ptr" %}
6963
6964 ins_encode %{
6965 int opcode = $mem->opcode();
6966 Register base = as_Register($mem$$base);
6967 int index = $mem$$index;
6968 int size = $mem$$scale;
6969 int disp = $mem$$disp;
6970 Register reg = as_Register($src$$reg);
6971
6972 // we sometimes get asked to store the stack pointer into the
6973 // current thread -- we cannot do that directly on AArch64
6974 if (reg == r31_sp) {
6975 MacroAssembler _masm(&cbuf);
6976 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
6977 __ mov(rscratch2, sp);
6978 reg = rscratch2;
6979 }
6980 Address::extend scale;
6981
6982 // Hooboy, this is fugly. We need a way to communicate to the
6983 // encoder that the index needs to be sign extended, so we have to
6984 // enumerate all the cases.
6985 switch (opcode) {
6986 case INDINDEXSCALEDOFFSETI2L:
6987 case INDINDEXSCALEDI2L:
6988 case INDINDEXSCALEDOFFSETI2LN:
6989 case INDINDEXSCALEDI2LN:
6990 case INDINDEXOFFSETI2L:
6991 case INDINDEXOFFSETI2LN:
6992 scale = Address::sxtw(size);
6993 break;
6994 default:
6995 scale = Address::lsl(size);
6996 }
6997 Address adr;
6998 if (index == -1) {
6999 adr = Address(base, disp);
7000 } else {
7001 if (disp == 0) {
7002 adr = Address(base, as_Register(index), scale);
7003 } else {
7004 __ lea(rscratch1, Address(base, disp));
7005 adr = Address(rscratch1, as_Register(index), scale);
7006 }
7007 }
7008
7009 __ str(reg, adr);
7010 %}
7011
7012 ins_pipe(istore_reg_mem);
7013 %}
7014
7015 // Store Pointer
7016 instruct storeimmP0(immP0 zero, memory mem)
7017 %{
7018 match(Set mem (StoreP mem zero));
7019 predicate(!needs_releasing_store(n));
7020
7021 ins_cost(INSN_COST);
7022 format %{ "str zr, $mem\t# ptr" %}
7023
7024 ins_encode(aarch64_enc_str0(mem));
7025
7026 ins_pipe(istore_mem);
7027 %}
7028
7029 // Store Compressed Pointer
7030 instruct storeN(iRegN src, memory mem)
8114 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8115
8116 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8117 ins_cost(2 * VOLATILE_REF_COST);
8118
8119 effect(KILL cr);
8120
8121 format %{
8122 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8123 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8124 %}
8125
8126 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8127 aarch64_enc_cset_eq(res));
8128
8129 ins_pipe(pipe_slow);
8130 %}
8131
8132 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8133
8134 predicate(!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR);
8135 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8136 ins_cost(2 * VOLATILE_REF_COST);
8137
8138 effect(KILL cr);
8139
8140 format %{
8141 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8142 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8143 %}
8144
8145 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8146 aarch64_enc_cset_eq(res));
8147
8148 ins_pipe(pipe_slow);
8149 %}
8150
8151 instruct compareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
8152
8153 predicate(UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypePtr::NULL_PTR);
8154 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8155 ins_cost(2 * VOLATILE_REF_COST);
8156
8157 effect(TEMP tmp, KILL cr);
8158
8159 format %{
8160 "cmpxchg_shenandoah_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
8161 %}
8162
8163 ins_encode(aarch64_enc_cmpxchg_oop_shenandoah(mem, oldval, newval, tmp, res));
8164
8165 ins_pipe(pipe_slow);
8166 %}
8167
8168 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8169
8170 predicate(!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypeNarrowOop::NULL_PTR);
8171 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8172 ins_cost(2 * VOLATILE_REF_COST);
8173
8174 effect(KILL cr);
8175
8176 format %{
8177 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8178 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8179 %}
8180
8181 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8182 aarch64_enc_cset_eq(res));
8183
8184 ins_pipe(pipe_slow);
8185 %}
8186
8187 instruct compareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
8188
8189 predicate(UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypeNarrowOop::NULL_PTR);
8190 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8191 ins_cost(2 * VOLATILE_REF_COST);
8192
8193 effect(TEMP tmp, KILL cr);
8194
8195 format %{
8196 "cmpxchgw_shenandoah_narrow_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
8197 %}
8198
8199 ins_encode %{
8200 Register tmp = $tmp$$Register;
8201 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
8202 ShenandoahBarrierSetAssembler::bsasm()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ false, $res$$Register);
8203 %}
8204
8205 ins_pipe(pipe_slow);
8206 %}
8207
8208 // alternative CompareAndSwapX when we are eliding barriers
8209
8210 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8211
8212 predicate(needs_acquiring_load_exclusive(n));
8213 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8214 ins_cost(VOLATILE_REF_COST);
8215
8216 effect(KILL cr);
8217
8218 format %{
8219 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8220 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8221 %}
8222
8223 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8224 aarch64_enc_cset_eq(res));
8225
8226 ins_pipe(pipe_slow);
8230
8231 predicate(needs_acquiring_load_exclusive(n));
8232 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8233 ins_cost(VOLATILE_REF_COST);
8234
8235 effect(KILL cr);
8236
8237 format %{
8238 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8239 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8240 %}
8241
8242 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8243 aarch64_enc_cset_eq(res));
8244
8245 ins_pipe(pipe_slow);
8246 %}
8247
8248 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8249
8250 predicate(needs_acquiring_load_exclusive(n) && (!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR));
8251 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8252 ins_cost(VOLATILE_REF_COST);
8253
8254 effect(KILL cr);
8255
8256 format %{
8257 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8258 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8259 %}
8260
8261 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8262 aarch64_enc_cset_eq(res));
8263
8264 ins_pipe(pipe_slow);
8265 %}
8266
8267 instruct compareAndSwapPAcq_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
8268
8269 predicate(needs_acquiring_load_exclusive(n) && UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypePtr::NULL_PTR);
8270 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8271 ins_cost(VOLATILE_REF_COST);
8272
8273 effect(TEMP tmp, KILL cr);
8274
8275 format %{
8276 "cmpxchg_acq_shenandoah_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
8277 %}
8278
8279 ins_encode(aarch64_enc_cmpxchg_acq_oop_shenandoah(mem, oldval, newval, tmp, res));
8280
8281 ins_pipe(pipe_slow);
8282 %}
8283
8284 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8285
8286 predicate(needs_acquiring_load_exclusive(n) && (!UseShenandoahGC || !ShenandoahCASBarrier|| n->in(3)->in(1)->bottom_type() == TypeNarrowOop::NULL_PTR));
8287 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8288 ins_cost(VOLATILE_REF_COST);
8289
8290 effect(KILL cr);
8291
8292 format %{
8293 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8294 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8295 %}
8296
8297 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8298 aarch64_enc_cset_eq(res));
8299
8300 ins_pipe(pipe_slow);
8301 %}
8302
8303 instruct compareAndSwapNAcq_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
8304
8305 predicate(needs_acquiring_load_exclusive(n) && UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypeNarrowOop::NULL_PTR);
8306 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8307 ins_cost(VOLATILE_REF_COST);
8308
8309 effect(TEMP tmp, KILL cr);
8310
8311 format %{
8312 "cmpxchgw_acq_shenandoah_narrow_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
8313 %}
8314
8315 ins_encode %{
8316 Register tmp = $tmp$$Register;
8317 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
8318 ShenandoahBarrierSetAssembler::bsasm()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, /*acquire*/ true, /*release*/ true, /*weak*/ false, /*is_cae*/ false, $res$$Register);
8319 %}
8320
8321 ins_pipe(pipe_slow);
8322 %}
8323
8324 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
8325 match(Set prev (GetAndSetI mem newv));
8326 ins_cost(2 * VOLATILE_REF_COST);
8327 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8328 ins_encode %{
8329 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8330 %}
8331 ins_pipe(pipe_serial);
8332 %}
8333
8334 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
8335 match(Set prev (GetAndSetL mem newv));
8336 ins_cost(2 * VOLATILE_REF_COST);
8337 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
8338 ins_encode %{
8339 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8340 %}
8341 ins_pipe(pipe_serial);
8342 %}
|