< prev index next >

src/hotspot/cpu/aarch64/aarch64.ad

Print this page


   1 //
   2 // Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
   3 // Copyright (c) 2014, 2019, Red Hat, Inc. All rights reserved.
   4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5 //
   6 // This code is free software; you can redistribute it and/or modify it
   7 // under the terms of the GNU General Public License version 2 only, as
   8 // published by the Free Software Foundation.
   9 //
  10 // This code is distributed in the hope that it will be useful, but WITHOUT
  11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13 // version 2 for more details (a copy is included in the LICENSE file that
  14 // accompanied this code).
  15 //
  16 // You should have received a copy of the GNU General Public License version
  17 // 2 along with this work; if not, write to the Free Software Foundation,
  18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19 //
  20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21 // or visit www.oracle.com if you need additional information or have any
  22 // questions.


 983 definitions %{
 984   // The default cost (of a register move instruction).
 985   int_def INSN_COST            (    100,     100);
 986   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 987   int_def CALL_COST            (    200,     2 * INSN_COST);
 988   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 989 %}
 990 
 991 
 992 //----------SOURCE BLOCK-------------------------------------------------------
 993 // This is a block of C++ code which provides values, functions, and
 994 // definitions necessary in the rest of the architecture description
 995 
 996 source_hpp %{
 997 
 998 #include "asm/macroAssembler.hpp"
 999 #include "gc/shared/cardTable.hpp"
1000 #include "gc/shared/cardTableBarrierSet.hpp"
1001 #include "gc/shared/collectedHeap.hpp"
1002 #include "opto/addnode.hpp"





1003 
1004 class CallStubImpl {
1005 
1006   //--------------------------------------------------------------
1007   //---<  Used for optimization in Compile::shorten_branches  >---
1008   //--------------------------------------------------------------
1009 
1010  public:
1011   // Size of call trampoline stub.
1012   static uint size_call_trampoline() {
1013     return 0; // no call trampolines on this platform
1014   }
1015 
1016   // number of relocations needed by a call trampoline stub
1017   static uint reloc_call_trampoline() {
1018     return 0; // no call trampolines on this platform
1019   }
1020 };
1021 
1022 class HandlerImpl {


1833   // Mem slices (from the StoreCM and other card mark queue stores).
1834   // However in this case the AliasIdxBot Mem slice does not come
1835   // direct from the card mark membar. It is merged through a series
1836   // of Phi nodes. These are needed to merge the AliasIdxBot Mem flow
1837   // from the leading membar with the Mem feed from the card mark
1838   // membar. Each Phi corresponds to one of the Ifs which may skip
1839   // around the card mark membar. So when the If implementing the NULL
1840   // value check has been elided the total number of Phis is 2
1841   // otherwise it is 3.
1842   //
1843   // The CAS graph when using G1GC also includes a pre-write subgraph
1844   // and an optional post-write subgraph. The same variations are
1845   // introduced as for CMS with conditional card marking i.e. the
1846   // StoreP/N is swapped for a CompareAndSwapP/N with a following
1847   // SCMemProj, the trailing MemBarVolatile for a MemBarCPUOrder +
1848   // MemBarAcquire pair. There may be an extra If test introduced in
1849   // the CAS case, when the boolean result of the CAS is tested by the
1850   // caller. In that case an extra Region and AliasIdxBot Phi may be
1851   // introduced before the MergeMem
1852   //






1853   // So, the upshot is that in all cases the subgraph will include a
1854   // *normal* memory subgraph betwen the leading membar and its child
1855   // membar: either a normal volatile put graph including a releasing
1856   // StoreX and terminating with a trailing volatile membar or card
1857   // mark volatile membar; or a normal CAS graph including a
1858   // CompareAndSwapX + SCMemProj pair and terminating with a card mark
1859   // volatile membar or a trailing cpu order and acquire membar
1860   // pair. If the child membar is not a (volatile) card mark membar
1861   // then it marks the end of the volatile put or CAS subgraph. If the
1862   // child is a card mark membar then the normal subgraph will form
1863   // part of a larger volatile put or CAS subgraph if and only if the
1864   // child feeds an AliasIdxBot Mem feed to a trailing barrier via a
1865   // MergeMem. That feed is either direct (for CMS) or via 2, 3 or 4
1866   // Phi nodes merging the leading barrier memory flow (for G1).
1867   //
1868   // The predicates controlling generation of instructions for store
1869   // and barrier nodes employ a few simple helper functions (described
1870   // below) which identify the presence or absence of all these
1871   // subgraph configurations and provide a means of traversing from
1872   // one node in the subgraph to another.


1967   // the returned value may be a card mark or trailing membar
1968   //
1969 
1970   MemBarNode *leading_to_normal(MemBarNode *leading)
1971   {
1972     assert((leading->Opcode() == Op_MemBarRelease ||
1973             leading->Opcode() == Op_MemBarCPUOrder),
1974            "expecting a volatile or cpuroder membar!");
1975 
1976     // check the mem flow
1977     ProjNode *mem = leading->proj_out(TypeFunc::Memory);
1978 
1979     if (!mem) {
1980       return NULL;
1981     }
1982 
1983     Node *x = NULL;
1984     StoreNode * st = NULL;
1985     LoadStoreNode *cas = NULL;
1986     MergeMemNode *mm = NULL;

1987 
1988     for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) {
1989       x = mem->fast_out(i);
1990       if (x->is_MergeMem()) {
1991         if (mm != NULL) {
1992           return NULL;
1993         }
1994         // two merge mems is one too many
1995         mm = x->as_MergeMem();











1996       } else if (x->is_Store() && x->as_Store()->is_release() && x->Opcode() != Op_StoreCM) {
1997         // two releasing stores/CAS nodes is one too many
1998         if (st != NULL || cas != NULL) {
1999           return NULL;
2000         }
2001         st = x->as_Store();
2002       } else if (is_CAS(x->Opcode())) {
2003         if (st != NULL || cas != NULL) {
2004           return NULL;
2005         }
2006         cas = x->as_LoadStore();
2007       }
2008     }
2009 
2010     // must have a store or a cas
2011     if (!st && !cas) {
2012       return NULL;
2013     }
2014 
2015     // must have a merge
2016     if (!mm) {

2017       return NULL;
2018     }
2019 
2020     Node *feed = NULL;
2021     if (cas) {
2022       // look for an SCMemProj
2023       for (DUIterator_Fast imax, i = cas->fast_outs(imax); i < imax; i++) {
2024         x = cas->fast_out(i);
2025         if (x->Opcode() == Op_SCMemProj) {
2026           feed = x;
2027           break;
2028         }
2029       }
2030       if (feed == NULL) {
2031         return NULL;
2032       }
2033     } else {
2034       feed = st;
2035     }
2036     // ensure the feed node feeds the existing mergemem;
2037     for (DUIterator_Fast imax, i = feed->fast_outs(imax); i < imax; i++) {
2038       x = feed->fast_out(i);
2039       if (x == mm) {
2040         break;
2041       }
2042     }

2043     if (x != mm) {
2044       return NULL;










2045     }
2046 
2047     MemBarNode *mbar = NULL;
2048     // ensure the merge feeds to the expected type of membar
2049     for (DUIterator_Fast imax, i = mm->fast_outs(imax); i < imax; i++) {
2050       x = mm->fast_out(i);
2051       if (x->is_MemBar()) {
2052         if (x->Opcode() == Op_MemBarCPUOrder) {
2053           // with a store any cpu order membar should precede a
2054           // trailing volatile membar. with a cas it should precede a
2055           // trailing acquire membar. in either case try to skip to
2056           // that next membar
2057           MemBarNode *y =  x->as_MemBar();
2058           y = child_membar(y);
2059           if (y != NULL) {
2060             // skip to this new membar to do the check
2061             x = y;
2062           }
2063           
2064         }


4333                /*weak*/ false, noreg);
4334   %}
4335 
4336   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
4337     MacroAssembler _masm(&cbuf);
4338     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4339     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
4340                Assembler::halfword, /*acquire*/ false, /*release*/ true,
4341                /*weak*/ false, noreg);
4342   %}
4343 
4344   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
4345     MacroAssembler _masm(&cbuf);
4346     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4347     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
4348                Assembler::byte, /*acquire*/ false, /*release*/ true,
4349                /*weak*/ false, noreg);
4350   %}
4351 
4352 









4353   // The only difference between aarch64_enc_cmpxchg and
4354   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
4355   // CompareAndSwap sequence to serve as a barrier on acquiring a
4356   // lock.
4357   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
4358     MacroAssembler _masm(&cbuf);
4359     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4360     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
4361                Assembler::xword, /*acquire*/ true, /*release*/ true,
4362                /*weak*/ false, noreg);
4363   %}
4364 
4365   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
4366     MacroAssembler _masm(&cbuf);
4367     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4368     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
4369                Assembler::word, /*acquire*/ true, /*release*/ true,
4370                /*weak*/ false, noreg);
4371   %}
4372 
4373 










4374   // auxiliary used for CompareAndSwapX to set result register
4375   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
4376     MacroAssembler _masm(&cbuf);
4377     Register res_reg = as_Register($res$$reg);
4378     __ cset(res_reg, Assembler::EQ);
4379   %}
4380 
4381   // prefetch encodings
4382 
4383   enc_class aarch64_enc_prefetchw(memory mem) %{
4384     MacroAssembler _masm(&cbuf);
4385     Register base = as_Register($mem$$base);
4386     int index = $mem$$index;
4387     int scale = $mem$$scale;
4388     int disp = $mem$$disp;
4389     if (index == -1) {
4390       __ prfm(Address(base, disp), PSTL1KEEP);
4391     } else {
4392       Register index_reg = as_Register(index);
4393       if (disp == 0) {


9274 
9275   ins_pipe(ialu_reg);
9276 %}
9277 
9278 // Convert compressed oop into int for vectors alignment masking
9279 // in case of 32bit oops (heap < 4Gb).
9280 instruct convN2I(iRegINoSp dst, iRegN src)
9281 %{
9282   predicate(Universe::narrow_oop_shift() == 0);
9283   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
9284 
9285   ins_cost(INSN_COST);
9286   format %{ "mov dst, $src\t# compressed ptr -> int" %}
9287   ins_encode %{
9288     __ movw($dst$$Register, $src$$Register);
9289   %}
9290 
9291   ins_pipe(ialu_reg);
9292 %}
9293 















9294 
9295 // Convert oop pointer into compressed form
9296 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
9297   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
9298   match(Set dst (EncodeP src));
9299   effect(KILL cr);
9300   ins_cost(INSN_COST * 3);
9301   format %{ "encode_heap_oop $dst, $src" %}
9302   ins_encode %{
9303     Register s = $src$$Register;
9304     Register d = $dst$$Register;
9305     __ encode_heap_oop(d, s);
9306   %}
9307   ins_pipe(ialu_reg);
9308 %}
9309 
9310 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
9311   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
9312   match(Set dst (EncodeP src));
9313   ins_cost(INSN_COST * 3);


9584 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
9585 
9586   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
9587   ins_cost(2 * VOLATILE_REF_COST);
9588 
9589   effect(KILL cr);
9590 
9591  format %{
9592     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
9593     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9594  %}
9595 
9596  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
9597             aarch64_enc_cset_eq(res));
9598 
9599   ins_pipe(pipe_slow);
9600 %}
9601 
9602 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9603 

9604   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9605   ins_cost(2 * VOLATILE_REF_COST);
9606 
9607   effect(KILL cr);
9608 
9609  format %{
9610     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
9611     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9612  %}
9613 
9614  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
9615             aarch64_enc_cset_eq(res));
9616 
9617   ins_pipe(pipe_slow);
9618 %}
9619 

















9620 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
9621 

9622   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9623   ins_cost(2 * VOLATILE_REF_COST);
9624 
9625   effect(KILL cr);
9626 
9627  format %{
9628     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
9629     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9630  %}
9631 
9632  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
9633             aarch64_enc_cset_eq(res));
9634 
9635   ins_pipe(pipe_slow);
9636 %}
9637 





















9638 // alternative CompareAndSwapX when we are eliding barriers
9639 
9640 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9641 
9642   predicate(needs_acquiring_load_exclusive(n));
9643   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
9644   ins_cost(VOLATILE_REF_COST);
9645 
9646   effect(KILL cr);
9647 
9648  format %{
9649     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9650     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9651  %}
9652 
9653  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
9654             aarch64_enc_cset_eq(res));
9655 
9656   ins_pipe(pipe_slow);
9657 %}


9660 
9661   predicate(needs_acquiring_load_exclusive(n));
9662   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
9663   ins_cost(VOLATILE_REF_COST);
9664 
9665   effect(KILL cr);
9666 
9667  format %{
9668     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
9669     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9670  %}
9671 
9672  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
9673             aarch64_enc_cset_eq(res));
9674 
9675   ins_pipe(pipe_slow);
9676 %}
9677 
9678 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9679 
9680   predicate(needs_acquiring_load_exclusive(n));
9681   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9682   ins_cost(VOLATILE_REF_COST);
9683 
9684   effect(KILL cr);
9685 
9686  format %{
9687     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
9688     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9689  %}
9690 
9691  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
9692             aarch64_enc_cset_eq(res));
9693 
9694   ins_pipe(pipe_slow);
9695 %}
9696 

















9697 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
9698 
9699   predicate(needs_acquiring_load_exclusive(n));
9700   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9701   ins_cost(VOLATILE_REF_COST);
9702 
9703   effect(KILL cr);
9704 
9705  format %{
9706     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
9707     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9708  %}
9709 
9710  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
9711             aarch64_enc_cset_eq(res));
9712 
9713   ins_pipe(pipe_slow);
9714 %}
9715 




















9716 
9717 // ---------------------------------------------------------------------
9718 
9719 
9720 // BEGIN This section of the file is automatically generated. Do not edit --------------
9721 
9722 // Sundry CAS operations.  Note that release is always true,
9723 // regardless of the memory ordering of the CAS.  This is because we
9724 // need the volatile case to be sequentially consistent but there is
9725 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
9726 // can't check the type of memory ordering here, so we always emit a
9727 // STLXR.
9728 
9729 // This section is generated from aarch64_ad_cas.m4
9730 
9731 
9732 
9733 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9734   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
9735   ins_cost(2 * VOLATILE_REF_COST);


9776   %}
9777   ins_pipe(pipe_slow);
9778 %}
9779 
9780 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9781   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
9782   ins_cost(2 * VOLATILE_REF_COST);
9783   effect(TEMP_DEF res, KILL cr);
9784   format %{
9785     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9786   %}
9787   ins_encode %{
9788     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9789                Assembler::xword, /*acquire*/ false, /*release*/ true,
9790                /*weak*/ false, $res$$Register);
9791   %}
9792   ins_pipe(pipe_slow);
9793 %}
9794 
9795 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{

9796   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9797   ins_cost(2 * VOLATILE_REF_COST);
9798   effect(TEMP_DEF res, KILL cr);
9799   format %{
9800     "cmpxchg $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9801   %}
9802   ins_encode %{
9803     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9804                Assembler::word, /*acquire*/ false, /*release*/ true,
9805                /*weak*/ false, $res$$Register);
9806   %}
9807   ins_pipe(pipe_slow);
9808 %}
9809 

















9810 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{

9811   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9812   ins_cost(2 * VOLATILE_REF_COST);
9813   effect(TEMP_DEF res, KILL cr);
9814   format %{
9815     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9816   %}
9817   ins_encode %{
9818     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9819                Assembler::xword, /*acquire*/ false, /*release*/ true,
9820                /*weak*/ false, $res$$Register);
9821   %}
9822   ins_pipe(pipe_slow);
9823 %}
9824 

















9825 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9826   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9827   ins_cost(2 * VOLATILE_REF_COST);
9828   effect(KILL cr);
9829   format %{
9830     "cmpxchg $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9831     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9832   %}
9833   ins_encode %{
9834     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9835                Assembler::byte, /*acquire*/ false, /*release*/ true,
9836                /*weak*/ true, noreg);
9837     __ csetw($res$$Register, Assembler::EQ);
9838   %}
9839   ins_pipe(pipe_slow);
9840 %}
9841 
9842 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9843   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9844   ins_cost(2 * VOLATILE_REF_COST);


9874 %}
9875 
9876 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9877   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9878   ins_cost(2 * VOLATILE_REF_COST);
9879   effect(KILL cr);
9880   format %{
9881     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9882     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9883   %}
9884   ins_encode %{
9885     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9886                Assembler::xword, /*acquire*/ false, /*release*/ true,
9887                /*weak*/ true, noreg);
9888     __ csetw($res$$Register, Assembler::EQ);
9889   %}
9890   ins_pipe(pipe_slow);
9891 %}
9892 
9893 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{

9894   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9895   ins_cost(2 * VOLATILE_REF_COST);
9896   effect(KILL cr);
9897   format %{
9898     "cmpxchg $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9899     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9900   %}
9901   ins_encode %{
9902     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9903                Assembler::word, /*acquire*/ false, /*release*/ true,
9904                /*weak*/ true, noreg);
9905     __ csetw($res$$Register, Assembler::EQ);
9906   %}
9907   ins_pipe(pipe_slow);
9908 %}
9909 

















9910 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{

9911   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9912   ins_cost(2 * VOLATILE_REF_COST);
9913   effect(KILL cr);
9914   format %{
9915     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9916     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9917   %}
9918   ins_encode %{
9919     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9920                Assembler::xword, /*acquire*/ false, /*release*/ true,
9921                /*weak*/ true, noreg);
9922     __ csetw($res$$Register, Assembler::EQ);
9923   %}
9924   ins_pipe(pipe_slow);
9925 %}
9926 
















9927 // END This section of the file is automatically generated. Do not edit --------------
9928 // ---------------------------------------------------------------------
9929 
9930 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
9931   match(Set prev (GetAndSetI mem newv));
9932   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
9933   ins_encode %{
9934     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9935   %}
9936   ins_pipe(pipe_serial);
9937 %}
9938 
9939 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
9940   match(Set prev (GetAndSetL mem newv));
9941   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
9942   ins_encode %{
9943     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9944   %}
9945   ins_pipe(pipe_serial);
9946 %}


   1 //
   2 // Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved.
   3 // Copyright (c) 2014, 2019, Red Hat, Inc. All rights reserved.
   4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5 //
   6 // This code is free software; you can redistribute it and/or modify it
   7 // under the terms of the GNU General Public License version 2 only, as
   8 // published by the Free Software Foundation.
   9 //
  10 // This code is distributed in the hope that it will be useful, but WITHOUT
  11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13 // version 2 for more details (a copy is included in the LICENSE file that
  14 // accompanied this code).
  15 //
  16 // You should have received a copy of the GNU General Public License version
  17 // 2 along with this work; if not, write to the Free Software Foundation,
  18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19 //
  20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21 // or visit www.oracle.com if you need additional information or have any
  22 // questions.


 983 definitions %{
 984   // The default cost (of a register move instruction).
 985   int_def INSN_COST            (    100,     100);
 986   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 987   int_def CALL_COST            (    200,     2 * INSN_COST);
 988   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 989 %}
 990 
 991 
 992 //----------SOURCE BLOCK-------------------------------------------------------
 993 // This is a block of C++ code which provides values, functions, and
 994 // definitions necessary in the rest of the architecture description
 995 
 996 source_hpp %{
 997 
 998 #include "asm/macroAssembler.hpp"
 999 #include "gc/shared/cardTable.hpp"
1000 #include "gc/shared/cardTableBarrierSet.hpp"
1001 #include "gc/shared/collectedHeap.hpp"
1002 #include "opto/addnode.hpp"
1003 #if INCLUDE_SHENANDOAHGC
1004 #include "gc/shenandoah/shenandoahBrooksPointer.hpp"
1005 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
1006 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
1007 #endif
1008 
1009 class CallStubImpl {
1010 
1011   //--------------------------------------------------------------
1012   //---<  Used for optimization in Compile::shorten_branches  >---
1013   //--------------------------------------------------------------
1014 
1015  public:
1016   // Size of call trampoline stub.
1017   static uint size_call_trampoline() {
1018     return 0; // no call trampolines on this platform
1019   }
1020 
1021   // number of relocations needed by a call trampoline stub
1022   static uint reloc_call_trampoline() {
1023     return 0; // no call trampolines on this platform
1024   }
1025 };
1026 
1027 class HandlerImpl {


1838   // Mem slices (from the StoreCM and other card mark queue stores).
1839   // However in this case the AliasIdxBot Mem slice does not come
1840   // direct from the card mark membar. It is merged through a series
1841   // of Phi nodes. These are needed to merge the AliasIdxBot Mem flow
1842   // from the leading membar with the Mem feed from the card mark
1843   // membar. Each Phi corresponds to one of the Ifs which may skip
1844   // around the card mark membar. So when the If implementing the NULL
1845   // value check has been elided the total number of Phis is 2
1846   // otherwise it is 3.
1847   //
1848   // The CAS graph when using G1GC also includes a pre-write subgraph
1849   // and an optional post-write subgraph. The same variations are
1850   // introduced as for CMS with conditional card marking i.e. the
1851   // StoreP/N is swapped for a CompareAndSwapP/N with a following
1852   // SCMemProj, the trailing MemBarVolatile for a MemBarCPUOrder +
1853   // MemBarAcquire pair. There may be an extra If test introduced in
1854   // the CAS case, when the boolean result of the CAS is tested by the
1855   // caller. In that case an extra Region and AliasIdxBot Phi may be
1856   // introduced before the MergeMem
1857   //
1858   // Shenandoah includes a pre barrier between the leading membar and
1859   // the volatile object store/cas but its effect on the memory
1860   // subgraph between the leading and trailing barriers doesn't
1861   // require special predicates: the predicates for the default case
1862   // work unmodified.
1863   //
1864   // So, the upshot is that in all cases the subgraph will include a
1865   // *normal* memory subgraph betwen the leading membar and its child
1866   // membar: either a normal volatile put graph including a releasing
1867   // StoreX and terminating with a trailing volatile membar or card
1868   // mark volatile membar; or a normal CAS graph including a
1869   // CompareAndSwapX + SCMemProj pair and terminating with a card mark
1870   // volatile membar or a trailing cpu order and acquire membar
1871   // pair. If the child membar is not a (volatile) card mark membar
1872   // then it marks the end of the volatile put or CAS subgraph. If the
1873   // child is a card mark membar then the normal subgraph will form
1874   // part of a larger volatile put or CAS subgraph if and only if the
1875   // child feeds an AliasIdxBot Mem feed to a trailing barrier via a
1876   // MergeMem. That feed is either direct (for CMS) or via 2, 3 or 4
1877   // Phi nodes merging the leading barrier memory flow (for G1).
1878   //
1879   // The predicates controlling generation of instructions for store
1880   // and barrier nodes employ a few simple helper functions (described
1881   // below) which identify the presence or absence of all these
1882   // subgraph configurations and provide a means of traversing from
1883   // one node in the subgraph to another.


1978   // the returned value may be a card mark or trailing membar
1979   //
1980 
1981   MemBarNode *leading_to_normal(MemBarNode *leading)
1982   {
1983     assert((leading->Opcode() == Op_MemBarRelease ||
1984             leading->Opcode() == Op_MemBarCPUOrder),
1985            "expecting a volatile or cpuroder membar!");
1986 
1987     // check the mem flow
1988     ProjNode *mem = leading->proj_out(TypeFunc::Memory);
1989 
1990     if (!mem) {
1991       return NULL;
1992     }
1993 
1994     Node *x = NULL;
1995     StoreNode * st = NULL;
1996     LoadStoreNode *cas = NULL;
1997     MergeMemNode *mm = NULL;
1998     MergeMemNode *mm2 = NULL;
1999 
2000     for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) {
2001       x = mem->fast_out(i);
2002       if (x->is_MergeMem()) {
2003         if (UseShenandoahGC) {
2004           // three merge mems is one too many for Shenandoah
2005           if (mm == NULL) {
2006             mm = x->as_MergeMem();
2007           } else if (mm2 == NULL) {
2008             mm2 = x->as_MergeMem();
2009           } else {
2010             return NULL;
2011           }
2012         } else {
2013           // two merge mems is one too many
2014           if (mm != NULL) {
2015             return NULL;
2016           }
2017           mm = x->as_MergeMem();
2018         }
2019       } else if (x->is_Store() && x->as_Store()->is_release() && x->Opcode() != Op_StoreCM) {
2020         // two releasing stores/CAS nodes is one too many
2021         if (st != NULL || cas != NULL) {
2022           return NULL;
2023         }
2024         st = x->as_Store();
2025       } else if (is_CAS(x->Opcode())) {
2026         if (st != NULL || cas != NULL) {
2027           return NULL;
2028         }
2029         cas = x->as_LoadStore();
2030       }
2031     }
2032 
2033     // must have a store or a cas
2034     if (!st && !cas) {
2035       return NULL;
2036     }
2037 
2038     // must have a merge if we also have st
2039     // can only have a second merge if UseShenandoahGC
2040     if (st && (!mm || (!UseShenandoahGC && mm2))) {
2041       return NULL;
2042     }
2043 
2044     Node *feed = NULL;
2045     if (cas) {
2046       // look for an SCMemProj
2047       for (DUIterator_Fast imax, i = cas->fast_outs(imax); i < imax; i++) {
2048         x = cas->fast_out(i);
2049         if (x->Opcode() == Op_SCMemProj) {
2050           feed = x;
2051           break;
2052         }
2053       }
2054       if (feed == NULL) {
2055         return NULL;
2056       }
2057     } else {
2058       feed = st;
2059     }
2060     // ensure the feed node feeds the existing mergemem;
2061     for (DUIterator_Fast imax, i = feed->fast_outs(imax); i < imax; i++) {
2062       x = feed->fast_out(i);
2063       if (x == mm || (UseShenandoahGC && x == mm2)) {
2064         break;
2065       }
2066     }
2067 
2068     if (x != mm) {
2069       if (!UseShenandoahGC) {
2070         // mm was our only chance
2071         return NULL;
2072       } else if (x != mm2) {
2073         // mm2 was our only other chance
2074         return NULL;
2075       } else {
2076         // mm2 is the merge fed by the store
2077         // search from there for the membar
2078         mm = mm2;
2079       }
2080     }
2081 
2082     MemBarNode *mbar = NULL;
2083     // ensure the merge feeds to the expected type of membar
2084     for (DUIterator_Fast imax, i = mm->fast_outs(imax); i < imax; i++) {
2085       x = mm->fast_out(i);
2086       if (x->is_MemBar()) {
2087         if (x->Opcode() == Op_MemBarCPUOrder) {
2088           // with a store any cpu order membar should precede a
2089           // trailing volatile membar. with a cas it should precede a
2090           // trailing acquire membar. in either case try to skip to
2091           // that next membar
2092           MemBarNode *y =  x->as_MemBar();
2093           y = child_membar(y);
2094           if (y != NULL) {
2095             // skip to this new membar to do the check
2096             x = y;
2097           }
2098           
2099         }


4368                /*weak*/ false, noreg);
4369   %}
4370 
4371   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
4372     MacroAssembler _masm(&cbuf);
4373     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4374     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
4375                Assembler::halfword, /*acquire*/ false, /*release*/ true,
4376                /*weak*/ false, noreg);
4377   %}
4378 
4379   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
4380     MacroAssembler _masm(&cbuf);
4381     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4382     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
4383                Assembler::byte, /*acquire*/ false, /*release*/ true,
4384                /*weak*/ false, noreg);
4385   %}
4386 
4387 
4388   enc_class aarch64_enc_cmpxchg_oop_shenandoah(memory mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, iRegINoSp res) %{
4389     MacroAssembler _masm(&cbuf);
4390     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4391     Register tmp = $tmp$$Register;
4392     __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
4393     ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
4394                                                    /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ false, $res$$Register);
4395   %}
4396 
4397   // The only difference between aarch64_enc_cmpxchg and
4398   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
4399   // CompareAndSwap sequence to serve as a barrier on acquiring a
4400   // lock.
4401   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
4402     MacroAssembler _masm(&cbuf);
4403     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4404     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
4405                Assembler::xword, /*acquire*/ true, /*release*/ true,
4406                /*weak*/ false, noreg);
4407   %}
4408 
4409   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
4410     MacroAssembler _masm(&cbuf);
4411     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4412     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
4413                Assembler::word, /*acquire*/ true, /*release*/ true,
4414                /*weak*/ false, noreg);
4415   %}
4416 
4417 
4418   enc_class aarch64_enc_cmpxchg_acq_oop_shenandoah(memory mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, iRegINoSp res) %{
4419     MacroAssembler _masm(&cbuf);
4420     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
4421     Register tmp = $tmp$$Register;
4422     __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
4423     ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
4424                                                    /*acquire*/ true, /*release*/ true, /*weak*/ false, /*is_cae*/ false,
4425                                                    $res$$Register);
4426   %}
4427 
4428   // auxiliary used for CompareAndSwapX to set result register
4429   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
4430     MacroAssembler _masm(&cbuf);
4431     Register res_reg = as_Register($res$$reg);
4432     __ cset(res_reg, Assembler::EQ);
4433   %}
4434 
4435   // prefetch encodings
4436 
4437   enc_class aarch64_enc_prefetchw(memory mem) %{
4438     MacroAssembler _masm(&cbuf);
4439     Register base = as_Register($mem$$base);
4440     int index = $mem$$index;
4441     int scale = $mem$$scale;
4442     int disp = $mem$$disp;
4443     if (index == -1) {
4444       __ prfm(Address(base, disp), PSTL1KEEP);
4445     } else {
4446       Register index_reg = as_Register(index);
4447       if (disp == 0) {


9328 
9329   ins_pipe(ialu_reg);
9330 %}
9331 
9332 // Convert compressed oop into int for vectors alignment masking
9333 // in case of 32bit oops (heap < 4Gb).
9334 instruct convN2I(iRegINoSp dst, iRegN src)
9335 %{
9336   predicate(Universe::narrow_oop_shift() == 0);
9337   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
9338 
9339   ins_cost(INSN_COST);
9340   format %{ "mov dst, $src\t# compressed ptr -> int" %}
9341   ins_encode %{
9342     __ movw($dst$$Register, $src$$Register);
9343   %}
9344 
9345   ins_pipe(ialu_reg);
9346 %}
9347 
9348 instruct shenandoahRB(iRegPNoSp dst, iRegP src, rFlagsReg cr) %{
9349   match(Set dst (ShenandoahReadBarrier src));
9350   format %{ "shenandoah_rb $dst,$src" %}
9351   ins_encode %{
9352 #if INCLUDE_SHENANDOAHGC
9353     Register s = $src$$Register;
9354     Register d = $dst$$Register;
9355     __ ldr(d, Address(s, ShenandoahBrooksPointer::byte_offset()));
9356 #else
9357     ShouldNotReachHere();
9358 #endif
9359   %}
9360   ins_pipe(pipe_class_memory);
9361 %}
9362 
9363 
9364 // Convert oop pointer into compressed form
9365 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
9366   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
9367   match(Set dst (EncodeP src));
9368   effect(KILL cr);
9369   ins_cost(INSN_COST * 3);
9370   format %{ "encode_heap_oop $dst, $src" %}
9371   ins_encode %{
9372     Register s = $src$$Register;
9373     Register d = $dst$$Register;
9374     __ encode_heap_oop(d, s);
9375   %}
9376   ins_pipe(ialu_reg);
9377 %}
9378 
9379 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
9380   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
9381   match(Set dst (EncodeP src));
9382   ins_cost(INSN_COST * 3);


9653 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
9654 
9655   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
9656   ins_cost(2 * VOLATILE_REF_COST);
9657 
9658   effect(KILL cr);
9659 
9660  format %{
9661     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
9662     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9663  %}
9664 
9665  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
9666             aarch64_enc_cset_eq(res));
9667 
9668   ins_pipe(pipe_slow);
9669 %}
9670 
9671 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9672 
9673   predicate(!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR);
9674   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9675   ins_cost(2 * VOLATILE_REF_COST);
9676 
9677   effect(KILL cr);
9678 
9679  format %{
9680     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
9681     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9682  %}
9683 
9684  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
9685             aarch64_enc_cset_eq(res));
9686 
9687   ins_pipe(pipe_slow);
9688 %}
9689 
9690 instruct compareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
9691 
9692   predicate(UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypePtr::NULL_PTR);
9693   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9694   ins_cost(2 * VOLATILE_REF_COST);
9695 
9696   effect(TEMP tmp, KILL cr);
9697 
9698   format %{
9699     "cmpxchg_shenandoah_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
9700   %}
9701 
9702   ins_encode(aarch64_enc_cmpxchg_oop_shenandoah(mem, oldval, newval, tmp, res));
9703 
9704   ins_pipe(pipe_slow);
9705 %}
9706 
9707 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
9708 
9709   predicate(!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypeNarrowOop::NULL_PTR);
9710   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9711   ins_cost(2 * VOLATILE_REF_COST);
9712 
9713   effect(KILL cr);
9714 
9715  format %{
9716     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
9717     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9718  %}
9719 
9720  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
9721             aarch64_enc_cset_eq(res));
9722 
9723   ins_pipe(pipe_slow);
9724 %}
9725 
9726 instruct compareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
9727 
9728   predicate(UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypeNarrowOop::NULL_PTR);
9729   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9730   ins_cost(2 * VOLATILE_REF_COST);
9731 
9732   effect(TEMP tmp, KILL cr);
9733 
9734   format %{
9735     "cmpxchgw_shenandoah_narrow_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
9736   %}
9737 
9738   ins_encode %{
9739     Register tmp = $tmp$$Register;
9740     __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
9741     ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ false, $res$$Register);
9742   %}
9743 
9744   ins_pipe(pipe_slow);
9745 %}
9746 
9747 // alternative CompareAndSwapX when we are eliding barriers
9748 
9749 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9750 
9751   predicate(needs_acquiring_load_exclusive(n));
9752   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
9753   ins_cost(VOLATILE_REF_COST);
9754 
9755   effect(KILL cr);
9756 
9757  format %{
9758     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9759     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9760  %}
9761 
9762  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
9763             aarch64_enc_cset_eq(res));
9764 
9765   ins_pipe(pipe_slow);
9766 %}


9769 
9770   predicate(needs_acquiring_load_exclusive(n));
9771   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
9772   ins_cost(VOLATILE_REF_COST);
9773 
9774   effect(KILL cr);
9775 
9776  format %{
9777     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
9778     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9779  %}
9780 
9781  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
9782             aarch64_enc_cset_eq(res));
9783 
9784   ins_pipe(pipe_slow);
9785 %}
9786 
9787 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9788 
9789   predicate(needs_acquiring_load_exclusive(n) && (!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR));
9790   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9791   ins_cost(VOLATILE_REF_COST);
9792 
9793   effect(KILL cr);
9794 
9795  format %{
9796     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
9797     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9798  %}
9799 
9800  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
9801             aarch64_enc_cset_eq(res));
9802 
9803   ins_pipe(pipe_slow);
9804 %}
9805 
9806 instruct compareAndSwapPAcq_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
9807 
9808   predicate(needs_acquiring_load_exclusive(n) && UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypePtr::NULL_PTR);
9809   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9810   ins_cost(VOLATILE_REF_COST);
9811 
9812   effect(TEMP tmp, KILL cr);
9813 
9814   format %{
9815     "cmpxchg_acq_shenandoah_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
9816   %}
9817 
9818   ins_encode(aarch64_enc_cmpxchg_acq_oop_shenandoah(mem, oldval, newval, tmp, res));
9819 
9820   ins_pipe(pipe_slow);
9821 %}
9822 
9823 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
9824 
9825   predicate(needs_acquiring_load_exclusive(n) && (!UseShenandoahGC || !ShenandoahCASBarrier|| n->in(3)->in(1)->bottom_type() == TypeNarrowOop::NULL_PTR));
9826   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9827   ins_cost(VOLATILE_REF_COST);
9828 
9829   effect(KILL cr);
9830 
9831  format %{
9832     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
9833     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9834  %}
9835 
9836  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
9837             aarch64_enc_cset_eq(res));
9838 
9839   ins_pipe(pipe_slow);
9840 %}
9841 
9842 instruct compareAndSwapNAcq_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
9843 
9844   predicate(needs_acquiring_load_exclusive(n) && UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypeNarrowOop::NULL_PTR);
9845   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9846   ins_cost(VOLATILE_REF_COST);
9847 
9848   effect(TEMP tmp, KILL cr);
9849 
9850  format %{
9851     "cmpxchgw_acq_shenandoah_narrow_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
9852  %}
9853 
9854   ins_encode %{
9855     Register tmp = $tmp$$Register;
9856     __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
9857     ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, /*acquire*/ true, /*release*/ true, /*weak*/ false, /*is_cae*/ false, $res$$Register);
9858   %}
9859 
9860   ins_pipe(pipe_slow);
9861 %}
9862 
9863 // ---------------------------------------------------------------------
9864 
9865 
9866 // BEGIN This section of the file is automatically generated. Do not edit --------------
9867 
9868 // Sundry CAS operations.  Note that release is always true,
9869 // regardless of the memory ordering of the CAS.  This is because we
9870 // need the volatile case to be sequentially consistent but there is
9871 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
9872 // can't check the type of memory ordering here, so we always emit a
9873 // STLXR.
9874 
9875 // This section is generated from aarch64_ad_cas.m4
9876 
9877 
9878 
9879 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9880   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
9881   ins_cost(2 * VOLATILE_REF_COST);


9922   %}
9923   ins_pipe(pipe_slow);
9924 %}
9925 
9926 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9927   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
9928   ins_cost(2 * VOLATILE_REF_COST);
9929   effect(TEMP_DEF res, KILL cr);
9930   format %{
9931     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9932   %}
9933   ins_encode %{
9934     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9935                Assembler::xword, /*acquire*/ false, /*release*/ true,
9936                /*weak*/ false, $res$$Register);
9937   %}
9938   ins_pipe(pipe_slow);
9939 %}
9940 
9941 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9942   predicate(!UseShenandoahGC || !ShenandoahCASBarrier);
9943   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9944   ins_cost(2 * VOLATILE_REF_COST);
9945   effect(TEMP_DEF res, KILL cr);
9946   format %{
9947     "cmpxchg $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9948   %}
9949   ins_encode %{
9950     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9951                Assembler::word, /*acquire*/ false, /*release*/ true,
9952                /*weak*/ false, $res$$Register);
9953   %}
9954   ins_pipe(pipe_slow);
9955 %}
9956 
9957 instruct compareAndExchangeN_shenandoah(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
9958   predicate(UseShenandoahGC && ShenandoahCASBarrier);
9959   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9960   ins_cost(3 * VOLATILE_REF_COST);
9961   effect(TEMP_DEF res, TEMP tmp, KILL cr);
9962   format %{
9963     "cmpxchg_oop_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9964   %}
9965   ins_encode %{
9966     Register tmp = $tmp$$Register;
9967     __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
9968     ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
9969                    /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ true, $res$$Register);
9970   %}
9971   ins_pipe(pipe_slow);
9972 %}
9973 
9974 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9975   predicate(!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR);
9976   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9977   ins_cost(2 * VOLATILE_REF_COST);
9978   effect(TEMP_DEF res, KILL cr);
9979   format %{
9980     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9981   %}
9982   ins_encode %{
9983     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9984                Assembler::xword, /*acquire*/ false, /*release*/ true,
9985                /*weak*/ false, $res$$Register);
9986   %}
9987   ins_pipe(pipe_slow);
9988 %}
9989 
9990 instruct compareAndExchangeP_shenandoah(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
9991   predicate(UseShenandoahGC && ShenandoahCASBarrier);
9992   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9993   ins_cost(3 * VOLATILE_REF_COST);
9994   effect(TEMP_DEF res, TEMP tmp, KILL cr);
9995   format %{
9996     "cmpxchg_oop_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
9997   %}
9998   ins_encode %{
9999     Register tmp = $tmp$$Register;
10000     __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
10001     ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
10002                    /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ true, $res$$Register);
10003   %}
10004   ins_pipe(pipe_slow);
10005 %}
10006 
10007 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
10008   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
10009   ins_cost(2 * VOLATILE_REF_COST);
10010   effect(KILL cr);
10011   format %{
10012     "cmpxchg $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
10013     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
10014   %}
10015   ins_encode %{
10016     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
10017                Assembler::byte, /*acquire*/ false, /*release*/ true,
10018                /*weak*/ true, noreg);
10019     __ csetw($res$$Register, Assembler::EQ);
10020   %}
10021   ins_pipe(pipe_slow);
10022 %}
10023 
10024 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
10025   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
10026   ins_cost(2 * VOLATILE_REF_COST);


10056 %}
10057 
10058 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
10059   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
10060   ins_cost(2 * VOLATILE_REF_COST);
10061   effect(KILL cr);
10062   format %{
10063     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
10064     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
10065   %}
10066   ins_encode %{
10067     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
10068                Assembler::xword, /*acquire*/ false, /*release*/ true,
10069                /*weak*/ true, noreg);
10070     __ csetw($res$$Register, Assembler::EQ);
10071   %}
10072   ins_pipe(pipe_slow);
10073 %}
10074 
10075 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
10076   predicate(!UseShenandoahGC || !ShenandoahCASBarrier);
10077   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
10078   ins_cost(2 * VOLATILE_REF_COST);
10079   effect(KILL cr);
10080   format %{
10081     "cmpxchg $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
10082     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
10083   %}
10084   ins_encode %{
10085     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
10086                Assembler::word, /*acquire*/ false, /*release*/ true,
10087                /*weak*/ true, noreg);
10088     __ csetw($res$$Register, Assembler::EQ);
10089   %}
10090   ins_pipe(pipe_slow);
10091 %}
10092 
10093 instruct weakCompareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
10094   predicate(UseShenandoahGC && ShenandoahCASBarrier);
10095   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
10096   ins_cost(3 * VOLATILE_REF_COST);
10097   effect(TEMP tmp, KILL cr);
10098   format %{
10099     "cmpxchg_oop_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
10100   %}
10101   ins_encode %{
10102     Register tmp = $tmp$$Register;
10103     __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
10104     ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
10105                    /*acquire*/ false, /*release*/ true, /*weak*/ true, /*is_cae*/ false, $res$$Register);
10106   %}
10107   ins_pipe(pipe_slow);
10108 %}
10109 
10110 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
10111   predicate(!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR);
10112   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
10113   ins_cost(2 * VOLATILE_REF_COST);
10114   effect(KILL cr);
10115   format %{
10116     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
10117     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
10118   %}
10119   ins_encode %{
10120     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
10121                Assembler::xword, /*acquire*/ false, /*release*/ true,
10122                /*weak*/ true, noreg);
10123     __ csetw($res$$Register, Assembler::EQ);
10124   %}
10125   ins_pipe(pipe_slow);
10126 %}
10127 
10128 instruct weakCompareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
10129   predicate(UseShenandoahGC && ShenandoahCASBarrier);
10130   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
10131   ins_cost(3 * VOLATILE_REF_COST);
10132   effect(TEMP tmp, KILL cr);
10133   format %{
10134     "cmpxchg_oop_shenandoah $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
10135   %}
10136   ins_encode %{
10137     Register tmp = $tmp$$Register;
10138     __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
10139     ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
10140                    /*acquire*/ false, /*release*/ true, /*weak*/ true, /*is_cae*/ false, $res$$Register);
10141   %}
10142   ins_pipe(pipe_slow);
10143 %}
10144 // END This section of the file is automatically generated. Do not edit --------------
10145 // ---------------------------------------------------------------------
10146 
10147 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
10148   match(Set prev (GetAndSetI mem newv));
10149   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
10150   ins_encode %{
10151     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
10152   %}
10153   ins_pipe(pipe_serial);
10154 %}
10155 
10156 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
10157   match(Set prev (GetAndSetL mem newv));
10158   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
10159   ins_encode %{
10160     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
10161   %}
10162   ins_pipe(pipe_serial);
10163 %}


< prev index next >