< prev index next >

src/hotspot/cpu/aarch64/aarch64.ad

Print this page

        

*** 1,7 **** // ! // Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved. // Copyright (c) 2014, 2019, Red Hat, Inc. All rights reserved. // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. // // This code is free software; you can redistribute it and/or modify it // under the terms of the GNU General Public License version 2 only, as --- 1,7 ---- // ! // Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved. // Copyright (c) 2014, 2019, Red Hat, Inc. All rights reserved. // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. // // This code is free software; you can redistribute it and/or modify it // under the terms of the GNU General Public License version 2 only, as
*** 998,1007 **** --- 998,1012 ---- #include "asm/macroAssembler.hpp" #include "gc/shared/cardTable.hpp" #include "gc/shared/cardTableBarrierSet.hpp" #include "gc/shared/collectedHeap.hpp" #include "opto/addnode.hpp" + #if INCLUDE_SHENANDOAHGC + #include "gc/shenandoah/shenandoahBrooksPointer.hpp" + #include "gc/shenandoah/shenandoahBarrierSet.hpp" + #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp" + #endif class CallStubImpl { //-------------------------------------------------------------- //---< Used for optimization in Compile::shorten_branches >---
*** 1848,1857 **** --- 1853,1868 ---- // MemBarAcquire pair. There may be an extra If test introduced in // the CAS case, when the boolean result of the CAS is tested by the // caller. In that case an extra Region and AliasIdxBot Phi may be // introduced before the MergeMem // + // Shenandoah includes a pre barrier between the leading membar and + // the volatile object store/cas but its effect on the memory + // subgraph between the leading and trailing barriers doesn't + // require special predicates: the predicates for the default case + // work unmodified. + // // So, the upshot is that in all cases the subgraph will include a // *normal* memory subgraph betwen the leading membar and its child // membar: either a normal volatile put graph including a releasing // StoreX and terminating with a trailing volatile membar or card // mark volatile membar; or a normal CAS graph including a
*** 1982,2000 **** Node *x = NULL; StoreNode * st = NULL; LoadStoreNode *cas = NULL; MergeMemNode *mm = NULL; for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) { x = mem->fast_out(i); if (x->is_MergeMem()) { ! if (mm != NULL) { ! return NULL; ! } ! // two merge mems is one too many ! mm = x->as_MergeMem(); } else if (x->is_Store() && x->as_Store()->is_release() && x->Opcode() != Op_StoreCM) { // two releasing stores/CAS nodes is one too many if (st != NULL || cas != NULL) { return NULL; } --- 1993,2023 ---- Node *x = NULL; StoreNode * st = NULL; LoadStoreNode *cas = NULL; MergeMemNode *mm = NULL; + MergeMemNode *mm2 = NULL; for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) { x = mem->fast_out(i); if (x->is_MergeMem()) { ! if (UseShenandoahGC) { ! // three merge mems is one too many for Shenandoah ! if (mm == NULL) { ! mm = x->as_MergeMem(); ! } else if (mm2 == NULL) { ! mm2 = x->as_MergeMem(); ! } else { ! return NULL; ! } ! } else { ! // two merge mems is one too many ! if (mm != NULL) { ! return NULL; ! } ! mm = x->as_MergeMem(); ! } } else if (x->is_Store() && x->as_Store()->is_release() && x->Opcode() != Op_StoreCM) { // two releasing stores/CAS nodes is one too many if (st != NULL || cas != NULL) { return NULL; }
*** 2010,2021 **** // must have a store or a cas if (!st && !cas) { return NULL; } ! // must have a merge ! if (!mm) { return NULL; } Node *feed = NULL; if (cas) { --- 2033,2045 ---- // must have a store or a cas if (!st && !cas) { return NULL; } ! // must have a merge if we also have st ! // can only have a second merge if UseShenandoahGC ! if (st && (!mm || (!UseShenandoahGC && mm2))) { return NULL; } Node *feed = NULL; if (cas) {
*** 2034,2049 **** feed = st; } // ensure the feed node feeds the existing mergemem; for (DUIterator_Fast imax, i = feed->fast_outs(imax); i < imax; i++) { x = feed->fast_out(i); ! if (x == mm) { break; } } if (x != mm) { ! return NULL; } MemBarNode *mbar = NULL; // ensure the merge feeds to the expected type of membar for (DUIterator_Fast imax, i = mm->fast_outs(imax); i < imax; i++) { --- 2058,2084 ---- feed = st; } // ensure the feed node feeds the existing mergemem; for (DUIterator_Fast imax, i = feed->fast_outs(imax); i < imax; i++) { x = feed->fast_out(i); ! if (x == mm || (UseShenandoahGC && x == mm2)) { break; } } + if (x != mm) { ! if (!UseShenandoahGC) { ! // mm was our only chance ! return NULL; ! } else if (x != mm2) { ! // mm2 was our only other chance ! return NULL; ! } else { ! // mm2 is the merge fed by the store ! // search from there for the membar ! mm = mm2; ! } } MemBarNode *mbar = NULL; // ensure the merge feeds to the expected type of membar for (DUIterator_Fast imax, i = mm->fast_outs(imax); i < imax; i++) {
*** 4348,4357 **** --- 4383,4401 ---- Assembler::byte, /*acquire*/ false, /*release*/ true, /*weak*/ false, noreg); %} + enc_class aarch64_enc_cmpxchg_oop_shenandoah(memory mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, iRegINoSp res) %{ + MacroAssembler _masm(&cbuf); + guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); + Register tmp = $tmp$$Register; + __ mov(tmp, $oldval$$Register); // Must not clobber oldval. + ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, + /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ false, $res$$Register); + %} + // The only difference between aarch64_enc_cmpxchg and // aarch64_enc_cmpxchg_acq is that we use load-acquire in the // CompareAndSwap sequence to serve as a barrier on acquiring a // lock. enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
*** 4369,4378 **** --- 4413,4432 ---- Assembler::word, /*acquire*/ true, /*release*/ true, /*weak*/ false, noreg); %} + enc_class aarch64_enc_cmpxchg_acq_oop_shenandoah(memory mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, iRegINoSp res) %{ + MacroAssembler _masm(&cbuf); + guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); + Register tmp = $tmp$$Register; + __ mov(tmp, $oldval$$Register); // Must not clobber oldval. + ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, + /*acquire*/ true, /*release*/ true, /*weak*/ false, /*is_cae*/ false, + $res$$Register); + %} + // auxiliary used for CompareAndSwapX to set result register enc_class aarch64_enc_cset_eq(iRegINoSp res) %{ MacroAssembler _masm(&cbuf); Register res_reg = as_Register($res$$reg); __ cset(res_reg, Assembler::EQ);
*** 9289,9298 **** --- 9343,9367 ---- %} ins_pipe(ialu_reg); %} + instruct shenandoahRB(iRegPNoSp dst, iRegP src, rFlagsReg cr) %{ + match(Set dst (ShenandoahReadBarrier src)); + format %{ "shenandoah_rb $dst,$src" %} + ins_encode %{ + #if INCLUDE_SHENANDOAHGC + Register s = $src$$Register; + Register d = $dst$$Register; + __ ldr(d, Address(s, ShenandoahBrooksPointer::byte_offset())); + #else + ShouldNotReachHere(); + #endif + %} + ins_pipe(pipe_class_memory); + %} + // Convert oop pointer into compressed form instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{ predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull); match(Set dst (EncodeP src));
*** 9599,9608 **** --- 9668,9678 ---- ins_pipe(pipe_slow); %} instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ + predicate(!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR); match(Set res (CompareAndSwapP mem (Binary oldval newval))); ins_cost(2 * VOLATILE_REF_COST); effect(KILL cr);
*** 9615,9626 **** --- 9685,9714 ---- aarch64_enc_cset_eq(res)); ins_pipe(pipe_slow); %} + instruct compareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{ + + predicate(UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypePtr::NULL_PTR); + match(Set res (CompareAndSwapP mem (Binary oldval newval))); + ins_cost(2 * VOLATILE_REF_COST); + + effect(TEMP tmp, KILL cr); + + format %{ + "cmpxchg_shenandoah_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp" + %} + + ins_encode(aarch64_enc_cmpxchg_oop_shenandoah(mem, oldval, newval, tmp, res)); + + ins_pipe(pipe_slow); + %} + instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{ + predicate(!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypeNarrowOop::NULL_PTR); match(Set res (CompareAndSwapN mem (Binary oldval newval))); ins_cost(2 * VOLATILE_REF_COST); effect(KILL cr);
*** 9633,9642 **** --- 9721,9751 ---- aarch64_enc_cset_eq(res)); ins_pipe(pipe_slow); %} + instruct compareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{ + + predicate(UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypeNarrowOop::NULL_PTR); + match(Set res (CompareAndSwapN mem (Binary oldval newval))); + ins_cost(2 * VOLATILE_REF_COST); + + effect(TEMP tmp, KILL cr); + + format %{ + "cmpxchgw_shenandoah_narrow_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp" + %} + + ins_encode %{ + Register tmp = $tmp$$Register; + __ mov(tmp, $oldval$$Register); // Must not clobber oldval. + ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ false, $res$$Register); + %} + + ins_pipe(pipe_slow); + %} + // alternative CompareAndSwapX when we are eliding barriers instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{ predicate(needs_acquiring_load_exclusive(n));
*** 9675,9685 **** ins_pipe(pipe_slow); %} instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ ! predicate(needs_acquiring_load_exclusive(n)); match(Set res (CompareAndSwapP mem (Binary oldval newval))); ins_cost(VOLATILE_REF_COST); effect(KILL cr); --- 9784,9794 ---- ins_pipe(pipe_slow); %} instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ ! predicate(needs_acquiring_load_exclusive(n) && (!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR)); match(Set res (CompareAndSwapP mem (Binary oldval newval))); ins_cost(VOLATILE_REF_COST); effect(KILL cr);
*** 9692,9704 **** aarch64_enc_cset_eq(res)); ins_pipe(pipe_slow); %} instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{ ! predicate(needs_acquiring_load_exclusive(n)); match(Set res (CompareAndSwapN mem (Binary oldval newval))); ins_cost(VOLATILE_REF_COST); effect(KILL cr); --- 9801,9830 ---- aarch64_enc_cset_eq(res)); ins_pipe(pipe_slow); %} + instruct compareAndSwapPAcq_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{ + + predicate(needs_acquiring_load_exclusive(n) && UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypePtr::NULL_PTR); + match(Set res (CompareAndSwapP mem (Binary oldval newval))); + ins_cost(VOLATILE_REF_COST); + + effect(TEMP tmp, KILL cr); + + format %{ + "cmpxchg_acq_shenandoah_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp" + %} + + ins_encode(aarch64_enc_cmpxchg_acq_oop_shenandoah(mem, oldval, newval, tmp, res)); + + ins_pipe(pipe_slow); + %} + instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{ ! predicate(needs_acquiring_load_exclusive(n) && (!UseShenandoahGC || !ShenandoahCASBarrier|| n->in(3)->in(1)->bottom_type() == TypeNarrowOop::NULL_PTR)); match(Set res (CompareAndSwapN mem (Binary oldval newval))); ins_cost(VOLATILE_REF_COST); effect(KILL cr);
*** 9711,9720 **** --- 9837,9866 ---- aarch64_enc_cset_eq(res)); ins_pipe(pipe_slow); %} + instruct compareAndSwapNAcq_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{ + + predicate(needs_acquiring_load_exclusive(n) && UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypeNarrowOop::NULL_PTR); + match(Set res (CompareAndSwapN mem (Binary oldval newval))); + ins_cost(VOLATILE_REF_COST); + + effect(TEMP tmp, KILL cr); + + format %{ + "cmpxchgw_acq_shenandoah_narrow_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp" + %} + + ins_encode %{ + Register tmp = $tmp$$Register; + __ mov(tmp, $oldval$$Register); // Must not clobber oldval. + ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, /*acquire*/ true, /*release*/ true, /*weak*/ false, /*is_cae*/ false, $res$$Register); + %} + + ins_pipe(pipe_slow); + %} // --------------------------------------------------------------------- // BEGIN This section of the file is automatically generated. Do not edit --------------
*** 9791,9800 **** --- 9937,9947 ---- %} ins_pipe(pipe_slow); %} instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{ + predicate(!UseShenandoahGC || !ShenandoahCASBarrier); match(Set res (CompareAndExchangeN mem (Binary oldval newval))); ins_cost(2 * VOLATILE_REF_COST); effect(TEMP_DEF res, KILL cr); format %{ "cmpxchg $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
*** 9805,9815 **** --- 9952,9980 ---- /*weak*/ false, $res$$Register); %} ins_pipe(pipe_slow); %} + instruct compareAndExchangeN_shenandoah(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{ + predicate(UseShenandoahGC && ShenandoahCASBarrier); + match(Set res (CompareAndExchangeN mem (Binary oldval newval))); + ins_cost(3 * VOLATILE_REF_COST); + effect(TEMP_DEF res, TEMP tmp, KILL cr); + format %{ + "cmpxchg_oop_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval" + %} + ins_encode %{ + Register tmp = $tmp$$Register; + __ mov(tmp, $oldval$$Register); // Must not clobber oldval. + ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, + /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ true, $res$$Register); + %} + ins_pipe(pipe_slow); + %} + instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ + predicate(!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR); match(Set res (CompareAndExchangeP mem (Binary oldval newval))); ins_cost(2 * VOLATILE_REF_COST); effect(TEMP_DEF res, KILL cr); format %{ "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
*** 9820,9829 **** --- 9985,10011 ---- /*weak*/ false, $res$$Register); %} ins_pipe(pipe_slow); %} + instruct compareAndExchangeP_shenandoah(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{ + predicate(UseShenandoahGC && ShenandoahCASBarrier); + match(Set res (CompareAndExchangeP mem (Binary oldval newval))); + ins_cost(3 * VOLATILE_REF_COST); + effect(TEMP_DEF res, TEMP tmp, KILL cr); + format %{ + "cmpxchg_oop_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp" + %} + ins_encode %{ + Register tmp = $tmp$$Register; + __ mov(tmp, $oldval$$Register); // Must not clobber oldval. + ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, + /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ true, $res$$Register); + %} + ins_pipe(pipe_slow); + %} + instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{ match(Set res (WeakCompareAndSwapB mem (Binary oldval newval))); ins_cost(2 * VOLATILE_REF_COST); effect(KILL cr); format %{
*** 9889,9898 **** --- 10071,10081 ---- %} ins_pipe(pipe_slow); %} instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{ + predicate(!UseShenandoahGC || !ShenandoahCASBarrier); match(Set res (WeakCompareAndSwapN mem (Binary oldval newval))); ins_cost(2 * VOLATILE_REF_COST); effect(KILL cr); format %{ "cmpxchg $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
*** 9905,9915 **** --- 10088,10116 ---- __ csetw($res$$Register, Assembler::EQ); %} ins_pipe(pipe_slow); %} + instruct weakCompareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{ + predicate(UseShenandoahGC && ShenandoahCASBarrier); + match(Set res (WeakCompareAndSwapN mem (Binary oldval newval))); + ins_cost(3 * VOLATILE_REF_COST); + effect(TEMP tmp, KILL cr); + format %{ + "cmpxchg_oop_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval" + %} + ins_encode %{ + Register tmp = $tmp$$Register; + __ mov(tmp, $oldval$$Register); // Must not clobber oldval. + ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, + /*acquire*/ false, /*release*/ true, /*weak*/ true, /*is_cae*/ false, $res$$Register); + %} + ins_pipe(pipe_slow); + %} + instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ + predicate(!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR); match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); ins_cost(2 * VOLATILE_REF_COST); effect(KILL cr); format %{ "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
*** 9922,9931 **** --- 10123,10148 ---- __ csetw($res$$Register, Assembler::EQ); %} ins_pipe(pipe_slow); %} + instruct weakCompareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{ + predicate(UseShenandoahGC && ShenandoahCASBarrier); + match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); + ins_cost(3 * VOLATILE_REF_COST); + effect(TEMP tmp, KILL cr); + format %{ + "cmpxchg_oop_shenandoah $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval" + %} + ins_encode %{ + Register tmp = $tmp$$Register; + __ mov(tmp, $oldval$$Register); // Must not clobber oldval. + ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, + /*acquire*/ false, /*release*/ true, /*weak*/ true, /*is_cae*/ false, $res$$Register); + %} + ins_pipe(pipe_slow); + %} // END This section of the file is automatically generated. Do not edit -------------- // --------------------------------------------------------------------- instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{ match(Set prev (GetAndSetI mem newv));
< prev index next >