< prev index next >

src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp

Print this page

  24  */
  25 
  26 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
  27 #include "gc/shenandoah/mode/shenandoahMode.hpp"
  28 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  29 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
  30 #include "gc/shenandoah/shenandoahForwarding.hpp"
  31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
  33 #include "gc/shenandoah/shenandoahRuntime.hpp"
  34 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
  35 #include "interpreter/interp_masm.hpp"
  36 #include "interpreter/interpreter.hpp"
  37 #include "runtime/javaThread.hpp"
  38 #include "runtime/sharedRuntime.hpp"
  39 #ifdef COMPILER1
  40 #include "c1/c1_LIRAssembler.hpp"
  41 #include "c1/c1_MacroAssembler.hpp"
  42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
  43 #endif



  44 
  45 #define __ masm->
  46 






  47 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
  48                                                        Register src, Register dst, Register count, RegSet saved_regs) {
  49   if (is_oop) {
  50     bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
  51     if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
  52 
  53       Label done;
  54 
  55       // Avoid calling runtime if count == 0
  56       __ cbz(count, done);
  57 
  58       // Is GC active?
  59       Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
  60       __ ldrb(rscratch1, gc_state);
  61       if (ShenandoahSATBBarrier && dest_uninitialized) {
  62         __ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
  63       } else {
  64         __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING);
  65         __ tst(rscratch1, rscratch2);
  66         __ br(Assembler::EQ, done);

 585   if (is_cae) {
 586     // We're falling through to done to indicate success.  Success
 587     // with is_cae is denoted by returning the value of expected as
 588     // result.
 589     __ mov(tmp2, expected);
 590   }
 591 
 592   __ bind(done);
 593   // At entry to done, the Z (EQ) flag is on iff if the CAS
 594   // operation was successful.  Additionally, if is_cae, tmp2 holds
 595   // the value most recently fetched from addr. In this case, success
 596   // is denoted by tmp2 matching expected.
 597 
 598   if (is_cae) {
 599     __ mov(result, tmp2);
 600   } else {
 601     __ cset(result, Assembler::EQ);
 602   }
 603 }
 604 









































































































































































































































































































































































































































































































































 605 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
 606                                                                      Register start, Register count, Register scratch) {
 607   assert(ShenandoahCardBarrier, "Should have been checked by caller");
 608 
 609   Label L_loop, L_done;
 610   const Register end = count;
 611 
 612   // Zero count? Nothing to do.
 613   __ cbz(count, L_done);
 614 
 615   // end = start + count << LogBytesPerHeapOop
 616   // last element address to make inclusive
 617   __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop)));
 618   __ sub(end, end, BytesPerHeapOop);
 619   __ lsr(start, start, CardTable::card_shift());
 620   __ lsr(end, end, CardTable::card_shift());
 621 
 622   // number of bytes to copy
 623   __ sub(count, end, start);
 624 

 784       __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow));
 785     } else {
 786       __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
 787     }
 788   } else {
 789     assert(is_phantom, "only remaining strength");
 790     assert(is_native, "phantom must only be called off-heap");
 791     __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom));
 792   }
 793   __ blr(lr);
 794   __ mov(rscratch1, r0);
 795   __ pop_call_clobbered_registers();
 796   __ mov(r0, rscratch1);
 797 
 798   __ epilogue();
 799 }
 800 
 801 #undef __
 802 
 803 #endif // COMPILER1

































  24  */
  25 
  26 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
  27 #include "gc/shenandoah/mode/shenandoahMode.hpp"
  28 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  29 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
  30 #include "gc/shenandoah/shenandoahForwarding.hpp"
  31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
  33 #include "gc/shenandoah/shenandoahRuntime.hpp"
  34 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
  35 #include "interpreter/interp_masm.hpp"
  36 #include "interpreter/interpreter.hpp"
  37 #include "runtime/javaThread.hpp"
  38 #include "runtime/sharedRuntime.hpp"
  39 #ifdef COMPILER1
  40 #include "c1/c1_LIRAssembler.hpp"
  41 #include "c1/c1_MacroAssembler.hpp"
  42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
  43 #endif
  44 #ifdef COMPILER2
  45 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
  46 #endif
  47 
  48 #define __ masm->
  49 
  50 #ifdef PRODUCT
  51 #define BLOCK_COMMENT(str) /* nothing */
  52 #else
  53 #define BLOCK_COMMENT(str) __ block_comment(str)
  54 #endif
  55 
  56 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
  57                                                        Register src, Register dst, Register count, RegSet saved_regs) {
  58   if (is_oop) {
  59     bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
  60     if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
  61 
  62       Label done;
  63 
  64       // Avoid calling runtime if count == 0
  65       __ cbz(count, done);
  66 
  67       // Is GC active?
  68       Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
  69       __ ldrb(rscratch1, gc_state);
  70       if (ShenandoahSATBBarrier && dest_uninitialized) {
  71         __ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
  72       } else {
  73         __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING);
  74         __ tst(rscratch1, rscratch2);
  75         __ br(Assembler::EQ, done);

 594   if (is_cae) {
 595     // We're falling through to done to indicate success.  Success
 596     // with is_cae is denoted by returning the value of expected as
 597     // result.
 598     __ mov(tmp2, expected);
 599   }
 600 
 601   __ bind(done);
 602   // At entry to done, the Z (EQ) flag is on iff if the CAS
 603   // operation was successful.  Additionally, if is_cae, tmp2 holds
 604   // the value most recently fetched from addr. In this case, success
 605   // is denoted by tmp2 matching expected.
 606 
 607   if (is_cae) {
 608     __ mov(result, tmp2);
 609   } else {
 610     __ cset(result, Assembler::EQ);
 611   }
 612 }
 613 
 614 #ifdef COMPILER2
 615 void ShenandoahBarrierSetAssembler::load_ref_barrier_c2(const MachNode* node, MacroAssembler* masm, Register obj, Register addr, bool narrow, bool maybe_null, Register gc_state) {
 616   assert_different_registers(obj, addr);
 617   BLOCK_COMMENT("load_ref_barrier_c2 {");
 618   if (!ShenandoahLoadRefBarrierStubC2::needs_barrier(node)) {
 619     return;
 620   }
 621   Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
 622   ShenandoahLoadRefBarrierStubC2* const stub = ShenandoahLoadRefBarrierStubC2::create(node, obj, addr, gc_state, noreg, noreg, narrow);
 623 
 624   // Don't preserve the obj across the runtime call, we override it from the
 625   // return value anyway.
 626   stub->dont_preserve(obj);
 627   stub->dont_preserve(gc_state);
 628 
 629   // Check if GC marking is in progress or we are handling a weak reference,
 630   // otherwise we don't have to do anything. The code below was optimized to
 631   // use less registers and instructions as possible at the expense of always
 632   // having a branch instruction. The reason why we use this particular branch
 633   // scheme is because the stub entry may be too far for the tbnz to jump to.
 634   bool is_strong = (node->barrier_data() & ShenandoahBarrierStrong) != 0;
 635   if (is_strong) {
 636     __ tbz(gc_state, ShenandoahHeap::HAS_FORWARDED_BITPOS, *stub->continuation());
 637     __ b(*stub->entry());
 638   } else {
 639     static_assert(ShenandoahHeap::HAS_FORWARDED_BITPOS == 0, "Relied on in LRB check below.");
 640     __ orr(gc_state, gc_state, gc_state, Assembler::LSR, ShenandoahHeap::WEAK_ROOTS_BITPOS);
 641     __ tbz(gc_state, ShenandoahHeap::HAS_FORWARDED_BITPOS, *stub->continuation());
 642     __ b(*stub->entry());
 643   }
 644 
 645   __ bind(*stub->continuation());
 646   BLOCK_COMMENT("} load_ref_barrier_c2");
 647 }
 648 
 649 /**
 650  * The logic implemented here relies on certain flags being on specific
 651  * positions of the GCState. Also note that all pointer values in register are
 652  * guaranteed to be 'to-space' addresses. The algorithm is as follows. If the
 653  * CAS succeed:
 654  *    - 'res' will be set to 1.
 655  *    - We need to check SATB flag (index 1 of GCState). If the flag is active
 656  *      need to store 'oldval' in the buffer.
 657  *    - We wrote 'newval' to 'addr', therefore we need to mark the corresponding
 658  *      card in the card table for 'addr' as dirty.
 659  * If the CAS failed:
 660  *    - 'res' will be set to 0.
 661  *    - If the GCState FORWARDING bit (index 0 of GCState) is set we'll need to
 662  *      retry the CAS, because the failure may be because the value in 'addr' is
 663  *      the (outdated) 'from-space' version of 'expected'. The retry is done in a
 664  *      stub. If the retry succeed then we need to do the steps described above
 665  *      too for CAS succeed too.
 666  *    - If FORWARDING bit is clear there is nothing else to do.
 667  */
 668 void ShenandoahBarrierSetAssembler::cas_c2(const MachNode* node, MacroAssembler* masm, Register res, Register addr, Register oldval, Register newval, Register tmp1, Register tmp2, bool narrow, bool acquire, bool release, bool weak) {
 669   Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
 670   Assembler::operand_size op_size = UseCompressedOops ? Assembler::word : Assembler::xword;
 671 
 672   // Assuming just for now that we need both barriers
 673   NewShenandoahCASBarrierSlowStubC2* const cmpx = NewShenandoahCASBarrierSlowStubC2::create(node, addr, oldval, newval, res, tmp1, tmp2, narrow, false, acquire, release, weak);
 674 
 675   // Issue cmpxchg first
 676   __ cmpxchg(addr, oldval, newval, op_size, acquire, release, weak, tmp2);
 677 
 678   // Load GC state in tmp1
 679   Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 680   __ ldrb(tmp1, gcs_addr);
 681 
 682   // First CAS attempt. If successful, then we are done.
 683   // EQ flag set iff success. 'res' holds value fetched.
 684   __ cset(res, Assembler::EQ);
 685 
 686   // Shift GCState right according to result of previous CAS. Luckily, the
 687   // boolean result of the CAS also matches the index of the bit that we need
 688   // to test later on.
 689   __ lsrv(tmp1, tmp1, res);
 690 
 691   // Test bit '0' of tmp1, which at this point will be FORWARDING bit if CAS
 692   // failed, or SATB bit if CAS succeded.
 693   __ tbnz(tmp1, 0x0, *cmpx->entry());
 694 
 695   // Would be nice to fold this in the comparison above, but how?
 696   // Skip Card Table dirtying if CAS failed.
 697   __ cbz(res, *cmpx->continuation());
 698   card_barrier_c2(node, masm, addr, tmp1);
 699 
 700   __ bind(*cmpx->continuation());
 701 }
 702 
 703 void ShenandoahBarrierSetAssembler::cae_c2(const MachNode* node, MacroAssembler* masm, Register res, Register addr, Register oldval, Register newval, Register tmp1, Register tmp2, bool maybe_null, bool narrow, bool acquire, bool release, bool weak) {
 704   Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
 705   Assembler::operand_size op_size = UseCompressedOops ? Assembler::word : Assembler::xword;
 706 
 707   Label succeded;
 708 
 709   // Assuming just for now that we need both barriers
 710   ShenandoahCASBarrierSlowStubC2* const cmpx = ShenandoahCASBarrierSlowStubC2::create(node, addr, oldval, newval, res, tmp1, tmp2, false, acquire, release, weak);
 711 
 712   // Load GC state in tmp1
 713   Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 714   __ ldrb(tmp1, gcs_addr);
 715 
 716   // Fast-path: Try to CAS optimistically. If successful, then we are done.
 717   // EQ flag set iff success. 'res' holds value fetched.
 718   __ cmpxchg(addr, oldval, newval, op_size, acquire, release, weak, res);
 719   __ cset(res, Assembler::EQ);
 720   __ lsrv(tmp1, tmp1, res);
 721   __ tbnz(tmp1, 0x0, *cmpx->entry());
 722 
 723   __ tbz(res, 0x0, *cmpx->continuation());
 724   card_barrier_c2(node, masm, addr, tmp1);
 725 
 726   __ bind(*cmpx->continuation());
 727 }
 728 
 729 
 730 
 731 void ShenandoahBarrierSetAssembler::store_c2(const MachNode* node, MacroAssembler* masm,
 732                                              Register dst, bool dst_narrow,
 733                                              Register src, bool src_narrow,
 734                                              Register tmp, Register pre_val,
 735                                              bool is_volatile) {
 736 
 737   Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 738   __ ldrb(tmp, gcs_addr);
 739 
 740   satb_barrier_c2(node, masm, dst, pre_val, tmp, src_narrow);
 741 
 742   card_barrier_c2(node, masm, dst, tmp);
 743 
 744   // Need to encode into tmp, because we cannot clobber src.
 745   // TODO: Maybe there is a matcher way to test that src is unused after this?
 746   if (dst_narrow && !src_narrow) {
 747     __ mov(tmp, src);
 748     if (ShenandoahStoreBarrierStubC2::src_not_null(node)) {
 749       __ encode_heap_oop_not_null(tmp);
 750     } else {
 751       __ encode_heap_oop(tmp);
 752     }
 753     src = tmp;
 754   }
 755 
 756   // Do the actual store
 757   if (dst_narrow) {
 758     if (is_volatile) {
 759       __ stlrw(src, dst);
 760     } else {
 761       __ strw(src, dst);
 762     }
 763   } else {
 764     if (is_volatile) {
 765       __ stlr(src, dst);
 766     } else {
 767       __ str(src, dst);
 768     }
 769   }
 770 }
 771 
 772 void ShenandoahBarrierSetAssembler::satb_barrier_c2(const MachNode* node, MacroAssembler* masm, Register addr, Register pre_val,
 773                                                     Register gc_state, bool encoded_preval) {
 774   BLOCK_COMMENT("satb_barrier_c2 {");
 775   assert_different_registers(addr, pre_val, rscratch1, rscratch2);
 776   if (!ShenandoahSATBBarrierStubC2::needs_barrier(node)) {
 777     return;
 778   }
 779   Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
 780   ShenandoahSATBBarrierStubC2* const stub = ShenandoahSATBBarrierStubC2::create(node, addr, pre_val, gc_state, encoded_preval);
 781 
 782   // Check if GC marking is in progress, otherwise we don't have to do
 783   // anything.
 784   __ tstw(gc_state, ShenandoahHeap::MARKING);
 785   __ br(Assembler::NE, *stub->entry());
 786   __ bind(*stub->continuation());
 787   BLOCK_COMMENT("} satb_barrier_c2");
 788 }
 789 
 790 void ShenandoahBarrierSetAssembler::card_barrier_c2(const MachNode* node, MacroAssembler* masm, Register addr, Register tmp) {
 791   if (!ShenandoahCardBarrier ||
 792       (node->barrier_data() & ShenandoahBarrierCardMark) == 0) {
 793     return;
 794   }
 795 
 796   Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
 797   __ lsr(tmp, addr, CardTable::card_shift());
 798 
 799   assert(CardTable::dirty_card_val() == 0, "must be");
 800 
 801   Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
 802   __ ldr(rscratch1, curr_ct_holder_addr);
 803 
 804   if (UseCondCardMark) {
 805     Label L_already_dirty;
 806     __ ldrb(rscratch2, Address(tmp, rscratch1));
 807     __ cbz(rscratch2, L_already_dirty);
 808     __ strb(zr, Address(tmp, rscratch1));
 809     __ bind(L_already_dirty);
 810   } else {
 811     __ strb(zr, Address(tmp, rscratch1));
 812   }
 813 }
 814 
 815 void ShenandoahBarrierSetAssembler::cmpxchg_oop_c2(const MachNode* node,
 816                                                    MacroAssembler* masm,
 817                                                    Register addr, Register oldval,
 818                                                    Register newval, Register res,
 819                                                    Register gc_state, Register tmp,
 820                                                    bool acquire, bool release, bool weak, bool exchange) {
 821   assert(res != noreg, "need result register");
 822   assert_different_registers(oldval, addr, res, gc_state, tmp);
 823   assert_different_registers(newval, addr, res, gc_state, tmp);
 824 
 825   // Fast-path: Try to CAS optimistically. If successful, then we are done.
 826   // EQ flag set iff success. 'tmp' holds value fetched.
 827   Assembler::operand_size size = UseCompressedOops ? Assembler::word : Assembler::xword;
 828   __ cmpxchg(addr, oldval, newval, size, acquire, release, weak, tmp);
 829 
 830   // If we need a boolean result out of CAS, set the flag appropriately.  This
 831   // would be the final result if we do not go slow.
 832   if (!exchange) {
 833     __ cset(res, Assembler::EQ);
 834   } else {
 835     __ mov(res, tmp);
 836   }
 837 
 838   if (ShenandoahCASBarrier) {
 839     ShenandoahCASBarrierSlowStubC2* const slow_stub =
 840       ShenandoahCASBarrierSlowStubC2::create(node, addr, oldval, newval, res, gc_state, tmp, exchange, acquire, release, weak);
 841 
 842     slow_stub->preserve(gc_state);    // this really need to be preserved as we
 843                                       // try to use it in subsequent barriers
 844 
 845     slow_stub->dont_preserve(res);    // set at the end, no need to save
 846     slow_stub->dont_preserve(oldval); // saved explicitly
 847     slow_stub->dont_preserve(tmp);    // temp, no need to save
 848 
 849     // On success, we do not need any additional handling.
 850     __ br(Assembler::EQ, *slow_stub->continuation());
 851 
 852     // If GC is in progress, it is likely we need additional handling for false negatives.
 853     __ tbz(gc_state, ShenandoahHeap::HAS_FORWARDED_BITPOS, *slow_stub->continuation());
 854     __ b(*slow_stub->entry());
 855 
 856     // Slow stub re-enters with result set correctly.
 857     __ bind(*slow_stub->continuation());
 858   }
 859 }
 860 
 861 #undef __
 862 #define __ masm.
 863 
 864 void ShenandoahLoadRefBarrierStubC2::emit_code(MacroAssembler& masm) {
 865   BLOCK_COMMENT("ShenandoahLoadRefBarrierStubC2::emit_code {");
 866   Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
 867   __ bind(*entry());
 868   Register obj = _obj;
 869   if (_narrow) {
 870     __ decode_heap_oop(_tmp1, _obj);
 871     obj = _tmp1;
 872   }
 873   // Weak/phantom loads always need to go to runtime.
 874   if ((_node->barrier_data() & ShenandoahBarrierStrong) != 0) {
 875     // Check for object in cset.
 876     __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
 877     __ lsr(rscratch1, obj, ShenandoahHeapRegion::region_size_bytes_shift_jint());
 878     __ ldrb(rscratch2, Address(rscratch2, rscratch1));
 879     __ cbz(rscratch2, *continuation());
 880   }
 881   {
 882     SaveLiveRegisters save_registers(&masm, this);
 883     if (c_rarg0 != obj) {
 884       if (c_rarg0 == _addr) {
 885         __ mov(rscratch1, _addr);
 886         _addr = rscratch1;
 887       }
 888       __ mov(c_rarg0, obj);
 889     }
 890     __ mov(c_rarg1, _addr);
 891 
 892     if (_narrow) {
 893       if ((_node->barrier_data() & ShenandoahBarrierStrong) != 0) {
 894         __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow));
 895       } else if ((_node->barrier_data() & ShenandoahBarrierWeak) != 0) {
 896         __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow));
 897       } else if ((_node->barrier_data() & ShenandoahBarrierPhantom) != 0) {
 898         __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom_narrow));
 899       }
 900     } else {
 901       if ((_node->barrier_data() & ShenandoahBarrierStrong) != 0) {
 902         __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong));
 903       } else if ((_node->barrier_data() & ShenandoahBarrierWeak) != 0) {
 904         __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
 905       } else if ((_node->barrier_data() & ShenandoahBarrierPhantom) != 0) {
 906         __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom));
 907       }
 908     }
 909     __ blr(rscratch1);
 910     __ mov(_obj, r0);
 911   }
 912   if (_narrow) {
 913     __ encode_heap_oop(_obj);
 914   }
 915   __ b(*continuation());
 916   BLOCK_COMMENT("} ShenandoahLoadRefBarrierStubC2::emit_code");
 917 }
 918 
 919 void ShenandoahSATBBarrierStubC2::emit_code(MacroAssembler& masm) {
 920   BLOCK_COMMENT("ShenandoahSATBBarrierStubC2::emit_code {");
 921   Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
 922   __ bind(*entry());
 923 
 924   // The tmp register that we receive is usually a register holding the
 925   // "gc_state" which may be required by subsequent memory operations in their
 926   // fastpath.
 927   RegSet saved = RegSet::of(_tmp);
 928   __ push(saved, sp);
 929 
 930   // Do we need to load the previous value?
 931   if (_addr != noreg) {
 932     __ load_heap_oop(_tmp, Address(_addr, 0), noreg, noreg, AS_RAW);
 933   } else {
 934     if (_encoded_preval) {
 935       __ decode_heap_oop(_tmp, _preval);
 936     } else {
 937       _tmp = _preval;
 938     }
 939   }
 940 
 941   Address index(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
 942   Address buffer(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
 943   Label runtime;
 944   __ ldr(rscratch1, index);
 945   // If buffer is full, call into runtime.
 946   __ cbz(rscratch1, runtime);
 947 
 948   // The buffer is not full, store value into it.
 949   __ sub(rscratch1, rscratch1, wordSize);
 950   __ str(rscratch1, index);
 951   __ ldr(rscratch2, buffer);
 952   __ str(_tmp, Address(rscratch2, rscratch1));
 953   __ pop(saved, sp);
 954   __ b(*continuation());
 955 
 956   // Runtime call
 957   __ bind(runtime);
 958   {
 959     SaveLiveRegisters save_registers(&masm, this);
 960     __ mov(c_rarg0, _tmp);
 961     __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre_c2));
 962     __ blr(rscratch1);
 963   }
 964   __ pop(saved, sp);
 965   __ b(*continuation());
 966   BLOCK_COMMENT("} ShenandoahSATBBarrierStubC2::emit_code");
 967 }
 968 
 969 void ShenandoahLoadBarrierStubC2::emit_code(MacroAssembler& masm) {
 970   Unimplemented();
 971 }
 972 
 973 void ShenandoahStoreBarrierStubC2::emit_code(MacroAssembler& masm) {
 974   Unimplemented();
 975 }
 976 
 977 void ShenandoahCASBarrierSlowStubC2::emit_code(MacroAssembler& masm) {
 978   __ bind(*entry());
 979 
 980   // CAS has failed because the value held at addr does not match expected.
 981   // This may be a false negative because the version in memory might be
 982   // the from-space version of the same object we currently hold to-space
 983   // reference for.
 984   //
 985   // To resolve this, we need to pass the location through the LRB fixup,
 986   // this will make sure that the location has only to-space pointers.
 987   // To avoid calling into runtime often, we cset-check the object first.
 988   // We can inline most of the work here, but there is little point,
 989   // as CAS failures over cset locations must be rare. This fast-slow split
 990   // matches what we do for normal LRB.
 991 
 992   // Non-strong references should always go to runtime. We do not expect
 993   // CASes over non-strong locations.
 994   assert((_node->barrier_data() & ShenandoahBarrierStrong) != 0, "Only strong references for CASes");
 995 
 996   Label L_final;
 997 
 998   // (Compressed) failure witness is in _tmp2.
 999   // Unpack it and check if it is in collection set.
1000   // We need to backup the compressed version to use in the LRB.
1001   __ mov(_result, _tmp2);
1002   if (UseCompressedOops) {
1003     __ decode_heap_oop(_tmp2);
1004   }
1005 
1006   __ mov(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
1007   __ lsr(_tmp2, _tmp2, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1008   __ ldrb(_tmp1, Address(_tmp1, _tmp2));
1009   __ cbz(_tmp1, L_final);
1010 
1011   {
1012     SaveLiveRegisters save_registers(&masm, this);
1013     // Load up failure witness again.
1014     __ mov(c_rarg0, _result);
1015     if (UseCompressedOops) {
1016       __ decode_heap_oop(c_rarg0);
1017     }
1018     __ mov(c_rarg1, _addr_reg);
1019 
1020     if (UseCompressedOops) {
1021       __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow), 2);
1022     } else {
1023       __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong), 2);
1024     }
1025     // We have called LRB to fix up the heap location. We do not care about its
1026     // result, as we will just try to CAS the location again.
1027   }
1028 
1029   __ bind(L_final);
1030 
1031   Assembler::operand_size size = UseCompressedOops ? Assembler::word : Assembler::xword;
1032   __ cmpxchg(_addr_reg, _expected, _new_val, size, _acquire, _release, _weak, _result);
1033 
1034   if (!_cae) {
1035     __ cset(_result, Assembler::EQ);
1036   }
1037   __ b(*continuation());
1038 }
1039 
1040 void NewShenandoahCASBarrierSlowStubC2::emit_code(MacroAssembler& masm) {
1041   __ bind(*entry());
1042 
1043   // Non-strong references should always go to runtime. We do not expect
1044   // CASes over non-strong locations.
1045   assert((_node->barrier_data() & ShenandoahBarrierStrong) != 0, "Only strong references for CASes");
1046 
1047   Label L_final;
1048   Label L_succeded;
1049 
1050   // check if first CAS succeded, if it did we just need to write to SATB
1051   __ cbnz(_result, L_succeded);
1052 
1053           // First CAS attempt did not succed. Execute LRB on 'addr' and retry CAS.
1054 
1055           // (Compressed) failure witness is in _tmp2.
1056           // Unpack it and check if it is in collection set.
1057           // We need to backup the compressed version to use in the LRB.
1058           __ mov(_result, _tmp2);
1059           if (UseCompressedOops) {
1060             __ decode_heap_oop(_tmp2);
1061           }
1062 
1063           __ mov(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
1064           __ lsr(_tmp2, _tmp2, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1065           __ ldrb(_tmp1, Address(_tmp1, _tmp2));
1066           __ cbz(_tmp1, L_final);
1067 
1068           {
1069             SaveLiveRegisters save_registers(&masm, this);
1070             // Load up failure witness again.
1071             __ mov(c_rarg0, _result);
1072             if (UseCompressedOops) {
1073               __ decode_heap_oop(c_rarg0);
1074             }
1075             __ mov(c_rarg1, _addr_reg);
1076 
1077             if (UseCompressedOops) {
1078               __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow), 2);
1079             } else {
1080               __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong), 2);
1081             }
1082             // We have called LRB to fix up the heap location. We do not care about its
1083             // result, as we will just try to CAS the location again.
1084           }
1085 
1086           __ bind(L_final);
1087 
1088           Assembler::operand_size size = UseCompressedOops ? Assembler::word : Assembler::xword;
1089           __ cmpxchg(_addr_reg, _expected, _new_val, size, _acquire, _release, _weak, _result);
1090 
1091           __ cset(_result, Assembler::EQ);
1092 
1093           // If the retry did not succeed skip SATB
1094           __ cbz(_result, *continuation());
1095 
1096 
1097 
1098     __ bind(L_succeded);
1099               if (_narrow) {
1100                 __ decode_heap_oop(_tmp1, _expected);
1101               } else {
1102                 _tmp1 = _expected;
1103               }
1104 
1105               Address index(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
1106               Address buffer(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
1107               Label runtime;
1108               __ ldr(rscratch1, index);
1109               // If buffer is full, call into runtime.
1110               __ cbz(rscratch1, runtime);
1111 
1112               // The buffer is not full, store value into it.
1113               __ sub(rscratch1, rscratch1, wordSize);
1114               __ str(rscratch1, index);
1115               __ ldr(rscratch2, buffer);
1116               __ str(_tmp1, Address(rscratch2, rscratch1));
1117               __ b(*continuation());
1118 
1119               // Runtime call
1120               __ bind(runtime);
1121               {
1122                 SaveLiveRegisters save_registers(&masm, this);
1123                 __ mov(c_rarg0, _tmp1);
1124                 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre_c2));
1125                 __ blr(rscratch1);
1126               }
1127 
1128 
1129     __ b(*continuation());
1130   }
1131 #undef __
1132 #define __ masm->
1133 #endif // COMPILER2
1134 
1135 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
1136                                                                      Register start, Register count, Register scratch) {
1137   assert(ShenandoahCardBarrier, "Should have been checked by caller");
1138 
1139   Label L_loop, L_done;
1140   const Register end = count;
1141 
1142   // Zero count? Nothing to do.
1143   __ cbz(count, L_done);
1144 
1145   // end = start + count << LogBytesPerHeapOop
1146   // last element address to make inclusive
1147   __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop)));
1148   __ sub(end, end, BytesPerHeapOop);
1149   __ lsr(start, start, CardTable::card_shift());
1150   __ lsr(end, end, CardTable::card_shift());
1151 
1152   // number of bytes to copy
1153   __ sub(count, end, start);
1154 

1314       __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow));
1315     } else {
1316       __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
1317     }
1318   } else {
1319     assert(is_phantom, "only remaining strength");
1320     assert(is_native, "phantom must only be called off-heap");
1321     __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom));
1322   }
1323   __ blr(lr);
1324   __ mov(rscratch1, r0);
1325   __ pop_call_clobbered_registers();
1326   __ mov(r0, rscratch1);
1327 
1328   __ epilogue();
1329 }
1330 
1331 #undef __
1332 
1333 #endif // COMPILER1
1334 
1335 //void ShenandoahBarrierSetAssembler::cas_c2(const MachNode* node, MacroAssembler* masm, Register res, Register addr, Register oldval, Register newval, Register tmp1, Register tmp2, bool narrow, bool acquire, bool release, bool weak) {
1336 //  Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
1337 //  Assembler::operand_size op_size = UseCompressedOops ? Assembler::word : Assembler::xword;
1338 //
1339 //  Label succeded;
1340 //
1341 //  ShenandoahSATBBarrierStubC2* const satb = ShenandoahSATBBarrierStubC2::create(node, addr, oldval, tmp1, narrow);
1342 //
1343 //  // Assuming just for now that we need both barriers
1344 //  ShenandoahCASBarrierSlowStubC2* const cmpx = ShenandoahCASBarrierSlowStubC2::create(node, addr, oldval, newval, res, tmp1, tmp2, false, acquire, release, weak);
1345 //
1346 //  // Load GC state in tmp1
1347 //  Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1348 //  __ ldrb(tmp1, gcs_addr);
1349 //
1350 //  // Fast-path: Try to CAS optimistically. If successful, then we are done.
1351 //  // EQ flag set iff success. 'res' holds value fetched.
1352 //  __ cmpxchg(addr, oldval, newval, op_size, acquire, release, weak, res);
1353 //  __ cset(res, Assembler::EQ);
1354 //  __ br(Assembler::EQ, succeded);
1355 //
1356 //  __ tbnz(tmp1, ShenandoahHeap::HAS_FORWARDED_BITPOS, *cmpx->entry());
1357 //  __ b(*cmpx->continuation());
1358 //
1359 //  __ bind(succeded);
1360 //  __ tbnz(tmp1, ShenandoahHeap::MARKING_BITPOS, *satb->entry());
1361 //  __ bind(*satb->continuation());
1362 //  card_barrier_c2(node, masm, addr, tmp1);
1363 //
1364 //  __ bind(*cmpx->continuation());
1365 //}
< prev index next >