25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahForwarding.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interp_masm.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "runtime/javaThread.hpp"
39 #include "runtime/sharedRuntime.hpp"
40 #ifdef COMPILER1
41 #include "c1/c1_LIRAssembler.hpp"
42 #include "c1/c1_MacroAssembler.hpp"
43 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
44 #endif
45
46 #define __ masm->
47
48 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
49 Register src, Register dst, Register count, RegSet saved_regs) {
50 if (is_oop) {
51 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
52 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
53
54 Label done;
55
56 // Avoid calling runtime if count == 0
57 __ cbz(count, done);
58
59 // Is GC active?
60 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
61 __ ldrb(rscratch1, gc_state);
62 if (ShenandoahSATBBarrier && dest_uninitialized) {
63 __ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
64 } else {
610 if (is_cae) {
611 // We're falling through to done to indicate success. Success
612 // with is_cae is denoted by returning the value of expected as
613 // result.
614 __ mov(tmp2, expected);
615 }
616
617 __ bind(done);
618 // At entry to done, the Z (EQ) flag is on iff if the CAS
619 // operation was successful. Additionally, if is_cae, tmp2 holds
620 // the value most recently fetched from addr. In this case, success
621 // is denoted by tmp2 matching expected.
622
623 if (is_cae) {
624 __ mov(result, tmp2);
625 } else {
626 __ cset(result, Assembler::EQ);
627 }
628 }
629
630 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
631 Register start, Register count, Register scratch) {
632 assert(ShenandoahCardBarrier, "Should have been checked by caller");
633
634 Label L_loop, L_done;
635 const Register end = count;
636
637 // Zero count? Nothing to do.
638 __ cbz(count, L_done);
639
640 // end = start + count << LogBytesPerHeapOop
641 // last element address to make inclusive
642 __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop)));
643 __ sub(end, end, BytesPerHeapOop);
644 __ lsr(start, start, CardTable::card_shift());
645 __ lsr(end, end, CardTable::card_shift());
646
647 // number of bytes to copy
648 __ sub(count, end, start);
649
|
25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahForwarding.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interp_masm.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "runtime/javaThread.hpp"
39 #include "runtime/sharedRuntime.hpp"
40 #ifdef COMPILER1
41 #include "c1/c1_LIRAssembler.hpp"
42 #include "c1/c1_MacroAssembler.hpp"
43 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
44 #endif
45 #ifdef COMPILER2
46 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
47 #include "opto/output.hpp"
48 #endif
49
50 #define __ masm->
51
52 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
53 Register src, Register dst, Register count, RegSet saved_regs) {
54 if (is_oop) {
55 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
56 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
57
58 Label done;
59
60 // Avoid calling runtime if count == 0
61 __ cbz(count, done);
62
63 // Is GC active?
64 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
65 __ ldrb(rscratch1, gc_state);
66 if (ShenandoahSATBBarrier && dest_uninitialized) {
67 __ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
68 } else {
614 if (is_cae) {
615 // We're falling through to done to indicate success. Success
616 // with is_cae is denoted by returning the value of expected as
617 // result.
618 __ mov(tmp2, expected);
619 }
620
621 __ bind(done);
622 // At entry to done, the Z (EQ) flag is on iff if the CAS
623 // operation was successful. Additionally, if is_cae, tmp2 holds
624 // the value most recently fetched from addr. In this case, success
625 // is denoted by tmp2 matching expected.
626
627 if (is_cae) {
628 __ mov(result, tmp2);
629 } else {
630 __ cset(result, Assembler::EQ);
631 }
632 }
633
634 #ifdef COMPILER2
635 void ShenandoahBarrierStubC2::gc_state_check_c2(MacroAssembler* masm, Register gcstate, const unsigned char test_state, ShenandoahBarrierStubC2* slow_stub) {
636 if (ShenandoahGCStateCheckRemove) {
637 // Unrealistic: remove all barrier fastpath checks.
638 } else if (ShenandoahGCStateCheckHotpatch) {
639 // In the ideal world, we would hot-patch the branch to slow stub with a single
640 // (unconditional) jump or nop, based on our current GC state.
641 __ nop();
642 } else {
643 int bit_to_check = ShenandoahThreadLocalData::gc_state_to_fast_bit(test_state);
644 Address gc_state_fast(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_offset()));
645 __ ldrb(gcstate, gc_state_fast);
646 if (slow_stub->_test_and_branch_reachable) {
647 __ tbnz(gcstate, bit_to_check, *slow_stub->entry());
648 } else {
649 __ tbz(gcstate, bit_to_check, *slow_stub->continuation());
650 __ b(*slow_stub->entry());
651 }
652
653 // This is were the slowpath stub will return to or the code above will
654 // jump to if the checks are false
655 __ bind(*slow_stub->continuation());
656 }
657 }
658
659 bool needs_acquiring_load_exclusive(const MachNode *n) {
660 assert(n->is_CAS(true), "expecting a compare and swap");
661 if (n->is_CAS(false)) {
662 assert(n->has_trailing_membar(), "expected trailing membar");
663 } else {
664 return n->has_trailing_membar();
665 }
666
667 // so we can just return true here
668 return true;
669 }
670
671 void ShenandoahBarrierSetAssembler::compare_and_set_c2(const MachNode* node, MacroAssembler* masm, Register res, Register addr,
672 Register oldval, Register newval, Register tmp, bool exchange, bool maybe_null, bool narrow, bool weak) {
673 bool acquire = needs_acquiring_load_exclusive(node);
674 Assembler::operand_size op_size = narrow ? Assembler::word : Assembler::xword;
675
676 // Pre-barrier covers several things:
677 // a. Avoids false positives from CAS encountering to-space memory values.
678 // b. Satisfies the need for LRB for the CAE result.
679 // c. Records old value for the sake of SATB.
680 //
681 // (a) and (b) are covered because load barrier does memory location fixup.
682 // (c) is covered by KA on the current memory value.
683 if (ShenandoahBarrierStubC2::needs_slow_barrier(node)) {
684 ShenandoahBarrierStubC2* const stub = ShenandoahBarrierStubC2::create(node, tmp, addr, narrow, /* do_load: */ true, __ offset());
685 char check = 0;
686 check |= ShenandoahBarrierStubC2::needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
687 check |= ShenandoahBarrierStubC2::needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
688 assert(!ShenandoahBarrierStubC2::needs_load_ref_barrier_weak(node), "Not supported for CAS");
689 ShenandoahBarrierStubC2::gc_state_check_c2(masm, rscratch1, check, stub);
690 }
691
692 // CAS!
693 __ cmpxchg(addr, oldval, newval, op_size, acquire, /* release */ true, weak, exchange ? res : noreg);
694
695 // If we need a boolean result out of CAS, set the flag appropriately and promote the result.
696 if (!exchange) {
697 assert(res != noreg, "need result register");
698 __ cset(res, Assembler::EQ);
699 }
700
701 // Post-barrier deals with card updates.
702 card_barrier_c2(node, masm, Address(addr, 0));
703 }
704
705 void ShenandoahBarrierSetAssembler::get_and_set_c2(const MachNode* node, MacroAssembler* masm, Register preval,
706 Register newval, Register addr, Register tmp) {
707 bool acquire = needs_acquiring_load_exclusive(node);
708 bool narrow = node->bottom_type()->isa_narrowoop();
709
710 // Pre-barrier covers several things:
711 // a. Satisfies the need for LRB for the GAS result.
712 // b. Records old value for the sake of SATB.
713 //
714 // (a) is covered because load barrier does memory location fixup.
715 // (b) is covered by KA on the current memory value.
716 if (ShenandoahBarrierStubC2::needs_slow_barrier(node)) {
717 ShenandoahBarrierStubC2* const stub = ShenandoahBarrierStubC2::create(node, tmp, addr, narrow, /* do_load: */ true, __ offset());
718 char check = 0;
719 check |= ShenandoahBarrierStubC2::needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
720 check |= ShenandoahBarrierStubC2::needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
721 assert(!ShenandoahBarrierStubC2::needs_load_ref_barrier_weak(node), "Not supported for GAS");
722 ShenandoahBarrierStubC2::gc_state_check_c2(masm, rscratch1, check, stub);
723 }
724
725 if (narrow) {
726 if (acquire) {
727 __ atomic_xchgalw(preval, newval, addr);
728 } else {
729 __ atomic_xchgw(preval, newval, addr);
730 }
731 } else {
732 if (acquire) {
733 __ atomic_xchgal(preval, newval, addr);
734 } else {
735 __ atomic_xchg(preval, newval, addr);
736 }
737 }
738
739 // Post-barrier deals with card updates.
740 card_barrier_c2(node, masm, Address(addr, 0));
741 }
742
743 void ShenandoahBarrierSetAssembler::store_c2(const MachNode* node, MacroAssembler* masm, Address dst, bool dst_narrow,
744 Register src, bool src_narrow, Register tmp) {
745
746 // Pre-barrier: SATB, keep-alive the current memory value.
747 if (ShenandoahBarrierStubC2::needs_slow_barrier(node)) {
748 assert(!ShenandoahBarrierStubC2::needs_load_ref_barrier(node), "Should not be required for stores");
749 ShenandoahBarrierStubC2* const stub = ShenandoahBarrierStubC2::create(node, tmp, dst, dst_narrow, /* do_load: */ true, __ offset());
750 ShenandoahBarrierStubC2::gc_state_check_c2(masm, rscratch1, ShenandoahHeap::MARKING, stub);
751 }
752
753 // Do the actual store
754 bool is_volatile = node->has_trailing_membar();
755 if (dst_narrow) {
756 if (!src_narrow) {
757 // Need to encode into rscratch, because we cannot clobber src.
758 // TODO: Maybe there is a matcher way to test that src is unused after this?
759 __ mov(rscratch1, src);
760 if (ShenandoahBarrierStubC2::src_not_null(node)) {
761 __ encode_heap_oop_not_null(rscratch1);
762 } else {
763 __ encode_heap_oop(rscratch1);
764 }
765 src = rscratch1;
766 }
767
768 if (is_volatile) {
769 __ stlrw(src, dst.base());
770 } else {
771 __ strw(src, dst);
772 }
773 } else {
774 if (is_volatile) {
775 __ stlr(src, dst.base());
776 } else {
777 __ str(src, dst);
778 }
779 }
780
781 // Post-barrier: card updates.
782 card_barrier_c2(node, masm, dst);
783 }
784
785 void ShenandoahBarrierSetAssembler::load_c2(const MachNode* node, MacroAssembler* masm, Register dst, Address src) {
786 bool acquire = node->memory_order() == MemNode::MemOrd::acquire;
787 bool narrow = node->bottom_type()->isa_narrowoop();
788
789 // Do the actual load. This load is the candidate for implicit null check, and MUST come first.
790 if (narrow) {
791 if (acquire) {
792 __ ldarw(dst, src.base());
793 } else {
794 __ ldrw(dst, src);
795 }
796 } else {
797 if (acquire) {
798 __ ldar(dst, src.base());
799 } else {
800 __ ldr(dst, src);
801 }
802 }
803
804 // Post-barrier: LRB / KA / weak-root processing.
805 if (ShenandoahBarrierStubC2::needs_slow_barrier(node)) {
806 ShenandoahBarrierStubC2* const stub = ShenandoahBarrierStubC2::create(node, dst, src, narrow, /* do_load: */ false, __ offset());
807 char check = 0;
808 check |= ShenandoahBarrierStubC2::needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
809 check |= ShenandoahBarrierStubC2::needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
810 check |= ShenandoahBarrierStubC2::needs_load_ref_barrier_weak(node) ? ShenandoahHeap::WEAK_ROOTS : 0;
811 ShenandoahBarrierStubC2::gc_state_check_c2(masm, rscratch1, check, stub);
812 }
813 }
814
815 void ShenandoahBarrierSetAssembler::card_barrier_c2(const MachNode* node, MacroAssembler* masm, Address address) {
816 if (ShenandoahSkipBarriers || (node->barrier_data() & ShenandoahBitCardMark) == 0) {
817 return;
818 }
819
820 assert(CardTable::dirty_card_val() == 0, "must be");
821 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
822
823 // rscratch2 = addr >> CardTable::card_shift()
824 __ lea(rscratch2, address);
825 __ lsr(rscratch2, rscratch2, CardTable::card_shift());
826
827 // rscratch1 = card table base (holder)
828 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
829 __ ldr(rscratch1, curr_ct_holder_addr);
830
831 // rscratch2 = &card_table[card_index]
832 __ add(rscratch2, rscratch1, rscratch2);
833
834 if (UseCondCardMark) {
835 Label L_already_dirty;
836 __ ldrb(rscratch1, Address(rscratch2));
837 __ cbz(rscratch1, L_already_dirty);
838 __ strb(zr, Address(rscratch2));
839 __ bind(L_already_dirty);
840 } else {
841 __ strb(zr, Address(rscratch2));
842 }
843 }
844 #undef __
845 #define __ masm.
846
847 // Only handles forward branch jumps, target_offset >= branch_offset
848 // FIXME: copied verbatim from ZGC, duplicated code.
849 static bool aarch64_test_and_branch_reachable(int branch_offset, int target_offset) {
850 assert(branch_offset >= 0, "branch to stub offsets must be positive");
851 assert(target_offset >= 0, "offset in stubs section must be positive");
852 assert(target_offset >= branch_offset, "forward branches only, branch_offset -> target_offset");
853
854 const int test_and_branch_delta_limit = 32 * K;
855
856 const int test_and_branch_to_trampoline_delta = target_offset - branch_offset;
857
858 return test_and_branch_to_trampoline_delta < test_and_branch_delta_limit;
859 }
860
861 ShenandoahBarrierStubC2::ShenandoahBarrierStubC2(const MachNode* node, Register obj, Address addr, bool narrow, bool do_load, int offset) :
862 BarrierStubC2(node),
863 _obj(obj),
864 _addr(addr),
865 _do_load(do_load),
866 _narrow(narrow),
867 _maybe_null(!src_not_null(node)),
868 _needs_load_ref_barrier(needs_load_ref_barrier(node)),
869 _needs_load_ref_weak_barrier(needs_load_ref_barrier_weak(node)),
870 _needs_keep_alive_barrier(needs_keep_alive_barrier(node)),
871 _fastpath_branch_offset(offset),
872 _test_and_branch_reachable(),
873 _skip_trampoline(),
874 _test_and_branch_reachable_entry() {
875
876 assert(!ShenandoahSkipBarriers, "Do not touch stubs when disabled");
877 assert(!_narrow || is_heap_access(node), "Only heap accesses can be narrow");
878
879 // If we are in scratch emit mode we assume worse case by leaving
880 // _test_and_branch_reachable false.
881 PhaseOutput* const output = Compile::current()->output();
882 if (output->in_scratch_emit_size()) {
883 return;
884 }
885
886 // Assume that each trampoline is one single instruction and that the stubs
887 // will follow immediatelly after the _code section. Therefore, we are
888 // checking if the distance between the fastpath branch and the
889 // trampoline/entry of the current Stub is less than 32K.
890 const int code_size = output->buffer_sizing_data()->_code;
891 const int trampoline_offset = trampoline_stubs_count() * NativeInstruction::instruction_size;
892 _test_and_branch_reachable = aarch64_test_and_branch_reachable(_fastpath_branch_offset, code_size + trampoline_offset);
893 if (_test_and_branch_reachable) {
894 inc_trampoline_stubs_count();
895 }
896 }
897
898 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
899 // If we reach here with _skip_trampoline set it means that earlier we
900 // emitted a trampoline to this stub and now we need to emit the actual stub.
901 if (_skip_trampoline) {
902 emit_code_actual(masm);
903 return;
904 }
905 _skip_trampoline = true;
906
907 // The fastpath executes two branch instructions to reach this stub, let's
908 // just emit the stub here and not add a third one.
909 if (!_test_and_branch_reachable) {
910 // By registering the stub again, after setting _skip_trampoline, we'll
911 // effectivelly cause the stub to be emitted the next time ::emit_code is
912 // called.
913 ShenandoahBarrierStubC2::register_stub(this);
914 return;
915 }
916
917 // This is entry point when coming from fastpath, IFF it's able to reach here
918 // with a test and branch instruction, otherwise the entry is
919 // ShenandoahBarrierStubC2::entry();
920 const int target_offset = __ offset();
921 __ bind(_test_and_branch_reachable_entry);
922
923 #ifdef ASSERT
924 // Current assumption is that the barrier stubs are the first stubs emitted
925 // after the actual code
926 PhaseOutput* const output = Compile::current()->output();
927 assert(stubs_start_offset() <= output->buffer_sizing_data()->_code, "stubs are assumed to be emitted directly after code and code_size is a hard limit on where it can start");
928 assert(aarch64_test_and_branch_reachable(_fastpath_branch_offset, target_offset), "trampoline should be reachable");
929 #endif
930
931 // Next fastpath branch's offset is unknown, but it's > current _fastpath_branch_offset
932 const int next_branch_offset = _fastpath_branch_offset + NativeInstruction::instruction_size;
933
934 // If emitting the current stub directly does not interfere with emission of
935 // the next potential trampoline then do it to avoid executing additional
936 // branch when coming from fastpath.
937 if (aarch64_test_and_branch_reachable(next_branch_offset, target_offset + get_stub_size())) {
938 emit_code_actual(masm);
939 } else {
940 __ b(*BarrierStubC2::entry());
941 // By registering the stub again, after setting _skip_trampoline to true,
942 // we'll effectivelly cause the stub to be emitted the next time
943 // ::emit_code is called.
944 ShenandoahBarrierStubC2::register_stub(this);
945 }
946 }
947
948 Label* ShenandoahBarrierStubC2::entry() {
949 if (_test_and_branch_reachable) {
950 return &_test_and_branch_reachable_entry;
951 }
952 return BarrierStubC2::entry();
953 }
954
955 int ShenandoahBarrierStubC2::get_stub_size() {
956 PhaseOutput* const output = Compile::current()->output();
957 assert(!output->in_scratch_emit_size(), "only used when emitting stubs");
958 BufferBlob* const blob = output->scratch_buffer_blob();
959 CodeBuffer cb(blob->content_begin(), (address)output->scratch_locs_memory() - blob->content_begin());
960 MacroAssembler masm(&cb);
961 output->set_in_scratch_emit_size(true);
962 emit_code_actual(masm);
963 output->set_in_scratch_emit_size(false);
964 return cb.insts_size();
965 }
966
967 void ShenandoahBarrierStubC2::emit_code_actual(MacroAssembler& masm) {
968 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
969
970 // Stub entry
971 if (!Compile::current()->output()->in_scratch_emit_size()) {
972 __ bind(*BarrierStubC2::entry());
973 }
974
975 // If we need to load ourselves, do it here.
976 if (_do_load) {
977 // This does the load and the decode if necessary
978 __ load_heap_oop(_obj, _addr, noreg, noreg, AS_RAW);
979 } else if (_narrow) {
980 // If object is narrow, we need to decode it first: barrier checks need full oops.
981 if (_maybe_null) {
982 __ decode_heap_oop(_obj);
983 } else {
984 __ decode_heap_oop_not_null(_obj);
985 }
986 }
987
988 if (_do_load || _maybe_null) {
989 __ cbz(_obj, *continuation());
990 }
991
992 keepalive(&masm, _obj, rscratch1, rscratch2);
993
994 lrb(&masm, _obj, _addr, noreg);
995
996 // If object is narrow, we need to encode it before exiting.
997 // For encoding, dst can only turn null if we are dealing with weak loads.
998 // Otherwise, we have already null-checked. We can skip all this if we performed
999 // the load ourselves, which means the value is not used by caller.
1000 if (_narrow && !_do_load) {
1001 if (_needs_load_ref_weak_barrier) {
1002 __ encode_heap_oop(_obj);
1003 } else {
1004 __ encode_heap_oop_not_null(_obj);
1005 }
1006 }
1007
1008 // Go back to fast path
1009 __ b(*continuation());
1010 }
1011
1012 #undef __
1013 #define __ masm->
1014
1015 void ShenandoahBarrierStubC2::keepalive(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2) {
1016 Address index(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
1017 Address buffer(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
1018 Label L_runtime;
1019 Label L_done;
1020
1021 // The node doesn't even need keepalive barrier, just don't check anything else
1022 if (!_needs_keep_alive_barrier) {
1023 return ;
1024 }
1025
1026 // If both LRB and KeepAlive barriers are required (rare), do a runtime check
1027 // for enabled barrier.
1028 if (_needs_load_ref_barrier) {
1029 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1030 __ ldrb(rscratch1, gcs_addr);
1031 __ tbz(rscratch1, ShenandoahHeap::MARKING_BITPOS, L_done);
1032 }
1033
1034 // If buffer is full, call into runtime.
1035 __ ldr(tmp1, index);
1036 __ cbz(tmp1, L_runtime);
1037
1038 // The buffer is not full, store value into it.
1039 __ sub(tmp1, tmp1, wordSize);
1040 __ str(tmp1, index);
1041 __ ldr(tmp2, buffer);
1042 __ str(obj, Address(tmp2, tmp1));
1043 __ b(L_done);
1044
1045 // Runtime call
1046 __ bind(L_runtime);
1047
1048 preserve(obj);
1049 {
1050 SaveLiveRegisters save_registers(masm, this);
1051 __ mov(c_rarg0, obj);
1052 __ mov(tmp1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre));
1053 __ blr(tmp1);
1054 }
1055
1056 __ bind(L_done);
1057 }
1058
1059 void ShenandoahBarrierStubC2::lrb(MacroAssembler* masm, Register obj, Address addr, Register tmp) {
1060 Label L_done;
1061
1062 // The node doesn't even need LRB barrier, just don't check anything else
1063 if (!_needs_load_ref_barrier) {
1064 return ;
1065 }
1066
1067 if ((_node->barrier_data() & ShenandoahBitStrong) != 0) {
1068 // If both LRB and KeepAlive barriers are required (rare), do a runtime
1069 // check for enabled barrier.
1070 if (_needs_keep_alive_barrier) {
1071 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1072 __ ldrb(rscratch1, gcs_addr);
1073 if (_needs_load_ref_weak_barrier) {
1074 __ orr(rscratch1, rscratch1, rscratch1, Assembler::LSR, ShenandoahHeap::WEAK_ROOTS_BITPOS);
1075 }
1076 __ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, L_done);
1077 }
1078
1079 // Weak/phantom loads always need to go to runtime. For strong refs we
1080 // check if the object in cset, if they are not, then we are done with LRB.
1081 __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
1082 __ lsr(rscratch1, obj, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1083 __ ldrb(rscratch2, Address(rscratch2, rscratch1));
1084 __ cbz(rscratch2, L_done);
1085 }
1086
1087 dont_preserve(obj);
1088 {
1089 SaveLiveRegisters save_registers(masm, this);
1090
1091 // Shuffle in the arguments. The end result should be:
1092 // c_rarg0 <-- obj
1093 // c_rarg1 <-- lea(addr)
1094 if (c_rarg0 == obj) {
1095 __ lea(c_rarg1, addr);
1096 } else if (c_rarg1 == obj) {
1097 // Set up arguments in reverse, and then flip them
1098 __ lea(c_rarg0, addr);
1099 // flip them
1100 __ mov(rscratch1, c_rarg0);
1101 __ mov(c_rarg0, c_rarg1);
1102 __ mov(c_rarg1, rscratch1);
1103 } else {
1104 assert_different_registers(c_rarg1, obj);
1105 __ lea(c_rarg1, addr);
1106 __ mov(c_rarg0, obj);
1107 }
1108
1109 // Get address of runtime LRB entry and call it
1110 __ mov(rscratch1, lrb_runtime_entry_addr());
1111 __ blr(rscratch1);
1112
1113 // If we loaded the object in the stub it means we don't need to return it
1114 // to fastpath, so no need to make this mov.
1115 if (!_do_load) {
1116 __ mov(obj, r0);
1117 }
1118 }
1119
1120 __ bind(L_done);
1121 }
1122
1123 #endif // COMPILER2
1124
1125 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
1126 Register start, Register count, Register scratch) {
1127 assert(ShenandoahCardBarrier, "Should have been checked by caller");
1128
1129 Label L_loop, L_done;
1130 const Register end = count;
1131
1132 // Zero count? Nothing to do.
1133 __ cbz(count, L_done);
1134
1135 // end = start + count << LogBytesPerHeapOop
1136 // last element address to make inclusive
1137 __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop)));
1138 __ sub(end, end, BytesPerHeapOop);
1139 __ lsr(start, start, CardTable::card_shift());
1140 __ lsr(end, end, CardTable::card_shift());
1141
1142 // number of bytes to copy
1143 __ sub(count, end, start);
1144
|