24 */
25
26 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
27 #include "gc/shenandoah/mode/shenandoahMode.hpp"
28 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
30 #include "gc/shenandoah/shenandoahForwarding.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
33 #include "gc/shenandoah/shenandoahRuntime.hpp"
34 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
35 #include "interpreter/interp_masm.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "runtime/javaThread.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #ifdef COMPILER1
40 #include "c1/c1_LIRAssembler.hpp"
41 #include "c1/c1_MacroAssembler.hpp"
42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
43 #endif
44
45 #define __ masm->
46
47 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
48 Register src, Register dst, Register count, RegSet saved_regs) {
49 if (is_oop) {
50 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
51 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
52
53 Label done;
54
55 // Avoid calling runtime if count == 0
56 __ cbz(count, done);
57
58 // Is GC active?
59 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
60 __ ldrb(rscratch1, gc_state);
61 if (ShenandoahSATBBarrier && dest_uninitialized) {
62 __ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
63 } else {
64 __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING);
65 __ tst(rscratch1, rscratch2);
66 __ br(Assembler::EQ, done);
585 if (is_cae) {
586 // We're falling through to done to indicate success. Success
587 // with is_cae is denoted by returning the value of expected as
588 // result.
589 __ mov(tmp2, expected);
590 }
591
592 __ bind(done);
593 // At entry to done, the Z (EQ) flag is on iff if the CAS
594 // operation was successful. Additionally, if is_cae, tmp2 holds
595 // the value most recently fetched from addr. In this case, success
596 // is denoted by tmp2 matching expected.
597
598 if (is_cae) {
599 __ mov(result, tmp2);
600 } else {
601 __ cset(result, Assembler::EQ);
602 }
603 }
604
605 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
606 Register start, Register count, Register scratch) {
607 assert(ShenandoahCardBarrier, "Should have been checked by caller");
608
609 Label L_loop, L_done;
610 const Register end = count;
611
612 // Zero count? Nothing to do.
613 __ cbz(count, L_done);
614
615 // end = start + count << LogBytesPerHeapOop
616 // last element address to make inclusive
617 __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop)));
618 __ sub(end, end, BytesPerHeapOop);
619 __ lsr(start, start, CardTable::card_shift());
620 __ lsr(end, end, CardTable::card_shift());
621
622 // number of bytes to copy
623 __ sub(count, end, start);
624
784 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow));
785 } else {
786 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
787 }
788 } else {
789 assert(is_phantom, "only remaining strength");
790 assert(is_native, "phantom must only be called off-heap");
791 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom));
792 }
793 __ blr(lr);
794 __ mov(rscratch1, r0);
795 __ pop_call_clobbered_registers();
796 __ mov(r0, rscratch1);
797
798 __ epilogue();
799 }
800
801 #undef __
802
803 #endif // COMPILER1
|
24 */
25
26 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
27 #include "gc/shenandoah/mode/shenandoahMode.hpp"
28 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
30 #include "gc/shenandoah/shenandoahForwarding.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
33 #include "gc/shenandoah/shenandoahRuntime.hpp"
34 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
35 #include "interpreter/interp_masm.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "runtime/javaThread.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #ifdef COMPILER1
40 #include "c1/c1_LIRAssembler.hpp"
41 #include "c1/c1_MacroAssembler.hpp"
42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
43 #endif
44 #ifdef COMPILER2
45 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
46 #endif
47
48 #define __ masm->
49
50 #ifdef PRODUCT
51 #define BLOCK_COMMENT(str) /* nothing */
52 #else
53 #define BLOCK_COMMENT(str) __ block_comment(str)
54 #endif
55
56 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
57 Register src, Register dst, Register count, RegSet saved_regs) {
58 if (is_oop) {
59 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
60 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
61
62 Label done;
63
64 // Avoid calling runtime if count == 0
65 __ cbz(count, done);
66
67 // Is GC active?
68 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
69 __ ldrb(rscratch1, gc_state);
70 if (ShenandoahSATBBarrier && dest_uninitialized) {
71 __ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
72 } else {
73 __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING);
74 __ tst(rscratch1, rscratch2);
75 __ br(Assembler::EQ, done);
594 if (is_cae) {
595 // We're falling through to done to indicate success. Success
596 // with is_cae is denoted by returning the value of expected as
597 // result.
598 __ mov(tmp2, expected);
599 }
600
601 __ bind(done);
602 // At entry to done, the Z (EQ) flag is on iff if the CAS
603 // operation was successful. Additionally, if is_cae, tmp2 holds
604 // the value most recently fetched from addr. In this case, success
605 // is denoted by tmp2 matching expected.
606
607 if (is_cae) {
608 __ mov(result, tmp2);
609 } else {
610 __ cset(result, Assembler::EQ);
611 }
612 }
613
614 #ifdef COMPILER2
615 void ShenandoahBarrierSetAssembler::load_ref_barrier_c2(const MachNode* node, MacroAssembler* masm, Register obj, Register addr, bool narrow, bool maybe_null, Register gc_state) {
616 assert_different_registers(obj, addr);
617 BLOCK_COMMENT("load_ref_barrier_c2 {");
618 if (!ShenandoahLoadRefBarrierStubC2::needs_barrier(node)) {
619 return;
620 }
621 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
622 ShenandoahLoadRefBarrierStubC2* const stub = ShenandoahLoadRefBarrierStubC2::create(node, obj, addr, gc_state, noreg, noreg, narrow);
623
624 // Don't preserve the obj across the runtime call, we override it from the
625 // return value anyway.
626 stub->dont_preserve(obj);
627 stub->dont_preserve(gc_state);
628
629 // Check if GC marking is in progress or we are handling a weak reference,
630 // otherwise we don't have to do anything. The code below was optimized to
631 // use less registers and instructions as possible at the expense of always
632 // having a branch instruction. The reason why we use this particular branch
633 // scheme is because the stub entry may be too far for the tbnz to jump to.
634 bool is_strong = (node->barrier_data() & ShenandoahBarrierStrong) != 0;
635 if (is_strong) {
636 __ tbz(gc_state, ShenandoahHeap::HAS_FORWARDED_BITPOS, *stub->continuation());
637 __ b(*stub->entry());
638 } else {
639 static_assert(ShenandoahHeap::HAS_FORWARDED_BITPOS == 0, "Relied on in LRB check below.");
640 __ orr(gc_state, gc_state, gc_state, Assembler::LSR, ShenandoahHeap::WEAK_ROOTS_BITPOS);
641 __ tbz(gc_state, ShenandoahHeap::HAS_FORWARDED_BITPOS, *stub->continuation());
642 __ b(*stub->entry());
643 }
644
645 __ bind(*stub->continuation());
646 BLOCK_COMMENT("} load_ref_barrier_c2");
647 }
648
649 /**
650 * The logic implemented here relies on certain flags being on specific
651 * positions of the GCState. Also note that all pointer values in register are
652 * guaranteed to be 'to-space' addresses. The algorithm is as follows. If the
653 * CAS succeed:
654 * - 'res' will be set to 1.
655 * - We need to check SATB flag (index 1 of GCState). If the flag is active
656 * need to store 'oldval' in the buffer.
657 * - We wrote 'newval' to 'addr', therefore we need to mark the corresponding
658 * card in the card table for 'addr' as dirty.
659 * If the CAS failed:
660 * - 'res' will be set to 0.
661 * - If the GCState FORWARDING bit (index 0 of GCState) is set we'll need to
662 * retry the CAS, because the failure may be because the value in 'addr' is
663 * the (outdated) 'from-space' version of 'expected'. The retry is done in a
664 * stub. If the retry succeed then we need to do the steps described above
665 * too for CAS succeed too.
666 * - If FORWARDING bit is clear there is nothing else to do.
667 */
668 void ShenandoahBarrierSetAssembler::cas_c2(const MachNode* node, MacroAssembler* masm, Register res, Register addr, Register oldval, Register newval, Register tmp1, Register tmp2, bool narrow, bool acquire, bool release, bool weak) {
669 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
670 Assembler::operand_size op_size = UseCompressedOops ? Assembler::word : Assembler::xword;
671
672 // Assuming just for now that we need both barriers
673 NewShenandoahCASBarrierSlowStubC2* const cmpx = NewShenandoahCASBarrierSlowStubC2::create(node, addr, oldval, newval, res, tmp1, tmp2, narrow, false, acquire, release, weak);
674
675 // Issue cmpxchg first
676 __ cmpxchg(addr, oldval, newval, op_size, acquire, release, weak, tmp2);
677
678 // Load GC state in tmp1
679 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
680 __ ldrb(tmp1, gcs_addr);
681
682 // First CAS attempt. If successful, then we are done.
683 // EQ flag set iff success. 'res' holds value fetched.
684 __ cset(res, Assembler::EQ);
685
686 // Shift GCState right according to result of previous CAS. Luckily, the
687 // boolean result of the CAS also matches the index of the bit that we need
688 // to test later on.
689 __ lsrv(tmp1, tmp1, res);
690
691 // Test bit '0' of tmp1, which at this point will be FORWARDING bit if CAS
692 // failed, or SATB bit if CAS succeded.
693 __ tbnz(tmp1, 0x0, *cmpx->entry());
694
695 // Would be nice to fold this in the comparison above, but how?
696 // Skip Card Table dirtying if CAS failed.
697 __ cbz(res, *cmpx->continuation());
698 card_barrier_c2(node, masm, addr, tmp1);
699
700 __ bind(*cmpx->continuation());
701 }
702
703 void ShenandoahBarrierSetAssembler::cae_c2(const MachNode* node, MacroAssembler* masm, Register res, Register addr, Register oldval, Register newval, Register tmp1, Register tmp2, bool maybe_null, bool narrow, bool acquire, bool release, bool weak) {
704 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
705 Assembler::operand_size op_size = UseCompressedOops ? Assembler::word : Assembler::xword;
706
707 Label succeded;
708
709 // Assuming just for now that we need both barriers
710 ShenandoahCASBarrierSlowStubC2* const cmpx = ShenandoahCASBarrierSlowStubC2::create(node, addr, oldval, newval, res, tmp1, tmp2, false, acquire, release, weak);
711
712 // Load GC state in tmp1
713 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
714 __ ldrb(tmp1, gcs_addr);
715
716 // Fast-path: Try to CAS optimistically. If successful, then we are done.
717 // EQ flag set iff success. 'res' holds value fetched.
718 __ cmpxchg(addr, oldval, newval, op_size, acquire, release, weak, res);
719 __ cset(res, Assembler::EQ);
720 __ lsrv(tmp1, tmp1, res);
721 __ tbnz(tmp1, 0x0, *cmpx->entry());
722
723 __ tbz(res, 0x0, *cmpx->continuation());
724 card_barrier_c2(node, masm, addr, tmp1);
725
726 __ bind(*cmpx->continuation());
727 }
728
729
730
731 void ShenandoahBarrierSetAssembler::store_c2(const MachNode* node, MacroAssembler* masm,
732 Register dst, bool dst_narrow,
733 Register src, bool src_narrow,
734 Register tmp, Register pre_val,
735 bool is_volatile) {
736
737 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
738 __ ldrb(tmp, gcs_addr);
739
740 satb_barrier_c2(node, masm, dst, pre_val, tmp, src_narrow);
741
742 card_barrier_c2(node, masm, dst, tmp);
743
744 // Need to encode into tmp, because we cannot clobber src.
745 // TODO: Maybe there is a matcher way to test that src is unused after this?
746 if (dst_narrow && !src_narrow) {
747 __ mov(tmp, src);
748 if (ShenandoahStoreBarrierStubC2::src_not_null(node)) {
749 __ encode_heap_oop_not_null(tmp);
750 } else {
751 __ encode_heap_oop(tmp);
752 }
753 src = tmp;
754 }
755
756 // Do the actual store
757 if (dst_narrow) {
758 if (is_volatile) {
759 __ stlrw(src, dst);
760 } else {
761 __ strw(src, dst);
762 }
763 } else {
764 if (is_volatile) {
765 __ stlr(src, dst);
766 } else {
767 __ str(src, dst);
768 }
769 }
770 }
771
772 void ShenandoahBarrierSetAssembler::satb_barrier_c2(const MachNode* node, MacroAssembler* masm, Register addr, Register pre_val,
773 Register gc_state, bool encoded_preval) {
774 BLOCK_COMMENT("satb_barrier_c2 {");
775 assert_different_registers(addr, pre_val, rscratch1, rscratch2);
776 if (!ShenandoahSATBBarrierStubC2::needs_barrier(node)) {
777 return;
778 }
779 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
780 ShenandoahSATBBarrierStubC2* const stub = ShenandoahSATBBarrierStubC2::create(node, addr, pre_val, gc_state, encoded_preval);
781
782 // Check if GC marking is in progress, otherwise we don't have to do
783 // anything.
784 __ tstw(gc_state, ShenandoahHeap::MARKING);
785 __ br(Assembler::NE, *stub->entry());
786 __ bind(*stub->continuation());
787 BLOCK_COMMENT("} satb_barrier_c2");
788 }
789
790 void ShenandoahBarrierSetAssembler::card_barrier_c2(const MachNode* node, MacroAssembler* masm, Register addr, Register tmp) {
791 if (!ShenandoahCardBarrier ||
792 (node->barrier_data() & ShenandoahBarrierCardMark) == 0) {
793 return;
794 }
795
796 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
797 __ lsr(tmp, addr, CardTable::card_shift());
798
799 assert(CardTable::dirty_card_val() == 0, "must be");
800
801 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
802 __ ldr(rscratch1, curr_ct_holder_addr);
803
804 if (UseCondCardMark) {
805 Label L_already_dirty;
806 __ ldrb(rscratch2, Address(tmp, rscratch1));
807 __ cbz(rscratch2, L_already_dirty);
808 __ strb(zr, Address(tmp, rscratch1));
809 __ bind(L_already_dirty);
810 } else {
811 __ strb(zr, Address(tmp, rscratch1));
812 }
813 }
814
815 void ShenandoahBarrierSetAssembler::cmpxchg_oop_c2(const MachNode* node,
816 MacroAssembler* masm,
817 Register addr, Register oldval,
818 Register newval, Register res,
819 Register gc_state, Register tmp,
820 bool acquire, bool release, bool weak, bool exchange) {
821 assert(res != noreg, "need result register");
822 assert_different_registers(oldval, addr, res, gc_state, tmp);
823 assert_different_registers(newval, addr, res, gc_state, tmp);
824
825 // Fast-path: Try to CAS optimistically. If successful, then we are done.
826 // EQ flag set iff success. 'tmp' holds value fetched.
827 Assembler::operand_size size = UseCompressedOops ? Assembler::word : Assembler::xword;
828 __ cmpxchg(addr, oldval, newval, size, acquire, release, weak, tmp);
829
830 // If we need a boolean result out of CAS, set the flag appropriately. This
831 // would be the final result if we do not go slow.
832 if (!exchange) {
833 __ cset(res, Assembler::EQ);
834 } else {
835 __ mov(res, tmp);
836 }
837
838 if (ShenandoahCASBarrier) {
839 ShenandoahCASBarrierSlowStubC2* const slow_stub =
840 ShenandoahCASBarrierSlowStubC2::create(node, addr, oldval, newval, res, gc_state, tmp, exchange, acquire, release, weak);
841
842 slow_stub->preserve(gc_state); // this really need to be preserved as we
843 // try to use it in subsequent barriers
844
845 slow_stub->dont_preserve(res); // set at the end, no need to save
846 slow_stub->dont_preserve(oldval); // saved explicitly
847 slow_stub->dont_preserve(tmp); // temp, no need to save
848
849 // On success, we do not need any additional handling.
850 __ br(Assembler::EQ, *slow_stub->continuation());
851
852 // If GC is in progress, it is likely we need additional handling for false negatives.
853 __ tbz(gc_state, ShenandoahHeap::HAS_FORWARDED_BITPOS, *slow_stub->continuation());
854 __ b(*slow_stub->entry());
855
856 // Slow stub re-enters with result set correctly.
857 __ bind(*slow_stub->continuation());
858 }
859 }
860
861 #undef __
862 #define __ masm.
863
864 void ShenandoahLoadRefBarrierStubC2::emit_code(MacroAssembler& masm) {
865 BLOCK_COMMENT("ShenandoahLoadRefBarrierStubC2::emit_code {");
866 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
867 __ bind(*entry());
868 Register obj = _obj;
869 if (_narrow) {
870 __ decode_heap_oop(_tmp1, _obj);
871 obj = _tmp1;
872 }
873 // Weak/phantom loads always need to go to runtime.
874 if ((_node->barrier_data() & ShenandoahBarrierStrong) != 0) {
875 // Check for object in cset.
876 __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
877 __ lsr(rscratch1, obj, ShenandoahHeapRegion::region_size_bytes_shift_jint());
878 __ ldrb(rscratch2, Address(rscratch2, rscratch1));
879 __ cbz(rscratch2, *continuation());
880 }
881 {
882 SaveLiveRegisters save_registers(&masm, this);
883 if (c_rarg0 != obj) {
884 if (c_rarg0 == _addr) {
885 __ mov(rscratch1, _addr);
886 _addr = rscratch1;
887 }
888 __ mov(c_rarg0, obj);
889 }
890 __ mov(c_rarg1, _addr);
891
892 if (_narrow) {
893 if ((_node->barrier_data() & ShenandoahBarrierStrong) != 0) {
894 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow));
895 } else if ((_node->barrier_data() & ShenandoahBarrierWeak) != 0) {
896 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow));
897 } else if ((_node->barrier_data() & ShenandoahBarrierPhantom) != 0) {
898 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom_narrow));
899 }
900 } else {
901 if ((_node->barrier_data() & ShenandoahBarrierStrong) != 0) {
902 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong));
903 } else if ((_node->barrier_data() & ShenandoahBarrierWeak) != 0) {
904 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
905 } else if ((_node->barrier_data() & ShenandoahBarrierPhantom) != 0) {
906 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom));
907 }
908 }
909 __ blr(rscratch1);
910 __ mov(_obj, r0);
911 }
912 if (_narrow) {
913 __ encode_heap_oop(_obj);
914 }
915 __ b(*continuation());
916 BLOCK_COMMENT("} ShenandoahLoadRefBarrierStubC2::emit_code");
917 }
918
919 void ShenandoahSATBBarrierStubC2::emit_code(MacroAssembler& masm) {
920 BLOCK_COMMENT("ShenandoahSATBBarrierStubC2::emit_code {");
921 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
922 __ bind(*entry());
923
924 // The tmp register that we receive is usually a register holding the
925 // "gc_state" which may be required by subsequent memory operations in their
926 // fastpath.
927 RegSet saved = RegSet::of(_tmp);
928 __ push(saved, sp);
929
930 // Do we need to load the previous value?
931 if (_addr != noreg) {
932 __ load_heap_oop(_tmp, Address(_addr, 0), noreg, noreg, AS_RAW);
933 } else {
934 if (_encoded_preval) {
935 __ decode_heap_oop(_tmp, _preval);
936 } else {
937 _tmp = _preval;
938 }
939 }
940
941 Address index(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
942 Address buffer(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
943 Label runtime;
944 __ ldr(rscratch1, index);
945 // If buffer is full, call into runtime.
946 __ cbz(rscratch1, runtime);
947
948 // The buffer is not full, store value into it.
949 __ sub(rscratch1, rscratch1, wordSize);
950 __ str(rscratch1, index);
951 __ ldr(rscratch2, buffer);
952 __ str(_tmp, Address(rscratch2, rscratch1));
953 __ pop(saved, sp);
954 __ b(*continuation());
955
956 // Runtime call
957 __ bind(runtime);
958 {
959 SaveLiveRegisters save_registers(&masm, this);
960 __ mov(c_rarg0, _tmp);
961 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre_c2));
962 __ blr(rscratch1);
963 }
964 __ pop(saved, sp);
965 __ b(*continuation());
966 BLOCK_COMMENT("} ShenandoahSATBBarrierStubC2::emit_code");
967 }
968
969 void ShenandoahLoadBarrierStubC2::emit_code(MacroAssembler& masm) {
970 Unimplemented();
971 }
972
973 void ShenandoahStoreBarrierStubC2::emit_code(MacroAssembler& masm) {
974 Unimplemented();
975 }
976
977 void ShenandoahCASBarrierSlowStubC2::emit_code(MacroAssembler& masm) {
978 __ bind(*entry());
979
980 // CAS has failed because the value held at addr does not match expected.
981 // This may be a false negative because the version in memory might be
982 // the from-space version of the same object we currently hold to-space
983 // reference for.
984 //
985 // To resolve this, we need to pass the location through the LRB fixup,
986 // this will make sure that the location has only to-space pointers.
987 // To avoid calling into runtime often, we cset-check the object first.
988 // We can inline most of the work here, but there is little point,
989 // as CAS failures over cset locations must be rare. This fast-slow split
990 // matches what we do for normal LRB.
991
992 // Non-strong references should always go to runtime. We do not expect
993 // CASes over non-strong locations.
994 assert((_node->barrier_data() & ShenandoahBarrierStrong) != 0, "Only strong references for CASes");
995
996 Label L_final;
997
998 // (Compressed) failure witness is in _tmp2.
999 // Unpack it and check if it is in collection set.
1000 // We need to backup the compressed version to use in the LRB.
1001 __ mov(_result, _tmp2);
1002 if (UseCompressedOops) {
1003 __ decode_heap_oop(_tmp2);
1004 }
1005
1006 __ mov(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
1007 __ lsr(_tmp2, _tmp2, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1008 __ ldrb(_tmp1, Address(_tmp1, _tmp2));
1009 __ cbz(_tmp1, L_final);
1010
1011 {
1012 SaveLiveRegisters save_registers(&masm, this);
1013 // Load up failure witness again.
1014 __ mov(c_rarg0, _result);
1015 if (UseCompressedOops) {
1016 __ decode_heap_oop(c_rarg0);
1017 }
1018 __ mov(c_rarg1, _addr_reg);
1019
1020 if (UseCompressedOops) {
1021 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow), 2);
1022 } else {
1023 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong), 2);
1024 }
1025 // We have called LRB to fix up the heap location. We do not care about its
1026 // result, as we will just try to CAS the location again.
1027 }
1028
1029 __ bind(L_final);
1030
1031 Assembler::operand_size size = UseCompressedOops ? Assembler::word : Assembler::xword;
1032 __ cmpxchg(_addr_reg, _expected, _new_val, size, _acquire, _release, _weak, _result);
1033
1034 if (!_cae) {
1035 __ cset(_result, Assembler::EQ);
1036 }
1037 __ b(*continuation());
1038 }
1039
1040 void NewShenandoahCASBarrierSlowStubC2::emit_code(MacroAssembler& masm) {
1041 __ bind(*entry());
1042
1043 // Non-strong references should always go to runtime. We do not expect
1044 // CASes over non-strong locations.
1045 assert((_node->barrier_data() & ShenandoahBarrierStrong) != 0, "Only strong references for CASes");
1046
1047 Label L_final;
1048 Label L_succeded;
1049
1050 // check if first CAS succeded, if it did we just need to write to SATB
1051 __ cbnz(_result, L_succeded);
1052
1053 // First CAS attempt did not succed. Execute LRB on 'addr' and retry CAS.
1054
1055 // (Compressed) failure witness is in _tmp2.
1056 // Unpack it and check if it is in collection set.
1057 // We need to backup the compressed version to use in the LRB.
1058 __ mov(_result, _tmp2);
1059 if (UseCompressedOops) {
1060 __ decode_heap_oop(_tmp2);
1061 }
1062
1063 __ mov(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
1064 __ lsr(_tmp2, _tmp2, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1065 __ ldrb(_tmp1, Address(_tmp1, _tmp2));
1066 __ cbz(_tmp1, L_final);
1067
1068 {
1069 SaveLiveRegisters save_registers(&masm, this);
1070 // Load up failure witness again.
1071 __ mov(c_rarg0, _result);
1072 if (UseCompressedOops) {
1073 __ decode_heap_oop(c_rarg0);
1074 }
1075 __ mov(c_rarg1, _addr_reg);
1076
1077 if (UseCompressedOops) {
1078 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow), 2);
1079 } else {
1080 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong), 2);
1081 }
1082 // We have called LRB to fix up the heap location. We do not care about its
1083 // result, as we will just try to CAS the location again.
1084 }
1085
1086 __ bind(L_final);
1087
1088 Assembler::operand_size size = UseCompressedOops ? Assembler::word : Assembler::xword;
1089 __ cmpxchg(_addr_reg, _expected, _new_val, size, _acquire, _release, _weak, _result);
1090
1091 __ cset(_result, Assembler::EQ);
1092
1093 // If the retry did not succeed skip SATB
1094 __ cbz(_result, *continuation());
1095
1096
1097
1098 __ bind(L_succeded);
1099 if (_narrow) {
1100 __ decode_heap_oop(_tmp1, _expected);
1101 } else {
1102 _tmp1 = _expected;
1103 }
1104
1105 Address index(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
1106 Address buffer(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
1107 Label runtime;
1108 __ ldr(rscratch1, index);
1109 // If buffer is full, call into runtime.
1110 __ cbz(rscratch1, runtime);
1111
1112 // The buffer is not full, store value into it.
1113 __ sub(rscratch1, rscratch1, wordSize);
1114 __ str(rscratch1, index);
1115 __ ldr(rscratch2, buffer);
1116 __ str(_tmp1, Address(rscratch2, rscratch1));
1117 __ b(*continuation());
1118
1119 // Runtime call
1120 __ bind(runtime);
1121 {
1122 SaveLiveRegisters save_registers(&masm, this);
1123 __ mov(c_rarg0, _tmp1);
1124 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre_c2));
1125 __ blr(rscratch1);
1126 }
1127
1128
1129 __ b(*continuation());
1130 }
1131 #undef __
1132 #define __ masm->
1133 #endif // COMPILER2
1134
1135 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
1136 Register start, Register count, Register scratch) {
1137 assert(ShenandoahCardBarrier, "Should have been checked by caller");
1138
1139 Label L_loop, L_done;
1140 const Register end = count;
1141
1142 // Zero count? Nothing to do.
1143 __ cbz(count, L_done);
1144
1145 // end = start + count << LogBytesPerHeapOop
1146 // last element address to make inclusive
1147 __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop)));
1148 __ sub(end, end, BytesPerHeapOop);
1149 __ lsr(start, start, CardTable::card_shift());
1150 __ lsr(end, end, CardTable::card_shift());
1151
1152 // number of bytes to copy
1153 __ sub(count, end, start);
1154
1314 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow));
1315 } else {
1316 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
1317 }
1318 } else {
1319 assert(is_phantom, "only remaining strength");
1320 assert(is_native, "phantom must only be called off-heap");
1321 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom));
1322 }
1323 __ blr(lr);
1324 __ mov(rscratch1, r0);
1325 __ pop_call_clobbered_registers();
1326 __ mov(r0, rscratch1);
1327
1328 __ epilogue();
1329 }
1330
1331 #undef __
1332
1333 #endif // COMPILER1
1334
1335 //void ShenandoahBarrierSetAssembler::cas_c2(const MachNode* node, MacroAssembler* masm, Register res, Register addr, Register oldval, Register newval, Register tmp1, Register tmp2, bool narrow, bool acquire, bool release, bool weak) {
1336 // Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
1337 // Assembler::operand_size op_size = UseCompressedOops ? Assembler::word : Assembler::xword;
1338 //
1339 // Label succeded;
1340 //
1341 // ShenandoahSATBBarrierStubC2* const satb = ShenandoahSATBBarrierStubC2::create(node, addr, oldval, tmp1, narrow);
1342 //
1343 // // Assuming just for now that we need both barriers
1344 // ShenandoahCASBarrierSlowStubC2* const cmpx = ShenandoahCASBarrierSlowStubC2::create(node, addr, oldval, newval, res, tmp1, tmp2, false, acquire, release, weak);
1345 //
1346 // // Load GC state in tmp1
1347 // Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1348 // __ ldrb(tmp1, gcs_addr);
1349 //
1350 // // Fast-path: Try to CAS optimistically. If successful, then we are done.
1351 // // EQ flag set iff success. 'res' holds value fetched.
1352 // __ cmpxchg(addr, oldval, newval, op_size, acquire, release, weak, res);
1353 // __ cset(res, Assembler::EQ);
1354 // __ br(Assembler::EQ, succeded);
1355 //
1356 // __ tbnz(tmp1, ShenandoahHeap::HAS_FORWARDED_BITPOS, *cmpx->entry());
1357 // __ b(*cmpx->continuation());
1358 //
1359 // __ bind(succeded);
1360 // __ tbnz(tmp1, ShenandoahHeap::MARKING_BITPOS, *satb->entry());
1361 // __ bind(*satb->continuation());
1362 // card_barrier_c2(node, masm, addr, tmp1);
1363 //
1364 // __ bind(*cmpx->continuation());
1365 //}
|