24 */
25
26 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
27 #include "gc/shenandoah/mode/shenandoahMode.hpp"
28 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
30 #include "gc/shenandoah/shenandoahForwarding.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
33 #include "gc/shenandoah/shenandoahRuntime.hpp"
34 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
35 #include "interpreter/interpreter.hpp"
36 #include "runtime/javaThread.hpp"
37 #include "runtime/sharedRuntime.hpp"
38 #include "utilities/macros.hpp"
39 #ifdef COMPILER1
40 #include "c1/c1_LIRAssembler.hpp"
41 #include "c1/c1_MacroAssembler.hpp"
42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
43 #endif
44
45 #define __ masm->
46
47 static void save_machine_state(MacroAssembler* masm, bool handle_gpr, bool handle_fp) {
48 if (handle_gpr) {
49 __ push_IU_state();
50 }
51
52 if (handle_fp) {
53 // Some paths can be reached from the c2i adapter with live fp arguments in registers.
54 assert(Argument::n_float_register_parameters_j == 8, "8 fp registers to save at java call");
55
56 const int xmm_size = wordSize * 2;
57 __ subptr(rsp, xmm_size * 8);
58 __ movdbl(Address(rsp, xmm_size * 0), xmm0);
59 __ movdbl(Address(rsp, xmm_size * 1), xmm1);
60 __ movdbl(Address(rsp, xmm_size * 2), xmm2);
61 __ movdbl(Address(rsp, xmm_size * 3), xmm3);
62 __ movdbl(Address(rsp, xmm_size * 4), xmm4);
63 __ movdbl(Address(rsp, xmm_size * 5), xmm5);
644 if (UseCompressedOops) {
645 __ lock();
646 __ cmpxchgl(newval, addr);
647 } else {
648 __ lock();
649 __ cmpxchgptr(newval, addr);
650 }
651 __ jcc(Assembler::equal, L_success);
652
653 // Step 2. CAS had failed. This may be a false negative.
654 //
655 // The trouble comes when we compare the to-space pointer with the from-space
656 // pointer to the same object. To resolve this, it will suffice to resolve
657 // the value from memory -- this will give both to-space pointers.
658 // If they mismatch, then it was a legitimate failure.
659 //
660 // Before reaching to resolve sequence, see if we can avoid the whole shebang
661 // with filters.
662
663 // Filter: when offending in-memory value is null, the failure is definitely legitimate
664 __ testptr(oldval, oldval);
665 __ jcc(Assembler::zero, L_failure);
666
667 // Filter: when heap is stable, the failure is definitely legitimate
668 const Register thread = r15_thread;
669 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
670 __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED);
671 __ jcc(Assembler::zero, L_failure);
672
673 if (UseCompressedOops) {
674 __ movl(tmp2, oldval);
675 __ decode_heap_oop(tmp2);
676 } else {
677 __ movptr(tmp2, oldval);
678 }
679
680 // Decode offending in-memory value.
681 // Test if-forwarded
682 __ testb(Address(tmp2, oopDesc::mark_offset_in_bytes()), markWord::marked_value);
683 __ jcc(Assembler::noParity, L_failure); // When odd number of bits, then not forwarded
684 __ jcc(Assembler::zero, L_failure); // When it is 00, then also not forwarded
746 // and promote the result. Note that we handle the flag from both the 1st and 2nd CAS.
747 // Otherwise, failure witness for CAE is in oldval on all paths, and we can return.
748
749 if (exchange) {
750 __ bind(L_failure);
751 __ bind(L_success);
752 } else {
753 assert(res != noreg, "need result register");
754
755 Label exit;
756 __ bind(L_failure);
757 __ xorptr(res, res);
758 __ jmpb(exit);
759
760 __ bind(L_success);
761 __ movptr(res, 1);
762 __ bind(exit);
763 }
764 }
765
766 #ifdef PRODUCT
767 #define BLOCK_COMMENT(str) /* nothing */
768 #else
769 #define BLOCK_COMMENT(str) __ block_comment(str)
770 #endif
771
772 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
773
774 #define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8)
775
776 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
777 Register addr, Register count,
778 Register tmp) {
779 assert(ShenandoahCardBarrier, "Should have been checked by caller");
780
781 Label L_loop, L_done;
782 const Register end = count;
783 assert_different_registers(addr, end);
784
785 // Zero count? Nothing to do.
|
24 */
25
26 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
27 #include "gc/shenandoah/mode/shenandoahMode.hpp"
28 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
30 #include "gc/shenandoah/shenandoahForwarding.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
33 #include "gc/shenandoah/shenandoahRuntime.hpp"
34 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
35 #include "interpreter/interpreter.hpp"
36 #include "runtime/javaThread.hpp"
37 #include "runtime/sharedRuntime.hpp"
38 #include "utilities/macros.hpp"
39 #ifdef COMPILER1
40 #include "c1/c1_LIRAssembler.hpp"
41 #include "c1/c1_MacroAssembler.hpp"
42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
43 #endif
44 #ifdef COMPILER2
45 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
46 #endif
47
48 #define __ masm->
49
50 static void save_machine_state(MacroAssembler* masm, bool handle_gpr, bool handle_fp) {
51 if (handle_gpr) {
52 __ push_IU_state();
53 }
54
55 if (handle_fp) {
56 // Some paths can be reached from the c2i adapter with live fp arguments in registers.
57 assert(Argument::n_float_register_parameters_j == 8, "8 fp registers to save at java call");
58
59 const int xmm_size = wordSize * 2;
60 __ subptr(rsp, xmm_size * 8);
61 __ movdbl(Address(rsp, xmm_size * 0), xmm0);
62 __ movdbl(Address(rsp, xmm_size * 1), xmm1);
63 __ movdbl(Address(rsp, xmm_size * 2), xmm2);
64 __ movdbl(Address(rsp, xmm_size * 3), xmm3);
65 __ movdbl(Address(rsp, xmm_size * 4), xmm4);
66 __ movdbl(Address(rsp, xmm_size * 5), xmm5);
647 if (UseCompressedOops) {
648 __ lock();
649 __ cmpxchgl(newval, addr);
650 } else {
651 __ lock();
652 __ cmpxchgptr(newval, addr);
653 }
654 __ jcc(Assembler::equal, L_success);
655
656 // Step 2. CAS had failed. This may be a false negative.
657 //
658 // The trouble comes when we compare the to-space pointer with the from-space
659 // pointer to the same object. To resolve this, it will suffice to resolve
660 // the value from memory -- this will give both to-space pointers.
661 // If they mismatch, then it was a legitimate failure.
662 //
663 // Before reaching to resolve sequence, see if we can avoid the whole shebang
664 // with filters.
665
666 // Filter: when offending in-memory value is null, the failure is definitely legitimate
667 if (UseCompressedOops) {
668 __ testl(oldval, oldval);
669 } else {
670 __ testptr(oldval, oldval);
671 }
672 __ jcc(Assembler::zero, L_failure);
673
674 // Filter: when heap is stable, the failure is definitely legitimate
675 const Register thread = r15_thread;
676 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
677 __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED);
678 __ jcc(Assembler::zero, L_failure);
679
680 if (UseCompressedOops) {
681 __ movl(tmp2, oldval);
682 __ decode_heap_oop(tmp2);
683 } else {
684 __ movptr(tmp2, oldval);
685 }
686
687 // Decode offending in-memory value.
688 // Test if-forwarded
689 __ testb(Address(tmp2, oopDesc::mark_offset_in_bytes()), markWord::marked_value);
690 __ jcc(Assembler::noParity, L_failure); // When odd number of bits, then not forwarded
691 __ jcc(Assembler::zero, L_failure); // When it is 00, then also not forwarded
753 // and promote the result. Note that we handle the flag from both the 1st and 2nd CAS.
754 // Otherwise, failure witness for CAE is in oldval on all paths, and we can return.
755
756 if (exchange) {
757 __ bind(L_failure);
758 __ bind(L_success);
759 } else {
760 assert(res != noreg, "need result register");
761
762 Label exit;
763 __ bind(L_failure);
764 __ xorptr(res, res);
765 __ jmpb(exit);
766
767 __ bind(L_success);
768 __ movptr(res, 1);
769 __ bind(exit);
770 }
771 }
772
773 #ifdef COMPILER2
774 void ShenandoahBarrierSetAssembler::gc_state_check_c2(MacroAssembler* masm, const char test_state, BarrierStubC2* slow_stub) {
775 const int size = 11;
776 if (ShenandoahNopGCState) {
777 __ nop(size);
778 return;
779 }
780 #ifdef ASSERT
781 address start = __ pc();
782 #endif
783
784 Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
785 __ testb(gc_state, test_state);
786 __ jcc(Assembler::notZero, *slow_stub->entry());
787
788 #ifdef ASSERT
789 int actual_size = __ pc() - start;
790 assert(actual_size == size, "Should be: %d == %d", actual_size, size);
791 #endif
792 }
793
794 void ShenandoahBarrierSetAssembler::load_ref_barrier_c2(const MachNode* node, MacroAssembler* masm, Register obj, Register addr, Register tmp1, Register tmp2, Register tmp3, bool narrow) {
795 if (!ShenandoahLoadRefBarrierStubC2::needs_barrier(node)) {
796 return;
797 }
798 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
799
800 ShenandoahLoadRefBarrierStubC2* const stub = ShenandoahLoadRefBarrierStubC2::create(node, obj, addr, tmp1, tmp2, tmp3, narrow);
801 stub->dont_preserve(obj); // set at the end, no need to save
802 if (tmp1 != noreg) {
803 stub->dont_preserve(tmp1); // temp, no need to save
804 }
805 if (tmp2 != noreg) {
806 stub->dont_preserve(tmp2); // temp, no need to save
807 }
808 if (tmp3 != noreg) {
809 stub->dont_preserve(tmp3); // temp, no need to save
810 }
811
812 int flags = ShenandoahHeap::HAS_FORWARDED;
813 if ((node->barrier_data() & ShenandoahBarrierStrong) == 0) {
814 flags |= ShenandoahHeap::WEAK_ROOTS;
815 }
816 gc_state_check_c2(masm, flags, stub);
817 __ bind(*stub->continuation());
818 }
819
820 void ShenandoahBarrierSetAssembler::load_c2(const MachNode* node, MacroAssembler* masm,
821 Register dst,
822 Address src,
823 bool narrow,
824 Register tmp) {
825 // Do the actual load. This load is the candidate for implicit null check, and MUST come first.
826 if (narrow) {
827 __ movl(dst, src);
828 } else {
829 __ movq(dst, src);
830 }
831
832 // Emit barrier if needed
833 if (ShenandoahLoadBarrierStubC2::needs_barrier(node)) {
834 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
835
836 ShenandoahLoadBarrierStubC2* const stub = ShenandoahLoadBarrierStubC2::create(node, dst, src, narrow, tmp);
837 stub->dont_preserve(tmp); // temp, no need to save
838
839 char check = 0;
840 check |= ShenandoahLoadBarrierStubC2::needs_satb_barrier(node) ? ShenandoahHeap::MARKING : 0;
841 check |= ShenandoahLoadBarrierStubC2::needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
842 check |= ShenandoahLoadBarrierStubC2::needs_load_ref_barrier_weak(node) ? ShenandoahHeap::WEAK_ROOTS : 0;
843 gc_state_check_c2(masm, check, stub);
844 __ bind(*stub->continuation());
845 }
846 }
847
848 void ShenandoahBarrierSetAssembler::store_c2(const MachNode* node, MacroAssembler* masm,
849 Address dst, bool dst_narrow,
850 Register src, bool src_narrow,
851 Register tmp) {
852 // Emit barrier if needed
853 if (ShenandoahStoreBarrierStubC2::needs_barrier(node)) {
854 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
855
856 if (ShenandoahStoreBarrierStubC2::needs_satb_barrier(node)) {
857 ShenandoahStoreBarrierStubC2* const stub = ShenandoahStoreBarrierStubC2::create(node, dst, dst_narrow, src, src_narrow, tmp);
858 stub->dont_preserve(tmp); // temp, no need to preserve it
859
860 gc_state_check_c2(masm, ShenandoahHeap::MARKING, stub);
861 __ bind(*stub->continuation());
862 }
863
864 if (ShenandoahStoreBarrierStubC2::needs_card_barrier(node)) {
865 // Card table barrier is not conditional on GC state.
866 // You might think this needs to be a post-barrier. But I don't think it does: the card table updates
867 // and stores are not expected to be ordered. As long as there is no safepoint between these stores, we are
868 // free to do them in any order.
869
870 // So it is convenient to pull card table update here. It also follows the stencil we want:
871 // there should be a single gc-state check for every possible fast path. If card table barrier needed
872 // a gc state check, we would have commoned it with gc state check for SATB barrier above, and _then_
873 // called to the slowpath.
874
875 // Using this address compute sequence allows us to use only one temp register.
876 // TODO: Upstream this separately, mainline Shenandoah might benefit from this already?
877 __ lea(tmp, dst);
878 __ shrptr(tmp, CardTable::card_shift());
879 __ addptr(tmp, Address(r15_thread, in_bytes(ShenandoahThreadLocalData::card_table_offset())));
880
881 int dirty = CardTable::dirty_card_val();
882 if (UseCondCardMark) {
883 Label L_already_dirty;
884 __ cmpb(Address(tmp, 0), dirty);
885 __ jccb(Assembler::equal, L_already_dirty);
886 __ movb(Address(tmp, 0), dirty);
887 __ bind(L_already_dirty);
888 } else {
889 __ movb(Address(tmp, 0), dirty);
890 }
891 }
892 }
893
894 // Need to encode into tmp, because we cannot clobber src.
895 // TODO: Maybe there is a matcher way to test that src is unused after this?
896 if (dst_narrow && !src_narrow) {
897 __ movq(tmp, src);
898 if (ShenandoahStoreBarrierStubC2::src_not_null(node)) {
899 __ encode_heap_oop_not_null(tmp);
900 } else {
901 __ encode_heap_oop(tmp);
902 }
903 src = tmp;
904 }
905
906 // Do the actual store
907 if (dst_narrow) {
908 __ movl(dst, src);
909 } else {
910 __ movq(dst, src);
911 }
912 }
913
914 void ShenandoahBarrierSetAssembler::satb_barrier_c2(const MachNode* node, MacroAssembler* masm,
915 Register addr, Register preval, Register tmp) {
916 if (!ShenandoahSATBBarrierStubC2::needs_barrier(node)) {
917 return;
918 }
919 ShenandoahSATBBarrierStubC2* const stub = ShenandoahSATBBarrierStubC2::create(node, addr, preval, tmp, /* TODO: */ false);
920 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
921
922 gc_state_check_c2(masm, ShenandoahHeap::MARKING, stub);
923 __ bind(*stub->continuation());
924 }
925
926 void ShenandoahBarrierSetAssembler::card_barrier_c2(const MachNode* node, MacroAssembler* masm,
927 Register addr, Register addr_tmp, Register tmp) {
928 if ((node->barrier_data() & ShenandoahBarrierCardMark) == 0) {
929 return;
930 }
931 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
932 if (addr != noreg) {
933 __ mov(addr_tmp, addr);
934 }
935 __ shrptr(addr_tmp, CardTable::card_shift());
936
937 Address curr_ct_holder_addr(r15_thread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
938 __ movptr(tmp, curr_ct_holder_addr);
939 Address card_addr(tmp, addr_tmp, Address::times_1);
940
941 int dirty = CardTable::dirty_card_val();
942 if (UseCondCardMark) {
943 Label L_already_dirty;
944 __ cmpb(card_addr, dirty);
945 __ jccb(Assembler::equal, L_already_dirty);
946 __ movb(card_addr, dirty);
947 __ bind(L_already_dirty);
948 } else {
949 __ movb(card_addr, dirty);
950 }
951 }
952
953 void ShenandoahBarrierSetAssembler::cmpxchg_oop_c2(const MachNode* node, MacroAssembler* masm,
954 Register res, Address addr, Register oldval, Register newval, Register tmp1, Register tmp2,
955 bool exchange) {
956 assert(oldval == rax, "must be in rax for implicit use in cmpxchg");
957 assert_different_registers(oldval, tmp1, tmp2);
958 assert_different_registers(newval, tmp1, tmp2);
959
960 // Remember oldval for retry logic in slow path. We need to do it here,
961 // because it will be overwritten by the fast-path CAS.
962 if (ShenandoahCASBarrier) {
963 __ movptr(tmp2, oldval);
964 }
965
966 // Fast-path: Try to CAS optimistically. If successful, then we are done.
967 __ lock();
968 if (UseCompressedOops) {
969 __ cmpxchgl(newval, addr);
970 } else {
971 __ cmpxchgptr(newval, addr);
972 }
973
974 // If we need a boolean result out of CAS, set the flag appropriately and promote the result.
975 // This would be the final result if we do not go slow.
976 if (!exchange) {
977 assert(res != noreg, "need result register");
978 __ setcc(Assembler::equal, res);
979 } else {
980 assert(res == noreg, "no result expected");
981 }
982
983 if (ShenandoahCASBarrier) {
984 ShenandoahCASBarrierSlowStubC2* const stub =
985 ShenandoahCASBarrierSlowStubC2::create(node, addr, oldval, newval, res, tmp1, tmp2, exchange);
986 if (res != noreg) {
987 stub->dont_preserve(res); // set at the end, no need to save
988 }
989 stub->dont_preserve(oldval); // saved explicitly
990 stub->dont_preserve(tmp1); // temp, no need to save
991 stub->dont_preserve(tmp2); // temp, no need to save
992
993 // On success, we do not need any additional handling.
994 __ jccb(Assembler::equal, *stub->continuation());
995
996 // If GC is in progress, it is likely we need additional handling for false negatives.
997 // Slow stub re-enters with result set correctly.
998 gc_state_check_c2(masm, ShenandoahHeap::HAS_FORWARDED, stub);
999 __ bind(*stub->continuation());
1000 }
1001 }
1002
1003 #undef __
1004 #define __ masm.
1005
1006 void ShenandoahLoadBarrierStubC2::emit_code(MacroAssembler& masm) {
1007 __ bind(*entry());
1008
1009 assert_different_registers(_tmp, _dst);
1010
1011 Label L_end;
1012
1013 // If the object is null, there is no point in applying barriers.
1014 if (_narrow) {
1015 __ testl(_dst, _dst);
1016 } else {
1017 __ testptr(_dst, _dst);
1018 }
1019 __ jcc(Assembler::equal, *continuation());
1020
1021 // If object is narrow, we need to decode it first.
1022 if (_narrow) {
1023 __ decode_heap_oop_not_null(_dst);
1024 }
1025
1026 if (_needs_load_ref_barrier) {
1027 Label L_lrb_done;
1028
1029 bool is_weak = (_node->barrier_data() & ShenandoahBarrierStrong) == 0;
1030
1031 // Runtime check for LRB
1032 Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1033 __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED | (is_weak ? ShenandoahHeap::WEAK_ROOTS : 0));
1034 __ jcc(Assembler::zero, L_lrb_done);
1035
1036 // Weak/phantom loads always need to go to runtime.
1037 if (!is_weak) {
1038 __ movptr(_tmp, _dst);
1039 __ shrptr(_tmp, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1040 __ addptr(_tmp, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
1041 __ testb(Address(_tmp, 0), 0xFF);
1042 __ jcc(Assembler::zero, L_lrb_done);
1043 }
1044
1045 dont_preserve(_dst); // For LRB we must not preserve _dst
1046 {
1047 SaveLiveRegisters save_registers(&masm, this);
1048
1049 // Shuffle in the arguments. The end result should be:
1050 // c_rarg0 <-- _dst
1051 // c_rarg1 <-- lea(_src)
1052 if (c_rarg0 == _dst) {
1053 __ lea(c_rarg1, _src);
1054 } else if (c_rarg1 == _dst) {
1055 // Set up arguments in reverse, and then flip them
1056 __ lea(c_rarg0, _src);
1057 __ xchgptr(c_rarg0, c_rarg1);
1058 } else {
1059 assert_different_registers(c_rarg1, _dst);
1060 __ lea(c_rarg1, _src);
1061 __ movptr(c_rarg0, _dst);
1062 }
1063
1064 address entry;
1065 if (_narrow) {
1066 if ((_node->barrier_data() & ShenandoahBarrierStrong) != 0) {
1067 entry = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
1068 } else if ((_node->barrier_data() & ShenandoahBarrierWeak) != 0) {
1069 entry = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
1070 } else if ((_node->barrier_data() & ShenandoahBarrierPhantom) != 0) {
1071 entry = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom_narrow);
1072 }
1073 } else {
1074 if ((_node->barrier_data() & ShenandoahBarrierStrong) != 0) {
1075 entry = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
1076 } else if ((_node->barrier_data() & ShenandoahBarrierWeak) != 0) {
1077 entry = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
1078 } else if ((_node->barrier_data() & ShenandoahBarrierPhantom) != 0) {
1079 entry = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
1080 }
1081 }
1082 __ call(RuntimeAddress(entry), rax);
1083 assert(!save_registers.contains(_dst), "must not save result register");
1084 __ movptr(_dst, rax);
1085 }
1086
1087 // Paranoia: if LRB returns null for a weak access, do NOT feed it into SATB, which does not accept null pointers.
1088 __ testptr(_dst, _dst);
1089 __ jcc(Assembler::equal, L_end);
1090
1091 __ bind(L_lrb_done);
1092 }
1093
1094 if (_needs_satb_barrier) {
1095 // Push obj to SATB, if needed.
1096
1097 Label L_satb_done, L_satb_runtime;
1098
1099 // Runtime check for SATB
1100 Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1101 __ testb(gc_state, ShenandoahHeap::MARKING);
1102 __ jcc(Assembler::zero, L_satb_done);
1103
1104 // Can we store a value in the given thread's buffer?
1105 // (The index field is typed as size_t.)
1106 Address index(r15_thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
1107 Address buffer(r15_thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
1108
1109 __ movptr(_tmp, index);
1110 __ testptr(_tmp, _tmp);
1111 __ jccb(Assembler::zero, L_satb_runtime);
1112 // The buffer is not full, store value into it.
1113 __ subptr(_tmp, wordSize);
1114 __ movptr(index, _tmp);
1115 __ addptr(_tmp, buffer);
1116 __ movptr(Address(_tmp, 0), _dst);
1117 __ jmp(L_satb_done);
1118
1119 __ bind(L_satb_runtime);
1120
1121 preserve(_dst); // For SATB we must preserve _dst
1122 {
1123 SaveLiveRegisters save_registers(&masm, this);
1124 if (c_rarg0 != _dst) {
1125 __ mov(c_rarg0, _dst);
1126 }
1127 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre_c2)), rax);
1128 }
1129
1130 __ bind(L_satb_done);
1131 }
1132
1133 __ bind(L_end);
1134 if (_narrow) {
1135 __ encode_heap_oop(_dst);
1136 }
1137
1138 __ jmp(*continuation());
1139 }
1140
1141 void ShenandoahStoreBarrierStubC2::emit_code(MacroAssembler& masm) {
1142 __ bind(*entry());
1143
1144 Label L_runtime, L_preval_null;
1145
1146 // We need 2 temp registers for this code to work.
1147 // _tmp is already allocated and will carry preval for the call.
1148 // Allocate the other one now.
1149 Register tmp2 = noreg;
1150 for (int i = 0; i < 8; i++) {
1151 Register r = as_Register(i);
1152 if (r != rsp && r != rbp && r != _src && r != _tmp) {
1153 if (tmp2 == noreg) {
1154 tmp2 = r;
1155 break;
1156 }
1157 }
1158 }
1159
1160 assert(tmp2 != noreg, "tmp2 allocated");
1161 assert_different_registers(_tmp, tmp2, _src);
1162
1163 Register preval = _tmp;
1164 Register slot = tmp2;
1165
1166 // Load value from memory
1167 if (_dst_narrow) {
1168 __ movl(preval, _dst);
1169 } else {
1170 __ movq(preval, _dst);
1171 }
1172
1173 // Is the previous value null?
1174 __ cmpptr(preval, NULL_WORD);
1175 __ jccb(Assembler::equal, L_preval_null);
1176
1177 if (_dst_narrow) {
1178 __ decode_heap_oop_not_null(preval);
1179 }
1180
1181 // Can we store a value in the given thread's buffer?
1182 // (The index field is typed as size_t.)
1183 Address index(r15_thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
1184 Address buffer(r15_thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
1185
1186 __ push(tmp2);
1187 __ movptr(slot, index);
1188 __ testptr(slot, slot);
1189 __ jccb(Assembler::zero, L_runtime);
1190 // The buffer is not full, store value into it.
1191 __ subptr(slot, wordSize);
1192 __ movptr(index, slot);
1193 __ addptr(slot, buffer);
1194 __ movptr(Address(slot, 0), preval);
1195
1196 // Pop temps and exit
1197 __ pop(tmp2);
1198 __ bind(L_preval_null);
1199 __ jmp(*continuation());
1200
1201 __ bind(L_runtime);
1202 __ pop(tmp2);
1203 {
1204 SaveLiveRegisters save_registers(&masm, this);
1205 if (c_rarg0 != preval) {
1206 __ mov(c_rarg0, preval);
1207 }
1208 // rax is a caller-saved, non-argument-passing register, so it does not
1209 // interfere with c_rarg0 or c_rarg1. If it contained any live value before
1210 // entering this stub, it is saved at this point, and restored after the
1211 // call. If it did not contain any live value, it is free to be used. In
1212 // either case, it is safe to use it here as a call scratch register.
1213 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre_c2)), rax);
1214 }
1215 __ jmp(*continuation());
1216 }
1217
1218 void ShenandoahLoadRefBarrierStubC2::emit_code(MacroAssembler& masm) {
1219 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1220 __ bind(*entry());
1221
1222 Register obj = _obj;
1223 if (_narrow) {
1224 __ movl(_tmp1, _obj);
1225 __ decode_heap_oop(_tmp1);
1226 obj = _tmp1;
1227 }
1228
1229 // Weak/phantom loads always need to go to runtime.
1230 if ((_node->barrier_data() & ShenandoahBarrierStrong) != 0) {
1231 __ movptr(_tmp2, obj);
1232 __ shrptr(_tmp2, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1233 __ movptr(_tmp3, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
1234 __ movbool(_tmp2, Address(_tmp2, _tmp3, Address::times_1));
1235 __ testbool(_tmp2);
1236 __ jcc(Assembler::zero, *continuation());
1237 }
1238
1239 {
1240 SaveLiveRegisters save_registers(&masm, this);
1241 if (c_rarg0 != obj) {
1242 if (c_rarg0 == _addr) {
1243 __ movptr(_tmp2, _addr);
1244 _addr = _tmp2;
1245 }
1246 __ movptr(c_rarg0, obj);
1247 }
1248 if (c_rarg1 != _addr) {
1249 __ movptr(c_rarg1, _addr);
1250 }
1251
1252 address entry;
1253 if (_narrow) {
1254 if ((_node->barrier_data() & ShenandoahBarrierStrong) != 0) {
1255 entry = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
1256 } else if ((_node->barrier_data() & ShenandoahBarrierWeak) != 0) {
1257 entry = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
1258 } else if ((_node->barrier_data() & ShenandoahBarrierPhantom) != 0) {
1259 entry = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom_narrow);
1260 }
1261 } else {
1262 if ((_node->barrier_data() & ShenandoahBarrierStrong) != 0) {
1263 entry = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
1264 } else if ((_node->barrier_data() & ShenandoahBarrierWeak) != 0) {
1265 entry = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
1266 } else if ((_node->barrier_data() & ShenandoahBarrierPhantom) != 0) {
1267 entry = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
1268 }
1269 }
1270 __ call(RuntimeAddress(entry), rax);
1271 assert(!save_registers.contains(_obj), "must not save result register");
1272 __ movptr(_obj, rax);
1273 }
1274 if (_narrow) {
1275 __ encode_heap_oop(_obj);
1276 }
1277
1278 __ jmp(*continuation());
1279 }
1280
1281 void ShenandoahSATBBarrierStubC2::emit_code(MacroAssembler& masm) {
1282 __ bind(*entry());
1283 Address index(r15_thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
1284 Address buffer(r15_thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
1285
1286 Label runtime;
1287
1288 // Do we need to load the previous value?
1289 if (_addr != noreg) {
1290 __ load_heap_oop(_preval, Address(_addr, 0), noreg, AS_RAW);
1291 }
1292 // Is the previous value null?
1293 __ cmpptr(_preval, NULL_WORD);
1294 __ jcc(Assembler::equal, *continuation());
1295
1296 // Can we store a value in the given thread's buffer?
1297 // (The index field is typed as size_t.)
1298 __ movptr(_tmp, index);
1299 __ testptr(_tmp, _tmp);
1300 __ jccb(Assembler::zero, runtime);
1301 // The buffer is not full, store value into it.
1302 __ subptr(_tmp, wordSize);
1303 __ movptr(index, _tmp);
1304 __ addptr(_tmp, buffer);
1305 __ movptr(Address(_tmp, 0), _preval);
1306
1307 __ jmp(*continuation());
1308
1309 __ bind(runtime);
1310 {
1311 SaveLiveRegisters save_registers(&masm, this);
1312 if (c_rarg0 != _preval) {
1313 __ mov(c_rarg0, _preval);
1314 }
1315 // rax is a caller-saved, non-argument-passing register, so it does not
1316 // interfere with c_rarg0 or c_rarg1. If it contained any live value before
1317 // entering this stub, it is saved at this point, and restored after the
1318 // call. If it did not contain any live value, it is free to be used. In
1319 // either case, it is safe to use it here as a call scratch register.
1320 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre_c2)), rax);
1321 }
1322 __ jmp(*continuation());
1323 }
1324
1325 void ShenandoahCASBarrierSlowStubC2::emit_code(MacroAssembler& masm) {
1326 __ bind(*entry());
1327
1328 // CAS has failed because the value held at addr does not match expected.
1329 // This may be a false negative because the version in memory might be
1330 // the from-space version of the same object we currently hold to-space
1331 // reference for.
1332 //
1333 // To resolve this, we need to pass the location through the LRB fixup,
1334 // this will make sure that the location has only to-space pointers.
1335 // To avoid calling into runtime often, we cset-check the object first.
1336 // We can inline most of the work here, but there is little point,
1337 // as CAS failures over cset locations must be rare. This fast-slow split
1338 // matches what we do for normal LRB.
1339
1340 assert(_expected == rax, "expected must be rax");
1341
1342 // Non-strong references should always go to runtime. We do not expect
1343 // CASes over non-strong locations.
1344 assert((_node->barrier_data() & ShenandoahBarrierStrong) != 0, "Only strong references for CASes");
1345
1346 Label L_final;
1347
1348 // Fast-path stashed original oldval to tmp2 for us. We need to save it
1349 // for the final retry. This frees up tmp2 for cset check below.
1350 __ push(_tmp2);
1351
1352 // (Compressed) failure witness is in _expected.
1353 // Unpack it and check if it is in collection set.
1354 __ movptr(_tmp1, _expected);
1355 if (UseCompressedOops) {
1356 __ decode_heap_oop(_tmp1);
1357 }
1358 __ shrptr(_tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1359 __ movptr(_tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
1360 __ movbool(_tmp1, Address(_tmp1, _tmp2, Address::times_1));
1361 __ testbool(_tmp1);
1362 __ jcc(Assembler::zero, L_final);
1363
1364 {
1365 SaveLiveRegisters save_registers(&masm, this);
1366 // Load up failure witness again.
1367 if (c_rarg0 != _expected) {
1368 __ movptr(c_rarg0, _expected);
1369 }
1370 if (UseCompressedOops) {
1371 __ decode_heap_oop(c_rarg0);
1372 }
1373 __ lea(c_rarg1, _addr);
1374
1375 if (UseCompressedOops) {
1376 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow), 2);
1377 } else {
1378 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong), 2);
1379 }
1380 // We have called LRB to fix up the heap location. We do not care about its result,
1381 // as we will just try to CAS the location again.
1382 }
1383
1384 __ bind(L_final);
1385
1386 // Try to CAS again with the original expected value.
1387 // At this point, there can no longer be false negatives.
1388 __ pop(_expected);
1389 __ lock();
1390 if (UseCompressedOops) {
1391 __ cmpxchgl(_new_val, _addr);
1392 } else {
1393 __ cmpxchgptr(_new_val, _addr);
1394 }
1395 if (!_cae) {
1396 assert(_result != noreg, "need result register");
1397 __ setcc(Assembler::equal, _result);
1398 } else {
1399 assert(_result == noreg, "no result expected");
1400 }
1401 __ jmp(*continuation());
1402 }
1403
1404 #undef __
1405 #define __ masm->
1406 #endif
1407
1408 #ifdef PRODUCT
1409 #define BLOCK_COMMENT(str) /* nothing */
1410 #else
1411 #define BLOCK_COMMENT(str) __ block_comment(str)
1412 #endif
1413
1414 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
1415
1416 #define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8)
1417
1418 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
1419 Register addr, Register count,
1420 Register tmp) {
1421 assert(ShenandoahCardBarrier, "Should have been checked by caller");
1422
1423 Label L_loop, L_done;
1424 const Register end = count;
1425 assert_different_registers(addr, end);
1426
1427 // Zero count? Nothing to do.
|