24 */
25
26 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
27 #include "gc/shenandoah/mode/shenandoahMode.hpp"
28 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
30 #include "gc/shenandoah/shenandoahForwarding.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
33 #include "gc/shenandoah/shenandoahRuntime.hpp"
34 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
35 #include "interpreter/interp_masm.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "runtime/javaThread.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #ifdef COMPILER1
40 #include "c1/c1_LIRAssembler.hpp"
41 #include "c1/c1_MacroAssembler.hpp"
42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
43 #endif
44
45 #define __ masm->
46
47 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
48 Register src, Register dst, Register count, RegSet saved_regs) {
49 if (is_oop) {
50 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
51 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
52
53 Label done;
54
55 // Avoid calling runtime if count == 0
56 __ cbz(count, done);
57
58 // Is GC active?
59 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
60 __ ldrb(rscratch1, gc_state);
61 if (ShenandoahSATBBarrier && dest_uninitialized) {
62 __ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
63 } else {
589 if (is_cae) {
590 // We're falling through to done to indicate success. Success
591 // with is_cae is denoted by returning the value of expected as
592 // result.
593 __ mov(tmp2, expected);
594 }
595
596 __ bind(done);
597 // At entry to done, the Z (EQ) flag is on iff if the CAS
598 // operation was successful. Additionally, if is_cae, tmp2 holds
599 // the value most recently fetched from addr. In this case, success
600 // is denoted by tmp2 matching expected.
601
602 if (is_cae) {
603 __ mov(result, tmp2);
604 } else {
605 __ cset(result, Assembler::EQ);
606 }
607 }
608
609 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
610 Register start, Register count, Register scratch, RegSet saved_regs) {
611 assert(ShenandoahCardBarrier, "Should have been checked by caller");
612
613 Label L_loop, L_done;
614 const Register end = count;
615
616 // Zero count? Nothing to do.
617 __ cbz(count, L_done);
618
619 // end = start + count << LogBytesPerHeapOop
620 // last element address to make inclusive
621 __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop)));
622 __ sub(end, end, BytesPerHeapOop);
623 __ lsr(start, start, CardTable::card_shift());
624 __ lsr(end, end, CardTable::card_shift());
625
626 // number of bytes to copy
627 __ sub(count, end, start);
628
|
24 */
25
26 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
27 #include "gc/shenandoah/mode/shenandoahMode.hpp"
28 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
30 #include "gc/shenandoah/shenandoahForwarding.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
33 #include "gc/shenandoah/shenandoahRuntime.hpp"
34 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
35 #include "interpreter/interp_masm.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "runtime/javaThread.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #ifdef COMPILER1
40 #include "c1/c1_LIRAssembler.hpp"
41 #include "c1/c1_MacroAssembler.hpp"
42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
43 #endif
44 #ifdef COMPILER2
45 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
46 #endif
47
48 #define __ masm->
49
50 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
51 Register src, Register dst, Register count, RegSet saved_regs) {
52 if (is_oop) {
53 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
54 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
55
56 Label done;
57
58 // Avoid calling runtime if count == 0
59 __ cbz(count, done);
60
61 // Is GC active?
62 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
63 __ ldrb(rscratch1, gc_state);
64 if (ShenandoahSATBBarrier && dest_uninitialized) {
65 __ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
66 } else {
592 if (is_cae) {
593 // We're falling through to done to indicate success. Success
594 // with is_cae is denoted by returning the value of expected as
595 // result.
596 __ mov(tmp2, expected);
597 }
598
599 __ bind(done);
600 // At entry to done, the Z (EQ) flag is on iff if the CAS
601 // operation was successful. Additionally, if is_cae, tmp2 holds
602 // the value most recently fetched from addr. In this case, success
603 // is denoted by tmp2 matching expected.
604
605 if (is_cae) {
606 __ mov(result, tmp2);
607 } else {
608 __ cset(result, Assembler::EQ);
609 }
610 }
611
612 #ifdef COMPILER2
613 void ShenandoahBarrierSetAssembler::load_ref_barrier_c2(const MachNode* node, MacroAssembler* masm, Register obj, Register addr, Register tmp, bool narrow, bool maybe_null) {
614 if (!ShenandoahLoadRefBarrierStubC2::needs_barrier(node)) {
615 return;
616 }
617 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
618 Label done;
619 if (maybe_null) {
620 __ cbz(obj, done);
621 }
622 ShenandoahLoadRefBarrierStubC2* const stub = ShenandoahLoadRefBarrierStubC2::create(node, obj, addr, tmp, noreg, noreg, narrow);
623 // Don't preserve the obj across the runtime call, we override it from the return value anyway.
624 stub->dont_preserve(obj);
625 // Check if GC marking is in progress, otherwise we don't have to do anything.
626 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
627 __ ldrb(rscratch1, gc_state);
628 __ tstw(rscratch1, ShenandoahHeap::HAS_FORWARDED);
629 __ br(Assembler::NE, *stub->entry());
630 __ bind(*stub->continuation());
631 __ bind(done);
632 }
633
634 void ShenandoahBarrierSetAssembler::satb_barrier_c2(const MachNode* node, MacroAssembler* masm, Register addr, Register pre_val) {
635 assert_different_registers(addr, pre_val);
636 if (!ShenandoahSATBBarrierStubC2::needs_barrier(node)) {
637 return;
638 }
639 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
640 ShenandoahSATBBarrierStubC2* const stub = ShenandoahSATBBarrierStubC2::create(node, addr, pre_val);
641
642 // Check if GC marking is in progress, otherwise we don't have to do anything.
643 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
644 __ ldrb(rscratch1, gc_state);
645 __ tstw(rscratch1, ShenandoahHeap::MARKING);
646 __ br(Assembler::NE, *stub->entry());
647 __ bind(*stub->continuation());
648 }
649
650 void ShenandoahBarrierSetAssembler::card_barrier_c2(const MachNode* node, MacroAssembler* masm, Register addr, Register tmp) {
651 if (!ShenandoahCardBarrier ||
652 (node->barrier_data() & (ShenandoahBarrierCardMark | ShenandoahBarrierCardMarkNotNull)) == 0) {
653 return;
654 }
655
656 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
657 __ lsr(tmp, addr, CardTable::card_shift());
658
659 assert(CardTable::dirty_card_val() == 0, "must be");
660
661 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
662 __ ldr(rscratch1, curr_ct_holder_addr);
663
664 if (UseCondCardMark) {
665 Label L_already_dirty;
666 __ ldrb(rscratch2, Address(tmp, rscratch1));
667 __ cbz(rscratch2, L_already_dirty);
668 __ strb(zr, Address(tmp, rscratch1));
669 __ bind(L_already_dirty);
670 } else {
671 __ strb(zr, Address(tmp, rscratch1));
672 }
673 }
674
675 void ShenandoahBarrierSetAssembler::cmpxchg_oop_c2(const MachNode* node,
676 MacroAssembler* masm,
677 Register addr,
678 Register expected,
679 Register new_val,
680 Register result,
681 bool acquire, bool release, bool weak,
682 bool is_cae) {
683 Register tmp = rscratch2;
684 Assembler::operand_size size = UseCompressedOops ? Assembler::word : Assembler::xword;
685
686 assert_different_registers(addr, expected, result, tmp);
687 assert_different_registers(addr, new_val, result, tmp);
688
689 ShenandoahCASBarrierSlowStubC2* const slow_stub = ShenandoahCASBarrierSlowStubC2::create(node, addr, expected, new_val, result, tmp, is_cae, acquire, release, weak);
690 ShenandoahCASBarrierMidStubC2* const mid_stub = ShenandoahCASBarrierMidStubC2::create(node, slow_stub, expected, result, tmp, is_cae);
691
692 // Step 1. Fast-path.
693 //
694 // Try to CAS with given arguments. If successful, then we are done.
695 __ cmpxchg(addr, expected, new_val, size, acquire, release, weak, result);
696 // EQ flag set iff success. result holds value fetched.
697
698 __ br(Assembler::NE, *mid_stub->entry());
699
700 // Slow-stub re-enters with condition flags according to CAS, we may need to
701 // set result accordingly.
702 __ bind(*slow_stub->continuation());
703 if (!is_cae) {
704 __ cset(result, Assembler::EQ);
705 }
706
707 // Mid-stub re-enters with result set correctly.
708 __ bind(*mid_stub->continuation());
709 }
710
711 #undef __
712 #define __ masm.
713
714 void ShenandoahLoadRefBarrierStubC2::emit_code(MacroAssembler& masm) {
715 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
716 __ bind(*entry());
717 Register obj = _obj;
718 if (_narrow) {
719 __ decode_heap_oop(_tmp1, _obj);
720 obj = _tmp1;
721 }
722 // Weak/phantom loads always need to go to runtime.
723 if ((_node->barrier_data() & ShenandoahBarrierStrong) != 0) {
724 // Check for object in cset.
725 __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
726 __ lsr(rscratch1, obj, ShenandoahHeapRegion::region_size_bytes_shift_jint());
727 __ ldrb(rscratch2, Address(rscratch2, rscratch1));
728 __ cbz(rscratch2, *continuation());
729 }
730 {
731 SaveLiveRegisters save_registers(&masm, this);
732 if (c_rarg0 != obj) {
733 if (c_rarg0 == _addr) {
734 __ mov(rscratch1, _addr);
735 _addr = rscratch1;
736 }
737 __ mov(c_rarg0, obj);
738 }
739 __ mov(c_rarg1, _addr);
740
741 if (_narrow) {
742 if ((_node->barrier_data() & ShenandoahBarrierStrong) != 0) {
743 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow));
744 } else if ((_node->barrier_data() & ShenandoahBarrierWeak) != 0) {
745 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow));
746 } else if ((_node->barrier_data() & ShenandoahBarrierPhantom) != 0) {
747 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom_narrow));
748 }
749 } else {
750 if ((_node->barrier_data() & ShenandoahBarrierStrong) != 0) {
751 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong));
752 } else if ((_node->barrier_data() & ShenandoahBarrierWeak) != 0) {
753 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
754 } else if ((_node->barrier_data() & ShenandoahBarrierPhantom) != 0) {
755 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom));
756 }
757 }
758 __ blr(rscratch1);
759 __ mov(_obj, r0);
760 }
761 if (_narrow) {
762 __ encode_heap_oop(_obj);
763 }
764 __ b(*continuation());
765 }
766
767 void ShenandoahSATBBarrierStubC2::emit_code(MacroAssembler& masm) {
768 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
769 __ bind(*entry());
770 // Do we need to load the previous value?
771 if (_addr != noreg) {
772 __ load_heap_oop(_preval, Address(_addr, 0), noreg, noreg, AS_RAW);
773 }
774 // Is the previous value null?
775 __ cbz(_preval, *continuation());
776
777 Address index(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
778 Address buffer(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
779 Label runtime;
780 __ ldr(rscratch1, index);
781 // If buffer is full, call into runtime.
782 __ cbz(rscratch1, runtime);
783
784 // The buffer is not full, store value into it.
785 __ sub(rscratch1, rscratch1, wordSize);
786 __ str(rscratch1, index);
787 __ ldr(rscratch2, buffer);
788 __ str(_preval, Address(rscratch2, rscratch1));
789 __ b(*continuation());
790
791 // Runtime call
792 __ bind(runtime);
793 {
794 SaveLiveRegisters save_registers(&masm, this);
795 if (c_rarg0 != _preval) {
796 __ mov(c_rarg0, _preval);
797 }
798 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre_c2));
799 __ blr(rscratch1);
800 }
801 __ b(*continuation());
802 }
803
804 void ShenandoahCASBarrierMidStubC2::emit_code(MacroAssembler& masm) {
805 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
806 __ bind(*entry());
807
808 // Check if CAS result is null. If it is, then we must have a legitimate failure.
809 // This makes loading the fwdptr in the slow-path simpler.
810 __ tst(_result, _result);
811 // In case of !CAE, this has the correct value for legitimate failure (0/false)
812 // in result register.
813 __ br(Assembler::EQ, *continuation());
814
815 // Check if GC is in progress, otherwise we must have a legitimate failure.
816 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
817 __ ldrb(_tmp, gc_state);
818 __ tstw(_tmp, ShenandoahHeap::HAS_FORWARDED);
819 __ br(Assembler::NE, *_slow_stub->entry());
820
821 if (!_cae) {
822 __ mov(_result, 0); // result = false
823 }
824 __ b(*continuation());
825 }
826
827 void ShenandoahCASBarrierSlowStubC2::emit_code(MacroAssembler& masm) {
828 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
829 __ bind(*entry());
830 Assembler::operand_size size = UseCompressedOops ? Assembler::word : Assembler::xword;
831
832 // Step 2. CAS has failed because the value held at addr does not
833 // match expected. This may be a false negative because the value fetched
834 // from addr (now held in result) may be a from-space pointer to the
835 // original copy of same object referenced by to-space pointer expected.
836 //
837 // To resolve this, it suffices to find the forward pointer associated
838 // with fetched value. If this matches expected, retry CAS with new
839 // parameters. If this mismatches, then we have a legitimate
840 // failure, and we're done.
841
842 // overwrite tmp with from-space pointer fetched from memory
843 __ mov(_tmp1, _result);
844
845 if (UseCompressedOops) {
846 // Decode tmp in order to resolve its forward pointer
847 __ decode_heap_oop_not_null(_tmp1, _tmp1);
848 }
849
850 // Load/decode forwarding pointer.
851 __ ldr(_tmp1, Address(_tmp1, oopDesc::mark_offset_in_bytes()));
852 // Negate the mark-word. This allows us to test lowest 2 bits easily while preserving the upper bits.
853 __ eon(_tmp1, _tmp1, zr);
854 __ ands(zr, _tmp1, markWord::lock_mask_in_place);
855 // Not forwarded, must have a legit CAS failure.
856 __ br(Assembler::NE, *continuation());
857 // Set the lowest two bits. This is equivalent to clearing the two bits after
858 // the subsequent inversion.
859 __ orr(_tmp1, _tmp1, markWord::marked_value);
860 // And invert back to get the forwardee.
861 __ eon(_tmp1, _tmp1, zr);
862
863 if (UseCompressedOops) {
864 // Encode tmp to compare against expected.
865 __ encode_heap_oop_not_null(_tmp1, _tmp1);
866 }
867
868 // Does forwarded value of fetched from-space pointer match original
869 // value of expected? If result holds null, this comparison will fail
870 // because we know from step1 that expected is not null. There is
871 // no need for a separate test for result (the value originally held
872 // in memory) equal to null.
873 __ cmp(_tmp1, _expected);
874
875 // If not, then the failure was legitimate and we're done.
876 // Branching to continuation with NE condition denotes failure.
877 __ br(Assembler::NE, *continuation());
878
879 // Fall through to step 3.
880
881 // Step 3. We've confirmed that the value originally held in memory
882 // (now held in result) pointed to from-space version of original
883 // expected value. Try the CAS again with the from-space expected
884 // value. If it now succeeds, we're good.
885 //
886 // Note: result holds encoded from-space pointer that matches to-space
887 // object residing at expected. result is the new "expected".
888
889 // Note that macro implementation of __cmpxchg cannot use same register
890 // tmp2 for result and expected since it overwrites result before it
891 // compares result with expected.
892 __ mov(_tmp1, _result);
893 __ cmpxchg(_addr_reg, _tmp1, _new_val, size, _acquire, _release, _weak, _result);
894 // EQ flag set iff success. result holds value fetched, rscratch1 clobbered.
895
896 // If fetched value did not equal the new expected, this could
897 // still be a false negative because some other thread may have
898 // newly overwritten the memory value with its to-space equivalent.
899 __ br(Assembler::EQ, *continuation());
900
901 // Step 4. Retry CAS with original to-space expected.
902 __ cmpxchg(_addr_reg, _expected, _new_val, size, _acquire, _release, _weak, _result);
903
904 __ b(*continuation());
905 }
906 #undef __
907 #define __ masm->
908 #endif // COMPILER2
909
910 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
911 Register start, Register count, Register scratch, RegSet saved_regs) {
912 assert(ShenandoahCardBarrier, "Should have been checked by caller");
913
914 Label L_loop, L_done;
915 const Register end = count;
916
917 // Zero count? Nothing to do.
918 __ cbz(count, L_done);
919
920 // end = start + count << LogBytesPerHeapOop
921 // last element address to make inclusive
922 __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop)));
923 __ sub(end, end, BytesPerHeapOop);
924 __ lsr(start, start, CardTable::card_shift());
925 __ lsr(end, end, CardTable::card_shift());
926
927 // number of bytes to copy
928 __ sub(count, end, start);
929
|