13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
33 #include "gc/shenandoah/shenandoahRuntime.hpp"
34 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
35 #include "interpreter/interp_masm.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "runtime/javaThread.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #ifdef COMPILER1
40 #include "c1/c1_LIRAssembler.hpp"
41 #include "c1/c1_MacroAssembler.hpp"
42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
43 #endif
44 #ifdef COMPILER2
45 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
46 #include "opto/output.hpp"
47 #endif
48
49 #define __ masm->
50
51 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
52 Register src, Register dst, Register count, RegSet saved_regs) {
53 if (is_oop) {
54 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
55 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
56
649 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
650 __ ldr(tmp1, curr_ct_holder_addr);
651
652 // tmp2 = effective address
653 __ lea(tmp2, address);
654
655 // tmp2 = &card_table[ addr >> CardTable::card_shift() ] ; card index
656 __ add(tmp2, tmp1, tmp2, Assembler::LSR, CardTable::card_shift());
657
658 if (UseCondCardMark) {
659 Label L_already_dirty;
660 __ ldrb(tmp1, Address(tmp2));
661 __ cbz(tmp1, L_already_dirty);
662 __ strb(zr, Address(tmp2));
663 __ bind(L_already_dirty);
664 } else {
665 __ strb(zr, Address(tmp2));
666 }
667 }
668
669 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
670 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
671 PhaseOutput* const output = Compile::current()->output();
672 Address gc_state_fast(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(test_state)));
673
674 // We piggyback on scratch_emit_size mode to compute the slowpath stub size.
675 // We'll use that information to decide whether we need a far jump to the
676 // stub entry point or not. In scratch_emit_size mode we don't bind entry()
677 // because otherwise it will be rebound when we later emit the instructions
678 // for real.
679 if (_needs_far_jump) {
680 __ ldrb(tmp, gc_state_fast);
681 __ cbz(tmp, *continuation());
682 __ b(output->in_scratch_emit_size() ? *continuation() : *entry());
683 } else {
684 __ ldrb(tmp, gc_state_fast);
685 __ cbnz(tmp, output->in_scratch_emit_size() ? *continuation() : *entry());
686 }
687
688 // This is were the slowpath stub will return to or the code above will
689 // jump to if the checks are false
690 __ bind(*continuation());
691 }
692
693 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
694 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
695 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
696 PhaseOutput* const output = Compile::current()->output();
697
698 // We piggyback on scratch_emit_size mode to compute the slowpath stub size.
699 // We'll use that information to decide whether we need a far jump to the
700 // stub entry point or not. In scratch_emit_size mode we don't bind entry()
701 // because otherwise it will be rebound when we later emit the instructions
702 // for real.
703 if (!output->in_scratch_emit_size()) {
704 __ bind(*entry());
705 }
706
707 // If we need to load ourselves, do it here.
708 if (_do_load) {
709 if (_narrow) {
710 __ ldrw(_obj, _addr);
711 } else {
712 __ ldr(_obj, _addr);
743 keepalive(masm, continuation());
744 } else if (_needs_load_ref_barrier) {
745 lrb(masm);
746 } else {
747 ShouldNotReachHere();
748 }
749 }
750
751 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
752 if (_needs_far_jump) {
753 Label L_short_jump;
754 __ cbnz(reg, L_short_jump);
755 __ b(*continuation());
756 __ bind(L_short_jump);
757 } else {
758 __ cbz(reg, *continuation());
759 }
760 }
761
762 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
763 Address gcstate(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::MARKING)));
764 Address index(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
765 Address buffer(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
766 Label L_through, L_slowpath;
767
768 // If another barrier is enabled as well, do a runtime check for a specific barrier.
769 if (_needs_load_ref_barrier) {
770 assert(L_done == nullptr, "L_done is always null when _needs_load_ref_barrier is true");
771 __ ldrb(_tmp1, gcstate);
772 __ cbz(_tmp1, L_through);
773 }
774
775 // Fast-path: put object into buffer.
776 // If buffer is already full, go slow.
777 __ ldr(_tmp1, index);
778 __ cbz(_tmp1, L_slowpath);
779 __ sub(_tmp1, _tmp1, wordSize);
780 __ str(_tmp1, index);
781 __ ldr(_tmp2, buffer);
782
783 // Store the object in queue.
784 // If object is narrow, we need to decode it before inserting.
785 if (_narrow) {
786 __ add(_tmp2, _tmp2, _tmp1);
787 __ decode_heap_oop_not_null(_tmp1, _obj);
788 __ str(_tmp1, Address(_tmp2));
789 } else {
790 // Buffer is 64-bit address, must be in base register.
791 __ str(_obj, Address(_tmp2, _tmp1));
792 }
802 __ bind(L_slowpath);
803
804 {
805 SaveLiveRegisters slr(&masm, this);
806
807 // Go to runtime and handle the rest there.
808 __ mov(c_rarg0, _obj);
809 __ lea(lr, RuntimeAddress(keepalive_runtime_entry_addr()));
810 __ blr(lr);
811 }
812 if (L_done != nullptr) {
813 __ b(*L_done);
814 } else {
815 __ bind(L_through);
816 }
817 }
818
819 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
820 Label L_slow;
821
822 // If another barrier is enabled as well, do a runtime check for a specific barrier.
823 if (_needs_keep_alive_barrier) {
824 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
825 Address gc_state_fast(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(state_to_check)));
826 __ ldrb(_tmp1, gc_state_fast);
827 maybe_far_jump_if_zero(masm, _tmp1);
828 }
829
830 // If weak references are being processed, weak/phantom loads need to go slow,
831 // regardless of their cset status.
832 if (_needs_load_ref_weak_barrier) {
833 Address gc_state_fast(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::WEAK_ROOTS)));
834 __ ldrb(_tmp1, gc_state_fast);
835 __ cbnz(_tmp1, L_slow);
836 }
837
838 // Cset-check. Fall-through to slow if in collection set.
839 bool is_aot = AOTCodeCache::is_on_for_dump();
840 if (!is_aot) {
841 __ mov(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
842 if (_narrow) {
843 __ decode_heap_oop_not_null(_tmp2, _obj);
844 __ add(_tmp1, _tmp1, _tmp2, Assembler::LSR, ShenandoahHeapRegion::region_size_bytes_shift_jint());
845 } else {
846 __ add(_tmp1, _tmp1, _obj, Assembler::LSR, ShenandoahHeapRegion::region_size_bytes_shift_jint());
847 }
848 } else {
849 // Generating AOT code, pull the cset bitmap and region shift from AOT table.
850 if (_narrow) {
851 __ decode_heap_oop_not_null(_tmp1, _obj);
852 } else {
853 __ mov(_tmp1, _obj);
854 }
855 __ lea(_tmp2, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
|
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
33 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interp_masm.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "nativeInst_aarch64.hpp"
39 #include "runtime/javaThread.hpp"
40 #include "runtime/sharedRuntime.hpp"
41 #ifdef COMPILER1
42 #include "c1/c1_LIRAssembler.hpp"
43 #include "c1/c1_MacroAssembler.hpp"
44 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
45 #endif
46 #ifdef COMPILER2
47 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
48 #include "opto/output.hpp"
49 #endif
50
51 #define __ masm->
52
53 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
54 Register src, Register dst, Register count, RegSet saved_regs) {
55 if (is_oop) {
56 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
57 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
58
651 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
652 __ ldr(tmp1, curr_ct_holder_addr);
653
654 // tmp2 = effective address
655 __ lea(tmp2, address);
656
657 // tmp2 = &card_table[ addr >> CardTable::card_shift() ] ; card index
658 __ add(tmp2, tmp1, tmp2, Assembler::LSR, CardTable::card_shift());
659
660 if (UseCondCardMark) {
661 Label L_already_dirty;
662 __ ldrb(tmp1, Address(tmp2));
663 __ cbz(tmp1, L_already_dirty);
664 __ strb(zr, Address(tmp2));
665 __ bind(L_already_dirty);
666 } else {
667 __ strb(zr, Address(tmp2));
668 }
669 }
670
671 void ShenandoahBarrierStubC2::patchable_jump_if_gc_state(MacroAssembler& masm, const char test_state, Label* L_target) {
672 // Emit the unconditional branch in the first version of the method.
673 // Let the rest of runtime figure out how to manage it.
674 __ relocate(patchable_barrier_Relocation::spec(ShenandoahNMethod::encode_to_reloc(test_state, false)));
675 __ b(*L_target);
676 }
677
678 void ShenandoahBarrierStubC2::patchable_jump_if_not_gc_state(MacroAssembler& masm, const char test_state, Label* L_target) {
679 // Emit the unconditional branch in the first version of the method.
680 // Let the rest of runtime figure out how to manage it.
681 __ relocate(patchable_barrier_Relocation::spec(ShenandoahNMethod::encode_to_reloc(test_state, true)));
682 __ b(*L_target);
683 }
684
685 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
686 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
687 PhaseOutput* const output = Compile::current()->output();
688
689 if (output->in_scratch_emit_size()) {
690 // We piggyback on scratch_emit_size mode to compute the slowpath stub size.
691 // We'll use that information to decide whether we need a far jump to the
692 // stub entry point or not. In scratch_emit_size mode we don't bind entry()
693 // because otherwise it will be rebound when we later emit the instructions
694 // for real.
695 __ nop();
696 } else {
697 patchable_jump_if_gc_state(masm, test_state, entry());
698 }
699
700 // This is were the slowpath stub will return to or the code above will
701 // jump to if the checks are false
702 __ bind(*continuation());
703 }
704
705 address ShenandoahBarrierSetAssembler::parse_stub_address(address pc) {
706 NativeInstruction* ni = nativeInstruction_at(pc);
707 assert(ni->is_jump(), "Initial code version: GC barrier fastpath must be a jump");
708 NativeJump* jmp = nativeJump_at(pc);
709 return jmp->jump_destination();
710 }
711
712 static bool is_patchable_nop(address pc) {
713 if (*(pc + 0) != 0x1F) return false;
714 if (*(pc + 1) != 0x20) return false;
715 if (*(pc + 2) != 0x03) return false;
716 if (*(pc + 3) != 0xD5) return false;
717 return true;
718 }
719
720 static void insert_patchable_nop(address pc) {
721 *reinterpret_cast<int32_t*>(pc) = 0xD503201F;
722 }
723
724 static void insert_patchable_jump(address code_pos, address entry) {
725 intptr_t disp = (intptr_t)entry - ((intptr_t)code_pos);
726 int64_t imm26 = disp >> 2;
727 guarantee(Assembler::is_simm(imm26, 26), "maximum offset is 128MiB, requested %ld", imm26);
728
729 uint32_t new_val = 0x14000000 | (imm26 & 0x03FFFFFF);
730 AtomicAccess::store((uint32_t*)code_pos, new_val);
731 }
732
733 static void check_at(bool cond, address pc, const char* msg) {
734 assert(cond, "%s: at PC " PTR_FORMAT ": %02x%02x%02x%02x",
735 msg, p2i(pc), *(pc + 0), *(pc + 1), *(pc + 2), *(pc + 3));
736 }
737
738 bool ShenandoahBarrierSetAssembler::patch_branch_to_nop(address pc) {
739 NativeInstruction* ni = nativeInstruction_at(pc);
740 bool patching = ni->is_jump();
741 if (patching) {
742 insert_patchable_nop(pc);
743 }
744 check_at(is_patchable_nop(pc), pc, "Should be nop");
745 return patching;
746 }
747
748 bool ShenandoahBarrierSetAssembler::patch_nop_to_branch(address pc, address stub_addr) {
749 bool patching = is_patchable_nop(pc);
750 if (patching) {
751 insert_patchable_jump(pc, stub_addr);
752 }
753
754 NativeInstruction* ni = nativeInstruction_at(pc);
755 check_at(ni->is_jump(), pc, "Should be jump");
756 check_at(nativeJump_at(pc)->jump_destination() == stub_addr, pc, "Jump should be to the same address");
757 return patching;
758 }
759
760 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
761 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
762 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
763 PhaseOutput* const output = Compile::current()->output();
764
765 // We piggyback on scratch_emit_size mode to compute the slowpath stub size.
766 // We'll use that information to decide whether we need a far jump to the
767 // stub entry point or not. In scratch_emit_size mode we don't bind entry()
768 // because otherwise it will be rebound when we later emit the instructions
769 // for real.
770 if (!output->in_scratch_emit_size()) {
771 __ bind(*entry());
772 }
773
774 // If we need to load ourselves, do it here.
775 if (_do_load) {
776 if (_narrow) {
777 __ ldrw(_obj, _addr);
778 } else {
779 __ ldr(_obj, _addr);
810 keepalive(masm, continuation());
811 } else if (_needs_load_ref_barrier) {
812 lrb(masm);
813 } else {
814 ShouldNotReachHere();
815 }
816 }
817
818 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
819 if (_needs_far_jump) {
820 Label L_short_jump;
821 __ cbnz(reg, L_short_jump);
822 __ b(*continuation());
823 __ bind(L_short_jump);
824 } else {
825 __ cbz(reg, *continuation());
826 }
827 }
828
829 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
830 Address index(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
831 Address buffer(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
832 Label L_through, L_slowpath;
833
834 // If another barrier is enabled as well, do a check for a specific barrier.
835 if (_needs_load_ref_barrier) {
836 assert(L_done == nullptr, "Should be");
837 char state_to_check = ShenandoahHeap::MARKING;
838 patchable_jump_if_not_gc_state(masm, state_to_check, &L_through);
839 }
840
841 // Fast-path: put object into buffer.
842 // If buffer is already full, go slow.
843 __ ldr(_tmp1, index);
844 __ cbz(_tmp1, L_slowpath);
845 __ sub(_tmp1, _tmp1, wordSize);
846 __ str(_tmp1, index);
847 __ ldr(_tmp2, buffer);
848
849 // Store the object in queue.
850 // If object is narrow, we need to decode it before inserting.
851 if (_narrow) {
852 __ add(_tmp2, _tmp2, _tmp1);
853 __ decode_heap_oop_not_null(_tmp1, _obj);
854 __ str(_tmp1, Address(_tmp2));
855 } else {
856 // Buffer is 64-bit address, must be in base register.
857 __ str(_obj, Address(_tmp2, _tmp1));
858 }
868 __ bind(L_slowpath);
869
870 {
871 SaveLiveRegisters slr(&masm, this);
872
873 // Go to runtime and handle the rest there.
874 __ mov(c_rarg0, _obj);
875 __ lea(lr, RuntimeAddress(keepalive_runtime_entry_addr()));
876 __ blr(lr);
877 }
878 if (L_done != nullptr) {
879 __ b(*L_done);
880 } else {
881 __ bind(L_through);
882 }
883 }
884
885 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
886 Label L_slow;
887
888 // If another barrier is enabled as well, do a check for a specific barrier.
889 if (_needs_keep_alive_barrier) {
890 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
891 patchable_jump_if_not_gc_state(masm, state_to_check, continuation());
892 }
893
894 // If weak references are being processed, weak/phantom loads need to go slow,
895 // regardless of their cset status.
896 if (_needs_load_ref_weak_barrier) {
897 char state_to_check = ShenandoahHeap::WEAK_ROOTS;
898 patchable_jump_if_gc_state(masm, state_to_check, &L_slow);
899 }
900
901 // Cset-check. Fall-through to slow if in collection set.
902 bool is_aot = AOTCodeCache::is_on_for_dump();
903 if (!is_aot) {
904 __ mov(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
905 if (_narrow) {
906 __ decode_heap_oop_not_null(_tmp2, _obj);
907 __ add(_tmp1, _tmp1, _tmp2, Assembler::LSR, ShenandoahHeapRegion::region_size_bytes_shift_jint());
908 } else {
909 __ add(_tmp1, _tmp1, _obj, Assembler::LSR, ShenandoahHeapRegion::region_size_bytes_shift_jint());
910 }
911 } else {
912 // Generating AOT code, pull the cset bitmap and region shift from AOT table.
913 if (_narrow) {
914 __ decode_heap_oop_not_null(_tmp1, _obj);
915 } else {
916 __ mov(_tmp1, _obj);
917 }
918 __ lea(_tmp2, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
|