13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
33 #include "gc/shenandoah/shenandoahRuntime.hpp"
34 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
35 #include "interpreter/interpreter.hpp"
36 #include "runtime/javaThread.hpp"
37 #include "runtime/sharedRuntime.hpp"
38 #include "utilities/macros.hpp"
39 #ifdef COMPILER1
40 #include "c1/c1_LIRAssembler.hpp"
41 #include "c1/c1_MacroAssembler.hpp"
42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
43 #endif
44 #ifdef COMPILER2
45 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
46 #endif
47
48 #define __ masm->
49
50 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
51 Register src, Register dst, Register count) {
52
53 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
54
55 if (is_reference_type(type)) {
56 if (ShenandoahCardBarrier) {
57 bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
58 bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
59 bool obj_int = (type == T_OBJECT) && UseCompressedOops;
60
61 // We need to save the original element count because the array copy stub
62 // will destroy the value and we need it for the card marking barrier.
63 if (!checkcast) {
64 if (!obj_int) {
65 // Save count for barrier
561 __ movptr(tmp, curr_ct_holder_addr);
562
563 __ leaq(end, Address(addr, count, TIMES_OOP, 0)); // end == addr+count*oop_size
564 __ subptr(end, BytesPerHeapOop); // end - 1 to make inclusive
565 __ shrptr(addr, CardTable::card_shift());
566 __ shrptr(end, CardTable::card_shift());
567 __ subptr(end, addr); // end --> cards count
568
569 __ addptr(addr, tmp);
570
571 __ BIND(L_loop);
572 __ movb(Address(addr, count, Address::times_1), 0);
573 __ decrement(count);
574 __ jccb(Assembler::greaterEqual, L_loop);
575
576 __ BIND(L_done);
577 }
578
579 #undef __
580
581 #ifdef COMPILER1
582
583 #define __ ce->masm()->
584
585 void ShenandoahBarrierSetAssembler::keepalive_barrier_c1_stub(LIR_Assembler* ce, ShenandoahKeepaliveBarrierStub* stub) {
586 __ bind(*stub->entry());
587
588 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
589
590 Register obj = stub->obj()->as_register();
591
592 if (stub->do_load()) {
593 ce->mem2reg(stub->addr(), stub->obj(), T_OBJECT, lir_patch_none, nullptr, /* wide = */ false);
594 }
595 __ cmpptr(obj, NULL_WORD);
596 __ jcc(Assembler::equal, *stub->continuation());
597
598 ce->store_parameter(obj, 0);
599 __ call(RuntimeAddress(bs->keepalive_barrier_stub()));
600 __ jmp(*stub->continuation());
749
750 __ lea(tmp1, addr);
751 __ shrptr(tmp1, CardTable::card_shift());
752 __ addptr(tmp1, Address(r15_thread, in_bytes(ShenandoahThreadLocalData::card_table_offset())));
753 Address card_address(tmp1, 0);
754
755 assert(CardTable::dirty_card_val() == 0, "Encoding assumption");
756 Label L_done;
757 if (UseCondCardMark) {
758 __ cmpb(card_address, 0);
759 __ jccb(Assembler::equal, L_done);
760 }
761 if (UseCompressedOops && CompressedOops::base() == nullptr) {
762 __ movb(card_address, r12);
763 } else {
764 __ movb(card_address, 0);
765 }
766 __ bind(L_done);
767 }
768
769 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
770 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
771
772 Address gc_state_fast(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(test_state)));
773 __ cmpb(gc_state_fast, 0);
774 __ jcc(Assembler::notEqual, *entry());
775 __ bind(*continuation());
776 }
777
778 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
779 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
780 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
781
782 // On x86, there is a significant penalty with unaligned branch target, for example
783 // when the target instruction straggles the fetch line. It makes (performance) sense
784 // to spend some code size to align the target better.
785 __ align(16);
786 __ bind(*entry());
787
788 // If we need to load ourselves, do it here.
789 if (_do_load) {
790 if (_narrow) {
791 __ movl(_obj, _addr);
792 } else {
793 __ movq(_obj, _addr);
794 }
795 }
796
797 // If the object is null, there is no point in applying barriers.
802 bool needs_both_barriers = _needs_keep_alive_barrier && _needs_load_ref_barrier;
803 if (!_do_load || needs_both_barriers) {
804 preserve(_obj);
805 }
806
807 // Go for barriers. Barriers can return straight to continuation, as long
808 // as another barrier is not needed.
809 if (needs_both_barriers) {
810 keepalive(masm, nullptr);
811 lrb(masm);
812 } else if (_needs_keep_alive_barrier) {
813 keepalive(masm, continuation());
814 } else if (_needs_load_ref_barrier) {
815 lrb(masm);
816 } else {
817 ShouldNotReachHere();
818 }
819 }
820
821 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
822 Address gc_state_fast(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::MARKING)));
823 Address index(r15_thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
824 Address buffer(r15_thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
825
826 Label L_through, L_pop_and_slow;
827
828 // If another barrier is enabled as well, do a runtime check for a specific barrier.
829 if (_needs_load_ref_barrier) {
830 assert(L_done == nullptr, "L_done is always null when _needs_load_ref_barrier is true");
831 __ cmpb(gc_state_fast, 0);
832 __ jcc(Assembler::equal, L_through);
833 }
834
835 // Need temp to work, allocate one now.
836 bool tmp_live;
837 Register tmp = select_temp_register(tmp_live);
838 if (tmp_live) {
839 __ push(tmp);
840 }
841
842 // Fast-path: put object into buffer.
843 // If buffer is already full, go slow.
844 __ movptr(tmp, index);
845 __ subptr(tmp, wordSize);
846 __ jccb(Assembler::below, L_pop_and_slow);
847 __ movptr(index, tmp);
848 __ addptr(tmp, buffer);
849
850 // Store the object in queue.
851 // If object is narrow, we need to decode it before inserting.
852 // We can skip the re-encoding if we know that object is not preserved.
881 // Shuffle in the arguments. The end result should be:
882 // c_rarg0 <-- obj
883 if (c_rarg0 != _obj) {
884 __ mov(c_rarg0, _obj);
885 }
886
887 // Go to runtime and handle the rest there.
888 // Use rax as scratch, as it will be saved if live.
889 __ call(RuntimeAddress(keepalive_runtime_entry_addr()), rax);
890 }
891 if (L_done != nullptr) {
892 __ jmp(*L_done);
893 } else {
894 __ bind(L_through);
895 }
896 }
897
898 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
899 Label L_pop_and_slow, L_slow;
900
901 // If another barrier is enabled as well, do a runtime check for a specific barrier.
902 if (_needs_keep_alive_barrier) {
903 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
904 Address gc_state_fast(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(state_to_check)));
905 __ cmpb(gc_state_fast, 0);
906 __ jcc(Assembler::equal, *continuation());
907 }
908
909 // If weak references are being processed, weak/phantom loads need to go slow,
910 // regardless of their cset status.
911 if (_needs_load_ref_weak_barrier) {
912 Address gc_state_fast(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::WEAK_ROOTS)));
913 __ cmpb(gc_state_fast, 0);
914 __ jccb(Assembler::notEqual, L_slow);
915 }
916
917 bool is_aot = AOTCodeCache::is_on_for_dump();
918
919 // Need temp to work, allocate one now.
920 bool tmp_live;
921 Register tmp = select_temp_register(tmp_live, /* skip_reg1 = */ is_aot ? rcx : noreg);
922 if (tmp_live) {
923 __ push(tmp);
924 }
925
926 // Compute the cset bitmap index
927 if (_narrow) {
928 __ decode_heap_oop_not_null(tmp, _obj);
929 } else {
930 __ movptr(tmp, _obj);
931 }
932
933 Address cset_addr_arg;
934 intptr_t cset_addr = reinterpret_cast<intptr_t>(ShenandoahHeap::in_cset_fast_test_addr());
|
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
33 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "nativeInst_x86.hpp"
38 #include "runtime/javaThread.hpp"
39 #include "runtime/sharedRuntime.hpp"
40 #include "utilities/macros.hpp"
41 #ifdef COMPILER1
42 #include "c1/c1_LIRAssembler.hpp"
43 #include "c1/c1_MacroAssembler.hpp"
44 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
45 #endif
46 #ifdef COMPILER2
47 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
48 #include "opto/output.hpp"
49 #endif
50
51 #define __ masm->
52
53 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
54 Register src, Register dst, Register count) {
55
56 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
57
58 if (is_reference_type(type)) {
59 if (ShenandoahCardBarrier) {
60 bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
61 bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
62 bool obj_int = (type == T_OBJECT) && UseCompressedOops;
63
64 // We need to save the original element count because the array copy stub
65 // will destroy the value and we need it for the card marking barrier.
66 if (!checkcast) {
67 if (!obj_int) {
68 // Save count for barrier
564 __ movptr(tmp, curr_ct_holder_addr);
565
566 __ leaq(end, Address(addr, count, TIMES_OOP, 0)); // end == addr+count*oop_size
567 __ subptr(end, BytesPerHeapOop); // end - 1 to make inclusive
568 __ shrptr(addr, CardTable::card_shift());
569 __ shrptr(end, CardTable::card_shift());
570 __ subptr(end, addr); // end --> cards count
571
572 __ addptr(addr, tmp);
573
574 __ BIND(L_loop);
575 __ movb(Address(addr, count, Address::times_1), 0);
576 __ decrement(count);
577 __ jccb(Assembler::greaterEqual, L_loop);
578
579 __ BIND(L_done);
580 }
581
582 #undef __
583
584 address ShenandoahBarrierSetAssembler::parse_jump_address(address pc) {
585 NativeInstruction* ni = nativeInstruction_at(pc);
586 assert(ni->is_jump(), "Must be a jump");
587 NativeJump* jmp = nativeJump_at(pc);
588 return jmp->jump_destination();
589 }
590
591 void ShenandoahBarrierSetAssembler::insert_patchable_nop(address pc) {
592 *(pc + 0) = 0x0F;
593 *(pc + 1) = 0x1F;
594 *(pc + 2) = 0x44;
595 *(pc + 3) = 0x00;
596 *(pc + 4) = 0x00;
597 }
598
599 bool ShenandoahBarrierSetAssembler::is_patchable_nop(address pc) {
600 if (*(pc + 0) != 0x0F) return false;
601 if (*(pc + 1) != 0x1F) return false;
602 if (*(pc + 2) != 0x44) return false;
603 if (*(pc + 3) != 0x00) return false;
604 if (*(pc + 4) != 0x00) return false;
605 return true;
606 }
607
608 void ShenandoahBarrierSetAssembler::insert_patchable_jump(address pc, address target_pc) {
609 int32_t disp = checked_cast<int32_t>((intptr_t)target_pc - ((intptr_t)pc + 5));
610
611 *(pc + 0) = 0xE9;
612 *(pc + 1) = (disp >> 0) & 0xFF;
613 *(pc + 2) = (disp >> 8) & 0xFF;
614 *(pc + 3) = (disp >> 16) & 0xFF;
615 *(pc + 4) = (disp >> 24) & 0xFF;
616 }
617
618 bool ShenandoahBarrierSetAssembler::is_patchable_jump(address pc, address target_pc) {
619 int32_t disp = checked_cast<int32_t>((intptr_t)target_pc - ((intptr_t)pc + 5));
620
621 if (*(pc + 0) != 0xE9) return false;
622 if (*(pc + 1) != ((disp >> 0) & 0xFF)) return false;
623 if (*(pc + 2) != ((disp >> 8) & 0xFF)) return false;
624 if (*(pc + 3) != ((disp >> 16) & 0xFF)) return false;
625 if (*(pc + 4) != ((disp >> 24) & 0xFF)) return false;
626 return true;
627 }
628
629 #ifdef COMPILER1
630
631 #define __ ce->masm()->
632
633 void ShenandoahBarrierSetAssembler::keepalive_barrier_c1_stub(LIR_Assembler* ce, ShenandoahKeepaliveBarrierStub* stub) {
634 __ bind(*stub->entry());
635
636 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
637
638 Register obj = stub->obj()->as_register();
639
640 if (stub->do_load()) {
641 ce->mem2reg(stub->addr(), stub->obj(), T_OBJECT, lir_patch_none, nullptr, /* wide = */ false);
642 }
643 __ cmpptr(obj, NULL_WORD);
644 __ jcc(Assembler::equal, *stub->continuation());
645
646 ce->store_parameter(obj, 0);
647 __ call(RuntimeAddress(bs->keepalive_barrier_stub()));
648 __ jmp(*stub->continuation());
797
798 __ lea(tmp1, addr);
799 __ shrptr(tmp1, CardTable::card_shift());
800 __ addptr(tmp1, Address(r15_thread, in_bytes(ShenandoahThreadLocalData::card_table_offset())));
801 Address card_address(tmp1, 0);
802
803 assert(CardTable::dirty_card_val() == 0, "Encoding assumption");
804 Label L_done;
805 if (UseCondCardMark) {
806 __ cmpb(card_address, 0);
807 __ jccb(Assembler::equal, L_done);
808 }
809 if (UseCompressedOops && CompressedOops::base() == nullptr) {
810 __ movb(card_address, r12);
811 } else {
812 __ movb(card_address, 0);
813 }
814 __ bind(L_done);
815 }
816
817 void ShenandoahBarrierStubC2::patchable_jump(MacroAssembler& masm, const char gc_state, bool jump_when_state, Label* L_target) {
818 PhaseOutput* const output = Compile::current()->output();
819 if (output->in_scratch_emit_size()) {
820 // Avoid binding L_target in scratch emits.
821 // We know the patchable check is exactly 5 bytes long.
822 __ nop(5);
823 return;
824 }
825
826 // Emit the unconditional branch in the first version of the method.
827 // Let the rest of runtime figure out how to manage it.
828 __ relocate(patchable_barrier_Relocation::spec(ShenandoahNMethod::encode_to_reloc(gc_state, jump_when_state)));
829 __ jmp(*L_target, /* maybe_short = */ false);
830 }
831
832 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state) {
833 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
834 patchable_jump_if_gc_state(masm, test_state, entry());
835 __ bind(*continuation());
836 }
837
838
839 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
840 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
841 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
842
843 // On x86, there is a significant penalty with unaligned branch target, for example
844 // when the target instruction straggles the fetch line. It makes (performance) sense
845 // to spend some code size to align the target better.
846 __ align(16);
847 __ bind(*entry());
848
849 // If we need to load ourselves, do it here.
850 if (_do_load) {
851 if (_narrow) {
852 __ movl(_obj, _addr);
853 } else {
854 __ movq(_obj, _addr);
855 }
856 }
857
858 // If the object is null, there is no point in applying barriers.
863 bool needs_both_barriers = _needs_keep_alive_barrier && _needs_load_ref_barrier;
864 if (!_do_load || needs_both_barriers) {
865 preserve(_obj);
866 }
867
868 // Go for barriers. Barriers can return straight to continuation, as long
869 // as another barrier is not needed.
870 if (needs_both_barriers) {
871 keepalive(masm, nullptr);
872 lrb(masm);
873 } else if (_needs_keep_alive_barrier) {
874 keepalive(masm, continuation());
875 } else if (_needs_load_ref_barrier) {
876 lrb(masm);
877 } else {
878 ShouldNotReachHere();
879 }
880 }
881
882 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
883 Address index(r15_thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
884 Address buffer(r15_thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
885
886 Label L_through, L_pop_and_slow;
887
888 // If another barrier is enabled as well, do a check for a specific barrier.
889 if (_needs_load_ref_barrier) {
890 assert(L_done == nullptr, "Should be");
891 char state_to_check = ShenandoahHeap::MARKING;
892 patchable_jump_if_not_gc_state(masm, state_to_check, &L_through);
893 }
894
895 // Need temp to work, allocate one now.
896 bool tmp_live;
897 Register tmp = select_temp_register(tmp_live);
898 if (tmp_live) {
899 __ push(tmp);
900 }
901
902 // Fast-path: put object into buffer.
903 // If buffer is already full, go slow.
904 __ movptr(tmp, index);
905 __ subptr(tmp, wordSize);
906 __ jccb(Assembler::below, L_pop_and_slow);
907 __ movptr(index, tmp);
908 __ addptr(tmp, buffer);
909
910 // Store the object in queue.
911 // If object is narrow, we need to decode it before inserting.
912 // We can skip the re-encoding if we know that object is not preserved.
941 // Shuffle in the arguments. The end result should be:
942 // c_rarg0 <-- obj
943 if (c_rarg0 != _obj) {
944 __ mov(c_rarg0, _obj);
945 }
946
947 // Go to runtime and handle the rest there.
948 // Use rax as scratch, as it will be saved if live.
949 __ call(RuntimeAddress(keepalive_runtime_entry_addr()), rax);
950 }
951 if (L_done != nullptr) {
952 __ jmp(*L_done);
953 } else {
954 __ bind(L_through);
955 }
956 }
957
958 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
959 Label L_pop_and_slow, L_slow;
960
961 // If another barrier is enabled as well, do a check for a specific barrier.
962 if (_needs_keep_alive_barrier) {
963 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
964 patchable_jump_if_not_gc_state(masm, state_to_check, continuation());
965 }
966
967 // If weak references are being processed, weak/phantom loads need to go slow,
968 // regardless of their cset status.
969 if (_needs_load_ref_weak_barrier) {
970 char state_to_check = ShenandoahHeap::WEAK_ROOTS;
971 patchable_jump_if_gc_state(masm, state_to_check, &L_slow);
972 }
973
974 bool is_aot = AOTCodeCache::is_on_for_dump();
975
976 // Need temp to work, allocate one now.
977 bool tmp_live;
978 Register tmp = select_temp_register(tmp_live, /* skip_reg1 = */ is_aot ? rcx : noreg);
979 if (tmp_live) {
980 __ push(tmp);
981 }
982
983 // Compute the cset bitmap index
984 if (_narrow) {
985 __ decode_heap_oop_not_null(tmp, _obj);
986 } else {
987 __ movptr(tmp, _obj);
988 }
989
990 Address cset_addr_arg;
991 intptr_t cset_addr = reinterpret_cast<intptr_t>(ShenandoahHeap::in_cset_fast_test_addr());
|