13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
33 #include "gc/shenandoah/shenandoahRuntime.hpp"
34 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
35 #include "interpreter/interpreter.hpp"
36 #include "runtime/javaThread.hpp"
37 #include "runtime/sharedRuntime.hpp"
38 #include "utilities/macros.hpp"
39 #ifdef COMPILER1
40 #include "c1/c1_LIRAssembler.hpp"
41 #include "c1/c1_MacroAssembler.hpp"
42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
43 #endif
44 #ifdef COMPILER2
45 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
46 #endif
47
48 #define __ masm->
49
50 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
51 Register src, Register dst, Register count) {
52
53 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
54
55 if (is_reference_type(type)) {
724
725 __ lea(tmp1, addr);
726 __ shrptr(tmp1, CardTable::card_shift());
727 __ addptr(tmp1, Address(r15_thread, in_bytes(ShenandoahThreadLocalData::card_table_offset())));
728 Address card_address(tmp1, 0);
729
730 assert(CardTable::dirty_card_val() == 0, "Encoding assumption");
731 Label L_done;
732 if (UseCondCardMark) {
733 __ cmpb(card_address, 0);
734 __ jccb(Assembler::equal, L_done);
735 }
736 if (UseCompressedOops && CompressedOops::base() == nullptr) {
737 __ movb(card_address, r12);
738 } else {
739 __ movb(card_address, 0);
740 }
741 __ bind(L_done);
742 }
743
744 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
745 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
746
747 Address gc_state_fast(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(test_state)));
748 __ cmpb(gc_state_fast, 0);
749 __ jcc(Assembler::notEqual, *entry());
750 __ bind(*continuation());
751 }
752
753 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
754 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
755 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
756
757 // On x86, there is a significant penalty with unaligned branch target, for example
758 // when the target instruction straggles the fetch line. It makes (performance) sense
759 // to spend some code size to align the target better.
760 __ align(16);
761 __ bind(*entry());
762
763 // If we need to load ourselves, do it here.
764 if (_do_load) {
765 if (_narrow) {
766 __ movl(_obj, _addr);
767 } else {
768 __ movq(_obj, _addr);
769 }
770 }
771
772 // If the object is null, there is no point in applying barriers.
777 bool needs_both_barriers = _needs_keep_alive_barrier && _needs_load_ref_barrier;
778 if (!_do_load || needs_both_barriers) {
779 preserve(_obj);
780 }
781
782 // Go for barriers. Barriers can return straight to continuation, as long
783 // as another barrier is not needed.
784 if (needs_both_barriers) {
785 keepalive(masm, nullptr);
786 lrb(masm);
787 } else if (_needs_keep_alive_barrier) {
788 keepalive(masm, continuation());
789 } else if (_needs_load_ref_barrier) {
790 lrb(masm);
791 } else {
792 ShouldNotReachHere();
793 }
794 }
795
796 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
797 Address gc_state_fast(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::MARKING)));
798 Address index(r15_thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
799 Address buffer(r15_thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
800
801 Label L_through, L_pop_and_slow;
802
803 // If another barrier is enabled as well, do a runtime check for a specific barrier.
804 if (_needs_load_ref_barrier) {
805 assert(L_done == nullptr, "L_done is always null when _needs_load_ref_barrier is true");
806 __ cmpb(gc_state_fast, 0);
807 __ jcc(Assembler::equal, L_through);
808 }
809
810 // Need temp to work, allocate one now.
811 bool tmp_live;
812 Register tmp = select_temp_register(tmp_live);
813 if (tmp_live) {
814 __ push(tmp);
815 }
816
817 // Fast-path: put object into buffer.
818 // If buffer is already full, go slow.
819 __ movptr(tmp, index);
820 __ subptr(tmp, wordSize);
821 __ jccb(Assembler::below, L_pop_and_slow);
822 __ movptr(index, tmp);
823 __ addptr(tmp, buffer);
824
825 // Store the object in queue.
826 // If object is narrow, we need to decode it before inserting.
827 // We can skip the re-encoding if we know that object is not preserved.
856 // Shuffle in the arguments. The end result should be:
857 // c_rarg0 <-- obj
858 if (c_rarg0 != _obj) {
859 __ mov(c_rarg0, _obj);
860 }
861
862 // Go to runtime and handle the rest there.
863 // Use rax as scratch, as it will be saved if live.
864 __ call(RuntimeAddress(keepalive_runtime_entry_addr()), rax);
865 }
866 if (L_done != nullptr) {
867 __ jmp(*L_done);
868 } else {
869 __ bind(L_through);
870 }
871 }
872
873 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
874 Label L_pop_and_slow, L_slow;
875
876 // If another barrier is enabled as well, do a runtime check for a specific barrier.
877 if (_needs_keep_alive_barrier) {
878 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
879 Address gc_state_fast(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(state_to_check)));
880 __ cmpb(gc_state_fast, 0);
881 __ jcc(Assembler::equal, *continuation());
882 }
883
884 // If weak references are being processed, weak/phantom loads need to go slow,
885 // regardless of their cset status.
886 if (_needs_load_ref_weak_barrier) {
887 Address gc_state_fast(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::WEAK_ROOTS)));
888 __ cmpb(gc_state_fast, 0);
889 __ jccb(Assembler::notEqual, L_slow);
890 }
891
892 bool is_aot = AOTCodeCache::is_on_for_dump();
893
894 // Need temp to work, allocate one now.
895 bool tmp_live;
896 Register tmp = select_temp_register(tmp_live, /* skip_reg1 = */ is_aot ? rcx : noreg);
897 if (tmp_live) {
898 __ push(tmp);
899 }
900
901 // Compute the cset bitmap index
902 if (_narrow) {
903 __ decode_heap_oop_not_null(tmp, _obj);
904 } else {
905 __ movptr(tmp, _obj);
906 }
907
908 Address cset_addr_arg;
909 intptr_t cset_addr = reinterpret_cast<intptr_t>(ShenandoahHeap::in_cset_fast_test_addr());
|
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
33 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "nativeInst_x86.hpp"
38 #include "runtime/javaThread.hpp"
39 #include "runtime/sharedRuntime.hpp"
40 #include "utilities/macros.hpp"
41 #ifdef COMPILER1
42 #include "c1/c1_LIRAssembler.hpp"
43 #include "c1/c1_MacroAssembler.hpp"
44 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
45 #endif
46 #ifdef COMPILER2
47 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
48 #endif
49
50 #define __ masm->
51
52 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
53 Register src, Register dst, Register count) {
54
55 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
56
57 if (is_reference_type(type)) {
726
727 __ lea(tmp1, addr);
728 __ shrptr(tmp1, CardTable::card_shift());
729 __ addptr(tmp1, Address(r15_thread, in_bytes(ShenandoahThreadLocalData::card_table_offset())));
730 Address card_address(tmp1, 0);
731
732 assert(CardTable::dirty_card_val() == 0, "Encoding assumption");
733 Label L_done;
734 if (UseCondCardMark) {
735 __ cmpb(card_address, 0);
736 __ jccb(Assembler::equal, L_done);
737 }
738 if (UseCompressedOops && CompressedOops::base() == nullptr) {
739 __ movb(card_address, r12);
740 } else {
741 __ movb(card_address, 0);
742 }
743 __ bind(L_done);
744 }
745
746 void ShenandoahBarrierStubC2::patchable_jump_if_gc_state(MacroAssembler& masm, const char test_state, Label* L_target) {
747 // Emit the unconditional branch in the first version of the method.
748 // Let the rest of runtime figure out how to manage it.
749 __ relocate(patchable_barrier_Relocation::spec(ShenandoahNMethod::encode_to_reloc(test_state, false)));
750 __ jmp(*L_target, /* maybe_short = */ false);
751 }
752
753 void ShenandoahBarrierStubC2::patchable_jump_if_not_gc_state(MacroAssembler& masm, const char test_state, Label* L_target) {
754 // Emit the unconditional branch in the first version of the method.
755 // Let the rest of runtime figure out how to manage it.
756 __ relocate(patchable_barrier_Relocation::spec(ShenandoahNMethod::encode_to_reloc(test_state, true)));
757 __ jmp(*L_target, /* maybe_short = */ false);
758 }
759
760 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
761 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
762 patchable_jump_if_gc_state(masm, test_state, entry());
763 __ bind(*continuation());
764 }
765
766 address ShenandoahBarrierSetAssembler::parse_stub_address(address pc) {
767 NativeInstruction* ni = nativeInstruction_at(pc);
768 assert(ni->is_jump(), "Initial code version: GC barrier fastpath must be a jump");
769 NativeJump* jmp = nativeJump_at(pc);
770 return jmp->jump_destination();
771 }
772
773 static void insert_patchable_nop(address pc) {
774 *(pc + 0) = 0x0F;
775 *(pc + 1) = 0x1F;
776 *(pc + 2) = 0x44;
777 *(pc + 3) = 0x00;
778 *(pc + 4) = 0x00;
779 }
780
781 static bool is_patchable_nop(address pc) {
782 if (*(pc + 0) != 0x0F) return false;
783 if (*(pc + 1) != 0x1F) return false;
784 if (*(pc + 2) != 0x44) return false;
785 if (*(pc + 3) != 0x00) return false;
786 if (*(pc + 4) != 0x00) return false;
787 return true;
788 }
789
790 static void insert_patchable_jump(address code_pos, address entry) {
791 // Borrowed from NativeJump::insert
792 intptr_t disp = (intptr_t)entry - ((intptr_t)code_pos + 1 + 4);
793 guarantee(disp == (intptr_t)(int32_t)disp, "must be 32-bit offset");
794 *code_pos = 0xE9;
795 *((int32_t*)(code_pos + 1)) = (int32_t)disp;
796 }
797
798 static void check_at(bool cond, address pc, const char* msg) {
799 assert(cond, "%s: at PC " PTR_FORMAT ": %02x%02x%02x%02x%02x",
800 msg, p2i(pc), *(pc + 0), *(pc + 1), *(pc + 2), *(pc + 3), *(pc + 4));
801 }
802
803 bool ShenandoahBarrierSetAssembler::patch_branch_to_nop(address pc) {
804 NativeInstruction* ni = nativeInstruction_at(pc);
805 bool patching = ni->is_jump();
806 if (patching) {
807 insert_patchable_nop(pc);
808 }
809 check_at(is_patchable_nop(pc), pc, "Should be nop");
810 return patching;
811 }
812
813 bool ShenandoahBarrierSetAssembler::patch_nop_to_branch(address pc, address stub_addr) {
814 bool patching = is_patchable_nop(pc);
815 if (patching) {
816 insert_patchable_jump(pc, stub_addr);
817 }
818
819 NativeInstruction* ni = nativeInstruction_at(pc);
820 check_at(ni->is_jump(), pc, "Should be jump");
821 check_at(nativeJump_at(pc)->jump_destination() == stub_addr, pc, "Jump should be to the same address");
822 return patching;
823 }
824
825 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
826 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
827 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
828
829 // On x86, there is a significant penalty with unaligned branch target, for example
830 // when the target instruction straggles the fetch line. It makes (performance) sense
831 // to spend some code size to align the target better.
832 __ align(16);
833 __ bind(*entry());
834
835 // If we need to load ourselves, do it here.
836 if (_do_load) {
837 if (_narrow) {
838 __ movl(_obj, _addr);
839 } else {
840 __ movq(_obj, _addr);
841 }
842 }
843
844 // If the object is null, there is no point in applying barriers.
849 bool needs_both_barriers = _needs_keep_alive_barrier && _needs_load_ref_barrier;
850 if (!_do_load || needs_both_barriers) {
851 preserve(_obj);
852 }
853
854 // Go for barriers. Barriers can return straight to continuation, as long
855 // as another barrier is not needed.
856 if (needs_both_barriers) {
857 keepalive(masm, nullptr);
858 lrb(masm);
859 } else if (_needs_keep_alive_barrier) {
860 keepalive(masm, continuation());
861 } else if (_needs_load_ref_barrier) {
862 lrb(masm);
863 } else {
864 ShouldNotReachHere();
865 }
866 }
867
868 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
869 Address index(r15_thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
870 Address buffer(r15_thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
871
872 Label L_through, L_pop_and_slow;
873
874 // If another barrier is enabled as well, do a check for a specific barrier.
875 if (_needs_load_ref_barrier) {
876 assert(L_done == nullptr, "Should be");
877 char state_to_check = ShenandoahHeap::MARKING;
878 patchable_jump_if_not_gc_state(masm, state_to_check, &L_through);
879 }
880
881 // Need temp to work, allocate one now.
882 bool tmp_live;
883 Register tmp = select_temp_register(tmp_live);
884 if (tmp_live) {
885 __ push(tmp);
886 }
887
888 // Fast-path: put object into buffer.
889 // If buffer is already full, go slow.
890 __ movptr(tmp, index);
891 __ subptr(tmp, wordSize);
892 __ jccb(Assembler::below, L_pop_and_slow);
893 __ movptr(index, tmp);
894 __ addptr(tmp, buffer);
895
896 // Store the object in queue.
897 // If object is narrow, we need to decode it before inserting.
898 // We can skip the re-encoding if we know that object is not preserved.
927 // Shuffle in the arguments. The end result should be:
928 // c_rarg0 <-- obj
929 if (c_rarg0 != _obj) {
930 __ mov(c_rarg0, _obj);
931 }
932
933 // Go to runtime and handle the rest there.
934 // Use rax as scratch, as it will be saved if live.
935 __ call(RuntimeAddress(keepalive_runtime_entry_addr()), rax);
936 }
937 if (L_done != nullptr) {
938 __ jmp(*L_done);
939 } else {
940 __ bind(L_through);
941 }
942 }
943
944 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
945 Label L_pop_and_slow, L_slow;
946
947 // If another barrier is enabled as well, do a check for a specific barrier.
948 if (_needs_keep_alive_barrier) {
949 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
950 patchable_jump_if_not_gc_state(masm, state_to_check, continuation());
951 }
952
953 // If weak references are being processed, weak/phantom loads need to go slow,
954 // regardless of their cset status.
955 if (_needs_load_ref_weak_barrier) {
956 char state_to_check = ShenandoahHeap::WEAK_ROOTS;
957 patchable_jump_if_gc_state(masm, state_to_check, &L_slow);
958 }
959
960 bool is_aot = AOTCodeCache::is_on_for_dump();
961
962 // Need temp to work, allocate one now.
963 bool tmp_live;
964 Register tmp = select_temp_register(tmp_live, /* skip_reg1 = */ is_aot ? rcx : noreg);
965 if (tmp_live) {
966 __ push(tmp);
967 }
968
969 // Compute the cset bitmap index
970 if (_narrow) {
971 __ decode_heap_oop_not_null(tmp, _obj);
972 } else {
973 __ movptr(tmp, _obj);
974 }
975
976 Address cset_addr_arg;
977 intptr_t cset_addr = reinterpret_cast<intptr_t>(ShenandoahHeap::in_cset_fast_test_addr());
|