13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
33 #include "gc/shenandoah/shenandoahRuntime.hpp"
34 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
35 #include "interpreter/interp_masm.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "runtime/javaThread.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #ifdef COMPILER1
40 #include "c1/c1_LIRAssembler.hpp"
41 #include "c1/c1_MacroAssembler.hpp"
42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
43 #endif
44 #ifdef COMPILER2
45 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
46 #include "opto/output.hpp"
47 #endif
48
49 #define __ masm->
50
51 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
52 Register src, Register dst, Register count, RegSet saved_regs) {
53 if (is_oop) {
54 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
55 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
56
461 __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop)));
462 __ sub(end, end, BytesPerHeapOop);
463 __ lsr(start, start, CardTable::card_shift());
464 __ lsr(end, end, CardTable::card_shift());
465
466 // number of bytes to copy
467 __ sub(count, end, start);
468
469 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
470 __ ldr(scratch, curr_ct_holder_addr);
471 __ add(start, start, scratch);
472 __ bind(L_loop);
473 __ strb(zr, Address(start, count));
474 __ subs(count, count, 1);
475 __ br(Assembler::GE, L_loop);
476 __ bind(L_done);
477 }
478
479 #undef __
480
481 #ifdef COMPILER1
482
483 #define __ ce->masm()->
484
485 void ShenandoahBarrierSetAssembler::keepalive_barrier_c1_stub(LIR_Assembler* ce, ShenandoahKeepaliveBarrierStub* stub) {
486 __ bind(*stub->entry());
487
488 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
489
490 Register obj = stub->obj()->as_register();
491
492 if (stub->do_load()) {
493 ce->mem2reg(stub->addr(), stub->obj(), T_OBJECT, lir_patch_none, nullptr, /* wide = */ false);
494 }
495 __ cbz(obj, *stub->continuation());
496 ce->store_parameter(obj, 0);
497 __ far_call(RuntimeAddress(bs->keepalive_barrier_stub()));
498 __ b(*stub->continuation());
499 }
500
675 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
676 __ ldr(tmp1, curr_ct_holder_addr);
677
678 // tmp2 = effective address
679 __ lea(tmp2, address);
680
681 // tmp2 = &card_table[ addr >> CardTable::card_shift() ] ; card index
682 __ add(tmp2, tmp1, tmp2, Assembler::LSR, CardTable::card_shift());
683
684 if (UseCondCardMark) {
685 Label L_already_dirty;
686 __ ldrb(tmp1, Address(tmp2));
687 __ cbz(tmp1, L_already_dirty);
688 __ strb(zr, Address(tmp2));
689 __ bind(L_already_dirty);
690 } else {
691 __ strb(zr, Address(tmp2));
692 }
693 }
694
695 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
696 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
697 PhaseOutput* const output = Compile::current()->output();
698 Address gc_state_fast(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(test_state)));
699
700 // We piggyback on scratch_emit_size mode to compute the slowpath stub size.
701 // We'll use that information to decide whether we need a far jump to the
702 // stub entry point or not. In scratch_emit_size mode we don't bind entry()
703 // because otherwise it will be rebound when we later emit the instructions
704 // for real.
705 if (_needs_far_jump) {
706 __ ldrb(tmp, gc_state_fast);
707 __ cbz(tmp, *continuation());
708 __ b(output->in_scratch_emit_size() ? *continuation() : *entry());
709 } else {
710 __ ldrb(tmp, gc_state_fast);
711 __ cbnz(tmp, output->in_scratch_emit_size() ? *continuation() : *entry());
712 }
713
714 // This is were the slowpath stub will return to or the code above will
715 // jump to if the checks are false
716 __ bind(*continuation());
717 }
718
719 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
720 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
721 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
722 PhaseOutput* const output = Compile::current()->output();
723
724 // We piggyback on scratch_emit_size mode to compute the slowpath stub size.
725 // We'll use that information to decide whether we need a far jump to the
726 // stub entry point or not. In scratch_emit_size mode we don't bind entry()
727 // because otherwise it will be rebound when we later emit the instructions
728 // for real.
729 if (!output->in_scratch_emit_size()) {
730 __ bind(*entry());
731 }
732
733 // If we need to load ourselves, do it here.
734 if (_do_load) {
735 if (_narrow) {
769 keepalive(masm, continuation());
770 } else if (_needs_load_ref_barrier) {
771 lrb(masm);
772 } else {
773 ShouldNotReachHere();
774 }
775 }
776
777 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
778 if (_needs_far_jump) {
779 Label L_short_jump;
780 __ cbnz(reg, L_short_jump);
781 __ b(*continuation());
782 __ bind(L_short_jump);
783 } else {
784 __ cbz(reg, *continuation());
785 }
786 }
787
788 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
789 Address gcstate(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::MARKING)));
790 Address index(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
791 Address buffer(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
792 Label L_through, L_slowpath;
793
794 // If another barrier is enabled as well, do a runtime check for a specific barrier.
795 if (_needs_load_ref_barrier) {
796 assert(L_done == nullptr, "L_done is always null when _needs_load_ref_barrier is true");
797 __ ldrb(_tmp1, gcstate);
798 __ cbz(_tmp1, L_through);
799 }
800
801 // Fast-path: put object into buffer.
802 // If buffer is already full, go slow.
803 __ ldr(_tmp1, index);
804 __ cbz(_tmp1, L_slowpath);
805 __ sub(_tmp1, _tmp1, wordSize);
806 __ str(_tmp1, index);
807 __ ldr(_tmp2, buffer);
808
809 // Store the object in queue.
810 // If object is narrow, we need to decode it before inserting.
811 if (_narrow) {
812 __ add(_tmp2, _tmp2, _tmp1);
813 __ decode_heap_oop_not_null(_tmp1, _obj);
814 __ str(_tmp1, Address(_tmp2));
815 } else {
816 // Buffer is 64-bit address, must be in base register.
817 __ str(_obj, Address(_tmp2, _tmp1));
818 }
828 __ bind(L_slowpath);
829
830 {
831 SaveLiveRegisters slr(&masm, this);
832
833 // Go to runtime and handle the rest there.
834 __ mov(c_rarg0, _obj);
835 __ lea(lr, RuntimeAddress(keepalive_runtime_entry_addr()));
836 __ blr(lr);
837 }
838 if (L_done != nullptr) {
839 __ b(*L_done);
840 } else {
841 __ bind(L_through);
842 }
843 }
844
845 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
846 Label L_slow;
847
848 // If another barrier is enabled as well, do a runtime check for a specific barrier.
849 if (_needs_keep_alive_barrier) {
850 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
851 Address gc_state_fast(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(state_to_check)));
852 __ ldrb(_tmp1, gc_state_fast);
853 maybe_far_jump_if_zero(masm, _tmp1);
854 }
855
856 // If weak references are being processed, weak/phantom loads need to go slow,
857 // regardless of their cset status.
858 if (_needs_load_ref_weak_barrier) {
859 Address gc_state_fast(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::WEAK_ROOTS)));
860 __ ldrb(_tmp1, gc_state_fast);
861 __ cbnz(_tmp1, L_slow);
862 }
863
864 // Cset-check. Fall-through to slow if in collection set.
865 bool is_aot = AOTCodeCache::is_on_for_dump();
866 if (!is_aot) {
867 __ mov(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
868 if (_narrow) {
869 __ decode_heap_oop_not_null(_tmp2, _obj);
870 __ add(_tmp1, _tmp1, _tmp2, Assembler::LSR, ShenandoahHeapRegion::region_size_bytes_shift_jint());
871 } else {
872 __ add(_tmp1, _tmp1, _obj, Assembler::LSR, ShenandoahHeapRegion::region_size_bytes_shift_jint());
873 }
874 } else {
875 // Generating AOT code, pull the cset bitmap and region shift from AOT table.
876 if (_narrow) {
877 __ decode_heap_oop_not_null(_tmp1, _obj);
878 } else {
879 __ mov(_tmp1, _obj);
880 }
881 __ lea(_tmp2, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
|
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
33 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interp_masm.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "nativeInst_aarch64.hpp"
39 #include "runtime/javaThread.hpp"
40 #include "runtime/sharedRuntime.hpp"
41 #ifdef COMPILER1
42 #include "c1/c1_LIRAssembler.hpp"
43 #include "c1/c1_MacroAssembler.hpp"
44 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
45 #endif
46 #ifdef COMPILER2
47 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
48 #include "opto/output.hpp"
49 #endif
50
51 #define __ masm->
52
53 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
54 Register src, Register dst, Register count, RegSet saved_regs) {
55 if (is_oop) {
56 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
57 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
58
463 __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop)));
464 __ sub(end, end, BytesPerHeapOop);
465 __ lsr(start, start, CardTable::card_shift());
466 __ lsr(end, end, CardTable::card_shift());
467
468 // number of bytes to copy
469 __ sub(count, end, start);
470
471 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
472 __ ldr(scratch, curr_ct_holder_addr);
473 __ add(start, start, scratch);
474 __ bind(L_loop);
475 __ strb(zr, Address(start, count));
476 __ subs(count, count, 1);
477 __ br(Assembler::GE, L_loop);
478 __ bind(L_done);
479 }
480
481 #undef __
482
483 address ShenandoahBarrierSetAssembler::parse_jump_address(address pc) {
484 NativeInstruction* ni = nativeInstruction_at(pc);
485 assert(ni->is_jump(), "Initial code version: GC barrier fastpath must be a jump");
486 NativeJump* jmp = nativeJump_at(pc);
487 return jmp->jump_destination();
488 }
489
490 static uint32_t encode_patchable_nop() {
491 return 0xD503201F;
492 }
493
494 static uint32_t encode_patchable_jump(address pc, address target_pc) {
495 int32_t disp = checked_cast<int32_t>((intptr_t)target_pc - (intptr_t)pc);
496 int64_t imm26 = disp >> 2;
497 assert(Assembler::is_simm(imm26, 26), "maximum offset is 128MiB");
498 return 0x14000000 | (imm26 & 0x03FFFFFF);
499 }
500
501 void ShenandoahBarrierSetAssembler::insert_patchable_nop(address pc) {
502 *((uint32_t*)pc) = encode_patchable_nop();
503 }
504
505 void ShenandoahBarrierSetAssembler::insert_patchable_jump(address pc, address target_pc) {
506 *((uint32_t*)pc) = encode_patchable_jump(pc, target_pc);
507 }
508
509 bool ShenandoahBarrierSetAssembler::is_patchable_nop(address pc) {
510 return *((uint32_t*)pc) == encode_patchable_nop();
511 }
512
513 bool ShenandoahBarrierSetAssembler::is_patchable_jump(address pc, address target_pc) {
514 return *((uint32_t*)pc) == encode_patchable_jump(pc, target_pc);
515 }
516
517 #ifdef COMPILER1
518
519 #define __ ce->masm()->
520
521 void ShenandoahBarrierSetAssembler::keepalive_barrier_c1_stub(LIR_Assembler* ce, ShenandoahKeepaliveBarrierStub* stub) {
522 __ bind(*stub->entry());
523
524 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
525
526 Register obj = stub->obj()->as_register();
527
528 if (stub->do_load()) {
529 ce->mem2reg(stub->addr(), stub->obj(), T_OBJECT, lir_patch_none, nullptr, /* wide = */ false);
530 }
531 __ cbz(obj, *stub->continuation());
532 ce->store_parameter(obj, 0);
533 __ far_call(RuntimeAddress(bs->keepalive_barrier_stub()));
534 __ b(*stub->continuation());
535 }
536
711 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
712 __ ldr(tmp1, curr_ct_holder_addr);
713
714 // tmp2 = effective address
715 __ lea(tmp2, address);
716
717 // tmp2 = &card_table[ addr >> CardTable::card_shift() ] ; card index
718 __ add(tmp2, tmp1, tmp2, Assembler::LSR, CardTable::card_shift());
719
720 if (UseCondCardMark) {
721 Label L_already_dirty;
722 __ ldrb(tmp1, Address(tmp2));
723 __ cbz(tmp1, L_already_dirty);
724 __ strb(zr, Address(tmp2));
725 __ bind(L_already_dirty);
726 } else {
727 __ strb(zr, Address(tmp2));
728 }
729 }
730
731 void ShenandoahBarrierStubC2::patchable_jump(MacroAssembler& masm, const char gc_state, bool jump_when_state, Label* L_target) {
732 PhaseOutput* const output = Compile::current()->output();
733 if (output->in_scratch_emit_size()) {
734 // Avoid binding L_target in scratch emits.
735 // We know the patched check is exactly one instruction long.
736 __ nop();
737 return;
738 }
739
740 // Emit the unconditional branch in the first version of the method.
741 // Let the rest of runtime figure out how to manage it.
742 __ relocate(patchable_barrier_Relocation::spec(ShenandoahNMethod::encode_to_reloc(gc_state, jump_when_state)));
743 __ b(*L_target);
744 }
745
746 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state) {
747 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
748 patchable_jump_if_gc_state(masm, test_state, entry());
749 __ bind(*continuation());
750 }
751
752 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
753 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
754 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
755 PhaseOutput* const output = Compile::current()->output();
756
757 // We piggyback on scratch_emit_size mode to compute the slowpath stub size.
758 // We'll use that information to decide whether we need a far jump to the
759 // stub entry point or not. In scratch_emit_size mode we don't bind entry()
760 // because otherwise it will be rebound when we later emit the instructions
761 // for real.
762 if (!output->in_scratch_emit_size()) {
763 __ bind(*entry());
764 }
765
766 // If we need to load ourselves, do it here.
767 if (_do_load) {
768 if (_narrow) {
802 keepalive(masm, continuation());
803 } else if (_needs_load_ref_barrier) {
804 lrb(masm);
805 } else {
806 ShouldNotReachHere();
807 }
808 }
809
810 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
811 if (_needs_far_jump) {
812 Label L_short_jump;
813 __ cbnz(reg, L_short_jump);
814 __ b(*continuation());
815 __ bind(L_short_jump);
816 } else {
817 __ cbz(reg, *continuation());
818 }
819 }
820
821 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
822 Address index(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
823 Address buffer(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
824 Label L_through, L_slowpath;
825
826 // If another barrier is enabled as well, do a check for a specific barrier.
827 if (_needs_load_ref_barrier) {
828 assert(L_done == nullptr, "Should be");
829 char state_to_check = ShenandoahHeap::MARKING;
830 patchable_jump_if_not_gc_state(masm, state_to_check, &L_through);
831 }
832
833 // Fast-path: put object into buffer.
834 // If buffer is already full, go slow.
835 __ ldr(_tmp1, index);
836 __ cbz(_tmp1, L_slowpath);
837 __ sub(_tmp1, _tmp1, wordSize);
838 __ str(_tmp1, index);
839 __ ldr(_tmp2, buffer);
840
841 // Store the object in queue.
842 // If object is narrow, we need to decode it before inserting.
843 if (_narrow) {
844 __ add(_tmp2, _tmp2, _tmp1);
845 __ decode_heap_oop_not_null(_tmp1, _obj);
846 __ str(_tmp1, Address(_tmp2));
847 } else {
848 // Buffer is 64-bit address, must be in base register.
849 __ str(_obj, Address(_tmp2, _tmp1));
850 }
860 __ bind(L_slowpath);
861
862 {
863 SaveLiveRegisters slr(&masm, this);
864
865 // Go to runtime and handle the rest there.
866 __ mov(c_rarg0, _obj);
867 __ lea(lr, RuntimeAddress(keepalive_runtime_entry_addr()));
868 __ blr(lr);
869 }
870 if (L_done != nullptr) {
871 __ b(*L_done);
872 } else {
873 __ bind(L_through);
874 }
875 }
876
877 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
878 Label L_slow;
879
880 // If another barrier is enabled as well, do a check for a specific barrier.
881 if (_needs_keep_alive_barrier) {
882 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
883 patchable_jump_if_not_gc_state(masm, state_to_check, continuation());
884 }
885
886 // If weak references are being processed, weak/phantom loads need to go slow,
887 // regardless of their cset status.
888 if (_needs_load_ref_weak_barrier) {
889 char state_to_check = ShenandoahHeap::WEAK_ROOTS;
890 patchable_jump_if_gc_state(masm, state_to_check, &L_slow);
891 }
892
893 // Cset-check. Fall-through to slow if in collection set.
894 bool is_aot = AOTCodeCache::is_on_for_dump();
895 if (!is_aot) {
896 __ mov(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
897 if (_narrow) {
898 __ decode_heap_oop_not_null(_tmp2, _obj);
899 __ add(_tmp1, _tmp1, _tmp2, Assembler::LSR, ShenandoahHeapRegion::region_size_bytes_shift_jint());
900 } else {
901 __ add(_tmp1, _tmp1, _obj, Assembler::LSR, ShenandoahHeapRegion::region_size_bytes_shift_jint());
902 }
903 } else {
904 // Generating AOT code, pull the cset bitmap and region shift from AOT table.
905 if (_narrow) {
906 __ decode_heap_oop_not_null(_tmp1, _obj);
907 } else {
908 __ mov(_tmp1, _obj);
909 }
910 __ lea(_tmp2, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
|