14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "code/aotCodeCache.hpp"
28 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
29 #include "gc/shenandoah/mode/shenandoahMode.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
31 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interp_masm.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "runtime/javaThread.hpp"
39 #include "runtime/sharedRuntime.hpp"
40 #ifdef COMPILER1
41 #include "c1/c1_LIRAssembler.hpp"
42 #include "c1/c1_MacroAssembler.hpp"
43 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
44 #endif
45 #ifdef COMPILER2
46 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
47 #include "opto/output.hpp"
48 #endif
49
50 #define __ masm->
51
52 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
53 Register src, Register dst, Register count, RegSet saved_regs) {
54 if (is_oop) {
55 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
56 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
57
496 __ srli(start, start, CardTable::card_shift());
497 __ srli(end, end, CardTable::card_shift());
498
499 // number of bytes to copy
500 __ sub(count, end, start);
501
502 Address curr_ct_holder_addr(xthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
503 __ ld(tmp, curr_ct_holder_addr);
504 __ add(start, start, tmp);
505
506 __ bind(L_loop);
507 __ add(tmp, start, count);
508 __ sb(zr, Address(tmp));
509 __ subi(count, count, 1);
510 __ bgez(count, L_loop);
511 __ bind(L_done);
512 }
513
514 #undef __
515
516 #ifdef COMPILER1
517
518 #define __ ce->masm()->
519
520 void ShenandoahBarrierSetAssembler::keepalive_barrier_c1_stub(LIR_Assembler* ce, ShenandoahKeepaliveBarrierStub* stub) {
521 __ bind(*stub->entry());
522
523 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
524
525 Register obj = stub->obj()->as_register();
526
527 if (stub->do_load()) {
528 ce->mem2reg(stub->addr(), stub->obj(), T_OBJECT, lir_patch_none, nullptr, false /* wide */);
529 }
530 __ beqz(obj, *stub->continuation(), /* is_far */ true);
531
532 ce->store_parameter(obj, 0);
533 __ far_call(RuntimeAddress(bs->keepalive_barrier_stub()));
534 __ j(*stub->continuation());
535 }
677 __ ld(tmp1, curr_ct_holder_addr);
678
679 // tmp1 = effective address
680 __ la(tmp2, address);
681
682 // tmp2 = &card_table[ addr >> CardTable::card_shift() ] ; card index
683 __ srli(tmp2, tmp2, CardTable::card_shift());
684 __ add(tmp2, tmp2, tmp1);
685
686 if (UseCondCardMark) {
687 Label L_already_dirty;
688 __ lbu(tmp1, Address(tmp2));
689 __ beqz(tmp1, L_already_dirty);
690 __ sb(zr, Address(tmp2));
691 __ bind(L_already_dirty);
692 } else {
693 __ sb(zr, Address(tmp2));
694 }
695 }
696
697 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
698 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
699
700 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(test_state)));
701 __ lbu(tmp, gc_state_fast);
702 __ beqz(tmp, *continuation());
703 __ j(*entry());
704
705 // This is were the slowpath stub will return to or the code above will
706 // jump to if the checks are false
707 __ bind(*continuation());
708 }
709
710 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
711 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
712 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
713
714 __ bind(*entry());
715
716 // If we need to load ourselves, do it here.
717 if (_do_load) {
718 if (_narrow) {
719 __ lwu(_obj, _addr);
720 } else {
721 __ ld(_obj, _addr);
722 }
723 }
724
725 // If the object is null, there is no point in applying barriers.
726 maybe_far_jump_if_zero(masm, _obj);
741 keepalive(masm, continuation());
742 } else if (_needs_load_ref_barrier) {
743 lrb(masm);
744 } else {
745 ShouldNotReachHere();
746 }
747 }
748
749 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
750 Label L_short_jump;
751 __ bnez(reg, L_short_jump);
752 __ j(*continuation());
753 __ bind(L_short_jump);
754 }
755
756 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
757 Address index(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
758 Address buffer(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
759 Label L_through, L_slowpath;
760
761 // If another barrier is enabled as well, do a runtime check for a specific barrier.
762 if (_needs_load_ref_barrier) {
763 assert(L_done == nullptr, "L_done is always null when _needs_load_ref_barrier is true");
764 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::MARKING)));
765 __ lbu(_tmp1, gc_state_fast);
766 __ beqz(_tmp1, L_through);
767 }
768
769 // Fast-path: put object into buffer.
770 // If buffer is already full, go slow.
771 __ ld(_tmp1, index);
772 __ beqz(_tmp1, L_slowpath);
773 __ subi(_tmp1, _tmp1, wordSize);
774 __ sd(_tmp1, index);
775 __ ld(_tmp2, buffer);
776
777 // Store the object in queue.
778 // If object is narrow, we need to decode it before inserting.
779 __ add(_tmp1, _tmp1, _tmp2);
780 if (_narrow) {
781 __ decode_heap_oop_not_null(_tmp2, _obj);
782 __ sd(_tmp2, Address(_tmp1));
783 } else {
784 __ sd(_obj, Address(_tmp1));
785 }
786
795 __ bind(L_slowpath);
796
797 {
798 SaveLiveRegisters slr(&masm, this);
799
800 // Go to runtime and handle the rest there.
801 __ mv(c_rarg0, _obj);
802 __ la(ra, RuntimeAddress(keepalive_runtime_entry_addr()));
803 __ jalr(ra);
804 }
805 if (L_done != nullptr) {
806 __ j(*L_done);
807 } else {
808 __ bind(L_through);
809 }
810 }
811
812 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
813 Label L_slow;
814
815 // If another barrier is enabled as well, do a runtime check for a specific barrier.
816 if (_needs_keep_alive_barrier) {
817 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
818 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(state_to_check)));
819 __ lbu(_tmp1, gc_state_fast);
820 maybe_far_jump_if_zero(masm, _tmp1);
821 }
822
823 // If weak references are being processed, weak/phantom loads need to go slow,
824 // regardless of their cset status.
825 if (_needs_load_ref_weak_barrier) {
826 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::WEAK_ROOTS)));
827 __ lbu(_tmp1, gc_state_fast);
828 __ bnez(_tmp1, L_slow);
829 }
830
831 // Cset-check. Fall-through to slow if in collection set.
832 if (_narrow) {
833 __ decode_heap_oop_not_null(_tmp2, _obj);
834 } else {
835 __ mv(_tmp2, _obj);
836 }
837
838 if (AOTCodeCache::is_on_for_dump()) {
839 __ lwu(_tmp1, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
840 __ srl(_tmp2, _tmp2, _tmp1);
841 __ ld(_tmp1, ExternalAddress(AOTRuntimeConstants::cset_base_address()));
842 } else {
843 __ mv(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
844 __ srli(_tmp2, _tmp2, ShenandoahHeapRegion::region_size_bytes_shift_jint());
845 }
846 __ add(_tmp1, _tmp1, _tmp2);
847 __ lbu(_tmp1, Address(_tmp1, 0));
848 maybe_far_jump_if_zero(masm, _tmp1);
|
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "code/aotCodeCache.hpp"
28 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
29 #include "gc/shenandoah/mode/shenandoahMode.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
31 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
35 #include "gc/shenandoah/shenandoahRuntime.hpp"
36 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
37 #include "interpreter/interp_masm.hpp"
38 #include "interpreter/interpreter.hpp"
39 #include "nativeInst_riscv.hpp"
40 #include "runtime/icache.hpp"
41 #include "runtime/javaThread.hpp"
42 #include "runtime/sharedRuntime.hpp"
43 #ifdef COMPILER1
44 #include "c1/c1_LIRAssembler.hpp"
45 #include "c1/c1_MacroAssembler.hpp"
46 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
47 #endif
48 #ifdef COMPILER2
49 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
50 #include "opto/output.hpp"
51 #endif
52
53 #define __ masm->
54
55 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
56 Register src, Register dst, Register count, RegSet saved_regs) {
57 if (is_oop) {
58 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
59 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
60
499 __ srli(start, start, CardTable::card_shift());
500 __ srli(end, end, CardTable::card_shift());
501
502 // number of bytes to copy
503 __ sub(count, end, start);
504
505 Address curr_ct_holder_addr(xthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
506 __ ld(tmp, curr_ct_holder_addr);
507 __ add(start, start, tmp);
508
509 __ bind(L_loop);
510 __ add(tmp, start, count);
511 __ sb(zr, Address(tmp));
512 __ subi(count, count, 1);
513 __ bgez(count, L_loop);
514 __ bind(L_done);
515 }
516
517 #undef __
518
519 address ShenandoahBarrierSetAssembler::parse_jump_address(address pc) {
520 NativeInstruction* ni = nativeInstruction_at(pc);
521 assert(ni->is_jump(), "Must be a jump");
522 NativeJump* jmp = nativeJump_at(pc);
523 return jmp->jump_destination();
524 }
525
526 static uint32_t encode_patchable_nop() {
527 return 0x00000013;
528 }
529
530 static uint32_t encode_patchable_jump(address pc, address target_pc) {
531 int32_t disp = checked_cast<int32_t>((intptr_t)target_pc - (intptr_t)pc);
532 return Assembler::encode_jal(x0, disp);
533 }
534
535 void ShenandoahBarrierSetAssembler::insert_patchable_nop(address pc) {
536 *((uint32_t*)pc) = encode_patchable_nop();
537 assert(nativeInstruction_at(pc)->is_nop(), "Sanity");
538 if (!UseCtxFencei) {
539 ICache::invalidate_word(pc);
540 }
541 }
542
543 void ShenandoahBarrierSetAssembler::insert_patchable_jump(address pc, address target_pc) {
544 *((uint32_t*)pc) = encode_patchable_jump(pc, target_pc);
545 if (!UseCtxFencei) {
546 ICache::invalidate_word(pc);
547 }
548 }
549
550 bool ShenandoahBarrierSetAssembler::is_patchable_nop(address pc) {
551 return *((uint32_t*)pc) == encode_patchable_nop();
552 }
553
554 bool ShenandoahBarrierSetAssembler::is_patchable_jump(address pc, address target_pc) {
555 return *((uint32_t*)pc) == encode_patchable_jump(pc, target_pc);
556 }
557
558 #ifdef COMPILER1
559
560 #define __ ce->masm()->
561
562 void ShenandoahBarrierSetAssembler::keepalive_barrier_c1_stub(LIR_Assembler* ce, ShenandoahKeepaliveBarrierStub* stub) {
563 __ bind(*stub->entry());
564
565 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
566
567 Register obj = stub->obj()->as_register();
568
569 if (stub->do_load()) {
570 ce->mem2reg(stub->addr(), stub->obj(), T_OBJECT, lir_patch_none, nullptr, false /* wide */);
571 }
572 __ beqz(obj, *stub->continuation(), /* is_far */ true);
573
574 ce->store_parameter(obj, 0);
575 __ far_call(RuntimeAddress(bs->keepalive_barrier_stub()));
576 __ j(*stub->continuation());
577 }
719 __ ld(tmp1, curr_ct_holder_addr);
720
721 // tmp1 = effective address
722 __ la(tmp2, address);
723
724 // tmp2 = &card_table[ addr >> CardTable::card_shift() ] ; card index
725 __ srli(tmp2, tmp2, CardTable::card_shift());
726 __ add(tmp2, tmp2, tmp1);
727
728 if (UseCondCardMark) {
729 Label L_already_dirty;
730 __ lbu(tmp1, Address(tmp2));
731 __ beqz(tmp1, L_already_dirty);
732 __ sb(zr, Address(tmp2));
733 __ bind(L_already_dirty);
734 } else {
735 __ sb(zr, Address(tmp2));
736 }
737 }
738
739 void ShenandoahBarrierStubC2::patchable_jump(MacroAssembler& masm, const char gc_state, bool jump_when_state, Register tmp1, Register tmp2, Label* L_target) {
740 PhaseOutput* const output = Compile::current()->output();
741 if (output->in_scratch_emit_size()) {
742 // Avoid binding L_target in scratch emits.
743 // We know the patched check is exactly one incompressible instruction long.
744 Assembler::IncompressibleScope scope(&masm);
745 __ nop();
746 return;
747 }
748
749 // Emit the unconditional branch in the first version of the method.
750 // Let the rest of runtime figure out how to manage it.
751 __ relocate(patchable_barrier_Relocation::spec(ShenandoahNMethod::encode_to_reloc(gc_state, jump_when_state)));
752 __ j(*L_target);
753 }
754
755 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp1, Register tmp2) {
756 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
757 patchable_jump_if_gc_state(masm, test_state, tmp1, tmp2, entry());
758 __ bind(*continuation());
759 }
760
761 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
762 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
763 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
764
765 __ bind(*entry());
766
767 // If we need to load ourselves, do it here.
768 if (_do_load) {
769 if (_narrow) {
770 __ lwu(_obj, _addr);
771 } else {
772 __ ld(_obj, _addr);
773 }
774 }
775
776 // If the object is null, there is no point in applying barriers.
777 maybe_far_jump_if_zero(masm, _obj);
792 keepalive(masm, continuation());
793 } else if (_needs_load_ref_barrier) {
794 lrb(masm);
795 } else {
796 ShouldNotReachHere();
797 }
798 }
799
800 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
801 Label L_short_jump;
802 __ bnez(reg, L_short_jump);
803 __ j(*continuation());
804 __ bind(L_short_jump);
805 }
806
807 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
808 Address index(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
809 Address buffer(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
810 Label L_through, L_slowpath;
811
812 // If another barrier is enabled as well, do a check for a specific barrier.
813 if (_needs_load_ref_barrier) {
814 assert(L_done == nullptr, "Should be");
815 char state_to_check = ShenandoahHeap::MARKING;
816 patchable_jump_if_not_gc_state(masm, state_to_check, _tmp1, _tmp2, &L_through);
817 }
818
819 // Fast-path: put object into buffer.
820 // If buffer is already full, go slow.
821 __ ld(_tmp1, index);
822 __ beqz(_tmp1, L_slowpath);
823 __ subi(_tmp1, _tmp1, wordSize);
824 __ sd(_tmp1, index);
825 __ ld(_tmp2, buffer);
826
827 // Store the object in queue.
828 // If object is narrow, we need to decode it before inserting.
829 __ add(_tmp1, _tmp1, _tmp2);
830 if (_narrow) {
831 __ decode_heap_oop_not_null(_tmp2, _obj);
832 __ sd(_tmp2, Address(_tmp1));
833 } else {
834 __ sd(_obj, Address(_tmp1));
835 }
836
845 __ bind(L_slowpath);
846
847 {
848 SaveLiveRegisters slr(&masm, this);
849
850 // Go to runtime and handle the rest there.
851 __ mv(c_rarg0, _obj);
852 __ la(ra, RuntimeAddress(keepalive_runtime_entry_addr()));
853 __ jalr(ra);
854 }
855 if (L_done != nullptr) {
856 __ j(*L_done);
857 } else {
858 __ bind(L_through);
859 }
860 }
861
862 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
863 Label L_slow;
864
865 // If another barrier is enabled as well, do a check for a specific barrier.
866 if (_needs_keep_alive_barrier) {
867 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
868 patchable_jump_if_not_gc_state(masm, state_to_check, _tmp1, _tmp2, continuation());
869 }
870
871 // If weak references are being processed, weak/phantom loads need to go slow,
872 // regardless of their cset status.
873 if (_needs_load_ref_weak_barrier) {
874 char state_to_check = ShenandoahHeap::WEAK_ROOTS;
875 patchable_jump_if_gc_state(masm, state_to_check, _tmp1, _tmp2, &L_slow);
876 }
877
878 // Cset-check. Fall-through to slow if in collection set.
879 if (_narrow) {
880 __ decode_heap_oop_not_null(_tmp2, _obj);
881 } else {
882 __ mv(_tmp2, _obj);
883 }
884
885 if (AOTCodeCache::is_on_for_dump()) {
886 __ lwu(_tmp1, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
887 __ srl(_tmp2, _tmp2, _tmp1);
888 __ ld(_tmp1, ExternalAddress(AOTRuntimeConstants::cset_base_address()));
889 } else {
890 __ mv(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
891 __ srli(_tmp2, _tmp2, ShenandoahHeapRegion::region_size_bytes_shift_jint());
892 }
893 __ add(_tmp1, _tmp1, _tmp2);
894 __ lbu(_tmp1, Address(_tmp1, 0));
895 maybe_far_jump_if_zero(masm, _tmp1);
|