13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
33 #include "gc/shenandoah/shenandoahRuntime.hpp"
34 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
35 #include "interpreter/interp_masm.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "runtime/javaThread.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #ifdef COMPILER1
40 #include "c1/c1_LIRAssembler.hpp"
41 #include "c1/c1_MacroAssembler.hpp"
42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
43 #endif
44 #ifdef COMPILER2
45 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
46 #include "opto/output.hpp"
47 #endif
48
49 #define __ masm->
50
51 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
52 Register src, Register dst, Register count, RegSet saved_regs) {
53 if (is_oop) {
54 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
55 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
56
482 __ srli(start, start, CardTable::card_shift());
483 __ srli(end, end, CardTable::card_shift());
484
485 // number of bytes to copy
486 __ sub(count, end, start);
487
488 Address curr_ct_holder_addr(xthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
489 __ ld(tmp, curr_ct_holder_addr);
490 __ add(start, start, tmp);
491
492 __ bind(L_loop);
493 __ add(tmp, start, count);
494 __ sb(zr, Address(tmp));
495 __ subi(count, count, 1);
496 __ bgez(count, L_loop);
497 __ bind(L_done);
498 }
499
500 #undef __
501
502 #ifdef COMPILER1
503
504 #define __ ce->masm()->
505
506 void ShenandoahBarrierSetAssembler::keepalive_barrier_c1_stub(LIR_Assembler* ce, ShenandoahKeepaliveBarrierStub* stub) {
507 __ bind(*stub->entry());
508
509 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
510
511 Register obj = stub->obj()->as_register();
512
513 if (stub->do_load()) {
514 ce->mem2reg(stub->addr(), stub->obj(), T_OBJECT, lir_patch_none, nullptr, false /* wide */);
515 }
516 __ beqz(obj, *stub->continuation(), /* is_far */ true);
517
518 ce->store_parameter(obj, 0);
519 __ far_call(RuntimeAddress(bs->keepalive_barrier_stub()));
520 __ j(*stub->continuation());
521 }
663 __ ld(tmp1, curr_ct_holder_addr);
664
665 // tmp1 = effective address
666 __ la(tmp2, address);
667
668 // tmp2 = &card_table[ addr >> CardTable::card_shift() ] ; card index
669 __ srli(tmp2, tmp2, CardTable::card_shift());
670 __ add(tmp2, tmp2, tmp1);
671
672 if (UseCondCardMark) {
673 Label L_already_dirty;
674 __ lbu(tmp1, Address(tmp2));
675 __ beqz(tmp1, L_already_dirty);
676 __ sb(zr, Address(tmp2));
677 __ bind(L_already_dirty);
678 } else {
679 __ sb(zr, Address(tmp2));
680 }
681 }
682
683 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
684 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
685
686 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(test_state)));
687 __ lbu(tmp, gc_state_fast);
688 __ beqz(tmp, *continuation());
689 __ j(*entry());
690
691 // This is were the slowpath stub will return to or the code above will
692 // jump to if the checks are false
693 __ bind(*continuation());
694 }
695
696 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
697 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
698 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
699
700 __ bind(*entry());
701
702 // If we need to load ourselves, do it here.
703 if (_do_load) {
704 if (_narrow) {
705 __ lwu(_obj, _addr);
706 } else {
707 __ ld(_obj, _addr);
708 }
709 }
710
711 // If the object is null, there is no point in applying barriers.
712 maybe_far_jump_if_zero(masm, _obj);
727 keepalive(masm, continuation());
728 } else if (_needs_load_ref_barrier) {
729 lrb(masm);
730 } else {
731 ShouldNotReachHere();
732 }
733 }
734
735 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
736 Label L_short_jump;
737 __ bnez(reg, L_short_jump);
738 __ j(*continuation());
739 __ bind(L_short_jump);
740 }
741
742 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
743 Address index(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
744 Address buffer(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
745 Label L_through, L_slowpath;
746
747 // If another barrier is enabled as well, do a runtime check for a specific barrier.
748 if (_needs_load_ref_barrier) {
749 assert(L_done == nullptr, "L_done is always null when _needs_load_ref_barrier is true");
750 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::MARKING)));
751 __ lbu(_tmp1, gc_state_fast);
752 __ beqz(_tmp1, L_through);
753 }
754
755 // Fast-path: put object into buffer.
756 // If buffer is already full, go slow.
757 __ ld(_tmp1, index);
758 __ beqz(_tmp1, L_slowpath);
759 __ subi(_tmp1, _tmp1, wordSize);
760 __ sd(_tmp1, index);
761 __ ld(_tmp2, buffer);
762
763 // Store the object in queue.
764 // If object is narrow, we need to decode it before inserting.
765 __ add(_tmp1, _tmp1, _tmp2);
766 if (_narrow) {
767 __ decode_heap_oop_not_null(_tmp2, _obj);
768 __ sd(_tmp2, Address(_tmp1));
769 } else {
770 __ sd(_obj, Address(_tmp1));
771 }
772
781 __ bind(L_slowpath);
782
783 {
784 SaveLiveRegisters slr(&masm, this);
785
786 // Go to runtime and handle the rest there.
787 __ mv(c_rarg0, _obj);
788 __ la(ra, RuntimeAddress(keepalive_runtime_entry_addr()));
789 __ jalr(ra);
790 }
791 if (L_done != nullptr) {
792 __ j(*L_done);
793 } else {
794 __ bind(L_through);
795 }
796 }
797
798 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
799 Label L_slow;
800
801 // If another barrier is enabled as well, do a runtime check for a specific barrier.
802 if (_needs_keep_alive_barrier) {
803 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
804 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(state_to_check)));
805 __ lbu(_tmp1, gc_state_fast);
806 maybe_far_jump_if_zero(masm, _tmp1);
807 }
808
809 // If weak references are being processed, weak/phantom loads need to go slow,
810 // regardless of their cset status.
811 if (_needs_load_ref_weak_barrier) {
812 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::WEAK_ROOTS)));
813 __ lbu(_tmp1, gc_state_fast);
814 __ bnez(_tmp1, L_slow);
815 }
816
817 // Cset-check. Fall-through to slow if in collection set.
818 if (_narrow) {
819 __ decode_heap_oop_not_null(_tmp2, _obj);
820 } else {
821 __ mv(_tmp2, _obj);
822 }
823
824 if (AOTCodeCache::is_on_for_dump()) {
825 __ lwu(_tmp1, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
826 __ srl(_tmp2, _tmp2, _tmp1);
827 __ ld(_tmp1, ExternalAddress(AOTRuntimeConstants::cset_base_address()));
828 } else {
829 __ mv(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
830 __ srli(_tmp2, _tmp2, ShenandoahHeapRegion::region_size_bytes_shift_jint());
831 }
832 __ add(_tmp1, _tmp1, _tmp2);
833 __ lbu(_tmp1, Address(_tmp1, 0));
834 maybe_far_jump_if_zero(masm, _tmp1);
|
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
33 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interp_masm.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "nativeInst_riscv.hpp"
39 #include "runtime/javaThread.hpp"
40 #include "runtime/sharedRuntime.hpp"
41 #ifdef COMPILER1
42 #include "c1/c1_LIRAssembler.hpp"
43 #include "c1/c1_MacroAssembler.hpp"
44 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
45 #endif
46 #ifdef COMPILER2
47 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
48 #include "opto/output.hpp"
49 #endif
50
51 #define __ masm->
52
53 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
54 Register src, Register dst, Register count, RegSet saved_regs) {
55 if (is_oop) {
56 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
57 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
58
484 __ srli(start, start, CardTable::card_shift());
485 __ srli(end, end, CardTable::card_shift());
486
487 // number of bytes to copy
488 __ sub(count, end, start);
489
490 Address curr_ct_holder_addr(xthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
491 __ ld(tmp, curr_ct_holder_addr);
492 __ add(start, start, tmp);
493
494 __ bind(L_loop);
495 __ add(tmp, start, count);
496 __ sb(zr, Address(tmp));
497 __ subi(count, count, 1);
498 __ bgez(count, L_loop);
499 __ bind(L_done);
500 }
501
502 #undef __
503
504 address ShenandoahBarrierSetAssembler::parse_jump_address(address pc) {
505 NativeInstruction* ni = nativeInstruction_at(pc);
506 assert(ni->is_jump(), "Must be a jump");
507 NativeJump* jmp = nativeJump_at(pc);
508 return jmp->jump_destination();
509 }
510
511 static uint32_t encode_patchable_nop() {
512 return 0x00000013;
513 }
514
515 static uint32_t encode_patchable_jump(address pc, address target_pc) {
516 int32_t disp = checked_cast<int32_t>((intptr_t)target_pc - (intptr_t)pc);
517 return Assembler::encode_jal(x0, disp);
518 }
519
520 void ShenandoahBarrierSetAssembler::insert_patchable_nop(address pc) {
521 *((uint32_t*)pc) = encode_patchable_nop();
522 assert(nativeInstruction_at(pc)->is_nop(), "Sanity");
523 }
524
525 void ShenandoahBarrierSetAssembler::insert_patchable_jump(address pc, address target_pc) {
526 *((uint32_t*)pc) = encode_patchable_jump(pc, target_pc);
527 }
528
529 bool ShenandoahBarrierSetAssembler::is_patchable_nop(address pc) {
530 return *((uint32_t*)pc) == encode_patchable_nop();
531 }
532
533 bool ShenandoahBarrierSetAssembler::is_patchable_jump(address pc, address target_pc) {
534 return *((uint32_t*)pc) == encode_patchable_jump(pc, target_pc);
535 }
536
537 #ifdef COMPILER1
538
539 #define __ ce->masm()->
540
541 void ShenandoahBarrierSetAssembler::keepalive_barrier_c1_stub(LIR_Assembler* ce, ShenandoahKeepaliveBarrierStub* stub) {
542 __ bind(*stub->entry());
543
544 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
545
546 Register obj = stub->obj()->as_register();
547
548 if (stub->do_load()) {
549 ce->mem2reg(stub->addr(), stub->obj(), T_OBJECT, lir_patch_none, nullptr, false /* wide */);
550 }
551 __ beqz(obj, *stub->continuation(), /* is_far */ true);
552
553 ce->store_parameter(obj, 0);
554 __ far_call(RuntimeAddress(bs->keepalive_barrier_stub()));
555 __ j(*stub->continuation());
556 }
698 __ ld(tmp1, curr_ct_holder_addr);
699
700 // tmp1 = effective address
701 __ la(tmp2, address);
702
703 // tmp2 = &card_table[ addr >> CardTable::card_shift() ] ; card index
704 __ srli(tmp2, tmp2, CardTable::card_shift());
705 __ add(tmp2, tmp2, tmp1);
706
707 if (UseCondCardMark) {
708 Label L_already_dirty;
709 __ lbu(tmp1, Address(tmp2));
710 __ beqz(tmp1, L_already_dirty);
711 __ sb(zr, Address(tmp2));
712 __ bind(L_already_dirty);
713 } else {
714 __ sb(zr, Address(tmp2));
715 }
716 }
717
718 void ShenandoahBarrierStubC2::patchable_jump(MacroAssembler& masm, const char gc_state, bool jump_when_state, Register tmp1, Register tmp2, Label* L_target) {
719 PhaseOutput* const output = Compile::current()->output();
720 if (output->in_scratch_emit_size()) {
721 // Avoid binding L_target in scratch emits.
722 // We know the patched check is exactly one incompressible instruction long.
723 Assembler::IncompressibleScope scope(&masm);
724 __ nop();
725 return;
726 }
727
728 // Emit the unconditional branch in the first version of the method.
729 // Let the rest of runtime figure out how to manage it.
730 __ relocate(patchable_barrier_Relocation::spec(ShenandoahNMethod::encode_to_reloc(gc_state, jump_when_state)));
731 __ j(*L_target);
732 }
733
734 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp1, Register tmp2) {
735 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
736 patchable_jump_if_gc_state(masm, test_state, tmp1, tmp2, entry());
737 __ bind(*continuation());
738 }
739
740 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
741 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
742 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
743
744 __ bind(*entry());
745
746 // If we need to load ourselves, do it here.
747 if (_do_load) {
748 if (_narrow) {
749 __ lwu(_obj, _addr);
750 } else {
751 __ ld(_obj, _addr);
752 }
753 }
754
755 // If the object is null, there is no point in applying barriers.
756 maybe_far_jump_if_zero(masm, _obj);
771 keepalive(masm, continuation());
772 } else if (_needs_load_ref_barrier) {
773 lrb(masm);
774 } else {
775 ShouldNotReachHere();
776 }
777 }
778
779 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
780 Label L_short_jump;
781 __ bnez(reg, L_short_jump);
782 __ j(*continuation());
783 __ bind(L_short_jump);
784 }
785
786 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
787 Address index(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
788 Address buffer(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
789 Label L_through, L_slowpath;
790
791 // If another barrier is enabled as well, do a check for a specific barrier.
792 if (_needs_load_ref_barrier) {
793 assert(L_done == nullptr, "Should be");
794 char state_to_check = ShenandoahHeap::MARKING;
795 patchable_jump_if_not_gc_state(masm, state_to_check, _tmp1, _tmp2, &L_through);
796 }
797
798 // Fast-path: put object into buffer.
799 // If buffer is already full, go slow.
800 __ ld(_tmp1, index);
801 __ beqz(_tmp1, L_slowpath);
802 __ subi(_tmp1, _tmp1, wordSize);
803 __ sd(_tmp1, index);
804 __ ld(_tmp2, buffer);
805
806 // Store the object in queue.
807 // If object is narrow, we need to decode it before inserting.
808 __ add(_tmp1, _tmp1, _tmp2);
809 if (_narrow) {
810 __ decode_heap_oop_not_null(_tmp2, _obj);
811 __ sd(_tmp2, Address(_tmp1));
812 } else {
813 __ sd(_obj, Address(_tmp1));
814 }
815
824 __ bind(L_slowpath);
825
826 {
827 SaveLiveRegisters slr(&masm, this);
828
829 // Go to runtime and handle the rest there.
830 __ mv(c_rarg0, _obj);
831 __ la(ra, RuntimeAddress(keepalive_runtime_entry_addr()));
832 __ jalr(ra);
833 }
834 if (L_done != nullptr) {
835 __ j(*L_done);
836 } else {
837 __ bind(L_through);
838 }
839 }
840
841 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
842 Label L_slow;
843
844 // If another barrier is enabled as well, do a check for a specific barrier.
845 if (_needs_keep_alive_barrier) {
846 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
847 patchable_jump_if_not_gc_state(masm, state_to_check, _tmp1, _tmp2, continuation());
848 }
849
850 // If weak references are being processed, weak/phantom loads need to go slow,
851 // regardless of their cset status.
852 if (_needs_load_ref_weak_barrier) {
853 char state_to_check = ShenandoahHeap::WEAK_ROOTS;
854 patchable_jump_if_gc_state(masm, state_to_check, _tmp1, _tmp2, &L_slow);
855 }
856
857 // Cset-check. Fall-through to slow if in collection set.
858 if (_narrow) {
859 __ decode_heap_oop_not_null(_tmp2, _obj);
860 } else {
861 __ mv(_tmp2, _obj);
862 }
863
864 if (AOTCodeCache::is_on_for_dump()) {
865 __ lwu(_tmp1, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
866 __ srl(_tmp2, _tmp2, _tmp1);
867 __ ld(_tmp1, ExternalAddress(AOTRuntimeConstants::cset_base_address()));
868 } else {
869 __ mv(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
870 __ srli(_tmp2, _tmp2, ShenandoahHeapRegion::region_size_bytes_shift_jint());
871 }
872 __ add(_tmp1, _tmp1, _tmp2);
873 __ lbu(_tmp1, Address(_tmp1, 0));
874 maybe_far_jump_if_zero(masm, _tmp1);
|