13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
33 #include "gc/shenandoah/shenandoahRuntime.hpp"
34 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
35 #include "interpreter/interp_masm.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "runtime/javaThread.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #ifdef COMPILER1
40 #include "c1/c1_LIRAssembler.hpp"
41 #include "c1/c1_MacroAssembler.hpp"
42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
43 #endif
44 #ifdef COMPILER2
45 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
46 #include "opto/output.hpp"
47 #endif
48
49 #define __ masm->
50
51 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
52 Register src, Register dst, Register count, RegSet saved_regs) {
53 if (is_oop) {
54 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
55 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
56
476 __ srli(start, start, CardTable::card_shift());
477 __ srli(end, end, CardTable::card_shift());
478
479 // number of bytes to copy
480 __ sub(count, end, start);
481
482 Address curr_ct_holder_addr(xthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
483 __ ld(tmp, curr_ct_holder_addr);
484 __ add(start, start, tmp);
485
486 __ bind(L_loop);
487 __ add(tmp, start, count);
488 __ sb(zr, Address(tmp));
489 __ subi(count, count, 1);
490 __ bgez(count, L_loop);
491 __ bind(L_done);
492 }
493
494 #undef __
495
496 #ifdef COMPILER1
497
498 #define __ ce->masm()->
499
500 void ShenandoahBarrierSetAssembler::keepalive_barrier_c1_stub(LIR_Assembler* ce, ShenandoahKeepaliveBarrierStub* stub) {
501 __ bind(*stub->entry());
502
503 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
504
505 Register obj = stub->obj()->as_register();
506
507 if (stub->do_load()) {
508 ce->mem2reg(stub->addr(), stub->obj(), T_OBJECT, lir_patch_none, nullptr, false /* wide */);
509 }
510 __ beqz(obj, *stub->continuation(), /* is_far */ true);
511
512 ce->store_parameter(obj, 0);
513 __ far_call(RuntimeAddress(bs->keepalive_barrier_stub()));
514 __ j(*stub->continuation());
515 }
657 __ ld(tmp1, curr_ct_holder_addr);
658
659 // tmp1 = effective address
660 __ la(tmp2, address);
661
662 // tmp2 = &card_table[ addr >> CardTable::card_shift() ] ; card index
663 __ srli(tmp2, tmp2, CardTable::card_shift());
664 __ add(tmp2, tmp2, tmp1);
665
666 if (UseCondCardMark) {
667 Label L_already_dirty;
668 __ lbu(tmp1, Address(tmp2));
669 __ beqz(tmp1, L_already_dirty);
670 __ sb(zr, Address(tmp2));
671 __ bind(L_already_dirty);
672 } else {
673 __ sb(zr, Address(tmp2));
674 }
675 }
676
677 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
678 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
679
680 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(test_state)));
681 __ lbu(tmp, gc_state_fast);
682 __ beqz(tmp, *continuation());
683 __ j(*entry());
684
685 // This is were the slowpath stub will return to or the code above will
686 // jump to if the checks are false
687 __ bind(*continuation());
688 }
689
690 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
691 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
692 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
693
694 __ bind(*entry());
695
696 // If we need to load ourselves, do it here.
697 if (_do_load) {
698 if (_narrow) {
699 __ lwu(_obj, _addr);
700 } else {
701 __ ld(_obj, _addr);
702 }
703 }
704
705 // If the object is null, there is no point in applying barriers.
706 maybe_far_jump_if_zero(masm, _obj);
721 keepalive(masm, continuation());
722 } else if (_needs_load_ref_barrier) {
723 lrb(masm);
724 } else {
725 ShouldNotReachHere();
726 }
727 }
728
729 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
730 Label L_short_jump;
731 __ bnez(reg, L_short_jump);
732 __ j(*continuation());
733 __ bind(L_short_jump);
734 }
735
736 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
737 Address index(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
738 Address buffer(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
739 Label L_through, L_slowpath;
740
741 // If another barrier is enabled as well, do a runtime check for a specific barrier.
742 if (_needs_load_ref_barrier) {
743 assert(L_done == nullptr, "L_done is always null when _needs_load_ref_barrier is true");
744 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::MARKING)));
745 __ lbu(_tmp1, gc_state_fast);
746 __ beqz(_tmp1, L_through);
747 }
748
749 // Fast-path: put object into buffer.
750 // If buffer is already full, go slow.
751 __ ld(_tmp1, index);
752 __ beqz(_tmp1, L_slowpath);
753 __ subi(_tmp1, _tmp1, wordSize);
754 __ sd(_tmp1, index);
755 __ ld(_tmp2, buffer);
756
757 // Store the object in queue.
758 // If object is narrow, we need to decode it before inserting.
759 __ add(_tmp1, _tmp1, _tmp2);
760 if (_narrow) {
761 __ decode_heap_oop_not_null(_tmp2, _obj);
762 __ sd(_tmp2, Address(_tmp1));
763 } else {
764 __ sd(_obj, Address(_tmp1));
765 }
766
775 __ bind(L_slowpath);
776
777 {
778 SaveLiveRegisters slr(&masm, this);
779
780 // Go to runtime and handle the rest there.
781 __ mv(c_rarg0, _obj);
782 __ la(ra, RuntimeAddress(keepalive_runtime_entry_addr()));
783 __ jalr(ra);
784 }
785 if (L_done != nullptr) {
786 __ j(*L_done);
787 } else {
788 __ bind(L_through);
789 }
790 }
791
792 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
793 Label L_slow;
794
795 // If another barrier is enabled as well, do a runtime check for a specific barrier.
796 if (_needs_keep_alive_barrier) {
797 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
798 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(state_to_check)));
799 __ lbu(_tmp1, gc_state_fast);
800 maybe_far_jump_if_zero(masm, _tmp1);
801 }
802
803 // If weak references are being processed, weak/phantom loads need to go slow,
804 // regardless of their cset status.
805 if (_needs_load_ref_weak_barrier) {
806 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::WEAK_ROOTS)));
807 __ lbu(_tmp1, gc_state_fast);
808 __ bnez(_tmp1, L_slow);
809 }
810
811 // Cset-check. Fall-through to slow if in collection set.
812 if (_narrow) {
813 __ decode_heap_oop_not_null(_tmp2, _obj);
814 } else {
815 __ mv(_tmp2, _obj);
816 }
817
818 __ mv(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
819 __ srli(_tmp2, _tmp2, ShenandoahHeapRegion::region_size_bytes_shift_jint());
820 __ add(_tmp1, _tmp1, _tmp2);
821 __ lbu(_tmp1, Address(_tmp1, 0));
822 maybe_far_jump_if_zero(masm, _tmp1);
823
824 // Slow path
825 __ bind(L_slow);
826
827 // Obj is the result, need to temporarily stop preserving it.
828 bool is_obj_preserved = is_preserved(_obj);
|
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
33 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interp_masm.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "nativeInst_riscv.hpp"
39 #include "runtime/javaThread.hpp"
40 #include "runtime/sharedRuntime.hpp"
41 #ifdef COMPILER1
42 #include "c1/c1_LIRAssembler.hpp"
43 #include "c1/c1_MacroAssembler.hpp"
44 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
45 #endif
46 #ifdef COMPILER2
47 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
48 #include "opto/output.hpp"
49 #endif
50
51 #define __ masm->
52
53 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
54 Register src, Register dst, Register count, RegSet saved_regs) {
55 if (is_oop) {
56 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
57 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
58
478 __ srli(start, start, CardTable::card_shift());
479 __ srli(end, end, CardTable::card_shift());
480
481 // number of bytes to copy
482 __ sub(count, end, start);
483
484 Address curr_ct_holder_addr(xthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
485 __ ld(tmp, curr_ct_holder_addr);
486 __ add(start, start, tmp);
487
488 __ bind(L_loop);
489 __ add(tmp, start, count);
490 __ sb(zr, Address(tmp));
491 __ subi(count, count, 1);
492 __ bgez(count, L_loop);
493 __ bind(L_done);
494 }
495
496 #undef __
497
498 address ShenandoahBarrierSetAssembler::parse_jump_address(address pc) {
499 NativeInstruction* ni = nativeInstruction_at(pc);
500 assert(ni->is_jump(), "Must be a jump");
501 NativeJump* jmp = nativeJump_at(pc);
502 return jmp->jump_destination();
503 }
504
505 static uint32_t encode_patchable_nop() {
506 return 0x00000013;
507 }
508
509 static uint32_t encode_patchable_jump(address pc, address target_pc) {
510 int32_t disp = checked_cast<int32_t>((intptr_t)target_pc - (intptr_t)pc);
511 return Assembler::encode_jal(x0, disp);
512 }
513
514 void ShenandoahBarrierSetAssembler::insert_patchable_nop(address pc) {
515 *((uint32_t*)pc) = encode_patchable_nop();
516 }
517
518 void ShenandoahBarrierSetAssembler::insert_patchable_jump(address pc, address target_pc) {
519 *((uint32_t*)pc) = encode_patchable_jump(pc, target_pc);
520 }
521
522 bool ShenandoahBarrierSetAssembler::is_patchable_nop(address pc) {
523 return *((uint32_t*)pc) == encode_patchable_nop();
524 }
525
526 bool ShenandoahBarrierSetAssembler::is_patchable_jump(address pc, address target_pc) {
527 return *((uint32_t*)pc) == encode_patchable_jump(pc, target_pc);
528 }
529
530 #ifdef COMPILER1
531
532 #define __ ce->masm()->
533
534 void ShenandoahBarrierSetAssembler::keepalive_barrier_c1_stub(LIR_Assembler* ce, ShenandoahKeepaliveBarrierStub* stub) {
535 __ bind(*stub->entry());
536
537 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
538
539 Register obj = stub->obj()->as_register();
540
541 if (stub->do_load()) {
542 ce->mem2reg(stub->addr(), stub->obj(), T_OBJECT, lir_patch_none, nullptr, false /* wide */);
543 }
544 __ beqz(obj, *stub->continuation(), /* is_far */ true);
545
546 ce->store_parameter(obj, 0);
547 __ far_call(RuntimeAddress(bs->keepalive_barrier_stub()));
548 __ j(*stub->continuation());
549 }
691 __ ld(tmp1, curr_ct_holder_addr);
692
693 // tmp1 = effective address
694 __ la(tmp2, address);
695
696 // tmp2 = &card_table[ addr >> CardTable::card_shift() ] ; card index
697 __ srli(tmp2, tmp2, CardTable::card_shift());
698 __ add(tmp2, tmp2, tmp1);
699
700 if (UseCondCardMark) {
701 Label L_already_dirty;
702 __ lbu(tmp1, Address(tmp2));
703 __ beqz(tmp1, L_already_dirty);
704 __ sb(zr, Address(tmp2));
705 __ bind(L_already_dirty);
706 } else {
707 __ sb(zr, Address(tmp2));
708 }
709 }
710
711 void ShenandoahBarrierStubC2::patchable_jump(MacroAssembler& masm, const char gc_state, bool jump_when_state, Label* L_target) {
712 PhaseOutput* const output = Compile::current()->output();
713 if (output->in_scratch_emit_size()) {
714 // Avoid binding L_target in scratch emits.
715 // We know the patched check is exactly one incompressible instruction long.
716 Assembler::IncompressibleScope scope(&masm);
717 __ nop();
718 return;
719 }
720
721 // Emit the unconditional branch in the first version of the method.
722 // Let the rest of runtime figure out how to manage it.
723 __ relocate(patchable_barrier_Relocation::spec(ShenandoahNMethod::encode_to_reloc(gc_state, jump_when_state)));
724 __ j(*L_target);
725 }
726
727 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state) {
728 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
729 patchable_jump_if_gc_state(masm, test_state, entry());
730 __ bind(*continuation());
731 }
732
733 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
734 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
735 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
736
737 __ bind(*entry());
738
739 // If we need to load ourselves, do it here.
740 if (_do_load) {
741 if (_narrow) {
742 __ lwu(_obj, _addr);
743 } else {
744 __ ld(_obj, _addr);
745 }
746 }
747
748 // If the object is null, there is no point in applying barriers.
749 maybe_far_jump_if_zero(masm, _obj);
764 keepalive(masm, continuation());
765 } else if (_needs_load_ref_barrier) {
766 lrb(masm);
767 } else {
768 ShouldNotReachHere();
769 }
770 }
771
772 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
773 Label L_short_jump;
774 __ bnez(reg, L_short_jump);
775 __ j(*continuation());
776 __ bind(L_short_jump);
777 }
778
779 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
780 Address index(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
781 Address buffer(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
782 Label L_through, L_slowpath;
783
784 // If another barrier is enabled as well, do a check for a specific barrier.
785 if (_needs_load_ref_barrier) {
786 assert(L_done == nullptr, "Should be");
787 char state_to_check = ShenandoahHeap::MARKING;
788 patchable_jump_if_not_gc_state(masm, state_to_check, &L_through);
789 }
790
791 // Fast-path: put object into buffer.
792 // If buffer is already full, go slow.
793 __ ld(_tmp1, index);
794 __ beqz(_tmp1, L_slowpath);
795 __ subi(_tmp1, _tmp1, wordSize);
796 __ sd(_tmp1, index);
797 __ ld(_tmp2, buffer);
798
799 // Store the object in queue.
800 // If object is narrow, we need to decode it before inserting.
801 __ add(_tmp1, _tmp1, _tmp2);
802 if (_narrow) {
803 __ decode_heap_oop_not_null(_tmp2, _obj);
804 __ sd(_tmp2, Address(_tmp1));
805 } else {
806 __ sd(_obj, Address(_tmp1));
807 }
808
817 __ bind(L_slowpath);
818
819 {
820 SaveLiveRegisters slr(&masm, this);
821
822 // Go to runtime and handle the rest there.
823 __ mv(c_rarg0, _obj);
824 __ la(ra, RuntimeAddress(keepalive_runtime_entry_addr()));
825 __ jalr(ra);
826 }
827 if (L_done != nullptr) {
828 __ j(*L_done);
829 } else {
830 __ bind(L_through);
831 }
832 }
833
834 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
835 Label L_slow;
836
837 // If another barrier is enabled as well, do a check for a specific barrier.
838 if (_needs_keep_alive_barrier) {
839 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
840 patchable_jump_if_not_gc_state(masm, state_to_check, continuation());
841 }
842
843 // If weak references are being processed, weak/phantom loads need to go slow,
844 // regardless of their cset status.
845 if (_needs_load_ref_weak_barrier) {
846 char state_to_check = ShenandoahHeap::WEAK_ROOTS;
847 patchable_jump_if_gc_state(masm, state_to_check, &L_slow);
848 }
849
850 // Cset-check. Fall-through to slow if in collection set.
851 if (_narrow) {
852 __ decode_heap_oop_not_null(_tmp2, _obj);
853 } else {
854 __ mv(_tmp2, _obj);
855 }
856
857 __ mv(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
858 __ srli(_tmp2, _tmp2, ShenandoahHeapRegion::region_size_bytes_shift_jint());
859 __ add(_tmp1, _tmp1, _tmp2);
860 __ lbu(_tmp1, Address(_tmp1, 0));
861 maybe_far_jump_if_zero(masm, _tmp1);
862
863 // Slow path
864 __ bind(L_slow);
865
866 // Obj is the result, need to temporarily stop preserving it.
867 bool is_obj_preserved = is_preserved(_obj);
|