14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "code/aotCodeCache.hpp"
28 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
29 #include "gc/shenandoah/mode/shenandoahMode.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
31 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interp_masm.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "runtime/javaThread.hpp"
39 #include "runtime/sharedRuntime.hpp"
40 #ifdef COMPILER1
41 #include "c1/c1_LIRAssembler.hpp"
42 #include "c1/c1_MacroAssembler.hpp"
43 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
44 #endif
45 #ifdef COMPILER2
46 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
47 #include "opto/output.hpp"
48 #endif
49
50 #define __ masm->
51
52 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
53 Register src, Register dst, Register count, RegSet saved_regs) {
54 if (is_oop) {
55 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
56 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
57
476 __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop)));
477 __ sub(end, end, BytesPerHeapOop);
478 __ lsr(start, start, CardTable::card_shift());
479 __ lsr(end, end, CardTable::card_shift());
480
481 // number of bytes to copy
482 __ sub(count, end, start);
483
484 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
485 __ ldr(scratch, curr_ct_holder_addr);
486 __ add(start, start, scratch);
487 __ bind(L_loop);
488 __ strb(zr, Address(start, count));
489 __ subs(count, count, 1);
490 __ br(Assembler::GE, L_loop);
491 __ bind(L_done);
492 }
493
494 #undef __
495
496 #ifdef COMPILER1
497
498 #define __ ce->masm()->
499
500 void ShenandoahBarrierSetAssembler::keepalive_barrier_c1_stub(LIR_Assembler* ce, ShenandoahKeepaliveBarrierStub* stub) {
501 __ bind(*stub->entry());
502
503 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
504
505 Register obj = stub->obj()->as_register();
506
507 if (stub->do_load()) {
508 ce->mem2reg(stub->addr(), stub->obj(), T_OBJECT, lir_patch_none, nullptr, /* wide = */ false);
509 }
510 __ cbz(obj, *stub->continuation());
511 ce->store_parameter(obj, 0);
512 __ far_call(RuntimeAddress(bs->keepalive_barrier_stub()));
513 __ b(*stub->continuation());
514 }
515
690 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
691 __ ldr(tmp1, curr_ct_holder_addr);
692
693 // tmp2 = effective address
694 __ lea(tmp2, address);
695
696 // tmp2 = &card_table[ addr >> CardTable::card_shift() ] ; card index
697 __ add(tmp2, tmp1, tmp2, Assembler::LSR, CardTable::card_shift());
698
699 if (UseCondCardMark) {
700 Label L_already_dirty;
701 __ ldrb(tmp1, Address(tmp2));
702 __ cbz(tmp1, L_already_dirty);
703 __ strb(zr, Address(tmp2));
704 __ bind(L_already_dirty);
705 } else {
706 __ strb(zr, Address(tmp2));
707 }
708 }
709
710 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
711 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
712 PhaseOutput* const output = Compile::current()->output();
713 Address gc_state_fast(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(test_state)));
714
715 // We piggyback on scratch_emit_size mode to compute the slowpath stub size.
716 // We'll use that information to decide whether we need a far jump to the
717 // stub entry point or not. In scratch_emit_size mode we don't bind entry()
718 // because otherwise it will be rebound when we later emit the instructions
719 // for real.
720 if (_needs_far_jump) {
721 __ ldrb(tmp, gc_state_fast);
722 __ cbz(tmp, *continuation());
723 __ b(output->in_scratch_emit_size() ? *continuation() : *entry());
724 } else {
725 __ ldrb(tmp, gc_state_fast);
726 __ cbnz(tmp, output->in_scratch_emit_size() ? *continuation() : *entry());
727 }
728
729 // This is were the slowpath stub will return to or the code above will
730 // jump to if the checks are false
731 __ bind(*continuation());
732 }
733
734 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
735 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
736 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
737 PhaseOutput* const output = Compile::current()->output();
738
739 // We piggyback on scratch_emit_size mode to compute the slowpath stub size.
740 // We'll use that information to decide whether we need a far jump to the
741 // stub entry point or not. In scratch_emit_size mode we don't bind entry()
742 // because otherwise it will be rebound when we later emit the instructions
743 // for real.
744 if (!output->in_scratch_emit_size()) {
745 __ bind(*entry());
746 }
747
748 // If we need to load ourselves, do it here.
749 if (_do_load) {
750 if (_narrow) {
784 keepalive(masm, continuation());
785 } else if (_needs_load_ref_barrier) {
786 lrb(masm);
787 } else {
788 ShouldNotReachHere();
789 }
790 }
791
792 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
793 if (_needs_far_jump) {
794 Label L_short_jump;
795 __ cbnz(reg, L_short_jump);
796 __ b(*continuation());
797 __ bind(L_short_jump);
798 } else {
799 __ cbz(reg, *continuation());
800 }
801 }
802
803 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
804 Address gcstate(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::MARKING)));
805 Address index(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
806 Address buffer(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
807 Label L_through, L_slowpath;
808
809 // If another barrier is enabled as well, do a runtime check for a specific barrier.
810 if (_needs_load_ref_barrier) {
811 assert(L_done == nullptr, "L_done is always null when _needs_load_ref_barrier is true");
812 __ ldrb(_tmp1, gcstate);
813 __ cbz(_tmp1, L_through);
814 }
815
816 // Fast-path: put object into buffer.
817 // If buffer is already full, go slow.
818 __ ldr(_tmp1, index);
819 __ cbz(_tmp1, L_slowpath);
820 __ sub(_tmp1, _tmp1, wordSize);
821 __ str(_tmp1, index);
822 __ ldr(_tmp2, buffer);
823
824 // Store the object in queue.
825 // If object is narrow, we need to decode it before inserting.
826 if (_narrow) {
827 __ add(_tmp2, _tmp2, _tmp1);
828 __ decode_heap_oop_not_null(_tmp1, _obj);
829 __ str(_tmp1, Address(_tmp2));
830 } else {
831 // Buffer is 64-bit address, must be in base register.
832 __ str(_obj, Address(_tmp2, _tmp1));
833 }
843 __ bind(L_slowpath);
844
845 {
846 SaveLiveRegisters slr(&masm, this);
847
848 // Go to runtime and handle the rest there.
849 __ mov(c_rarg0, _obj);
850 __ lea(lr, RuntimeAddress(keepalive_runtime_entry_addr()));
851 __ blr(lr);
852 }
853 if (L_done != nullptr) {
854 __ b(*L_done);
855 } else {
856 __ bind(L_through);
857 }
858 }
859
860 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
861 Label L_slow;
862
863 // If another barrier is enabled as well, do a runtime check for a specific barrier.
864 if (_needs_keep_alive_barrier) {
865 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
866 Address gc_state_fast(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(state_to_check)));
867 __ ldrb(_tmp1, gc_state_fast);
868 maybe_far_jump_if_zero(masm, _tmp1);
869 }
870
871 // If weak references are being processed, weak/phantom loads need to go slow,
872 // regardless of their cset status.
873 if (_needs_load_ref_weak_barrier) {
874 Address gc_state_fast(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::WEAK_ROOTS)));
875 __ ldrb(_tmp1, gc_state_fast);
876 __ cbnz(_tmp1, L_slow);
877 }
878
879 // Cset-check. Fall-through to slow if in collection set.
880 bool is_aot = AOTCodeCache::is_on_for_dump();
881 if (!is_aot) {
882 __ mov(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
883 if (_narrow) {
884 __ decode_heap_oop_not_null(_tmp2, _obj);
885 __ add(_tmp1, _tmp1, _tmp2, Assembler::LSR, ShenandoahHeapRegion::region_size_bytes_shift_jint());
886 } else {
887 __ add(_tmp1, _tmp1, _obj, Assembler::LSR, ShenandoahHeapRegion::region_size_bytes_shift_jint());
888 }
889 } else {
890 // Generating AOT code, pull the cset bitmap and region shift from AOT table.
891 if (_narrow) {
892 __ decode_heap_oop_not_null(_tmp1, _obj);
893 } else {
894 __ mov(_tmp1, _obj);
895 }
896 __ lea(_tmp2, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
|
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "code/aotCodeCache.hpp"
28 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
29 #include "gc/shenandoah/mode/shenandoahMode.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
31 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
35 #include "gc/shenandoah/shenandoahRuntime.hpp"
36 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
37 #include "interpreter/interp_masm.hpp"
38 #include "interpreter/interpreter.hpp"
39 #include "nativeInst_aarch64.hpp"
40 #include "runtime/icache.hpp"
41 #include "runtime/javaThread.hpp"
42 #include "runtime/sharedRuntime.hpp"
43 #ifdef COMPILER1
44 #include "c1/c1_LIRAssembler.hpp"
45 #include "c1/c1_MacroAssembler.hpp"
46 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
47 #endif
48 #ifdef COMPILER2
49 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
50 #include "opto/output.hpp"
51 #endif
52
53 #define __ masm->
54
55 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
56 Register src, Register dst, Register count, RegSet saved_regs) {
57 if (is_oop) {
58 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
59 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
60
479 __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop)));
480 __ sub(end, end, BytesPerHeapOop);
481 __ lsr(start, start, CardTable::card_shift());
482 __ lsr(end, end, CardTable::card_shift());
483
484 // number of bytes to copy
485 __ sub(count, end, start);
486
487 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
488 __ ldr(scratch, curr_ct_holder_addr);
489 __ add(start, start, scratch);
490 __ bind(L_loop);
491 __ strb(zr, Address(start, count));
492 __ subs(count, count, 1);
493 __ br(Assembler::GE, L_loop);
494 __ bind(L_done);
495 }
496
497 #undef __
498
499 address ShenandoahBarrierSetAssembler::parse_jump_address(address pc) {
500 NativeInstruction* ni = nativeInstruction_at(pc);
501 assert(ni->is_jump(), "Initial code version: GC barrier fastpath must be a jump");
502 NativeJump* jmp = nativeJump_at(pc);
503 return jmp->jump_destination();
504 }
505
506 static uint32_t encode_patchable_nop() {
507 return 0xD503201F;
508 }
509
510 static uint32_t encode_patchable_jump(address pc, address target_pc) {
511 int32_t disp = checked_cast<int32_t>((intptr_t)target_pc - (intptr_t)pc);
512 int64_t imm26 = disp >> 2;
513 guarantee(Assembler::is_simm(imm26, 26), "Maximum offset is 128MiB");
514 return 0x14000000 | (imm26 & 0x03FFFFFF);
515 }
516
517 void ShenandoahBarrierSetAssembler::insert_patchable_nop(address pc) {
518 *((uint32_t*)pc) = encode_patchable_nop();
519 assert(nativeInstruction_at(pc)->is_nop(), "Sanity");
520 if (!UseSingleICacheInvalidation) {
521 ICache::invalidate_word(pc);
522 }
523 }
524
525 void ShenandoahBarrierSetAssembler::insert_patchable_jump(address pc, address target_pc) {
526 *((uint32_t*)pc) = encode_patchable_jump(pc, target_pc);
527 if (!UseSingleICacheInvalidation) {
528 ICache::invalidate_word(pc);
529 }
530 }
531
532 bool ShenandoahBarrierSetAssembler::is_patchable_nop(address pc) {
533 return *((uint32_t*)pc) == encode_patchable_nop();
534 }
535
536 bool ShenandoahBarrierSetAssembler::is_patchable_jump(address pc, address target_pc) {
537 return *((uint32_t*)pc) == encode_patchable_jump(pc, target_pc);
538 }
539
540 #ifdef COMPILER1
541
542 #define __ ce->masm()->
543
544 void ShenandoahBarrierSetAssembler::keepalive_barrier_c1_stub(LIR_Assembler* ce, ShenandoahKeepaliveBarrierStub* stub) {
545 __ bind(*stub->entry());
546
547 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
548
549 Register obj = stub->obj()->as_register();
550
551 if (stub->do_load()) {
552 ce->mem2reg(stub->addr(), stub->obj(), T_OBJECT, lir_patch_none, nullptr, /* wide = */ false);
553 }
554 __ cbz(obj, *stub->continuation());
555 ce->store_parameter(obj, 0);
556 __ far_call(RuntimeAddress(bs->keepalive_barrier_stub()));
557 __ b(*stub->continuation());
558 }
559
734 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
735 __ ldr(tmp1, curr_ct_holder_addr);
736
737 // tmp2 = effective address
738 __ lea(tmp2, address);
739
740 // tmp2 = &card_table[ addr >> CardTable::card_shift() ] ; card index
741 __ add(tmp2, tmp1, tmp2, Assembler::LSR, CardTable::card_shift());
742
743 if (UseCondCardMark) {
744 Label L_already_dirty;
745 __ ldrb(tmp1, Address(tmp2));
746 __ cbz(tmp1, L_already_dirty);
747 __ strb(zr, Address(tmp2));
748 __ bind(L_already_dirty);
749 } else {
750 __ strb(zr, Address(tmp2));
751 }
752 }
753
754 void ShenandoahBarrierStubC2::patchable_jump(MacroAssembler& masm, const char gc_state, bool jump_when_state, Register tmp1, Register tmp2, Label* L_target) {
755 const size_t check_size = 8;
756
757 PhaseOutput* const output = Compile::current()->output();
758 if (output->in_scratch_emit_size()) {
759 // Avoid binding L_target and emitting more branches in scratch emits.
760 // We know the patched check is exactly 1 instruction long in release,
761 // and verification adds more instructions.
762 for (size_t c = 0; c < 1 DEBUG_ONLY(+ check_size); c++) {
763 __ nop();
764 }
765 return;
766 }
767
768 #ifdef ASSERT
769 Label L_fake_entry, L_skip;
770 Address gc_state_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
771
772 address check_start = __ pc();
773
774 __ ldrb(tmp1, gc_state_addr);
775 __ mov(tmp2, gc_state);
776 __ andr(tmp2, tmp1, tmp2);
777
778 // Emit the secondary jump and use it to cross-check against the actual GC state.
779 // This also checks that all interesting GC state transitions are done non-racily
780 // from the perspective of the thread executing the nmethod.
781 __ relocate(patchable_barrier_Relocation::spec(ShenandoahNMethod::encode_to_reloc(gc_state, jump_when_state)));
782 __ b(L_fake_entry);
783
784 // Currently hot-patched to NOP.
785 if (jump_when_state) {
786 __ cbz(tmp2, L_skip);
787 } else {
788 __ cbnz(tmp2, L_skip);
789 }
790 __ hlt(0);
791
792 // Currently hot-patched to JUMP.
793 __ bind(L_fake_entry);
794 if (jump_when_state) {
795 __ cbnz(tmp2, L_skip);
796 } else {
797 __ cbz(tmp2, L_skip);
798 }
799 __ hlt(0);
800
801 __ bind(L_skip);
802
803 address check_end = __ pc();
804 size_t actual_check_size = pointer_delta(check_end, check_start, Assembler::instruction_size);
805 assert(check_size == actual_check_size, "Must match: %zu != %zu", check_size, actual_check_size);
806 #endif
807
808 // Emit the unconditional branch in the first version of the method.
809 // Let the rest of runtime figure out how to manage it.
810 __ relocate(patchable_barrier_Relocation::spec(ShenandoahNMethod::encode_to_reloc(gc_state, jump_when_state)));
811 __ b(*L_target);
812 }
813
814 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp1, Register tmp2) {
815 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
816 patchable_jump_if_gc_state(masm, test_state, tmp1, tmp2, entry());
817 __ bind(*continuation());
818 }
819
820 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
821 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
822 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
823 PhaseOutput* const output = Compile::current()->output();
824
825 // We piggyback on scratch_emit_size mode to compute the slowpath stub size.
826 // We'll use that information to decide whether we need a far jump to the
827 // stub entry point or not. In scratch_emit_size mode we don't bind entry()
828 // because otherwise it will be rebound when we later emit the instructions
829 // for real.
830 if (!output->in_scratch_emit_size()) {
831 __ bind(*entry());
832 }
833
834 // If we need to load ourselves, do it here.
835 if (_do_load) {
836 if (_narrow) {
870 keepalive(masm, continuation());
871 } else if (_needs_load_ref_barrier) {
872 lrb(masm);
873 } else {
874 ShouldNotReachHere();
875 }
876 }
877
878 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
879 if (_needs_far_jump) {
880 Label L_short_jump;
881 __ cbnz(reg, L_short_jump);
882 __ b(*continuation());
883 __ bind(L_short_jump);
884 } else {
885 __ cbz(reg, *continuation());
886 }
887 }
888
889 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
890 Address index(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
891 Address buffer(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
892 Label L_through, L_slowpath;
893
894 // If another barrier is enabled as well, do a check for a specific barrier.
895 if (_needs_load_ref_barrier) {
896 assert(L_done == nullptr, "Should be");
897 char state_to_check = ShenandoahHeap::MARKING;
898 patchable_jump_if_not_gc_state(masm, state_to_check, _tmp1, _tmp2, &L_through);
899 }
900
901 // Fast-path: put object into buffer.
902 // If buffer is already full, go slow.
903 __ ldr(_tmp1, index);
904 __ cbz(_tmp1, L_slowpath);
905 __ sub(_tmp1, _tmp1, wordSize);
906 __ str(_tmp1, index);
907 __ ldr(_tmp2, buffer);
908
909 // Store the object in queue.
910 // If object is narrow, we need to decode it before inserting.
911 if (_narrow) {
912 __ add(_tmp2, _tmp2, _tmp1);
913 __ decode_heap_oop_not_null(_tmp1, _obj);
914 __ str(_tmp1, Address(_tmp2));
915 } else {
916 // Buffer is 64-bit address, must be in base register.
917 __ str(_obj, Address(_tmp2, _tmp1));
918 }
928 __ bind(L_slowpath);
929
930 {
931 SaveLiveRegisters slr(&masm, this);
932
933 // Go to runtime and handle the rest there.
934 __ mov(c_rarg0, _obj);
935 __ lea(lr, RuntimeAddress(keepalive_runtime_entry_addr()));
936 __ blr(lr);
937 }
938 if (L_done != nullptr) {
939 __ b(*L_done);
940 } else {
941 __ bind(L_through);
942 }
943 }
944
945 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
946 Label L_slow;
947
948 // If another barrier is enabled as well, do a check for a specific barrier.
949 if (_needs_keep_alive_barrier) {
950 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
951 patchable_jump_if_not_gc_state(masm, state_to_check, _tmp1, _tmp2, continuation());
952 }
953
954 // If weak references are being processed, weak/phantom loads need to go slow,
955 // regardless of their cset status.
956 if (_needs_load_ref_weak_barrier) {
957 char state_to_check = ShenandoahHeap::WEAK_ROOTS;
958 patchable_jump_if_gc_state(masm, state_to_check, _tmp1, _tmp2, &L_slow);
959 }
960
961 // Cset-check. Fall-through to slow if in collection set.
962 bool is_aot = AOTCodeCache::is_on_for_dump();
963 if (!is_aot) {
964 __ mov(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
965 if (_narrow) {
966 __ decode_heap_oop_not_null(_tmp2, _obj);
967 __ add(_tmp1, _tmp1, _tmp2, Assembler::LSR, ShenandoahHeapRegion::region_size_bytes_shift_jint());
968 } else {
969 __ add(_tmp1, _tmp1, _obj, Assembler::LSR, ShenandoahHeapRegion::region_size_bytes_shift_jint());
970 }
971 } else {
972 // Generating AOT code, pull the cset bitmap and region shift from AOT table.
973 if (_narrow) {
974 __ decode_heap_oop_not_null(_tmp1, _obj);
975 } else {
976 __ mov(_tmp1, _obj);
977 }
978 __ lea(_tmp2, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
|