14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "code/aotCodeCache.hpp"
28 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
29 #include "gc/shenandoah/mode/shenandoahMode.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
31 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interp_masm.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "runtime/javaThread.hpp"
39 #include "runtime/sharedRuntime.hpp"
40 #ifdef COMPILER1
41 #include "c1/c1_LIRAssembler.hpp"
42 #include "c1/c1_MacroAssembler.hpp"
43 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
44 #endif
45 #ifdef COMPILER2
46 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
47 #include "opto/output.hpp"
48 #endif
49
50 #define __ masm->
51
52 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
53 Register src, Register dst, Register count, RegSet saved_regs) {
54 if (is_oop) {
55 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
56 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
57
476 __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop)));
477 __ sub(end, end, BytesPerHeapOop);
478 __ lsr(start, start, CardTable::card_shift());
479 __ lsr(end, end, CardTable::card_shift());
480
481 // number of bytes to copy
482 __ sub(count, end, start);
483
484 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
485 __ ldr(scratch, curr_ct_holder_addr);
486 __ add(start, start, scratch);
487 __ bind(L_loop);
488 __ strb(zr, Address(start, count));
489 __ subs(count, count, 1);
490 __ br(Assembler::GE, L_loop);
491 __ bind(L_done);
492 }
493
494 #undef __
495
496 #ifdef COMPILER1
497
498 #define __ ce->masm()->
499
500 void ShenandoahBarrierSetAssembler::keepalive_barrier_c1_stub(LIR_Assembler* ce, ShenandoahKeepaliveBarrierStub* stub) {
501 __ bind(*stub->entry());
502
503 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
504
505 Register obj = stub->obj()->as_register();
506
507 if (stub->do_load()) {
508 ce->mem2reg(stub->addr(), stub->obj(), T_OBJECT, lir_patch_none, nullptr, /* wide = */ false);
509 }
510 __ cbz(obj, *stub->continuation());
511 ce->store_parameter(obj, 0);
512 __ far_call(RuntimeAddress(bs->keepalive_barrier_stub()));
513 __ b(*stub->continuation());
514 }
515
690 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
691 __ ldr(tmp1, curr_ct_holder_addr);
692
693 // tmp2 = effective address
694 __ lea(tmp2, address);
695
696 // tmp2 = &card_table[ addr >> CardTable::card_shift() ] ; card index
697 __ add(tmp2, tmp1, tmp2, Assembler::LSR, CardTable::card_shift());
698
699 if (UseCondCardMark) {
700 Label L_already_dirty;
701 __ ldrb(tmp1, Address(tmp2));
702 __ cbz(tmp1, L_already_dirty);
703 __ strb(zr, Address(tmp2));
704 __ bind(L_already_dirty);
705 } else {
706 __ strb(zr, Address(tmp2));
707 }
708 }
709
710 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
711 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
712 PhaseOutput* const output = Compile::current()->output();
713 Address gc_state_fast(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(test_state)));
714
715 // We piggyback on scratch_emit_size mode to compute the slowpath stub size.
716 // We'll use that information to decide whether we need a far jump to the
717 // stub entry point or not. In scratch_emit_size mode we don't bind entry()
718 // because otherwise it will be rebound when we later emit the instructions
719 // for real.
720 if (_needs_far_jump) {
721 __ ldrb(tmp, gc_state_fast);
722 __ cbz(tmp, *continuation());
723 __ b(output->in_scratch_emit_size() ? *continuation() : *entry());
724 } else {
725 __ ldrb(tmp, gc_state_fast);
726 __ cbnz(tmp, output->in_scratch_emit_size() ? *continuation() : *entry());
727 }
728
729 // This is were the slowpath stub will return to or the code above will
730 // jump to if the checks are false
731 __ bind(*continuation());
732 }
733
734 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
735 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
736 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
737 PhaseOutput* const output = Compile::current()->output();
738
739 // We piggyback on scratch_emit_size mode to compute the slowpath stub size.
740 // We'll use that information to decide whether we need a far jump to the
741 // stub entry point or not. In scratch_emit_size mode we don't bind entry()
742 // because otherwise it will be rebound when we later emit the instructions
743 // for real.
744 if (!output->in_scratch_emit_size()) {
745 __ bind(*entry());
746 }
747
748 // If we need to load ourselves, do it here.
749 if (_do_load) {
750 if (_narrow) {
784 keepalive(masm, continuation());
785 } else if (_needs_load_ref_barrier) {
786 lrb(masm);
787 } else {
788 ShouldNotReachHere();
789 }
790 }
791
792 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
793 if (_needs_far_jump) {
794 Label L_short_jump;
795 __ cbnz(reg, L_short_jump);
796 __ b(*continuation());
797 __ bind(L_short_jump);
798 } else {
799 __ cbz(reg, *continuation());
800 }
801 }
802
803 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
804 Address gcstate(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::MARKING)));
805 Address index(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
806 Address buffer(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
807 Label L_through, L_slowpath;
808
809 // If another barrier is enabled as well, do a runtime check for a specific barrier.
810 if (_needs_load_ref_barrier) {
811 assert(L_done == nullptr, "L_done is always null when _needs_load_ref_barrier is true");
812 __ ldrb(_tmp1, gcstate);
813 __ cbz(_tmp1, L_through);
814 }
815
816 // Fast-path: put object into buffer.
817 // If buffer is already full, go slow.
818 __ ldr(_tmp1, index);
819 __ cbz(_tmp1, L_slowpath);
820 __ sub(_tmp1, _tmp1, wordSize);
821 __ str(_tmp1, index);
822 __ ldr(_tmp2, buffer);
823
824 // Store the object in queue.
825 // If object is narrow, we need to decode it before inserting.
826 if (_narrow) {
827 __ add(_tmp2, _tmp2, _tmp1);
828 __ decode_heap_oop_not_null(_tmp1, _obj);
829 __ str(_tmp1, Address(_tmp2));
830 } else {
831 // Buffer is 64-bit address, must be in base register.
832 __ str(_obj, Address(_tmp2, _tmp1));
833 }
843 __ bind(L_slowpath);
844
845 {
846 SaveLiveRegisters slr(&masm, this);
847
848 // Go to runtime and handle the rest there.
849 __ mov(c_rarg0, _obj);
850 __ lea(lr, RuntimeAddress(keepalive_runtime_entry_addr()));
851 __ blr(lr);
852 }
853 if (L_done != nullptr) {
854 __ b(*L_done);
855 } else {
856 __ bind(L_through);
857 }
858 }
859
860 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
861 Label L_slow;
862
863 // If another barrier is enabled as well, do a runtime check for a specific barrier.
864 if (_needs_keep_alive_barrier) {
865 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
866 Address gc_state_fast(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(state_to_check)));
867 __ ldrb(_tmp1, gc_state_fast);
868 maybe_far_jump_if_zero(masm, _tmp1);
869 }
870
871 // If weak references are being processed, weak/phantom loads need to go slow,
872 // regardless of their cset status.
873 if (_needs_load_ref_weak_barrier) {
874 Address gc_state_fast(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::WEAK_ROOTS)));
875 __ ldrb(_tmp1, gc_state_fast);
876 __ cbnz(_tmp1, L_slow);
877 }
878
879 // Cset-check. Fall-through to slow if in collection set.
880 bool is_aot = AOTCodeCache::is_on_for_dump();
881 if (!is_aot) {
882 __ mov(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
883 if (_narrow) {
884 __ decode_heap_oop_not_null(_tmp2, _obj);
885 __ add(_tmp1, _tmp1, _tmp2, Assembler::LSR, ShenandoahHeapRegion::region_size_bytes_shift_jint());
886 } else {
887 __ add(_tmp1, _tmp1, _obj, Assembler::LSR, ShenandoahHeapRegion::region_size_bytes_shift_jint());
888 }
889 } else {
890 // Generating AOT code, pull the cset bitmap and region shift from AOT table.
891 if (_narrow) {
892 __ decode_heap_oop_not_null(_tmp1, _obj);
893 } else {
894 __ mov(_tmp1, _obj);
895 }
896 __ lea(_tmp2, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
|
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "code/aotCodeCache.hpp"
28 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
29 #include "gc/shenandoah/mode/shenandoahMode.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
31 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
35 #include "gc/shenandoah/shenandoahRuntime.hpp"
36 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
37 #include "interpreter/interp_masm.hpp"
38 #include "interpreter/interpreter.hpp"
39 #include "nativeInst_aarch64.hpp"
40 #include "runtime/javaThread.hpp"
41 #include "runtime/sharedRuntime.hpp"
42 #ifdef COMPILER1
43 #include "c1/c1_LIRAssembler.hpp"
44 #include "c1/c1_MacroAssembler.hpp"
45 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
46 #endif
47 #ifdef COMPILER2
48 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
49 #include "opto/output.hpp"
50 #endif
51
52 #define __ masm->
53
54 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
55 Register src, Register dst, Register count, RegSet saved_regs) {
56 if (is_oop) {
57 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
58 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
59
478 __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop)));
479 __ sub(end, end, BytesPerHeapOop);
480 __ lsr(start, start, CardTable::card_shift());
481 __ lsr(end, end, CardTable::card_shift());
482
483 // number of bytes to copy
484 __ sub(count, end, start);
485
486 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
487 __ ldr(scratch, curr_ct_holder_addr);
488 __ add(start, start, scratch);
489 __ bind(L_loop);
490 __ strb(zr, Address(start, count));
491 __ subs(count, count, 1);
492 __ br(Assembler::GE, L_loop);
493 __ bind(L_done);
494 }
495
496 #undef __
497
498 address ShenandoahBarrierSetAssembler::parse_jump_address(address pc) {
499 NativeInstruction* ni = nativeInstruction_at(pc);
500 assert(ni->is_jump(), "Initial code version: GC barrier fastpath must be a jump");
501 NativeJump* jmp = nativeJump_at(pc);
502 return jmp->jump_destination();
503 }
504
505 static uint32_t encode_patchable_nop() {
506 return 0xD503201F;
507 }
508
509 static uint32_t encode_patchable_jump(address pc, address target_pc) {
510 int32_t disp = checked_cast<int32_t>((intptr_t)target_pc - (intptr_t)pc);
511 int64_t imm26 = disp >> 2;
512 guarantee(Assembler::is_simm(imm26, 26), "Maximum offset is 128MiB");
513 return 0x14000000 | (imm26 & 0x03FFFFFF);
514 }
515
516 void ShenandoahBarrierSetAssembler::insert_patchable_nop(address pc) {
517 *((uint32_t*)pc) = encode_patchable_nop();
518 assert(nativeInstruction_at(pc)->is_nop(), "Sanity");
519 }
520
521 void ShenandoahBarrierSetAssembler::insert_patchable_jump(address pc, address target_pc) {
522 *((uint32_t*)pc) = encode_patchable_jump(pc, target_pc);
523 }
524
525 bool ShenandoahBarrierSetAssembler::is_patchable_nop(address pc) {
526 return *((uint32_t*)pc) == encode_patchable_nop();
527 }
528
529 bool ShenandoahBarrierSetAssembler::is_patchable_jump(address pc, address target_pc) {
530 return *((uint32_t*)pc) == encode_patchable_jump(pc, target_pc);
531 }
532
533 #ifdef COMPILER1
534
535 #define __ ce->masm()->
536
537 void ShenandoahBarrierSetAssembler::keepalive_barrier_c1_stub(LIR_Assembler* ce, ShenandoahKeepaliveBarrierStub* stub) {
538 __ bind(*stub->entry());
539
540 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
541
542 Register obj = stub->obj()->as_register();
543
544 if (stub->do_load()) {
545 ce->mem2reg(stub->addr(), stub->obj(), T_OBJECT, lir_patch_none, nullptr, /* wide = */ false);
546 }
547 __ cbz(obj, *stub->continuation());
548 ce->store_parameter(obj, 0);
549 __ far_call(RuntimeAddress(bs->keepalive_barrier_stub()));
550 __ b(*stub->continuation());
551 }
552
727 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
728 __ ldr(tmp1, curr_ct_holder_addr);
729
730 // tmp2 = effective address
731 __ lea(tmp2, address);
732
733 // tmp2 = &card_table[ addr >> CardTable::card_shift() ] ; card index
734 __ add(tmp2, tmp1, tmp2, Assembler::LSR, CardTable::card_shift());
735
736 if (UseCondCardMark) {
737 Label L_already_dirty;
738 __ ldrb(tmp1, Address(tmp2));
739 __ cbz(tmp1, L_already_dirty);
740 __ strb(zr, Address(tmp2));
741 __ bind(L_already_dirty);
742 } else {
743 __ strb(zr, Address(tmp2));
744 }
745 }
746
747 void ShenandoahBarrierStubC2::patchable_jump(MacroAssembler& masm, const char gc_state, bool jump_when_state, Register tmp1, Register tmp2, Label* L_target) {
748 const size_t check_size = 8;
749
750 PhaseOutput* const output = Compile::current()->output();
751 if (output->in_scratch_emit_size()) {
752 // Avoid binding L_target and emitting more branches in scratch emits.
753 // We know the patched check is exactly 1 instruction long in release,
754 // and verification adds more instructions.
755 for (size_t c = 0; c < 1 DEBUG_ONLY(+ check_size); c++) {
756 __ nop();
757 }
758 return;
759 }
760
761 #ifdef ASSERT
762 Label L_fake_entry, L_skip;
763 Address gc_state_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
764
765 address check_start = __ pc();
766
767 __ ldrb(tmp1, gc_state_addr);
768 __ mov(tmp2, gc_state);
769 __ andr(tmp2, tmp1, tmp2);
770
771 // Emit the secondary jump and use it to cross-check against the actual GC state.
772 // This also checks that all interesting GC state transitions are done non-racily
773 // from the perspective of the thread executing the nmethod.
774 __ relocate(patchable_barrier_Relocation::spec(ShenandoahNMethod::encode_to_reloc(gc_state, jump_when_state)));
775 __ b(L_fake_entry);
776
777 // Currently hot-patched to NOP.
778 if (jump_when_state) {
779 __ cbz(tmp2, L_skip);
780 } else {
781 __ cbnz(tmp2, L_skip);
782 }
783 __ hlt(0);
784
785 // Currently hot-patched to JUMP.
786 __ bind(L_fake_entry);
787 if (jump_when_state) {
788 __ cbnz(tmp2, L_skip);
789 } else {
790 __ cbz(tmp2, L_skip);
791 }
792 __ hlt(0);
793
794 __ bind(L_skip);
795
796 address check_end = __ pc();
797 size_t actual_check_size = pointer_delta(check_end, check_start, Assembler::instruction_size);
798 assert(check_size == actual_check_size, "Must match: %zu != %zu", check_size, actual_check_size);
799 #endif
800
801 // Emit the unconditional branch in the first version of the method.
802 // Let the rest of runtime figure out how to manage it.
803 __ relocate(patchable_barrier_Relocation::spec(ShenandoahNMethod::encode_to_reloc(gc_state, jump_when_state)));
804 __ b(*L_target);
805 }
806
807 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp1, Register tmp2) {
808 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
809 patchable_jump_if_gc_state(masm, test_state, tmp1, tmp2, entry());
810 __ bind(*continuation());
811 }
812
813 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
814 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
815 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
816 PhaseOutput* const output = Compile::current()->output();
817
818 // We piggyback on scratch_emit_size mode to compute the slowpath stub size.
819 // We'll use that information to decide whether we need a far jump to the
820 // stub entry point or not. In scratch_emit_size mode we don't bind entry()
821 // because otherwise it will be rebound when we later emit the instructions
822 // for real.
823 if (!output->in_scratch_emit_size()) {
824 __ bind(*entry());
825 }
826
827 // If we need to load ourselves, do it here.
828 if (_do_load) {
829 if (_narrow) {
863 keepalive(masm, continuation());
864 } else if (_needs_load_ref_barrier) {
865 lrb(masm);
866 } else {
867 ShouldNotReachHere();
868 }
869 }
870
871 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
872 if (_needs_far_jump) {
873 Label L_short_jump;
874 __ cbnz(reg, L_short_jump);
875 __ b(*continuation());
876 __ bind(L_short_jump);
877 } else {
878 __ cbz(reg, *continuation());
879 }
880 }
881
882 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
883 Address index(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
884 Address buffer(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
885 Label L_through, L_slowpath;
886
887 // If another barrier is enabled as well, do a check for a specific barrier.
888 if (_needs_load_ref_barrier) {
889 assert(L_done == nullptr, "Should be");
890 char state_to_check = ShenandoahHeap::MARKING;
891 patchable_jump_if_not_gc_state(masm, state_to_check, _tmp1, _tmp2, &L_through);
892 }
893
894 // Fast-path: put object into buffer.
895 // If buffer is already full, go slow.
896 __ ldr(_tmp1, index);
897 __ cbz(_tmp1, L_slowpath);
898 __ sub(_tmp1, _tmp1, wordSize);
899 __ str(_tmp1, index);
900 __ ldr(_tmp2, buffer);
901
902 // Store the object in queue.
903 // If object is narrow, we need to decode it before inserting.
904 if (_narrow) {
905 __ add(_tmp2, _tmp2, _tmp1);
906 __ decode_heap_oop_not_null(_tmp1, _obj);
907 __ str(_tmp1, Address(_tmp2));
908 } else {
909 // Buffer is 64-bit address, must be in base register.
910 __ str(_obj, Address(_tmp2, _tmp1));
911 }
921 __ bind(L_slowpath);
922
923 {
924 SaveLiveRegisters slr(&masm, this);
925
926 // Go to runtime and handle the rest there.
927 __ mov(c_rarg0, _obj);
928 __ lea(lr, RuntimeAddress(keepalive_runtime_entry_addr()));
929 __ blr(lr);
930 }
931 if (L_done != nullptr) {
932 __ b(*L_done);
933 } else {
934 __ bind(L_through);
935 }
936 }
937
938 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
939 Label L_slow;
940
941 // If another barrier is enabled as well, do a check for a specific barrier.
942 if (_needs_keep_alive_barrier) {
943 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
944 patchable_jump_if_not_gc_state(masm, state_to_check, _tmp1, _tmp2, continuation());
945 }
946
947 // If weak references are being processed, weak/phantom loads need to go slow,
948 // regardless of their cset status.
949 if (_needs_load_ref_weak_barrier) {
950 char state_to_check = ShenandoahHeap::WEAK_ROOTS;
951 patchable_jump_if_gc_state(masm, state_to_check, _tmp1, _tmp2, &L_slow);
952 }
953
954 // Cset-check. Fall-through to slow if in collection set.
955 bool is_aot = AOTCodeCache::is_on_for_dump();
956 if (!is_aot) {
957 __ mov(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
958 if (_narrow) {
959 __ decode_heap_oop_not_null(_tmp2, _obj);
960 __ add(_tmp1, _tmp1, _tmp2, Assembler::LSR, ShenandoahHeapRegion::region_size_bytes_shift_jint());
961 } else {
962 __ add(_tmp1, _tmp1, _obj, Assembler::LSR, ShenandoahHeapRegion::region_size_bytes_shift_jint());
963 }
964 } else {
965 // Generating AOT code, pull the cset bitmap and region shift from AOT table.
966 if (_narrow) {
967 __ decode_heap_oop_not_null(_tmp1, _obj);
968 } else {
969 __ mov(_tmp1, _obj);
970 }
971 __ lea(_tmp2, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
|