785 #endif // COMPILER1
786
787 #ifdef COMPILER2
788
789 #define __ masm->
790
791 static bool needs_acquiring_load_reserved(const MachNode* n) {
792 assert(n->is_CAS(true), "expecting a compare and swap");
793 if (n->is_CAS(false)) {
794 assert(n->has_trailing_membar(), "expected trailing membar");
795 } else {
796 return n->has_trailing_membar();
797 }
798 return true;
799 }
800
801 void ShenandoahBarrierStubC2::gc_state_check_c2(MacroAssembler* masm,
802 Register gcstate,
803 const unsigned char test_state,
804 ShenandoahBarrierStubC2* slow_stub) {
805 int bit_to_check = ShenandoahThreadLocalData::gc_state_to_fast_bit(test_state);
806 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_offset()));
807 __ lbu(gcstate, gc_state_fast);
808 __ test_bit(gcstate, gcstate, bit_to_check);
809 __ bnez(gcstate, *slow_stub->entry());
810
811 // Fast path falls through here when the barrier is not needed.
812 __ bind(*slow_stub->continuation());
813 }
814
815 void ShenandoahBarrierSetAssembler::compare_and_set_c2(const MachNode* node, MacroAssembler* masm, Register res, Register addr,
816 Register oldval, Register newval, Register tmp, bool exchange, bool maybe_null, bool narrow, bool weak) {
817 const Assembler::Aqrl acquire = needs_acquiring_load_reserved(node) ? Assembler::aq : Assembler::relaxed;
818 const Assembler::Aqrl release = Assembler::rl;
819
820 // Pre-barrier covers several things:
821 // a. Avoids false positives from CAS encountering to-space memory values.
822 // b. Satisfies the need for LRB for the CAE result.
823 // c. Records old value for the sake of SATB.
824 //
825 // (a) and (b) are covered because load barrier does memory location fixup.
826 // (c) is covered by KA on the current memory value.
827 if (ShenandoahBarrierStubC2::needs_slow_barrier(node)) {
828 ShenandoahBarrierStubC2* const stub = ShenandoahBarrierStubC2::create(node, tmp, Address(addr, 0), narrow, /* do_load: */ true);
829 char check = 0;
830 check |= ShenandoahBarrierStubC2::needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
831 check |= ShenandoahBarrierStubC2::needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
832 assert(!ShenandoahBarrierStubC2::needs_load_ref_barrier_weak(node), "Not supported for CAS/CAE");
937 }
938 } else {
939 __ ld(dst, src);
940 if (acquire) {
941 __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
942 }
943 }
944
945 // Post-barrier: LRB / KA / weak-root processing.
946 if (ShenandoahBarrierStubC2::needs_slow_barrier(node)) {
947 ShenandoahBarrierStubC2* const stub = ShenandoahBarrierStubC2::create(node, dst, src, narrow, /* do_load: */ false);
948 char check = 0;
949 check |= ShenandoahBarrierStubC2::needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
950 check |= ShenandoahBarrierStubC2::needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
951 check |= ShenandoahBarrierStubC2::needs_load_ref_barrier_weak(node) ? ShenandoahHeap::WEAK_ROOTS : 0;
952 ShenandoahBarrierStubC2::gc_state_check_c2(masm, t0, check, stub);
953 }
954 }
955
956 void ShenandoahBarrierSetAssembler::card_barrier_c2(const MachNode* node, MacroAssembler* masm, Address address) {
957 if (!ShenandoahBarrierStubC2::needs_card_table_barrier(node)) {
958 return;
959 }
960
961 assert(CardTable::dirty_card_val() == 0, "must be");
962 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
963
964 // t1 = effective address
965 __ la(t1, address);
966
967 // t1 = card index
968 __ srli(t1, t1, CardTable::card_shift());
969
970 // t0 = card table base
971 Address curr_ct_holder_addr(xthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
972 __ ld(t0, curr_ct_holder_addr);
973
974 // t1 = &card_table[card_index]
975 __ add(t1, t1, t0);
976
977 if (UseCondCardMark) {
|
785 #endif // COMPILER1
786
787 #ifdef COMPILER2
788
789 #define __ masm->
790
791 static bool needs_acquiring_load_reserved(const MachNode* n) {
792 assert(n->is_CAS(true), "expecting a compare and swap");
793 if (n->is_CAS(false)) {
794 assert(n->has_trailing_membar(), "expected trailing membar");
795 } else {
796 return n->has_trailing_membar();
797 }
798 return true;
799 }
800
801 void ShenandoahBarrierStubC2::gc_state_check_c2(MacroAssembler* masm,
802 Register gcstate,
803 const unsigned char test_state,
804 ShenandoahBarrierStubC2* slow_stub) {
805 if (ShenandoahGCStateCheckRemove) {
806 // Unrealistic: remove all barrier fastpath checks.
807 } else if (ShenandoahGCStateCheckHotpatch) {
808 __ nop();
809 } else {
810 int bit_to_check = ShenandoahThreadLocalData::gc_state_to_fast_bit(test_state);
811 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_offset()));
812 __ lbu(gcstate, gc_state_fast);
813 __ test_bit(gcstate, gcstate, bit_to_check);
814 __ bnez(gcstate, *slow_stub->entry());
815
816 // Fast path falls through here when the barrier is not needed.
817 __ bind(*slow_stub->continuation());
818 }
819 }
820
821 void ShenandoahBarrierSetAssembler::compare_and_set_c2(const MachNode* node, MacroAssembler* masm, Register res, Register addr,
822 Register oldval, Register newval, Register tmp, bool exchange, bool maybe_null, bool narrow, bool weak) {
823 const Assembler::Aqrl acquire = needs_acquiring_load_reserved(node) ? Assembler::aq : Assembler::relaxed;
824 const Assembler::Aqrl release = Assembler::rl;
825
826 // Pre-barrier covers several things:
827 // a. Avoids false positives from CAS encountering to-space memory values.
828 // b. Satisfies the need for LRB for the CAE result.
829 // c. Records old value for the sake of SATB.
830 //
831 // (a) and (b) are covered because load barrier does memory location fixup.
832 // (c) is covered by KA on the current memory value.
833 if (ShenandoahBarrierStubC2::needs_slow_barrier(node)) {
834 ShenandoahBarrierStubC2* const stub = ShenandoahBarrierStubC2::create(node, tmp, Address(addr, 0), narrow, /* do_load: */ true);
835 char check = 0;
836 check |= ShenandoahBarrierStubC2::needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
837 check |= ShenandoahBarrierStubC2::needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
838 assert(!ShenandoahBarrierStubC2::needs_load_ref_barrier_weak(node), "Not supported for CAS/CAE");
943 }
944 } else {
945 __ ld(dst, src);
946 if (acquire) {
947 __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
948 }
949 }
950
951 // Post-barrier: LRB / KA / weak-root processing.
952 if (ShenandoahBarrierStubC2::needs_slow_barrier(node)) {
953 ShenandoahBarrierStubC2* const stub = ShenandoahBarrierStubC2::create(node, dst, src, narrow, /* do_load: */ false);
954 char check = 0;
955 check |= ShenandoahBarrierStubC2::needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
956 check |= ShenandoahBarrierStubC2::needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
957 check |= ShenandoahBarrierStubC2::needs_load_ref_barrier_weak(node) ? ShenandoahHeap::WEAK_ROOTS : 0;
958 ShenandoahBarrierStubC2::gc_state_check_c2(masm, t0, check, stub);
959 }
960 }
961
962 void ShenandoahBarrierSetAssembler::card_barrier_c2(const MachNode* node, MacroAssembler* masm, Address address) {
963 if (ShenandoahSkipBarriers || (node->barrier_data() & ShenandoahBitCardMark) == 0) {
964 return;
965 }
966
967 assert(CardTable::dirty_card_val() == 0, "must be");
968 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
969
970 // t1 = effective address
971 __ la(t1, address);
972
973 // t1 = card index
974 __ srli(t1, t1, CardTable::card_shift());
975
976 // t0 = card table base
977 Address curr_ct_holder_addr(xthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
978 __ ld(t0, curr_ct_holder_addr);
979
980 // t1 = &card_table[card_index]
981 __ add(t1, t1, t0);
982
983 if (UseCondCardMark) {
|