< prev index next >

src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp

Print this page

        

*** 22,32 **** */ #include "precompiled.hpp" #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp" #include "gc/shenandoah/shenandoahForwarding.hpp" ! #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahHeapRegion.hpp" #include "gc/shenandoah/shenandoahHeuristics.hpp" #include "gc/shenandoah/shenandoahRuntime.hpp" #include "gc/shenandoah/shenandoahThreadLocalData.hpp" #include "interpreter/interpreter.hpp" --- 22,32 ---- */ #include "precompiled.hpp" #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp" #include "gc/shenandoah/shenandoahForwarding.hpp" ! #include "gc/shenandoah/shenandoahHeap.hpp" #include "gc/shenandoah/shenandoahHeapRegion.hpp" #include "gc/shenandoah/shenandoahHeuristics.hpp" #include "gc/shenandoah/shenandoahRuntime.hpp" #include "gc/shenandoah/shenandoahThreadLocalData.hpp" #include "interpreter/interpreter.hpp"
*** 45,55 **** void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, Register src, Register dst, Register count, RegSet saved_regs) { if (is_oop) { bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; ! if (ShenandoahSATBBarrier && !dest_uninitialized) { Label done; // Avoid calling runtime if count == 0 __ cbz(count, done); --- 45,55 ---- void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, Register src, Register dst, Register count, RegSet saved_regs) { if (is_oop) { bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; ! if (ShenandoahSATBBarrier && !dest_uninitialized && !ShenandoahHeap::heap()->heuristics()->can_do_traversal_gc()) { Label done; // Avoid calling runtime if count == 0 __ cbz(count, done);
*** 240,252 **** } Label done; __ ldr(tmp, Address(dst, oopDesc::mark_offset_in_bytes())); __ eon(tmp, tmp, zr); ! __ ands(zr, tmp, markWord::lock_mask_in_place); __ br(Assembler::NE, done); ! __ orr(tmp, tmp, markWord::marked_value); __ eon(dst, tmp, zr); __ bind(done); if (borrow_reg) { __ pop(RegSet::of(tmp), sp); --- 240,252 ---- } Label done; __ ldr(tmp, Address(dst, oopDesc::mark_offset_in_bytes())); __ eon(tmp, tmp, zr); ! __ ands(zr, tmp, markOopDesc::lock_mask_in_place); __ br(Assembler::NE, done); ! __ orr(tmp, tmp, markOopDesc::marked_value); __ eon(dst, tmp, zr); __ bind(done); if (borrow_reg) { __ pop(RegSet::of(tmp), sp);
*** 280,323 **** __ bind(done); __ leave(); } - void ShenandoahBarrierSetAssembler::load_reference_barrier_native(MacroAssembler* masm, Register dst, Register tmp) { - if (!ShenandoahLoadRefBarrier) { - return; - } - - assert(dst != rscratch2, "need rscratch2"); - - Label is_null; - Label done; - - __ cbz(dst, is_null); - - __ enter(); - - Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); - __ ldrb(rscratch2, gc_state); - - // Check for heap in evacuation phase - __ tbz(rscratch2, ShenandoahHeap::EVACUATION_BITPOS, done); - - __ mov(rscratch2, dst); - __ push_call_clobbered_registers(); - __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_native)); - __ mov(r0, rscratch2); - __ blr(lr); - __ mov(rscratch2, r0); - __ pop_call_clobbered_registers(); - __ mov(dst, rscratch2); - - __ bind(done); - __ leave(); - __ bind(is_null); - } - void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) { if (ShenandoahStoreValEnqueueBarrier) { // Save possibly live regs. RegSet live_regs = RegSet::range(r0, r4) - dst; __ push(live_regs, sp); --- 280,289 ----
*** 341,369 **** } void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp_thread) { bool on_oop = type == T_OBJECT || type == T_ARRAY; - bool not_in_heap = (decorators & IN_NATIVE) != 0; bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0; bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0; bool on_reference = on_weak || on_phantom; - bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0; BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); if (on_oop) { ! if (not_in_heap) { ! if (ShenandoahHeap::heap()->is_traversal_mode()) { ! load_reference_barrier(masm, dst, tmp1); ! keep_alive = true; ! } else { ! load_reference_barrier_native(masm, dst, tmp1); ! } ! } else { ! load_reference_barrier(masm, dst, tmp1); ! } ! if (ShenandoahKeepAliveBarrier && on_reference && keep_alive) { __ enter(); satb_write_barrier_pre(masm /* masm */, noreg /* obj */, dst /* pre_val */, rthread /* thread */, --- 307,325 ---- } void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp_thread) { bool on_oop = type == T_OBJECT || type == T_ARRAY; bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0; bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0; bool on_reference = on_weak || on_phantom; BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); if (on_oop) { ! load_reference_barrier(masm, dst, tmp1); ! ! if (ShenandoahKeepAliveBarrier && on_reference) { __ enter(); satb_write_barrier_pre(masm /* masm */, noreg /* obj */, dst /* pre_val */, rthread /* thread */,
*** 413,442 **** BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), val, noreg, noreg); } } - void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env, - Register obj, Register tmp, Label& slowpath) { - Label done; - // Resolve jobject - BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath); - - // Check for null. - __ cbz(obj, done); - - assert(obj != rscratch2, "need rscratch2"); - Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); - __ ldrb(rscratch2, gc_state); - - // Check for heap in evacuation phase - __ tbnz(rscratch2, ShenandoahHeap::EVACUATION_BITPOS, slowpath); - - __ bind(done); - } - - void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, Register addr, Register expected, Register new_val, bool acquire, bool release, bool weak, bool is_cae, Register result) { Register tmp1 = rscratch1; Register tmp2 = rscratch2; --- 369,378 ----
*** 519,567 **** __ far_call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin())); __ b(*stub->continuation()); } void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) { - ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); - __ bind(*stub->entry()); Register obj = stub->obj()->as_register(); Register res = stub->result()->as_register(); - Register tmp1 = stub->tmp1()->as_register(); - Register tmp2 = stub->tmp2()->as_register(); ! assert(res == r0, "result must arrive in r0"); if (res != obj) { __ mov(res, obj); } - // Check for null. ! __ cbz(res, *stub->continuation()); ! ! // Check for object in cset. ! __ mov(tmp2, ShenandoahHeap::in_cset_fast_test_addr()); ! __ lsr(tmp1, res, ShenandoahHeapRegion::region_size_bytes_shift_jint()); ! __ ldrb(tmp2, Address(tmp2, tmp1)); ! __ cbz(tmp2, *stub->continuation()); ! ! // Check if object is already forwarded. ! Label slow_path; ! __ ldr(tmp1, Address(res, oopDesc::mark_offset_in_bytes())); ! __ eon(tmp1, tmp1, zr); ! __ ands(zr, tmp1, markWord::lock_mask_in_place); ! __ br(Assembler::NE, slow_path); ! ! // Decode forwarded object. ! __ orr(tmp1, tmp1, markWord::marked_value); ! __ eon(res, tmp1, zr); ! __ b(*stub->continuation()); ! __ bind(slow_path); ! ce->store_parameter(res, 0); ! __ far_call(RuntimeAddress(bs->load_reference_barrier_rt_code_blob()->code_begin())); __ b(*stub->continuation()); } #undef __ --- 455,483 ---- __ far_call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin())); __ b(*stub->continuation()); } void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) { Register obj = stub->obj()->as_register(); Register res = stub->result()->as_register(); ! Label done; ! ! __ bind(*stub->entry()); if (res != obj) { __ mov(res, obj); } // Check for null. ! if (stub->needs_null_check()) { ! __ cbz(res, done); ! } ! load_reference_barrier_not_null(ce->masm(), res, rscratch1); + __ bind(done); __ b(*stub->continuation()); } #undef __
*** 611,635 **** __ bind(done); __ epilogue(); } - void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm) { - __ prologue("shenandoah_load_reference_barrier", false); - // arg0 : object to be resolved - - __ push_call_clobbered_registers(); - __ load_parameter(0, r0); - __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier)); - __ blr(lr); - __ mov(rscratch1, r0); - __ pop_call_clobbered_registers(); - __ mov(r0, rscratch1); - - __ epilogue(); - } - #undef __ #endif // COMPILER1 address ShenandoahBarrierSetAssembler::shenandoah_lrb() { --- 527,536 ----
*** 660,687 **** __ ldrb(rscratch2, Address(rscratch2, rscratch1)); __ tbnz(rscratch2, 0, work); __ ret(lr); __ bind(work); ! Label slow_path; ! __ ldr(rscratch1, Address(r0, oopDesc::mark_offset_in_bytes())); ! __ eon(rscratch1, rscratch1, zr); ! __ ands(zr, rscratch1, markWord::lock_mask_in_place); ! __ br(Assembler::NE, slow_path); ! ! // Decode forwarded object. ! __ orr(rscratch1, rscratch1, markWord::marked_value); ! __ eon(r0, rscratch1, zr); ! __ ret(lr); - __ bind(slow_path); __ enter(); // required for proper stackwalking of RuntimeStub frame __ push_call_clobbered_registers(); ! __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier)); ! __ blr(lr); __ mov(rscratch1, r0); __ pop_call_clobbered_registers(); __ mov(r0, rscratch1); __ leave(); // required for proper stackwalking of RuntimeStub frame --- 561,581 ---- __ ldrb(rscratch2, Address(rscratch2, rscratch1)); __ tbnz(rscratch2, 0, work); __ ret(lr); __ bind(work); ! __ mov(rscratch2, r0); ! resolve_forward_pointer_not_null(cgen->assembler(), r0, rscratch1); ! __ cmp(rscratch2, r0); ! __ br(Assembler::NE, done); __ enter(); // required for proper stackwalking of RuntimeStub frame __ push_call_clobbered_registers(); ! __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT)); ! __ blrt(lr, 1, 0, MacroAssembler::ret_type_integral); __ mov(rscratch1, r0); __ pop_call_clobbered_registers(); __ mov(r0, rscratch1); __ leave(); // required for proper stackwalking of RuntimeStub frame
< prev index next >