< prev index next >

src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp

Print this page

        

*** 39,49 **** #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp" #endif #define __ masm-> ! address ShenandoahBarrierSetAssembler::_shenandoah_wb = NULL; void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) { bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0; --- 39,49 ---- #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp" #endif #define __ masm-> ! address ShenandoahBarrierSetAssembler::_shenandoah_lrb = NULL; void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) { bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
*** 294,356 **** if(tosca_live) __ pop(rax); __ bind(done); } ! void ShenandoahBarrierSetAssembler::read_barrier(MacroAssembler* masm, Register dst) { ! if (ShenandoahReadBarrier) { ! read_barrier_impl(masm, dst); ! } ! } ! ! void ShenandoahBarrierSetAssembler::read_barrier_impl(MacroAssembler* masm, Register dst) { ! assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled"); Label is_null; __ testptr(dst, dst); __ jcc(Assembler::zero, is_null); ! read_barrier_not_null_impl(masm, dst); __ bind(is_null); } ! void ShenandoahBarrierSetAssembler::read_barrier_not_null(MacroAssembler* masm, Register dst) { ! if (ShenandoahReadBarrier) { ! read_barrier_not_null_impl(masm, dst); ! } ! } ! ! void ShenandoahBarrierSetAssembler::read_barrier_not_null_impl(MacroAssembler* masm, Register dst) { ! assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled"); __ movptr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset())); } ! void ShenandoahBarrierSetAssembler::write_barrier(MacroAssembler* masm, Register dst) { ! if (ShenandoahWriteBarrier) { ! write_barrier_impl(masm, dst); ! } ! } ! ! void ShenandoahBarrierSetAssembler::write_barrier_impl(MacroAssembler* masm, Register dst) { ! assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier), "Should be enabled"); #ifdef _LP64 Label done; Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL); __ jccb(Assembler::zero, done); // Heap is unstable, need to perform the read-barrier even if WB is inactive ! read_barrier_not_null(masm, dst); __ testb(gc_state, ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL); __ jccb(Assembler::zero, done); if (dst != rax) { __ xchgptr(dst, rax); // Move obj into rax and save rax into obj. } ! __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_wb()))); if (dst != rax) { __ xchgptr(rax, dst); // Swap back obj with rax. } --- 294,338 ---- if(tosca_live) __ pop(rax); __ bind(done); } ! void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst) { ! assert(ShenandoahCASBarrier, "should be enabled"); Label is_null; __ testptr(dst, dst); __ jcc(Assembler::zero, is_null); ! resolve_forward_pointer_not_null(masm, dst); __ bind(is_null); } ! void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst) { ! assert(ShenandoahCASBarrier || ShenandoahLoadRefBarrier, "should be enabled"); __ movptr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset())); } ! void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembler* masm, Register dst) { ! assert(ShenandoahLoadRefBarrier, "Should be enabled"); #ifdef _LP64 Label done; Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL); __ jccb(Assembler::zero, done); // Heap is unstable, need to perform the read-barrier even if WB is inactive ! resolve_forward_pointer_not_null(masm, dst); __ testb(gc_state, ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL); __ jccb(Assembler::zero, done); if (dst != rax) { __ xchgptr(dst, rax); // Move obj into rax and save rax into obj. } ! __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_lrb()))); if (dst != rax) { __ xchgptr(rax, dst); // Swap back obj with rax. }
*** 359,386 **** Unimplemented(); #endif } void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) { ! if (ShenandoahStoreValReadBarrier || ShenandoahStoreValEnqueueBarrier) { storeval_barrier_impl(masm, dst, tmp); } } void ShenandoahBarrierSetAssembler::storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp) { ! assert(UseShenandoahGC && (ShenandoahStoreValReadBarrier || ShenandoahStoreValEnqueueBarrier), "should be enabled"); if (dst == noreg) return; #ifdef _LP64 if (ShenandoahStoreValEnqueueBarrier) { - Label is_null; - __ testptr(dst, dst); - __ jcc(Assembler::zero, is_null); - write_barrier_impl(masm, dst); - __ bind(is_null); - // The set of registers to be saved+restored is the same as in the write-barrier above. // Those are the commonly used registers in the interpreter. __ pusha(); // __ push_callee_saved_registers(); __ subptr(rsp, 2 * Interpreter::stackElementSize); --- 341,362 ---- Unimplemented(); #endif } void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) { ! if (ShenandoahStoreValEnqueueBarrier) { storeval_barrier_impl(masm, dst, tmp); } } void ShenandoahBarrierSetAssembler::storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp) { ! assert(ShenandoahStoreValEnqueueBarrier, "should be enabled"); if (dst == noreg) return; #ifdef _LP64 if (ShenandoahStoreValEnqueueBarrier) { // The set of registers to be saved+restored is the same as in the write-barrier above. // Those are the commonly used registers in the interpreter. __ pusha(); // __ push_callee_saved_registers(); __ subptr(rsp, 2 * Interpreter::stackElementSize);
*** 390,443 **** __ movdbl(xmm0, Address(rsp, 0)); __ addptr(rsp, 2 * Interpreter::stackElementSize); //__ pop_callee_saved_registers(); __ popa(); } - if (ShenandoahStoreValReadBarrier) { - read_barrier_impl(masm, dst); - } #else Unimplemented(); #endif } void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp_thread) { bool on_oop = type == T_OBJECT || type == T_ARRAY; - bool in_heap = (decorators & IN_HEAP) != 0; bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0; bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0; bool on_reference = on_weak || on_phantom; ! if (in_heap) { ! read_barrier_not_null(masm, src.base()); ! } ! BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); ! if (ShenandoahKeepAliveBarrier && on_oop && on_reference) { ! const Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread); ! NOT_LP64(__ get_thread(thread)); ! ! // Generate the SATB pre-barrier code to log the value of ! // the referent field in an SATB buffer. ! shenandoah_write_barrier_pre(masm /* masm */, ! noreg /* obj */, ! dst /* pre_val */, ! thread /* thread */, ! tmp1 /* tmp */, ! true /* tosca_live */, ! true /* expand_call */); } } void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Address dst, Register val, Register tmp1, Register tmp2) { bool in_heap = (decorators & IN_HEAP) != 0; bool as_normal = (decorators & AS_NORMAL) != 0; ! if (in_heap) { ! write_barrier(masm, dst.base()); ! } ! if (type == T_OBJECT || type == T_ARRAY) { bool needs_pre_barrier = as_normal; Register tmp3 = LP64_ONLY(r8) NOT_LP64(rsi); Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx); // flatten object address if needed --- 366,424 ---- __ movdbl(xmm0, Address(rsp, 0)); __ addptr(rsp, 2 * Interpreter::stackElementSize); //__ pop_callee_saved_registers(); __ popa(); } #else Unimplemented(); #endif } + void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst) { + if (ShenandoahLoadRefBarrier) { + Label done; + __ testptr(dst, dst); + __ jcc(Assembler::zero, done); + load_reference_barrier_not_null(masm, dst); + __ bind(done); + } + } + void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp_thread) { bool on_oop = type == T_OBJECT || type == T_ARRAY; bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0; bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0; bool on_reference = on_weak || on_phantom; ! BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); ! if (on_oop) { ! load_reference_barrier(masm, dst); ! ! if (ShenandoahKeepAliveBarrier && on_reference) { ! const Register thread = NOT_LP64(tmp_thread) ! LP64_ONLY(r15_thread); ! NOT_LP64(__ get_thread(thread)); ! // Generate the SATB pre-barrier code to log the value of ! // the referent field in an SATB buffer. ! shenandoah_write_barrier_pre(masm /* masm */, ! noreg /* obj */, ! dst /* pre_val */, ! thread /* thread */, ! tmp1 /* tmp */, ! true /* tosca_live */, ! true /* expand_call */); ! } } } void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Address dst, Register val, Register tmp1, Register tmp2) { + bool on_oop = type == T_OBJECT || type == T_ARRAY; bool in_heap = (decorators & IN_HEAP) != 0; bool as_normal = (decorators & AS_NORMAL) != 0; ! if (on_oop && in_heap) { bool needs_pre_barrier = as_normal; Register tmp3 = LP64_ONLY(r8) NOT_LP64(rsi); Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx); // flatten object address if needed
*** 476,523 **** } else { BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2); } } - #ifndef _LP64 - void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, - Address obj1, jobject obj2) { - Unimplemented(); - } - - void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, - Register obj1, jobject obj2) { - Unimplemented(); - } - #endif - - - void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, Register op1, Register op2) { - __ cmpptr(op1, op2); - if (ShenandoahAcmpBarrier) { - Label done; - __ jccb(Assembler::equal, done); - read_barrier(masm, op1); - read_barrier(masm, op2); - __ cmpptr(op1, op2); - __ bind(done); - } - } - - void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, Register src1, Address src2) { - __ cmpptr(src1, src2); - if (ShenandoahAcmpBarrier) { - Label done; - __ jccb(Assembler::equal, done); - __ movptr(rscratch2, src2); - read_barrier(masm, src1); - read_barrier(masm, rscratch2); - __ cmpptr(src1, rscratch2); - __ bind(done); - } - } - void ShenandoahBarrierSetAssembler::tlab_allocate(MacroAssembler* masm, Register thread, Register obj, Register var_size_in_bytes, int con_size_in_bytes, Register t1, Register t2, --- 457,466 ----
*** 563,657 **** __ subptr(var_size_in_bytes, obj); } __ verify_tlab(); } - void ShenandoahBarrierSetAssembler::resolve(MacroAssembler* masm, DecoratorSet decorators, Register obj) { - bool oop_not_null = (decorators & IS_NOT_NULL) != 0; - bool is_write = (decorators & ACCESS_WRITE) != 0; - if (is_write) { - if (oop_not_null) { - write_barrier(masm, obj); - } else { - Label done; - __ testptr(obj, obj); - __ jcc(Assembler::zero, done); - write_barrier(masm, obj); - __ bind(done); - } - } else { - if (oop_not_null) { - read_barrier_not_null(masm, obj); - } else { - read_barrier(masm, obj); - } - } - } - // Special Shenandoah CAS implementation that handles false negatives // due to concurrent evacuation. #ifndef _LP64 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, Register res, Address addr, Register oldval, Register newval, ! bool exchange, bool encode, Register tmp1, Register tmp2) { // Shenandoah has no 32-bit version for this. Unimplemented(); } #else void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, Register res, Address addr, Register oldval, Register newval, ! bool exchange, bool encode, Register tmp1, Register tmp2) { ! if (!ShenandoahCASBarrier) { ! #ifdef _LP64 ! if (UseCompressedOops) { ! if (encode) { ! __ encode_heap_oop(oldval); ! __ mov(rscratch1, newval); ! __ encode_heap_oop(rscratch1); ! newval = rscratch1; ! } ! if (os::is_MP()) { ! __ lock(); ! } ! // oldval (rax) is implicitly used by this instruction ! __ cmpxchgl(newval, addr); ! } else ! #endif ! { ! if (os::is_MP()) { ! __ lock(); ! } ! __ cmpxchgptr(newval, addr); ! } ! ! if (!exchange) { ! assert(res != NULL, "need result register"); ! __ setb(Assembler::equal, res); ! __ movzbl(res, res); ! } ! return; ! } ! assert(ShenandoahCASBarrier, "Should only be used when CAS barrier is enabled"); assert(oldval == rax, "must be in rax for implicit use in cmpxchg"); Label retry, done; - // Apply storeval barrier to newval. - if (encode) { - storeval_barrier(masm, newval, tmp1); - } - - if (UseCompressedOops) { - if (encode) { - __ encode_heap_oop(oldval); - __ mov(rscratch1, newval); - __ encode_heap_oop(rscratch1); - newval = rscratch1; - } - } - // Remember oldval for retry logic below if (UseCompressedOops) { __ movl(tmp1, oldval); } else { __ movptr(tmp1, oldval); --- 506,533 ---- __ subptr(var_size_in_bytes, obj); } __ verify_tlab(); } // Special Shenandoah CAS implementation that handles false negatives // due to concurrent evacuation. #ifndef _LP64 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, Register res, Address addr, Register oldval, Register newval, ! bool exchange, Register tmp1, Register tmp2) { // Shenandoah has no 32-bit version for this. Unimplemented(); } #else void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, Register res, Address addr, Register oldval, Register newval, ! bool exchange, Register tmp1, Register tmp2) { assert(ShenandoahCASBarrier, "Should only be used when CAS barrier is enabled"); assert(oldval == rax, "must be in rax for implicit use in cmpxchg"); Label retry, done; // Remember oldval for retry logic below if (UseCompressedOops) { __ movl(tmp1, oldval); } else { __ movptr(tmp1, oldval);
*** 675,693 **** // both to-space pointers. If they mismatch, then it was a legitimate failure. // if (UseCompressedOops) { __ decode_heap_oop(tmp1); } ! read_barrier_impl(masm, tmp1); if (UseCompressedOops) { __ movl(tmp2, oldval); __ decode_heap_oop(tmp2); } else { __ movptr(tmp2, oldval); } ! read_barrier_impl(masm, tmp2); __ cmpptr(tmp1, tmp2); __ jcc(Assembler::notEqual, done, true); // Step 3. Try to CAS again with resolved to-space pointers. --- 551,569 ---- // both to-space pointers. If they mismatch, then it was a legitimate failure. // if (UseCompressedOops) { __ decode_heap_oop(tmp1); } ! resolve_forward_pointer(masm, tmp1); if (UseCompressedOops) { __ movl(tmp2, oldval); __ decode_heap_oop(tmp2); } else { __ movptr(tmp2, oldval); } ! resolve_forward_pointer(masm, tmp2); __ cmpptr(tmp1, tmp2); __ jcc(Assembler::notEqual, done, true); // Step 3. Try to CAS again with resolved to-space pointers.
*** 709,719 **** __ movl(tmp2, oldval); __ decode_heap_oop(tmp2); } else { __ movptr(tmp2, oldval); } ! read_barrier_impl(masm, tmp2); __ cmpptr(tmp1, tmp2); __ jcc(Assembler::equal, retry, true); // Step 4. If we need a boolean result out of CAS, check the flag again, --- 585,595 ---- __ movl(tmp2, oldval); __ decode_heap_oop(tmp2); } else { __ movptr(tmp2, oldval); } ! resolve_forward_pointer(masm, tmp2); __ cmpptr(tmp1, tmp2); __ jcc(Assembler::equal, retry, true); // Step 4. If we need a boolean result out of CAS, check the flag again,
*** 857,867 **** __ call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin())); __ jmp(*stub->continuation()); } ! void ShenandoahBarrierSetAssembler::gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub) { __ bind(*stub->entry()); Label done; Register obj = stub->obj()->as_register(); Register res = stub->result()->as_register(); --- 733,743 ---- __ call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin())); __ jmp(*stub->continuation()); } ! void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) { __ bind(*stub->entry()); Label done; Register obj = stub->obj()->as_register(); Register res = stub->result()->as_register();
*** 874,884 **** if (stub->needs_null_check()) { __ testptr(res, res); __ jcc(Assembler::zero, done); } ! write_barrier(ce->masm(), res); __ bind(done); __ jmp(*stub->continuation()); } --- 750,760 ---- if (stub->needs_null_check()) { __ testptr(res, res); __ jcc(Assembler::zero, done); } ! load_reference_barrier_not_null(ce->masm(), res); __ bind(done); __ jmp(*stub->continuation()); }
*** 944,963 **** #undef __ #endif // COMPILER1 ! address ShenandoahBarrierSetAssembler::shenandoah_wb() { ! assert(_shenandoah_wb != NULL, "need write barrier stub"); ! return _shenandoah_wb; } #define __ cgen->assembler()-> ! address ShenandoahBarrierSetAssembler::generate_shenandoah_wb(StubCodeGenerator* cgen) { __ align(CodeEntryAlignment); ! StubCodeMark mark(cgen, "StubRoutines", "shenandoah_wb"); address start = __ pc(); #ifdef _LP64 Label not_done; --- 820,839 ---- #undef __ #endif // COMPILER1 ! address ShenandoahBarrierSetAssembler::shenandoah_lrb() { ! assert(_shenandoah_lrb != NULL, "need load reference barrier stub"); ! return _shenandoah_lrb; } #define __ cgen->assembler()-> ! address ShenandoahBarrierSetAssembler::generate_shenandoah_lrb(StubCodeGenerator* cgen) { __ align(CodeEntryAlignment); ! StubCodeMark mark(cgen, "StubRoutines", "shenandoah_lrb"); address start = __ pc(); #ifdef _LP64 Label not_done;
*** 1001,1011 **** __ push(r13); __ push(r14); __ push(r15); save_vector_registers(cgen->assembler()); __ movptr(rdi, rax); ! __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT), rdi); restore_vector_registers(cgen->assembler()); __ pop(r15); __ pop(r14); __ pop(r13); __ pop(r12); --- 877,887 ---- __ push(r13); __ push(r14); __ push(r15); save_vector_registers(cgen->assembler()); __ movptr(rdi, rax); ! __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT), rdi); restore_vector_registers(cgen->assembler()); __ pop(r15); __ pop(r14); __ pop(r13); __ pop(r12);
*** 1028,1041 **** } #undef __ void ShenandoahBarrierSetAssembler::barrier_stubs_init() { ! if (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier) { int stub_code_size = 4096; ResourceMark rm; BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size); CodeBuffer buf(bb); StubCodeGenerator cgen(&buf); ! _shenandoah_wb = generate_shenandoah_wb(&cgen); } } --- 904,917 ---- } #undef __ void ShenandoahBarrierSetAssembler::barrier_stubs_init() { ! if (ShenandoahLoadRefBarrier) { int stub_code_size = 4096; ResourceMark rm; BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size); CodeBuffer buf(bb); StubCodeGenerator cgen(&buf); ! _shenandoah_lrb = generate_shenandoah_lrb(&cgen); } }
< prev index next >