< prev index next >

src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp

Print this page

 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
 27 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
 28 #include "gc/shenandoah/shenandoahForwarding.hpp"
 29 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 30 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
 31 #include "gc/shenandoah/shenandoahRuntime.hpp"
 32 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
 33 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"

 34 #include "interpreter/interpreter.hpp"
 35 #include "interpreter/interp_masm.hpp"
 36 #include "runtime/javaThread.hpp"
 37 #include "runtime/sharedRuntime.hpp"
 38 #ifdef COMPILER1
 39 #include "c1/c1_LIRAssembler.hpp"
 40 #include "c1/c1_MacroAssembler.hpp"
 41 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
 42 #endif
 43 
 44 #define __ masm->
 45 
 46 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
 47                                                        Register src, Register dst, Register count, RegSet saved_regs) {
 48   if (is_oop) {
 49     bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
 50     if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahIUBarrier || ShenandoahLoadRefBarrier) {
 51 
 52       Label done;
 53 
 54       // Avoid calling runtime if count == 0
 55       __ cbz(count, done);
 56 
 57       // Is GC active?
 58       Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 59       __ ldrb(rscratch1, gc_state);
 60       if (ShenandoahSATBBarrier && dest_uninitialized) {
 61         __ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
 62       } else {
 63         __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING);
 64         __ tst(rscratch1, rscratch2);
 65         __ br(Assembler::EQ, done);
 66       }
 67 
 68       __ push(saved_regs, sp);
 69       if (UseCompressedOops) {
 70         __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop_entry), src, dst, count);
 71       } else {
 72         __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop_entry), src, dst, count);
 73       }
 74       __ pop(saved_regs, sp);
 75       __ bind(done);
 76     }
 77   }
 78 }
 79 







 80 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
 81                                                                  Register obj,
 82                                                                  Register pre_val,
 83                                                                  Register thread,
 84                                                                  Register tmp,
 85                                                                  bool tosca_live,
 86                                                                  bool expand_call) {
 87   if (ShenandoahSATBBarrier) {
 88     satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, rscratch1, tosca_live, expand_call);
 89   }
 90 }
 91 
 92 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
 93                                                            Register obj,
 94                                                            Register pre_val,
 95                                                            Register thread,
 96                                                            Register tmp1,
 97                                                            Register tmp2,
 98                                                            bool tosca_live,
 99                                                            bool expand_call) {

358     BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
359   }
360 
361   // 3: apply keep-alive barrier if needed
362   if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
363     __ enter(/*strip_ret_addr*/true);
364     __ push_call_clobbered_registers();
365     satb_write_barrier_pre(masm /* masm */,
366                            noreg /* obj */,
367                            dst /* pre_val */,
368                            rthread /* thread */,
369                            tmp1 /* tmp1 */,
370                            tmp2 /* tmp2 */,
371                            true /* tosca_live */,
372                            true /* expand_call */);
373     __ pop_call_clobbered_registers();
374     __ leave();
375   }
376 }
377 

























378 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
379                                              Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
380   bool on_oop = is_reference_type(type);
381   if (!on_oop) {
382     BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
383     return;
384   }
385 
386   // flatten object address if needed
387   if (dst.index() == noreg && dst.offset() == 0) {
388     if (dst.base() != tmp3) {
389       __ mov(tmp3, dst.base());
390     }
391   } else {
392     __ lea(tmp3, dst);
393   }
394 
395   shenandoah_write_barrier_pre(masm,
396                                tmp3 /* obj */,
397                                tmp2 /* pre_val */,
398                                rthread /* thread */,
399                                tmp1  /* tmp */,
400                                val != noreg /* tosca_live */,
401                                false /* expand_call */);
402 
403   if (val == noreg) {
404     BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), noreg, noreg, noreg, noreg);
405   } else {
406     iu_barrier(masm, val, tmp1);
407     // G1 barrier needs uncompressed oop for region cross check.
408     Register new_val = val;
409     if (UseCompressedOops) {
410       new_val = rscratch2;
411       __ mov(new_val, val);
412     }
413     BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);

414   }
415 
416 }
417 
418 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
419                                                                   Register obj, Register tmp, Label& slowpath) {
420   Label done;
421   // Resolve jobject
422   BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
423 
424   // Check for null.
425   __ cbz(obj, done);
426 
427   assert(obj != rscratch2, "need rscratch2");
428   Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset());
429   __ lea(rscratch2, gc_state);
430   __ ldrb(rscratch2, Address(rscratch2));
431 
432   // Check for heap in evacuation phase
433   __ tbnz(rscratch2, ShenandoahHeap::EVACUATION_BITPOS, slowpath);

578   if (is_cae) {
579     // We're falling through to done to indicate success.  Success
580     // with is_cae is denoted by returning the value of expected as
581     // result.
582     __ mov(tmp2, expected);
583   }
584 
585   __ bind(done);
586   // At entry to done, the Z (EQ) flag is on iff if the CAS
587   // operation was successful.  Additionally, if is_cae, tmp2 holds
588   // the value most recently fetched from addr. In this case, success
589   // is denoted by tmp2 matching expected.
590 
591   if (is_cae) {
592     __ mov(result, tmp2);
593   } else {
594     __ cset(result, Assembler::EQ);
595   }
596 }
597 





























598 #undef __
599 
600 #ifdef COMPILER1
601 
602 #define __ ce->masm()->
603 
604 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
605   ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
606   // At this point we know that marking is in progress.
607   // If do_load() is true then we have to emit the
608   // load of the previous value; otherwise it has already
609   // been loaded into _pre_val.
610 
611   __ bind(*stub->entry());
612 
613   assert(stub->pre_val()->is_register(), "Precondition.");
614 
615   Register pre_val_reg = stub->pre_val()->as_register();
616 
617   if (stub->do_load()) {

678 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
679   __ prologue("shenandoah_pre_barrier", false);
680 
681   // arg0 : previous value of memory
682 
683   BarrierSet* bs = BarrierSet::barrier_set();
684 
685   const Register pre_val = r0;
686   const Register thread = rthread;
687   const Register tmp = rscratch1;
688 
689   Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
690   Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
691 
692   Label done;
693   Label runtime;
694 
695   // Is marking still active?
696   Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
697   __ ldrb(tmp, gc_state);
698   __ tbz(tmp, ShenandoahHeap::MARKING_BITPOS, done);






699 
700   // Can we store original value in the thread's buffer?
701   __ ldr(tmp, queue_index);
702   __ cbz(tmp, runtime);
703 
704   __ sub(tmp, tmp, wordSize);
705   __ str(tmp, queue_index);
706   __ ldr(rscratch2, buffer);
707   __ add(tmp, tmp, rscratch2);
708   __ load_parameter(0, rscratch2);
709   __ str(rscratch2, Address(tmp, 0));
710   __ b(done);
711 
712   __ bind(runtime);
713   __ push_call_clobbered_registers();
714   __ load_parameter(0, pre_val);
715   __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread);
716   __ pop_call_clobbered_registers();
717   __ bind(done);
718 

 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
 27 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
 28 #include "gc/shenandoah/shenandoahForwarding.hpp"
 29 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 30 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
 31 #include "gc/shenandoah/shenandoahRuntime.hpp"
 32 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
 33 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
 34 #include "gc/shenandoah/mode/shenandoahMode.hpp"
 35 #include "interpreter/interpreter.hpp"
 36 #include "interpreter/interp_masm.hpp"
 37 #include "runtime/javaThread.hpp"
 38 #include "runtime/sharedRuntime.hpp"
 39 #ifdef COMPILER1
 40 #include "c1/c1_LIRAssembler.hpp"
 41 #include "c1/c1_MacroAssembler.hpp"
 42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
 43 #endif
 44 
 45 #define __ masm->
 46 
 47 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
 48                                                        Register src, Register dst, Register count, RegSet saved_regs) {
 49   if (is_oop) {
 50     bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
 51     if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahIUBarrier || ShenandoahLoadRefBarrier) {
 52 
 53       Label done;
 54 
 55       // Avoid calling runtime if count == 0
 56       __ cbz(count, done);
 57 
 58       // Is GC active?
 59       Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 60       __ ldrb(rscratch1, gc_state);
 61       if (ShenandoahSATBBarrier && dest_uninitialized) {
 62         __ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
 63       } else {
 64         __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING);
 65         __ tst(rscratch1, rscratch2);
 66         __ br(Assembler::EQ, done);
 67       }
 68 
 69       __ push(saved_regs, sp);
 70       if (UseCompressedOops) {
 71         __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop_entry), src, dst, count);
 72       } else {
 73         __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop_entry), src, dst, count);
 74       }
 75       __ pop(saved_regs, sp);
 76       __ bind(done);
 77     }
 78   }
 79 }
 80 
 81 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
 82                                                        Register start, Register count, Register tmp, RegSet saved_regs) {
 83   if (is_oop) {
 84     gen_write_ref_array_post_barrier(masm, decorators, start, count, tmp, saved_regs);
 85   }
 86 }
 87 
 88 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
 89                                                                  Register obj,
 90                                                                  Register pre_val,
 91                                                                  Register thread,
 92                                                                  Register tmp,
 93                                                                  bool tosca_live,
 94                                                                  bool expand_call) {
 95   if (ShenandoahSATBBarrier) {
 96     satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, rscratch1, tosca_live, expand_call);
 97   }
 98 }
 99 
100 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
101                                                            Register obj,
102                                                            Register pre_val,
103                                                            Register thread,
104                                                            Register tmp1,
105                                                            Register tmp2,
106                                                            bool tosca_live,
107                                                            bool expand_call) {

366     BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
367   }
368 
369   // 3: apply keep-alive barrier if needed
370   if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
371     __ enter(/*strip_ret_addr*/true);
372     __ push_call_clobbered_registers();
373     satb_write_barrier_pre(masm /* masm */,
374                            noreg /* obj */,
375                            dst /* pre_val */,
376                            rthread /* thread */,
377                            tmp1 /* tmp1 */,
378                            tmp2 /* tmp2 */,
379                            true /* tosca_live */,
380                            true /* expand_call */);
381     __ pop_call_clobbered_registers();
382     __ leave();
383   }
384 }
385 
386 void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj) {
387   if (!ShenandoahHeap::heap()->mode()->is_generational()) {
388       return;
389   }
390 
391   ShenandoahBarrierSet* ctbs = ShenandoahBarrierSet::barrier_set();
392   CardTable* ct = ctbs->card_table();
393 
394   __ lsr(obj, obj, CardTable::card_shift());
395 
396   assert(CardTable::dirty_card_val() == 0, "must be");
397 
398   __ load_byte_map_base(rscratch1);
399 
400   if (UseCondCardMark) {
401     Label L_already_dirty;
402     __ ldrb(rscratch2,  Address(obj, rscratch1));
403     __ cbz(rscratch2, L_already_dirty);
404     __ strb(zr, Address(obj, rscratch1));
405     __ bind(L_already_dirty);
406   } else {
407     __ strb(zr, Address(obj, rscratch1));
408   }
409 }
410 
411 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
412                                              Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
413   bool on_oop = is_reference_type(type);
414   if (!on_oop) {
415     BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
416     return;
417   }
418 
419   // flatten object address if needed
420   if (dst.index() == noreg && dst.offset() == 0) {
421     if (dst.base() != tmp3) {
422       __ mov(tmp3, dst.base());
423     }
424   } else {
425     __ lea(tmp3, dst);
426   }
427 
428   shenandoah_write_barrier_pre(masm,
429                                tmp3 /* obj */,
430                                tmp2 /* pre_val */,
431                                rthread /* thread */,
432                                tmp1  /* tmp */,
433                                val != noreg /* tosca_live */,
434                                false /* expand_call */);
435 
436   if (val == noreg) {
437     BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), noreg, noreg, noreg, noreg);
438   } else {
439     iu_barrier(masm, val, tmp1);
440     // G1 barrier needs uncompressed oop for region cross check.
441     Register new_val = val;
442     if (UseCompressedOops) {
443       new_val = rscratch2;
444       __ mov(new_val, val);
445     }
446     BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
447     store_check(masm, r3);
448   }
449 
450 }
451 
452 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
453                                                                   Register obj, Register tmp, Label& slowpath) {
454   Label done;
455   // Resolve jobject
456   BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
457 
458   // Check for null.
459   __ cbz(obj, done);
460 
461   assert(obj != rscratch2, "need rscratch2");
462   Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset());
463   __ lea(rscratch2, gc_state);
464   __ ldrb(rscratch2, Address(rscratch2));
465 
466   // Check for heap in evacuation phase
467   __ tbnz(rscratch2, ShenandoahHeap::EVACUATION_BITPOS, slowpath);

612   if (is_cae) {
613     // We're falling through to done to indicate success.  Success
614     // with is_cae is denoted by returning the value of expected as
615     // result.
616     __ mov(tmp2, expected);
617   }
618 
619   __ bind(done);
620   // At entry to done, the Z (EQ) flag is on iff if the CAS
621   // operation was successful.  Additionally, if is_cae, tmp2 holds
622   // the value most recently fetched from addr. In this case, success
623   // is denoted by tmp2 matching expected.
624 
625   if (is_cae) {
626     __ mov(result, tmp2);
627   } else {
628     __ cset(result, Assembler::EQ);
629   }
630 }
631 
632 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
633                                                                      Register start, Register count, Register scratch, RegSet saved_regs) {
634   if (!ShenandoahHeap::heap()->mode()->is_generational()) {
635     return;
636   }
637 
638   ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
639   CardTable* ct = bs->card_table();
640 
641   Label L_loop, L_done;
642   const Register end = count;
643 
644   __ cbz(count, L_done); // zero count - nothing to do
645 
646   __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop))); // end = start + count << LogBytesPerHeapOop
647   __ sub(end, end, BytesPerHeapOop); // last element address to make inclusive
648   __ lsr(start, start, CardTable::card_shift());
649   __ lsr(end, end, CardTable::card_shift());
650   __ sub(count, end, start); // number of bytes to copy
651 
652   __ load_byte_map_base(scratch);
653   __ add(start, start, scratch);
654   __ bind(L_loop);
655   __ strb(zr, Address(start, count));
656   __ subs(count, count, 1);
657   __ br(Assembler::GE, L_loop);
658   __ bind(L_done);
659 }
660 
661 #undef __
662 
663 #ifdef COMPILER1
664 
665 #define __ ce->masm()->
666 
667 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
668   ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
669   // At this point we know that marking is in progress.
670   // If do_load() is true then we have to emit the
671   // load of the previous value; otherwise it has already
672   // been loaded into _pre_val.
673 
674   __ bind(*stub->entry());
675 
676   assert(stub->pre_val()->is_register(), "Precondition.");
677 
678   Register pre_val_reg = stub->pre_val()->as_register();
679 
680   if (stub->do_load()) {

741 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
742   __ prologue("shenandoah_pre_barrier", false);
743 
744   // arg0 : previous value of memory
745 
746   BarrierSet* bs = BarrierSet::barrier_set();
747 
748   const Register pre_val = r0;
749   const Register thread = rthread;
750   const Register tmp = rscratch1;
751 
752   Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
753   Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
754 
755   Label done;
756   Label runtime;
757 
758   // Is marking still active?
759   Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
760   __ ldrb(tmp, gc_state);
761   if (!ShenandoahHeap::heap()->mode()->is_generational()) {
762     __ tbz(tmp, ShenandoahHeap::YOUNG_MARKING_BITPOS, done);
763   } else {
764     __ mov(rscratch2, ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING);
765     __ tst(tmp, rscratch2);
766     __ br(Assembler::EQ, done);
767   }
768 
769   // Can we store original value in the thread's buffer?
770   __ ldr(tmp, queue_index);
771   __ cbz(tmp, runtime);
772 
773   __ sub(tmp, tmp, wordSize);
774   __ str(tmp, queue_index);
775   __ ldr(rscratch2, buffer);
776   __ add(tmp, tmp, rscratch2);
777   __ load_parameter(0, rscratch2);
778   __ str(rscratch2, Address(tmp, 0));
779   __ b(done);
780 
781   __ bind(runtime);
782   __ push_call_clobbered_registers();
783   __ load_parameter(0, pre_val);
784   __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread);
785   __ pop_call_clobbered_registers();
786   __ bind(done);
787 
< prev index next >