< prev index next >

src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp

Print this page

 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
 27 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
 28 #include "gc/shenandoah/shenandoahForwarding.hpp"
 29 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 30 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
 31 #include "gc/shenandoah/shenandoahRuntime.hpp"
 32 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
 33 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"

 34 #include "interpreter/interpreter.hpp"
 35 #include "interpreter/interp_masm.hpp"
 36 #include "runtime/sharedRuntime.hpp"
 37 #include "runtime/thread.hpp"
 38 #ifdef COMPILER1
 39 #include "c1/c1_LIRAssembler.hpp"
 40 #include "c1/c1_MacroAssembler.hpp"
 41 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
 42 #endif
 43 
 44 #define __ masm->
 45 
 46 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
 47                                                        Register src, Register dst, Register count, RegSet saved_regs) {
 48   if (is_oop) {
 49     bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
 50     if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahIUBarrier || ShenandoahLoadRefBarrier) {
 51 
 52       Label done;
 53 
 54       // Avoid calling runtime if count == 0
 55       __ cbz(count, done);
 56 
 57       // Is GC active?
 58       Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 59       __ ldrb(rscratch1, gc_state);
 60       if (ShenandoahSATBBarrier && dest_uninitialized) {
 61         __ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
 62       } else {
 63         __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING);
 64         __ tst(rscratch1, rscratch2);
 65         __ br(Assembler::EQ, done);
 66       }
 67 
 68       __ push(saved_regs, sp);
 69       if (UseCompressedOops) {
 70         __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop_entry), src, dst, count);
 71       } else {
 72         __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop_entry), src, dst, count);
 73       }
 74       __ pop(saved_regs, sp);
 75       __ bind(done);
 76     }
 77   }
 78 }
 79 







 80 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
 81                                                                  Register obj,
 82                                                                  Register pre_val,
 83                                                                  Register thread,
 84                                                                  Register tmp,
 85                                                                  bool tosca_live,
 86                                                                  bool expand_call) {
 87   if (ShenandoahSATBBarrier) {
 88     satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
 89   }
 90 }
 91 
 92 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
 93                                                            Register obj,
 94                                                            Register pre_val,
 95                                                            Register thread,
 96                                                            Register tmp,
 97                                                            bool tosca_live,
 98                                                            bool expand_call) {
 99   // If expand_call is true then we expand the call_VM_leaf macro

356   } else {
357     BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
358   }
359 
360   // 3: apply keep-alive barrier if needed
361   if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
362     __ enter();
363     __ push_call_clobbered_registers();
364     satb_write_barrier_pre(masm /* masm */,
365                            noreg /* obj */,
366                            dst /* pre_val */,
367                            rthread /* thread */,
368                            tmp1 /* tmp */,
369                            true /* tosca_live */,
370                            true /* expand_call */);
371     __ pop_call_clobbered_registers();
372     __ leave();
373   }
374 }
375 

























376 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
377                                              Address dst, Register val, Register tmp1, Register tmp2) {
378   bool on_oop = is_reference_type(type);
379   if (!on_oop) {
380     BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
381     return;
382   }
383 
384   // flatten object address if needed
385   if (dst.index() == noreg && dst.offset() == 0) {
386     if (dst.base() != r3) {
387       __ mov(r3, dst.base());
388     }
389   } else {
390     __ lea(r3, dst);
391   }
392 
393   shenandoah_write_barrier_pre(masm,
394                                r3 /* obj */,
395                                tmp2 /* pre_val */,
396                                rthread /* thread */,
397                                tmp1  /* tmp */,
398                                val != noreg /* tosca_live */,
399                                false /* expand_call */);
400 
401   if (val == noreg) {
402     BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), noreg, noreg, noreg);
403   } else {
404     iu_barrier(masm, val, tmp1);
405     // G1 barrier needs uncompressed oop for region cross check.
406     Register new_val = val;
407     if (UseCompressedOops) {
408       new_val = rscratch2;
409       __ mov(new_val, val);
410     }
411     BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), val, noreg, noreg);

412   }
413 
414 }
415 
416 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
417                                                                   Register obj, Register tmp, Label& slowpath) {
418   Label done;
419   // Resolve jobject
420   BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
421 
422   // Check for null.
423   __ cbz(obj, done);
424 
425   assert(obj != rscratch2, "need rscratch2");
426   Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset());
427   __ lea(rscratch2, gc_state);
428   __ ldrb(rscratch2, Address(rscratch2));
429 
430   // Check for heap in evacuation phase
431   __ tbnz(rscratch2, ShenandoahHeap::EVACUATION_BITPOS, slowpath);

576   if (is_cae) {
577     // We're falling through to done to indicate success.  Success
578     // with is_cae is denoted by returning the value of expected as
579     // result.
580     __ mov(tmp2, expected);
581   }
582 
583   __ bind(done);
584   // At entry to done, the Z (EQ) flag is on iff if the CAS
585   // operation was successful.  Additionally, if is_cae, tmp2 holds
586   // the value most recently fetched from addr. In this case, success
587   // is denoted by tmp2 matching expected.
588 
589   if (is_cae) {
590     __ mov(result, tmp2);
591   } else {
592     __ cset(result, Assembler::EQ);
593   }
594 }
595 





























596 #undef __
597 
598 #ifdef COMPILER1
599 
600 #define __ ce->masm()->
601 
602 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
603   ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
604   // At this point we know that marking is in progress.
605   // If do_load() is true then we have to emit the
606   // load of the previous value; otherwise it has already
607   // been loaded into _pre_val.
608 
609   __ bind(*stub->entry());
610 
611   assert(stub->pre_val()->is_register(), "Precondition.");
612 
613   Register pre_val_reg = stub->pre_val()->as_register();
614 
615   if (stub->do_load()) {

676 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
677   __ prologue("shenandoah_pre_barrier", false);
678 
679   // arg0 : previous value of memory
680 
681   BarrierSet* bs = BarrierSet::barrier_set();
682 
683   const Register pre_val = r0;
684   const Register thread = rthread;
685   const Register tmp = rscratch1;
686 
687   Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
688   Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
689 
690   Label done;
691   Label runtime;
692 
693   // Is marking still active?
694   Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
695   __ ldrb(tmp, gc_state);
696   __ tbz(tmp, ShenandoahHeap::MARKING_BITPOS, done);






697 
698   // Can we store original value in the thread's buffer?
699   __ ldr(tmp, queue_index);
700   __ cbz(tmp, runtime);
701 
702   __ sub(tmp, tmp, wordSize);
703   __ str(tmp, queue_index);
704   __ ldr(rscratch2, buffer);
705   __ add(tmp, tmp, rscratch2);
706   __ load_parameter(0, rscratch2);
707   __ str(rscratch2, Address(tmp, 0));
708   __ b(done);
709 
710   __ bind(runtime);
711   __ push_call_clobbered_registers();
712   __ load_parameter(0, pre_val);
713   __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread);
714   __ pop_call_clobbered_registers();
715   __ bind(done);
716 

 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
 27 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
 28 #include "gc/shenandoah/shenandoahForwarding.hpp"
 29 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 30 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
 31 #include "gc/shenandoah/shenandoahRuntime.hpp"
 32 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
 33 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
 34 #include "gc/shenandoah/mode/shenandoahMode.hpp"
 35 #include "interpreter/interpreter.hpp"
 36 #include "interpreter/interp_masm.hpp"
 37 #include "runtime/sharedRuntime.hpp"
 38 #include "runtime/thread.hpp"
 39 #ifdef COMPILER1
 40 #include "c1/c1_LIRAssembler.hpp"
 41 #include "c1/c1_MacroAssembler.hpp"
 42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
 43 #endif
 44 
 45 #define __ masm->
 46 
 47 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
 48                                                        Register src, Register dst, Register count, RegSet saved_regs) {
 49   if (is_oop) {
 50     bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
 51     if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahIUBarrier || ShenandoahLoadRefBarrier) {
 52 
 53       Label done;
 54 
 55       // Avoid calling runtime if count == 0
 56       __ cbz(count, done);
 57 
 58       // Is GC active?
 59       Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 60       __ ldrb(rscratch1, gc_state);
 61       if (ShenandoahSATBBarrier && dest_uninitialized) {
 62         __ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
 63       } else {
 64         __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING);
 65         __ tst(rscratch1, rscratch2);
 66         __ br(Assembler::EQ, done);
 67       }
 68 
 69       __ push(saved_regs, sp);
 70       if (UseCompressedOops) {
 71         __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop_entry), src, dst, count);
 72       } else {
 73         __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop_entry), src, dst, count);
 74       }
 75       __ pop(saved_regs, sp);
 76       __ bind(done);
 77     }
 78   }
 79 }
 80 
 81 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
 82                                                        Register start, Register count, Register tmp, RegSet saved_regs) {
 83   if (is_oop) {
 84     gen_write_ref_array_post_barrier(masm, decorators, start, count, tmp, saved_regs);
 85   }
 86 }
 87 
 88 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
 89                                                                  Register obj,
 90                                                                  Register pre_val,
 91                                                                  Register thread,
 92                                                                  Register tmp,
 93                                                                  bool tosca_live,
 94                                                                  bool expand_call) {
 95   if (ShenandoahSATBBarrier) {
 96     satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
 97   }
 98 }
 99 
100 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
101                                                            Register obj,
102                                                            Register pre_val,
103                                                            Register thread,
104                                                            Register tmp,
105                                                            bool tosca_live,
106                                                            bool expand_call) {
107   // If expand_call is true then we expand the call_VM_leaf macro

364   } else {
365     BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
366   }
367 
368   // 3: apply keep-alive barrier if needed
369   if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
370     __ enter();
371     __ push_call_clobbered_registers();
372     satb_write_barrier_pre(masm /* masm */,
373                            noreg /* obj */,
374                            dst /* pre_val */,
375                            rthread /* thread */,
376                            tmp1 /* tmp */,
377                            true /* tosca_live */,
378                            true /* expand_call */);
379     __ pop_call_clobbered_registers();
380     __ leave();
381   }
382 }
383 
384 void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj) {
385   if (!ShenandoahHeap::heap()->mode()->is_generational()) {
386       return;
387   }
388 
389   ShenandoahBarrierSet* ctbs = ShenandoahBarrierSet::barrier_set();
390   CardTable* ct = ctbs->card_table();
391 
392   __ lsr(obj, obj, CardTable::card_shift);
393 
394   assert(CardTable::dirty_card_val() == 0, "must be");
395 
396   __ load_byte_map_base(rscratch1);
397 
398   if (UseCondCardMark) {
399     Label L_already_dirty;
400     __ ldrb(rscratch2,  Address(obj, rscratch1));
401     __ cbz(rscratch2, L_already_dirty);
402     __ strb(zr, Address(obj, rscratch1));
403     __ bind(L_already_dirty);
404   } else {
405     __ strb(zr, Address(obj, rscratch1));
406   }
407 }
408 
409 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
410                                              Address dst, Register val, Register tmp1, Register tmp2) {
411   bool on_oop = is_reference_type(type);
412   if (!on_oop) {
413     BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
414     return;
415   }
416 
417   // flatten object address if needed
418   if (dst.index() == noreg && dst.offset() == 0) {
419     if (dst.base() != r3) {
420       __ mov(r3, dst.base());
421     }
422   } else {
423     __ lea(r3, dst);
424   }
425 
426   shenandoah_write_barrier_pre(masm,
427                                r3 /* obj */,
428                                tmp2 /* pre_val */,
429                                rthread /* thread */,
430                                tmp1  /* tmp */,
431                                val != noreg /* tosca_live */,
432                                false /* expand_call */);
433 
434   if (val == noreg) {
435     BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), noreg, noreg, noreg);
436   } else {
437     iu_barrier(masm, val, tmp1);
438     // G1 barrier needs uncompressed oop for region cross check.
439     Register new_val = val;
440     if (UseCompressedOops) {
441       new_val = rscratch2;
442       __ mov(new_val, val);
443     }
444     BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), val, noreg, noreg);
445     store_check(masm, r3);
446   }
447 
448 }
449 
450 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
451                                                                   Register obj, Register tmp, Label& slowpath) {
452   Label done;
453   // Resolve jobject
454   BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
455 
456   // Check for null.
457   __ cbz(obj, done);
458 
459   assert(obj != rscratch2, "need rscratch2");
460   Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset());
461   __ lea(rscratch2, gc_state);
462   __ ldrb(rscratch2, Address(rscratch2));
463 
464   // Check for heap in evacuation phase
465   __ tbnz(rscratch2, ShenandoahHeap::EVACUATION_BITPOS, slowpath);

610   if (is_cae) {
611     // We're falling through to done to indicate success.  Success
612     // with is_cae is denoted by returning the value of expected as
613     // result.
614     __ mov(tmp2, expected);
615   }
616 
617   __ bind(done);
618   // At entry to done, the Z (EQ) flag is on iff if the CAS
619   // operation was successful.  Additionally, if is_cae, tmp2 holds
620   // the value most recently fetched from addr. In this case, success
621   // is denoted by tmp2 matching expected.
622 
623   if (is_cae) {
624     __ mov(result, tmp2);
625   } else {
626     __ cset(result, Assembler::EQ);
627   }
628 }
629 
630 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
631                                                                      Register start, Register count, Register scratch, RegSet saved_regs) {
632   if (!ShenandoahHeap::heap()->mode()->is_generational()) {
633     return;
634   }
635 
636   ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
637   CardTable* ct = bs->card_table();
638 
639   Label L_loop, L_done;
640   const Register end = count;
641 
642   __ cbz(count, L_done); // zero count - nothing to do
643 
644   __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop))); // end = start + count << LogBytesPerHeapOop
645   __ sub(end, end, BytesPerHeapOop); // last element address to make inclusive
646   __ lsr(start, start, CardTable::card_shift);
647   __ lsr(end, end, CardTable::card_shift);
648   __ sub(count, end, start); // number of bytes to copy
649 
650   __ load_byte_map_base(scratch);
651   __ add(start, start, scratch);
652   __ bind(L_loop);
653   __ strb(zr, Address(start, count));
654   __ subs(count, count, 1);
655   __ br(Assembler::GE, L_loop);
656   __ bind(L_done);
657 }
658 
659 #undef __
660 
661 #ifdef COMPILER1
662 
663 #define __ ce->masm()->
664 
665 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
666   ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
667   // At this point we know that marking is in progress.
668   // If do_load() is true then we have to emit the
669   // load of the previous value; otherwise it has already
670   // been loaded into _pre_val.
671 
672   __ bind(*stub->entry());
673 
674   assert(stub->pre_val()->is_register(), "Precondition.");
675 
676   Register pre_val_reg = stub->pre_val()->as_register();
677 
678   if (stub->do_load()) {

739 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
740   __ prologue("shenandoah_pre_barrier", false);
741 
742   // arg0 : previous value of memory
743 
744   BarrierSet* bs = BarrierSet::barrier_set();
745 
746   const Register pre_val = r0;
747   const Register thread = rthread;
748   const Register tmp = rscratch1;
749 
750   Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
751   Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
752 
753   Label done;
754   Label runtime;
755 
756   // Is marking still active?
757   Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
758   __ ldrb(tmp, gc_state);
759   if (!ShenandoahHeap::heap()->mode()->is_generational()) {
760     __ tbz(tmp, ShenandoahHeap::YOUNG_MARKING_BITPOS, done);
761   } else {
762     __ mov(rscratch2, ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING);
763     __ tst(tmp, rscratch2);
764     __ br(Assembler::EQ, done);
765   }
766 
767   // Can we store original value in the thread's buffer?
768   __ ldr(tmp, queue_index);
769   __ cbz(tmp, runtime);
770 
771   __ sub(tmp, tmp, wordSize);
772   __ str(tmp, queue_index);
773   __ ldr(rscratch2, buffer);
774   __ add(tmp, tmp, rscratch2);
775   __ load_parameter(0, rscratch2);
776   __ str(rscratch2, Address(tmp, 0));
777   __ b(done);
778 
779   __ bind(runtime);
780   __ push_call_clobbered_registers();
781   __ load_parameter(0, pre_val);
782   __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread);
783   __ pop_call_clobbered_registers();
784   __ bind(done);
785 
< prev index next >