14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
27 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
28 #include "gc/shenandoah/shenandoahForwarding.hpp"
29 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
30 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
31 #include "gc/shenandoah/shenandoahRuntime.hpp"
32 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
33 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
34 #include "interpreter/interpreter.hpp"
35 #include "interpreter/interp_masm.hpp"
36 #include "runtime/javaThread.hpp"
37 #include "runtime/sharedRuntime.hpp"
38 #ifdef COMPILER1
39 #include "c1/c1_LIRAssembler.hpp"
40 #include "c1/c1_MacroAssembler.hpp"
41 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
42 #endif
43
44 #define __ masm->
45
46 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
47 Register src, Register dst, Register count, RegSet saved_regs) {
48 if (is_oop) {
49 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
50 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahIUBarrier || ShenandoahLoadRefBarrier) {
51
52 Label done;
53
54 // Avoid calling runtime if count == 0
55 __ cbz(count, done);
56
57 // Is GC active?
58 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
59 __ ldrb(rscratch1, gc_state);
60 if (ShenandoahSATBBarrier && dest_uninitialized) {
61 __ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
62 } else {
63 __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING);
64 __ tst(rscratch1, rscratch2);
65 __ br(Assembler::EQ, done);
66 }
67
68 __ push(saved_regs, sp);
69 if (UseCompressedOops) {
70 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop_entry), src, dst, count);
71 } else {
72 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop_entry), src, dst, count);
73 }
74 __ pop(saved_regs, sp);
75 __ bind(done);
76 }
77 }
78 }
79
80 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
81 Register obj,
82 Register pre_val,
83 Register thread,
84 Register tmp,
85 bool tosca_live,
86 bool expand_call) {
87 if (ShenandoahSATBBarrier) {
88 satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
89 }
90 }
91
92 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
93 Register obj,
94 Register pre_val,
95 Register thread,
96 Register tmp,
97 bool tosca_live,
98 bool expand_call) {
99 // If expand_call is true then we expand the call_VM_leaf macro
356 } else {
357 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
358 }
359
360 // 3: apply keep-alive barrier if needed
361 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
362 __ enter(/*strip_ret_addr*/true);
363 __ push_call_clobbered_registers();
364 satb_write_barrier_pre(masm /* masm */,
365 noreg /* obj */,
366 dst /* pre_val */,
367 rthread /* thread */,
368 tmp1 /* tmp */,
369 true /* tosca_live */,
370 true /* expand_call */);
371 __ pop_call_clobbered_registers();
372 __ leave();
373 }
374 }
375
376 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
377 Address dst, Register val, Register tmp1, Register tmp2) {
378 bool on_oop = is_reference_type(type);
379 if (!on_oop) {
380 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
381 return;
382 }
383
384 // flatten object address if needed
385 if (dst.index() == noreg && dst.offset() == 0) {
386 if (dst.base() != r3) {
387 __ mov(r3, dst.base());
388 }
389 } else {
390 __ lea(r3, dst);
391 }
392
393 shenandoah_write_barrier_pre(masm,
394 r3 /* obj */,
395 tmp2 /* pre_val */,
396 rthread /* thread */,
397 tmp1 /* tmp */,
398 val != noreg /* tosca_live */,
399 false /* expand_call */);
400
401 if (val == noreg) {
402 BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), noreg, noreg, noreg);
403 } else {
404 iu_barrier(masm, val, tmp1);
405 // G1 barrier needs uncompressed oop for region cross check.
406 Register new_val = val;
407 if (UseCompressedOops) {
408 new_val = rscratch2;
409 __ mov(new_val, val);
410 }
411 BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), val, noreg, noreg);
412 }
413
414 }
415
416 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
417 Register obj, Register tmp, Label& slowpath) {
418 Label done;
419 // Resolve jobject
420 BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
421
422 // Check for null.
423 __ cbz(obj, done);
424
425 assert(obj != rscratch2, "need rscratch2");
426 Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset());
427 __ lea(rscratch2, gc_state);
428 __ ldrb(rscratch2, Address(rscratch2));
429
430 // Check for heap in evacuation phase
431 __ tbnz(rscratch2, ShenandoahHeap::EVACUATION_BITPOS, slowpath);
576 if (is_cae) {
577 // We're falling through to done to indicate success. Success
578 // with is_cae is denoted by returning the value of expected as
579 // result.
580 __ mov(tmp2, expected);
581 }
582
583 __ bind(done);
584 // At entry to done, the Z (EQ) flag is on iff if the CAS
585 // operation was successful. Additionally, if is_cae, tmp2 holds
586 // the value most recently fetched from addr. In this case, success
587 // is denoted by tmp2 matching expected.
588
589 if (is_cae) {
590 __ mov(result, tmp2);
591 } else {
592 __ cset(result, Assembler::EQ);
593 }
594 }
595
596 #undef __
597
598 #ifdef COMPILER1
599
600 #define __ ce->masm()->
601
602 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
603 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
604 // At this point we know that marking is in progress.
605 // If do_load() is true then we have to emit the
606 // load of the previous value; otherwise it has already
607 // been loaded into _pre_val.
608
609 __ bind(*stub->entry());
610
611 assert(stub->pre_val()->is_register(), "Precondition.");
612
613 Register pre_val_reg = stub->pre_val()->as_register();
614
615 if (stub->do_load()) {
676 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
677 __ prologue("shenandoah_pre_barrier", false);
678
679 // arg0 : previous value of memory
680
681 BarrierSet* bs = BarrierSet::barrier_set();
682
683 const Register pre_val = r0;
684 const Register thread = rthread;
685 const Register tmp = rscratch1;
686
687 Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
688 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
689
690 Label done;
691 Label runtime;
692
693 // Is marking still active?
694 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
695 __ ldrb(tmp, gc_state);
696 __ tbz(tmp, ShenandoahHeap::MARKING_BITPOS, done);
697
698 // Can we store original value in the thread's buffer?
699 __ ldr(tmp, queue_index);
700 __ cbz(tmp, runtime);
701
702 __ sub(tmp, tmp, wordSize);
703 __ str(tmp, queue_index);
704 __ ldr(rscratch2, buffer);
705 __ add(tmp, tmp, rscratch2);
706 __ load_parameter(0, rscratch2);
707 __ str(rscratch2, Address(tmp, 0));
708 __ b(done);
709
710 __ bind(runtime);
711 __ push_call_clobbered_registers();
712 __ load_parameter(0, pre_val);
713 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread);
714 __ pop_call_clobbered_registers();
715 __ bind(done);
716
|
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
27 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
28 #include "gc/shenandoah/shenandoahForwarding.hpp"
29 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
30 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
31 #include "gc/shenandoah/shenandoahRuntime.hpp"
32 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
33 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
34 #include "gc/shenandoah/mode/shenandoahMode.hpp"
35 #include "interpreter/interpreter.hpp"
36 #include "interpreter/interp_masm.hpp"
37 #include "runtime/javaThread.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #ifdef COMPILER1
40 #include "c1/c1_LIRAssembler.hpp"
41 #include "c1/c1_MacroAssembler.hpp"
42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
43 #endif
44
45 #define __ masm->
46
47 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
48 Register src, Register dst, Register count, RegSet saved_regs) {
49 if (is_oop) {
50 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
51 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahIUBarrier || ShenandoahLoadRefBarrier) {
52
53 Label done;
54
55 // Avoid calling runtime if count == 0
56 __ cbz(count, done);
57
58 // Is GC active?
59 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
60 __ ldrb(rscratch1, gc_state);
61 if (ShenandoahSATBBarrier && dest_uninitialized) {
62 __ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
63 } else {
64 __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING);
65 __ tst(rscratch1, rscratch2);
66 __ br(Assembler::EQ, done);
67 }
68
69 __ push(saved_regs, sp);
70 if (UseCompressedOops) {
71 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop_entry), src, dst, count);
72 } else {
73 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop_entry), src, dst, count);
74 }
75 __ pop(saved_regs, sp);
76 __ bind(done);
77 }
78 }
79 }
80
81 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
82 Register start, Register count, Register tmp, RegSet saved_regs) {
83 if (is_oop) {
84 gen_write_ref_array_post_barrier(masm, decorators, start, count, tmp, saved_regs);
85 }
86 }
87
88 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
89 Register obj,
90 Register pre_val,
91 Register thread,
92 Register tmp,
93 bool tosca_live,
94 bool expand_call) {
95 if (ShenandoahSATBBarrier) {
96 satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
97 }
98 }
99
100 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
101 Register obj,
102 Register pre_val,
103 Register thread,
104 Register tmp,
105 bool tosca_live,
106 bool expand_call) {
107 // If expand_call is true then we expand the call_VM_leaf macro
364 } else {
365 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
366 }
367
368 // 3: apply keep-alive barrier if needed
369 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
370 __ enter(/*strip_ret_addr*/true);
371 __ push_call_clobbered_registers();
372 satb_write_barrier_pre(masm /* masm */,
373 noreg /* obj */,
374 dst /* pre_val */,
375 rthread /* thread */,
376 tmp1 /* tmp */,
377 true /* tosca_live */,
378 true /* expand_call */);
379 __ pop_call_clobbered_registers();
380 __ leave();
381 }
382 }
383
384 void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj) {
385 if (!ShenandoahHeap::heap()->mode()->is_generational()) {
386 return;
387 }
388
389 ShenandoahBarrierSet* ctbs = ShenandoahBarrierSet::barrier_set();
390 CardTable* ct = ctbs->card_table();
391
392 __ lsr(obj, obj, CardTable::card_shift());
393
394 assert(CardTable::dirty_card_val() == 0, "must be");
395
396 __ load_byte_map_base(rscratch1);
397
398 if (UseCondCardMark) {
399 Label L_already_dirty;
400 __ ldrb(rscratch2, Address(obj, rscratch1));
401 __ cbz(rscratch2, L_already_dirty);
402 __ strb(zr, Address(obj, rscratch1));
403 __ bind(L_already_dirty);
404 } else {
405 __ strb(zr, Address(obj, rscratch1));
406 }
407 }
408
409 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
410 Address dst, Register val, Register tmp1, Register tmp2) {
411 bool on_oop = is_reference_type(type);
412 if (!on_oop) {
413 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
414 return;
415 }
416
417 // flatten object address if needed
418 if (dst.index() == noreg && dst.offset() == 0) {
419 if (dst.base() != r3) {
420 __ mov(r3, dst.base());
421 }
422 } else {
423 __ lea(r3, dst);
424 }
425
426 shenandoah_write_barrier_pre(masm,
427 r3 /* obj */,
428 tmp2 /* pre_val */,
429 rthread /* thread */,
430 tmp1 /* tmp */,
431 val != noreg /* tosca_live */,
432 false /* expand_call */);
433
434 if (val == noreg) {
435 BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), noreg, noreg, noreg);
436 } else {
437 iu_barrier(masm, val, tmp1);
438 // G1 barrier needs uncompressed oop for region cross check.
439 Register new_val = val;
440 if (UseCompressedOops) {
441 new_val = rscratch2;
442 __ mov(new_val, val);
443 }
444 BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), val, noreg, noreg);
445 store_check(masm, r3);
446 }
447
448 }
449
450 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
451 Register obj, Register tmp, Label& slowpath) {
452 Label done;
453 // Resolve jobject
454 BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
455
456 // Check for null.
457 __ cbz(obj, done);
458
459 assert(obj != rscratch2, "need rscratch2");
460 Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset());
461 __ lea(rscratch2, gc_state);
462 __ ldrb(rscratch2, Address(rscratch2));
463
464 // Check for heap in evacuation phase
465 __ tbnz(rscratch2, ShenandoahHeap::EVACUATION_BITPOS, slowpath);
610 if (is_cae) {
611 // We're falling through to done to indicate success. Success
612 // with is_cae is denoted by returning the value of expected as
613 // result.
614 __ mov(tmp2, expected);
615 }
616
617 __ bind(done);
618 // At entry to done, the Z (EQ) flag is on iff if the CAS
619 // operation was successful. Additionally, if is_cae, tmp2 holds
620 // the value most recently fetched from addr. In this case, success
621 // is denoted by tmp2 matching expected.
622
623 if (is_cae) {
624 __ mov(result, tmp2);
625 } else {
626 __ cset(result, Assembler::EQ);
627 }
628 }
629
630 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
631 Register start, Register count, Register scratch, RegSet saved_regs) {
632 if (!ShenandoahHeap::heap()->mode()->is_generational()) {
633 return;
634 }
635
636 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
637 CardTable* ct = bs->card_table();
638
639 Label L_loop, L_done;
640 const Register end = count;
641
642 __ cbz(count, L_done); // zero count - nothing to do
643
644 __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop))); // end = start + count << LogBytesPerHeapOop
645 __ sub(end, end, BytesPerHeapOop); // last element address to make inclusive
646 __ lsr(start, start, CardTable::card_shift());
647 __ lsr(end, end, CardTable::card_shift());
648 __ sub(count, end, start); // number of bytes to copy
649
650 __ load_byte_map_base(scratch);
651 __ add(start, start, scratch);
652 __ bind(L_loop);
653 __ strb(zr, Address(start, count));
654 __ subs(count, count, 1);
655 __ br(Assembler::GE, L_loop);
656 __ bind(L_done);
657 }
658
659 #undef __
660
661 #ifdef COMPILER1
662
663 #define __ ce->masm()->
664
665 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
666 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
667 // At this point we know that marking is in progress.
668 // If do_load() is true then we have to emit the
669 // load of the previous value; otherwise it has already
670 // been loaded into _pre_val.
671
672 __ bind(*stub->entry());
673
674 assert(stub->pre_val()->is_register(), "Precondition.");
675
676 Register pre_val_reg = stub->pre_val()->as_register();
677
678 if (stub->do_load()) {
739 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
740 __ prologue("shenandoah_pre_barrier", false);
741
742 // arg0 : previous value of memory
743
744 BarrierSet* bs = BarrierSet::barrier_set();
745
746 const Register pre_val = r0;
747 const Register thread = rthread;
748 const Register tmp = rscratch1;
749
750 Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
751 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
752
753 Label done;
754 Label runtime;
755
756 // Is marking still active?
757 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
758 __ ldrb(tmp, gc_state);
759 if (!ShenandoahHeap::heap()->mode()->is_generational()) {
760 __ tbz(tmp, ShenandoahHeap::YOUNG_MARKING_BITPOS, done);
761 } else {
762 __ mov(rscratch2, ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING);
763 __ tst(tmp, rscratch2);
764 __ br(Assembler::EQ, done);
765 }
766
767 // Can we store original value in the thread's buffer?
768 __ ldr(tmp, queue_index);
769 __ cbz(tmp, runtime);
770
771 __ sub(tmp, tmp, wordSize);
772 __ str(tmp, queue_index);
773 __ ldr(rscratch2, buffer);
774 __ add(tmp, tmp, rscratch2);
775 __ load_parameter(0, rscratch2);
776 __ str(rscratch2, Address(tmp, 0));
777 __ b(done);
778
779 __ bind(runtime);
780 __ push_call_clobbered_registers();
781 __ load_parameter(0, pre_val);
782 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread);
783 __ pop_call_clobbered_registers();
784 __ bind(done);
785
|