15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
28 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
29 #include "gc/shenandoah/shenandoahForwarding.hpp"
30 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
31 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
32 #include "gc/shenandoah/shenandoahRuntime.hpp"
33 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
34 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
35 #include "interpreter/interpreter.hpp"
36 #include "interpreter/interp_masm.hpp"
37 #include "runtime/javaThread.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #ifdef COMPILER1
40 #include "c1/c1_LIRAssembler.hpp"
41 #include "c1/c1_MacroAssembler.hpp"
42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
43 #endif
44
45 #define __ masm->
46
47 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
48 Register src, Register dst, Register count, RegSet saved_regs) {
49 if (is_oop) {
50 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
51 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
52
53 Label done;
54
64 __ test_bit(t0, t0, ShenandoahHeap::HAS_FORWARDED_BITPOS);
65 __ beqz(t0, done);
66 } else {
67 __ andi(t0, t0, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING);
68 __ beqz(t0, done);
69 }
70
71 __ push_reg(saved_regs, sp);
72 if (UseCompressedOops) {
73 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop),
74 src, dst, count);
75 } else {
76 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop), src, dst, count);
77 }
78 __ pop_reg(saved_regs, sp);
79 __ bind(done);
80 }
81 }
82 }
83
84 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
85 Register obj,
86 Register pre_val,
87 Register thread,
88 Register tmp,
89 bool tosca_live,
90 bool expand_call) {
91 if (ShenandoahSATBBarrier) {
92 satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, t0, tosca_live, expand_call);
93 }
94 }
95
96 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
97 Register obj,
98 Register pre_val,
99 Register thread,
100 Register tmp1,
101 Register tmp2,
102 bool tosca_live,
103 bool expand_call) {
365 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
366 }
367
368 // 3: apply keep-alive barrier if needed
369 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
370 __ enter();
371 __ push_call_clobbered_registers();
372 satb_write_barrier_pre(masm /* masm */,
373 noreg /* obj */,
374 dst /* pre_val */,
375 xthread /* thread */,
376 tmp1 /* tmp1 */,
377 tmp2 /* tmp2 */,
378 true /* tosca_live */,
379 true /* expand_call */);
380 __ pop_call_clobbered_registers();
381 __ leave();
382 }
383 }
384
385 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
386 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
387 bool on_oop = is_reference_type(type);
388 if (!on_oop) {
389 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
390 return;
391 }
392
393 // flatten object address if needed
394 if (dst.offset() == 0) {
395 if (dst.base() != tmp3) {
396 __ mv(tmp3, dst.base());
397 }
398 } else {
399 __ la(tmp3, dst);
400 }
401
402 shenandoah_write_barrier_pre(masm,
403 tmp3 /* obj */,
404 tmp2 /* pre_val */,
405 xthread /* thread */,
406 tmp1 /* tmp */,
407 val != noreg /* tosca_live */,
408 false /* expand_call */);
409
410 if (val == noreg) {
411 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), noreg, noreg, noreg, noreg);
412 } else {
413 // Barrier needs uncompressed oop for region cross check.
414 Register new_val = val;
415 if (UseCompressedOops) {
416 new_val = t1;
417 __ mv(new_val, val);
418 }
419 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
420 }
421 }
422
423 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
424 Register obj, Register tmp, Label& slowpath) {
425 Label done;
426 // Resolve jobject
427 BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
428
429 // Check for null.
430 __ beqz(obj, done);
431
432 assert(obj != t1, "need t1");
433 Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset());
434 __ lbu(t1, gc_state);
435
436 // Check for heap in evacuation phase
437 __ test_bit(t0, t1, ShenandoahHeap::EVACUATION_BITPOS);
438 __ bnez(t0, slowpath);
439
507 __ bne(t0, t1, retry);
508
509 __ bind(success);
510 if (is_cae) {
511 __ mv(result, expected);
512 } else {
513 __ mv(result, 1);
514 }
515 __ j(done);
516
517 __ bind(fail);
518 if (is_cae) {
519 __ mv(result, t0);
520 } else {
521 __ mv(result, zr);
522 }
523
524 __ bind(done);
525 }
526
527 #undef __
528
529 #ifdef COMPILER1
530
531 #define __ ce->masm()->
532
533 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
534 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
535 // At this point we know that marking is in progress.
536 // If do_load() is true then we have to emit the
537 // load of the previous value; otherwise it has already
538 // been loaded into _pre_val.
539 __ bind(*stub->entry());
540
541 assert(stub->pre_val()->is_register(), "Precondition.");
542
543 Register pre_val_reg = stub->pre_val()->as_register();
544
545 if (stub->do_load()) {
546 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /* wide */);
|
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
28 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
29 #include "gc/shenandoah/shenandoahForwarding.hpp"
30 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
31 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
32 #include "gc/shenandoah/shenandoahRuntime.hpp"
33 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
34 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
35 #include "gc/shenandoah/mode/shenandoahMode.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "interpreter/interp_masm.hpp"
38 #include "runtime/javaThread.hpp"
39 #include "runtime/sharedRuntime.hpp"
40 #ifdef COMPILER1
41 #include "c1/c1_LIRAssembler.hpp"
42 #include "c1/c1_MacroAssembler.hpp"
43 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
44 #endif
45
46 #define __ masm->
47
48 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
49 Register src, Register dst, Register count, RegSet saved_regs) {
50 if (is_oop) {
51 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
52 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
53
54 Label done;
55
65 __ test_bit(t0, t0, ShenandoahHeap::HAS_FORWARDED_BITPOS);
66 __ beqz(t0, done);
67 } else {
68 __ andi(t0, t0, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING);
69 __ beqz(t0, done);
70 }
71
72 __ push_reg(saved_regs, sp);
73 if (UseCompressedOops) {
74 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop),
75 src, dst, count);
76 } else {
77 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop), src, dst, count);
78 }
79 __ pop_reg(saved_regs, sp);
80 __ bind(done);
81 }
82 }
83 }
84
85 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
86 Register start, Register count, Register tmp, RegSet saved_regs) {
87 if (ShenandoahCardBarrier && is_oop) {
88 gen_write_ref_array_post_barrier(masm, decorators, start, count, tmp, saved_regs);
89 }
90 }
91
92 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
93 Register obj,
94 Register pre_val,
95 Register thread,
96 Register tmp,
97 bool tosca_live,
98 bool expand_call) {
99 if (ShenandoahSATBBarrier) {
100 satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, t0, tosca_live, expand_call);
101 }
102 }
103
104 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
105 Register obj,
106 Register pre_val,
107 Register thread,
108 Register tmp1,
109 Register tmp2,
110 bool tosca_live,
111 bool expand_call) {
373 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
374 }
375
376 // 3: apply keep-alive barrier if needed
377 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
378 __ enter();
379 __ push_call_clobbered_registers();
380 satb_write_barrier_pre(masm /* masm */,
381 noreg /* obj */,
382 dst /* pre_val */,
383 xthread /* thread */,
384 tmp1 /* tmp1 */,
385 tmp2 /* tmp2 */,
386 true /* tosca_live */,
387 true /* expand_call */);
388 __ pop_call_clobbered_registers();
389 __ leave();
390 }
391 }
392
393 void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj) {
394 assert(ShenandoahCardBarrier, "Did you mean to enable ShenandoahCardBarrier?");
395
396 __ srli(obj, obj, CardTable::card_shift());
397
398 assert(CardTable::dirty_card_val() == 0, "must be");
399
400 __ load_byte_map_base(t1);
401 __ add(t1, obj, t1);
402
403 if (UseCondCardMark) {
404 Label L_already_dirty;
405 __ lbu(t0, Address(t1));
406 __ beqz(t0, L_already_dirty);
407 __ sb(zr, Address(t1));
408 __ bind(L_already_dirty);
409 } else {
410 __ sb(zr, Address(t1));
411 }
412 }
413
414 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
415 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
416 bool on_oop = is_reference_type(type);
417 if (!on_oop) {
418 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
419 return;
420 }
421
422 // flatten object address if needed
423 if (dst.offset() == 0) {
424 if (dst.base() != tmp3) {
425 __ mv(tmp3, dst.base());
426 }
427 } else {
428 __ la(tmp3, dst);
429 }
430
431 shenandoah_write_barrier_pre(masm,
432 tmp3 /* obj */,
433 tmp2 /* pre_val */,
434 xthread /* thread */,
435 tmp1 /* tmp */,
436 val != noreg /* tosca_live */,
437 false /* expand_call */);
438
439 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
440
441 bool in_heap = (decorators & IN_HEAP) != 0;
442 bool needs_post_barrier = (val != noreg) && in_heap && ShenandoahCardBarrier;
443 if (needs_post_barrier) {
444 store_check(masm, tmp3);
445 }
446 }
447
448 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
449 Register obj, Register tmp, Label& slowpath) {
450 Label done;
451 // Resolve jobject
452 BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
453
454 // Check for null.
455 __ beqz(obj, done);
456
457 assert(obj != t1, "need t1");
458 Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset());
459 __ lbu(t1, gc_state);
460
461 // Check for heap in evacuation phase
462 __ test_bit(t0, t1, ShenandoahHeap::EVACUATION_BITPOS);
463 __ bnez(t0, slowpath);
464
532 __ bne(t0, t1, retry);
533
534 __ bind(success);
535 if (is_cae) {
536 __ mv(result, expected);
537 } else {
538 __ mv(result, 1);
539 }
540 __ j(done);
541
542 __ bind(fail);
543 if (is_cae) {
544 __ mv(result, t0);
545 } else {
546 __ mv(result, zr);
547 }
548
549 __ bind(done);
550 }
551
552 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
553 Register start, Register count, Register tmp, RegSet saved_regs) {
554 assert(ShenandoahCardBarrier, "Did you mean to enable ShenandoahCardBarrier?");
555
556 Label L_loop, L_done;
557 const Register end = count;
558
559 // Zero count? Nothing to do.
560 __ beqz(count, L_done);
561
562 // end = start + count << LogBytesPerHeapOop
563 // last element address to make inclusive
564 __ shadd(end, count, start, tmp, LogBytesPerHeapOop);
565 __ sub(end, end, BytesPerHeapOop);
566 __ srli(start, start, CardTable::card_shift());
567 __ srli(end, end, CardTable::card_shift());
568
569 // number of bytes to copy
570 __ sub(count, end, start);
571
572 __ load_byte_map_base(tmp);
573 __ add(start, start, tmp);
574
575 __ bind(L_loop);
576 __ add(tmp, start, count);
577 __ sb(zr, Address(tmp));
578 __ sub(count, count, 1);
579 __ bgez(count, L_loop);
580 __ bind(L_done);
581 }
582
583 #undef __
584
585 #ifdef COMPILER1
586
587 #define __ ce->masm()->
588
589 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
590 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
591 // At this point we know that marking is in progress.
592 // If do_load() is true then we have to emit the
593 // load of the previous value; otherwise it has already
594 // been loaded into _pre_val.
595 __ bind(*stub->entry());
596
597 assert(stub->pre_val()->is_register(), "Precondition.");
598
599 Register pre_val_reg = stub->pre_val()->as_register();
600
601 if (stub->do_load()) {
602 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /* wide */);
|