1 /*
2 * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/javaClasses.hpp"
27 #include "gc/shared/barrierSet.hpp"
28 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
29 #include "gc/shenandoah/shenandoahForwarding.hpp"
30 #include "gc/shenandoah/shenandoahHeap.hpp"
31 #include "gc/shenandoah/shenandoahRuntime.hpp"
32 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
33 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
34 #include "gc/shenandoah/c2/shenandoahSupport.hpp"
35 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
36 #include "opto/arraycopynode.hpp"
37 #include "opto/escape.hpp"
38 #include "opto/graphKit.hpp"
39 #include "opto/idealKit.hpp"
40 #include "opto/macro.hpp"
41 #include "opto/movenode.hpp"
42 #include "opto/narrowptrnode.hpp"
43 #include "opto/rootnode.hpp"
44 #include "opto/runtime.hpp"
45
46 ShenandoahBarrierSetC2* ShenandoahBarrierSetC2::bsc2() {
47 return reinterpret_cast<ShenandoahBarrierSetC2*>(BarrierSet::barrier_set()->barrier_set_c2());
48 }
279 } __ end_if(); // (!index)
280 } __ end_if(); // (pre_val != nullptr)
281 } __ end_if(); // (!marking)
282
283 // Final sync IdealKit and GraphKit.
284 kit->final_sync(ideal);
285
286 if (ShenandoahSATBBarrier && adr != nullptr) {
287 Node* c = kit->control();
288 Node* call = c->in(1)->in(1)->in(1)->in(0);
289 assert(is_shenandoah_wb_pre_call(call), "shenandoah_wb_pre call expected");
290 call->add_req(adr);
291 }
292 }
293
294 bool ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(Node* call) {
295 return call->is_CallLeaf() &&
296 call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry);
297 }
298
299 bool ShenandoahBarrierSetC2::is_shenandoah_lrb_call(Node* call) {
300 if (!call->is_CallLeaf()) {
301 return false;
302 }
303
304 address entry_point = call->as_CallLeaf()->entry_point();
305 return (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong)) ||
306 (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow)) ||
307 (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak)) ||
308 (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow)) ||
309 (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom));
310 }
311
312 bool ShenandoahBarrierSetC2::is_shenandoah_marking_if(PhaseValues* phase, Node* n) {
313 if (n->Opcode() != Op_If) {
314 return false;
315 }
316
317 Node* bol = n->in(1);
318 assert(bol->is_Bool(), "");
319 Node* cmpx = bol->in(1);
320 if (bol->as_Bool()->_test._test == BoolTest::ne &&
321 cmpx->is_Cmp() && cmpx->in(2) == phase->intcon(0) &&
322 is_shenandoah_state_load(cmpx->in(1)->in(1)) &&
323 cmpx->in(1)->in(2)->is_Con() &&
324 cmpx->in(1)->in(2) == phase->intcon(ShenandoahHeap::MARKING)) {
325 return true;
326 }
327
328 return false;
329 }
433 // Use the pre-barrier to record the value in the referent field
434 satb_write_barrier_pre(kit, false /* do_load */,
435 nullptr /* obj */, nullptr /* adr */, max_juint /* alias_idx */, nullptr /* val */, nullptr /* val_type */,
436 pre_val /* pre_val */,
437 T_OBJECT);
438 if (need_mem_bar) {
439 // Add memory barrier to prevent commoning reads from this field
440 // across safepoint since GC can change its value.
441 kit->insert_mem_bar(Op_MemBarCPUOrder);
442 }
443 // Update IdealKit from graphKit.
444 __ sync_kit(kit);
445
446 } __ end_if(); // _ref_type != ref_none
447 } __ end_if(); // offset == referent_offset
448
449 // Final sync IdealKit and GraphKit.
450 kit->final_sync(ideal);
451 }
452
453 #undef __
454
455 const TypeFunc* ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type() {
456 const Type **fields = TypeTuple::fields(2);
457 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value
458 fields[TypeFunc::Parms+1] = TypeRawPtr::NOTNULL; // thread
459 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
460
461 // create result type (range)
462 fields = TypeTuple::fields(0);
463 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
464
465 return TypeFunc::make(domain, range);
466 }
467
468 const TypeFunc* ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type() {
469 const Type **fields = TypeTuple::fields(1);
470 fields[TypeFunc::Parms+0] = TypeOopPtr::NOTNULL; // src oop
471 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);
472
496 DecoratorSet decorators = access.decorators();
497
498 const TypePtr* adr_type = access.addr().type();
499 Node* adr = access.addr().node();
500
501 if (!access.is_oop()) {
502 return BarrierSetC2::store_at_resolved(access, val);
503 }
504
505 if (access.is_parse_access()) {
506 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
507 GraphKit* kit = parse_access.kit();
508
509 uint adr_idx = kit->C->get_alias_index(adr_type);
510 assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" );
511 Node* value = val.node();
512 value = shenandoah_iu_barrier(kit, value);
513 val.set_node(value);
514 shenandoah_write_barrier_pre(kit, true /* do_load */, /*kit->control(),*/ access.base(), adr, adr_idx, val.node(),
515 static_cast<const TypeOopPtr*>(val.type()), nullptr /* pre_val */, access.type());
516 } else {
517 assert(access.is_opt_access(), "only for optimization passes");
518 assert(((decorators & C2_TIGHTLY_COUPLED_ALLOC) != 0 || !ShenandoahSATBBarrier) && (decorators & C2_ARRAY_COPY) != 0, "unexpected caller of this code");
519 C2OptAccess& opt_access = static_cast<C2OptAccess&>(access);
520 PhaseGVN& gvn = opt_access.gvn();
521
522 if (ShenandoahIUBarrier) {
523 Node* enqueue = gvn.transform(new ShenandoahIUBarrierNode(val.node()));
524 val.set_node(enqueue);
525 }
526 }
527 return BarrierSetC2::store_at_resolved(access, val);
528 }
529
530 Node* ShenandoahBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
531 // 1: non-reference load, no additional barrier is needed
532 if (!access.is_oop()) {
533 return BarrierSetC2::load_at_resolved(access, val_type);
534 }
535
536 Node* load = BarrierSetC2::load_at_resolved(access, val_type);
537 DecoratorSet decorators = access.decorators();
538 BasicType type = access.type();
539
540 // 2: apply LRB if needed
541 if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
542 load = new ShenandoahLoadReferenceBarrierNode(nullptr, load, decorators);
543 if (access.is_parse_access()) {
544 load = static_cast<C2ParseAccess &>(access).kit()->gvn().transform(load);
545 } else {
546 load = static_cast<C2OptAccess &>(access).gvn().transform(load);
547 }
578
579 if (on_weak_ref) {
580 // Use the pre-barrier to record the value in the referent field
581 satb_write_barrier_pre(kit, false /* do_load */,
582 nullptr /* obj */, nullptr /* adr */, max_juint /* alias_idx */, nullptr /* val */, nullptr /* val_type */,
583 load /* pre_val */, T_OBJECT);
584 // Add memory barrier to prevent commoning reads from this field
585 // across safepoint since GC can change its value.
586 kit->insert_mem_bar(Op_MemBarCPUOrder);
587 } else if (unknown) {
588 // We do not require a mem bar inside pre_barrier if need_mem_bar
589 // is set: the barriers would be emitted by us.
590 insert_pre_barrier(kit, obj, offset, load, !need_cpu_mem_bar);
591 }
592 }
593
594 return load;
595 }
596
597 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
598 Node* new_val, const Type* value_type) const {
599 GraphKit* kit = access.kit();
600 if (access.is_oop()) {
601 new_val = shenandoah_iu_barrier(kit, new_val);
602 shenandoah_write_barrier_pre(kit, false /* do_load */,
603 nullptr, nullptr, max_juint, nullptr, nullptr,
604 expected_val /* pre_val */, T_OBJECT);
605
606 MemNode::MemOrd mo = access.mem_node_mo();
607 Node* mem = access.memory();
608 Node* adr = access.addr().node();
609 const TypePtr* adr_type = access.addr().type();
610 Node* load_store = nullptr;
611
612 #ifdef _LP64
613 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
614 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
615 Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
616 if (ShenandoahCASBarrier) {
617 load_store = kit->gvn().transform(new ShenandoahCompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo));
618 } else {
620 }
621 } else
622 #endif
623 {
624 if (ShenandoahCASBarrier) {
625 load_store = kit->gvn().transform(new ShenandoahCompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo));
626 } else {
627 load_store = kit->gvn().transform(new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo));
628 }
629 }
630
631 access.set_raw_access(load_store);
632 pin_atomic_op(access);
633
634 #ifdef _LP64
635 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
636 load_store = kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
637 }
638 #endif
639 load_store = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(nullptr, load_store, access.decorators()));
640 return load_store;
641 }
642 return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
643 }
644
645 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
646 Node* new_val, const Type* value_type) const {
647 GraphKit* kit = access.kit();
648 if (access.is_oop()) {
649 new_val = shenandoah_iu_barrier(kit, new_val);
650 shenandoah_write_barrier_pre(kit, false /* do_load */,
651 nullptr, nullptr, max_juint, nullptr, nullptr,
652 expected_val /* pre_val */, T_OBJECT);
653 DecoratorSet decorators = access.decorators();
654 MemNode::MemOrd mo = access.mem_node_mo();
655 Node* mem = access.memory();
656 bool is_weak_cas = (decorators & C2_WEAK_CMPXCHG) != 0;
657 Node* load_store = nullptr;
658 Node* adr = access.addr().node();
659 #ifdef _LP64
675 }
676 } else
677 #endif
678 {
679 if (ShenandoahCASBarrier) {
680 if (is_weak_cas) {
681 load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
682 } else {
683 load_store = kit->gvn().transform(new ShenandoahCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
684 }
685 } else {
686 if (is_weak_cas) {
687 load_store = kit->gvn().transform(new WeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
688 } else {
689 load_store = kit->gvn().transform(new CompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
690 }
691 }
692 }
693 access.set_raw_access(load_store);
694 pin_atomic_op(access);
695 return load_store;
696 }
697 return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
698 }
699
700 Node* ShenandoahBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* val, const Type* value_type) const {
701 GraphKit* kit = access.kit();
702 if (access.is_oop()) {
703 val = shenandoah_iu_barrier(kit, val);
704 }
705 Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type);
706 if (access.is_oop()) {
707 result = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(nullptr, result, access.decorators()));
708 shenandoah_write_barrier_pre(kit, false /* do_load */,
709 nullptr, nullptr, max_juint, nullptr, nullptr,
710 result /* pre_val */, T_OBJECT);
711 }
712 return result;
713 }
714
715
716 bool ShenandoahBarrierSetC2::is_gc_pre_barrier_node(Node* node) const {
717 return is_shenandoah_wb_pre_call(node);
718 }
719
720 // Support for GC barriers emitted during parsing
721 bool ShenandoahBarrierSetC2::is_gc_barrier_node(Node* node) const {
722 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier || node->Opcode() == Op_ShenandoahIUBarrier) return true;
723 if (node->Opcode() != Op_CallLeaf && node->Opcode() != Op_CallLeafNoFP) {
724 return false;
725 }
726 CallLeafNode *call = node->as_CallLeaf();
727 if (call->_name == nullptr) {
728 return false;
729 }
730
731 return strcmp(call->_name, "shenandoah_clone_barrier") == 0 ||
732 strcmp(call->_name, "shenandoah_cas_obj") == 0 ||
733 strcmp(call->_name, "shenandoah_wb_pre") == 0;
734 }
735
736 Node* ShenandoahBarrierSetC2::step_over_gc_barrier(Node* c) const {
737 if (c == nullptr) {
738 return c;
739 }
740 if (c->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
741 return c->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
742 }
743 if (c->Opcode() == Op_ShenandoahIUBarrier) {
744 c = c->in(1);
745 }
746 return c;
747 }
748
749 bool ShenandoahBarrierSetC2::expand_barriers(Compile* C, PhaseIterGVN& igvn) const {
750 return !ShenandoahBarrierC2Support::expand(C, igvn);
751 }
752
753 bool ShenandoahBarrierSetC2::optimize_loops(PhaseIdealLoop* phase, LoopOptsMode mode, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const {
889
890 // Support for macro expanded GC barriers
891 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
892 if (node->Opcode() == Op_ShenandoahIUBarrier) {
893 state()->add_iu_barrier((ShenandoahIUBarrierNode*) node);
894 }
895 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
896 state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
897 }
898 }
899
900 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
901 if (node->Opcode() == Op_ShenandoahIUBarrier) {
902 state()->remove_iu_barrier((ShenandoahIUBarrierNode*) node);
903 }
904 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
905 state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
906 }
907 }
908
909 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* n) const {
910 if (is_shenandoah_wb_pre_call(n)) {
911 shenandoah_eliminate_wb_pre(n, ¯o->igvn());
912 }
913 }
914
915 void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
916 assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
917 Node* c = call->as_Call()->proj_out(TypeFunc::Control);
918 c = c->unique_ctrl_out();
919 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
920 c = c->unique_ctrl_out();
921 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
922 Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
923 assert(iff->is_If(), "expect test");
924 if (!is_shenandoah_marking_if(igvn, iff)) {
925 c = c->unique_ctrl_out();
926 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
927 iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
928 assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
929 }
930 Node* cmpx = iff->in(1)->in(1);
931 igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ));
|
1 /*
2 * Copyright (c) 2018, 2023, Red Hat, Inc. All rights reserved.
3 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "classfile/javaClasses.hpp"
28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahCardTable.hpp"
31 #include "gc/shenandoah/shenandoahForwarding.hpp"
32 #include "gc/shenandoah/shenandoahHeap.hpp"
33 #include "gc/shenandoah/shenandoahRuntime.hpp"
34 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
35 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
36 #include "gc/shenandoah/c2/shenandoahSupport.hpp"
37 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
38 #include "opto/arraycopynode.hpp"
39 #include "opto/escape.hpp"
40 #include "opto/graphKit.hpp"
41 #include "opto/idealKit.hpp"
42 #include "opto/macro.hpp"
43 #include "opto/movenode.hpp"
44 #include "opto/narrowptrnode.hpp"
45 #include "opto/rootnode.hpp"
46 #include "opto/runtime.hpp"
47
48 ShenandoahBarrierSetC2* ShenandoahBarrierSetC2::bsc2() {
49 return reinterpret_cast<ShenandoahBarrierSetC2*>(BarrierSet::barrier_set()->barrier_set_c2());
50 }
281 } __ end_if(); // (!index)
282 } __ end_if(); // (pre_val != nullptr)
283 } __ end_if(); // (!marking)
284
285 // Final sync IdealKit and GraphKit.
286 kit->final_sync(ideal);
287
288 if (ShenandoahSATBBarrier && adr != nullptr) {
289 Node* c = kit->control();
290 Node* call = c->in(1)->in(1)->in(1)->in(0);
291 assert(is_shenandoah_wb_pre_call(call), "shenandoah_wb_pre call expected");
292 call->add_req(adr);
293 }
294 }
295
296 bool ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(Node* call) {
297 return call->is_CallLeaf() &&
298 call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry);
299 }
300
301 bool ShenandoahBarrierSetC2::is_shenandoah_clone_call(Node* call) {
302 return call->is_CallLeaf() &&
303 call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::shenandoah_clone_barrier);
304 }
305
306 bool ShenandoahBarrierSetC2::is_shenandoah_lrb_call(Node* call) {
307 if (!call->is_CallLeaf()) {
308 return false;
309 }
310
311 address entry_point = call->as_CallLeaf()->entry_point();
312 return (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong)) ||
313 (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow)) ||
314 (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak)) ||
315 (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow)) ||
316 (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom)) ||
317 (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom_narrow));
318 }
319
320 bool ShenandoahBarrierSetC2::is_shenandoah_marking_if(PhaseValues* phase, Node* n) {
321 if (n->Opcode() != Op_If) {
322 return false;
323 }
324
325 Node* bol = n->in(1);
326 assert(bol->is_Bool(), "");
327 Node* cmpx = bol->in(1);
328 if (bol->as_Bool()->_test._test == BoolTest::ne &&
329 cmpx->is_Cmp() && cmpx->in(2) == phase->intcon(0) &&
330 is_shenandoah_state_load(cmpx->in(1)->in(1)) &&
331 cmpx->in(1)->in(2)->is_Con() &&
332 cmpx->in(1)->in(2) == phase->intcon(ShenandoahHeap::MARKING)) {
333 return true;
334 }
335
336 return false;
337 }
441 // Use the pre-barrier to record the value in the referent field
442 satb_write_barrier_pre(kit, false /* do_load */,
443 nullptr /* obj */, nullptr /* adr */, max_juint /* alias_idx */, nullptr /* val */, nullptr /* val_type */,
444 pre_val /* pre_val */,
445 T_OBJECT);
446 if (need_mem_bar) {
447 // Add memory barrier to prevent commoning reads from this field
448 // across safepoint since GC can change its value.
449 kit->insert_mem_bar(Op_MemBarCPUOrder);
450 }
451 // Update IdealKit from graphKit.
452 __ sync_kit(kit);
453
454 } __ end_if(); // _ref_type != ref_none
455 } __ end_if(); // offset == referent_offset
456
457 // Final sync IdealKit and GraphKit.
458 kit->final_sync(ideal);
459 }
460
461 Node* ShenandoahBarrierSetC2::byte_map_base_node(GraphKit* kit) const {
462 BarrierSet* bs = BarrierSet::barrier_set();
463 ShenandoahBarrierSet* ctbs = barrier_set_cast<ShenandoahBarrierSet>(bs);
464 CardTable::CardValue* card_table_base = ctbs->card_table()->byte_map_base();
465 if (card_table_base != nullptr) {
466 return kit->makecon(TypeRawPtr::make((address)card_table_base));
467 } else {
468 return kit->null();
469 }
470 }
471
472 void ShenandoahBarrierSetC2::post_barrier(GraphKit* kit,
473 Node* ctl,
474 Node* oop_store,
475 Node* obj,
476 Node* adr,
477 uint adr_idx,
478 Node* val,
479 BasicType bt,
480 bool use_precise) const {
481 assert(ShenandoahCardBarrier, "Should have been checked by caller");
482
483 // No store check needed if we're storing a null.
484 if (val != nullptr && val->is_Con()) {
485 // must be either an oop or NULL
486 const Type* t = val->bottom_type();
487 if (t == TypePtr::NULL_PTR || t == Type::TOP)
488 return;
489 }
490
491 if (ReduceInitialCardMarks && obj == kit->just_allocated_object(kit->control())) {
492 // We can skip marks on a freshly-allocated object in Eden.
493 // Keep this code in sync with new_deferred_store_barrier() in runtime.cpp.
494 // That routine informs GC to take appropriate compensating steps,
495 // upon a slow-path allocation, so as to make this card-mark
496 // elision safe.
497 return;
498 }
499
500 if (!use_precise) {
501 // All card marks for a (non-array) instance are in one place:
502 adr = obj;
503 }
504 // (Else it's an array (or unknown), and we want more precise card marks.)
505 assert(adr != nullptr, "");
506
507 IdealKit ideal(kit, true);
508
509 // Convert the pointer to an int prior to doing math on it
510 Node* cast = __ CastPX(__ ctrl(), adr);
511
512 // Divide by card size
513 Node* card_offset = __ URShiftX( cast, __ ConI(CardTable::card_shift()) );
514
515 // Combine card table base and card offset
516 Node* card_adr = __ AddP(__ top(), byte_map_base_node(kit), card_offset );
517
518 // Get the alias_index for raw card-mark memory
519 int adr_type = Compile::AliasIdxRaw;
520 Node* zero = __ ConI(0); // Dirty card value
521
522 if (UseCondCardMark) {
523 // The classic GC reference write barrier is typically implemented
524 // as a store into the global card mark table. Unfortunately
525 // unconditional stores can result in false sharing and excessive
526 // coherence traffic as well as false transactional aborts.
527 // UseCondCardMark enables MP "polite" conditional card mark
528 // stores. In theory we could relax the load from ctrl() to
529 // no_ctrl, but that doesn't buy much latitude.
530 Node* card_val = __ load( __ ctrl(), card_adr, TypeInt::BYTE, T_BYTE, adr_type);
531 __ if_then(card_val, BoolTest::ne, zero);
532 }
533
534 // Smash zero into card
535 __ store(__ ctrl(), card_adr, zero, T_BYTE, adr_type, MemNode::unordered);
536
537 if (UseCondCardMark) {
538 __ end_if();
539 }
540
541 // Final sync IdealKit and GraphKit.
542 kit->final_sync(ideal);
543 }
544
545 #undef __
546
547 const TypeFunc* ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type() {
548 const Type **fields = TypeTuple::fields(2);
549 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value
550 fields[TypeFunc::Parms+1] = TypeRawPtr::NOTNULL; // thread
551 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
552
553 // create result type (range)
554 fields = TypeTuple::fields(0);
555 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
556
557 return TypeFunc::make(domain, range);
558 }
559
560 const TypeFunc* ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type() {
561 const Type **fields = TypeTuple::fields(1);
562 fields[TypeFunc::Parms+0] = TypeOopPtr::NOTNULL; // src oop
563 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);
564
588 DecoratorSet decorators = access.decorators();
589
590 const TypePtr* adr_type = access.addr().type();
591 Node* adr = access.addr().node();
592
593 if (!access.is_oop()) {
594 return BarrierSetC2::store_at_resolved(access, val);
595 }
596
597 if (access.is_parse_access()) {
598 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
599 GraphKit* kit = parse_access.kit();
600
601 uint adr_idx = kit->C->get_alias_index(adr_type);
602 assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" );
603 Node* value = val.node();
604 value = shenandoah_iu_barrier(kit, value);
605 val.set_node(value);
606 shenandoah_write_barrier_pre(kit, true /* do_load */, /*kit->control(),*/ access.base(), adr, adr_idx, val.node(),
607 static_cast<const TypeOopPtr*>(val.type()), nullptr /* pre_val */, access.type());
608
609 Node* result = BarrierSetC2::store_at_resolved(access, val);
610
611 if (ShenandoahCardBarrier) {
612 const bool anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0;
613 const bool is_array = (decorators & IS_ARRAY) != 0;
614 const bool use_precise = is_array || anonymous;
615 post_barrier(kit, kit->control(), access.raw_access(), access.base(),
616 adr, adr_idx, val.node(), access.type(), use_precise);
617 }
618 return result;
619 } else {
620 assert(access.is_opt_access(), "only for optimization passes");
621 assert(((decorators & C2_TIGHTLY_COUPLED_ALLOC) != 0 || !ShenandoahSATBBarrier) && (decorators & C2_ARRAY_COPY) != 0, "unexpected caller of this code");
622 C2OptAccess& opt_access = static_cast<C2OptAccess&>(access);
623 PhaseGVN& gvn = opt_access.gvn();
624
625 if (ShenandoahIUBarrier) {
626 Node* enqueue = gvn.transform(new ShenandoahIUBarrierNode(val.node()));
627 val.set_node(enqueue);
628 }
629 return BarrierSetC2::store_at_resolved(access, val);
630 }
631 }
632
633 Node* ShenandoahBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
634 // 1: non-reference load, no additional barrier is needed
635 if (!access.is_oop()) {
636 return BarrierSetC2::load_at_resolved(access, val_type);
637 }
638
639 Node* load = BarrierSetC2::load_at_resolved(access, val_type);
640 DecoratorSet decorators = access.decorators();
641 BasicType type = access.type();
642
643 // 2: apply LRB if needed
644 if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
645 load = new ShenandoahLoadReferenceBarrierNode(nullptr, load, decorators);
646 if (access.is_parse_access()) {
647 load = static_cast<C2ParseAccess &>(access).kit()->gvn().transform(load);
648 } else {
649 load = static_cast<C2OptAccess &>(access).gvn().transform(load);
650 }
681
682 if (on_weak_ref) {
683 // Use the pre-barrier to record the value in the referent field
684 satb_write_barrier_pre(kit, false /* do_load */,
685 nullptr /* obj */, nullptr /* adr */, max_juint /* alias_idx */, nullptr /* val */, nullptr /* val_type */,
686 load /* pre_val */, T_OBJECT);
687 // Add memory barrier to prevent commoning reads from this field
688 // across safepoint since GC can change its value.
689 kit->insert_mem_bar(Op_MemBarCPUOrder);
690 } else if (unknown) {
691 // We do not require a mem bar inside pre_barrier if need_mem_bar
692 // is set: the barriers would be emitted by us.
693 insert_pre_barrier(kit, obj, offset, load, !need_cpu_mem_bar);
694 }
695 }
696
697 return load;
698 }
699
700 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
701 Node* new_val, const Type* value_type) const {
702 GraphKit* kit = access.kit();
703 if (access.is_oop()) {
704 new_val = shenandoah_iu_barrier(kit, new_val);
705 shenandoah_write_barrier_pre(kit, false /* do_load */,
706 nullptr, nullptr, max_juint, nullptr, nullptr,
707 expected_val /* pre_val */, T_OBJECT);
708
709 MemNode::MemOrd mo = access.mem_node_mo();
710 Node* mem = access.memory();
711 Node* adr = access.addr().node();
712 const TypePtr* adr_type = access.addr().type();
713 Node* load_store = nullptr;
714
715 #ifdef _LP64
716 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
717 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
718 Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
719 if (ShenandoahCASBarrier) {
720 load_store = kit->gvn().transform(new ShenandoahCompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo));
721 } else {
723 }
724 } else
725 #endif
726 {
727 if (ShenandoahCASBarrier) {
728 load_store = kit->gvn().transform(new ShenandoahCompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo));
729 } else {
730 load_store = kit->gvn().transform(new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo));
731 }
732 }
733
734 access.set_raw_access(load_store);
735 pin_atomic_op(access);
736
737 #ifdef _LP64
738 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
739 load_store = kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
740 }
741 #endif
742 load_store = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(nullptr, load_store, access.decorators()));
743 if (ShenandoahCardBarrier) {
744 post_barrier(kit, kit->control(), access.raw_access(), access.base(),
745 access.addr().node(), access.alias_idx(), new_val, T_OBJECT, true);
746 }
747 return load_store;
748 }
749 return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
750 }
751
752 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
753 Node* new_val, const Type* value_type) const {
754 GraphKit* kit = access.kit();
755 if (access.is_oop()) {
756 new_val = shenandoah_iu_barrier(kit, new_val);
757 shenandoah_write_barrier_pre(kit, false /* do_load */,
758 nullptr, nullptr, max_juint, nullptr, nullptr,
759 expected_val /* pre_val */, T_OBJECT);
760 DecoratorSet decorators = access.decorators();
761 MemNode::MemOrd mo = access.mem_node_mo();
762 Node* mem = access.memory();
763 bool is_weak_cas = (decorators & C2_WEAK_CMPXCHG) != 0;
764 Node* load_store = nullptr;
765 Node* adr = access.addr().node();
766 #ifdef _LP64
782 }
783 } else
784 #endif
785 {
786 if (ShenandoahCASBarrier) {
787 if (is_weak_cas) {
788 load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
789 } else {
790 load_store = kit->gvn().transform(new ShenandoahCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
791 }
792 } else {
793 if (is_weak_cas) {
794 load_store = kit->gvn().transform(new WeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
795 } else {
796 load_store = kit->gvn().transform(new CompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
797 }
798 }
799 }
800 access.set_raw_access(load_store);
801 pin_atomic_op(access);
802 if (ShenandoahCardBarrier) {
803 post_barrier(kit, kit->control(), access.raw_access(), access.base(),
804 access.addr().node(), access.alias_idx(), new_val, T_OBJECT, true);
805 }
806 return load_store;
807 }
808 return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
809 }
810
811 Node* ShenandoahBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* val, const Type* value_type) const {
812 GraphKit* kit = access.kit();
813 if (access.is_oop()) {
814 val = shenandoah_iu_barrier(kit, val);
815 }
816 Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type);
817 if (access.is_oop()) {
818 result = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(nullptr, result, access.decorators()));
819 shenandoah_write_barrier_pre(kit, false /* do_load */,
820 nullptr, nullptr, max_juint, nullptr, nullptr,
821 result /* pre_val */, T_OBJECT);
822 if (ShenandoahCardBarrier) {
823 post_barrier(kit, kit->control(), access.raw_access(), access.base(),
824 access.addr().node(), access.alias_idx(), val, T_OBJECT, true);
825 }
826 }
827 return result;
828 }
829
830
831 bool ShenandoahBarrierSetC2::is_gc_pre_barrier_node(Node* node) const {
832 return is_shenandoah_wb_pre_call(node);
833 }
834
835 bool ShenandoahBarrierSetC2::is_gc_barrier_node(Node* node) const {
836 return (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) || (node->Opcode() == Op_ShenandoahIUBarrier) ||
837 is_shenandoah_lrb_call(node) ||
838 is_shenandoah_wb_pre_call(node) ||
839 is_shenandoah_clone_call(node);
840 }
841
842 Node* ShenandoahBarrierSetC2::step_over_gc_barrier(Node* c) const {
843 if (c == nullptr) {
844 return c;
845 }
846 if (c->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
847 return c->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
848 }
849 if (c->Opcode() == Op_ShenandoahIUBarrier) {
850 c = c->in(1);
851 }
852 return c;
853 }
854
855 bool ShenandoahBarrierSetC2::expand_barriers(Compile* C, PhaseIterGVN& igvn) const {
856 return !ShenandoahBarrierC2Support::expand(C, igvn);
857 }
858
859 bool ShenandoahBarrierSetC2::optimize_loops(PhaseIdealLoop* phase, LoopOptsMode mode, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const {
995
996 // Support for macro expanded GC barriers
997 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
998 if (node->Opcode() == Op_ShenandoahIUBarrier) {
999 state()->add_iu_barrier((ShenandoahIUBarrierNode*) node);
1000 }
1001 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
1002 state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
1003 }
1004 }
1005
1006 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
1007 if (node->Opcode() == Op_ShenandoahIUBarrier) {
1008 state()->remove_iu_barrier((ShenandoahIUBarrierNode*) node);
1009 }
1010 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
1011 state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
1012 }
1013 }
1014
1015 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const {
1016 if (is_shenandoah_wb_pre_call(node)) {
1017 shenandoah_eliminate_wb_pre(node, ¯o->igvn());
1018 }
1019 if (ShenandoahCardBarrier && node->Opcode() == Op_CastP2X) {
1020 Node* shift = node->unique_out();
1021 Node* addp = shift->unique_out();
1022 for (DUIterator_Last jmin, j = addp->last_outs(jmin); j >= jmin; --j) {
1023 Node* mem = addp->last_out(j);
1024 if (UseCondCardMark && mem->is_Load()) {
1025 assert(mem->Opcode() == Op_LoadB, "unexpected code shape");
1026 // The load is checking if the card has been written so
1027 // replace it with zero to fold the test.
1028 macro->replace_node(mem, macro->intcon(0));
1029 continue;
1030 }
1031 assert(mem->is_Store(), "store required");
1032 macro->replace_node(mem, mem->in(MemNode::Memory));
1033 }
1034 }
1035 }
1036
1037 void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
1038 assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
1039 Node* c = call->as_Call()->proj_out(TypeFunc::Control);
1040 c = c->unique_ctrl_out();
1041 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
1042 c = c->unique_ctrl_out();
1043 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
1044 Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
1045 assert(iff->is_If(), "expect test");
1046 if (!is_shenandoah_marking_if(igvn, iff)) {
1047 c = c->unique_ctrl_out();
1048 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
1049 iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
1050 assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
1051 }
1052 Node* cmpx = iff->in(1)->in(1);
1053 igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ));
|