1 /*
2 * Copyright (c) 2018, 2023, Red Hat, Inc. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/javaClasses.hpp"
27 #include "gc/shared/barrierSet.hpp"
28 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
29 #include "gc/shenandoah/shenandoahForwarding.hpp"
30 #include "gc/shenandoah/shenandoahHeap.hpp"
31 #include "gc/shenandoah/shenandoahRuntime.hpp"
32 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
33 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
34 #include "gc/shenandoah/c2/shenandoahSupport.hpp"
35 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
36 #include "opto/arraycopynode.hpp"
37 #include "opto/escape.hpp"
38 #include "opto/graphKit.hpp"
39 #include "opto/idealKit.hpp"
40 #include "opto/macro.hpp"
41 #include "opto/movenode.hpp"
42 #include "opto/narrowptrnode.hpp"
43 #include "opto/rootnode.hpp"
44 #include "opto/runtime.hpp"
45
46 ShenandoahBarrierSetC2* ShenandoahBarrierSetC2::bsc2() {
47 return reinterpret_cast<ShenandoahBarrierSetC2*>(BarrierSet::barrier_set()->barrier_set_c2());
48 }
415 // Use the pre-barrier to record the value in the referent field
416 satb_write_barrier_pre(kit, false /* do_load */,
417 nullptr /* obj */, nullptr /* adr */, max_juint /* alias_idx */, nullptr /* val */, nullptr /* val_type */,
418 pre_val /* pre_val */,
419 T_OBJECT);
420 if (need_mem_bar) {
421 // Add memory barrier to prevent commoning reads from this field
422 // across safepoint since GC can change its value.
423 kit->insert_mem_bar(Op_MemBarCPUOrder);
424 }
425 // Update IdealKit from graphKit.
426 __ sync_kit(kit);
427
428 } __ end_if(); // _ref_type != ref_none
429 } __ end_if(); // offset == referent_offset
430
431 // Final sync IdealKit and GraphKit.
432 kit->final_sync(ideal);
433 }
434
435 #undef __
436
437 const TypeFunc* ShenandoahBarrierSetC2::write_ref_field_pre_Type() {
438 const Type **fields = TypeTuple::fields(2);
439 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value
440 fields[TypeFunc::Parms+1] = TypeRawPtr::NOTNULL; // thread
441 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
442
443 // create result type (range)
444 fields = TypeTuple::fields(0);
445 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
446
447 return TypeFunc::make(domain, range);
448 }
449
450 const TypeFunc* ShenandoahBarrierSetC2::clone_barrier_Type() {
451 const Type **fields = TypeTuple::fields(1);
452 fields[TypeFunc::Parms+0] = TypeOopPtr::NOTNULL; // src oop
453 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);
454
482
483 bool no_keepalive = (decorators & AS_NO_KEEPALIVE) != 0;
484
485 if (!access.is_oop()) {
486 return BarrierSetC2::store_at_resolved(access, val);
487 }
488
489 if (no_keepalive) {
490 // No keep-alive means no need for the pre-barrier.
491 return BarrierSetC2::store_at_resolved(access, val);
492 }
493
494 if (access.is_parse_access()) {
495 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
496 GraphKit* kit = parse_access.kit();
497
498 uint adr_idx = kit->C->get_alias_index(adr_type);
499 assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" );
500 shenandoah_write_barrier_pre(kit, true /* do_load */, /*kit->control(),*/ access.base(), adr, adr_idx, val.node(),
501 static_cast<const TypeOopPtr*>(val.type()), nullptr /* pre_val */, access.type());
502 }
503 return BarrierSetC2::store_at_resolved(access, val);
504 }
505
506 Node* ShenandoahBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
507 // 1: non-reference load, no additional barrier is needed
508 if (!access.is_oop()) {
509 return BarrierSetC2::load_at_resolved(access, val_type);
510 }
511
512 Node* load = BarrierSetC2::load_at_resolved(access, val_type);
513 DecoratorSet decorators = access.decorators();
514 BasicType type = access.type();
515
516 // 2: apply LRB if needed
517 if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
518 load = new ShenandoahLoadReferenceBarrierNode(nullptr, load, decorators);
519 if (access.is_parse_access()) {
520 load = static_cast<C2ParseAccess &>(access).kit()->gvn().transform(load);
521 } else {
522 load = static_cast<C2OptAccess &>(access).gvn().transform(load);
523 }
554
555 if (on_weak_ref) {
556 // Use the pre-barrier to record the value in the referent field
557 satb_write_barrier_pre(kit, false /* do_load */,
558 nullptr /* obj */, nullptr /* adr */, max_juint /* alias_idx */, nullptr /* val */, nullptr /* val_type */,
559 load /* pre_val */, T_OBJECT);
560 // Add memory barrier to prevent commoning reads from this field
561 // across safepoint since GC can change its value.
562 kit->insert_mem_bar(Op_MemBarCPUOrder);
563 } else if (unknown) {
564 // We do not require a mem bar inside pre_barrier if need_mem_bar
565 // is set: the barriers would be emitted by us.
566 insert_pre_barrier(kit, obj, offset, load, !need_cpu_mem_bar);
567 }
568 }
569
570 return load;
571 }
572
573 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
574 Node* new_val, const Type* value_type) const {
575 GraphKit* kit = access.kit();
576 if (access.is_oop()) {
577 shenandoah_write_barrier_pre(kit, false /* do_load */,
578 nullptr, nullptr, max_juint, nullptr, nullptr,
579 expected_val /* pre_val */, T_OBJECT);
580
581 MemNode::MemOrd mo = access.mem_node_mo();
582 Node* mem = access.memory();
583 Node* adr = access.addr().node();
584 const TypePtr* adr_type = access.addr().type();
585 Node* load_store = nullptr;
586
587 #ifdef _LP64
588 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
589 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
590 Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
591 if (ShenandoahCASBarrier) {
592 load_store = kit->gvn().transform(new ShenandoahCompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo));
593 } else {
594 load_store = kit->gvn().transform(new CompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo));
595 }
596 } else
597 #endif
598 {
599 if (ShenandoahCASBarrier) {
600 load_store = kit->gvn().transform(new ShenandoahCompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo));
601 } else {
602 load_store = kit->gvn().transform(new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo));
603 }
604 }
605
606 access.set_raw_access(load_store);
607 pin_atomic_op(access);
608
609 #ifdef _LP64
610 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
611 load_store = kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
612 }
613 #endif
614 load_store = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(nullptr, load_store, access.decorators()));
615 return load_store;
616 }
617 return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
618 }
619
620 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
621 Node* new_val, const Type* value_type) const {
622 GraphKit* kit = access.kit();
623 if (access.is_oop()) {
624 shenandoah_write_barrier_pre(kit, false /* do_load */,
625 nullptr, nullptr, max_juint, nullptr, nullptr,
626 expected_val /* pre_val */, T_OBJECT);
627 DecoratorSet decorators = access.decorators();
628 MemNode::MemOrd mo = access.mem_node_mo();
629 Node* mem = access.memory();
630 bool is_weak_cas = (decorators & C2_WEAK_CMPXCHG) != 0;
631 Node* load_store = nullptr;
632 Node* adr = access.addr().node();
633 #ifdef _LP64
634 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
649 }
650 } else
651 #endif
652 {
653 if (ShenandoahCASBarrier) {
654 if (is_weak_cas) {
655 load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
656 } else {
657 load_store = kit->gvn().transform(new ShenandoahCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
658 }
659 } else {
660 if (is_weak_cas) {
661 load_store = kit->gvn().transform(new WeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
662 } else {
663 load_store = kit->gvn().transform(new CompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
664 }
665 }
666 }
667 access.set_raw_access(load_store);
668 pin_atomic_op(access);
669 return load_store;
670 }
671 return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
672 }
673
674 Node* ShenandoahBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* val, const Type* value_type) const {
675 GraphKit* kit = access.kit();
676 Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type);
677 if (access.is_oop()) {
678 result = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(nullptr, result, access.decorators()));
679 shenandoah_write_barrier_pre(kit, false /* do_load */,
680 nullptr, nullptr, max_juint, nullptr, nullptr,
681 result /* pre_val */, T_OBJECT);
682 }
683 return result;
684 }
685
686
687 bool ShenandoahBarrierSetC2::is_gc_pre_barrier_node(Node* node) const {
688 return is_shenandoah_wb_pre_call(node);
689 }
690
691 bool ShenandoahBarrierSetC2::is_gc_barrier_node(Node* node) const {
692 return (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) ||
693 is_shenandoah_lrb_call(node) ||
694 is_shenandoah_wb_pre_call(node) ||
695 is_shenandoah_clone_call(node);
696 }
697
698 Node* ShenandoahBarrierSetC2::step_over_gc_barrier(Node* c) const {
699 if (c == nullptr) {
700 return c;
701 }
835 phase->igvn().replace_node(ac, call);
836 } else {
837 BarrierSetC2::clone_at_expansion(phase, ac);
838 }
839 }
840
841
842 // Support for macro expanded GC barriers
843 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
844 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
845 state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
846 }
847 }
848
849 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
850 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
851 state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
852 }
853 }
854
855 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* n) const {
856 if (is_shenandoah_wb_pre_call(n)) {
857 shenandoah_eliminate_wb_pre(n, ¯o->igvn());
858 }
859 }
860
861 void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
862 assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
863 Node* c = call->as_Call()->proj_out(TypeFunc::Control);
864 c = c->unique_ctrl_out();
865 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
866 c = c->unique_ctrl_out();
867 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
868 Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
869 assert(iff->is_If(), "expect test");
870 if (!is_shenandoah_marking_if(igvn, iff)) {
871 c = c->unique_ctrl_out();
872 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
873 iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
874 assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
875 }
876 Node* cmpx = iff->in(1)->in(1);
877 igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ));
|
1 /*
2 * Copyright (c) 2018, 2023, Red Hat, Inc. All rights reserved.
3 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "classfile/javaClasses.hpp"
28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahCardTable.hpp"
31 #include "gc/shenandoah/shenandoahForwarding.hpp"
32 #include "gc/shenandoah/shenandoahHeap.hpp"
33 #include "gc/shenandoah/shenandoahRuntime.hpp"
34 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
35 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
36 #include "gc/shenandoah/c2/shenandoahSupport.hpp"
37 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
38 #include "opto/arraycopynode.hpp"
39 #include "opto/escape.hpp"
40 #include "opto/graphKit.hpp"
41 #include "opto/idealKit.hpp"
42 #include "opto/macro.hpp"
43 #include "opto/movenode.hpp"
44 #include "opto/narrowptrnode.hpp"
45 #include "opto/rootnode.hpp"
46 #include "opto/runtime.hpp"
47
48 ShenandoahBarrierSetC2* ShenandoahBarrierSetC2::bsc2() {
49 return reinterpret_cast<ShenandoahBarrierSetC2*>(BarrierSet::barrier_set()->barrier_set_c2());
50 }
417 // Use the pre-barrier to record the value in the referent field
418 satb_write_barrier_pre(kit, false /* do_load */,
419 nullptr /* obj */, nullptr /* adr */, max_juint /* alias_idx */, nullptr /* val */, nullptr /* val_type */,
420 pre_val /* pre_val */,
421 T_OBJECT);
422 if (need_mem_bar) {
423 // Add memory barrier to prevent commoning reads from this field
424 // across safepoint since GC can change its value.
425 kit->insert_mem_bar(Op_MemBarCPUOrder);
426 }
427 // Update IdealKit from graphKit.
428 __ sync_kit(kit);
429
430 } __ end_if(); // _ref_type != ref_none
431 } __ end_if(); // offset == referent_offset
432
433 // Final sync IdealKit and GraphKit.
434 kit->final_sync(ideal);
435 }
436
437 Node* ShenandoahBarrierSetC2::byte_map_base_node(GraphKit* kit) const {
438 BarrierSet* bs = BarrierSet::barrier_set();
439 ShenandoahBarrierSet* ctbs = barrier_set_cast<ShenandoahBarrierSet>(bs);
440 CardTable::CardValue* card_table_base = ctbs->card_table()->byte_map_base();
441 if (card_table_base != nullptr) {
442 return kit->makecon(TypeRawPtr::make((address)card_table_base));
443 } else {
444 return kit->null();
445 }
446 }
447
448 void ShenandoahBarrierSetC2::post_barrier(GraphKit* kit,
449 Node* ctl,
450 Node* oop_store,
451 Node* obj,
452 Node* adr,
453 uint adr_idx,
454 Node* val,
455 BasicType bt,
456 bool use_precise) const {
457 assert(ShenandoahCardBarrier, "Should have been checked by caller");
458
459 // No store check needed if we're storing a null.
460 if (val != nullptr && val->is_Con()) {
461 // must be either an oop or NULL
462 const Type* t = val->bottom_type();
463 if (t == TypePtr::NULL_PTR || t == Type::TOP)
464 return;
465 }
466
467 if (ReduceInitialCardMarks && obj == kit->just_allocated_object(kit->control())) {
468 // We can skip marks on a freshly-allocated object in Eden.
469 // Keep this code in sync with new_deferred_store_barrier() in runtime.cpp.
470 // That routine informs GC to take appropriate compensating steps,
471 // upon a slow-path allocation, so as to make this card-mark
472 // elision safe.
473 return;
474 }
475
476 if (!use_precise) {
477 // All card marks for a (non-array) instance are in one place:
478 adr = obj;
479 }
480 // (Else it's an array (or unknown), and we want more precise card marks.)
481 assert(adr != nullptr, "");
482
483 IdealKit ideal(kit, true);
484
485 // Convert the pointer to an int prior to doing math on it
486 Node* cast = __ CastPX(__ ctrl(), adr);
487
488 // Divide by card size
489 Node* card_offset = __ URShiftX( cast, __ ConI(CardTable::card_shift()) );
490
491 // Combine card table base and card offset
492 Node* card_adr = __ AddP(__ top(), byte_map_base_node(kit), card_offset );
493
494 // Get the alias_index for raw card-mark memory
495 int adr_type = Compile::AliasIdxRaw;
496 Node* zero = __ ConI(0); // Dirty card value
497
498 if (UseCondCardMark) {
499 // The classic GC reference write barrier is typically implemented
500 // as a store into the global card mark table. Unfortunately
501 // unconditional stores can result in false sharing and excessive
502 // coherence traffic as well as false transactional aborts.
503 // UseCondCardMark enables MP "polite" conditional card mark
504 // stores. In theory we could relax the load from ctrl() to
505 // no_ctrl, but that doesn't buy much latitude.
506 Node* card_val = __ load( __ ctrl(), card_adr, TypeInt::BYTE, T_BYTE, adr_type);
507 __ if_then(card_val, BoolTest::ne, zero);
508 }
509
510 // Smash zero into card
511 __ store(__ ctrl(), card_adr, zero, T_BYTE, adr_type, MemNode::unordered);
512
513 if (UseCondCardMark) {
514 __ end_if();
515 }
516
517 // Final sync IdealKit and GraphKit.
518 kit->final_sync(ideal);
519 }
520
521 #undef __
522
523 const TypeFunc* ShenandoahBarrierSetC2::write_ref_field_pre_Type() {
524 const Type **fields = TypeTuple::fields(2);
525 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value
526 fields[TypeFunc::Parms+1] = TypeRawPtr::NOTNULL; // thread
527 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
528
529 // create result type (range)
530 fields = TypeTuple::fields(0);
531 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
532
533 return TypeFunc::make(domain, range);
534 }
535
536 const TypeFunc* ShenandoahBarrierSetC2::clone_barrier_Type() {
537 const Type **fields = TypeTuple::fields(1);
538 fields[TypeFunc::Parms+0] = TypeOopPtr::NOTNULL; // src oop
539 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);
540
568
569 bool no_keepalive = (decorators & AS_NO_KEEPALIVE) != 0;
570
571 if (!access.is_oop()) {
572 return BarrierSetC2::store_at_resolved(access, val);
573 }
574
575 if (no_keepalive) {
576 // No keep-alive means no need for the pre-barrier.
577 return BarrierSetC2::store_at_resolved(access, val);
578 }
579
580 if (access.is_parse_access()) {
581 C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
582 GraphKit* kit = parse_access.kit();
583
584 uint adr_idx = kit->C->get_alias_index(adr_type);
585 assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" );
586 shenandoah_write_barrier_pre(kit, true /* do_load */, /*kit->control(),*/ access.base(), adr, adr_idx, val.node(),
587 static_cast<const TypeOopPtr*>(val.type()), nullptr /* pre_val */, access.type());
588
589 Node* result = BarrierSetC2::store_at_resolved(access, val);
590
591 if (ShenandoahCardBarrier) {
592 const bool anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0;
593 const bool is_array = (decorators & IS_ARRAY) != 0;
594 const bool use_precise = is_array || anonymous;
595 post_barrier(kit, kit->control(), access.raw_access(), access.base(),
596 adr, adr_idx, val.node(), access.type(), use_precise);
597 }
598 return result;
599 } else {
600 assert(access.is_opt_access(), "only for optimization passes");
601 assert(((decorators & C2_TIGHTLY_COUPLED_ALLOC) != 0 || !ShenandoahSATBBarrier) && (decorators & C2_ARRAY_COPY) != 0, "unexpected caller of this code");
602 return BarrierSetC2::store_at_resolved(access, val);
603 }
604 }
605
606 Node* ShenandoahBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
607 // 1: non-reference load, no additional barrier is needed
608 if (!access.is_oop()) {
609 return BarrierSetC2::load_at_resolved(access, val_type);
610 }
611
612 Node* load = BarrierSetC2::load_at_resolved(access, val_type);
613 DecoratorSet decorators = access.decorators();
614 BasicType type = access.type();
615
616 // 2: apply LRB if needed
617 if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
618 load = new ShenandoahLoadReferenceBarrierNode(nullptr, load, decorators);
619 if (access.is_parse_access()) {
620 load = static_cast<C2ParseAccess &>(access).kit()->gvn().transform(load);
621 } else {
622 load = static_cast<C2OptAccess &>(access).gvn().transform(load);
623 }
654
655 if (on_weak_ref) {
656 // Use the pre-barrier to record the value in the referent field
657 satb_write_barrier_pre(kit, false /* do_load */,
658 nullptr /* obj */, nullptr /* adr */, max_juint /* alias_idx */, nullptr /* val */, nullptr /* val_type */,
659 load /* pre_val */, T_OBJECT);
660 // Add memory barrier to prevent commoning reads from this field
661 // across safepoint since GC can change its value.
662 kit->insert_mem_bar(Op_MemBarCPUOrder);
663 } else if (unknown) {
664 // We do not require a mem bar inside pre_barrier if need_mem_bar
665 // is set: the barriers would be emitted by us.
666 insert_pre_barrier(kit, obj, offset, load, !need_cpu_mem_bar);
667 }
668 }
669
670 return load;
671 }
672
673 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
674 Node* new_val, const Type* value_type) const {
675 GraphKit* kit = access.kit();
676 if (access.is_oop()) {
677 shenandoah_write_barrier_pre(kit, false /* do_load */,
678 nullptr, nullptr, max_juint, nullptr, nullptr,
679 expected_val /* pre_val */, T_OBJECT);
680
681 MemNode::MemOrd mo = access.mem_node_mo();
682 Node* mem = access.memory();
683 Node* adr = access.addr().node();
684 const TypePtr* adr_type = access.addr().type();
685 Node* load_store = nullptr;
686
687 #ifdef _LP64
688 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
689 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
690 Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
691 if (ShenandoahCASBarrier) {
692 load_store = kit->gvn().transform(new ShenandoahCompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo));
693 } else {
694 load_store = kit->gvn().transform(new CompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo));
695 }
696 } else
697 #endif
698 {
699 if (ShenandoahCASBarrier) {
700 load_store = kit->gvn().transform(new ShenandoahCompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo));
701 } else {
702 load_store = kit->gvn().transform(new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo));
703 }
704 }
705
706 access.set_raw_access(load_store);
707 pin_atomic_op(access);
708
709 #ifdef _LP64
710 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
711 load_store = kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
712 }
713 #endif
714 load_store = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(nullptr, load_store, access.decorators()));
715 if (ShenandoahCardBarrier) {
716 post_barrier(kit, kit->control(), access.raw_access(), access.base(),
717 access.addr().node(), access.alias_idx(), new_val, T_OBJECT, true);
718 }
719 return load_store;
720 }
721 return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
722 }
723
724 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
725 Node* new_val, const Type* value_type) const {
726 GraphKit* kit = access.kit();
727 if (access.is_oop()) {
728 shenandoah_write_barrier_pre(kit, false /* do_load */,
729 nullptr, nullptr, max_juint, nullptr, nullptr,
730 expected_val /* pre_val */, T_OBJECT);
731 DecoratorSet decorators = access.decorators();
732 MemNode::MemOrd mo = access.mem_node_mo();
733 Node* mem = access.memory();
734 bool is_weak_cas = (decorators & C2_WEAK_CMPXCHG) != 0;
735 Node* load_store = nullptr;
736 Node* adr = access.addr().node();
737 #ifdef _LP64
738 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
753 }
754 } else
755 #endif
756 {
757 if (ShenandoahCASBarrier) {
758 if (is_weak_cas) {
759 load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
760 } else {
761 load_store = kit->gvn().transform(new ShenandoahCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
762 }
763 } else {
764 if (is_weak_cas) {
765 load_store = kit->gvn().transform(new WeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
766 } else {
767 load_store = kit->gvn().transform(new CompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
768 }
769 }
770 }
771 access.set_raw_access(load_store);
772 pin_atomic_op(access);
773 if (ShenandoahCardBarrier) {
774 post_barrier(kit, kit->control(), access.raw_access(), access.base(),
775 access.addr().node(), access.alias_idx(), new_val, T_OBJECT, true);
776 }
777 return load_store;
778 }
779 return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
780 }
781
782 Node* ShenandoahBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* val, const Type* value_type) const {
783 GraphKit* kit = access.kit();
784 Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type);
785 if (access.is_oop()) {
786 result = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(nullptr, result, access.decorators()));
787 shenandoah_write_barrier_pre(kit, false /* do_load */,
788 nullptr, nullptr, max_juint, nullptr, nullptr,
789 result /* pre_val */, T_OBJECT);
790 if (ShenandoahCardBarrier) {
791 post_barrier(kit, kit->control(), access.raw_access(), access.base(),
792 access.addr().node(), access.alias_idx(), val, T_OBJECT, true);
793 }
794 }
795 return result;
796 }
797
798
799 bool ShenandoahBarrierSetC2::is_gc_pre_barrier_node(Node* node) const {
800 return is_shenandoah_wb_pre_call(node);
801 }
802
803 bool ShenandoahBarrierSetC2::is_gc_barrier_node(Node* node) const {
804 return (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) ||
805 is_shenandoah_lrb_call(node) ||
806 is_shenandoah_wb_pre_call(node) ||
807 is_shenandoah_clone_call(node);
808 }
809
810 Node* ShenandoahBarrierSetC2::step_over_gc_barrier(Node* c) const {
811 if (c == nullptr) {
812 return c;
813 }
947 phase->igvn().replace_node(ac, call);
948 } else {
949 BarrierSetC2::clone_at_expansion(phase, ac);
950 }
951 }
952
953
954 // Support for macro expanded GC barriers
955 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
956 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
957 state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
958 }
959 }
960
961 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
962 if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
963 state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
964 }
965 }
966
967 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const {
968 if (is_shenandoah_wb_pre_call(node)) {
969 shenandoah_eliminate_wb_pre(node, ¯o->igvn());
970 }
971 if (ShenandoahCardBarrier && node->Opcode() == Op_CastP2X) {
972 Node* shift = node->unique_out();
973 Node* addp = shift->unique_out();
974 for (DUIterator_Last jmin, j = addp->last_outs(jmin); j >= jmin; --j) {
975 Node* mem = addp->last_out(j);
976 if (UseCondCardMark && mem->is_Load()) {
977 assert(mem->Opcode() == Op_LoadB, "unexpected code shape");
978 // The load is checking if the card has been written so
979 // replace it with zero to fold the test.
980 macro->replace_node(mem, macro->intcon(0));
981 continue;
982 }
983 assert(mem->is_Store(), "store required");
984 macro->replace_node(mem, mem->in(MemNode::Memory));
985 }
986 }
987 }
988
989 void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
990 assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
991 Node* c = call->as_Call()->proj_out(TypeFunc::Control);
992 c = c->unique_ctrl_out();
993 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
994 c = c->unique_ctrl_out();
995 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
996 Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
997 assert(iff->is_If(), "expect test");
998 if (!is_shenandoah_marking_if(igvn, iff)) {
999 c = c->unique_ctrl_out();
1000 assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
1001 iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
1002 assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
1003 }
1004 Node* cmpx = iff->in(1)->in(1);
1005 igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ));
|