< prev index next >

src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp

Print this page

  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "classfile/javaClasses.hpp"
  27 #include "gc/shared/barrierSet.hpp"
  28 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  29 #include "gc/shenandoah/shenandoahForwarding.hpp"
  30 #include "gc/shenandoah/shenandoahHeap.hpp"
  31 #include "gc/shenandoah/shenandoahRuntime.hpp"
  32 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
  33 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
  34 #include "gc/shenandoah/c2/shenandoahSupport.hpp"
  35 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"

  36 #include "opto/arraycopynode.hpp"
  37 #include "opto/escape.hpp"
  38 #include "opto/graphKit.hpp"
  39 #include "opto/idealKit.hpp"
  40 #include "opto/macro.hpp"
  41 #include "opto/movenode.hpp"
  42 #include "opto/narrowptrnode.hpp"
  43 #include "opto/rootnode.hpp"
  44 #include "opto/runtime.hpp"
  45 
  46 ShenandoahBarrierSetC2* ShenandoahBarrierSetC2::bsc2() {
  47   return reinterpret_cast<ShenandoahBarrierSetC2*>(BarrierSet::barrier_set()->barrier_set_c2());
  48 }
  49 
  50 ShenandoahBarrierSetC2State::ShenandoahBarrierSetC2State(Arena* comp_arena)
  51   : _iu_barriers(new (comp_arena) GrowableArray<ShenandoahIUBarrierNode*>(comp_arena, 8,  0, NULL)),
  52     _load_reference_barriers(new (comp_arena) GrowableArray<ShenandoahLoadReferenceBarrierNode*>(comp_arena, 8,  0, NULL)) {
  53 }
  54 
  55 int ShenandoahBarrierSetC2State::iu_barriers_count() const {

 224 
 225   Node* no_base = __ top();
 226   Node* zero  = __ ConI(0);
 227   Node* zeroX = __ ConX(0);
 228 
 229   float likely  = PROB_LIKELY(0.999);
 230   float unlikely  = PROB_UNLIKELY(0.999);
 231 
 232   // Offsets into the thread
 233   const int index_offset   = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());
 234   const int buffer_offset  = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
 235 
 236   // Now the actual pointers into the thread
 237   Node* buffer_adr  = __ AddP(no_base, tls, __ ConX(buffer_offset));
 238   Node* index_adr   = __ AddP(no_base, tls, __ ConX(index_offset));
 239 
 240   // Now some of the values
 241   Node* marking;
 242   Node* gc_state = __ AddP(no_base, tls, __ ConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset())));
 243   Node* ld = __ load(__ ctrl(), gc_state, TypeInt::BYTE, T_BYTE, Compile::AliasIdxRaw);
 244   marking = __ AndI(ld, __ ConI(ShenandoahHeap::MARKING));
 245   assert(ShenandoahBarrierC2Support::is_gc_state_load(ld), "Should match the shape");
 246 
 247   // if (!marking)
 248   __ if_then(marking, BoolTest::ne, zero, unlikely); {
 249     BasicType index_bt = TypeX_X->basic_type();
 250     assert(sizeof(size_t) == type2aelembytes(index_bt), "Loading Shenandoah SATBMarkQueue::_index with wrong size.");
 251     Node* index   = __ load(__ ctrl(), index_adr, TypeX_X, index_bt, Compile::AliasIdxRaw);
 252 
 253     if (do_load) {
 254       // load original value
 255       // alias_idx correct??
 256       pre_val = __ load(__ ctrl(), adr, val_type, bt, alias_idx);
 257     }
 258 
 259     // if (pre_val != NULL)
 260     __ if_then(pre_val, BoolTest::ne, kit->null()); {
 261       Node* buffer  = __ load(__ ctrl(), buffer_adr, TypeRawPtr::NOTNULL, T_ADDRESS, Compile::AliasIdxRaw);
 262 
 263       // is the queue for this thread full?
 264       __ if_then(index, BoolTest::ne, zeroX, likely); {

 305   address entry_point = call->as_CallLeaf()->entry_point();
 306   return (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong)) ||
 307          (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow)) ||
 308          (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak)) ||
 309          (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow)) ||
 310          (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom));
 311 }
 312 
 313 bool ShenandoahBarrierSetC2::is_shenandoah_marking_if(PhaseTransform *phase, Node* n) {
 314   if (n->Opcode() != Op_If) {
 315     return false;
 316   }
 317 
 318   Node* bol = n->in(1);
 319   assert(bol->is_Bool(), "");
 320   Node* cmpx = bol->in(1);
 321   if (bol->as_Bool()->_test._test == BoolTest::ne &&
 322       cmpx->is_Cmp() && cmpx->in(2) == phase->intcon(0) &&
 323       is_shenandoah_state_load(cmpx->in(1)->in(1)) &&
 324       cmpx->in(1)->in(2)->is_Con() &&
 325       cmpx->in(1)->in(2) == phase->intcon(ShenandoahHeap::MARKING)) {
 326     return true;
 327   }
 328 
 329   return false;
 330 }
 331 
 332 bool ShenandoahBarrierSetC2::is_shenandoah_state_load(Node* n) {
 333   if (!n->is_Load()) return false;
 334   const int state_offset = in_bytes(ShenandoahThreadLocalData::gc_state_offset());
 335   return n->in(2)->is_AddP() && n->in(2)->in(2)->Opcode() == Op_ThreadLocal
 336          && n->in(2)->in(3)->is_Con()
 337          && n->in(2)->in(3)->bottom_type()->is_intptr_t()->get_con() == state_offset;
 338 }
 339 
 340 void ShenandoahBarrierSetC2::shenandoah_write_barrier_pre(GraphKit* kit,
 341                                                           bool do_load,
 342                                                           Node* obj,
 343                                                           Node* adr,
 344                                                           uint alias_idx,
 345                                                           Node* val,

 434         // Use the pre-barrier to record the value in the referent field
 435         satb_write_barrier_pre(kit, false /* do_load */,
 436                                NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
 437                                pre_val /* pre_val */,
 438                                T_OBJECT);
 439         if (need_mem_bar) {
 440           // Add memory barrier to prevent commoning reads from this field
 441           // across safepoint since GC can change its value.
 442           kit->insert_mem_bar(Op_MemBarCPUOrder);
 443         }
 444         // Update IdealKit from graphKit.
 445         __ sync_kit(kit);
 446 
 447       } __ end_if(); // _ref_type != ref_none
 448   } __ end_if(); // offset == referent_offset
 449 
 450   // Final sync IdealKit and GraphKit.
 451   kit->final_sync(ideal);
 452 }
 453 




























































































 454 #undef __
 455 
 456 const TypeFunc* ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type() {
 457   const Type **fields = TypeTuple::fields(2);
 458   fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value
 459   fields[TypeFunc::Parms+1] = TypeRawPtr::NOTNULL; // thread
 460   const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
 461 
 462   // create result type (range)
 463   fields = TypeTuple::fields(0);
 464   const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
 465 
 466   return TypeFunc::make(domain, range);
 467 }
 468 
 469 const TypeFunc* ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type() {
 470   const Type **fields = TypeTuple::fields(1);
 471   fields[TypeFunc::Parms+0] = TypeOopPtr::NOTNULL; // src oop
 472   const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);
 473 

 500   Node* adr = access.addr().node();
 501 
 502   bool anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0;
 503   bool on_heap = (decorators & IN_HEAP) != 0;
 504 
 505   if (!access.is_oop() || (!on_heap && !anonymous)) {
 506     return BarrierSetC2::store_at_resolved(access, val);
 507   }
 508 
 509   if (access.is_parse_access()) {
 510     C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
 511     GraphKit* kit = parse_access.kit();
 512 
 513     uint adr_idx = kit->C->get_alias_index(adr_type);
 514     assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" );
 515     Node* value = val.node();
 516     value = shenandoah_iu_barrier(kit, value);
 517     val.set_node(value);
 518     shenandoah_write_barrier_pre(kit, true /* do_load */, /*kit->control(),*/ access.base(), adr, adr_idx, val.node(),
 519                                  static_cast<const TypeOopPtr*>(val.type()), NULL /* pre_val */, access.type());






 520   } else {
 521     assert(access.is_opt_access(), "only for optimization passes");
 522     assert(((decorators & C2_TIGHTLY_COUPLED_ALLOC) != 0 || !ShenandoahSATBBarrier) && (decorators & C2_ARRAY_COPY) != 0, "unexpected caller of this code");
 523     C2OptAccess& opt_access = static_cast<C2OptAccess&>(access);
 524     PhaseGVN& gvn =  opt_access.gvn();
 525 
 526     if (ShenandoahIUBarrier) {
 527       Node* enqueue = gvn.transform(new ShenandoahIUBarrierNode(val.node()));
 528       val.set_node(enqueue);
 529     }

 530   }
 531   return BarrierSetC2::store_at_resolved(access, val);
 532 }
 533 
 534 Node* ShenandoahBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
 535   // 1: non-reference load, no additional barrier is needed
 536   if (!access.is_oop()) {
 537     return BarrierSetC2::load_at_resolved(access, val_type);;
 538   }
 539 
 540   Node* load = BarrierSetC2::load_at_resolved(access, val_type);
 541   DecoratorSet decorators = access.decorators();
 542   BasicType type = access.type();
 543 
 544   // 2: apply LRB if needed
 545   if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
 546     load = new ShenandoahLoadReferenceBarrierNode(NULL, load, decorators);
 547     if (access.is_parse_access()) {
 548       load = static_cast<C2ParseAccess &>(access).kit()->gvn().transform(load);
 549     } else {
 550       load = static_cast<C2OptAccess &>(access).gvn().transform(load);
 551     }

 582 
 583     if (on_weak_ref) {
 584       // Use the pre-barrier to record the value in the referent field
 585       satb_write_barrier_pre(kit, false /* do_load */,
 586                              NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
 587                              load /* pre_val */, T_OBJECT);
 588       // Add memory barrier to prevent commoning reads from this field
 589       // across safepoint since GC can change its value.
 590       kit->insert_mem_bar(Op_MemBarCPUOrder);
 591     } else if (unknown) {
 592       // We do not require a mem bar inside pre_barrier if need_mem_bar
 593       // is set: the barriers would be emitted by us.
 594       insert_pre_barrier(kit, obj, offset, load, !need_cpu_mem_bar);
 595     }
 596   }
 597 
 598   return load;
 599 }
 600 
 601 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
 602                                                    Node* new_val, const Type* value_type) const {
 603   GraphKit* kit = access.kit();
 604   if (access.is_oop()) {
 605     new_val = shenandoah_iu_barrier(kit, new_val);
 606     shenandoah_write_barrier_pre(kit, false /* do_load */,
 607                                  NULL, NULL, max_juint, NULL, NULL,
 608                                  expected_val /* pre_val */, T_OBJECT);
 609 
 610     MemNode::MemOrd mo = access.mem_node_mo();
 611     Node* mem = access.memory();
 612     Node* adr = access.addr().node();
 613     const TypePtr* adr_type = access.addr().type();
 614     Node* load_store = NULL;
 615 
 616 #ifdef _LP64
 617     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
 618       Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
 619       Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
 620       if (ShenandoahCASBarrier) {
 621         load_store = kit->gvn().transform(new ShenandoahCompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo));
 622       } else {

 624       }
 625     } else
 626 #endif
 627     {
 628       if (ShenandoahCASBarrier) {
 629         load_store = kit->gvn().transform(new ShenandoahCompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo));
 630       } else {
 631         load_store = kit->gvn().transform(new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo));
 632       }
 633     }
 634 
 635     access.set_raw_access(load_store);
 636     pin_atomic_op(access);
 637 
 638 #ifdef _LP64
 639     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
 640       load_store = kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
 641     }
 642 #endif
 643     load_store = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, load_store, access.decorators()));

 644     return load_store;
 645   }
 646   return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
 647 }
 648 
 649 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
 650                                                               Node* new_val, const Type* value_type) const {
 651   GraphKit* kit = access.kit();
 652   if (access.is_oop()) {
 653     new_val = shenandoah_iu_barrier(kit, new_val);
 654     shenandoah_write_barrier_pre(kit, false /* do_load */,
 655                                  NULL, NULL, max_juint, NULL, NULL,
 656                                  expected_val /* pre_val */, T_OBJECT);
 657     DecoratorSet decorators = access.decorators();
 658     MemNode::MemOrd mo = access.mem_node_mo();
 659     Node* mem = access.memory();
 660     bool is_weak_cas = (decorators & C2_WEAK_CMPXCHG) != 0;
 661     Node* load_store = NULL;
 662     Node* adr = access.addr().node();
 663 #ifdef _LP64

 679       }
 680     } else
 681 #endif
 682     {
 683       if (ShenandoahCASBarrier) {
 684         if (is_weak_cas) {
 685           load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
 686         } else {
 687           load_store = kit->gvn().transform(new ShenandoahCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
 688         }
 689       } else {
 690         if (is_weak_cas) {
 691           load_store = kit->gvn().transform(new WeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
 692         } else {
 693           load_store = kit->gvn().transform(new CompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
 694         }
 695       }
 696     }
 697     access.set_raw_access(load_store);
 698     pin_atomic_op(access);


 699     return load_store;
 700   }
 701   return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
 702 }
 703 
 704 Node* ShenandoahBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* val, const Type* value_type) const {
 705   GraphKit* kit = access.kit();
 706   if (access.is_oop()) {
 707     val = shenandoah_iu_barrier(kit, val);
 708   }
 709   Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type);
 710   if (access.is_oop()) {
 711     result = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, result, access.decorators()));
 712     shenandoah_write_barrier_pre(kit, false /* do_load */,
 713                                  NULL, NULL, max_juint, NULL, NULL,
 714                                  result /* pre_val */, T_OBJECT);


 715   }
 716   return result;
 717 }
 718 
 719 // Support for GC barriers emitted during parsing
 720 bool ShenandoahBarrierSetC2::is_gc_barrier_node(Node* node) const {
 721   if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) return true;
 722   if (node->Opcode() != Op_CallLeaf && node->Opcode() != Op_CallLeafNoFP) {
 723     return false;
 724   }
 725   CallLeafNode *call = node->as_CallLeaf();
 726   if (call->_name == NULL) {
 727     return false;
 728   }
 729 
 730   return strcmp(call->_name, "shenandoah_clone_barrier") == 0 ||
 731          strcmp(call->_name, "shenandoah_cas_obj") == 0 ||
 732          strcmp(call->_name, "shenandoah_wb_pre") == 0;
 733 }
 734 

 777   if (phase == Optimization) {
 778     return !ShenandoahIUBarrier;
 779   }
 780   return true;
 781 }
 782 
 783 bool ShenandoahBarrierSetC2::clone_needs_barrier(Node* src, PhaseGVN& gvn) {
 784   const TypeOopPtr* src_type = gvn.type(src)->is_oopptr();
 785   if (src_type->isa_instptr() != NULL) {
 786     ciInstanceKlass* ik = src_type->klass()->as_instance_klass();
 787     if ((src_type->klass_is_exact() || (!ik->is_interface() && !ik->has_subklass())) && !ik->has_injected_fields()) {
 788       if (ik->has_object_fields()) {
 789         return true;
 790       } else {
 791         if (!src_type->klass_is_exact()) {
 792           Compile::current()->dependencies()->assert_leaf_type(ik);
 793         }
 794       }
 795     } else {
 796       return true;
 797         }
 798   } else if (src_type->isa_aryptr()) {
 799     BasicType src_elem  = src_type->klass()->as_array_klass()->element_type()->basic_type();
 800     if (is_reference_type(src_elem)) {
 801       return true;
 802     }
 803   } else {
 804     return true;
 805   }
 806   return false;
 807 }
 808 
 809 void ShenandoahBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
 810   Node* ctrl = ac->in(TypeFunc::Control);
 811   Node* mem = ac->in(TypeFunc::Memory);
 812   Node* src_base = ac->in(ArrayCopyNode::Src);
 813   Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
 814   Node* dest_base = ac->in(ArrayCopyNode::Dest);
 815   Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
 816   Node* length = ac->in(ArrayCopyNode::Length);
 817 

 820 
 821   if (ShenandoahCloneBarrier && clone_needs_barrier(src, phase->igvn())) {
 822     // Check if heap is has forwarded objects. If it does, we need to call into the special
 823     // routine that would fix up source references before we can continue.
 824 
 825     enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
 826     Node* region = new RegionNode(PATH_LIMIT);
 827     Node* mem_phi = new PhiNode(region, Type::MEMORY, TypeRawPtr::BOTTOM);
 828 
 829     Node* thread = phase->transform_later(new ThreadLocalNode());
 830     Node* offset = phase->igvn().MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 831     Node* gc_state_addr = phase->transform_later(new AddPNode(phase->C->top(), thread, offset));
 832 
 833     uint gc_state_idx = Compile::AliasIdxRaw;
 834     const TypePtr* gc_state_adr_type = NULL; // debug-mode-only argument
 835     debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
 836 
 837     Node* gc_state    = phase->transform_later(new LoadBNode(ctrl, mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered));
 838     int flags = ShenandoahHeap::HAS_FORWARDED;
 839     if (ShenandoahIUBarrier) {
 840       flags |= ShenandoahHeap::MARKING;
 841     }
 842     Node* stable_and  = phase->transform_later(new AndINode(gc_state, phase->igvn().intcon(flags)));
 843     Node* stable_cmp  = phase->transform_later(new CmpINode(stable_and, phase->igvn().zerocon(T_INT)));
 844     Node* stable_test = phase->transform_later(new BoolNode(stable_cmp, BoolTest::ne));
 845 
 846     IfNode* stable_iff  = phase->transform_later(new IfNode(ctrl, stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN))->as_If();
 847     Node* stable_ctrl   = phase->transform_later(new IfFalseNode(stable_iff));
 848     Node* unstable_ctrl = phase->transform_later(new IfTrueNode(stable_iff));
 849 
 850     // Heap is stable, no need to do anything additional
 851     region->init_req(_heap_stable, stable_ctrl);
 852     mem_phi->init_req(_heap_stable, mem);
 853 
 854     // Heap is unstable, call into clone barrier stub
 855     Node* call = phase->make_leaf_call(unstable_ctrl, mem,
 856                     ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type(),
 857                     CAST_FROM_FN_PTR(address, ShenandoahRuntime::shenandoah_clone_barrier),
 858                     "shenandoah_clone",
 859                     TypeRawPtr::BOTTOM,
 860                     src_base);

 888 
 889 // Support for macro expanded GC barriers
 890 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
 891   if (node->Opcode() == Op_ShenandoahIUBarrier) {
 892     state()->add_iu_barrier((ShenandoahIUBarrierNode*) node);
 893   }
 894   if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
 895     state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
 896   }
 897 }
 898 
 899 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
 900   if (node->Opcode() == Op_ShenandoahIUBarrier) {
 901     state()->remove_iu_barrier((ShenandoahIUBarrierNode*) node);
 902   }
 903   if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
 904     state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
 905   }
 906 }
 907 
 908 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* n) const {
 909   if (is_shenandoah_wb_pre_call(n)) {
 910     shenandoah_eliminate_wb_pre(n, &macro->igvn());

















 911   }
 912 }
 913 
 914 void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
 915   assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
 916   Node* c = call->as_Call()->proj_out(TypeFunc::Control);
 917   c = c->unique_ctrl_out();
 918   assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
 919   c = c->unique_ctrl_out();
 920   assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
 921   Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
 922   assert(iff->is_If(), "expect test");
 923   if (!is_shenandoah_marking_if(igvn, iff)) {
 924     c = c->unique_ctrl_out();
 925     assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
 926     iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
 927     assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
 928   }
 929   Node* cmpx = iff->in(1)->in(1);
 930   igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ));

  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "classfile/javaClasses.hpp"
  27 #include "gc/shared/barrierSet.hpp"
  28 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  29 #include "gc/shenandoah/shenandoahForwarding.hpp"
  30 #include "gc/shenandoah/shenandoahHeap.hpp"
  31 #include "gc/shenandoah/shenandoahRuntime.hpp"
  32 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
  33 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
  34 #include "gc/shenandoah/c2/shenandoahSupport.hpp"
  35 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
  36 #include "gc/shenandoah/mode/shenandoahMode.hpp"
  37 #include "opto/arraycopynode.hpp"
  38 #include "opto/escape.hpp"
  39 #include "opto/graphKit.hpp"
  40 #include "opto/idealKit.hpp"
  41 #include "opto/macro.hpp"
  42 #include "opto/movenode.hpp"
  43 #include "opto/narrowptrnode.hpp"
  44 #include "opto/rootnode.hpp"
  45 #include "opto/runtime.hpp"
  46 
  47 ShenandoahBarrierSetC2* ShenandoahBarrierSetC2::bsc2() {
  48   return reinterpret_cast<ShenandoahBarrierSetC2*>(BarrierSet::barrier_set()->barrier_set_c2());
  49 }
  50 
  51 ShenandoahBarrierSetC2State::ShenandoahBarrierSetC2State(Arena* comp_arena)
  52   : _iu_barriers(new (comp_arena) GrowableArray<ShenandoahIUBarrierNode*>(comp_arena, 8,  0, NULL)),
  53     _load_reference_barriers(new (comp_arena) GrowableArray<ShenandoahLoadReferenceBarrierNode*>(comp_arena, 8,  0, NULL)) {
  54 }
  55 
  56 int ShenandoahBarrierSetC2State::iu_barriers_count() const {

 225 
 226   Node* no_base = __ top();
 227   Node* zero  = __ ConI(0);
 228   Node* zeroX = __ ConX(0);
 229 
 230   float likely  = PROB_LIKELY(0.999);
 231   float unlikely  = PROB_UNLIKELY(0.999);
 232 
 233   // Offsets into the thread
 234   const int index_offset   = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());
 235   const int buffer_offset  = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
 236 
 237   // Now the actual pointers into the thread
 238   Node* buffer_adr  = __ AddP(no_base, tls, __ ConX(buffer_offset));
 239   Node* index_adr   = __ AddP(no_base, tls, __ ConX(index_offset));
 240 
 241   // Now some of the values
 242   Node* marking;
 243   Node* gc_state = __ AddP(no_base, tls, __ ConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset())));
 244   Node* ld = __ load(__ ctrl(), gc_state, TypeInt::BYTE, T_BYTE, Compile::AliasIdxRaw);
 245   marking = __ AndI(ld, __ ConI(ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING));
 246   assert(ShenandoahBarrierC2Support::is_gc_state_load(ld), "Should match the shape");
 247 
 248   // if (!marking)
 249   __ if_then(marking, BoolTest::ne, zero, unlikely); {
 250     BasicType index_bt = TypeX_X->basic_type();
 251     assert(sizeof(size_t) == type2aelembytes(index_bt), "Loading Shenandoah SATBMarkQueue::_index with wrong size.");
 252     Node* index   = __ load(__ ctrl(), index_adr, TypeX_X, index_bt, Compile::AliasIdxRaw);
 253 
 254     if (do_load) {
 255       // load original value
 256       // alias_idx correct??
 257       pre_val = __ load(__ ctrl(), adr, val_type, bt, alias_idx);
 258     }
 259 
 260     // if (pre_val != NULL)
 261     __ if_then(pre_val, BoolTest::ne, kit->null()); {
 262       Node* buffer  = __ load(__ ctrl(), buffer_adr, TypeRawPtr::NOTNULL, T_ADDRESS, Compile::AliasIdxRaw);
 263 
 264       // is the queue for this thread full?
 265       __ if_then(index, BoolTest::ne, zeroX, likely); {

 306   address entry_point = call->as_CallLeaf()->entry_point();
 307   return (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong)) ||
 308          (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow)) ||
 309          (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak)) ||
 310          (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow)) ||
 311          (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom));
 312 }
 313 
 314 bool ShenandoahBarrierSetC2::is_shenandoah_marking_if(PhaseTransform *phase, Node* n) {
 315   if (n->Opcode() != Op_If) {
 316     return false;
 317   }
 318 
 319   Node* bol = n->in(1);
 320   assert(bol->is_Bool(), "");
 321   Node* cmpx = bol->in(1);
 322   if (bol->as_Bool()->_test._test == BoolTest::ne &&
 323       cmpx->is_Cmp() && cmpx->in(2) == phase->intcon(0) &&
 324       is_shenandoah_state_load(cmpx->in(1)->in(1)) &&
 325       cmpx->in(1)->in(2)->is_Con() &&
 326       cmpx->in(1)->in(2) == phase->intcon(ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING)) {
 327     return true;
 328   }
 329 
 330   return false;
 331 }
 332 
 333 bool ShenandoahBarrierSetC2::is_shenandoah_state_load(Node* n) {
 334   if (!n->is_Load()) return false;
 335   const int state_offset = in_bytes(ShenandoahThreadLocalData::gc_state_offset());
 336   return n->in(2)->is_AddP() && n->in(2)->in(2)->Opcode() == Op_ThreadLocal
 337          && n->in(2)->in(3)->is_Con()
 338          && n->in(2)->in(3)->bottom_type()->is_intptr_t()->get_con() == state_offset;
 339 }
 340 
 341 void ShenandoahBarrierSetC2::shenandoah_write_barrier_pre(GraphKit* kit,
 342                                                           bool do_load,
 343                                                           Node* obj,
 344                                                           Node* adr,
 345                                                           uint alias_idx,
 346                                                           Node* val,

 435         // Use the pre-barrier to record the value in the referent field
 436         satb_write_barrier_pre(kit, false /* do_load */,
 437                                NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
 438                                pre_val /* pre_val */,
 439                                T_OBJECT);
 440         if (need_mem_bar) {
 441           // Add memory barrier to prevent commoning reads from this field
 442           // across safepoint since GC can change its value.
 443           kit->insert_mem_bar(Op_MemBarCPUOrder);
 444         }
 445         // Update IdealKit from graphKit.
 446         __ sync_kit(kit);
 447 
 448       } __ end_if(); // _ref_type != ref_none
 449   } __ end_if(); // offset == referent_offset
 450 
 451   // Final sync IdealKit and GraphKit.
 452   kit->final_sync(ideal);
 453 }
 454 
 455 Node* ShenandoahBarrierSetC2::byte_map_base_node(GraphKit* kit) const {
 456   BarrierSet* bs = BarrierSet::barrier_set();
 457   ShenandoahBarrierSet* ctbs = barrier_set_cast<ShenandoahBarrierSet>(bs);
 458   CardTable::CardValue* card_table_base = ctbs->card_table()->byte_map_base();
 459   if (card_table_base != NULL) {
 460     return kit->makecon(TypeRawPtr::make((address)card_table_base));
 461   } else {
 462     return kit->null();
 463   }
 464 }
 465 
 466 void ShenandoahBarrierSetC2::post_barrier(GraphKit* kit,
 467                                           Node* ctl,
 468                                           Node* oop_store,
 469                                           Node* obj,
 470                                           Node* adr,
 471                                           uint  adr_idx,
 472                                           Node* val,
 473                                           BasicType bt,
 474                                           bool use_precise) const {
 475   if (!ShenandoahHeap::heap()->mode()->is_generational()) {
 476     return;
 477   }
 478 
 479   ShenandoahBarrierSet* ctbs = barrier_set_cast<ShenandoahBarrierSet>(BarrierSet::barrier_set());
 480   CardTable* ct = ctbs->card_table();
 481   // No store check needed if we're storing a NULL or an old object
 482   // (latter case is probably a string constant). The concurrent
 483   // mark sweep garbage collector, however, needs to have all nonNull
 484   // oop updates flagged via card-marks.
 485   if (val != NULL && val->is_Con()) {
 486     // must be either an oop or NULL
 487     const Type* t = val->bottom_type();
 488     if (t == TypePtr::NULL_PTR || t == Type::TOP)
 489       // stores of null never (?) need barriers
 490       return;
 491   }
 492 
 493   if (ReduceInitialCardMarks && obj == kit->just_allocated_object(kit->control())) {
 494     // We can skip marks on a freshly-allocated object in Eden.
 495     // Keep this code in sync with new_deferred_store_barrier() in runtime.cpp.
 496     // That routine informs GC to take appropriate compensating steps,
 497     // upon a slow-path allocation, so as to make this card-mark
 498     // elision safe.
 499     return;
 500   }
 501 
 502   if (!use_precise) {
 503     // All card marks for a (non-array) instance are in one place:
 504     adr = obj;
 505   }
 506   // (Else it's an array (or unknown), and we want more precise card marks.)
 507   assert(adr != NULL, "");
 508 
 509   IdealKit ideal(kit, true);
 510 
 511   // Convert the pointer to an int prior to doing math on it
 512   Node* cast = __ CastPX(__ ctrl(), adr);
 513 
 514   // Divide by card size
 515   Node* card_offset = __ URShiftX( cast, __ ConI(CardTable::card_shift) );
 516 
 517   // Combine card table base and card offset
 518   Node* card_adr = __ AddP(__ top(), byte_map_base_node(kit), card_offset );
 519 
 520   // Get the alias_index for raw card-mark memory
 521   int adr_type = Compile::AliasIdxRaw;
 522   Node*   zero = __ ConI(0); // Dirty card value
 523 
 524   if (UseCondCardMark) {
 525     // The classic GC reference write barrier is typically implemented
 526     // as a store into the global card mark table.  Unfortunately
 527     // unconditional stores can result in false sharing and excessive
 528     // coherence traffic as well as false transactional aborts.
 529     // UseCondCardMark enables MP "polite" conditional card mark
 530     // stores.  In theory we could relax the load from ctrl() to
 531     // no_ctrl, but that doesn't buy much latitude.
 532     Node* card_val = __ load( __ ctrl(), card_adr, TypeInt::BYTE, T_BYTE, adr_type);
 533     __ if_then(card_val, BoolTest::ne, zero);
 534   }
 535 
 536   // Smash zero into card
 537   __ store(__ ctrl(), card_adr, zero, T_BYTE, adr_type, MemNode::unordered);
 538 
 539   if (UseCondCardMark) {
 540     __ end_if();
 541   }
 542 
 543   // Final sync IdealKit and GraphKit.
 544   kit->final_sync(ideal);
 545 }
 546 
 547 #undef __
 548 
 549 const TypeFunc* ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type() {
 550   const Type **fields = TypeTuple::fields(2);
 551   fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value
 552   fields[TypeFunc::Parms+1] = TypeRawPtr::NOTNULL; // thread
 553   const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
 554 
 555   // create result type (range)
 556   fields = TypeTuple::fields(0);
 557   const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
 558 
 559   return TypeFunc::make(domain, range);
 560 }
 561 
 562 const TypeFunc* ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type() {
 563   const Type **fields = TypeTuple::fields(1);
 564   fields[TypeFunc::Parms+0] = TypeOopPtr::NOTNULL; // src oop
 565   const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);
 566 

 593   Node* adr = access.addr().node();
 594 
 595   bool anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0;
 596   bool on_heap = (decorators & IN_HEAP) != 0;
 597 
 598   if (!access.is_oop() || (!on_heap && !anonymous)) {
 599     return BarrierSetC2::store_at_resolved(access, val);
 600   }
 601 
 602   if (access.is_parse_access()) {
 603     C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
 604     GraphKit* kit = parse_access.kit();
 605 
 606     uint adr_idx = kit->C->get_alias_index(adr_type);
 607     assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" );
 608     Node* value = val.node();
 609     value = shenandoah_iu_barrier(kit, value);
 610     val.set_node(value);
 611     shenandoah_write_barrier_pre(kit, true /* do_load */, /*kit->control(),*/ access.base(), adr, adr_idx, val.node(),
 612                                  static_cast<const TypeOopPtr*>(val.type()), NULL /* pre_val */, access.type());
 613 
 614     Node* result = BarrierSetC2::store_at_resolved(access, val);
 615     bool is_array = (decorators & IS_ARRAY) != 0;
 616     bool use_precise = is_array || anonymous;
 617     post_barrier(kit, kit->control(), access.raw_access(), access.base(), adr, adr_idx, val.node(), access.type(), use_precise);
 618     return result;
 619   } else {
 620     assert(access.is_opt_access(), "only for optimization passes");
 621     assert(((decorators & C2_TIGHTLY_COUPLED_ALLOC) != 0 || !ShenandoahSATBBarrier) && (decorators & C2_ARRAY_COPY) != 0, "unexpected caller of this code");
 622     C2OptAccess& opt_access = static_cast<C2OptAccess&>(access);
 623     PhaseGVN& gvn =  opt_access.gvn();
 624 
 625     if (ShenandoahIUBarrier) {
 626       Node* enqueue = gvn.transform(new ShenandoahIUBarrierNode(val.node()));
 627       val.set_node(enqueue);
 628     }
 629     return BarrierSetC2::store_at_resolved(access, val);
 630   }

 631 }
 632 
 633 Node* ShenandoahBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
 634   // 1: non-reference load, no additional barrier is needed
 635   if (!access.is_oop()) {
 636     return BarrierSetC2::load_at_resolved(access, val_type);;
 637   }
 638 
 639   Node* load = BarrierSetC2::load_at_resolved(access, val_type);
 640   DecoratorSet decorators = access.decorators();
 641   BasicType type = access.type();
 642 
 643   // 2: apply LRB if needed
 644   if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
 645     load = new ShenandoahLoadReferenceBarrierNode(NULL, load, decorators);
 646     if (access.is_parse_access()) {
 647       load = static_cast<C2ParseAccess &>(access).kit()->gvn().transform(load);
 648     } else {
 649       load = static_cast<C2OptAccess &>(access).gvn().transform(load);
 650     }

 681 
 682     if (on_weak_ref) {
 683       // Use the pre-barrier to record the value in the referent field
 684       satb_write_barrier_pre(kit, false /* do_load */,
 685                              NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
 686                              load /* pre_val */, T_OBJECT);
 687       // Add memory barrier to prevent commoning reads from this field
 688       // across safepoint since GC can change its value.
 689       kit->insert_mem_bar(Op_MemBarCPUOrder);
 690     } else if (unknown) {
 691       // We do not require a mem bar inside pre_barrier if need_mem_bar
 692       // is set: the barriers would be emitted by us.
 693       insert_pre_barrier(kit, obj, offset, load, !need_cpu_mem_bar);
 694     }
 695   }
 696 
 697   return load;
 698 }
 699 
 700 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
 701                                                              Node* new_val, const Type* value_type) const {
 702   GraphKit* kit = access.kit();
 703   if (access.is_oop()) {
 704     new_val = shenandoah_iu_barrier(kit, new_val);
 705     shenandoah_write_barrier_pre(kit, false /* do_load */,
 706                                  NULL, NULL, max_juint, NULL, NULL,
 707                                  expected_val /* pre_val */, T_OBJECT);
 708 
 709     MemNode::MemOrd mo = access.mem_node_mo();
 710     Node* mem = access.memory();
 711     Node* adr = access.addr().node();
 712     const TypePtr* adr_type = access.addr().type();
 713     Node* load_store = NULL;
 714 
 715 #ifdef _LP64
 716     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
 717       Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
 718       Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
 719       if (ShenandoahCASBarrier) {
 720         load_store = kit->gvn().transform(new ShenandoahCompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo));
 721       } else {

 723       }
 724     } else
 725 #endif
 726     {
 727       if (ShenandoahCASBarrier) {
 728         load_store = kit->gvn().transform(new ShenandoahCompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo));
 729       } else {
 730         load_store = kit->gvn().transform(new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo));
 731       }
 732     }
 733 
 734     access.set_raw_access(load_store);
 735     pin_atomic_op(access);
 736 
 737 #ifdef _LP64
 738     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
 739       load_store = kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
 740     }
 741 #endif
 742     load_store = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, load_store, access.decorators()));
 743     post_barrier(kit, kit->control(), access.raw_access(), access.base(), access.addr().node(), access.alias_idx(), new_val, T_OBJECT, true);
 744     return load_store;
 745   }
 746   return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
 747 }
 748 
 749 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
 750                                                               Node* new_val, const Type* value_type) const {
 751   GraphKit* kit = access.kit();
 752   if (access.is_oop()) {
 753     new_val = shenandoah_iu_barrier(kit, new_val);
 754     shenandoah_write_barrier_pre(kit, false /* do_load */,
 755                                  NULL, NULL, max_juint, NULL, NULL,
 756                                  expected_val /* pre_val */, T_OBJECT);
 757     DecoratorSet decorators = access.decorators();
 758     MemNode::MemOrd mo = access.mem_node_mo();
 759     Node* mem = access.memory();
 760     bool is_weak_cas = (decorators & C2_WEAK_CMPXCHG) != 0;
 761     Node* load_store = NULL;
 762     Node* adr = access.addr().node();
 763 #ifdef _LP64

 779       }
 780     } else
 781 #endif
 782     {
 783       if (ShenandoahCASBarrier) {
 784         if (is_weak_cas) {
 785           load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
 786         } else {
 787           load_store = kit->gvn().transform(new ShenandoahCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
 788         }
 789       } else {
 790         if (is_weak_cas) {
 791           load_store = kit->gvn().transform(new WeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
 792         } else {
 793           load_store = kit->gvn().transform(new CompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
 794         }
 795       }
 796     }
 797     access.set_raw_access(load_store);
 798     pin_atomic_op(access);
 799     post_barrier(kit, kit->control(), access.raw_access(), access.base(),
 800                  access.addr().node(), access.alias_idx(), new_val, T_OBJECT, true);
 801     return load_store;
 802   }
 803   return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
 804 }
 805 
 806 Node* ShenandoahBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* val, const Type* value_type) const {
 807   GraphKit* kit = access.kit();
 808   if (access.is_oop()) {
 809     val = shenandoah_iu_barrier(kit, val);
 810   }
 811   Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type);
 812   if (access.is_oop()) {
 813     result = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, result, access.decorators()));
 814     shenandoah_write_barrier_pre(kit, false /* do_load */,
 815                                  NULL, NULL, max_juint, NULL, NULL,
 816                                  result /* pre_val */, T_OBJECT);
 817     post_barrier(kit, kit->control(), access.raw_access(), access.base(),
 818                  access.addr().node(), access.alias_idx(), val, T_OBJECT, true);
 819   }
 820   return result;
 821 }
 822 
 823 // Support for GC barriers emitted during parsing
 824 bool ShenandoahBarrierSetC2::is_gc_barrier_node(Node* node) const {
 825   if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) return true;
 826   if (node->Opcode() != Op_CallLeaf && node->Opcode() != Op_CallLeafNoFP) {
 827     return false;
 828   }
 829   CallLeafNode *call = node->as_CallLeaf();
 830   if (call->_name == NULL) {
 831     return false;
 832   }
 833 
 834   return strcmp(call->_name, "shenandoah_clone_barrier") == 0 ||
 835          strcmp(call->_name, "shenandoah_cas_obj") == 0 ||
 836          strcmp(call->_name, "shenandoah_wb_pre") == 0;
 837 }
 838 

 881   if (phase == Optimization) {
 882     return !ShenandoahIUBarrier;
 883   }
 884   return true;
 885 }
 886 
 887 bool ShenandoahBarrierSetC2::clone_needs_barrier(Node* src, PhaseGVN& gvn) {
 888   const TypeOopPtr* src_type = gvn.type(src)->is_oopptr();
 889   if (src_type->isa_instptr() != NULL) {
 890     ciInstanceKlass* ik = src_type->klass()->as_instance_klass();
 891     if ((src_type->klass_is_exact() || (!ik->is_interface() && !ik->has_subklass())) && !ik->has_injected_fields()) {
 892       if (ik->has_object_fields()) {
 893         return true;
 894       } else {
 895         if (!src_type->klass_is_exact()) {
 896           Compile::current()->dependencies()->assert_leaf_type(ik);
 897         }
 898       }
 899     } else {
 900       return true;
 901     }
 902   } else if (src_type->isa_aryptr()) {
 903     BasicType src_elem  = src_type->klass()->as_array_klass()->element_type()->basic_type();
 904     if (is_reference_type(src_elem)) {
 905       return true;
 906     }
 907   } else {
 908     return true;
 909   }
 910   return false;
 911 }
 912 
 913 void ShenandoahBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
 914   Node* ctrl = ac->in(TypeFunc::Control);
 915   Node* mem = ac->in(TypeFunc::Memory);
 916   Node* src_base = ac->in(ArrayCopyNode::Src);
 917   Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
 918   Node* dest_base = ac->in(ArrayCopyNode::Dest);
 919   Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
 920   Node* length = ac->in(ArrayCopyNode::Length);
 921 

 924 
 925   if (ShenandoahCloneBarrier && clone_needs_barrier(src, phase->igvn())) {
 926     // Check if heap is has forwarded objects. If it does, we need to call into the special
 927     // routine that would fix up source references before we can continue.
 928 
 929     enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
 930     Node* region = new RegionNode(PATH_LIMIT);
 931     Node* mem_phi = new PhiNode(region, Type::MEMORY, TypeRawPtr::BOTTOM);
 932 
 933     Node* thread = phase->transform_later(new ThreadLocalNode());
 934     Node* offset = phase->igvn().MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 935     Node* gc_state_addr = phase->transform_later(new AddPNode(phase->C->top(), thread, offset));
 936 
 937     uint gc_state_idx = Compile::AliasIdxRaw;
 938     const TypePtr* gc_state_adr_type = NULL; // debug-mode-only argument
 939     debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
 940 
 941     Node* gc_state    = phase->transform_later(new LoadBNode(ctrl, mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered));
 942     int flags = ShenandoahHeap::HAS_FORWARDED;
 943     if (ShenandoahIUBarrier) {
 944       flags |= ShenandoahHeap::YOUNG_MARKING;
 945     }
 946     Node* stable_and  = phase->transform_later(new AndINode(gc_state, phase->igvn().intcon(flags)));
 947     Node* stable_cmp  = phase->transform_later(new CmpINode(stable_and, phase->igvn().zerocon(T_INT)));
 948     Node* stable_test = phase->transform_later(new BoolNode(stable_cmp, BoolTest::ne));
 949 
 950     IfNode* stable_iff  = phase->transform_later(new IfNode(ctrl, stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN))->as_If();
 951     Node* stable_ctrl   = phase->transform_later(new IfFalseNode(stable_iff));
 952     Node* unstable_ctrl = phase->transform_later(new IfTrueNode(stable_iff));
 953 
 954     // Heap is stable, no need to do anything additional
 955     region->init_req(_heap_stable, stable_ctrl);
 956     mem_phi->init_req(_heap_stable, mem);
 957 
 958     // Heap is unstable, call into clone barrier stub
 959     Node* call = phase->make_leaf_call(unstable_ctrl, mem,
 960                     ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type(),
 961                     CAST_FROM_FN_PTR(address, ShenandoahRuntime::shenandoah_clone_barrier),
 962                     "shenandoah_clone",
 963                     TypeRawPtr::BOTTOM,
 964                     src_base);

 992 
 993 // Support for macro expanded GC barriers
 994 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
 995   if (node->Opcode() == Op_ShenandoahIUBarrier) {
 996     state()->add_iu_barrier((ShenandoahIUBarrierNode*) node);
 997   }
 998   if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
 999     state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
1000   }
1001 }
1002 
1003 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
1004   if (node->Opcode() == Op_ShenandoahIUBarrier) {
1005     state()->remove_iu_barrier((ShenandoahIUBarrierNode*) node);
1006   }
1007   if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
1008     state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
1009   }
1010 }
1011 
1012 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const {
1013   if (is_shenandoah_wb_pre_call(node)) {
1014     shenandoah_eliminate_wb_pre(node, &macro->igvn());
1015   }
1016   if (node->Opcode() == Op_CastP2X && ShenandoahHeap::heap()->mode()->is_generational()) {
1017     assert(node->Opcode() == Op_CastP2X, "ConvP2XNode required");
1018      Node *shift = node->unique_out();
1019      Node *addp = shift->unique_out();
1020      for (DUIterator_Last jmin, j = addp->last_outs(jmin); j >= jmin; --j) {
1021        Node *mem = addp->last_out(j);
1022        if (UseCondCardMark && mem->is_Load()) {
1023          assert(mem->Opcode() == Op_LoadB, "unexpected code shape");
1024          // The load is checking if the card has been written so
1025          // replace it with zero to fold the test.
1026          macro->replace_node(mem, macro->intcon(0));
1027          continue;
1028        }
1029        assert(mem->is_Store(), "store required");
1030        macro->replace_node(mem, mem->in(MemNode::Memory));
1031      }
1032   }
1033 }
1034 
1035 void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
1036   assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
1037   Node* c = call->as_Call()->proj_out(TypeFunc::Control);
1038   c = c->unique_ctrl_out();
1039   assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
1040   c = c->unique_ctrl_out();
1041   assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
1042   Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
1043   assert(iff->is_If(), "expect test");
1044   if (!is_shenandoah_marking_if(igvn, iff)) {
1045     c = c->unique_ctrl_out();
1046     assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
1047     iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
1048     assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
1049   }
1050   Node* cmpx = iff->in(1)->in(1);
1051   igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ));
< prev index next >