< prev index next >

src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp

Print this page

  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "classfile/javaClasses.hpp"
  27 #include "gc/shared/barrierSet.hpp"
  28 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  29 #include "gc/shenandoah/shenandoahForwarding.hpp"
  30 #include "gc/shenandoah/shenandoahHeap.hpp"
  31 #include "gc/shenandoah/shenandoahRuntime.hpp"
  32 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
  33 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
  34 #include "gc/shenandoah/c2/shenandoahSupport.hpp"
  35 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"

  36 #include "opto/arraycopynode.hpp"
  37 #include "opto/escape.hpp"
  38 #include "opto/graphKit.hpp"
  39 #include "opto/idealKit.hpp"
  40 #include "opto/macro.hpp"
  41 #include "opto/movenode.hpp"
  42 #include "opto/narrowptrnode.hpp"
  43 #include "opto/rootnode.hpp"
  44 #include "opto/runtime.hpp"
  45 
  46 ShenandoahBarrierSetC2* ShenandoahBarrierSetC2::bsc2() {
  47   return reinterpret_cast<ShenandoahBarrierSetC2*>(BarrierSet::barrier_set()->barrier_set_c2());
  48 }
  49 
  50 ShenandoahBarrierSetC2State::ShenandoahBarrierSetC2State(Arena* comp_arena)
  51   : _iu_barriers(new (comp_arena) GrowableArray<ShenandoahIUBarrierNode*>(comp_arena, 8,  0, NULL)),
  52     _load_reference_barriers(new (comp_arena) GrowableArray<ShenandoahLoadReferenceBarrierNode*>(comp_arena, 8,  0, NULL)) {
  53 }
  54 
  55 int ShenandoahBarrierSetC2State::iu_barriers_count() const {

 224 
 225   Node* no_base = __ top();
 226   Node* zero  = __ ConI(0);
 227   Node* zeroX = __ ConX(0);
 228 
 229   float likely  = PROB_LIKELY(0.999);
 230   float unlikely  = PROB_UNLIKELY(0.999);
 231 
 232   // Offsets into the thread
 233   const int index_offset   = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());
 234   const int buffer_offset  = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
 235 
 236   // Now the actual pointers into the thread
 237   Node* buffer_adr  = __ AddP(no_base, tls, __ ConX(buffer_offset));
 238   Node* index_adr   = __ AddP(no_base, tls, __ ConX(index_offset));
 239 
 240   // Now some of the values
 241   Node* marking;
 242   Node* gc_state = __ AddP(no_base, tls, __ ConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset())));
 243   Node* ld = __ load(__ ctrl(), gc_state, TypeInt::BYTE, T_BYTE, Compile::AliasIdxRaw);
 244   marking = __ AndI(ld, __ ConI(ShenandoahHeap::MARKING));
 245   assert(ShenandoahBarrierC2Support::is_gc_state_load(ld), "Should match the shape");
 246 
 247   // if (!marking)
 248   __ if_then(marking, BoolTest::ne, zero, unlikely); {
 249     BasicType index_bt = TypeX_X->basic_type();
 250     assert(sizeof(size_t) == type2aelembytes(index_bt), "Loading Shenandoah SATBMarkQueue::_index with wrong size.");
 251     Node* index   = __ load(__ ctrl(), index_adr, TypeX_X, index_bt, Compile::AliasIdxRaw);
 252 
 253     if (do_load) {
 254       // load original value
 255       // alias_idx correct??
 256       pre_val = __ load(__ ctrl(), adr, val_type, bt, alias_idx);
 257     }
 258 
 259     // if (pre_val != NULL)
 260     __ if_then(pre_val, BoolTest::ne, kit->null()); {
 261       Node* buffer  = __ load(__ ctrl(), buffer_adr, TypeRawPtr::NOTNULL, T_ADDRESS, Compile::AliasIdxRaw);
 262 
 263       // is the queue for this thread full?
 264       __ if_then(index, BoolTest::ne, zeroX, likely); {

 305   address entry_point = call->as_CallLeaf()->entry_point();
 306   return (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong)) ||
 307          (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow)) ||
 308          (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak)) ||
 309          (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow)) ||
 310          (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom));
 311 }
 312 
 313 bool ShenandoahBarrierSetC2::is_shenandoah_marking_if(PhaseTransform *phase, Node* n) {
 314   if (n->Opcode() != Op_If) {
 315     return false;
 316   }
 317 
 318   Node* bol = n->in(1);
 319   assert(bol->is_Bool(), "");
 320   Node* cmpx = bol->in(1);
 321   if (bol->as_Bool()->_test._test == BoolTest::ne &&
 322       cmpx->is_Cmp() && cmpx->in(2) == phase->intcon(0) &&
 323       is_shenandoah_state_load(cmpx->in(1)->in(1)) &&
 324       cmpx->in(1)->in(2)->is_Con() &&
 325       cmpx->in(1)->in(2) == phase->intcon(ShenandoahHeap::MARKING)) {
 326     return true;
 327   }
 328 
 329   return false;
 330 }
 331 
 332 bool ShenandoahBarrierSetC2::is_shenandoah_state_load(Node* n) {
 333   if (!n->is_Load()) return false;
 334   const int state_offset = in_bytes(ShenandoahThreadLocalData::gc_state_offset());
 335   return n->in(2)->is_AddP() && n->in(2)->in(2)->Opcode() == Op_ThreadLocal
 336          && n->in(2)->in(3)->is_Con()
 337          && n->in(2)->in(3)->bottom_type()->is_intptr_t()->get_con() == state_offset;
 338 }
 339 
 340 void ShenandoahBarrierSetC2::shenandoah_write_barrier_pre(GraphKit* kit,
 341                                                           bool do_load,
 342                                                           Node* obj,
 343                                                           Node* adr,
 344                                                           uint alias_idx,
 345                                                           Node* val,

 434         // Use the pre-barrier to record the value in the referent field
 435         satb_write_barrier_pre(kit, false /* do_load */,
 436                                NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
 437                                pre_val /* pre_val */,
 438                                T_OBJECT);
 439         if (need_mem_bar) {
 440           // Add memory barrier to prevent commoning reads from this field
 441           // across safepoint since GC can change its value.
 442           kit->insert_mem_bar(Op_MemBarCPUOrder);
 443         }
 444         // Update IdealKit from graphKit.
 445         __ sync_kit(kit);
 446 
 447       } __ end_if(); // _ref_type != ref_none
 448   } __ end_if(); // offset == referent_offset
 449 
 450   // Final sync IdealKit and GraphKit.
 451   kit->final_sync(ideal);
 452 }
 453 




























































































 454 #undef __
 455 
 456 const TypeFunc* ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type() {
 457   const Type **fields = TypeTuple::fields(2);
 458   fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value
 459   fields[TypeFunc::Parms+1] = TypeRawPtr::NOTNULL; // thread
 460   const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
 461 
 462   // create result type (range)
 463   fields = TypeTuple::fields(0);
 464   const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
 465 
 466   return TypeFunc::make(domain, range);
 467 }
 468 
 469 const TypeFunc* ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type() {
 470   const Type **fields = TypeTuple::fields(1);
 471   fields[TypeFunc::Parms+0] = TypeOopPtr::NOTNULL; // src oop
 472   const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);
 473 

 500   Node* adr = access.addr().node();
 501 
 502   bool anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0;
 503   bool on_heap = (decorators & IN_HEAP) != 0;
 504 
 505   if (!access.is_oop() || (!on_heap && !anonymous)) {
 506     return BarrierSetC2::store_at_resolved(access, val);
 507   }
 508 
 509   if (access.is_parse_access()) {
 510     C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
 511     GraphKit* kit = parse_access.kit();
 512 
 513     uint adr_idx = kit->C->get_alias_index(adr_type);
 514     assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" );
 515     Node* value = val.node();
 516     value = shenandoah_iu_barrier(kit, value);
 517     val.set_node(value);
 518     shenandoah_write_barrier_pre(kit, true /* do_load */, /*kit->control(),*/ access.base(), adr, adr_idx, val.node(),
 519                                  static_cast<const TypeOopPtr*>(val.type()), NULL /* pre_val */, access.type());






 520   } else {
 521     assert(access.is_opt_access(), "only for optimization passes");
 522     assert(((decorators & C2_TIGHTLY_COUPLED_ALLOC) != 0 || !ShenandoahSATBBarrier) && (decorators & C2_ARRAY_COPY) != 0, "unexpected caller of this code");
 523     C2OptAccess& opt_access = static_cast<C2OptAccess&>(access);
 524     PhaseGVN& gvn =  opt_access.gvn();
 525 
 526     if (ShenandoahIUBarrier) {
 527       Node* enqueue = gvn.transform(new ShenandoahIUBarrierNode(val.node()));
 528       val.set_node(enqueue);
 529     }

 530   }
 531   return BarrierSetC2::store_at_resolved(access, val);
 532 }
 533 
 534 Node* ShenandoahBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
 535   // 1: non-reference load, no additional barrier is needed
 536   if (!access.is_oop()) {
 537     return BarrierSetC2::load_at_resolved(access, val_type);;
 538   }
 539 
 540   Node* load = BarrierSetC2::load_at_resolved(access, val_type);
 541   DecoratorSet decorators = access.decorators();
 542   BasicType type = access.type();
 543 
 544   // 2: apply LRB if needed
 545   if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
 546     load = new ShenandoahLoadReferenceBarrierNode(NULL, load, decorators);
 547     if (access.is_parse_access()) {
 548       load = static_cast<C2ParseAccess &>(access).kit()->gvn().transform(load);
 549     } else {
 550       load = static_cast<C2OptAccess &>(access).gvn().transform(load);
 551     }

 582 
 583     if (on_weak_ref) {
 584       // Use the pre-barrier to record the value in the referent field
 585       satb_write_barrier_pre(kit, false /* do_load */,
 586                              NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
 587                              load /* pre_val */, T_OBJECT);
 588       // Add memory barrier to prevent commoning reads from this field
 589       // across safepoint since GC can change its value.
 590       kit->insert_mem_bar(Op_MemBarCPUOrder);
 591     } else if (unknown) {
 592       // We do not require a mem bar inside pre_barrier if need_mem_bar
 593       // is set: the barriers would be emitted by us.
 594       insert_pre_barrier(kit, obj, offset, load, !need_cpu_mem_bar);
 595     }
 596   }
 597 
 598   return load;
 599 }
 600 
 601 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
 602                                                    Node* new_val, const Type* value_type) const {
 603   GraphKit* kit = access.kit();
 604   if (access.is_oop()) {
 605     new_val = shenandoah_iu_barrier(kit, new_val);
 606     shenandoah_write_barrier_pre(kit, false /* do_load */,
 607                                  NULL, NULL, max_juint, NULL, NULL,
 608                                  expected_val /* pre_val */, T_OBJECT);
 609 
 610     MemNode::MemOrd mo = access.mem_node_mo();
 611     Node* mem = access.memory();
 612     Node* adr = access.addr().node();
 613     const TypePtr* adr_type = access.addr().type();
 614     Node* load_store = NULL;
 615 
 616 #ifdef _LP64
 617     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
 618       Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
 619       Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
 620       if (ShenandoahCASBarrier) {
 621         load_store = kit->gvn().transform(new ShenandoahCompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo));
 622       } else {

 624       }
 625     } else
 626 #endif
 627     {
 628       if (ShenandoahCASBarrier) {
 629         load_store = kit->gvn().transform(new ShenandoahCompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo));
 630       } else {
 631         load_store = kit->gvn().transform(new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo));
 632       }
 633     }
 634 
 635     access.set_raw_access(load_store);
 636     pin_atomic_op(access);
 637 
 638 #ifdef _LP64
 639     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
 640       load_store = kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
 641     }
 642 #endif
 643     load_store = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, load_store, access.decorators()));

 644     return load_store;
 645   }
 646   return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
 647 }
 648 
 649 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
 650                                                               Node* new_val, const Type* value_type) const {
 651   GraphKit* kit = access.kit();
 652   if (access.is_oop()) {
 653     new_val = shenandoah_iu_barrier(kit, new_val);
 654     shenandoah_write_barrier_pre(kit, false /* do_load */,
 655                                  NULL, NULL, max_juint, NULL, NULL,
 656                                  expected_val /* pre_val */, T_OBJECT);
 657     DecoratorSet decorators = access.decorators();
 658     MemNode::MemOrd mo = access.mem_node_mo();
 659     Node* mem = access.memory();
 660     bool is_weak_cas = (decorators & C2_WEAK_CMPXCHG) != 0;
 661     Node* load_store = NULL;
 662     Node* adr = access.addr().node();
 663 #ifdef _LP64

 679       }
 680     } else
 681 #endif
 682     {
 683       if (ShenandoahCASBarrier) {
 684         if (is_weak_cas) {
 685           load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
 686         } else {
 687           load_store = kit->gvn().transform(new ShenandoahCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
 688         }
 689       } else {
 690         if (is_weak_cas) {
 691           load_store = kit->gvn().transform(new WeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
 692         } else {
 693           load_store = kit->gvn().transform(new CompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
 694         }
 695       }
 696     }
 697     access.set_raw_access(load_store);
 698     pin_atomic_op(access);


 699     return load_store;
 700   }
 701   return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
 702 }
 703 
 704 Node* ShenandoahBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* val, const Type* value_type) const {
 705   GraphKit* kit = access.kit();
 706   if (access.is_oop()) {
 707     val = shenandoah_iu_barrier(kit, val);
 708   }
 709   Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type);
 710   if (access.is_oop()) {
 711     result = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, result, access.decorators()));
 712     shenandoah_write_barrier_pre(kit, false /* do_load */,
 713                                  NULL, NULL, max_juint, NULL, NULL,
 714                                  result /* pre_val */, T_OBJECT);


 715   }
 716   return result;
 717 }
 718 
 719 
 720 bool ShenandoahBarrierSetC2::is_gc_pre_barrier_node(Node* node) const {
 721   return is_shenandoah_wb_pre_call(node);
 722 }
 723 
 724 // Support for GC barriers emitted during parsing
 725 bool ShenandoahBarrierSetC2::is_gc_barrier_node(Node* node) const {
 726   if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier || node->Opcode() == Op_ShenandoahIUBarrier) return true;
 727   if (node->Opcode() != Op_CallLeaf && node->Opcode() != Op_CallLeafNoFP) {
 728     return false;
 729   }
 730   CallLeafNode *call = node->as_CallLeaf();
 731   if (call->_name == NULL) {
 732     return false;
 733   }
 734 

 782   if (phase == Optimization) {
 783     return !ShenandoahIUBarrier;
 784   }
 785   return true;
 786 }
 787 
 788 bool ShenandoahBarrierSetC2::clone_needs_barrier(Node* src, PhaseGVN& gvn) {
 789   const TypeOopPtr* src_type = gvn.type(src)->is_oopptr();
 790   if (src_type->isa_instptr() != NULL) {
 791     ciInstanceKlass* ik = src_type->is_instptr()->instance_klass();
 792     if ((src_type->klass_is_exact() || !ik->has_subklass()) && !ik->has_injected_fields()) {
 793       if (ik->has_object_fields()) {
 794         return true;
 795       } else {
 796         if (!src_type->klass_is_exact()) {
 797           Compile::current()->dependencies()->assert_leaf_type(ik);
 798         }
 799       }
 800     } else {
 801       return true;
 802         }
 803   } else if (src_type->isa_aryptr()) {
 804     BasicType src_elem = src_type->isa_aryptr()->elem()->array_element_basic_type();
 805     if (is_reference_type(src_elem, true)) {
 806       return true;
 807     }
 808   } else {
 809     return true;
 810   }
 811   return false;
 812 }
 813 
 814 void ShenandoahBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
 815   Node* ctrl = ac->in(TypeFunc::Control);
 816   Node* mem = ac->in(TypeFunc::Memory);
 817   Node* src_base = ac->in(ArrayCopyNode::Src);
 818   Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
 819   Node* dest_base = ac->in(ArrayCopyNode::Dest);
 820   Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
 821   Node* length = ac->in(ArrayCopyNode::Length);
 822 

 825 
 826   if (ShenandoahCloneBarrier && clone_needs_barrier(src, phase->igvn())) {
 827     // Check if heap is has forwarded objects. If it does, we need to call into the special
 828     // routine that would fix up source references before we can continue.
 829 
 830     enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
 831     Node* region = new RegionNode(PATH_LIMIT);
 832     Node* mem_phi = new PhiNode(region, Type::MEMORY, TypeRawPtr::BOTTOM);
 833 
 834     Node* thread = phase->transform_later(new ThreadLocalNode());
 835     Node* offset = phase->igvn().MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 836     Node* gc_state_addr = phase->transform_later(new AddPNode(phase->C->top(), thread, offset));
 837 
 838     uint gc_state_idx = Compile::AliasIdxRaw;
 839     const TypePtr* gc_state_adr_type = NULL; // debug-mode-only argument
 840     debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
 841 
 842     Node* gc_state    = phase->transform_later(new LoadBNode(ctrl, mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered));
 843     int flags = ShenandoahHeap::HAS_FORWARDED;
 844     if (ShenandoahIUBarrier) {
 845       flags |= ShenandoahHeap::MARKING;
 846     }
 847     Node* stable_and  = phase->transform_later(new AndINode(gc_state, phase->igvn().intcon(flags)));
 848     Node* stable_cmp  = phase->transform_later(new CmpINode(stable_and, phase->igvn().zerocon(T_INT)));
 849     Node* stable_test = phase->transform_later(new BoolNode(stable_cmp, BoolTest::ne));
 850 
 851     IfNode* stable_iff  = phase->transform_later(new IfNode(ctrl, stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN))->as_If();
 852     Node* stable_ctrl   = phase->transform_later(new IfFalseNode(stable_iff));
 853     Node* unstable_ctrl = phase->transform_later(new IfTrueNode(stable_iff));
 854 
 855     // Heap is stable, no need to do anything additional
 856     region->init_req(_heap_stable, stable_ctrl);
 857     mem_phi->init_req(_heap_stable, mem);
 858 
 859     // Heap is unstable, call into clone barrier stub
 860     Node* call = phase->make_leaf_call(unstable_ctrl, mem,
 861                     ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type(),
 862                     CAST_FROM_FN_PTR(address, ShenandoahRuntime::shenandoah_clone_barrier),
 863                     "shenandoah_clone",
 864                     TypeRawPtr::BOTTOM,
 865                     src_base);

 893 
 894 // Support for macro expanded GC barriers
 895 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
 896   if (node->Opcode() == Op_ShenandoahIUBarrier) {
 897     state()->add_iu_barrier((ShenandoahIUBarrierNode*) node);
 898   }
 899   if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
 900     state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
 901   }
 902 }
 903 
 904 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
 905   if (node->Opcode() == Op_ShenandoahIUBarrier) {
 906     state()->remove_iu_barrier((ShenandoahIUBarrierNode*) node);
 907   }
 908   if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
 909     state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
 910   }
 911 }
 912 
 913 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* n) const {
 914   if (is_shenandoah_wb_pre_call(n)) {
 915     shenandoah_eliminate_wb_pre(n, &macro->igvn());

















 916   }
 917 }
 918 
 919 void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
 920   assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
 921   Node* c = call->as_Call()->proj_out(TypeFunc::Control);
 922   c = c->unique_ctrl_out();
 923   assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
 924   c = c->unique_ctrl_out();
 925   assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
 926   Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
 927   assert(iff->is_If(), "expect test");
 928   if (!is_shenandoah_marking_if(igvn, iff)) {
 929     c = c->unique_ctrl_out();
 930     assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
 931     iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
 932     assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
 933   }
 934   Node* cmpx = iff->in(1)->in(1);
 935   igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ));

  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "classfile/javaClasses.hpp"
  27 #include "gc/shared/barrierSet.hpp"
  28 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  29 #include "gc/shenandoah/shenandoahForwarding.hpp"
  30 #include "gc/shenandoah/shenandoahHeap.hpp"
  31 #include "gc/shenandoah/shenandoahRuntime.hpp"
  32 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
  33 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
  34 #include "gc/shenandoah/c2/shenandoahSupport.hpp"
  35 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
  36 #include "gc/shenandoah/mode/shenandoahMode.hpp"
  37 #include "opto/arraycopynode.hpp"
  38 #include "opto/escape.hpp"
  39 #include "opto/graphKit.hpp"
  40 #include "opto/idealKit.hpp"
  41 #include "opto/macro.hpp"
  42 #include "opto/movenode.hpp"
  43 #include "opto/narrowptrnode.hpp"
  44 #include "opto/rootnode.hpp"
  45 #include "opto/runtime.hpp"
  46 
  47 ShenandoahBarrierSetC2* ShenandoahBarrierSetC2::bsc2() {
  48   return reinterpret_cast<ShenandoahBarrierSetC2*>(BarrierSet::barrier_set()->barrier_set_c2());
  49 }
  50 
  51 ShenandoahBarrierSetC2State::ShenandoahBarrierSetC2State(Arena* comp_arena)
  52   : _iu_barriers(new (comp_arena) GrowableArray<ShenandoahIUBarrierNode*>(comp_arena, 8,  0, NULL)),
  53     _load_reference_barriers(new (comp_arena) GrowableArray<ShenandoahLoadReferenceBarrierNode*>(comp_arena, 8,  0, NULL)) {
  54 }
  55 
  56 int ShenandoahBarrierSetC2State::iu_barriers_count() const {

 225 
 226   Node* no_base = __ top();
 227   Node* zero  = __ ConI(0);
 228   Node* zeroX = __ ConX(0);
 229 
 230   float likely  = PROB_LIKELY(0.999);
 231   float unlikely  = PROB_UNLIKELY(0.999);
 232 
 233   // Offsets into the thread
 234   const int index_offset   = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());
 235   const int buffer_offset  = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
 236 
 237   // Now the actual pointers into the thread
 238   Node* buffer_adr  = __ AddP(no_base, tls, __ ConX(buffer_offset));
 239   Node* index_adr   = __ AddP(no_base, tls, __ ConX(index_offset));
 240 
 241   // Now some of the values
 242   Node* marking;
 243   Node* gc_state = __ AddP(no_base, tls, __ ConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset())));
 244   Node* ld = __ load(__ ctrl(), gc_state, TypeInt::BYTE, T_BYTE, Compile::AliasIdxRaw);
 245   marking = __ AndI(ld, __ ConI(ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING));
 246   assert(ShenandoahBarrierC2Support::is_gc_state_load(ld), "Should match the shape");
 247 
 248   // if (!marking)
 249   __ if_then(marking, BoolTest::ne, zero, unlikely); {
 250     BasicType index_bt = TypeX_X->basic_type();
 251     assert(sizeof(size_t) == type2aelembytes(index_bt), "Loading Shenandoah SATBMarkQueue::_index with wrong size.");
 252     Node* index   = __ load(__ ctrl(), index_adr, TypeX_X, index_bt, Compile::AliasIdxRaw);
 253 
 254     if (do_load) {
 255       // load original value
 256       // alias_idx correct??
 257       pre_val = __ load(__ ctrl(), adr, val_type, bt, alias_idx);
 258     }
 259 
 260     // if (pre_val != NULL)
 261     __ if_then(pre_val, BoolTest::ne, kit->null()); {
 262       Node* buffer  = __ load(__ ctrl(), buffer_adr, TypeRawPtr::NOTNULL, T_ADDRESS, Compile::AliasIdxRaw);
 263 
 264       // is the queue for this thread full?
 265       __ if_then(index, BoolTest::ne, zeroX, likely); {

 306   address entry_point = call->as_CallLeaf()->entry_point();
 307   return (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong)) ||
 308          (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow)) ||
 309          (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak)) ||
 310          (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow)) ||
 311          (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom));
 312 }
 313 
 314 bool ShenandoahBarrierSetC2::is_shenandoah_marking_if(PhaseTransform *phase, Node* n) {
 315   if (n->Opcode() != Op_If) {
 316     return false;
 317   }
 318 
 319   Node* bol = n->in(1);
 320   assert(bol->is_Bool(), "");
 321   Node* cmpx = bol->in(1);
 322   if (bol->as_Bool()->_test._test == BoolTest::ne &&
 323       cmpx->is_Cmp() && cmpx->in(2) == phase->intcon(0) &&
 324       is_shenandoah_state_load(cmpx->in(1)->in(1)) &&
 325       cmpx->in(1)->in(2)->is_Con() &&
 326       cmpx->in(1)->in(2) == phase->intcon(ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING)) {
 327     return true;
 328   }
 329 
 330   return false;
 331 }
 332 
 333 bool ShenandoahBarrierSetC2::is_shenandoah_state_load(Node* n) {
 334   if (!n->is_Load()) return false;
 335   const int state_offset = in_bytes(ShenandoahThreadLocalData::gc_state_offset());
 336   return n->in(2)->is_AddP() && n->in(2)->in(2)->Opcode() == Op_ThreadLocal
 337          && n->in(2)->in(3)->is_Con()
 338          && n->in(2)->in(3)->bottom_type()->is_intptr_t()->get_con() == state_offset;
 339 }
 340 
 341 void ShenandoahBarrierSetC2::shenandoah_write_barrier_pre(GraphKit* kit,
 342                                                           bool do_load,
 343                                                           Node* obj,
 344                                                           Node* adr,
 345                                                           uint alias_idx,
 346                                                           Node* val,

 435         // Use the pre-barrier to record the value in the referent field
 436         satb_write_barrier_pre(kit, false /* do_load */,
 437                                NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
 438                                pre_val /* pre_val */,
 439                                T_OBJECT);
 440         if (need_mem_bar) {
 441           // Add memory barrier to prevent commoning reads from this field
 442           // across safepoint since GC can change its value.
 443           kit->insert_mem_bar(Op_MemBarCPUOrder);
 444         }
 445         // Update IdealKit from graphKit.
 446         __ sync_kit(kit);
 447 
 448       } __ end_if(); // _ref_type != ref_none
 449   } __ end_if(); // offset == referent_offset
 450 
 451   // Final sync IdealKit and GraphKit.
 452   kit->final_sync(ideal);
 453 }
 454 
 455 Node* ShenandoahBarrierSetC2::byte_map_base_node(GraphKit* kit) const {
 456   BarrierSet* bs = BarrierSet::barrier_set();
 457   ShenandoahBarrierSet* ctbs = barrier_set_cast<ShenandoahBarrierSet>(bs);
 458   CardTable::CardValue* card_table_base = ctbs->card_table()->byte_map_base();
 459   if (card_table_base != NULL) {
 460     return kit->makecon(TypeRawPtr::make((address)card_table_base));
 461   } else {
 462     return kit->null();
 463   }
 464 }
 465 
 466 void ShenandoahBarrierSetC2::post_barrier(GraphKit* kit,
 467                                           Node* ctl,
 468                                           Node* oop_store,
 469                                           Node* obj,
 470                                           Node* adr,
 471                                           uint  adr_idx,
 472                                           Node* val,
 473                                           BasicType bt,
 474                                           bool use_precise) const {
 475   if (!ShenandoahHeap::heap()->mode()->is_generational()) {
 476     return;
 477   }
 478 
 479   ShenandoahBarrierSet* ctbs = barrier_set_cast<ShenandoahBarrierSet>(BarrierSet::barrier_set());
 480   CardTable* ct = ctbs->card_table();
 481   // No store check needed if we're storing a NULL or an old object
 482   // (latter case is probably a string constant). The concurrent
 483   // mark sweep garbage collector, however, needs to have all nonNull
 484   // oop updates flagged via card-marks.
 485   if (val != NULL && val->is_Con()) {
 486     // must be either an oop or NULL
 487     const Type* t = val->bottom_type();
 488     if (t == TypePtr::NULL_PTR || t == Type::TOP)
 489       // stores of null never (?) need barriers
 490       return;
 491   }
 492 
 493   if (ReduceInitialCardMarks && obj == kit->just_allocated_object(kit->control())) {
 494     // We can skip marks on a freshly-allocated object in Eden.
 495     // Keep this code in sync with new_deferred_store_barrier() in runtime.cpp.
 496     // That routine informs GC to take appropriate compensating steps,
 497     // upon a slow-path allocation, so as to make this card-mark
 498     // elision safe.
 499     return;
 500   }
 501 
 502   if (!use_precise) {
 503     // All card marks for a (non-array) instance are in one place:
 504     adr = obj;
 505   }
 506   // (Else it's an array (or unknown), and we want more precise card marks.)
 507   assert(adr != NULL, "");
 508 
 509   IdealKit ideal(kit, true);
 510 
 511   // Convert the pointer to an int prior to doing math on it
 512   Node* cast = __ CastPX(__ ctrl(), adr);
 513 
 514   // Divide by card size
 515   Node* card_offset = __ URShiftX( cast, __ ConI(CardTable::card_shift()) );
 516 
 517   // Combine card table base and card offset
 518   Node* card_adr = __ AddP(__ top(), byte_map_base_node(kit), card_offset );
 519 
 520   // Get the alias_index for raw card-mark memory
 521   int adr_type = Compile::AliasIdxRaw;
 522   Node*   zero = __ ConI(0); // Dirty card value
 523 
 524   if (UseCondCardMark) {
 525     // The classic GC reference write barrier is typically implemented
 526     // as a store into the global card mark table.  Unfortunately
 527     // unconditional stores can result in false sharing and excessive
 528     // coherence traffic as well as false transactional aborts.
 529     // UseCondCardMark enables MP "polite" conditional card mark
 530     // stores.  In theory we could relax the load from ctrl() to
 531     // no_ctrl, but that doesn't buy much latitude.
 532     Node* card_val = __ load( __ ctrl(), card_adr, TypeInt::BYTE, T_BYTE, adr_type);
 533     __ if_then(card_val, BoolTest::ne, zero);
 534   }
 535 
 536   // Smash zero into card
 537   __ store(__ ctrl(), card_adr, zero, T_BYTE, adr_type, MemNode::unordered);
 538 
 539   if (UseCondCardMark) {
 540     __ end_if();
 541   }
 542 
 543   // Final sync IdealKit and GraphKit.
 544   kit->final_sync(ideal);
 545 }
 546 
 547 #undef __
 548 
 549 const TypeFunc* ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type() {
 550   const Type **fields = TypeTuple::fields(2);
 551   fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value
 552   fields[TypeFunc::Parms+1] = TypeRawPtr::NOTNULL; // thread
 553   const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
 554 
 555   // create result type (range)
 556   fields = TypeTuple::fields(0);
 557   const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
 558 
 559   return TypeFunc::make(domain, range);
 560 }
 561 
 562 const TypeFunc* ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type() {
 563   const Type **fields = TypeTuple::fields(1);
 564   fields[TypeFunc::Parms+0] = TypeOopPtr::NOTNULL; // src oop
 565   const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);
 566 

 593   Node* adr = access.addr().node();
 594 
 595   bool anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0;
 596   bool on_heap = (decorators & IN_HEAP) != 0;
 597 
 598   if (!access.is_oop() || (!on_heap && !anonymous)) {
 599     return BarrierSetC2::store_at_resolved(access, val);
 600   }
 601 
 602   if (access.is_parse_access()) {
 603     C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
 604     GraphKit* kit = parse_access.kit();
 605 
 606     uint adr_idx = kit->C->get_alias_index(adr_type);
 607     assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" );
 608     Node* value = val.node();
 609     value = shenandoah_iu_barrier(kit, value);
 610     val.set_node(value);
 611     shenandoah_write_barrier_pre(kit, true /* do_load */, /*kit->control(),*/ access.base(), adr, adr_idx, val.node(),
 612                                  static_cast<const TypeOopPtr*>(val.type()), NULL /* pre_val */, access.type());
 613 
 614     Node* result = BarrierSetC2::store_at_resolved(access, val);
 615     bool is_array = (decorators & IS_ARRAY) != 0;
 616     bool use_precise = is_array || anonymous;
 617     post_barrier(kit, kit->control(), access.raw_access(), access.base(), adr, adr_idx, val.node(), access.type(), use_precise);
 618     return result;
 619   } else {
 620     assert(access.is_opt_access(), "only for optimization passes");
 621     assert(((decorators & C2_TIGHTLY_COUPLED_ALLOC) != 0 || !ShenandoahSATBBarrier) && (decorators & C2_ARRAY_COPY) != 0, "unexpected caller of this code");
 622     C2OptAccess& opt_access = static_cast<C2OptAccess&>(access);
 623     PhaseGVN& gvn =  opt_access.gvn();
 624 
 625     if (ShenandoahIUBarrier) {
 626       Node* enqueue = gvn.transform(new ShenandoahIUBarrierNode(val.node()));
 627       val.set_node(enqueue);
 628     }
 629     return BarrierSetC2::store_at_resolved(access, val);
 630   }

 631 }
 632 
 633 Node* ShenandoahBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
 634   // 1: non-reference load, no additional barrier is needed
 635   if (!access.is_oop()) {
 636     return BarrierSetC2::load_at_resolved(access, val_type);;
 637   }
 638 
 639   Node* load = BarrierSetC2::load_at_resolved(access, val_type);
 640   DecoratorSet decorators = access.decorators();
 641   BasicType type = access.type();
 642 
 643   // 2: apply LRB if needed
 644   if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
 645     load = new ShenandoahLoadReferenceBarrierNode(NULL, load, decorators);
 646     if (access.is_parse_access()) {
 647       load = static_cast<C2ParseAccess &>(access).kit()->gvn().transform(load);
 648     } else {
 649       load = static_cast<C2OptAccess &>(access).gvn().transform(load);
 650     }

 681 
 682     if (on_weak_ref) {
 683       // Use the pre-barrier to record the value in the referent field
 684       satb_write_barrier_pre(kit, false /* do_load */,
 685                              NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
 686                              load /* pre_val */, T_OBJECT);
 687       // Add memory barrier to prevent commoning reads from this field
 688       // across safepoint since GC can change its value.
 689       kit->insert_mem_bar(Op_MemBarCPUOrder);
 690     } else if (unknown) {
 691       // We do not require a mem bar inside pre_barrier if need_mem_bar
 692       // is set: the barriers would be emitted by us.
 693       insert_pre_barrier(kit, obj, offset, load, !need_cpu_mem_bar);
 694     }
 695   }
 696 
 697   return load;
 698 }
 699 
 700 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
 701                                                              Node* new_val, const Type* value_type) const {
 702   GraphKit* kit = access.kit();
 703   if (access.is_oop()) {
 704     new_val = shenandoah_iu_barrier(kit, new_val);
 705     shenandoah_write_barrier_pre(kit, false /* do_load */,
 706                                  NULL, NULL, max_juint, NULL, NULL,
 707                                  expected_val /* pre_val */, T_OBJECT);
 708 
 709     MemNode::MemOrd mo = access.mem_node_mo();
 710     Node* mem = access.memory();
 711     Node* adr = access.addr().node();
 712     const TypePtr* adr_type = access.addr().type();
 713     Node* load_store = NULL;
 714 
 715 #ifdef _LP64
 716     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
 717       Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
 718       Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
 719       if (ShenandoahCASBarrier) {
 720         load_store = kit->gvn().transform(new ShenandoahCompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo));
 721       } else {

 723       }
 724     } else
 725 #endif
 726     {
 727       if (ShenandoahCASBarrier) {
 728         load_store = kit->gvn().transform(new ShenandoahCompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo));
 729       } else {
 730         load_store = kit->gvn().transform(new CompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo));
 731       }
 732     }
 733 
 734     access.set_raw_access(load_store);
 735     pin_atomic_op(access);
 736 
 737 #ifdef _LP64
 738     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
 739       load_store = kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
 740     }
 741 #endif
 742     load_store = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, load_store, access.decorators()));
 743     post_barrier(kit, kit->control(), access.raw_access(), access.base(), access.addr().node(), access.alias_idx(), new_val, T_OBJECT, true);
 744     return load_store;
 745   }
 746   return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
 747 }
 748 
 749 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
 750                                                               Node* new_val, const Type* value_type) const {
 751   GraphKit* kit = access.kit();
 752   if (access.is_oop()) {
 753     new_val = shenandoah_iu_barrier(kit, new_val);
 754     shenandoah_write_barrier_pre(kit, false /* do_load */,
 755                                  NULL, NULL, max_juint, NULL, NULL,
 756                                  expected_val /* pre_val */, T_OBJECT);
 757     DecoratorSet decorators = access.decorators();
 758     MemNode::MemOrd mo = access.mem_node_mo();
 759     Node* mem = access.memory();
 760     bool is_weak_cas = (decorators & C2_WEAK_CMPXCHG) != 0;
 761     Node* load_store = NULL;
 762     Node* adr = access.addr().node();
 763 #ifdef _LP64

 779       }
 780     } else
 781 #endif
 782     {
 783       if (ShenandoahCASBarrier) {
 784         if (is_weak_cas) {
 785           load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
 786         } else {
 787           load_store = kit->gvn().transform(new ShenandoahCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
 788         }
 789       } else {
 790         if (is_weak_cas) {
 791           load_store = kit->gvn().transform(new WeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
 792         } else {
 793           load_store = kit->gvn().transform(new CompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));
 794         }
 795       }
 796     }
 797     access.set_raw_access(load_store);
 798     pin_atomic_op(access);
 799     post_barrier(kit, kit->control(), access.raw_access(), access.base(),
 800                  access.addr().node(), access.alias_idx(), new_val, T_OBJECT, true);
 801     return load_store;
 802   }
 803   return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
 804 }
 805 
 806 Node* ShenandoahBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* val, const Type* value_type) const {
 807   GraphKit* kit = access.kit();
 808   if (access.is_oop()) {
 809     val = shenandoah_iu_barrier(kit, val);
 810   }
 811   Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type);
 812   if (access.is_oop()) {
 813     result = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, result, access.decorators()));
 814     shenandoah_write_barrier_pre(kit, false /* do_load */,
 815                                  NULL, NULL, max_juint, NULL, NULL,
 816                                  result /* pre_val */, T_OBJECT);
 817     post_barrier(kit, kit->control(), access.raw_access(), access.base(),
 818                  access.addr().node(), access.alias_idx(), val, T_OBJECT, true);
 819   }
 820   return result;
 821 }
 822 
 823 
 824 bool ShenandoahBarrierSetC2::is_gc_pre_barrier_node(Node* node) const {
 825   return is_shenandoah_wb_pre_call(node);
 826 }
 827 
 828 // Support for GC barriers emitted during parsing
 829 bool ShenandoahBarrierSetC2::is_gc_barrier_node(Node* node) const {
 830   if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier || node->Opcode() == Op_ShenandoahIUBarrier) return true;
 831   if (node->Opcode() != Op_CallLeaf && node->Opcode() != Op_CallLeafNoFP) {
 832     return false;
 833   }
 834   CallLeafNode *call = node->as_CallLeaf();
 835   if (call->_name == NULL) {
 836     return false;
 837   }
 838 

 886   if (phase == Optimization) {
 887     return !ShenandoahIUBarrier;
 888   }
 889   return true;
 890 }
 891 
 892 bool ShenandoahBarrierSetC2::clone_needs_barrier(Node* src, PhaseGVN& gvn) {
 893   const TypeOopPtr* src_type = gvn.type(src)->is_oopptr();
 894   if (src_type->isa_instptr() != NULL) {
 895     ciInstanceKlass* ik = src_type->is_instptr()->instance_klass();
 896     if ((src_type->klass_is_exact() || !ik->has_subklass()) && !ik->has_injected_fields()) {
 897       if (ik->has_object_fields()) {
 898         return true;
 899       } else {
 900         if (!src_type->klass_is_exact()) {
 901           Compile::current()->dependencies()->assert_leaf_type(ik);
 902         }
 903       }
 904     } else {
 905       return true;
 906     }
 907   } else if (src_type->isa_aryptr()) {
 908     BasicType src_elem = src_type->isa_aryptr()->elem()->array_element_basic_type();
 909     if (is_reference_type(src_elem, true)) {
 910       return true;
 911     }
 912   } else {
 913     return true;
 914   }
 915   return false;
 916 }
 917 
 918 void ShenandoahBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
 919   Node* ctrl = ac->in(TypeFunc::Control);
 920   Node* mem = ac->in(TypeFunc::Memory);
 921   Node* src_base = ac->in(ArrayCopyNode::Src);
 922   Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
 923   Node* dest_base = ac->in(ArrayCopyNode::Dest);
 924   Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
 925   Node* length = ac->in(ArrayCopyNode::Length);
 926 

 929 
 930   if (ShenandoahCloneBarrier && clone_needs_barrier(src, phase->igvn())) {
 931     // Check if heap is has forwarded objects. If it does, we need to call into the special
 932     // routine that would fix up source references before we can continue.
 933 
 934     enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
 935     Node* region = new RegionNode(PATH_LIMIT);
 936     Node* mem_phi = new PhiNode(region, Type::MEMORY, TypeRawPtr::BOTTOM);
 937 
 938     Node* thread = phase->transform_later(new ThreadLocalNode());
 939     Node* offset = phase->igvn().MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
 940     Node* gc_state_addr = phase->transform_later(new AddPNode(phase->C->top(), thread, offset));
 941 
 942     uint gc_state_idx = Compile::AliasIdxRaw;
 943     const TypePtr* gc_state_adr_type = NULL; // debug-mode-only argument
 944     debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
 945 
 946     Node* gc_state    = phase->transform_later(new LoadBNode(ctrl, mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered));
 947     int flags = ShenandoahHeap::HAS_FORWARDED;
 948     if (ShenandoahIUBarrier) {
 949       flags |= ShenandoahHeap::YOUNG_MARKING;
 950     }
 951     Node* stable_and  = phase->transform_later(new AndINode(gc_state, phase->igvn().intcon(flags)));
 952     Node* stable_cmp  = phase->transform_later(new CmpINode(stable_and, phase->igvn().zerocon(T_INT)));
 953     Node* stable_test = phase->transform_later(new BoolNode(stable_cmp, BoolTest::ne));
 954 
 955     IfNode* stable_iff  = phase->transform_later(new IfNode(ctrl, stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN))->as_If();
 956     Node* stable_ctrl   = phase->transform_later(new IfFalseNode(stable_iff));
 957     Node* unstable_ctrl = phase->transform_later(new IfTrueNode(stable_iff));
 958 
 959     // Heap is stable, no need to do anything additional
 960     region->init_req(_heap_stable, stable_ctrl);
 961     mem_phi->init_req(_heap_stable, mem);
 962 
 963     // Heap is unstable, call into clone barrier stub
 964     Node* call = phase->make_leaf_call(unstable_ctrl, mem,
 965                     ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type(),
 966                     CAST_FROM_FN_PTR(address, ShenandoahRuntime::shenandoah_clone_barrier),
 967                     "shenandoah_clone",
 968                     TypeRawPtr::BOTTOM,
 969                     src_base);

 997 
 998 // Support for macro expanded GC barriers
 999 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
1000   if (node->Opcode() == Op_ShenandoahIUBarrier) {
1001     state()->add_iu_barrier((ShenandoahIUBarrierNode*) node);
1002   }
1003   if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
1004     state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
1005   }
1006 }
1007 
1008 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
1009   if (node->Opcode() == Op_ShenandoahIUBarrier) {
1010     state()->remove_iu_barrier((ShenandoahIUBarrierNode*) node);
1011   }
1012   if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
1013     state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
1014   }
1015 }
1016 
1017 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const {
1018   if (is_shenandoah_wb_pre_call(node)) {
1019     shenandoah_eliminate_wb_pre(node, &macro->igvn());
1020   }
1021   if (node->Opcode() == Op_CastP2X && ShenandoahHeap::heap()->mode()->is_generational()) {
1022     assert(node->Opcode() == Op_CastP2X, "ConvP2XNode required");
1023      Node *shift = node->unique_out();
1024      Node *addp = shift->unique_out();
1025      for (DUIterator_Last jmin, j = addp->last_outs(jmin); j >= jmin; --j) {
1026        Node *mem = addp->last_out(j);
1027        if (UseCondCardMark && mem->is_Load()) {
1028          assert(mem->Opcode() == Op_LoadB, "unexpected code shape");
1029          // The load is checking if the card has been written so
1030          // replace it with zero to fold the test.
1031          macro->replace_node(mem, macro->intcon(0));
1032          continue;
1033        }
1034        assert(mem->is_Store(), "store required");
1035        macro->replace_node(mem, mem->in(MemNode::Memory));
1036      }
1037   }
1038 }
1039 
1040 void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
1041   assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
1042   Node* c = call->as_Call()->proj_out(TypeFunc::Control);
1043   c = c->unique_ctrl_out();
1044   assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
1045   c = c->unique_ctrl_out();
1046   assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
1047   Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
1048   assert(iff->is_If(), "expect test");
1049   if (!is_shenandoah_marking_if(igvn, iff)) {
1050     c = c->unique_ctrl_out();
1051     assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");
1052     iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);
1053     assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");
1054   }
1055   Node* cmpx = iff->in(1)->in(1);
1056   igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ));
< prev index next >