401 for (uint j = 0; j < n->req(); j++) {
402 Node* in = n->in(j);
403 if (in != nullptr) {
404 wq.push(in);
405 }
406 }
407 }
408 }
409
410 // Support for macro expanded GC barriers
411 void ShenandoahBarrierSetC2::eliminate_gc_barrier_data(Node* node) const {
412 if (node->is_LoadStore()) {
413 LoadStoreNode* loadstore = node->as_LoadStore();
414 loadstore->set_barrier_data(0);
415 } else if (node->is_Mem()) {
416 MemNode* mem = node->as_Mem();
417 mem->set_barrier_data(0);
418 }
419 }
420
421 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const {
422 eliminate_gc_barrier_data(node);
423 }
424
425 void ShenandoahBarrierSetC2::elide_dominated_barrier(MachNode* node, MachNode* dominator) const {
426 uint8_t orig_bd = node->barrier_data();
427 if (orig_bd == 0) {
428 // Nothing to do.
429 return;
430 }
431
432 uint8_t bd = orig_bd;
433 int node_opcode = node->ideal_Opcode();
434
435 if (dominator == nullptr) {
436 // Must be allocation node.
437 if (is_Load(node_opcode) || is_LoadStore(node_opcode)) {
438 // Loads from recent allocations do not need LRBs.
439 bd &= ~ShenandoahBitStrong;
440 }
441 if (is_Store(node_opcode) || is_LoadStore(node_opcode)) {
540 bool ShenandoahBarrierSetC2::clone_needs_barrier(const TypeOopPtr* src_type, bool& is_oop_array) {
541 if (!ShenandoahCloneBarrier) {
542 return false;
543 }
544
545 if (src_type->isa_instptr() != nullptr) {
546 // Instance: need barrier only if there is a possibility of having an oop anywhere in it.
547 ciInstanceKlass* ik = src_type->is_instptr()->instance_klass();
548 if ((src_type->klass_is_exact() || !ik->has_subklass()) &&
549 !ik->has_injected_fields() && !ik->has_object_fields()) {
550 if (!src_type->klass_is_exact()) {
551 // Class is *currently* the leaf in the hierarchy.
552 // Record the dependency so that we deopt if this does not hold in future.
553 Compile::current()->dependencies()->assert_leaf_type(ik);
554 }
555 return false;
556 }
557 } else if (src_type->isa_aryptr() != nullptr) {
558 // Array: need barrier only if array is oop-bearing.
559 BasicType src_elem = src_type->isa_aryptr()->elem()->array_element_basic_type();
560 if (is_reference_type(src_elem, true)) {
561 is_oop_array = true;
562 } else {
563 return false;
564 }
565 }
566
567 // Assume the worst.
568 return true;
569 }
570
571 void ShenandoahBarrierSetC2::clone(GraphKit* kit, Node* src_base, Node* dst_base, Node* size, bool is_array) const {
572 const TypeOopPtr* src_type = kit->gvn().type(src_base)->is_oopptr();
573
574 bool is_oop_array = false;
575 if (!clone_needs_barrier(src_type, is_oop_array)) {
576 // No barrier is needed? Just do what common BarrierSetC2 wants with it.
577 BarrierSetC2::clone(kit, src_base, dst_base, size, is_array);
578 return;
579 }
580
581 if (ShenandoahCloneRuntime || !is_array || !is_oop_array) {
619 // No barrier is needed? Expand to normal HeapWord-sized arraycopy.
620 BarrierSetC2::clone_at_expansion(phase, ac);
621 return;
622 }
623
624 if (ShenandoahCloneRuntime || !ac->is_clone_array() || !is_oop_array) {
625 // Still looks like an instance? Likely a large instance or reflective
626 // clone with unknown length. Go to runtime and handle it there.
627 clone_in_runtime(phase, ac, ShenandoahRuntime::clone_addr(), "ShenandoahRuntime::clone");
628 return;
629 }
630
631 // We are cloning the oop array. Call into normal oop array copy stubs.
632 // Those stubs would call BarrierSetAssembler to handle GC barriers.
633
634 // This is the full clone, so offsets should equal each other and be at array base.
635 assert(src_offset == dest_offset, "should be equal");
636 const jlong offset = src_offset->get_long();
637 const TypeAryPtr* const ary_ptr = src->get_ptr_type()->isa_aryptr();
638 BasicType bt = ary_ptr->elem()->array_element_basic_type();
639 assert(offset == arrayOopDesc::base_offset_in_bytes(bt), "should match");
640
641 const char* copyfunc_name = "arraycopy";
642 const address copyfunc_addr = phase->basictype2arraycopy(T_OBJECT, nullptr, nullptr, true, copyfunc_name, true);
643
644 Node* const call = phase->make_leaf_call(ctrl, mem,
645 OptoRuntime::fast_arraycopy_Type(),
646 copyfunc_addr, copyfunc_name,
647 TypeRawPtr::BOTTOM,
648 phase->basic_plus_adr(src, src_offset),
649 phase->basic_plus_adr(dest, dest_offset),
650 length,
651 phase->top()
652 );
653 phase->transform_later(call);
654
655 phase->igvn().replace_node(ac, call);
656 }
657
658 void* ShenandoahBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
|
401 for (uint j = 0; j < n->req(); j++) {
402 Node* in = n->in(j);
403 if (in != nullptr) {
404 wq.push(in);
405 }
406 }
407 }
408 }
409
410 // Support for macro expanded GC barriers
411 void ShenandoahBarrierSetC2::eliminate_gc_barrier_data(Node* node) const {
412 if (node->is_LoadStore()) {
413 LoadStoreNode* loadstore = node->as_LoadStore();
414 loadstore->set_barrier_data(0);
415 } else if (node->is_Mem()) {
416 MemNode* mem = node->as_Mem();
417 mem->set_barrier_data(0);
418 }
419 }
420
421 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseIterGVN* macro, Node* node) const {
422 eliminate_gc_barrier_data(node);
423 }
424
425 void ShenandoahBarrierSetC2::elide_dominated_barrier(MachNode* node, MachNode* dominator) const {
426 uint8_t orig_bd = node->barrier_data();
427 if (orig_bd == 0) {
428 // Nothing to do.
429 return;
430 }
431
432 uint8_t bd = orig_bd;
433 int node_opcode = node->ideal_Opcode();
434
435 if (dominator == nullptr) {
436 // Must be allocation node.
437 if (is_Load(node_opcode) || is_LoadStore(node_opcode)) {
438 // Loads from recent allocations do not need LRBs.
439 bd &= ~ShenandoahBitStrong;
440 }
441 if (is_Store(node_opcode) || is_LoadStore(node_opcode)) {
540 bool ShenandoahBarrierSetC2::clone_needs_barrier(const TypeOopPtr* src_type, bool& is_oop_array) {
541 if (!ShenandoahCloneBarrier) {
542 return false;
543 }
544
545 if (src_type->isa_instptr() != nullptr) {
546 // Instance: need barrier only if there is a possibility of having an oop anywhere in it.
547 ciInstanceKlass* ik = src_type->is_instptr()->instance_klass();
548 if ((src_type->klass_is_exact() || !ik->has_subklass()) &&
549 !ik->has_injected_fields() && !ik->has_object_fields()) {
550 if (!src_type->klass_is_exact()) {
551 // Class is *currently* the leaf in the hierarchy.
552 // Record the dependency so that we deopt if this does not hold in future.
553 Compile::current()->dependencies()->assert_leaf_type(ik);
554 }
555 return false;
556 }
557 } else if (src_type->isa_aryptr() != nullptr) {
558 // Array: need barrier only if array is oop-bearing.
559 BasicType src_elem = src_type->isa_aryptr()->elem()->array_element_basic_type();
560 if (is_reference_type(src_elem, true) && src_type->is_not_flat()) {
561 is_oop_array = true;
562 } else if (!src_type->is_not_flat()) {
563 // Maybe flat, assume the worst.
564 } else {
565 return false;
566 }
567 }
568
569 // Assume the worst.
570 return true;
571 }
572
573 void ShenandoahBarrierSetC2::clone(GraphKit* kit, Node* src_base, Node* dst_base, Node* size, bool is_array) const {
574 const TypeOopPtr* src_type = kit->gvn().type(src_base)->is_oopptr();
575
576 bool is_oop_array = false;
577 if (!clone_needs_barrier(src_type, is_oop_array)) {
578 // No barrier is needed? Just do what common BarrierSetC2 wants with it.
579 BarrierSetC2::clone(kit, src_base, dst_base, size, is_array);
580 return;
581 }
582
583 if (ShenandoahCloneRuntime || !is_array || !is_oop_array) {
621 // No barrier is needed? Expand to normal HeapWord-sized arraycopy.
622 BarrierSetC2::clone_at_expansion(phase, ac);
623 return;
624 }
625
626 if (ShenandoahCloneRuntime || !ac->is_clone_array() || !is_oop_array) {
627 // Still looks like an instance? Likely a large instance or reflective
628 // clone with unknown length. Go to runtime and handle it there.
629 clone_in_runtime(phase, ac, ShenandoahRuntime::clone_addr(), "ShenandoahRuntime::clone");
630 return;
631 }
632
633 // We are cloning the oop array. Call into normal oop array copy stubs.
634 // Those stubs would call BarrierSetAssembler to handle GC barriers.
635
636 // This is the full clone, so offsets should equal each other and be at array base.
637 assert(src_offset == dest_offset, "should be equal");
638 const jlong offset = src_offset->get_long();
639 const TypeAryPtr* const ary_ptr = src->get_ptr_type()->isa_aryptr();
640 BasicType bt = ary_ptr->elem()->array_element_basic_type();
641 if (offset != arrayOopDesc::base_offset_in_bytes(bt)) {
642 // Something is off with flat arrays. Go to runtime instead.
643 // TODO: Figure this out.
644 clone_in_runtime(phase, ac, ShenandoahRuntime::clone_addr(), "ShenandoahRuntime::clone");
645 return;
646 }
647 assert(offset == arrayOopDesc::base_offset_in_bytes(bt), "should match");
648
649 const char* copyfunc_name = "arraycopy";
650 const address copyfunc_addr = phase->basictype2arraycopy(T_OBJECT, nullptr, nullptr, true, copyfunc_name, true);
651
652 Node* const call = phase->make_leaf_call(ctrl, mem,
653 OptoRuntime::fast_arraycopy_Type(),
654 copyfunc_addr, copyfunc_name,
655 TypeRawPtr::BOTTOM,
656 phase->basic_plus_adr(src, src_offset),
657 phase->basic_plus_adr(dest, dest_offset),
658 length,
659 phase->top()
660 );
661 phase->transform_later(call);
662
663 phase->igvn().replace_node(ac, call);
664 }
665
666 void* ShenandoahBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
|