404 bool is_clone, bool is_clone_instance,
405 ArrayCopyPhase phase) const {
406 if (phase == ArrayCopyPhase::Parsing) {
407 return false;
408 }
409 if (phase == ArrayCopyPhase::Optimization) {
410 return is_clone_instance;
411 }
412 // else ArrayCopyPhase::Expansion
413 return type == T_OBJECT || type == T_ARRAY;
414 }
415
416 #define XTOP LP64_ONLY(COMMA phase->top())
417
418 void ZBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
419 Node* const src = ac->in(ArrayCopyNode::Src);
420 const TypeAryPtr* const ary_ptr = src->get_ptr_type()->isa_aryptr();
421
422 if (ac->is_clone_array() && ary_ptr != nullptr) {
423 BasicType bt = ary_ptr->elem()->array_element_basic_type();
424 if (is_reference_type(bt)) {
425 // Clone object array
426 bt = T_OBJECT;
427 } else {
428 // Clone primitive array
429 bt = T_LONG;
430 }
431
432 Node* const ctrl = ac->in(TypeFunc::Control);
433 Node* const mem = ac->in(TypeFunc::Memory);
434 Node* const src = ac->in(ArrayCopyNode::Src);
435 Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
436 Node* const dest = ac->in(ArrayCopyNode::Dest);
437 Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
438 Node* length = ac->in(ArrayCopyNode::Length);
439
440 if (bt == T_OBJECT) {
441 // BarrierSetC2::clone sets the offsets via BarrierSetC2::arraycopy_payload_base_offset
442 // which 8-byte aligns them to allow for word size copies. Make sure the offsets point
443 // to the first element in the array when cloning object arrays. Otherwise, load
444 // barriers are applied to parts of the header. Also adjust the length accordingly.
445 assert(src_offset == dest_offset, "should be equal");
446 const jlong offset = src_offset->get_long();
447 if (offset != arrayOopDesc::base_offset_in_bytes(T_OBJECT)) {
448 assert(!UseCompressedClassPointers, "should only happen without compressed class pointers");
449 assert((arrayOopDesc::base_offset_in_bytes(T_OBJECT) - offset) == BytesPerLong, "unexpected offset");
450 length = phase->transform_later(new SubLNode(length, phase->longcon(1))); // Size is in longs
451 src_offset = phase->longcon(arrayOopDesc::base_offset_in_bytes(T_OBJECT));
452 dest_offset = src_offset;
453 }
454 }
455 Node* const payload_src = phase->basic_plus_adr(src, src_offset);
456 Node* const payload_dst = phase->basic_plus_adr(dest, dest_offset);
457
458 const char* copyfunc_name = "arraycopy";
459 const address copyfunc_addr = phase->basictype2arraycopy(bt, nullptr, nullptr, true, copyfunc_name, true);
460
461 const TypePtr* const raw_adr_type = TypeRawPtr::BOTTOM;
462 const TypeFunc* const call_type = OptoRuntime::fast_arraycopy_Type();
463
464 Node* const call = phase->make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, payload_src, payload_dst, length XTOP);
465 phase->transform_later(call);
466
467 phase->igvn().replace_node(ac, call);
468 return;
469 }
470
794 if (mach->barrier_data() != 0) {
795 atomics.push(mach);
796 load_dominators.push(mach);
797 store_dominators.push(mach);
798 atomic_dominators.push(mach);
799 }
800 break;
801
802 default:
803 break;
804 }
805 }
806 }
807
808 // Step 2 - Find dominating accesses or allocations for each access
809 analyze_dominating_barriers_impl(loads, load_dominators);
810 analyze_dominating_barriers_impl(stores, store_dominators);
811 analyze_dominating_barriers_impl(atomics, atomic_dominators);
812 }
813
814 void ZBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const {
815 eliminate_gc_barrier_data(node);
816 }
817
818 void ZBarrierSetC2::eliminate_gc_barrier_data(Node* node) const {
819 if (node->is_LoadStore()) {
820 LoadStoreNode* loadstore = node->as_LoadStore();
821 loadstore->set_barrier_data(ZBarrierElided);
822 } else if (node->is_Mem()) {
823 MemNode* mem = node->as_Mem();
824 mem->set_barrier_data(ZBarrierElided);
825 }
826 }
827
828 #ifndef PRODUCT
829 void ZBarrierSetC2::dump_barrier_data(const MachNode* mach, outputStream* st) const {
830 if ((mach->barrier_data() & ZBarrierStrong) != 0) {
831 st->print("strong ");
832 }
833 if ((mach->barrier_data() & ZBarrierWeak) != 0) {
834 st->print("weak ");
|
404 bool is_clone, bool is_clone_instance,
405 ArrayCopyPhase phase) const {
406 if (phase == ArrayCopyPhase::Parsing) {
407 return false;
408 }
409 if (phase == ArrayCopyPhase::Optimization) {
410 return is_clone_instance;
411 }
412 // else ArrayCopyPhase::Expansion
413 return type == T_OBJECT || type == T_ARRAY;
414 }
415
416 #define XTOP LP64_ONLY(COMMA phase->top())
417
418 void ZBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
419 Node* const src = ac->in(ArrayCopyNode::Src);
420 const TypeAryPtr* const ary_ptr = src->get_ptr_type()->isa_aryptr();
421
422 if (ac->is_clone_array() && ary_ptr != nullptr) {
423 BasicType bt = ary_ptr->elem()->array_element_basic_type();
424 if (is_reference_type(bt) && !ary_ptr->is_flat()) {
425 // Clone object array
426 bt = T_OBJECT;
427 } else {
428 // Clone primitive array
429 bt = T_LONG;
430 }
431
432 Node* const ctrl = ac->in(TypeFunc::Control);
433 Node* const mem = ac->in(TypeFunc::Memory);
434 Node* const src = ac->in(ArrayCopyNode::Src);
435 Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
436 Node* const dest = ac->in(ArrayCopyNode::Dest);
437 Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
438 Node* length = ac->in(ArrayCopyNode::Length);
439
440 if (bt == T_OBJECT) {
441 // BarrierSetC2::clone sets the offsets via BarrierSetC2::arraycopy_payload_base_offset
442 // which 8-byte aligns them to allow for word size copies. Make sure the offsets point
443 // to the first element in the array when cloning object arrays. Otherwise, load
444 // barriers are applied to parts of the header. Also adjust the length accordingly.
445 assert(src_offset == dest_offset, "should be equal");
446 const jlong offset = src_offset->get_long();
447 if (offset != arrayOopDesc::base_offset_in_bytes(T_OBJECT)) {
448 assert(!UseCompressedClassPointers, "should only happen without compressed class pointers");
449 assert((arrayOopDesc::base_offset_in_bytes(T_OBJECT) - offset) == BytesPerLong, "unexpected offset");
450 length = phase->transform_later(new SubXNode(length, phase->longcon(1))); // Size is in longs
451 src_offset = phase->longcon(arrayOopDesc::base_offset_in_bytes(T_OBJECT));
452 dest_offset = src_offset;
453 }
454 }
455 Node* const payload_src = phase->basic_plus_adr(src, src_offset);
456 Node* const payload_dst = phase->basic_plus_adr(dest, dest_offset);
457
458 const char* copyfunc_name = "arraycopy";
459 const address copyfunc_addr = phase->basictype2arraycopy(bt, nullptr, nullptr, true, copyfunc_name, true);
460
461 const TypePtr* const raw_adr_type = TypeRawPtr::BOTTOM;
462 const TypeFunc* const call_type = OptoRuntime::fast_arraycopy_Type();
463
464 Node* const call = phase->make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, payload_src, payload_dst, length XTOP);
465 phase->transform_later(call);
466
467 phase->igvn().replace_node(ac, call);
468 return;
469 }
470
794 if (mach->barrier_data() != 0) {
795 atomics.push(mach);
796 load_dominators.push(mach);
797 store_dominators.push(mach);
798 atomic_dominators.push(mach);
799 }
800 break;
801
802 default:
803 break;
804 }
805 }
806 }
807
808 // Step 2 - Find dominating accesses or allocations for each access
809 analyze_dominating_barriers_impl(loads, load_dominators);
810 analyze_dominating_barriers_impl(stores, store_dominators);
811 analyze_dominating_barriers_impl(atomics, atomic_dominators);
812 }
813
814
815 void ZBarrierSetC2::eliminate_gc_barrier(PhaseIterGVN* igvn, Node* node) const {
816 eliminate_gc_barrier_data(node);
817 }
818
819 void ZBarrierSetC2::eliminate_gc_barrier_data(Node* node) const {
820 if (node->is_LoadStore()) {
821 LoadStoreNode* loadstore = node->as_LoadStore();
822 loadstore->set_barrier_data(ZBarrierElided);
823 } else if (node->is_Mem()) {
824 MemNode* mem = node->as_Mem();
825 mem->set_barrier_data(ZBarrierElided);
826 }
827 }
828
829 #ifndef PRODUCT
830 void ZBarrierSetC2::dump_barrier_data(const MachNode* mach, outputStream* st) const {
831 if ((mach->barrier_data() & ZBarrierStrong) != 0) {
832 st->print("strong ");
833 }
834 if ((mach->barrier_data() & ZBarrierWeak) != 0) {
835 st->print("weak ");
|