456 domain_fields[TypeFunc::Parms + 1] = TypeInstPtr::NOTNULL; // dst
457 domain_fields[TypeFunc::Parms + 2] = TypeLong::LONG; // size lower
458 domain_fields[TypeFunc::Parms + 3] = Type::HALF; // size upper
459 const TypeTuple* const domain = TypeTuple::make(TypeFunc::Parms + 4, domain_fields);
460
461 // Create result type (range)
462 const Type** const range_fields = TypeTuple::fields(0);
463 const TypeTuple* const range = TypeTuple::make(TypeFunc::Parms + 0, range_fields);
464
465 return TypeFunc::make(domain, range);
466 }
467
468 #define XTOP LP64_ONLY(COMMA phase->top())
469
470 void ZBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
471 Node* const src = ac->in(ArrayCopyNode::Src);
472 const TypeAryPtr* const ary_ptr = src->get_ptr_type()->isa_aryptr();
473
474 if (ac->is_clone_array() && ary_ptr != nullptr) {
475 BasicType bt = ary_ptr->elem()->array_element_basic_type();
476 if (is_reference_type(bt)) {
477 // Clone object array
478 bt = T_OBJECT;
479 } else {
480 // Clone primitive array
481 bt = T_LONG;
482 }
483
484 Node* const ctrl = ac->in(TypeFunc::Control);
485 Node* const mem = ac->in(TypeFunc::Memory);
486 Node* const src = ac->in(ArrayCopyNode::Src);
487 Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
488 Node* const dest = ac->in(ArrayCopyNode::Dest);
489 Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
490 Node* length = ac->in(ArrayCopyNode::Length);
491
492 if (bt == T_OBJECT) {
493 // BarrierSetC2::clone sets the offsets via BarrierSetC2::arraycopy_payload_base_offset
494 // which 8-byte aligns them to allow for word size copies. Make sure the offsets point
495 // to the first element in the array when cloning object arrays. Otherwise, load
496 // barriers are applied to parts of the header. Also adjust the length accordingly.
497 assert(src_offset == dest_offset, "should be equal");
498 const jlong offset = src_offset->get_long();
499 if (offset != arrayOopDesc::base_offset_in_bytes(T_OBJECT)) {
500 assert(!UseCompressedClassPointers, "should only happen without compressed class pointers");
501 assert((arrayOopDesc::base_offset_in_bytes(T_OBJECT) - offset) == BytesPerLong, "unexpected offset");
502 length = phase->transform_later(new SubLNode(length, phase->longcon(1))); // Size is in longs
503 src_offset = phase->longcon(arrayOopDesc::base_offset_in_bytes(T_OBJECT));
504 dest_offset = src_offset;
505 }
506 }
507 Node* const payload_src = phase->basic_plus_adr(src, src_offset);
508 Node* const payload_dst = phase->basic_plus_adr(dest, dest_offset);
509
510 const char* copyfunc_name = "arraycopy";
511 const address copyfunc_addr = phase->basictype2arraycopy(bt, nullptr, nullptr, true, copyfunc_name, true);
512
513 const TypePtr* const raw_adr_type = TypeRawPtr::BOTTOM;
514 const TypeFunc* const call_type = OptoRuntime::fast_arraycopy_Type();
515
516 Node* const call = phase->make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, payload_src, payload_dst, length XTOP);
517 phase->transform_later(call);
518
519 phase->igvn().replace_node(ac, call);
520 return;
521 }
522
527 Node* const size = ac->in(ArrayCopyNode::Length);
528
529 assert(size->bottom_type()->is_long(), "Should be long");
530
531 // The native clone we are calling here expects the instance size in words
532 // Add header/offset size to payload size to get instance size.
533 Node* const base_offset = phase->longcon(arraycopy_payload_base_offset(ac->is_clone_array()) >> LogBytesPerLong);
534 Node* const full_size = phase->transform_later(new AddLNode(size, base_offset));
535
536 Node* const call = phase->make_leaf_call(ctrl,
537 mem,
538 clone_type(),
539 ZBarrierSetRuntime::clone_addr(),
540 "ZBarrierSetRuntime::clone",
541 TypeRawPtr::BOTTOM,
542 src,
543 dst,
544 full_size,
545 phase->top());
546 phase->transform_later(call);
547 phase->igvn().replace_node(ac, call);
548 }
549
550 #undef XTOP
551
552 // == Dominating barrier elision ==
553
554 static bool block_has_safepoint(const Block* block, uint from, uint to) {
555 for (uint i = from; i < to; i++) {
556 if (block->get_node(i)->is_MachSafePoint()) {
557 // Safepoint found
558 return true;
559 }
560 }
561
562 // Safepoint not found
563 return false;
564 }
565
566 static bool block_has_safepoint(const Block* block) {
567 return block_has_safepoint(block, 0, block->number_of_nodes());
942 // If this node tracks liveness, update it
943 RegMask* const regs = barrier_set_state()->live(node);
944 if (regs != nullptr) {
945 regs->OR(new_live);
946 }
947 }
948
949 // Now at block top, see if we have any changes
950 new_live.SUBTRACT(old_live);
951 if (new_live.is_NotEmpty()) {
952 // Liveness has refined, update and propagate to prior blocks
953 old_live.OR(new_live);
954 for (uint i = 1; i < block->num_preds(); ++i) {
955 Block* const pred = cfg->get_block_for_node(block->pred(i));
956 worklist.push(pred);
957 }
958 }
959 }
960 }
961
962 void ZBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const {
963 eliminate_gc_barrier_data(node);
964 }
965
966 void ZBarrierSetC2::eliminate_gc_barrier_data(Node* node) const {
967 if (node->is_LoadStore()) {
968 LoadStoreNode* loadstore = node->as_LoadStore();
969 loadstore->set_barrier_data(ZBarrierElided);
970 } else if (node->is_Mem()) {
971 MemNode* mem = node->as_Mem();
972 mem->set_barrier_data(ZBarrierElided);
973 }
974 }
975
976 #ifndef PRODUCT
977 void ZBarrierSetC2::dump_barrier_data(const MachNode* mach, outputStream* st) const {
978 if ((mach->barrier_data() & ZBarrierStrong) != 0) {
979 st->print("strong ");
980 }
981 if ((mach->barrier_data() & ZBarrierWeak) != 0) {
982 st->print("weak ");
|
456 domain_fields[TypeFunc::Parms + 1] = TypeInstPtr::NOTNULL; // dst
457 domain_fields[TypeFunc::Parms + 2] = TypeLong::LONG; // size lower
458 domain_fields[TypeFunc::Parms + 3] = Type::HALF; // size upper
459 const TypeTuple* const domain = TypeTuple::make(TypeFunc::Parms + 4, domain_fields);
460
461 // Create result type (range)
462 const Type** const range_fields = TypeTuple::fields(0);
463 const TypeTuple* const range = TypeTuple::make(TypeFunc::Parms + 0, range_fields);
464
465 return TypeFunc::make(domain, range);
466 }
467
468 #define XTOP LP64_ONLY(COMMA phase->top())
469
470 void ZBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
471 Node* const src = ac->in(ArrayCopyNode::Src);
472 const TypeAryPtr* const ary_ptr = src->get_ptr_type()->isa_aryptr();
473
474 if (ac->is_clone_array() && ary_ptr != nullptr) {
475 BasicType bt = ary_ptr->elem()->array_element_basic_type();
476 if (is_reference_type(bt) && !ary_ptr->is_flat()) {
477 // Clone object array
478 bt = T_OBJECT;
479 } else {
480 // Clone primitive array
481 bt = T_LONG;
482 }
483
484 Node* const ctrl = ac->in(TypeFunc::Control);
485 Node* const mem = ac->in(TypeFunc::Memory);
486 Node* const src = ac->in(ArrayCopyNode::Src);
487 Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
488 Node* const dest = ac->in(ArrayCopyNode::Dest);
489 Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
490 Node* length = ac->in(ArrayCopyNode::Length);
491
492 if (bt == T_OBJECT) {
493 // BarrierSetC2::clone sets the offsets via BarrierSetC2::arraycopy_payload_base_offset
494 // which 8-byte aligns them to allow for word size copies. Make sure the offsets point
495 // to the first element in the array when cloning object arrays. Otherwise, load
496 // barriers are applied to parts of the header. Also adjust the length accordingly.
497 assert(src_offset == dest_offset, "should be equal");
498 const jlong offset = src_offset->get_long();
499 if (offset != arrayOopDesc::base_offset_in_bytes(T_OBJECT)) {
500 assert(!UseCompressedClassPointers, "should only happen without compressed class pointers");
501 assert((arrayOopDesc::base_offset_in_bytes(T_OBJECT) - offset) == BytesPerLong, "unexpected offset");
502 length = phase->transform_later(new SubXNode(length, phase->longcon(1))); // Size is in longs
503 src_offset = phase->longcon(arrayOopDesc::base_offset_in_bytes(T_OBJECT));
504 dest_offset = src_offset;
505 }
506 }
507 Node* const payload_src = phase->basic_plus_adr(src, src_offset);
508 Node* const payload_dst = phase->basic_plus_adr(dest, dest_offset);
509
510 const char* copyfunc_name = "arraycopy";
511 const address copyfunc_addr = phase->basictype2arraycopy(bt, nullptr, nullptr, true, copyfunc_name, true);
512
513 const TypePtr* const raw_adr_type = TypeRawPtr::BOTTOM;
514 const TypeFunc* const call_type = OptoRuntime::fast_arraycopy_Type();
515
516 Node* const call = phase->make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, payload_src, payload_dst, length XTOP);
517 phase->transform_later(call);
518
519 phase->igvn().replace_node(ac, call);
520 return;
521 }
522
527 Node* const size = ac->in(ArrayCopyNode::Length);
528
529 assert(size->bottom_type()->is_long(), "Should be long");
530
531 // The native clone we are calling here expects the instance size in words
532 // Add header/offset size to payload size to get instance size.
533 Node* const base_offset = phase->longcon(arraycopy_payload_base_offset(ac->is_clone_array()) >> LogBytesPerLong);
534 Node* const full_size = phase->transform_later(new AddLNode(size, base_offset));
535
536 Node* const call = phase->make_leaf_call(ctrl,
537 mem,
538 clone_type(),
539 ZBarrierSetRuntime::clone_addr(),
540 "ZBarrierSetRuntime::clone",
541 TypeRawPtr::BOTTOM,
542 src,
543 dst,
544 full_size,
545 phase->top());
546 phase->transform_later(call);
547 phase->replace_node(ac, call);
548 }
549
550 #undef XTOP
551
552 // == Dominating barrier elision ==
553
554 static bool block_has_safepoint(const Block* block, uint from, uint to) {
555 for (uint i = from; i < to; i++) {
556 if (block->get_node(i)->is_MachSafePoint()) {
557 // Safepoint found
558 return true;
559 }
560 }
561
562 // Safepoint not found
563 return false;
564 }
565
566 static bool block_has_safepoint(const Block* block) {
567 return block_has_safepoint(block, 0, block->number_of_nodes());
942 // If this node tracks liveness, update it
943 RegMask* const regs = barrier_set_state()->live(node);
944 if (regs != nullptr) {
945 regs->OR(new_live);
946 }
947 }
948
949 // Now at block top, see if we have any changes
950 new_live.SUBTRACT(old_live);
951 if (new_live.is_NotEmpty()) {
952 // Liveness has refined, update and propagate to prior blocks
953 old_live.OR(new_live);
954 for (uint i = 1; i < block->num_preds(); ++i) {
955 Block* const pred = cfg->get_block_for_node(block->pred(i));
956 worklist.push(pred);
957 }
958 }
959 }
960 }
961
962 void ZBarrierSetC2::eliminate_gc_barrier(PhaseIterGVN* igvn, Node* node) const {
963 eliminate_gc_barrier_data(node);
964 }
965
966 void ZBarrierSetC2::eliminate_gc_barrier_data(Node* node) const {
967 if (node->is_LoadStore()) {
968 LoadStoreNode* loadstore = node->as_LoadStore();
969 loadstore->set_barrier_data(ZBarrierElided);
970 } else if (node->is_Mem()) {
971 MemNode* mem = node->as_Mem();
972 mem->set_barrier_data(ZBarrierElided);
973 }
974 }
975
976 #ifndef PRODUCT
977 void ZBarrierSetC2::dump_barrier_data(const MachNode* mach, outputStream* st) const {
978 if ((mach->barrier_data() & ZBarrierStrong) != 0) {
979 st->print("strong ");
980 }
981 if ((mach->barrier_data() & ZBarrierWeak) != 0) {
982 st->print("weak ");
|