442 domain_fields[TypeFunc::Parms + 1] = TypeInstPtr::NOTNULL; // dst
443 domain_fields[TypeFunc::Parms + 2] = TypeLong::LONG; // size lower
444 domain_fields[TypeFunc::Parms + 3] = Type::HALF; // size upper
445 const TypeTuple* const domain = TypeTuple::make(TypeFunc::Parms + 4, domain_fields);
446
447 // Create result type (range)
448 const Type** const range_fields = TypeTuple::fields(0);
449 const TypeTuple* const range = TypeTuple::make(TypeFunc::Parms + 0, range_fields);
450
451 return TypeFunc::make(domain, range);
452 }
453
454 #define XTOP LP64_ONLY(COMMA phase->top())
455
456 void ZBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
457 Node* const src = ac->in(ArrayCopyNode::Src);
458 const TypeAryPtr* const ary_ptr = src->get_ptr_type()->isa_aryptr();
459
460 if (ac->is_clone_array() && ary_ptr != nullptr) {
461 BasicType bt = ary_ptr->elem()->array_element_basic_type();
462 if (is_reference_type(bt)) {
463 // Clone object array
464 bt = T_OBJECT;
465 } else {
466 // Clone primitive array
467 bt = T_LONG;
468 }
469
470 Node* const ctrl = ac->in(TypeFunc::Control);
471 Node* const mem = ac->in(TypeFunc::Memory);
472 Node* const src = ac->in(ArrayCopyNode::Src);
473 Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
474 Node* const dest = ac->in(ArrayCopyNode::Dest);
475 Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
476 Node* length = ac->in(ArrayCopyNode::Length);
477
478 if (bt == T_OBJECT) {
479 // BarrierSetC2::clone sets the offsets via BarrierSetC2::arraycopy_payload_base_offset
480 // which 8-byte aligns them to allow for word size copies. Make sure the offsets point
481 // to the first element in the array when cloning object arrays. Otherwise, load
482 // barriers are applied to parts of the header. Also adjust the length accordingly.
483 assert(src_offset == dest_offset, "should be equal");
484 const jlong offset = src_offset->get_long();
485 if (offset != arrayOopDesc::base_offset_in_bytes(T_OBJECT)) {
486 assert(!UseCompressedClassPointers, "should only happen without compressed class pointers");
487 assert((arrayOopDesc::base_offset_in_bytes(T_OBJECT) - offset) == BytesPerLong, "unexpected offset");
488 length = phase->transform_later(new SubLNode(length, phase->longcon(1))); // Size is in longs
489 src_offset = phase->longcon(arrayOopDesc::base_offset_in_bytes(T_OBJECT));
490 dest_offset = src_offset;
491 }
492 }
493 Node* const payload_src = phase->basic_plus_adr(src, src_offset);
494 Node* const payload_dst = phase->basic_plus_adr(dest, dest_offset);
495
496 const char* copyfunc_name = "arraycopy";
497 const address copyfunc_addr = phase->basictype2arraycopy(bt, nullptr, nullptr, true, copyfunc_name, true);
498
499 const TypePtr* const raw_adr_type = TypeRawPtr::BOTTOM;
500 const TypeFunc* const call_type = OptoRuntime::fast_arraycopy_Type();
501
502 Node* const call = phase->make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, payload_src, payload_dst, length XTOP);
503 phase->transform_later(call);
504
505 phase->igvn().replace_node(ac, call);
506 return;
507 }
508
513 Node* const size = ac->in(ArrayCopyNode::Length);
514
515 assert(size->bottom_type()->is_long(), "Should be long");
516
517 // The native clone we are calling here expects the instance size in words
518 // Add header/offset size to payload size to get instance size.
519 Node* const base_offset = phase->longcon(arraycopy_payload_base_offset(ac->is_clone_array()) >> LogBytesPerLong);
520 Node* const full_size = phase->transform_later(new AddLNode(size, base_offset));
521
522 Node* const call = phase->make_leaf_call(ctrl,
523 mem,
524 clone_type(),
525 ZBarrierSetRuntime::clone_addr(),
526 "ZBarrierSetRuntime::clone",
527 TypeRawPtr::BOTTOM,
528 src,
529 dst,
530 full_size,
531 phase->top());
532 phase->transform_later(call);
533 phase->igvn().replace_node(ac, call);
534 }
535
536 #undef XTOP
537
538 // == Dominating barrier elision ==
539
540 static bool block_has_safepoint(const Block* block, uint from, uint to) {
541 for (uint i = from; i < to; i++) {
542 if (block->get_node(i)->is_MachSafePoint()) {
543 // Safepoint found
544 return true;
545 }
546 }
547
548 // Safepoint not found
549 return false;
550 }
551
552 static bool block_has_safepoint(const Block* block) {
553 return block_has_safepoint(block, 0, block->number_of_nodes());
928 // If this node tracks liveness, update it
929 RegMask* const regs = barrier_set_state()->live(node);
930 if (regs != nullptr) {
931 regs->OR(new_live);
932 }
933 }
934
935 // Now at block top, see if we have any changes
936 new_live.SUBTRACT(old_live);
937 if (new_live.is_NotEmpty()) {
938 // Liveness has refined, update and propagate to prior blocks
939 old_live.OR(new_live);
940 for (uint i = 1; i < block->num_preds(); ++i) {
941 Block* const pred = cfg->get_block_for_node(block->pred(i));
942 worklist.push(pred);
943 }
944 }
945 }
946 }
947
948 void ZBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const {
949 eliminate_gc_barrier_data(node);
950 }
951
952 void ZBarrierSetC2::eliminate_gc_barrier_data(Node* node) const {
953 if (node->is_LoadStore()) {
954 LoadStoreNode* loadstore = node->as_LoadStore();
955 loadstore->set_barrier_data(ZBarrierElided);
956 } else if (node->is_Mem()) {
957 MemNode* mem = node->as_Mem();
958 mem->set_barrier_data(ZBarrierElided);
959 }
960 }
961
962 #ifndef PRODUCT
963 void ZBarrierSetC2::dump_barrier_data(const MachNode* mach, outputStream* st) const {
964 if ((mach->barrier_data() & ZBarrierStrong) != 0) {
965 st->print("strong ");
966 }
967 if ((mach->barrier_data() & ZBarrierWeak) != 0) {
968 st->print("weak ");
|
442 domain_fields[TypeFunc::Parms + 1] = TypeInstPtr::NOTNULL; // dst
443 domain_fields[TypeFunc::Parms + 2] = TypeLong::LONG; // size lower
444 domain_fields[TypeFunc::Parms + 3] = Type::HALF; // size upper
445 const TypeTuple* const domain = TypeTuple::make(TypeFunc::Parms + 4, domain_fields);
446
447 // Create result type (range)
448 const Type** const range_fields = TypeTuple::fields(0);
449 const TypeTuple* const range = TypeTuple::make(TypeFunc::Parms + 0, range_fields);
450
451 return TypeFunc::make(domain, range);
452 }
453
454 #define XTOP LP64_ONLY(COMMA phase->top())
455
456 void ZBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
457 Node* const src = ac->in(ArrayCopyNode::Src);
458 const TypeAryPtr* const ary_ptr = src->get_ptr_type()->isa_aryptr();
459
460 if (ac->is_clone_array() && ary_ptr != nullptr) {
461 BasicType bt = ary_ptr->elem()->array_element_basic_type();
462 if (is_reference_type(bt) && !ary_ptr->is_flat()) {
463 // Clone object array
464 bt = T_OBJECT;
465 } else {
466 // Clone primitive array
467 bt = T_LONG;
468 }
469
470 Node* const ctrl = ac->in(TypeFunc::Control);
471 Node* const mem = ac->in(TypeFunc::Memory);
472 Node* const src = ac->in(ArrayCopyNode::Src);
473 Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
474 Node* const dest = ac->in(ArrayCopyNode::Dest);
475 Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
476 Node* length = ac->in(ArrayCopyNode::Length);
477
478 if (bt == T_OBJECT) {
479 // BarrierSetC2::clone sets the offsets via BarrierSetC2::arraycopy_payload_base_offset
480 // which 8-byte aligns them to allow for word size copies. Make sure the offsets point
481 // to the first element in the array when cloning object arrays. Otherwise, load
482 // barriers are applied to parts of the header. Also adjust the length accordingly.
483 assert(src_offset == dest_offset, "should be equal");
484 const jlong offset = src_offset->get_long();
485 if (offset != arrayOopDesc::base_offset_in_bytes(T_OBJECT)) {
486 assert(!UseCompressedClassPointers, "should only happen without compressed class pointers");
487 assert((arrayOopDesc::base_offset_in_bytes(T_OBJECT) - offset) == BytesPerLong, "unexpected offset");
488 length = phase->transform_later(new SubXNode(length, phase->longcon(1))); // Size is in longs
489 src_offset = phase->longcon(arrayOopDesc::base_offset_in_bytes(T_OBJECT));
490 dest_offset = src_offset;
491 }
492 }
493 Node* const payload_src = phase->basic_plus_adr(src, src_offset);
494 Node* const payload_dst = phase->basic_plus_adr(dest, dest_offset);
495
496 const char* copyfunc_name = "arraycopy";
497 const address copyfunc_addr = phase->basictype2arraycopy(bt, nullptr, nullptr, true, copyfunc_name, true);
498
499 const TypePtr* const raw_adr_type = TypeRawPtr::BOTTOM;
500 const TypeFunc* const call_type = OptoRuntime::fast_arraycopy_Type();
501
502 Node* const call = phase->make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, payload_src, payload_dst, length XTOP);
503 phase->transform_later(call);
504
505 phase->igvn().replace_node(ac, call);
506 return;
507 }
508
513 Node* const size = ac->in(ArrayCopyNode::Length);
514
515 assert(size->bottom_type()->is_long(), "Should be long");
516
517 // The native clone we are calling here expects the instance size in words
518 // Add header/offset size to payload size to get instance size.
519 Node* const base_offset = phase->longcon(arraycopy_payload_base_offset(ac->is_clone_array()) >> LogBytesPerLong);
520 Node* const full_size = phase->transform_later(new AddLNode(size, base_offset));
521
522 Node* const call = phase->make_leaf_call(ctrl,
523 mem,
524 clone_type(),
525 ZBarrierSetRuntime::clone_addr(),
526 "ZBarrierSetRuntime::clone",
527 TypeRawPtr::BOTTOM,
528 src,
529 dst,
530 full_size,
531 phase->top());
532 phase->transform_later(call);
533 phase->replace_node(ac, call);
534 }
535
536 #undef XTOP
537
538 // == Dominating barrier elision ==
539
540 static bool block_has_safepoint(const Block* block, uint from, uint to) {
541 for (uint i = from; i < to; i++) {
542 if (block->get_node(i)->is_MachSafePoint()) {
543 // Safepoint found
544 return true;
545 }
546 }
547
548 // Safepoint not found
549 return false;
550 }
551
552 static bool block_has_safepoint(const Block* block) {
553 return block_has_safepoint(block, 0, block->number_of_nodes());
928 // If this node tracks liveness, update it
929 RegMask* const regs = barrier_set_state()->live(node);
930 if (regs != nullptr) {
931 regs->OR(new_live);
932 }
933 }
934
935 // Now at block top, see if we have any changes
936 new_live.SUBTRACT(old_live);
937 if (new_live.is_NotEmpty()) {
938 // Liveness has refined, update and propagate to prior blocks
939 old_live.OR(new_live);
940 for (uint i = 1; i < block->num_preds(); ++i) {
941 Block* const pred = cfg->get_block_for_node(block->pred(i));
942 worklist.push(pred);
943 }
944 }
945 }
946 }
947
948 void ZBarrierSetC2::eliminate_gc_barrier(PhaseIterGVN* igvn, Node* node) const {
949 eliminate_gc_barrier_data(node);
950 }
951
952 void ZBarrierSetC2::eliminate_gc_barrier_data(Node* node) const {
953 if (node->is_LoadStore()) {
954 LoadStoreNode* loadstore = node->as_LoadStore();
955 loadstore->set_barrier_data(ZBarrierElided);
956 } else if (node->is_Mem()) {
957 MemNode* mem = node->as_Mem();
958 mem->set_barrier_data(ZBarrierElided);
959 }
960 }
961
962 #ifndef PRODUCT
963 void ZBarrierSetC2::dump_barrier_data(const MachNode* mach, outputStream* st) const {
964 if ((mach->barrier_data() & ZBarrierStrong) != 0) {
965 st->print("strong ");
966 }
967 if ((mach->barrier_data() & ZBarrierWeak) != 0) {
968 st->print("weak ");
|