681 }
682
683 Node* BarrierSetC2::atomic_xchg_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
684 C2AccessFence fence(access);
685 resolve_address(access);
686 return atomic_xchg_at_resolved(access, new_val, value_type);
687 }
688
689 Node* BarrierSetC2::atomic_add_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
690 C2AccessFence fence(access);
691 resolve_address(access);
692 return atomic_add_at_resolved(access, new_val, value_type);
693 }
694
695 int BarrierSetC2::arraycopy_payload_base_offset(bool is_array) {
696 // Exclude the header but include array length to copy by 8 bytes words.
697 // Can't use base_offset_in_bytes(bt) since basic type is unknown.
698 int base_off = is_array ? arrayOopDesc::length_offset_in_bytes() :
699 instanceOopDesc::base_offset_in_bytes();
700 // base_off:
701 // 8 - 32-bit VM or 64-bit VM, compact headers
702 // 12 - 64-bit VM, compressed klass
703 // 16 - 64-bit VM, normal klass
704 if (base_off % BytesPerLong != 0) {
705 assert(!UseCompactObjectHeaders, "");
706 if (is_array) {
707 // Exclude length to copy by 8 bytes words.
708 base_off += sizeof(int);
709 } else {
710 // Include klass to copy by 8 bytes words.
711 base_off = instanceOopDesc::klass_offset_in_bytes();
712 }
713 assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment");
714 }
715 return base_off;
716 }
717
718 void BarrierSetC2::clone(GraphKit* kit, Node* src_base, Node* dst_base, Node* size, bool is_array) const {
719 int base_off = arraycopy_payload_base_offset(is_array);
720 Node* payload_size = size;
721 Node* offset = kit->MakeConX(base_off);
722 payload_size = kit->gvn().transform(new SubXNode(payload_size, offset));
723 if (is_array) {
724 // Ensure the array payload size is rounded up to the next BytesPerLong
725 // multiple when converting to double-words. This is necessary because array
726 // size does not include object alignment padding, so it might not be a
727 // multiple of BytesPerLong for sub-long element types.
728 payload_size = kit->gvn().transform(new AddXNode(payload_size, kit->MakeConX(BytesPerLong - 1)));
729 }
730 payload_size = kit->gvn().transform(new URShiftXNode(payload_size, kit->intcon(LogBytesPerLong)));
731 ArrayCopyNode* ac = ArrayCopyNode::make(kit, false, src_base, offset, dst_base, offset, payload_size, true, false);
732 if (is_array) {
733 ac->set_clone_array();
734 } else {
735 ac->set_clone_inst();
736 }
737 Node* n = kit->gvn().transform(ac);
738 if (n == ac) {
739 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
831 inline const TypeFunc* BarrierSetC2::clone_type() {
832 assert(BarrierSetC2::_clone_type_Type != nullptr, "should be initialized");
833 return BarrierSetC2::_clone_type_Type;
834 }
835
836 #define XTOP LP64_ONLY(COMMA phase->top())
837
838 void BarrierSetC2::clone_in_runtime(PhaseMacroExpand* phase, ArrayCopyNode* ac,
839 address clone_addr, const char* clone_name) const {
840 Node* const ctrl = ac->in(TypeFunc::Control);
841 Node* const mem = ac->in(TypeFunc::Memory);
842 Node* const src = ac->in(ArrayCopyNode::Src);
843 Node* const dst = ac->in(ArrayCopyNode::Dest);
844 Node* const size = ac->in(ArrayCopyNode::Length);
845
846 assert(size->bottom_type()->base() == Type_X,
847 "Should be of object size type (int for 32 bits, long for 64 bits)");
848
849 // The native clone we are calling here expects the object size in words.
850 // Add header/offset size to payload size to get object size.
851 Node* const base_offset = phase->MakeConX(arraycopy_payload_base_offset(ac->is_clone_array()) >> LogBytesPerLong);
852 Node* const full_size = phase->transform_later(new AddXNode(size, base_offset));
853 // HeapAccess<>::clone expects size in heap words.
854 // For 64-bits platforms, this is a no-operation.
855 // For 32-bits platforms, we need to multiply full_size by HeapWordsPerLong (2).
856 Node* const full_size_in_heap_words = phase->transform_later(new LShiftXNode(full_size, phase->intcon(LogHeapWordsPerLong)));
857
858 Node* const call = phase->make_leaf_call(ctrl,
859 mem,
860 clone_type(),
861 clone_addr,
862 clone_name,
863 TypeRawPtr::BOTTOM,
864 src, dst, full_size_in_heap_words XTOP);
865 phase->transform_later(call);
866 phase->igvn().replace_node(ac, call);
867 }
868
869 void BarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
870 Node* ctrl = ac->in(TypeFunc::Control);
871 Node* mem = ac->in(TypeFunc::Memory);
872 Node* src = ac->in(ArrayCopyNode::Src);
873 Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
874 Node* dest = ac->in(ArrayCopyNode::Dest);
875 Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
876 Node* length = ac->in(ArrayCopyNode::Length);
877
878 Node* payload_src = phase->basic_plus_adr(src, src_offset);
879 Node* payload_dst = phase->basic_plus_adr(dest, dest_offset);
880
881 const char* copyfunc_name = "arraycopy";
882 address copyfunc_addr = phase->basictype2arraycopy(T_LONG, nullptr, nullptr, true, copyfunc_name, true);
883
884 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
885 const TypeFunc* call_type = OptoRuntime::fast_arraycopy_Type();
886
887 Node* call = phase->make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, payload_src, payload_dst, length XTOP);
888 phase->transform_later(call);
889
890 phase->igvn().replace_node(ac, call);
891 }
892
893 #undef XTOP
894
895 static bool block_has_safepoint(const Block* block, uint from, uint to) {
896 for (uint i = from; i < to; i++) {
897 if (block->get_node(i)->is_MachSafePoint()) {
898 // Safepoint found
899 return true;
900 }
901 }
902
903 // Safepoint not found
904 return false;
905 }
906
907 static bool block_has_safepoint(const Block* block) {
908 return block_has_safepoint(block, 0, block->number_of_nodes());
909 }
910
911 static uint block_index(const Block* block, const Node* node) {
912 for (uint j = 0; j < block->number_of_nodes(); ++j) {
|
681 }
682
683 Node* BarrierSetC2::atomic_xchg_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
684 C2AccessFence fence(access);
685 resolve_address(access);
686 return atomic_xchg_at_resolved(access, new_val, value_type);
687 }
688
689 Node* BarrierSetC2::atomic_add_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
690 C2AccessFence fence(access);
691 resolve_address(access);
692 return atomic_add_at_resolved(access, new_val, value_type);
693 }
694
695 int BarrierSetC2::arraycopy_payload_base_offset(bool is_array) {
696 // Exclude the header but include array length to copy by 8 bytes words.
697 // Can't use base_offset_in_bytes(bt) since basic type is unknown.
698 int base_off = is_array ? arrayOopDesc::length_offset_in_bytes() :
699 instanceOopDesc::base_offset_in_bytes();
700 // base_off:
701 // 4 - compact headers
702 // 8 - 32-bit VM
703 // 12 - 64-bit VM, compressed klass
704 // 16 - 64-bit VM, normal klass
705 if (base_off % BytesPerLong != 0) {
706 if (is_array) {
707 // Exclude length to copy by 8 bytes words.
708 base_off += sizeof(int);
709 } else {
710 if (!UseCompactObjectHeaders) {
711 // Include klass to copy by 8 bytes words.
712 base_off = instanceOopDesc::klass_offset_in_bytes();
713 }
714 }
715 assert(base_off % BytesPerLong == 0 || UseCompactObjectHeaders, "expect 8 bytes alignment");
716 }
717 return base_off;
718 }
719
720 void BarrierSetC2::clone(GraphKit* kit, Node* src_base, Node* dst_base, Node* size, bool is_array) const {
721 int base_off = arraycopy_payload_base_offset(is_array);
722
723 Node* payload_size = size;
724 Node* offset = kit->MakeConX(base_off);
725 payload_size = kit->gvn().transform(new SubXNode(payload_size, offset));
726 if (is_array) {
727 // Ensure the array payload size is rounded up to the next BytesPerLong
728 // multiple when converting to double-words. This is necessary because array
729 // size does not include object alignment padding, so it might not be a
730 // multiple of BytesPerLong for sub-long element types.
731 payload_size = kit->gvn().transform(new AddXNode(payload_size, kit->MakeConX(BytesPerLong - 1)));
732 }
733 payload_size = kit->gvn().transform(new URShiftXNode(payload_size, kit->intcon(LogBytesPerLong)));
734 ArrayCopyNode* ac = ArrayCopyNode::make(kit, false, src_base, offset, dst_base, offset, payload_size, true, false);
735 if (is_array) {
736 ac->set_clone_array();
737 } else {
738 ac->set_clone_inst();
739 }
740 Node* n = kit->gvn().transform(ac);
741 if (n == ac) {
742 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
834 inline const TypeFunc* BarrierSetC2::clone_type() {
835 assert(BarrierSetC2::_clone_type_Type != nullptr, "should be initialized");
836 return BarrierSetC2::_clone_type_Type;
837 }
838
839 #define XTOP LP64_ONLY(COMMA phase->top())
840
841 void BarrierSetC2::clone_in_runtime(PhaseMacroExpand* phase, ArrayCopyNode* ac,
842 address clone_addr, const char* clone_name) const {
843 Node* const ctrl = ac->in(TypeFunc::Control);
844 Node* const mem = ac->in(TypeFunc::Memory);
845 Node* const src = ac->in(ArrayCopyNode::Src);
846 Node* const dst = ac->in(ArrayCopyNode::Dest);
847 Node* const size = ac->in(ArrayCopyNode::Length);
848
849 assert(size->bottom_type()->base() == Type_X,
850 "Should be of object size type (int for 32 bits, long for 64 bits)");
851
852 // The native clone we are calling here expects the object size in words.
853 // Add header/offset size to payload size to get object size.
854
855 // We need the full object size - payload (already aligned) plus base offset (which is not always aligned, so round *up*),
856 // because clone_in_runtime copies the whole object from 0 to end.
857 Node* const base_offset = phase->MakeConX((arraycopy_payload_base_offset(ac->is_clone_array()) + (BytesPerLong - 1)) >> LogBytesPerLong);
858 Node* const full_size = phase->transform_later(new AddXNode(size, base_offset));
859
860 // HeapAccess<>::clone expects size in heap words.
861 // For 64-bits platforms, this is a no-operation.
862 // For 32-bits platforms, we need to multiply full_size by HeapWordsPerLong (2).
863 Node* const full_size_in_heap_words = phase->transform_later(new LShiftXNode(full_size, phase->intcon(LogHeapWordsPerLong)));
864
865 Node* const call = phase->make_leaf_call(ctrl,
866 mem,
867 clone_type(),
868 clone_addr,
869 clone_name,
870 TypeRawPtr::BOTTOM,
871 src, dst, full_size_in_heap_words XTOP);
872 phase->transform_later(call);
873 phase->igvn().replace_node(ac, call);
874 }
875
876 void BarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
877 Node* ctrl = ac->in(TypeFunc::Control);
878 Node* mem = ac->in(TypeFunc::Memory);
879 Node* src = ac->in(ArrayCopyNode::Src);
880 Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
881 Node* dest = ac->in(ArrayCopyNode::Dest);
882 Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
883 Node* length = ac->in(ArrayCopyNode::Length);
884
885 Node* payload_src = phase->basic_plus_adr(src, src_offset);
886 Node* payload_dst = phase->basic_plus_adr(dest, dest_offset);
887
888 if (should_copy_int_prefix(phase, ac)) {
889 mem = arraycopy_copy_int_prefix(phase, ctrl, mem, payload_src, payload_dst);
890
891 // We've copied the prefix, bump the pointers.
892 payload_src = phase->basic_plus_adr(src, payload_src, BytesPerInt);
893 payload_dst = phase->basic_plus_adr(dest, payload_dst, BytesPerInt);
894 }
895
896 // Bulk copy.
897 const char* copyfunc_name = "arraycopy";
898 address copyfunc_addr = phase->basictype2arraycopy(T_LONG, nullptr, nullptr, true, copyfunc_name, true);
899
900 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
901 const TypeFunc* call_type = OptoRuntime::fast_arraycopy_Type();
902
903 Node* call = phase->make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, payload_src, payload_dst, length XTOP);
904 phase->transform_later(call);
905
906 phase->igvn().replace_node(ac, call);
907 }
908
909 bool BarrierSetC2::should_copy_int_prefix(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
910 // We do our bulk copy in longs. If base offset is not aligned, then we must copy the prefix separately.
911 // With CompactObjectHeaders, the base offset for an instance is 4 bytes.
912 // We cannot simply expand the copy to the previous long-alignment, as that will copy the object header,
913 // which is stateful with COH - it contains hash and lock bits that are specific to the instance.
914
915 // Skip this when src has an array type. With StressReflectiveCode, the
916 // instance path of the clone can be live in the IR even when the type system
917 // knows src is an array. The pre-copy is unnecessary on such paths (they
918 // are unreachable at runtime), and creating a LoadNode at the array length
919 // offset would assert (LoadRangeNode required).
920 Node* src = ac->in(ArrayCopyNode::Src);
921 if (phase->igvn().type(src)->isa_aryptr()) {
922 return false;
923 }
924
925 int base_off = arraycopy_payload_base_offset(ac->is_clone_array());
926 if (is_aligned(base_off, BytesPerLong)) {
927 // We're aligned, no need to copy anything separately.
928 return false;
929 }
930
931 assert(UseCompactObjectHeaders, "non-aligned base offset only possible with compact object headers");
932 assert(is_aligned(base_off, BytesPerInt), "must be 4-bytes aligned");
933 return true;
934 }
935
936 MergeMemNode* BarrierSetC2::arraycopy_copy_int_prefix(PhaseMacroExpand* phase, Node* ctrl, Node* mem, Node* src, Node* dst) const {
937 // Manual load/store of one int.
938 MergeMemNode* mm = phase->transform_later(MergeMemNode::make(mem))->as_MergeMem();
939 const TypePtr* s_adr_type = phase->igvn().type(src)->is_ptr();
940 const TypePtr* d_adr_type = phase->igvn().type(dst)->is_ptr();
941 uint s_alias_idx = phase->C->get_alias_index(s_adr_type);
942 uint d_alias_idx = phase->C->get_alias_index(d_adr_type);
943 // This copies the first 4 bytes after the compact header (hash field or first instance field) as a raw int.
944 // The actual field at this offset may be a narrowOop, so the load/store must be marked as mismatched to
945 // avoid StoreN-vs-StoreI assertion failures during IGVN.
946 Node* load_prefix = phase->transform_later(
947 LoadNode::make(phase->igvn(), ctrl, mm->memory_at(s_alias_idx), src, s_adr_type,
948 TypeInt::INT, T_INT, MemNode::unordered, LoadNode::DependsOnlyOnTest,
949 false /*require_atomic_access*/, false /*unaligned*/, true /*mismatched*/));
950 Node* store_prefix = phase->transform_later(
951 StoreNode::make(phase->igvn(), ctrl, mm->memory_at(d_alias_idx), dst, d_adr_type,
952 load_prefix, T_INT, MemNode::unordered));
953 store_prefix->as_Store()->set_mismatched_access();
954 mm->set_memory_at(d_alias_idx, store_prefix);
955 return mm;
956 }
957
958 #undef XTOP
959
960 static bool block_has_safepoint(const Block* block, uint from, uint to) {
961 for (uint i = from; i < to; i++) {
962 if (block->get_node(i)->is_MachSafePoint()) {
963 // Safepoint found
964 return true;
965 }
966 }
967
968 // Safepoint not found
969 return false;
970 }
971
972 static bool block_has_safepoint(const Block* block) {
973 return block_has_safepoint(block, 0, block->number_of_nodes());
974 }
975
976 static uint block_index(const Block* block, const Node* node) {
977 for (uint j = 0; j < block->number_of_nodes(); ++j) {
|