687 }
688
689 Node* BarrierSetC2::atomic_xchg_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
690 C2AccessFence fence(access);
691 resolve_address(access);
692 return atomic_xchg_at_resolved(access, new_val, value_type);
693 }
694
695 Node* BarrierSetC2::atomic_add_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
696 C2AccessFence fence(access);
697 resolve_address(access);
698 return atomic_add_at_resolved(access, new_val, value_type);
699 }
700
701 int BarrierSetC2::arraycopy_payload_base_offset(bool is_array) {
702 // Exclude the header but include array length to copy by 8 bytes words.
703 // Can't use base_offset_in_bytes(bt) since basic type is unknown.
704 int base_off = is_array ? arrayOopDesc::length_offset_in_bytes() :
705 instanceOopDesc::base_offset_in_bytes();
706 // base_off:
707 // 8 - 32-bit VM or 64-bit VM, compact headers
708 // 12 - 64-bit VM, compressed klass
709 // 16 - 64-bit VM, normal klass
710 if (base_off % BytesPerLong != 0) {
711 assert(!UseCompactObjectHeaders, "");
712 if (is_array) {
713 // Exclude length to copy by 8 bytes words.
714 base_off += sizeof(int);
715 } else {
716 // Include klass to copy by 8 bytes words.
717 base_off = instanceOopDesc::klass_offset_in_bytes();
718 }
719 assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment");
720 }
721 return base_off;
722 }
723
724 void BarrierSetC2::clone(GraphKit* kit, Node* src_base, Node* dst_base, Node* size, bool is_array) const {
725 int base_off = arraycopy_payload_base_offset(is_array);
726 Node* payload_size = size;
727 Node* offset = kit->MakeConX(base_off);
728 payload_size = kit->gvn().transform(new SubXNode(payload_size, offset));
729 if (is_array) {
730 // Ensure the array payload size is rounded up to the next BytesPerLong
731 // multiple when converting to double-words. This is necessary because array
732 // size does not include object alignment padding, so it might not be a
733 // multiple of BytesPerLong for sub-long element types.
734 payload_size = kit->gvn().transform(new AddXNode(payload_size, kit->MakeConX(BytesPerLong - 1)));
735 }
736 payload_size = kit->gvn().transform(new URShiftXNode(payload_size, kit->intcon(LogBytesPerLong)));
737 ArrayCopyNode* ac = ArrayCopyNode::make(kit, false, src_base, offset, dst_base, offset, payload_size, true, false);
738 if (is_array) {
739 ac->set_clone_array();
740 } else {
741 ac->set_clone_inst();
742 }
743 Node* n = kit->gvn().transform(ac);
744 if (n == ac) {
745 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
837 inline const TypeFunc* BarrierSetC2::clone_type() {
838 assert(BarrierSetC2::_clone_type_Type != nullptr, "should be initialized");
839 return BarrierSetC2::_clone_type_Type;
840 }
841
842 #define XTOP LP64_ONLY(COMMA phase->top())
843
844 void BarrierSetC2::clone_in_runtime(PhaseMacroExpand* phase, ArrayCopyNode* ac,
845 address clone_addr, const char* clone_name) const {
846 Node* const ctrl = ac->in(TypeFunc::Control);
847 Node* const mem = ac->in(TypeFunc::Memory);
848 Node* const src = ac->in(ArrayCopyNode::Src);
849 Node* const dst = ac->in(ArrayCopyNode::Dest);
850 Node* const size = ac->in(ArrayCopyNode::Length);
851
852 assert(size->bottom_type()->base() == Type_X,
853 "Should be of object size type (int for 32 bits, long for 64 bits)");
854
855 // The native clone we are calling here expects the object size in words.
856 // Add header/offset size to payload size to get object size.
857 Node* const base_offset = phase->MakeConX(arraycopy_payload_base_offset(ac->is_clone_array()) >> LogBytesPerLong);
858 Node* const full_size = phase->transform_later(new AddXNode(size, base_offset));
859 // HeapAccess<>::clone expects size in heap words.
860 // For 64-bits platforms, this is a no-operation.
861 // For 32-bits platforms, we need to multiply full_size by HeapWordsPerLong (2).
862 Node* const full_size_in_heap_words = phase->transform_later(new LShiftXNode(full_size, phase->intcon(LogHeapWordsPerLong)));
863
864 Node* const call = phase->make_leaf_call(ctrl,
865 mem,
866 clone_type(),
867 clone_addr,
868 clone_name,
869 TypeRawPtr::BOTTOM,
870 src, dst, full_size_in_heap_words XTOP);
871 phase->transform_later(call);
872 phase->igvn().replace_node(ac, call);
873 }
874
875 void BarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
876 Node* ctrl = ac->in(TypeFunc::Control);
877 Node* mem = ac->in(TypeFunc::Memory);
878 Node* src = ac->in(ArrayCopyNode::Src);
879 Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
880 Node* dest = ac->in(ArrayCopyNode::Dest);
881 Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
882 Node* length = ac->in(ArrayCopyNode::Length);
883
884 Node* payload_src = phase->basic_plus_adr(src, src_offset);
885 Node* payload_dst = phase->basic_plus_adr(dest, dest_offset);
886
887 const char* copyfunc_name = "arraycopy";
888 address copyfunc_addr = phase->basictype2arraycopy(T_LONG, nullptr, nullptr, true, copyfunc_name, true);
889
890 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
891 const TypeFunc* call_type = OptoRuntime::fast_arraycopy_Type();
892
893 Node* call = phase->make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, payload_src, payload_dst, length XTOP);
894 phase->transform_later(call);
895
896 phase->igvn().replace_node(ac, call);
897 }
898
899 #undef XTOP
900
901 static bool block_has_safepoint(const Block* block, uint from, uint to) {
902 for (uint i = from; i < to; i++) {
903 if (block->get_node(i)->is_MachSafePoint()) {
904 // Safepoint found
905 return true;
906 }
907 }
908
909 // Safepoint not found
910 return false;
911 }
912
913 static bool block_has_safepoint(const Block* block) {
914 return block_has_safepoint(block, 0, block->number_of_nodes());
915 }
916
917 static uint block_index(const Block* block, const Node* node) {
918 for (uint j = 0; j < block->number_of_nodes(); ++j) {
|
687 }
688
689 Node* BarrierSetC2::atomic_xchg_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
690 C2AccessFence fence(access);
691 resolve_address(access);
692 return atomic_xchg_at_resolved(access, new_val, value_type);
693 }
694
695 Node* BarrierSetC2::atomic_add_at(C2AtomicParseAccess& access, Node* new_val, const Type* value_type) const {
696 C2AccessFence fence(access);
697 resolve_address(access);
698 return atomic_add_at_resolved(access, new_val, value_type);
699 }
700
701 int BarrierSetC2::arraycopy_payload_base_offset(bool is_array) {
702 // Exclude the header but include array length to copy by 8 bytes words.
703 // Can't use base_offset_in_bytes(bt) since basic type is unknown.
704 int base_off = is_array ? arrayOopDesc::length_offset_in_bytes() :
705 instanceOopDesc::base_offset_in_bytes();
706 // base_off:
707 // 4 - compact headers
708 // 8 - 32-bit VM
709 // 12 - 64-bit VM, compressed klass
710 // 16 - 64-bit VM, normal klass
711 if (base_off % BytesPerLong != 0) {
712 if (is_array) {
713 // Exclude length to copy by 8 bytes words.
714 base_off += sizeof(int);
715 } else {
716 if (!UseCompactObjectHeaders) {
717 // Include klass to copy by 8 bytes words.
718 base_off = instanceOopDesc::klass_offset_in_bytes();
719 }
720 }
721 assert(base_off % BytesPerLong == 0 || UseCompactObjectHeaders, "expect 8 bytes alignment");
722 }
723 return base_off;
724 }
725
726 void BarrierSetC2::clone(GraphKit* kit, Node* src_base, Node* dst_base, Node* size, bool is_array) const {
727 int base_off = arraycopy_payload_base_offset(is_array);
728
729 Node* payload_size = size;
730 Node* offset = kit->MakeConX(base_off);
731 payload_size = kit->gvn().transform(new SubXNode(payload_size, offset));
732 if (is_array) {
733 // Ensure the array payload size is rounded up to the next BytesPerLong
734 // multiple when converting to double-words. This is necessary because array
735 // size does not include object alignment padding, so it might not be a
736 // multiple of BytesPerLong for sub-long element types.
737 payload_size = kit->gvn().transform(new AddXNode(payload_size, kit->MakeConX(BytesPerLong - 1)));
738 }
739 payload_size = kit->gvn().transform(new URShiftXNode(payload_size, kit->intcon(LogBytesPerLong)));
740 ArrayCopyNode* ac = ArrayCopyNode::make(kit, false, src_base, offset, dst_base, offset, payload_size, true, false);
741 if (is_array) {
742 ac->set_clone_array();
743 } else {
744 ac->set_clone_inst();
745 }
746 Node* n = kit->gvn().transform(ac);
747 if (n == ac) {
748 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
840 inline const TypeFunc* BarrierSetC2::clone_type() {
841 assert(BarrierSetC2::_clone_type_Type != nullptr, "should be initialized");
842 return BarrierSetC2::_clone_type_Type;
843 }
844
845 #define XTOP LP64_ONLY(COMMA phase->top())
846
847 void BarrierSetC2::clone_in_runtime(PhaseMacroExpand* phase, ArrayCopyNode* ac,
848 address clone_addr, const char* clone_name) const {
849 Node* const ctrl = ac->in(TypeFunc::Control);
850 Node* const mem = ac->in(TypeFunc::Memory);
851 Node* const src = ac->in(ArrayCopyNode::Src);
852 Node* const dst = ac->in(ArrayCopyNode::Dest);
853 Node* const size = ac->in(ArrayCopyNode::Length);
854
855 assert(size->bottom_type()->base() == Type_X,
856 "Should be of object size type (int for 32 bits, long for 64 bits)");
857
858 // The native clone we are calling here expects the object size in words.
859 // Add header/offset size to payload size to get object size.
860
861 // We need the full object size - payload (already aligned) plus base offset (which is not always aligned, so round *up*),
862 // because clone_in_runtime copies the whole object from 0 to end.
863 Node* const base_offset = phase->MakeConX((arraycopy_payload_base_offset(ac->is_clone_array()) + (BytesPerLong - 1)) >> LogBytesPerLong);
864 Node* const full_size = phase->transform_later(new AddXNode(size, base_offset));
865
866 // HeapAccess<>::clone expects size in heap words.
867 // For 64-bits platforms, this is a no-operation.
868 // For 32-bits platforms, we need to multiply full_size by HeapWordsPerLong (2).
869 Node* const full_size_in_heap_words = phase->transform_later(new LShiftXNode(full_size, phase->intcon(LogHeapWordsPerLong)));
870
871 Node* const call = phase->make_leaf_call(ctrl,
872 mem,
873 clone_type(),
874 clone_addr,
875 clone_name,
876 TypeRawPtr::BOTTOM,
877 src, dst, full_size_in_heap_words XTOP);
878 phase->transform_later(call);
879 phase->igvn().replace_node(ac, call);
880 }
881
882 void BarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
883 Node* ctrl = ac->in(TypeFunc::Control);
884 Node* mem = ac->in(TypeFunc::Memory);
885 Node* src = ac->in(ArrayCopyNode::Src);
886 Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
887 Node* dest = ac->in(ArrayCopyNode::Dest);
888 Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
889 Node* length = ac->in(ArrayCopyNode::Length);
890
891 Node* payload_src = phase->basic_plus_adr(src, src_offset);
892 Node* payload_dst = phase->basic_plus_adr(dest, dest_offset);
893
894 if (should_copy_int_prefix(phase, ac)) {
895 mem = arraycopy_copy_int_prefix(phase, ctrl, mem, payload_src, payload_dst);
896
897 // We've copied the prefix, bump the pointers.
898 payload_src = phase->basic_plus_adr(src, payload_src, BytesPerInt);
899 payload_dst = phase->basic_plus_adr(dest, payload_dst, BytesPerInt);
900 }
901
902 // Bulk copy.
903 const char* copyfunc_name = "arraycopy";
904 address copyfunc_addr = phase->basictype2arraycopy(T_LONG, nullptr, nullptr, true, copyfunc_name, true);
905
906 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
907 const TypeFunc* call_type = OptoRuntime::fast_arraycopy_Type();
908
909 Node* call = phase->make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, payload_src, payload_dst, length XTOP);
910 phase->transform_later(call);
911
912 phase->igvn().replace_node(ac, call);
913 }
914
915 bool BarrierSetC2::should_copy_int_prefix(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
916 // We do our bulk copy in longs. If base offset is not aligned, then we must copy the prefix separately.
917 // With CompactObjectHeaders, the base offset for an instance is 4 bytes.
918 // We cannot simply expand the copy to the previous long-alignment, as that will copy the object header,
919 // which is stateful with COH - it contains hash and lock bits that are specific to the instance.
920
921 // Skip this when src has an array type. With StressReflectiveCode, the
922 // instance path of the clone can be live in the IR even when the type system
923 // knows src is an array. The pre-copy is unnecessary on such paths (they
924 // are unreachable at runtime), and creating a LoadNode at the array length
925 // offset would assert (LoadRangeNode required).
926 Node* src = ac->in(ArrayCopyNode::Src);
927 if (phase->igvn().type(src)->isa_aryptr()) {
928 return false;
929 }
930
931 int base_off = arraycopy_payload_base_offset(ac->is_clone_array());
932 if (is_aligned(base_off, BytesPerLong)) {
933 // We're aligned, no need to copy anything separately.
934 return false;
935 }
936
937 assert(UseCompactObjectHeaders, "non-aligned base offset only possible with compact object headers");
938 assert(is_aligned(base_off, BytesPerInt), "must be 4-bytes aligned");
939 return true;
940 }
941
942 MergeMemNode* BarrierSetC2::arraycopy_copy_int_prefix(PhaseMacroExpand* phase, Node* ctrl, Node* mem, Node* src, Node* dst) const {
943 // Manual load/store of one int.
944 MergeMemNode* mm = phase->transform_later(MergeMemNode::make(mem))->as_MergeMem();
945 const TypePtr* s_adr_type = phase->igvn().type(src)->is_ptr();
946 const TypePtr* d_adr_type = phase->igvn().type(dst)->is_ptr();
947 uint s_alias_idx = phase->C->get_alias_index(s_adr_type);
948 uint d_alias_idx = phase->C->get_alias_index(d_adr_type);
949 // This copies the first 4 bytes after the compact header (hash field or first instance field) as a raw int.
950 // The actual field at this offset may be a narrowOop, so the load/store must be marked as mismatched to
951 // avoid StoreN-vs-StoreI assertion failures during IGVN.
952 Node* load_prefix = phase->transform_later(
953 LoadNode::make(phase->igvn(), ctrl, mm->memory_at(s_alias_idx), src, s_adr_type,
954 TypeInt::INT, T_INT, MemNode::unordered, LoadNode::DependsOnlyOnTest,
955 false /*require_atomic_access*/, false /*unaligned*/, true /*mismatched*/));
956 Node* store_prefix = phase->transform_later(
957 StoreNode::make(phase->igvn(), ctrl, mm->memory_at(d_alias_idx), dst, d_adr_type,
958 load_prefix, T_INT, MemNode::unordered));
959 store_prefix->as_Store()->set_mismatched_access();
960 mm->set_memory_at(d_alias_idx, store_prefix);
961 return mm;
962 }
963
964 #undef XTOP
965
966 static bool block_has_safepoint(const Block* block, uint from, uint to) {
967 for (uint i = from; i < to; i++) {
968 if (block->get_node(i)->is_MachSafePoint()) {
969 // Safepoint found
970 return true;
971 }
972 }
973
974 // Safepoint not found
975 return false;
976 }
977
978 static bool block_has_safepoint(const Block* block) {
979 return block_has_safepoint(block, 0, block->number_of_nodes());
980 }
981
982 static uint block_index(const Block* block, const Node* node) {
983 for (uint j = 0; j < block->number_of_nodes(); ++j) {
|