< prev index next >

src/hotspot/share/opto/macroArrayCopy.cpp

Print this page


   1 /*
   2  * Copyright (c) 2012, 2016, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "gc/shared/barrierSet.hpp"
  27 #include "opto/arraycopynode.hpp"
  28 #include "oops/objArrayKlass.hpp"
  29 #include "opto/convertnode.hpp"
  30 #include "opto/graphKit.hpp"
  31 #include "opto/macro.hpp"
  32 #include "opto/runtime.hpp"
  33 #include "utilities/align.hpp"




  34 
  35 
  36 void PhaseMacroExpand::insert_mem_bar(Node** ctrl, Node** mem, int opcode, Node* precedent) {
  37   MemBarNode* mb = MemBarNode::make(C, opcode, Compile::AliasIdxBot, precedent);
  38   mb->init_req(TypeFunc::Control, *ctrl);
  39   mb->init_req(TypeFunc::Memory, *mem);
  40   transform_later(mb);
  41   *ctrl = new ProjNode(mb,TypeFunc::Control);
  42   transform_later(*ctrl);
  43   Node* mem_proj = new ProjNode(mb,TypeFunc::Memory);
  44   transform_later(mem_proj);
  45   *mem = mem_proj;
  46 }
  47 
  48 Node* PhaseMacroExpand::array_element_address(Node* ary, Node* idx, BasicType elembt) {
  49   uint shift  = exact_log2(type2aelembytes(elembt));
  50   uint header = arrayOopDesc::base_offset_in_bytes(elembt);
  51   Node* base =  basic_plus_adr(ary, header);
  52 #ifdef _LP64
  53   // see comment in GraphKit::array_element_address


 535         // (At this point we can assume disjoint_bases, since types differ.)
 536         int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
 537         Node* p1 = basic_plus_adr(dest_klass, ek_offset);
 538         Node* n1 = LoadKlassNode::make(_igvn, NULL, C->immutable_memory(), p1, TypeRawPtr::BOTTOM);
 539         Node* dest_elem_klass = transform_later(n1);
 540         Node* cv = generate_checkcast_arraycopy(&local_ctrl, &local_mem,
 541                                                 adr_type,
 542                                                 dest_elem_klass,
 543                                                 src, src_offset, dest, dest_offset,
 544                                                 ConvI2X(copy_length), dest_uninitialized);
 545         if (cv == NULL)  cv = intcon(-1);  // failure (no stub available)
 546         checked_control = local_ctrl;
 547         checked_i_o     = *io;
 548         checked_mem     = local_mem->memory_at(alias_idx);
 549         checked_value   = cv;
 550       }
 551     }
 552     // At this point we know we do not need type checks on oop stores.
 553 
 554     BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
 555     if (alloc != NULL && !bs->array_copy_requires_gc_barriers(copy_type)) {
 556       // If we do not need gc barriers, copy using the jint or jlong stub.
 557       copy_type = LP64_ONLY(UseCompressedOops ? T_INT : T_LONG) NOT_LP64(T_INT);
 558       assert(type2aelembytes(basic_elem_type) == type2aelembytes(copy_type),
 559              "sizes agree");
 560     }
 561   }
 562 
 563   if (!(*ctrl)->is_top()) {
 564     // Generate the fast path, if possible.
 565     Node* local_ctrl = *ctrl;
 566     MergeMemNode* local_mem = MergeMemNode::make(mem);
 567     transform_later(local_mem);
 568 
 569     generate_unchecked_arraycopy(&local_ctrl, &local_mem,
 570                                  adr_type, copy_type, disjoint_bases,
 571                                  src, src_offset, dest, dest_offset,
 572                                  ConvI2X(copy_length), dest_uninitialized);
 573 
 574     // Present the results of the fast call.
 575     result_region->init_req(fast_path, local_ctrl);


1066   Node* src_start  = src;
1067   Node* dest_start = dest;
1068   if (src_offset != NULL || dest_offset != NULL) {
1069     src_start =  array_element_address(src, src_offset, basic_elem_type);
1070     dest_start = array_element_address(dest, dest_offset, basic_elem_type);
1071   }
1072 
1073   // Figure out which arraycopy runtime method to call.
1074   const char* copyfunc_name = "arraycopy";
1075   address     copyfunc_addr =
1076       basictype2arraycopy(basic_elem_type, src_offset, dest_offset,
1077                           disjoint_bases, copyfunc_name, dest_uninitialized);
1078 
1079   const TypeFunc* call_type = OptoRuntime::fast_arraycopy_Type();
1080   Node* call = make_leaf_call(*ctrl, *mem, call_type, copyfunc_addr, copyfunc_name, adr_type,
1081                               src_start, dest_start, copy_length XTOP);
1082 
1083   finish_arraycopy_call(call, ctrl, mem, adr_type);
1084 }
1085 

















1086 void PhaseMacroExpand::expand_arraycopy_node(ArrayCopyNode *ac) {
1087   Node* ctrl = ac->in(TypeFunc::Control);
1088   Node* io = ac->in(TypeFunc::I_O);
1089   Node* src = ac->in(ArrayCopyNode::Src);
1090   Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
1091   Node* dest = ac->in(ArrayCopyNode::Dest);
1092   Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
1093   Node* length = ac->in(ArrayCopyNode::Length);
1094   MergeMemNode* merge_mem = NULL;
1095 
1096   if (ac->is_clonebasic()) {
1097     assert (src_offset == NULL && dest_offset == NULL, "for clone offsets should be null");
1098     Node* mem = ac->in(TypeFunc::Memory);
1099     const char* copyfunc_name = "arraycopy";
1100     address     copyfunc_addr =
1101       basictype2arraycopy(T_LONG, NULL, NULL,
1102                           true, copyfunc_name, true);
1103 
1104     const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
1105     const TypeFunc* call_type = OptoRuntime::fast_arraycopy_Type();
1106 
1107     Node* call = make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, src, dest, length XTOP);
1108     transform_later(call);
1109 



























1110     _igvn.replace_node(ac, call);
1111     return;
1112   } else if (ac->is_copyof() || ac->is_copyofrange() || ac->is_cloneoop()) {
1113     Node* mem = ac->in(TypeFunc::Memory);
1114     merge_mem = MergeMemNode::make(mem);
1115     transform_later(merge_mem);
1116 
1117     RegionNode* slow_region = new RegionNode(1);
1118     transform_later(slow_region);
1119 
1120     AllocateArrayNode* alloc = NULL;
1121     if (ac->is_alloc_tightly_coupled()) {
1122       alloc = AllocateArrayNode::Ideal_array_allocation(dest, &_igvn);
1123       assert(alloc != NULL, "expect alloc");
1124     }
1125 
1126     const TypePtr* adr_type = _igvn.type(dest)->is_oopptr()->add_offset(Type::OffsetBot);
1127     if (ac->_dest_type != TypeOopPtr::BOTTOM) {
1128       adr_type = ac->_dest_type->add_offset(Type::OffsetBot)->is_ptr();
1129     }
1130     if (ac->_src_type != ac->_dest_type) {
1131       adr_type = TypeRawPtr::BOTTOM;
1132     }
1133     generate_arraycopy(ac, alloc, &ctrl, merge_mem, &io,
1134                        adr_type, T_OBJECT,
1135                        src, src_offset, dest, dest_offset, length,
1136                        true, !ac->is_copyofrange());
1137 
1138     return;
1139   }
1140 
1141   AllocateArrayNode* alloc = NULL;
1142   if (ac->is_alloc_tightly_coupled()) {
1143     alloc = AllocateArrayNode::Ideal_array_allocation(dest, &_igvn);
1144     assert(alloc != NULL, "expect alloc");
1145   }
1146 
1147   assert(ac->is_arraycopy() || ac->is_arraycopy_validated(), "should be an arraycopy");
1148 
1149   // Compile time checks.  If any of these checks cannot be verified at compile time,
1150   // we do not make a fast path for this call.  Instead, we let the call remain as it
1151   // is.  The checks we choose to mandate at compile time are:


   1 /*
   2  * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "gc/shared/barrierSet.hpp"
  27 #include "opto/arraycopynode.hpp"
  28 #include "oops/objArrayKlass.hpp"
  29 #include "opto/convertnode.hpp"
  30 #include "opto/graphKit.hpp"
  31 #include "opto/macro.hpp"
  32 #include "opto/runtime.hpp"
  33 #include "utilities/align.hpp"
  34 #if INCLUDE_SHENANDOAHGC
  35 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
  36 #include "gc/shenandoah/shenandoahRuntime.hpp"
  37 #endif
  38 
  39 
  40 void PhaseMacroExpand::insert_mem_bar(Node** ctrl, Node** mem, int opcode, Node* precedent) {
  41   MemBarNode* mb = MemBarNode::make(C, opcode, Compile::AliasIdxBot, precedent);
  42   mb->init_req(TypeFunc::Control, *ctrl);
  43   mb->init_req(TypeFunc::Memory, *mem);
  44   transform_later(mb);
  45   *ctrl = new ProjNode(mb,TypeFunc::Control);
  46   transform_later(*ctrl);
  47   Node* mem_proj = new ProjNode(mb,TypeFunc::Memory);
  48   transform_later(mem_proj);
  49   *mem = mem_proj;
  50 }
  51 
  52 Node* PhaseMacroExpand::array_element_address(Node* ary, Node* idx, BasicType elembt) {
  53   uint shift  = exact_log2(type2aelembytes(elembt));
  54   uint header = arrayOopDesc::base_offset_in_bytes(elembt);
  55   Node* base =  basic_plus_adr(ary, header);
  56 #ifdef _LP64
  57   // see comment in GraphKit::array_element_address


 539         // (At this point we can assume disjoint_bases, since types differ.)
 540         int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
 541         Node* p1 = basic_plus_adr(dest_klass, ek_offset);
 542         Node* n1 = LoadKlassNode::make(_igvn, NULL, C->immutable_memory(), p1, TypeRawPtr::BOTTOM);
 543         Node* dest_elem_klass = transform_later(n1);
 544         Node* cv = generate_checkcast_arraycopy(&local_ctrl, &local_mem,
 545                                                 adr_type,
 546                                                 dest_elem_klass,
 547                                                 src, src_offset, dest, dest_offset,
 548                                                 ConvI2X(copy_length), dest_uninitialized);
 549         if (cv == NULL)  cv = intcon(-1);  // failure (no stub available)
 550         checked_control = local_ctrl;
 551         checked_i_o     = *io;
 552         checked_mem     = local_mem->memory_at(alias_idx);
 553         checked_value   = cv;
 554       }
 555     }
 556     // At this point we know we do not need type checks on oop stores.
 557 
 558     BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
 559     if (alloc != NULL && !bs->array_copy_requires_gc_barriers(copy_type) && !UseShenandoahGC) {
 560       // If we do not need gc barriers, copy using the jint or jlong stub.
 561       copy_type = LP64_ONLY(UseCompressedOops ? T_INT : T_LONG) NOT_LP64(T_INT);
 562       assert(type2aelembytes(basic_elem_type) == type2aelembytes(copy_type),
 563              "sizes agree");
 564     }
 565   }
 566 
 567   if (!(*ctrl)->is_top()) {
 568     // Generate the fast path, if possible.
 569     Node* local_ctrl = *ctrl;
 570     MergeMemNode* local_mem = MergeMemNode::make(mem);
 571     transform_later(local_mem);
 572 
 573     generate_unchecked_arraycopy(&local_ctrl, &local_mem,
 574                                  adr_type, copy_type, disjoint_bases,
 575                                  src, src_offset, dest, dest_offset,
 576                                  ConvI2X(copy_length), dest_uninitialized);
 577 
 578     // Present the results of the fast call.
 579     result_region->init_req(fast_path, local_ctrl);


1070   Node* src_start  = src;
1071   Node* dest_start = dest;
1072   if (src_offset != NULL || dest_offset != NULL) {
1073     src_start =  array_element_address(src, src_offset, basic_elem_type);
1074     dest_start = array_element_address(dest, dest_offset, basic_elem_type);
1075   }
1076 
1077   // Figure out which arraycopy runtime method to call.
1078   const char* copyfunc_name = "arraycopy";
1079   address     copyfunc_addr =
1080       basictype2arraycopy(basic_elem_type, src_offset, dest_offset,
1081                           disjoint_bases, copyfunc_name, dest_uninitialized);
1082 
1083   const TypeFunc* call_type = OptoRuntime::fast_arraycopy_Type();
1084   Node* call = make_leaf_call(*ctrl, *mem, call_type, copyfunc_addr, copyfunc_name, adr_type,
1085                               src_start, dest_start, copy_length XTOP);
1086 
1087   finish_arraycopy_call(call, ctrl, mem, adr_type);
1088 }
1089 
1090 #if INCLUDE_SHENANDOAHGC
1091 Node* PhaseMacroExpand::shenandoah_call_clone_barrier(Node* call, Node* dest) {
1092   assert (UseShenandoahGC && ShenandoahCloneBarrier, "Should be enabled");
1093   const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
1094   Node* c = new ProjNode(call,TypeFunc::Control);
1095   transform_later(c);
1096   Node* m = new ProjNode(call, TypeFunc::Memory);
1097   transform_later(m);
1098   assert(dest->is_AddP(), "bad input");
1099   call = make_leaf_call(c, m, ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type(),
1100                         CAST_FROM_FN_PTR(address, ShenandoahRuntime::shenandoah_clone_barrier),
1101                         "shenandoah_clone_barrier", raw_adr_type, dest->in(AddPNode::Base));
1102   transform_later(call);
1103   return call;
1104 }
1105 #endif
1106 
1107 void PhaseMacroExpand::expand_arraycopy_node(ArrayCopyNode *ac) {
1108   Node* ctrl = ac->in(TypeFunc::Control);
1109   Node* io = ac->in(TypeFunc::I_O);
1110   Node* src = ac->in(ArrayCopyNode::Src);
1111   Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
1112   Node* dest = ac->in(ArrayCopyNode::Dest);
1113   Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
1114   Node* length = ac->in(ArrayCopyNode::Length);
1115   MergeMemNode* merge_mem = NULL;
1116 
1117   if (ac->is_clonebasic()) {
1118     assert (src_offset == NULL && dest_offset == NULL, "for clone offsets should be null");
1119     Node* mem = ac->in(TypeFunc::Memory);
1120     const char* copyfunc_name = "arraycopy";
1121     address     copyfunc_addr =
1122       basictype2arraycopy(T_LONG, NULL, NULL,
1123                           true, copyfunc_name, true);
1124 
1125     const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
1126     const TypeFunc* call_type = OptoRuntime::fast_arraycopy_Type();
1127 
1128     Node* call = make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, src, dest, length XTOP);
1129     transform_later(call);
1130 
1131 #if INCLUDE_SHENANDOAHGC
1132     if (UseShenandoahGC && ShenandoahCloneBarrier) {
1133       const TypeOopPtr* src_type = _igvn.type(src)->is_oopptr();
1134       if (src_type->isa_instptr() != NULL) {
1135         ciInstanceKlass* ik = src_type->klass()->as_instance_klass();
1136         if ((src_type->klass_is_exact() || (!ik->is_interface() && !ik->has_subklass())) && !ik->has_injected_fields()) {
1137           if (ik->has_object_fields()) {
1138             call = shenandoah_call_clone_barrier(call, dest);
1139           } else {
1140             if (!src_type->klass_is_exact()) {
1141               C->dependencies()->assert_leaf_type(ik);
1142             }
1143           }
1144         } else {
1145           call = shenandoah_call_clone_barrier(call, dest);
1146         }
1147       } else if (src_type->isa_aryptr()) {
1148         BasicType src_elem  = src_type->klass()->as_array_klass()->element_type()->basic_type();
1149         if (src_elem == T_OBJECT || src_elem == T_ARRAY) {
1150           call = shenandoah_call_clone_barrier(call, dest);
1151         }
1152       } else {
1153         call = shenandoah_call_clone_barrier(call, dest);
1154       }
1155     }
1156 #endif
1157 
1158     _igvn.replace_node(ac, call);
1159     return;
1160   } else if (ac->is_copyof() || ac->is_copyofrange() || ac->is_cloneoop()) {
1161     Node* mem = ac->in(TypeFunc::Memory);
1162     merge_mem = MergeMemNode::make(mem);
1163     transform_later(merge_mem);
1164 
1165     RegionNode* slow_region = new RegionNode(1);
1166     transform_later(slow_region);
1167 
1168     AllocateArrayNode* alloc = NULL;
1169     if (ac->is_alloc_tightly_coupled()) {
1170       alloc = AllocateArrayNode::Ideal_array_allocation(dest, &_igvn);
1171       assert(alloc != NULL, "expect alloc");
1172     }
1173 
1174     const TypePtr* adr_type = _igvn.type(dest)->is_oopptr()->add_offset(Type::OffsetBot);
1175     if (ac->_dest_type != TypeOopPtr::BOTTOM) {
1176       adr_type = ac->_dest_type->add_offset(Type::OffsetBot)->is_ptr();



1177     }
1178     generate_arraycopy(ac, alloc, &ctrl, merge_mem, &io,
1179                        adr_type, T_OBJECT,
1180                        src, src_offset, dest, dest_offset, length,
1181                        true, !ac->is_copyofrange());
1182 
1183     return;
1184   }
1185 
1186   AllocateArrayNode* alloc = NULL;
1187   if (ac->is_alloc_tightly_coupled()) {
1188     alloc = AllocateArrayNode::Ideal_array_allocation(dest, &_igvn);
1189     assert(alloc != NULL, "expect alloc");
1190   }
1191 
1192   assert(ac->is_arraycopy() || ac->is_arraycopy_validated(), "should be an arraycopy");
1193 
1194   // Compile time checks.  If any of these checks cannot be verified at compile time,
1195   // we do not make a fast path for this call.  Instead, we let the call remain as it
1196   // is.  The checks we choose to mandate at compile time are:


< prev index next >