< prev index next >

src/cpu/x86/vm/stubGenerator_x86_64.cpp

Print this page




  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/macroAssembler.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "interpreter/interpreter.hpp"
  29 #include "nativeInst_x86.hpp"
  30 #include "oops/instanceOop.hpp"
  31 #include "oops/method.hpp"
  32 #include "oops/objArrayKlass.hpp"
  33 #include "oops/oop.inline.hpp"
  34 #include "prims/methodHandles.hpp"
  35 #include "runtime/frame.inline.hpp"
  36 #include "runtime/handles.inline.hpp"
  37 #include "runtime/sharedRuntime.hpp"
  38 #include "runtime/stubCodeGenerator.hpp"
  39 #include "runtime/stubRoutines.hpp"
  40 #include "runtime/thread.inline.hpp"

  41 #include "utilities/top.hpp"
  42 #ifdef COMPILER2
  43 #include "opto/runtime.hpp"
  44 #endif



  45 
  46 // Declaration and definition of StubGenerator (no .hpp file).
  47 // For a more detailed description of the stub routine structure
  48 // see the comment in stubRoutines.hpp
  49 
  50 #define __ _masm->
  51 #define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8)
  52 #define a__ ((Assembler*)_masm)->
  53 
  54 #ifdef PRODUCT
  55 #define BLOCK_COMMENT(str) /* nothing */
  56 #else
  57 #define BLOCK_COMMENT(str) __ block_comment(str)
  58 #endif
  59 
  60 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  61 const int MXCSR_MASK = 0xFFC0;  // Mask out any pending exceptions
  62 
  63 // Stub Code definitions
  64 


1161 #endif
1162   }
1163 
1164   void restore_arg_regs() {
1165     const Register saved_rdi = r9;
1166     const Register saved_rsi = r10;
1167 #ifdef _WIN64
1168     __ movptr(rdi, saved_rdi);
1169     __ movptr(rsi, saved_rsi);
1170 #endif
1171   }
1172 
1173   // Generate code for an array write pre barrier
1174   //
1175   //     addr    -  starting address
1176   //     count   -  element count
1177   //     tmp     - scratch register
1178   //
1179   //     Destroy no registers!
1180   //
1181   void  gen_write_ref_array_pre_barrier(Register addr, Register count, bool dest_uninitialized) {
1182     BarrierSet* bs = Universe::heap()->barrier_set();
1183     switch (bs->kind()) {
1184       case BarrierSet::G1SATBCT:
1185       case BarrierSet::G1SATBCTLogging:
1186         // With G1, don't generate the call if we statically know that the target in uninitialized
1187         if (!dest_uninitialized) {
1188            __ pusha();                      // push registers
1189            if (count == c_rarg0) {
1190              if (addr == c_rarg1) {
1191                // exactly backwards!!
1192                __ xchgptr(c_rarg1, c_rarg0);
1193              } else {
1194                __ movptr(c_rarg1, count);
1195                __ movptr(c_rarg0, addr);
1196              }
1197            } else {
1198              __ movptr(c_rarg0, addr);
1199              __ movptr(c_rarg1, count);
1200            }
1201            __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre), 2);
1202            __ popa();
1203         }
1204          break;
1205       case BarrierSet::CardTableModRef:
1206       case BarrierSet::CardTableExtension:
1207       case BarrierSet::ModRef:
1208         break;





1209       default:
1210         ShouldNotReachHere();
1211 
1212     }
1213   }
1214 
1215   //
1216   // Generate code for an array write post barrier
1217   //
1218   //  Input:
1219   //     start    - register containing starting address of destination array
1220   //     count    - elements count
1221   //     scratch  - scratch register
1222   //
1223   //  The input registers are overwritten.
1224   //
1225   void  gen_write_ref_array_post_barrier(Register start, Register count, Register scratch) {
1226     assert_different_registers(start, count, scratch);
1227     BarrierSet* bs = Universe::heap()->barrier_set();
1228     switch (bs->kind()) {


1250           assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
1251 
1252           Label L_loop;
1253           const Register end = count;
1254 
1255           __ leaq(end, Address(start, count, TIMES_OOP, 0));  // end == start+count*oop_size
1256           __ subptr(end, BytesPerHeapOop); // end - 1 to make inclusive
1257           __ shrptr(start, CardTableModRefBS::card_shift);
1258           __ shrptr(end,   CardTableModRefBS::card_shift);
1259           __ subptr(end, start); // end --> cards count
1260 
1261           int64_t disp = (int64_t) ct->byte_map_base;
1262           __ mov64(scratch, disp);
1263           __ addptr(start, scratch);
1264         __ BIND(L_loop);
1265           __ movb(Address(start, count, Address::times_1), 0);
1266           __ decrement(count);
1267           __ jcc(Assembler::greaterEqual, L_loop);
1268         }
1269         break;




1270       default:
1271         ShouldNotReachHere();
1272 
1273     }
1274   }
1275 
1276 
1277   // Copy big chunks forward
1278   //
1279   // Inputs:
1280   //   end_from     - source arrays end address
1281   //   end_to       - destination array end address
1282   //   qword_count  - 64-bits element count, negative
1283   //   to           - scratch
1284   //   L_copy_bytes - entry label
1285   //   L_copy_8_bytes  - exit  label
1286   //
1287   void copy_bytes_forward(Register end_from, Register end_to,
1288                              Register qword_count, Register to,
1289                              Label& L_copy_bytes, Label& L_copy_8_bytes) {


1867     const Register qword_count = count;
1868     const Register end_from    = from; // source array end address
1869     const Register end_to      = to;   // destination array end address
1870     const Register saved_to    = r11;  // saved destination array address
1871     // End pointers are inclusive, and if count is not zero they point
1872     // to the last unit copied:  end_to[0] := end_from[0]
1873 
1874     __ enter(); // required for proper stackwalking of RuntimeStub frame
1875     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1876 
1877     if (entry != NULL) {
1878       *entry = __ pc();
1879       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1880       BLOCK_COMMENT("Entry:");
1881     }
1882 
1883     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1884                       // r9 and r10 may be used to save non-volatile registers
1885     if (is_oop) {
1886       __ movq(saved_to, to);
1887       gen_write_ref_array_pre_barrier(to, count, dest_uninitialized);
1888     }
1889 
1890     // 'from', 'to' and 'count' are now valid
1891     __ movptr(dword_count, count);
1892     __ shrptr(count, 1); // count => qword_count
1893 
1894     // Copy from low to high addresses.  Use 'to' as scratch.
1895     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
1896     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
1897     __ negptr(qword_count);
1898     __ jmp(L_copy_bytes);
1899 
1900     // Copy trailing qwords
1901   __ BIND(L_copy_8_bytes);
1902     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
1903     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
1904     __ increment(qword_count);
1905     __ jcc(Assembler::notZero, L_copy_8_bytes);
1906 
1907     // Check for and copy trailing dword


1955     const Register to          = rsi;  // destination array address
1956     const Register count       = rdx;  // elements count
1957     const Register dword_count = rcx;
1958     const Register qword_count = count;
1959 
1960     __ enter(); // required for proper stackwalking of RuntimeStub frame
1961     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1962 
1963     if (entry != NULL) {
1964       *entry = __ pc();
1965        // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1966       BLOCK_COMMENT("Entry:");
1967     }
1968 
1969     array_overlap_test(nooverlap_target, Address::times_4);
1970     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1971                       // r9 and r10 may be used to save non-volatile registers
1972 
1973     if (is_oop) {
1974       // no registers are destroyed by this call
1975       gen_write_ref_array_pre_barrier(to, count, dest_uninitialized);
1976     }
1977 
1978     assert_clean_int(count, rax); // Make sure 'count' is clean int.
1979     // 'from', 'to' and 'count' are now valid
1980     __ movptr(dword_count, count);
1981     __ shrptr(count, 1); // count => qword_count
1982 
1983     // Copy from high to low addresses.  Use 'to' as scratch.
1984 
1985     // Check for and copy trailing dword
1986     __ testl(dword_count, 1);
1987     __ jcc(Assembler::zero, L_copy_bytes);
1988     __ movl(rax, Address(from, dword_count, Address::times_4, -4));
1989     __ movl(Address(to, dword_count, Address::times_4, -4), rax);
1990     __ jmp(L_copy_bytes);
1991 
1992     // Copy trailing qwords
1993   __ BIND(L_copy_8_bytes);
1994     __ movq(rax, Address(from, qword_count, Address::times_8, -8));
1995     __ movq(Address(to, qword_count, Address::times_8, -8), rax);


2053     // End pointers are inclusive, and if count is not zero they point
2054     // to the last unit copied:  end_to[0] := end_from[0]
2055 
2056     __ enter(); // required for proper stackwalking of RuntimeStub frame
2057     // Save no-overlap entry point for generate_conjoint_long_oop_copy()
2058     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
2059 
2060     if (entry != NULL) {
2061       *entry = __ pc();
2062       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
2063       BLOCK_COMMENT("Entry:");
2064     }
2065 
2066     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
2067                       // r9 and r10 may be used to save non-volatile registers
2068     // 'from', 'to' and 'qword_count' are now valid
2069     if (is_oop) {
2070       // Save to and count for store barrier
2071       __ movptr(saved_count, qword_count);
2072       // no registers are destroyed by this call
2073       gen_write_ref_array_pre_barrier(to, qword_count, dest_uninitialized);
2074     }
2075 
2076     // Copy from low to high addresses.  Use 'to' as scratch.
2077     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
2078     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
2079     __ negptr(qword_count);
2080     __ jmp(L_copy_bytes);
2081 
2082     // Copy trailing qwords
2083   __ BIND(L_copy_8_bytes);
2084     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
2085     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
2086     __ increment(qword_count);
2087     __ jcc(Assembler::notZero, L_copy_8_bytes);
2088 
2089     if (is_oop) {
2090       __ jmp(L_exit);
2091     } else {
2092       restore_arg_regs();
2093       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free


2140     const Register qword_count = rdx;  // elements count
2141     const Register saved_count = rcx;
2142 
2143     __ enter(); // required for proper stackwalking of RuntimeStub frame
2144     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
2145 
2146     if (entry != NULL) {
2147       *entry = __ pc();
2148       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
2149       BLOCK_COMMENT("Entry:");
2150     }
2151 
2152     array_overlap_test(nooverlap_target, Address::times_8);
2153     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
2154                       // r9 and r10 may be used to save non-volatile registers
2155     // 'from', 'to' and 'qword_count' are now valid
2156     if (is_oop) {
2157       // Save to and count for store barrier
2158       __ movptr(saved_count, qword_count);
2159       // No registers are destroyed by this call
2160       gen_write_ref_array_pre_barrier(to, saved_count, dest_uninitialized);
2161     }
2162 
2163     __ jmp(L_copy_bytes);
2164 
2165     // Copy trailing qwords
2166   __ BIND(L_copy_8_bytes);
2167     __ movq(rax, Address(from, qword_count, Address::times_8, -8));
2168     __ movq(Address(to, qword_count, Address::times_8, -8), rax);
2169     __ decrement(qword_count);
2170     __ jcc(Assembler::notZero, L_copy_8_bytes);
2171 
2172     if (is_oop) {
2173       __ jmp(L_exit);
2174     } else {
2175       restore_arg_regs();
2176       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
2177       __ xorptr(rax, rax); // return 0
2178       __ leave(); // required for proper stackwalking of RuntimeStub frame
2179       __ ret(0);
2180     }


2314 #ifdef ASSERT
2315     BLOCK_COMMENT("assert consistent ckoff/ckval");
2316     // The ckoff and ckval must be mutually consistent,
2317     // even though caller generates both.
2318     { Label L;
2319       int sco_offset = in_bytes(Klass::super_check_offset_offset());
2320       __ cmpl(ckoff, Address(ckval, sco_offset));
2321       __ jcc(Assembler::equal, L);
2322       __ stop("super_check_offset inconsistent");
2323       __ bind(L);
2324     }
2325 #endif //ASSERT
2326 
2327     // Loop-invariant addresses.  They are exclusive end pointers.
2328     Address end_from_addr(from, length, TIMES_OOP, 0);
2329     Address   end_to_addr(to,   length, TIMES_OOP, 0);
2330     // Loop-variant addresses.  They assume post-incremented count < 0.
2331     Address from_element_addr(end_from, count, TIMES_OOP, 0);
2332     Address   to_element_addr(end_to,   count, TIMES_OOP, 0);
2333 
2334     gen_write_ref_array_pre_barrier(to, count, dest_uninitialized);
2335 
2336     // Copy from low to high addresses, indexed from the end of each array.
2337     __ lea(end_from, end_from_addr);
2338     __ lea(end_to,   end_to_addr);
2339     __ movptr(r14_length, length);        // save a copy of the length
2340     assert(length == count, "");          // else fix next line:
2341     __ negptr(count);                     // negate and test the length
2342     __ jcc(Assembler::notZero, L_load_element);
2343 
2344     // Empty array:  Nothing to do.
2345     __ xorptr(rax, rax);                  // return 0 on (trivial) success
2346     __ jmp(L_done);
2347 
2348     // ======== begin loop ========
2349     // (Loop is rotated; its entry is L_load_element.)
2350     // Loop control:
2351     //   for (count = -count; count != 0; count++)
2352     // Base pointers src, dst are biased by 8*(count-1),to last element.
2353     __ align(OptoLoopAlignment);
2354 




  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/macroAssembler.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "interpreter/interpreter.hpp"
  29 #include "nativeInst_x86.hpp"
  30 #include "oops/instanceOop.hpp"
  31 #include "oops/method.hpp"
  32 #include "oops/objArrayKlass.hpp"
  33 #include "oops/oop.inline.hpp"
  34 #include "prims/methodHandles.hpp"
  35 #include "runtime/frame.inline.hpp"
  36 #include "runtime/handles.inline.hpp"
  37 #include "runtime/sharedRuntime.hpp"
  38 #include "runtime/stubCodeGenerator.hpp"
  39 #include "runtime/stubRoutines.hpp"
  40 #include "runtime/thread.inline.hpp"
  41 #include "utilities/macros.hpp"
  42 #include "utilities/top.hpp"
  43 #ifdef COMPILER2
  44 #include "opto/runtime.hpp"
  45 #endif
  46 #if INCLUDE_ALL_GCS
  47 #include "shenandoahBarrierSetAssembler_x86.hpp"
  48 #endif
  49 
  50 // Declaration and definition of StubGenerator (no .hpp file).
  51 // For a more detailed description of the stub routine structure
  52 // see the comment in stubRoutines.hpp
  53 
  54 #define __ _masm->
  55 #define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8)
  56 #define a__ ((Assembler*)_masm)->
  57 
  58 #ifdef PRODUCT
  59 #define BLOCK_COMMENT(str) /* nothing */
  60 #else
  61 #define BLOCK_COMMENT(str) __ block_comment(str)
  62 #endif
  63 
  64 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  65 const int MXCSR_MASK = 0xFFC0;  // Mask out any pending exceptions
  66 
  67 // Stub Code definitions
  68 


1165 #endif
1166   }
1167 
1168   void restore_arg_regs() {
1169     const Register saved_rdi = r9;
1170     const Register saved_rsi = r10;
1171 #ifdef _WIN64
1172     __ movptr(rdi, saved_rdi);
1173     __ movptr(rsi, saved_rsi);
1174 #endif
1175   }
1176 
1177   // Generate code for an array write pre barrier
1178   //
1179   //     addr    -  starting address
1180   //     count   -  element count
1181   //     tmp     - scratch register
1182   //
1183   //     Destroy no registers!
1184   //
1185   void  gen_write_ref_array_pre_barrier(Register src, Register addr, Register count, bool dest_uninitialized) {
1186     BarrierSet* bs = Universe::heap()->barrier_set();
1187     switch (bs->kind()) {
1188       case BarrierSet::G1SATBCT:
1189       case BarrierSet::G1SATBCTLogging:
1190         // With G1, don't generate the call if we statically know that the target in uninitialized
1191         if (!dest_uninitialized) {
1192            __ pusha();                      // push registers
1193            if (count == c_rarg0) {
1194              if (addr == c_rarg1) {
1195                // exactly backwards!!
1196                __ xchgptr(c_rarg1, c_rarg0);
1197              } else {
1198                __ movptr(c_rarg1, count);
1199                __ movptr(c_rarg0, addr);
1200              }
1201            } else {
1202              __ movptr(c_rarg0, addr);
1203              __ movptr(c_rarg1, count);
1204            }
1205            __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre), 2);
1206            __ popa();
1207         }
1208          break;
1209       case BarrierSet::CardTableModRef:
1210       case BarrierSet::CardTableExtension:
1211       case BarrierSet::ModRef:
1212         break;
1213 #if INCLUDE_ALL_GCS
1214       case BarrierSet::ShenandoahBarrierSet:
1215         ShenandoahBarrierSetAssembler::bsasm()->arraycopy_prologue(_masm, dest_uninitialized, src, addr, count);
1216         break;
1217 #endif
1218       default:
1219         ShouldNotReachHere();
1220 
1221     }
1222   }
1223 
1224   //
1225   // Generate code for an array write post barrier
1226   //
1227   //  Input:
1228   //     start    - register containing starting address of destination array
1229   //     count    - elements count
1230   //     scratch  - scratch register
1231   //
1232   //  The input registers are overwritten.
1233   //
1234   void  gen_write_ref_array_post_barrier(Register start, Register count, Register scratch) {
1235     assert_different_registers(start, count, scratch);
1236     BarrierSet* bs = Universe::heap()->barrier_set();
1237     switch (bs->kind()) {


1259           assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
1260 
1261           Label L_loop;
1262           const Register end = count;
1263 
1264           __ leaq(end, Address(start, count, TIMES_OOP, 0));  // end == start+count*oop_size
1265           __ subptr(end, BytesPerHeapOop); // end - 1 to make inclusive
1266           __ shrptr(start, CardTableModRefBS::card_shift);
1267           __ shrptr(end,   CardTableModRefBS::card_shift);
1268           __ subptr(end, start); // end --> cards count
1269 
1270           int64_t disp = (int64_t) ct->byte_map_base;
1271           __ mov64(scratch, disp);
1272           __ addptr(start, scratch);
1273         __ BIND(L_loop);
1274           __ movb(Address(start, count, Address::times_1), 0);
1275           __ decrement(count);
1276           __ jcc(Assembler::greaterEqual, L_loop);
1277         }
1278         break;
1279 #if INCLUDE_ALL_GCS
1280       case BarrierSet::ShenandoahBarrierSet:
1281         break;
1282 #endif
1283       default:
1284         ShouldNotReachHere();
1285 
1286     }
1287   }
1288 
1289 
1290   // Copy big chunks forward
1291   //
1292   // Inputs:
1293   //   end_from     - source arrays end address
1294   //   end_to       - destination array end address
1295   //   qword_count  - 64-bits element count, negative
1296   //   to           - scratch
1297   //   L_copy_bytes - entry label
1298   //   L_copy_8_bytes  - exit  label
1299   //
1300   void copy_bytes_forward(Register end_from, Register end_to,
1301                              Register qword_count, Register to,
1302                              Label& L_copy_bytes, Label& L_copy_8_bytes) {


1880     const Register qword_count = count;
1881     const Register end_from    = from; // source array end address
1882     const Register end_to      = to;   // destination array end address
1883     const Register saved_to    = r11;  // saved destination array address
1884     // End pointers are inclusive, and if count is not zero they point
1885     // to the last unit copied:  end_to[0] := end_from[0]
1886 
1887     __ enter(); // required for proper stackwalking of RuntimeStub frame
1888     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1889 
1890     if (entry != NULL) {
1891       *entry = __ pc();
1892       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1893       BLOCK_COMMENT("Entry:");
1894     }
1895 
1896     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1897                       // r9 and r10 may be used to save non-volatile registers
1898     if (is_oop) {
1899       __ movq(saved_to, to);
1900       gen_write_ref_array_pre_barrier(from, to, count, dest_uninitialized);
1901     }
1902 
1903     // 'from', 'to' and 'count' are now valid
1904     __ movptr(dword_count, count);
1905     __ shrptr(count, 1); // count => qword_count
1906 
1907     // Copy from low to high addresses.  Use 'to' as scratch.
1908     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
1909     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
1910     __ negptr(qword_count);
1911     __ jmp(L_copy_bytes);
1912 
1913     // Copy trailing qwords
1914   __ BIND(L_copy_8_bytes);
1915     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
1916     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
1917     __ increment(qword_count);
1918     __ jcc(Assembler::notZero, L_copy_8_bytes);
1919 
1920     // Check for and copy trailing dword


1968     const Register to          = rsi;  // destination array address
1969     const Register count       = rdx;  // elements count
1970     const Register dword_count = rcx;
1971     const Register qword_count = count;
1972 
1973     __ enter(); // required for proper stackwalking of RuntimeStub frame
1974     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1975 
1976     if (entry != NULL) {
1977       *entry = __ pc();
1978        // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1979       BLOCK_COMMENT("Entry:");
1980     }
1981 
1982     array_overlap_test(nooverlap_target, Address::times_4);
1983     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1984                       // r9 and r10 may be used to save non-volatile registers
1985 
1986     if (is_oop) {
1987       // no registers are destroyed by this call
1988       gen_write_ref_array_pre_barrier(from, to, count, dest_uninitialized);
1989     }
1990 
1991     assert_clean_int(count, rax); // Make sure 'count' is clean int.
1992     // 'from', 'to' and 'count' are now valid
1993     __ movptr(dword_count, count);
1994     __ shrptr(count, 1); // count => qword_count
1995 
1996     // Copy from high to low addresses.  Use 'to' as scratch.
1997 
1998     // Check for and copy trailing dword
1999     __ testl(dword_count, 1);
2000     __ jcc(Assembler::zero, L_copy_bytes);
2001     __ movl(rax, Address(from, dword_count, Address::times_4, -4));
2002     __ movl(Address(to, dword_count, Address::times_4, -4), rax);
2003     __ jmp(L_copy_bytes);
2004 
2005     // Copy trailing qwords
2006   __ BIND(L_copy_8_bytes);
2007     __ movq(rax, Address(from, qword_count, Address::times_8, -8));
2008     __ movq(Address(to, qword_count, Address::times_8, -8), rax);


2066     // End pointers are inclusive, and if count is not zero they point
2067     // to the last unit copied:  end_to[0] := end_from[0]
2068 
2069     __ enter(); // required for proper stackwalking of RuntimeStub frame
2070     // Save no-overlap entry point for generate_conjoint_long_oop_copy()
2071     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
2072 
2073     if (entry != NULL) {
2074       *entry = __ pc();
2075       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
2076       BLOCK_COMMENT("Entry:");
2077     }
2078 
2079     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
2080                       // r9 and r10 may be used to save non-volatile registers
2081     // 'from', 'to' and 'qword_count' are now valid
2082     if (is_oop) {
2083       // Save to and count for store barrier
2084       __ movptr(saved_count, qword_count);
2085       // no registers are destroyed by this call
2086       gen_write_ref_array_pre_barrier(from, to, qword_count, dest_uninitialized);
2087     }
2088 
2089     // Copy from low to high addresses.  Use 'to' as scratch.
2090     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
2091     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
2092     __ negptr(qword_count);
2093     __ jmp(L_copy_bytes);
2094 
2095     // Copy trailing qwords
2096   __ BIND(L_copy_8_bytes);
2097     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
2098     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
2099     __ increment(qword_count);
2100     __ jcc(Assembler::notZero, L_copy_8_bytes);
2101 
2102     if (is_oop) {
2103       __ jmp(L_exit);
2104     } else {
2105       restore_arg_regs();
2106       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free


2153     const Register qword_count = rdx;  // elements count
2154     const Register saved_count = rcx;
2155 
2156     __ enter(); // required for proper stackwalking of RuntimeStub frame
2157     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
2158 
2159     if (entry != NULL) {
2160       *entry = __ pc();
2161       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
2162       BLOCK_COMMENT("Entry:");
2163     }
2164 
2165     array_overlap_test(nooverlap_target, Address::times_8);
2166     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
2167                       // r9 and r10 may be used to save non-volatile registers
2168     // 'from', 'to' and 'qword_count' are now valid
2169     if (is_oop) {
2170       // Save to and count for store barrier
2171       __ movptr(saved_count, qword_count);
2172       // No registers are destroyed by this call
2173       gen_write_ref_array_pre_barrier(from, to, saved_count, dest_uninitialized);
2174     }
2175 
2176     __ jmp(L_copy_bytes);
2177 
2178     // Copy trailing qwords
2179   __ BIND(L_copy_8_bytes);
2180     __ movq(rax, Address(from, qword_count, Address::times_8, -8));
2181     __ movq(Address(to, qword_count, Address::times_8, -8), rax);
2182     __ decrement(qword_count);
2183     __ jcc(Assembler::notZero, L_copy_8_bytes);
2184 
2185     if (is_oop) {
2186       __ jmp(L_exit);
2187     } else {
2188       restore_arg_regs();
2189       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
2190       __ xorptr(rax, rax); // return 0
2191       __ leave(); // required for proper stackwalking of RuntimeStub frame
2192       __ ret(0);
2193     }


2327 #ifdef ASSERT
2328     BLOCK_COMMENT("assert consistent ckoff/ckval");
2329     // The ckoff and ckval must be mutually consistent,
2330     // even though caller generates both.
2331     { Label L;
2332       int sco_offset = in_bytes(Klass::super_check_offset_offset());
2333       __ cmpl(ckoff, Address(ckval, sco_offset));
2334       __ jcc(Assembler::equal, L);
2335       __ stop("super_check_offset inconsistent");
2336       __ bind(L);
2337     }
2338 #endif //ASSERT
2339 
2340     // Loop-invariant addresses.  They are exclusive end pointers.
2341     Address end_from_addr(from, length, TIMES_OOP, 0);
2342     Address   end_to_addr(to,   length, TIMES_OOP, 0);
2343     // Loop-variant addresses.  They assume post-incremented count < 0.
2344     Address from_element_addr(end_from, count, TIMES_OOP, 0);
2345     Address   to_element_addr(end_to,   count, TIMES_OOP, 0);
2346 
2347     gen_write_ref_array_pre_barrier(from, to, count, dest_uninitialized);
2348 
2349     // Copy from low to high addresses, indexed from the end of each array.
2350     __ lea(end_from, end_from_addr);
2351     __ lea(end_to,   end_to_addr);
2352     __ movptr(r14_length, length);        // save a copy of the length
2353     assert(length == count, "");          // else fix next line:
2354     __ negptr(count);                     // negate and test the length
2355     __ jcc(Assembler::notZero, L_load_element);
2356 
2357     // Empty array:  Nothing to do.
2358     __ xorptr(rax, rax);                  // return 0 on (trivial) success
2359     __ jmp(L_done);
2360 
2361     // ======== begin loop ========
2362     // (Loop is rotated; its entry is L_load_element.)
2363     // Loop control:
2364     //   for (count = -count; count != 0; count++)
2365     // Base pointers src, dst are biased by 8*(count-1),to last element.
2366     __ align(OptoLoopAlignment);
2367 


< prev index next >