1 /*
   2  * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/macroAssembler.hpp"
  27 #include "gc/shared/barrierSet.hpp"
  28 #include "gc/shared/barrierSetAssembler.hpp"
  29 #include "oops/objArrayKlass.hpp"
  30 #include "runtime/sharedRuntime.hpp"
  31 #include "runtime/stubRoutines.hpp"
  32 #include "stubGenerator_x86_64.hpp"
  33 #ifdef COMPILER2
  34 #include "opto/c2_globals.hpp"
  35 #endif
  36 #if INCLUDE_JVMCI
  37 #include "jvmci/jvmci_globals.hpp"
  38 #endif
  39 
  40 #define __ _masm->
  41 
  42 #define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8)
  43 
  44 #ifdef PRODUCT
  45 #define BLOCK_COMMENT(str) /* nothing */
  46 #else
  47 #define BLOCK_COMMENT(str) __ block_comment(str)
  48 #endif // PRODUCT
  49 
  50 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  51 
  52 #ifdef PRODUCT
  53 #define INC_COUNTER_NP(counter, rscratch) ((void)0)
  54 #else
  55 #define INC_COUNTER_NP(counter, rscratch) \
  56 BLOCK_COMMENT("inc_counter " #counter); \
  57 inc_counter_np(_masm, counter, rscratch);
  58 
  59 static void inc_counter_np(MacroAssembler* _masm, uint& counter, Register rscratch) {
  60   __ incrementl(ExternalAddress((address)&counter), rscratch);
  61 }
  62 
  63 #if COMPILER2_OR_JVMCI
  64 static uint& get_profile_ctr(int shift) {
  65   if (shift == 0) {
  66     return SharedRuntime::_jbyte_array_copy_ctr;
  67   } else if (shift == 1) {
  68     return SharedRuntime::_jshort_array_copy_ctr;
  69   } else if (shift == 2) {
  70     return SharedRuntime::_jint_array_copy_ctr;
  71   } else {
  72     assert(shift == 3, "");
  73     return SharedRuntime::_jlong_array_copy_ctr;
  74   }
  75 }
  76 #endif // COMPILER2_OR_JVMCI
  77 #endif // !PRODUCT
  78 
  79 void StubGenerator::generate_arraycopy_stubs() {
  80   address entry;
  81   address entry_jbyte_arraycopy;
  82   address entry_jshort_arraycopy;
  83   address entry_jint_arraycopy;
  84   address entry_oop_arraycopy;
  85   address entry_jlong_arraycopy;
  86   address entry_checkcast_arraycopy;
  87 
  88   StubRoutines::_jbyte_disjoint_arraycopy  = generate_disjoint_byte_copy(false, &entry,
  89                                                                          "jbyte_disjoint_arraycopy");
  90   StubRoutines::_jbyte_arraycopy           = generate_conjoint_byte_copy(false, entry, &entry_jbyte_arraycopy,
  91                                                                          "jbyte_arraycopy");
  92 
  93   StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_short_copy(false, &entry,
  94                                                                           "jshort_disjoint_arraycopy");
  95   StubRoutines::_jshort_arraycopy          = generate_conjoint_short_copy(false, entry, &entry_jshort_arraycopy,
  96                                                                           "jshort_arraycopy");
  97 
  98   StubRoutines::_jint_disjoint_arraycopy   = generate_disjoint_int_oop_copy(false, false, &entry,
  99                                                                             "jint_disjoint_arraycopy");
 100   StubRoutines::_jint_arraycopy            = generate_conjoint_int_oop_copy(false, false, entry,
 101                                                                             &entry_jint_arraycopy, "jint_arraycopy");
 102 
 103   StubRoutines::_jlong_disjoint_arraycopy  = generate_disjoint_long_oop_copy(false, false, &entry,
 104                                                                              "jlong_disjoint_arraycopy");
 105   StubRoutines::_jlong_arraycopy           = generate_conjoint_long_oop_copy(false, false, entry,
 106                                                                              &entry_jlong_arraycopy, "jlong_arraycopy");
 107   if (UseCompressedOops) {
 108     StubRoutines::_oop_disjoint_arraycopy  = generate_disjoint_int_oop_copy(false, true, &entry,
 109                                                                             "oop_disjoint_arraycopy");
 110     StubRoutines::_oop_arraycopy           = generate_conjoint_int_oop_copy(false, true, entry,
 111                                                                             &entry_oop_arraycopy, "oop_arraycopy");
 112     StubRoutines::_oop_disjoint_arraycopy_uninit  = generate_disjoint_int_oop_copy(false, true, &entry,
 113                                                                                    "oop_disjoint_arraycopy_uninit",
 114                                                                                    /*dest_uninitialized*/true);
 115     StubRoutines::_oop_arraycopy_uninit           = generate_conjoint_int_oop_copy(false, true, entry,
 116                                                                                    nullptr, "oop_arraycopy_uninit",
 117                                                                                    /*dest_uninitialized*/true);
 118   } else {
 119     StubRoutines::_oop_disjoint_arraycopy  = generate_disjoint_long_oop_copy(false, true, &entry,
 120                                                                              "oop_disjoint_arraycopy");
 121     StubRoutines::_oop_arraycopy           = generate_conjoint_long_oop_copy(false, true, entry,
 122                                                                              &entry_oop_arraycopy, "oop_arraycopy");
 123     StubRoutines::_oop_disjoint_arraycopy_uninit  = generate_disjoint_long_oop_copy(false, true, &entry,
 124                                                                                     "oop_disjoint_arraycopy_uninit",
 125                                                                                     /*dest_uninitialized*/true);
 126     StubRoutines::_oop_arraycopy_uninit           = generate_conjoint_long_oop_copy(false, true, entry,
 127                                                                                     nullptr, "oop_arraycopy_uninit",
 128                                                                                     /*dest_uninitialized*/true);
 129   }
 130 
 131   StubRoutines::_checkcast_arraycopy        = generate_checkcast_copy("checkcast_arraycopy", &entry_checkcast_arraycopy);
 132   StubRoutines::_checkcast_arraycopy_uninit = generate_checkcast_copy("checkcast_arraycopy_uninit", nullptr,
 133                                                                       /*dest_uninitialized*/true);
 134 
 135   StubRoutines::_unsafe_arraycopy    = generate_unsafe_copy("unsafe_arraycopy",
 136                                                             entry_jbyte_arraycopy,
 137                                                             entry_jshort_arraycopy,
 138                                                             entry_jint_arraycopy,
 139                                                             entry_jlong_arraycopy);
 140   StubRoutines::_generic_arraycopy   = generate_generic_copy("generic_arraycopy",
 141                                                              entry_jbyte_arraycopy,
 142                                                              entry_jshort_arraycopy,
 143                                                              entry_jint_arraycopy,
 144                                                              entry_oop_arraycopy,
 145                                                              entry_jlong_arraycopy,
 146                                                              entry_checkcast_arraycopy);
 147 
 148   StubRoutines::_jbyte_fill = generate_fill(T_BYTE, false, "jbyte_fill");
 149   StubRoutines::_jshort_fill = generate_fill(T_SHORT, false, "jshort_fill");
 150   StubRoutines::_jint_fill = generate_fill(T_INT, false, "jint_fill");
 151   StubRoutines::_arrayof_jbyte_fill = generate_fill(T_BYTE, true, "arrayof_jbyte_fill");
 152   StubRoutines::_arrayof_jshort_fill = generate_fill(T_SHORT, true, "arrayof_jshort_fill");
 153   StubRoutines::_arrayof_jint_fill = generate_fill(T_INT, true, "arrayof_jint_fill");
 154 
 155   // We don't generate specialized code for HeapWord-aligned source
 156   // arrays, so just use the code we've already generated
 157   StubRoutines::_arrayof_jbyte_disjoint_arraycopy  = StubRoutines::_jbyte_disjoint_arraycopy;
 158   StubRoutines::_arrayof_jbyte_arraycopy           = StubRoutines::_jbyte_arraycopy;
 159 
 160   StubRoutines::_arrayof_jshort_disjoint_arraycopy = StubRoutines::_jshort_disjoint_arraycopy;
 161   StubRoutines::_arrayof_jshort_arraycopy          = StubRoutines::_jshort_arraycopy;
 162 
 163   StubRoutines::_arrayof_jint_disjoint_arraycopy   = StubRoutines::_jint_disjoint_arraycopy;
 164   StubRoutines::_arrayof_jint_arraycopy            = StubRoutines::_jint_arraycopy;
 165 
 166   StubRoutines::_arrayof_jlong_disjoint_arraycopy  = StubRoutines::_jlong_disjoint_arraycopy;
 167   StubRoutines::_arrayof_jlong_arraycopy           = StubRoutines::_jlong_arraycopy;
 168 
 169   StubRoutines::_arrayof_oop_disjoint_arraycopy    = StubRoutines::_oop_disjoint_arraycopy;
 170   StubRoutines::_arrayof_oop_arraycopy             = StubRoutines::_oop_arraycopy;
 171 
 172   StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit    = StubRoutines::_oop_disjoint_arraycopy_uninit;
 173   StubRoutines::_arrayof_oop_arraycopy_uninit             = StubRoutines::_oop_arraycopy_uninit;
 174 }
 175 
 176 
 177 // Verify that a register contains clean 32-bits positive value
 178 // (high 32-bits are 0) so it could be used in 64-bits shifts.
 179 //
 180 //  Input:
 181 //    Rint  -  32-bits value
 182 //    Rtmp  -  scratch
 183 //
 184 void StubGenerator::assert_clean_int(Register Rint, Register Rtmp) {
 185 #ifdef ASSERT
 186   Label L;
 187   assert_different_registers(Rtmp, Rint);
 188   __ movslq(Rtmp, Rint);
 189   __ cmpq(Rtmp, Rint);
 190   __ jcc(Assembler::equal, L);
 191   __ stop("high 32-bits of int value are not 0");
 192   __ bind(L);
 193 #endif
 194 }
 195 
 196 
 197 //  Generate overlap test for array copy stubs
 198 //
 199 //  Input:
 200 //     c_rarg0 - from
 201 //     c_rarg1 - to
 202 //     c_rarg2 - element count
 203 //
 204 //  Output:
 205 //     rax   - &from[element count - 1]
 206 //
 207 void StubGenerator::array_overlap_test(address no_overlap_target, Label* NOLp, Address::ScaleFactor sf) {
 208   const Register from     = c_rarg0;
 209   const Register to       = c_rarg1;
 210   const Register count    = c_rarg2;
 211   const Register end_from = rax;
 212 
 213   __ cmpptr(to, from);
 214   __ lea(end_from, Address(from, count, sf, 0));
 215   if (NOLp == nullptr) {
 216     ExternalAddress no_overlap(no_overlap_target);
 217     __ jump_cc(Assembler::belowEqual, no_overlap);
 218     __ cmpptr(to, end_from);
 219     __ jump_cc(Assembler::aboveEqual, no_overlap);
 220   } else {
 221     __ jcc(Assembler::belowEqual, (*NOLp));
 222     __ cmpptr(to, end_from);
 223     __ jcc(Assembler::aboveEqual, (*NOLp));
 224   }
 225 }
 226 
 227 
 228 // Copy big chunks forward
 229 //
 230 // Inputs:
 231 //   end_from     - source arrays end address
 232 //   end_to       - destination array end address
 233 //   qword_count  - 64-bits element count, negative
 234 //   tmp1         - scratch
 235 //   L_copy_bytes - entry label
 236 //   L_copy_8_bytes  - exit  label
 237 //
 238 void StubGenerator::copy_bytes_forward(Register end_from, Register end_to,
 239                                        Register qword_count, Register tmp1,
 240                                        Register tmp2, Label& L_copy_bytes,
 241                                        Label& L_copy_8_bytes, DecoratorSet decorators,
 242                                        BasicType type) {
 243   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 244   DEBUG_ONLY(__ stop("enter at entry label, not here"));
 245   Label L_loop;
 246   __ align(OptoLoopAlignment);
 247   if (UseUnalignedLoadStores) {
 248     Label L_end;
 249     __ BIND(L_loop);
 250     if (UseAVX >= 2) {
 251       bs->copy_load_at(_masm, decorators, type, 32,
 252                        xmm0, Address(end_from, qword_count, Address::times_8, -56),
 253                        tmp1, xmm1);
 254       bs->copy_store_at(_masm, decorators, type, 32,
 255                         Address(end_to, qword_count, Address::times_8, -56), xmm0,
 256                         tmp1, tmp2, xmm1);
 257 
 258       bs->copy_load_at(_masm, decorators, type, 32,
 259                        xmm0, Address(end_from, qword_count, Address::times_8, -24),
 260                        tmp1, xmm1);
 261       bs->copy_store_at(_masm, decorators, type, 32,
 262                         Address(end_to, qword_count, Address::times_8, -24), xmm0,
 263                         tmp1, tmp2, xmm1);
 264     } else {
 265       bs->copy_load_at(_masm, decorators, type, 16,
 266                        xmm0, Address(end_from, qword_count, Address::times_8, -56),
 267                        tmp1, xmm1);
 268       bs->copy_store_at(_masm, decorators, type, 16,
 269                         Address(end_to, qword_count, Address::times_8, -56), xmm0,
 270                         tmp1, tmp2, xmm1);
 271       bs->copy_load_at(_masm, decorators, type, 16,
 272                        xmm0, Address(end_from, qword_count, Address::times_8, -40),
 273                        tmp1, xmm1);
 274       bs->copy_store_at(_masm, decorators, type, 16,
 275                         Address(end_to, qword_count, Address::times_8, -40), xmm0,
 276                         tmp1, tmp2, xmm1);
 277       bs->copy_load_at(_masm, decorators, type, 16,
 278                        xmm0, Address(end_from, qword_count, Address::times_8, -24),
 279                        tmp1, xmm1);
 280       bs->copy_store_at(_masm, decorators, type, 16,
 281                         Address(end_to, qword_count, Address::times_8, -24), xmm0,
 282                         tmp1, tmp2, xmm1);
 283       bs->copy_load_at(_masm, decorators, type, 16,
 284                        xmm0, Address(end_from, qword_count, Address::times_8, -8),
 285                        tmp1, xmm1);
 286       bs->copy_store_at(_masm, decorators, type, 16,
 287                         Address(end_to, qword_count, Address::times_8, -8), xmm0,
 288                         tmp1, tmp2, xmm1);
 289     }
 290 
 291     __ BIND(L_copy_bytes);
 292     __ addptr(qword_count, 8);
 293     __ jcc(Assembler::lessEqual, L_loop);
 294     __ subptr(qword_count, 4);  // sub(8) and add(4)
 295     __ jcc(Assembler::greater, L_end);
 296     // Copy trailing 32 bytes
 297     if (UseAVX >= 2) {
 298       bs->copy_load_at(_masm, decorators, type, 32,
 299                        xmm0, Address(end_from, qword_count, Address::times_8, -24),
 300                        tmp1, xmm1);
 301       bs->copy_store_at(_masm, decorators, type, 32,
 302                         Address(end_to, qword_count, Address::times_8, -24), xmm0,
 303                         tmp1, tmp2, xmm1);
 304     } else {
 305       bs->copy_load_at(_masm, decorators, type, 16,
 306                        xmm0, Address(end_from, qword_count, Address::times_8, -24),
 307                        tmp1, xmm1);
 308       bs->copy_store_at(_masm, decorators, type, 16,
 309                         Address(end_to, qword_count, Address::times_8, -24), xmm0,
 310                         tmp1, tmp2, xmm1);
 311       bs->copy_load_at(_masm, decorators, type, 16,
 312                        xmm0, Address(end_from, qword_count, Address::times_8, -8),
 313                        tmp1, xmm1);
 314       bs->copy_store_at(_masm, decorators, type, 16,
 315                         Address(end_to, qword_count, Address::times_8, -8), xmm0,
 316                         tmp1, tmp2, xmm1);
 317     }
 318     __ addptr(qword_count, 4);
 319     __ BIND(L_end);
 320   } else {
 321     // Copy 32-bytes per iteration
 322     __ BIND(L_loop);
 323     bs->copy_load_at(_masm, decorators, type, 8,
 324                      tmp1, Address(end_from, qword_count, Address::times_8, -24),
 325                      tmp2);
 326     bs->copy_store_at(_masm, decorators, type, 8,
 327                       Address(end_to, qword_count, Address::times_8, -24), tmp1,
 328                       tmp2);
 329     bs->copy_load_at(_masm, decorators, type, 8,
 330                      tmp1, Address(end_from, qword_count, Address::times_8, -16),
 331                      tmp2);
 332     bs->copy_store_at(_masm, decorators, type, 8,
 333                       Address(end_to, qword_count, Address::times_8, -16), tmp1,
 334                       tmp2);
 335     bs->copy_load_at(_masm, decorators, type, 8,
 336                      tmp1, Address(end_from, qword_count, Address::times_8, -8),
 337                      tmp2);
 338     bs->copy_store_at(_masm, decorators, type, 8,
 339                       Address(end_to, qword_count, Address::times_8, -8), tmp1,
 340                       tmp2);
 341     bs->copy_load_at(_masm, decorators, type, 8,
 342                      tmp1, Address(end_from, qword_count, Address::times_8, 0),
 343                      tmp2);
 344     bs->copy_store_at(_masm, decorators, type, 8,
 345                       Address(end_to, qword_count, Address::times_8, 0), tmp1,
 346                       tmp2);
 347 
 348     __ BIND(L_copy_bytes);
 349     __ addptr(qword_count, 4);
 350     __ jcc(Assembler::lessEqual, L_loop);
 351   }
 352   __ subptr(qword_count, 4);
 353   __ jcc(Assembler::less, L_copy_8_bytes); // Copy trailing qwords
 354 }
 355 
 356 
 357 // Copy big chunks backward
 358 //
 359 // Inputs:
 360 //   from         - source arrays address
 361 //   dest         - destination array address
 362 //   qword_count  - 64-bits element count
 363 //   tmp1         - scratch
 364 //   L_copy_bytes - entry label
 365 //   L_copy_8_bytes  - exit  label
 366 //
 367 void StubGenerator::copy_bytes_backward(Register from, Register dest,
 368                                         Register qword_count, Register tmp1,
 369                                         Register tmp2, Label& L_copy_bytes,
 370                                         Label& L_copy_8_bytes, DecoratorSet decorators,
 371                                         BasicType type) {
 372   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 373   DEBUG_ONLY(__ stop("enter at entry label, not here"));
 374   Label L_loop;
 375   __ align(OptoLoopAlignment);
 376   if (UseUnalignedLoadStores) {
 377     Label L_end;
 378     __ BIND(L_loop);
 379     if (UseAVX >= 2) {
 380       bs->copy_load_at(_masm, decorators, type, 32,
 381                        xmm0, Address(from, qword_count, Address::times_8, 32),
 382                        tmp1, xmm1);
 383       bs->copy_store_at(_masm, decorators, type, 32,
 384                         Address(dest, qword_count, Address::times_8, 32), xmm0,
 385                         tmp1, tmp2, xmm1);
 386       bs->copy_load_at(_masm, decorators, type, 32,
 387                        xmm0, Address(from, qword_count, Address::times_8, 0),
 388                        tmp1, xmm1);
 389       bs->copy_store_at(_masm, decorators, type, 32,
 390                         Address(dest, qword_count, Address::times_8, 0), xmm0,
 391                         tmp1, tmp2, xmm1);
 392     } else {
 393       bs->copy_load_at(_masm, decorators, type, 16,
 394                        xmm0, Address(from, qword_count, Address::times_8, 48),
 395                        tmp1, xmm1);
 396       bs->copy_store_at(_masm, decorators, type, 16,
 397                         Address(dest, qword_count, Address::times_8, 48), xmm0,
 398                         tmp1, tmp2, xmm1);
 399       bs->copy_load_at(_masm, decorators, type, 16,
 400                        xmm0, Address(from, qword_count, Address::times_8, 32),
 401                        tmp1, xmm1);
 402       bs->copy_store_at(_masm, decorators, type, 16,
 403                         Address(dest, qword_count, Address::times_8, 32), xmm0,
 404                         tmp1, tmp2, xmm1);
 405       bs->copy_load_at(_masm, decorators, type, 16,
 406                        xmm0, Address(from, qword_count, Address::times_8, 16),
 407                        tmp1, xmm1);
 408       bs->copy_store_at(_masm, decorators, type, 16,
 409                         Address(dest, qword_count, Address::times_8, 16), xmm0,
 410                         tmp1, tmp2, xmm1);
 411       bs->copy_load_at(_masm, decorators, type, 16,
 412                        xmm0, Address(from, qword_count, Address::times_8, 0),
 413                        tmp1, xmm1);
 414       bs->copy_store_at(_masm, decorators, type, 16,
 415                         Address(dest, qword_count, Address::times_8, 0), xmm0,
 416                         tmp1, tmp2, xmm1);
 417     }
 418 
 419     __ BIND(L_copy_bytes);
 420     __ subptr(qword_count, 8);
 421     __ jcc(Assembler::greaterEqual, L_loop);
 422 
 423     __ addptr(qword_count, 4);  // add(8) and sub(4)
 424     __ jcc(Assembler::less, L_end);
 425     // Copy trailing 32 bytes
 426     if (UseAVX >= 2) {
 427       bs->copy_load_at(_masm, decorators, type, 32,
 428                        xmm0, Address(from, qword_count, Address::times_8, 0),
 429                        tmp1, xmm1);
 430       bs->copy_store_at(_masm, decorators, type, 32,
 431                         Address(dest, qword_count, Address::times_8, 0), xmm0,
 432                         tmp1, tmp2, xmm1);
 433     } else {
 434       bs->copy_load_at(_masm, decorators, type, 16,
 435                        xmm0, Address(from, qword_count, Address::times_8, 16),
 436                        tmp1, xmm1);
 437       bs->copy_store_at(_masm, decorators, type, 16,
 438                         Address(dest, qword_count, Address::times_8, 16), xmm0,
 439                         tmp1, tmp2, xmm1);
 440       bs->copy_load_at(_masm, decorators, type, 16,
 441                        xmm0, Address(from, qword_count, Address::times_8, 0),
 442                        tmp1, xmm1);
 443       bs->copy_store_at(_masm, decorators, type, 16,
 444                         Address(dest, qword_count, Address::times_8, 0), xmm0,
 445                         tmp1, tmp2, xmm1);
 446     }
 447     __ subptr(qword_count, 4);
 448     __ BIND(L_end);
 449   } else {
 450     // Copy 32-bytes per iteration
 451     __ BIND(L_loop);
 452     bs->copy_load_at(_masm, decorators, type, 8,
 453                      tmp1, Address(from, qword_count, Address::times_8, 24),
 454                      tmp2);
 455     bs->copy_store_at(_masm, decorators, type, 8,
 456                       Address(dest, qword_count, Address::times_8, 24), tmp1,
 457                       tmp2);
 458     bs->copy_load_at(_masm, decorators, type, 8,
 459                      tmp1, Address(from, qword_count, Address::times_8, 16),
 460                      tmp2);
 461     bs->copy_store_at(_masm, decorators, type, 8,
 462                       Address(dest, qword_count, Address::times_8, 16), tmp1,
 463                       tmp2);
 464     bs->copy_load_at(_masm, decorators, type, 8,
 465                      tmp1, Address(from, qword_count, Address::times_8, 8),
 466                      tmp2);
 467     bs->copy_store_at(_masm, decorators, type, 8,
 468                       Address(dest, qword_count, Address::times_8, 8), tmp1,
 469                       tmp2);
 470     bs->copy_load_at(_masm, decorators, type, 8,
 471                      tmp1, Address(from, qword_count, Address::times_8, 0),
 472                      tmp2);
 473     bs->copy_store_at(_masm, decorators, type, 8,
 474                       Address(dest, qword_count, Address::times_8, 0), tmp1,
 475                       tmp2);
 476 
 477     __ BIND(L_copy_bytes);
 478     __ subptr(qword_count, 4);
 479     __ jcc(Assembler::greaterEqual, L_loop);
 480   }
 481   __ addptr(qword_count, 4);
 482   __ jcc(Assembler::greater, L_copy_8_bytes); // Copy trailing qwords
 483 }
 484 
 485 #if COMPILER2_OR_JVMCI
 486 
 487 // Note: Following rules apply to AVX3 optimized arraycopy stubs:-
 488 // - If target supports AVX3 features (BW+VL+F) then implementation uses 32 byte vectors (YMMs)
 489 //   for both special cases (various small block sizes) and aligned copy loop. This is the
 490 //   default configuration.
 491 // - If copy length is above AVX3Threshold, then implementation use 64 byte vectors (ZMMs)
 492 //   for main copy loop (and subsequent tail) since bulk of the cycles will be consumed in it.
 493 // - If user forces MaxVectorSize=32 then above 4096 bytes its seen that REP MOVs shows a
 494 //   better performance for disjoint copies. For conjoint/backward copy vector based
 495 //   copy performs better.
 496 // - If user sets AVX3Threshold=0, then special cases for small blocks sizes operate over
 497 //   64 byte vector registers (ZMMs).
 498 
 499 // Inputs:
 500 //   c_rarg0   - source array address
 501 //   c_rarg1   - destination array address
 502 //   c_rarg2   - element count, treated as ssize_t, can be zero
 503 //
 504 //
 505 // Side Effects:
 506 //   disjoint_copy_avx3_masked is set to the no-overlap entry point
 507 //   used by generate_conjoint_[byte/int/short/long]_copy().
 508 //
 509 address StubGenerator::generate_disjoint_copy_avx3_masked(address* entry, const char *name,
 510                                                           int shift, bool aligned, bool is_oop,
 511                                                           bool dest_uninitialized) {
 512   __ align(CodeEntryAlignment);
 513   StubCodeMark mark(this, "StubRoutines", name);
 514   address start = __ pc();
 515 
 516   int avx3threshold = VM_Version::avx3_threshold();
 517   bool use64byteVector = (MaxVectorSize > 32) && (avx3threshold == 0);
 518   const int large_threshold = 2621440; // 2.5 MB
 519   Label L_main_loop, L_main_loop_64bytes, L_tail, L_tail64, L_exit, L_entry;
 520   Label L_repmovs, L_main_pre_loop, L_main_pre_loop_64bytes, L_pre_main_post_64;
 521   Label L_copy_large, L_finish;
 522   const Register from        = rdi;  // source array address
 523   const Register to          = rsi;  // destination array address
 524   const Register count       = rdx;  // elements count
 525   const Register temp1       = r8;
 526   const Register temp2       = r11;
 527   const Register temp3       = rax;
 528   const Register temp4       = rcx;
 529   // End pointers are inclusive, and if count is not zero they point
 530   // to the last unit copied:  end_to[0] := end_from[0]
 531 
 532   __ enter(); // required for proper stackwalking of RuntimeStub frame
 533   assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
 534 
 535   if (entry != nullptr) {
 536     *entry = __ pc();
 537      // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
 538     BLOCK_COMMENT("Entry:");
 539   }
 540 
 541   BasicType type_vec[] = { T_BYTE,  T_SHORT,  T_INT,   T_LONG};
 542   BasicType type = is_oop ? T_OBJECT : type_vec[shift];
 543 
 544   setup_argument_regs(type);
 545 
 546   DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT;
 547   if (dest_uninitialized) {
 548     decorators |= IS_DEST_UNINITIALIZED;
 549   }
 550   if (aligned) {
 551     decorators |= ARRAYCOPY_ALIGNED;
 552   }
 553   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
 554   bs->arraycopy_prologue(_masm, decorators, type, from, to, count);
 555 
 556   {
 557     // Type(shift)           byte(0), short(1), int(2),   long(3)
 558     int loop_size[]        = { 192,     96,       48,      24};
 559     int threshold[]        = { 4096,    2048,     1024,    512};
 560 
 561     // UnsafeCopyMemory page error: continue after ucm
 562     UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
 563     // 'from', 'to' and 'count' are now valid
 564 
 565     // temp1 holds remaining count and temp4 holds running count used to compute
 566     // next address offset for start of to/from addresses (temp4 * scale).
 567     __ mov64(temp4, 0);
 568     __ movq(temp1, count);
 569 
 570     // Zero length check.
 571     __ BIND(L_tail);
 572     __ cmpq(temp1, 0);
 573     __ jcc(Assembler::lessEqual, L_exit);
 574 
 575     // Special cases using 32 byte [masked] vector copy operations.
 576     arraycopy_avx3_special_cases(xmm1, k2, from, to, temp1, shift,
 577                                  temp4, temp3, use64byteVector, L_entry, L_exit);
 578 
 579     // PRE-MAIN-POST loop for aligned copy.
 580     __ BIND(L_entry);
 581 
 582     if (MaxVectorSize == 64) {
 583       __ movq(temp2, temp1);
 584       __ shlq(temp2, shift);
 585       __ cmpq(temp2, large_threshold);
 586       __ jcc(Assembler::greaterEqual, L_copy_large);
 587     }
 588     if (avx3threshold != 0) {
 589       __ cmpq(count, threshold[shift]);
 590       if (MaxVectorSize == 64) {
 591         // Copy using 64 byte vectors.
 592         __ jcc(Assembler::greaterEqual, L_pre_main_post_64);
 593       } else {
 594         assert(MaxVectorSize < 64, "vector size should be < 64 bytes");
 595         // REP MOVS offer a faster copy path.
 596         __ jcc(Assembler::greaterEqual, L_repmovs);
 597       }
 598     }
 599 
 600     if ((MaxVectorSize < 64)  || (avx3threshold != 0)) {
 601       // Partial copy to make dst address 32 byte aligned.
 602       __ movq(temp2, to);
 603       __ andq(temp2, 31);
 604       __ jcc(Assembler::equal, L_main_pre_loop);
 605 
 606       __ negptr(temp2);
 607       __ addq(temp2, 32);
 608       if (shift) {
 609         __ shrq(temp2, shift);
 610       }
 611       __ movq(temp3, temp2);
 612       copy32_masked_avx(to, from, xmm1, k2, temp3, temp4, temp1, shift);
 613       __ movq(temp4, temp2);
 614       __ movq(temp1, count);
 615       __ subq(temp1, temp2);
 616 
 617       __ cmpq(temp1, loop_size[shift]);
 618       __ jcc(Assembler::less, L_tail);
 619 
 620       __ BIND(L_main_pre_loop);
 621       __ subq(temp1, loop_size[shift]);
 622 
 623       // Main loop with aligned copy block size of 192 bytes at 32 byte granularity.
 624       __ align32();
 625       __ BIND(L_main_loop);
 626          copy64_avx(to, from, temp4, xmm1, false, shift, 0);
 627          copy64_avx(to, from, temp4, xmm1, false, shift, 64);
 628          copy64_avx(to, from, temp4, xmm1, false, shift, 128);
 629          __ addptr(temp4, loop_size[shift]);
 630          __ subq(temp1, loop_size[shift]);
 631          __ jcc(Assembler::greater, L_main_loop);
 632 
 633       __ addq(temp1, loop_size[shift]);
 634 
 635       // Tail loop.
 636       __ jmp(L_tail);
 637 
 638       __ BIND(L_repmovs);
 639         __ movq(temp2, temp1);
 640         // Swap to(RSI) and from(RDI) addresses to comply with REP MOVs semantics.
 641         __ movq(temp3, to);
 642         __ movq(to,  from);
 643         __ movq(from, temp3);
 644         // Save to/from for restoration post rep_mov.
 645         __ movq(temp1, to);
 646         __ movq(temp3, from);
 647         if(shift < 3) {
 648           __ shrq(temp2, 3-shift);     // quad word count
 649         }
 650         __ movq(temp4 , temp2);        // move quad ward count into temp4(RCX).
 651         __ rep_mov();
 652         __ shlq(temp2, 3);             // convert quad words into byte count.
 653         if(shift) {
 654           __ shrq(temp2, shift);       // type specific count.
 655         }
 656         // Restore original addresses in to/from.
 657         __ movq(to, temp3);
 658         __ movq(from, temp1);
 659         __ movq(temp4, temp2);
 660         __ movq(temp1, count);
 661         __ subq(temp1, temp2);         // tailing part (less than a quad ward size).
 662         __ jmp(L_tail);
 663     }
 664 
 665     if (MaxVectorSize > 32) {
 666       __ BIND(L_pre_main_post_64);
 667       // Partial copy to make dst address 64 byte aligned.
 668       __ movq(temp2, to);
 669       __ andq(temp2, 63);
 670       __ jcc(Assembler::equal, L_main_pre_loop_64bytes);
 671 
 672       __ negptr(temp2);
 673       __ addq(temp2, 64);
 674       if (shift) {
 675         __ shrq(temp2, shift);
 676       }
 677       __ movq(temp3, temp2);
 678       copy64_masked_avx(to, from, xmm1, k2, temp3, temp4, temp1, shift, 0 , true);
 679       __ movq(temp4, temp2);
 680       __ movq(temp1, count);
 681       __ subq(temp1, temp2);
 682 
 683       __ cmpq(temp1, loop_size[shift]);
 684       __ jcc(Assembler::less, L_tail64);
 685 
 686       __ BIND(L_main_pre_loop_64bytes);
 687       __ subq(temp1, loop_size[shift]);
 688 
 689       // Main loop with aligned copy block size of 192 bytes at
 690       // 64 byte copy granularity.
 691       __ align32();
 692       __ BIND(L_main_loop_64bytes);
 693          copy64_avx(to, from, temp4, xmm1, false, shift, 0 , true);
 694          copy64_avx(to, from, temp4, xmm1, false, shift, 64, true);
 695          copy64_avx(to, from, temp4, xmm1, false, shift, 128, true);
 696          __ addptr(temp4, loop_size[shift]);
 697          __ subq(temp1, loop_size[shift]);
 698          __ jcc(Assembler::greater, L_main_loop_64bytes);
 699 
 700       __ addq(temp1, loop_size[shift]);
 701       // Zero length check.
 702       __ jcc(Assembler::lessEqual, L_exit);
 703 
 704       __ BIND(L_tail64);
 705 
 706       // Tail handling using 64 byte [masked] vector copy operations.
 707       use64byteVector = true;
 708       arraycopy_avx3_special_cases(xmm1, k2, from, to, temp1, shift,
 709                                    temp4, temp3, use64byteVector, L_entry, L_exit);
 710     }
 711     __ BIND(L_exit);
 712   }
 713 
 714   __ BIND(L_finish);
 715   address ucme_exit_pc = __ pc();
 716   // When called from generic_arraycopy r11 contains specific values
 717   // used during arraycopy epilogue, re-initializing r11.
 718   if (is_oop) {
 719     __ movq(r11, shift == 3 ? count : to);
 720   }
 721   bs->arraycopy_epilogue(_masm, decorators, type, from, to, count);
 722   restore_argument_regs(type);
 723   INC_COUNTER_NP(get_profile_ctr(shift), rscratch1); // Update counter after rscratch1 is free
 724   __ xorptr(rax, rax); // return 0
 725   __ vzeroupper();
 726   __ leave(); // required for proper stackwalking of RuntimeStub frame
 727   __ ret(0);
 728 
 729   if (MaxVectorSize == 64) {
 730     __ BIND(L_copy_large);
 731     arraycopy_avx3_large(to, from, temp1, temp2, temp3, temp4, count, xmm1, xmm2, xmm3, xmm4, shift);
 732     __ jmp(L_finish);
 733   }
 734   return start;
 735 }
 736 
 737 void StubGenerator::arraycopy_avx3_large(Register to, Register from, Register temp1, Register temp2,
 738                                          Register temp3, Register temp4, Register count,
 739                                          XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
 740                                          XMMRegister xmm4, int shift) {
 741 
 742   // Type(shift)           byte(0), short(1), int(2),   long(3)
 743   int loop_size[]        = { 256,     128,       64,      32};
 744   int threshold[]        = { 4096,    2048,     1024,    512};
 745 
 746   Label L_main_loop_large;
 747   Label L_tail_large;
 748   Label L_exit_large;
 749   Label L_entry_large;
 750   Label L_main_pre_loop_large;
 751   Label L_pre_main_post_large;
 752 
 753   assert(MaxVectorSize == 64, "vector length != 64");
 754   __ BIND(L_entry_large);
 755 
 756   __ BIND(L_pre_main_post_large);
 757   // Partial copy to make dst address 64 byte aligned.
 758   __ movq(temp2, to);
 759   __ andq(temp2, 63);
 760   __ jcc(Assembler::equal, L_main_pre_loop_large);
 761 
 762   __ negptr(temp2);
 763   __ addq(temp2, 64);
 764   if (shift) {
 765     __ shrq(temp2, shift);
 766   }
 767   __ movq(temp3, temp2);
 768   copy64_masked_avx(to, from, xmm1, k2, temp3, temp4, temp1, shift, 0, true);
 769   __ movq(temp4, temp2);
 770   __ movq(temp1, count);
 771   __ subq(temp1, temp2);
 772 
 773   __ cmpq(temp1, loop_size[shift]);
 774   __ jcc(Assembler::less, L_tail_large);
 775 
 776   __ BIND(L_main_pre_loop_large);
 777   __ subq(temp1, loop_size[shift]);
 778 
 779   // Main loop with aligned copy block size of 256 bytes at 64 byte copy granularity.
 780   __ align32();
 781   __ BIND(L_main_loop_large);
 782   copy256_avx3(to, from, temp4, xmm1, xmm2, xmm3, xmm4, shift, 0);
 783   __ addptr(temp4, loop_size[shift]);
 784   __ subq(temp1, loop_size[shift]);
 785   __ jcc(Assembler::greater, L_main_loop_large);
 786   // fence needed because copy256_avx3 uses non-temporal stores
 787   __ sfence();
 788 
 789   __ addq(temp1, loop_size[shift]);
 790   // Zero length check.
 791   __ jcc(Assembler::lessEqual, L_exit_large);
 792   __ BIND(L_tail_large);
 793   // Tail handling using 64 byte [masked] vector copy operations.
 794   __ cmpq(temp1, 0);
 795   __ jcc(Assembler::lessEqual, L_exit_large);
 796   arraycopy_avx3_special_cases_256(xmm1, k2, from, to, temp1, shift,
 797                                temp4, temp3, L_exit_large);
 798   __ BIND(L_exit_large);
 799 }
 800 
 801 // Inputs:
 802 //   c_rarg0   - source array address
 803 //   c_rarg1   - destination array address
 804 //   c_rarg2   - element count, treated as ssize_t, can be zero
 805 //
 806 //
 807 address StubGenerator::generate_conjoint_copy_avx3_masked(address* entry, const char *name, int shift,
 808                                                           address nooverlap_target, bool aligned,
 809                                                           bool is_oop, bool dest_uninitialized) {
 810   __ align(CodeEntryAlignment);
 811   StubCodeMark mark(this, "StubRoutines", name);
 812   address start = __ pc();
 813 
 814   int avx3threshold = VM_Version::avx3_threshold();
 815   bool use64byteVector = (MaxVectorSize > 32) && (avx3threshold == 0);
 816 
 817   Label L_main_pre_loop, L_main_pre_loop_64bytes, L_pre_main_post_64;
 818   Label L_main_loop, L_main_loop_64bytes, L_tail, L_tail64, L_exit, L_entry;
 819   const Register from        = rdi;  // source array address
 820   const Register to          = rsi;  // destination array address
 821   const Register count       = rdx;  // elements count
 822   const Register temp1       = r8;
 823   const Register temp2       = rcx;
 824   const Register temp3       = r11;
 825   const Register temp4       = rax;
 826   // End pointers are inclusive, and if count is not zero they point
 827   // to the last unit copied:  end_to[0] := end_from[0]
 828 
 829   __ enter(); // required for proper stackwalking of RuntimeStub frame
 830   assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
 831 
 832   if (entry != nullptr) {
 833     *entry = __ pc();
 834      // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
 835     BLOCK_COMMENT("Entry:");
 836   }
 837 
 838   array_overlap_test(nooverlap_target, (Address::ScaleFactor)(shift));
 839 
 840   BasicType type_vec[] = { T_BYTE,  T_SHORT,  T_INT,   T_LONG};
 841   BasicType type = is_oop ? T_OBJECT : type_vec[shift];
 842 
 843   setup_argument_regs(type);
 844 
 845   DecoratorSet decorators = IN_HEAP | IS_ARRAY;
 846   if (dest_uninitialized) {
 847     decorators |= IS_DEST_UNINITIALIZED;
 848   }
 849   if (aligned) {
 850     decorators |= ARRAYCOPY_ALIGNED;
 851   }
 852   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
 853   bs->arraycopy_prologue(_masm, decorators, type, from, to, count);
 854   {
 855     // Type(shift)       byte(0), short(1), int(2),   long(3)
 856     int loop_size[]   = { 192,     96,       48,      24};
 857     int threshold[]   = { 4096,    2048,     1024,    512};
 858 
 859     // UnsafeCopyMemory page error: continue after ucm
 860     UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
 861     // 'from', 'to' and 'count' are now valid
 862 
 863     // temp1 holds remaining count.
 864     __ movq(temp1, count);
 865 
 866     // Zero length check.
 867     __ BIND(L_tail);
 868     __ cmpq(temp1, 0);
 869     __ jcc(Assembler::lessEqual, L_exit);
 870 
 871     __ mov64(temp2, 0);
 872     __ movq(temp3, temp1);
 873     // Special cases using 32 byte [masked] vector copy operations.
 874     arraycopy_avx3_special_cases_conjoint(xmm1, k2, from, to, temp2, temp3, temp1, shift,
 875                                           temp4, use64byteVector, L_entry, L_exit);
 876 
 877     // PRE-MAIN-POST loop for aligned copy.
 878     __ BIND(L_entry);
 879 
 880     if ((MaxVectorSize > 32) && (avx3threshold != 0)) {
 881       __ cmpq(temp1, threshold[shift]);
 882       __ jcc(Assembler::greaterEqual, L_pre_main_post_64);
 883     }
 884 
 885     if ((MaxVectorSize < 64)  || (avx3threshold != 0)) {
 886       // Partial copy to make dst address 32 byte aligned.
 887       __ leaq(temp2, Address(to, temp1, (Address::ScaleFactor)(shift), 0));
 888       __ andq(temp2, 31);
 889       __ jcc(Assembler::equal, L_main_pre_loop);
 890 
 891       if (shift) {
 892         __ shrq(temp2, shift);
 893       }
 894       __ subq(temp1, temp2);
 895       copy32_masked_avx(to, from, xmm1, k2, temp2, temp1, temp3, shift);
 896 
 897       __ cmpq(temp1, loop_size[shift]);
 898       __ jcc(Assembler::less, L_tail);
 899 
 900       __ BIND(L_main_pre_loop);
 901 
 902       // Main loop with aligned copy block size of 192 bytes at 32 byte granularity.
 903       __ align32();
 904       __ BIND(L_main_loop);
 905          copy64_avx(to, from, temp1, xmm1, true, shift, -64);
 906          copy64_avx(to, from, temp1, xmm1, true, shift, -128);
 907          copy64_avx(to, from, temp1, xmm1, true, shift, -192);
 908          __ subptr(temp1, loop_size[shift]);
 909          __ cmpq(temp1, loop_size[shift]);
 910          __ jcc(Assembler::greater, L_main_loop);
 911 
 912       // Tail loop.
 913       __ jmp(L_tail);
 914     }
 915 
 916     if (MaxVectorSize > 32) {
 917       __ BIND(L_pre_main_post_64);
 918       // Partial copy to make dst address 64 byte aligned.
 919       __ leaq(temp2, Address(to, temp1, (Address::ScaleFactor)(shift), 0));
 920       __ andq(temp2, 63);
 921       __ jcc(Assembler::equal, L_main_pre_loop_64bytes);
 922 
 923       if (shift) {
 924         __ shrq(temp2, shift);
 925       }
 926       __ subq(temp1, temp2);
 927       copy64_masked_avx(to, from, xmm1, k2, temp2, temp1, temp3, shift, 0 , true);
 928 
 929       __ cmpq(temp1, loop_size[shift]);
 930       __ jcc(Assembler::less, L_tail64);
 931 
 932       __ BIND(L_main_pre_loop_64bytes);
 933 
 934       // Main loop with aligned copy block size of 192 bytes at
 935       // 64 byte copy granularity.
 936       __ align32();
 937       __ BIND(L_main_loop_64bytes);
 938          copy64_avx(to, from, temp1, xmm1, true, shift, -64 , true);
 939          copy64_avx(to, from, temp1, xmm1, true, shift, -128, true);
 940          copy64_avx(to, from, temp1, xmm1, true, shift, -192, true);
 941          __ subq(temp1, loop_size[shift]);
 942          __ cmpq(temp1, loop_size[shift]);
 943          __ jcc(Assembler::greater, L_main_loop_64bytes);
 944 
 945       // Zero length check.
 946       __ cmpq(temp1, 0);
 947       __ jcc(Assembler::lessEqual, L_exit);
 948 
 949       __ BIND(L_tail64);
 950 
 951       // Tail handling using 64 byte [masked] vector copy operations.
 952       use64byteVector = true;
 953       __ mov64(temp2, 0);
 954       __ movq(temp3, temp1);
 955       arraycopy_avx3_special_cases_conjoint(xmm1, k2, from, to, temp2, temp3, temp1, shift,
 956                                             temp4, use64byteVector, L_entry, L_exit);
 957     }
 958     __ BIND(L_exit);
 959   }
 960   address ucme_exit_pc = __ pc();
 961   // When called from generic_arraycopy r11 contains specific values
 962   // used during arraycopy epilogue, re-initializing r11.
 963   if(is_oop) {
 964     __ movq(r11, count);
 965   }
 966   bs->arraycopy_epilogue(_masm, decorators, type, from, to, count);
 967   restore_argument_regs(type);
 968   INC_COUNTER_NP(get_profile_ctr(shift), rscratch1); // Update counter after rscratch1 is free
 969   __ xorptr(rax, rax); // return 0
 970   __ vzeroupper();
 971   __ leave(); // required for proper stackwalking of RuntimeStub frame
 972   __ ret(0);
 973 
 974   return start;
 975 }
 976 
 977 void StubGenerator::arraycopy_avx3_special_cases(XMMRegister xmm, KRegister mask, Register from,
 978                                                  Register to, Register count, int shift,
 979                                                  Register index, Register temp,
 980                                                  bool use64byteVector, Label& L_entry, Label& L_exit) {
 981   Label L_entry_64, L_entry_96, L_entry_128;
 982   Label L_entry_160, L_entry_192;
 983 
 984   int size_mat[][6] = {
 985   /* T_BYTE */ {32 , 64,  96 , 128 , 160 , 192 },
 986   /* T_SHORT*/ {16 , 32,  48 , 64  , 80  , 96  },
 987   /* T_INT  */ {8  , 16,  24 , 32  , 40  , 48  },
 988   /* T_LONG */ {4  ,  8,  12 , 16  , 20  , 24  }
 989   };
 990 
 991   // Case A) Special case for length less than equal to 32 bytes.
 992   __ cmpq(count, size_mat[shift][0]);
 993   __ jccb(Assembler::greater, L_entry_64);
 994   copy32_masked_avx(to, from, xmm, mask, count, index, temp, shift);
 995   __ jmp(L_exit);
 996 
 997   // Case B) Special case for length less than equal to 64 bytes.
 998   __ BIND(L_entry_64);
 999   __ cmpq(count, size_mat[shift][1]);
1000   __ jccb(Assembler::greater, L_entry_96);
1001   copy64_masked_avx(to, from, xmm, mask, count, index, temp, shift, 0, use64byteVector);
1002   __ jmp(L_exit);
1003 
1004   // Case C) Special case for length less than equal to 96 bytes.
1005   __ BIND(L_entry_96);
1006   __ cmpq(count, size_mat[shift][2]);
1007   __ jccb(Assembler::greater, L_entry_128);
1008   copy64_avx(to, from, index, xmm, false, shift, 0, use64byteVector);
1009   __ subq(count, 64 >> shift);
1010   copy32_masked_avx(to, from, xmm, mask, count, index, temp, shift, 64);
1011   __ jmp(L_exit);
1012 
1013   // Case D) Special case for length less than equal to 128 bytes.
1014   __ BIND(L_entry_128);
1015   __ cmpq(count, size_mat[shift][3]);
1016   __ jccb(Assembler::greater, L_entry_160);
1017   copy64_avx(to, from, index, xmm, false, shift, 0, use64byteVector);
1018   copy32_avx(to, from, index, xmm, shift, 64);
1019   __ subq(count, 96 >> shift);
1020   copy32_masked_avx(to, from, xmm, mask, count, index, temp, shift, 96);
1021   __ jmp(L_exit);
1022 
1023   // Case E) Special case for length less than equal to 160 bytes.
1024   __ BIND(L_entry_160);
1025   __ cmpq(count, size_mat[shift][4]);
1026   __ jccb(Assembler::greater, L_entry_192);
1027   copy64_avx(to, from, index, xmm, false, shift, 0, use64byteVector);
1028   copy64_avx(to, from, index, xmm, false, shift, 64, use64byteVector);
1029   __ subq(count, 128 >> shift);
1030   copy32_masked_avx(to, from, xmm, mask, count, index, temp, shift, 128);
1031   __ jmp(L_exit);
1032 
1033   // Case F) Special case for length less than equal to 192 bytes.
1034   __ BIND(L_entry_192);
1035   __ cmpq(count, size_mat[shift][5]);
1036   __ jcc(Assembler::greater, L_entry);
1037   copy64_avx(to, from, index, xmm, false, shift, 0, use64byteVector);
1038   copy64_avx(to, from, index, xmm, false, shift, 64, use64byteVector);
1039   copy32_avx(to, from, index, xmm, shift, 128);
1040   __ subq(count, 160 >> shift);
1041   copy32_masked_avx(to, from, xmm, mask, count, index, temp, shift, 160);
1042   __ jmp(L_exit);
1043 }
1044 
1045 void StubGenerator::arraycopy_avx3_special_cases_256(XMMRegister xmm, KRegister mask, Register from,
1046                                                      Register to, Register count, int shift, Register index,
1047                                                      Register temp, Label& L_exit) {
1048   Label L_entry_64, L_entry_128, L_entry_192, L_entry_256;
1049 
1050   int size_mat[][4] = {
1051   /* T_BYTE */ {64, 128, 192, 256},
1052   /* T_SHORT*/ {32, 64 , 96 , 128},
1053   /* T_INT  */ {16, 32 , 48 ,  64},
1054   /* T_LONG */ { 8, 16 , 24 ,  32}
1055   };
1056 
1057   assert(MaxVectorSize == 64, "vector length != 64");
1058   // Case A) Special case for length less than or equal to 64 bytes.
1059   __ BIND(L_entry_64);
1060   __ cmpq(count, size_mat[shift][0]);
1061   __ jccb(Assembler::greater, L_entry_128);
1062   copy64_masked_avx(to, from, xmm, mask, count, index, temp, shift, 0, true);
1063   __ jmp(L_exit);
1064 
1065   // Case B) Special case for length less than or equal to 128 bytes.
1066   __ BIND(L_entry_128);
1067   __ cmpq(count, size_mat[shift][1]);
1068   __ jccb(Assembler::greater, L_entry_192);
1069   copy64_avx(to, from, index, xmm, false, shift, 0, true);
1070   __ subq(count, 64 >> shift);
1071   copy64_masked_avx(to, from, xmm, mask, count, index, temp, shift, 64, true);
1072   __ jmp(L_exit);
1073 
1074   // Case C) Special case for length less than or equal to 192 bytes.
1075   __ BIND(L_entry_192);
1076   __ cmpq(count, size_mat[shift][2]);
1077   __ jcc(Assembler::greater, L_entry_256);
1078   copy64_avx(to, from, index, xmm, false, shift, 0, true);
1079   copy64_avx(to, from, index, xmm, false, shift, 64, true);
1080   __ subq(count, 128 >> shift);
1081   copy64_masked_avx(to, from, xmm, mask, count, index, temp, shift, 128, true);
1082   __ jmp(L_exit);
1083 
1084   // Case D) Special case for length less than or equal to 256 bytes.
1085   __ BIND(L_entry_256);
1086   copy64_avx(to, from, index, xmm, false, shift, 0, true);
1087   copy64_avx(to, from, index, xmm, false, shift, 64, true);
1088   copy64_avx(to, from, index, xmm, false, shift, 128, true);
1089   __ subq(count, 192 >> shift);
1090   copy64_masked_avx(to, from, xmm, mask, count, index, temp, shift, 192, true);
1091   __ jmp(L_exit);
1092 }
1093 
1094 void StubGenerator::arraycopy_avx3_special_cases_conjoint(XMMRegister xmm, KRegister mask, Register from,
1095                                                            Register to, Register start_index, Register end_index,
1096                                                            Register count, int shift, Register temp,
1097                                                            bool use64byteVector, Label& L_entry, Label& L_exit) {
1098   Label L_entry_64, L_entry_96, L_entry_128;
1099   Label L_entry_160, L_entry_192;
1100   bool avx3 = (MaxVectorSize > 32) && (VM_Version::avx3_threshold() == 0);
1101 
1102   int size_mat[][6] = {
1103   /* T_BYTE */ {32 , 64,  96 , 128 , 160 , 192 },
1104   /* T_SHORT*/ {16 , 32,  48 , 64  , 80  , 96  },
1105   /* T_INT  */ {8  , 16,  24 , 32  , 40  , 48  },
1106   /* T_LONG */ {4  ,  8,  12 , 16  , 20  , 24  }
1107   };
1108 
1109   // Case A) Special case for length less than equal to 32 bytes.
1110   __ cmpq(count, size_mat[shift][0]);
1111   __ jccb(Assembler::greater, L_entry_64);
1112   copy32_masked_avx(to, from, xmm, mask, count, start_index, temp, shift);
1113   __ jmp(L_exit);
1114 
1115   // Case B) Special case for length less than equal to 64 bytes.
1116   __ BIND(L_entry_64);
1117   __ cmpq(count, size_mat[shift][1]);
1118   __ jccb(Assembler::greater, L_entry_96);
1119   if (avx3) {
1120      copy64_masked_avx(to, from, xmm, mask, count, start_index, temp, shift, 0, true);
1121   } else {
1122      copy32_avx(to, from, end_index, xmm, shift, -32);
1123      __ subq(count, 32 >> shift);
1124      copy32_masked_avx(to, from, xmm, mask, count, start_index, temp, shift);
1125   }
1126   __ jmp(L_exit);
1127 
1128   // Case C) Special case for length less than equal to 96 bytes.
1129   __ BIND(L_entry_96);
1130   __ cmpq(count, size_mat[shift][2]);
1131   __ jccb(Assembler::greater, L_entry_128);
1132   copy64_avx(to, from, end_index, xmm, true, shift, -64, use64byteVector);
1133   __ subq(count, 64 >> shift);
1134   copy32_masked_avx(to, from, xmm, mask, count, start_index, temp, shift);
1135   __ jmp(L_exit);
1136 
1137   // Case D) Special case for length less than equal to 128 bytes.
1138   __ BIND(L_entry_128);
1139   __ cmpq(count, size_mat[shift][3]);
1140   __ jccb(Assembler::greater, L_entry_160);
1141   copy64_avx(to, from, end_index, xmm, true, shift, -64, use64byteVector);
1142   copy32_avx(to, from, end_index, xmm, shift, -96);
1143   __ subq(count, 96 >> shift);
1144   copy32_masked_avx(to, from, xmm, mask, count, start_index, temp, shift);
1145   __ jmp(L_exit);
1146 
1147   // Case E) Special case for length less than equal to 160 bytes.
1148   __ BIND(L_entry_160);
1149   __ cmpq(count, size_mat[shift][4]);
1150   __ jccb(Assembler::greater, L_entry_192);
1151   copy64_avx(to, from, end_index, xmm, true, shift, -64, use64byteVector);
1152   copy64_avx(to, from, end_index, xmm, true, shift, -128, use64byteVector);
1153   __ subq(count, 128 >> shift);
1154   copy32_masked_avx(to, from, xmm, mask, count, start_index, temp, shift);
1155   __ jmp(L_exit);
1156 
1157   // Case F) Special case for length less than equal to 192 bytes.
1158   __ BIND(L_entry_192);
1159   __ cmpq(count, size_mat[shift][5]);
1160   __ jcc(Assembler::greater, L_entry);
1161   copy64_avx(to, from, end_index, xmm, true, shift, -64, use64byteVector);
1162   copy64_avx(to, from, end_index, xmm, true, shift, -128, use64byteVector);
1163   copy32_avx(to, from, end_index, xmm, shift, -160);
1164   __ subq(count, 160 >> shift);
1165   copy32_masked_avx(to, from, xmm, mask, count, start_index, temp, shift);
1166   __ jmp(L_exit);
1167 }
1168 
1169 void StubGenerator::copy256_avx3(Register dst, Register src, Register index, XMMRegister xmm1,
1170                                 XMMRegister xmm2, XMMRegister xmm3, XMMRegister xmm4,
1171                                 int shift, int offset) {
1172   if (MaxVectorSize == 64) {
1173     Address::ScaleFactor scale = (Address::ScaleFactor)(shift);
1174     __ prefetcht0(Address(src, index, scale, offset + 0x200));
1175     __ prefetcht0(Address(src, index, scale, offset + 0x240));
1176     __ prefetcht0(Address(src, index, scale, offset + 0x280));
1177     __ prefetcht0(Address(src, index, scale, offset + 0x2C0));
1178 
1179     __ prefetcht0(Address(src, index, scale, offset + 0x400));
1180     __ prefetcht0(Address(src, index, scale, offset + 0x440));
1181     __ prefetcht0(Address(src, index, scale, offset + 0x480));
1182     __ prefetcht0(Address(src, index, scale, offset + 0x4C0));
1183 
1184     __ evmovdquq(xmm1, Address(src, index, scale, offset), Assembler::AVX_512bit);
1185     __ evmovdquq(xmm2, Address(src, index, scale, offset + 0x40), Assembler::AVX_512bit);
1186     __ evmovdquq(xmm3, Address(src, index, scale, offset + 0x80), Assembler::AVX_512bit);
1187     __ evmovdquq(xmm4, Address(src, index, scale, offset + 0xC0), Assembler::AVX_512bit);
1188 
1189     __ evmovntdquq(Address(dst, index, scale, offset), xmm1, Assembler::AVX_512bit);
1190     __ evmovntdquq(Address(dst, index, scale, offset + 0x40), xmm2, Assembler::AVX_512bit);
1191     __ evmovntdquq(Address(dst, index, scale, offset + 0x80), xmm3, Assembler::AVX_512bit);
1192     __ evmovntdquq(Address(dst, index, scale, offset + 0xC0), xmm4, Assembler::AVX_512bit);
1193   }
1194 }
1195 
1196 void StubGenerator::copy64_masked_avx(Register dst, Register src, XMMRegister xmm,
1197                                        KRegister mask, Register length, Register index,
1198                                        Register temp, int shift, int offset,
1199                                        bool use64byteVector) {
1200   BasicType type[] = { T_BYTE,  T_SHORT,  T_INT,   T_LONG};
1201   assert(MaxVectorSize >= 32, "vector length should be >= 32");
1202   if (!use64byteVector) {
1203     copy32_avx(dst, src, index, xmm, shift, offset);
1204     __ subptr(length, 32 >> shift);
1205     copy32_masked_avx(dst, src, xmm, mask, length, index, temp, shift, offset+32);
1206   } else {
1207     Address::ScaleFactor scale = (Address::ScaleFactor)(shift);
1208     assert(MaxVectorSize == 64, "vector length != 64");
1209     __ mov64(temp, -1L);
1210     __ bzhiq(temp, temp, length);
1211     __ kmovql(mask, temp);
1212     __ evmovdqu(type[shift], mask, xmm, Address(src, index, scale, offset), false, Assembler::AVX_512bit);
1213     __ evmovdqu(type[shift], mask, Address(dst, index, scale, offset), xmm, true, Assembler::AVX_512bit);
1214   }
1215 }
1216 
1217 
1218 void StubGenerator::copy32_masked_avx(Register dst, Register src, XMMRegister xmm,
1219                                        KRegister mask, Register length, Register index,
1220                                        Register temp, int shift, int offset) {
1221   assert(MaxVectorSize >= 32, "vector length should be >= 32");
1222   BasicType type[] = { T_BYTE,  T_SHORT,  T_INT,   T_LONG};
1223   Address::ScaleFactor scale = (Address::ScaleFactor)(shift);
1224   __ mov64(temp, -1L);
1225   __ bzhiq(temp, temp, length);
1226   __ kmovql(mask, temp);
1227   __ evmovdqu(type[shift], mask, xmm, Address(src, index, scale, offset), false, Assembler::AVX_256bit);
1228   __ evmovdqu(type[shift], mask, Address(dst, index, scale, offset), xmm, true, Assembler::AVX_256bit);
1229 }
1230 
1231 
1232 void StubGenerator::copy32_avx(Register dst, Register src, Register index, XMMRegister xmm,
1233                                 int shift, int offset) {
1234   assert(MaxVectorSize >= 32, "vector length should be >= 32");
1235   Address::ScaleFactor scale = (Address::ScaleFactor)(shift);
1236   __ vmovdqu(xmm, Address(src, index, scale, offset));
1237   __ vmovdqu(Address(dst, index, scale, offset), xmm);
1238 }
1239 
1240 
1241 void StubGenerator::copy64_avx(Register dst, Register src, Register index, XMMRegister xmm,
1242                                 bool conjoint, int shift, int offset, bool use64byteVector) {
1243   assert(MaxVectorSize == 64 || MaxVectorSize == 32, "vector length mismatch");
1244   if (!use64byteVector) {
1245     if (conjoint) {
1246       copy32_avx(dst, src, index, xmm, shift, offset+32);
1247       copy32_avx(dst, src, index, xmm, shift, offset);
1248     } else {
1249       copy32_avx(dst, src, index, xmm, shift, offset);
1250       copy32_avx(dst, src, index, xmm, shift, offset+32);
1251     }
1252   } else {
1253     Address::ScaleFactor scale = (Address::ScaleFactor)(shift);
1254     __ evmovdquq(xmm, Address(src, index, scale, offset), Assembler::AVX_512bit);
1255     __ evmovdquq(Address(dst, index, scale, offset), xmm, Assembler::AVX_512bit);
1256   }
1257 }
1258 
1259 #endif // COMPILER2_OR_JVMCI
1260 
1261 
1262 // Arguments:
1263 //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1264 //             ignored
1265 //   name    - stub name string
1266 //
1267 // Inputs:
1268 //   c_rarg0   - source array address
1269 //   c_rarg1   - destination array address
1270 //   c_rarg2   - element count, treated as ssize_t, can be zero
1271 //
1272 // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries,
1273 // we let the hardware handle it.  The one to eight bytes within words,
1274 // dwords or qwords that span cache line boundaries will still be loaded
1275 // and stored atomically.
1276 //
1277 // Side Effects:
1278 //   disjoint_byte_copy_entry is set to the no-overlap entry point
1279 //   used by generate_conjoint_byte_copy().
1280 //
1281 address StubGenerator::generate_disjoint_byte_copy(bool aligned, address* entry, const char *name) {
1282 #if COMPILER2_OR_JVMCI
1283   if (VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2() && MaxVectorSize  >= 32) {
1284      return generate_disjoint_copy_avx3_masked(entry, "jbyte_disjoint_arraycopy_avx3", 0,
1285                                                aligned, false, false);
1286   }
1287 #endif
1288   __ align(CodeEntryAlignment);
1289   StubCodeMark mark(this, "StubRoutines", name);
1290   address start = __ pc();
1291   DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT;
1292 
1293   Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_copy_2_bytes;
1294   Label L_copy_byte, L_exit;
1295   const Register from        = rdi;  // source array address
1296   const Register to          = rsi;  // destination array address
1297   const Register count       = rdx;  // elements count
1298   const Register byte_count  = rcx;
1299   const Register qword_count = count;
1300   const Register end_from    = from; // source array end address
1301   const Register end_to      = to;   // destination array end address
1302   // End pointers are inclusive, and if count is not zero they point
1303   // to the last unit copied:  end_to[0] := end_from[0]
1304 
1305   __ enter(); // required for proper stackwalking of RuntimeStub frame
1306   assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1307 
1308   if (entry != nullptr) {
1309     *entry = __ pc();
1310      // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1311     BLOCK_COMMENT("Entry:");
1312   }
1313 
1314   setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1315                     // r9 and r10 may be used to save non-volatile registers
1316 
1317   {
1318     // UnsafeCopyMemory page error: continue after ucm
1319     UnsafeCopyMemoryMark ucmm(this, !aligned, true);
1320     // 'from', 'to' and 'count' are now valid
1321     __ movptr(byte_count, count);
1322     __ shrptr(count, 3); // count => qword_count
1323 
1324     // Copy from low to high addresses.  Use 'to' as scratch.
1325     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
1326     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
1327     __ negptr(qword_count); // make the count negative
1328     __ jmp(L_copy_bytes);
1329 
1330     // Copy trailing qwords
1331   __ BIND(L_copy_8_bytes);
1332     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
1333     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
1334     __ increment(qword_count);
1335     __ jcc(Assembler::notZero, L_copy_8_bytes);
1336 
1337     // Check for and copy trailing dword
1338   __ BIND(L_copy_4_bytes);
1339     __ testl(byte_count, 4);
1340     __ jccb(Assembler::zero, L_copy_2_bytes);
1341     __ movl(rax, Address(end_from, 8));
1342     __ movl(Address(end_to, 8), rax);
1343 
1344     __ addptr(end_from, 4);
1345     __ addptr(end_to, 4);
1346 
1347     // Check for and copy trailing word
1348   __ BIND(L_copy_2_bytes);
1349     __ testl(byte_count, 2);
1350     __ jccb(Assembler::zero, L_copy_byte);
1351     __ movw(rax, Address(end_from, 8));
1352     __ movw(Address(end_to, 8), rax);
1353 
1354     __ addptr(end_from, 2);
1355     __ addptr(end_to, 2);
1356 
1357     // Check for and copy trailing byte
1358   __ BIND(L_copy_byte);
1359     __ testl(byte_count, 1);
1360     __ jccb(Assembler::zero, L_exit);
1361     __ movb(rax, Address(end_from, 8));
1362     __ movb(Address(end_to, 8), rax);
1363   }
1364 __ BIND(L_exit);
1365   address ucme_exit_pc = __ pc();
1366   restore_arg_regs();
1367   INC_COUNTER_NP(SharedRuntime::_jbyte_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free
1368   __ xorptr(rax, rax); // return 0
1369   __ vzeroupper();
1370   __ leave(); // required for proper stackwalking of RuntimeStub frame
1371   __ ret(0);
1372 
1373   {
1374     UnsafeCopyMemoryMark ucmm(this, !aligned, false, ucme_exit_pc);
1375     // Copy in multi-bytes chunks
1376     copy_bytes_forward(end_from, end_to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, T_BYTE);
1377     __ jmp(L_copy_4_bytes);
1378   }
1379   return start;
1380 }
1381 
1382 
1383 // Arguments:
1384 //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1385 //             ignored
1386 //   name    - stub name string
1387 //
1388 // Inputs:
1389 //   c_rarg0   - source array address
1390 //   c_rarg1   - destination array address
1391 //   c_rarg2   - element count, treated as ssize_t, can be zero
1392 //
1393 // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries,
1394 // we let the hardware handle it.  The one to eight bytes within words,
1395 // dwords or qwords that span cache line boundaries will still be loaded
1396 // and stored atomically.
1397 //
1398 address StubGenerator::generate_conjoint_byte_copy(bool aligned, address nooverlap_target,
1399                                                    address* entry, const char *name) {
1400 #if COMPILER2_OR_JVMCI
1401   if (VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2() && MaxVectorSize  >= 32) {
1402      return generate_conjoint_copy_avx3_masked(entry, "jbyte_conjoint_arraycopy_avx3", 0,
1403                                                nooverlap_target, aligned, false, false);
1404   }
1405 #endif
1406   __ align(CodeEntryAlignment);
1407   StubCodeMark mark(this, "StubRoutines", name);
1408   address start = __ pc();
1409   DecoratorSet decorators = IN_HEAP | IS_ARRAY;
1410 
1411   Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_copy_2_bytes;
1412   const Register from        = rdi;  // source array address
1413   const Register to          = rsi;  // destination array address
1414   const Register count       = rdx;  // elements count
1415   const Register byte_count  = rcx;
1416   const Register qword_count = count;
1417 
1418   __ enter(); // required for proper stackwalking of RuntimeStub frame
1419   assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1420 
1421   if (entry != nullptr) {
1422     *entry = __ pc();
1423     // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1424     BLOCK_COMMENT("Entry:");
1425   }
1426 
1427   array_overlap_test(nooverlap_target, Address::times_1);
1428   setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1429                     // r9 and r10 may be used to save non-volatile registers
1430 
1431   {
1432     // UnsafeCopyMemory page error: continue after ucm
1433     UnsafeCopyMemoryMark ucmm(this, !aligned, true);
1434     // 'from', 'to' and 'count' are now valid
1435     __ movptr(byte_count, count);
1436     __ shrptr(count, 3);   // count => qword_count
1437 
1438     // Copy from high to low addresses.
1439 
1440     // Check for and copy trailing byte
1441     __ testl(byte_count, 1);
1442     __ jcc(Assembler::zero, L_copy_2_bytes);
1443     __ movb(rax, Address(from, byte_count, Address::times_1, -1));
1444     __ movb(Address(to, byte_count, Address::times_1, -1), rax);
1445     __ decrement(byte_count); // Adjust for possible trailing word
1446 
1447     // Check for and copy trailing word
1448   __ BIND(L_copy_2_bytes);
1449     __ testl(byte_count, 2);
1450     __ jcc(Assembler::zero, L_copy_4_bytes);
1451     __ movw(rax, Address(from, byte_count, Address::times_1, -2));
1452     __ movw(Address(to, byte_count, Address::times_1, -2), rax);
1453 
1454     // Check for and copy trailing dword
1455   __ BIND(L_copy_4_bytes);
1456     __ testl(byte_count, 4);
1457     __ jcc(Assembler::zero, L_copy_bytes);
1458     __ movl(rax, Address(from, qword_count, Address::times_8));
1459     __ movl(Address(to, qword_count, Address::times_8), rax);
1460     __ jmp(L_copy_bytes);
1461 
1462     // Copy trailing qwords
1463   __ BIND(L_copy_8_bytes);
1464     __ movq(rax, Address(from, qword_count, Address::times_8, -8));
1465     __ movq(Address(to, qword_count, Address::times_8, -8), rax);
1466     __ decrement(qword_count);
1467     __ jcc(Assembler::notZero, L_copy_8_bytes);
1468   }
1469   restore_arg_regs();
1470   INC_COUNTER_NP(SharedRuntime::_jbyte_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free
1471   __ xorptr(rax, rax); // return 0
1472   __ vzeroupper();
1473   __ leave(); // required for proper stackwalking of RuntimeStub frame
1474   __ ret(0);
1475 
1476   {
1477     // UnsafeCopyMemory page error: continue after ucm
1478     UnsafeCopyMemoryMark ucmm(this, !aligned, true);
1479     // Copy in multi-bytes chunks
1480     copy_bytes_backward(from, to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, T_BYTE);
1481   }
1482   restore_arg_regs();
1483   INC_COUNTER_NP(SharedRuntime::_jbyte_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free
1484   __ xorptr(rax, rax); // return 0
1485   __ vzeroupper();
1486   __ leave(); // required for proper stackwalking of RuntimeStub frame
1487   __ ret(0);
1488 
1489   return start;
1490 }
1491 
1492 
1493 // Arguments:
1494 //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1495 //             ignored
1496 //   name    - stub name string
1497 //
1498 // Inputs:
1499 //   c_rarg0   - source array address
1500 //   c_rarg1   - destination array address
1501 //   c_rarg2   - element count, treated as ssize_t, can be zero
1502 //
1503 // If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we
1504 // let the hardware handle it.  The two or four words within dwords
1505 // or qwords that span cache line boundaries will still be loaded
1506 // and stored atomically.
1507 //
1508 // Side Effects:
1509 //   disjoint_short_copy_entry is set to the no-overlap entry point
1510 //   used by generate_conjoint_short_copy().
1511 //
1512 address StubGenerator::generate_disjoint_short_copy(bool aligned, address *entry, const char *name) {
1513 #if COMPILER2_OR_JVMCI
1514   if (VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2() && MaxVectorSize  >= 32) {
1515      return generate_disjoint_copy_avx3_masked(entry, "jshort_disjoint_arraycopy_avx3", 1,
1516                                                aligned, false, false);
1517   }
1518 #endif
1519 
1520   __ align(CodeEntryAlignment);
1521   StubCodeMark mark(this, "StubRoutines", name);
1522   address start = __ pc();
1523   DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT;
1524 
1525   Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes,L_copy_2_bytes,L_exit;
1526   const Register from        = rdi;  // source array address
1527   const Register to          = rsi;  // destination array address
1528   const Register count       = rdx;  // elements count
1529   const Register word_count  = rcx;
1530   const Register qword_count = count;
1531   const Register end_from    = from; // source array end address
1532   const Register end_to      = to;   // destination array end address
1533   // End pointers are inclusive, and if count is not zero they point
1534   // to the last unit copied:  end_to[0] := end_from[0]
1535 
1536   __ enter(); // required for proper stackwalking of RuntimeStub frame
1537   assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1538 
1539   if (entry != nullptr) {
1540     *entry = __ pc();
1541     // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1542     BLOCK_COMMENT("Entry:");
1543   }
1544 
1545   setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1546                     // r9 and r10 may be used to save non-volatile registers
1547 
1548   {
1549     // UnsafeCopyMemory page error: continue after ucm
1550     UnsafeCopyMemoryMark ucmm(this, !aligned, true);
1551     // 'from', 'to' and 'count' are now valid
1552     __ movptr(word_count, count);
1553     __ shrptr(count, 2); // count => qword_count
1554 
1555     // Copy from low to high addresses.  Use 'to' as scratch.
1556     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
1557     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
1558     __ negptr(qword_count);
1559     __ jmp(L_copy_bytes);
1560 
1561     // Copy trailing qwords
1562   __ BIND(L_copy_8_bytes);
1563     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
1564     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
1565     __ increment(qword_count);
1566     __ jcc(Assembler::notZero, L_copy_8_bytes);
1567 
1568     // Original 'dest' is trashed, so we can't use it as a
1569     // base register for a possible trailing word copy
1570 
1571     // Check for and copy trailing dword
1572   __ BIND(L_copy_4_bytes);
1573     __ testl(word_count, 2);
1574     __ jccb(Assembler::zero, L_copy_2_bytes);
1575     __ movl(rax, Address(end_from, 8));
1576     __ movl(Address(end_to, 8), rax);
1577 
1578     __ addptr(end_from, 4);
1579     __ addptr(end_to, 4);
1580 
1581     // Check for and copy trailing word
1582   __ BIND(L_copy_2_bytes);
1583     __ testl(word_count, 1);
1584     __ jccb(Assembler::zero, L_exit);
1585     __ movw(rax, Address(end_from, 8));
1586     __ movw(Address(end_to, 8), rax);
1587   }
1588 __ BIND(L_exit);
1589   address ucme_exit_pc = __ pc();
1590   restore_arg_regs();
1591   INC_COUNTER_NP(SharedRuntime::_jshort_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free
1592   __ xorptr(rax, rax); // return 0
1593   __ vzeroupper();
1594   __ leave(); // required for proper stackwalking of RuntimeStub frame
1595   __ ret(0);
1596 
1597   {
1598     UnsafeCopyMemoryMark ucmm(this, !aligned, false, ucme_exit_pc);
1599     // Copy in multi-bytes chunks
1600     copy_bytes_forward(end_from, end_to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, T_SHORT);
1601     __ jmp(L_copy_4_bytes);
1602   }
1603 
1604   return start;
1605 }
1606 
1607 
1608 address StubGenerator::generate_fill(BasicType t, bool aligned, const char *name) {
1609   __ align(CodeEntryAlignment);
1610   StubCodeMark mark(this, "StubRoutines", name);
1611   address start = __ pc();
1612 
1613   BLOCK_COMMENT("Entry:");
1614 
1615   const Register to       = c_rarg0;  // destination array address
1616   const Register value    = c_rarg1;  // value
1617   const Register count    = c_rarg2;  // elements count
1618   __ mov(r11, count);
1619 
1620   __ enter(); // required for proper stackwalking of RuntimeStub frame
1621 
1622   __ generate_fill(t, aligned, to, value, r11, rax, xmm0);
1623 
1624   __ vzeroupper();
1625   __ leave(); // required for proper stackwalking of RuntimeStub frame
1626   __ ret(0);
1627 
1628   return start;
1629 }
1630 
1631 
1632 // Arguments:
1633 //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1634 //             ignored
1635 //   name    - stub name string
1636 //
1637 // Inputs:
1638 //   c_rarg0   - source array address
1639 //   c_rarg1   - destination array address
1640 //   c_rarg2   - element count, treated as ssize_t, can be zero
1641 //
1642 // If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we
1643 // let the hardware handle it.  The two or four words within dwords
1644 // or qwords that span cache line boundaries will still be loaded
1645 // and stored atomically.
1646 //
1647 address StubGenerator::generate_conjoint_short_copy(bool aligned, address nooverlap_target,
1648                                                     address *entry, const char *name) {
1649 #if COMPILER2_OR_JVMCI
1650   if (VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2() && MaxVectorSize  >= 32) {
1651      return generate_conjoint_copy_avx3_masked(entry, "jshort_conjoint_arraycopy_avx3", 1,
1652                                                nooverlap_target, aligned, false, false);
1653   }
1654 #endif
1655   __ align(CodeEntryAlignment);
1656   StubCodeMark mark(this, "StubRoutines", name);
1657   address start = __ pc();
1658   DecoratorSet decorators = IN_HEAP | IS_ARRAY;
1659 
1660   Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes;
1661   const Register from        = rdi;  // source array address
1662   const Register to          = rsi;  // destination array address
1663   const Register count       = rdx;  // elements count
1664   const Register word_count  = rcx;
1665   const Register qword_count = count;
1666 
1667   __ enter(); // required for proper stackwalking of RuntimeStub frame
1668   assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1669 
1670   if (entry != nullptr) {
1671     *entry = __ pc();
1672     // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1673     BLOCK_COMMENT("Entry:");
1674   }
1675 
1676   array_overlap_test(nooverlap_target, Address::times_2);
1677   setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1678                     // r9 and r10 may be used to save non-volatile registers
1679 
1680   {
1681     // UnsafeCopyMemory page error: continue after ucm
1682     UnsafeCopyMemoryMark ucmm(this, !aligned, true);
1683     // 'from', 'to' and 'count' are now valid
1684     __ movptr(word_count, count);
1685     __ shrptr(count, 2); // count => qword_count
1686 
1687     // Copy from high to low addresses.  Use 'to' as scratch.
1688 
1689     // Check for and copy trailing word
1690     __ testl(word_count, 1);
1691     __ jccb(Assembler::zero, L_copy_4_bytes);
1692     __ movw(rax, Address(from, word_count, Address::times_2, -2));
1693     __ movw(Address(to, word_count, Address::times_2, -2), rax);
1694 
1695    // Check for and copy trailing dword
1696   __ BIND(L_copy_4_bytes);
1697     __ testl(word_count, 2);
1698     __ jcc(Assembler::zero, L_copy_bytes);
1699     __ movl(rax, Address(from, qword_count, Address::times_8));
1700     __ movl(Address(to, qword_count, Address::times_8), rax);
1701     __ jmp(L_copy_bytes);
1702 
1703     // Copy trailing qwords
1704   __ BIND(L_copy_8_bytes);
1705     __ movq(rax, Address(from, qword_count, Address::times_8, -8));
1706     __ movq(Address(to, qword_count, Address::times_8, -8), rax);
1707     __ decrement(qword_count);
1708     __ jcc(Assembler::notZero, L_copy_8_bytes);
1709   }
1710   restore_arg_regs();
1711   INC_COUNTER_NP(SharedRuntime::_jshort_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free
1712   __ xorptr(rax, rax); // return 0
1713   __ vzeroupper();
1714   __ leave(); // required for proper stackwalking of RuntimeStub frame
1715   __ ret(0);
1716 
1717   {
1718     // UnsafeCopyMemory page error: continue after ucm
1719     UnsafeCopyMemoryMark ucmm(this, !aligned, true);
1720     // Copy in multi-bytes chunks
1721     copy_bytes_backward(from, to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, T_SHORT);
1722   }
1723   restore_arg_regs();
1724   INC_COUNTER_NP(SharedRuntime::_jshort_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free
1725   __ xorptr(rax, rax); // return 0
1726   __ vzeroupper();
1727   __ leave(); // required for proper stackwalking of RuntimeStub frame
1728   __ ret(0);
1729 
1730   return start;
1731 }
1732 
1733 
1734 // Arguments:
1735 //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1736 //             ignored
1737 //   is_oop  - true => oop array, so generate store check code
1738 //   name    - stub name string
1739 //
1740 // Inputs:
1741 //   c_rarg0   - source array address
1742 //   c_rarg1   - destination array address
1743 //   c_rarg2   - element count, treated as ssize_t, can be zero
1744 //
1745 // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
1746 // the hardware handle it.  The two dwords within qwords that span
1747 // cache line boundaries will still be loaded and stored atomically.
1748 //
1749 // Side Effects:
1750 //   disjoint_int_copy_entry is set to the no-overlap entry point
1751 //   used by generate_conjoint_int_oop_copy().
1752 //
1753 address StubGenerator::generate_disjoint_int_oop_copy(bool aligned, bool is_oop, address* entry,
1754                                                       const char *name, bool dest_uninitialized) {
1755   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
1756 #if COMPILER2_OR_JVMCI
1757   if ((!is_oop || bs->supports_avx3_masked_arraycopy()) && VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2() && MaxVectorSize  >= 32) {
1758      return generate_disjoint_copy_avx3_masked(entry, "jint_disjoint_arraycopy_avx3", 2,
1759                                                aligned, is_oop, dest_uninitialized);
1760   }
1761 #endif
1762 
1763   __ align(CodeEntryAlignment);
1764   StubCodeMark mark(this, "StubRoutines", name);
1765   address start = __ pc();
1766 
1767   Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_exit;
1768   const Register from        = rdi;  // source array address
1769   const Register to          = rsi;  // destination array address
1770   const Register count       = rdx;  // elements count
1771   const Register dword_count = rcx;
1772   const Register qword_count = count;
1773   const Register end_from    = from; // source array end address
1774   const Register end_to      = to;   // destination array end address
1775   // End pointers are inclusive, and if count is not zero they point
1776   // to the last unit copied:  end_to[0] := end_from[0]
1777 
1778   __ enter(); // required for proper stackwalking of RuntimeStub frame
1779   assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1780 
1781   if (entry != nullptr) {
1782     *entry = __ pc();
1783     // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1784     BLOCK_COMMENT("Entry:");
1785   }
1786 
1787   setup_arg_regs_using_thread(); // from => rdi, to => rsi, count => rdx
1788                                  // r9 is used to save r15_thread
1789 
1790   DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT;
1791   if (dest_uninitialized) {
1792     decorators |= IS_DEST_UNINITIALIZED;
1793   }
1794   if (aligned) {
1795     decorators |= ARRAYCOPY_ALIGNED;
1796   }
1797 
1798   BasicType type = is_oop ? T_OBJECT : T_INT;
1799   bs->arraycopy_prologue(_masm, decorators, type, from, to, count);
1800 
1801   {
1802     // UnsafeCopyMemory page error: continue after ucm
1803     UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
1804     // 'from', 'to' and 'count' are now valid
1805     __ movptr(dword_count, count);
1806     __ shrptr(count, 1); // count => qword_count
1807 
1808     // Copy from low to high addresses.  Use 'to' as scratch.
1809     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
1810     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
1811     __ negptr(qword_count);
1812     __ jmp(L_copy_bytes);
1813 
1814     // Copy trailing qwords
1815   __ BIND(L_copy_8_bytes);
1816     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
1817     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
1818     __ increment(qword_count);
1819     __ jcc(Assembler::notZero, L_copy_8_bytes);
1820 
1821     // Check for and copy trailing dword
1822   __ BIND(L_copy_4_bytes);
1823     __ testl(dword_count, 1); // Only byte test since the value is 0 or 1
1824     __ jccb(Assembler::zero, L_exit);
1825     __ movl(rax, Address(end_from, 8));
1826     __ movl(Address(end_to, 8), rax);
1827   }
1828 __ BIND(L_exit);
1829   address ucme_exit_pc = __ pc();
1830   bs->arraycopy_epilogue(_masm, decorators, type, from, to, dword_count);
1831   restore_arg_regs_using_thread();
1832   INC_COUNTER_NP(SharedRuntime::_jint_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free
1833   __ vzeroupper();
1834   __ xorptr(rax, rax); // return 0
1835   __ leave(); // required for proper stackwalking of RuntimeStub frame
1836   __ ret(0);
1837 
1838   {
1839     UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, false, ucme_exit_pc);
1840     // Copy in multi-bytes chunks
1841     copy_bytes_forward(end_from, end_to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, is_oop ? T_OBJECT : T_INT);
1842     __ jmp(L_copy_4_bytes);
1843   }
1844 
1845   return start;
1846 }
1847 
1848 
1849 // Arguments:
1850 //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1851 //             ignored
1852 //   is_oop  - true => oop array, so generate store check code
1853 //   name    - stub name string
1854 //
1855 // Inputs:
1856 //   c_rarg0   - source array address
1857 //   c_rarg1   - destination array address
1858 //   c_rarg2   - element count, treated as ssize_t, can be zero
1859 //
1860 // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
1861 // the hardware handle it.  The two dwords within qwords that span
1862 // cache line boundaries will still be loaded and stored atomically.
1863 //
1864 address StubGenerator::generate_conjoint_int_oop_copy(bool aligned, bool is_oop, address nooverlap_target,
1865                                                       address *entry, const char *name,
1866                                                       bool dest_uninitialized) {
1867   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
1868 #if COMPILER2_OR_JVMCI
1869   if ((!is_oop || bs->supports_avx3_masked_arraycopy()) && VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2() && MaxVectorSize  >= 32) {
1870      return generate_conjoint_copy_avx3_masked(entry, "jint_conjoint_arraycopy_avx3", 2,
1871                                                nooverlap_target, aligned, is_oop, dest_uninitialized);
1872   }
1873 #endif
1874   __ align(CodeEntryAlignment);
1875   StubCodeMark mark(this, "StubRoutines", name);
1876   address start = __ pc();
1877 
1878   Label L_copy_bytes, L_copy_8_bytes, L_exit;
1879   const Register from        = rdi;  // source array address
1880   const Register to          = rsi;  // destination array address
1881   const Register count       = rdx;  // elements count
1882   const Register dword_count = rcx;
1883   const Register qword_count = count;
1884 
1885   __ enter(); // required for proper stackwalking of RuntimeStub frame
1886   assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1887 
1888   if (entry != nullptr) {
1889     *entry = __ pc();
1890      // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1891     BLOCK_COMMENT("Entry:");
1892   }
1893 
1894   array_overlap_test(nooverlap_target, Address::times_4);
1895   setup_arg_regs_using_thread(); // from => rdi, to => rsi, count => rdx
1896                                  // r9 is used to save r15_thread
1897 
1898   DecoratorSet decorators = IN_HEAP | IS_ARRAY;
1899   if (dest_uninitialized) {
1900     decorators |= IS_DEST_UNINITIALIZED;
1901   }
1902   if (aligned) {
1903     decorators |= ARRAYCOPY_ALIGNED;
1904   }
1905 
1906   BasicType type = is_oop ? T_OBJECT : T_INT;
1907   // no registers are destroyed by this call
1908   bs->arraycopy_prologue(_masm, decorators, type, from, to, count);
1909 
1910   assert_clean_int(count, rax); // Make sure 'count' is clean int.
1911   {
1912     // UnsafeCopyMemory page error: continue after ucm
1913     UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
1914     // 'from', 'to' and 'count' are now valid
1915     __ movptr(dword_count, count);
1916     __ shrptr(count, 1); // count => qword_count
1917 
1918     // Copy from high to low addresses.  Use 'to' as scratch.
1919 
1920     // Check for and copy trailing dword
1921     __ testl(dword_count, 1);
1922     __ jcc(Assembler::zero, L_copy_bytes);
1923     __ movl(rax, Address(from, dword_count, Address::times_4, -4));
1924     __ movl(Address(to, dword_count, Address::times_4, -4), rax);
1925     __ jmp(L_copy_bytes);
1926 
1927     // Copy trailing qwords
1928   __ BIND(L_copy_8_bytes);
1929     __ movq(rax, Address(from, qword_count, Address::times_8, -8));
1930     __ movq(Address(to, qword_count, Address::times_8, -8), rax);
1931     __ decrement(qword_count);
1932     __ jcc(Assembler::notZero, L_copy_8_bytes);
1933   }
1934   if (is_oop) {
1935     __ jmp(L_exit);
1936   }
1937   restore_arg_regs_using_thread();
1938   INC_COUNTER_NP(SharedRuntime::_jint_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free
1939   __ xorptr(rax, rax); // return 0
1940   __ vzeroupper();
1941   __ leave(); // required for proper stackwalking of RuntimeStub frame
1942   __ ret(0);
1943 
1944   {
1945     // UnsafeCopyMemory page error: continue after ucm
1946     UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
1947     // Copy in multi-bytes chunks
1948     copy_bytes_backward(from, to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, is_oop ? T_OBJECT : T_INT);
1949   }
1950 
1951 __ BIND(L_exit);
1952   bs->arraycopy_epilogue(_masm, decorators, type, from, to, dword_count);
1953   restore_arg_regs_using_thread();
1954   INC_COUNTER_NP(SharedRuntime::_jint_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free
1955   __ xorptr(rax, rax); // return 0
1956   __ vzeroupper();
1957   __ leave(); // required for proper stackwalking of RuntimeStub frame
1958   __ ret(0);
1959 
1960   return start;
1961 }
1962 
1963 
1964 // Arguments:
1965 //   aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes
1966 //             ignored
1967 //   is_oop  - true => oop array, so generate store check code
1968 //   name    - stub name string
1969 //
1970 // Inputs:
1971 //   c_rarg0   - source array address
1972 //   c_rarg1   - destination array address
1973 //   c_rarg2   - element count, treated as ssize_t, can be zero
1974 //
1975  // Side Effects:
1976 //   disjoint_oop_copy_entry or disjoint_long_copy_entry is set to the
1977 //   no-overlap entry point used by generate_conjoint_long_oop_copy().
1978 //
1979 address StubGenerator::generate_disjoint_long_oop_copy(bool aligned, bool is_oop, address *entry,
1980                                                        const char *name, bool dest_uninitialized) {
1981   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
1982 #if COMPILER2_OR_JVMCI
1983   if ((!is_oop || bs->supports_avx3_masked_arraycopy()) && VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2() && MaxVectorSize >= 32) {
1984      return generate_disjoint_copy_avx3_masked(entry, "jlong_disjoint_arraycopy_avx3", 3,
1985                                                aligned, is_oop, dest_uninitialized);
1986   }
1987 #endif
1988   __ align(CodeEntryAlignment);
1989   StubCodeMark mark(this, "StubRoutines", name);
1990   address start = __ pc();
1991 
1992   Label L_copy_bytes, L_copy_8_bytes, L_exit;
1993   const Register from        = rdi;  // source array address
1994   const Register to          = rsi;  // destination array address
1995   const Register qword_count = rdx;  // elements count
1996   const Register end_from    = from; // source array end address
1997   const Register end_to      = rcx;  // destination array end address
1998   const Register saved_count = r11;
1999   // End pointers are inclusive, and if count is not zero they point
2000   // to the last unit copied:  end_to[0] := end_from[0]
2001 
2002   __ enter(); // required for proper stackwalking of RuntimeStub frame
2003   // Save no-overlap entry point for generate_conjoint_long_oop_copy()
2004   assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
2005 
2006   if (entry != nullptr) {
2007     *entry = __ pc();
2008     // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
2009     BLOCK_COMMENT("Entry:");
2010   }
2011 
2012   setup_arg_regs_using_thread(); // from => rdi, to => rsi, count => rdx
2013                                    // r9 is used to save r15_thread
2014   // 'from', 'to' and 'qword_count' are now valid
2015 
2016   DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT;
2017   if (dest_uninitialized) {
2018     decorators |= IS_DEST_UNINITIALIZED;
2019   }
2020   if (aligned) {
2021     decorators |= ARRAYCOPY_ALIGNED;
2022   }
2023 
2024   BasicType type = is_oop ? T_OBJECT : T_LONG;
2025   bs->arraycopy_prologue(_masm, decorators, type, from, to, qword_count);
2026   {
2027     // UnsafeCopyMemory page error: continue after ucm
2028     UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
2029 
2030     // Copy from low to high addresses.  Use 'to' as scratch.
2031     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
2032     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
2033     __ negptr(qword_count);
2034     __ jmp(L_copy_bytes);
2035 
2036     // Copy trailing qwords
2037   __ BIND(L_copy_8_bytes);
2038     bs->copy_load_at(_masm, decorators, type, 8,
2039                      rax, Address(end_from, qword_count, Address::times_8, 8),
2040                      r10);
2041     bs->copy_store_at(_masm, decorators, type, 8,
2042                       Address(end_to, qword_count, Address::times_8, 8), rax,
2043                       r10);
2044     __ increment(qword_count);
2045     __ jcc(Assembler::notZero, L_copy_8_bytes);
2046   }
2047   if (is_oop) {
2048     __ jmp(L_exit);
2049   } else {
2050     restore_arg_regs_using_thread();
2051     INC_COUNTER_NP(SharedRuntime::_jlong_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free
2052     __ xorptr(rax, rax); // return 0
2053     __ vzeroupper();
2054     __ leave(); // required for proper stackwalking of RuntimeStub frame
2055     __ ret(0);
2056   }
2057 
2058   {
2059     // UnsafeCopyMemory page error: continue after ucm
2060     UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
2061     // Copy in multi-bytes chunks
2062     copy_bytes_forward(end_from, end_to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, is_oop ? T_OBJECT : T_LONG);
2063   }
2064 
2065   __ BIND(L_exit);
2066   bs->arraycopy_epilogue(_masm, decorators, type, from, to, qword_count);
2067   restore_arg_regs_using_thread();
2068   INC_COUNTER_NP(is_oop ? SharedRuntime::_oop_array_copy_ctr :
2069                           SharedRuntime::_jlong_array_copy_ctr,
2070                  rscratch1); // Update counter after rscratch1 is free
2071   __ vzeroupper();
2072   __ xorptr(rax, rax); // return 0
2073   __ leave(); // required for proper stackwalking of RuntimeStub frame
2074   __ ret(0);
2075 
2076   return start;
2077 }
2078 
2079 
2080 // Arguments:
2081 //   aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes
2082 //             ignored
2083 //   is_oop  - true => oop array, so generate store check code
2084 //   name    - stub name string
2085 //
2086 // Inputs:
2087 //   c_rarg0   - source array address
2088 //   c_rarg1   - destination array address
2089 //   c_rarg2   - element count, treated as ssize_t, can be zero
2090 //
2091 address StubGenerator::generate_conjoint_long_oop_copy(bool aligned, bool is_oop, address nooverlap_target,
2092                                                        address *entry, const char *name,
2093                                                        bool dest_uninitialized) {
2094   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
2095 #if COMPILER2_OR_JVMCI
2096   if ((!is_oop || bs->supports_avx3_masked_arraycopy()) && VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2() && MaxVectorSize  >= 32) {
2097      return generate_conjoint_copy_avx3_masked(entry, "jlong_conjoint_arraycopy_avx3", 3,
2098                                                nooverlap_target, aligned, is_oop, dest_uninitialized);
2099   }
2100 #endif
2101   __ align(CodeEntryAlignment);
2102   StubCodeMark mark(this, "StubRoutines", name);
2103   address start = __ pc();
2104 
2105   Label L_copy_bytes, L_copy_8_bytes, L_exit;
2106   const Register from        = rdi;  // source array address
2107   const Register to          = rsi;  // destination array address
2108   const Register qword_count = rdx;  // elements count
2109   const Register saved_count = rcx;
2110 
2111   __ enter(); // required for proper stackwalking of RuntimeStub frame
2112   assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
2113 
2114   if (entry != nullptr) {
2115     *entry = __ pc();
2116     // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
2117     BLOCK_COMMENT("Entry:");
2118   }
2119 
2120   array_overlap_test(nooverlap_target, Address::times_8);
2121   setup_arg_regs_using_thread(); // from => rdi, to => rsi, count => rdx
2122                                  // r9 is used to save r15_thread
2123   // 'from', 'to' and 'qword_count' are now valid
2124 
2125   DecoratorSet decorators = IN_HEAP | IS_ARRAY;
2126   if (dest_uninitialized) {
2127     decorators |= IS_DEST_UNINITIALIZED;
2128   }
2129   if (aligned) {
2130     decorators |= ARRAYCOPY_ALIGNED;
2131   }
2132 
2133   BasicType type = is_oop ? T_OBJECT : T_LONG;
2134   bs->arraycopy_prologue(_masm, decorators, type, from, to, qword_count);
2135   {
2136     // UnsafeCopyMemory page error: continue after ucm
2137     UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
2138 
2139     __ jmp(L_copy_bytes);
2140 
2141     // Copy trailing qwords
2142   __ BIND(L_copy_8_bytes);
2143     bs->copy_load_at(_masm, decorators, type, 8,
2144                      rax, Address(from, qword_count, Address::times_8, -8),
2145                      r10);
2146     bs->copy_store_at(_masm, decorators, type, 8,
2147                       Address(to, qword_count, Address::times_8, -8), rax,
2148                       r10);
2149     __ decrement(qword_count);
2150     __ jcc(Assembler::notZero, L_copy_8_bytes);
2151   }
2152   if (is_oop) {
2153     __ jmp(L_exit);
2154   } else {
2155     restore_arg_regs_using_thread();
2156     INC_COUNTER_NP(SharedRuntime::_jlong_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free
2157     __ xorptr(rax, rax); // return 0
2158     __ vzeroupper();
2159     __ leave(); // required for proper stackwalking of RuntimeStub frame
2160     __ ret(0);
2161   }
2162   {
2163     // UnsafeCopyMemory page error: continue after ucm
2164     UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
2165 
2166     // Copy in multi-bytes chunks
2167     copy_bytes_backward(from, to, qword_count, rax, r10, L_copy_bytes, L_copy_8_bytes, decorators, is_oop ? T_OBJECT : T_LONG);
2168   }
2169   __ BIND(L_exit);
2170   bs->arraycopy_epilogue(_masm, decorators, type, from, to, qword_count);
2171   restore_arg_regs_using_thread();
2172   INC_COUNTER_NP(is_oop ? SharedRuntime::_oop_array_copy_ctr :
2173                           SharedRuntime::_jlong_array_copy_ctr,
2174                  rscratch1); // Update counter after rscratch1 is free
2175   __ vzeroupper();
2176   __ xorptr(rax, rax); // return 0
2177   __ leave(); // required for proper stackwalking of RuntimeStub frame
2178   __ ret(0);
2179 
2180   return start;
2181 }
2182 
2183 
2184 // Helper for generating a dynamic type check.
2185 // Smashes no registers.
2186 void StubGenerator::generate_type_check(Register sub_klass,
2187                                         Register super_check_offset,
2188                                         Register super_klass,
2189                                         Label& L_success) {
2190   assert_different_registers(sub_klass, super_check_offset, super_klass);
2191 
2192   BLOCK_COMMENT("type_check:");
2193 
2194   Label L_miss;
2195 
2196   __ check_klass_subtype_fast_path(sub_klass, super_klass, noreg,        &L_success, &L_miss, nullptr,
2197                                    super_check_offset);
2198   __ check_klass_subtype_slow_path(sub_klass, super_klass, noreg, noreg, &L_success, nullptr);
2199 
2200   // Fall through on failure!
2201   __ BIND(L_miss);
2202 }
2203 
2204 //
2205 //  Generate checkcasting array copy stub
2206 //
2207 //  Input:
2208 //    c_rarg0   - source array address
2209 //    c_rarg1   - destination array address
2210 //    c_rarg2   - element count, treated as ssize_t, can be zero
2211 //    c_rarg3   - size_t ckoff (super_check_offset)
2212 // not Win64
2213 //    c_rarg4   - oop ckval (super_klass)
2214 // Win64
2215 //    rsp+40    - oop ckval (super_klass)
2216 //
2217 //  Output:
2218 //    rax ==  0  -  success
2219 //    rax == -1^K - failure, where K is partial transfer count
2220 //
2221 address StubGenerator::generate_checkcast_copy(const char *name, address *entry, bool dest_uninitialized) {
2222 
2223   Label L_load_element, L_store_element, L_do_card_marks, L_done;
2224 
2225   // Input registers (after setup_arg_regs)
2226   const Register from        = rdi;   // source array address
2227   const Register to          = rsi;   // destination array address
2228   const Register length      = rdx;   // elements count
2229   const Register ckoff       = rcx;   // super_check_offset
2230   const Register ckval       = r8;    // super_klass
2231 
2232   // Registers used as temps (r13, r14 are save-on-entry)
2233   const Register end_from    = from;  // source array end address
2234   const Register end_to      = r13;   // destination array end address
2235   const Register count       = rdx;   // -(count_remaining)
2236   const Register r14_length  = r14;   // saved copy of length
2237   // End pointers are inclusive, and if length is not zero they point
2238   // to the last unit copied:  end_to[0] := end_from[0]
2239 
2240   const Register rax_oop    = rax;    // actual oop copied
2241   const Register r11_klass  = r11;    // oop._klass
2242 
2243   //---------------------------------------------------------------
2244   // Assembler stub will be used for this call to arraycopy
2245   // if the two arrays are subtypes of Object[] but the
2246   // destination array type is not equal to or a supertype
2247   // of the source type.  Each element must be separately
2248   // checked.
2249 
2250   __ align(CodeEntryAlignment);
2251   StubCodeMark mark(this, "StubRoutines", name);
2252   address start = __ pc();
2253 
2254   __ enter(); // required for proper stackwalking of RuntimeStub frame
2255 
2256 #ifdef ASSERT
2257   // caller guarantees that the arrays really are different
2258   // otherwise, we would have to make conjoint checks
2259   { Label L;
2260     array_overlap_test(L, TIMES_OOP);
2261     __ stop("checkcast_copy within a single array");
2262     __ bind(L);
2263   }
2264 #endif //ASSERT
2265 
2266   setup_arg_regs_using_thread(4); // from => rdi, to => rsi, length => rdx
2267                                   // ckoff => rcx, ckval => r8
2268                                   // r9 is used to save r15_thread
2269 #ifdef _WIN64
2270   // last argument (#4) is on stack on Win64
2271   __ movptr(ckval, Address(rsp, 6 * wordSize));
2272 #endif
2273 
2274   // Caller of this entry point must set up the argument registers.
2275   if (entry != nullptr) {
2276     *entry = __ pc();
2277     BLOCK_COMMENT("Entry:");
2278   }
2279 
2280   // allocate spill slots for r13, r14
2281   enum {
2282     saved_r13_offset,
2283     saved_r14_offset,
2284     saved_r10_offset,
2285     saved_rbp_offset
2286   };
2287   __ subptr(rsp, saved_rbp_offset * wordSize);
2288   __ movptr(Address(rsp, saved_r13_offset * wordSize), r13);
2289   __ movptr(Address(rsp, saved_r14_offset * wordSize), r14);
2290   __ movptr(Address(rsp, saved_r10_offset * wordSize), r10);
2291 
2292 #ifdef ASSERT
2293     Label L2;
2294     __ get_thread(r14);
2295     __ cmpptr(r15_thread, r14);
2296     __ jcc(Assembler::equal, L2);
2297     __ stop("StubRoutines::call_stub: r15_thread is modified by call");
2298     __ bind(L2);
2299 #endif // ASSERT
2300 
2301   // check that int operands are properly extended to size_t
2302   assert_clean_int(length, rax);
2303   assert_clean_int(ckoff, rax);
2304 
2305 #ifdef ASSERT
2306   BLOCK_COMMENT("assert consistent ckoff/ckval");
2307   // The ckoff and ckval must be mutually consistent,
2308   // even though caller generates both.
2309   { Label L;
2310     int sco_offset = in_bytes(Klass::super_check_offset_offset());
2311     __ cmpl(ckoff, Address(ckval, sco_offset));
2312     __ jcc(Assembler::equal, L);
2313     __ stop("super_check_offset inconsistent");
2314     __ bind(L);
2315   }
2316 #endif //ASSERT
2317 
2318   // Loop-invariant addresses.  They are exclusive end pointers.
2319   Address end_from_addr(from, length, TIMES_OOP, 0);
2320   Address   end_to_addr(to,   length, TIMES_OOP, 0);
2321   // Loop-variant addresses.  They assume post-incremented count < 0.
2322   Address from_element_addr(end_from, count, TIMES_OOP, 0);
2323   Address   to_element_addr(end_to,   count, TIMES_OOP, 0);
2324 
2325   DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_CHECKCAST | ARRAYCOPY_DISJOINT;
2326   if (dest_uninitialized) {
2327     decorators |= IS_DEST_UNINITIALIZED;
2328   }
2329 
2330   BasicType type = T_OBJECT;
2331   size_t element_size = UseCompressedOops ? 4 : 8;
2332 
2333   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
2334   bs->arraycopy_prologue(_masm, decorators, type, from, to, count);
2335 
2336   // Copy from low to high addresses, indexed from the end of each array.
2337   __ lea(end_from, end_from_addr);
2338   __ lea(end_to,   end_to_addr);
2339   __ movptr(r14_length, length);        // save a copy of the length
2340   assert(length == count, "");          // else fix next line:
2341   __ negptr(count);                     // negate and test the length
2342   __ jcc(Assembler::notZero, L_load_element);
2343 
2344   // Empty array:  Nothing to do.
2345   __ xorptr(rax, rax);                  // return 0 on (trivial) success
2346   __ jmp(L_done);
2347 
2348   // ======== begin loop ========
2349   // (Loop is rotated; its entry is L_load_element.)
2350   // Loop control:
2351   //   for (count = -count; count != 0; count++)
2352   // Base pointers src, dst are biased by 8*(count-1),to last element.
2353   __ align(OptoLoopAlignment);
2354 
2355   __ BIND(L_store_element);
2356   bs->copy_store_at(_masm,
2357                     decorators,
2358                     type,
2359                     element_size,
2360                     to_element_addr,
2361                     rax_oop,
2362                     r10);
2363   __ increment(count);               // increment the count toward zero
2364   __ jcc(Assembler::zero, L_do_card_marks);
2365 
2366   // ======== loop entry is here ========
2367   __ BIND(L_load_element);
2368   bs->copy_load_at(_masm,
2369                    decorators,
2370                    type,
2371                    element_size,
2372                    rax_oop,
2373                    from_element_addr,
2374                    r10);
2375   __ testptr(rax_oop, rax_oop);
2376   __ jcc(Assembler::zero, L_store_element);
2377 
2378   __ load_klass(r11_klass, rax_oop, rscratch1);// query the object klass
2379   generate_type_check(r11_klass, ckoff, ckval, L_store_element);
2380   // ======== end loop ========
2381 
2382   // It was a real error; we must depend on the caller to finish the job.
2383   // Register rdx = -1 * number of *remaining* oops, r14 = *total* oops.
2384   // Emit GC store barriers for the oops we have copied (r14 + rdx),
2385   // and report their number to the caller.
2386   assert_different_registers(rax, r14_length, count, to, end_to, rcx, rscratch1);
2387   Label L_post_barrier;
2388   __ addptr(r14_length, count);     // K = (original - remaining) oops
2389   __ movptr(rax, r14_length);       // save the value
2390   __ notptr(rax);                   // report (-1^K) to caller (does not affect flags)
2391   __ jccb(Assembler::notZero, L_post_barrier);
2392   __ jmp(L_done); // K == 0, nothing was copied, skip post barrier
2393 
2394   // Come here on success only.
2395   __ BIND(L_do_card_marks);
2396   __ xorptr(rax, rax);              // return 0 on success
2397 
2398   __ BIND(L_post_barrier);
2399   bs->arraycopy_epilogue(_masm, decorators, type, from, to, r14_length);
2400 
2401   // Common exit point (success or failure).
2402   __ BIND(L_done);
2403   __ movptr(r13, Address(rsp, saved_r13_offset * wordSize));
2404   __ movptr(r14, Address(rsp, saved_r14_offset * wordSize));
2405   __ movptr(r10, Address(rsp, saved_r10_offset * wordSize));
2406   restore_arg_regs_using_thread();
2407   INC_COUNTER_NP(SharedRuntime::_checkcast_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free
2408   __ leave(); // required for proper stackwalking of RuntimeStub frame
2409   __ ret(0);
2410 
2411   return start;
2412 }
2413 
2414 
2415 //  Generate 'unsafe' array copy stub
2416 //  Though just as safe as the other stubs, it takes an unscaled
2417 //  size_t argument instead of an element count.
2418 //
2419 //  Input:
2420 //    c_rarg0   - source array address
2421 //    c_rarg1   - destination array address
2422 //    c_rarg2   - byte count, treated as ssize_t, can be zero
2423 //
2424 // Examines the alignment of the operands and dispatches
2425 // to a long, int, short, or byte copy loop.
2426 //
2427 address StubGenerator::generate_unsafe_copy(const char *name,
2428                                             address byte_copy_entry, address short_copy_entry,
2429                                             address int_copy_entry, address long_copy_entry) {
2430 
2431   Label L_long_aligned, L_int_aligned, L_short_aligned;
2432 
2433   // Input registers (before setup_arg_regs)
2434   const Register from        = c_rarg0;  // source array address
2435   const Register to          = c_rarg1;  // destination array address
2436   const Register size        = c_rarg2;  // byte count (size_t)
2437 
2438   // Register used as a temp
2439   const Register bits        = rax;      // test copy of low bits
2440 
2441   __ align(CodeEntryAlignment);
2442   StubCodeMark mark(this, "StubRoutines", name);
2443   address start = __ pc();
2444 
2445   __ enter(); // required for proper stackwalking of RuntimeStub frame
2446 
2447   // bump this on entry, not on exit:
2448   INC_COUNTER_NP(SharedRuntime::_unsafe_array_copy_ctr, rscratch1);
2449 
2450   __ mov(bits, from);
2451   __ orptr(bits, to);
2452   __ orptr(bits, size);
2453 
2454   __ testb(bits, BytesPerLong-1);
2455   __ jccb(Assembler::zero, L_long_aligned);
2456 
2457   __ testb(bits, BytesPerInt-1);
2458   __ jccb(Assembler::zero, L_int_aligned);
2459 
2460   __ testb(bits, BytesPerShort-1);
2461   __ jump_cc(Assembler::notZero, RuntimeAddress(byte_copy_entry));
2462 
2463   __ BIND(L_short_aligned);
2464   __ shrptr(size, LogBytesPerShort); // size => short_count
2465   __ jump(RuntimeAddress(short_copy_entry));
2466 
2467   __ BIND(L_int_aligned);
2468   __ shrptr(size, LogBytesPerInt); // size => int_count
2469   __ jump(RuntimeAddress(int_copy_entry));
2470 
2471   __ BIND(L_long_aligned);
2472   __ shrptr(size, LogBytesPerLong); // size => qword_count
2473   __ jump(RuntimeAddress(long_copy_entry));
2474 
2475   return start;
2476 }
2477 
2478 
2479 // Perform range checks on the proposed arraycopy.
2480 // Kills temp, but nothing else.
2481 // Also, clean the sign bits of src_pos and dst_pos.
2482 void StubGenerator::arraycopy_range_checks(Register src,     // source array oop (c_rarg0)
2483                                            Register src_pos, // source position (c_rarg1)
2484                                            Register dst,     // destination array oo (c_rarg2)
2485                                            Register dst_pos, // destination position (c_rarg3)
2486                                            Register length,
2487                                            Register temp,
2488                                            Label& L_failed) {
2489   BLOCK_COMMENT("arraycopy_range_checks:");
2490 
2491   //  if (src_pos + length > arrayOop(src)->length())  FAIL;
2492   __ movl(temp, length);
2493   __ addl(temp, src_pos);             // src_pos + length
2494   __ cmpl(temp, Address(src, arrayOopDesc::length_offset_in_bytes()));
2495   __ jcc(Assembler::above, L_failed);
2496 
2497   //  if (dst_pos + length > arrayOop(dst)->length())  FAIL;
2498   __ movl(temp, length);
2499   __ addl(temp, dst_pos);             // dst_pos + length
2500   __ cmpl(temp, Address(dst, arrayOopDesc::length_offset_in_bytes()));
2501   __ jcc(Assembler::above, L_failed);
2502 
2503   // Have to clean up high 32-bits of 'src_pos' and 'dst_pos'.
2504   // Move with sign extension can be used since they are positive.
2505   __ movslq(src_pos, src_pos);
2506   __ movslq(dst_pos, dst_pos);
2507 
2508   BLOCK_COMMENT("arraycopy_range_checks done");
2509 }
2510 
2511 
2512 //  Generate generic array copy stubs
2513 //
2514 //  Input:
2515 //    c_rarg0    -  src oop
2516 //    c_rarg1    -  src_pos (32-bits)
2517 //    c_rarg2    -  dst oop
2518 //    c_rarg3    -  dst_pos (32-bits)
2519 // not Win64
2520 //    c_rarg4    -  element count (32-bits)
2521 // Win64
2522 //    rsp+40     -  element count (32-bits)
2523 //
2524 //  Output:
2525 //    rax ==  0  -  success
2526 //    rax == -1^K - failure, where K is partial transfer count
2527 //
2528 address StubGenerator::generate_generic_copy(const char *name,
2529                                              address byte_copy_entry, address short_copy_entry,
2530                                              address int_copy_entry, address oop_copy_entry,
2531                                              address long_copy_entry, address checkcast_copy_entry) {
2532 
2533   Label L_failed, L_failed_0, L_objArray;
2534   Label L_copy_shorts, L_copy_ints, L_copy_longs;
2535 
2536   // Input registers
2537   const Register src        = c_rarg0;  // source array oop
2538   const Register src_pos    = c_rarg1;  // source position
2539   const Register dst        = c_rarg2;  // destination array oop
2540   const Register dst_pos    = c_rarg3;  // destination position
2541 #ifndef _WIN64
2542   const Register length     = c_rarg4;
2543   const Register rklass_tmp = r9;  // load_klass
2544 #else
2545   const Address  length(rsp, 7 * wordSize);  // elements count is on stack on Win64
2546   const Register rklass_tmp = rdi;  // load_klass
2547 #endif
2548 
2549   { int modulus = CodeEntryAlignment;
2550     int target  = modulus - 5; // 5 = sizeof jmp(L_failed)
2551     int advance = target - (__ offset() % modulus);
2552     if (advance < 0)  advance += modulus;
2553     if (advance > 0)  __ nop(advance);
2554   }
2555   StubCodeMark mark(this, "StubRoutines", name);
2556 
2557   // Short-hop target to L_failed.  Makes for denser prologue code.
2558   __ BIND(L_failed_0);
2559   __ jmp(L_failed);
2560   assert(__ offset() % CodeEntryAlignment == 0, "no further alignment needed");
2561 
2562   __ align(CodeEntryAlignment);
2563   address start = __ pc();
2564 
2565   __ enter(); // required for proper stackwalking of RuntimeStub frame
2566 
2567 #ifdef _WIN64
2568   __ push(rklass_tmp); // rdi is callee-save on Windows
2569 #endif
2570 
2571   // bump this on entry, not on exit:
2572   INC_COUNTER_NP(SharedRuntime::_generic_array_copy_ctr, rscratch1);
2573 
2574   //-----------------------------------------------------------------------
2575   // Assembler stub will be used for this call to arraycopy
2576   // if the following conditions are met:
2577   //
2578   // (1) src and dst must not be null.
2579   // (2) src_pos must not be negative.
2580   // (3) dst_pos must not be negative.
2581   // (4) length  must not be negative.
2582   // (5) src klass and dst klass should be the same and not null.
2583   // (6) src and dst should be arrays.
2584   // (7) src_pos + length must not exceed length of src.
2585   // (8) dst_pos + length must not exceed length of dst.
2586   //
2587 
2588   //  if (src == nullptr) return -1;
2589   __ testptr(src, src);         // src oop
2590   size_t j1off = __ offset();
2591   __ jccb(Assembler::zero, L_failed_0);
2592 
2593   //  if (src_pos < 0) return -1;
2594   __ testl(src_pos, src_pos); // src_pos (32-bits)
2595   __ jccb(Assembler::negative, L_failed_0);
2596 
2597   //  if (dst == nullptr) return -1;
2598   __ testptr(dst, dst);         // dst oop
2599   __ jccb(Assembler::zero, L_failed_0);
2600 
2601   //  if (dst_pos < 0) return -1;
2602   __ testl(dst_pos, dst_pos); // dst_pos (32-bits)
2603   size_t j4off = __ offset();
2604   __ jccb(Assembler::negative, L_failed_0);
2605 
2606   // The first four tests are very dense code,
2607   // but not quite dense enough to put four
2608   // jumps in a 16-byte instruction fetch buffer.
2609   // That's good, because some branch predicters
2610   // do not like jumps so close together.
2611   // Make sure of this.
2612   guarantee(((j1off ^ j4off) & ~15) != 0, "I$ line of 1st & 4th jumps");
2613 
2614   // registers used as temp
2615   const Register r11_length    = r11; // elements count to copy
2616   const Register r10_src_klass = r10; // array klass
2617 
2618   //  if (length < 0) return -1;
2619   __ movl(r11_length, length);        // length (elements count, 32-bits value)
2620   __ testl(r11_length, r11_length);
2621   __ jccb(Assembler::negative, L_failed_0);
2622 
2623   __ load_klass(r10_src_klass, src, rklass_tmp);
2624 #ifdef ASSERT
2625   //  assert(src->klass() != nullptr);
2626   {
2627     BLOCK_COMMENT("assert klasses not null {");
2628     Label L1, L2;
2629     __ testptr(r10_src_klass, r10_src_klass);
2630     __ jcc(Assembler::notZero, L2);   // it is broken if klass is null
2631     __ bind(L1);
2632     __ stop("broken null klass");
2633     __ bind(L2);
2634     __ load_klass(rax, dst, rklass_tmp);
2635     __ cmpq(rax, 0);
2636     __ jcc(Assembler::equal, L1);     // this would be broken also
2637     BLOCK_COMMENT("} assert klasses not null done");
2638   }
2639 #endif
2640 
2641   // Load layout helper (32-bits)
2642   //
2643   //  |array_tag|     | header_size | element_type |     |log2_element_size|
2644   // 32        30    24            16              8     2                 0
2645   //
2646   //   array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
2647   //
2648 
2649   const int lh_offset = in_bytes(Klass::layout_helper_offset());
2650 
2651   // Handle objArrays completely differently...
2652   const jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2653   __ cmpl(Address(r10_src_klass, lh_offset), objArray_lh);
2654   __ jcc(Assembler::equal, L_objArray);
2655 
2656   //  if (src->klass() != dst->klass()) return -1;
2657   __ load_klass(rax, dst, rklass_tmp);
2658   __ cmpq(r10_src_klass, rax);
2659   __ jcc(Assembler::notEqual, L_failed);
2660 
2661   // Check for flat inline type array -> return -1
2662   __ test_flat_array_oop(src, rax, L_failed);
2663 
2664   // Check for null-free (non-flat) inline type array -> handle as object array
2665   __ test_null_free_array_oop(src, rax, L_objArray);
2666 
2667   const Register rax_lh = rax;  // layout helper
2668   __ movl(rax_lh, Address(r10_src_klass, lh_offset));
2669 
2670   // Check for flat inline type array -> return -1
2671   __ testl(rax_lh, Klass::_lh_array_tag_flat_value_bit_inplace);
2672   __ jcc(Assembler::notZero, L_failed);
2673 
2674   //  if (!src->is_Array()) return -1;
2675   __ cmpl(rax_lh, Klass::_lh_neutral_value);
2676   __ jcc(Assembler::greaterEqual, L_failed);
2677 
2678   // At this point, it is known to be a typeArray (array_tag 0x3).
2679 #ifdef ASSERT
2680   {
2681     BLOCK_COMMENT("assert primitive array {");
2682     Label L;
2683     __ movl(rklass_tmp, rax_lh);
2684     __ sarl(rklass_tmp, Klass::_lh_array_tag_shift);
2685     __ cmpl(rklass_tmp, Klass::_lh_array_tag_type_value);
2686     __ jcc(Assembler::equal, L);
2687     __ stop("must be a primitive array");
2688     __ bind(L);
2689     BLOCK_COMMENT("} assert primitive array done");
2690   }
2691 #endif
2692 
2693   arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
2694                          r10, L_failed);
2695 
2696   // TypeArrayKlass
2697   //
2698   // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize);
2699   // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize);
2700   //
2701 
2702   const Register r10_offset = r10;    // array offset
2703   const Register rax_elsize = rax_lh; // element size
2704 
2705   __ movl(r10_offset, rax_lh);
2706   __ shrl(r10_offset, Klass::_lh_header_size_shift);
2707   __ andptr(r10_offset, Klass::_lh_header_size_mask);   // array_offset
2708   __ addptr(src, r10_offset);           // src array offset
2709   __ addptr(dst, r10_offset);           // dst array offset
2710   BLOCK_COMMENT("choose copy loop based on element size");
2711   __ andl(rax_lh, Klass::_lh_log2_element_size_mask); // rax_lh -> rax_elsize
2712 
2713 #ifdef _WIN64
2714   __ pop(rklass_tmp); // Restore callee-save rdi
2715 #endif
2716 
2717   // next registers should be set before the jump to corresponding stub
2718   const Register from     = c_rarg0;  // source array address
2719   const Register to       = c_rarg1;  // destination array address
2720   const Register count    = c_rarg2;  // elements count
2721 
2722   // 'from', 'to', 'count' registers should be set in such order
2723   // since they are the same as 'src', 'src_pos', 'dst'.
2724 
2725   __ cmpl(rax_elsize, 0);
2726   __ jccb(Assembler::notEqual, L_copy_shorts);
2727   __ lea(from, Address(src, src_pos, Address::times_1, 0));// src_addr
2728   __ lea(to,   Address(dst, dst_pos, Address::times_1, 0));// dst_addr
2729   __ movl2ptr(count, r11_length); // length
2730   __ jump(RuntimeAddress(byte_copy_entry));
2731 
2732 __ BIND(L_copy_shorts);
2733   __ cmpl(rax_elsize, LogBytesPerShort);
2734   __ jccb(Assembler::notEqual, L_copy_ints);
2735   __ lea(from, Address(src, src_pos, Address::times_2, 0));// src_addr
2736   __ lea(to,   Address(dst, dst_pos, Address::times_2, 0));// dst_addr
2737   __ movl2ptr(count, r11_length); // length
2738   __ jump(RuntimeAddress(short_copy_entry));
2739 
2740 __ BIND(L_copy_ints);
2741   __ cmpl(rax_elsize, LogBytesPerInt);
2742   __ jccb(Assembler::notEqual, L_copy_longs);
2743   __ lea(from, Address(src, src_pos, Address::times_4, 0));// src_addr
2744   __ lea(to,   Address(dst, dst_pos, Address::times_4, 0));// dst_addr
2745   __ movl2ptr(count, r11_length); // length
2746   __ jump(RuntimeAddress(int_copy_entry));
2747 
2748 __ BIND(L_copy_longs);
2749 #ifdef ASSERT
2750   {
2751     BLOCK_COMMENT("assert long copy {");
2752     Label L;
2753     __ cmpl(rax_elsize, LogBytesPerLong);
2754     __ jcc(Assembler::equal, L);
2755     __ stop("must be long copy, but elsize is wrong");
2756     __ bind(L);
2757     BLOCK_COMMENT("} assert long copy done");
2758   }
2759 #endif
2760   __ lea(from, Address(src, src_pos, Address::times_8, 0));// src_addr
2761   __ lea(to,   Address(dst, dst_pos, Address::times_8, 0));// dst_addr
2762   __ movl2ptr(count, r11_length); // length
2763   __ jump(RuntimeAddress(long_copy_entry));
2764 
2765   // ObjArrayKlass
2766 __ BIND(L_objArray);
2767   // live at this point:  r10_src_klass, r11_length, src[_pos], dst[_pos]
2768 
2769   Label L_plain_copy, L_checkcast_copy;
2770   //  test array classes for subtyping
2771   __ load_klass(rax, dst, rklass_tmp);
2772   __ cmpq(r10_src_klass, rax); // usual case is exact equality
2773   __ jcc(Assembler::notEqual, L_checkcast_copy);
2774 
2775   // Identically typed arrays can be copied without element-wise checks.
2776   arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
2777                          r10, L_failed);
2778 
2779   __ lea(from, Address(src, src_pos, TIMES_OOP,
2780                arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // src_addr
2781   __ lea(to,   Address(dst, dst_pos, TIMES_OOP,
2782                arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // dst_addr
2783   __ movl2ptr(count, r11_length); // length
2784 __ BIND(L_plain_copy);
2785 #ifdef _WIN64
2786   __ pop(rklass_tmp); // Restore callee-save rdi
2787 #endif
2788   __ jump(RuntimeAddress(oop_copy_entry));
2789 
2790 __ BIND(L_checkcast_copy);
2791   // live at this point:  r10_src_klass, r11_length, rax (dst_klass)
2792   {
2793     // Before looking at dst.length, make sure dst is also an objArray.
2794     // This check also fails for flat arrays which are not supported.
2795     __ cmpl(Address(rax, lh_offset), objArray_lh);
2796     __ jcc(Assembler::notEqual, L_failed);
2797 
2798 #ifdef ASSERT
2799     {
2800       BLOCK_COMMENT("assert not null-free array {");
2801       Label L;
2802       __ test_non_null_free_array_oop(dst, rklass_tmp, L);
2803       __ stop("unexpected null-free array");
2804       __ bind(L);
2805       BLOCK_COMMENT("} assert not null-free array");
2806     }
2807 #endif
2808 
2809     // It is safe to examine both src.length and dst.length.
2810     arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
2811                            rax, L_failed);
2812 
2813     const Register r11_dst_klass = r11;
2814     __ load_klass(r11_dst_klass, dst, rklass_tmp); // reload
2815 
2816     // Marshal the base address arguments now, freeing registers.
2817     __ lea(from, Address(src, src_pos, TIMES_OOP,
2818                  arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
2819     __ lea(to,   Address(dst, dst_pos, TIMES_OOP,
2820                  arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
2821     __ movl(count, length);           // length (reloaded)
2822     Register sco_temp = c_rarg3;      // this register is free now
2823     assert_different_registers(from, to, count, sco_temp,
2824                                r11_dst_klass, r10_src_klass);
2825     assert_clean_int(count, sco_temp);
2826 
2827     // Generate the type check.
2828     const int sco_offset = in_bytes(Klass::super_check_offset_offset());
2829     __ movl(sco_temp, Address(r11_dst_klass, sco_offset));
2830     assert_clean_int(sco_temp, rax);
2831     generate_type_check(r10_src_klass, sco_temp, r11_dst_klass, L_plain_copy);
2832 
2833     // Fetch destination element klass from the ObjArrayKlass header.
2834     int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
2835     __ movptr(r11_dst_klass, Address(r11_dst_klass, ek_offset));
2836     __ movl(  sco_temp,      Address(r11_dst_klass, sco_offset));
2837     assert_clean_int(sco_temp, rax);
2838 
2839 #ifdef _WIN64
2840     __ pop(rklass_tmp); // Restore callee-save rdi
2841 #endif
2842 
2843     // the checkcast_copy loop needs two extra arguments:
2844     assert(c_rarg3 == sco_temp, "#3 already in place");
2845     // Set up arguments for checkcast_copy_entry.
2846     setup_arg_regs_using_thread(4);
2847     __ movptr(r8, r11_dst_klass);  // dst.klass.element_klass, r8 is c_rarg4 on Linux/Solaris
2848     __ jump(RuntimeAddress(checkcast_copy_entry));
2849   }
2850 
2851 __ BIND(L_failed);
2852 #ifdef _WIN64
2853   __ pop(rklass_tmp); // Restore callee-save rdi
2854 #endif
2855   __ xorptr(rax, rax);
2856   __ notptr(rax); // return -1
2857   __ leave();   // required for proper stackwalking of RuntimeStub frame
2858   __ ret(0);
2859 
2860   return start;
2861 }
2862 
2863 #undef __