1 /*
   2  * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/assembler.hpp"
  27 #include "asm/assembler.inline.hpp"
  28 #include "opto/c2_MacroAssembler.hpp"
  29 #include "opto/compile.hpp"
  30 #include "opto/intrinsicnode.hpp"
  31 #include "opto/matcher.hpp"
  32 #include "opto/output.hpp"
  33 #include "opto/subnode.hpp"
  34 #include "runtime/stubRoutines.hpp"
  35 #include "utilities/globalDefinitions.hpp"
  36 
  37 #ifdef PRODUCT
  38 #define BLOCK_COMMENT(str) /* nothing */
  39 #define STOP(error) stop(error)
  40 #else
  41 #define BLOCK_COMMENT(str) block_comment(str)
  42 #define STOP(error) block_comment(error); stop(error)
  43 #endif
  44 
  45 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  46 
  47 typedef void (MacroAssembler::* chr_insn)(Register Rt, const Address &adr);
  48 
  49 void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg, Register tmpReg,
  50                                   Register tmp2Reg, Register tmp3Reg) {
  51   Register oop = objectReg;
  52   Register box = boxReg;
  53   Register disp_hdr = tmpReg;
  54   Register tmp = tmp2Reg;
  55   Label object_has_monitor;
  56   Label count, no_count;
  57 
  58   assert(LockingMode != LM_LIGHTWEIGHT, "lightweight locking should use fast_lock_lightweight");
  59   assert_different_registers(oop, box, tmp, disp_hdr, rscratch1);
  60 
  61   // Load markWord from object into displaced_header.
  62   ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
  63 
  64   if (DiagnoseSyncOnValueBasedClasses != 0) {
  65     load_klass(tmp, oop);
  66     ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
  67     tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
  68     br(Assembler::NE, no_count);
  69   }
  70 
  71   // Check for existing monitor
  72   tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
  73 
  74   if (LockingMode == LM_MONITOR) {
  75     tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
  76     b(no_count);
  77   } else {
  78     assert(LockingMode == LM_LEGACY, "must be");
  79     // Set tmp to be (markWord of object | UNLOCK_VALUE).
  80     orr(tmp, disp_hdr, markWord::unlocked_value);
  81 
  82     // Initialize the box. (Must happen before we update the object mark!)
  83     str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
  84 
  85     // Compare object markWord with an unlocked value (tmp) and if
  86     // equal exchange the stack address of our box with object markWord.
  87     // On failure disp_hdr contains the possibly locked markWord.
  88     cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
  89             /*release*/ true, /*weak*/ false, disp_hdr);
  90     br(Assembler::EQ, count);
  91 
  92     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
  93 
  94     // If the compare-and-exchange succeeded, then we found an unlocked
  95     // object, will have now locked it will continue at label cont
  96 
  97     // Check if the owner is self by comparing the value in the
  98     // markWord of object (disp_hdr) with the stack pointer.
  99     mov(rscratch1, sp);
 100     sub(disp_hdr, disp_hdr, rscratch1);
 101     mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
 102     // If condition is true we are cont and hence we can store 0 as the
 103     // displaced header in the box, which indicates that it is a recursive lock.
 104     ands(tmp/*==0?*/, disp_hdr, tmp);   // Sets flags for result
 105     str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 106     b(no_count);
 107   }
 108 
 109   // Handle existing monitor.
 110   bind(object_has_monitor);
 111 
 112   // The object's monitor m is unlocked iff m->owner == nullptr,
 113   // otherwise m->owner may contain a thread id, a stack address for LM_LEGACY,
 114   // or the ANONYMOUS_OWNER constant for LM_LIGHTWEIGHT.
 115   //
 116   // Try to CAS m->owner from null to current thread.
 117   ldr(rscratch2, Address(rthread, JavaThread::lock_id_offset()));
 118   add(tmp, disp_hdr, (in_bytes(ObjectMonitor::owner_offset())-markWord::monitor_value));
 119   cmpxchg(tmp, zr, rscratch2, Assembler::xword, /*acquire*/ true,
 120           /*release*/ true, /*weak*/ false, tmp3Reg); // Sets flags for result
 121 
 122   // Store a non-null value into the box to avoid looking like a re-entrant
 123   // lock. The fast-path monitor unlock code checks for
 124   // markWord::monitor_value so use markWord::unused_mark which has the
 125   // relevant bit set, and also matches ObjectSynchronizer::enter.
 126   mov(tmp, (address)markWord::unused_mark().value());
 127   str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 128 
 129   br(Assembler::EQ, no_count); // CAS success means locking succeeded
 130 
 131   cmp(tmp3Reg, rscratch2);
 132   br(Assembler::NE, no_count); // Check for recursive locking
 133 
 134   // Recursive lock case
 135   increment(Address(disp_hdr, in_bytes(ObjectMonitor::recursions_offset()) - markWord::monitor_value), 1);
 136   // flag == EQ still from the cmp above, checking if this is a reentrant lock
 137   b(no_count);
 138 
 139   bind(count);
 140   inc_held_monitor_count();
 141 
 142   bind(no_count);
 143   // flag == EQ indicates success
 144   // flag == NE indicates failure
 145 }
 146 
 147 void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg, Register tmpReg,
 148                                     Register tmp2Reg) {
 149   Register oop = objectReg;
 150   Register box = boxReg;
 151   Register disp_hdr = tmpReg;
 152   Register tmp = tmp2Reg;
 153   Label cont;
 154   Label object_has_monitor;
 155   Label no_count;
 156 
 157   assert(LockingMode != LM_LIGHTWEIGHT, "lightweight locking should use fast_unlock_lightweight");
 158   assert_different_registers(oop, box, tmp, disp_hdr);
 159 
 160   if (LockingMode == LM_LEGACY) {
 161     // Find the lock address and load the displaced header from the stack.
 162     ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 163 
 164     // If the displaced header is 0, we have a recursive unlock.
 165     cmp(disp_hdr, zr);
 166     br(Assembler::EQ, no_count);
 167   }
 168 
 169   // Handle existing monitor.
 170   ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
 171   tbnz(tmp, exact_log2(markWord::monitor_value), object_has_monitor);
 172 
 173   if (LockingMode == LM_MONITOR) {
 174     tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
 175     b(no_count);
 176   } else {
 177     assert(LockingMode == LM_LEGACY, "must be");
 178     // Check if it is still a light weight lock, this is is true if we
 179     // see the stack address of the basicLock in the markWord of the
 180     // object.
 181 
 182     cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
 183             /*release*/ true, /*weak*/ false, tmp);
 184     b(cont);
 185   }
 186 
 187   assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 188 
 189   // Handle existing monitor.
 190   bind(object_has_monitor);
 191   STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
 192   add(tmp, tmp, -(int)markWord::monitor_value); // monitor
 193 
 194   // If the owner is anonymous, we need to fix it -- in an outline stub.
 195   Register tmp2 = disp_hdr;
 196   ldr(tmp2, Address(tmp, ObjectMonitor::owner_offset()));
 197   // We cannot use tbnz here, the target might be too far away and cannot
 198   // be encoded.
 199   mov(rscratch1, (uint64_t)ObjectMonitor::ANONYMOUS_OWNER);
 200   cmp(tmp2, rscratch1);
 201   C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmp, tmp2);
 202   Compile::current()->output()->add_stub(stub);
 203   br(Assembler::EQ, stub->entry());
 204   bind(stub->continuation());
 205 
 206   ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
 207 
 208   Label notRecursive;
 209   cbz(disp_hdr, notRecursive);
 210 
 211   // Recursive lock
 212   sub(disp_hdr, disp_hdr, 1u);
 213   str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
 214   cmp(disp_hdr, disp_hdr); // Sets flags for result
 215   b(no_count);
 216 
 217   bind(notRecursive);
 218   ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset()));
 219   ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset()));
 220   orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
 221   cmp(rscratch1, zr); // Sets flags for result
 222   cbnz(rscratch1, no_count);
 223   // need a release store here
 224   lea(tmp, Address(tmp, ObjectMonitor::owner_offset()));
 225   stlr(zr, tmp); // set unowned
 226   b(no_count);
 227 
 228   bind(cont);
 229   // flag == EQ indicates success
 230   // flag == NE indicates failure
 231   br(Assembler::NE, no_count);
 232 
 233   dec_held_monitor_count();
 234 
 235   bind(no_count);
 236 }
 237 
 238 void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register t1,
 239                                               Register t2, Register t3, Register t4) {
 240   assert(LockingMode == LM_LIGHTWEIGHT, "must be");
 241   assert_different_registers(obj, t1, t2, t3, t4);
 242 
 243   // Handle inflated monitor.
 244   Label inflated;
 245   // Finish fast lock successfully. MUST branch to with flag == EQ
 246   Label locked;
 247   // Finish fast lock unsuccessfully. MUST branch to with flag == NE
 248   Label slow_path;
 249 
 250   if (DiagnoseSyncOnValueBasedClasses != 0) {
 251     load_klass(t1, obj);
 252     ldrw(t1, Address(t1, Klass::access_flags_offset()));
 253     tstw(t1, JVM_ACC_IS_VALUE_BASED_CLASS);
 254     br(Assembler::NE, slow_path);
 255   }
 256 
 257   const Register t1_mark = t1;
 258 
 259   { // Lightweight locking
 260 
 261     // Push lock to the lock stack and finish successfully. MUST branch to with flag == EQ
 262     Label push;
 263 
 264     const Register t2_top = t2;
 265     const Register t3_t = t3;
 266 
 267     // Check if lock-stack is full.
 268     ldrw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
 269     cmpw(t2_top, (unsigned)LockStack::end_offset() - 1);
 270     br(Assembler::GT, slow_path);
 271 
 272     // Check if recursive.
 273     subw(t3_t, t2_top, oopSize);
 274     ldr(t3_t, Address(rthread, t3_t));
 275     cmp(obj, t3_t);
 276     br(Assembler::EQ, push);
 277 
 278     // Relaxed normal load to check for monitor. Optimization for monitor case.
 279     ldr(t1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
 280     tbnz(t1_mark, exact_log2(markWord::monitor_value), inflated);
 281 
 282     // Not inflated
 283     assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid a lea");
 284 
 285     // Try to lock. Transition lock-bits 0b01 => 0b00
 286     orr(t1_mark, t1_mark, markWord::unlocked_value);
 287     eor(t3_t, t1_mark, markWord::unlocked_value);
 288     cmpxchg(/*addr*/ obj, /*expected*/ t1_mark, /*new*/ t3_t, Assembler::xword,
 289             /*acquire*/ true, /*release*/ false, /*weak*/ false, noreg);
 290     br(Assembler::NE, slow_path);
 291 
 292     bind(push);
 293     // After successful lock, push object on lock-stack.
 294     str(obj, Address(rthread, t2_top));
 295     addw(t2_top, t2_top, oopSize);
 296     strw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
 297     b(locked);
 298   }
 299 
 300   { // Handle inflated monitor.
 301     bind(inflated);
 302 
 303     // mark contains the tagged ObjectMonitor*.
 304     const Register t1_tagged_monitor = t1_mark;
 305     const uintptr_t monitor_tag = markWord::monitor_value;
 306     const Register t2_owner_addr = t2;
 307     const Register t3_owner = t3;
 308 
 309     // Compute owner address.
 310     lea(t2_owner_addr, Address(t1_tagged_monitor, (in_bytes(ObjectMonitor::owner_offset()) - monitor_tag)));
 311 
 312     // CAS owner (null => current thread id).
 313     ldr(t4, Address(rthread, JavaThread::lock_id_offset()));
 314     cmpxchg(t2_owner_addr, zr, t4, Assembler::xword, /*acquire*/ true,
 315             /*release*/ false, /*weak*/ false, t3_owner);
 316     br(Assembler::EQ, locked);
 317 
 318     // Check if recursive.
 319     cmp(t3_owner, t4);
 320     br(Assembler::NE, slow_path);
 321 
 322     // Recursive.
 323     increment(Address(t1_tagged_monitor, in_bytes(ObjectMonitor::recursions_offset()) - monitor_tag), 1);
 324   }
 325 
 326   bind(locked);
 327 
 328 #ifdef ASSERT
 329   // Check that locked label is reached with Flags == EQ.
 330   Label flag_correct;
 331   br(Assembler::EQ, flag_correct);
 332   stop("Fast Lock Flag != EQ");
 333 #endif
 334 
 335   bind(slow_path);
 336 #ifdef ASSERT
 337   // Check that slow_path label is reached with Flags == NE.
 338   br(Assembler::NE, flag_correct);
 339   stop("Fast Lock Flag != NE");
 340   bind(flag_correct);
 341 #endif
 342   // C2 uses the value of Flags (NE vs EQ) to determine the continuation.
 343 }
 344 
 345 void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register t1, Register t2,
 346                                                 Register t3) {
 347   assert(LockingMode == LM_LIGHTWEIGHT, "must be");
 348   assert_different_registers(obj, t1, t2, t3);
 349 
 350   // Handle inflated monitor.
 351   Label inflated, inflated_load_monitor;
 352   // Finish fast unlock successfully. MUST branch to with flag == EQ
 353   Label unlocked;
 354   // Finish fast unlock unsuccessfully. MUST branch to with flag == NE
 355   Label slow_path;
 356 
 357   const Register t1_mark = t1;
 358   const Register t2_top = t2;
 359   const Register t3_t = t3;
 360 
 361   { // Lightweight unlock
 362 
 363     // Check if obj is top of lock-stack.
 364     ldrw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
 365     subw(t2_top, t2_top, oopSize);
 366     ldr(t3_t, Address(rthread, t2_top));
 367     cmp(obj, t3_t);
 368     // Top of lock stack was not obj. Must be monitor.
 369     br(Assembler::NE, inflated_load_monitor);
 370 
 371     // Pop lock-stack.
 372     DEBUG_ONLY(str(zr, Address(rthread, t2_top));)
 373     strw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
 374 
 375     // Check if recursive.
 376     subw(t3_t, t2_top, oopSize);
 377     ldr(t3_t, Address(rthread, t3_t));
 378     cmp(obj, t3_t);
 379     br(Assembler::EQ, unlocked);
 380 
 381     // Not recursive.
 382     // Load Mark.
 383     ldr(t1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
 384 
 385     // Check header for monitor (0b10).
 386     tbnz(t1_mark, exact_log2(markWord::monitor_value), inflated);
 387 
 388     // Try to unlock. Transition lock bits 0b00 => 0b01
 389     assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
 390     orr(t3_t, t1_mark, markWord::unlocked_value);
 391     cmpxchg(/*addr*/ obj, /*expected*/ t1_mark, /*new*/ t3_t, Assembler::xword,
 392             /*acquire*/ false, /*release*/ true, /*weak*/ false, noreg);
 393     br(Assembler::EQ, unlocked);
 394 
 395     // Compare and exchange failed.
 396     // Restore lock-stack and handle the unlock in runtime.
 397     DEBUG_ONLY(str(obj, Address(rthread, t2_top));)
 398     addw(t2_top, t2_top, oopSize);
 399     str(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
 400     b(slow_path);
 401   }
 402 
 403 
 404   { // Handle inflated monitor.
 405     bind(inflated_load_monitor);
 406     ldr(t1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
 407 #ifdef ASSERT
 408     tbnz(t1_mark, exact_log2(markWord::monitor_value), inflated);
 409     stop("Fast Unlock not monitor");
 410 #endif
 411 
 412     bind(inflated);
 413 
 414 #ifdef ASSERT
 415     Label check_done;
 416     subw(t2_top, t2_top, oopSize);
 417     cmpw(t2_top, in_bytes(JavaThread::lock_stack_base_offset()));
 418     br(Assembler::LT, check_done);
 419     ldr(t3_t, Address(rthread, t2_top));
 420     cmp(obj, t3_t);
 421     br(Assembler::NE, inflated);
 422     stop("Fast Unlock lock on stack");
 423     bind(check_done);
 424 #endif
 425 
 426     // mark contains the tagged ObjectMonitor*.
 427     const Register t1_monitor = t1_mark;
 428     const uintptr_t monitor_tag = markWord::monitor_value;
 429 
 430     // Untag the monitor.
 431     sub(t1_monitor, t1_mark, monitor_tag);
 432 
 433     const Register t2_recursions = t2;
 434     Label not_recursive;
 435 
 436     // Check if recursive.
 437     ldr(t2_recursions, Address(t1_monitor, ObjectMonitor::recursions_offset()));
 438     cbz(t2_recursions, not_recursive);
 439 
 440     // Recursive unlock.
 441     sub(t2_recursions, t2_recursions, 1u);
 442     str(t2_recursions, Address(t1_monitor, ObjectMonitor::recursions_offset()));
 443     // Set flag == EQ
 444     cmp(t2_recursions, t2_recursions);
 445     b(unlocked);
 446 
 447     bind(not_recursive);
 448 
 449     Label release;
 450     const Register t2_owner_addr = t2;
 451 
 452     // Compute owner address.
 453     lea(t2_owner_addr, Address(t1_monitor, ObjectMonitor::owner_offset()));
 454 
 455     // Check if the entry lists are empty.
 456     ldr(rscratch1, Address(t1_monitor, ObjectMonitor::EntryList_offset()));
 457     ldr(t3_t, Address(t1_monitor, ObjectMonitor::cxq_offset()));
 458     orr(rscratch1, rscratch1, t3_t);
 459     cmp(rscratch1, zr);
 460     br(Assembler::EQ, release);
 461 
 462     // The owner may be anonymous and we removed the last obj entry in
 463     // the lock-stack. This loses the information about the owner.
 464     // Write the thread id to the owner field so the runtime knows the owner.
 465     ldr(t3_t, Address(rthread, JavaThread::lock_id_offset()));
 466     str(t3_t, Address(t2_owner_addr));
 467     b(slow_path);
 468 
 469     bind(release);
 470     // Set owner to null.
 471     // Release to satisfy the JMM
 472     stlr(zr, t2_owner_addr);
 473   }
 474 
 475   bind(unlocked);
 476 
 477 #ifdef ASSERT
 478   // Check that unlocked label is reached with Flags == EQ.
 479   Label flag_correct;
 480   br(Assembler::EQ, flag_correct);
 481   stop("Fast Unlock Flag != EQ");
 482 #endif
 483 
 484   bind(slow_path);
 485 #ifdef ASSERT
 486   // Check that slow_path label is reached with Flags == NE.
 487   br(Assembler::NE, flag_correct);
 488   stop("Fast Unlock Flag != NE");
 489   bind(flag_correct);
 490 #endif
 491   // C2 uses the value of Flags (NE vs EQ) to determine the continuation.
 492 }
 493 
 494 // Search for str1 in str2 and return index or -1
 495 // Clobbers: rscratch1, rscratch2, rflags. May also clobber v0-v1, when icnt1==-1.
 496 void C2_MacroAssembler::string_indexof(Register str2, Register str1,
 497                                        Register cnt2, Register cnt1,
 498                                        Register tmp1, Register tmp2,
 499                                        Register tmp3, Register tmp4,
 500                                        Register tmp5, Register tmp6,
 501                                        int icnt1, Register result, int ae) {
 502   // NOTE: tmp5, tmp6 can be zr depending on specific method version
 503   Label LINEARSEARCH, LINEARSTUB, LINEAR_MEDIUM, DONE, NOMATCH, MATCH;
 504 
 505   Register ch1 = rscratch1;
 506   Register ch2 = rscratch2;
 507   Register cnt1tmp = tmp1;
 508   Register cnt2tmp = tmp2;
 509   Register cnt1_neg = cnt1;
 510   Register cnt2_neg = cnt2;
 511   Register result_tmp = tmp4;
 512 
 513   bool isL = ae == StrIntrinsicNode::LL;
 514 
 515   bool str1_isL = ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UL;
 516   bool str2_isL = ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::LU;
 517   int str1_chr_shift = str1_isL ? 0:1;
 518   int str2_chr_shift = str2_isL ? 0:1;
 519   int str1_chr_size = str1_isL ? 1:2;
 520   int str2_chr_size = str2_isL ? 1:2;
 521   chr_insn str1_load_1chr = str1_isL ? (chr_insn)&MacroAssembler::ldrb :
 522                                       (chr_insn)&MacroAssembler::ldrh;
 523   chr_insn str2_load_1chr = str2_isL ? (chr_insn)&MacroAssembler::ldrb :
 524                                       (chr_insn)&MacroAssembler::ldrh;
 525   chr_insn load_2chr = isL ? (chr_insn)&MacroAssembler::ldrh : (chr_insn)&MacroAssembler::ldrw;
 526   chr_insn load_4chr = isL ? (chr_insn)&MacroAssembler::ldrw : (chr_insn)&MacroAssembler::ldr;
 527 
 528   // Note, inline_string_indexOf() generates checks:
 529   // if (substr.count > string.count) return -1;
 530   // if (substr.count == 0) return 0;
 531 
 532   // We have two strings, a source string in str2, cnt2 and a pattern string
 533   // in str1, cnt1. Find the 1st occurrence of pattern in source or return -1.
 534 
 535   // For larger pattern and source we use a simplified Boyer Moore algorithm.
 536   // With a small pattern and source we use linear scan.
 537 
 538   if (icnt1 == -1) {
 539     sub(result_tmp, cnt2, cnt1);
 540     cmp(cnt1, (u1)8);             // Use Linear Scan if cnt1 < 8 || cnt1 >= 256
 541     br(LT, LINEARSEARCH);
 542     dup(v0, T16B, cnt1); // done in separate FPU pipeline. Almost no penalty
 543     subs(zr, cnt1, 256);
 544     lsr(tmp1, cnt2, 2);
 545     ccmp(cnt1, tmp1, 0b0000, LT); // Source must be 4 * pattern for BM
 546     br(GE, LINEARSTUB);
 547   }
 548 
 549 // The Boyer Moore alogorithm is based on the description here:-
 550 //
 551 // http://en.wikipedia.org/wiki/Boyer%E2%80%93Moore_string_search_algorithm
 552 //
 553 // This describes and algorithm with 2 shift rules. The 'Bad Character' rule
 554 // and the 'Good Suffix' rule.
 555 //
 556 // These rules are essentially heuristics for how far we can shift the
 557 // pattern along the search string.
 558 //
 559 // The implementation here uses the 'Bad Character' rule only because of the
 560 // complexity of initialisation for the 'Good Suffix' rule.
 561 //
 562 // This is also known as the Boyer-Moore-Horspool algorithm:-
 563 //
 564 // http://en.wikipedia.org/wiki/Boyer-Moore-Horspool_algorithm
 565 //
 566 // This particular implementation has few java-specific optimizations.
 567 //
 568 // #define ASIZE 256
 569 //
 570 //    int bm(unsigned char *x, int m, unsigned char *y, int n) {
 571 //       int i, j;
 572 //       unsigned c;
 573 //       unsigned char bc[ASIZE];
 574 //
 575 //       /* Preprocessing */
 576 //       for (i = 0; i < ASIZE; ++i)
 577 //          bc[i] = m;
 578 //       for (i = 0; i < m - 1; ) {
 579 //          c = x[i];
 580 //          ++i;
 581 //          // c < 256 for Latin1 string, so, no need for branch
 582 //          #ifdef PATTERN_STRING_IS_LATIN1
 583 //          bc[c] = m - i;
 584 //          #else
 585 //          if (c < ASIZE) bc[c] = m - i;
 586 //          #endif
 587 //       }
 588 //
 589 //       /* Searching */
 590 //       j = 0;
 591 //       while (j <= n - m) {
 592 //          c = y[i+j];
 593 //          if (x[m-1] == c)
 594 //            for (i = m - 2; i >= 0 && x[i] == y[i + j]; --i);
 595 //          if (i < 0) return j;
 596 //          // c < 256 for Latin1 string, so, no need for branch
 597 //          #ifdef SOURCE_STRING_IS_LATIN1
 598 //          // LL case: (c< 256) always true. Remove branch
 599 //          j += bc[y[j+m-1]];
 600 //          #endif
 601 //          #ifndef PATTERN_STRING_IS_UTF
 602 //          // UU case: need if (c<ASIZE) check. Skip 1 character if not.
 603 //          if (c < ASIZE)
 604 //            j += bc[y[j+m-1]];
 605 //          else
 606 //            j += 1
 607 //          #endif
 608 //          #ifdef PATTERN_IS_LATIN1_AND_SOURCE_IS_UTF
 609 //          // UL case: need if (c<ASIZE) check. Skip <pattern length> if not.
 610 //          if (c < ASIZE)
 611 //            j += bc[y[j+m-1]];
 612 //          else
 613 //            j += m
 614 //          #endif
 615 //       }
 616 //    }
 617 
 618   if (icnt1 == -1) {
 619     Label BCLOOP, BCSKIP, BMLOOPSTR2, BMLOOPSTR1, BMSKIP, BMADV, BMMATCH,
 620         BMLOOPSTR1_LASTCMP, BMLOOPSTR1_CMP, BMLOOPSTR1_AFTER_LOAD, BM_INIT_LOOP;
 621     Register cnt1end = tmp2;
 622     Register str2end = cnt2;
 623     Register skipch = tmp2;
 624 
 625     // str1 length is >=8, so, we can read at least 1 register for cases when
 626     // UTF->Latin1 conversion is not needed(8 LL or 4UU) and half register for
 627     // UL case. We'll re-read last character in inner pre-loop code to have
 628     // single outer pre-loop load
 629     const int firstStep = isL ? 7 : 3;
 630 
 631     const int ASIZE = 256;
 632     const int STORED_BYTES = 32; // amount of bytes stored per instruction
 633     sub(sp, sp, ASIZE);
 634     mov(tmp5, ASIZE/STORED_BYTES); // loop iterations
 635     mov(ch1, sp);
 636     BIND(BM_INIT_LOOP);
 637       stpq(v0, v0, Address(post(ch1, STORED_BYTES)));
 638       subs(tmp5, tmp5, 1);
 639       br(GT, BM_INIT_LOOP);
 640 
 641       sub(cnt1tmp, cnt1, 1);
 642       mov(tmp5, str2);
 643       add(str2end, str2, result_tmp, LSL, str2_chr_shift);
 644       sub(ch2, cnt1, 1);
 645       mov(tmp3, str1);
 646     BIND(BCLOOP);
 647       (this->*str1_load_1chr)(ch1, Address(post(tmp3, str1_chr_size)));
 648       if (!str1_isL) {
 649         subs(zr, ch1, ASIZE);
 650         br(HS, BCSKIP);
 651       }
 652       strb(ch2, Address(sp, ch1));
 653     BIND(BCSKIP);
 654       subs(ch2, ch2, 1);
 655       br(GT, BCLOOP);
 656 
 657       add(tmp6, str1, cnt1, LSL, str1_chr_shift); // address after str1
 658       if (str1_isL == str2_isL) {
 659         // load last 8 bytes (8LL/4UU symbols)
 660         ldr(tmp6, Address(tmp6, -wordSize));
 661       } else {
 662         ldrw(tmp6, Address(tmp6, -wordSize/2)); // load last 4 bytes(4 symbols)
 663         // convert Latin1 to UTF. We'll have to wait until load completed, but
 664         // it's still faster than per-character loads+checks
 665         lsr(tmp3, tmp6, BitsPerByte * (wordSize/2 - str1_chr_size)); // str1[N-1]
 666         ubfx(ch1, tmp6, 8, 8); // str1[N-2]
 667         ubfx(ch2, tmp6, 16, 8); // str1[N-3]
 668         andr(tmp6, tmp6, 0xFF); // str1[N-4]
 669         orr(ch2, ch1, ch2, LSL, 16);
 670         orr(tmp6, tmp6, tmp3, LSL, 48);
 671         orr(tmp6, tmp6, ch2, LSL, 16);
 672       }
 673     BIND(BMLOOPSTR2);
 674       (this->*str2_load_1chr)(skipch, Address(str2, cnt1tmp, Address::lsl(str2_chr_shift)));
 675       sub(cnt1tmp, cnt1tmp, firstStep); // cnt1tmp is positive here, because cnt1 >= 8
 676       if (str1_isL == str2_isL) {
 677         // re-init tmp3. It's for free because it's executed in parallel with
 678         // load above. Alternative is to initialize it before loop, but it'll
 679         // affect performance on in-order systems with 2 or more ld/st pipelines
 680         lsr(tmp3, tmp6, BitsPerByte * (wordSize - str1_chr_size));
 681       }
 682       if (!isL) { // UU/UL case
 683         lsl(ch2, cnt1tmp, 1); // offset in bytes
 684       }
 685       cmp(tmp3, skipch);
 686       br(NE, BMSKIP);
 687       ldr(ch2, Address(str2, isL ? cnt1tmp : ch2));
 688       mov(ch1, tmp6);
 689       if (isL) {
 690         b(BMLOOPSTR1_AFTER_LOAD);
 691       } else {
 692         sub(cnt1tmp, cnt1tmp, 1); // no need to branch for UU/UL case. cnt1 >= 8
 693         b(BMLOOPSTR1_CMP);
 694       }
 695     BIND(BMLOOPSTR1);
 696       (this->*str1_load_1chr)(ch1, Address(str1, cnt1tmp, Address::lsl(str1_chr_shift)));
 697       (this->*str2_load_1chr)(ch2, Address(str2, cnt1tmp, Address::lsl(str2_chr_shift)));
 698     BIND(BMLOOPSTR1_AFTER_LOAD);
 699       subs(cnt1tmp, cnt1tmp, 1);
 700       br(LT, BMLOOPSTR1_LASTCMP);
 701     BIND(BMLOOPSTR1_CMP);
 702       cmp(ch1, ch2);
 703       br(EQ, BMLOOPSTR1);
 704     BIND(BMSKIP);
 705       if (!isL) {
 706         // if we've met UTF symbol while searching Latin1 pattern, then we can
 707         // skip cnt1 symbols
 708         if (str1_isL != str2_isL) {
 709           mov(result_tmp, cnt1);
 710         } else {
 711           mov(result_tmp, 1);
 712         }
 713         subs(zr, skipch, ASIZE);
 714         br(HS, BMADV);
 715       }
 716       ldrb(result_tmp, Address(sp, skipch)); // load skip distance
 717     BIND(BMADV);
 718       sub(cnt1tmp, cnt1, 1);
 719       add(str2, str2, result_tmp, LSL, str2_chr_shift);
 720       cmp(str2, str2end);
 721       br(LE, BMLOOPSTR2);
 722       add(sp, sp, ASIZE);
 723       b(NOMATCH);
 724     BIND(BMLOOPSTR1_LASTCMP);
 725       cmp(ch1, ch2);
 726       br(NE, BMSKIP);
 727     BIND(BMMATCH);
 728       sub(result, str2, tmp5);
 729       if (!str2_isL) lsr(result, result, 1);
 730       add(sp, sp, ASIZE);
 731       b(DONE);
 732 
 733     BIND(LINEARSTUB);
 734     cmp(cnt1, (u1)16); // small patterns still should be handled by simple algorithm
 735     br(LT, LINEAR_MEDIUM);
 736     mov(result, zr);
 737     RuntimeAddress stub = nullptr;
 738     if (isL) {
 739       stub = RuntimeAddress(StubRoutines::aarch64::string_indexof_linear_ll());
 740       assert(stub.target() != nullptr, "string_indexof_linear_ll stub has not been generated");
 741     } else if (str1_isL) {
 742       stub = RuntimeAddress(StubRoutines::aarch64::string_indexof_linear_ul());
 743        assert(stub.target() != nullptr, "string_indexof_linear_ul stub has not been generated");
 744     } else {
 745       stub = RuntimeAddress(StubRoutines::aarch64::string_indexof_linear_uu());
 746       assert(stub.target() != nullptr, "string_indexof_linear_uu stub has not been generated");
 747     }
 748     address call = trampoline_call(stub);
 749     if (call == nullptr) {
 750       DEBUG_ONLY(reset_labels(LINEARSEARCH, LINEAR_MEDIUM, DONE, NOMATCH, MATCH));
 751       ciEnv::current()->record_failure("CodeCache is full");
 752       return;
 753     }
 754     b(DONE);
 755   }
 756 
 757   BIND(LINEARSEARCH);
 758   {
 759     Label DO1, DO2, DO3;
 760 
 761     Register str2tmp = tmp2;
 762     Register first = tmp3;
 763 
 764     if (icnt1 == -1)
 765     {
 766         Label DOSHORT, FIRST_LOOP, STR2_NEXT, STR1_LOOP, STR1_NEXT;
 767 
 768         cmp(cnt1, u1(str1_isL == str2_isL ? 4 : 2));
 769         br(LT, DOSHORT);
 770       BIND(LINEAR_MEDIUM);
 771         (this->*str1_load_1chr)(first, Address(str1));
 772         lea(str1, Address(str1, cnt1, Address::lsl(str1_chr_shift)));
 773         sub(cnt1_neg, zr, cnt1, LSL, str1_chr_shift);
 774         lea(str2, Address(str2, result_tmp, Address::lsl(str2_chr_shift)));
 775         sub(cnt2_neg, zr, result_tmp, LSL, str2_chr_shift);
 776 
 777       BIND(FIRST_LOOP);
 778         (this->*str2_load_1chr)(ch2, Address(str2, cnt2_neg));
 779         cmp(first, ch2);
 780         br(EQ, STR1_LOOP);
 781       BIND(STR2_NEXT);
 782         adds(cnt2_neg, cnt2_neg, str2_chr_size);
 783         br(LE, FIRST_LOOP);
 784         b(NOMATCH);
 785 
 786       BIND(STR1_LOOP);
 787         adds(cnt1tmp, cnt1_neg, str1_chr_size);
 788         add(cnt2tmp, cnt2_neg, str2_chr_size);
 789         br(GE, MATCH);
 790 
 791       BIND(STR1_NEXT);
 792         (this->*str1_load_1chr)(ch1, Address(str1, cnt1tmp));
 793         (this->*str2_load_1chr)(ch2, Address(str2, cnt2tmp));
 794         cmp(ch1, ch2);
 795         br(NE, STR2_NEXT);
 796         adds(cnt1tmp, cnt1tmp, str1_chr_size);
 797         add(cnt2tmp, cnt2tmp, str2_chr_size);
 798         br(LT, STR1_NEXT);
 799         b(MATCH);
 800 
 801       BIND(DOSHORT);
 802       if (str1_isL == str2_isL) {
 803         cmp(cnt1, (u1)2);
 804         br(LT, DO1);
 805         br(GT, DO3);
 806       }
 807     }
 808 
 809     if (icnt1 == 4) {
 810       Label CH1_LOOP;
 811 
 812         (this->*load_4chr)(ch1, str1);
 813         sub(result_tmp, cnt2, 4);
 814         lea(str2, Address(str2, result_tmp, Address::lsl(str2_chr_shift)));
 815         sub(cnt2_neg, zr, result_tmp, LSL, str2_chr_shift);
 816 
 817       BIND(CH1_LOOP);
 818         (this->*load_4chr)(ch2, Address(str2, cnt2_neg));
 819         cmp(ch1, ch2);
 820         br(EQ, MATCH);
 821         adds(cnt2_neg, cnt2_neg, str2_chr_size);
 822         br(LE, CH1_LOOP);
 823         b(NOMATCH);
 824       }
 825 
 826     if ((icnt1 == -1 && str1_isL == str2_isL) || icnt1 == 2) {
 827       Label CH1_LOOP;
 828 
 829       BIND(DO2);
 830         (this->*load_2chr)(ch1, str1);
 831         if (icnt1 == 2) {
 832           sub(result_tmp, cnt2, 2);
 833         }
 834         lea(str2, Address(str2, result_tmp, Address::lsl(str2_chr_shift)));
 835         sub(cnt2_neg, zr, result_tmp, LSL, str2_chr_shift);
 836       BIND(CH1_LOOP);
 837         (this->*load_2chr)(ch2, Address(str2, cnt2_neg));
 838         cmp(ch1, ch2);
 839         br(EQ, MATCH);
 840         adds(cnt2_neg, cnt2_neg, str2_chr_size);
 841         br(LE, CH1_LOOP);
 842         b(NOMATCH);
 843     }
 844 
 845     if ((icnt1 == -1 && str1_isL == str2_isL) || icnt1 == 3) {
 846       Label FIRST_LOOP, STR2_NEXT, STR1_LOOP;
 847 
 848       BIND(DO3);
 849         (this->*load_2chr)(first, str1);
 850         (this->*str1_load_1chr)(ch1, Address(str1, 2*str1_chr_size));
 851         if (icnt1 == 3) {
 852           sub(result_tmp, cnt2, 3);
 853         }
 854         lea(str2, Address(str2, result_tmp, Address::lsl(str2_chr_shift)));
 855         sub(cnt2_neg, zr, result_tmp, LSL, str2_chr_shift);
 856       BIND(FIRST_LOOP);
 857         (this->*load_2chr)(ch2, Address(str2, cnt2_neg));
 858         cmpw(first, ch2);
 859         br(EQ, STR1_LOOP);
 860       BIND(STR2_NEXT);
 861         adds(cnt2_neg, cnt2_neg, str2_chr_size);
 862         br(LE, FIRST_LOOP);
 863         b(NOMATCH);
 864 
 865       BIND(STR1_LOOP);
 866         add(cnt2tmp, cnt2_neg, 2*str2_chr_size);
 867         (this->*str2_load_1chr)(ch2, Address(str2, cnt2tmp));
 868         cmp(ch1, ch2);
 869         br(NE, STR2_NEXT);
 870         b(MATCH);
 871     }
 872 
 873     if (icnt1 == -1 || icnt1 == 1) {
 874       Label CH1_LOOP, HAS_ZERO, DO1_SHORT, DO1_LOOP;
 875 
 876       BIND(DO1);
 877         (this->*str1_load_1chr)(ch1, str1);
 878         cmp(cnt2, (u1)8);
 879         br(LT, DO1_SHORT);
 880 
 881         sub(result_tmp, cnt2, 8/str2_chr_size);
 882         sub(cnt2_neg, zr, result_tmp, LSL, str2_chr_shift);
 883         mov(tmp3, str2_isL ? 0x0101010101010101 : 0x0001000100010001);
 884         lea(str2, Address(str2, result_tmp, Address::lsl(str2_chr_shift)));
 885 
 886         if (str2_isL) {
 887           orr(ch1, ch1, ch1, LSL, 8);
 888         }
 889         orr(ch1, ch1, ch1, LSL, 16);
 890         orr(ch1, ch1, ch1, LSL, 32);
 891       BIND(CH1_LOOP);
 892         ldr(ch2, Address(str2, cnt2_neg));
 893         eor(ch2, ch1, ch2);
 894         sub(tmp1, ch2, tmp3);
 895         orr(tmp2, ch2, str2_isL ? 0x7f7f7f7f7f7f7f7f : 0x7fff7fff7fff7fff);
 896         bics(tmp1, tmp1, tmp2);
 897         br(NE, HAS_ZERO);
 898         adds(cnt2_neg, cnt2_neg, 8);
 899         br(LT, CH1_LOOP);
 900 
 901         cmp(cnt2_neg, (u1)8);
 902         mov(cnt2_neg, 0);
 903         br(LT, CH1_LOOP);
 904         b(NOMATCH);
 905 
 906       BIND(HAS_ZERO);
 907         rev(tmp1, tmp1);
 908         clz(tmp1, tmp1);
 909         add(cnt2_neg, cnt2_neg, tmp1, LSR, 3);
 910         b(MATCH);
 911 
 912       BIND(DO1_SHORT);
 913         mov(result_tmp, cnt2);
 914         lea(str2, Address(str2, cnt2, Address::lsl(str2_chr_shift)));
 915         sub(cnt2_neg, zr, cnt2, LSL, str2_chr_shift);
 916       BIND(DO1_LOOP);
 917         (this->*str2_load_1chr)(ch2, Address(str2, cnt2_neg));
 918         cmpw(ch1, ch2);
 919         br(EQ, MATCH);
 920         adds(cnt2_neg, cnt2_neg, str2_chr_size);
 921         br(LT, DO1_LOOP);
 922     }
 923   }
 924   BIND(NOMATCH);
 925     mov(result, -1);
 926     b(DONE);
 927   BIND(MATCH);
 928     add(result, result_tmp, cnt2_neg, ASR, str2_chr_shift);
 929   BIND(DONE);
 930 }
 931 
 932 typedef void (MacroAssembler::* chr_insn)(Register Rt, const Address &adr);
 933 typedef void (MacroAssembler::* uxt_insn)(Register Rd, Register Rn);
 934 
 935 void C2_MacroAssembler::string_indexof_char(Register str1, Register cnt1,
 936                                             Register ch, Register result,
 937                                             Register tmp1, Register tmp2, Register tmp3)
 938 {
 939   Label CH1_LOOP, HAS_ZERO, DO1_SHORT, DO1_LOOP, MATCH, NOMATCH, DONE;
 940   Register cnt1_neg = cnt1;
 941   Register ch1 = rscratch1;
 942   Register result_tmp = rscratch2;
 943 
 944   cbz(cnt1, NOMATCH);
 945 
 946   cmp(cnt1, (u1)4);
 947   br(LT, DO1_SHORT);
 948 
 949   orr(ch, ch, ch, LSL, 16);
 950   orr(ch, ch, ch, LSL, 32);
 951 
 952   sub(cnt1, cnt1, 4);
 953   mov(result_tmp, cnt1);
 954   lea(str1, Address(str1, cnt1, Address::uxtw(1)));
 955   sub(cnt1_neg, zr, cnt1, LSL, 1);
 956 
 957   mov(tmp3, 0x0001000100010001);
 958 
 959   BIND(CH1_LOOP);
 960     ldr(ch1, Address(str1, cnt1_neg));
 961     eor(ch1, ch, ch1);
 962     sub(tmp1, ch1, tmp3);
 963     orr(tmp2, ch1, 0x7fff7fff7fff7fff);
 964     bics(tmp1, tmp1, tmp2);
 965     br(NE, HAS_ZERO);
 966     adds(cnt1_neg, cnt1_neg, 8);
 967     br(LT, CH1_LOOP);
 968 
 969     cmp(cnt1_neg, (u1)8);
 970     mov(cnt1_neg, 0);
 971     br(LT, CH1_LOOP);
 972     b(NOMATCH);
 973 
 974   BIND(HAS_ZERO);
 975     rev(tmp1, tmp1);
 976     clz(tmp1, tmp1);
 977     add(cnt1_neg, cnt1_neg, tmp1, LSR, 3);
 978     b(MATCH);
 979 
 980   BIND(DO1_SHORT);
 981     mov(result_tmp, cnt1);
 982     lea(str1, Address(str1, cnt1, Address::uxtw(1)));
 983     sub(cnt1_neg, zr, cnt1, LSL, 1);
 984   BIND(DO1_LOOP);
 985     ldrh(ch1, Address(str1, cnt1_neg));
 986     cmpw(ch, ch1);
 987     br(EQ, MATCH);
 988     adds(cnt1_neg, cnt1_neg, 2);
 989     br(LT, DO1_LOOP);
 990   BIND(NOMATCH);
 991     mov(result, -1);
 992     b(DONE);
 993   BIND(MATCH);
 994     add(result, result_tmp, cnt1_neg, ASR, 1);
 995   BIND(DONE);
 996 }
 997 
 998 void C2_MacroAssembler::string_indexof_char_sve(Register str1, Register cnt1,
 999                                                 Register ch, Register result,
1000                                                 FloatRegister ztmp1,
1001                                                 FloatRegister ztmp2,
1002                                                 PRegister tmp_pg,
1003                                                 PRegister tmp_pdn, bool isL)
1004 {
1005   // Note that `tmp_pdn` should *NOT* be used as governing predicate register.
1006   assert(tmp_pg->is_governing(),
1007          "this register has to be a governing predicate register");
1008 
1009   Label LOOP, MATCH, DONE, NOMATCH;
1010   Register vec_len = rscratch1;
1011   Register idx = rscratch2;
1012 
1013   SIMD_RegVariant T = (isL == true) ? B : H;
1014 
1015   cbz(cnt1, NOMATCH);
1016 
1017   // Assign the particular char throughout the vector.
1018   sve_dup(ztmp2, T, ch);
1019   if (isL) {
1020     sve_cntb(vec_len);
1021   } else {
1022     sve_cnth(vec_len);
1023   }
1024   mov(idx, 0);
1025 
1026   // Generate a predicate to control the reading of input string.
1027   sve_whilelt(tmp_pg, T, idx, cnt1);
1028 
1029   BIND(LOOP);
1030     // Read a vector of 8- or 16-bit data depending on the string type. Note
1031     // that inactive elements indicated by the predicate register won't cause
1032     // a data read from memory to the destination vector.
1033     if (isL) {
1034       sve_ld1b(ztmp1, T, tmp_pg, Address(str1, idx));
1035     } else {
1036       sve_ld1h(ztmp1, T, tmp_pg, Address(str1, idx, Address::lsl(1)));
1037     }
1038     add(idx, idx, vec_len);
1039 
1040     // Perform the comparison. An element of the destination predicate is set
1041     // to active if the particular char is matched.
1042     sve_cmp(Assembler::EQ, tmp_pdn, T, tmp_pg, ztmp1, ztmp2);
1043 
1044     // Branch if the particular char is found.
1045     br(NE, MATCH);
1046 
1047     sve_whilelt(tmp_pg, T, idx, cnt1);
1048 
1049     // Loop back if the particular char not found.
1050     br(MI, LOOP);
1051 
1052   BIND(NOMATCH);
1053     mov(result, -1);
1054     b(DONE);
1055 
1056   BIND(MATCH);
1057     // Undo the index increment.
1058     sub(idx, idx, vec_len);
1059 
1060     // Crop the vector to find its location.
1061     sve_brka(tmp_pdn, tmp_pg, tmp_pdn, false /* isMerge */);
1062     add(result, idx, -1);
1063     sve_incp(result, T, tmp_pdn);
1064   BIND(DONE);
1065 }
1066 
1067 void C2_MacroAssembler::stringL_indexof_char(Register str1, Register cnt1,
1068                                             Register ch, Register result,
1069                                             Register tmp1, Register tmp2, Register tmp3)
1070 {
1071   Label CH1_LOOP, HAS_ZERO, DO1_SHORT, DO1_LOOP, MATCH, NOMATCH, DONE;
1072   Register cnt1_neg = cnt1;
1073   Register ch1 = rscratch1;
1074   Register result_tmp = rscratch2;
1075 
1076   cbz(cnt1, NOMATCH);
1077 
1078   cmp(cnt1, (u1)8);
1079   br(LT, DO1_SHORT);
1080 
1081   orr(ch, ch, ch, LSL, 8);
1082   orr(ch, ch, ch, LSL, 16);
1083   orr(ch, ch, ch, LSL, 32);
1084 
1085   sub(cnt1, cnt1, 8);
1086   mov(result_tmp, cnt1);
1087   lea(str1, Address(str1, cnt1));
1088   sub(cnt1_neg, zr, cnt1);
1089 
1090   mov(tmp3, 0x0101010101010101);
1091 
1092   BIND(CH1_LOOP);
1093     ldr(ch1, Address(str1, cnt1_neg));
1094     eor(ch1, ch, ch1);
1095     sub(tmp1, ch1, tmp3);
1096     orr(tmp2, ch1, 0x7f7f7f7f7f7f7f7f);
1097     bics(tmp1, tmp1, tmp2);
1098     br(NE, HAS_ZERO);
1099     adds(cnt1_neg, cnt1_neg, 8);
1100     br(LT, CH1_LOOP);
1101 
1102     cmp(cnt1_neg, (u1)8);
1103     mov(cnt1_neg, 0);
1104     br(LT, CH1_LOOP);
1105     b(NOMATCH);
1106 
1107   BIND(HAS_ZERO);
1108     rev(tmp1, tmp1);
1109     clz(tmp1, tmp1);
1110     add(cnt1_neg, cnt1_neg, tmp1, LSR, 3);
1111     b(MATCH);
1112 
1113   BIND(DO1_SHORT);
1114     mov(result_tmp, cnt1);
1115     lea(str1, Address(str1, cnt1));
1116     sub(cnt1_neg, zr, cnt1);
1117   BIND(DO1_LOOP);
1118     ldrb(ch1, Address(str1, cnt1_neg));
1119     cmp(ch, ch1);
1120     br(EQ, MATCH);
1121     adds(cnt1_neg, cnt1_neg, 1);
1122     br(LT, DO1_LOOP);
1123   BIND(NOMATCH);
1124     mov(result, -1);
1125     b(DONE);
1126   BIND(MATCH);
1127     add(result, result_tmp, cnt1_neg);
1128   BIND(DONE);
1129 }
1130 
1131 // Compare strings.
1132 void C2_MacroAssembler::string_compare(Register str1, Register str2,
1133     Register cnt1, Register cnt2, Register result, Register tmp1, Register tmp2,
1134     FloatRegister vtmp1, FloatRegister vtmp2, FloatRegister vtmp3,
1135     PRegister pgtmp1, PRegister pgtmp2, int ae) {
1136   Label DONE, SHORT_LOOP, SHORT_STRING, SHORT_LAST, TAIL, STUB,
1137       DIFF, NEXT_WORD, SHORT_LOOP_TAIL, SHORT_LAST2, SHORT_LAST_INIT,
1138       SHORT_LOOP_START, TAIL_CHECK;
1139 
1140   bool isLL = ae == StrIntrinsicNode::LL;
1141   bool isLU = ae == StrIntrinsicNode::LU;
1142   bool isUL = ae == StrIntrinsicNode::UL;
1143 
1144   // The stub threshold for LL strings is: 72 (64 + 8) chars
1145   // UU: 36 chars, or 72 bytes (valid for the 64-byte large loop with prefetch)
1146   // LU/UL: 24 chars, or 48 bytes (valid for the 16-character loop at least)
1147   const u1 stub_threshold = isLL ? 72 : ((isLU || isUL) ? 24 : 36);
1148 
1149   bool str1_isL = isLL || isLU;
1150   bool str2_isL = isLL || isUL;
1151 
1152   int str1_chr_shift = str1_isL ? 0 : 1;
1153   int str2_chr_shift = str2_isL ? 0 : 1;
1154   int str1_chr_size = str1_isL ? 1 : 2;
1155   int str2_chr_size = str2_isL ? 1 : 2;
1156   int minCharsInWord = isLL ? wordSize : wordSize/2;
1157 
1158   FloatRegister vtmpZ = vtmp1, vtmp = vtmp2;
1159   chr_insn str1_load_chr = str1_isL ? (chr_insn)&MacroAssembler::ldrb :
1160                                       (chr_insn)&MacroAssembler::ldrh;
1161   chr_insn str2_load_chr = str2_isL ? (chr_insn)&MacroAssembler::ldrb :
1162                                       (chr_insn)&MacroAssembler::ldrh;
1163   uxt_insn ext_chr = isLL ? (uxt_insn)&MacroAssembler::uxtbw :
1164                             (uxt_insn)&MacroAssembler::uxthw;
1165 
1166   BLOCK_COMMENT("string_compare {");
1167 
1168   // Bizzarely, the counts are passed in bytes, regardless of whether they
1169   // are L or U strings, however the result is always in characters.
1170   if (!str1_isL) asrw(cnt1, cnt1, 1);
1171   if (!str2_isL) asrw(cnt2, cnt2, 1);
1172 
1173   // Compute the minimum of the string lengths and save the difference.
1174   subsw(result, cnt1, cnt2);
1175   cselw(cnt2, cnt1, cnt2, Assembler::LE); // min
1176 
1177   // A very short string
1178   cmpw(cnt2, minCharsInWord);
1179   br(Assembler::LE, SHORT_STRING);
1180 
1181   // Compare longwords
1182   // load first parts of strings and finish initialization while loading
1183   {
1184     if (str1_isL == str2_isL) { // LL or UU
1185       ldr(tmp1, Address(str1));
1186       cmp(str1, str2);
1187       br(Assembler::EQ, DONE);
1188       ldr(tmp2, Address(str2));
1189       cmp(cnt2, stub_threshold);
1190       br(GE, STUB);
1191       subsw(cnt2, cnt2, minCharsInWord);
1192       br(EQ, TAIL_CHECK);
1193       lea(str2, Address(str2, cnt2, Address::uxtw(str2_chr_shift)));
1194       lea(str1, Address(str1, cnt2, Address::uxtw(str1_chr_shift)));
1195       sub(cnt2, zr, cnt2, LSL, str2_chr_shift);
1196     } else if (isLU) {
1197       ldrs(vtmp, Address(str1));
1198       ldr(tmp2, Address(str2));
1199       cmp(cnt2, stub_threshold);
1200       br(GE, STUB);
1201       subw(cnt2, cnt2, 4);
1202       eor(vtmpZ, T16B, vtmpZ, vtmpZ);
1203       lea(str1, Address(str1, cnt2, Address::uxtw(str1_chr_shift)));
1204       lea(str2, Address(str2, cnt2, Address::uxtw(str2_chr_shift)));
1205       zip1(vtmp, T8B, vtmp, vtmpZ);
1206       sub(cnt1, zr, cnt2, LSL, str1_chr_shift);
1207       sub(cnt2, zr, cnt2, LSL, str2_chr_shift);
1208       add(cnt1, cnt1, 4);
1209       fmovd(tmp1, vtmp);
1210     } else { // UL case
1211       ldr(tmp1, Address(str1));
1212       ldrs(vtmp, Address(str2));
1213       cmp(cnt2, stub_threshold);
1214       br(GE, STUB);
1215       subw(cnt2, cnt2, 4);
1216       lea(str1, Address(str1, cnt2, Address::uxtw(str1_chr_shift)));
1217       eor(vtmpZ, T16B, vtmpZ, vtmpZ);
1218       lea(str2, Address(str2, cnt2, Address::uxtw(str2_chr_shift)));
1219       sub(cnt1, zr, cnt2, LSL, str1_chr_shift);
1220       zip1(vtmp, T8B, vtmp, vtmpZ);
1221       sub(cnt2, zr, cnt2, LSL, str2_chr_shift);
1222       add(cnt1, cnt1, 8);
1223       fmovd(tmp2, vtmp);
1224     }
1225     adds(cnt2, cnt2, isUL ? 4 : 8);
1226     br(GE, TAIL);
1227     eor(rscratch2, tmp1, tmp2);
1228     cbnz(rscratch2, DIFF);
1229     // main loop
1230     bind(NEXT_WORD);
1231     if (str1_isL == str2_isL) {
1232       ldr(tmp1, Address(str1, cnt2));
1233       ldr(tmp2, Address(str2, cnt2));
1234       adds(cnt2, cnt2, 8);
1235     } else if (isLU) {
1236       ldrs(vtmp, Address(str1, cnt1));
1237       ldr(tmp2, Address(str2, cnt2));
1238       add(cnt1, cnt1, 4);
1239       zip1(vtmp, T8B, vtmp, vtmpZ);
1240       fmovd(tmp1, vtmp);
1241       adds(cnt2, cnt2, 8);
1242     } else { // UL
1243       ldrs(vtmp, Address(str2, cnt2));
1244       ldr(tmp1, Address(str1, cnt1));
1245       zip1(vtmp, T8B, vtmp, vtmpZ);
1246       add(cnt1, cnt1, 8);
1247       fmovd(tmp2, vtmp);
1248       adds(cnt2, cnt2, 4);
1249     }
1250     br(GE, TAIL);
1251 
1252     eor(rscratch2, tmp1, tmp2);
1253     cbz(rscratch2, NEXT_WORD);
1254     b(DIFF);
1255     bind(TAIL);
1256     eor(rscratch2, tmp1, tmp2);
1257     cbnz(rscratch2, DIFF);
1258     // Last longword.  In the case where length == 4 we compare the
1259     // same longword twice, but that's still faster than another
1260     // conditional branch.
1261     if (str1_isL == str2_isL) {
1262       ldr(tmp1, Address(str1));
1263       ldr(tmp2, Address(str2));
1264     } else if (isLU) {
1265       ldrs(vtmp, Address(str1));
1266       ldr(tmp2, Address(str2));
1267       zip1(vtmp, T8B, vtmp, vtmpZ);
1268       fmovd(tmp1, vtmp);
1269     } else { // UL
1270       ldrs(vtmp, Address(str2));
1271       ldr(tmp1, Address(str1));
1272       zip1(vtmp, T8B, vtmp, vtmpZ);
1273       fmovd(tmp2, vtmp);
1274     }
1275     bind(TAIL_CHECK);
1276     eor(rscratch2, tmp1, tmp2);
1277     cbz(rscratch2, DONE);
1278 
1279     // Find the first different characters in the longwords and
1280     // compute their difference.
1281     bind(DIFF);
1282     rev(rscratch2, rscratch2);
1283     clz(rscratch2, rscratch2);
1284     andr(rscratch2, rscratch2, isLL ? -8 : -16);
1285     lsrv(tmp1, tmp1, rscratch2);
1286     (this->*ext_chr)(tmp1, tmp1);
1287     lsrv(tmp2, tmp2, rscratch2);
1288     (this->*ext_chr)(tmp2, tmp2);
1289     subw(result, tmp1, tmp2);
1290     b(DONE);
1291   }
1292 
1293   bind(STUB);
1294     RuntimeAddress stub = nullptr;
1295     switch(ae) {
1296       case StrIntrinsicNode::LL:
1297         stub = RuntimeAddress(StubRoutines::aarch64::compare_long_string_LL());
1298         break;
1299       case StrIntrinsicNode::UU:
1300         stub = RuntimeAddress(StubRoutines::aarch64::compare_long_string_UU());
1301         break;
1302       case StrIntrinsicNode::LU:
1303         stub = RuntimeAddress(StubRoutines::aarch64::compare_long_string_LU());
1304         break;
1305       case StrIntrinsicNode::UL:
1306         stub = RuntimeAddress(StubRoutines::aarch64::compare_long_string_UL());
1307         break;
1308       default:
1309         ShouldNotReachHere();
1310      }
1311     assert(stub.target() != nullptr, "compare_long_string stub has not been generated");
1312     address call = trampoline_call(stub);
1313     if (call == nullptr) {
1314       DEBUG_ONLY(reset_labels(DONE, SHORT_LOOP, SHORT_STRING, SHORT_LAST, SHORT_LOOP_TAIL, SHORT_LAST2, SHORT_LAST_INIT, SHORT_LOOP_START));
1315       ciEnv::current()->record_failure("CodeCache is full");
1316       return;
1317     }
1318     b(DONE);
1319 
1320   bind(SHORT_STRING);
1321   // Is the minimum length zero?
1322   cbz(cnt2, DONE);
1323   // arrange code to do most branches while loading and loading next characters
1324   // while comparing previous
1325   (this->*str1_load_chr)(tmp1, Address(post(str1, str1_chr_size)));
1326   subs(cnt2, cnt2, 1);
1327   br(EQ, SHORT_LAST_INIT);
1328   (this->*str2_load_chr)(cnt1, Address(post(str2, str2_chr_size)));
1329   b(SHORT_LOOP_START);
1330   bind(SHORT_LOOP);
1331   subs(cnt2, cnt2, 1);
1332   br(EQ, SHORT_LAST);
1333   bind(SHORT_LOOP_START);
1334   (this->*str1_load_chr)(tmp2, Address(post(str1, str1_chr_size)));
1335   (this->*str2_load_chr)(rscratch1, Address(post(str2, str2_chr_size)));
1336   cmp(tmp1, cnt1);
1337   br(NE, SHORT_LOOP_TAIL);
1338   subs(cnt2, cnt2, 1);
1339   br(EQ, SHORT_LAST2);
1340   (this->*str1_load_chr)(tmp1, Address(post(str1, str1_chr_size)));
1341   (this->*str2_load_chr)(cnt1, Address(post(str2, str2_chr_size)));
1342   cmp(tmp2, rscratch1);
1343   br(EQ, SHORT_LOOP);
1344   sub(result, tmp2, rscratch1);
1345   b(DONE);
1346   bind(SHORT_LOOP_TAIL);
1347   sub(result, tmp1, cnt1);
1348   b(DONE);
1349   bind(SHORT_LAST2);
1350   cmp(tmp2, rscratch1);
1351   br(EQ, DONE);
1352   sub(result, tmp2, rscratch1);
1353 
1354   b(DONE);
1355   bind(SHORT_LAST_INIT);
1356   (this->*str2_load_chr)(cnt1, Address(post(str2, str2_chr_size)));
1357   bind(SHORT_LAST);
1358   cmp(tmp1, cnt1);
1359   br(EQ, DONE);
1360   sub(result, tmp1, cnt1);
1361 
1362   bind(DONE);
1363 
1364   BLOCK_COMMENT("} string_compare");
1365 }
1366 
1367 void C2_MacroAssembler::neon_compare(FloatRegister dst, BasicType bt, FloatRegister src1,
1368                                      FloatRegister src2, Condition cond, bool isQ) {
1369   SIMD_Arrangement size = esize2arrangement((unsigned)type2aelembytes(bt), isQ);
1370   FloatRegister zn = src1, zm = src2;
1371   bool needs_negation = false;
1372   switch (cond) {
1373     case LT: cond = GT; zn = src2; zm = src1; break;
1374     case LE: cond = GE; zn = src2; zm = src1; break;
1375     case LO: cond = HI; zn = src2; zm = src1; break;
1376     case LS: cond = HS; zn = src2; zm = src1; break;
1377     case NE: cond = EQ; needs_negation = true; break;
1378     default:
1379       break;
1380   }
1381 
1382   if (is_floating_point_type(bt)) {
1383     fcm(cond, dst, size, zn, zm);
1384   } else {
1385     cm(cond, dst, size, zn, zm);
1386   }
1387 
1388   if (needs_negation) {
1389     notr(dst, isQ ? T16B : T8B, dst);
1390   }
1391 }
1392 
1393 void C2_MacroAssembler::neon_compare_zero(FloatRegister dst, BasicType bt, FloatRegister src,
1394                                           Condition cond, bool isQ) {
1395   SIMD_Arrangement size = esize2arrangement((unsigned)type2aelembytes(bt), isQ);
1396   if (bt == T_FLOAT || bt == T_DOUBLE) {
1397     if (cond == Assembler::NE) {
1398       fcm(Assembler::EQ, dst, size, src);
1399       notr(dst, isQ ? T16B : T8B, dst);
1400     } else {
1401       fcm(cond, dst, size, src);
1402     }
1403   } else {
1404     if (cond == Assembler::NE) {
1405       cm(Assembler::EQ, dst, size, src);
1406       notr(dst, isQ ? T16B : T8B, dst);
1407     } else {
1408       cm(cond, dst, size, src);
1409     }
1410   }
1411 }
1412 
1413 // Compress the least significant bit of each byte to the rightmost and clear
1414 // the higher garbage bits.
1415 void C2_MacroAssembler::bytemask_compress(Register dst) {
1416   // Example input, dst = 0x01 00 00 00 01 01 00 01
1417   // The "??" bytes are garbage.
1418   orr(dst, dst, dst, Assembler::LSR, 7);  // dst = 0x?? 02 ?? 00 ?? 03 ?? 01
1419   orr(dst, dst, dst, Assembler::LSR, 14); // dst = 0x????????08 ??????0D
1420   orr(dst, dst, dst, Assembler::LSR, 28); // dst = 0x????????????????8D
1421   andr(dst, dst, 0xff);                   // dst = 0x8D
1422 }
1423 
1424 // Pack the lowest-numbered bit of each mask element in src into a long value
1425 // in dst, at most the first 64 lane elements.
1426 // Clobbers: rscratch1, if UseSVE=1 or the hardware doesn't support FEAT_BITPERM.
1427 void C2_MacroAssembler::sve_vmask_tolong(Register dst, PRegister src, BasicType bt, int lane_cnt,
1428                                          FloatRegister vtmp1, FloatRegister vtmp2) {
1429   assert(lane_cnt <= 64 && is_power_of_2(lane_cnt), "Unsupported lane count");
1430   assert_different_registers(dst, rscratch1);
1431   assert_different_registers(vtmp1, vtmp2);
1432 
1433   Assembler::SIMD_RegVariant size = elemType_to_regVariant(bt);
1434   // Example:   src = 0b01100101 10001101, bt = T_BYTE, lane_cnt = 16
1435   // Expected:  dst = 0x658D
1436 
1437   // Convert the mask into vector with sequential bytes.
1438   // vtmp1 = 0x00010100 0x00010001 0x01000000 0x01010001
1439   sve_cpy(vtmp1, size, src, 1, false);
1440   if (bt != T_BYTE) {
1441     sve_vector_narrow(vtmp1, B, vtmp1, size, vtmp2);
1442   }
1443 
1444   if (UseSVE > 1 && VM_Version::supports_svebitperm()) {
1445     // Given a vector with the value 0x00 or 0x01 in each byte, the basic idea
1446     // is to compress each significant bit of the byte in a cross-lane way. Due
1447     // to the lack of a cross-lane bit-compress instruction, we use BEXT
1448     // (bit-compress in each lane) with the biggest lane size (T = D) then
1449     // concatenate the results.
1450 
1451     // The second source input of BEXT, initialized with 0x01 in each byte.
1452     // vtmp2 = 0x01010101 0x01010101 0x01010101 0x01010101
1453     sve_dup(vtmp2, B, 1);
1454 
1455     // BEXT vtmp1.D, vtmp1.D, vtmp2.D
1456     // vtmp1 = 0x0001010000010001 | 0x0100000001010001
1457     // vtmp2 = 0x0101010101010101 | 0x0101010101010101
1458     //         ---------------------------------------
1459     // vtmp1 = 0x0000000000000065 | 0x000000000000008D
1460     sve_bext(vtmp1, D, vtmp1, vtmp2);
1461 
1462     // Concatenate the lowest significant 8 bits in each 8 bytes, and extract the
1463     // result to dst.
1464     // vtmp1 = 0x0000000000000000 | 0x000000000000658D
1465     // dst   = 0x658D
1466     if (lane_cnt <= 8) {
1467       // No need to concatenate.
1468       umov(dst, vtmp1, B, 0);
1469     } else if (lane_cnt <= 16) {
1470       ins(vtmp1, B, vtmp1, 1, 8);
1471       umov(dst, vtmp1, H, 0);
1472     } else {
1473       // As the lane count is 64 at most, the final expected value must be in
1474       // the lowest 64 bits after narrowing vtmp1 from D to B.
1475       sve_vector_narrow(vtmp1, B, vtmp1, D, vtmp2);
1476       umov(dst, vtmp1, D, 0);
1477     }
1478   } else if (UseSVE > 0) {
1479     // Compress the lowest 8 bytes.
1480     fmovd(dst, vtmp1);
1481     bytemask_compress(dst);
1482     if (lane_cnt <= 8) return;
1483 
1484     // Repeat on higher bytes and join the results.
1485     // Compress 8 bytes in each iteration.
1486     for (int idx = 1; idx < (lane_cnt / 8); idx++) {
1487       sve_extract_integral(rscratch1, T_LONG, vtmp1, idx, vtmp2);
1488       bytemask_compress(rscratch1);
1489       orr(dst, dst, rscratch1, Assembler::LSL, idx << 3);
1490     }
1491   } else {
1492     assert(false, "unsupported");
1493     ShouldNotReachHere();
1494   }
1495 }
1496 
1497 // Unpack the mask, a long value in src, into predicate register dst based on the
1498 // corresponding data type. Note that dst can support at most 64 lanes.
1499 // Below example gives the expected dst predicate register in different types, with
1500 // a valid src(0x658D) on a 1024-bit vector size machine.
1501 // BYTE:  dst = 0x00 00 00 00 00 00 00 00 00 00 00 00 00 00 65 8D
1502 // SHORT: dst = 0x00 00 00 00 00 00 00 00 00 00 00 00 14 11 40 51
1503 // INT:   dst = 0x00 00 00 00 00 00 00 00 01 10 01 01 10 00 11 01
1504 // LONG:  dst = 0x00 01 01 00 00 01 00 01 01 00 00 00 01 01 00 01
1505 //
1506 // The number of significant bits of src must be equal to lane_cnt. E.g., 0xFF658D which
1507 // has 24 significant bits would be an invalid input if dst predicate register refers to
1508 // a LONG type 1024-bit vector, which has at most 16 lanes.
1509 void C2_MacroAssembler::sve_vmask_fromlong(PRegister dst, Register src, BasicType bt, int lane_cnt,
1510                                            FloatRegister vtmp1, FloatRegister vtmp2) {
1511   assert(UseSVE == 2 && VM_Version::supports_svebitperm() &&
1512          lane_cnt <= 64 && is_power_of_2(lane_cnt), "unsupported");
1513   Assembler::SIMD_RegVariant size = elemType_to_regVariant(bt);
1514   // Example:   src = 0x658D, bt = T_BYTE, size = B, lane_cnt = 16
1515   // Expected:  dst = 0b01101001 10001101
1516 
1517   // Put long value from general purpose register into the first lane of vector.
1518   // vtmp1 = 0x0000000000000000 | 0x000000000000658D
1519   sve_dup(vtmp1, B, 0);
1520   mov(vtmp1, D, 0, src);
1521 
1522   // As sve_cmp generates mask value with the minimum unit in byte, we should
1523   // transform the value in the first lane which is mask in bit now to the
1524   // mask in byte, which can be done by SVE2's BDEP instruction.
1525 
1526   // The first source input of BDEP instruction. Deposite each byte in every 8 bytes.
1527   // vtmp1 = 0x0000000000000065 | 0x000000000000008D
1528   if (lane_cnt <= 8) {
1529     // Nothing. As only one byte exsits.
1530   } else if (lane_cnt <= 16) {
1531     ins(vtmp1, B, vtmp1, 8, 1);
1532     mov(vtmp1, B, 1, zr);
1533   } else {
1534     sve_vector_extend(vtmp1, D, vtmp1, B);
1535   }
1536 
1537   // The second source input of BDEP instruction, initialized with 0x01 for each byte.
1538   // vtmp2 = 0x01010101 0x01010101 0x01010101 0x01010101
1539   sve_dup(vtmp2, B, 1);
1540 
1541   // BDEP vtmp1.D, vtmp1.D, vtmp2.D
1542   // vtmp1 = 0x0000000000000065 | 0x000000000000008D
1543   // vtmp2 = 0x0101010101010101 | 0x0101010101010101
1544   //         ---------------------------------------
1545   // vtmp1 = 0x0001010000010001 | 0x0100000001010001
1546   sve_bdep(vtmp1, D, vtmp1, vtmp2);
1547 
1548   if (bt != T_BYTE) {
1549     sve_vector_extend(vtmp1, size, vtmp1, B);
1550   }
1551   // Generate mask according to the given vector, in which the elements have been
1552   // extended to expected type.
1553   // dst = 0b01101001 10001101
1554   sve_cmp(Assembler::NE, dst, size, ptrue, vtmp1, 0);
1555 }
1556 
1557 // Clobbers: rflags
1558 void C2_MacroAssembler::sve_compare(PRegister pd, BasicType bt, PRegister pg,
1559                                     FloatRegister zn, FloatRegister zm, Condition cond) {
1560   assert(pg->is_governing(), "This register has to be a governing predicate register");
1561   FloatRegister z1 = zn, z2 = zm;
1562   switch (cond) {
1563     case LE: z1 = zm; z2 = zn; cond = GE; break;
1564     case LT: z1 = zm; z2 = zn; cond = GT; break;
1565     case LO: z1 = zm; z2 = zn; cond = HI; break;
1566     case LS: z1 = zm; z2 = zn; cond = HS; break;
1567     default:
1568       break;
1569   }
1570 
1571   SIMD_RegVariant size = elemType_to_regVariant(bt);
1572   if (is_floating_point_type(bt)) {
1573     sve_fcm(cond, pd, size, pg, z1, z2);
1574   } else {
1575     assert(is_integral_type(bt), "unsupported element type");
1576     sve_cmp(cond, pd, size, pg, z1, z2);
1577   }
1578 }
1579 
1580 // Get index of the last mask lane that is set
1581 void C2_MacroAssembler::sve_vmask_lasttrue(Register dst, BasicType bt, PRegister src, PRegister ptmp) {
1582   SIMD_RegVariant size = elemType_to_regVariant(bt);
1583   sve_rev(ptmp, size, src);
1584   sve_brkb(ptmp, ptrue, ptmp, false);
1585   sve_cntp(dst, size, ptrue, ptmp);
1586   movw(rscratch1, MaxVectorSize / type2aelembytes(bt) - 1);
1587   subw(dst, rscratch1, dst);
1588 }
1589 
1590 // Extend integer vector src to dst with the same lane count
1591 // but larger element size, e.g. 4B -> 4I
1592 void C2_MacroAssembler::neon_vector_extend(FloatRegister dst, BasicType dst_bt, unsigned dst_vlen_in_bytes,
1593                                            FloatRegister src, BasicType src_bt, bool is_unsigned) {
1594   if (src_bt == T_BYTE) {
1595     if (dst_bt == T_SHORT) {
1596       // 4B/8B to 4S/8S
1597       _xshll(is_unsigned, dst, T8H, src, T8B, 0);
1598     } else {
1599       // 4B to 4I
1600       assert(dst_vlen_in_bytes == 16 && dst_bt == T_INT, "unsupported");
1601       _xshll(is_unsigned, dst, T8H, src, T8B, 0);
1602       _xshll(is_unsigned, dst, T4S, dst, T4H, 0);
1603     }
1604   } else if (src_bt == T_SHORT) {
1605     // 4S to 4I
1606     assert(dst_vlen_in_bytes == 16 && dst_bt == T_INT, "unsupported");
1607     _xshll(is_unsigned, dst, T4S, src, T4H, 0);
1608   } else if (src_bt == T_INT) {
1609     // 2I to 2L
1610     assert(dst_vlen_in_bytes == 16 && dst_bt == T_LONG, "unsupported");
1611     _xshll(is_unsigned, dst, T2D, src, T2S, 0);
1612   } else {
1613     ShouldNotReachHere();
1614   }
1615 }
1616 
1617 // Narrow integer vector src down to dst with the same lane count
1618 // but smaller element size, e.g. 4I -> 4B
1619 void C2_MacroAssembler::neon_vector_narrow(FloatRegister dst, BasicType dst_bt,
1620                                            FloatRegister src, BasicType src_bt, unsigned src_vlen_in_bytes) {
1621   if (src_bt == T_SHORT) {
1622     // 4S/8S to 4B/8B
1623     assert(src_vlen_in_bytes == 8 || src_vlen_in_bytes == 16, "unsupported");
1624     assert(dst_bt == T_BYTE, "unsupported");
1625     xtn(dst, T8B, src, T8H);
1626   } else if (src_bt == T_INT) {
1627     // 4I to 4B/4S
1628     assert(src_vlen_in_bytes == 16, "unsupported");
1629     assert(dst_bt == T_BYTE || dst_bt == T_SHORT, "unsupported");
1630     xtn(dst, T4H, src, T4S);
1631     if (dst_bt == T_BYTE) {
1632       xtn(dst, T8B, dst, T8H);
1633     }
1634   } else if (src_bt == T_LONG) {
1635     // 2L to 2I
1636     assert(src_vlen_in_bytes == 16, "unsupported");
1637     assert(dst_bt == T_INT, "unsupported");
1638     xtn(dst, T2S, src, T2D);
1639   } else {
1640     ShouldNotReachHere();
1641   }
1642 }
1643 
1644 void C2_MacroAssembler::sve_vector_extend(FloatRegister dst, SIMD_RegVariant dst_size,
1645                                           FloatRegister src, SIMD_RegVariant src_size,
1646                                           bool is_unsigned) {
1647   assert(dst_size > src_size && dst_size <= D && src_size <= S, "invalid element size");
1648 
1649   if (src_size == B) {
1650     switch (dst_size) {
1651     case H:
1652       _sve_xunpk(is_unsigned, /* is_high */ false, dst, H, src);
1653       break;
1654     case S:
1655       _sve_xunpk(is_unsigned, /* is_high */ false, dst, H, src);
1656       _sve_xunpk(is_unsigned, /* is_high */ false, dst, S, dst);
1657       break;
1658     case D:
1659       _sve_xunpk(is_unsigned, /* is_high */ false, dst, H, src);
1660       _sve_xunpk(is_unsigned, /* is_high */ false, dst, S, dst);
1661       _sve_xunpk(is_unsigned, /* is_high */ false, dst, D, dst);
1662       break;
1663     default:
1664       ShouldNotReachHere();
1665     }
1666   } else if (src_size == H) {
1667     if (dst_size == S) {
1668       _sve_xunpk(is_unsigned, /* is_high */ false, dst, S, src);
1669     } else { // D
1670       _sve_xunpk(is_unsigned, /* is_high */ false, dst, S, src);
1671       _sve_xunpk(is_unsigned, /* is_high */ false, dst, D, dst);
1672     }
1673   } else if (src_size == S) {
1674     _sve_xunpk(is_unsigned, /* is_high */ false, dst, D, src);
1675   }
1676 }
1677 
1678 // Vector narrow from src to dst with specified element sizes.
1679 // High part of dst vector will be filled with zero.
1680 void C2_MacroAssembler::sve_vector_narrow(FloatRegister dst, SIMD_RegVariant dst_size,
1681                                           FloatRegister src, SIMD_RegVariant src_size,
1682                                           FloatRegister tmp) {
1683   assert(dst_size < src_size && dst_size <= S && src_size <= D, "invalid element size");
1684   assert_different_registers(src, tmp);
1685   sve_dup(tmp, src_size, 0);
1686   if (src_size == D) {
1687     switch (dst_size) {
1688     case S:
1689       sve_uzp1(dst, S, src, tmp);
1690       break;
1691     case H:
1692       assert_different_registers(dst, tmp);
1693       sve_uzp1(dst, S, src, tmp);
1694       sve_uzp1(dst, H, dst, tmp);
1695       break;
1696     case B:
1697       assert_different_registers(dst, tmp);
1698       sve_uzp1(dst, S, src, tmp);
1699       sve_uzp1(dst, H, dst, tmp);
1700       sve_uzp1(dst, B, dst, tmp);
1701       break;
1702     default:
1703       ShouldNotReachHere();
1704     }
1705   } else if (src_size == S) {
1706     if (dst_size == H) {
1707       sve_uzp1(dst, H, src, tmp);
1708     } else { // B
1709       assert_different_registers(dst, tmp);
1710       sve_uzp1(dst, H, src, tmp);
1711       sve_uzp1(dst, B, dst, tmp);
1712     }
1713   } else if (src_size == H) {
1714     sve_uzp1(dst, B, src, tmp);
1715   }
1716 }
1717 
1718 // Extend src predicate to dst predicate with the same lane count but larger
1719 // element size, e.g. 64Byte -> 512Long
1720 void C2_MacroAssembler::sve_vmaskcast_extend(PRegister dst, PRegister src,
1721                                              uint dst_element_length_in_bytes,
1722                                              uint src_element_length_in_bytes) {
1723   if (dst_element_length_in_bytes == 2 * src_element_length_in_bytes) {
1724     sve_punpklo(dst, src);
1725   } else if (dst_element_length_in_bytes == 4 * src_element_length_in_bytes) {
1726     sve_punpklo(dst, src);
1727     sve_punpklo(dst, dst);
1728   } else if (dst_element_length_in_bytes == 8 * src_element_length_in_bytes) {
1729     sve_punpklo(dst, src);
1730     sve_punpklo(dst, dst);
1731     sve_punpklo(dst, dst);
1732   } else {
1733     assert(false, "unsupported");
1734     ShouldNotReachHere();
1735   }
1736 }
1737 
1738 // Narrow src predicate to dst predicate with the same lane count but
1739 // smaller element size, e.g. 512Long -> 64Byte
1740 void C2_MacroAssembler::sve_vmaskcast_narrow(PRegister dst, PRegister src, PRegister ptmp,
1741                                              uint dst_element_length_in_bytes, uint src_element_length_in_bytes) {
1742   // The insignificant bits in src predicate are expected to be zero.
1743   // To ensure the higher order bits of the resultant narrowed vector are 0, an all-zero predicate is
1744   // passed as the second argument. An example narrowing operation with a given mask would be -
1745   // 128Long -> 64Int on a 128-bit machine i.e 2L -> 2I
1746   // Mask (for 2 Longs) : TF
1747   // Predicate register for the above mask (16 bits) : 00000001 00000000
1748   // After narrowing (uzp1 dst.b, src.b, ptmp.b) : 0000 0000 0001 0000
1749   // Which translates to mask for 2 integers as : TF (lower half is considered while upper half is 0)
1750   assert_different_registers(src, ptmp);
1751   assert_different_registers(dst, ptmp);
1752   sve_pfalse(ptmp);
1753   if (dst_element_length_in_bytes * 2 == src_element_length_in_bytes) {
1754     sve_uzp1(dst, B, src, ptmp);
1755   } else if (dst_element_length_in_bytes * 4 == src_element_length_in_bytes) {
1756     sve_uzp1(dst, H, src, ptmp);
1757     sve_uzp1(dst, B, dst, ptmp);
1758   } else if (dst_element_length_in_bytes * 8 == src_element_length_in_bytes) {
1759     sve_uzp1(dst, S, src, ptmp);
1760     sve_uzp1(dst, H, dst, ptmp);
1761     sve_uzp1(dst, B, dst, ptmp);
1762   } else {
1763     assert(false, "unsupported");
1764     ShouldNotReachHere();
1765   }
1766 }
1767 
1768 // Vector reduction add for integral type with ASIMD instructions.
1769 void C2_MacroAssembler::neon_reduce_add_integral(Register dst, BasicType bt,
1770                                                  Register isrc, FloatRegister vsrc,
1771                                                  unsigned vector_length_in_bytes,
1772                                                  FloatRegister vtmp) {
1773   assert(vector_length_in_bytes == 8 || vector_length_in_bytes == 16, "unsupported");
1774   assert_different_registers(dst, isrc);
1775   bool isQ = vector_length_in_bytes == 16;
1776 
1777   BLOCK_COMMENT("neon_reduce_add_integral {");
1778     switch(bt) {
1779       case T_BYTE:
1780         addv(vtmp, isQ ? T16B : T8B, vsrc);
1781         smov(dst, vtmp, B, 0);
1782         addw(dst, dst, isrc, ext::sxtb);
1783         break;
1784       case T_SHORT:
1785         addv(vtmp, isQ ? T8H : T4H, vsrc);
1786         smov(dst, vtmp, H, 0);
1787         addw(dst, dst, isrc, ext::sxth);
1788         break;
1789       case T_INT:
1790         isQ ? addv(vtmp, T4S, vsrc) : addpv(vtmp, T2S, vsrc, vsrc);
1791         umov(dst, vtmp, S, 0);
1792         addw(dst, dst, isrc);
1793         break;
1794       case T_LONG:
1795         assert(isQ, "unsupported");
1796         addpd(vtmp, vsrc);
1797         umov(dst, vtmp, D, 0);
1798         add(dst, dst, isrc);
1799         break;
1800       default:
1801         assert(false, "unsupported");
1802         ShouldNotReachHere();
1803     }
1804   BLOCK_COMMENT("} neon_reduce_add_integral");
1805 }
1806 
1807 // Vector reduction multiply for integral type with ASIMD instructions.
1808 // Note: temporary registers vtmp1 and vtmp2 are not used in some cases.
1809 // Clobbers: rscratch1
1810 void C2_MacroAssembler::neon_reduce_mul_integral(Register dst, BasicType bt,
1811                                                  Register isrc, FloatRegister vsrc,
1812                                                  unsigned vector_length_in_bytes,
1813                                                  FloatRegister vtmp1, FloatRegister vtmp2) {
1814   assert(vector_length_in_bytes == 8 || vector_length_in_bytes == 16, "unsupported");
1815   bool isQ = vector_length_in_bytes == 16;
1816 
1817   BLOCK_COMMENT("neon_reduce_mul_integral {");
1818     switch(bt) {
1819       case T_BYTE:
1820         if (isQ) {
1821           // Multiply the lower half and higher half of vector iteratively.
1822           // vtmp1 = vsrc[8:15]
1823           ins(vtmp1, D, vsrc, 0, 1);
1824           // vtmp1[n] = vsrc[n] * vsrc[n + 8], where n=[0, 7]
1825           mulv(vtmp1, T8B, vtmp1, vsrc);
1826           // vtmp2 = vtmp1[4:7]
1827           ins(vtmp2, S, vtmp1, 0, 1);
1828           // vtmp1[n] = vtmp1[n] * vtmp1[n + 4], where n=[0, 3]
1829           mulv(vtmp1, T8B, vtmp2, vtmp1);
1830         } else {
1831           ins(vtmp1, S, vsrc, 0, 1);
1832           mulv(vtmp1, T8B, vtmp1, vsrc);
1833         }
1834         // vtmp2 = vtmp1[2:3]
1835         ins(vtmp2, H, vtmp1, 0, 1);
1836         // vtmp2[n] = vtmp1[n] * vtmp1[n + 2], where n=[0, 1]
1837         mulv(vtmp2, T8B, vtmp2, vtmp1);
1838         // dst = vtmp2[0] * isrc * vtmp2[1]
1839         umov(rscratch1, vtmp2, B, 0);
1840         mulw(dst, rscratch1, isrc);
1841         sxtb(dst, dst);
1842         umov(rscratch1, vtmp2, B, 1);
1843         mulw(dst, rscratch1, dst);
1844         sxtb(dst, dst);
1845         break;
1846       case T_SHORT:
1847         if (isQ) {
1848           ins(vtmp2, D, vsrc, 0, 1);
1849           mulv(vtmp2, T4H, vtmp2, vsrc);
1850           ins(vtmp1, S, vtmp2, 0, 1);
1851           mulv(vtmp1, T4H, vtmp1, vtmp2);
1852         } else {
1853           ins(vtmp1, S, vsrc, 0, 1);
1854           mulv(vtmp1, T4H, vtmp1, vsrc);
1855         }
1856         umov(rscratch1, vtmp1, H, 0);
1857         mulw(dst, rscratch1, isrc);
1858         sxth(dst, dst);
1859         umov(rscratch1, vtmp1, H, 1);
1860         mulw(dst, rscratch1, dst);
1861         sxth(dst, dst);
1862         break;
1863       case T_INT:
1864         if (isQ) {
1865           ins(vtmp1, D, vsrc, 0, 1);
1866           mulv(vtmp1, T2S, vtmp1, vsrc);
1867         } else {
1868           vtmp1 = vsrc;
1869         }
1870         umov(rscratch1, vtmp1, S, 0);
1871         mul(dst, rscratch1, isrc);
1872         umov(rscratch1, vtmp1, S, 1);
1873         mul(dst, rscratch1, dst);
1874         break;
1875       case T_LONG:
1876         umov(rscratch1, vsrc, D, 0);
1877         mul(dst, isrc, rscratch1);
1878         umov(rscratch1, vsrc, D, 1);
1879         mul(dst, dst, rscratch1);
1880         break;
1881       default:
1882         assert(false, "unsupported");
1883         ShouldNotReachHere();
1884     }
1885   BLOCK_COMMENT("} neon_reduce_mul_integral");
1886 }
1887 
1888 // Vector reduction multiply for floating-point type with ASIMD instructions.
1889 void C2_MacroAssembler::neon_reduce_mul_fp(FloatRegister dst, BasicType bt,
1890                                            FloatRegister fsrc, FloatRegister vsrc,
1891                                            unsigned vector_length_in_bytes,
1892                                            FloatRegister vtmp) {
1893   assert(vector_length_in_bytes == 8 || vector_length_in_bytes == 16, "unsupported");
1894   bool isQ = vector_length_in_bytes == 16;
1895 
1896   BLOCK_COMMENT("neon_reduce_mul_fp {");
1897     switch(bt) {
1898       case T_FLOAT:
1899         fmuls(dst, fsrc, vsrc);
1900         ins(vtmp, S, vsrc, 0, 1);
1901         fmuls(dst, dst, vtmp);
1902         if (isQ) {
1903           ins(vtmp, S, vsrc, 0, 2);
1904           fmuls(dst, dst, vtmp);
1905           ins(vtmp, S, vsrc, 0, 3);
1906           fmuls(dst, dst, vtmp);
1907          }
1908         break;
1909       case T_DOUBLE:
1910         assert(isQ, "unsupported");
1911         fmuld(dst, fsrc, vsrc);
1912         ins(vtmp, D, vsrc, 0, 1);
1913         fmuld(dst, dst, vtmp);
1914         break;
1915       default:
1916         assert(false, "unsupported");
1917         ShouldNotReachHere();
1918     }
1919   BLOCK_COMMENT("} neon_reduce_mul_fp");
1920 }
1921 
1922 // Helper to select logical instruction
1923 void C2_MacroAssembler::neon_reduce_logical_helper(int opc, bool is64, Register Rd,
1924                                                    Register Rn, Register Rm,
1925                                                    enum shift_kind kind, unsigned shift) {
1926   switch(opc) {
1927     case Op_AndReductionV:
1928       is64 ? andr(Rd, Rn, Rm, kind, shift) : andw(Rd, Rn, Rm, kind, shift);
1929       break;
1930     case Op_OrReductionV:
1931       is64 ? orr(Rd, Rn, Rm, kind, shift) : orrw(Rd, Rn, Rm, kind, shift);
1932       break;
1933     case Op_XorReductionV:
1934       is64 ? eor(Rd, Rn, Rm, kind, shift) : eorw(Rd, Rn, Rm, kind, shift);
1935       break;
1936     default:
1937       assert(false, "unsupported");
1938       ShouldNotReachHere();
1939   }
1940 }
1941 
1942 // Vector reduction logical operations And, Or, Xor
1943 // Clobbers: rscratch1
1944 void C2_MacroAssembler::neon_reduce_logical(int opc, Register dst, BasicType bt,
1945                                             Register isrc, FloatRegister vsrc,
1946                                             unsigned vector_length_in_bytes) {
1947   assert(opc == Op_AndReductionV || opc == Op_OrReductionV || opc == Op_XorReductionV,
1948          "unsupported");
1949   assert(vector_length_in_bytes == 8 || vector_length_in_bytes == 16, "unsupported");
1950   assert_different_registers(dst, isrc);
1951   bool isQ = vector_length_in_bytes == 16;
1952 
1953   BLOCK_COMMENT("neon_reduce_logical {");
1954     umov(rscratch1, vsrc, isQ ? D : S, 0);
1955     umov(dst, vsrc, isQ ? D : S, 1);
1956     neon_reduce_logical_helper(opc, /* is64 */ true, dst, dst, rscratch1);
1957     switch(bt) {
1958       case T_BYTE:
1959         if (isQ) {
1960           neon_reduce_logical_helper(opc, /* is64 */ true, dst, dst, dst, Assembler::LSR, 32);
1961         }
1962         neon_reduce_logical_helper(opc, /* is64 */ false, dst, dst, dst, Assembler::LSR, 16);
1963         neon_reduce_logical_helper(opc, /* is64 */ false, dst, dst, dst, Assembler::LSR, 8);
1964         neon_reduce_logical_helper(opc, /* is64 */ false, dst, isrc, dst);
1965         sxtb(dst, dst);
1966         break;
1967       case T_SHORT:
1968         if (isQ) {
1969           neon_reduce_logical_helper(opc, /* is64 */ true, dst, dst, dst, Assembler::LSR, 32);
1970         }
1971         neon_reduce_logical_helper(opc, /* is64 */ false, dst, dst, dst, Assembler::LSR, 16);
1972         neon_reduce_logical_helper(opc, /* is64 */ false, dst, isrc, dst);
1973         sxth(dst, dst);
1974         break;
1975       case T_INT:
1976         if (isQ) {
1977           neon_reduce_logical_helper(opc, /* is64 */ true, dst, dst, dst, Assembler::LSR, 32);
1978         }
1979         neon_reduce_logical_helper(opc, /* is64 */ false, dst, isrc, dst);
1980         break;
1981       case T_LONG:
1982         assert(isQ, "unsupported");
1983         neon_reduce_logical_helper(opc, /* is64 */ true, dst, isrc, dst);
1984         break;
1985       default:
1986         assert(false, "unsupported");
1987         ShouldNotReachHere();
1988     }
1989   BLOCK_COMMENT("} neon_reduce_logical");
1990 }
1991 
1992 // Vector reduction min/max for integral type with ASIMD instructions.
1993 // Note: vtmp is not used and expected to be fnoreg for T_LONG case.
1994 // Clobbers: rscratch1, rflags
1995 void C2_MacroAssembler::neon_reduce_minmax_integral(int opc, Register dst, BasicType bt,
1996                                                     Register isrc, FloatRegister vsrc,
1997                                                     unsigned vector_length_in_bytes,
1998                                                     FloatRegister vtmp) {
1999   assert(opc == Op_MinReductionV || opc == Op_MaxReductionV, "unsupported");
2000   assert(vector_length_in_bytes == 8 || vector_length_in_bytes == 16, "unsupported");
2001   assert(bt == T_BYTE || bt == T_SHORT || bt == T_INT || bt == T_LONG, "unsupported");
2002   assert_different_registers(dst, isrc);
2003   bool isQ = vector_length_in_bytes == 16;
2004   bool is_min = opc == Op_MinReductionV;
2005 
2006   BLOCK_COMMENT("neon_reduce_minmax_integral {");
2007     if (bt == T_LONG) {
2008       assert(vtmp == fnoreg, "should be");
2009       assert(isQ, "should be");
2010       umov(rscratch1, vsrc, D, 0);
2011       cmp(isrc, rscratch1);
2012       csel(dst, isrc, rscratch1, is_min ? LT : GT);
2013       umov(rscratch1, vsrc, D, 1);
2014       cmp(dst, rscratch1);
2015       csel(dst, dst, rscratch1, is_min ? LT : GT);
2016     } else {
2017       SIMD_Arrangement size = esize2arrangement((unsigned)type2aelembytes(bt), isQ);
2018       if (size == T2S) {
2019         is_min ? sminp(vtmp, size, vsrc, vsrc) : smaxp(vtmp, size, vsrc, vsrc);
2020       } else {
2021         is_min ? sminv(vtmp, size, vsrc) : smaxv(vtmp, size, vsrc);
2022       }
2023       if (bt == T_INT) {
2024         umov(dst, vtmp, S, 0);
2025       } else {
2026         smov(dst, vtmp, elemType_to_regVariant(bt), 0);
2027       }
2028       cmpw(dst, isrc);
2029       cselw(dst, dst, isrc, is_min ? LT : GT);
2030     }
2031   BLOCK_COMMENT("} neon_reduce_minmax_integral");
2032 }
2033 
2034 // Vector reduction for integral type with SVE instruction.
2035 // Supported operations are Add, And, Or, Xor, Max, Min.
2036 // rflags would be clobbered if opc is Op_MaxReductionV or Op_MinReductionV.
2037 void C2_MacroAssembler::sve_reduce_integral(int opc, Register dst, BasicType bt, Register src1,
2038                                             FloatRegister src2, PRegister pg, FloatRegister tmp) {
2039   assert(bt == T_BYTE || bt == T_SHORT || bt == T_INT || bt == T_LONG, "unsupported element type");
2040   assert(pg->is_governing(), "This register has to be a governing predicate register");
2041   assert_different_registers(src1, dst);
2042   // Register "dst" and "tmp" are to be clobbered, and "src1" and "src2" should be preserved.
2043   Assembler::SIMD_RegVariant size = elemType_to_regVariant(bt);
2044   switch (opc) {
2045     case Op_AddReductionVI: {
2046       sve_uaddv(tmp, size, pg, src2);
2047       if (bt == T_BYTE) {
2048         smov(dst, tmp, size, 0);
2049         addw(dst, src1, dst, ext::sxtb);
2050       } else if (bt == T_SHORT) {
2051         smov(dst, tmp, size, 0);
2052         addw(dst, src1, dst, ext::sxth);
2053       } else {
2054         umov(dst, tmp, size, 0);
2055         addw(dst, dst, src1);
2056       }
2057       break;
2058     }
2059     case Op_AddReductionVL: {
2060       sve_uaddv(tmp, size, pg, src2);
2061       umov(dst, tmp, size, 0);
2062       add(dst, dst, src1);
2063       break;
2064     }
2065     case Op_AndReductionV: {
2066       sve_andv(tmp, size, pg, src2);
2067       if (bt == T_INT || bt == T_LONG) {
2068         umov(dst, tmp, size, 0);
2069       } else {
2070         smov(dst, tmp, size, 0);
2071       }
2072       if (bt == T_LONG) {
2073         andr(dst, dst, src1);
2074       } else {
2075         andw(dst, dst, src1);
2076       }
2077       break;
2078     }
2079     case Op_OrReductionV: {
2080       sve_orv(tmp, size, pg, src2);
2081       if (bt == T_INT || bt == T_LONG) {
2082         umov(dst, tmp, size, 0);
2083       } else {
2084         smov(dst, tmp, size, 0);
2085       }
2086       if (bt == T_LONG) {
2087         orr(dst, dst, src1);
2088       } else {
2089         orrw(dst, dst, src1);
2090       }
2091       break;
2092     }
2093     case Op_XorReductionV: {
2094       sve_eorv(tmp, size, pg, src2);
2095       if (bt == T_INT || bt == T_LONG) {
2096         umov(dst, tmp, size, 0);
2097       } else {
2098         smov(dst, tmp, size, 0);
2099       }
2100       if (bt == T_LONG) {
2101         eor(dst, dst, src1);
2102       } else {
2103         eorw(dst, dst, src1);
2104       }
2105       break;
2106     }
2107     case Op_MaxReductionV: {
2108       sve_smaxv(tmp, size, pg, src2);
2109       if (bt == T_INT || bt == T_LONG) {
2110         umov(dst, tmp, size, 0);
2111       } else {
2112         smov(dst, tmp, size, 0);
2113       }
2114       if (bt == T_LONG) {
2115         cmp(dst, src1);
2116         csel(dst, dst, src1, Assembler::GT);
2117       } else {
2118         cmpw(dst, src1);
2119         cselw(dst, dst, src1, Assembler::GT);
2120       }
2121       break;
2122     }
2123     case Op_MinReductionV: {
2124       sve_sminv(tmp, size, pg, src2);
2125       if (bt == T_INT || bt == T_LONG) {
2126         umov(dst, tmp, size, 0);
2127       } else {
2128         smov(dst, tmp, size, 0);
2129       }
2130       if (bt == T_LONG) {
2131         cmp(dst, src1);
2132         csel(dst, dst, src1, Assembler::LT);
2133       } else {
2134         cmpw(dst, src1);
2135         cselw(dst, dst, src1, Assembler::LT);
2136       }
2137       break;
2138     }
2139     default:
2140       assert(false, "unsupported");
2141       ShouldNotReachHere();
2142   }
2143 
2144   if (opc == Op_AndReductionV || opc == Op_OrReductionV || opc == Op_XorReductionV) {
2145     if (bt == T_BYTE) {
2146       sxtb(dst, dst);
2147     } else if (bt == T_SHORT) {
2148       sxth(dst, dst);
2149     }
2150   }
2151 }
2152 
2153 // Set elements of the dst predicate to true for lanes in the range of [0, lane_cnt), or
2154 // to false otherwise. The input "lane_cnt" should be smaller than or equal to the supported
2155 // max vector length of the basic type. Clobbers: rscratch1 and the rFlagsReg.
2156 void C2_MacroAssembler::sve_gen_mask_imm(PRegister dst, BasicType bt, uint32_t lane_cnt) {
2157   uint32_t max_vector_length = Matcher::max_vector_size(bt);
2158   assert(lane_cnt <= max_vector_length, "unsupported input lane_cnt");
2159 
2160   // Set all elements to false if the input "lane_cnt" is zero.
2161   if (lane_cnt == 0) {
2162     sve_pfalse(dst);
2163     return;
2164   }
2165 
2166   SIMD_RegVariant size = elemType_to_regVariant(bt);
2167   assert(size != Q, "invalid size");
2168 
2169   // Set all true if "lane_cnt" equals to the max lane count.
2170   if (lane_cnt == max_vector_length) {
2171     sve_ptrue(dst, size, /* ALL */ 0b11111);
2172     return;
2173   }
2174 
2175   // Fixed numbers for "ptrue".
2176   switch(lane_cnt) {
2177   case 1: /* VL1 */
2178   case 2: /* VL2 */
2179   case 3: /* VL3 */
2180   case 4: /* VL4 */
2181   case 5: /* VL5 */
2182   case 6: /* VL6 */
2183   case 7: /* VL7 */
2184   case 8: /* VL8 */
2185     sve_ptrue(dst, size, lane_cnt);
2186     return;
2187   case 16:
2188     sve_ptrue(dst, size, /* VL16 */ 0b01001);
2189     return;
2190   case 32:
2191     sve_ptrue(dst, size, /* VL32 */ 0b01010);
2192     return;
2193   case 64:
2194     sve_ptrue(dst, size, /* VL64 */ 0b01011);
2195     return;
2196   case 128:
2197     sve_ptrue(dst, size, /* VL128 */ 0b01100);
2198     return;
2199   case 256:
2200     sve_ptrue(dst, size, /* VL256 */ 0b01101);
2201     return;
2202   default:
2203     break;
2204   }
2205 
2206   // Special patterns for "ptrue".
2207   if (lane_cnt == round_down_power_of_2(max_vector_length)) {
2208     sve_ptrue(dst, size, /* POW2 */ 0b00000);
2209   } else if (lane_cnt == max_vector_length - (max_vector_length % 4)) {
2210     sve_ptrue(dst, size, /* MUL4 */ 0b11101);
2211   } else if (lane_cnt == max_vector_length - (max_vector_length % 3)) {
2212     sve_ptrue(dst, size, /* MUL3 */ 0b11110);
2213   } else {
2214     // Encode to "whileltw" for the remaining cases.
2215     mov(rscratch1, lane_cnt);
2216     sve_whileltw(dst, size, zr, rscratch1);
2217   }
2218 }
2219 
2220 // Pack active elements of src, under the control of mask, into the lowest-numbered elements of dst.
2221 // Any remaining elements of dst will be filled with zero.
2222 // Clobbers: rscratch1
2223 // Preserves: src, mask
2224 void C2_MacroAssembler::sve_compress_short(FloatRegister dst, FloatRegister src, PRegister mask,
2225                                            FloatRegister vtmp1, FloatRegister vtmp2,
2226                                            PRegister pgtmp) {
2227   assert(pgtmp->is_governing(), "This register has to be a governing predicate register");
2228   assert_different_registers(dst, src, vtmp1, vtmp2);
2229   assert_different_registers(mask, pgtmp);
2230 
2231   // Example input:   src   = 8888 7777 6666 5555 4444 3333 2222 1111
2232   //                  mask  = 0001 0000 0000 0001 0001 0000 0001 0001
2233   // Expected result: dst   = 0000 0000 0000 8888 5555 4444 2222 1111
2234   sve_dup(vtmp2, H, 0);
2235 
2236   // Extend lowest half to type INT.
2237   // dst = 00004444 00003333 00002222 00001111
2238   sve_uunpklo(dst, S, src);
2239   // pgtmp = 00000001 00000000 00000001 00000001
2240   sve_punpklo(pgtmp, mask);
2241   // Pack the active elements in size of type INT to the right,
2242   // and fill the remainings with zero.
2243   // dst = 00000000 00004444 00002222 00001111
2244   sve_compact(dst, S, dst, pgtmp);
2245   // Narrow the result back to type SHORT.
2246   // dst = 0000 0000 0000 0000 0000 4444 2222 1111
2247   sve_uzp1(dst, H, dst, vtmp2);
2248   // Count the active elements of lowest half.
2249   // rscratch1 = 3
2250   sve_cntp(rscratch1, S, ptrue, pgtmp);
2251 
2252   // Repeat to the highest half.
2253   // pgtmp = 00000001 00000000 00000000 00000001
2254   sve_punpkhi(pgtmp, mask);
2255   // vtmp1 = 00008888 00007777 00006666 00005555
2256   sve_uunpkhi(vtmp1, S, src);
2257   // vtmp1 = 00000000 00000000 00008888 00005555
2258   sve_compact(vtmp1, S, vtmp1, pgtmp);
2259   // vtmp1 = 0000 0000 0000 0000 0000 0000 8888 5555
2260   sve_uzp1(vtmp1, H, vtmp1, vtmp2);
2261 
2262   // Compressed low:   dst   = 0000 0000 0000 0000 0000 4444 2222 1111
2263   // Compressed high:  vtmp1 = 0000 0000 0000 0000 0000 0000 8888  5555
2264   // Left shift(cross lane) compressed high with TRUE_CNT lanes,
2265   // TRUE_CNT is the number of active elements in the compressed low.
2266   neg(rscratch1, rscratch1);
2267   // vtmp2 = {4 3 2 1 0 -1 -2 -3}
2268   sve_index(vtmp2, H, rscratch1, 1);
2269   // vtmp1 = 0000 0000 0000 8888 5555 0000 0000 0000
2270   sve_tbl(vtmp1, H, vtmp1, vtmp2);
2271 
2272   // Combine the compressed high(after shifted) with the compressed low.
2273   // dst = 0000 0000 0000 8888 5555 4444 2222 1111
2274   sve_orr(dst, dst, vtmp1);
2275 }
2276 
2277 // Clobbers: rscratch1, rscratch2
2278 // Preserves: src, mask
2279 void C2_MacroAssembler::sve_compress_byte(FloatRegister dst, FloatRegister src, PRegister mask,
2280                                           FloatRegister vtmp1, FloatRegister vtmp2,
2281                                           FloatRegister vtmp3, FloatRegister vtmp4,
2282                                           PRegister ptmp, PRegister pgtmp) {
2283   assert(pgtmp->is_governing(), "This register has to be a governing predicate register");
2284   assert_different_registers(dst, src, vtmp1, vtmp2, vtmp3, vtmp4);
2285   assert_different_registers(mask, ptmp, pgtmp);
2286   // Example input:   src   = 88 77 66 55 44 33 22 11
2287   //                  mask  = 01 00 00 01 01 00 01 01
2288   // Expected result: dst   = 00 00 00 88 55 44 22 11
2289 
2290   sve_dup(vtmp4, B, 0);
2291   // Extend lowest half to type SHORT.
2292   // vtmp1 = 0044 0033 0022 0011
2293   sve_uunpklo(vtmp1, H, src);
2294   // ptmp = 0001 0000 0001 0001
2295   sve_punpklo(ptmp, mask);
2296   // Count the active elements of lowest half.
2297   // rscratch2 = 3
2298   sve_cntp(rscratch2, H, ptrue, ptmp);
2299   // Pack the active elements in size of type SHORT to the right,
2300   // and fill the remainings with zero.
2301   // dst = 0000 0044 0022 0011
2302   sve_compress_short(dst, vtmp1, ptmp, vtmp2, vtmp3, pgtmp);
2303   // Narrow the result back to type BYTE.
2304   // dst = 00 00 00 00 00 44 22 11
2305   sve_uzp1(dst, B, dst, vtmp4);
2306 
2307   // Repeat to the highest half.
2308   // ptmp = 0001 0000 0000 0001
2309   sve_punpkhi(ptmp, mask);
2310   // vtmp1 = 0088 0077 0066 0055
2311   sve_uunpkhi(vtmp2, H, src);
2312   // vtmp1 = 0000 0000 0088 0055
2313   sve_compress_short(vtmp1, vtmp2, ptmp, vtmp3, vtmp4, pgtmp);
2314 
2315   sve_dup(vtmp4, B, 0);
2316   // vtmp1 = 00 00 00 00 00 00 88 55
2317   sve_uzp1(vtmp1, B, vtmp1, vtmp4);
2318 
2319   // Compressed low:   dst   = 00 00 00 00 00 44 22 11
2320   // Compressed high:  vtmp1 = 00 00 00 00 00 00 88 55
2321   // Left shift(cross lane) compressed high with TRUE_CNT lanes,
2322   // TRUE_CNT is the number of active elements in the compressed low.
2323   neg(rscratch2, rscratch2);
2324   // vtmp2 = {4 3 2 1 0 -1 -2 -3}
2325   sve_index(vtmp2, B, rscratch2, 1);
2326   // vtmp1 = 00 00 00 88 55 00 00 00
2327   sve_tbl(vtmp1, B, vtmp1, vtmp2);
2328   // Combine the compressed high(after shifted) with the compressed low.
2329   // dst = 00 00 00 88 55 44 22 11
2330   sve_orr(dst, dst, vtmp1);
2331 }
2332 
2333 void C2_MacroAssembler::neon_reverse_bits(FloatRegister dst, FloatRegister src, BasicType bt, bool isQ) {
2334   assert(bt == T_BYTE || bt == T_SHORT || bt == T_INT || bt == T_LONG, "unsupported basic type");
2335   SIMD_Arrangement size = isQ ? T16B : T8B;
2336   if (bt == T_BYTE) {
2337     rbit(dst, size, src);
2338   } else {
2339     neon_reverse_bytes(dst, src, bt, isQ);
2340     rbit(dst, size, dst);
2341   }
2342 }
2343 
2344 void C2_MacroAssembler::neon_reverse_bytes(FloatRegister dst, FloatRegister src, BasicType bt, bool isQ) {
2345   assert(bt == T_BYTE || bt == T_SHORT || bt == T_INT || bt == T_LONG, "unsupported basic type");
2346   SIMD_Arrangement size = isQ ? T16B : T8B;
2347   switch (bt) {
2348     case T_BYTE:
2349       if (dst != src) {
2350         orr(dst, size, src, src);
2351       }
2352       break;
2353     case T_SHORT:
2354       rev16(dst, size, src);
2355       break;
2356     case T_INT:
2357       rev32(dst, size, src);
2358       break;
2359     case T_LONG:
2360       rev64(dst, size, src);
2361       break;
2362     default:
2363       assert(false, "unsupported");
2364       ShouldNotReachHere();
2365   }
2366 }
2367 
2368 // Extract a scalar element from an sve vector at position 'idx'.
2369 // The input elements in src are expected to be of integral type.
2370 void C2_MacroAssembler::sve_extract_integral(Register dst, BasicType bt, FloatRegister src,
2371                                              int idx, FloatRegister vtmp) {
2372   assert(bt == T_BYTE || bt == T_SHORT || bt == T_INT || bt == T_LONG, "unsupported element type");
2373   Assembler::SIMD_RegVariant size = elemType_to_regVariant(bt);
2374   if (regVariant_to_elemBits(size) * idx < 128) { // generate lower cost NEON instruction
2375     if (bt == T_INT || bt == T_LONG) {
2376       umov(dst, src, size, idx);
2377     } else {
2378       smov(dst, src, size, idx);
2379     }
2380   } else {
2381     sve_orr(vtmp, src, src);
2382     sve_ext(vtmp, vtmp, idx << size);
2383     if (bt == T_INT || bt == T_LONG) {
2384       umov(dst, vtmp, size, 0);
2385     } else {
2386       smov(dst, vtmp, size, 0);
2387     }
2388   }
2389 }
2390 
2391 // java.lang.Math::round intrinsics
2392 
2393 // Clobbers: rscratch1, rflags
2394 void C2_MacroAssembler::vector_round_neon(FloatRegister dst, FloatRegister src, FloatRegister tmp1,
2395                                           FloatRegister tmp2, FloatRegister tmp3, SIMD_Arrangement T) {
2396   assert_different_registers(tmp1, tmp2, tmp3, src, dst);
2397   switch (T) {
2398     case T2S:
2399     case T4S:
2400       fmovs(tmp1, T, 0.5f);
2401       mov(rscratch1, jint_cast(0x1.0p23f));
2402       break;
2403     case T2D:
2404       fmovd(tmp1, T, 0.5);
2405       mov(rscratch1, julong_cast(0x1.0p52));
2406       break;
2407     default:
2408       assert(T == T2S || T == T4S || T == T2D, "invalid arrangement");
2409   }
2410   fadd(tmp1, T, tmp1, src);
2411   fcvtms(tmp1, T, tmp1);
2412   // tmp1 = floor(src + 0.5, ties to even)
2413 
2414   fcvtas(dst, T, src);
2415   // dst = round(src), ties to away
2416 
2417   fneg(tmp3, T, src);
2418   dup(tmp2, T, rscratch1);
2419   cm(HS, tmp3, T, tmp3, tmp2);
2420   // tmp3 is now a set of flags
2421 
2422   bif(dst, T16B, tmp1, tmp3);
2423   // result in dst
2424 }
2425 
2426 // Clobbers: rscratch1, rflags
2427 void C2_MacroAssembler::vector_round_sve(FloatRegister dst, FloatRegister src, FloatRegister tmp1,
2428                                          FloatRegister tmp2, PRegister pgtmp, SIMD_RegVariant T) {
2429   assert(pgtmp->is_governing(), "This register has to be a governing predicate register");
2430   assert_different_registers(tmp1, tmp2, src, dst);
2431 
2432   switch (T) {
2433     case S:
2434       mov(rscratch1, jint_cast(0x1.0p23f));
2435       break;
2436     case D:
2437       mov(rscratch1, julong_cast(0x1.0p52));
2438       break;
2439     default:
2440       assert(T == S || T == D, "invalid register variant");
2441   }
2442 
2443   sve_frinta(dst, T, ptrue, src);
2444   // dst = round(src), ties to away
2445 
2446   Label none;
2447 
2448   sve_fneg(tmp1, T, ptrue, src);
2449   sve_dup(tmp2, T, rscratch1);
2450   sve_cmp(HS, pgtmp, T, ptrue, tmp2, tmp1);
2451   br(EQ, none);
2452   {
2453     sve_cpy(tmp1, T, pgtmp, 0.5);
2454     sve_fadd(tmp1, T, pgtmp, src);
2455     sve_frintm(dst, T, pgtmp, tmp1);
2456     // dst = floor(src + 0.5, ties to even)
2457   }
2458   bind(none);
2459 
2460   sve_fcvtzs(dst, T, ptrue, dst, T);
2461   // result in dst
2462 }
2463 
2464 void C2_MacroAssembler::vector_signum_neon(FloatRegister dst, FloatRegister src, FloatRegister zero,
2465                                            FloatRegister one, SIMD_Arrangement T) {
2466   assert_different_registers(dst, src, zero, one);
2467   assert(T == T2S || T == T4S || T == T2D, "invalid arrangement");
2468 
2469   facgt(dst, T, src, zero);
2470   ushr(dst, T, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
2471   bsl(dst, T == T2S ? T8B : T16B, one, src); // Result in dst
2472 }
2473 
2474 void C2_MacroAssembler::vector_signum_sve(FloatRegister dst, FloatRegister src, FloatRegister zero,
2475                                           FloatRegister one, FloatRegister vtmp, PRegister pgtmp, SIMD_RegVariant T) {
2476     assert_different_registers(dst, src, zero, one, vtmp);
2477     assert(pgtmp->is_governing(), "This register has to be a governing predicate register");
2478 
2479     sve_orr(vtmp, src, src);
2480     sve_fac(Assembler::GT, pgtmp, T, ptrue, src, zero); // pmtp=0 for +-0.0 and NaN. 0x1 otherwise
2481     switch (T) {
2482     case S:
2483       sve_and(vtmp, T, min_jint); // Extract the sign bit of float value in every lane of src
2484       sve_orr(vtmp, T, jint_cast(1.0)); // OR it with +1 to make the final result +1 or -1 depending
2485                                         // on the sign of the float value
2486       break;
2487     case D:
2488       sve_and(vtmp, T, min_jlong);
2489       sve_orr(vtmp, T, jlong_cast(1.0));
2490       break;
2491     default:
2492       assert(false, "unsupported");
2493       ShouldNotReachHere();
2494     }
2495     sve_sel(dst, T, pgtmp, vtmp, src); // Select either from src or vtmp based on the predicate register pgtmp
2496                                        // Result in dst
2497 }
2498 
2499 bool C2_MacroAssembler::in_scratch_emit_size() {
2500   if (ciEnv::current()->task() != nullptr) {
2501     PhaseOutput* phase_output = Compile::current()->output();
2502     if (phase_output != nullptr && phase_output->in_scratch_emit_size()) {
2503       return true;
2504     }
2505   }
2506   return MacroAssembler::in_scratch_emit_size();
2507 }