1 /*
2 * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/assembler.hpp"
27 #include "asm/assembler.inline.hpp"
28 #include "opto/c2_MacroAssembler.hpp"
29 #include "opto/compile.hpp"
30 #include "opto/intrinsicnode.hpp"
31 #include "opto/matcher.hpp"
32 #include "opto/output.hpp"
33 #include "opto/subnode.hpp"
34 #include "runtime/stubRoutines.hpp"
35 #include "utilities/globalDefinitions.hpp"
36
37 #ifdef PRODUCT
38 #define BLOCK_COMMENT(str) /* nothing */
39 #define STOP(error) stop(error)
40 #else
41 #define BLOCK_COMMENT(str) block_comment(str)
42 #define STOP(error) block_comment(error); stop(error)
43 #endif
44
45 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
46
47 typedef void (MacroAssembler::* chr_insn)(Register Rt, const Address &adr);
48
49 void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg, Register tmpReg,
50 Register tmp2Reg, Register tmp3Reg) {
51 Register oop = objectReg;
52 Register box = boxReg;
53 Register disp_hdr = tmpReg;
54 Register tmp = tmp2Reg;
55 Label cont;
56 Label object_has_monitor;
57 Label count, no_count;
58
59 assert(LockingMode != LM_LIGHTWEIGHT, "lightweight locking should use fast_lock_lightweight");
60 assert_different_registers(oop, box, tmp, disp_hdr);
61
62 // Load markWord from object into displaced_header.
63 ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
64
65 if (DiagnoseSyncOnValueBasedClasses != 0) {
66 load_klass(tmp, oop);
67 ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
68 tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
69 br(Assembler::NE, cont);
70 }
71
72 // Check for existing monitor
73 tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
74
75 if (LockingMode == LM_MONITOR) {
76 tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
77 b(cont);
78 } else {
79 assert(LockingMode == LM_LEGACY, "must be");
80 // Set tmp to be (markWord of object | UNLOCK_VALUE).
81 orr(tmp, disp_hdr, markWord::unlocked_value);
82
83 // Initialize the box. (Must happen before we update the object mark!)
84 str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
85
86 // Compare object markWord with an unlocked value (tmp) and if
87 // equal exchange the stack address of our box with object markWord.
88 // On failure disp_hdr contains the possibly locked markWord.
89 cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
90 /*release*/ true, /*weak*/ false, disp_hdr);
91 br(Assembler::EQ, cont);
92
93 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
94
95 // If the compare-and-exchange succeeded, then we found an unlocked
96 // object, will have now locked it will continue at label cont
97
98 // Check if the owner is self by comparing the value in the
99 // markWord of object (disp_hdr) with the stack pointer.
100 mov(rscratch1, sp);
101 sub(disp_hdr, disp_hdr, rscratch1);
102 mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
103 // If condition is true we are cont and hence we can store 0 as the
104 // displaced header in the box, which indicates that it is a recursive lock.
105 ands(tmp/*==0?*/, disp_hdr, tmp); // Sets flags for result
106 str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
107 b(cont);
108 }
109
110 // Handle existing monitor.
111 bind(object_has_monitor);
112
113 // The object's monitor m is unlocked iff m->owner == nullptr,
114 // otherwise m->owner may contain a thread or a stack address.
115 //
116 // Try to CAS m->owner from null to current thread.
117 add(tmp, disp_hdr, (in_bytes(ObjectMonitor::owner_offset())-markWord::monitor_value));
118 cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
119 /*release*/ true, /*weak*/ false, tmp3Reg); // Sets flags for result
120
121 // Store a non-null value into the box to avoid looking like a re-entrant
122 // lock. The fast-path monitor unlock code checks for
123 // markWord::monitor_value so use markWord::unused_mark which has the
124 // relevant bit set, and also matches ObjectSynchronizer::enter.
125 mov(tmp, (address)markWord::unused_mark().value());
126 str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
127
128 br(Assembler::EQ, cont); // CAS success means locking succeeded
129
130 cmp(tmp3Reg, rthread);
131 br(Assembler::NE, cont); // Check for recursive locking
132
133 // Recursive lock case
134 increment(Address(disp_hdr, in_bytes(ObjectMonitor::recursions_offset()) - markWord::monitor_value), 1);
135 // flag == EQ still from the cmp above, checking if this is a reentrant lock
136
137 bind(cont);
138 // flag == EQ indicates success
139 // flag == NE indicates failure
140 br(Assembler::NE, no_count);
141
142 bind(count);
143 increment(Address(rthread, JavaThread::held_monitor_count_offset()));
144
145 bind(no_count);
146 }
147
148 void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg, Register tmpReg,
149 Register tmp2Reg) {
150 Register oop = objectReg;
151 Register box = boxReg;
152 Register disp_hdr = tmpReg;
153 Register tmp = tmp2Reg;
154 Label cont;
155 Label object_has_monitor;
156 Label count, no_count;
157
158 assert(LockingMode != LM_LIGHTWEIGHT, "lightweight locking should use fast_unlock_lightweight");
159 assert_different_registers(oop, box, tmp, disp_hdr);
160
161 if (LockingMode == LM_LEGACY) {
162 // Find the lock address and load the displaced header from the stack.
163 ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
164
165 // If the displaced header is 0, we have a recursive unlock.
166 cmp(disp_hdr, zr);
167 br(Assembler::EQ, cont);
168 }
169
170 // Handle existing monitor.
171 ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
172 tbnz(tmp, exact_log2(markWord::monitor_value), object_has_monitor);
173
174 if (LockingMode == LM_MONITOR) {
175 tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
176 b(cont);
177 } else {
178 assert(LockingMode == LM_LEGACY, "must be");
179 // Check if it is still a light weight lock, this is is true if we
180 // see the stack address of the basicLock in the markWord of the
181 // object.
182
183 cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
184 /*release*/ true, /*weak*/ false, tmp);
185 b(cont);
186 }
187
188 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
189
190 // Handle existing monitor.
191 bind(object_has_monitor);
192 STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
193 add(tmp, tmp, -(int)markWord::monitor_value); // monitor
194
195 ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
196
197 Label notRecursive;
198 cbz(disp_hdr, notRecursive);
199
200 // Recursive lock
201 sub(disp_hdr, disp_hdr, 1u);
202 str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
203 cmp(disp_hdr, disp_hdr); // Sets flags for result
204 b(cont);
205
206 bind(notRecursive);
207 ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset()));
208 ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset()));
209 orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
210 cmp(rscratch1, zr); // Sets flags for result
211 cbnz(rscratch1, cont);
212 // need a release store here
213 lea(tmp, Address(tmp, ObjectMonitor::owner_offset()));
214 stlr(zr, tmp); // set unowned
215
216 bind(cont);
217 // flag == EQ indicates success
218 // flag == NE indicates failure
219 br(Assembler::NE, no_count);
220
221 bind(count);
222 decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
223
224 bind(no_count);
225 }
226
227 void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register t1,
228 Register t2, Register t3) {
229 assert(LockingMode == LM_LIGHTWEIGHT, "must be");
230 assert_different_registers(obj, t1, t2, t3);
231
232 // Handle inflated monitor.
233 Label inflated;
234 // Finish fast lock successfully. MUST branch to with flag == EQ
235 Label locked;
236 // Finish fast lock unsuccessfully. MUST branch to with flag == NE
237 Label slow_path;
238
239 if (DiagnoseSyncOnValueBasedClasses != 0) {
240 load_klass(t1, obj);
241 ldrw(t1, Address(t1, Klass::access_flags_offset()));
242 tstw(t1, JVM_ACC_IS_VALUE_BASED_CLASS);
243 br(Assembler::NE, slow_path);
244 }
245
246 const Register t1_mark = t1;
247
248 { // Lightweight locking
249
250 // Push lock to the lock stack and finish successfully. MUST branch to with flag == EQ
251 Label push;
252
253 const Register t2_top = t2;
254 const Register t3_t = t3;
255
256 // Check if lock-stack is full.
257 ldrw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
258 cmpw(t2_top, (unsigned)LockStack::end_offset() - 1);
259 br(Assembler::GT, slow_path);
260
261 // Check if recursive.
262 subw(t3_t, t2_top, oopSize);
263 ldr(t3_t, Address(rthread, t3_t));
264 cmp(obj, t3_t);
265 br(Assembler::EQ, push);
266
267 // Relaxed normal load to check for monitor. Optimization for monitor case.
268 ldr(t1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
269 tbnz(t1_mark, exact_log2(markWord::monitor_value), inflated);
270
271 // Not inflated
272 assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid a lea");
273
274 // Try to lock. Transition lock-bits 0b01 => 0b00
275 orr(t1_mark, t1_mark, markWord::unlocked_value);
276 eor(t3_t, t1_mark, markWord::unlocked_value);
277 cmpxchg(/*addr*/ obj, /*expected*/ t1_mark, /*new*/ t3_t, Assembler::xword,
278 /*acquire*/ true, /*release*/ false, /*weak*/ false, noreg);
279 br(Assembler::NE, slow_path);
280
281 bind(push);
282 // After successful lock, push object on lock-stack.
283 str(obj, Address(rthread, t2_top));
284 addw(t2_top, t2_top, oopSize);
285 strw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
286 b(locked);
287 }
288
289 { // Handle inflated monitor.
290 bind(inflated);
291
292 // mark contains the tagged ObjectMonitor*.
293 const Register t1_tagged_monitor = t1_mark;
294 const uintptr_t monitor_tag = markWord::monitor_value;
295 const Register t2_owner_addr = t2;
296 const Register t3_owner = t3;
297
298 // Compute owner address.
299 lea(t2_owner_addr, Address(t1_tagged_monitor, (in_bytes(ObjectMonitor::owner_offset()) - monitor_tag)));
300
301 // CAS owner (null => current thread).
302 cmpxchg(t2_owner_addr, zr, rthread, Assembler::xword, /*acquire*/ true,
303 /*release*/ false, /*weak*/ false, t3_owner);
304 br(Assembler::EQ, locked);
305
306 // Check if recursive.
307 cmp(t3_owner, rthread);
308 br(Assembler::NE, slow_path);
309
310 // Recursive.
311 increment(Address(t1_tagged_monitor, in_bytes(ObjectMonitor::recursions_offset()) - monitor_tag), 1);
312 }
313
314 bind(locked);
315 increment(Address(rthread, JavaThread::held_monitor_count_offset()));
316
317 #ifdef ASSERT
318 // Check that locked label is reached with Flags == EQ.
319 Label flag_correct;
320 br(Assembler::EQ, flag_correct);
321 stop("Fast Lock Flag != EQ");
322 #endif
323
324 bind(slow_path);
325 #ifdef ASSERT
326 // Check that slow_path label is reached with Flags == NE.
327 br(Assembler::NE, flag_correct);
328 stop("Fast Lock Flag != NE");
329 bind(flag_correct);
330 #endif
331 // C2 uses the value of Flags (NE vs EQ) to determine the continuation.
332 }
333
334 void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register t1, Register t2,
335 Register t3) {
336 assert(LockingMode == LM_LIGHTWEIGHT, "must be");
337 assert_different_registers(obj, t1, t2, t3);
338
339 // Handle inflated monitor.
340 Label inflated, inflated_load_monitor;
341 // Finish fast unlock successfully. MUST branch to with flag == EQ
342 Label unlocked;
343 // Finish fast unlock unsuccessfully. MUST branch to with flag == NE
344 Label slow_path;
345
346 const Register t1_mark = t1;
347 const Register t2_top = t2;
348 const Register t3_t = t3;
349
350 { // Lightweight unlock
351
352 // Check if obj is top of lock-stack.
353 ldrw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
354 subw(t2_top, t2_top, oopSize);
355 ldr(t3_t, Address(rthread, t2_top));
356 cmp(obj, t3_t);
357 // Top of lock stack was not obj. Must be monitor.
358 br(Assembler::NE, inflated_load_monitor);
359
360 // Pop lock-stack.
361 DEBUG_ONLY(str(zr, Address(rthread, t2_top));)
362 strw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
363
364 // Check if recursive.
365 subw(t3_t, t2_top, oopSize);
366 ldr(t3_t, Address(rthread, t3_t));
367 cmp(obj, t3_t);
368 br(Assembler::EQ, unlocked);
369
370 // Not recursive.
371 // Load Mark.
372 ldr(t1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
373
374 // Check header for monitor (0b10).
375 tbnz(t1_mark, exact_log2(markWord::monitor_value), inflated);
376
377 // Try to unlock. Transition lock bits 0b00 => 0b01
378 assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
379 orr(t3_t, t1_mark, markWord::unlocked_value);
380 cmpxchg(/*addr*/ obj, /*expected*/ t1_mark, /*new*/ t3_t, Assembler::xword,
381 /*acquire*/ false, /*release*/ true, /*weak*/ false, noreg);
382 br(Assembler::EQ, unlocked);
383
384 // Compare and exchange failed.
385 // Restore lock-stack and handle the unlock in runtime.
386 DEBUG_ONLY(str(obj, Address(rthread, t2_top));)
387 addw(t2_top, t2_top, oopSize);
388 str(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
389 b(slow_path);
390 }
391
392
393 { // Handle inflated monitor.
394 bind(inflated_load_monitor);
395 ldr(t1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
396 #ifdef ASSERT
397 tbnz(t1_mark, exact_log2(markWord::monitor_value), inflated);
398 stop("Fast Unlock not monitor");
399 #endif
400
401 bind(inflated);
402
403 #ifdef ASSERT
404 Label check_done;
405 subw(t2_top, t2_top, oopSize);
406 cmpw(t2_top, in_bytes(JavaThread::lock_stack_base_offset()));
407 br(Assembler::LT, check_done);
408 ldr(t3_t, Address(rthread, t2_top));
409 cmp(obj, t3_t);
410 br(Assembler::NE, inflated);
411 stop("Fast Unlock lock on stack");
412 bind(check_done);
413 #endif
414
415 // mark contains the tagged ObjectMonitor*.
416 const Register t1_monitor = t1_mark;
417 const uintptr_t monitor_tag = markWord::monitor_value;
418
419 // Untag the monitor.
420 sub(t1_monitor, t1_mark, monitor_tag);
421
422 const Register t2_recursions = t2;
423 Label not_recursive;
424
425 // Check if recursive.
426 ldr(t2_recursions, Address(t1_monitor, ObjectMonitor::recursions_offset()));
427 cbz(t2_recursions, not_recursive);
428
429 // Recursive unlock.
430 sub(t2_recursions, t2_recursions, 1u);
431 str(t2_recursions, Address(t1_monitor, ObjectMonitor::recursions_offset()));
432 // Set flag == EQ
433 cmp(t2_recursions, t2_recursions);
434 b(unlocked);
435
436 bind(not_recursive);
437
438 Label release;
439 const Register t2_owner_addr = t2;
440
441 // Compute owner address.
442 lea(t2_owner_addr, Address(t1_monitor, ObjectMonitor::owner_offset()));
443
444 // Check if the entry lists are empty.
445 ldr(rscratch1, Address(t1_monitor, ObjectMonitor::EntryList_offset()));
446 ldr(t3_t, Address(t1_monitor, ObjectMonitor::cxq_offset()));
447 orr(rscratch1, rscratch1, t3_t);
448 cmp(rscratch1, zr);
449 br(Assembler::EQ, release);
450
451 // The owner may be anonymous and we removed the last obj entry in
452 // the lock-stack. This loses the information about the owner.
453 // Write the thread to the owner field so the runtime knows the owner.
454 str(rthread, Address(t2_owner_addr));
455 b(slow_path);
456
457 bind(release);
458 // Set owner to null.
459 // Release to satisfy the JMM
460 stlr(zr, t2_owner_addr);
461 }
462
463 bind(unlocked);
464 decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
465
466 #ifdef ASSERT
467 // Check that unlocked label is reached with Flags == EQ.
468 Label flag_correct;
469 br(Assembler::EQ, flag_correct);
470 stop("Fast Unlock Flag != EQ");
471 #endif
472
473 bind(slow_path);
474 #ifdef ASSERT
475 // Check that slow_path label is reached with Flags == NE.
476 br(Assembler::NE, flag_correct);
477 stop("Fast Unlock Flag != NE");
478 bind(flag_correct);
479 #endif
480 // C2 uses the value of Flags (NE vs EQ) to determine the continuation.
481 }
482
483 // Search for str1 in str2 and return index or -1
484 // Clobbers: rscratch1, rscratch2, rflags. May also clobber v0-v1, when icnt1==-1.
485 void C2_MacroAssembler::string_indexof(Register str2, Register str1,
486 Register cnt2, Register cnt1,
487 Register tmp1, Register tmp2,
488 Register tmp3, Register tmp4,
489 Register tmp5, Register tmp6,
490 int icnt1, Register result, int ae) {
491 // NOTE: tmp5, tmp6 can be zr depending on specific method version
492 Label LINEARSEARCH, LINEARSTUB, LINEAR_MEDIUM, DONE, NOMATCH, MATCH;
493
494 Register ch1 = rscratch1;
495 Register ch2 = rscratch2;
496 Register cnt1tmp = tmp1;
497 Register cnt2tmp = tmp2;
498 Register cnt1_neg = cnt1;
499 Register cnt2_neg = cnt2;
500 Register result_tmp = tmp4;
501
502 bool isL = ae == StrIntrinsicNode::LL;
503
504 bool str1_isL = ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UL;
505 bool str2_isL = ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::LU;
506 int str1_chr_shift = str1_isL ? 0:1;
507 int str2_chr_shift = str2_isL ? 0:1;
508 int str1_chr_size = str1_isL ? 1:2;
509 int str2_chr_size = str2_isL ? 1:2;
510 chr_insn str1_load_1chr = str1_isL ? (chr_insn)&MacroAssembler::ldrb :
511 (chr_insn)&MacroAssembler::ldrh;
512 chr_insn str2_load_1chr = str2_isL ? (chr_insn)&MacroAssembler::ldrb :
513 (chr_insn)&MacroAssembler::ldrh;
514 chr_insn load_2chr = isL ? (chr_insn)&MacroAssembler::ldrh : (chr_insn)&MacroAssembler::ldrw;
515 chr_insn load_4chr = isL ? (chr_insn)&MacroAssembler::ldrw : (chr_insn)&MacroAssembler::ldr;
516
517 // Note, inline_string_indexOf() generates checks:
518 // if (substr.count > string.count) return -1;
519 // if (substr.count == 0) return 0;
520
521 // We have two strings, a source string in str2, cnt2 and a pattern string
522 // in str1, cnt1. Find the 1st occurrence of pattern in source or return -1.
523
524 // For larger pattern and source we use a simplified Boyer Moore algorithm.
525 // With a small pattern and source we use linear scan.
526
527 if (icnt1 == -1) {
528 sub(result_tmp, cnt2, cnt1);
529 cmp(cnt1, (u1)8); // Use Linear Scan if cnt1 < 8 || cnt1 >= 256
530 br(LT, LINEARSEARCH);
531 dup(v0, T16B, cnt1); // done in separate FPU pipeline. Almost no penalty
532 subs(zr, cnt1, 256);
533 lsr(tmp1, cnt2, 2);
534 ccmp(cnt1, tmp1, 0b0000, LT); // Source must be 4 * pattern for BM
535 br(GE, LINEARSTUB);
536 }
537
538 // The Boyer Moore alogorithm is based on the description here:-
539 //
540 // http://en.wikipedia.org/wiki/Boyer%E2%80%93Moore_string_search_algorithm
541 //
542 // This describes and algorithm with 2 shift rules. The 'Bad Character' rule
543 // and the 'Good Suffix' rule.
544 //
545 // These rules are essentially heuristics for how far we can shift the
546 // pattern along the search string.
547 //
548 // The implementation here uses the 'Bad Character' rule only because of the
549 // complexity of initialisation for the 'Good Suffix' rule.
550 //
551 // This is also known as the Boyer-Moore-Horspool algorithm:-
552 //
553 // http://en.wikipedia.org/wiki/Boyer-Moore-Horspool_algorithm
554 //
555 // This particular implementation has few java-specific optimizations.
556 //
557 // #define ASIZE 256
558 //
559 // int bm(unsigned char *x, int m, unsigned char *y, int n) {
560 // int i, j;
561 // unsigned c;
562 // unsigned char bc[ASIZE];
563 //
564 // /* Preprocessing */
565 // for (i = 0; i < ASIZE; ++i)
566 // bc[i] = m;
567 // for (i = 0; i < m - 1; ) {
568 // c = x[i];
569 // ++i;
570 // // c < 256 for Latin1 string, so, no need for branch
571 // #ifdef PATTERN_STRING_IS_LATIN1
572 // bc[c] = m - i;
573 // #else
574 // if (c < ASIZE) bc[c] = m - i;
575 // #endif
576 // }
577 //
578 // /* Searching */
579 // j = 0;
580 // while (j <= n - m) {
581 // c = y[i+j];
582 // if (x[m-1] == c)
583 // for (i = m - 2; i >= 0 && x[i] == y[i + j]; --i);
584 // if (i < 0) return j;
585 // // c < 256 for Latin1 string, so, no need for branch
586 // #ifdef SOURCE_STRING_IS_LATIN1
587 // // LL case: (c< 256) always true. Remove branch
588 // j += bc[y[j+m-1]];
589 // #endif
590 // #ifndef PATTERN_STRING_IS_UTF
591 // // UU case: need if (c<ASIZE) check. Skip 1 character if not.
592 // if (c < ASIZE)
593 // j += bc[y[j+m-1]];
594 // else
595 // j += 1
596 // #endif
597 // #ifdef PATTERN_IS_LATIN1_AND_SOURCE_IS_UTF
598 // // UL case: need if (c<ASIZE) check. Skip <pattern length> if not.
599 // if (c < ASIZE)
600 // j += bc[y[j+m-1]];
601 // else
602 // j += m
603 // #endif
604 // }
605 // }
606
607 if (icnt1 == -1) {
608 Label BCLOOP, BCSKIP, BMLOOPSTR2, BMLOOPSTR1, BMSKIP, BMADV, BMMATCH,
609 BMLOOPSTR1_LASTCMP, BMLOOPSTR1_CMP, BMLOOPSTR1_AFTER_LOAD, BM_INIT_LOOP;
610 Register cnt1end = tmp2;
611 Register str2end = cnt2;
612 Register skipch = tmp2;
613
614 // str1 length is >=8, so, we can read at least 1 register for cases when
615 // UTF->Latin1 conversion is not needed(8 LL or 4UU) and half register for
616 // UL case. We'll re-read last character in inner pre-loop code to have
617 // single outer pre-loop load
618 const int firstStep = isL ? 7 : 3;
619
620 const int ASIZE = 256;
621 const int STORED_BYTES = 32; // amount of bytes stored per instruction
622 sub(sp, sp, ASIZE);
623 mov(tmp5, ASIZE/STORED_BYTES); // loop iterations
624 mov(ch1, sp);
625 BIND(BM_INIT_LOOP);
626 stpq(v0, v0, Address(post(ch1, STORED_BYTES)));
627 subs(tmp5, tmp5, 1);
628 br(GT, BM_INIT_LOOP);
629
630 sub(cnt1tmp, cnt1, 1);
631 mov(tmp5, str2);
632 add(str2end, str2, result_tmp, LSL, str2_chr_shift);
633 sub(ch2, cnt1, 1);
634 mov(tmp3, str1);
635 BIND(BCLOOP);
636 (this->*str1_load_1chr)(ch1, Address(post(tmp3, str1_chr_size)));
637 if (!str1_isL) {
638 subs(zr, ch1, ASIZE);
639 br(HS, BCSKIP);
640 }
641 strb(ch2, Address(sp, ch1));
642 BIND(BCSKIP);
643 subs(ch2, ch2, 1);
644 br(GT, BCLOOP);
645
646 add(tmp6, str1, cnt1, LSL, str1_chr_shift); // address after str1
647 if (str1_isL == str2_isL) {
648 // load last 8 bytes (8LL/4UU symbols)
649 ldr(tmp6, Address(tmp6, -wordSize));
650 } else {
651 ldrw(tmp6, Address(tmp6, -wordSize/2)); // load last 4 bytes(4 symbols)
652 // convert Latin1 to UTF. We'll have to wait until load completed, but
653 // it's still faster than per-character loads+checks
654 lsr(tmp3, tmp6, BitsPerByte * (wordSize/2 - str1_chr_size)); // str1[N-1]
655 ubfx(ch1, tmp6, 8, 8); // str1[N-2]
656 ubfx(ch2, tmp6, 16, 8); // str1[N-3]
657 andr(tmp6, tmp6, 0xFF); // str1[N-4]
658 orr(ch2, ch1, ch2, LSL, 16);
659 orr(tmp6, tmp6, tmp3, LSL, 48);
660 orr(tmp6, tmp6, ch2, LSL, 16);
661 }
662 BIND(BMLOOPSTR2);
663 (this->*str2_load_1chr)(skipch, Address(str2, cnt1tmp, Address::lsl(str2_chr_shift)));
664 sub(cnt1tmp, cnt1tmp, firstStep); // cnt1tmp is positive here, because cnt1 >= 8
665 if (str1_isL == str2_isL) {
666 // re-init tmp3. It's for free because it's executed in parallel with
667 // load above. Alternative is to initialize it before loop, but it'll
668 // affect performance on in-order systems with 2 or more ld/st pipelines
669 lsr(tmp3, tmp6, BitsPerByte * (wordSize - str1_chr_size));
670 }
671 if (!isL) { // UU/UL case
672 lsl(ch2, cnt1tmp, 1); // offset in bytes
673 }
674 cmp(tmp3, skipch);
675 br(NE, BMSKIP);
676 ldr(ch2, Address(str2, isL ? cnt1tmp : ch2));
677 mov(ch1, tmp6);
678 if (isL) {
679 b(BMLOOPSTR1_AFTER_LOAD);
680 } else {
681 sub(cnt1tmp, cnt1tmp, 1); // no need to branch for UU/UL case. cnt1 >= 8
682 b(BMLOOPSTR1_CMP);
683 }
684 BIND(BMLOOPSTR1);
685 (this->*str1_load_1chr)(ch1, Address(str1, cnt1tmp, Address::lsl(str1_chr_shift)));
686 (this->*str2_load_1chr)(ch2, Address(str2, cnt1tmp, Address::lsl(str2_chr_shift)));
687 BIND(BMLOOPSTR1_AFTER_LOAD);
688 subs(cnt1tmp, cnt1tmp, 1);
689 br(LT, BMLOOPSTR1_LASTCMP);
690 BIND(BMLOOPSTR1_CMP);
691 cmp(ch1, ch2);
692 br(EQ, BMLOOPSTR1);
693 BIND(BMSKIP);
694 if (!isL) {
695 // if we've met UTF symbol while searching Latin1 pattern, then we can
696 // skip cnt1 symbols
697 if (str1_isL != str2_isL) {
698 mov(result_tmp, cnt1);
699 } else {
700 mov(result_tmp, 1);
701 }
702 subs(zr, skipch, ASIZE);
703 br(HS, BMADV);
704 }
705 ldrb(result_tmp, Address(sp, skipch)); // load skip distance
706 BIND(BMADV);
707 sub(cnt1tmp, cnt1, 1);
708 add(str2, str2, result_tmp, LSL, str2_chr_shift);
709 cmp(str2, str2end);
710 br(LE, BMLOOPSTR2);
711 add(sp, sp, ASIZE);
712 b(NOMATCH);
713 BIND(BMLOOPSTR1_LASTCMP);
714 cmp(ch1, ch2);
715 br(NE, BMSKIP);
716 BIND(BMMATCH);
717 sub(result, str2, tmp5);
718 if (!str2_isL) lsr(result, result, 1);
719 add(sp, sp, ASIZE);
720 b(DONE);
721
722 BIND(LINEARSTUB);
723 cmp(cnt1, (u1)16); // small patterns still should be handled by simple algorithm
724 br(LT, LINEAR_MEDIUM);
725 mov(result, zr);
726 RuntimeAddress stub = nullptr;
727 if (isL) {
728 stub = RuntimeAddress(StubRoutines::aarch64::string_indexof_linear_ll());
729 assert(stub.target() != nullptr, "string_indexof_linear_ll stub has not been generated");
730 } else if (str1_isL) {
731 stub = RuntimeAddress(StubRoutines::aarch64::string_indexof_linear_ul());
732 assert(stub.target() != nullptr, "string_indexof_linear_ul stub has not been generated");
733 } else {
734 stub = RuntimeAddress(StubRoutines::aarch64::string_indexof_linear_uu());
735 assert(stub.target() != nullptr, "string_indexof_linear_uu stub has not been generated");
736 }
737 address call = trampoline_call(stub);
738 if (call == nullptr) {
739 DEBUG_ONLY(reset_labels(LINEARSEARCH, LINEAR_MEDIUM, DONE, NOMATCH, MATCH));
740 ciEnv::current()->record_failure("CodeCache is full");
741 return;
742 }
743 b(DONE);
744 }
745
746 BIND(LINEARSEARCH);
747 {
748 Label DO1, DO2, DO3;
749
750 Register str2tmp = tmp2;
751 Register first = tmp3;
752
753 if (icnt1 == -1)
754 {
755 Label DOSHORT, FIRST_LOOP, STR2_NEXT, STR1_LOOP, STR1_NEXT;
756
757 cmp(cnt1, u1(str1_isL == str2_isL ? 4 : 2));
758 br(LT, DOSHORT);
759 BIND(LINEAR_MEDIUM);
760 (this->*str1_load_1chr)(first, Address(str1));
761 lea(str1, Address(str1, cnt1, Address::lsl(str1_chr_shift)));
762 sub(cnt1_neg, zr, cnt1, LSL, str1_chr_shift);
763 lea(str2, Address(str2, result_tmp, Address::lsl(str2_chr_shift)));
764 sub(cnt2_neg, zr, result_tmp, LSL, str2_chr_shift);
765
766 BIND(FIRST_LOOP);
767 (this->*str2_load_1chr)(ch2, Address(str2, cnt2_neg));
768 cmp(first, ch2);
769 br(EQ, STR1_LOOP);
770 BIND(STR2_NEXT);
771 adds(cnt2_neg, cnt2_neg, str2_chr_size);
772 br(LE, FIRST_LOOP);
773 b(NOMATCH);
774
775 BIND(STR1_LOOP);
776 adds(cnt1tmp, cnt1_neg, str1_chr_size);
777 add(cnt2tmp, cnt2_neg, str2_chr_size);
778 br(GE, MATCH);
779
780 BIND(STR1_NEXT);
781 (this->*str1_load_1chr)(ch1, Address(str1, cnt1tmp));
782 (this->*str2_load_1chr)(ch2, Address(str2, cnt2tmp));
783 cmp(ch1, ch2);
784 br(NE, STR2_NEXT);
785 adds(cnt1tmp, cnt1tmp, str1_chr_size);
786 add(cnt2tmp, cnt2tmp, str2_chr_size);
787 br(LT, STR1_NEXT);
788 b(MATCH);
789
790 BIND(DOSHORT);
791 if (str1_isL == str2_isL) {
792 cmp(cnt1, (u1)2);
793 br(LT, DO1);
794 br(GT, DO3);
795 }
796 }
797
798 if (icnt1 == 4) {
799 Label CH1_LOOP;
800
801 (this->*load_4chr)(ch1, str1);
802 sub(result_tmp, cnt2, 4);
803 lea(str2, Address(str2, result_tmp, Address::lsl(str2_chr_shift)));
804 sub(cnt2_neg, zr, result_tmp, LSL, str2_chr_shift);
805
806 BIND(CH1_LOOP);
807 (this->*load_4chr)(ch2, Address(str2, cnt2_neg));
808 cmp(ch1, ch2);
809 br(EQ, MATCH);
810 adds(cnt2_neg, cnt2_neg, str2_chr_size);
811 br(LE, CH1_LOOP);
812 b(NOMATCH);
813 }
814
815 if ((icnt1 == -1 && str1_isL == str2_isL) || icnt1 == 2) {
816 Label CH1_LOOP;
817
818 BIND(DO2);
819 (this->*load_2chr)(ch1, str1);
820 if (icnt1 == 2) {
821 sub(result_tmp, cnt2, 2);
822 }
823 lea(str2, Address(str2, result_tmp, Address::lsl(str2_chr_shift)));
824 sub(cnt2_neg, zr, result_tmp, LSL, str2_chr_shift);
825 BIND(CH1_LOOP);
826 (this->*load_2chr)(ch2, Address(str2, cnt2_neg));
827 cmp(ch1, ch2);
828 br(EQ, MATCH);
829 adds(cnt2_neg, cnt2_neg, str2_chr_size);
830 br(LE, CH1_LOOP);
831 b(NOMATCH);
832 }
833
834 if ((icnt1 == -1 && str1_isL == str2_isL) || icnt1 == 3) {
835 Label FIRST_LOOP, STR2_NEXT, STR1_LOOP;
836
837 BIND(DO3);
838 (this->*load_2chr)(first, str1);
839 (this->*str1_load_1chr)(ch1, Address(str1, 2*str1_chr_size));
840 if (icnt1 == 3) {
841 sub(result_tmp, cnt2, 3);
842 }
843 lea(str2, Address(str2, result_tmp, Address::lsl(str2_chr_shift)));
844 sub(cnt2_neg, zr, result_tmp, LSL, str2_chr_shift);
845 BIND(FIRST_LOOP);
846 (this->*load_2chr)(ch2, Address(str2, cnt2_neg));
847 cmpw(first, ch2);
848 br(EQ, STR1_LOOP);
849 BIND(STR2_NEXT);
850 adds(cnt2_neg, cnt2_neg, str2_chr_size);
851 br(LE, FIRST_LOOP);
852 b(NOMATCH);
853
854 BIND(STR1_LOOP);
855 add(cnt2tmp, cnt2_neg, 2*str2_chr_size);
856 (this->*str2_load_1chr)(ch2, Address(str2, cnt2tmp));
857 cmp(ch1, ch2);
858 br(NE, STR2_NEXT);
859 b(MATCH);
860 }
861
862 if (icnt1 == -1 || icnt1 == 1) {
863 Label CH1_LOOP, HAS_ZERO, DO1_SHORT, DO1_LOOP;
864
865 BIND(DO1);
866 (this->*str1_load_1chr)(ch1, str1);
867 cmp(cnt2, (u1)8);
868 br(LT, DO1_SHORT);
869
870 sub(result_tmp, cnt2, 8/str2_chr_size);
871 sub(cnt2_neg, zr, result_tmp, LSL, str2_chr_shift);
872 mov(tmp3, str2_isL ? 0x0101010101010101 : 0x0001000100010001);
873 lea(str2, Address(str2, result_tmp, Address::lsl(str2_chr_shift)));
874
875 if (str2_isL) {
876 orr(ch1, ch1, ch1, LSL, 8);
877 }
878 orr(ch1, ch1, ch1, LSL, 16);
879 orr(ch1, ch1, ch1, LSL, 32);
880 BIND(CH1_LOOP);
881 ldr(ch2, Address(str2, cnt2_neg));
882 eor(ch2, ch1, ch2);
883 sub(tmp1, ch2, tmp3);
884 orr(tmp2, ch2, str2_isL ? 0x7f7f7f7f7f7f7f7f : 0x7fff7fff7fff7fff);
885 bics(tmp1, tmp1, tmp2);
886 br(NE, HAS_ZERO);
887 adds(cnt2_neg, cnt2_neg, 8);
888 br(LT, CH1_LOOP);
889
890 cmp(cnt2_neg, (u1)8);
891 mov(cnt2_neg, 0);
892 br(LT, CH1_LOOP);
893 b(NOMATCH);
894
895 BIND(HAS_ZERO);
896 rev(tmp1, tmp1);
897 clz(tmp1, tmp1);
898 add(cnt2_neg, cnt2_neg, tmp1, LSR, 3);
899 b(MATCH);
900
901 BIND(DO1_SHORT);
902 mov(result_tmp, cnt2);
903 lea(str2, Address(str2, cnt2, Address::lsl(str2_chr_shift)));
904 sub(cnt2_neg, zr, cnt2, LSL, str2_chr_shift);
905 BIND(DO1_LOOP);
906 (this->*str2_load_1chr)(ch2, Address(str2, cnt2_neg));
907 cmpw(ch1, ch2);
908 br(EQ, MATCH);
909 adds(cnt2_neg, cnt2_neg, str2_chr_size);
910 br(LT, DO1_LOOP);
911 }
912 }
913 BIND(NOMATCH);
914 mov(result, -1);
915 b(DONE);
916 BIND(MATCH);
917 add(result, result_tmp, cnt2_neg, ASR, str2_chr_shift);
918 BIND(DONE);
919 }
920
921 typedef void (MacroAssembler::* chr_insn)(Register Rt, const Address &adr);
922 typedef void (MacroAssembler::* uxt_insn)(Register Rd, Register Rn);
923
924 void C2_MacroAssembler::string_indexof_char(Register str1, Register cnt1,
925 Register ch, Register result,
926 Register tmp1, Register tmp2, Register tmp3)
927 {
928 Label CH1_LOOP, HAS_ZERO, DO1_SHORT, DO1_LOOP, MATCH, NOMATCH, DONE;
929 Register cnt1_neg = cnt1;
930 Register ch1 = rscratch1;
931 Register result_tmp = rscratch2;
932
933 cbz(cnt1, NOMATCH);
934
935 cmp(cnt1, (u1)4);
936 br(LT, DO1_SHORT);
937
938 orr(ch, ch, ch, LSL, 16);
939 orr(ch, ch, ch, LSL, 32);
940
941 sub(cnt1, cnt1, 4);
942 mov(result_tmp, cnt1);
943 lea(str1, Address(str1, cnt1, Address::uxtw(1)));
944 sub(cnt1_neg, zr, cnt1, LSL, 1);
945
946 mov(tmp3, 0x0001000100010001);
947
948 BIND(CH1_LOOP);
949 ldr(ch1, Address(str1, cnt1_neg));
950 eor(ch1, ch, ch1);
951 sub(tmp1, ch1, tmp3);
952 orr(tmp2, ch1, 0x7fff7fff7fff7fff);
953 bics(tmp1, tmp1, tmp2);
954 br(NE, HAS_ZERO);
955 adds(cnt1_neg, cnt1_neg, 8);
956 br(LT, CH1_LOOP);
957
958 cmp(cnt1_neg, (u1)8);
959 mov(cnt1_neg, 0);
960 br(LT, CH1_LOOP);
961 b(NOMATCH);
962
963 BIND(HAS_ZERO);
964 rev(tmp1, tmp1);
965 clz(tmp1, tmp1);
966 add(cnt1_neg, cnt1_neg, tmp1, LSR, 3);
967 b(MATCH);
968
969 BIND(DO1_SHORT);
970 mov(result_tmp, cnt1);
971 lea(str1, Address(str1, cnt1, Address::uxtw(1)));
972 sub(cnt1_neg, zr, cnt1, LSL, 1);
973 BIND(DO1_LOOP);
974 ldrh(ch1, Address(str1, cnt1_neg));
975 cmpw(ch, ch1);
976 br(EQ, MATCH);
977 adds(cnt1_neg, cnt1_neg, 2);
978 br(LT, DO1_LOOP);
979 BIND(NOMATCH);
980 mov(result, -1);
981 b(DONE);
982 BIND(MATCH);
983 add(result, result_tmp, cnt1_neg, ASR, 1);
984 BIND(DONE);
985 }
986
987 void C2_MacroAssembler::string_indexof_char_sve(Register str1, Register cnt1,
988 Register ch, Register result,
989 FloatRegister ztmp1,
990 FloatRegister ztmp2,
991 PRegister tmp_pg,
992 PRegister tmp_pdn, bool isL)
993 {
994 // Note that `tmp_pdn` should *NOT* be used as governing predicate register.
995 assert(tmp_pg->is_governing(),
996 "this register has to be a governing predicate register");
997
998 Label LOOP, MATCH, DONE, NOMATCH;
999 Register vec_len = rscratch1;
1000 Register idx = rscratch2;
1001
1002 SIMD_RegVariant T = (isL == true) ? B : H;
1003
1004 cbz(cnt1, NOMATCH);
1005
1006 // Assign the particular char throughout the vector.
1007 sve_dup(ztmp2, T, ch);
1008 if (isL) {
1009 sve_cntb(vec_len);
1010 } else {
1011 sve_cnth(vec_len);
1012 }
1013 mov(idx, 0);
1014
1015 // Generate a predicate to control the reading of input string.
1016 sve_whilelt(tmp_pg, T, idx, cnt1);
1017
1018 BIND(LOOP);
1019 // Read a vector of 8- or 16-bit data depending on the string type. Note
1020 // that inactive elements indicated by the predicate register won't cause
1021 // a data read from memory to the destination vector.
1022 if (isL) {
1023 sve_ld1b(ztmp1, T, tmp_pg, Address(str1, idx));
1024 } else {
1025 sve_ld1h(ztmp1, T, tmp_pg, Address(str1, idx, Address::lsl(1)));
1026 }
1027 add(idx, idx, vec_len);
1028
1029 // Perform the comparison. An element of the destination predicate is set
1030 // to active if the particular char is matched.
1031 sve_cmp(Assembler::EQ, tmp_pdn, T, tmp_pg, ztmp1, ztmp2);
1032
1033 // Branch if the particular char is found.
1034 br(NE, MATCH);
1035
1036 sve_whilelt(tmp_pg, T, idx, cnt1);
1037
1038 // Loop back if the particular char not found.
1039 br(MI, LOOP);
1040
1041 BIND(NOMATCH);
1042 mov(result, -1);
1043 b(DONE);
1044
1045 BIND(MATCH);
1046 // Undo the index increment.
1047 sub(idx, idx, vec_len);
1048
1049 // Crop the vector to find its location.
1050 sve_brka(tmp_pdn, tmp_pg, tmp_pdn, false /* isMerge */);
1051 add(result, idx, -1);
1052 sve_incp(result, T, tmp_pdn);
1053 BIND(DONE);
1054 }
1055
1056 void C2_MacroAssembler::stringL_indexof_char(Register str1, Register cnt1,
1057 Register ch, Register result,
1058 Register tmp1, Register tmp2, Register tmp3)
1059 {
1060 Label CH1_LOOP, HAS_ZERO, DO1_SHORT, DO1_LOOP, MATCH, NOMATCH, DONE;
1061 Register cnt1_neg = cnt1;
1062 Register ch1 = rscratch1;
1063 Register result_tmp = rscratch2;
1064
1065 cbz(cnt1, NOMATCH);
1066
1067 cmp(cnt1, (u1)8);
1068 br(LT, DO1_SHORT);
1069
1070 orr(ch, ch, ch, LSL, 8);
1071 orr(ch, ch, ch, LSL, 16);
1072 orr(ch, ch, ch, LSL, 32);
1073
1074 sub(cnt1, cnt1, 8);
1075 mov(result_tmp, cnt1);
1076 lea(str1, Address(str1, cnt1));
1077 sub(cnt1_neg, zr, cnt1);
1078
1079 mov(tmp3, 0x0101010101010101);
1080
1081 BIND(CH1_LOOP);
1082 ldr(ch1, Address(str1, cnt1_neg));
1083 eor(ch1, ch, ch1);
1084 sub(tmp1, ch1, tmp3);
1085 orr(tmp2, ch1, 0x7f7f7f7f7f7f7f7f);
1086 bics(tmp1, tmp1, tmp2);
1087 br(NE, HAS_ZERO);
1088 adds(cnt1_neg, cnt1_neg, 8);
1089 br(LT, CH1_LOOP);
1090
1091 cmp(cnt1_neg, (u1)8);
1092 mov(cnt1_neg, 0);
1093 br(LT, CH1_LOOP);
1094 b(NOMATCH);
1095
1096 BIND(HAS_ZERO);
1097 rev(tmp1, tmp1);
1098 clz(tmp1, tmp1);
1099 add(cnt1_neg, cnt1_neg, tmp1, LSR, 3);
1100 b(MATCH);
1101
1102 BIND(DO1_SHORT);
1103 mov(result_tmp, cnt1);
1104 lea(str1, Address(str1, cnt1));
1105 sub(cnt1_neg, zr, cnt1);
1106 BIND(DO1_LOOP);
1107 ldrb(ch1, Address(str1, cnt1_neg));
1108 cmp(ch, ch1);
1109 br(EQ, MATCH);
1110 adds(cnt1_neg, cnt1_neg, 1);
1111 br(LT, DO1_LOOP);
1112 BIND(NOMATCH);
1113 mov(result, -1);
1114 b(DONE);
1115 BIND(MATCH);
1116 add(result, result_tmp, cnt1_neg);
1117 BIND(DONE);
1118 }
1119
1120 // Compare strings.
1121 void C2_MacroAssembler::string_compare(Register str1, Register str2,
1122 Register cnt1, Register cnt2, Register result, Register tmp1, Register tmp2,
1123 FloatRegister vtmp1, FloatRegister vtmp2, FloatRegister vtmp3,
1124 PRegister pgtmp1, PRegister pgtmp2, int ae) {
1125 Label DONE, SHORT_LOOP, SHORT_STRING, SHORT_LAST, TAIL, STUB,
1126 DIFF, NEXT_WORD, SHORT_LOOP_TAIL, SHORT_LAST2, SHORT_LAST_INIT,
1127 SHORT_LOOP_START, TAIL_CHECK;
1128
1129 bool isLL = ae == StrIntrinsicNode::LL;
1130 bool isLU = ae == StrIntrinsicNode::LU;
1131 bool isUL = ae == StrIntrinsicNode::UL;
1132
1133 // The stub threshold for LL strings is: 72 (64 + 8) chars
1134 // UU: 36 chars, or 72 bytes (valid for the 64-byte large loop with prefetch)
1135 // LU/UL: 24 chars, or 48 bytes (valid for the 16-character loop at least)
1136 const u1 stub_threshold = isLL ? 72 : ((isLU || isUL) ? 24 : 36);
1137
1138 bool str1_isL = isLL || isLU;
1139 bool str2_isL = isLL || isUL;
1140
1141 int str1_chr_shift = str1_isL ? 0 : 1;
1142 int str2_chr_shift = str2_isL ? 0 : 1;
1143 int str1_chr_size = str1_isL ? 1 : 2;
1144 int str2_chr_size = str2_isL ? 1 : 2;
1145 int minCharsInWord = isLL ? wordSize : wordSize/2;
1146
1147 FloatRegister vtmpZ = vtmp1, vtmp = vtmp2;
1148 chr_insn str1_load_chr = str1_isL ? (chr_insn)&MacroAssembler::ldrb :
1149 (chr_insn)&MacroAssembler::ldrh;
1150 chr_insn str2_load_chr = str2_isL ? (chr_insn)&MacroAssembler::ldrb :
1151 (chr_insn)&MacroAssembler::ldrh;
1152 uxt_insn ext_chr = isLL ? (uxt_insn)&MacroAssembler::uxtbw :
1153 (uxt_insn)&MacroAssembler::uxthw;
1154
1155 BLOCK_COMMENT("string_compare {");
1156
1157 // Bizzarely, the counts are passed in bytes, regardless of whether they
1158 // are L or U strings, however the result is always in characters.
1159 if (!str1_isL) asrw(cnt1, cnt1, 1);
1160 if (!str2_isL) asrw(cnt2, cnt2, 1);
1161
1162 // Compute the minimum of the string lengths and save the difference.
1163 subsw(result, cnt1, cnt2);
1164 cselw(cnt2, cnt1, cnt2, Assembler::LE); // min
1165
1166 // A very short string
1167 cmpw(cnt2, minCharsInWord);
1168 br(Assembler::LE, SHORT_STRING);
1169
1170 // Compare longwords
1171 // load first parts of strings and finish initialization while loading
1172 {
1173 if (str1_isL == str2_isL) { // LL or UU
1174 ldr(tmp1, Address(str1));
1175 cmp(str1, str2);
1176 br(Assembler::EQ, DONE);
1177 ldr(tmp2, Address(str2));
1178 cmp(cnt2, stub_threshold);
1179 br(GE, STUB);
1180 subsw(cnt2, cnt2, minCharsInWord);
1181 br(EQ, TAIL_CHECK);
1182 lea(str2, Address(str2, cnt2, Address::uxtw(str2_chr_shift)));
1183 lea(str1, Address(str1, cnt2, Address::uxtw(str1_chr_shift)));
1184 sub(cnt2, zr, cnt2, LSL, str2_chr_shift);
1185 } else if (isLU) {
1186 ldrs(vtmp, Address(str1));
1187 ldr(tmp2, Address(str2));
1188 cmp(cnt2, stub_threshold);
1189 br(GE, STUB);
1190 subw(cnt2, cnt2, 4);
1191 eor(vtmpZ, T16B, vtmpZ, vtmpZ);
1192 lea(str1, Address(str1, cnt2, Address::uxtw(str1_chr_shift)));
1193 lea(str2, Address(str2, cnt2, Address::uxtw(str2_chr_shift)));
1194 zip1(vtmp, T8B, vtmp, vtmpZ);
1195 sub(cnt1, zr, cnt2, LSL, str1_chr_shift);
1196 sub(cnt2, zr, cnt2, LSL, str2_chr_shift);
1197 add(cnt1, cnt1, 4);
1198 fmovd(tmp1, vtmp);
1199 } else { // UL case
1200 ldr(tmp1, Address(str1));
1201 ldrs(vtmp, Address(str2));
1202 cmp(cnt2, stub_threshold);
1203 br(GE, STUB);
1204 subw(cnt2, cnt2, 4);
1205 lea(str1, Address(str1, cnt2, Address::uxtw(str1_chr_shift)));
1206 eor(vtmpZ, T16B, vtmpZ, vtmpZ);
1207 lea(str2, Address(str2, cnt2, Address::uxtw(str2_chr_shift)));
1208 sub(cnt1, zr, cnt2, LSL, str1_chr_shift);
1209 zip1(vtmp, T8B, vtmp, vtmpZ);
1210 sub(cnt2, zr, cnt2, LSL, str2_chr_shift);
1211 add(cnt1, cnt1, 8);
1212 fmovd(tmp2, vtmp);
1213 }
1214 adds(cnt2, cnt2, isUL ? 4 : 8);
1215 br(GE, TAIL);
1216 eor(rscratch2, tmp1, tmp2);
1217 cbnz(rscratch2, DIFF);
1218 // main loop
1219 bind(NEXT_WORD);
1220 if (str1_isL == str2_isL) {
1221 ldr(tmp1, Address(str1, cnt2));
1222 ldr(tmp2, Address(str2, cnt2));
1223 adds(cnt2, cnt2, 8);
1224 } else if (isLU) {
1225 ldrs(vtmp, Address(str1, cnt1));
1226 ldr(tmp2, Address(str2, cnt2));
1227 add(cnt1, cnt1, 4);
1228 zip1(vtmp, T8B, vtmp, vtmpZ);
1229 fmovd(tmp1, vtmp);
1230 adds(cnt2, cnt2, 8);
1231 } else { // UL
1232 ldrs(vtmp, Address(str2, cnt2));
1233 ldr(tmp1, Address(str1, cnt1));
1234 zip1(vtmp, T8B, vtmp, vtmpZ);
1235 add(cnt1, cnt1, 8);
1236 fmovd(tmp2, vtmp);
1237 adds(cnt2, cnt2, 4);
1238 }
1239 br(GE, TAIL);
1240
1241 eor(rscratch2, tmp1, tmp2);
1242 cbz(rscratch2, NEXT_WORD);
1243 b(DIFF);
1244 bind(TAIL);
1245 eor(rscratch2, tmp1, tmp2);
1246 cbnz(rscratch2, DIFF);
1247 // Last longword. In the case where length == 4 we compare the
1248 // same longword twice, but that's still faster than another
1249 // conditional branch.
1250 if (str1_isL == str2_isL) {
1251 ldr(tmp1, Address(str1));
1252 ldr(tmp2, Address(str2));
1253 } else if (isLU) {
1254 ldrs(vtmp, Address(str1));
1255 ldr(tmp2, Address(str2));
1256 zip1(vtmp, T8B, vtmp, vtmpZ);
1257 fmovd(tmp1, vtmp);
1258 } else { // UL
1259 ldrs(vtmp, Address(str2));
1260 ldr(tmp1, Address(str1));
1261 zip1(vtmp, T8B, vtmp, vtmpZ);
1262 fmovd(tmp2, vtmp);
1263 }
1264 bind(TAIL_CHECK);
1265 eor(rscratch2, tmp1, tmp2);
1266 cbz(rscratch2, DONE);
1267
1268 // Find the first different characters in the longwords and
1269 // compute their difference.
1270 bind(DIFF);
1271 rev(rscratch2, rscratch2);
1272 clz(rscratch2, rscratch2);
1273 andr(rscratch2, rscratch2, isLL ? -8 : -16);
1274 lsrv(tmp1, tmp1, rscratch2);
1275 (this->*ext_chr)(tmp1, tmp1);
1276 lsrv(tmp2, tmp2, rscratch2);
1277 (this->*ext_chr)(tmp2, tmp2);
1278 subw(result, tmp1, tmp2);
1279 b(DONE);
1280 }
1281
1282 bind(STUB);
1283 RuntimeAddress stub = nullptr;
1284 switch(ae) {
1285 case StrIntrinsicNode::LL:
1286 stub = RuntimeAddress(StubRoutines::aarch64::compare_long_string_LL());
1287 break;
1288 case StrIntrinsicNode::UU:
1289 stub = RuntimeAddress(StubRoutines::aarch64::compare_long_string_UU());
1290 break;
1291 case StrIntrinsicNode::LU:
1292 stub = RuntimeAddress(StubRoutines::aarch64::compare_long_string_LU());
1293 break;
1294 case StrIntrinsicNode::UL:
1295 stub = RuntimeAddress(StubRoutines::aarch64::compare_long_string_UL());
1296 break;
1297 default:
1298 ShouldNotReachHere();
1299 }
1300 assert(stub.target() != nullptr, "compare_long_string stub has not been generated");
1301 address call = trampoline_call(stub);
1302 if (call == nullptr) {
1303 DEBUG_ONLY(reset_labels(DONE, SHORT_LOOP, SHORT_STRING, SHORT_LAST, SHORT_LOOP_TAIL, SHORT_LAST2, SHORT_LAST_INIT, SHORT_LOOP_START));
1304 ciEnv::current()->record_failure("CodeCache is full");
1305 return;
1306 }
1307 b(DONE);
1308
1309 bind(SHORT_STRING);
1310 // Is the minimum length zero?
1311 cbz(cnt2, DONE);
1312 // arrange code to do most branches while loading and loading next characters
1313 // while comparing previous
1314 (this->*str1_load_chr)(tmp1, Address(post(str1, str1_chr_size)));
1315 subs(cnt2, cnt2, 1);
1316 br(EQ, SHORT_LAST_INIT);
1317 (this->*str2_load_chr)(cnt1, Address(post(str2, str2_chr_size)));
1318 b(SHORT_LOOP_START);
1319 bind(SHORT_LOOP);
1320 subs(cnt2, cnt2, 1);
1321 br(EQ, SHORT_LAST);
1322 bind(SHORT_LOOP_START);
1323 (this->*str1_load_chr)(tmp2, Address(post(str1, str1_chr_size)));
1324 (this->*str2_load_chr)(rscratch1, Address(post(str2, str2_chr_size)));
1325 cmp(tmp1, cnt1);
1326 br(NE, SHORT_LOOP_TAIL);
1327 subs(cnt2, cnt2, 1);
1328 br(EQ, SHORT_LAST2);
1329 (this->*str1_load_chr)(tmp1, Address(post(str1, str1_chr_size)));
1330 (this->*str2_load_chr)(cnt1, Address(post(str2, str2_chr_size)));
1331 cmp(tmp2, rscratch1);
1332 br(EQ, SHORT_LOOP);
1333 sub(result, tmp2, rscratch1);
1334 b(DONE);
1335 bind(SHORT_LOOP_TAIL);
1336 sub(result, tmp1, cnt1);
1337 b(DONE);
1338 bind(SHORT_LAST2);
1339 cmp(tmp2, rscratch1);
1340 br(EQ, DONE);
1341 sub(result, tmp2, rscratch1);
1342
1343 b(DONE);
1344 bind(SHORT_LAST_INIT);
1345 (this->*str2_load_chr)(cnt1, Address(post(str2, str2_chr_size)));
1346 bind(SHORT_LAST);
1347 cmp(tmp1, cnt1);
1348 br(EQ, DONE);
1349 sub(result, tmp1, cnt1);
1350
1351 bind(DONE);
1352
1353 BLOCK_COMMENT("} string_compare");
1354 }
1355
1356 void C2_MacroAssembler::neon_compare(FloatRegister dst, BasicType bt, FloatRegister src1,
1357 FloatRegister src2, Condition cond, bool isQ) {
1358 SIMD_Arrangement size = esize2arrangement((unsigned)type2aelembytes(bt), isQ);
1359 FloatRegister zn = src1, zm = src2;
1360 bool needs_negation = false;
1361 switch (cond) {
1362 case LT: cond = GT; zn = src2; zm = src1; break;
1363 case LE: cond = GE; zn = src2; zm = src1; break;
1364 case LO: cond = HI; zn = src2; zm = src1; break;
1365 case LS: cond = HS; zn = src2; zm = src1; break;
1366 case NE: cond = EQ; needs_negation = true; break;
1367 default:
1368 break;
1369 }
1370
1371 if (is_floating_point_type(bt)) {
1372 fcm(cond, dst, size, zn, zm);
1373 } else {
1374 cm(cond, dst, size, zn, zm);
1375 }
1376
1377 if (needs_negation) {
1378 notr(dst, isQ ? T16B : T8B, dst);
1379 }
1380 }
1381
1382 void C2_MacroAssembler::neon_compare_zero(FloatRegister dst, BasicType bt, FloatRegister src,
1383 Condition cond, bool isQ) {
1384 SIMD_Arrangement size = esize2arrangement((unsigned)type2aelembytes(bt), isQ);
1385 if (bt == T_FLOAT || bt == T_DOUBLE) {
1386 if (cond == Assembler::NE) {
1387 fcm(Assembler::EQ, dst, size, src);
1388 notr(dst, isQ ? T16B : T8B, dst);
1389 } else {
1390 fcm(cond, dst, size, src);
1391 }
1392 } else {
1393 if (cond == Assembler::NE) {
1394 cm(Assembler::EQ, dst, size, src);
1395 notr(dst, isQ ? T16B : T8B, dst);
1396 } else {
1397 cm(cond, dst, size, src);
1398 }
1399 }
1400 }
1401
1402 // Compress the least significant bit of each byte to the rightmost and clear
1403 // the higher garbage bits.
1404 void C2_MacroAssembler::bytemask_compress(Register dst) {
1405 // Example input, dst = 0x01 00 00 00 01 01 00 01
1406 // The "??" bytes are garbage.
1407 orr(dst, dst, dst, Assembler::LSR, 7); // dst = 0x?? 02 ?? 00 ?? 03 ?? 01
1408 orr(dst, dst, dst, Assembler::LSR, 14); // dst = 0x????????08 ??????0D
1409 orr(dst, dst, dst, Assembler::LSR, 28); // dst = 0x????????????????8D
1410 andr(dst, dst, 0xff); // dst = 0x8D
1411 }
1412
1413 // Pack the lowest-numbered bit of each mask element in src into a long value
1414 // in dst, at most the first 64 lane elements.
1415 // Clobbers: rscratch1, if UseSVE=1 or the hardware doesn't support FEAT_BITPERM.
1416 void C2_MacroAssembler::sve_vmask_tolong(Register dst, PRegister src, BasicType bt, int lane_cnt,
1417 FloatRegister vtmp1, FloatRegister vtmp2) {
1418 assert(lane_cnt <= 64 && is_power_of_2(lane_cnt), "Unsupported lane count");
1419 assert_different_registers(dst, rscratch1);
1420 assert_different_registers(vtmp1, vtmp2);
1421
1422 Assembler::SIMD_RegVariant size = elemType_to_regVariant(bt);
1423 // Example: src = 0b01100101 10001101, bt = T_BYTE, lane_cnt = 16
1424 // Expected: dst = 0x658D
1425
1426 // Convert the mask into vector with sequential bytes.
1427 // vtmp1 = 0x00010100 0x00010001 0x01000000 0x01010001
1428 sve_cpy(vtmp1, size, src, 1, false);
1429 if (bt != T_BYTE) {
1430 sve_vector_narrow(vtmp1, B, vtmp1, size, vtmp2);
1431 }
1432
1433 if (UseSVE > 1 && VM_Version::supports_svebitperm()) {
1434 // Given a vector with the value 0x00 or 0x01 in each byte, the basic idea
1435 // is to compress each significant bit of the byte in a cross-lane way. Due
1436 // to the lack of a cross-lane bit-compress instruction, we use BEXT
1437 // (bit-compress in each lane) with the biggest lane size (T = D) then
1438 // concatenate the results.
1439
1440 // The second source input of BEXT, initialized with 0x01 in each byte.
1441 // vtmp2 = 0x01010101 0x01010101 0x01010101 0x01010101
1442 sve_dup(vtmp2, B, 1);
1443
1444 // BEXT vtmp1.D, vtmp1.D, vtmp2.D
1445 // vtmp1 = 0x0001010000010001 | 0x0100000001010001
1446 // vtmp2 = 0x0101010101010101 | 0x0101010101010101
1447 // ---------------------------------------
1448 // vtmp1 = 0x0000000000000065 | 0x000000000000008D
1449 sve_bext(vtmp1, D, vtmp1, vtmp2);
1450
1451 // Concatenate the lowest significant 8 bits in each 8 bytes, and extract the
1452 // result to dst.
1453 // vtmp1 = 0x0000000000000000 | 0x000000000000658D
1454 // dst = 0x658D
1455 if (lane_cnt <= 8) {
1456 // No need to concatenate.
1457 umov(dst, vtmp1, B, 0);
1458 } else if (lane_cnt <= 16) {
1459 ins(vtmp1, B, vtmp1, 1, 8);
1460 umov(dst, vtmp1, H, 0);
1461 } else {
1462 // As the lane count is 64 at most, the final expected value must be in
1463 // the lowest 64 bits after narrowing vtmp1 from D to B.
1464 sve_vector_narrow(vtmp1, B, vtmp1, D, vtmp2);
1465 umov(dst, vtmp1, D, 0);
1466 }
1467 } else if (UseSVE > 0) {
1468 // Compress the lowest 8 bytes.
1469 fmovd(dst, vtmp1);
1470 bytemask_compress(dst);
1471 if (lane_cnt <= 8) return;
1472
1473 // Repeat on higher bytes and join the results.
1474 // Compress 8 bytes in each iteration.
1475 for (int idx = 1; idx < (lane_cnt / 8); idx++) {
1476 sve_extract_integral(rscratch1, T_LONG, vtmp1, idx, vtmp2);
1477 bytemask_compress(rscratch1);
1478 orr(dst, dst, rscratch1, Assembler::LSL, idx << 3);
1479 }
1480 } else {
1481 assert(false, "unsupported");
1482 ShouldNotReachHere();
1483 }
1484 }
1485
1486 // Unpack the mask, a long value in src, into predicate register dst based on the
1487 // corresponding data type. Note that dst can support at most 64 lanes.
1488 // Below example gives the expected dst predicate register in different types, with
1489 // a valid src(0x658D) on a 1024-bit vector size machine.
1490 // BYTE: dst = 0x00 00 00 00 00 00 00 00 00 00 00 00 00 00 65 8D
1491 // SHORT: dst = 0x00 00 00 00 00 00 00 00 00 00 00 00 14 11 40 51
1492 // INT: dst = 0x00 00 00 00 00 00 00 00 01 10 01 01 10 00 11 01
1493 // LONG: dst = 0x00 01 01 00 00 01 00 01 01 00 00 00 01 01 00 01
1494 //
1495 // The number of significant bits of src must be equal to lane_cnt. E.g., 0xFF658D which
1496 // has 24 significant bits would be an invalid input if dst predicate register refers to
1497 // a LONG type 1024-bit vector, which has at most 16 lanes.
1498 void C2_MacroAssembler::sve_vmask_fromlong(PRegister dst, Register src, BasicType bt, int lane_cnt,
1499 FloatRegister vtmp1, FloatRegister vtmp2) {
1500 assert(UseSVE == 2 && VM_Version::supports_svebitperm() &&
1501 lane_cnt <= 64 && is_power_of_2(lane_cnt), "unsupported");
1502 Assembler::SIMD_RegVariant size = elemType_to_regVariant(bt);
1503 // Example: src = 0x658D, bt = T_BYTE, size = B, lane_cnt = 16
1504 // Expected: dst = 0b01101001 10001101
1505
1506 // Put long value from general purpose register into the first lane of vector.
1507 // vtmp1 = 0x0000000000000000 | 0x000000000000658D
1508 sve_dup(vtmp1, B, 0);
1509 mov(vtmp1, D, 0, src);
1510
1511 // As sve_cmp generates mask value with the minimum unit in byte, we should
1512 // transform the value in the first lane which is mask in bit now to the
1513 // mask in byte, which can be done by SVE2's BDEP instruction.
1514
1515 // The first source input of BDEP instruction. Deposite each byte in every 8 bytes.
1516 // vtmp1 = 0x0000000000000065 | 0x000000000000008D
1517 if (lane_cnt <= 8) {
1518 // Nothing. As only one byte exsits.
1519 } else if (lane_cnt <= 16) {
1520 ins(vtmp1, B, vtmp1, 8, 1);
1521 mov(vtmp1, B, 1, zr);
1522 } else {
1523 sve_vector_extend(vtmp1, D, vtmp1, B);
1524 }
1525
1526 // The second source input of BDEP instruction, initialized with 0x01 for each byte.
1527 // vtmp2 = 0x01010101 0x01010101 0x01010101 0x01010101
1528 sve_dup(vtmp2, B, 1);
1529
1530 // BDEP vtmp1.D, vtmp1.D, vtmp2.D
1531 // vtmp1 = 0x0000000000000065 | 0x000000000000008D
1532 // vtmp2 = 0x0101010101010101 | 0x0101010101010101
1533 // ---------------------------------------
1534 // vtmp1 = 0x0001010000010001 | 0x0100000001010001
1535 sve_bdep(vtmp1, D, vtmp1, vtmp2);
1536
1537 if (bt != T_BYTE) {
1538 sve_vector_extend(vtmp1, size, vtmp1, B);
1539 }
1540 // Generate mask according to the given vector, in which the elements have been
1541 // extended to expected type.
1542 // dst = 0b01101001 10001101
1543 sve_cmp(Assembler::NE, dst, size, ptrue, vtmp1, 0);
1544 }
1545
1546 // Clobbers: rflags
1547 void C2_MacroAssembler::sve_compare(PRegister pd, BasicType bt, PRegister pg,
1548 FloatRegister zn, FloatRegister zm, Condition cond) {
1549 assert(pg->is_governing(), "This register has to be a governing predicate register");
1550 FloatRegister z1 = zn, z2 = zm;
1551 switch (cond) {
1552 case LE: z1 = zm; z2 = zn; cond = GE; break;
1553 case LT: z1 = zm; z2 = zn; cond = GT; break;
1554 case LO: z1 = zm; z2 = zn; cond = HI; break;
1555 case LS: z1 = zm; z2 = zn; cond = HS; break;
1556 default:
1557 break;
1558 }
1559
1560 SIMD_RegVariant size = elemType_to_regVariant(bt);
1561 if (is_floating_point_type(bt)) {
1562 sve_fcm(cond, pd, size, pg, z1, z2);
1563 } else {
1564 assert(is_integral_type(bt), "unsupported element type");
1565 sve_cmp(cond, pd, size, pg, z1, z2);
1566 }
1567 }
1568
1569 // Get index of the last mask lane that is set
1570 void C2_MacroAssembler::sve_vmask_lasttrue(Register dst, BasicType bt, PRegister src, PRegister ptmp) {
1571 SIMD_RegVariant size = elemType_to_regVariant(bt);
1572 sve_rev(ptmp, size, src);
1573 sve_brkb(ptmp, ptrue, ptmp, false);
1574 sve_cntp(dst, size, ptrue, ptmp);
1575 movw(rscratch1, MaxVectorSize / type2aelembytes(bt) - 1);
1576 subw(dst, rscratch1, dst);
1577 }
1578
1579 // Extend integer vector src to dst with the same lane count
1580 // but larger element size, e.g. 4B -> 4I
1581 void C2_MacroAssembler::neon_vector_extend(FloatRegister dst, BasicType dst_bt, unsigned dst_vlen_in_bytes,
1582 FloatRegister src, BasicType src_bt, bool is_unsigned) {
1583 if (src_bt == T_BYTE) {
1584 if (dst_bt == T_SHORT) {
1585 // 4B/8B to 4S/8S
1586 _xshll(is_unsigned, dst, T8H, src, T8B, 0);
1587 } else {
1588 // 4B to 4I
1589 assert(dst_vlen_in_bytes == 16 && dst_bt == T_INT, "unsupported");
1590 _xshll(is_unsigned, dst, T8H, src, T8B, 0);
1591 _xshll(is_unsigned, dst, T4S, dst, T4H, 0);
1592 }
1593 } else if (src_bt == T_SHORT) {
1594 // 4S to 4I
1595 assert(dst_vlen_in_bytes == 16 && dst_bt == T_INT, "unsupported");
1596 _xshll(is_unsigned, dst, T4S, src, T4H, 0);
1597 } else if (src_bt == T_INT) {
1598 // 2I to 2L
1599 assert(dst_vlen_in_bytes == 16 && dst_bt == T_LONG, "unsupported");
1600 _xshll(is_unsigned, dst, T2D, src, T2S, 0);
1601 } else {
1602 ShouldNotReachHere();
1603 }
1604 }
1605
1606 // Narrow integer vector src down to dst with the same lane count
1607 // but smaller element size, e.g. 4I -> 4B
1608 void C2_MacroAssembler::neon_vector_narrow(FloatRegister dst, BasicType dst_bt,
1609 FloatRegister src, BasicType src_bt, unsigned src_vlen_in_bytes) {
1610 if (src_bt == T_SHORT) {
1611 // 4S/8S to 4B/8B
1612 assert(src_vlen_in_bytes == 8 || src_vlen_in_bytes == 16, "unsupported");
1613 assert(dst_bt == T_BYTE, "unsupported");
1614 xtn(dst, T8B, src, T8H);
1615 } else if (src_bt == T_INT) {
1616 // 4I to 4B/4S
1617 assert(src_vlen_in_bytes == 16, "unsupported");
1618 assert(dst_bt == T_BYTE || dst_bt == T_SHORT, "unsupported");
1619 xtn(dst, T4H, src, T4S);
1620 if (dst_bt == T_BYTE) {
1621 xtn(dst, T8B, dst, T8H);
1622 }
1623 } else if (src_bt == T_LONG) {
1624 // 2L to 2I
1625 assert(src_vlen_in_bytes == 16, "unsupported");
1626 assert(dst_bt == T_INT, "unsupported");
1627 xtn(dst, T2S, src, T2D);
1628 } else {
1629 ShouldNotReachHere();
1630 }
1631 }
1632
1633 void C2_MacroAssembler::sve_vector_extend(FloatRegister dst, SIMD_RegVariant dst_size,
1634 FloatRegister src, SIMD_RegVariant src_size,
1635 bool is_unsigned) {
1636 assert(dst_size > src_size && dst_size <= D && src_size <= S, "invalid element size");
1637
1638 if (src_size == B) {
1639 switch (dst_size) {
1640 case H:
1641 _sve_xunpk(is_unsigned, /* is_high */ false, dst, H, src);
1642 break;
1643 case S:
1644 _sve_xunpk(is_unsigned, /* is_high */ false, dst, H, src);
1645 _sve_xunpk(is_unsigned, /* is_high */ false, dst, S, dst);
1646 break;
1647 case D:
1648 _sve_xunpk(is_unsigned, /* is_high */ false, dst, H, src);
1649 _sve_xunpk(is_unsigned, /* is_high */ false, dst, S, dst);
1650 _sve_xunpk(is_unsigned, /* is_high */ false, dst, D, dst);
1651 break;
1652 default:
1653 ShouldNotReachHere();
1654 }
1655 } else if (src_size == H) {
1656 if (dst_size == S) {
1657 _sve_xunpk(is_unsigned, /* is_high */ false, dst, S, src);
1658 } else { // D
1659 _sve_xunpk(is_unsigned, /* is_high */ false, dst, S, src);
1660 _sve_xunpk(is_unsigned, /* is_high */ false, dst, D, dst);
1661 }
1662 } else if (src_size == S) {
1663 _sve_xunpk(is_unsigned, /* is_high */ false, dst, D, src);
1664 }
1665 }
1666
1667 // Vector narrow from src to dst with specified element sizes.
1668 // High part of dst vector will be filled with zero.
1669 void C2_MacroAssembler::sve_vector_narrow(FloatRegister dst, SIMD_RegVariant dst_size,
1670 FloatRegister src, SIMD_RegVariant src_size,
1671 FloatRegister tmp) {
1672 assert(dst_size < src_size && dst_size <= S && src_size <= D, "invalid element size");
1673 assert_different_registers(src, tmp);
1674 sve_dup(tmp, src_size, 0);
1675 if (src_size == D) {
1676 switch (dst_size) {
1677 case S:
1678 sve_uzp1(dst, S, src, tmp);
1679 break;
1680 case H:
1681 assert_different_registers(dst, tmp);
1682 sve_uzp1(dst, S, src, tmp);
1683 sve_uzp1(dst, H, dst, tmp);
1684 break;
1685 case B:
1686 assert_different_registers(dst, tmp);
1687 sve_uzp1(dst, S, src, tmp);
1688 sve_uzp1(dst, H, dst, tmp);
1689 sve_uzp1(dst, B, dst, tmp);
1690 break;
1691 default:
1692 ShouldNotReachHere();
1693 }
1694 } else if (src_size == S) {
1695 if (dst_size == H) {
1696 sve_uzp1(dst, H, src, tmp);
1697 } else { // B
1698 assert_different_registers(dst, tmp);
1699 sve_uzp1(dst, H, src, tmp);
1700 sve_uzp1(dst, B, dst, tmp);
1701 }
1702 } else if (src_size == H) {
1703 sve_uzp1(dst, B, src, tmp);
1704 }
1705 }
1706
1707 // Extend src predicate to dst predicate with the same lane count but larger
1708 // element size, e.g. 64Byte -> 512Long
1709 void C2_MacroAssembler::sve_vmaskcast_extend(PRegister dst, PRegister src,
1710 uint dst_element_length_in_bytes,
1711 uint src_element_length_in_bytes) {
1712 if (dst_element_length_in_bytes == 2 * src_element_length_in_bytes) {
1713 sve_punpklo(dst, src);
1714 } else if (dst_element_length_in_bytes == 4 * src_element_length_in_bytes) {
1715 sve_punpklo(dst, src);
1716 sve_punpklo(dst, dst);
1717 } else if (dst_element_length_in_bytes == 8 * src_element_length_in_bytes) {
1718 sve_punpklo(dst, src);
1719 sve_punpklo(dst, dst);
1720 sve_punpklo(dst, dst);
1721 } else {
1722 assert(false, "unsupported");
1723 ShouldNotReachHere();
1724 }
1725 }
1726
1727 // Narrow src predicate to dst predicate with the same lane count but
1728 // smaller element size, e.g. 512Long -> 64Byte
1729 void C2_MacroAssembler::sve_vmaskcast_narrow(PRegister dst, PRegister src, PRegister ptmp,
1730 uint dst_element_length_in_bytes, uint src_element_length_in_bytes) {
1731 // The insignificant bits in src predicate are expected to be zero.
1732 // To ensure the higher order bits of the resultant narrowed vector are 0, an all-zero predicate is
1733 // passed as the second argument. An example narrowing operation with a given mask would be -
1734 // 128Long -> 64Int on a 128-bit machine i.e 2L -> 2I
1735 // Mask (for 2 Longs) : TF
1736 // Predicate register for the above mask (16 bits) : 00000001 00000000
1737 // After narrowing (uzp1 dst.b, src.b, ptmp.b) : 0000 0000 0001 0000
1738 // Which translates to mask for 2 integers as : TF (lower half is considered while upper half is 0)
1739 assert_different_registers(src, ptmp);
1740 assert_different_registers(dst, ptmp);
1741 sve_pfalse(ptmp);
1742 if (dst_element_length_in_bytes * 2 == src_element_length_in_bytes) {
1743 sve_uzp1(dst, B, src, ptmp);
1744 } else if (dst_element_length_in_bytes * 4 == src_element_length_in_bytes) {
1745 sve_uzp1(dst, H, src, ptmp);
1746 sve_uzp1(dst, B, dst, ptmp);
1747 } else if (dst_element_length_in_bytes * 8 == src_element_length_in_bytes) {
1748 sve_uzp1(dst, S, src, ptmp);
1749 sve_uzp1(dst, H, dst, ptmp);
1750 sve_uzp1(dst, B, dst, ptmp);
1751 } else {
1752 assert(false, "unsupported");
1753 ShouldNotReachHere();
1754 }
1755 }
1756
1757 // Vector reduction add for integral type with ASIMD instructions.
1758 void C2_MacroAssembler::neon_reduce_add_integral(Register dst, BasicType bt,
1759 Register isrc, FloatRegister vsrc,
1760 unsigned vector_length_in_bytes,
1761 FloatRegister vtmp) {
1762 assert(vector_length_in_bytes == 8 || vector_length_in_bytes == 16, "unsupported");
1763 assert_different_registers(dst, isrc);
1764 bool isQ = vector_length_in_bytes == 16;
1765
1766 BLOCK_COMMENT("neon_reduce_add_integral {");
1767 switch(bt) {
1768 case T_BYTE:
1769 addv(vtmp, isQ ? T16B : T8B, vsrc);
1770 smov(dst, vtmp, B, 0);
1771 addw(dst, dst, isrc, ext::sxtb);
1772 break;
1773 case T_SHORT:
1774 addv(vtmp, isQ ? T8H : T4H, vsrc);
1775 smov(dst, vtmp, H, 0);
1776 addw(dst, dst, isrc, ext::sxth);
1777 break;
1778 case T_INT:
1779 isQ ? addv(vtmp, T4S, vsrc) : addpv(vtmp, T2S, vsrc, vsrc);
1780 umov(dst, vtmp, S, 0);
1781 addw(dst, dst, isrc);
1782 break;
1783 case T_LONG:
1784 assert(isQ, "unsupported");
1785 addpd(vtmp, vsrc);
1786 umov(dst, vtmp, D, 0);
1787 add(dst, dst, isrc);
1788 break;
1789 default:
1790 assert(false, "unsupported");
1791 ShouldNotReachHere();
1792 }
1793 BLOCK_COMMENT("} neon_reduce_add_integral");
1794 }
1795
1796 // Vector reduction multiply for integral type with ASIMD instructions.
1797 // Note: temporary registers vtmp1 and vtmp2 are not used in some cases.
1798 // Clobbers: rscratch1
1799 void C2_MacroAssembler::neon_reduce_mul_integral(Register dst, BasicType bt,
1800 Register isrc, FloatRegister vsrc,
1801 unsigned vector_length_in_bytes,
1802 FloatRegister vtmp1, FloatRegister vtmp2) {
1803 assert(vector_length_in_bytes == 8 || vector_length_in_bytes == 16, "unsupported");
1804 bool isQ = vector_length_in_bytes == 16;
1805
1806 BLOCK_COMMENT("neon_reduce_mul_integral {");
1807 switch(bt) {
1808 case T_BYTE:
1809 if (isQ) {
1810 // Multiply the lower half and higher half of vector iteratively.
1811 // vtmp1 = vsrc[8:15]
1812 ins(vtmp1, D, vsrc, 0, 1);
1813 // vtmp1[n] = vsrc[n] * vsrc[n + 8], where n=[0, 7]
1814 mulv(vtmp1, T8B, vtmp1, vsrc);
1815 // vtmp2 = vtmp1[4:7]
1816 ins(vtmp2, S, vtmp1, 0, 1);
1817 // vtmp1[n] = vtmp1[n] * vtmp1[n + 4], where n=[0, 3]
1818 mulv(vtmp1, T8B, vtmp2, vtmp1);
1819 } else {
1820 ins(vtmp1, S, vsrc, 0, 1);
1821 mulv(vtmp1, T8B, vtmp1, vsrc);
1822 }
1823 // vtmp2 = vtmp1[2:3]
1824 ins(vtmp2, H, vtmp1, 0, 1);
1825 // vtmp2[n] = vtmp1[n] * vtmp1[n + 2], where n=[0, 1]
1826 mulv(vtmp2, T8B, vtmp2, vtmp1);
1827 // dst = vtmp2[0] * isrc * vtmp2[1]
1828 umov(rscratch1, vtmp2, B, 0);
1829 mulw(dst, rscratch1, isrc);
1830 sxtb(dst, dst);
1831 umov(rscratch1, vtmp2, B, 1);
1832 mulw(dst, rscratch1, dst);
1833 sxtb(dst, dst);
1834 break;
1835 case T_SHORT:
1836 if (isQ) {
1837 ins(vtmp2, D, vsrc, 0, 1);
1838 mulv(vtmp2, T4H, vtmp2, vsrc);
1839 ins(vtmp1, S, vtmp2, 0, 1);
1840 mulv(vtmp1, T4H, vtmp1, vtmp2);
1841 } else {
1842 ins(vtmp1, S, vsrc, 0, 1);
1843 mulv(vtmp1, T4H, vtmp1, vsrc);
1844 }
1845 umov(rscratch1, vtmp1, H, 0);
1846 mulw(dst, rscratch1, isrc);
1847 sxth(dst, dst);
1848 umov(rscratch1, vtmp1, H, 1);
1849 mulw(dst, rscratch1, dst);
1850 sxth(dst, dst);
1851 break;
1852 case T_INT:
1853 if (isQ) {
1854 ins(vtmp1, D, vsrc, 0, 1);
1855 mulv(vtmp1, T2S, vtmp1, vsrc);
1856 } else {
1857 vtmp1 = vsrc;
1858 }
1859 umov(rscratch1, vtmp1, S, 0);
1860 mul(dst, rscratch1, isrc);
1861 umov(rscratch1, vtmp1, S, 1);
1862 mul(dst, rscratch1, dst);
1863 break;
1864 case T_LONG:
1865 umov(rscratch1, vsrc, D, 0);
1866 mul(dst, isrc, rscratch1);
1867 umov(rscratch1, vsrc, D, 1);
1868 mul(dst, dst, rscratch1);
1869 break;
1870 default:
1871 assert(false, "unsupported");
1872 ShouldNotReachHere();
1873 }
1874 BLOCK_COMMENT("} neon_reduce_mul_integral");
1875 }
1876
1877 // Vector reduction multiply for floating-point type with ASIMD instructions.
1878 void C2_MacroAssembler::neon_reduce_mul_fp(FloatRegister dst, BasicType bt,
1879 FloatRegister fsrc, FloatRegister vsrc,
1880 unsigned vector_length_in_bytes,
1881 FloatRegister vtmp) {
1882 assert(vector_length_in_bytes == 8 || vector_length_in_bytes == 16, "unsupported");
1883 bool isQ = vector_length_in_bytes == 16;
1884
1885 BLOCK_COMMENT("neon_reduce_mul_fp {");
1886 switch(bt) {
1887 case T_FLOAT:
1888 fmuls(dst, fsrc, vsrc);
1889 ins(vtmp, S, vsrc, 0, 1);
1890 fmuls(dst, dst, vtmp);
1891 if (isQ) {
1892 ins(vtmp, S, vsrc, 0, 2);
1893 fmuls(dst, dst, vtmp);
1894 ins(vtmp, S, vsrc, 0, 3);
1895 fmuls(dst, dst, vtmp);
1896 }
1897 break;
1898 case T_DOUBLE:
1899 assert(isQ, "unsupported");
1900 fmuld(dst, fsrc, vsrc);
1901 ins(vtmp, D, vsrc, 0, 1);
1902 fmuld(dst, dst, vtmp);
1903 break;
1904 default:
1905 assert(false, "unsupported");
1906 ShouldNotReachHere();
1907 }
1908 BLOCK_COMMENT("} neon_reduce_mul_fp");
1909 }
1910
1911 // Helper to select logical instruction
1912 void C2_MacroAssembler::neon_reduce_logical_helper(int opc, bool is64, Register Rd,
1913 Register Rn, Register Rm,
1914 enum shift_kind kind, unsigned shift) {
1915 switch(opc) {
1916 case Op_AndReductionV:
1917 is64 ? andr(Rd, Rn, Rm, kind, shift) : andw(Rd, Rn, Rm, kind, shift);
1918 break;
1919 case Op_OrReductionV:
1920 is64 ? orr(Rd, Rn, Rm, kind, shift) : orrw(Rd, Rn, Rm, kind, shift);
1921 break;
1922 case Op_XorReductionV:
1923 is64 ? eor(Rd, Rn, Rm, kind, shift) : eorw(Rd, Rn, Rm, kind, shift);
1924 break;
1925 default:
1926 assert(false, "unsupported");
1927 ShouldNotReachHere();
1928 }
1929 }
1930
1931 // Vector reduction logical operations And, Or, Xor
1932 // Clobbers: rscratch1
1933 void C2_MacroAssembler::neon_reduce_logical(int opc, Register dst, BasicType bt,
1934 Register isrc, FloatRegister vsrc,
1935 unsigned vector_length_in_bytes) {
1936 assert(opc == Op_AndReductionV || opc == Op_OrReductionV || opc == Op_XorReductionV,
1937 "unsupported");
1938 assert(vector_length_in_bytes == 8 || vector_length_in_bytes == 16, "unsupported");
1939 assert_different_registers(dst, isrc);
1940 bool isQ = vector_length_in_bytes == 16;
1941
1942 BLOCK_COMMENT("neon_reduce_logical {");
1943 umov(rscratch1, vsrc, isQ ? D : S, 0);
1944 umov(dst, vsrc, isQ ? D : S, 1);
1945 neon_reduce_logical_helper(opc, /* is64 */ true, dst, dst, rscratch1);
1946 switch(bt) {
1947 case T_BYTE:
1948 if (isQ) {
1949 neon_reduce_logical_helper(opc, /* is64 */ true, dst, dst, dst, Assembler::LSR, 32);
1950 }
1951 neon_reduce_logical_helper(opc, /* is64 */ false, dst, dst, dst, Assembler::LSR, 16);
1952 neon_reduce_logical_helper(opc, /* is64 */ false, dst, dst, dst, Assembler::LSR, 8);
1953 neon_reduce_logical_helper(opc, /* is64 */ false, dst, isrc, dst);
1954 sxtb(dst, dst);
1955 break;
1956 case T_SHORT:
1957 if (isQ) {
1958 neon_reduce_logical_helper(opc, /* is64 */ true, dst, dst, dst, Assembler::LSR, 32);
1959 }
1960 neon_reduce_logical_helper(opc, /* is64 */ false, dst, dst, dst, Assembler::LSR, 16);
1961 neon_reduce_logical_helper(opc, /* is64 */ false, dst, isrc, dst);
1962 sxth(dst, dst);
1963 break;
1964 case T_INT:
1965 if (isQ) {
1966 neon_reduce_logical_helper(opc, /* is64 */ true, dst, dst, dst, Assembler::LSR, 32);
1967 }
1968 neon_reduce_logical_helper(opc, /* is64 */ false, dst, isrc, dst);
1969 break;
1970 case T_LONG:
1971 assert(isQ, "unsupported");
1972 neon_reduce_logical_helper(opc, /* is64 */ true, dst, isrc, dst);
1973 break;
1974 default:
1975 assert(false, "unsupported");
1976 ShouldNotReachHere();
1977 }
1978 BLOCK_COMMENT("} neon_reduce_logical");
1979 }
1980
1981 // Vector reduction min/max for integral type with ASIMD instructions.
1982 // Note: vtmp is not used and expected to be fnoreg for T_LONG case.
1983 // Clobbers: rscratch1, rflags
1984 void C2_MacroAssembler::neon_reduce_minmax_integral(int opc, Register dst, BasicType bt,
1985 Register isrc, FloatRegister vsrc,
1986 unsigned vector_length_in_bytes,
1987 FloatRegister vtmp) {
1988 assert(opc == Op_MinReductionV || opc == Op_MaxReductionV, "unsupported");
1989 assert(vector_length_in_bytes == 8 || vector_length_in_bytes == 16, "unsupported");
1990 assert(bt == T_BYTE || bt == T_SHORT || bt == T_INT || bt == T_LONG, "unsupported");
1991 assert_different_registers(dst, isrc);
1992 bool isQ = vector_length_in_bytes == 16;
1993 bool is_min = opc == Op_MinReductionV;
1994
1995 BLOCK_COMMENT("neon_reduce_minmax_integral {");
1996 if (bt == T_LONG) {
1997 assert(vtmp == fnoreg, "should be");
1998 assert(isQ, "should be");
1999 umov(rscratch1, vsrc, D, 0);
2000 cmp(isrc, rscratch1);
2001 csel(dst, isrc, rscratch1, is_min ? LT : GT);
2002 umov(rscratch1, vsrc, D, 1);
2003 cmp(dst, rscratch1);
2004 csel(dst, dst, rscratch1, is_min ? LT : GT);
2005 } else {
2006 SIMD_Arrangement size = esize2arrangement((unsigned)type2aelembytes(bt), isQ);
2007 if (size == T2S) {
2008 is_min ? sminp(vtmp, size, vsrc, vsrc) : smaxp(vtmp, size, vsrc, vsrc);
2009 } else {
2010 is_min ? sminv(vtmp, size, vsrc) : smaxv(vtmp, size, vsrc);
2011 }
2012 if (bt == T_INT) {
2013 umov(dst, vtmp, S, 0);
2014 } else {
2015 smov(dst, vtmp, elemType_to_regVariant(bt), 0);
2016 }
2017 cmpw(dst, isrc);
2018 cselw(dst, dst, isrc, is_min ? LT : GT);
2019 }
2020 BLOCK_COMMENT("} neon_reduce_minmax_integral");
2021 }
2022
2023 // Vector reduction for integral type with SVE instruction.
2024 // Supported operations are Add, And, Or, Xor, Max, Min.
2025 // rflags would be clobbered if opc is Op_MaxReductionV or Op_MinReductionV.
2026 void C2_MacroAssembler::sve_reduce_integral(int opc, Register dst, BasicType bt, Register src1,
2027 FloatRegister src2, PRegister pg, FloatRegister tmp) {
2028 assert(bt == T_BYTE || bt == T_SHORT || bt == T_INT || bt == T_LONG, "unsupported element type");
2029 assert(pg->is_governing(), "This register has to be a governing predicate register");
2030 assert_different_registers(src1, dst);
2031 // Register "dst" and "tmp" are to be clobbered, and "src1" and "src2" should be preserved.
2032 Assembler::SIMD_RegVariant size = elemType_to_regVariant(bt);
2033 switch (opc) {
2034 case Op_AddReductionVI: {
2035 sve_uaddv(tmp, size, pg, src2);
2036 if (bt == T_BYTE) {
2037 smov(dst, tmp, size, 0);
2038 addw(dst, src1, dst, ext::sxtb);
2039 } else if (bt == T_SHORT) {
2040 smov(dst, tmp, size, 0);
2041 addw(dst, src1, dst, ext::sxth);
2042 } else {
2043 umov(dst, tmp, size, 0);
2044 addw(dst, dst, src1);
2045 }
2046 break;
2047 }
2048 case Op_AddReductionVL: {
2049 sve_uaddv(tmp, size, pg, src2);
2050 umov(dst, tmp, size, 0);
2051 add(dst, dst, src1);
2052 break;
2053 }
2054 case Op_AndReductionV: {
2055 sve_andv(tmp, size, pg, src2);
2056 if (bt == T_INT || bt == T_LONG) {
2057 umov(dst, tmp, size, 0);
2058 } else {
2059 smov(dst, tmp, size, 0);
2060 }
2061 if (bt == T_LONG) {
2062 andr(dst, dst, src1);
2063 } else {
2064 andw(dst, dst, src1);
2065 }
2066 break;
2067 }
2068 case Op_OrReductionV: {
2069 sve_orv(tmp, size, pg, src2);
2070 if (bt == T_INT || bt == T_LONG) {
2071 umov(dst, tmp, size, 0);
2072 } else {
2073 smov(dst, tmp, size, 0);
2074 }
2075 if (bt == T_LONG) {
2076 orr(dst, dst, src1);
2077 } else {
2078 orrw(dst, dst, src1);
2079 }
2080 break;
2081 }
2082 case Op_XorReductionV: {
2083 sve_eorv(tmp, size, pg, src2);
2084 if (bt == T_INT || bt == T_LONG) {
2085 umov(dst, tmp, size, 0);
2086 } else {
2087 smov(dst, tmp, size, 0);
2088 }
2089 if (bt == T_LONG) {
2090 eor(dst, dst, src1);
2091 } else {
2092 eorw(dst, dst, src1);
2093 }
2094 break;
2095 }
2096 case Op_MaxReductionV: {
2097 sve_smaxv(tmp, size, pg, src2);
2098 if (bt == T_INT || bt == T_LONG) {
2099 umov(dst, tmp, size, 0);
2100 } else {
2101 smov(dst, tmp, size, 0);
2102 }
2103 if (bt == T_LONG) {
2104 cmp(dst, src1);
2105 csel(dst, dst, src1, Assembler::GT);
2106 } else {
2107 cmpw(dst, src1);
2108 cselw(dst, dst, src1, Assembler::GT);
2109 }
2110 break;
2111 }
2112 case Op_MinReductionV: {
2113 sve_sminv(tmp, size, pg, src2);
2114 if (bt == T_INT || bt == T_LONG) {
2115 umov(dst, tmp, size, 0);
2116 } else {
2117 smov(dst, tmp, size, 0);
2118 }
2119 if (bt == T_LONG) {
2120 cmp(dst, src1);
2121 csel(dst, dst, src1, Assembler::LT);
2122 } else {
2123 cmpw(dst, src1);
2124 cselw(dst, dst, src1, Assembler::LT);
2125 }
2126 break;
2127 }
2128 default:
2129 assert(false, "unsupported");
2130 ShouldNotReachHere();
2131 }
2132
2133 if (opc == Op_AndReductionV || opc == Op_OrReductionV || opc == Op_XorReductionV) {
2134 if (bt == T_BYTE) {
2135 sxtb(dst, dst);
2136 } else if (bt == T_SHORT) {
2137 sxth(dst, dst);
2138 }
2139 }
2140 }
2141
2142 // Set elements of the dst predicate to true for lanes in the range of [0, lane_cnt), or
2143 // to false otherwise. The input "lane_cnt" should be smaller than or equal to the supported
2144 // max vector length of the basic type. Clobbers: rscratch1 and the rFlagsReg.
2145 void C2_MacroAssembler::sve_gen_mask_imm(PRegister dst, BasicType bt, uint32_t lane_cnt) {
2146 uint32_t max_vector_length = Matcher::max_vector_size(bt);
2147 assert(lane_cnt <= max_vector_length, "unsupported input lane_cnt");
2148
2149 // Set all elements to false if the input "lane_cnt" is zero.
2150 if (lane_cnt == 0) {
2151 sve_pfalse(dst);
2152 return;
2153 }
2154
2155 SIMD_RegVariant size = elemType_to_regVariant(bt);
2156 assert(size != Q, "invalid size");
2157
2158 // Set all true if "lane_cnt" equals to the max lane count.
2159 if (lane_cnt == max_vector_length) {
2160 sve_ptrue(dst, size, /* ALL */ 0b11111);
2161 return;
2162 }
2163
2164 // Fixed numbers for "ptrue".
2165 switch(lane_cnt) {
2166 case 1: /* VL1 */
2167 case 2: /* VL2 */
2168 case 3: /* VL3 */
2169 case 4: /* VL4 */
2170 case 5: /* VL5 */
2171 case 6: /* VL6 */
2172 case 7: /* VL7 */
2173 case 8: /* VL8 */
2174 sve_ptrue(dst, size, lane_cnt);
2175 return;
2176 case 16:
2177 sve_ptrue(dst, size, /* VL16 */ 0b01001);
2178 return;
2179 case 32:
2180 sve_ptrue(dst, size, /* VL32 */ 0b01010);
2181 return;
2182 case 64:
2183 sve_ptrue(dst, size, /* VL64 */ 0b01011);
2184 return;
2185 case 128:
2186 sve_ptrue(dst, size, /* VL128 */ 0b01100);
2187 return;
2188 case 256:
2189 sve_ptrue(dst, size, /* VL256 */ 0b01101);
2190 return;
2191 default:
2192 break;
2193 }
2194
2195 // Special patterns for "ptrue".
2196 if (lane_cnt == round_down_power_of_2(max_vector_length)) {
2197 sve_ptrue(dst, size, /* POW2 */ 0b00000);
2198 } else if (lane_cnt == max_vector_length - (max_vector_length % 4)) {
2199 sve_ptrue(dst, size, /* MUL4 */ 0b11101);
2200 } else if (lane_cnt == max_vector_length - (max_vector_length % 3)) {
2201 sve_ptrue(dst, size, /* MUL3 */ 0b11110);
2202 } else {
2203 // Encode to "whileltw" for the remaining cases.
2204 mov(rscratch1, lane_cnt);
2205 sve_whileltw(dst, size, zr, rscratch1);
2206 }
2207 }
2208
2209 // Pack active elements of src, under the control of mask, into the lowest-numbered elements of dst.
2210 // Any remaining elements of dst will be filled with zero.
2211 // Clobbers: rscratch1
2212 // Preserves: src, mask
2213 void C2_MacroAssembler::sve_compress_short(FloatRegister dst, FloatRegister src, PRegister mask,
2214 FloatRegister vtmp1, FloatRegister vtmp2,
2215 PRegister pgtmp) {
2216 assert(pgtmp->is_governing(), "This register has to be a governing predicate register");
2217 assert_different_registers(dst, src, vtmp1, vtmp2);
2218 assert_different_registers(mask, pgtmp);
2219
2220 // Example input: src = 8888 7777 6666 5555 4444 3333 2222 1111
2221 // mask = 0001 0000 0000 0001 0001 0000 0001 0001
2222 // Expected result: dst = 0000 0000 0000 8888 5555 4444 2222 1111
2223 sve_dup(vtmp2, H, 0);
2224
2225 // Extend lowest half to type INT.
2226 // dst = 00004444 00003333 00002222 00001111
2227 sve_uunpklo(dst, S, src);
2228 // pgtmp = 00000001 00000000 00000001 00000001
2229 sve_punpklo(pgtmp, mask);
2230 // Pack the active elements in size of type INT to the right,
2231 // and fill the remainings with zero.
2232 // dst = 00000000 00004444 00002222 00001111
2233 sve_compact(dst, S, dst, pgtmp);
2234 // Narrow the result back to type SHORT.
2235 // dst = 0000 0000 0000 0000 0000 4444 2222 1111
2236 sve_uzp1(dst, H, dst, vtmp2);
2237 // Count the active elements of lowest half.
2238 // rscratch1 = 3
2239 sve_cntp(rscratch1, S, ptrue, pgtmp);
2240
2241 // Repeat to the highest half.
2242 // pgtmp = 00000001 00000000 00000000 00000001
2243 sve_punpkhi(pgtmp, mask);
2244 // vtmp1 = 00008888 00007777 00006666 00005555
2245 sve_uunpkhi(vtmp1, S, src);
2246 // vtmp1 = 00000000 00000000 00008888 00005555
2247 sve_compact(vtmp1, S, vtmp1, pgtmp);
2248 // vtmp1 = 0000 0000 0000 0000 0000 0000 8888 5555
2249 sve_uzp1(vtmp1, H, vtmp1, vtmp2);
2250
2251 // Compressed low: dst = 0000 0000 0000 0000 0000 4444 2222 1111
2252 // Compressed high: vtmp1 = 0000 0000 0000 0000 0000 0000 8888 5555
2253 // Left shift(cross lane) compressed high with TRUE_CNT lanes,
2254 // TRUE_CNT is the number of active elements in the compressed low.
2255 neg(rscratch1, rscratch1);
2256 // vtmp2 = {4 3 2 1 0 -1 -2 -3}
2257 sve_index(vtmp2, H, rscratch1, 1);
2258 // vtmp1 = 0000 0000 0000 8888 5555 0000 0000 0000
2259 sve_tbl(vtmp1, H, vtmp1, vtmp2);
2260
2261 // Combine the compressed high(after shifted) with the compressed low.
2262 // dst = 0000 0000 0000 8888 5555 4444 2222 1111
2263 sve_orr(dst, dst, vtmp1);
2264 }
2265
2266 // Clobbers: rscratch1, rscratch2
2267 // Preserves: src, mask
2268 void C2_MacroAssembler::sve_compress_byte(FloatRegister dst, FloatRegister src, PRegister mask,
2269 FloatRegister vtmp1, FloatRegister vtmp2,
2270 FloatRegister vtmp3, FloatRegister vtmp4,
2271 PRegister ptmp, PRegister pgtmp) {
2272 assert(pgtmp->is_governing(), "This register has to be a governing predicate register");
2273 assert_different_registers(dst, src, vtmp1, vtmp2, vtmp3, vtmp4);
2274 assert_different_registers(mask, ptmp, pgtmp);
2275 // Example input: src = 88 77 66 55 44 33 22 11
2276 // mask = 01 00 00 01 01 00 01 01
2277 // Expected result: dst = 00 00 00 88 55 44 22 11
2278
2279 sve_dup(vtmp4, B, 0);
2280 // Extend lowest half to type SHORT.
2281 // vtmp1 = 0044 0033 0022 0011
2282 sve_uunpklo(vtmp1, H, src);
2283 // ptmp = 0001 0000 0001 0001
2284 sve_punpklo(ptmp, mask);
2285 // Count the active elements of lowest half.
2286 // rscratch2 = 3
2287 sve_cntp(rscratch2, H, ptrue, ptmp);
2288 // Pack the active elements in size of type SHORT to the right,
2289 // and fill the remainings with zero.
2290 // dst = 0000 0044 0022 0011
2291 sve_compress_short(dst, vtmp1, ptmp, vtmp2, vtmp3, pgtmp);
2292 // Narrow the result back to type BYTE.
2293 // dst = 00 00 00 00 00 44 22 11
2294 sve_uzp1(dst, B, dst, vtmp4);
2295
2296 // Repeat to the highest half.
2297 // ptmp = 0001 0000 0000 0001
2298 sve_punpkhi(ptmp, mask);
2299 // vtmp1 = 0088 0077 0066 0055
2300 sve_uunpkhi(vtmp2, H, src);
2301 // vtmp1 = 0000 0000 0088 0055
2302 sve_compress_short(vtmp1, vtmp2, ptmp, vtmp3, vtmp4, pgtmp);
2303
2304 sve_dup(vtmp4, B, 0);
2305 // vtmp1 = 00 00 00 00 00 00 88 55
2306 sve_uzp1(vtmp1, B, vtmp1, vtmp4);
2307
2308 // Compressed low: dst = 00 00 00 00 00 44 22 11
2309 // Compressed high: vtmp1 = 00 00 00 00 00 00 88 55
2310 // Left shift(cross lane) compressed high with TRUE_CNT lanes,
2311 // TRUE_CNT is the number of active elements in the compressed low.
2312 neg(rscratch2, rscratch2);
2313 // vtmp2 = {4 3 2 1 0 -1 -2 -3}
2314 sve_index(vtmp2, B, rscratch2, 1);
2315 // vtmp1 = 00 00 00 88 55 00 00 00
2316 sve_tbl(vtmp1, B, vtmp1, vtmp2);
2317 // Combine the compressed high(after shifted) with the compressed low.
2318 // dst = 00 00 00 88 55 44 22 11
2319 sve_orr(dst, dst, vtmp1);
2320 }
2321
2322 void C2_MacroAssembler::neon_reverse_bits(FloatRegister dst, FloatRegister src, BasicType bt, bool isQ) {
2323 assert(bt == T_BYTE || bt == T_SHORT || bt == T_INT || bt == T_LONG, "unsupported basic type");
2324 SIMD_Arrangement size = isQ ? T16B : T8B;
2325 if (bt == T_BYTE) {
2326 rbit(dst, size, src);
2327 } else {
2328 neon_reverse_bytes(dst, src, bt, isQ);
2329 rbit(dst, size, dst);
2330 }
2331 }
2332
2333 void C2_MacroAssembler::neon_reverse_bytes(FloatRegister dst, FloatRegister src, BasicType bt, bool isQ) {
2334 assert(bt == T_BYTE || bt == T_SHORT || bt == T_INT || bt == T_LONG, "unsupported basic type");
2335 SIMD_Arrangement size = isQ ? T16B : T8B;
2336 switch (bt) {
2337 case T_BYTE:
2338 if (dst != src) {
2339 orr(dst, size, src, src);
2340 }
2341 break;
2342 case T_SHORT:
2343 rev16(dst, size, src);
2344 break;
2345 case T_INT:
2346 rev32(dst, size, src);
2347 break;
2348 case T_LONG:
2349 rev64(dst, size, src);
2350 break;
2351 default:
2352 assert(false, "unsupported");
2353 ShouldNotReachHere();
2354 }
2355 }
2356
2357 // Extract a scalar element from an sve vector at position 'idx'.
2358 // The input elements in src are expected to be of integral type.
2359 void C2_MacroAssembler::sve_extract_integral(Register dst, BasicType bt, FloatRegister src,
2360 int idx, FloatRegister vtmp) {
2361 assert(bt == T_BYTE || bt == T_SHORT || bt == T_INT || bt == T_LONG, "unsupported element type");
2362 Assembler::SIMD_RegVariant size = elemType_to_regVariant(bt);
2363 if (regVariant_to_elemBits(size) * idx < 128) { // generate lower cost NEON instruction
2364 if (bt == T_INT || bt == T_LONG) {
2365 umov(dst, src, size, idx);
2366 } else {
2367 smov(dst, src, size, idx);
2368 }
2369 } else {
2370 sve_orr(vtmp, src, src);
2371 sve_ext(vtmp, vtmp, idx << size);
2372 if (bt == T_INT || bt == T_LONG) {
2373 umov(dst, vtmp, size, 0);
2374 } else {
2375 smov(dst, vtmp, size, 0);
2376 }
2377 }
2378 }
2379
2380 // java.lang.Math::round intrinsics
2381
2382 // Clobbers: rscratch1, rflags
2383 void C2_MacroAssembler::vector_round_neon(FloatRegister dst, FloatRegister src, FloatRegister tmp1,
2384 FloatRegister tmp2, FloatRegister tmp3, SIMD_Arrangement T) {
2385 assert_different_registers(tmp1, tmp2, tmp3, src, dst);
2386 switch (T) {
2387 case T2S:
2388 case T4S:
2389 fmovs(tmp1, T, 0.5f);
2390 mov(rscratch1, jint_cast(0x1.0p23f));
2391 break;
2392 case T2D:
2393 fmovd(tmp1, T, 0.5);
2394 mov(rscratch1, julong_cast(0x1.0p52));
2395 break;
2396 default:
2397 assert(T == T2S || T == T4S || T == T2D, "invalid arrangement");
2398 }
2399 fadd(tmp1, T, tmp1, src);
2400 fcvtms(tmp1, T, tmp1);
2401 // tmp1 = floor(src + 0.5, ties to even)
2402
2403 fcvtas(dst, T, src);
2404 // dst = round(src), ties to away
2405
2406 fneg(tmp3, T, src);
2407 dup(tmp2, T, rscratch1);
2408 cm(HS, tmp3, T, tmp3, tmp2);
2409 // tmp3 is now a set of flags
2410
2411 bif(dst, T16B, tmp1, tmp3);
2412 // result in dst
2413 }
2414
2415 // Clobbers: rscratch1, rflags
2416 void C2_MacroAssembler::vector_round_sve(FloatRegister dst, FloatRegister src, FloatRegister tmp1,
2417 FloatRegister tmp2, PRegister pgtmp, SIMD_RegVariant T) {
2418 assert(pgtmp->is_governing(), "This register has to be a governing predicate register");
2419 assert_different_registers(tmp1, tmp2, src, dst);
2420
2421 switch (T) {
2422 case S:
2423 mov(rscratch1, jint_cast(0x1.0p23f));
2424 break;
2425 case D:
2426 mov(rscratch1, julong_cast(0x1.0p52));
2427 break;
2428 default:
2429 assert(T == S || T == D, "invalid register variant");
2430 }
2431
2432 sve_frinta(dst, T, ptrue, src);
2433 // dst = round(src), ties to away
2434
2435 Label none;
2436
2437 sve_fneg(tmp1, T, ptrue, src);
2438 sve_dup(tmp2, T, rscratch1);
2439 sve_cmp(HS, pgtmp, T, ptrue, tmp2, tmp1);
2440 br(EQ, none);
2441 {
2442 sve_cpy(tmp1, T, pgtmp, 0.5);
2443 sve_fadd(tmp1, T, pgtmp, src);
2444 sve_frintm(dst, T, pgtmp, tmp1);
2445 // dst = floor(src + 0.5, ties to even)
2446 }
2447 bind(none);
2448
2449 sve_fcvtzs(dst, T, ptrue, dst, T);
2450 // result in dst
2451 }
2452
2453 void C2_MacroAssembler::vector_signum_neon(FloatRegister dst, FloatRegister src, FloatRegister zero,
2454 FloatRegister one, SIMD_Arrangement T) {
2455 assert_different_registers(dst, src, zero, one);
2456 assert(T == T2S || T == T4S || T == T2D, "invalid arrangement");
2457
2458 facgt(dst, T, src, zero);
2459 ushr(dst, T, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
2460 bsl(dst, T == T2S ? T8B : T16B, one, src); // Result in dst
2461 }
2462
2463 void C2_MacroAssembler::vector_signum_sve(FloatRegister dst, FloatRegister src, FloatRegister zero,
2464 FloatRegister one, FloatRegister vtmp, PRegister pgtmp, SIMD_RegVariant T) {
2465 assert_different_registers(dst, src, zero, one, vtmp);
2466 assert(pgtmp->is_governing(), "This register has to be a governing predicate register");
2467
2468 sve_orr(vtmp, src, src);
2469 sve_fac(Assembler::GT, pgtmp, T, ptrue, src, zero); // pmtp=0 for +-0.0 and NaN. 0x1 otherwise
2470 switch (T) {
2471 case S:
2472 sve_and(vtmp, T, min_jint); // Extract the sign bit of float value in every lane of src
2473 sve_orr(vtmp, T, jint_cast(1.0)); // OR it with +1 to make the final result +1 or -1 depending
2474 // on the sign of the float value
2475 break;
2476 case D:
2477 sve_and(vtmp, T, min_jlong);
2478 sve_orr(vtmp, T, jlong_cast(1.0));
2479 break;
2480 default:
2481 assert(false, "unsupported");
2482 ShouldNotReachHere();
2483 }
2484 sve_sel(dst, T, pgtmp, vtmp, src); // Select either from src or vtmp based on the predicate register pgtmp
2485 // Result in dst
2486 }
2487
2488 bool C2_MacroAssembler::in_scratch_emit_size() {
2489 if (ciEnv::current()->task() != nullptr) {
2490 PhaseOutput* phase_output = Compile::current()->output();
2491 if (phase_output != nullptr && phase_output->in_scratch_emit_size()) {
2492 return true;
2493 }
2494 }
2495 return MacroAssembler::in_scratch_emit_size();
2496 }