1 /*
2 * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/assembler.hpp"
27 #include "asm/assembler.inline.hpp"
28 #include "opto/c2_MacroAssembler.hpp"
29 #include "opto/compile.hpp"
30 #include "opto/intrinsicnode.hpp"
31 #include "opto/matcher.hpp"
32 #include "opto/output.hpp"
33 #include "opto/subnode.hpp"
34 #include "runtime/stubRoutines.hpp"
35 #include "utilities/globalDefinitions.hpp"
36
37 #ifdef PRODUCT
38 #define BLOCK_COMMENT(str) /* nothing */
39 #define STOP(error) stop(error)
40 #else
41 #define BLOCK_COMMENT(str) block_comment(str)
42 #define STOP(error) block_comment(error); stop(error)
43 #endif
44
45 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
46
47 typedef void (MacroAssembler::* chr_insn)(Register Rt, const Address &adr);
48
49 void C2_MacroAssembler::entry_barrier() {
50 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
51 if (BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
52 // Dummy labels for just measuring the code size
53 Label dummy_slow_path;
54 Label dummy_continuation;
55 Label dummy_guard;
56 Label* slow_path = &dummy_slow_path;
57 Label* continuation = &dummy_continuation;
58 Label* guard = &dummy_guard;
59 if (!Compile::current()->output()->in_scratch_emit_size()) {
60 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
61 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
62 Compile::current()->output()->add_stub(stub);
63 slow_path = &stub->entry();
64 continuation = &stub->continuation();
65 guard = &stub->guard();
66 }
67 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
68 bs->nmethod_entry_barrier(this, slow_path, continuation, guard);
69 }
70 }
71
72 void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg, Register tmpReg,
73 Register tmp2Reg, Register tmp3Reg) {
74 Register oop = objectReg;
75 Register box = boxReg;
76 Register disp_hdr = tmpReg;
77 Register tmp = tmp2Reg;
78 Label cont;
79 Label object_has_monitor;
80 Label count, no_count;
81
82 assert(LockingMode != LM_LIGHTWEIGHT, "lightweight locking should use fast_lock_lightweight");
83 assert_different_registers(oop, box, tmp, disp_hdr);
84
85 // Load markWord from object into displaced_header.
86 ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
87
88 if (DiagnoseSyncOnValueBasedClasses != 0) {
89 load_klass(tmp, oop);
90 ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
91 tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
92 br(Assembler::NE, cont);
93 }
94
95 // Check for existing monitor
96 tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
97
98 if (LockingMode == LM_MONITOR) {
99 tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
100 b(cont);
101 } else {
102 assert(LockingMode == LM_LEGACY, "must be");
103 // Set tmp to be (markWord of object | UNLOCK_VALUE).
104 orr(tmp, disp_hdr, markWord::unlocked_value);
105
106 if (EnableValhalla) {
107 // Mask inline_type bit such that we go to the slow path if object is an inline type
108 andr(tmp, tmp, ~((int) markWord::inline_type_bit_in_place));
109 }
110
111 // Initialize the box. (Must happen before we update the object mark!)
112 str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
113
114 // Compare object markWord with an unlocked value (tmp) and if
115 // equal exchange the stack address of our box with object markWord.
116 // On failure disp_hdr contains the possibly locked markWord.
117 cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
118 /*release*/ true, /*weak*/ false, disp_hdr);
119 br(Assembler::EQ, cont);
120
121 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
122
123 // If the compare-and-exchange succeeded, then we found an unlocked
124 // object, will have now locked it will continue at label cont
125
126 // Check if the owner is self by comparing the value in the
127 // markWord of object (disp_hdr) with the stack pointer.
128 mov(rscratch1, sp);
129 sub(disp_hdr, disp_hdr, rscratch1);
130 mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
131 // If condition is true we are cont and hence we can store 0 as the
132 // displaced header in the box, which indicates that it is a recursive lock.
133 ands(tmp/*==0?*/, disp_hdr, tmp); // Sets flags for result
134 str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
135 b(cont);
136 }
137
138 // Handle existing monitor.
139 bind(object_has_monitor);
140
141 // The object's monitor m is unlocked iff m->owner == nullptr,
142 // otherwise m->owner may contain a thread or a stack address.
143 //
144 // Try to CAS m->owner from null to current thread.
145 add(tmp, disp_hdr, (in_bytes(ObjectMonitor::owner_offset())-markWord::monitor_value));
146 cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
147 /*release*/ true, /*weak*/ false, tmp3Reg); // Sets flags for result
148
149 // Store a non-null value into the box to avoid looking like a re-entrant
150 // lock. The fast-path monitor unlock code checks for
151 // markWord::monitor_value so use markWord::unused_mark which has the
152 // relevant bit set, and also matches ObjectSynchronizer::enter.
153 mov(tmp, (address)markWord::unused_mark().value());
154 str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
155
156 br(Assembler::EQ, cont); // CAS success means locking succeeded
157
158 cmp(tmp3Reg, rthread);
159 br(Assembler::NE, cont); // Check for recursive locking
160
161 // Recursive lock case
162 increment(Address(disp_hdr, in_bytes(ObjectMonitor::recursions_offset()) - markWord::monitor_value), 1);
163 // flag == EQ still from the cmp above, checking if this is a reentrant lock
164
165 bind(cont);
166 // flag == EQ indicates success
167 // flag == NE indicates failure
168 br(Assembler::NE, no_count);
169
170 bind(count);
171 increment(Address(rthread, JavaThread::held_monitor_count_offset()));
172
173 bind(no_count);
174 }
175
176 void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg, Register tmpReg,
177 Register tmp2Reg) {
178 Register oop = objectReg;
179 Register box = boxReg;
180 Register disp_hdr = tmpReg;
181 Register tmp = tmp2Reg;
182 Label cont;
183 Label object_has_monitor;
184 Label count, no_count;
185
186 assert(LockingMode != LM_LIGHTWEIGHT, "lightweight locking should use fast_unlock_lightweight");
187 assert_different_registers(oop, box, tmp, disp_hdr);
188
189 if (LockingMode == LM_LEGACY) {
190 // Find the lock address and load the displaced header from the stack.
191 ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
192
193 // If the displaced header is 0, we have a recursive unlock.
194 cmp(disp_hdr, zr);
195 br(Assembler::EQ, cont);
196 }
197
198 // Handle existing monitor.
199 ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
200 tbnz(tmp, exact_log2(markWord::monitor_value), object_has_monitor);
201
202 if (LockingMode == LM_MONITOR) {
203 tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
204 b(cont);
205 } else {
206 assert(LockingMode == LM_LEGACY, "must be");
207 // Check if it is still a light weight lock, this is is true if we
208 // see the stack address of the basicLock in the markWord of the
209 // object.
210
211 cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
212 /*release*/ true, /*weak*/ false, tmp);
213 b(cont);
214 }
215
216 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
217
218 // Handle existing monitor.
219 bind(object_has_monitor);
220 STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
221 add(tmp, tmp, -(int)markWord::monitor_value); // monitor
222
223 ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
224
225 Label notRecursive;
226 cbz(disp_hdr, notRecursive);
227
228 // Recursive lock
229 sub(disp_hdr, disp_hdr, 1u);
230 str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
231 cmp(disp_hdr, disp_hdr); // Sets flags for result
232 b(cont);
233
234 bind(notRecursive);
235 ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset()));
236 ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset()));
237 orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
238 cmp(rscratch1, zr); // Sets flags for result
239 cbnz(rscratch1, cont);
240 // need a release store here
241 lea(tmp, Address(tmp, ObjectMonitor::owner_offset()));
242 stlr(zr, tmp); // set unowned
243
244 bind(cont);
245 // flag == EQ indicates success
246 // flag == NE indicates failure
247 br(Assembler::NE, no_count);
248
249 bind(count);
250 decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
251
252 bind(no_count);
253 }
254
255 void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register t1,
256 Register t2, Register t3) {
257 assert(LockingMode == LM_LIGHTWEIGHT, "must be");
258 assert_different_registers(obj, t1, t2, t3);
259
260 // Handle inflated monitor.
261 Label inflated;
262 // Finish fast lock successfully. MUST branch to with flag == EQ
263 Label locked;
264 // Finish fast lock unsuccessfully. MUST branch to with flag == NE
265 Label slow_path;
266
267 if (DiagnoseSyncOnValueBasedClasses != 0) {
268 load_klass(t1, obj);
269 ldrw(t1, Address(t1, Klass::access_flags_offset()));
270 tstw(t1, JVM_ACC_IS_VALUE_BASED_CLASS);
271 br(Assembler::NE, slow_path);
272 }
273
274 const Register t1_mark = t1;
275
276 { // Lightweight locking
277
278 // Push lock to the lock stack and finish successfully. MUST branch to with flag == EQ
279 Label push;
280
281 const Register t2_top = t2;
282 const Register t3_t = t3;
283
284 // Check if lock-stack is full.
285 ldrw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
286 cmpw(t2_top, (unsigned)LockStack::end_offset() - 1);
287 br(Assembler::GT, slow_path);
288
289 // Check if recursive.
290 subw(t3_t, t2_top, oopSize);
291 ldr(t3_t, Address(rthread, t3_t));
292 cmp(obj, t3_t);
293 br(Assembler::EQ, push);
294
295 // Relaxed normal load to check for monitor. Optimization for monitor case.
296 ldr(t1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
297 tbnz(t1_mark, exact_log2(markWord::monitor_value), inflated);
298
299 // Not inflated
300 assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid a lea");
301
302 // Try to lock. Transition lock-bits 0b01 => 0b00
303 orr(t1_mark, t1_mark, markWord::unlocked_value);
304 eor(t3_t, t1_mark, markWord::unlocked_value);
305 cmpxchg(/*addr*/ obj, /*expected*/ t1_mark, /*new*/ t3_t, Assembler::xword,
306 /*acquire*/ true, /*release*/ false, /*weak*/ false, noreg);
307 br(Assembler::NE, slow_path);
308
309 bind(push);
310 // After successful lock, push object on lock-stack.
311 str(obj, Address(rthread, t2_top));
312 addw(t2_top, t2_top, oopSize);
313 strw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
314 b(locked);
315 }
316
317 { // Handle inflated monitor.
318 bind(inflated);
319
320 // mark contains the tagged ObjectMonitor*.
321 const Register t1_tagged_monitor = t1_mark;
322 const uintptr_t monitor_tag = markWord::monitor_value;
323 const Register t2_owner_addr = t2;
324 const Register t3_owner = t3;
325
326 // Compute owner address.
327 lea(t2_owner_addr, Address(t1_tagged_monitor, (in_bytes(ObjectMonitor::owner_offset()) - monitor_tag)));
328
329 // CAS owner (null => current thread).
330 cmpxchg(t2_owner_addr, zr, rthread, Assembler::xword, /*acquire*/ true,
331 /*release*/ false, /*weak*/ false, t3_owner);
332 br(Assembler::EQ, locked);
333
334 // Check if recursive.
335 cmp(t3_owner, rthread);
336 br(Assembler::NE, slow_path);
337
338 // Recursive.
339 increment(Address(t1_tagged_monitor, in_bytes(ObjectMonitor::recursions_offset()) - monitor_tag), 1);
340 }
341
342 bind(locked);
343 increment(Address(rthread, JavaThread::held_monitor_count_offset()));
344
345 #ifdef ASSERT
346 // Check that locked label is reached with Flags == EQ.
347 Label flag_correct;
348 br(Assembler::EQ, flag_correct);
349 stop("Fast Lock Flag != EQ");
350 #endif
351
352 bind(slow_path);
353 #ifdef ASSERT
354 // Check that slow_path label is reached with Flags == NE.
355 br(Assembler::NE, flag_correct);
356 stop("Fast Lock Flag != NE");
357 bind(flag_correct);
358 #endif
359 // C2 uses the value of Flags (NE vs EQ) to determine the continuation.
360 }
361
362 void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register t1, Register t2,
363 Register t3) {
364 assert(LockingMode == LM_LIGHTWEIGHT, "must be");
365 assert_different_registers(obj, t1, t2, t3);
366
367 // Handle inflated monitor.
368 Label inflated, inflated_load_monitor;
369 // Finish fast unlock successfully. MUST branch to with flag == EQ
370 Label unlocked;
371 // Finish fast unlock unsuccessfully. MUST branch to with flag == NE
372 Label slow_path;
373
374 const Register t1_mark = t1;
375 const Register t2_top = t2;
376 const Register t3_t = t3;
377
378 { // Lightweight unlock
379
380 // Check if obj is top of lock-stack.
381 ldrw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
382 subw(t2_top, t2_top, oopSize);
383 ldr(t3_t, Address(rthread, t2_top));
384 cmp(obj, t3_t);
385 // Top of lock stack was not obj. Must be monitor.
386 br(Assembler::NE, inflated_load_monitor);
387
388 // Pop lock-stack.
389 DEBUG_ONLY(str(zr, Address(rthread, t2_top));)
390 strw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
391
392 // Check if recursive.
393 subw(t3_t, t2_top, oopSize);
394 ldr(t3_t, Address(rthread, t3_t));
395 cmp(obj, t3_t);
396 br(Assembler::EQ, unlocked);
397
398 // Not recursive.
399 // Load Mark.
400 ldr(t1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
401
402 // Check header for monitor (0b10).
403 tbnz(t1_mark, exact_log2(markWord::monitor_value), inflated);
404
405 // Try to unlock. Transition lock bits 0b00 => 0b01
406 assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
407 orr(t3_t, t1_mark, markWord::unlocked_value);
408 cmpxchg(/*addr*/ obj, /*expected*/ t1_mark, /*new*/ t3_t, Assembler::xword,
409 /*acquire*/ false, /*release*/ true, /*weak*/ false, noreg);
410 br(Assembler::EQ, unlocked);
411
412 // Compare and exchange failed.
413 // Restore lock-stack and handle the unlock in runtime.
414 DEBUG_ONLY(str(obj, Address(rthread, t2_top));)
415 addw(t2_top, t2_top, oopSize);
416 str(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
417 b(slow_path);
418 }
419
420
421 { // Handle inflated monitor.
422 bind(inflated_load_monitor);
423 ldr(t1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
424 #ifdef ASSERT
425 tbnz(t1_mark, exact_log2(markWord::monitor_value), inflated);
426 stop("Fast Unlock not monitor");
427 #endif
428
429 bind(inflated);
430
431 #ifdef ASSERT
432 Label check_done;
433 subw(t2_top, t2_top, oopSize);
434 cmpw(t2_top, in_bytes(JavaThread::lock_stack_base_offset()));
435 br(Assembler::LT, check_done);
436 ldr(t3_t, Address(rthread, t2_top));
437 cmp(obj, t3_t);
438 br(Assembler::NE, inflated);
439 stop("Fast Unlock lock on stack");
440 bind(check_done);
441 #endif
442
443 // mark contains the tagged ObjectMonitor*.
444 const Register t1_monitor = t1_mark;
445 const uintptr_t monitor_tag = markWord::monitor_value;
446
447 // Untag the monitor.
448 sub(t1_monitor, t1_mark, monitor_tag);
449
450 const Register t2_recursions = t2;
451 Label not_recursive;
452
453 // Check if recursive.
454 ldr(t2_recursions, Address(t1_monitor, ObjectMonitor::recursions_offset()));
455 cbz(t2_recursions, not_recursive);
456
457 // Recursive unlock.
458 sub(t2_recursions, t2_recursions, 1u);
459 str(t2_recursions, Address(t1_monitor, ObjectMonitor::recursions_offset()));
460 // Set flag == EQ
461 cmp(t2_recursions, t2_recursions);
462 b(unlocked);
463
464 bind(not_recursive);
465
466 Label release;
467 const Register t2_owner_addr = t2;
468
469 // Compute owner address.
470 lea(t2_owner_addr, Address(t1_monitor, ObjectMonitor::owner_offset()));
471
472 // Check if the entry lists are empty.
473 ldr(rscratch1, Address(t1_monitor, ObjectMonitor::EntryList_offset()));
474 ldr(t3_t, Address(t1_monitor, ObjectMonitor::cxq_offset()));
475 orr(rscratch1, rscratch1, t3_t);
476 cmp(rscratch1, zr);
477 br(Assembler::EQ, release);
478
479 // The owner may be anonymous and we removed the last obj entry in
480 // the lock-stack. This loses the information about the owner.
481 // Write the thread to the owner field so the runtime knows the owner.
482 str(rthread, Address(t2_owner_addr));
483 b(slow_path);
484
485 bind(release);
486 // Set owner to null.
487 // Release to satisfy the JMM
488 stlr(zr, t2_owner_addr);
489 }
490
491 bind(unlocked);
492 decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
493
494 #ifdef ASSERT
495 // Check that unlocked label is reached with Flags == EQ.
496 Label flag_correct;
497 br(Assembler::EQ, flag_correct);
498 stop("Fast Unlock Flag != EQ");
499 #endif
500
501 bind(slow_path);
502 #ifdef ASSERT
503 // Check that slow_path label is reached with Flags == NE.
504 br(Assembler::NE, flag_correct);
505 stop("Fast Unlock Flag != NE");
506 bind(flag_correct);
507 #endif
508 // C2 uses the value of Flags (NE vs EQ) to determine the continuation.
509 }
510
511 // Search for str1 in str2 and return index or -1
512 // Clobbers: rscratch1, rscratch2, rflags. May also clobber v0-v1, when icnt1==-1.
513 void C2_MacroAssembler::string_indexof(Register str2, Register str1,
514 Register cnt2, Register cnt1,
515 Register tmp1, Register tmp2,
516 Register tmp3, Register tmp4,
517 Register tmp5, Register tmp6,
518 int icnt1, Register result, int ae) {
519 // NOTE: tmp5, tmp6 can be zr depending on specific method version
520 Label LINEARSEARCH, LINEARSTUB, LINEAR_MEDIUM, DONE, NOMATCH, MATCH;
521
522 Register ch1 = rscratch1;
523 Register ch2 = rscratch2;
524 Register cnt1tmp = tmp1;
525 Register cnt2tmp = tmp2;
526 Register cnt1_neg = cnt1;
527 Register cnt2_neg = cnt2;
528 Register result_tmp = tmp4;
529
530 bool isL = ae == StrIntrinsicNode::LL;
531
532 bool str1_isL = ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UL;
533 bool str2_isL = ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::LU;
534 int str1_chr_shift = str1_isL ? 0:1;
535 int str2_chr_shift = str2_isL ? 0:1;
536 int str1_chr_size = str1_isL ? 1:2;
537 int str2_chr_size = str2_isL ? 1:2;
538 chr_insn str1_load_1chr = str1_isL ? (chr_insn)&MacroAssembler::ldrb :
539 (chr_insn)&MacroAssembler::ldrh;
540 chr_insn str2_load_1chr = str2_isL ? (chr_insn)&MacroAssembler::ldrb :
541 (chr_insn)&MacroAssembler::ldrh;
542 chr_insn load_2chr = isL ? (chr_insn)&MacroAssembler::ldrh : (chr_insn)&MacroAssembler::ldrw;
543 chr_insn load_4chr = isL ? (chr_insn)&MacroAssembler::ldrw : (chr_insn)&MacroAssembler::ldr;
544
545 // Note, inline_string_indexOf() generates checks:
546 // if (substr.count > string.count) return -1;
547 // if (substr.count == 0) return 0;
548
549 // We have two strings, a source string in str2, cnt2 and a pattern string
550 // in str1, cnt1. Find the 1st occurrence of pattern in source or return -1.
551
552 // For larger pattern and source we use a simplified Boyer Moore algorithm.
553 // With a small pattern and source we use linear scan.
554
555 if (icnt1 == -1) {
556 sub(result_tmp, cnt2, cnt1);
557 cmp(cnt1, (u1)8); // Use Linear Scan if cnt1 < 8 || cnt1 >= 256
558 br(LT, LINEARSEARCH);
559 dup(v0, T16B, cnt1); // done in separate FPU pipeline. Almost no penalty
560 subs(zr, cnt1, 256);
561 lsr(tmp1, cnt2, 2);
562 ccmp(cnt1, tmp1, 0b0000, LT); // Source must be 4 * pattern for BM
563 br(GE, LINEARSTUB);
564 }
565
566 // The Boyer Moore alogorithm is based on the description here:-
567 //
568 // http://en.wikipedia.org/wiki/Boyer%E2%80%93Moore_string_search_algorithm
569 //
570 // This describes and algorithm with 2 shift rules. The 'Bad Character' rule
571 // and the 'Good Suffix' rule.
572 //
573 // These rules are essentially heuristics for how far we can shift the
574 // pattern along the search string.
575 //
576 // The implementation here uses the 'Bad Character' rule only because of the
577 // complexity of initialisation for the 'Good Suffix' rule.
578 //
579 // This is also known as the Boyer-Moore-Horspool algorithm:-
580 //
581 // http://en.wikipedia.org/wiki/Boyer-Moore-Horspool_algorithm
582 //
583 // This particular implementation has few java-specific optimizations.
584 //
585 // #define ASIZE 256
586 //
587 // int bm(unsigned char *x, int m, unsigned char *y, int n) {
588 // int i, j;
589 // unsigned c;
590 // unsigned char bc[ASIZE];
591 //
592 // /* Preprocessing */
593 // for (i = 0; i < ASIZE; ++i)
594 // bc[i] = m;
595 // for (i = 0; i < m - 1; ) {
596 // c = x[i];
597 // ++i;
598 // // c < 256 for Latin1 string, so, no need for branch
599 // #ifdef PATTERN_STRING_IS_LATIN1
600 // bc[c] = m - i;
601 // #else
602 // if (c < ASIZE) bc[c] = m - i;
603 // #endif
604 // }
605 //
606 // /* Searching */
607 // j = 0;
608 // while (j <= n - m) {
609 // c = y[i+j];
610 // if (x[m-1] == c)
611 // for (i = m - 2; i >= 0 && x[i] == y[i + j]; --i);
612 // if (i < 0) return j;
613 // // c < 256 for Latin1 string, so, no need for branch
614 // #ifdef SOURCE_STRING_IS_LATIN1
615 // // LL case: (c< 256) always true. Remove branch
616 // j += bc[y[j+m-1]];
617 // #endif
618 // #ifndef PATTERN_STRING_IS_UTF
619 // // UU case: need if (c<ASIZE) check. Skip 1 character if not.
620 // if (c < ASIZE)
621 // j += bc[y[j+m-1]];
622 // else
623 // j += 1
624 // #endif
625 // #ifdef PATTERN_IS_LATIN1_AND_SOURCE_IS_UTF
626 // // UL case: need if (c<ASIZE) check. Skip <pattern length> if not.
627 // if (c < ASIZE)
628 // j += bc[y[j+m-1]];
629 // else
630 // j += m
631 // #endif
632 // }
633 // }
634
635 if (icnt1 == -1) {
636 Label BCLOOP, BCSKIP, BMLOOPSTR2, BMLOOPSTR1, BMSKIP, BMADV, BMMATCH,
637 BMLOOPSTR1_LASTCMP, BMLOOPSTR1_CMP, BMLOOPSTR1_AFTER_LOAD, BM_INIT_LOOP;
638 Register cnt1end = tmp2;
639 Register str2end = cnt2;
640 Register skipch = tmp2;
641
642 // str1 length is >=8, so, we can read at least 1 register for cases when
643 // UTF->Latin1 conversion is not needed(8 LL or 4UU) and half register for
644 // UL case. We'll re-read last character in inner pre-loop code to have
645 // single outer pre-loop load
646 const int firstStep = isL ? 7 : 3;
647
648 const int ASIZE = 256;
649 const int STORED_BYTES = 32; // amount of bytes stored per instruction
650 sub(sp, sp, ASIZE);
651 mov(tmp5, ASIZE/STORED_BYTES); // loop iterations
652 mov(ch1, sp);
653 BIND(BM_INIT_LOOP);
654 stpq(v0, v0, Address(post(ch1, STORED_BYTES)));
655 subs(tmp5, tmp5, 1);
656 br(GT, BM_INIT_LOOP);
657
658 sub(cnt1tmp, cnt1, 1);
659 mov(tmp5, str2);
660 add(str2end, str2, result_tmp, LSL, str2_chr_shift);
661 sub(ch2, cnt1, 1);
662 mov(tmp3, str1);
663 BIND(BCLOOP);
664 (this->*str1_load_1chr)(ch1, Address(post(tmp3, str1_chr_size)));
665 if (!str1_isL) {
666 subs(zr, ch1, ASIZE);
667 br(HS, BCSKIP);
668 }
669 strb(ch2, Address(sp, ch1));
670 BIND(BCSKIP);
671 subs(ch2, ch2, 1);
672 br(GT, BCLOOP);
673
674 add(tmp6, str1, cnt1, LSL, str1_chr_shift); // address after str1
675 if (str1_isL == str2_isL) {
676 // load last 8 bytes (8LL/4UU symbols)
677 ldr(tmp6, Address(tmp6, -wordSize));
678 } else {
679 ldrw(tmp6, Address(tmp6, -wordSize/2)); // load last 4 bytes(4 symbols)
680 // convert Latin1 to UTF. We'll have to wait until load completed, but
681 // it's still faster than per-character loads+checks
682 lsr(tmp3, tmp6, BitsPerByte * (wordSize/2 - str1_chr_size)); // str1[N-1]
683 ubfx(ch1, tmp6, 8, 8); // str1[N-2]
684 ubfx(ch2, tmp6, 16, 8); // str1[N-3]
685 andr(tmp6, tmp6, 0xFF); // str1[N-4]
686 orr(ch2, ch1, ch2, LSL, 16);
687 orr(tmp6, tmp6, tmp3, LSL, 48);
688 orr(tmp6, tmp6, ch2, LSL, 16);
689 }
690 BIND(BMLOOPSTR2);
691 (this->*str2_load_1chr)(skipch, Address(str2, cnt1tmp, Address::lsl(str2_chr_shift)));
692 sub(cnt1tmp, cnt1tmp, firstStep); // cnt1tmp is positive here, because cnt1 >= 8
693 if (str1_isL == str2_isL) {
694 // re-init tmp3. It's for free because it's executed in parallel with
695 // load above. Alternative is to initialize it before loop, but it'll
696 // affect performance on in-order systems with 2 or more ld/st pipelines
697 lsr(tmp3, tmp6, BitsPerByte * (wordSize - str1_chr_size));
698 }
699 if (!isL) { // UU/UL case
700 lsl(ch2, cnt1tmp, 1); // offset in bytes
701 }
702 cmp(tmp3, skipch);
703 br(NE, BMSKIP);
704 ldr(ch2, Address(str2, isL ? cnt1tmp : ch2));
705 mov(ch1, tmp6);
706 if (isL) {
707 b(BMLOOPSTR1_AFTER_LOAD);
708 } else {
709 sub(cnt1tmp, cnt1tmp, 1); // no need to branch for UU/UL case. cnt1 >= 8
710 b(BMLOOPSTR1_CMP);
711 }
712 BIND(BMLOOPSTR1);
713 (this->*str1_load_1chr)(ch1, Address(str1, cnt1tmp, Address::lsl(str1_chr_shift)));
714 (this->*str2_load_1chr)(ch2, Address(str2, cnt1tmp, Address::lsl(str2_chr_shift)));
715 BIND(BMLOOPSTR1_AFTER_LOAD);
716 subs(cnt1tmp, cnt1tmp, 1);
717 br(LT, BMLOOPSTR1_LASTCMP);
718 BIND(BMLOOPSTR1_CMP);
719 cmp(ch1, ch2);
720 br(EQ, BMLOOPSTR1);
721 BIND(BMSKIP);
722 if (!isL) {
723 // if we've met UTF symbol while searching Latin1 pattern, then we can
724 // skip cnt1 symbols
725 if (str1_isL != str2_isL) {
726 mov(result_tmp, cnt1);
727 } else {
728 mov(result_tmp, 1);
729 }
730 subs(zr, skipch, ASIZE);
731 br(HS, BMADV);
732 }
733 ldrb(result_tmp, Address(sp, skipch)); // load skip distance
734 BIND(BMADV);
735 sub(cnt1tmp, cnt1, 1);
736 add(str2, str2, result_tmp, LSL, str2_chr_shift);
737 cmp(str2, str2end);
738 br(LE, BMLOOPSTR2);
739 add(sp, sp, ASIZE);
740 b(NOMATCH);
741 BIND(BMLOOPSTR1_LASTCMP);
742 cmp(ch1, ch2);
743 br(NE, BMSKIP);
744 BIND(BMMATCH);
745 sub(result, str2, tmp5);
746 if (!str2_isL) lsr(result, result, 1);
747 add(sp, sp, ASIZE);
748 b(DONE);
749
750 BIND(LINEARSTUB);
751 cmp(cnt1, (u1)16); // small patterns still should be handled by simple algorithm
752 br(LT, LINEAR_MEDIUM);
753 mov(result, zr);
754 RuntimeAddress stub = nullptr;
755 if (isL) {
756 stub = RuntimeAddress(StubRoutines::aarch64::string_indexof_linear_ll());
757 assert(stub.target() != nullptr, "string_indexof_linear_ll stub has not been generated");
758 } else if (str1_isL) {
759 stub = RuntimeAddress(StubRoutines::aarch64::string_indexof_linear_ul());
760 assert(stub.target() != nullptr, "string_indexof_linear_ul stub has not been generated");
761 } else {
762 stub = RuntimeAddress(StubRoutines::aarch64::string_indexof_linear_uu());
763 assert(stub.target() != nullptr, "string_indexof_linear_uu stub has not been generated");
764 }
765 address call = trampoline_call(stub);
766 if (call == nullptr) {
767 DEBUG_ONLY(reset_labels(LINEARSEARCH, LINEAR_MEDIUM, DONE, NOMATCH, MATCH));
768 ciEnv::current()->record_failure("CodeCache is full");
769 return;
770 }
771 b(DONE);
772 }
773
774 BIND(LINEARSEARCH);
775 {
776 Label DO1, DO2, DO3;
777
778 Register str2tmp = tmp2;
779 Register first = tmp3;
780
781 if (icnt1 == -1)
782 {
783 Label DOSHORT, FIRST_LOOP, STR2_NEXT, STR1_LOOP, STR1_NEXT;
784
785 cmp(cnt1, u1(str1_isL == str2_isL ? 4 : 2));
786 br(LT, DOSHORT);
787 BIND(LINEAR_MEDIUM);
788 (this->*str1_load_1chr)(first, Address(str1));
789 lea(str1, Address(str1, cnt1, Address::lsl(str1_chr_shift)));
790 sub(cnt1_neg, zr, cnt1, LSL, str1_chr_shift);
791 lea(str2, Address(str2, result_tmp, Address::lsl(str2_chr_shift)));
792 sub(cnt2_neg, zr, result_tmp, LSL, str2_chr_shift);
793
794 BIND(FIRST_LOOP);
795 (this->*str2_load_1chr)(ch2, Address(str2, cnt2_neg));
796 cmp(first, ch2);
797 br(EQ, STR1_LOOP);
798 BIND(STR2_NEXT);
799 adds(cnt2_neg, cnt2_neg, str2_chr_size);
800 br(LE, FIRST_LOOP);
801 b(NOMATCH);
802
803 BIND(STR1_LOOP);
804 adds(cnt1tmp, cnt1_neg, str1_chr_size);
805 add(cnt2tmp, cnt2_neg, str2_chr_size);
806 br(GE, MATCH);
807
808 BIND(STR1_NEXT);
809 (this->*str1_load_1chr)(ch1, Address(str1, cnt1tmp));
810 (this->*str2_load_1chr)(ch2, Address(str2, cnt2tmp));
811 cmp(ch1, ch2);
812 br(NE, STR2_NEXT);
813 adds(cnt1tmp, cnt1tmp, str1_chr_size);
814 add(cnt2tmp, cnt2tmp, str2_chr_size);
815 br(LT, STR1_NEXT);
816 b(MATCH);
817
818 BIND(DOSHORT);
819 if (str1_isL == str2_isL) {
820 cmp(cnt1, (u1)2);
821 br(LT, DO1);
822 br(GT, DO3);
823 }
824 }
825
826 if (icnt1 == 4) {
827 Label CH1_LOOP;
828
829 (this->*load_4chr)(ch1, str1);
830 sub(result_tmp, cnt2, 4);
831 lea(str2, Address(str2, result_tmp, Address::lsl(str2_chr_shift)));
832 sub(cnt2_neg, zr, result_tmp, LSL, str2_chr_shift);
833
834 BIND(CH1_LOOP);
835 (this->*load_4chr)(ch2, Address(str2, cnt2_neg));
836 cmp(ch1, ch2);
837 br(EQ, MATCH);
838 adds(cnt2_neg, cnt2_neg, str2_chr_size);
839 br(LE, CH1_LOOP);
840 b(NOMATCH);
841 }
842
843 if ((icnt1 == -1 && str1_isL == str2_isL) || icnt1 == 2) {
844 Label CH1_LOOP;
845
846 BIND(DO2);
847 (this->*load_2chr)(ch1, str1);
848 if (icnt1 == 2) {
849 sub(result_tmp, cnt2, 2);
850 }
851 lea(str2, Address(str2, result_tmp, Address::lsl(str2_chr_shift)));
852 sub(cnt2_neg, zr, result_tmp, LSL, str2_chr_shift);
853 BIND(CH1_LOOP);
854 (this->*load_2chr)(ch2, Address(str2, cnt2_neg));
855 cmp(ch1, ch2);
856 br(EQ, MATCH);
857 adds(cnt2_neg, cnt2_neg, str2_chr_size);
858 br(LE, CH1_LOOP);
859 b(NOMATCH);
860 }
861
862 if ((icnt1 == -1 && str1_isL == str2_isL) || icnt1 == 3) {
863 Label FIRST_LOOP, STR2_NEXT, STR1_LOOP;
864
865 BIND(DO3);
866 (this->*load_2chr)(first, str1);
867 (this->*str1_load_1chr)(ch1, Address(str1, 2*str1_chr_size));
868 if (icnt1 == 3) {
869 sub(result_tmp, cnt2, 3);
870 }
871 lea(str2, Address(str2, result_tmp, Address::lsl(str2_chr_shift)));
872 sub(cnt2_neg, zr, result_tmp, LSL, str2_chr_shift);
873 BIND(FIRST_LOOP);
874 (this->*load_2chr)(ch2, Address(str2, cnt2_neg));
875 cmpw(first, ch2);
876 br(EQ, STR1_LOOP);
877 BIND(STR2_NEXT);
878 adds(cnt2_neg, cnt2_neg, str2_chr_size);
879 br(LE, FIRST_LOOP);
880 b(NOMATCH);
881
882 BIND(STR1_LOOP);
883 add(cnt2tmp, cnt2_neg, 2*str2_chr_size);
884 (this->*str2_load_1chr)(ch2, Address(str2, cnt2tmp));
885 cmp(ch1, ch2);
886 br(NE, STR2_NEXT);
887 b(MATCH);
888 }
889
890 if (icnt1 == -1 || icnt1 == 1) {
891 Label CH1_LOOP, HAS_ZERO, DO1_SHORT, DO1_LOOP;
892
893 BIND(DO1);
894 (this->*str1_load_1chr)(ch1, str1);
895 cmp(cnt2, (u1)8);
896 br(LT, DO1_SHORT);
897
898 sub(result_tmp, cnt2, 8/str2_chr_size);
899 sub(cnt2_neg, zr, result_tmp, LSL, str2_chr_shift);
900 mov(tmp3, str2_isL ? 0x0101010101010101 : 0x0001000100010001);
901 lea(str2, Address(str2, result_tmp, Address::lsl(str2_chr_shift)));
902
903 if (str2_isL) {
904 orr(ch1, ch1, ch1, LSL, 8);
905 }
906 orr(ch1, ch1, ch1, LSL, 16);
907 orr(ch1, ch1, ch1, LSL, 32);
908 BIND(CH1_LOOP);
909 ldr(ch2, Address(str2, cnt2_neg));
910 eor(ch2, ch1, ch2);
911 sub(tmp1, ch2, tmp3);
912 orr(tmp2, ch2, str2_isL ? 0x7f7f7f7f7f7f7f7f : 0x7fff7fff7fff7fff);
913 bics(tmp1, tmp1, tmp2);
914 br(NE, HAS_ZERO);
915 adds(cnt2_neg, cnt2_neg, 8);
916 br(LT, CH1_LOOP);
917
918 cmp(cnt2_neg, (u1)8);
919 mov(cnt2_neg, 0);
920 br(LT, CH1_LOOP);
921 b(NOMATCH);
922
923 BIND(HAS_ZERO);
924 rev(tmp1, tmp1);
925 clz(tmp1, tmp1);
926 add(cnt2_neg, cnt2_neg, tmp1, LSR, 3);
927 b(MATCH);
928
929 BIND(DO1_SHORT);
930 mov(result_tmp, cnt2);
931 lea(str2, Address(str2, cnt2, Address::lsl(str2_chr_shift)));
932 sub(cnt2_neg, zr, cnt2, LSL, str2_chr_shift);
933 BIND(DO1_LOOP);
934 (this->*str2_load_1chr)(ch2, Address(str2, cnt2_neg));
935 cmpw(ch1, ch2);
936 br(EQ, MATCH);
937 adds(cnt2_neg, cnt2_neg, str2_chr_size);
938 br(LT, DO1_LOOP);
939 }
940 }
941 BIND(NOMATCH);
942 mov(result, -1);
943 b(DONE);
944 BIND(MATCH);
945 add(result, result_tmp, cnt2_neg, ASR, str2_chr_shift);
946 BIND(DONE);
947 }
948
949 typedef void (MacroAssembler::* chr_insn)(Register Rt, const Address &adr);
950 typedef void (MacroAssembler::* uxt_insn)(Register Rd, Register Rn);
951
952 void C2_MacroAssembler::string_indexof_char(Register str1, Register cnt1,
953 Register ch, Register result,
954 Register tmp1, Register tmp2, Register tmp3)
955 {
956 Label CH1_LOOP, HAS_ZERO, DO1_SHORT, DO1_LOOP, MATCH, NOMATCH, DONE;
957 Register cnt1_neg = cnt1;
958 Register ch1 = rscratch1;
959 Register result_tmp = rscratch2;
960
961 cbz(cnt1, NOMATCH);
962
963 cmp(cnt1, (u1)4);
964 br(LT, DO1_SHORT);
965
966 orr(ch, ch, ch, LSL, 16);
967 orr(ch, ch, ch, LSL, 32);
968
969 sub(cnt1, cnt1, 4);
970 mov(result_tmp, cnt1);
971 lea(str1, Address(str1, cnt1, Address::uxtw(1)));
972 sub(cnt1_neg, zr, cnt1, LSL, 1);
973
974 mov(tmp3, 0x0001000100010001);
975
976 BIND(CH1_LOOP);
977 ldr(ch1, Address(str1, cnt1_neg));
978 eor(ch1, ch, ch1);
979 sub(tmp1, ch1, tmp3);
980 orr(tmp2, ch1, 0x7fff7fff7fff7fff);
981 bics(tmp1, tmp1, tmp2);
982 br(NE, HAS_ZERO);
983 adds(cnt1_neg, cnt1_neg, 8);
984 br(LT, CH1_LOOP);
985
986 cmp(cnt1_neg, (u1)8);
987 mov(cnt1_neg, 0);
988 br(LT, CH1_LOOP);
989 b(NOMATCH);
990
991 BIND(HAS_ZERO);
992 rev(tmp1, tmp1);
993 clz(tmp1, tmp1);
994 add(cnt1_neg, cnt1_neg, tmp1, LSR, 3);
995 b(MATCH);
996
997 BIND(DO1_SHORT);
998 mov(result_tmp, cnt1);
999 lea(str1, Address(str1, cnt1, Address::uxtw(1)));
1000 sub(cnt1_neg, zr, cnt1, LSL, 1);
1001 BIND(DO1_LOOP);
1002 ldrh(ch1, Address(str1, cnt1_neg));
1003 cmpw(ch, ch1);
1004 br(EQ, MATCH);
1005 adds(cnt1_neg, cnt1_neg, 2);
1006 br(LT, DO1_LOOP);
1007 BIND(NOMATCH);
1008 mov(result, -1);
1009 b(DONE);
1010 BIND(MATCH);
1011 add(result, result_tmp, cnt1_neg, ASR, 1);
1012 BIND(DONE);
1013 }
1014
1015 void C2_MacroAssembler::string_indexof_char_sve(Register str1, Register cnt1,
1016 Register ch, Register result,
1017 FloatRegister ztmp1,
1018 FloatRegister ztmp2,
1019 PRegister tmp_pg,
1020 PRegister tmp_pdn, bool isL)
1021 {
1022 // Note that `tmp_pdn` should *NOT* be used as governing predicate register.
1023 assert(tmp_pg->is_governing(),
1024 "this register has to be a governing predicate register");
1025
1026 Label LOOP, MATCH, DONE, NOMATCH;
1027 Register vec_len = rscratch1;
1028 Register idx = rscratch2;
1029
1030 SIMD_RegVariant T = (isL == true) ? B : H;
1031
1032 cbz(cnt1, NOMATCH);
1033
1034 // Assign the particular char throughout the vector.
1035 sve_dup(ztmp2, T, ch);
1036 if (isL) {
1037 sve_cntb(vec_len);
1038 } else {
1039 sve_cnth(vec_len);
1040 }
1041 mov(idx, 0);
1042
1043 // Generate a predicate to control the reading of input string.
1044 sve_whilelt(tmp_pg, T, idx, cnt1);
1045
1046 BIND(LOOP);
1047 // Read a vector of 8- or 16-bit data depending on the string type. Note
1048 // that inactive elements indicated by the predicate register won't cause
1049 // a data read from memory to the destination vector.
1050 if (isL) {
1051 sve_ld1b(ztmp1, T, tmp_pg, Address(str1, idx));
1052 } else {
1053 sve_ld1h(ztmp1, T, tmp_pg, Address(str1, idx, Address::lsl(1)));
1054 }
1055 add(idx, idx, vec_len);
1056
1057 // Perform the comparison. An element of the destination predicate is set
1058 // to active if the particular char is matched.
1059 sve_cmp(Assembler::EQ, tmp_pdn, T, tmp_pg, ztmp1, ztmp2);
1060
1061 // Branch if the particular char is found.
1062 br(NE, MATCH);
1063
1064 sve_whilelt(tmp_pg, T, idx, cnt1);
1065
1066 // Loop back if the particular char not found.
1067 br(MI, LOOP);
1068
1069 BIND(NOMATCH);
1070 mov(result, -1);
1071 b(DONE);
1072
1073 BIND(MATCH);
1074 // Undo the index increment.
1075 sub(idx, idx, vec_len);
1076
1077 // Crop the vector to find its location.
1078 sve_brka(tmp_pdn, tmp_pg, tmp_pdn, false /* isMerge */);
1079 add(result, idx, -1);
1080 sve_incp(result, T, tmp_pdn);
1081 BIND(DONE);
1082 }
1083
1084 void C2_MacroAssembler::stringL_indexof_char(Register str1, Register cnt1,
1085 Register ch, Register result,
1086 Register tmp1, Register tmp2, Register tmp3)
1087 {
1088 Label CH1_LOOP, HAS_ZERO, DO1_SHORT, DO1_LOOP, MATCH, NOMATCH, DONE;
1089 Register cnt1_neg = cnt1;
1090 Register ch1 = rscratch1;
1091 Register result_tmp = rscratch2;
1092
1093 cbz(cnt1, NOMATCH);
1094
1095 cmp(cnt1, (u1)8);
1096 br(LT, DO1_SHORT);
1097
1098 orr(ch, ch, ch, LSL, 8);
1099 orr(ch, ch, ch, LSL, 16);
1100 orr(ch, ch, ch, LSL, 32);
1101
1102 sub(cnt1, cnt1, 8);
1103 mov(result_tmp, cnt1);
1104 lea(str1, Address(str1, cnt1));
1105 sub(cnt1_neg, zr, cnt1);
1106
1107 mov(tmp3, 0x0101010101010101);
1108
1109 BIND(CH1_LOOP);
1110 ldr(ch1, Address(str1, cnt1_neg));
1111 eor(ch1, ch, ch1);
1112 sub(tmp1, ch1, tmp3);
1113 orr(tmp2, ch1, 0x7f7f7f7f7f7f7f7f);
1114 bics(tmp1, tmp1, tmp2);
1115 br(NE, HAS_ZERO);
1116 adds(cnt1_neg, cnt1_neg, 8);
1117 br(LT, CH1_LOOP);
1118
1119 cmp(cnt1_neg, (u1)8);
1120 mov(cnt1_neg, 0);
1121 br(LT, CH1_LOOP);
1122 b(NOMATCH);
1123
1124 BIND(HAS_ZERO);
1125 rev(tmp1, tmp1);
1126 clz(tmp1, tmp1);
1127 add(cnt1_neg, cnt1_neg, tmp1, LSR, 3);
1128 b(MATCH);
1129
1130 BIND(DO1_SHORT);
1131 mov(result_tmp, cnt1);
1132 lea(str1, Address(str1, cnt1));
1133 sub(cnt1_neg, zr, cnt1);
1134 BIND(DO1_LOOP);
1135 ldrb(ch1, Address(str1, cnt1_neg));
1136 cmp(ch, ch1);
1137 br(EQ, MATCH);
1138 adds(cnt1_neg, cnt1_neg, 1);
1139 br(LT, DO1_LOOP);
1140 BIND(NOMATCH);
1141 mov(result, -1);
1142 b(DONE);
1143 BIND(MATCH);
1144 add(result, result_tmp, cnt1_neg);
1145 BIND(DONE);
1146 }
1147
1148 // Compare strings.
1149 void C2_MacroAssembler::string_compare(Register str1, Register str2,
1150 Register cnt1, Register cnt2, Register result, Register tmp1, Register tmp2,
1151 FloatRegister vtmp1, FloatRegister vtmp2, FloatRegister vtmp3,
1152 PRegister pgtmp1, PRegister pgtmp2, int ae) {
1153 Label DONE, SHORT_LOOP, SHORT_STRING, SHORT_LAST, TAIL, STUB,
1154 DIFF, NEXT_WORD, SHORT_LOOP_TAIL, SHORT_LAST2, SHORT_LAST_INIT,
1155 SHORT_LOOP_START, TAIL_CHECK;
1156
1157 bool isLL = ae == StrIntrinsicNode::LL;
1158 bool isLU = ae == StrIntrinsicNode::LU;
1159 bool isUL = ae == StrIntrinsicNode::UL;
1160
1161 // The stub threshold for LL strings is: 72 (64 + 8) chars
1162 // UU: 36 chars, or 72 bytes (valid for the 64-byte large loop with prefetch)
1163 // LU/UL: 24 chars, or 48 bytes (valid for the 16-character loop at least)
1164 const u1 stub_threshold = isLL ? 72 : ((isLU || isUL) ? 24 : 36);
1165
1166 bool str1_isL = isLL || isLU;
1167 bool str2_isL = isLL || isUL;
1168
1169 int str1_chr_shift = str1_isL ? 0 : 1;
1170 int str2_chr_shift = str2_isL ? 0 : 1;
1171 int str1_chr_size = str1_isL ? 1 : 2;
1172 int str2_chr_size = str2_isL ? 1 : 2;
1173 int minCharsInWord = isLL ? wordSize : wordSize/2;
1174
1175 FloatRegister vtmpZ = vtmp1, vtmp = vtmp2;
1176 chr_insn str1_load_chr = str1_isL ? (chr_insn)&MacroAssembler::ldrb :
1177 (chr_insn)&MacroAssembler::ldrh;
1178 chr_insn str2_load_chr = str2_isL ? (chr_insn)&MacroAssembler::ldrb :
1179 (chr_insn)&MacroAssembler::ldrh;
1180 uxt_insn ext_chr = isLL ? (uxt_insn)&MacroAssembler::uxtbw :
1181 (uxt_insn)&MacroAssembler::uxthw;
1182
1183 BLOCK_COMMENT("string_compare {");
1184
1185 // Bizzarely, the counts are passed in bytes, regardless of whether they
1186 // are L or U strings, however the result is always in characters.
1187 if (!str1_isL) asrw(cnt1, cnt1, 1);
1188 if (!str2_isL) asrw(cnt2, cnt2, 1);
1189
1190 // Compute the minimum of the string lengths and save the difference.
1191 subsw(result, cnt1, cnt2);
1192 cselw(cnt2, cnt1, cnt2, Assembler::LE); // min
1193
1194 // A very short string
1195 cmpw(cnt2, minCharsInWord);
1196 br(Assembler::LE, SHORT_STRING);
1197
1198 // Compare longwords
1199 // load first parts of strings and finish initialization while loading
1200 {
1201 if (str1_isL == str2_isL) { // LL or UU
1202 ldr(tmp1, Address(str1));
1203 cmp(str1, str2);
1204 br(Assembler::EQ, DONE);
1205 ldr(tmp2, Address(str2));
1206 cmp(cnt2, stub_threshold);
1207 br(GE, STUB);
1208 subsw(cnt2, cnt2, minCharsInWord);
1209 br(EQ, TAIL_CHECK);
1210 lea(str2, Address(str2, cnt2, Address::uxtw(str2_chr_shift)));
1211 lea(str1, Address(str1, cnt2, Address::uxtw(str1_chr_shift)));
1212 sub(cnt2, zr, cnt2, LSL, str2_chr_shift);
1213 } else if (isLU) {
1214 ldrs(vtmp, Address(str1));
1215 ldr(tmp2, Address(str2));
1216 cmp(cnt2, stub_threshold);
1217 br(GE, STUB);
1218 subw(cnt2, cnt2, 4);
1219 eor(vtmpZ, T16B, vtmpZ, vtmpZ);
1220 lea(str1, Address(str1, cnt2, Address::uxtw(str1_chr_shift)));
1221 lea(str2, Address(str2, cnt2, Address::uxtw(str2_chr_shift)));
1222 zip1(vtmp, T8B, vtmp, vtmpZ);
1223 sub(cnt1, zr, cnt2, LSL, str1_chr_shift);
1224 sub(cnt2, zr, cnt2, LSL, str2_chr_shift);
1225 add(cnt1, cnt1, 4);
1226 fmovd(tmp1, vtmp);
1227 } else { // UL case
1228 ldr(tmp1, Address(str1));
1229 ldrs(vtmp, Address(str2));
1230 cmp(cnt2, stub_threshold);
1231 br(GE, STUB);
1232 subw(cnt2, cnt2, 4);
1233 lea(str1, Address(str1, cnt2, Address::uxtw(str1_chr_shift)));
1234 eor(vtmpZ, T16B, vtmpZ, vtmpZ);
1235 lea(str2, Address(str2, cnt2, Address::uxtw(str2_chr_shift)));
1236 sub(cnt1, zr, cnt2, LSL, str1_chr_shift);
1237 zip1(vtmp, T8B, vtmp, vtmpZ);
1238 sub(cnt2, zr, cnt2, LSL, str2_chr_shift);
1239 add(cnt1, cnt1, 8);
1240 fmovd(tmp2, vtmp);
1241 }
1242 adds(cnt2, cnt2, isUL ? 4 : 8);
1243 br(GE, TAIL);
1244 eor(rscratch2, tmp1, tmp2);
1245 cbnz(rscratch2, DIFF);
1246 // main loop
1247 bind(NEXT_WORD);
1248 if (str1_isL == str2_isL) {
1249 ldr(tmp1, Address(str1, cnt2));
1250 ldr(tmp2, Address(str2, cnt2));
1251 adds(cnt2, cnt2, 8);
1252 } else if (isLU) {
1253 ldrs(vtmp, Address(str1, cnt1));
1254 ldr(tmp2, Address(str2, cnt2));
1255 add(cnt1, cnt1, 4);
1256 zip1(vtmp, T8B, vtmp, vtmpZ);
1257 fmovd(tmp1, vtmp);
1258 adds(cnt2, cnt2, 8);
1259 } else { // UL
1260 ldrs(vtmp, Address(str2, cnt2));
1261 ldr(tmp1, Address(str1, cnt1));
1262 zip1(vtmp, T8B, vtmp, vtmpZ);
1263 add(cnt1, cnt1, 8);
1264 fmovd(tmp2, vtmp);
1265 adds(cnt2, cnt2, 4);
1266 }
1267 br(GE, TAIL);
1268
1269 eor(rscratch2, tmp1, tmp2);
1270 cbz(rscratch2, NEXT_WORD);
1271 b(DIFF);
1272 bind(TAIL);
1273 eor(rscratch2, tmp1, tmp2);
1274 cbnz(rscratch2, DIFF);
1275 // Last longword. In the case where length == 4 we compare the
1276 // same longword twice, but that's still faster than another
1277 // conditional branch.
1278 if (str1_isL == str2_isL) {
1279 ldr(tmp1, Address(str1));
1280 ldr(tmp2, Address(str2));
1281 } else if (isLU) {
1282 ldrs(vtmp, Address(str1));
1283 ldr(tmp2, Address(str2));
1284 zip1(vtmp, T8B, vtmp, vtmpZ);
1285 fmovd(tmp1, vtmp);
1286 } else { // UL
1287 ldrs(vtmp, Address(str2));
1288 ldr(tmp1, Address(str1));
1289 zip1(vtmp, T8B, vtmp, vtmpZ);
1290 fmovd(tmp2, vtmp);
1291 }
1292 bind(TAIL_CHECK);
1293 eor(rscratch2, tmp1, tmp2);
1294 cbz(rscratch2, DONE);
1295
1296 // Find the first different characters in the longwords and
1297 // compute their difference.
1298 bind(DIFF);
1299 rev(rscratch2, rscratch2);
1300 clz(rscratch2, rscratch2);
1301 andr(rscratch2, rscratch2, isLL ? -8 : -16);
1302 lsrv(tmp1, tmp1, rscratch2);
1303 (this->*ext_chr)(tmp1, tmp1);
1304 lsrv(tmp2, tmp2, rscratch2);
1305 (this->*ext_chr)(tmp2, tmp2);
1306 subw(result, tmp1, tmp2);
1307 b(DONE);
1308 }
1309
1310 bind(STUB);
1311 RuntimeAddress stub = nullptr;
1312 switch(ae) {
1313 case StrIntrinsicNode::LL:
1314 stub = RuntimeAddress(StubRoutines::aarch64::compare_long_string_LL());
1315 break;
1316 case StrIntrinsicNode::UU:
1317 stub = RuntimeAddress(StubRoutines::aarch64::compare_long_string_UU());
1318 break;
1319 case StrIntrinsicNode::LU:
1320 stub = RuntimeAddress(StubRoutines::aarch64::compare_long_string_LU());
1321 break;
1322 case StrIntrinsicNode::UL:
1323 stub = RuntimeAddress(StubRoutines::aarch64::compare_long_string_UL());
1324 break;
1325 default:
1326 ShouldNotReachHere();
1327 }
1328 assert(stub.target() != nullptr, "compare_long_string stub has not been generated");
1329 address call = trampoline_call(stub);
1330 if (call == nullptr) {
1331 DEBUG_ONLY(reset_labels(DONE, SHORT_LOOP, SHORT_STRING, SHORT_LAST, SHORT_LOOP_TAIL, SHORT_LAST2, SHORT_LAST_INIT, SHORT_LOOP_START));
1332 ciEnv::current()->record_failure("CodeCache is full");
1333 return;
1334 }
1335 b(DONE);
1336
1337 bind(SHORT_STRING);
1338 // Is the minimum length zero?
1339 cbz(cnt2, DONE);
1340 // arrange code to do most branches while loading and loading next characters
1341 // while comparing previous
1342 (this->*str1_load_chr)(tmp1, Address(post(str1, str1_chr_size)));
1343 subs(cnt2, cnt2, 1);
1344 br(EQ, SHORT_LAST_INIT);
1345 (this->*str2_load_chr)(cnt1, Address(post(str2, str2_chr_size)));
1346 b(SHORT_LOOP_START);
1347 bind(SHORT_LOOP);
1348 subs(cnt2, cnt2, 1);
1349 br(EQ, SHORT_LAST);
1350 bind(SHORT_LOOP_START);
1351 (this->*str1_load_chr)(tmp2, Address(post(str1, str1_chr_size)));
1352 (this->*str2_load_chr)(rscratch1, Address(post(str2, str2_chr_size)));
1353 cmp(tmp1, cnt1);
1354 br(NE, SHORT_LOOP_TAIL);
1355 subs(cnt2, cnt2, 1);
1356 br(EQ, SHORT_LAST2);
1357 (this->*str1_load_chr)(tmp1, Address(post(str1, str1_chr_size)));
1358 (this->*str2_load_chr)(cnt1, Address(post(str2, str2_chr_size)));
1359 cmp(tmp2, rscratch1);
1360 br(EQ, SHORT_LOOP);
1361 sub(result, tmp2, rscratch1);
1362 b(DONE);
1363 bind(SHORT_LOOP_TAIL);
1364 sub(result, tmp1, cnt1);
1365 b(DONE);
1366 bind(SHORT_LAST2);
1367 cmp(tmp2, rscratch1);
1368 br(EQ, DONE);
1369 sub(result, tmp2, rscratch1);
1370
1371 b(DONE);
1372 bind(SHORT_LAST_INIT);
1373 (this->*str2_load_chr)(cnt1, Address(post(str2, str2_chr_size)));
1374 bind(SHORT_LAST);
1375 cmp(tmp1, cnt1);
1376 br(EQ, DONE);
1377 sub(result, tmp1, cnt1);
1378
1379 bind(DONE);
1380
1381 BLOCK_COMMENT("} string_compare");
1382 }
1383
1384 void C2_MacroAssembler::neon_compare(FloatRegister dst, BasicType bt, FloatRegister src1,
1385 FloatRegister src2, Condition cond, bool isQ) {
1386 SIMD_Arrangement size = esize2arrangement((unsigned)type2aelembytes(bt), isQ);
1387 FloatRegister zn = src1, zm = src2;
1388 bool needs_negation = false;
1389 switch (cond) {
1390 case LT: cond = GT; zn = src2; zm = src1; break;
1391 case LE: cond = GE; zn = src2; zm = src1; break;
1392 case LO: cond = HI; zn = src2; zm = src1; break;
1393 case LS: cond = HS; zn = src2; zm = src1; break;
1394 case NE: cond = EQ; needs_negation = true; break;
1395 default:
1396 break;
1397 }
1398
1399 if (is_floating_point_type(bt)) {
1400 fcm(cond, dst, size, zn, zm);
1401 } else {
1402 cm(cond, dst, size, zn, zm);
1403 }
1404
1405 if (needs_negation) {
1406 notr(dst, isQ ? T16B : T8B, dst);
1407 }
1408 }
1409
1410 void C2_MacroAssembler::neon_compare_zero(FloatRegister dst, BasicType bt, FloatRegister src,
1411 Condition cond, bool isQ) {
1412 SIMD_Arrangement size = esize2arrangement((unsigned)type2aelembytes(bt), isQ);
1413 if (bt == T_FLOAT || bt == T_DOUBLE) {
1414 if (cond == Assembler::NE) {
1415 fcm(Assembler::EQ, dst, size, src);
1416 notr(dst, isQ ? T16B : T8B, dst);
1417 } else {
1418 fcm(cond, dst, size, src);
1419 }
1420 } else {
1421 if (cond == Assembler::NE) {
1422 cm(Assembler::EQ, dst, size, src);
1423 notr(dst, isQ ? T16B : T8B, dst);
1424 } else {
1425 cm(cond, dst, size, src);
1426 }
1427 }
1428 }
1429
1430 // Compress the least significant bit of each byte to the rightmost and clear
1431 // the higher garbage bits.
1432 void C2_MacroAssembler::bytemask_compress(Register dst) {
1433 // Example input, dst = 0x01 00 00 00 01 01 00 01
1434 // The "??" bytes are garbage.
1435 orr(dst, dst, dst, Assembler::LSR, 7); // dst = 0x?? 02 ?? 00 ?? 03 ?? 01
1436 orr(dst, dst, dst, Assembler::LSR, 14); // dst = 0x????????08 ??????0D
1437 orr(dst, dst, dst, Assembler::LSR, 28); // dst = 0x????????????????8D
1438 andr(dst, dst, 0xff); // dst = 0x8D
1439 }
1440
1441 // Pack the lowest-numbered bit of each mask element in src into a long value
1442 // in dst, at most the first 64 lane elements.
1443 // Clobbers: rscratch1, if UseSVE=1 or the hardware doesn't support FEAT_BITPERM.
1444 void C2_MacroAssembler::sve_vmask_tolong(Register dst, PRegister src, BasicType bt, int lane_cnt,
1445 FloatRegister vtmp1, FloatRegister vtmp2) {
1446 assert(lane_cnt <= 64 && is_power_of_2(lane_cnt), "Unsupported lane count");
1447 assert_different_registers(dst, rscratch1);
1448 assert_different_registers(vtmp1, vtmp2);
1449
1450 Assembler::SIMD_RegVariant size = elemType_to_regVariant(bt);
1451 // Example: src = 0b01100101 10001101, bt = T_BYTE, lane_cnt = 16
1452 // Expected: dst = 0x658D
1453
1454 // Convert the mask into vector with sequential bytes.
1455 // vtmp1 = 0x00010100 0x00010001 0x01000000 0x01010001
1456 sve_cpy(vtmp1, size, src, 1, false);
1457 if (bt != T_BYTE) {
1458 sve_vector_narrow(vtmp1, B, vtmp1, size, vtmp2);
1459 }
1460
1461 if (UseSVE > 1 && VM_Version::supports_svebitperm()) {
1462 // Given a vector with the value 0x00 or 0x01 in each byte, the basic idea
1463 // is to compress each significant bit of the byte in a cross-lane way. Due
1464 // to the lack of a cross-lane bit-compress instruction, we use BEXT
1465 // (bit-compress in each lane) with the biggest lane size (T = D) then
1466 // concatenate the results.
1467
1468 // The second source input of BEXT, initialized with 0x01 in each byte.
1469 // vtmp2 = 0x01010101 0x01010101 0x01010101 0x01010101
1470 sve_dup(vtmp2, B, 1);
1471
1472 // BEXT vtmp1.D, vtmp1.D, vtmp2.D
1473 // vtmp1 = 0x0001010000010001 | 0x0100000001010001
1474 // vtmp2 = 0x0101010101010101 | 0x0101010101010101
1475 // ---------------------------------------
1476 // vtmp1 = 0x0000000000000065 | 0x000000000000008D
1477 sve_bext(vtmp1, D, vtmp1, vtmp2);
1478
1479 // Concatenate the lowest significant 8 bits in each 8 bytes, and extract the
1480 // result to dst.
1481 // vtmp1 = 0x0000000000000000 | 0x000000000000658D
1482 // dst = 0x658D
1483 if (lane_cnt <= 8) {
1484 // No need to concatenate.
1485 umov(dst, vtmp1, B, 0);
1486 } else if (lane_cnt <= 16) {
1487 ins(vtmp1, B, vtmp1, 1, 8);
1488 umov(dst, vtmp1, H, 0);
1489 } else {
1490 // As the lane count is 64 at most, the final expected value must be in
1491 // the lowest 64 bits after narrowing vtmp1 from D to B.
1492 sve_vector_narrow(vtmp1, B, vtmp1, D, vtmp2);
1493 umov(dst, vtmp1, D, 0);
1494 }
1495 } else if (UseSVE > 0) {
1496 // Compress the lowest 8 bytes.
1497 fmovd(dst, vtmp1);
1498 bytemask_compress(dst);
1499 if (lane_cnt <= 8) return;
1500
1501 // Repeat on higher bytes and join the results.
1502 // Compress 8 bytes in each iteration.
1503 for (int idx = 1; idx < (lane_cnt / 8); idx++) {
1504 sve_extract_integral(rscratch1, T_LONG, vtmp1, idx, vtmp2);
1505 bytemask_compress(rscratch1);
1506 orr(dst, dst, rscratch1, Assembler::LSL, idx << 3);
1507 }
1508 } else {
1509 assert(false, "unsupported");
1510 ShouldNotReachHere();
1511 }
1512 }
1513
1514 // Unpack the mask, a long value in src, into predicate register dst based on the
1515 // corresponding data type. Note that dst can support at most 64 lanes.
1516 // Below example gives the expected dst predicate register in different types, with
1517 // a valid src(0x658D) on a 1024-bit vector size machine.
1518 // BYTE: dst = 0x00 00 00 00 00 00 00 00 00 00 00 00 00 00 65 8D
1519 // SHORT: dst = 0x00 00 00 00 00 00 00 00 00 00 00 00 14 11 40 51
1520 // INT: dst = 0x00 00 00 00 00 00 00 00 01 10 01 01 10 00 11 01
1521 // LONG: dst = 0x00 01 01 00 00 01 00 01 01 00 00 00 01 01 00 01
1522 //
1523 // The number of significant bits of src must be equal to lane_cnt. E.g., 0xFF658D which
1524 // has 24 significant bits would be an invalid input if dst predicate register refers to
1525 // a LONG type 1024-bit vector, which has at most 16 lanes.
1526 void C2_MacroAssembler::sve_vmask_fromlong(PRegister dst, Register src, BasicType bt, int lane_cnt,
1527 FloatRegister vtmp1, FloatRegister vtmp2) {
1528 assert(UseSVE == 2 && VM_Version::supports_svebitperm() &&
1529 lane_cnt <= 64 && is_power_of_2(lane_cnt), "unsupported");
1530 Assembler::SIMD_RegVariant size = elemType_to_regVariant(bt);
1531 // Example: src = 0x658D, bt = T_BYTE, size = B, lane_cnt = 16
1532 // Expected: dst = 0b01101001 10001101
1533
1534 // Put long value from general purpose register into the first lane of vector.
1535 // vtmp1 = 0x0000000000000000 | 0x000000000000658D
1536 sve_dup(vtmp1, B, 0);
1537 mov(vtmp1, D, 0, src);
1538
1539 // As sve_cmp generates mask value with the minimum unit in byte, we should
1540 // transform the value in the first lane which is mask in bit now to the
1541 // mask in byte, which can be done by SVE2's BDEP instruction.
1542
1543 // The first source input of BDEP instruction. Deposite each byte in every 8 bytes.
1544 // vtmp1 = 0x0000000000000065 | 0x000000000000008D
1545 if (lane_cnt <= 8) {
1546 // Nothing. As only one byte exsits.
1547 } else if (lane_cnt <= 16) {
1548 ins(vtmp1, B, vtmp1, 8, 1);
1549 mov(vtmp1, B, 1, zr);
1550 } else {
1551 sve_vector_extend(vtmp1, D, vtmp1, B);
1552 }
1553
1554 // The second source input of BDEP instruction, initialized with 0x01 for each byte.
1555 // vtmp2 = 0x01010101 0x01010101 0x01010101 0x01010101
1556 sve_dup(vtmp2, B, 1);
1557
1558 // BDEP vtmp1.D, vtmp1.D, vtmp2.D
1559 // vtmp1 = 0x0000000000000065 | 0x000000000000008D
1560 // vtmp2 = 0x0101010101010101 | 0x0101010101010101
1561 // ---------------------------------------
1562 // vtmp1 = 0x0001010000010001 | 0x0100000001010001
1563 sve_bdep(vtmp1, D, vtmp1, vtmp2);
1564
1565 if (bt != T_BYTE) {
1566 sve_vector_extend(vtmp1, size, vtmp1, B);
1567 }
1568 // Generate mask according to the given vector, in which the elements have been
1569 // extended to expected type.
1570 // dst = 0b01101001 10001101
1571 sve_cmp(Assembler::NE, dst, size, ptrue, vtmp1, 0);
1572 }
1573
1574 // Clobbers: rflags
1575 void C2_MacroAssembler::sve_compare(PRegister pd, BasicType bt, PRegister pg,
1576 FloatRegister zn, FloatRegister zm, Condition cond) {
1577 assert(pg->is_governing(), "This register has to be a governing predicate register");
1578 FloatRegister z1 = zn, z2 = zm;
1579 switch (cond) {
1580 case LE: z1 = zm; z2 = zn; cond = GE; break;
1581 case LT: z1 = zm; z2 = zn; cond = GT; break;
1582 case LO: z1 = zm; z2 = zn; cond = HI; break;
1583 case LS: z1 = zm; z2 = zn; cond = HS; break;
1584 default:
1585 break;
1586 }
1587
1588 SIMD_RegVariant size = elemType_to_regVariant(bt);
1589 if (is_floating_point_type(bt)) {
1590 sve_fcm(cond, pd, size, pg, z1, z2);
1591 } else {
1592 assert(is_integral_type(bt), "unsupported element type");
1593 sve_cmp(cond, pd, size, pg, z1, z2);
1594 }
1595 }
1596
1597 // Get index of the last mask lane that is set
1598 void C2_MacroAssembler::sve_vmask_lasttrue(Register dst, BasicType bt, PRegister src, PRegister ptmp) {
1599 SIMD_RegVariant size = elemType_to_regVariant(bt);
1600 sve_rev(ptmp, size, src);
1601 sve_brkb(ptmp, ptrue, ptmp, false);
1602 sve_cntp(dst, size, ptrue, ptmp);
1603 movw(rscratch1, MaxVectorSize / type2aelembytes(bt) - 1);
1604 subw(dst, rscratch1, dst);
1605 }
1606
1607 // Extend integer vector src to dst with the same lane count
1608 // but larger element size, e.g. 4B -> 4I
1609 void C2_MacroAssembler::neon_vector_extend(FloatRegister dst, BasicType dst_bt, unsigned dst_vlen_in_bytes,
1610 FloatRegister src, BasicType src_bt, bool is_unsigned) {
1611 if (src_bt == T_BYTE) {
1612 if (dst_bt == T_SHORT) {
1613 // 4B/8B to 4S/8S
1614 _xshll(is_unsigned, dst, T8H, src, T8B, 0);
1615 } else {
1616 // 4B to 4I
1617 assert(dst_vlen_in_bytes == 16 && dst_bt == T_INT, "unsupported");
1618 _xshll(is_unsigned, dst, T8H, src, T8B, 0);
1619 _xshll(is_unsigned, dst, T4S, dst, T4H, 0);
1620 }
1621 } else if (src_bt == T_SHORT) {
1622 // 4S to 4I
1623 assert(dst_vlen_in_bytes == 16 && dst_bt == T_INT, "unsupported");
1624 _xshll(is_unsigned, dst, T4S, src, T4H, 0);
1625 } else if (src_bt == T_INT) {
1626 // 2I to 2L
1627 assert(dst_vlen_in_bytes == 16 && dst_bt == T_LONG, "unsupported");
1628 _xshll(is_unsigned, dst, T2D, src, T2S, 0);
1629 } else {
1630 ShouldNotReachHere();
1631 }
1632 }
1633
1634 // Narrow integer vector src down to dst with the same lane count
1635 // but smaller element size, e.g. 4I -> 4B
1636 void C2_MacroAssembler::neon_vector_narrow(FloatRegister dst, BasicType dst_bt,
1637 FloatRegister src, BasicType src_bt, unsigned src_vlen_in_bytes) {
1638 if (src_bt == T_SHORT) {
1639 // 4S/8S to 4B/8B
1640 assert(src_vlen_in_bytes == 8 || src_vlen_in_bytes == 16, "unsupported");
1641 assert(dst_bt == T_BYTE, "unsupported");
1642 xtn(dst, T8B, src, T8H);
1643 } else if (src_bt == T_INT) {
1644 // 4I to 4B/4S
1645 assert(src_vlen_in_bytes == 16, "unsupported");
1646 assert(dst_bt == T_BYTE || dst_bt == T_SHORT, "unsupported");
1647 xtn(dst, T4H, src, T4S);
1648 if (dst_bt == T_BYTE) {
1649 xtn(dst, T8B, dst, T8H);
1650 }
1651 } else if (src_bt == T_LONG) {
1652 // 2L to 2I
1653 assert(src_vlen_in_bytes == 16, "unsupported");
1654 assert(dst_bt == T_INT, "unsupported");
1655 xtn(dst, T2S, src, T2D);
1656 } else {
1657 ShouldNotReachHere();
1658 }
1659 }
1660
1661 void C2_MacroAssembler::sve_vector_extend(FloatRegister dst, SIMD_RegVariant dst_size,
1662 FloatRegister src, SIMD_RegVariant src_size,
1663 bool is_unsigned) {
1664 assert(dst_size > src_size && dst_size <= D && src_size <= S, "invalid element size");
1665
1666 if (src_size == B) {
1667 switch (dst_size) {
1668 case H:
1669 _sve_xunpk(is_unsigned, /* is_high */ false, dst, H, src);
1670 break;
1671 case S:
1672 _sve_xunpk(is_unsigned, /* is_high */ false, dst, H, src);
1673 _sve_xunpk(is_unsigned, /* is_high */ false, dst, S, dst);
1674 break;
1675 case D:
1676 _sve_xunpk(is_unsigned, /* is_high */ false, dst, H, src);
1677 _sve_xunpk(is_unsigned, /* is_high */ false, dst, S, dst);
1678 _sve_xunpk(is_unsigned, /* is_high */ false, dst, D, dst);
1679 break;
1680 default:
1681 ShouldNotReachHere();
1682 }
1683 } else if (src_size == H) {
1684 if (dst_size == S) {
1685 _sve_xunpk(is_unsigned, /* is_high */ false, dst, S, src);
1686 } else { // D
1687 _sve_xunpk(is_unsigned, /* is_high */ false, dst, S, src);
1688 _sve_xunpk(is_unsigned, /* is_high */ false, dst, D, dst);
1689 }
1690 } else if (src_size == S) {
1691 _sve_xunpk(is_unsigned, /* is_high */ false, dst, D, src);
1692 }
1693 }
1694
1695 // Vector narrow from src to dst with specified element sizes.
1696 // High part of dst vector will be filled with zero.
1697 void C2_MacroAssembler::sve_vector_narrow(FloatRegister dst, SIMD_RegVariant dst_size,
1698 FloatRegister src, SIMD_RegVariant src_size,
1699 FloatRegister tmp) {
1700 assert(dst_size < src_size && dst_size <= S && src_size <= D, "invalid element size");
1701 assert_different_registers(src, tmp);
1702 sve_dup(tmp, src_size, 0);
1703 if (src_size == D) {
1704 switch (dst_size) {
1705 case S:
1706 sve_uzp1(dst, S, src, tmp);
1707 break;
1708 case H:
1709 assert_different_registers(dst, tmp);
1710 sve_uzp1(dst, S, src, tmp);
1711 sve_uzp1(dst, H, dst, tmp);
1712 break;
1713 case B:
1714 assert_different_registers(dst, tmp);
1715 sve_uzp1(dst, S, src, tmp);
1716 sve_uzp1(dst, H, dst, tmp);
1717 sve_uzp1(dst, B, dst, tmp);
1718 break;
1719 default:
1720 ShouldNotReachHere();
1721 }
1722 } else if (src_size == S) {
1723 if (dst_size == H) {
1724 sve_uzp1(dst, H, src, tmp);
1725 } else { // B
1726 assert_different_registers(dst, tmp);
1727 sve_uzp1(dst, H, src, tmp);
1728 sve_uzp1(dst, B, dst, tmp);
1729 }
1730 } else if (src_size == H) {
1731 sve_uzp1(dst, B, src, tmp);
1732 }
1733 }
1734
1735 // Extend src predicate to dst predicate with the same lane count but larger
1736 // element size, e.g. 64Byte -> 512Long
1737 void C2_MacroAssembler::sve_vmaskcast_extend(PRegister dst, PRegister src,
1738 uint dst_element_length_in_bytes,
1739 uint src_element_length_in_bytes) {
1740 if (dst_element_length_in_bytes == 2 * src_element_length_in_bytes) {
1741 sve_punpklo(dst, src);
1742 } else if (dst_element_length_in_bytes == 4 * src_element_length_in_bytes) {
1743 sve_punpklo(dst, src);
1744 sve_punpklo(dst, dst);
1745 } else if (dst_element_length_in_bytes == 8 * src_element_length_in_bytes) {
1746 sve_punpklo(dst, src);
1747 sve_punpklo(dst, dst);
1748 sve_punpklo(dst, dst);
1749 } else {
1750 assert(false, "unsupported");
1751 ShouldNotReachHere();
1752 }
1753 }
1754
1755 // Narrow src predicate to dst predicate with the same lane count but
1756 // smaller element size, e.g. 512Long -> 64Byte
1757 void C2_MacroAssembler::sve_vmaskcast_narrow(PRegister dst, PRegister src, PRegister ptmp,
1758 uint dst_element_length_in_bytes, uint src_element_length_in_bytes) {
1759 // The insignificant bits in src predicate are expected to be zero.
1760 // To ensure the higher order bits of the resultant narrowed vector are 0, an all-zero predicate is
1761 // passed as the second argument. An example narrowing operation with a given mask would be -
1762 // 128Long -> 64Int on a 128-bit machine i.e 2L -> 2I
1763 // Mask (for 2 Longs) : TF
1764 // Predicate register for the above mask (16 bits) : 00000001 00000000
1765 // After narrowing (uzp1 dst.b, src.b, ptmp.b) : 0000 0000 0001 0000
1766 // Which translates to mask for 2 integers as : TF (lower half is considered while upper half is 0)
1767 assert_different_registers(src, ptmp);
1768 assert_different_registers(dst, ptmp);
1769 sve_pfalse(ptmp);
1770 if (dst_element_length_in_bytes * 2 == src_element_length_in_bytes) {
1771 sve_uzp1(dst, B, src, ptmp);
1772 } else if (dst_element_length_in_bytes * 4 == src_element_length_in_bytes) {
1773 sve_uzp1(dst, H, src, ptmp);
1774 sve_uzp1(dst, B, dst, ptmp);
1775 } else if (dst_element_length_in_bytes * 8 == src_element_length_in_bytes) {
1776 sve_uzp1(dst, S, src, ptmp);
1777 sve_uzp1(dst, H, dst, ptmp);
1778 sve_uzp1(dst, B, dst, ptmp);
1779 } else {
1780 assert(false, "unsupported");
1781 ShouldNotReachHere();
1782 }
1783 }
1784
1785 // Vector reduction add for integral type with ASIMD instructions.
1786 void C2_MacroAssembler::neon_reduce_add_integral(Register dst, BasicType bt,
1787 Register isrc, FloatRegister vsrc,
1788 unsigned vector_length_in_bytes,
1789 FloatRegister vtmp) {
1790 assert(vector_length_in_bytes == 8 || vector_length_in_bytes == 16, "unsupported");
1791 assert_different_registers(dst, isrc);
1792 bool isQ = vector_length_in_bytes == 16;
1793
1794 BLOCK_COMMENT("neon_reduce_add_integral {");
1795 switch(bt) {
1796 case T_BYTE:
1797 addv(vtmp, isQ ? T16B : T8B, vsrc);
1798 smov(dst, vtmp, B, 0);
1799 addw(dst, dst, isrc, ext::sxtb);
1800 break;
1801 case T_SHORT:
1802 addv(vtmp, isQ ? T8H : T4H, vsrc);
1803 smov(dst, vtmp, H, 0);
1804 addw(dst, dst, isrc, ext::sxth);
1805 break;
1806 case T_INT:
1807 isQ ? addv(vtmp, T4S, vsrc) : addpv(vtmp, T2S, vsrc, vsrc);
1808 umov(dst, vtmp, S, 0);
1809 addw(dst, dst, isrc);
1810 break;
1811 case T_LONG:
1812 assert(isQ, "unsupported");
1813 addpd(vtmp, vsrc);
1814 umov(dst, vtmp, D, 0);
1815 add(dst, dst, isrc);
1816 break;
1817 default:
1818 assert(false, "unsupported");
1819 ShouldNotReachHere();
1820 }
1821 BLOCK_COMMENT("} neon_reduce_add_integral");
1822 }
1823
1824 // Vector reduction multiply for integral type with ASIMD instructions.
1825 // Note: temporary registers vtmp1 and vtmp2 are not used in some cases.
1826 // Clobbers: rscratch1
1827 void C2_MacroAssembler::neon_reduce_mul_integral(Register dst, BasicType bt,
1828 Register isrc, FloatRegister vsrc,
1829 unsigned vector_length_in_bytes,
1830 FloatRegister vtmp1, FloatRegister vtmp2) {
1831 assert(vector_length_in_bytes == 8 || vector_length_in_bytes == 16, "unsupported");
1832 bool isQ = vector_length_in_bytes == 16;
1833
1834 BLOCK_COMMENT("neon_reduce_mul_integral {");
1835 switch(bt) {
1836 case T_BYTE:
1837 if (isQ) {
1838 // Multiply the lower half and higher half of vector iteratively.
1839 // vtmp1 = vsrc[8:15]
1840 ins(vtmp1, D, vsrc, 0, 1);
1841 // vtmp1[n] = vsrc[n] * vsrc[n + 8], where n=[0, 7]
1842 mulv(vtmp1, T8B, vtmp1, vsrc);
1843 // vtmp2 = vtmp1[4:7]
1844 ins(vtmp2, S, vtmp1, 0, 1);
1845 // vtmp1[n] = vtmp1[n] * vtmp1[n + 4], where n=[0, 3]
1846 mulv(vtmp1, T8B, vtmp2, vtmp1);
1847 } else {
1848 ins(vtmp1, S, vsrc, 0, 1);
1849 mulv(vtmp1, T8B, vtmp1, vsrc);
1850 }
1851 // vtmp2 = vtmp1[2:3]
1852 ins(vtmp2, H, vtmp1, 0, 1);
1853 // vtmp2[n] = vtmp1[n] * vtmp1[n + 2], where n=[0, 1]
1854 mulv(vtmp2, T8B, vtmp2, vtmp1);
1855 // dst = vtmp2[0] * isrc * vtmp2[1]
1856 umov(rscratch1, vtmp2, B, 0);
1857 mulw(dst, rscratch1, isrc);
1858 sxtb(dst, dst);
1859 umov(rscratch1, vtmp2, B, 1);
1860 mulw(dst, rscratch1, dst);
1861 sxtb(dst, dst);
1862 break;
1863 case T_SHORT:
1864 if (isQ) {
1865 ins(vtmp2, D, vsrc, 0, 1);
1866 mulv(vtmp2, T4H, vtmp2, vsrc);
1867 ins(vtmp1, S, vtmp2, 0, 1);
1868 mulv(vtmp1, T4H, vtmp1, vtmp2);
1869 } else {
1870 ins(vtmp1, S, vsrc, 0, 1);
1871 mulv(vtmp1, T4H, vtmp1, vsrc);
1872 }
1873 umov(rscratch1, vtmp1, H, 0);
1874 mulw(dst, rscratch1, isrc);
1875 sxth(dst, dst);
1876 umov(rscratch1, vtmp1, H, 1);
1877 mulw(dst, rscratch1, dst);
1878 sxth(dst, dst);
1879 break;
1880 case T_INT:
1881 if (isQ) {
1882 ins(vtmp1, D, vsrc, 0, 1);
1883 mulv(vtmp1, T2S, vtmp1, vsrc);
1884 } else {
1885 vtmp1 = vsrc;
1886 }
1887 umov(rscratch1, vtmp1, S, 0);
1888 mul(dst, rscratch1, isrc);
1889 umov(rscratch1, vtmp1, S, 1);
1890 mul(dst, rscratch1, dst);
1891 break;
1892 case T_LONG:
1893 umov(rscratch1, vsrc, D, 0);
1894 mul(dst, isrc, rscratch1);
1895 umov(rscratch1, vsrc, D, 1);
1896 mul(dst, dst, rscratch1);
1897 break;
1898 default:
1899 assert(false, "unsupported");
1900 ShouldNotReachHere();
1901 }
1902 BLOCK_COMMENT("} neon_reduce_mul_integral");
1903 }
1904
1905 // Vector reduction multiply for floating-point type with ASIMD instructions.
1906 void C2_MacroAssembler::neon_reduce_mul_fp(FloatRegister dst, BasicType bt,
1907 FloatRegister fsrc, FloatRegister vsrc,
1908 unsigned vector_length_in_bytes,
1909 FloatRegister vtmp) {
1910 assert(vector_length_in_bytes == 8 || vector_length_in_bytes == 16, "unsupported");
1911 bool isQ = vector_length_in_bytes == 16;
1912
1913 BLOCK_COMMENT("neon_reduce_mul_fp {");
1914 switch(bt) {
1915 case T_FLOAT:
1916 fmuls(dst, fsrc, vsrc);
1917 ins(vtmp, S, vsrc, 0, 1);
1918 fmuls(dst, dst, vtmp);
1919 if (isQ) {
1920 ins(vtmp, S, vsrc, 0, 2);
1921 fmuls(dst, dst, vtmp);
1922 ins(vtmp, S, vsrc, 0, 3);
1923 fmuls(dst, dst, vtmp);
1924 }
1925 break;
1926 case T_DOUBLE:
1927 assert(isQ, "unsupported");
1928 fmuld(dst, fsrc, vsrc);
1929 ins(vtmp, D, vsrc, 0, 1);
1930 fmuld(dst, dst, vtmp);
1931 break;
1932 default:
1933 assert(false, "unsupported");
1934 ShouldNotReachHere();
1935 }
1936 BLOCK_COMMENT("} neon_reduce_mul_fp");
1937 }
1938
1939 // Helper to select logical instruction
1940 void C2_MacroAssembler::neon_reduce_logical_helper(int opc, bool is64, Register Rd,
1941 Register Rn, Register Rm,
1942 enum shift_kind kind, unsigned shift) {
1943 switch(opc) {
1944 case Op_AndReductionV:
1945 is64 ? andr(Rd, Rn, Rm, kind, shift) : andw(Rd, Rn, Rm, kind, shift);
1946 break;
1947 case Op_OrReductionV:
1948 is64 ? orr(Rd, Rn, Rm, kind, shift) : orrw(Rd, Rn, Rm, kind, shift);
1949 break;
1950 case Op_XorReductionV:
1951 is64 ? eor(Rd, Rn, Rm, kind, shift) : eorw(Rd, Rn, Rm, kind, shift);
1952 break;
1953 default:
1954 assert(false, "unsupported");
1955 ShouldNotReachHere();
1956 }
1957 }
1958
1959 // Vector reduction logical operations And, Or, Xor
1960 // Clobbers: rscratch1
1961 void C2_MacroAssembler::neon_reduce_logical(int opc, Register dst, BasicType bt,
1962 Register isrc, FloatRegister vsrc,
1963 unsigned vector_length_in_bytes) {
1964 assert(opc == Op_AndReductionV || opc == Op_OrReductionV || opc == Op_XorReductionV,
1965 "unsupported");
1966 assert(vector_length_in_bytes == 8 || vector_length_in_bytes == 16, "unsupported");
1967 assert_different_registers(dst, isrc);
1968 bool isQ = vector_length_in_bytes == 16;
1969
1970 BLOCK_COMMENT("neon_reduce_logical {");
1971 umov(rscratch1, vsrc, isQ ? D : S, 0);
1972 umov(dst, vsrc, isQ ? D : S, 1);
1973 neon_reduce_logical_helper(opc, /* is64 */ true, dst, dst, rscratch1);
1974 switch(bt) {
1975 case T_BYTE:
1976 if (isQ) {
1977 neon_reduce_logical_helper(opc, /* is64 */ true, dst, dst, dst, Assembler::LSR, 32);
1978 }
1979 neon_reduce_logical_helper(opc, /* is64 */ false, dst, dst, dst, Assembler::LSR, 16);
1980 neon_reduce_logical_helper(opc, /* is64 */ false, dst, dst, dst, Assembler::LSR, 8);
1981 neon_reduce_logical_helper(opc, /* is64 */ false, dst, isrc, dst);
1982 sxtb(dst, dst);
1983 break;
1984 case T_SHORT:
1985 if (isQ) {
1986 neon_reduce_logical_helper(opc, /* is64 */ true, dst, dst, dst, Assembler::LSR, 32);
1987 }
1988 neon_reduce_logical_helper(opc, /* is64 */ false, dst, dst, dst, Assembler::LSR, 16);
1989 neon_reduce_logical_helper(opc, /* is64 */ false, dst, isrc, dst);
1990 sxth(dst, dst);
1991 break;
1992 case T_INT:
1993 if (isQ) {
1994 neon_reduce_logical_helper(opc, /* is64 */ true, dst, dst, dst, Assembler::LSR, 32);
1995 }
1996 neon_reduce_logical_helper(opc, /* is64 */ false, dst, isrc, dst);
1997 break;
1998 case T_LONG:
1999 assert(isQ, "unsupported");
2000 neon_reduce_logical_helper(opc, /* is64 */ true, dst, isrc, dst);
2001 break;
2002 default:
2003 assert(false, "unsupported");
2004 ShouldNotReachHere();
2005 }
2006 BLOCK_COMMENT("} neon_reduce_logical");
2007 }
2008
2009 // Vector reduction min/max for integral type with ASIMD instructions.
2010 // Note: vtmp is not used and expected to be fnoreg for T_LONG case.
2011 // Clobbers: rscratch1, rflags
2012 void C2_MacroAssembler::neon_reduce_minmax_integral(int opc, Register dst, BasicType bt,
2013 Register isrc, FloatRegister vsrc,
2014 unsigned vector_length_in_bytes,
2015 FloatRegister vtmp) {
2016 assert(opc == Op_MinReductionV || opc == Op_MaxReductionV, "unsupported");
2017 assert(vector_length_in_bytes == 8 || vector_length_in_bytes == 16, "unsupported");
2018 assert(bt == T_BYTE || bt == T_SHORT || bt == T_INT || bt == T_LONG, "unsupported");
2019 assert_different_registers(dst, isrc);
2020 bool isQ = vector_length_in_bytes == 16;
2021 bool is_min = opc == Op_MinReductionV;
2022
2023 BLOCK_COMMENT("neon_reduce_minmax_integral {");
2024 if (bt == T_LONG) {
2025 assert(vtmp == fnoreg, "should be");
2026 assert(isQ, "should be");
2027 umov(rscratch1, vsrc, D, 0);
2028 cmp(isrc, rscratch1);
2029 csel(dst, isrc, rscratch1, is_min ? LT : GT);
2030 umov(rscratch1, vsrc, D, 1);
2031 cmp(dst, rscratch1);
2032 csel(dst, dst, rscratch1, is_min ? LT : GT);
2033 } else {
2034 SIMD_Arrangement size = esize2arrangement((unsigned)type2aelembytes(bt), isQ);
2035 if (size == T2S) {
2036 is_min ? sminp(vtmp, size, vsrc, vsrc) : smaxp(vtmp, size, vsrc, vsrc);
2037 } else {
2038 is_min ? sminv(vtmp, size, vsrc) : smaxv(vtmp, size, vsrc);
2039 }
2040 if (bt == T_INT) {
2041 umov(dst, vtmp, S, 0);
2042 } else {
2043 smov(dst, vtmp, elemType_to_regVariant(bt), 0);
2044 }
2045 cmpw(dst, isrc);
2046 cselw(dst, dst, isrc, is_min ? LT : GT);
2047 }
2048 BLOCK_COMMENT("} neon_reduce_minmax_integral");
2049 }
2050
2051 // Vector reduction for integral type with SVE instruction.
2052 // Supported operations are Add, And, Or, Xor, Max, Min.
2053 // rflags would be clobbered if opc is Op_MaxReductionV or Op_MinReductionV.
2054 void C2_MacroAssembler::sve_reduce_integral(int opc, Register dst, BasicType bt, Register src1,
2055 FloatRegister src2, PRegister pg, FloatRegister tmp) {
2056 assert(bt == T_BYTE || bt == T_SHORT || bt == T_INT || bt == T_LONG, "unsupported element type");
2057 assert(pg->is_governing(), "This register has to be a governing predicate register");
2058 assert_different_registers(src1, dst);
2059 // Register "dst" and "tmp" are to be clobbered, and "src1" and "src2" should be preserved.
2060 Assembler::SIMD_RegVariant size = elemType_to_regVariant(bt);
2061 switch (opc) {
2062 case Op_AddReductionVI: {
2063 sve_uaddv(tmp, size, pg, src2);
2064 if (bt == T_BYTE) {
2065 smov(dst, tmp, size, 0);
2066 addw(dst, src1, dst, ext::sxtb);
2067 } else if (bt == T_SHORT) {
2068 smov(dst, tmp, size, 0);
2069 addw(dst, src1, dst, ext::sxth);
2070 } else {
2071 umov(dst, tmp, size, 0);
2072 addw(dst, dst, src1);
2073 }
2074 break;
2075 }
2076 case Op_AddReductionVL: {
2077 sve_uaddv(tmp, size, pg, src2);
2078 umov(dst, tmp, size, 0);
2079 add(dst, dst, src1);
2080 break;
2081 }
2082 case Op_AndReductionV: {
2083 sve_andv(tmp, size, pg, src2);
2084 if (bt == T_INT || bt == T_LONG) {
2085 umov(dst, tmp, size, 0);
2086 } else {
2087 smov(dst, tmp, size, 0);
2088 }
2089 if (bt == T_LONG) {
2090 andr(dst, dst, src1);
2091 } else {
2092 andw(dst, dst, src1);
2093 }
2094 break;
2095 }
2096 case Op_OrReductionV: {
2097 sve_orv(tmp, size, pg, src2);
2098 if (bt == T_INT || bt == T_LONG) {
2099 umov(dst, tmp, size, 0);
2100 } else {
2101 smov(dst, tmp, size, 0);
2102 }
2103 if (bt == T_LONG) {
2104 orr(dst, dst, src1);
2105 } else {
2106 orrw(dst, dst, src1);
2107 }
2108 break;
2109 }
2110 case Op_XorReductionV: {
2111 sve_eorv(tmp, size, pg, src2);
2112 if (bt == T_INT || bt == T_LONG) {
2113 umov(dst, tmp, size, 0);
2114 } else {
2115 smov(dst, tmp, size, 0);
2116 }
2117 if (bt == T_LONG) {
2118 eor(dst, dst, src1);
2119 } else {
2120 eorw(dst, dst, src1);
2121 }
2122 break;
2123 }
2124 case Op_MaxReductionV: {
2125 sve_smaxv(tmp, size, pg, src2);
2126 if (bt == T_INT || bt == T_LONG) {
2127 umov(dst, tmp, size, 0);
2128 } else {
2129 smov(dst, tmp, size, 0);
2130 }
2131 if (bt == T_LONG) {
2132 cmp(dst, src1);
2133 csel(dst, dst, src1, Assembler::GT);
2134 } else {
2135 cmpw(dst, src1);
2136 cselw(dst, dst, src1, Assembler::GT);
2137 }
2138 break;
2139 }
2140 case Op_MinReductionV: {
2141 sve_sminv(tmp, size, pg, src2);
2142 if (bt == T_INT || bt == T_LONG) {
2143 umov(dst, tmp, size, 0);
2144 } else {
2145 smov(dst, tmp, size, 0);
2146 }
2147 if (bt == T_LONG) {
2148 cmp(dst, src1);
2149 csel(dst, dst, src1, Assembler::LT);
2150 } else {
2151 cmpw(dst, src1);
2152 cselw(dst, dst, src1, Assembler::LT);
2153 }
2154 break;
2155 }
2156 default:
2157 assert(false, "unsupported");
2158 ShouldNotReachHere();
2159 }
2160
2161 if (opc == Op_AndReductionV || opc == Op_OrReductionV || opc == Op_XorReductionV) {
2162 if (bt == T_BYTE) {
2163 sxtb(dst, dst);
2164 } else if (bt == T_SHORT) {
2165 sxth(dst, dst);
2166 }
2167 }
2168 }
2169
2170 // Set elements of the dst predicate to true for lanes in the range of [0, lane_cnt), or
2171 // to false otherwise. The input "lane_cnt" should be smaller than or equal to the supported
2172 // max vector length of the basic type. Clobbers: rscratch1 and the rFlagsReg.
2173 void C2_MacroAssembler::sve_gen_mask_imm(PRegister dst, BasicType bt, uint32_t lane_cnt) {
2174 uint32_t max_vector_length = Matcher::max_vector_size(bt);
2175 assert(lane_cnt <= max_vector_length, "unsupported input lane_cnt");
2176
2177 // Set all elements to false if the input "lane_cnt" is zero.
2178 if (lane_cnt == 0) {
2179 sve_pfalse(dst);
2180 return;
2181 }
2182
2183 SIMD_RegVariant size = elemType_to_regVariant(bt);
2184 assert(size != Q, "invalid size");
2185
2186 // Set all true if "lane_cnt" equals to the max lane count.
2187 if (lane_cnt == max_vector_length) {
2188 sve_ptrue(dst, size, /* ALL */ 0b11111);
2189 return;
2190 }
2191
2192 // Fixed numbers for "ptrue".
2193 switch(lane_cnt) {
2194 case 1: /* VL1 */
2195 case 2: /* VL2 */
2196 case 3: /* VL3 */
2197 case 4: /* VL4 */
2198 case 5: /* VL5 */
2199 case 6: /* VL6 */
2200 case 7: /* VL7 */
2201 case 8: /* VL8 */
2202 sve_ptrue(dst, size, lane_cnt);
2203 return;
2204 case 16:
2205 sve_ptrue(dst, size, /* VL16 */ 0b01001);
2206 return;
2207 case 32:
2208 sve_ptrue(dst, size, /* VL32 */ 0b01010);
2209 return;
2210 case 64:
2211 sve_ptrue(dst, size, /* VL64 */ 0b01011);
2212 return;
2213 case 128:
2214 sve_ptrue(dst, size, /* VL128 */ 0b01100);
2215 return;
2216 case 256:
2217 sve_ptrue(dst, size, /* VL256 */ 0b01101);
2218 return;
2219 default:
2220 break;
2221 }
2222
2223 // Special patterns for "ptrue".
2224 if (lane_cnt == round_down_power_of_2(max_vector_length)) {
2225 sve_ptrue(dst, size, /* POW2 */ 0b00000);
2226 } else if (lane_cnt == max_vector_length - (max_vector_length % 4)) {
2227 sve_ptrue(dst, size, /* MUL4 */ 0b11101);
2228 } else if (lane_cnt == max_vector_length - (max_vector_length % 3)) {
2229 sve_ptrue(dst, size, /* MUL3 */ 0b11110);
2230 } else {
2231 // Encode to "whileltw" for the remaining cases.
2232 mov(rscratch1, lane_cnt);
2233 sve_whileltw(dst, size, zr, rscratch1);
2234 }
2235 }
2236
2237 // Pack active elements of src, under the control of mask, into the lowest-numbered elements of dst.
2238 // Any remaining elements of dst will be filled with zero.
2239 // Clobbers: rscratch1
2240 // Preserves: src, mask
2241 void C2_MacroAssembler::sve_compress_short(FloatRegister dst, FloatRegister src, PRegister mask,
2242 FloatRegister vtmp1, FloatRegister vtmp2,
2243 PRegister pgtmp) {
2244 assert(pgtmp->is_governing(), "This register has to be a governing predicate register");
2245 assert_different_registers(dst, src, vtmp1, vtmp2);
2246 assert_different_registers(mask, pgtmp);
2247
2248 // Example input: src = 8888 7777 6666 5555 4444 3333 2222 1111
2249 // mask = 0001 0000 0000 0001 0001 0000 0001 0001
2250 // Expected result: dst = 0000 0000 0000 8888 5555 4444 2222 1111
2251 sve_dup(vtmp2, H, 0);
2252
2253 // Extend lowest half to type INT.
2254 // dst = 00004444 00003333 00002222 00001111
2255 sve_uunpklo(dst, S, src);
2256 // pgtmp = 00000001 00000000 00000001 00000001
2257 sve_punpklo(pgtmp, mask);
2258 // Pack the active elements in size of type INT to the right,
2259 // and fill the remainings with zero.
2260 // dst = 00000000 00004444 00002222 00001111
2261 sve_compact(dst, S, dst, pgtmp);
2262 // Narrow the result back to type SHORT.
2263 // dst = 0000 0000 0000 0000 0000 4444 2222 1111
2264 sve_uzp1(dst, H, dst, vtmp2);
2265 // Count the active elements of lowest half.
2266 // rscratch1 = 3
2267 sve_cntp(rscratch1, S, ptrue, pgtmp);
2268
2269 // Repeat to the highest half.
2270 // pgtmp = 00000001 00000000 00000000 00000001
2271 sve_punpkhi(pgtmp, mask);
2272 // vtmp1 = 00008888 00007777 00006666 00005555
2273 sve_uunpkhi(vtmp1, S, src);
2274 // vtmp1 = 00000000 00000000 00008888 00005555
2275 sve_compact(vtmp1, S, vtmp1, pgtmp);
2276 // vtmp1 = 0000 0000 0000 0000 0000 0000 8888 5555
2277 sve_uzp1(vtmp1, H, vtmp1, vtmp2);
2278
2279 // Compressed low: dst = 0000 0000 0000 0000 0000 4444 2222 1111
2280 // Compressed high: vtmp1 = 0000 0000 0000 0000 0000 0000 8888 5555
2281 // Left shift(cross lane) compressed high with TRUE_CNT lanes,
2282 // TRUE_CNT is the number of active elements in the compressed low.
2283 neg(rscratch1, rscratch1);
2284 // vtmp2 = {4 3 2 1 0 -1 -2 -3}
2285 sve_index(vtmp2, H, rscratch1, 1);
2286 // vtmp1 = 0000 0000 0000 8888 5555 0000 0000 0000
2287 sve_tbl(vtmp1, H, vtmp1, vtmp2);
2288
2289 // Combine the compressed high(after shifted) with the compressed low.
2290 // dst = 0000 0000 0000 8888 5555 4444 2222 1111
2291 sve_orr(dst, dst, vtmp1);
2292 }
2293
2294 // Clobbers: rscratch1, rscratch2
2295 // Preserves: src, mask
2296 void C2_MacroAssembler::sve_compress_byte(FloatRegister dst, FloatRegister src, PRegister mask,
2297 FloatRegister vtmp1, FloatRegister vtmp2,
2298 FloatRegister vtmp3, FloatRegister vtmp4,
2299 PRegister ptmp, PRegister pgtmp) {
2300 assert(pgtmp->is_governing(), "This register has to be a governing predicate register");
2301 assert_different_registers(dst, src, vtmp1, vtmp2, vtmp3, vtmp4);
2302 assert_different_registers(mask, ptmp, pgtmp);
2303 // Example input: src = 88 77 66 55 44 33 22 11
2304 // mask = 01 00 00 01 01 00 01 01
2305 // Expected result: dst = 00 00 00 88 55 44 22 11
2306
2307 sve_dup(vtmp4, B, 0);
2308 // Extend lowest half to type SHORT.
2309 // vtmp1 = 0044 0033 0022 0011
2310 sve_uunpklo(vtmp1, H, src);
2311 // ptmp = 0001 0000 0001 0001
2312 sve_punpklo(ptmp, mask);
2313 // Count the active elements of lowest half.
2314 // rscratch2 = 3
2315 sve_cntp(rscratch2, H, ptrue, ptmp);
2316 // Pack the active elements in size of type SHORT to the right,
2317 // and fill the remainings with zero.
2318 // dst = 0000 0044 0022 0011
2319 sve_compress_short(dst, vtmp1, ptmp, vtmp2, vtmp3, pgtmp);
2320 // Narrow the result back to type BYTE.
2321 // dst = 00 00 00 00 00 44 22 11
2322 sve_uzp1(dst, B, dst, vtmp4);
2323
2324 // Repeat to the highest half.
2325 // ptmp = 0001 0000 0000 0001
2326 sve_punpkhi(ptmp, mask);
2327 // vtmp1 = 0088 0077 0066 0055
2328 sve_uunpkhi(vtmp2, H, src);
2329 // vtmp1 = 0000 0000 0088 0055
2330 sve_compress_short(vtmp1, vtmp2, ptmp, vtmp3, vtmp4, pgtmp);
2331
2332 sve_dup(vtmp4, B, 0);
2333 // vtmp1 = 00 00 00 00 00 00 88 55
2334 sve_uzp1(vtmp1, B, vtmp1, vtmp4);
2335
2336 // Compressed low: dst = 00 00 00 00 00 44 22 11
2337 // Compressed high: vtmp1 = 00 00 00 00 00 00 88 55
2338 // Left shift(cross lane) compressed high with TRUE_CNT lanes,
2339 // TRUE_CNT is the number of active elements in the compressed low.
2340 neg(rscratch2, rscratch2);
2341 // vtmp2 = {4 3 2 1 0 -1 -2 -3}
2342 sve_index(vtmp2, B, rscratch2, 1);
2343 // vtmp1 = 00 00 00 88 55 00 00 00
2344 sve_tbl(vtmp1, B, vtmp1, vtmp2);
2345 // Combine the compressed high(after shifted) with the compressed low.
2346 // dst = 00 00 00 88 55 44 22 11
2347 sve_orr(dst, dst, vtmp1);
2348 }
2349
2350 void C2_MacroAssembler::neon_reverse_bits(FloatRegister dst, FloatRegister src, BasicType bt, bool isQ) {
2351 assert(bt == T_BYTE || bt == T_SHORT || bt == T_INT || bt == T_LONG, "unsupported basic type");
2352 SIMD_Arrangement size = isQ ? T16B : T8B;
2353 if (bt == T_BYTE) {
2354 rbit(dst, size, src);
2355 } else {
2356 neon_reverse_bytes(dst, src, bt, isQ);
2357 rbit(dst, size, dst);
2358 }
2359 }
2360
2361 void C2_MacroAssembler::neon_reverse_bytes(FloatRegister dst, FloatRegister src, BasicType bt, bool isQ) {
2362 assert(bt == T_BYTE || bt == T_SHORT || bt == T_INT || bt == T_LONG, "unsupported basic type");
2363 SIMD_Arrangement size = isQ ? T16B : T8B;
2364 switch (bt) {
2365 case T_BYTE:
2366 if (dst != src) {
2367 orr(dst, size, src, src);
2368 }
2369 break;
2370 case T_SHORT:
2371 rev16(dst, size, src);
2372 break;
2373 case T_INT:
2374 rev32(dst, size, src);
2375 break;
2376 case T_LONG:
2377 rev64(dst, size, src);
2378 break;
2379 default:
2380 assert(false, "unsupported");
2381 ShouldNotReachHere();
2382 }
2383 }
2384
2385 // Extract a scalar element from an sve vector at position 'idx'.
2386 // The input elements in src are expected to be of integral type.
2387 void C2_MacroAssembler::sve_extract_integral(Register dst, BasicType bt, FloatRegister src,
2388 int idx, FloatRegister vtmp) {
2389 assert(bt == T_BYTE || bt == T_SHORT || bt == T_INT || bt == T_LONG, "unsupported element type");
2390 Assembler::SIMD_RegVariant size = elemType_to_regVariant(bt);
2391 if (regVariant_to_elemBits(size) * idx < 128) { // generate lower cost NEON instruction
2392 if (bt == T_INT || bt == T_LONG) {
2393 umov(dst, src, size, idx);
2394 } else {
2395 smov(dst, src, size, idx);
2396 }
2397 } else {
2398 sve_orr(vtmp, src, src);
2399 sve_ext(vtmp, vtmp, idx << size);
2400 if (bt == T_INT || bt == T_LONG) {
2401 umov(dst, vtmp, size, 0);
2402 } else {
2403 smov(dst, vtmp, size, 0);
2404 }
2405 }
2406 }
2407
2408 // java.lang.Math::round intrinsics
2409
2410 // Clobbers: rscratch1, rflags
2411 void C2_MacroAssembler::vector_round_neon(FloatRegister dst, FloatRegister src, FloatRegister tmp1,
2412 FloatRegister tmp2, FloatRegister tmp3, SIMD_Arrangement T) {
2413 assert_different_registers(tmp1, tmp2, tmp3, src, dst);
2414 switch (T) {
2415 case T2S:
2416 case T4S:
2417 fmovs(tmp1, T, 0.5f);
2418 mov(rscratch1, jint_cast(0x1.0p23f));
2419 break;
2420 case T2D:
2421 fmovd(tmp1, T, 0.5);
2422 mov(rscratch1, julong_cast(0x1.0p52));
2423 break;
2424 default:
2425 assert(T == T2S || T == T4S || T == T2D, "invalid arrangement");
2426 }
2427 fadd(tmp1, T, tmp1, src);
2428 fcvtms(tmp1, T, tmp1);
2429 // tmp1 = floor(src + 0.5, ties to even)
2430
2431 fcvtas(dst, T, src);
2432 // dst = round(src), ties to away
2433
2434 fneg(tmp3, T, src);
2435 dup(tmp2, T, rscratch1);
2436 cm(HS, tmp3, T, tmp3, tmp2);
2437 // tmp3 is now a set of flags
2438
2439 bif(dst, T16B, tmp1, tmp3);
2440 // result in dst
2441 }
2442
2443 // Clobbers: rscratch1, rflags
2444 void C2_MacroAssembler::vector_round_sve(FloatRegister dst, FloatRegister src, FloatRegister tmp1,
2445 FloatRegister tmp2, PRegister pgtmp, SIMD_RegVariant T) {
2446 assert(pgtmp->is_governing(), "This register has to be a governing predicate register");
2447 assert_different_registers(tmp1, tmp2, src, dst);
2448
2449 switch (T) {
2450 case S:
2451 mov(rscratch1, jint_cast(0x1.0p23f));
2452 break;
2453 case D:
2454 mov(rscratch1, julong_cast(0x1.0p52));
2455 break;
2456 default:
2457 assert(T == S || T == D, "invalid register variant");
2458 }
2459
2460 sve_frinta(dst, T, ptrue, src);
2461 // dst = round(src), ties to away
2462
2463 Label none;
2464
2465 sve_fneg(tmp1, T, ptrue, src);
2466 sve_dup(tmp2, T, rscratch1);
2467 sve_cmp(HS, pgtmp, T, ptrue, tmp2, tmp1);
2468 br(EQ, none);
2469 {
2470 sve_cpy(tmp1, T, pgtmp, 0.5);
2471 sve_fadd(tmp1, T, pgtmp, src);
2472 sve_frintm(dst, T, pgtmp, tmp1);
2473 // dst = floor(src + 0.5, ties to even)
2474 }
2475 bind(none);
2476
2477 sve_fcvtzs(dst, T, ptrue, dst, T);
2478 // result in dst
2479 }
2480
2481 void C2_MacroAssembler::vector_signum_neon(FloatRegister dst, FloatRegister src, FloatRegister zero,
2482 FloatRegister one, SIMD_Arrangement T) {
2483 assert_different_registers(dst, src, zero, one);
2484 assert(T == T2S || T == T4S || T == T2D, "invalid arrangement");
2485
2486 facgt(dst, T, src, zero);
2487 ushr(dst, T, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
2488 bsl(dst, T == T2S ? T8B : T16B, one, src); // Result in dst
2489 }
2490
2491 void C2_MacroAssembler::vector_signum_sve(FloatRegister dst, FloatRegister src, FloatRegister zero,
2492 FloatRegister one, FloatRegister vtmp, PRegister pgtmp, SIMD_RegVariant T) {
2493 assert_different_registers(dst, src, zero, one, vtmp);
2494 assert(pgtmp->is_governing(), "This register has to be a governing predicate register");
2495
2496 sve_orr(vtmp, src, src);
2497 sve_fac(Assembler::GT, pgtmp, T, ptrue, src, zero); // pmtp=0 for +-0.0 and NaN. 0x1 otherwise
2498 switch (T) {
2499 case S:
2500 sve_and(vtmp, T, min_jint); // Extract the sign bit of float value in every lane of src
2501 sve_orr(vtmp, T, jint_cast(1.0)); // OR it with +1 to make the final result +1 or -1 depending
2502 // on the sign of the float value
2503 break;
2504 case D:
2505 sve_and(vtmp, T, min_jlong);
2506 sve_orr(vtmp, T, jlong_cast(1.0));
2507 break;
2508 default:
2509 assert(false, "unsupported");
2510 ShouldNotReachHere();
2511 }
2512 sve_sel(dst, T, pgtmp, vtmp, src); // Select either from src or vtmp based on the predicate register pgtmp
2513 // Result in dst
2514 }
2515
2516 bool C2_MacroAssembler::in_scratch_emit_size() {
2517 if (ciEnv::current()->task() != nullptr) {
2518 PhaseOutput* phase_output = Compile::current()->output();
2519 if (phase_output != nullptr && phase_output->in_scratch_emit_size()) {
2520 return true;
2521 }
2522 }
2523 return MacroAssembler::in_scratch_emit_size();
2524 }