1 /*
2 * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "asm/assembler.hpp"
28 #include "asm/assembler.inline.hpp"
29 #include "opto/c2_MacroAssembler.hpp"
30 #include "opto/compile.hpp"
31 #include "opto/intrinsicnode.hpp"
32 #include "opto/output.hpp"
33 #include "opto/subnode.hpp"
34 #include "runtime/stubRoutines.hpp"
35 #include "utilities/globalDefinitions.hpp"
36
37 #ifdef PRODUCT
38 #define BLOCK_COMMENT(str) /* nothing */
39 #define STOP(error) stop(error)
40 #else
41 #define BLOCK_COMMENT(str) block_comment(str)
42 #define STOP(error) block_comment(error); stop(error)
43 #endif
44
45 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
46
47 void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg,
48 Register tmp1Reg, Register tmp2Reg, Register tmp3Reg) {
49 // Use cr register to indicate the fast_lock result: zero for success; non-zero for failure.
50 Register flag = t1;
51 Register oop = objectReg;
52 Register box = boxReg;
53 Register disp_hdr = tmp1Reg;
54 Register tmp = tmp2Reg;
55 Label object_has_monitor;
56 // Finish fast lock successfully. MUST branch to with flag == 0
57 Label locked;
58 // Finish fast lock unsuccessfully. slow_path MUST branch to with flag != 0
59 Label slow_path;
60
61 assert(LockingMode != LM_LIGHTWEIGHT, "lightweight locking should use fast_lock_lightweight");
62 assert_different_registers(oop, box, tmp, disp_hdr, flag, tmp3Reg, t0);
63
64 mv(flag, 1);
65
66 // Load markWord from object into displaced_header.
67 ld(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
68
69 if (DiagnoseSyncOnValueBasedClasses != 0) {
70 load_klass(tmp, oop);
71 lwu(tmp, Address(tmp, Klass::access_flags_offset()));
72 test_bit(tmp, tmp, exact_log2(JVM_ACC_IS_VALUE_BASED_CLASS));
73 bnez(tmp, slow_path);
74 }
75
76 // Check for existing monitor
77 test_bit(tmp, disp_hdr, exact_log2(markWord::monitor_value));
78 bnez(tmp, object_has_monitor);
79
80 if (LockingMode == LM_MONITOR) {
81 j(slow_path);
82 } else {
83 assert(LockingMode == LM_LEGACY, "must be");
84 // Set tmp to be (markWord of object | UNLOCK_VALUE).
85 ori(tmp, disp_hdr, markWord::unlocked_value);
86
87 // Initialize the box. (Must happen before we update the object mark!)
88 sd(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
89
90 // Compare object markWord with an unlocked value (tmp) and if
91 // equal exchange the stack address of our box with object markWord.
92 // On failure disp_hdr contains the possibly locked markWord.
93 cmpxchg(/*memory address*/oop, /*expected value*/tmp, /*new value*/box, Assembler::int64,
94 Assembler::aq, Assembler::rl, /*result*/disp_hdr);
95 beq(disp_hdr, tmp, locked);
96
97 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
98
99 // If the compare-and-exchange succeeded, then we found an unlocked
100 // object, will have now locked it will continue at label locked
101 // We did not see an unlocked object so try the fast recursive case.
102
103 // Check if the owner is self by comparing the value in the
104 // markWord of object (disp_hdr) with the stack pointer.
105 sub(disp_hdr, disp_hdr, sp);
106 mv(tmp, (intptr_t) (~(os::vm_page_size()-1) | (uintptr_t)markWord::lock_mask_in_place));
107 // If (mark & lock_mask) == 0 and mark - sp < page_size, we are stack-locking and goto label locked,
108 // hence we can store 0 as the displaced header in the box, which indicates that it is a
109 // recursive lock.
110 andr(tmp/*==0?*/, disp_hdr, tmp);
111 sd(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
112 beqz(tmp, locked);
113 j(slow_path);
114 }
115
116 // Handle existing monitor.
117 bind(object_has_monitor);
118 // The object's monitor m is unlocked iff m->owner == nullptr,
119 // otherwise m->owner may contain a thread or a stack address.
120 //
121 // Try to CAS m->owner from null to current thread id.
122 Register tid = flag;
123 mv(tid, Address(xthread, JavaThread::lock_id_offset()));
124 add(tmp, disp_hdr, (in_bytes(ObjectMonitor::owner_offset()) - markWord::monitor_value));
125 cmpxchg(/*memory address*/tmp, /*expected value*/zr, /*new value*/tid, Assembler::int64,
126 Assembler::aq, Assembler::rl, /*result*/tmp3Reg); // cas succeeds if tmp3Reg == zr(expected)
127
128 // Store a non-null value into the box to avoid looking like a re-entrant
129 // lock. The fast-path monitor unlock code checks for
130 // markWord::monitor_value so use markWord::unused_mark which has the
131 // relevant bit set, and also matches ObjectSynchronizer::slow_enter.
132 mv(tmp, (address)markWord::unused_mark().value());
133 sd(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
134
135 beqz(tmp3Reg, locked); // CAS success means locking succeeded
136
137 bne(tmp3Reg, tid, slow_path); // Check for recursive locking
138
139 // Recursive lock case
140 increment(Address(disp_hdr, in_bytes(ObjectMonitor::recursions_offset()) - markWord::monitor_value), 1, tmp2Reg, tmp3Reg);
141
142 bind(locked);
143 mv(flag, zr);
144 increment(Address(xthread, JavaThread::held_monitor_count_offset()), 1, tmp2Reg, tmp3Reg);
145
146 #ifdef ASSERT
147 // Check that locked label is reached with flag == 0.
148 Label flag_correct;
149 beqz(flag, flag_correct);
150 stop("Fast Lock Flag != 0");
151 #endif
152
153 bind(slow_path);
154 #ifdef ASSERT
155 // Check that slow_path label is reached with flag != 0.
156 bnez(flag, flag_correct);
157 stop("Fast Lock Flag == 0");
158 bind(flag_correct);
159 #endif
160 // C2 uses the value of flag (0 vs !0) to determine the continuation.
161 }
162
163 void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg,
164 Register tmp1Reg, Register tmp2Reg) {
165 // Use cr register to indicate the fast_unlock result: zero for success; non-zero for failure.
166 Register flag = t1;
167 Register oop = objectReg;
168 Register box = boxReg;
169 Register disp_hdr = tmp1Reg;
170 Register tmp = tmp2Reg;
171 Label object_has_monitor;
172 // Finish fast lock successfully. MUST branch to vwith flag == 0
173 Label unlocked;
174 // Finish fast lock unsuccessfully. slow_path MUST branch to with flag != 0
175 Label slow_path;
176
177 assert(LockingMode != LM_LIGHTWEIGHT, "lightweight locking should use fast_unlock_lightweight");
178 assert_different_registers(oop, box, tmp, disp_hdr, flag, t0);
179
180 mv(flag, 1);
181
182 if (LockingMode == LM_LEGACY) {
183 // Find the lock address and load the displaced header from the stack.
184 ld(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
185
186 // If the displaced header is 0, we have a recursive unlock.
187 beqz(disp_hdr, unlocked);
188 }
189
190 // Handle existing monitor.
191 ld(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
192 test_bit(t0, tmp, exact_log2(markWord::monitor_value));
193 bnez(t0, object_has_monitor);
194
195 if (LockingMode == LM_MONITOR) {
196 j(slow_path);
197 } else {
198 assert(LockingMode == LM_LEGACY, "must be");
199 // Check if it is still a light weight lock, this is true if we
200 // see the stack address of the basicLock in the markWord of the
201 // object.
202
203 cmpxchg(/*memory address*/oop, /*expected value*/box, /*new value*/disp_hdr, Assembler::int64,
204 Assembler::relaxed, Assembler::rl, /*result*/tmp);
205 beq(box, tmp, unlocked); // box == tmp if cas succeeds
206 j(slow_path);
207 }
208
209 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
210
211 // Handle existing monitor.
212 bind(object_has_monitor);
213 STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
214 add(tmp, tmp, -(int)markWord::monitor_value); // monitor
215
216 ld(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
217
218 Label notRecursive;
219 beqz(disp_hdr, notRecursive); // Will be 0 if not recursive.
220
221 // Recursive lock
222 addi(disp_hdr, disp_hdr, -1);
223 sd(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
224 j(unlocked);
225
226 bind(notRecursive);
227 ld(t0, Address(tmp, ObjectMonitor::EntryList_offset()));
228 ld(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset()));
229 orr(t0, t0, disp_hdr); // Will be 0 if both are 0.
230 bnez(t0, slow_path);
231
232 // need a release store here
233 la(tmp, Address(tmp, ObjectMonitor::owner_offset()));
234 membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore);
235 sd(zr, Address(tmp)); // set unowned
236
237 bind(unlocked);
238 mv(flag, zr);
239 decrement(Address(xthread, JavaThread::held_monitor_count_offset()), 1, tmp1Reg, tmp2Reg);
240
241 #ifdef ASSERT
242 // Check that unlocked label is reached with flag == 0.
243 Label flag_correct;
244 beqz(flag, flag_correct);
245 stop("Fast Lock Flag != 0");
246 #endif
247
248 bind(slow_path);
249 #ifdef ASSERT
250 // Check that slow_path label is reached with flag != 0.
251 bnez(flag, flag_correct);
252 stop("Fast Lock Flag == 0");
253 bind(flag_correct);
254 #endif
255 // C2 uses the value of flag (0 vs !0) to determine the continuation.
256 }
257
258 void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register tmp1, Register tmp2, Register tmp3) {
259 // Flag register, zero for success; non-zero for failure.
260 Register flag = t1;
261
262 assert(LockingMode == LM_LIGHTWEIGHT, "must be");
263 assert_different_registers(obj, tmp1, tmp2, tmp3, flag, t0);
264
265 mv(flag, 1);
266
267 // Handle inflated monitor.
268 Label inflated;
269 // Finish fast lock successfully. MUST branch to with flag == 0
270 Label locked;
271 // Finish fast lock unsuccessfully. slow_path MUST branch to with flag != 0
272 Label slow_path;
273
274 if (DiagnoseSyncOnValueBasedClasses != 0) {
275 load_klass(tmp1, obj);
276 lwu(tmp1, Address(tmp1, Klass::access_flags_offset()));
277 test_bit(tmp1, tmp1, exact_log2(JVM_ACC_IS_VALUE_BASED_CLASS));
278 bnez(tmp1, slow_path);
279 }
280
281 const Register tmp1_mark = tmp1;
282
283 { // Lightweight locking
284
285 // Push lock to the lock stack and finish successfully. MUST branch to with flag == 0
286 Label push;
287
288 const Register tmp2_top = tmp2;
289 const Register tmp3_t = tmp3;
290
291 // Check if lock-stack is full.
292 lwu(tmp2_top, Address(xthread, JavaThread::lock_stack_top_offset()));
293 mv(tmp3_t, (unsigned)LockStack::end_offset());
294 bge(tmp2_top, tmp3_t, slow_path);
295
296 // Check if recursive.
297 add(tmp3_t, xthread, tmp2_top);
298 ld(tmp3_t, Address(tmp3_t, -oopSize));
299 beq(obj, tmp3_t, push);
300
301 // Relaxed normal load to check for monitor. Optimization for monitor case.
302 ld(tmp1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
303 test_bit(tmp3_t, tmp1_mark, exact_log2(markWord::monitor_value));
304 bnez(tmp3_t, inflated);
305
306 // Not inflated
307 assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid a la");
308
309 // Try to lock. Transition lock-bits 0b01 => 0b00
310 ori(tmp1_mark, tmp1_mark, markWord::unlocked_value);
311 xori(tmp3_t, tmp1_mark, markWord::unlocked_value);
312 cmpxchg(/*addr*/ obj, /*expected*/ tmp1_mark, /*new*/ tmp3_t, Assembler::int64,
313 /*acquire*/ Assembler::aq, /*release*/ Assembler::relaxed, /*result*/ tmp3_t);
314 bne(tmp1_mark, tmp3_t, slow_path);
315
316 bind(push);
317 // After successful lock, push object on lock-stack.
318 add(tmp3_t, xthread, tmp2_top);
319 sd(obj, Address(tmp3_t));
320 addw(tmp2_top, tmp2_top, oopSize);
321 sw(tmp2_top, Address(xthread, JavaThread::lock_stack_top_offset()));
322 j(locked);
323 }
324
325 { // Handle inflated monitor.
326 bind(inflated);
327
328 // mark contains the tagged ObjectMonitor*.
329 const Register tmp1_tagged_monitor = tmp1_mark;
330 const uintptr_t monitor_tag = markWord::monitor_value;
331 const Register tmp2_owner_addr = tmp2;
332 const Register tmp3_owner = tmp3;
333
334 // Compute owner address.
335 la(tmp2_owner_addr, Address(tmp1_tagged_monitor, (in_bytes(ObjectMonitor::owner_offset()) - monitor_tag)));
336
337 // CAS owner (null => current thread id).
338 Register tid = flag;
339 mv(tid, Address(xthread, JavaThread::lock_id_offset()));
340 cmpxchg(/*addr*/ tmp2_owner_addr, /*expected*/ zr, /*new*/ tid, Assembler::int64,
341 /*acquire*/ Assembler::aq, /*release*/ Assembler::relaxed, /*result*/ tmp3_owner);
342 beqz(tmp3_owner, locked);
343
344 // Check if recursive.
345 bne(tmp3_owner, tid, slow_path);
346
347 // Recursive.
348 increment(Address(tmp1_tagged_monitor, in_bytes(ObjectMonitor::recursions_offset()) - monitor_tag), 1, tmp2, tmp3);
349 }
350
351 bind(locked);
352 mv(flag, zr);
353 increment(Address(xthread, JavaThread::held_monitor_count_offset()), 1, tmp2, tmp3);
354
355 #ifdef ASSERT
356 // Check that locked label is reached with flag == 0.
357 Label flag_correct;
358 beqz(flag, flag_correct);
359 stop("Fast Lock Flag != 0");
360 #endif
361
362 bind(slow_path);
363 #ifdef ASSERT
364 // Check that slow_path label is reached with flag != 0.
365 bnez(flag, flag_correct);
366 stop("Fast Lock Flag == 0");
367 bind(flag_correct);
368 #endif
369 // C2 uses the value of flag (0 vs !0) to determine the continuation.
370 }
371
372 void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register tmp1, Register tmp2,
373 Register tmp3) {
374 // Flag register, zero for success; non-zero for failure.
375 Register flag = t1;
376
377 assert(LockingMode == LM_LIGHTWEIGHT, "must be");
378 assert_different_registers(obj, tmp1, tmp2, tmp3, flag, t0);
379
380 mv(flag, 1);
381
382 // Handle inflated monitor.
383 Label inflated, inflated_load_monitor;
384 // Finish fast unlock successfully. unlocked MUST branch to with flag == 0
385 Label unlocked;
386 // Finish fast unlock unsuccessfully. MUST branch to with flag != 0
387 Label slow_path;
388
389 const Register tmp1_mark = tmp1;
390 const Register tmp2_top = tmp2;
391 const Register tmp3_t = tmp3;
392
393 { // Lightweight unlock
394
395 // Check if obj is top of lock-stack.
396 lwu(tmp2_top, Address(xthread, JavaThread::lock_stack_top_offset()));
397 subw(tmp2_top, tmp2_top, oopSize);
398 add(tmp3_t, xthread, tmp2_top);
399 ld(tmp3_t, Address(tmp3_t));
400 // Top of lock stack was not obj. Must be monitor.
401 bne(obj, tmp3_t, inflated_load_monitor);
402
403 // Pop lock-stack.
404 DEBUG_ONLY(add(tmp3_t, xthread, tmp2_top);)
405 DEBUG_ONLY(sd(zr, Address(tmp3_t));)
406 sw(tmp2_top, Address(xthread, JavaThread::lock_stack_top_offset()));
407
408 // Check if recursive.
409 add(tmp3_t, xthread, tmp2_top);
410 ld(tmp3_t, Address(tmp3_t, -oopSize));
411 beq(obj, tmp3_t, unlocked);
412
413 // Not recursive.
414 // Load Mark.
415 ld(tmp1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
416
417 // Check header for monitor (0b10).
418 test_bit(tmp3_t, tmp1_mark, exact_log2(markWord::monitor_value));
419 bnez(tmp3_t, inflated);
420
421 // Try to unlock. Transition lock bits 0b00 => 0b01
422 assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
423 ori(tmp3_t, tmp1_mark, markWord::unlocked_value);
424 cmpxchg(/*addr*/ obj, /*expected*/ tmp1_mark, /*new*/ tmp3_t, Assembler::int64,
425 /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, /*result*/ tmp3_t);
426 beq(tmp1_mark, tmp3_t, unlocked);
427
428 // Compare and exchange failed.
429 // Restore lock-stack and handle the unlock in runtime.
430 DEBUG_ONLY(add(tmp3_t, xthread, tmp2_top);)
431 DEBUG_ONLY(sd(obj, Address(tmp3_t));)
432 addw(tmp2_top, tmp2_top, oopSize);
433 sd(tmp2_top, Address(xthread, JavaThread::lock_stack_top_offset()));
434 j(slow_path);
435 }
436
437 { // Handle inflated monitor.
438 bind(inflated_load_monitor);
439 ld(tmp1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
440 #ifdef ASSERT
441 test_bit(tmp3_t, tmp1_mark, exact_log2(markWord::monitor_value));
442 bnez(tmp3_t, inflated);
443 stop("Fast Unlock not monitor");
444 #endif
445
446 bind(inflated);
447
448 #ifdef ASSERT
449 Label check_done;
450 subw(tmp2_top, tmp2_top, oopSize);
451 mv(tmp3_t, in_bytes(JavaThread::lock_stack_base_offset()));
452 blt(tmp2_top, tmp3_t, check_done);
453 add(tmp3_t, xthread, tmp2_top);
454 ld(tmp3_t, Address(tmp3_t));
455 bne(obj, tmp3_t, inflated);
456 stop("Fast Unlock lock on stack");
457 bind(check_done);
458 #endif
459
460 // mark contains the tagged ObjectMonitor*.
461 const Register tmp1_monitor = tmp1_mark;
462 const uintptr_t monitor_tag = markWord::monitor_value;
463
464 // Untag the monitor.
465 sub(tmp1_monitor, tmp1_mark, monitor_tag);
466
467 const Register tmp2_recursions = tmp2;
468 Label not_recursive;
469
470 // Check if recursive.
471 ld(tmp2_recursions, Address(tmp1_monitor, ObjectMonitor::recursions_offset()));
472 beqz(tmp2_recursions, not_recursive);
473
474 // Recursive unlock.
475 addi(tmp2_recursions, tmp2_recursions, -1);
476 sd(tmp2_recursions, Address(tmp1_monitor, ObjectMonitor::recursions_offset()));
477 j(unlocked);
478
479 bind(not_recursive);
480
481 Label release;
482 const Register tmp2_owner_addr = tmp2;
483
484 // Compute owner address.
485 la(tmp2_owner_addr, Address(tmp1_monitor, ObjectMonitor::owner_offset()));
486
487 // Check if the entry lists are empty.
488 ld(t0, Address(tmp1_monitor, ObjectMonitor::EntryList_offset()));
489 ld(tmp3_t, Address(tmp1_monitor, ObjectMonitor::cxq_offset()));
490 orr(t0, t0, tmp3_t);
491 beqz(t0, release);
492
493 // The owner may be anonymous and we removed the last obj entry in
494 // the lock-stack. This loses the information about the owner.
495 // Write the thread to the owner field so the runtime knows the owner.
496 Register tid = flag;
497 mv(tid, Address(xthread, JavaThread::lock_id_offset()));
498 sd(tid, Address(tmp2_owner_addr));
499 j(slow_path);
500
501 bind(release);
502 // Set owner to null.
503 membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore);
504 sd(zr, Address(tmp2_owner_addr));
505 }
506
507 bind(unlocked);
508 mv(flag, zr);
509 decrement(Address(xthread, JavaThread::held_monitor_count_offset()), 1, tmp2, tmp3);
510
511 #ifdef ASSERT
512 // Check that unlocked label is reached with flag == 0.
513 Label flag_correct;
514 beqz(flag, flag_correct);
515 stop("Fast Lock Flag != 0");
516 #endif
517
518 bind(slow_path);
519 #ifdef ASSERT
520 // Check that slow_path label is reached with flag != 0.
521 bnez(flag, flag_correct);
522 stop("Fast Lock Flag == 0");
523 bind(flag_correct);
524 #endif
525 // C2 uses the value of flag (0 vs !0) to determine the continuation.
526 }
527
528 // short string
529 // StringUTF16.indexOfChar
530 // StringLatin1.indexOfChar
531 void C2_MacroAssembler::string_indexof_char_short(Register str1, Register cnt1,
532 Register ch, Register result,
533 bool isL)
534 {
535 Register ch1 = t0;
536 Register index = t1;
537
538 BLOCK_COMMENT("string_indexof_char_short {");
539
540 Label LOOP, LOOP1, LOOP4, LOOP8;
541 Label MATCH, MATCH1, MATCH2, MATCH3,
542 MATCH4, MATCH5, MATCH6, MATCH7, NOMATCH;
543
544 mv(result, -1);
545 mv(index, zr);
546
547 bind(LOOP);
548 addi(t0, index, 8);
549 ble(t0, cnt1, LOOP8);
550 addi(t0, index, 4);
551 ble(t0, cnt1, LOOP4);
552 j(LOOP1);
553
554 bind(LOOP8);
555 isL ? lbu(ch1, Address(str1, 0)) : lhu(ch1, Address(str1, 0));
556 beq(ch, ch1, MATCH);
557 isL ? lbu(ch1, Address(str1, 1)) : lhu(ch1, Address(str1, 2));
558 beq(ch, ch1, MATCH1);
559 isL ? lbu(ch1, Address(str1, 2)) : lhu(ch1, Address(str1, 4));
560 beq(ch, ch1, MATCH2);
561 isL ? lbu(ch1, Address(str1, 3)) : lhu(ch1, Address(str1, 6));
562 beq(ch, ch1, MATCH3);
563 isL ? lbu(ch1, Address(str1, 4)) : lhu(ch1, Address(str1, 8));
564 beq(ch, ch1, MATCH4);
565 isL ? lbu(ch1, Address(str1, 5)) : lhu(ch1, Address(str1, 10));
566 beq(ch, ch1, MATCH5);
567 isL ? lbu(ch1, Address(str1, 6)) : lhu(ch1, Address(str1, 12));
568 beq(ch, ch1, MATCH6);
569 isL ? lbu(ch1, Address(str1, 7)) : lhu(ch1, Address(str1, 14));
570 beq(ch, ch1, MATCH7);
571 addi(index, index, 8);
572 addi(str1, str1, isL ? 8 : 16);
573 blt(index, cnt1, LOOP);
574 j(NOMATCH);
575
576 bind(LOOP4);
577 isL ? lbu(ch1, Address(str1, 0)) : lhu(ch1, Address(str1, 0));
578 beq(ch, ch1, MATCH);
579 isL ? lbu(ch1, Address(str1, 1)) : lhu(ch1, Address(str1, 2));
580 beq(ch, ch1, MATCH1);
581 isL ? lbu(ch1, Address(str1, 2)) : lhu(ch1, Address(str1, 4));
582 beq(ch, ch1, MATCH2);
583 isL ? lbu(ch1, Address(str1, 3)) : lhu(ch1, Address(str1, 6));
584 beq(ch, ch1, MATCH3);
585 addi(index, index, 4);
586 addi(str1, str1, isL ? 4 : 8);
587 bge(index, cnt1, NOMATCH);
588
589 bind(LOOP1);
590 isL ? lbu(ch1, Address(str1)) : lhu(ch1, Address(str1));
591 beq(ch, ch1, MATCH);
592 addi(index, index, 1);
593 addi(str1, str1, isL ? 1 : 2);
594 blt(index, cnt1, LOOP1);
595 j(NOMATCH);
596
597 bind(MATCH1);
598 addi(index, index, 1);
599 j(MATCH);
600
601 bind(MATCH2);
602 addi(index, index, 2);
603 j(MATCH);
604
605 bind(MATCH3);
606 addi(index, index, 3);
607 j(MATCH);
608
609 bind(MATCH4);
610 addi(index, index, 4);
611 j(MATCH);
612
613 bind(MATCH5);
614 addi(index, index, 5);
615 j(MATCH);
616
617 bind(MATCH6);
618 addi(index, index, 6);
619 j(MATCH);
620
621 bind(MATCH7);
622 addi(index, index, 7);
623
624 bind(MATCH);
625 mv(result, index);
626 bind(NOMATCH);
627 BLOCK_COMMENT("} string_indexof_char_short");
628 }
629
630 // StringUTF16.indexOfChar
631 // StringLatin1.indexOfChar
632 void C2_MacroAssembler::string_indexof_char(Register str1, Register cnt1,
633 Register ch, Register result,
634 Register tmp1, Register tmp2,
635 Register tmp3, Register tmp4,
636 bool isL)
637 {
638 Label CH1_LOOP, HIT, NOMATCH, DONE, DO_LONG;
639 Register ch1 = t0;
640 Register orig_cnt = t1;
641 Register mask1 = tmp3;
642 Register mask2 = tmp2;
643 Register match_mask = tmp1;
644 Register trailing_char = tmp4;
645 Register unaligned_elems = tmp4;
646
647 BLOCK_COMMENT("string_indexof_char {");
648 beqz(cnt1, NOMATCH);
649
650 addi(t0, cnt1, isL ? -32 : -16);
651 bgtz(t0, DO_LONG);
652 string_indexof_char_short(str1, cnt1, ch, result, isL);
653 j(DONE);
654
655 bind(DO_LONG);
656 mv(orig_cnt, cnt1);
657 if (AvoidUnalignedAccesses) {
658 Label ALIGNED;
659 andi(unaligned_elems, str1, 0x7);
660 beqz(unaligned_elems, ALIGNED);
661 sub(unaligned_elems, unaligned_elems, 8);
662 neg(unaligned_elems, unaligned_elems);
663 if (!isL) {
664 srli(unaligned_elems, unaligned_elems, 1);
665 }
666 // do unaligned part per element
667 string_indexof_char_short(str1, unaligned_elems, ch, result, isL);
668 bgez(result, DONE);
669 mv(orig_cnt, cnt1);
670 sub(cnt1, cnt1, unaligned_elems);
671 bind(ALIGNED);
672 }
673
674 // duplicate ch
675 if (isL) {
676 slli(ch1, ch, 8);
677 orr(ch, ch1, ch);
678 }
679 slli(ch1, ch, 16);
680 orr(ch, ch1, ch);
681 slli(ch1, ch, 32);
682 orr(ch, ch1, ch);
683
684 if (!isL) {
685 slli(cnt1, cnt1, 1);
686 }
687
688 uint64_t mask0101 = UCONST64(0x0101010101010101);
689 uint64_t mask0001 = UCONST64(0x0001000100010001);
690 mv(mask1, isL ? mask0101 : mask0001);
691 uint64_t mask7f7f = UCONST64(0x7f7f7f7f7f7f7f7f);
692 uint64_t mask7fff = UCONST64(0x7fff7fff7fff7fff);
693 mv(mask2, isL ? mask7f7f : mask7fff);
694
695 bind(CH1_LOOP);
696 ld(ch1, Address(str1));
697 addi(str1, str1, 8);
698 addi(cnt1, cnt1, -8);
699 compute_match_mask(ch1, ch, match_mask, mask1, mask2);
700 bnez(match_mask, HIT);
701 bgtz(cnt1, CH1_LOOP);
702 j(NOMATCH);
703
704 bind(HIT);
705 ctzc_bit(trailing_char, match_mask, isL, ch1, result);
706 srli(trailing_char, trailing_char, 3);
707 addi(cnt1, cnt1, 8);
708 ble(cnt1, trailing_char, NOMATCH);
709 // match case
710 if (!isL) {
711 srli(cnt1, cnt1, 1);
712 srli(trailing_char, trailing_char, 1);
713 }
714
715 sub(result, orig_cnt, cnt1);
716 add(result, result, trailing_char);
717 j(DONE);
718
719 bind(NOMATCH);
720 mv(result, -1);
721
722 bind(DONE);
723 BLOCK_COMMENT("} string_indexof_char");
724 }
725
726 typedef void (MacroAssembler::* load_chr_insn)(Register rd, const Address &adr, Register temp);
727
728 // Search for needle in haystack and return index or -1
729 // x10: result
730 // x11: haystack
731 // x12: haystack_len
732 // x13: needle
733 // x14: needle_len
734 void C2_MacroAssembler::string_indexof(Register haystack, Register needle,
735 Register haystack_len, Register needle_len,
736 Register tmp1, Register tmp2,
737 Register tmp3, Register tmp4,
738 Register tmp5, Register tmp6,
739 Register result, int ae)
740 {
741 assert(ae != StrIntrinsicNode::LU, "Invalid encoding");
742
743 Label LINEARSEARCH, LINEARSTUB, DONE, NOMATCH;
744
745 Register ch1 = t0;
746 Register ch2 = t1;
747 Register nlen_tmp = tmp1; // needle len tmp
748 Register hlen_tmp = tmp2; // haystack len tmp
749 Register result_tmp = tmp4;
750
751 bool isLL = ae == StrIntrinsicNode::LL;
752
753 bool needle_isL = ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UL;
754 bool haystack_isL = ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::LU;
755 int needle_chr_shift = needle_isL ? 0 : 1;
756 int haystack_chr_shift = haystack_isL ? 0 : 1;
757 int needle_chr_size = needle_isL ? 1 : 2;
758 int haystack_chr_size = haystack_isL ? 1 : 2;
759 load_chr_insn needle_load_1chr = needle_isL ? (load_chr_insn)&MacroAssembler::lbu :
760 (load_chr_insn)&MacroAssembler::lhu;
761 load_chr_insn haystack_load_1chr = haystack_isL ? (load_chr_insn)&MacroAssembler::lbu :
762 (load_chr_insn)&MacroAssembler::lhu;
763
764 BLOCK_COMMENT("string_indexof {");
765
766 // Note, inline_string_indexOf() generates checks:
767 // if (pattern.count > src.count) return -1;
768 // if (pattern.count == 0) return 0;
769
770 // We have two strings, a source string in haystack, haystack_len and a pattern string
771 // in needle, needle_len. Find the first occurrence of pattern in source or return -1.
772
773 // For larger pattern and source we use a simplified Boyer Moore algorithm.
774 // With a small pattern and source we use linear scan.
775
776 // needle_len >=8 && needle_len < 256 && needle_len < haystack_len/4, use bmh algorithm.
777 sub(result_tmp, haystack_len, needle_len);
778 // needle_len < 8, use linear scan
779 sub(t0, needle_len, 8);
780 bltz(t0, LINEARSEARCH);
781 // needle_len >= 256, use linear scan
782 sub(t0, needle_len, 256);
783 bgez(t0, LINEARSTUB);
784 // needle_len >= haystack_len/4, use linear scan
785 srli(t0, haystack_len, 2);
786 bge(needle_len, t0, LINEARSTUB);
787
788 // Boyer-Moore-Horspool introduction:
789 // The Boyer Moore alogorithm is based on the description here:-
790 //
791 // http://en.wikipedia.org/wiki/Boyer%E2%80%93Moore_string_search_algorithm
792 //
793 // This describes and algorithm with 2 shift rules. The 'Bad Character' rule
794 // and the 'Good Suffix' rule.
795 //
796 // These rules are essentially heuristics for how far we can shift the
797 // pattern along the search string.
798 //
799 // The implementation here uses the 'Bad Character' rule only because of the
800 // complexity of initialisation for the 'Good Suffix' rule.
801 //
802 // This is also known as the Boyer-Moore-Horspool algorithm:
803 //
804 // http://en.wikipedia.org/wiki/Boyer-Moore-Horspool_algorithm
805 //
806 // #define ASIZE 256
807 //
808 // int bm(unsigned char *pattern, int m, unsigned char *src, int n) {
809 // int i, j;
810 // unsigned c;
811 // unsigned char bc[ASIZE];
812 //
813 // /* Preprocessing */
814 // for (i = 0; i < ASIZE; ++i)
815 // bc[i] = m;
816 // for (i = 0; i < m - 1; ) {
817 // c = pattern[i];
818 // ++i;
819 // // c < 256 for Latin1 string, so, no need for branch
820 // #ifdef PATTERN_STRING_IS_LATIN1
821 // bc[c] = m - i;
822 // #else
823 // if (c < ASIZE) bc[c] = m - i;
824 // #endif
825 // }
826 //
827 // /* Searching */
828 // j = 0;
829 // while (j <= n - m) {
830 // c = src[i+j];
831 // if (pattern[m-1] == c)
832 // int k;
833 // for (k = m - 2; k >= 0 && pattern[k] == src[k + j]; --k);
834 // if (k < 0) return j;
835 // // c < 256 for Latin1 string, so, no need for branch
836 // #ifdef SOURCE_STRING_IS_LATIN1_AND_PATTERN_STRING_IS_LATIN1
837 // // LL case: (c< 256) always true. Remove branch
838 // j += bc[pattern[j+m-1]];
839 // #endif
840 // #ifdef SOURCE_STRING_IS_UTF_AND_PATTERN_STRING_IS_UTF
841 // // UU case: need if (c<ASIZE) check. Skip 1 character if not.
842 // if (c < ASIZE)
843 // j += bc[pattern[j+m-1]];
844 // else
845 // j += 1
846 // #endif
847 // #ifdef SOURCE_IS_UTF_AND_PATTERN_IS_LATIN1
848 // // UL case: need if (c<ASIZE) check. Skip <pattern length> if not.
849 // if (c < ASIZE)
850 // j += bc[pattern[j+m-1]];
851 // else
852 // j += m
853 // #endif
854 // }
855 // return -1;
856 // }
857
858 // temp register:t0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, result
859 Label BCLOOP, BCSKIP, BMLOOPSTR2, BMLOOPSTR1, BMSKIP, BMADV, BMMATCH,
860 BMLOOPSTR1_LASTCMP, BMLOOPSTR1_CMP, BMLOOPSTR1_AFTER_LOAD, BM_INIT_LOOP;
861
862 Register haystack_end = haystack_len;
863 Register skipch = tmp2;
864
865 // pattern length is >=8, so, we can read at least 1 register for cases when
866 // UTF->Latin1 conversion is not needed(8 LL or 4UU) and half register for
867 // UL case. We'll re-read last character in inner pre-loop code to have
868 // single outer pre-loop load
869 const int firstStep = isLL ? 7 : 3;
870
871 const int ASIZE = 256;
872 const int STORE_BYTES = 8; // 8 bytes stored per instruction(sd)
873
874 sub(sp, sp, ASIZE);
875
876 // init BC offset table with default value: needle_len
877 slli(t0, needle_len, 8);
878 orr(t0, t0, needle_len); // [63...16][needle_len][needle_len]
879 slli(tmp1, t0, 16);
880 orr(t0, tmp1, t0); // [63...32][needle_len][needle_len][needle_len][needle_len]
881 slli(tmp1, t0, 32);
882 orr(tmp5, tmp1, t0); // tmp5: 8 elements [needle_len]
883
884 mv(ch1, sp); // ch1 is t0
885 mv(tmp6, ASIZE / STORE_BYTES); // loop iterations
886
887 bind(BM_INIT_LOOP);
888 // for (i = 0; i < ASIZE; ++i)
889 // bc[i] = m;
890 for (int i = 0; i < 4; i++) {
891 sd(tmp5, Address(ch1, i * wordSize));
892 }
893 add(ch1, ch1, 32);
894 sub(tmp6, tmp6, 4);
895 bgtz(tmp6, BM_INIT_LOOP);
896
897 sub(nlen_tmp, needle_len, 1); // m - 1, index of the last element in pattern
898 Register orig_haystack = tmp5;
899 mv(orig_haystack, haystack);
900 // result_tmp = tmp4
901 shadd(haystack_end, result_tmp, haystack, haystack_end, haystack_chr_shift);
902 sub(ch2, needle_len, 1); // bc offset init value, ch2 is t1
903 mv(tmp3, needle);
904
905 // for (i = 0; i < m - 1; ) {
906 // c = pattern[i];
907 // ++i;
908 // // c < 256 for Latin1 string, so, no need for branch
909 // #ifdef PATTERN_STRING_IS_LATIN1
910 // bc[c] = m - i;
911 // #else
912 // if (c < ASIZE) bc[c] = m - i;
913 // #endif
914 // }
915 bind(BCLOOP);
916 (this->*needle_load_1chr)(ch1, Address(tmp3), noreg);
917 add(tmp3, tmp3, needle_chr_size);
918 if (!needle_isL) {
919 // ae == StrIntrinsicNode::UU
920 mv(tmp6, ASIZE);
921 bgeu(ch1, tmp6, BCSKIP);
922 }
923 add(tmp4, sp, ch1);
924 sb(ch2, Address(tmp4)); // store skip offset to BC offset table
925
926 bind(BCSKIP);
927 sub(ch2, ch2, 1); // for next pattern element, skip distance -1
928 bgtz(ch2, BCLOOP);
929
930 // tmp6: pattern end, address after needle
931 shadd(tmp6, needle_len, needle, tmp6, needle_chr_shift);
932 if (needle_isL == haystack_isL) {
933 // load last 8 bytes (8LL/4UU symbols)
934 ld(tmp6, Address(tmp6, -wordSize));
935 } else {
936 // UL: from UTF-16(source) search Latin1(pattern)
937 lwu(tmp6, Address(tmp6, -wordSize / 2)); // load last 4 bytes(4 symbols)
938 // convert Latin1 to UTF. eg: 0x0000abcd -> 0x0a0b0c0d
939 // We'll have to wait until load completed, but it's still faster than per-character loads+checks
940 srli(tmp3, tmp6, BitsPerByte * (wordSize / 2 - needle_chr_size)); // pattern[m-1], eg:0x0000000a
941 slli(ch2, tmp6, XLEN - 24);
942 srli(ch2, ch2, XLEN - 8); // pattern[m-2], 0x0000000b
943 slli(ch1, tmp6, XLEN - 16);
944 srli(ch1, ch1, XLEN - 8); // pattern[m-3], 0x0000000c
945 andi(tmp6, tmp6, 0xff); // pattern[m-4], 0x0000000d
946 slli(ch2, ch2, 16);
947 orr(ch2, ch2, ch1); // 0x00000b0c
948 slli(result, tmp3, 48); // use result as temp register
949 orr(tmp6, tmp6, result); // 0x0a00000d
950 slli(result, ch2, 16);
951 orr(tmp6, tmp6, result); // UTF-16:0x0a0b0c0d
952 }
953
954 // i = m - 1;
955 // skipch = j + i;
956 // if (skipch == pattern[m - 1]
957 // for (k = m - 2; k >= 0 && pattern[k] == src[k + j]; --k);
958 // else
959 // move j with bad char offset table
960 bind(BMLOOPSTR2);
961 // compare pattern to source string backward
962 shadd(result, nlen_tmp, haystack, result, haystack_chr_shift);
963 (this->*haystack_load_1chr)(skipch, Address(result), noreg);
964 sub(nlen_tmp, nlen_tmp, firstStep); // nlen_tmp is positive here, because needle_len >= 8
965 if (needle_isL == haystack_isL) {
966 // re-init tmp3. It's for free because it's executed in parallel with
967 // load above. Alternative is to initialize it before loop, but it'll
968 // affect performance on in-order systems with 2 or more ld/st pipelines
969 srli(tmp3, tmp6, BitsPerByte * (wordSize - needle_chr_size)); // UU/LL: pattern[m-1]
970 }
971 if (!isLL) { // UU/UL case
972 slli(ch2, nlen_tmp, 1); // offsets in bytes
973 }
974 bne(tmp3, skipch, BMSKIP); // if not equal, skipch is bad char
975 add(result, haystack, isLL ? nlen_tmp : ch2);
976 // load 8 bytes from source string
977 // if isLL is false then read granularity can be 2
978 load_long_misaligned(ch2, Address(result), ch1, isLL ? 1 : 2); // can use ch1 as temp register here as it will be trashed by next mv anyway
979 mv(ch1, tmp6);
980 if (isLL) {
981 j(BMLOOPSTR1_AFTER_LOAD);
982 } else {
983 sub(nlen_tmp, nlen_tmp, 1); // no need to branch for UU/UL case. cnt1 >= 8
984 j(BMLOOPSTR1_CMP);
985 }
986
987 bind(BMLOOPSTR1);
988 shadd(ch1, nlen_tmp, needle, ch1, needle_chr_shift);
989 (this->*needle_load_1chr)(ch1, Address(ch1), noreg);
990 shadd(ch2, nlen_tmp, haystack, ch2, haystack_chr_shift);
991 (this->*haystack_load_1chr)(ch2, Address(ch2), noreg);
992
993 bind(BMLOOPSTR1_AFTER_LOAD);
994 sub(nlen_tmp, nlen_tmp, 1);
995 bltz(nlen_tmp, BMLOOPSTR1_LASTCMP);
996
997 bind(BMLOOPSTR1_CMP);
998 beq(ch1, ch2, BMLOOPSTR1);
999
1000 bind(BMSKIP);
1001 if (!isLL) {
1002 // if we've met UTF symbol while searching Latin1 pattern, then we can
1003 // skip needle_len symbols
1004 if (needle_isL != haystack_isL) {
1005 mv(result_tmp, needle_len);
1006 } else {
1007 mv(result_tmp, 1);
1008 }
1009 mv(t0, ASIZE);
1010 bgeu(skipch, t0, BMADV);
1011 }
1012 add(result_tmp, sp, skipch);
1013 lbu(result_tmp, Address(result_tmp)); // load skip offset
1014
1015 bind(BMADV);
1016 sub(nlen_tmp, needle_len, 1);
1017 // move haystack after bad char skip offset
1018 shadd(haystack, result_tmp, haystack, result, haystack_chr_shift);
1019 ble(haystack, haystack_end, BMLOOPSTR2);
1020 add(sp, sp, ASIZE);
1021 j(NOMATCH);
1022
1023 bind(BMLOOPSTR1_LASTCMP);
1024 bne(ch1, ch2, BMSKIP);
1025
1026 bind(BMMATCH);
1027 sub(result, haystack, orig_haystack);
1028 if (!haystack_isL) {
1029 srli(result, result, 1);
1030 }
1031 add(sp, sp, ASIZE);
1032 j(DONE);
1033
1034 bind(LINEARSTUB);
1035 sub(t0, needle_len, 16); // small patterns still should be handled by simple algorithm
1036 bltz(t0, LINEARSEARCH);
1037 mv(result, zr);
1038 RuntimeAddress stub = nullptr;
1039 if (isLL) {
1040 stub = RuntimeAddress(StubRoutines::riscv::string_indexof_linear_ll());
1041 assert(stub.target() != nullptr, "string_indexof_linear_ll stub has not been generated");
1042 } else if (needle_isL) {
1043 stub = RuntimeAddress(StubRoutines::riscv::string_indexof_linear_ul());
1044 assert(stub.target() != nullptr, "string_indexof_linear_ul stub has not been generated");
1045 } else {
1046 stub = RuntimeAddress(StubRoutines::riscv::string_indexof_linear_uu());
1047 assert(stub.target() != nullptr, "string_indexof_linear_uu stub has not been generated");
1048 }
1049 address call = trampoline_call(stub);
1050 if (call == nullptr) {
1051 DEBUG_ONLY(reset_labels(LINEARSEARCH, DONE, NOMATCH));
1052 ciEnv::current()->record_failure("CodeCache is full");
1053 return;
1054 }
1055 j(DONE);
1056
1057 bind(NOMATCH);
1058 mv(result, -1);
1059 j(DONE);
1060
1061 bind(LINEARSEARCH);
1062 string_indexof_linearscan(haystack, needle, haystack_len, needle_len, tmp1, tmp2, tmp3, tmp4, -1, result, ae);
1063
1064 bind(DONE);
1065 BLOCK_COMMENT("} string_indexof");
1066 }
1067
1068 // string_indexof
1069 // result: x10
1070 // src: x11
1071 // src_count: x12
1072 // pattern: x13
1073 // pattern_count: x14 or 1/2/3/4
1074 void C2_MacroAssembler::string_indexof_linearscan(Register haystack, Register needle,
1075 Register haystack_len, Register needle_len,
1076 Register tmp1, Register tmp2,
1077 Register tmp3, Register tmp4,
1078 int needle_con_cnt, Register result, int ae)
1079 {
1080 // Note:
1081 // needle_con_cnt > 0 means needle_len register is invalid, needle length is constant
1082 // for UU/LL: needle_con_cnt[1, 4], UL: needle_con_cnt = 1
1083 assert(needle_con_cnt <= 4, "Invalid needle constant count");
1084 assert(ae != StrIntrinsicNode::LU, "Invalid encoding");
1085
1086 Register ch1 = t0;
1087 Register ch2 = t1;
1088 Register hlen_neg = haystack_len, nlen_neg = needle_len;
1089 Register nlen_tmp = tmp1, hlen_tmp = tmp2, result_tmp = tmp4;
1090
1091 bool isLL = ae == StrIntrinsicNode::LL;
1092
1093 bool needle_isL = ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UL;
1094 bool haystack_isL = ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::LU;
1095 int needle_chr_shift = needle_isL ? 0 : 1;
1096 int haystack_chr_shift = haystack_isL ? 0 : 1;
1097 int needle_chr_size = needle_isL ? 1 : 2;
1098 int haystack_chr_size = haystack_isL ? 1 : 2;
1099
1100 load_chr_insn needle_load_1chr = needle_isL ? (load_chr_insn)&MacroAssembler::lbu :
1101 (load_chr_insn)&MacroAssembler::lhu;
1102 load_chr_insn haystack_load_1chr = haystack_isL ? (load_chr_insn)&MacroAssembler::lbu :
1103 (load_chr_insn)&MacroAssembler::lhu;
1104 load_chr_insn load_2chr = isLL ? (load_chr_insn)&MacroAssembler::lhu : (load_chr_insn)&MacroAssembler::lwu;
1105 load_chr_insn load_4chr = isLL ? (load_chr_insn)&MacroAssembler::lwu : (load_chr_insn)&MacroAssembler::ld;
1106
1107 Label DO1, DO2, DO3, MATCH, NOMATCH, DONE;
1108
1109 Register first = tmp3;
1110
1111 if (needle_con_cnt == -1) {
1112 Label DOSHORT, FIRST_LOOP, STR2_NEXT, STR1_LOOP, STR1_NEXT;
1113
1114 sub(t0, needle_len, needle_isL == haystack_isL ? 4 : 2);
1115 bltz(t0, DOSHORT);
1116
1117 (this->*needle_load_1chr)(first, Address(needle), noreg);
1118 slli(t0, needle_len, needle_chr_shift);
1119 add(needle, needle, t0);
1120 neg(nlen_neg, t0);
1121 slli(t0, result_tmp, haystack_chr_shift);
1122 add(haystack, haystack, t0);
1123 neg(hlen_neg, t0);
1124
1125 bind(FIRST_LOOP);
1126 add(t0, haystack, hlen_neg);
1127 (this->*haystack_load_1chr)(ch2, Address(t0), noreg);
1128 beq(first, ch2, STR1_LOOP);
1129
1130 bind(STR2_NEXT);
1131 add(hlen_neg, hlen_neg, haystack_chr_size);
1132 blez(hlen_neg, FIRST_LOOP);
1133 j(NOMATCH);
1134
1135 bind(STR1_LOOP);
1136 add(nlen_tmp, nlen_neg, needle_chr_size);
1137 add(hlen_tmp, hlen_neg, haystack_chr_size);
1138 bgez(nlen_tmp, MATCH);
1139
1140 bind(STR1_NEXT);
1141 add(ch1, needle, nlen_tmp);
1142 (this->*needle_load_1chr)(ch1, Address(ch1), noreg);
1143 add(ch2, haystack, hlen_tmp);
1144 (this->*haystack_load_1chr)(ch2, Address(ch2), noreg);
1145 bne(ch1, ch2, STR2_NEXT);
1146 add(nlen_tmp, nlen_tmp, needle_chr_size);
1147 add(hlen_tmp, hlen_tmp, haystack_chr_size);
1148 bltz(nlen_tmp, STR1_NEXT);
1149 j(MATCH);
1150
1151 bind(DOSHORT);
1152 if (needle_isL == haystack_isL) {
1153 sub(t0, needle_len, 2);
1154 bltz(t0, DO1);
1155 bgtz(t0, DO3);
1156 }
1157 }
1158
1159 if (needle_con_cnt == 4) {
1160 Label CH1_LOOP;
1161 (this->*load_4chr)(ch1, Address(needle), noreg);
1162 sub(result_tmp, haystack_len, 4);
1163 slli(tmp3, result_tmp, haystack_chr_shift); // result as tmp
1164 add(haystack, haystack, tmp3);
1165 neg(hlen_neg, tmp3);
1166 if (AvoidUnalignedAccesses) {
1167 // preload first value, then we will read by 1 character per loop, instead of four
1168 // just shifting previous ch2 right by size of character in bits
1169 add(tmp3, haystack, hlen_neg);
1170 (this->*load_4chr)(ch2, Address(tmp3), noreg);
1171 if (isLL) {
1172 // need to erase 1 most significant byte in 32-bit value of ch2
1173 slli(ch2, ch2, 40);
1174 srli(ch2, ch2, 32);
1175 } else {
1176 slli(ch2, ch2, 16); // 2 most significant bytes will be erased by this operation
1177 }
1178 }
1179
1180 bind(CH1_LOOP);
1181 add(tmp3, haystack, hlen_neg);
1182 if (AvoidUnalignedAccesses) {
1183 srli(ch2, ch2, isLL ? 8 : 16);
1184 (this->*haystack_load_1chr)(tmp3, Address(tmp3, isLL ? 3 : 6), noreg);
1185 slli(tmp3, tmp3, isLL ? 24 : 48);
1186 add(ch2, ch2, tmp3);
1187 } else {
1188 (this->*load_4chr)(ch2, Address(tmp3), noreg);
1189 }
1190 beq(ch1, ch2, MATCH);
1191 add(hlen_neg, hlen_neg, haystack_chr_size);
1192 blez(hlen_neg, CH1_LOOP);
1193 j(NOMATCH);
1194 }
1195
1196 if ((needle_con_cnt == -1 && needle_isL == haystack_isL) || needle_con_cnt == 2) {
1197 Label CH1_LOOP;
1198 BLOCK_COMMENT("string_indexof DO2 {");
1199 bind(DO2);
1200 (this->*load_2chr)(ch1, Address(needle), noreg);
1201 if (needle_con_cnt == 2) {
1202 sub(result_tmp, haystack_len, 2);
1203 }
1204 slli(tmp3, result_tmp, haystack_chr_shift);
1205 add(haystack, haystack, tmp3);
1206 neg(hlen_neg, tmp3);
1207 if (AvoidUnalignedAccesses) {
1208 // preload first value, then we will read by 1 character per loop, instead of two
1209 // just shifting previous ch2 right by size of character in bits
1210 add(tmp3, haystack, hlen_neg);
1211 (this->*haystack_load_1chr)(ch2, Address(tmp3), noreg);
1212 slli(ch2, ch2, isLL ? 8 : 16);
1213 }
1214 bind(CH1_LOOP);
1215 add(tmp3, haystack, hlen_neg);
1216 if (AvoidUnalignedAccesses) {
1217 srli(ch2, ch2, isLL ? 8 : 16);
1218 (this->*haystack_load_1chr)(tmp3, Address(tmp3, isLL ? 1 : 2), noreg);
1219 slli(tmp3, tmp3, isLL ? 8 : 16);
1220 add(ch2, ch2, tmp3);
1221 } else {
1222 (this->*load_2chr)(ch2, Address(tmp3), noreg);
1223 }
1224 beq(ch1, ch2, MATCH);
1225 add(hlen_neg, hlen_neg, haystack_chr_size);
1226 blez(hlen_neg, CH1_LOOP);
1227 j(NOMATCH);
1228 BLOCK_COMMENT("} string_indexof DO2");
1229 }
1230
1231 if ((needle_con_cnt == -1 && needle_isL == haystack_isL) || needle_con_cnt == 3) {
1232 Label FIRST_LOOP, STR2_NEXT, STR1_LOOP;
1233 BLOCK_COMMENT("string_indexof DO3 {");
1234
1235 bind(DO3);
1236 (this->*load_2chr)(first, Address(needle), noreg);
1237 (this->*needle_load_1chr)(ch1, Address(needle, 2 * needle_chr_size), noreg);
1238 if (needle_con_cnt == 3) {
1239 sub(result_tmp, haystack_len, 3);
1240 }
1241 slli(hlen_tmp, result_tmp, haystack_chr_shift);
1242 add(haystack, haystack, hlen_tmp);
1243 neg(hlen_neg, hlen_tmp);
1244
1245 bind(FIRST_LOOP);
1246 add(ch2, haystack, hlen_neg);
1247 if (AvoidUnalignedAccesses) {
1248 (this->*haystack_load_1chr)(tmp2, Address(ch2, isLL ? 1 : 2), noreg); // we need a temp register, we can safely use hlen_tmp here, which is a synonym for tmp2
1249 (this->*haystack_load_1chr)(ch2, Address(ch2), noreg);
1250 slli(tmp2, tmp2, isLL ? 8 : 16);
1251 add(ch2, ch2, tmp2);
1252 } else {
1253 (this->*load_2chr)(ch2, Address(ch2), noreg);
1254 }
1255 beq(first, ch2, STR1_LOOP);
1256
1257 bind(STR2_NEXT);
1258 add(hlen_neg, hlen_neg, haystack_chr_size);
1259 blez(hlen_neg, FIRST_LOOP);
1260 j(NOMATCH);
1261
1262 bind(STR1_LOOP);
1263 add(hlen_tmp, hlen_neg, 2 * haystack_chr_size);
1264 add(ch2, haystack, hlen_tmp);
1265 (this->*haystack_load_1chr)(ch2, Address(ch2), noreg);
1266 bne(ch1, ch2, STR2_NEXT);
1267 j(MATCH);
1268 BLOCK_COMMENT("} string_indexof DO3");
1269 }
1270
1271 if (needle_con_cnt == -1 || needle_con_cnt == 1) {
1272 Label DO1_LOOP;
1273
1274 BLOCK_COMMENT("string_indexof DO1 {");
1275 bind(DO1);
1276 (this->*needle_load_1chr)(ch1, Address(needle), noreg);
1277 sub(result_tmp, haystack_len, 1);
1278 slli(tmp3, result_tmp, haystack_chr_shift);
1279 add(haystack, haystack, tmp3);
1280 neg(hlen_neg, tmp3);
1281
1282 bind(DO1_LOOP);
1283 add(tmp3, haystack, hlen_neg);
1284 (this->*haystack_load_1chr)(ch2, Address(tmp3), noreg);
1285 beq(ch1, ch2, MATCH);
1286 add(hlen_neg, hlen_neg, haystack_chr_size);
1287 blez(hlen_neg, DO1_LOOP);
1288 BLOCK_COMMENT("} string_indexof DO1");
1289 }
1290
1291 bind(NOMATCH);
1292 mv(result, -1);
1293 j(DONE);
1294
1295 bind(MATCH);
1296 srai(t0, hlen_neg, haystack_chr_shift);
1297 add(result, result_tmp, t0);
1298
1299 bind(DONE);
1300 }
1301
1302 // Compare strings.
1303 void C2_MacroAssembler::string_compare(Register str1, Register str2,
1304 Register cnt1, Register cnt2, Register result,
1305 Register tmp1, Register tmp2, Register tmp3,
1306 int ae)
1307 {
1308 Label DONE, SHORT_LOOP, SHORT_STRING, SHORT_LAST, TAIL, STUB,
1309 DIFFERENCE, NEXT_WORD, SHORT_LOOP_TAIL, SHORT_LAST2, SHORT_LAST_INIT,
1310 SHORT_LOOP_START, TAIL_CHECK, L;
1311
1312 const int STUB_THRESHOLD = 64 + 8;
1313 bool isLL = ae == StrIntrinsicNode::LL;
1314 bool isLU = ae == StrIntrinsicNode::LU;
1315 bool isUL = ae == StrIntrinsicNode::UL;
1316
1317 bool str1_isL = isLL || isLU;
1318 bool str2_isL = isLL || isUL;
1319
1320 // for L strings, 1 byte for 1 character
1321 // for U strings, 2 bytes for 1 character
1322 int str1_chr_size = str1_isL ? 1 : 2;
1323 int str2_chr_size = str2_isL ? 1 : 2;
1324 int minCharsInWord = isLL ? wordSize : wordSize / 2;
1325
1326 load_chr_insn str1_load_chr = str1_isL ? (load_chr_insn)&MacroAssembler::lbu : (load_chr_insn)&MacroAssembler::lhu;
1327 load_chr_insn str2_load_chr = str2_isL ? (load_chr_insn)&MacroAssembler::lbu : (load_chr_insn)&MacroAssembler::lhu;
1328
1329 BLOCK_COMMENT("string_compare {");
1330
1331 // Bizzarely, the counts are passed in bytes, regardless of whether they
1332 // are L or U strings, however the result is always in characters.
1333 if (!str1_isL) {
1334 sraiw(cnt1, cnt1, 1);
1335 }
1336 if (!str2_isL) {
1337 sraiw(cnt2, cnt2, 1);
1338 }
1339
1340 // Compute the minimum of the string lengths and save the difference in result.
1341 sub(result, cnt1, cnt2);
1342 bgt(cnt1, cnt2, L);
1343 mv(cnt2, cnt1);
1344 bind(L);
1345
1346 // A very short string
1347 mv(t0, minCharsInWord);
1348 ble(cnt2, t0, SHORT_STRING);
1349
1350 // Compare longwords
1351 // load first parts of strings and finish initialization while loading
1352 {
1353 if (str1_isL == str2_isL) { // LL or UU
1354 // check if str1 and str2 is same pointer
1355 beq(str1, str2, DONE);
1356 // load 8 bytes once to compare
1357 ld(tmp1, Address(str1));
1358 ld(tmp2, Address(str2));
1359 mv(t0, STUB_THRESHOLD);
1360 bge(cnt2, t0, STUB);
1361 sub(cnt2, cnt2, minCharsInWord);
1362 beqz(cnt2, TAIL_CHECK);
1363 // convert cnt2 from characters to bytes
1364 if (!str1_isL) {
1365 slli(cnt2, cnt2, 1);
1366 }
1367 add(str2, str2, cnt2);
1368 add(str1, str1, cnt2);
1369 sub(cnt2, zr, cnt2);
1370 } else if (isLU) { // LU case
1371 lwu(tmp1, Address(str1));
1372 ld(tmp2, Address(str2));
1373 mv(t0, STUB_THRESHOLD);
1374 bge(cnt2, t0, STUB);
1375 addi(cnt2, cnt2, -4);
1376 add(str1, str1, cnt2);
1377 sub(cnt1, zr, cnt2);
1378 slli(cnt2, cnt2, 1);
1379 add(str2, str2, cnt2);
1380 inflate_lo32(tmp3, tmp1);
1381 mv(tmp1, tmp3);
1382 sub(cnt2, zr, cnt2);
1383 addi(cnt1, cnt1, 4);
1384 } else { // UL case
1385 ld(tmp1, Address(str1));
1386 lwu(tmp2, Address(str2));
1387 mv(t0, STUB_THRESHOLD);
1388 bge(cnt2, t0, STUB);
1389 addi(cnt2, cnt2, -4);
1390 slli(t0, cnt2, 1);
1391 sub(cnt1, zr, t0);
1392 add(str1, str1, t0);
1393 add(str2, str2, cnt2);
1394 inflate_lo32(tmp3, tmp2);
1395 mv(tmp2, tmp3);
1396 sub(cnt2, zr, cnt2);
1397 addi(cnt1, cnt1, 8);
1398 }
1399 addi(cnt2, cnt2, isUL ? 4 : 8);
1400 bne(tmp1, tmp2, DIFFERENCE);
1401 bgez(cnt2, TAIL);
1402
1403 // main loop
1404 bind(NEXT_WORD);
1405 if (str1_isL == str2_isL) { // LL or UU
1406 add(t0, str1, cnt2);
1407 ld(tmp1, Address(t0));
1408 add(t0, str2, cnt2);
1409 ld(tmp2, Address(t0));
1410 addi(cnt2, cnt2, 8);
1411 } else if (isLU) { // LU case
1412 add(t0, str1, cnt1);
1413 lwu(tmp1, Address(t0));
1414 add(t0, str2, cnt2);
1415 ld(tmp2, Address(t0));
1416 addi(cnt1, cnt1, 4);
1417 inflate_lo32(tmp3, tmp1);
1418 mv(tmp1, tmp3);
1419 addi(cnt2, cnt2, 8);
1420 } else { // UL case
1421 add(t0, str2, cnt2);
1422 lwu(tmp2, Address(t0));
1423 add(t0, str1, cnt1);
1424 ld(tmp1, Address(t0));
1425 inflate_lo32(tmp3, tmp2);
1426 mv(tmp2, tmp3);
1427 addi(cnt1, cnt1, 8);
1428 addi(cnt2, cnt2, 4);
1429 }
1430 bne(tmp1, tmp2, DIFFERENCE);
1431 bltz(cnt2, NEXT_WORD);
1432 bind(TAIL);
1433 if (str1_isL == str2_isL) { // LL or UU
1434 load_long_misaligned(tmp1, Address(str1), tmp3, isLL ? 1 : 2);
1435 load_long_misaligned(tmp2, Address(str2), tmp3, isLL ? 1 : 2);
1436 } else if (isLU) { // LU case
1437 load_int_misaligned(tmp1, Address(str1), tmp3, false);
1438 load_long_misaligned(tmp2, Address(str2), tmp3, 2);
1439 inflate_lo32(tmp3, tmp1);
1440 mv(tmp1, tmp3);
1441 } else { // UL case
1442 load_int_misaligned(tmp2, Address(str2), tmp3, false);
1443 load_long_misaligned(tmp1, Address(str1), tmp3, 2);
1444 inflate_lo32(tmp3, tmp2);
1445 mv(tmp2, tmp3);
1446 }
1447 bind(TAIL_CHECK);
1448 beq(tmp1, tmp2, DONE);
1449
1450 // Find the first different characters in the longwords and
1451 // compute their difference.
1452 bind(DIFFERENCE);
1453 xorr(tmp3, tmp1, tmp2);
1454 ctzc_bit(result, tmp3, isLL); // count zero from lsb to msb
1455 srl(tmp1, tmp1, result);
1456 srl(tmp2, tmp2, result);
1457 if (isLL) {
1458 andi(tmp1, tmp1, 0xFF);
1459 andi(tmp2, tmp2, 0xFF);
1460 } else {
1461 andi(tmp1, tmp1, 0xFFFF);
1462 andi(tmp2, tmp2, 0xFFFF);
1463 }
1464 sub(result, tmp1, tmp2);
1465 j(DONE);
1466 }
1467
1468 bind(STUB);
1469 RuntimeAddress stub = nullptr;
1470 switch (ae) {
1471 case StrIntrinsicNode::LL:
1472 stub = RuntimeAddress(StubRoutines::riscv::compare_long_string_LL());
1473 break;
1474 case StrIntrinsicNode::UU:
1475 stub = RuntimeAddress(StubRoutines::riscv::compare_long_string_UU());
1476 break;
1477 case StrIntrinsicNode::LU:
1478 stub = RuntimeAddress(StubRoutines::riscv::compare_long_string_LU());
1479 break;
1480 case StrIntrinsicNode::UL:
1481 stub = RuntimeAddress(StubRoutines::riscv::compare_long_string_UL());
1482 break;
1483 default:
1484 ShouldNotReachHere();
1485 }
1486 assert(stub.target() != nullptr, "compare_long_string stub has not been generated");
1487 address call = trampoline_call(stub);
1488 if (call == nullptr) {
1489 DEBUG_ONLY(reset_labels(DONE, SHORT_LOOP, SHORT_STRING, SHORT_LAST, SHORT_LOOP_TAIL, SHORT_LAST2, SHORT_LAST_INIT, SHORT_LOOP_START));
1490 ciEnv::current()->record_failure("CodeCache is full");
1491 return;
1492 }
1493 j(DONE);
1494
1495 bind(SHORT_STRING);
1496 // Is the minimum length zero?
1497 beqz(cnt2, DONE);
1498 // arrange code to do most branches while loading and loading next characters
1499 // while comparing previous
1500 (this->*str1_load_chr)(tmp1, Address(str1), t0);
1501 addi(str1, str1, str1_chr_size);
1502 addi(cnt2, cnt2, -1);
1503 beqz(cnt2, SHORT_LAST_INIT);
1504 (this->*str2_load_chr)(cnt1, Address(str2), t0);
1505 addi(str2, str2, str2_chr_size);
1506 j(SHORT_LOOP_START);
1507 bind(SHORT_LOOP);
1508 addi(cnt2, cnt2, -1);
1509 beqz(cnt2, SHORT_LAST);
1510 bind(SHORT_LOOP_START);
1511 (this->*str1_load_chr)(tmp2, Address(str1), t0);
1512 addi(str1, str1, str1_chr_size);
1513 (this->*str2_load_chr)(t0, Address(str2), t0);
1514 addi(str2, str2, str2_chr_size);
1515 bne(tmp1, cnt1, SHORT_LOOP_TAIL);
1516 addi(cnt2, cnt2, -1);
1517 beqz(cnt2, SHORT_LAST2);
1518 (this->*str1_load_chr)(tmp1, Address(str1), t0);
1519 addi(str1, str1, str1_chr_size);
1520 (this->*str2_load_chr)(cnt1, Address(str2), t0);
1521 addi(str2, str2, str2_chr_size);
1522 beq(tmp2, t0, SHORT_LOOP);
1523 sub(result, tmp2, t0);
1524 j(DONE);
1525 bind(SHORT_LOOP_TAIL);
1526 sub(result, tmp1, cnt1);
1527 j(DONE);
1528 bind(SHORT_LAST2);
1529 beq(tmp2, t0, DONE);
1530 sub(result, tmp2, t0);
1531
1532 j(DONE);
1533 bind(SHORT_LAST_INIT);
1534 (this->*str2_load_chr)(cnt1, Address(str2), t0);
1535 addi(str2, str2, str2_chr_size);
1536 bind(SHORT_LAST);
1537 beq(tmp1, cnt1, DONE);
1538 sub(result, tmp1, cnt1);
1539
1540 bind(DONE);
1541
1542 BLOCK_COMMENT("} string_compare");
1543 }
1544
1545 void C2_MacroAssembler::arrays_equals(Register a1, Register a2,
1546 Register tmp1, Register tmp2, Register tmp3,
1547 Register result, int elem_size) {
1548 assert(elem_size == 1 || elem_size == 2, "must be char or byte");
1549 assert_different_registers(a1, a2, result, tmp1, tmp2, tmp3, t0);
1550
1551 int elem_per_word = wordSize/elem_size;
1552 int log_elem_size = exact_log2(elem_size);
1553 int length_offset = arrayOopDesc::length_offset_in_bytes();
1554 int base_offset = arrayOopDesc::base_offset_in_bytes(elem_size == 2 ? T_CHAR : T_BYTE);
1555
1556 Register cnt1 = tmp3;
1557 Register cnt2 = tmp1; // cnt2 only used in array length compare
1558 Label DONE, SAME, NEXT_WORD, SHORT, TAIL03, TAIL01;
1559
1560 BLOCK_COMMENT("arrays_equals {");
1561
1562 // if (a1 == a2), return true
1563 beq(a1, a2, SAME);
1564
1565 mv(result, false);
1566 // if (a1 == nullptr || a2 == nullptr)
1567 // return false;
1568 beqz(a1, DONE);
1569 beqz(a2, DONE);
1570
1571 // if (a1.length != a2.length)
1572 // return false;
1573 lwu(cnt1, Address(a1, length_offset));
1574 lwu(cnt2, Address(a2, length_offset));
1575 bne(cnt1, cnt2, DONE);
1576
1577 la(a1, Address(a1, base_offset));
1578 la(a2, Address(a2, base_offset));
1579 // Check for short strings, i.e. smaller than wordSize.
1580 addi(cnt1, cnt1, -elem_per_word);
1581 bltz(cnt1, SHORT);
1582
1583 // Main 8 byte comparison loop.
1584 bind(NEXT_WORD); {
1585 ld(tmp1, Address(a1));
1586 ld(tmp2, Address(a2));
1587 addi(cnt1, cnt1, -elem_per_word);
1588 addi(a1, a1, wordSize);
1589 addi(a2, a2, wordSize);
1590 bne(tmp1, tmp2, DONE);
1591 } bgez(cnt1, NEXT_WORD);
1592
1593 addi(tmp1, cnt1, elem_per_word);
1594 beqz(tmp1, SAME);
1595
1596 bind(SHORT);
1597 test_bit(tmp1, cnt1, 2 - log_elem_size);
1598 beqz(tmp1, TAIL03); // 0-7 bytes left.
1599 {
1600 lwu(tmp1, Address(a1));
1601 lwu(tmp2, Address(a2));
1602 addi(a1, a1, 4);
1603 addi(a2, a2, 4);
1604 bne(tmp1, tmp2, DONE);
1605 }
1606
1607 bind(TAIL03);
1608 test_bit(tmp1, cnt1, 1 - log_elem_size);
1609 beqz(tmp1, TAIL01); // 0-3 bytes left.
1610 {
1611 lhu(tmp1, Address(a1));
1612 lhu(tmp2, Address(a2));
1613 addi(a1, a1, 2);
1614 addi(a2, a2, 2);
1615 bne(tmp1, tmp2, DONE);
1616 }
1617
1618 bind(TAIL01);
1619 if (elem_size == 1) { // Only needed when comparing byte arrays.
1620 test_bit(tmp1, cnt1, 0);
1621 beqz(tmp1, SAME); // 0-1 bytes left.
1622 {
1623 lbu(tmp1, Address(a1));
1624 lbu(tmp2, Address(a2));
1625 bne(tmp1, tmp2, DONE);
1626 }
1627 }
1628
1629 bind(SAME);
1630 mv(result, true);
1631 // That's it.
1632 bind(DONE);
1633
1634 BLOCK_COMMENT("} arrays_equals");
1635 }
1636
1637 // Compare Strings
1638
1639 // For Strings we're passed the address of the first characters in a1 and a2
1640 // and the length in cnt1. There are two implementations.
1641 // For arrays >= 8 bytes, all comparisons (except for the tail) are performed
1642 // 8 bytes at a time. For the tail, we compare a halfword, then a short, and then a byte.
1643 // For strings < 8 bytes, we compare a halfword, then a short, and then a byte.
1644
1645 void C2_MacroAssembler::string_equals(Register a1, Register a2,
1646 Register result, Register cnt1)
1647 {
1648 Label SAME, DONE, SHORT, NEXT_WORD;
1649 Register tmp1 = t0;
1650 Register tmp2 = t1;
1651
1652 assert_different_registers(a1, a2, result, cnt1, tmp1, tmp2);
1653
1654 BLOCK_COMMENT("string_equals {");
1655
1656 mv(result, false);
1657
1658 // Check for short strings, i.e. smaller than wordSize.
1659 addi(cnt1, cnt1, -wordSize);
1660 bltz(cnt1, SHORT);
1661
1662 // Main 8 byte comparison loop.
1663 bind(NEXT_WORD); {
1664 ld(tmp1, Address(a1));
1665 ld(tmp2, Address(a2));
1666 addi(cnt1, cnt1, -wordSize);
1667 addi(a1, a1, wordSize);
1668 addi(a2, a2, wordSize);
1669 bne(tmp1, tmp2, DONE);
1670 } bgez(cnt1, NEXT_WORD);
1671
1672 addi(tmp1, cnt1, wordSize);
1673 beqz(tmp1, SAME);
1674
1675 bind(SHORT);
1676 Label TAIL03, TAIL01;
1677
1678 // 0-7 bytes left.
1679 test_bit(tmp1, cnt1, 2);
1680 beqz(tmp1, TAIL03);
1681 {
1682 lwu(tmp1, Address(a1));
1683 lwu(tmp2, Address(a2));
1684 addi(a1, a1, 4);
1685 addi(a2, a2, 4);
1686 bne(tmp1, tmp2, DONE);
1687 }
1688
1689 bind(TAIL03);
1690 // 0-3 bytes left.
1691 test_bit(tmp1, cnt1, 1);
1692 beqz(tmp1, TAIL01);
1693 {
1694 lhu(tmp1, Address(a1));
1695 lhu(tmp2, Address(a2));
1696 addi(a1, a1, 2);
1697 addi(a2, a2, 2);
1698 bne(tmp1, tmp2, DONE);
1699 }
1700
1701 bind(TAIL01);
1702 // 0-1 bytes left.
1703 test_bit(tmp1, cnt1, 0);
1704 beqz(tmp1, SAME);
1705 {
1706 lbu(tmp1, Address(a1));
1707 lbu(tmp2, Address(a2));
1708 bne(tmp1, tmp2, DONE);
1709 }
1710
1711 // Arrays are equal.
1712 bind(SAME);
1713 mv(result, true);
1714
1715 // That's it.
1716 bind(DONE);
1717 BLOCK_COMMENT("} string_equals");
1718 }
1719
1720 // jdk.internal.util.ArraysSupport.vectorizedHashCode
1721 void C2_MacroAssembler::arrays_hashcode(Register ary, Register cnt, Register result,
1722 Register tmp1, Register tmp2, Register tmp3,
1723 Register tmp4, Register tmp5, Register tmp6,
1724 BasicType eltype)
1725 {
1726 assert_different_registers(ary, cnt, result, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, t0, t1);
1727
1728 const int elsize = arrays_hashcode_elsize(eltype);
1729 const int chunks_end_shift = exact_log2(elsize);
1730
1731 switch (eltype) {
1732 case T_BOOLEAN: BLOCK_COMMENT("arrays_hashcode(unsigned byte) {"); break;
1733 case T_CHAR: BLOCK_COMMENT("arrays_hashcode(char) {"); break;
1734 case T_BYTE: BLOCK_COMMENT("arrays_hashcode(byte) {"); break;
1735 case T_SHORT: BLOCK_COMMENT("arrays_hashcode(short) {"); break;
1736 case T_INT: BLOCK_COMMENT("arrays_hashcode(int) {"); break;
1737 default:
1738 ShouldNotReachHere();
1739 }
1740
1741 const int stride = 4;
1742 const Register pow31_4 = tmp1;
1743 const Register pow31_3 = tmp2;
1744 const Register pow31_2 = tmp3;
1745 const Register chunks = tmp4;
1746 const Register chunks_end = chunks;
1747
1748 Label DONE, TAIL, TAIL_LOOP, WIDE_LOOP;
1749
1750 // result has a value initially
1751
1752 beqz(cnt, DONE);
1753
1754 andi(chunks, cnt, ~(stride-1));
1755 beqz(chunks, TAIL);
1756
1757 mv(pow31_4, 923521); // [31^^4]
1758 mv(pow31_3, 29791); // [31^^3]
1759 mv(pow31_2, 961); // [31^^2]
1760
1761 slli(chunks_end, chunks, chunks_end_shift);
1762 add(chunks_end, ary, chunks_end);
1763 andi(cnt, cnt, stride-1); // don't forget about tail!
1764
1765 bind(WIDE_LOOP);
1766 mulw(result, result, pow31_4); // 31^^4 * h
1767 arrays_hashcode_elload(t0, Address(ary, 0 * elsize), eltype);
1768 arrays_hashcode_elload(t1, Address(ary, 1 * elsize), eltype);
1769 arrays_hashcode_elload(tmp5, Address(ary, 2 * elsize), eltype);
1770 arrays_hashcode_elload(tmp6, Address(ary, 3 * elsize), eltype);
1771 mulw(t0, t0, pow31_3); // 31^^3 * ary[i+0]
1772 addw(result, result, t0);
1773 mulw(t1, t1, pow31_2); // 31^^2 * ary[i+1]
1774 addw(result, result, t1);
1775 slli(t0, tmp5, 5); // optimize 31^^1 * ary[i+2]
1776 subw(tmp5, t0, tmp5); // with ary[i+2]<<5 - ary[i+2]
1777 addw(result, result, tmp5);
1778 addw(result, result, tmp6); // 31^^4 * h + 31^^3 * ary[i+0] + 31^^2 * ary[i+1]
1779 // + 31^^1 * ary[i+2] + 31^^0 * ary[i+3]
1780 addi(ary, ary, elsize * stride);
1781 bne(ary, chunks_end, WIDE_LOOP);
1782 beqz(cnt, DONE);
1783
1784 bind(TAIL);
1785 slli(chunks_end, cnt, chunks_end_shift);
1786 add(chunks_end, ary, chunks_end);
1787
1788 bind(TAIL_LOOP);
1789 arrays_hashcode_elload(t0, Address(ary), eltype);
1790 slli(t1, result, 5); // optimize 31 * result
1791 subw(result, t1, result); // with result<<5 - result
1792 addw(result, result, t0);
1793 addi(ary, ary, elsize);
1794 bne(ary, chunks_end, TAIL_LOOP);
1795
1796 bind(DONE);
1797 BLOCK_COMMENT("} // arrays_hashcode");
1798 }
1799
1800 int C2_MacroAssembler::arrays_hashcode_elsize(BasicType eltype) {
1801 switch (eltype) {
1802 case T_BOOLEAN: return sizeof(jboolean);
1803 case T_BYTE: return sizeof(jbyte);
1804 case T_SHORT: return sizeof(jshort);
1805 case T_CHAR: return sizeof(jchar);
1806 case T_INT: return sizeof(jint);
1807 default:
1808 ShouldNotReachHere();
1809 return -1;
1810 }
1811 }
1812
1813 void C2_MacroAssembler::arrays_hashcode_elload(Register dst, Address src, BasicType eltype) {
1814 switch (eltype) {
1815 // T_BOOLEAN used as surrogate for unsigned byte
1816 case T_BOOLEAN: lbu(dst, src); break;
1817 case T_BYTE: lb(dst, src); break;
1818 case T_SHORT: lh(dst, src); break;
1819 case T_CHAR: lhu(dst, src); break;
1820 case T_INT: lw(dst, src); break;
1821 default:
1822 ShouldNotReachHere();
1823 }
1824 }
1825
1826 typedef void (Assembler::*conditional_branch_insn)(Register op1, Register op2, Label& label, bool is_far);
1827 typedef void (MacroAssembler::*float_conditional_branch_insn)(FloatRegister op1, FloatRegister op2, Label& label,
1828 bool is_far, bool is_unordered);
1829
1830 static conditional_branch_insn conditional_branches[] =
1831 {
1832 /* SHORT branches */
1833 (conditional_branch_insn)&MacroAssembler::beq,
1834 (conditional_branch_insn)&MacroAssembler::bgt,
1835 nullptr, // BoolTest::overflow
1836 (conditional_branch_insn)&MacroAssembler::blt,
1837 (conditional_branch_insn)&MacroAssembler::bne,
1838 (conditional_branch_insn)&MacroAssembler::ble,
1839 nullptr, // BoolTest::no_overflow
1840 (conditional_branch_insn)&MacroAssembler::bge,
1841
1842 /* UNSIGNED branches */
1843 (conditional_branch_insn)&MacroAssembler::beq,
1844 (conditional_branch_insn)&MacroAssembler::bgtu,
1845 nullptr,
1846 (conditional_branch_insn)&MacroAssembler::bltu,
1847 (conditional_branch_insn)&MacroAssembler::bne,
1848 (conditional_branch_insn)&MacroAssembler::bleu,
1849 nullptr,
1850 (conditional_branch_insn)&MacroAssembler::bgeu
1851 };
1852
1853 static float_conditional_branch_insn float_conditional_branches[] =
1854 {
1855 /* FLOAT SHORT branches */
1856 (float_conditional_branch_insn)&MacroAssembler::float_beq,
1857 (float_conditional_branch_insn)&MacroAssembler::float_bgt,
1858 nullptr, // BoolTest::overflow
1859 (float_conditional_branch_insn)&MacroAssembler::float_blt,
1860 (float_conditional_branch_insn)&MacroAssembler::float_bne,
1861 (float_conditional_branch_insn)&MacroAssembler::float_ble,
1862 nullptr, // BoolTest::no_overflow
1863 (float_conditional_branch_insn)&MacroAssembler::float_bge,
1864
1865 /* DOUBLE SHORT branches */
1866 (float_conditional_branch_insn)&MacroAssembler::double_beq,
1867 (float_conditional_branch_insn)&MacroAssembler::double_bgt,
1868 nullptr,
1869 (float_conditional_branch_insn)&MacroAssembler::double_blt,
1870 (float_conditional_branch_insn)&MacroAssembler::double_bne,
1871 (float_conditional_branch_insn)&MacroAssembler::double_ble,
1872 nullptr,
1873 (float_conditional_branch_insn)&MacroAssembler::double_bge
1874 };
1875
1876 void C2_MacroAssembler::cmp_branch(int cmpFlag, Register op1, Register op2, Label& label, bool is_far) {
1877 assert(cmpFlag >= 0 && cmpFlag < (int)(sizeof(conditional_branches) / sizeof(conditional_branches[0])),
1878 "invalid conditional branch index");
1879 (this->*conditional_branches[cmpFlag])(op1, op2, label, is_far);
1880 }
1881
1882 // This is a function should only be used by C2. Flip the unordered when unordered-greater, C2 would use
1883 // unordered-lesser instead of unordered-greater. Finally, commute the result bits at function do_one_bytecode().
1884 void C2_MacroAssembler::float_cmp_branch(int cmpFlag, FloatRegister op1, FloatRegister op2, Label& label, bool is_far) {
1885 assert(cmpFlag >= 0 && cmpFlag < (int)(sizeof(float_conditional_branches) / sizeof(float_conditional_branches[0])),
1886 "invalid float conditional branch index");
1887 int booltest_flag = cmpFlag & ~(C2_MacroAssembler::double_branch_mask);
1888 (this->*float_conditional_branches[cmpFlag])(op1, op2, label, is_far,
1889 (booltest_flag == (BoolTest::ge) || booltest_flag == (BoolTest::gt)) ? false : true);
1890 }
1891
1892 void C2_MacroAssembler::enc_cmpUEqNeLeGt_imm0_branch(int cmpFlag, Register op1, Label& L, bool is_far) {
1893 switch (cmpFlag) {
1894 case BoolTest::eq:
1895 case BoolTest::le:
1896 beqz(op1, L, is_far);
1897 break;
1898 case BoolTest::ne:
1899 case BoolTest::gt:
1900 bnez(op1, L, is_far);
1901 break;
1902 default:
1903 ShouldNotReachHere();
1904 }
1905 }
1906
1907 void C2_MacroAssembler::enc_cmpEqNe_imm0_branch(int cmpFlag, Register op1, Label& L, bool is_far) {
1908 switch (cmpFlag) {
1909 case BoolTest::eq:
1910 beqz(op1, L, is_far);
1911 break;
1912 case BoolTest::ne:
1913 bnez(op1, L, is_far);
1914 break;
1915 default:
1916 ShouldNotReachHere();
1917 }
1918 }
1919
1920 void C2_MacroAssembler::enc_cmove(int cmpFlag, Register op1, Register op2, Register dst, Register src) {
1921 Label L;
1922 cmp_branch(cmpFlag ^ (1 << neg_cond_bits), op1, op2, L);
1923 mv(dst, src);
1924 bind(L);
1925 }
1926
1927 // Set dst to NaN if any NaN input.
1928 void C2_MacroAssembler::minmax_fp(FloatRegister dst, FloatRegister src1, FloatRegister src2,
1929 bool is_double, bool is_min) {
1930 assert_different_registers(dst, src1, src2);
1931
1932 Label Done, Compare;
1933
1934 is_double ? fclass_d(t0, src1)
1935 : fclass_s(t0, src1);
1936 is_double ? fclass_d(t1, src2)
1937 : fclass_s(t1, src2);
1938 orr(t0, t0, t1);
1939 andi(t0, t0, fclass_mask::nan); // if src1 or src2 is quiet or signaling NaN then return NaN
1940 beqz(t0, Compare);
1941 is_double ? fadd_d(dst, src1, src2)
1942 : fadd_s(dst, src1, src2);
1943 j(Done);
1944
1945 bind(Compare);
1946 if (is_double) {
1947 is_min ? fmin_d(dst, src1, src2)
1948 : fmax_d(dst, src1, src2);
1949 } else {
1950 is_min ? fmin_s(dst, src1, src2)
1951 : fmax_s(dst, src1, src2);
1952 }
1953
1954 bind(Done);
1955 }
1956
1957 // According to Java SE specification, for floating-point round operations, if
1958 // the input is NaN, +/-infinity, or +/-0, the same input is returned as the
1959 // rounded result; this differs from behavior of RISC-V fcvt instructions (which
1960 // round out-of-range values to the nearest max or min value), therefore special
1961 // handling is needed by NaN, +/-Infinity, +/-0.
1962 void C2_MacroAssembler::round_double_mode(FloatRegister dst, FloatRegister src, int round_mode,
1963 Register tmp1, Register tmp2, Register tmp3) {
1964
1965 assert_different_registers(dst, src);
1966 assert_different_registers(tmp1, tmp2, tmp3);
1967
1968 // Set rounding mode for conversions
1969 // Here we use similar modes to double->long and long->double conversions
1970 // Different mode for long->double conversion matter only if long value was not representable as double,
1971 // we got long value as a result of double->long conversion so, it is definitely representable
1972 RoundingMode rm;
1973 switch (round_mode) {
1974 case RoundDoubleModeNode::rmode_ceil:
1975 rm = RoundingMode::rup;
1976 break;
1977 case RoundDoubleModeNode::rmode_floor:
1978 rm = RoundingMode::rdn;
1979 break;
1980 case RoundDoubleModeNode::rmode_rint:
1981 rm = RoundingMode::rne;
1982 break;
1983 default:
1984 ShouldNotReachHere();
1985 }
1986
1987 // tmp1 - is a register to store double converted to long int
1988 // tmp2 - is a register to create constant for comparison
1989 // tmp3 - is a register where we store modified result of double->long conversion
1990 Label done, bad_val;
1991
1992 // Conversion from double to long
1993 fcvt_l_d(tmp1, src, rm);
1994
1995 // Generate constant (tmp2)
1996 // tmp2 = 100...0000
1997 addi(tmp2, zr, 1);
1998 slli(tmp2, tmp2, 63);
1999
2000 // Prepare converted long (tmp1)
2001 // as a result when conversion overflow we got:
2002 // tmp1 = 011...1111 or 100...0000
2003 // Convert it to: tmp3 = 100...0000
2004 addi(tmp3, tmp1, 1);
2005 andi(tmp3, tmp3, -2);
2006 beq(tmp3, tmp2, bad_val);
2007
2008 // Conversion from long to double
2009 fcvt_d_l(dst, tmp1, rm);
2010 // Add sign of input value to result for +/- 0 cases
2011 fsgnj_d(dst, dst, src);
2012 j(done);
2013
2014 // If got conversion overflow return src
2015 bind(bad_val);
2016 fmv_d(dst, src);
2017
2018 bind(done);
2019 }
2020
2021 // According to Java SE specification, for floating-point signum operations, if
2022 // on input we have NaN or +/-0.0 value we should return it,
2023 // otherwise return +/- 1.0 using sign of input.
2024 // one - gives us a floating-point 1.0 (got from matching rule)
2025 // bool is_double - specifies single or double precision operations will be used.
2026 void C2_MacroAssembler::signum_fp(FloatRegister dst, FloatRegister one, bool is_double) {
2027 Label done;
2028
2029 is_double ? fclass_d(t0, dst)
2030 : fclass_s(t0, dst);
2031
2032 // check if input is -0, +0, signaling NaN or quiet NaN
2033 andi(t0, t0, fclass_mask::zero | fclass_mask::nan);
2034
2035 bnez(t0, done);
2036
2037 // use floating-point 1.0 with a sign of input
2038 is_double ? fsgnj_d(dst, one, dst)
2039 : fsgnj_s(dst, one, dst);
2040
2041 bind(done);
2042 }
2043
2044 static void float16_to_float_slow_path(C2_MacroAssembler& masm, C2GeneralStub<FloatRegister, Register, Register>& stub) {
2045 #define __ masm.
2046 FloatRegister dst = stub.data<0>();
2047 Register src = stub.data<1>();
2048 Register tmp = stub.data<2>();
2049 __ bind(stub.entry());
2050
2051 // following instructions mainly focus on NaN, as riscv does not handle
2052 // NaN well with fcvt, but the code also works for Inf at the same time.
2053
2054 // construct a NaN in 32 bits from the NaN in 16 bits,
2055 // we need the payloads of non-canonical NaNs to be preserved.
2056 __ mv(tmp, 0x7f800000);
2057 // sign-bit was already set via sign-extension if necessary.
2058 __ slli(t0, src, 13);
2059 __ orr(tmp, t0, tmp);
2060 __ fmv_w_x(dst, tmp);
2061
2062 __ j(stub.continuation());
2063 #undef __
2064 }
2065
2066 // j.l.Float.float16ToFloat
2067 void C2_MacroAssembler::float16_to_float(FloatRegister dst, Register src, Register tmp) {
2068 auto stub = C2CodeStub::make<FloatRegister, Register, Register>(dst, src, tmp, 20, float16_to_float_slow_path);
2069
2070 // On riscv, NaN needs a special process as fcvt does not work in that case.
2071 // On riscv, Inf does not need a special process as fcvt can handle it correctly.
2072 // but we consider to get the slow path to process NaN and Inf at the same time,
2073 // as both of them are rare cases, and if we try to get the slow path to handle
2074 // only NaN case it would sacrifise the performance for normal cases,
2075 // i.e. non-NaN and non-Inf cases.
2076
2077 // check whether it's a NaN or +/- Inf.
2078 mv(t0, 0x7c00);
2079 andr(tmp, src, t0);
2080 // jump to stub processing NaN and Inf cases.
2081 beq(t0, tmp, stub->entry());
2082
2083 // non-NaN or non-Inf cases, just use built-in instructions.
2084 fmv_h_x(dst, src);
2085 fcvt_s_h(dst, dst);
2086
2087 bind(stub->continuation());
2088 }
2089
2090 static void float_to_float16_slow_path(C2_MacroAssembler& masm, C2GeneralStub<Register, FloatRegister, Register>& stub) {
2091 #define __ masm.
2092 Register dst = stub.data<0>();
2093 FloatRegister src = stub.data<1>();
2094 Register tmp = stub.data<2>();
2095 __ bind(stub.entry());
2096
2097 __ fmv_x_w(dst, src);
2098
2099 // preserve the payloads of non-canonical NaNs.
2100 __ srai(dst, dst, 13);
2101 // preserve the sign bit.
2102 __ srai(tmp, dst, 13);
2103 __ slli(tmp, tmp, 10);
2104 __ mv(t0, 0x3ff);
2105 __ orr(tmp, tmp, t0);
2106
2107 // get the result by merging sign bit and payloads of preserved non-canonical NaNs.
2108 __ andr(dst, dst, tmp);
2109
2110 __ j(stub.continuation());
2111 #undef __
2112 }
2113
2114 // j.l.Float.floatToFloat16
2115 void C2_MacroAssembler::float_to_float16(Register dst, FloatRegister src, FloatRegister ftmp, Register xtmp) {
2116 auto stub = C2CodeStub::make<Register, FloatRegister, Register>(dst, src, xtmp, 130, float_to_float16_slow_path);
2117
2118 // On riscv, NaN needs a special process as fcvt does not work in that case.
2119
2120 // check whether it's a NaN.
2121 // replace fclass with feq as performance optimization.
2122 feq_s(t0, src, src);
2123 // jump to stub processing NaN cases.
2124 beqz(t0, stub->entry());
2125
2126 // non-NaN cases, just use built-in instructions.
2127 fcvt_h_s(ftmp, src);
2128 fmv_x_h(dst, ftmp);
2129
2130 bind(stub->continuation());
2131 }
2132
2133 static void float16_to_float_v_slow_path(C2_MacroAssembler& masm, C2GeneralStub<VectorRegister, VectorRegister, uint>& stub) {
2134 #define __ masm.
2135 VectorRegister dst = stub.data<0>();
2136 VectorRegister src = stub.data<1>();
2137 uint vector_length = stub.data<2>();
2138 __ bind(stub.entry());
2139
2140 // following instructions mainly focus on NaN, as riscv does not handle
2141 // NaN well with vfwcvt_f_f_v, but the code also works for Inf at the same time.
2142 //
2143 // construct NaN's in 32 bits from the NaN's in 16 bits,
2144 // we need the payloads of non-canonical NaNs to be preserved.
2145
2146 // adjust vector type to 2 * SEW.
2147 __ vsetvli_helper(T_FLOAT, vector_length, Assembler::m1);
2148 // widen and sign-extend src data.
2149 __ vsext_vf2(dst, src, Assembler::v0_t);
2150 __ mv(t0, 0x7f800000);
2151 // sign-bit was already set via sign-extension if necessary.
2152 __ vsll_vi(dst, dst, 13, Assembler::v0_t);
2153 __ vor_vx(dst, dst, t0, Assembler::v0_t);
2154
2155 __ j(stub.continuation());
2156 #undef __
2157 }
2158
2159 // j.l.Float.float16ToFloat
2160 void C2_MacroAssembler::float16_to_float_v(VectorRegister dst, VectorRegister src, uint vector_length) {
2161 auto stub = C2CodeStub::make<VectorRegister, VectorRegister, uint>
2162 (dst, src, vector_length, 24, float16_to_float_v_slow_path);
2163 assert_different_registers(dst, src);
2164
2165 // On riscv, NaN needs a special process as vfwcvt_f_f_v does not work in that case.
2166 // On riscv, Inf does not need a special process as vfwcvt_f_f_v can handle it correctly.
2167 // but we consider to get the slow path to process NaN and Inf at the same time,
2168 // as both of them are rare cases, and if we try to get the slow path to handle
2169 // only NaN case it would sacrifise the performance for normal cases,
2170 // i.e. non-NaN and non-Inf cases.
2171
2172 vsetvli_helper(BasicType::T_SHORT, vector_length, Assembler::mf2);
2173
2174 // check whether there is a NaN or +/- Inf.
2175 mv(t0, 0x7c00);
2176 vand_vx(v0, src, t0);
2177 // v0 will be used as mask in slow path.
2178 vmseq_vx(v0, v0, t0);
2179 vcpop_m(t0, v0);
2180
2181 // For non-NaN or non-Inf cases, just use built-in instructions.
2182 vfwcvt_f_f_v(dst, src);
2183
2184 // jump to stub processing NaN and Inf cases if there is any of them in the vector-wide.
2185 bnez(t0, stub->entry());
2186
2187 bind(stub->continuation());
2188 }
2189
2190 static void float_to_float16_v_slow_path(C2_MacroAssembler& masm,
2191 C2GeneralStub<VectorRegister, VectorRegister, VectorRegister>& stub) {
2192 #define __ masm.
2193 VectorRegister dst = stub.data<0>();
2194 VectorRegister src = stub.data<1>();
2195 VectorRegister tmp = stub.data<2>();
2196 __ bind(stub.entry());
2197
2198 // mul is already set to mf2 in float_to_float16_v.
2199
2200 // preserve the payloads of non-canonical NaNs.
2201 __ vnsra_wi(dst, src, 13, Assembler::v0_t);
2202
2203 // preserve the sign bit.
2204 __ vnsra_wi(tmp, src, 26, Assembler::v0_t);
2205 __ vsll_vi(tmp, tmp, 10, Assembler::v0_t);
2206 __ mv(t0, 0x3ff);
2207 __ vor_vx(tmp, tmp, t0, Assembler::v0_t);
2208
2209 // get the result by merging sign bit and payloads of preserved non-canonical NaNs.
2210 __ vand_vv(dst, dst, tmp, Assembler::v0_t);
2211
2212 __ j(stub.continuation());
2213 #undef __
2214 }
2215
2216 // j.l.Float.float16ToFloat
2217 void C2_MacroAssembler::float_to_float16_v(VectorRegister dst, VectorRegister src, VectorRegister vtmp,
2218 Register tmp, uint vector_length) {
2219 assert_different_registers(dst, src, vtmp);
2220
2221 auto stub = C2CodeStub::make<VectorRegister, VectorRegister, VectorRegister>
2222 (dst, src, vtmp, 28, float_to_float16_v_slow_path);
2223
2224 // On riscv, NaN needs a special process as vfncvt_f_f_w does not work in that case.
2225
2226 vsetvli_helper(BasicType::T_FLOAT, vector_length, Assembler::m1);
2227
2228 // check whether there is a NaN.
2229 // replace v_fclass with vmseq_vv as performance optimization.
2230 vmfne_vv(v0, src, src);
2231 vcpop_m(t0, v0);
2232
2233 vsetvli_helper(BasicType::T_SHORT, vector_length, Assembler::mf2, tmp);
2234
2235 // For non-NaN cases, just use built-in instructions.
2236 vfncvt_f_f_w(dst, src);
2237
2238 // jump to stub processing NaN cases.
2239 bnez(t0, stub->entry());
2240
2241 bind(stub->continuation());
2242 }
2243
2244 void C2_MacroAssembler::signum_fp_v(VectorRegister dst, VectorRegister one, BasicType bt, int vlen) {
2245 vsetvli_helper(bt, vlen);
2246
2247 // check if input is -0, +0, signaling NaN or quiet NaN
2248 vfclass_v(v0, dst);
2249 mv(t0, fclass_mask::zero | fclass_mask::nan);
2250 vand_vx(v0, v0, t0);
2251 vmseq_vi(v0, v0, 0);
2252
2253 // use floating-point 1.0 with a sign of input
2254 vfsgnj_vv(dst, one, dst, v0_t);
2255 }
2256
2257 void C2_MacroAssembler::compress_bits_v(Register dst, Register src, Register mask, bool is_long) {
2258 Assembler::SEW sew = is_long ? Assembler::e64 : Assembler::e32;
2259 // intrinsic is enabled when MaxVectorSize >= 16
2260 Assembler::LMUL lmul = is_long ? Assembler::m4 : Assembler::m2;
2261 long len = is_long ? 64 : 32;
2262
2263 // load the src data(in bits) to be compressed.
2264 vsetivli(x0, 1, sew, Assembler::m1);
2265 vmv_s_x(v0, src);
2266 // reset the src data(in bytes) to zero.
2267 mv(t0, len);
2268 vsetvli(x0, t0, Assembler::e8, lmul);
2269 vmv_v_i(v4, 0);
2270 // convert the src data from bits to bytes.
2271 vmerge_vim(v4, v4, 1); // v0 as the implicit mask register
2272 // reset the dst data(in bytes) to zero.
2273 vmv_v_i(v8, 0);
2274 // load the mask data(in bits).
2275 vsetivli(x0, 1, sew, Assembler::m1);
2276 vmv_s_x(v0, mask);
2277 // compress the src data(in bytes) to dst(in bytes).
2278 vsetvli(x0, t0, Assembler::e8, lmul);
2279 vcompress_vm(v8, v4, v0);
2280 // convert the dst data from bytes to bits.
2281 vmseq_vi(v0, v8, 1);
2282 // store result back.
2283 vsetivli(x0, 1, sew, Assembler::m1);
2284 vmv_x_s(dst, v0);
2285 }
2286
2287 void C2_MacroAssembler::compress_bits_i_v(Register dst, Register src, Register mask) {
2288 compress_bits_v(dst, src, mask, /* is_long */ false);
2289 }
2290
2291 void C2_MacroAssembler::compress_bits_l_v(Register dst, Register src, Register mask) {
2292 compress_bits_v(dst, src, mask, /* is_long */ true);
2293 }
2294
2295 void C2_MacroAssembler::expand_bits_v(Register dst, Register src, Register mask, bool is_long) {
2296 Assembler::SEW sew = is_long ? Assembler::e64 : Assembler::e32;
2297 // intrinsic is enabled when MaxVectorSize >= 16
2298 Assembler::LMUL lmul = is_long ? Assembler::m4 : Assembler::m2;
2299 long len = is_long ? 64 : 32;
2300
2301 // load the src data(in bits) to be expanded.
2302 vsetivli(x0, 1, sew, Assembler::m1);
2303 vmv_s_x(v0, src);
2304 // reset the src data(in bytes) to zero.
2305 mv(t0, len);
2306 vsetvli(x0, t0, Assembler::e8, lmul);
2307 vmv_v_i(v4, 0);
2308 // convert the src data from bits to bytes.
2309 vmerge_vim(v4, v4, 1); // v0 as implicit mask register
2310 // reset the dst data(in bytes) to zero.
2311 vmv_v_i(v12, 0);
2312 // load the mask data(in bits).
2313 vsetivli(x0, 1, sew, Assembler::m1);
2314 vmv_s_x(v0, mask);
2315 // expand the src data(in bytes) to dst(in bytes).
2316 vsetvli(x0, t0, Assembler::e8, lmul);
2317 viota_m(v8, v0);
2318 vrgather_vv(v12, v4, v8, VectorMask::v0_t); // v0 as implicit mask register
2319 // convert the dst data from bytes to bits.
2320 vmseq_vi(v0, v12, 1);
2321 // store result back.
2322 vsetivli(x0, 1, sew, Assembler::m1);
2323 vmv_x_s(dst, v0);
2324 }
2325
2326 void C2_MacroAssembler::expand_bits_i_v(Register dst, Register src, Register mask) {
2327 expand_bits_v(dst, src, mask, /* is_long */ false);
2328 }
2329
2330 void C2_MacroAssembler::expand_bits_l_v(Register dst, Register src, Register mask) {
2331 expand_bits_v(dst, src, mask, /* is_long */ true);
2332 }
2333
2334 void C2_MacroAssembler::element_compare(Register a1, Register a2, Register result, Register cnt, Register tmp1, Register tmp2,
2335 VectorRegister vr1, VectorRegister vr2, VectorRegister vrs, bool islatin, Label &DONE) {
2336 Label loop;
2337 Assembler::SEW sew = islatin ? Assembler::e8 : Assembler::e16;
2338
2339 bind(loop);
2340 vsetvli(tmp1, cnt, sew, Assembler::m2);
2341 vlex_v(vr1, a1, sew);
2342 vlex_v(vr2, a2, sew);
2343 vmsne_vv(vrs, vr1, vr2);
2344 vfirst_m(tmp2, vrs);
2345 bgez(tmp2, DONE);
2346 sub(cnt, cnt, tmp1);
2347 if (!islatin) {
2348 slli(tmp1, tmp1, 1); // get byte counts
2349 }
2350 add(a1, a1, tmp1);
2351 add(a2, a2, tmp1);
2352 bnez(cnt, loop);
2353
2354 mv(result, true);
2355 }
2356
2357 void C2_MacroAssembler::string_equals_v(Register a1, Register a2, Register result, Register cnt) {
2358 Label DONE;
2359 Register tmp1 = t0;
2360 Register tmp2 = t1;
2361
2362 BLOCK_COMMENT("string_equals_v {");
2363
2364 mv(result, false);
2365
2366 element_compare(a1, a2, result, cnt, tmp1, tmp2, v2, v4, v2, true, DONE);
2367
2368 bind(DONE);
2369 BLOCK_COMMENT("} string_equals_v");
2370 }
2371
2372 // used by C2 ClearArray patterns.
2373 // base: Address of a buffer to be zeroed
2374 // cnt: Count in HeapWords
2375 //
2376 // base, cnt, v4, v5, v6, v7 and t0 are clobbered.
2377 void C2_MacroAssembler::clear_array_v(Register base, Register cnt) {
2378 Label loop;
2379
2380 // making zero words
2381 vsetvli(t0, cnt, Assembler::e64, Assembler::m4);
2382 vxor_vv(v4, v4, v4);
2383
2384 bind(loop);
2385 vsetvli(t0, cnt, Assembler::e64, Assembler::m4);
2386 vse64_v(v4, base);
2387 sub(cnt, cnt, t0);
2388 shadd(base, t0, base, t0, 3);
2389 bnez(cnt, loop);
2390 }
2391
2392 void C2_MacroAssembler::arrays_equals_v(Register a1, Register a2, Register result,
2393 Register cnt1, int elem_size) {
2394 Label DONE;
2395 Register tmp1 = t0;
2396 Register tmp2 = t1;
2397 Register cnt2 = tmp2;
2398 int length_offset = arrayOopDesc::length_offset_in_bytes();
2399 int base_offset = arrayOopDesc::base_offset_in_bytes(elem_size == 2 ? T_CHAR : T_BYTE);
2400
2401 BLOCK_COMMENT("arrays_equals_v {");
2402
2403 // if (a1 == a2), return true
2404 mv(result, true);
2405 beq(a1, a2, DONE);
2406
2407 mv(result, false);
2408 // if a1 == null or a2 == null, return false
2409 beqz(a1, DONE);
2410 beqz(a2, DONE);
2411 // if (a1.length != a2.length), return false
2412 lwu(cnt1, Address(a1, length_offset));
2413 lwu(cnt2, Address(a2, length_offset));
2414 bne(cnt1, cnt2, DONE);
2415
2416 la(a1, Address(a1, base_offset));
2417 la(a2, Address(a2, base_offset));
2418
2419 element_compare(a1, a2, result, cnt1, tmp1, tmp2, v2, v4, v2, elem_size == 1, DONE);
2420
2421 bind(DONE);
2422
2423 BLOCK_COMMENT("} arrays_equals_v");
2424 }
2425
2426 void C2_MacroAssembler::string_compare_v(Register str1, Register str2, Register cnt1, Register cnt2,
2427 Register result, Register tmp1, Register tmp2, int encForm) {
2428 Label DIFFERENCE, DONE, L, loop;
2429 bool encLL = encForm == StrIntrinsicNode::LL;
2430 bool encLU = encForm == StrIntrinsicNode::LU;
2431 bool encUL = encForm == StrIntrinsicNode::UL;
2432
2433 bool str1_isL = encLL || encLU;
2434 bool str2_isL = encLL || encUL;
2435
2436 int minCharsInWord = encLL ? wordSize : wordSize / 2;
2437
2438 BLOCK_COMMENT("string_compare {");
2439
2440 // for Latin strings, 1 byte for 1 character
2441 // for UTF16 strings, 2 bytes for 1 character
2442 if (!str1_isL)
2443 sraiw(cnt1, cnt1, 1);
2444 if (!str2_isL)
2445 sraiw(cnt2, cnt2, 1);
2446
2447 // if str1 == str2, return the difference
2448 // save the minimum of the string lengths in cnt2.
2449 sub(result, cnt1, cnt2);
2450 bgt(cnt1, cnt2, L);
2451 mv(cnt2, cnt1);
2452 bind(L);
2453
2454 if (str1_isL == str2_isL) { // LL or UU
2455 element_compare(str1, str2, zr, cnt2, tmp1, tmp2, v2, v4, v2, encLL, DIFFERENCE);
2456 j(DONE);
2457 } else { // LU or UL
2458 Register strL = encLU ? str1 : str2;
2459 Register strU = encLU ? str2 : str1;
2460 VectorRegister vstr1 = encLU ? v8 : v4;
2461 VectorRegister vstr2 = encLU ? v4 : v8;
2462
2463 bind(loop);
2464 vsetvli(tmp1, cnt2, Assembler::e8, Assembler::m2);
2465 vle8_v(vstr1, strL);
2466 vsetvli(tmp1, cnt2, Assembler::e16, Assembler::m4);
2467 vzext_vf2(vstr2, vstr1);
2468 vle16_v(vstr1, strU);
2469 vmsne_vv(v4, vstr2, vstr1);
2470 vfirst_m(tmp2, v4);
2471 bgez(tmp2, DIFFERENCE);
2472 sub(cnt2, cnt2, tmp1);
2473 add(strL, strL, tmp1);
2474 shadd(strU, tmp1, strU, tmp1, 1);
2475 bnez(cnt2, loop);
2476 j(DONE);
2477 }
2478
2479 bind(DIFFERENCE);
2480 slli(tmp1, tmp2, 1);
2481 add(str1, str1, str1_isL ? tmp2 : tmp1);
2482 add(str2, str2, str2_isL ? tmp2 : tmp1);
2483 str1_isL ? lbu(tmp1, Address(str1, 0)) : lhu(tmp1, Address(str1, 0));
2484 str2_isL ? lbu(tmp2, Address(str2, 0)) : lhu(tmp2, Address(str2, 0));
2485 sub(result, tmp1, tmp2);
2486
2487 bind(DONE);
2488 }
2489
2490 void C2_MacroAssembler::byte_array_inflate_v(Register src, Register dst, Register len, Register tmp) {
2491 Label loop;
2492 assert_different_registers(src, dst, len, tmp, t0);
2493
2494 BLOCK_COMMENT("byte_array_inflate_v {");
2495 bind(loop);
2496 vsetvli(tmp, len, Assembler::e8, Assembler::m2);
2497 vle8_v(v6, src);
2498 vsetvli(t0, len, Assembler::e16, Assembler::m4);
2499 vzext_vf2(v4, v6);
2500 vse16_v(v4, dst);
2501 sub(len, len, tmp);
2502 add(src, src, tmp);
2503 shadd(dst, tmp, dst, tmp, 1);
2504 bnez(len, loop);
2505 BLOCK_COMMENT("} byte_array_inflate_v");
2506 }
2507
2508 // Compress char[] array to byte[].
2509 // Intrinsic for java.lang.StringUTF16.compress(char[] src, int srcOff, byte[] dst, int dstOff, int len)
2510 // result: the array length if every element in array can be encoded,
2511 // otherwise, the index of first non-latin1 (> 0xff) character.
2512 void C2_MacroAssembler::char_array_compress_v(Register src, Register dst, Register len,
2513 Register result, Register tmp) {
2514 encode_iso_array_v(src, dst, len, result, tmp, false);
2515 }
2516
2517 // Intrinsic for
2518 //
2519 // - sun/nio/cs/ISO_8859_1$Encoder.implEncodeISOArray
2520 // return the number of characters copied.
2521 // - java/lang/StringUTF16.compress
2522 // return index of non-latin1 character if copy fails, otherwise 'len'.
2523 //
2524 // This version always returns the number of characters copied. A successful
2525 // copy will complete with the post-condition: 'res' == 'len', while an
2526 // unsuccessful copy will exit with the post-condition: 0 <= 'res' < 'len'.
2527 //
2528 // Clobbers: src, dst, len, result, t0
2529 void C2_MacroAssembler::encode_iso_array_v(Register src, Register dst, Register len,
2530 Register result, Register tmp, bool ascii) {
2531 Label loop, fail, done;
2532
2533 BLOCK_COMMENT("encode_iso_array_v {");
2534 mv(result, 0);
2535
2536 bind(loop);
2537 mv(tmp, ascii ? 0x7f : 0xff);
2538 vsetvli(t0, len, Assembler::e16, Assembler::m2);
2539 vle16_v(v2, src);
2540
2541 vmsgtu_vx(v1, v2, tmp);
2542 vfirst_m(tmp, v1);
2543 vmsbf_m(v0, v1);
2544 // compress char to byte
2545 vsetvli(t0, len, Assembler::e8);
2546 vncvt_x_x_w(v1, v2, Assembler::v0_t);
2547 vse8_v(v1, dst, Assembler::v0_t);
2548
2549 // fail if char > 0x7f/0xff
2550 bgez(tmp, fail);
2551 add(result, result, t0);
2552 add(dst, dst, t0);
2553 sub(len, len, t0);
2554 shadd(src, t0, src, t0, 1);
2555 bnez(len, loop);
2556 j(done);
2557
2558 bind(fail);
2559 add(result, result, tmp);
2560
2561 bind(done);
2562 BLOCK_COMMENT("} encode_iso_array_v");
2563 }
2564
2565 void C2_MacroAssembler::count_positives_v(Register ary, Register len, Register result, Register tmp) {
2566 Label LOOP, SET_RESULT, DONE;
2567
2568 BLOCK_COMMENT("count_positives_v {");
2569 assert_different_registers(ary, len, result, tmp);
2570
2571 mv(result, zr);
2572
2573 bind(LOOP);
2574 vsetvli(t0, len, Assembler::e8, Assembler::m4);
2575 vle8_v(v4, ary);
2576 vmslt_vx(v4, v4, zr);
2577 vfirst_m(tmp, v4);
2578 bgez(tmp, SET_RESULT);
2579 // if tmp == -1, all bytes are positive
2580 add(result, result, t0);
2581
2582 sub(len, len, t0);
2583 add(ary, ary, t0);
2584 bnez(len, LOOP);
2585 j(DONE);
2586
2587 // add remaining positive bytes count
2588 bind(SET_RESULT);
2589 add(result, result, tmp);
2590
2591 bind(DONE);
2592 BLOCK_COMMENT("} count_positives_v");
2593 }
2594
2595 void C2_MacroAssembler::string_indexof_char_v(Register str1, Register cnt1,
2596 Register ch, Register result,
2597 Register tmp1, Register tmp2,
2598 bool isL) {
2599 mv(result, zr);
2600
2601 Label loop, MATCH, DONE;
2602 Assembler::SEW sew = isL ? Assembler::e8 : Assembler::e16;
2603 bind(loop);
2604 vsetvli(tmp1, cnt1, sew, Assembler::m4);
2605 vlex_v(v4, str1, sew);
2606 vmseq_vx(v4, v4, ch);
2607 vfirst_m(tmp2, v4);
2608 bgez(tmp2, MATCH); // if equal, return index
2609
2610 add(result, result, tmp1);
2611 sub(cnt1, cnt1, tmp1);
2612 if (!isL) slli(tmp1, tmp1, 1);
2613 add(str1, str1, tmp1);
2614 bnez(cnt1, loop);
2615
2616 mv(result, -1);
2617 j(DONE);
2618
2619 bind(MATCH);
2620 add(result, result, tmp2);
2621
2622 bind(DONE);
2623 }
2624
2625 // Set dst to NaN if any NaN input.
2626 void C2_MacroAssembler::minmax_fp_v(VectorRegister dst, VectorRegister src1, VectorRegister src2,
2627 BasicType bt, bool is_min, uint vector_length) {
2628 assert_different_registers(dst, src1, src2);
2629
2630 vsetvli_helper(bt, vector_length);
2631
2632 is_min ? vfmin_vv(dst, src1, src2)
2633 : vfmax_vv(dst, src1, src2);
2634
2635 vmfne_vv(v0, src1, src1);
2636 vfadd_vv(dst, src1, src1, Assembler::v0_t);
2637 vmfne_vv(v0, src2, src2);
2638 vfadd_vv(dst, src2, src2, Assembler::v0_t);
2639 }
2640
2641 // Set dst to NaN if any NaN input.
2642 // The destination vector register elements corresponding to masked-off elements
2643 // are handled with a mask-undisturbed policy.
2644 void C2_MacroAssembler::minmax_fp_masked_v(VectorRegister dst, VectorRegister src1, VectorRegister src2,
2645 VectorRegister vmask, VectorRegister tmp1, VectorRegister tmp2,
2646 BasicType bt, bool is_min, uint vector_length) {
2647 assert_different_registers(src1, src2, tmp1, tmp2);
2648 vsetvli_helper(bt, vector_length);
2649
2650 // Check vector elements of src1 and src2 for NaN.
2651 vmfeq_vv(tmp1, src1, src1);
2652 vmfeq_vv(tmp2, src2, src2);
2653
2654 vmandn_mm(v0, vmask, tmp1);
2655 vfadd_vv(dst, src1, src1, Assembler::v0_t);
2656 vmandn_mm(v0, vmask, tmp2);
2657 vfadd_vv(dst, src2, src2, Assembler::v0_t);
2658
2659 vmand_mm(tmp2, tmp1, tmp2);
2660 vmand_mm(v0, vmask, tmp2);
2661 is_min ? vfmin_vv(dst, src1, src2, Assembler::v0_t)
2662 : vfmax_vv(dst, src1, src2, Assembler::v0_t);
2663 }
2664
2665 // Set dst to NaN if any NaN input.
2666 void C2_MacroAssembler::reduce_minmax_fp_v(FloatRegister dst,
2667 FloatRegister src1, VectorRegister src2,
2668 VectorRegister tmp1, VectorRegister tmp2,
2669 bool is_double, bool is_min, uint vector_length, VectorMask vm) {
2670 assert_different_registers(dst, src1);
2671 assert_different_registers(src2, tmp1, tmp2);
2672
2673 Label L_done, L_NaN_1, L_NaN_2;
2674 // Set dst to src1 if src1 is NaN
2675 is_double ? feq_d(t0, src1, src1)
2676 : feq_s(t0, src1, src1);
2677 beqz(t0, L_NaN_2);
2678
2679 vsetvli_helper(is_double ? T_DOUBLE : T_FLOAT, vector_length);
2680 vfmv_s_f(tmp2, src1);
2681
2682 is_min ? vfredmin_vs(tmp1, src2, tmp2, vm)
2683 : vfredmax_vs(tmp1, src2, tmp2, vm);
2684 vfmv_f_s(dst, tmp1);
2685
2686 // Checking NaNs in src2
2687 vmfne_vv(tmp1, src2, src2, vm);
2688 vcpop_m(t0, tmp1, vm);
2689 beqz(t0, L_done);
2690
2691 bind(L_NaN_1);
2692 vfredusum_vs(tmp1, src2, tmp2, vm);
2693 vfmv_f_s(dst, tmp1);
2694 j(L_done);
2695
2696 bind(L_NaN_2);
2697 is_double ? fmv_d(dst, src1)
2698 : fmv_s(dst, src1);
2699 bind(L_done);
2700 }
2701
2702 bool C2_MacroAssembler::in_scratch_emit_size() {
2703 if (ciEnv::current()->task() != nullptr) {
2704 PhaseOutput* phase_output = Compile::current()->output();
2705 if (phase_output != nullptr && phase_output->in_scratch_emit_size()) {
2706 return true;
2707 }
2708 }
2709 return MacroAssembler::in_scratch_emit_size();
2710 }
2711
2712 void C2_MacroAssembler::reduce_integral_v(Register dst, Register src1,
2713 VectorRegister src2, VectorRegister tmp,
2714 int opc, BasicType bt, uint vector_length, VectorMask vm) {
2715 assert(bt == T_BYTE || bt == T_SHORT || bt == T_INT || bt == T_LONG, "unsupported element type");
2716 vsetvli_helper(bt, vector_length);
2717 vmv_s_x(tmp, src1);
2718 switch (opc) {
2719 case Op_AddReductionVI:
2720 case Op_AddReductionVL:
2721 vredsum_vs(tmp, src2, tmp, vm);
2722 break;
2723 case Op_AndReductionV:
2724 vredand_vs(tmp, src2, tmp, vm);
2725 break;
2726 case Op_OrReductionV:
2727 vredor_vs(tmp, src2, tmp, vm);
2728 break;
2729 case Op_XorReductionV:
2730 vredxor_vs(tmp, src2, tmp, vm);
2731 break;
2732 case Op_MaxReductionV:
2733 vredmax_vs(tmp, src2, tmp, vm);
2734 break;
2735 case Op_MinReductionV:
2736 vredmin_vs(tmp, src2, tmp, vm);
2737 break;
2738 default:
2739 ShouldNotReachHere();
2740 }
2741 vmv_x_s(dst, tmp);
2742 }
2743
2744 // Set vl and vtype for full and partial vector operations.
2745 // (vma = mu, vta = tu, vill = false)
2746 void C2_MacroAssembler::vsetvli_helper(BasicType bt, uint vector_length, LMUL vlmul, Register tmp) {
2747 Assembler::SEW sew = Assembler::elemtype_to_sew(bt);
2748 if (vector_length <= 31) {
2749 vsetivli(tmp, vector_length, sew, vlmul);
2750 } else if (vector_length == (MaxVectorSize / type2aelembytes(bt))) {
2751 vsetvli(tmp, x0, sew, vlmul);
2752 } else {
2753 mv(tmp, vector_length);
2754 vsetvli(tmp, tmp, sew, vlmul);
2755 }
2756 }
2757
2758 void C2_MacroAssembler::compare_integral_v(VectorRegister vd, VectorRegister src1, VectorRegister src2,
2759 int cond, BasicType bt, uint vector_length, VectorMask vm) {
2760 assert(is_integral_type(bt), "unsupported element type");
2761 assert(vm == Assembler::v0_t ? vd != v0 : true, "should be different registers");
2762 vsetvli_helper(bt, vector_length);
2763 vmclr_m(vd);
2764 switch (cond) {
2765 case BoolTest::eq: vmseq_vv(vd, src1, src2, vm); break;
2766 case BoolTest::ne: vmsne_vv(vd, src1, src2, vm); break;
2767 case BoolTest::le: vmsle_vv(vd, src1, src2, vm); break;
2768 case BoolTest::ge: vmsge_vv(vd, src1, src2, vm); break;
2769 case BoolTest::lt: vmslt_vv(vd, src1, src2, vm); break;
2770 case BoolTest::gt: vmsgt_vv(vd, src1, src2, vm); break;
2771 default:
2772 assert(false, "unsupported compare condition");
2773 ShouldNotReachHere();
2774 }
2775 }
2776
2777 void C2_MacroAssembler::compare_fp_v(VectorRegister vd, VectorRegister src1, VectorRegister src2,
2778 int cond, BasicType bt, uint vector_length, VectorMask vm) {
2779 assert(is_floating_point_type(bt), "unsupported element type");
2780 assert(vm == Assembler::v0_t ? vd != v0 : true, "should be different registers");
2781 vsetvli_helper(bt, vector_length);
2782 vmclr_m(vd);
2783 switch (cond) {
2784 case BoolTest::eq: vmfeq_vv(vd, src1, src2, vm); break;
2785 case BoolTest::ne: vmfne_vv(vd, src1, src2, vm); break;
2786 case BoolTest::le: vmfle_vv(vd, src1, src2, vm); break;
2787 case BoolTest::ge: vmfge_vv(vd, src1, src2, vm); break;
2788 case BoolTest::lt: vmflt_vv(vd, src1, src2, vm); break;
2789 case BoolTest::gt: vmfgt_vv(vd, src1, src2, vm); break;
2790 default:
2791 assert(false, "unsupported compare condition");
2792 ShouldNotReachHere();
2793 }
2794 }
2795
2796 void C2_MacroAssembler::integer_extend_v(VectorRegister dst, BasicType dst_bt, uint vector_length,
2797 VectorRegister src, BasicType src_bt, bool is_signed) {
2798 assert(type2aelembytes(dst_bt) > type2aelembytes(src_bt) && type2aelembytes(dst_bt) <= 8 && type2aelembytes(src_bt) <= 4, "invalid element size");
2799 assert(dst_bt != T_FLOAT && dst_bt != T_DOUBLE && src_bt != T_FLOAT && src_bt != T_DOUBLE, "unsupported element type");
2800 // https://github.com/riscv/riscv-v-spec/blob/master/v-spec.adoc#52-vector-operands
2801 // The destination EEW is greater than the source EEW, the source EMUL is at least 1,
2802 // and the overlap is in the highest-numbered part of the destination register group.
2803 // Since LMUL=1, vd and vs cannot be the same.
2804 assert_different_registers(dst, src);
2805
2806 vsetvli_helper(dst_bt, vector_length);
2807 if (is_signed) {
2808 if (src_bt == T_BYTE) {
2809 switch (dst_bt) {
2810 case T_SHORT:
2811 vsext_vf2(dst, src);
2812 break;
2813 case T_INT:
2814 vsext_vf4(dst, src);
2815 break;
2816 case T_LONG:
2817 vsext_vf8(dst, src);
2818 break;
2819 default:
2820 ShouldNotReachHere();
2821 }
2822 } else if (src_bt == T_SHORT) {
2823 if (dst_bt == T_INT) {
2824 vsext_vf2(dst, src);
2825 } else {
2826 vsext_vf4(dst, src);
2827 }
2828 } else if (src_bt == T_INT) {
2829 vsext_vf2(dst, src);
2830 }
2831 } else {
2832 if (src_bt == T_BYTE) {
2833 switch (dst_bt) {
2834 case T_SHORT:
2835 vzext_vf2(dst, src);
2836 break;
2837 case T_INT:
2838 vzext_vf4(dst, src);
2839 break;
2840 case T_LONG:
2841 vzext_vf8(dst, src);
2842 break;
2843 default:
2844 ShouldNotReachHere();
2845 }
2846 } else if (src_bt == T_SHORT) {
2847 if (dst_bt == T_INT) {
2848 vzext_vf2(dst, src);
2849 } else {
2850 vzext_vf4(dst, src);
2851 }
2852 } else if (src_bt == T_INT) {
2853 vzext_vf2(dst, src);
2854 }
2855 }
2856 }
2857
2858 // Vector narrow from src to dst with specified element sizes.
2859 // High part of dst vector will be filled with zero.
2860 void C2_MacroAssembler::integer_narrow_v(VectorRegister dst, BasicType dst_bt, uint vector_length,
2861 VectorRegister src, BasicType src_bt) {
2862 assert(type2aelembytes(dst_bt) < type2aelembytes(src_bt) && type2aelembytes(dst_bt) <= 4 && type2aelembytes(src_bt) <= 8, "invalid element size");
2863 assert(dst_bt != T_FLOAT && dst_bt != T_DOUBLE && src_bt != T_FLOAT && src_bt != T_DOUBLE, "unsupported element type");
2864 mv(t0, vector_length);
2865 if (src_bt == T_LONG) {
2866 // https://github.com/riscv/riscv-v-spec/blob/master/v-spec.adoc#117-vector-narrowing-integer-right-shift-instructions
2867 // Future extensions might add support for versions that narrow to a destination that is 1/4 the width of the source.
2868 // So we can currently only scale down by 1/2 the width at a time.
2869 vsetvli(t0, t0, Assembler::e32, Assembler::mf2);
2870 vncvt_x_x_w(dst, src);
2871 if (dst_bt == T_SHORT || dst_bt == T_BYTE) {
2872 vsetvli(t0, t0, Assembler::e16, Assembler::mf2);
2873 vncvt_x_x_w(dst, dst);
2874 if (dst_bt == T_BYTE) {
2875 vsetvli(t0, t0, Assembler::e8, Assembler::mf2);
2876 vncvt_x_x_w(dst, dst);
2877 }
2878 }
2879 } else if (src_bt == T_INT) {
2880 // T_SHORT
2881 vsetvli(t0, t0, Assembler::e16, Assembler::mf2);
2882 vncvt_x_x_w(dst, src);
2883 if (dst_bt == T_BYTE) {
2884 vsetvli(t0, t0, Assembler::e8, Assembler::mf2);
2885 vncvt_x_x_w(dst, dst);
2886 }
2887 } else if (src_bt == T_SHORT) {
2888 vsetvli(t0, t0, Assembler::e8, Assembler::mf2);
2889 vncvt_x_x_w(dst, src);
2890 }
2891 }
2892
2893 #define VFCVT_SAFE(VFLOATCVT) \
2894 void C2_MacroAssembler::VFLOATCVT##_safe(VectorRegister dst, VectorRegister src) { \
2895 assert_different_registers(dst, src); \
2896 vxor_vv(dst, dst, dst); \
2897 vmfeq_vv(v0, src, src); \
2898 VFLOATCVT(dst, src, Assembler::v0_t); \
2899 }
2900
2901 VFCVT_SAFE(vfcvt_rtz_x_f_v);
2902
2903 #undef VFCVT_SAFE
2904
2905 // Extract a scalar element from an vector at position 'idx'.
2906 // The input elements in src are expected to be of integral type.
2907 void C2_MacroAssembler::extract_v(Register dst, VectorRegister src, BasicType bt,
2908 int idx, VectorRegister tmp) {
2909 assert(is_integral_type(bt), "unsupported element type");
2910 assert(idx >= 0, "idx cannot be negative");
2911 // Only need the first element after vector slidedown
2912 vsetvli_helper(bt, 1);
2913 if (idx == 0) {
2914 vmv_x_s(dst, src);
2915 } else if (idx <= 31) {
2916 vslidedown_vi(tmp, src, idx);
2917 vmv_x_s(dst, tmp);
2918 } else {
2919 mv(t0, idx);
2920 vslidedown_vx(tmp, src, t0);
2921 vmv_x_s(dst, tmp);
2922 }
2923 }
2924
2925 // Extract a scalar element from an vector at position 'idx'.
2926 // The input elements in src are expected to be of floating point type.
2927 void C2_MacroAssembler::extract_fp_v(FloatRegister dst, VectorRegister src, BasicType bt,
2928 int idx, VectorRegister tmp) {
2929 assert(is_floating_point_type(bt), "unsupported element type");
2930 assert(idx >= 0, "idx cannot be negative");
2931 // Only need the first element after vector slidedown
2932 vsetvli_helper(bt, 1);
2933 if (idx == 0) {
2934 vfmv_f_s(dst, src);
2935 } else if (idx <= 31) {
2936 vslidedown_vi(tmp, src, idx);
2937 vfmv_f_s(dst, tmp);
2938 } else {
2939 mv(t0, idx);
2940 vslidedown_vx(tmp, src, t0);
2941 vfmv_f_s(dst, tmp);
2942 }
2943 }