1 /*
2 * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "asm/assembler.hpp"
27 #include "asm/assembler.inline.hpp"
28 #include "opto/c2_MacroAssembler.hpp"
29 #include "opto/compile.hpp"
30 #include "opto/intrinsicnode.hpp"
31 #include "opto/output.hpp"
32 #include "opto/subnode.hpp"
33 #include "runtime/objectMonitorTable.hpp"
34 #include "runtime/stubRoutines.hpp"
35 #include "runtime/synchronizer.hpp"
36 #include "utilities/globalDefinitions.hpp"
37
38 #ifdef PRODUCT
39 #define BLOCK_COMMENT(str) /* nothing */
40 #define STOP(error) stop(error)
41 #else
42 #define BLOCK_COMMENT(str) block_comment(str)
43 #define STOP(error) block_comment(error); stop(error)
44 #endif
45
46 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
47
48 void C2_MacroAssembler::fast_lock(Register obj, Register box,
49 Register tmp1, Register tmp2, Register tmp3, Register tmp4) {
50 // Flag register, zero for success; non-zero for failure.
51 Register flag = t1;
52
53 assert_different_registers(obj, box, tmp1, tmp2, tmp3, tmp4, flag, t0);
54
55 mv(flag, 1);
56
57 // Handle inflated monitor.
58 Label inflated;
59 // Finish fast lock successfully. MUST branch to with flag == 0
60 Label locked;
61 // Finish fast lock unsuccessfully. slow_path MUST branch to with flag != 0
62 Label slow_path;
63
64 if (UseObjectMonitorTable) {
65 // Clear cache in case fast locking succeeds or we need to take the slow-path.
66 sd(zr, Address(box, BasicLock::object_monitor_cache_offset_in_bytes()));
67 }
68
69 if (DiagnoseSyncOnValueBasedClasses != 0) {
70 load_klass(tmp1, obj);
71 lbu(tmp1, Address(tmp1, Klass::misc_flags_offset()));
72 test_bit(tmp1, tmp1, exact_log2(KlassFlags::_misc_is_value_based_class));
73 bnez(tmp1, slow_path);
74 }
75
76 const Register tmp1_mark = tmp1;
77 const Register tmp3_t = tmp3;
78
79 { // Fast locking
80
81 // Push lock to the lock stack and finish successfully. MUST branch to with flag == 0
82 Label push;
83
84 const Register tmp2_top = tmp2;
85
86 // Check if lock-stack is full.
87 lwu(tmp2_top, Address(xthread, JavaThread::lock_stack_top_offset()));
88 mv(tmp3_t, (unsigned)LockStack::end_offset());
89 bge(tmp2_top, tmp3_t, slow_path);
90
91 // Check if recursive.
92 add(tmp3_t, xthread, tmp2_top);
93 ld(tmp3_t, Address(tmp3_t, -oopSize));
94 beq(obj, tmp3_t, push);
95
96 // Relaxed normal load to check for monitor. Optimization for monitor case.
97 ld(tmp1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
98 test_bit(tmp3_t, tmp1_mark, exact_log2(markWord::monitor_value));
99 bnez(tmp3_t, inflated);
100
101 // Not inflated
102 assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid a la");
103
104 // Try to lock. Transition lock-bits 0b01 => 0b00
105 ori(tmp1_mark, tmp1_mark, markWord::unlocked_value);
106 xori(tmp3_t, tmp1_mark, markWord::unlocked_value);
107 cmpxchg(/*addr*/ obj, /*expected*/ tmp1_mark, /*new*/ tmp3_t, Assembler::int64,
108 /*acquire*/ Assembler::aq, /*release*/ Assembler::relaxed, /*result*/ tmp3_t);
109 bne(tmp1_mark, tmp3_t, slow_path);
110
111 bind(push);
112 // After successful lock, push object on lock-stack.
113 add(tmp3_t, xthread, tmp2_top);
114 sd(obj, Address(tmp3_t));
115 addw(tmp2_top, tmp2_top, oopSize);
116 sw(tmp2_top, Address(xthread, JavaThread::lock_stack_top_offset()));
117 j(locked);
118 }
119
120 { // Handle inflated monitor.
121 bind(inflated);
122
123 const Register tmp1_monitor = tmp1;
124 // Offsets into the current thread's object monitor cache (omc).
125 const ByteSize thr_omc_offset = JavaThread::om_cache_offset();
126 const ByteSize omc_monitor_offset = OMCache::monitor_offset();
127 const ByteSize omc_obj_offset = OMCache::obj_offset();
128
129 if (!UseObjectMonitorTable) {
130 assert(tmp1_monitor == tmp1_mark, "should be the same here");
131 } else {
132 const Register tmp2_hash = tmp2;
133 const Register tmp3_bucket = tmp3;
134 Label monitor_found;
135
136 // Save the mark, we might need it to extract the hash.
137 mv(tmp2_hash, tmp1_mark);
138
139 // Look for the monitor in the current thread's object monitor cache (omc).
140
141 ld(tmp1_monitor, Address(xthread, thr_omc_offset + omc_monitor_offset));
142 ld(tmp4, Address(xthread, thr_omc_offset + omc_obj_offset));
143 beq(obj, tmp4, monitor_found);
144
145 // Look for the monitor in the table.
146
147 // Get the hash code.
148 srli(tmp2_hash, tmp2_hash, markWord::hash_shift);
149
150 // Get the table and calculate the bucket's address.
151 la(tmp3_t, ExternalAddress(ObjectMonitorTable::current_table_address()));
152 ld(tmp3_t, Address(tmp3_t));
153 ld(tmp1, Address(tmp3_t, ObjectMonitorTable::table_capacity_mask_offset()));
154 andr(tmp2_hash, tmp2_hash, tmp1);
155 ld(tmp3_t, Address(tmp3_t, ObjectMonitorTable::table_buckets_offset()));
156
157 // Read the monitor from the bucket.
158 shadd(tmp3_bucket, tmp2_hash, tmp3_t, tmp4, LogBytesPerWord);
159 ld(tmp1_monitor, Address(tmp3_bucket));
160
161 // Check if the monitor in the bucket is special (empty, tombstone or removed).
162 mv(tmp2, ObjectMonitorTable::SpecialPointerValues::below_is_special);
163 bltu(tmp1_monitor, tmp2, slow_path);
164
165 // Check if object matches.
166 ld(tmp3, Address(tmp1_monitor, ObjectMonitor::object_offset()));
167 BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
168 bs_asm->try_peek_weak_handle_in_nmethod(this, tmp3, tmp3, tmp2, slow_path);
169 bne(tmp3, obj, slow_path);
170
171 // Store the monitor in the current thread's object monitor cache (omc).
172 sd(tmp1_monitor, Address(xthread, thr_omc_offset + omc_monitor_offset));
173 sd(obj, Address(xthread, thr_omc_offset + omc_obj_offset));
174
175 bind(monitor_found);
176 }
177
178 const Register tmp2_owner_addr = tmp2;
179 const Register tmp3_owner = tmp3;
180
181 const ByteSize monitor_tag = in_ByteSize(UseObjectMonitorTable ? 0 : checked_cast<int>(markWord::monitor_value));
182 const Address owner_address(tmp1_monitor, ObjectMonitor::owner_offset() - monitor_tag);
183 const Address recursions_address(tmp1_monitor, ObjectMonitor::recursions_offset() - monitor_tag);
184
185 Label monitor_locked;
186
187 // Compute owner address.
188 la(tmp2_owner_addr, owner_address);
189
190 // Try to CAS owner (no owner => current thread's _monitor_owner_id).
191 Register tid = tmp4;
192 ld(tid, Address(xthread, JavaThread::monitor_owner_id_offset()));
193 cmpxchg(/*addr*/ tmp2_owner_addr, /*expected*/ zr, /*new*/ tid, Assembler::int64,
194 /*acquire*/ Assembler::aq, /*release*/ Assembler::relaxed, /*result*/ tmp3_owner);
195 beqz(tmp3_owner, monitor_locked);
196
197 // Check if recursive.
198 bne(tmp3_owner, tid, slow_path);
199
200 // Recursive.
201 increment(recursions_address, 1, tmp2, tmp3);
202
203 bind(monitor_locked);
204 if (UseObjectMonitorTable) {
205 // Cache the monitor for unlock.
206 sd(tmp1_monitor, Address(box, BasicLock::object_monitor_cache_offset_in_bytes()));
207 }
208 }
209
210 bind(locked);
211 mv(flag, zr);
212
213 #ifdef ASSERT
214 // Check that locked label is reached with flag == 0.
215 Label flag_correct;
216 beqz(flag, flag_correct);
217 stop("Fast Lock Flag != 0");
218 #endif
219
220 bind(slow_path);
221 #ifdef ASSERT
222 // Check that slow_path label is reached with flag != 0.
223 bnez(flag, flag_correct);
224 stop("Fast Lock Flag == 0");
225 bind(flag_correct);
226 #endif
227 // C2 uses the value of flag (0 vs !0) to determine the continuation.
228 }
229
230 void C2_MacroAssembler::fast_unlock(Register obj, Register box,
231 Register tmp1, Register tmp2, Register tmp3) {
232 // Flag register, zero for success; non-zero for failure.
233 Register flag = t1;
234
235 assert_different_registers(obj, box, tmp1, tmp2, tmp3, flag, t0);
236
237 mv(flag, 1);
238
239 // Handle inflated monitor.
240 Label inflated, inflated_load_mark;
241 // Finish fast unlock successfully. unlocked MUST branch to with flag == 0
242 Label unlocked;
243 // Finish fast unlock unsuccessfully. MUST branch to with flag != 0
244 Label slow_path;
245
246 const Register tmp1_mark = tmp1;
247 const Register tmp2_top = tmp2;
248 const Register tmp3_t = tmp3;
249
250 { // Fast unlock
251 Label push_and_slow_path;
252
253 // Check if obj is top of lock-stack.
254 lwu(tmp2_top, Address(xthread, JavaThread::lock_stack_top_offset()));
255 subw(tmp2_top, tmp2_top, oopSize);
256 add(tmp3_t, xthread, tmp2_top);
257 ld(tmp3_t, Address(tmp3_t));
258 // Top of lock stack was not obj. Must be monitor.
259 bne(obj, tmp3_t, inflated_load_mark);
260
261 // Pop lock-stack.
262 DEBUG_ONLY(add(tmp3_t, xthread, tmp2_top);)
263 DEBUG_ONLY(sd(zr, Address(tmp3_t));)
264 sw(tmp2_top, Address(xthread, JavaThread::lock_stack_top_offset()));
265
266 // Check if recursive.
267 add(tmp3_t, xthread, tmp2_top);
268 ld(tmp3_t, Address(tmp3_t, -oopSize));
269 beq(obj, tmp3_t, unlocked);
270
271 // Not recursive.
272 // Load Mark.
273 ld(tmp1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
274
275 // Check header for monitor (0b10).
276 // Because we got here by popping (meaning we pushed in locked)
277 // there will be no monitor in the box. So we need to push back the obj
278 // so that the runtime can fix any potential anonymous owner.
279 test_bit(tmp3_t, tmp1_mark, exact_log2(markWord::monitor_value));
280 bnez(tmp3_t, UseObjectMonitorTable ? push_and_slow_path : inflated);
281
282 // Try to unlock. Transition lock bits 0b00 => 0b01
283 assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
284 ori(tmp3_t, tmp1_mark, markWord::unlocked_value);
285 cmpxchg(/*addr*/ obj, /*expected*/ tmp1_mark, /*new*/ tmp3_t, Assembler::int64,
286 /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, /*result*/ tmp3_t);
287 beq(tmp1_mark, tmp3_t, unlocked);
288
289 bind(push_and_slow_path);
290 // Compare and exchange failed.
291 // Restore lock-stack and handle the unlock in runtime.
292 DEBUG_ONLY(add(tmp3_t, xthread, tmp2_top);)
293 DEBUG_ONLY(sd(obj, Address(tmp3_t));)
294 addw(tmp2_top, tmp2_top, oopSize);
295 sd(tmp2_top, Address(xthread, JavaThread::lock_stack_top_offset()));
296 j(slow_path);
297 }
298
299 { // Handle inflated monitor.
300 bind(inflated_load_mark);
301 ld(tmp1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
302 #ifdef ASSERT
303 test_bit(tmp3_t, tmp1_mark, exact_log2(markWord::monitor_value));
304 bnez(tmp3_t, inflated);
305 stop("Fast Unlock not monitor");
306 #endif
307
308 bind(inflated);
309
310 #ifdef ASSERT
311 Label check_done;
312 subw(tmp2_top, tmp2_top, oopSize);
313 mv(tmp3_t, in_bytes(JavaThread::lock_stack_base_offset()));
314 blt(tmp2_top, tmp3_t, check_done);
315 add(tmp3_t, xthread, tmp2_top);
316 ld(tmp3_t, Address(tmp3_t));
317 bne(obj, tmp3_t, inflated);
318 stop("Fast Unlock lock on stack");
319 bind(check_done);
320 #endif
321
322 const Register tmp1_monitor = tmp1;
323
324 if (!UseObjectMonitorTable) {
325 assert(tmp1_monitor == tmp1_mark, "should be the same here");
326 // Untag the monitor.
327 subi(tmp1_monitor, tmp1_mark, (int)markWord::monitor_value);
328 } else {
329 ld(tmp1_monitor, Address(box, BasicLock::object_monitor_cache_offset_in_bytes()));
330 // No valid pointer below alignof(ObjectMonitor*). Take the slow path.
331 mv(tmp3_t, alignof(ObjectMonitor*));
332 bltu(tmp1_monitor, tmp3_t, slow_path);
333 }
334
335 const Register tmp2_recursions = tmp2;
336 Label not_recursive;
337
338 // Check if recursive.
339 ld(tmp2_recursions, Address(tmp1_monitor, ObjectMonitor::recursions_offset()));
340 beqz(tmp2_recursions, not_recursive);
341
342 // Recursive unlock.
343 subi(tmp2_recursions, tmp2_recursions, 1);
344 sd(tmp2_recursions, Address(tmp1_monitor, ObjectMonitor::recursions_offset()));
345 j(unlocked);
346
347 bind(not_recursive);
348
349 const Register tmp2_owner_addr = tmp2;
350
351 // Compute owner address.
352 la(tmp2_owner_addr, Address(tmp1_monitor, ObjectMonitor::owner_offset()));
353
354 // Set owner to null.
355 // Release to satisfy the JMM
356 membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore);
357 sd(zr, Address(tmp2_owner_addr));
358 // We need a full fence after clearing owner to avoid stranding.
359 // StoreLoad achieves this.
360 membar(StoreLoad);
361
362 // Check if the entry_list is empty.
363 ld(t0, Address(tmp1_monitor, ObjectMonitor::entry_list_offset()));
364 beqz(t0, unlocked); // If so we are done.
365
366 // Check if there is a successor.
367 ld(tmp3_t, Address(tmp1_monitor, ObjectMonitor::succ_offset()));
368 bnez(tmp3_t, unlocked); // If so we are done.
369
370 // Save the monitor pointer in the current thread, so we can try
371 // to reacquire the lock in SharedRuntime::monitor_exit_helper().
372 sd(tmp1_monitor, Address(xthread, JavaThread::unlocked_inflated_monitor_offset()));
373
374 mv(flag, 1);
375 j(slow_path);
376 }
377
378 bind(unlocked);
379 mv(flag, zr);
380
381 #ifdef ASSERT
382 // Check that unlocked label is reached with flag == 0.
383 Label flag_correct;
384 beqz(flag, flag_correct);
385 stop("Fast Lock Flag != 0");
386 #endif
387
388 bind(slow_path);
389 #ifdef ASSERT
390 // Check that slow_path label is reached with flag != 0.
391 bnez(flag, flag_correct);
392 stop("Fast Lock Flag == 0");
393 bind(flag_correct);
394 #endif
395 // C2 uses the value of flag (0 vs !0) to determine the continuation.
396 }
397
398 // short string
399 // StringUTF16.indexOfChar
400 // StringLatin1.indexOfChar
401 void C2_MacroAssembler::string_indexof_char_short(Register str1, Register cnt1,
402 Register ch, Register result,
403 bool isL)
404 {
405 Register ch1 = t0;
406 Register index = t1;
407
408 BLOCK_COMMENT("string_indexof_char_short {");
409
410 Label LOOP, LOOP1, LOOP4, LOOP8;
411 Label MATCH, MATCH1, MATCH2, MATCH3,
412 MATCH4, MATCH5, MATCH6, MATCH7, NOMATCH;
413
414 mv(result, -1);
415 mv(index, zr);
416
417 bind(LOOP);
418 addi(t0, index, 8);
419 ble(t0, cnt1, LOOP8);
420 addi(t0, index, 4);
421 ble(t0, cnt1, LOOP4);
422 j(LOOP1);
423
424 bind(LOOP8);
425 isL ? lbu(ch1, Address(str1, 0)) : lhu(ch1, Address(str1, 0));
426 beq(ch, ch1, MATCH);
427 isL ? lbu(ch1, Address(str1, 1)) : lhu(ch1, Address(str1, 2));
428 beq(ch, ch1, MATCH1);
429 isL ? lbu(ch1, Address(str1, 2)) : lhu(ch1, Address(str1, 4));
430 beq(ch, ch1, MATCH2);
431 isL ? lbu(ch1, Address(str1, 3)) : lhu(ch1, Address(str1, 6));
432 beq(ch, ch1, MATCH3);
433 isL ? lbu(ch1, Address(str1, 4)) : lhu(ch1, Address(str1, 8));
434 beq(ch, ch1, MATCH4);
435 isL ? lbu(ch1, Address(str1, 5)) : lhu(ch1, Address(str1, 10));
436 beq(ch, ch1, MATCH5);
437 isL ? lbu(ch1, Address(str1, 6)) : lhu(ch1, Address(str1, 12));
438 beq(ch, ch1, MATCH6);
439 isL ? lbu(ch1, Address(str1, 7)) : lhu(ch1, Address(str1, 14));
440 beq(ch, ch1, MATCH7);
441 addi(index, index, 8);
442 addi(str1, str1, isL ? 8 : 16);
443 blt(index, cnt1, LOOP);
444 j(NOMATCH);
445
446 bind(LOOP4);
447 isL ? lbu(ch1, Address(str1, 0)) : lhu(ch1, Address(str1, 0));
448 beq(ch, ch1, MATCH);
449 isL ? lbu(ch1, Address(str1, 1)) : lhu(ch1, Address(str1, 2));
450 beq(ch, ch1, MATCH1);
451 isL ? lbu(ch1, Address(str1, 2)) : lhu(ch1, Address(str1, 4));
452 beq(ch, ch1, MATCH2);
453 isL ? lbu(ch1, Address(str1, 3)) : lhu(ch1, Address(str1, 6));
454 beq(ch, ch1, MATCH3);
455 addi(index, index, 4);
456 addi(str1, str1, isL ? 4 : 8);
457 bge(index, cnt1, NOMATCH);
458
459 bind(LOOP1);
460 isL ? lbu(ch1, Address(str1)) : lhu(ch1, Address(str1));
461 beq(ch, ch1, MATCH);
462 addi(index, index, 1);
463 addi(str1, str1, isL ? 1 : 2);
464 blt(index, cnt1, LOOP1);
465 j(NOMATCH);
466
467 bind(MATCH1);
468 addi(index, index, 1);
469 j(MATCH);
470
471 bind(MATCH2);
472 addi(index, index, 2);
473 j(MATCH);
474
475 bind(MATCH3);
476 addi(index, index, 3);
477 j(MATCH);
478
479 bind(MATCH4);
480 addi(index, index, 4);
481 j(MATCH);
482
483 bind(MATCH5);
484 addi(index, index, 5);
485 j(MATCH);
486
487 bind(MATCH6);
488 addi(index, index, 6);
489 j(MATCH);
490
491 bind(MATCH7);
492 addi(index, index, 7);
493
494 bind(MATCH);
495 mv(result, index);
496 bind(NOMATCH);
497 BLOCK_COMMENT("} string_indexof_char_short");
498 }
499
500 // StringUTF16.indexOfChar
501 // StringLatin1.indexOfChar
502 void C2_MacroAssembler::string_indexof_char(Register str1, Register cnt1,
503 Register ch, Register result,
504 Register tmp1, Register tmp2,
505 Register tmp3, Register tmp4,
506 bool isL)
507 {
508 Label CH1_LOOP, HIT, NOMATCH, DONE, DO_LONG;
509 Register ch1 = t0;
510 Register orig_cnt = t1;
511 Register mask1 = tmp3;
512 Register mask2 = tmp2;
513 Register match_mask = tmp1;
514 Register trailing_char = tmp4;
515 Register unaligned_elems = tmp4;
516
517 BLOCK_COMMENT("string_indexof_char {");
518 beqz(cnt1, NOMATCH);
519
520 subi(t0, cnt1, isL ? 32 : 16);
521 bgtz(t0, DO_LONG);
522 string_indexof_char_short(str1, cnt1, ch, result, isL);
523 j(DONE);
524
525 bind(DO_LONG);
526 mv(orig_cnt, cnt1);
527 if (AvoidUnalignedAccesses) {
528 Label ALIGNED;
529 andi(unaligned_elems, str1, 0x7);
530 beqz(unaligned_elems, ALIGNED);
531 sub(unaligned_elems, unaligned_elems, 8);
532 neg(unaligned_elems, unaligned_elems);
533 if (!isL) {
534 srli(unaligned_elems, unaligned_elems, 1);
535 }
536 // do unaligned part per element
537 string_indexof_char_short(str1, unaligned_elems, ch, result, isL);
538 bgez(result, DONE);
539 mv(orig_cnt, cnt1);
540 sub(cnt1, cnt1, unaligned_elems);
541 bind(ALIGNED);
542 }
543
544 // duplicate ch
545 if (isL) {
546 slli(ch1, ch, 8);
547 orr(ch, ch1, ch);
548 }
549 slli(ch1, ch, 16);
550 orr(ch, ch1, ch);
551 slli(ch1, ch, 32);
552 orr(ch, ch1, ch);
553
554 if (!isL) {
555 slli(cnt1, cnt1, 1);
556 }
557
558 uint64_t mask0101 = UCONST64(0x0101010101010101);
559 uint64_t mask0001 = UCONST64(0x0001000100010001);
560 mv(mask1, isL ? mask0101 : mask0001);
561 uint64_t mask7f7f = UCONST64(0x7f7f7f7f7f7f7f7f);
562 uint64_t mask7fff = UCONST64(0x7fff7fff7fff7fff);
563 mv(mask2, isL ? mask7f7f : mask7fff);
564
565 bind(CH1_LOOP);
566 ld(ch1, Address(str1));
567 addi(str1, str1, 8);
568 subi(cnt1, cnt1, 8);
569 compute_match_mask(ch1, ch, match_mask, mask1, mask2);
570 bnez(match_mask, HIT);
571 bgtz(cnt1, CH1_LOOP);
572 j(NOMATCH);
573
574 bind(HIT);
575 // count bits of trailing zero chars
576 ctzc_bits(trailing_char, match_mask, isL, ch1, result);
577 srli(trailing_char, trailing_char, 3);
578 addi(cnt1, cnt1, 8);
579 ble(cnt1, trailing_char, NOMATCH);
580 // match case
581 if (!isL) {
582 srli(cnt1, cnt1, 1);
583 srli(trailing_char, trailing_char, 1);
584 }
585
586 sub(result, orig_cnt, cnt1);
587 add(result, result, trailing_char);
588 j(DONE);
589
590 bind(NOMATCH);
591 mv(result, -1);
592
593 bind(DONE);
594 BLOCK_COMMENT("} string_indexof_char");
595 }
596
597 typedef void (MacroAssembler::* load_chr_insn)(Register rd, const Address &adr, Register temp);
598
599 // Search for needle in haystack and return index or -1
600 // x10: result
601 // x11: haystack
602 // x12: haystack_len
603 // x13: needle
604 // x14: needle_len
605 void C2_MacroAssembler::string_indexof(Register haystack, Register needle,
606 Register haystack_len, Register needle_len,
607 Register tmp1, Register tmp2,
608 Register tmp3, Register tmp4,
609 Register tmp5, Register tmp6,
610 Register result, int ae)
611 {
612 assert(ae != StrIntrinsicNode::LU, "Invalid encoding");
613
614 Label LINEARSEARCH, LINEARSTUB, DONE, NOMATCH;
615
616 Register ch1 = t0;
617 Register ch2 = t1;
618 Register nlen_tmp = tmp1; // needle len tmp
619 Register hlen_tmp = tmp2; // haystack len tmp
620 Register result_tmp = tmp4;
621
622 bool isLL = ae == StrIntrinsicNode::LL;
623
624 bool needle_isL = ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UL;
625 bool haystack_isL = ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::LU;
626 int needle_chr_shift = needle_isL ? 0 : 1;
627 int haystack_chr_shift = haystack_isL ? 0 : 1;
628 int needle_chr_size = needle_isL ? 1 : 2;
629 int haystack_chr_size = haystack_isL ? 1 : 2;
630 load_chr_insn needle_load_1chr = needle_isL ? (load_chr_insn)&MacroAssembler::lbu :
631 (load_chr_insn)&MacroAssembler::lhu;
632 load_chr_insn haystack_load_1chr = haystack_isL ? (load_chr_insn)&MacroAssembler::lbu :
633 (load_chr_insn)&MacroAssembler::lhu;
634
635 BLOCK_COMMENT("string_indexof {");
636
637 // Note, inline_string_indexOf() generates checks:
638 // if (pattern.count > src.count) return -1;
639 // if (pattern.count == 0) return 0;
640
641 // We have two strings, a source string in haystack, haystack_len and a pattern string
642 // in needle, needle_len. Find the first occurrence of pattern in source or return -1.
643
644 // For larger pattern and source we use a simplified Boyer Moore algorithm.
645 // With a small pattern and source we use linear scan.
646
647 // needle_len >=8 && needle_len < 256 && needle_len < haystack_len/4, use bmh algorithm.
648 sub(result_tmp, haystack_len, needle_len);
649 // needle_len < 8, use linear scan
650 sub(t0, needle_len, 8);
651 bltz(t0, LINEARSEARCH);
652 // needle_len >= 256, use linear scan
653 sub(t0, needle_len, 256);
654 bgez(t0, LINEARSTUB);
655 // needle_len >= haystack_len/4, use linear scan
656 srli(t0, haystack_len, 2);
657 bge(needle_len, t0, LINEARSTUB);
658
659 // Boyer-Moore-Horspool introduction:
660 // The Boyer Moore alogorithm is based on the description here:-
661 //
662 // http://en.wikipedia.org/wiki/Boyer%E2%80%93Moore_string_search_algorithm
663 //
664 // This describes and algorithm with 2 shift rules. The 'Bad Character' rule
665 // and the 'Good Suffix' rule.
666 //
667 // These rules are essentially heuristics for how far we can shift the
668 // pattern along the search string.
669 //
670 // The implementation here uses the 'Bad Character' rule only because of the
671 // complexity of initialisation for the 'Good Suffix' rule.
672 //
673 // This is also known as the Boyer-Moore-Horspool algorithm:
674 //
675 // http://en.wikipedia.org/wiki/Boyer-Moore-Horspool_algorithm
676 //
677 // #define ASIZE 256
678 //
679 // int bm(unsigned char *pattern, int m, unsigned char *src, int n) {
680 // int i, j;
681 // unsigned c;
682 // unsigned char bc[ASIZE];
683 //
684 // /* Preprocessing */
685 // for (i = 0; i < ASIZE; ++i)
686 // bc[i] = m;
687 // for (i = 0; i < m - 1; ) {
688 // c = pattern[i];
689 // ++i;
690 // // c < 256 for Latin1 string, so, no need for branch
691 // #ifdef PATTERN_STRING_IS_LATIN1
692 // bc[c] = m - i;
693 // #else
694 // if (c < ASIZE) bc[c] = m - i;
695 // #endif
696 // }
697 //
698 // /* Searching */
699 // j = 0;
700 // while (j <= n - m) {
701 // c = src[i+j];
702 // if (pattern[m-1] == c)
703 // int k;
704 // for (k = m - 2; k >= 0 && pattern[k] == src[k + j]; --k);
705 // if (k < 0) return j;
706 // // c < 256 for Latin1 string, so, no need for branch
707 // #ifdef SOURCE_STRING_IS_LATIN1_AND_PATTERN_STRING_IS_LATIN1
708 // // LL case: (c< 256) always true. Remove branch
709 // j += bc[pattern[j+m-1]];
710 // #endif
711 // #ifdef SOURCE_STRING_IS_UTF_AND_PATTERN_STRING_IS_UTF
712 // // UU case: need if (c<ASIZE) check. Skip 1 character if not.
713 // if (c < ASIZE)
714 // j += bc[pattern[j+m-1]];
715 // else
716 // j += 1
717 // #endif
718 // #ifdef SOURCE_IS_UTF_AND_PATTERN_IS_LATIN1
719 // // UL case: need if (c<ASIZE) check. Skip <pattern length> if not.
720 // if (c < ASIZE)
721 // j += bc[pattern[j+m-1]];
722 // else
723 // j += m
724 // #endif
725 // }
726 // return -1;
727 // }
728
729 // temp register:t0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, result
730 Label BCLOOP, BCSKIP, BMLOOPSTR2, BMLOOPSTR1, BMSKIP, BMADV, BMMATCH,
731 BMLOOPSTR1_LASTCMP, BMLOOPSTR1_CMP, BMLOOPSTR1_AFTER_LOAD, BM_INIT_LOOP;
732
733 Register haystack_end = haystack_len;
734 Register skipch = tmp2;
735
736 // pattern length is >=8, so, we can read at least 1 register for cases when
737 // UTF->Latin1 conversion is not needed(8 LL or 4UU) and half register for
738 // UL case. We'll re-read last character in inner pre-loop code to have
739 // single outer pre-loop load
740 const int firstStep = isLL ? 7 : 3;
741
742 const int ASIZE = 256;
743 const int STORE_BYTES = 8; // 8 bytes stored per instruction(sd)
744
745 subi(sp, sp, ASIZE);
746
747 // init BC offset table with default value: needle_len
748 slli(t0, needle_len, 8);
749 orr(t0, t0, needle_len); // [63...16][needle_len][needle_len]
750 slli(tmp1, t0, 16);
751 orr(t0, tmp1, t0); // [63...32][needle_len][needle_len][needle_len][needle_len]
752 slli(tmp1, t0, 32);
753 orr(tmp5, tmp1, t0); // tmp5: 8 elements [needle_len]
754
755 mv(ch1, sp); // ch1 is t0
756 mv(tmp6, ASIZE / STORE_BYTES); // loop iterations
757
758 bind(BM_INIT_LOOP);
759 // for (i = 0; i < ASIZE; ++i)
760 // bc[i] = m;
761 for (int i = 0; i < 4; i++) {
762 sd(tmp5, Address(ch1, i * wordSize));
763 }
764 addi(ch1, ch1, 32);
765 subi(tmp6, tmp6, 4);
766 bgtz(tmp6, BM_INIT_LOOP);
767
768 subi(nlen_tmp, needle_len, 1); // m - 1, index of the last element in pattern
769 Register orig_haystack = tmp5;
770 mv(orig_haystack, haystack);
771 // result_tmp = tmp4
772 shadd(haystack_end, result_tmp, haystack, haystack_end, haystack_chr_shift);
773 subi(ch2, needle_len, 1); // bc offset init value, ch2 is t1
774 mv(tmp3, needle);
775
776 // for (i = 0; i < m - 1; ) {
777 // c = pattern[i];
778 // ++i;
779 // // c < 256 for Latin1 string, so, no need for branch
780 // #ifdef PATTERN_STRING_IS_LATIN1
781 // bc[c] = m - i;
782 // #else
783 // if (c < ASIZE) bc[c] = m - i;
784 // #endif
785 // }
786 bind(BCLOOP);
787 (this->*needle_load_1chr)(ch1, Address(tmp3), noreg);
788 addi(tmp3, tmp3, needle_chr_size);
789 if (!needle_isL) {
790 // ae == StrIntrinsicNode::UU
791 mv(tmp6, ASIZE);
792 bgeu(ch1, tmp6, BCSKIP);
793 }
794 add(tmp4, sp, ch1);
795 sb(ch2, Address(tmp4)); // store skip offset to BC offset table
796
797 bind(BCSKIP);
798 subi(ch2, ch2, 1); // for next pattern element, skip distance -1
799 bgtz(ch2, BCLOOP);
800
801 // tmp6: pattern end, address after needle
802 shadd(tmp6, needle_len, needle, tmp6, needle_chr_shift);
803 if (needle_isL == haystack_isL) {
804 // load last 8 bytes (8LL/4UU symbols)
805 ld(tmp6, Address(tmp6, -wordSize));
806 } else {
807 // UL: from UTF-16(source) search Latin1(pattern)
808 lwu(tmp6, Address(tmp6, -wordSize / 2)); // load last 4 bytes(4 symbols)
809 // convert Latin1 to UTF. eg: 0x0000abcd -> 0x0a0b0c0d
810 // We'll have to wait until load completed, but it's still faster than per-character loads+checks
811 srli(tmp3, tmp6, BitsPerByte * (wordSize / 2 - needle_chr_size)); // pattern[m-1], eg:0x0000000a
812 slli(ch2, tmp6, XLEN - 24);
813 srli(ch2, ch2, XLEN - 8); // pattern[m-2], 0x0000000b
814 slli(ch1, tmp6, XLEN - 16);
815 srli(ch1, ch1, XLEN - 8); // pattern[m-3], 0x0000000c
816 zext(tmp6, tmp6, 8); // pattern[m-4], 0x0000000d
817 slli(ch2, ch2, 16);
818 orr(ch2, ch2, ch1); // 0x00000b0c
819 slli(result, tmp3, 48); // use result as temp register
820 orr(tmp6, tmp6, result); // 0x0a00000d
821 slli(result, ch2, 16);
822 orr(tmp6, tmp6, result); // UTF-16:0x0a0b0c0d
823 }
824
825 // i = m - 1;
826 // skipch = j + i;
827 // if (skipch == pattern[m - 1]
828 // for (k = m - 2; k >= 0 && pattern[k] == src[k + j]; --k);
829 // else
830 // move j with bad char offset table
831 bind(BMLOOPSTR2);
832 // compare pattern to source string backward
833 shadd(result, nlen_tmp, haystack, result, haystack_chr_shift);
834 (this->*haystack_load_1chr)(skipch, Address(result), noreg);
835 subi(nlen_tmp, nlen_tmp, firstStep); // nlen_tmp is positive here, because needle_len >= 8
836 if (needle_isL == haystack_isL) {
837 // re-init tmp3. It's for free because it's executed in parallel with
838 // load above. Alternative is to initialize it before loop, but it'll
839 // affect performance on in-order systems with 2 or more ld/st pipelines
840 srli(tmp3, tmp6, BitsPerByte * (wordSize - needle_chr_size)); // UU/LL: pattern[m-1]
841 }
842 if (!isLL) { // UU/UL case
843 slli(ch2, nlen_tmp, 1); // offsets in bytes
844 }
845 bne(tmp3, skipch, BMSKIP); // if not equal, skipch is bad char
846 add(result, haystack, isLL ? nlen_tmp : ch2);
847 // load 8 bytes from source string
848 // if isLL is false then read granularity can be 2
849 load_long_misaligned(ch2, Address(result), ch1, isLL ? 1 : 2); // can use ch1 as temp register here as it will be trashed by next mv anyway
850 mv(ch1, tmp6);
851 if (isLL) {
852 j(BMLOOPSTR1_AFTER_LOAD);
853 } else {
854 subi(nlen_tmp, nlen_tmp, 1); // no need to branch for UU/UL case. cnt1 >= 8
855 j(BMLOOPSTR1_CMP);
856 }
857
858 bind(BMLOOPSTR1);
859 shadd(ch1, nlen_tmp, needle, ch1, needle_chr_shift);
860 (this->*needle_load_1chr)(ch1, Address(ch1), noreg);
861 shadd(ch2, nlen_tmp, haystack, ch2, haystack_chr_shift);
862 (this->*haystack_load_1chr)(ch2, Address(ch2), noreg);
863
864 bind(BMLOOPSTR1_AFTER_LOAD);
865 subi(nlen_tmp, nlen_tmp, 1);
866 bltz(nlen_tmp, BMLOOPSTR1_LASTCMP);
867
868 bind(BMLOOPSTR1_CMP);
869 beq(ch1, ch2, BMLOOPSTR1);
870
871 bind(BMSKIP);
872 if (!isLL) {
873 // if we've met UTF symbol while searching Latin1 pattern, then we can
874 // skip needle_len symbols
875 if (needle_isL != haystack_isL) {
876 mv(result_tmp, needle_len);
877 } else {
878 mv(result_tmp, 1);
879 }
880 mv(t0, ASIZE);
881 bgeu(skipch, t0, BMADV);
882 }
883 add(result_tmp, sp, skipch);
884 lbu(result_tmp, Address(result_tmp)); // load skip offset
885
886 bind(BMADV);
887 subi(nlen_tmp, needle_len, 1);
888 // move haystack after bad char skip offset
889 shadd(haystack, result_tmp, haystack, result, haystack_chr_shift);
890 ble(haystack, haystack_end, BMLOOPSTR2);
891 addi(sp, sp, ASIZE);
892 j(NOMATCH);
893
894 bind(BMLOOPSTR1_LASTCMP);
895 bne(ch1, ch2, BMSKIP);
896
897 bind(BMMATCH);
898 sub(result, haystack, orig_haystack);
899 if (!haystack_isL) {
900 srli(result, result, 1);
901 }
902 addi(sp, sp, ASIZE);
903 j(DONE);
904
905 bind(LINEARSTUB);
906 subi(t0, needle_len, 16); // small patterns still should be handled by simple algorithm
907 bltz(t0, LINEARSEARCH);
908 mv(result, zr);
909 RuntimeAddress stub = nullptr;
910 if (isLL) {
911 stub = RuntimeAddress(StubRoutines::riscv::string_indexof_linear_ll());
912 assert(stub.target() != nullptr, "string_indexof_linear_ll stub has not been generated");
913 } else if (needle_isL) {
914 stub = RuntimeAddress(StubRoutines::riscv::string_indexof_linear_ul());
915 assert(stub.target() != nullptr, "string_indexof_linear_ul stub has not been generated");
916 } else {
917 stub = RuntimeAddress(StubRoutines::riscv::string_indexof_linear_uu());
918 assert(stub.target() != nullptr, "string_indexof_linear_uu stub has not been generated");
919 }
920 address call = reloc_call(stub);
921 if (call == nullptr) {
922 DEBUG_ONLY(reset_labels(LINEARSEARCH, DONE, NOMATCH));
923 ciEnv::current()->record_failure("CodeCache is full");
924 return;
925 }
926 j(DONE);
927
928 bind(NOMATCH);
929 mv(result, -1);
930 j(DONE);
931
932 bind(LINEARSEARCH);
933 string_indexof_linearscan(haystack, needle, haystack_len, needle_len, tmp1, tmp2, tmp3, tmp4, -1, result, ae);
934
935 bind(DONE);
936 BLOCK_COMMENT("} string_indexof");
937 }
938
939 // string_indexof
940 // result: x10
941 // src: x11
942 // src_count: x12
943 // pattern: x13
944 // pattern_count: x14 or 1/2/3/4
945 void C2_MacroAssembler::string_indexof_linearscan(Register haystack, Register needle,
946 Register haystack_len, Register needle_len,
947 Register tmp1, Register tmp2,
948 Register tmp3, Register tmp4,
949 int needle_con_cnt, Register result, int ae)
950 {
951 // Note:
952 // needle_con_cnt > 0 means needle_len register is invalid, needle length is constant
953 // for UU/LL: needle_con_cnt[1, 4], UL: needle_con_cnt = 1
954 assert(needle_con_cnt <= 4, "Invalid needle constant count");
955 assert(ae != StrIntrinsicNode::LU, "Invalid encoding");
956
957 Register ch1 = t0;
958 Register ch2 = t1;
959 Register hlen_neg = haystack_len, nlen_neg = needle_len;
960 Register nlen_tmp = tmp1, hlen_tmp = tmp2, result_tmp = tmp4;
961
962 bool isLL = ae == StrIntrinsicNode::LL;
963
964 bool needle_isL = ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UL;
965 bool haystack_isL = ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::LU;
966 int needle_chr_shift = needle_isL ? 0 : 1;
967 int haystack_chr_shift = haystack_isL ? 0 : 1;
968 int needle_chr_size = needle_isL ? 1 : 2;
969 int haystack_chr_size = haystack_isL ? 1 : 2;
970
971 load_chr_insn needle_load_1chr = needle_isL ? (load_chr_insn)&MacroAssembler::lbu :
972 (load_chr_insn)&MacroAssembler::lhu;
973 load_chr_insn haystack_load_1chr = haystack_isL ? (load_chr_insn)&MacroAssembler::lbu :
974 (load_chr_insn)&MacroAssembler::lhu;
975 load_chr_insn load_2chr = isLL ? (load_chr_insn)&MacroAssembler::lhu : (load_chr_insn)&MacroAssembler::lwu;
976 load_chr_insn load_4chr = isLL ? (load_chr_insn)&MacroAssembler::lwu : (load_chr_insn)&MacroAssembler::ld;
977
978 Label DO1, DO2, DO3, MATCH, NOMATCH, DONE;
979
980 Register first = tmp3;
981
982 if (needle_con_cnt == -1) {
983 Label DOSHORT, FIRST_LOOP, STR2_NEXT, STR1_LOOP, STR1_NEXT;
984
985 subi(t0, needle_len, needle_isL == haystack_isL ? 4 : 2);
986 bltz(t0, DOSHORT);
987
988 (this->*needle_load_1chr)(first, Address(needle), noreg);
989 slli(t0, needle_len, needle_chr_shift);
990 add(needle, needle, t0);
991 neg(nlen_neg, t0);
992 slli(t0, result_tmp, haystack_chr_shift);
993 add(haystack, haystack, t0);
994 neg(hlen_neg, t0);
995
996 bind(FIRST_LOOP);
997 add(t0, haystack, hlen_neg);
998 (this->*haystack_load_1chr)(ch2, Address(t0), noreg);
999 beq(first, ch2, STR1_LOOP);
1000
1001 bind(STR2_NEXT);
1002 addi(hlen_neg, hlen_neg, haystack_chr_size);
1003 blez(hlen_neg, FIRST_LOOP);
1004 j(NOMATCH);
1005
1006 bind(STR1_LOOP);
1007 addi(nlen_tmp, nlen_neg, needle_chr_size);
1008 addi(hlen_tmp, hlen_neg, haystack_chr_size);
1009 bgez(nlen_tmp, MATCH);
1010
1011 bind(STR1_NEXT);
1012 add(ch1, needle, nlen_tmp);
1013 (this->*needle_load_1chr)(ch1, Address(ch1), noreg);
1014 add(ch2, haystack, hlen_tmp);
1015 (this->*haystack_load_1chr)(ch2, Address(ch2), noreg);
1016 bne(ch1, ch2, STR2_NEXT);
1017 addi(nlen_tmp, nlen_tmp, needle_chr_size);
1018 addi(hlen_tmp, hlen_tmp, haystack_chr_size);
1019 bltz(nlen_tmp, STR1_NEXT);
1020 j(MATCH);
1021
1022 bind(DOSHORT);
1023 if (needle_isL == haystack_isL) {
1024 subi(t0, needle_len, 2);
1025 bltz(t0, DO1);
1026 bgtz(t0, DO3);
1027 }
1028 }
1029
1030 if (needle_con_cnt == 4) {
1031 Label CH1_LOOP;
1032 (this->*load_4chr)(ch1, Address(needle), noreg);
1033 subi(result_tmp, haystack_len, 4);
1034 slli(tmp3, result_tmp, haystack_chr_shift); // result as tmp
1035 add(haystack, haystack, tmp3);
1036 neg(hlen_neg, tmp3);
1037 if (AvoidUnalignedAccesses) {
1038 // preload first value, then we will read by 1 character per loop, instead of four
1039 // just shifting previous ch2 right by size of character in bits
1040 add(tmp3, haystack, hlen_neg);
1041 (this->*load_4chr)(ch2, Address(tmp3), noreg);
1042 if (isLL) {
1043 // need to erase 1 most significant byte in 32-bit value of ch2
1044 slli(ch2, ch2, 40);
1045 srli(ch2, ch2, 32);
1046 } else {
1047 slli(ch2, ch2, 16); // 2 most significant bytes will be erased by this operation
1048 }
1049 }
1050
1051 bind(CH1_LOOP);
1052 add(tmp3, haystack, hlen_neg);
1053 if (AvoidUnalignedAccesses) {
1054 srli(ch2, ch2, isLL ? 8 : 16);
1055 (this->*haystack_load_1chr)(tmp3, Address(tmp3, isLL ? 3 : 6), noreg);
1056 slli(tmp3, tmp3, isLL ? 24 : 48);
1057 add(ch2, ch2, tmp3);
1058 } else {
1059 (this->*load_4chr)(ch2, Address(tmp3), noreg);
1060 }
1061 beq(ch1, ch2, MATCH);
1062 addi(hlen_neg, hlen_neg, haystack_chr_size);
1063 blez(hlen_neg, CH1_LOOP);
1064 j(NOMATCH);
1065 }
1066
1067 if ((needle_con_cnt == -1 && needle_isL == haystack_isL) || needle_con_cnt == 2) {
1068 Label CH1_LOOP;
1069 BLOCK_COMMENT("string_indexof DO2 {");
1070 bind(DO2);
1071 (this->*load_2chr)(ch1, Address(needle), noreg);
1072 if (needle_con_cnt == 2) {
1073 subi(result_tmp, haystack_len, 2);
1074 }
1075 slli(tmp3, result_tmp, haystack_chr_shift);
1076 add(haystack, haystack, tmp3);
1077 neg(hlen_neg, tmp3);
1078 if (AvoidUnalignedAccesses) {
1079 // preload first value, then we will read by 1 character per loop, instead of two
1080 // just shifting previous ch2 right by size of character in bits
1081 add(tmp3, haystack, hlen_neg);
1082 (this->*haystack_load_1chr)(ch2, Address(tmp3), noreg);
1083 slli(ch2, ch2, isLL ? 8 : 16);
1084 }
1085 bind(CH1_LOOP);
1086 add(tmp3, haystack, hlen_neg);
1087 if (AvoidUnalignedAccesses) {
1088 srli(ch2, ch2, isLL ? 8 : 16);
1089 (this->*haystack_load_1chr)(tmp3, Address(tmp3, isLL ? 1 : 2), noreg);
1090 slli(tmp3, tmp3, isLL ? 8 : 16);
1091 add(ch2, ch2, tmp3);
1092 } else {
1093 (this->*load_2chr)(ch2, Address(tmp3), noreg);
1094 }
1095 beq(ch1, ch2, MATCH);
1096 addi(hlen_neg, hlen_neg, haystack_chr_size);
1097 blez(hlen_neg, CH1_LOOP);
1098 j(NOMATCH);
1099 BLOCK_COMMENT("} string_indexof DO2");
1100 }
1101
1102 if ((needle_con_cnt == -1 && needle_isL == haystack_isL) || needle_con_cnt == 3) {
1103 Label FIRST_LOOP, STR2_NEXT, STR1_LOOP;
1104 BLOCK_COMMENT("string_indexof DO3 {");
1105
1106 bind(DO3);
1107 (this->*load_2chr)(first, Address(needle), noreg);
1108 (this->*needle_load_1chr)(ch1, Address(needle, 2 * needle_chr_size), noreg);
1109 if (needle_con_cnt == 3) {
1110 subi(result_tmp, haystack_len, 3);
1111 }
1112 slli(hlen_tmp, result_tmp, haystack_chr_shift);
1113 add(haystack, haystack, hlen_tmp);
1114 neg(hlen_neg, hlen_tmp);
1115
1116 bind(FIRST_LOOP);
1117 add(ch2, haystack, hlen_neg);
1118 if (AvoidUnalignedAccesses) {
1119 (this->*haystack_load_1chr)(tmp2, Address(ch2, isLL ? 1 : 2), noreg); // we need a temp register, we can safely use hlen_tmp here, which is a synonym for tmp2
1120 (this->*haystack_load_1chr)(ch2, Address(ch2), noreg);
1121 slli(tmp2, tmp2, isLL ? 8 : 16);
1122 add(ch2, ch2, tmp2);
1123 } else {
1124 (this->*load_2chr)(ch2, Address(ch2), noreg);
1125 }
1126 beq(first, ch2, STR1_LOOP);
1127
1128 bind(STR2_NEXT);
1129 addi(hlen_neg, hlen_neg, haystack_chr_size);
1130 blez(hlen_neg, FIRST_LOOP);
1131 j(NOMATCH);
1132
1133 bind(STR1_LOOP);
1134 addi(hlen_tmp, hlen_neg, 2 * haystack_chr_size);
1135 add(ch2, haystack, hlen_tmp);
1136 (this->*haystack_load_1chr)(ch2, Address(ch2), noreg);
1137 bne(ch1, ch2, STR2_NEXT);
1138 j(MATCH);
1139 BLOCK_COMMENT("} string_indexof DO3");
1140 }
1141
1142 if (needle_con_cnt == -1 || needle_con_cnt == 1) {
1143 Label DO1_LOOP;
1144
1145 BLOCK_COMMENT("string_indexof DO1 {");
1146 bind(DO1);
1147 (this->*needle_load_1chr)(ch1, Address(needle), noreg);
1148 subi(result_tmp, haystack_len, 1);
1149 slli(tmp3, result_tmp, haystack_chr_shift);
1150 add(haystack, haystack, tmp3);
1151 neg(hlen_neg, tmp3);
1152
1153 bind(DO1_LOOP);
1154 add(tmp3, haystack, hlen_neg);
1155 (this->*haystack_load_1chr)(ch2, Address(tmp3), noreg);
1156 beq(ch1, ch2, MATCH);
1157 addi(hlen_neg, hlen_neg, haystack_chr_size);
1158 blez(hlen_neg, DO1_LOOP);
1159 BLOCK_COMMENT("} string_indexof DO1");
1160 }
1161
1162 bind(NOMATCH);
1163 mv(result, -1);
1164 j(DONE);
1165
1166 bind(MATCH);
1167 srai(t0, hlen_neg, haystack_chr_shift);
1168 add(result, result_tmp, t0);
1169
1170 bind(DONE);
1171 }
1172
1173 // Compare longwords
1174 void C2_MacroAssembler::string_compare_long_same_encoding(Register result, Register str1, Register str2,
1175 const bool isLL, Register cnt1, Register cnt2,
1176 Register tmp1, Register tmp2, Register tmp3,
1177 const int STUB_THRESHOLD, Label *STUB, Label *SHORT_STRING, Label *DONE) {
1178 Label TAIL_CHECK, TAIL, NEXT_WORD, DIFFERENCE;
1179
1180 const int base_offset = arrayOopDesc::base_offset_in_bytes(T_BYTE);
1181 assert((base_offset % (UseCompactObjectHeaders ? 4 : 8)) == 0, "Must be");
1182
1183 const int minCharsInWord = isLL ? wordSize : wordSize / 2;
1184
1185 // load first parts of strings and finish initialization while loading
1186 beq(str1, str2, *DONE);
1187 // Alignment
1188 if (AvoidUnalignedAccesses && (base_offset % 8) != 0) {
1189 lwu(tmp1, Address(str1));
1190 lwu(tmp2, Address(str2));
1191 bne(tmp1, tmp2, DIFFERENCE);
1192 addi(str1, str1, 4);
1193 addi(str2, str2, 4);
1194 subi(cnt2, cnt2, minCharsInWord / 2);
1195
1196 // A very short string
1197 mv(t0, minCharsInWord);
1198 ble(cnt2, t0, *SHORT_STRING);
1199 }
1200 #ifdef ASSERT
1201 if (AvoidUnalignedAccesses) {
1202 Label align_ok;
1203 orr(t0, str1, str2);
1204 andi(t0, t0, 0x7);
1205 beqz(t0, align_ok);
1206 stop("bad alignment");
1207 bind(align_ok);
1208 }
1209 #endif
1210 // load 8 bytes once to compare
1211 ld(tmp1, Address(str1));
1212 ld(tmp2, Address(str2));
1213 mv(t0, STUB_THRESHOLD);
1214 bge(cnt2, t0, *STUB);
1215 subi(cnt2, cnt2, minCharsInWord);
1216 beqz(cnt2, TAIL_CHECK);
1217 // convert cnt2 from characters to bytes
1218 if (!isLL) {
1219 slli(cnt2, cnt2, 1);
1220 }
1221 add(str2, str2, cnt2);
1222 add(str1, str1, cnt2);
1223 sub(cnt2, zr, cnt2);
1224 addi(cnt2, cnt2, 8);
1225 bne(tmp1, tmp2, DIFFERENCE);
1226 bgez(cnt2, TAIL);
1227
1228 // main loop
1229 bind(NEXT_WORD);
1230 // 8-byte aligned loads when AvoidUnalignedAccesses is enabled
1231 add(t0, str1, cnt2);
1232 ld(tmp1, Address(t0));
1233 add(t0, str2, cnt2);
1234 ld(tmp2, Address(t0));
1235 addi(cnt2, cnt2, 8);
1236 bne(tmp1, tmp2, DIFFERENCE);
1237 bltz(cnt2, NEXT_WORD);
1238
1239 bind(TAIL);
1240 load_long_misaligned(tmp1, Address(str1), tmp3, isLL ? 1 : 2);
1241 load_long_misaligned(tmp2, Address(str2), tmp3, isLL ? 1 : 2);
1242
1243 bind(TAIL_CHECK);
1244 beq(tmp1, tmp2, *DONE);
1245
1246 // Find the first different characters in the longwords and
1247 // compute their difference.
1248 bind(DIFFERENCE);
1249 xorr(tmp3, tmp1, tmp2);
1250 // count bits of trailing zero chars
1251 ctzc_bits(result, tmp3, isLL);
1252 srl(tmp1, tmp1, result);
1253 srl(tmp2, tmp2, result);
1254 if (isLL) {
1255 zext(tmp1, tmp1, 8);
1256 zext(tmp2, tmp2, 8);
1257 } else {
1258 zext(tmp1, tmp1, 16);
1259 zext(tmp2, tmp2, 16);
1260 }
1261 sub(result, tmp1, tmp2);
1262
1263 j(*DONE);
1264 }
1265
1266 // Compare longwords
1267 void C2_MacroAssembler::string_compare_long_different_encoding(Register result, Register str1, Register str2,
1268 bool isLU, Register cnt1, Register cnt2,
1269 Register tmp1, Register tmp2, Register tmp3,
1270 const int STUB_THRESHOLD, Label *STUB, Label *DONE) {
1271 Label TAIL, NEXT_WORD, DIFFERENCE;
1272
1273 const int base_offset = arrayOopDesc::base_offset_in_bytes(T_BYTE);
1274 assert((base_offset % (UseCompactObjectHeaders ? 4 : 8)) == 0, "Must be");
1275
1276 Register strL = isLU ? str1 : str2;
1277 Register strU = isLU ? str2 : str1;
1278 Register tmpL = tmp1, tmpU = tmp2;
1279
1280 // load first parts of strings and finish initialization while loading
1281 mv(t0, STUB_THRESHOLD);
1282 bge(cnt2, t0, *STUB);
1283 lwu(tmpL, Address(strL));
1284 load_long_misaligned(tmpU, Address(strU), tmp3, (base_offset % 8) != 0 ? 4 : 8);
1285 subi(cnt2, cnt2, 4);
1286 add(strL, strL, cnt2);
1287 sub(cnt1, zr, cnt2);
1288 slli(cnt2, cnt2, 1);
1289 add(strU, strU, cnt2);
1290 inflate_lo32(tmp3, tmpL);
1291 mv(tmpL, tmp3);
1292 sub(cnt2, zr, cnt2);
1293 addi(cnt1, cnt1, 4);
1294 addi(cnt2, cnt2, 8);
1295 bne(tmpL, tmpU, DIFFERENCE);
1296 bgez(cnt2, TAIL);
1297
1298 // main loop
1299 bind(NEXT_WORD);
1300 add(t0, strL, cnt1);
1301 lwu(tmpL, Address(t0));
1302 add(t0, strU, cnt2);
1303 load_long_misaligned(tmpU, Address(t0), tmp3, (base_offset % 8) != 0 ? 4 : 8);
1304 addi(cnt1, cnt1, 4);
1305 inflate_lo32(tmp3, tmpL);
1306 mv(tmpL, tmp3);
1307 addi(cnt2, cnt2, 8);
1308 bne(tmpL, tmpU, DIFFERENCE);
1309 bltz(cnt2, NEXT_WORD);
1310
1311 bind(TAIL);
1312 load_int_misaligned(tmpL, Address(strL), tmp3, false);
1313 load_long_misaligned(tmpU, Address(strU), tmp3, 2);
1314 inflate_lo32(tmp3, tmpL);
1315 mv(tmpL, tmp3);
1316
1317 beq(tmpL, tmpU, *DONE);
1318
1319 // Find the first different characters in the longwords and
1320 // compute their difference.
1321 bind(DIFFERENCE);
1322 xorr(tmp3, tmpL, tmpU);
1323 // count bits of trailing zero chars
1324 ctzc_bits(result, tmp3);
1325 srl(tmpL, tmpL, result);
1326 srl(tmpU, tmpU, result);
1327 zext(tmpL, tmpL, 16);
1328 zext(tmpU, tmpU, 16);
1329 if (isLU) {
1330 sub(result, tmpL, tmpU);
1331 } else {
1332 sub(result, tmpU, tmpL);
1333 }
1334
1335 j(*DONE);
1336 }
1337
1338 // Compare strings.
1339 void C2_MacroAssembler::string_compare(Register str1, Register str2,
1340 Register cnt1, Register cnt2, Register result,
1341 Register tmp1, Register tmp2, Register tmp3,
1342 int ae)
1343 {
1344 Label DONE, SHORT_LOOP, SHORT_STRING, SHORT_LAST, STUB,
1345 SHORT_LOOP_TAIL, SHORT_LAST2, SHORT_LAST_INIT,
1346 SHORT_LOOP_START, L;
1347
1348 const int STUB_THRESHOLD = 64 + 8;
1349 bool isLL = ae == StrIntrinsicNode::LL;
1350 bool isLU = ae == StrIntrinsicNode::LU;
1351 bool isUL = ae == StrIntrinsicNode::UL;
1352
1353 bool str1_isL = isLL || isLU;
1354 bool str2_isL = isLL || isUL;
1355
1356 // for L strings, 1 byte for 1 character
1357 // for U strings, 2 bytes for 1 character
1358 int str1_chr_size = str1_isL ? 1 : 2;
1359 int str2_chr_size = str2_isL ? 1 : 2;
1360 int minCharsInWord = isLL ? wordSize : wordSize / 2;
1361
1362 load_chr_insn str1_load_chr = str1_isL ? (load_chr_insn)&MacroAssembler::lbu : (load_chr_insn)&MacroAssembler::lhu;
1363 load_chr_insn str2_load_chr = str2_isL ? (load_chr_insn)&MacroAssembler::lbu : (load_chr_insn)&MacroAssembler::lhu;
1364
1365 BLOCK_COMMENT("string_compare {");
1366
1367 // Bizarrely, the counts are passed in bytes, regardless of whether they
1368 // are L or U strings, however the result is always in characters.
1369 if (!str1_isL) {
1370 sraiw(cnt1, cnt1, 1);
1371 }
1372 if (!str2_isL) {
1373 sraiw(cnt2, cnt2, 1);
1374 }
1375
1376 // Compute the minimum of the string lengths and save the difference in result.
1377 sub(result, cnt1, cnt2);
1378 bgt(cnt1, cnt2, L);
1379 mv(cnt2, cnt1);
1380 bind(L);
1381
1382 // A very short string
1383 mv(t0, minCharsInWord);
1384 ble(cnt2, t0, SHORT_STRING);
1385
1386 // Compare longwords
1387 {
1388 if (str1_isL == str2_isL) { // LL or UU
1389 string_compare_long_same_encoding(result,
1390 str1, str2, isLL,
1391 cnt1, cnt2, tmp1, tmp2, tmp3,
1392 STUB_THRESHOLD, &STUB, &SHORT_STRING, &DONE);
1393 } else { // LU or UL
1394 string_compare_long_different_encoding(result,
1395 str1, str2, isLU,
1396 cnt1, cnt2, tmp1, tmp2, tmp3,
1397 STUB_THRESHOLD, &STUB, &DONE);
1398 }
1399 }
1400
1401 bind(STUB);
1402 RuntimeAddress stub = nullptr;
1403 switch (ae) {
1404 case StrIntrinsicNode::LL:
1405 stub = RuntimeAddress(StubRoutines::riscv::compare_long_string_LL());
1406 break;
1407 case StrIntrinsicNode::UU:
1408 stub = RuntimeAddress(StubRoutines::riscv::compare_long_string_UU());
1409 break;
1410 case StrIntrinsicNode::LU:
1411 stub = RuntimeAddress(StubRoutines::riscv::compare_long_string_LU());
1412 break;
1413 case StrIntrinsicNode::UL:
1414 stub = RuntimeAddress(StubRoutines::riscv::compare_long_string_UL());
1415 break;
1416 default:
1417 ShouldNotReachHere();
1418 }
1419 assert(stub.target() != nullptr, "compare_long_string stub has not been generated");
1420 address call = reloc_call(stub);
1421 if (call == nullptr) {
1422 DEBUG_ONLY(reset_labels(DONE, SHORT_LOOP, SHORT_STRING, SHORT_LAST, SHORT_LOOP_TAIL, SHORT_LAST2, SHORT_LAST_INIT, SHORT_LOOP_START));
1423 ciEnv::current()->record_failure("CodeCache is full");
1424 return;
1425 }
1426 j(DONE);
1427
1428 bind(SHORT_STRING);
1429 // Is the minimum length zero?
1430 beqz(cnt2, DONE);
1431 // arrange code to do most branches while loading and loading next characters
1432 // while comparing previous
1433 (this->*str1_load_chr)(tmp1, Address(str1), t0);
1434 addi(str1, str1, str1_chr_size);
1435 subi(cnt2, cnt2, 1);
1436 beqz(cnt2, SHORT_LAST_INIT);
1437 (this->*str2_load_chr)(cnt1, Address(str2), t0);
1438 addi(str2, str2, str2_chr_size);
1439 j(SHORT_LOOP_START);
1440 bind(SHORT_LOOP);
1441 subi(cnt2, cnt2, 1);
1442 beqz(cnt2, SHORT_LAST);
1443 bind(SHORT_LOOP_START);
1444 (this->*str1_load_chr)(tmp2, Address(str1), t0);
1445 addi(str1, str1, str1_chr_size);
1446 (this->*str2_load_chr)(t0, Address(str2), t0);
1447 addi(str2, str2, str2_chr_size);
1448 bne(tmp1, cnt1, SHORT_LOOP_TAIL);
1449 subi(cnt2, cnt2, 1);
1450 beqz(cnt2, SHORT_LAST2);
1451 (this->*str1_load_chr)(tmp1, Address(str1), t0);
1452 addi(str1, str1, str1_chr_size);
1453 (this->*str2_load_chr)(cnt1, Address(str2), t0);
1454 addi(str2, str2, str2_chr_size);
1455 beq(tmp2, t0, SHORT_LOOP);
1456 sub(result, tmp2, t0);
1457 j(DONE);
1458 bind(SHORT_LOOP_TAIL);
1459 sub(result, tmp1, cnt1);
1460 j(DONE);
1461 bind(SHORT_LAST2);
1462 beq(tmp2, t0, DONE);
1463 sub(result, tmp2, t0);
1464
1465 j(DONE);
1466 bind(SHORT_LAST_INIT);
1467 (this->*str2_load_chr)(cnt1, Address(str2), t0);
1468 addi(str2, str2, str2_chr_size);
1469 bind(SHORT_LAST);
1470 beq(tmp1, cnt1, DONE);
1471 sub(result, tmp1, cnt1);
1472
1473 bind(DONE);
1474
1475 BLOCK_COMMENT("} string_compare");
1476 }
1477
1478 void C2_MacroAssembler::arrays_equals(Register a1, Register a2,
1479 Register tmp1, Register tmp2, Register tmp3,
1480 Register result, int elem_size) {
1481 assert(elem_size == 1 || elem_size == 2, "must be char or byte");
1482 assert_different_registers(a1, a2, result, tmp1, tmp2, tmp3, t0);
1483
1484 int elem_per_word = wordSize / elem_size;
1485 int log_elem_size = exact_log2(elem_size);
1486 int length_offset = arrayOopDesc::length_offset_in_bytes();
1487 int base_offset = arrayOopDesc::base_offset_in_bytes(elem_size == 2 ? T_CHAR : T_BYTE);
1488
1489 assert((base_offset % (UseCompactObjectHeaders ? 4 : 8)) == 0, "Must be");
1490
1491 Register cnt1 = tmp3;
1492 Register cnt2 = tmp1; // cnt2 only used in array length compare
1493 Label DONE, SAME, NEXT_WORD, SHORT, TAIL03, TAIL01;
1494
1495 BLOCK_COMMENT("arrays_equals {");
1496
1497 // if (a1 == a2), return true
1498 beq(a1, a2, SAME);
1499
1500 mv(result, false);
1501 // if (a1 == nullptr || a2 == nullptr)
1502 // return false;
1503 beqz(a1, DONE);
1504 beqz(a2, DONE);
1505
1506 // if (a1.length != a2.length)
1507 // return false;
1508 lwu(cnt1, Address(a1, length_offset));
1509 lwu(cnt2, Address(a2, length_offset));
1510 bne(cnt1, cnt2, DONE);
1511
1512 la(a1, Address(a1, base_offset));
1513 la(a2, Address(a2, base_offset));
1514
1515 // Load 4 bytes once to compare for alignment before main loop.
1516 if (AvoidUnalignedAccesses && (base_offset % 8) != 0) {
1517 subi(cnt1, cnt1, elem_per_word / 2);
1518 bltz(cnt1, TAIL03);
1519 lwu(tmp1, Address(a1));
1520 lwu(tmp2, Address(a2));
1521 addi(a1, a1, 4);
1522 addi(a2, a2, 4);
1523 bne(tmp1, tmp2, DONE);
1524 }
1525
1526 // Check for short strings, i.e. smaller than wordSize.
1527 subi(cnt1, cnt1, elem_per_word);
1528 bltz(cnt1, SHORT);
1529
1530 #ifdef ASSERT
1531 if (AvoidUnalignedAccesses) {
1532 Label align_ok;
1533 orr(t0, a1, a2);
1534 andi(t0, t0, 0x7);
1535 beqz(t0, align_ok);
1536 stop("bad alignment");
1537 bind(align_ok);
1538 }
1539 #endif
1540
1541 // Main 8 byte comparison loop.
1542 bind(NEXT_WORD); {
1543 ld(tmp1, Address(a1));
1544 ld(tmp2, Address(a2));
1545 subi(cnt1, cnt1, elem_per_word);
1546 addi(a1, a1, wordSize);
1547 addi(a2, a2, wordSize);
1548 bne(tmp1, tmp2, DONE);
1549 } bgez(cnt1, NEXT_WORD);
1550
1551 addi(tmp1, cnt1, elem_per_word);
1552 beqz(tmp1, SAME);
1553
1554 bind(SHORT);
1555 test_bit(tmp1, cnt1, 2 - log_elem_size);
1556 beqz(tmp1, TAIL03); // 0-7 bytes left.
1557 {
1558 lwu(tmp1, Address(a1));
1559 lwu(tmp2, Address(a2));
1560 addi(a1, a1, 4);
1561 addi(a2, a2, 4);
1562 bne(tmp1, tmp2, DONE);
1563 }
1564
1565 bind(TAIL03);
1566 test_bit(tmp1, cnt1, 1 - log_elem_size);
1567 beqz(tmp1, TAIL01); // 0-3 bytes left.
1568 {
1569 lhu(tmp1, Address(a1));
1570 lhu(tmp2, Address(a2));
1571 addi(a1, a1, 2);
1572 addi(a2, a2, 2);
1573 bne(tmp1, tmp2, DONE);
1574 }
1575
1576 bind(TAIL01);
1577 if (elem_size == 1) { // Only needed when comparing byte arrays.
1578 test_bit(tmp1, cnt1, 0);
1579 beqz(tmp1, SAME); // 0-1 bytes left.
1580 {
1581 lbu(tmp1, Address(a1));
1582 lbu(tmp2, Address(a2));
1583 bne(tmp1, tmp2, DONE);
1584 }
1585 }
1586
1587 bind(SAME);
1588 mv(result, true);
1589 // That's it.
1590 bind(DONE);
1591
1592 BLOCK_COMMENT("} arrays_equals");
1593 }
1594
1595 // Compare Strings
1596
1597 // For Strings we're passed the address of the first characters in a1 and a2
1598 // and the length in cnt1. There are two implementations.
1599 // For arrays >= 8 bytes, all comparisons (except for the tail) are performed
1600 // 8 bytes at a time. For the tail, we compare a halfword, then a short, and then a byte.
1601 // For strings < 8 bytes, we compare a halfword, then a short, and then a byte.
1602
1603 void C2_MacroAssembler::string_equals(Register a1, Register a2,
1604 Register result, Register cnt1)
1605 {
1606 Label SAME, DONE, SHORT, NEXT_WORD, TAIL03, TAIL01;
1607 Register tmp1 = t0;
1608 Register tmp2 = t1;
1609
1610 assert_different_registers(a1, a2, result, cnt1, tmp1, tmp2);
1611
1612 int base_offset = arrayOopDesc::base_offset_in_bytes(T_BYTE);
1613
1614 assert((base_offset % (UseCompactObjectHeaders ? 4 : 8)) == 0, "Must be");
1615
1616 BLOCK_COMMENT("string_equals {");
1617
1618 mv(result, false);
1619
1620 // Load 4 bytes once to compare for alignment before main loop.
1621 if (AvoidUnalignedAccesses && (base_offset % 8) != 0) {
1622 subi(cnt1, cnt1, 4);
1623 bltz(cnt1, TAIL03);
1624 lwu(tmp1, Address(a1));
1625 lwu(tmp2, Address(a2));
1626 addi(a1, a1, 4);
1627 addi(a2, a2, 4);
1628 bne(tmp1, tmp2, DONE);
1629 }
1630
1631 // Check for short strings, i.e. smaller than wordSize.
1632 subi(cnt1, cnt1, wordSize);
1633 bltz(cnt1, SHORT);
1634
1635 #ifdef ASSERT
1636 if (AvoidUnalignedAccesses) {
1637 Label align_ok;
1638 orr(t0, a1, a2);
1639 andi(t0, t0, 0x7);
1640 beqz(t0, align_ok);
1641 stop("bad alignment");
1642 bind(align_ok);
1643 }
1644 #endif
1645
1646 // Main 8 byte comparison loop.
1647 bind(NEXT_WORD); {
1648 ld(tmp1, Address(a1));
1649 ld(tmp2, Address(a2));
1650 subi(cnt1, cnt1, wordSize);
1651 addi(a1, a1, wordSize);
1652 addi(a2, a2, wordSize);
1653 bne(tmp1, tmp2, DONE);
1654 } bgez(cnt1, NEXT_WORD);
1655
1656 addi(tmp1, cnt1, wordSize);
1657 beqz(tmp1, SAME);
1658
1659 bind(SHORT);
1660 // 0-7 bytes left.
1661 test_bit(tmp1, cnt1, 2);
1662 beqz(tmp1, TAIL03);
1663 {
1664 lwu(tmp1, Address(a1));
1665 lwu(tmp2, Address(a2));
1666 addi(a1, a1, 4);
1667 addi(a2, a2, 4);
1668 bne(tmp1, tmp2, DONE);
1669 }
1670
1671 bind(TAIL03);
1672 // 0-3 bytes left.
1673 test_bit(tmp1, cnt1, 1);
1674 beqz(tmp1, TAIL01);
1675 {
1676 lhu(tmp1, Address(a1));
1677 lhu(tmp2, Address(a2));
1678 addi(a1, a1, 2);
1679 addi(a2, a2, 2);
1680 bne(tmp1, tmp2, DONE);
1681 }
1682
1683 bind(TAIL01);
1684 // 0-1 bytes left.
1685 test_bit(tmp1, cnt1, 0);
1686 beqz(tmp1, SAME);
1687 {
1688 lbu(tmp1, Address(a1));
1689 lbu(tmp2, Address(a2));
1690 bne(tmp1, tmp2, DONE);
1691 }
1692
1693 // Arrays are equal.
1694 bind(SAME);
1695 mv(result, true);
1696
1697 // That's it.
1698 bind(DONE);
1699 BLOCK_COMMENT("} string_equals");
1700 }
1701
1702 // jdk.internal.util.ArraysSupport.vectorizedHashCode
1703 void C2_MacroAssembler::arrays_hashcode(Register ary, Register cnt, Register result,
1704 Register tmp1, Register tmp2, Register tmp3,
1705 Register tmp4, Register tmp5, Register tmp6,
1706 BasicType eltype)
1707 {
1708 assert(!UseRVV, "sanity");
1709 assert_different_registers(ary, cnt, result, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, t0, t1);
1710
1711 const int elsize = arrays_hashcode_elsize(eltype);
1712 const int chunks_end_shift = exact_log2(elsize);
1713
1714 switch (eltype) {
1715 case T_BOOLEAN: BLOCK_COMMENT("arrays_hashcode(unsigned byte) {"); break;
1716 case T_CHAR: BLOCK_COMMENT("arrays_hashcode(char) {"); break;
1717 case T_BYTE: BLOCK_COMMENT("arrays_hashcode(byte) {"); break;
1718 case T_SHORT: BLOCK_COMMENT("arrays_hashcode(short) {"); break;
1719 case T_INT: BLOCK_COMMENT("arrays_hashcode(int) {"); break;
1720 default:
1721 ShouldNotReachHere();
1722 }
1723
1724 const int stride = 4;
1725 const Register pow31_4 = tmp1;
1726 const Register pow31_3 = tmp2;
1727 const Register pow31_2 = tmp3;
1728 const Register chunks = tmp4;
1729 const Register chunks_end = chunks;
1730
1731 Label DONE, TAIL, TAIL_LOOP, WIDE_LOOP;
1732
1733 // result has a value initially
1734
1735 beqz(cnt, DONE);
1736
1737 andi(chunks, cnt, ~(stride - 1));
1738 beqz(chunks, TAIL);
1739
1740 mv(pow31_4, 923521); // [31^^4]
1741 mv(pow31_3, 29791); // [31^^3]
1742 mv(pow31_2, 961); // [31^^2]
1743
1744 shadd(chunks_end, chunks, ary, t0, chunks_end_shift);
1745 andi(cnt, cnt, stride - 1); // don't forget about tail!
1746
1747 bind(WIDE_LOOP);
1748 arrays_hashcode_elload(t0, Address(ary, 0 * elsize), eltype);
1749 arrays_hashcode_elload(t1, Address(ary, 1 * elsize), eltype);
1750 arrays_hashcode_elload(tmp5, Address(ary, 2 * elsize), eltype);
1751 arrays_hashcode_elload(tmp6, Address(ary, 3 * elsize), eltype);
1752 mulw(result, result, pow31_4); // 31^^4 * h
1753 mulw(t0, t0, pow31_3); // 31^^3 * ary[i+0]
1754 addw(result, result, t0);
1755 mulw(t1, t1, pow31_2); // 31^^2 * ary[i+1]
1756 addw(result, result, t1);
1757 slli(t0, tmp5, 5); // optimize 31^^1 * ary[i+2]
1758 subw(tmp5, t0, tmp5); // with ary[i+2]<<5 - ary[i+2]
1759 addw(result, result, tmp5);
1760 addw(result, result, tmp6); // 31^^4 * h + 31^^3 * ary[i+0] + 31^^2 * ary[i+1]
1761 // + 31^^1 * ary[i+2] + 31^^0 * ary[i+3]
1762 addi(ary, ary, elsize * stride);
1763 bne(ary, chunks_end, WIDE_LOOP);
1764 beqz(cnt, DONE);
1765
1766 bind(TAIL);
1767 shadd(chunks_end, cnt, ary, t0, chunks_end_shift);
1768
1769 bind(TAIL_LOOP);
1770 arrays_hashcode_elload(t0, Address(ary), eltype);
1771 slli(t1, result, 5); // optimize 31 * result
1772 subw(result, t1, result); // with result<<5 - result
1773 addw(result, result, t0);
1774 addi(ary, ary, elsize);
1775 bne(ary, chunks_end, TAIL_LOOP);
1776
1777 bind(DONE);
1778 BLOCK_COMMENT("} // arrays_hashcode");
1779 }
1780
1781 void C2_MacroAssembler::arrays_hashcode_v(Register ary, Register cnt, Register result,
1782 Register tmp1, Register tmp2, Register tmp3,
1783 BasicType eltype)
1784 {
1785 assert(UseRVV, "sanity");
1786 assert(StubRoutines::riscv::arrays_hashcode_powers_of_31() != nullptr, "sanity");
1787 assert_different_registers(ary, cnt, result, tmp1, tmp2, tmp3, t0, t1);
1788
1789 // The MaxVectorSize should have been set by detecting RVV max vector register
1790 // size when check UseRVV (i.e. MaxVectorSize == VM_Version::_initial_vector_length).
1791 // Let's use T_INT as all hashCode calculations eventually deal with ints.
1792 const int lmul = 2;
1793 const int stride = MaxVectorSize / sizeof(jint) * lmul;
1794
1795 const int elsize_bytes = arrays_hashcode_elsize(eltype);
1796 const int elsize_shift = exact_log2(elsize_bytes);
1797
1798 switch (eltype) {
1799 case T_BOOLEAN: BLOCK_COMMENT("arrays_hashcode_v(unsigned byte) {"); break;
1800 case T_CHAR: BLOCK_COMMENT("arrays_hashcode_v(char) {"); break;
1801 case T_BYTE: BLOCK_COMMENT("arrays_hashcode_v(byte) {"); break;
1802 case T_SHORT: BLOCK_COMMENT("arrays_hashcode_v(short) {"); break;
1803 case T_INT: BLOCK_COMMENT("arrays_hashcode_v(int) {"); break;
1804 default:
1805 ShouldNotReachHere();
1806 }
1807
1808 const Register pow31_highest = tmp1;
1809 const Register ary_end = tmp2;
1810 const Register consumed = tmp3;
1811
1812 const VectorRegister v_sum = v2;
1813 const VectorRegister v_src = v4;
1814 const VectorRegister v_coeffs = v6;
1815 const VectorRegister v_tmp = v8;
1816
1817 const address adr_pows31 = StubRoutines::riscv::arrays_hashcode_powers_of_31()
1818 + sizeof(jint);
1819 Label VEC_LOOP, DONE, SCALAR_TAIL, SCALAR_TAIL_LOOP;
1820
1821 // NB: at this point (a) 'result' already has some value,
1822 // (b) 'cnt' is not 0 or 1, see java code for details.
1823
1824 andi(t0, cnt, ~(stride - 1));
1825 beqz(t0, SCALAR_TAIL);
1826
1827 la(t1, ExternalAddress(adr_pows31));
1828 lw(pow31_highest, Address(t1, -1 * sizeof(jint)));
1829
1830 vsetvli(consumed, cnt, Assembler::e32, Assembler::m2);
1831 vle32_v(v_coeffs, t1); // 31^^(stride - 1) ... 31^^0
1832 vmv_v_x(v_sum, x0);
1833
1834 bind(VEC_LOOP);
1835 arrays_hashcode_elload_v(v_src, v_tmp, ary, eltype);
1836 vmul_vv(v_src, v_src, v_coeffs);
1837 vmadd_vx(v_sum, pow31_highest, v_src);
1838 mulw(result, result, pow31_highest);
1839 shadd(ary, consumed, ary, t0, elsize_shift);
1840 subw(cnt, cnt, consumed);
1841 andi(t1, cnt, ~(stride - 1));
1842 bnez(t1, VEC_LOOP);
1843
1844 vmv_s_x(v_tmp, x0);
1845 vredsum_vs(v_sum, v_sum, v_tmp);
1846 vmv_x_s(t0, v_sum);
1847 addw(result, result, t0);
1848 beqz(cnt, DONE);
1849
1850 bind(SCALAR_TAIL);
1851 shadd(ary_end, cnt, ary, t0, elsize_shift);
1852
1853 bind(SCALAR_TAIL_LOOP);
1854 arrays_hashcode_elload(t0, Address(ary), eltype);
1855 slli(t1, result, 5); // optimize 31 * result
1856 subw(result, t1, result); // with result<<5 - result
1857 addw(result, result, t0);
1858 addi(ary, ary, elsize_bytes);
1859 bne(ary, ary_end, SCALAR_TAIL_LOOP);
1860
1861 bind(DONE);
1862 BLOCK_COMMENT("} // arrays_hashcode_v");
1863 }
1864
1865 int C2_MacroAssembler::arrays_hashcode_elsize(BasicType eltype) {
1866 switch (eltype) {
1867 case T_BOOLEAN: return sizeof(jboolean);
1868 case T_BYTE: return sizeof(jbyte);
1869 case T_SHORT: return sizeof(jshort);
1870 case T_CHAR: return sizeof(jchar);
1871 case T_INT: return sizeof(jint);
1872 default:
1873 ShouldNotReachHere();
1874 return -1;
1875 }
1876 }
1877
1878 void C2_MacroAssembler::arrays_hashcode_elload(Register dst, Address src, BasicType eltype) {
1879 switch (eltype) {
1880 // T_BOOLEAN used as surrogate for unsigned byte
1881 case T_BOOLEAN: lbu(dst, src); break;
1882 case T_BYTE: lb(dst, src); break;
1883 case T_SHORT: lh(dst, src); break;
1884 case T_CHAR: lhu(dst, src); break;
1885 case T_INT: lw(dst, src); break;
1886 default:
1887 ShouldNotReachHere();
1888 }
1889 }
1890
1891 void C2_MacroAssembler::arrays_hashcode_elload_v(VectorRegister vdst,
1892 VectorRegister vtmp,
1893 Register src,
1894 BasicType eltype) {
1895 assert_different_registers(vdst, vtmp);
1896 switch (eltype) {
1897 case T_BOOLEAN:
1898 vle8_v(vtmp, src);
1899 vzext_vf4(vdst, vtmp);
1900 break;
1901 case T_BYTE:
1902 vle8_v(vtmp, src);
1903 vsext_vf4(vdst, vtmp);
1904 break;
1905 case T_CHAR:
1906 vle16_v(vtmp, src);
1907 vzext_vf2(vdst, vtmp);
1908 break;
1909 case T_SHORT:
1910 vle16_v(vtmp, src);
1911 vsext_vf2(vdst, vtmp);
1912 break;
1913 case T_INT:
1914 vle32_v(vdst, src);
1915 break;
1916 default:
1917 ShouldNotReachHere();
1918 }
1919 }
1920
1921 typedef void (Assembler::*conditional_branch_insn)(Register op1, Register op2, Label& label, bool is_far);
1922 typedef void (MacroAssembler::*float_conditional_branch_insn)(FloatRegister op1, FloatRegister op2, Label& label,
1923 bool is_far, bool is_unordered);
1924
1925 static conditional_branch_insn conditional_branches[] =
1926 {
1927 /* SHORT branches */
1928 (conditional_branch_insn)&MacroAssembler::beq,
1929 (conditional_branch_insn)&MacroAssembler::bgt,
1930 nullptr, // BoolTest::overflow
1931 (conditional_branch_insn)&MacroAssembler::blt,
1932 (conditional_branch_insn)&MacroAssembler::bne,
1933 (conditional_branch_insn)&MacroAssembler::ble,
1934 nullptr, // BoolTest::no_overflow
1935 (conditional_branch_insn)&MacroAssembler::bge,
1936
1937 /* UNSIGNED branches */
1938 (conditional_branch_insn)&MacroAssembler::beq,
1939 (conditional_branch_insn)&MacroAssembler::bgtu,
1940 nullptr,
1941 (conditional_branch_insn)&MacroAssembler::bltu,
1942 (conditional_branch_insn)&MacroAssembler::bne,
1943 (conditional_branch_insn)&MacroAssembler::bleu,
1944 nullptr,
1945 (conditional_branch_insn)&MacroAssembler::bgeu
1946 };
1947
1948 static float_conditional_branch_insn float_conditional_branches[] =
1949 {
1950 /* FLOAT SHORT branches */
1951 (float_conditional_branch_insn)&MacroAssembler::float_beq,
1952 (float_conditional_branch_insn)&MacroAssembler::float_bgt,
1953 nullptr, // BoolTest::overflow
1954 (float_conditional_branch_insn)&MacroAssembler::float_blt,
1955 (float_conditional_branch_insn)&MacroAssembler::float_bne,
1956 (float_conditional_branch_insn)&MacroAssembler::float_ble,
1957 nullptr, // BoolTest::no_overflow
1958 (float_conditional_branch_insn)&MacroAssembler::float_bge,
1959
1960 /* DOUBLE SHORT branches */
1961 (float_conditional_branch_insn)&MacroAssembler::double_beq,
1962 (float_conditional_branch_insn)&MacroAssembler::double_bgt,
1963 nullptr,
1964 (float_conditional_branch_insn)&MacroAssembler::double_blt,
1965 (float_conditional_branch_insn)&MacroAssembler::double_bne,
1966 (float_conditional_branch_insn)&MacroAssembler::double_ble,
1967 nullptr,
1968 (float_conditional_branch_insn)&MacroAssembler::double_bge
1969 };
1970
1971 void C2_MacroAssembler::cmp_branch(int cmpFlag, Register op1, Register op2, Label& label, bool is_far) {
1972 assert(cmpFlag >= 0 && cmpFlag < (int)(sizeof(conditional_branches) / sizeof(conditional_branches[0])),
1973 "invalid conditional branch index");
1974 (this->*conditional_branches[cmpFlag])(op1, op2, label, is_far);
1975 }
1976
1977 // This is a function should only be used by C2. Flip the unordered when unordered-greater, C2 would use
1978 // unordered-lesser instead of unordered-greater. Finally, commute the result bits at function do_one_bytecode().
1979 void C2_MacroAssembler::float_cmp_branch(int cmpFlag, FloatRegister op1, FloatRegister op2, Label& label, bool is_far) {
1980 assert(cmpFlag >= 0 && cmpFlag < (int)(sizeof(float_conditional_branches) / sizeof(float_conditional_branches[0])),
1981 "invalid float conditional branch index");
1982 int booltest_flag = cmpFlag & ~(C2_MacroAssembler::double_branch_mask);
1983 (this->*float_conditional_branches[cmpFlag])(op1, op2, label, is_far,
1984 (booltest_flag == (BoolTest::ge) || booltest_flag == (BoolTest::gt)) ? false : true);
1985 }
1986
1987 void C2_MacroAssembler::enc_cmpUEqNeLeGt_imm0_branch(int cmpFlag, Register op1, Label& L, bool is_far) {
1988 switch (cmpFlag) {
1989 case BoolTest::eq:
1990 case BoolTest::le:
1991 beqz(op1, L, is_far);
1992 break;
1993 case BoolTest::ne:
1994 case BoolTest::gt:
1995 bnez(op1, L, is_far);
1996 break;
1997 default:
1998 ShouldNotReachHere();
1999 }
2000 }
2001
2002 void C2_MacroAssembler::enc_cmpEqNe_imm0_branch(int cmpFlag, Register op1, Label& L, bool is_far) {
2003 switch (cmpFlag) {
2004 case BoolTest::eq:
2005 beqz(op1, L, is_far);
2006 break;
2007 case BoolTest::ne:
2008 bnez(op1, L, is_far);
2009 break;
2010 default:
2011 ShouldNotReachHere();
2012 }
2013 }
2014
2015 void C2_MacroAssembler::enc_cmove(int cmpFlag, Register op1, Register op2, Register dst, Register src) {
2016 bool is_unsigned = (cmpFlag & unsigned_branch_mask) == unsigned_branch_mask;
2017 int op_select = cmpFlag & (~unsigned_branch_mask);
2018
2019 switch (op_select) {
2020 case BoolTest::eq:
2021 cmov_eq(op1, op2, dst, src);
2022 break;
2023 case BoolTest::ne:
2024 cmov_ne(op1, op2, dst, src);
2025 break;
2026 case BoolTest::le:
2027 if (is_unsigned) {
2028 cmov_leu(op1, op2, dst, src);
2029 } else {
2030 cmov_le(op1, op2, dst, src);
2031 }
2032 break;
2033 case BoolTest::ge:
2034 if (is_unsigned) {
2035 cmov_geu(op1, op2, dst, src);
2036 } else {
2037 cmov_ge(op1, op2, dst, src);
2038 }
2039 break;
2040 case BoolTest::lt:
2041 if (is_unsigned) {
2042 cmov_ltu(op1, op2, dst, src);
2043 } else {
2044 cmov_lt(op1, op2, dst, src);
2045 }
2046 break;
2047 case BoolTest::gt:
2048 if (is_unsigned) {
2049 cmov_gtu(op1, op2, dst, src);
2050 } else {
2051 cmov_gt(op1, op2, dst, src);
2052 }
2053 break;
2054 default:
2055 assert(false, "unsupported compare condition");
2056 ShouldNotReachHere();
2057 }
2058 }
2059
2060 void C2_MacroAssembler::enc_cmove_cmp_fp(int cmpFlag, FloatRegister op1, FloatRegister op2, Register dst, Register src, bool is_single) {
2061 int op_select = cmpFlag & (~unsigned_branch_mask);
2062
2063 switch (op_select) {
2064 case BoolTest::eq:
2065 cmov_cmp_fp_eq(op1, op2, dst, src, is_single);
2066 break;
2067 case BoolTest::ne:
2068 cmov_cmp_fp_ne(op1, op2, dst, src, is_single);
2069 break;
2070 case BoolTest::le:
2071 cmov_cmp_fp_le(op1, op2, dst, src, is_single);
2072 break;
2073 case BoolTest::ge:
2074 cmov_cmp_fp_ge(op1, op2, dst, src, is_single);
2075 break;
2076 case BoolTest::lt:
2077 cmov_cmp_fp_lt(op1, op2, dst, src, is_single);
2078 break;
2079 case BoolTest::gt:
2080 cmov_cmp_fp_gt(op1, op2, dst, src, is_single);
2081 break;
2082 default:
2083 assert(false, "unsupported compare condition");
2084 ShouldNotReachHere();
2085 }
2086 }
2087
2088 void C2_MacroAssembler::enc_cmove_fp_cmp(int cmpFlag, Register op1, Register op2,
2089 FloatRegister dst, FloatRegister src, bool is_single) {
2090 bool is_unsigned = (cmpFlag & unsigned_branch_mask) == unsigned_branch_mask;
2091 int op_select = cmpFlag & (~unsigned_branch_mask);
2092
2093 switch (op_select) {
2094 case BoolTest::eq:
2095 cmov_fp_eq(op1, op2, dst, src, is_single);
2096 break;
2097 case BoolTest::ne:
2098 cmov_fp_ne(op1, op2, dst, src, is_single);
2099 break;
2100 case BoolTest::le:
2101 if (is_unsigned) {
2102 cmov_fp_leu(op1, op2, dst, src, is_single);
2103 } else {
2104 cmov_fp_le(op1, op2, dst, src, is_single);
2105 }
2106 break;
2107 case BoolTest::ge:
2108 if (is_unsigned) {
2109 cmov_fp_geu(op1, op2, dst, src, is_single);
2110 } else {
2111 cmov_fp_ge(op1, op2, dst, src, is_single);
2112 }
2113 break;
2114 case BoolTest::lt:
2115 if (is_unsigned) {
2116 cmov_fp_ltu(op1, op2, dst, src, is_single);
2117 } else {
2118 cmov_fp_lt(op1, op2, dst, src, is_single);
2119 }
2120 break;
2121 case BoolTest::gt:
2122 if (is_unsigned) {
2123 cmov_fp_gtu(op1, op2, dst, src, is_single);
2124 } else {
2125 cmov_fp_gt(op1, op2, dst, src, is_single);
2126 }
2127 break;
2128 default:
2129 assert(false, "unsupported compare condition");
2130 ShouldNotReachHere();
2131 }
2132 }
2133
2134 void C2_MacroAssembler::enc_cmove_fp_cmp_fp(int cmpFlag,
2135 FloatRegister op1, FloatRegister op2,
2136 FloatRegister dst, FloatRegister src,
2137 bool cmp_single, bool cmov_single) {
2138 int op_select = cmpFlag & (~unsigned_branch_mask);
2139
2140 switch (op_select) {
2141 case BoolTest::eq:
2142 cmov_fp_cmp_fp_eq(op1, op2, dst, src, cmp_single, cmov_single);
2143 break;
2144 case BoolTest::ne:
2145 cmov_fp_cmp_fp_ne(op1, op2, dst, src, cmp_single, cmov_single);
2146 break;
2147 case BoolTest::le:
2148 cmov_fp_cmp_fp_le(op1, op2, dst, src, cmp_single, cmov_single);
2149 break;
2150 case BoolTest::ge:
2151 cmov_fp_cmp_fp_ge(op1, op2, dst, src, cmp_single, cmov_single);
2152 break;
2153 case BoolTest::lt:
2154 cmov_fp_cmp_fp_lt(op1, op2, dst, src, cmp_single, cmov_single);
2155 break;
2156 case BoolTest::gt:
2157 cmov_fp_cmp_fp_gt(op1, op2, dst, src, cmp_single, cmov_single);
2158 break;
2159 default:
2160 assert(false, "unsupported compare condition");
2161 ShouldNotReachHere();
2162 }
2163 }
2164
2165 // Set dst to NaN if any NaN input.
2166 void C2_MacroAssembler::minmax_fp(FloatRegister dst, FloatRegister src1, FloatRegister src2,
2167 FLOAT_TYPE ft, bool is_min) {
2168 assert_cond((ft != FLOAT_TYPE::half_precision) || UseZfh);
2169
2170 Label Done, Compare;
2171
2172 switch (ft) {
2173 case FLOAT_TYPE::half_precision:
2174 fclass_h(t0, src1);
2175 fclass_h(t1, src2);
2176
2177 orr(t0, t0, t1);
2178 andi(t0, t0, FClassBits::nan); // if src1 or src2 is quiet or signaling NaN then return NaN
2179 beqz(t0, Compare);
2180
2181 fadd_h(dst, src1, src2);
2182 j(Done);
2183
2184 bind(Compare);
2185 if (is_min) {
2186 fmin_h(dst, src1, src2);
2187 } else {
2188 fmax_h(dst, src1, src2);
2189 }
2190 break;
2191 case FLOAT_TYPE::single_precision:
2192 fclass_s(t0, src1);
2193 fclass_s(t1, src2);
2194
2195 orr(t0, t0, t1);
2196 andi(t0, t0, FClassBits::nan); // if src1 or src2 is quiet or signaling NaN then return NaN
2197 beqz(t0, Compare);
2198
2199 fadd_s(dst, src1, src2);
2200 j(Done);
2201
2202 bind(Compare);
2203 if (is_min) {
2204 fmin_s(dst, src1, src2);
2205 } else {
2206 fmax_s(dst, src1, src2);
2207 }
2208 break;
2209 case FLOAT_TYPE::double_precision:
2210 fclass_d(t0, src1);
2211 fclass_d(t1, src2);
2212
2213 orr(t0, t0, t1);
2214 andi(t0, t0, FClassBits::nan); // if src1 or src2 is quiet or signaling NaN then return NaN
2215 beqz(t0, Compare);
2216
2217 fadd_d(dst, src1, src2);
2218 j(Done);
2219
2220 bind(Compare);
2221 if (is_min) {
2222 fmin_d(dst, src1, src2);
2223 } else {
2224 fmax_d(dst, src1, src2);
2225 }
2226 break;
2227 default:
2228 ShouldNotReachHere();
2229 }
2230
2231 bind(Done);
2232 }
2233
2234 // According to Java SE specification, for floating-point round operations, if
2235 // the input is NaN, +/-infinity, or +/-0, the same input is returned as the
2236 // rounded result; this differs from behavior of RISC-V fcvt instructions (which
2237 // round out-of-range values to the nearest max or min value), therefore special
2238 // handling is needed by NaN, +/-Infinity, +/-0.
2239 void C2_MacroAssembler::round_double_mode(FloatRegister dst, FloatRegister src, int round_mode,
2240 Register tmp1, Register tmp2, Register tmp3) {
2241
2242 assert_different_registers(dst, src);
2243 assert_different_registers(tmp1, tmp2, tmp3);
2244
2245 // Set rounding mode for conversions
2246 // Here we use similar modes to double->long and long->double conversions
2247 // Different mode for long->double conversion matter only if long value was not representable as double,
2248 // we got long value as a result of double->long conversion so, it is definitely representable
2249 RoundingMode rm;
2250 switch (round_mode) {
2251 case RoundDoubleModeNode::rmode_ceil:
2252 rm = RoundingMode::rup;
2253 break;
2254 case RoundDoubleModeNode::rmode_floor:
2255 rm = RoundingMode::rdn;
2256 break;
2257 case RoundDoubleModeNode::rmode_rint:
2258 rm = RoundingMode::rne;
2259 break;
2260 default:
2261 ShouldNotReachHere();
2262 }
2263
2264 // tmp1 - is a register to store double converted to long int
2265 // tmp2 - is a register to create constant for comparison
2266 // tmp3 - is a register where we store modified result of double->long conversion
2267 Label done, bad_val;
2268
2269 // Conversion from double to long
2270 fcvt_l_d(tmp1, src, rm);
2271
2272 // Generate constant (tmp2)
2273 // tmp2 = 100...0000
2274 addi(tmp2, zr, 1);
2275 slli(tmp2, tmp2, 63);
2276
2277 // Prepare converted long (tmp1)
2278 // as a result when conversion overflow we got:
2279 // tmp1 = 011...1111 or 100...0000
2280 // Convert it to: tmp3 = 100...0000
2281 addi(tmp3, tmp1, 1);
2282 andi(tmp3, tmp3, -2);
2283 beq(tmp3, tmp2, bad_val);
2284
2285 // Conversion from long to double
2286 fcvt_d_l(dst, tmp1, rm);
2287 // Add sign of input value to result for +/- 0 cases
2288 fsgnj_d(dst, dst, src);
2289 j(done);
2290
2291 // If got conversion overflow return src
2292 bind(bad_val);
2293 fmv_d(dst, src);
2294
2295 bind(done);
2296 }
2297
2298 // According to Java SE specification, for floating-point signum operations, if
2299 // on input we have NaN or +/-0.0 value we should return it,
2300 // otherwise return +/- 1.0 using sign of input.
2301 // one - gives us a floating-point 1.0 (got from matching rule)
2302 // bool is_double - specifies single or double precision operations will be used.
2303 void C2_MacroAssembler::signum_fp(FloatRegister dst, FloatRegister one, bool is_double) {
2304 Label done;
2305
2306 is_double ? fclass_d(t0, dst)
2307 : fclass_s(t0, dst);
2308
2309 // check if input is -0, +0, signaling NaN or quiet NaN
2310 andi(t0, t0, FClassBits::zero | FClassBits::nan);
2311
2312 bnez(t0, done);
2313
2314 // use floating-point 1.0 with a sign of input
2315 is_double ? fsgnj_d(dst, one, dst)
2316 : fsgnj_s(dst, one, dst);
2317
2318 bind(done);
2319 }
2320
2321 static void float16_to_float_slow_path(C2_MacroAssembler& masm, C2GeneralStub<FloatRegister, Register, Register>& stub) {
2322 #define __ masm.
2323 FloatRegister dst = stub.data<0>();
2324 Register src = stub.data<1>();
2325 Register tmp = stub.data<2>();
2326 __ bind(stub.entry());
2327
2328 // following instructions mainly focus on NaN, as riscv does not handle
2329 // NaN well with fcvt, but the code also works for Inf at the same time.
2330
2331 // construct a NaN in 32 bits from the NaN in 16 bits,
2332 // we need the payloads of non-canonical NaNs to be preserved.
2333 __ mv(tmp, 0x7f800000);
2334 // sign-bit was already set via sign-extension if necessary.
2335 __ slli(t0, src, 13);
2336 __ orr(tmp, t0, tmp);
2337 __ fmv_w_x(dst, tmp);
2338
2339 __ j(stub.continuation());
2340 #undef __
2341 }
2342
2343 // j.l.Float.float16ToFloat
2344 void C2_MacroAssembler::float16_to_float(FloatRegister dst, Register src, Register tmp) {
2345 auto stub = C2CodeStub::make<FloatRegister, Register, Register>(dst, src, tmp, 20, float16_to_float_slow_path);
2346
2347 // On riscv, NaN needs a special process as fcvt does not work in that case.
2348 // On riscv, Inf does not need a special process as fcvt can handle it correctly.
2349 // but we consider to get the slow path to process NaN and Inf at the same time,
2350 // as both of them are rare cases, and if we try to get the slow path to handle
2351 // only NaN case it would sacrifise the performance for normal cases,
2352 // i.e. non-NaN and non-Inf cases.
2353
2354 // check whether it's a NaN or +/- Inf.
2355 mv(t0, 0x7c00);
2356 andr(tmp, src, t0);
2357 // jump to stub processing NaN and Inf cases.
2358 beq(t0, tmp, stub->entry(), true);
2359
2360 // non-NaN or non-Inf cases, just use built-in instructions.
2361 fmv_h_x(dst, src);
2362 fcvt_s_h(dst, dst);
2363
2364 bind(stub->continuation());
2365 }
2366
2367 static void float_to_float16_slow_path(C2_MacroAssembler& masm, C2GeneralStub<Register, FloatRegister, Register>& stub) {
2368 #define __ masm.
2369 Register dst = stub.data<0>();
2370 FloatRegister src = stub.data<1>();
2371 Register tmp = stub.data<2>();
2372 __ bind(stub.entry());
2373
2374 __ float_to_float16_NaN(dst, src, t0, tmp);
2375
2376 __ j(stub.continuation());
2377 #undef __
2378 }
2379
2380 // j.l.Float.floatToFloat16
2381 void C2_MacroAssembler::float_to_float16(Register dst, FloatRegister src, FloatRegister ftmp, Register xtmp) {
2382 auto stub = C2CodeStub::make<Register, FloatRegister, Register>(dst, src, xtmp, 64, float_to_float16_slow_path);
2383
2384 // On riscv, NaN needs a special process as fcvt does not work in that case.
2385
2386 // check whether it's a NaN.
2387 // replace fclass with feq as performance optimization.
2388 feq_s(t0, src, src);
2389 // jump to stub processing NaN cases.
2390 beqz(t0, stub->entry(), true);
2391
2392 // non-NaN cases, just use built-in instructions.
2393 fcvt_h_s(ftmp, src);
2394 fmv_x_h(dst, ftmp);
2395
2396 bind(stub->continuation());
2397 }
2398
2399 static void float16_to_float_v_slow_path(C2_MacroAssembler& masm, C2GeneralStub<VectorRegister, VectorRegister, uint>& stub) {
2400 #define __ masm.
2401 VectorRegister dst = stub.data<0>();
2402 VectorRegister src = stub.data<1>();
2403 uint vector_length = stub.data<2>();
2404 __ bind(stub.entry());
2405
2406 // following instructions mainly focus on NaN, as riscv does not handle
2407 // NaN well with vfwcvt_f_f_v, but the code also works for Inf at the same time.
2408 //
2409 // construct NaN's in 32 bits from the NaN's in 16 bits,
2410 // we need the payloads of non-canonical NaNs to be preserved.
2411
2412 // adjust vector type to 2 * SEW.
2413 __ vsetvli_helper(T_FLOAT, vector_length, Assembler::m1);
2414 // widen and sign-extend src data.
2415 __ vsext_vf2(dst, src, Assembler::v0_t);
2416 __ mv(t0, 0x7f800000);
2417 // sign-bit was already set via sign-extension if necessary.
2418 __ vsll_vi(dst, dst, 13, Assembler::v0_t);
2419 __ vor_vx(dst, dst, t0, Assembler::v0_t);
2420
2421 __ j(stub.continuation());
2422 #undef __
2423 }
2424
2425 // j.l.Float.float16ToFloat
2426 void C2_MacroAssembler::float16_to_float_v(VectorRegister dst, VectorRegister src, uint vector_length) {
2427 auto stub = C2CodeStub::make<VectorRegister, VectorRegister, uint>
2428 (dst, src, vector_length, 24, float16_to_float_v_slow_path);
2429 assert_different_registers(dst, src);
2430
2431 // On riscv, NaN needs a special process as vfwcvt_f_f_v does not work in that case.
2432 // On riscv, Inf does not need a special process as vfwcvt_f_f_v can handle it correctly.
2433 // but we consider to get the slow path to process NaN and Inf at the same time,
2434 // as both of them are rare cases, and if we try to get the slow path to handle
2435 // only NaN case it would sacrifise the performance for normal cases,
2436 // i.e. non-NaN and non-Inf cases.
2437
2438 vsetvli_helper(BasicType::T_SHORT, vector_length, Assembler::mf2);
2439
2440 // check whether there is a NaN or +/- Inf.
2441 mv(t0, 0x7c00);
2442 vand_vx(v0, src, t0);
2443 // v0 will be used as mask in slow path.
2444 vmseq_vx(v0, v0, t0);
2445 vcpop_m(t0, v0);
2446
2447 // For non-NaN or non-Inf cases, just use built-in instructions.
2448 vfwcvt_f_f_v(dst, src);
2449
2450 // jump to stub processing NaN and Inf cases if there is any of them in the vector-wide.
2451 bnez(t0, stub->entry(), true);
2452
2453 bind(stub->continuation());
2454 }
2455
2456 static void float_to_float16_v_slow_path(C2_MacroAssembler& masm,
2457 C2GeneralStub<VectorRegister, VectorRegister, VectorRegister>& stub) {
2458 #define __ masm.
2459 VectorRegister dst = stub.data<0>();
2460 VectorRegister src = stub.data<1>();
2461 VectorRegister vtmp = stub.data<2>();
2462 assert_different_registers(dst, src, vtmp);
2463
2464 __ bind(stub.entry());
2465
2466 // Active elements (NaNs) are marked in v0 mask register.
2467 // mul is already set to mf2 in float_to_float16_v.
2468
2469 // Float (32 bits)
2470 // Bit: 31 30 to 23 22 to 0
2471 // +---+------------------+-----------------------------+
2472 // | S | Exponent | Mantissa (Fraction) |
2473 // +---+------------------+-----------------------------+
2474 // 1 bit 8 bits 23 bits
2475 //
2476 // Float (16 bits)
2477 // Bit: 15 14 to 10 9 to 0
2478 // +---+----------------+------------------+
2479 // | S | Exponent | Mantissa |
2480 // +---+----------------+------------------+
2481 // 1 bit 5 bits 10 bits
2482 const int fp_sign_bits = 1;
2483 const int fp32_bits = 32;
2484 const int fp32_mantissa_2nd_part_bits = 9;
2485 const int fp32_mantissa_3rd_part_bits = 4;
2486 const int fp16_exponent_bits = 5;
2487 const int fp16_mantissa_bits = 10;
2488
2489 // preserve the sign bit and exponent, clear mantissa.
2490 __ vnsra_wi(dst, src, fp32_bits - fp_sign_bits - fp16_exponent_bits, Assembler::v0_t);
2491 __ vsll_vi(dst, dst, fp16_mantissa_bits, Assembler::v0_t);
2492
2493 // Preserve high order bit of float NaN in the
2494 // binary16 result NaN (tenth bit); OR in remaining
2495 // bits into lower 9 bits of binary 16 significand.
2496 // | (doppel & 0x007f_e000) >> 13 // 10 bits
2497 // | (doppel & 0x0000_1ff0) >> 4 // 9 bits
2498 // | (doppel & 0x0000_000f)); // 4 bits
2499 //
2500 // Check j.l.Float.floatToFloat16 for more information.
2501 // 10 bits
2502 __ vnsrl_wi(vtmp, src, fp32_mantissa_2nd_part_bits + fp32_mantissa_3rd_part_bits, Assembler::v0_t);
2503 __ mv(t0, 0x3ff); // retain first part of mantissa in a float 32
2504 __ vand_vx(vtmp, vtmp, t0, Assembler::v0_t);
2505 __ vor_vv(dst, dst, vtmp, Assembler::v0_t);
2506 // 9 bits
2507 __ vnsrl_wi(vtmp, src, fp32_mantissa_3rd_part_bits, Assembler::v0_t);
2508 __ mv(t0, 0x1ff); // retain second part of mantissa in a float 32
2509 __ vand_vx(vtmp, vtmp, t0, Assembler::v0_t);
2510 __ vor_vv(dst, dst, vtmp, Assembler::v0_t);
2511 // 4 bits
2512 // Narrow shift is necessary to move data from 32 bits element to 16 bits element in vector register.
2513 __ vnsrl_wi(vtmp, src, 0, Assembler::v0_t);
2514 __ vand_vi(vtmp, vtmp, 0xf, Assembler::v0_t);
2515 __ vor_vv(dst, dst, vtmp, Assembler::v0_t);
2516
2517 __ j(stub.continuation());
2518 #undef __
2519 }
2520
2521 // j.l.Float.float16ToFloat
2522 void C2_MacroAssembler::float_to_float16_v(VectorRegister dst, VectorRegister src,
2523 VectorRegister vtmp, Register tmp, uint vector_length) {
2524 assert_different_registers(dst, src, vtmp);
2525
2526 auto stub = C2CodeStub::make<VectorRegister, VectorRegister, VectorRegister>
2527 (dst, src, vtmp, 56, float_to_float16_v_slow_path);
2528
2529 // On riscv, NaN needs a special process as vfncvt_f_f_w does not work in that case.
2530
2531 vsetvli_helper(BasicType::T_FLOAT, vector_length, Assembler::m1);
2532
2533 // check whether there is a NaN.
2534 // replace v_fclass with vmfne_vv as performance optimization.
2535 vmfne_vv(v0, src, src);
2536 vcpop_m(t0, v0);
2537
2538 vsetvli_helper(BasicType::T_SHORT, vector_length, Assembler::mf2, tmp);
2539
2540 // For non-NaN cases, just use built-in instructions.
2541 vfncvt_f_f_w(dst, src);
2542
2543 // jump to stub processing NaN cases.
2544 bnez(t0, stub->entry(), true);
2545
2546 bind(stub->continuation());
2547 }
2548
2549 void C2_MacroAssembler::signum_fp_v(VectorRegister dst, VectorRegister one, BasicType bt, int vlen) {
2550 vsetvli_helper(bt, vlen);
2551
2552 // check if input is -0, +0, signaling NaN or quiet NaN
2553 vfclass_v(v0, dst);
2554 mv(t0, FClassBits::zero | FClassBits::nan);
2555 vand_vx(v0, v0, t0);
2556 vmseq_vi(v0, v0, 0);
2557
2558 // use floating-point 1.0 with a sign of input
2559 vfsgnj_vv(dst, one, dst, v0_t);
2560 }
2561
2562 // j.l.Math.round(float)
2563 // Returns the closest int to the argument, with ties rounding to positive infinity.
2564 // We need to handle 3 special cases defined by java api spec:
2565 // NaN,
2566 // float >= Integer.MAX_VALUE,
2567 // float <= Integer.MIN_VALUE.
2568 void C2_MacroAssembler::java_round_float_v(VectorRegister dst, VectorRegister src, FloatRegister ftmp,
2569 BasicType bt, uint vector_length) {
2570 // In riscv, there is no straight corresponding rounding mode to satisfy the behaviour defined,
2571 // in java api spec, i.e. any rounding mode can not handle some corner cases, e.g.
2572 // RNE is the closest one, but it ties to "even", which means 1.5/2.5 both will be converted
2573 // to 2, instead of 2 and 3 respectively.
2574 // RUP does not work either, although java api requires "rounding to positive infinity",
2575 // but both 1.3/1.8 will be converted to 2, instead of 1 and 2 respectively.
2576 //
2577 // The optimal solution for non-NaN cases is:
2578 // src+0.5 => dst, with rdn rounding mode,
2579 // convert dst from float to int, with rnd rounding mode.
2580 // and, this solution works as expected for float >= Integer.MAX_VALUE and float <= Integer.MIN_VALUE.
2581 //
2582 // But, we still need to handle NaN explicilty with vector mask instructions.
2583 //
2584 // Check MacroAssembler::java_round_float and C2_MacroAssembler::vector_round_sve in aarch64 for more details.
2585
2586 csrwi(CSR_FRM, C2_MacroAssembler::rdn);
2587 vsetvli_helper(bt, vector_length);
2588
2589 // don't rearrage the instructions sequence order without performance testing.
2590 // check MacroAssembler::java_round_float in riscv64 for more details.
2591 mv(t0, jint_cast(0.5f));
2592 fmv_w_x(ftmp, t0);
2593
2594 // replacing vfclass with feq as performance optimization
2595 vmfeq_vv(v0, src, src);
2596 // set dst = 0 in cases of NaN
2597 vmv_v_x(dst, zr);
2598
2599 // dst = (src + 0.5) rounded down towards negative infinity
2600 vfadd_vf(dst, src, ftmp, Assembler::v0_t);
2601 vfcvt_x_f_v(dst, dst, Assembler::v0_t); // in RoundingMode::rdn
2602
2603 csrwi(CSR_FRM, C2_MacroAssembler::rne);
2604 }
2605
2606 // java.lang.Math.round(double a)
2607 // Returns the closest long to the argument, with ties rounding to positive infinity.
2608 void C2_MacroAssembler::java_round_double_v(VectorRegister dst, VectorRegister src, FloatRegister ftmp,
2609 BasicType bt, uint vector_length) {
2610 // check C2_MacroAssembler::java_round_float_v above for more details.
2611
2612 csrwi(CSR_FRM, C2_MacroAssembler::rdn);
2613 vsetvli_helper(bt, vector_length);
2614
2615 mv(t0, julong_cast(0.5));
2616 fmv_d_x(ftmp, t0);
2617
2618 // replacing vfclass with feq as performance optimization
2619 vmfeq_vv(v0, src, src);
2620 // set dst = 0 in cases of NaN
2621 vmv_v_x(dst, zr);
2622
2623 // dst = (src + 0.5) rounded down towards negative infinity
2624 vfadd_vf(dst, src, ftmp, Assembler::v0_t);
2625 vfcvt_x_f_v(dst, dst, Assembler::v0_t); // in RoundingMode::rdn
2626
2627 csrwi(CSR_FRM, C2_MacroAssembler::rne);
2628 }
2629
2630 void C2_MacroAssembler::element_compare(Register a1, Register a2, Register result, Register cnt, Register tmp1, Register tmp2,
2631 VectorRegister vr1, VectorRegister vr2, VectorRegister vrs, bool islatin, Label &DONE,
2632 Assembler::LMUL lmul) {
2633 Label loop;
2634 Assembler::SEW sew = islatin ? Assembler::e8 : Assembler::e16;
2635
2636 bind(loop);
2637 vsetvli(tmp1, cnt, sew, lmul);
2638 vlex_v(vr1, a1, sew);
2639 vlex_v(vr2, a2, sew);
2640 vmsne_vv(vrs, vr1, vr2);
2641 vfirst_m(tmp2, vrs);
2642 bgez(tmp2, DONE);
2643 sub(cnt, cnt, tmp1);
2644 if (!islatin) {
2645 slli(tmp1, tmp1, 1); // get byte counts
2646 }
2647 add(a1, a1, tmp1);
2648 add(a2, a2, tmp1);
2649 bnez(cnt, loop);
2650
2651 mv(result, true);
2652 }
2653
2654 void C2_MacroAssembler::string_equals_v(Register a1, Register a2, Register result, Register cnt) {
2655 Label DONE;
2656 Register tmp1 = t0;
2657 Register tmp2 = t1;
2658
2659 BLOCK_COMMENT("string_equals_v {");
2660
2661 mv(result, false);
2662
2663 element_compare(a1, a2, result, cnt, tmp1, tmp2, v2, v4, v2, true, DONE, Assembler::m2);
2664
2665 bind(DONE);
2666 BLOCK_COMMENT("} string_equals_v");
2667 }
2668
2669 // used by C2 ClearArray patterns.
2670 // base: Address of a buffer to be zeroed
2671 // cnt: Count in HeapWords
2672 //
2673 // base, cnt, v4, v5, v6, v7 and t0 are clobbered.
2674 void C2_MacroAssembler::clear_array_v(Register base, Register cnt) {
2675 Label loop;
2676
2677 // making zero words
2678 vsetvli(t0, cnt, Assembler::e64, Assembler::m4);
2679 vxor_vv(v4, v4, v4);
2680
2681 bind(loop);
2682 vsetvli(t0, cnt, Assembler::e64, Assembler::m4);
2683 vse64_v(v4, base);
2684 sub(cnt, cnt, t0);
2685 shadd(base, t0, base, t0, 3);
2686 bnez(cnt, loop);
2687 }
2688
2689 void C2_MacroAssembler::arrays_equals_v(Register a1, Register a2, Register result,
2690 Register cnt1, int elem_size) {
2691 assert(elem_size == 1 || elem_size == 2, "must be char or byte");
2692 assert_different_registers(a1, a2, result, cnt1, t0, t1);
2693
2694 Label DONE;
2695 Register tmp1 = t0;
2696 Register tmp2 = t1;
2697 Register cnt2 = tmp2;
2698 int length_offset = arrayOopDesc::length_offset_in_bytes();
2699 int base_offset = arrayOopDesc::base_offset_in_bytes(elem_size == 2 ? T_CHAR : T_BYTE);
2700
2701 assert((base_offset % (UseCompactObjectHeaders ? 4 : 8)) == 0, "Must be");
2702
2703 BLOCK_COMMENT("arrays_equals_v {");
2704
2705 // if (a1 == a2), return true
2706 mv(result, true);
2707 beq(a1, a2, DONE);
2708
2709 mv(result, false);
2710 // if a1 == null or a2 == null, return false
2711 beqz(a1, DONE);
2712 beqz(a2, DONE);
2713 // if (a1.length != a2.length), return false
2714 lwu(cnt1, Address(a1, length_offset));
2715 lwu(cnt2, Address(a2, length_offset));
2716 bne(cnt1, cnt2, DONE);
2717
2718 la(a1, Address(a1, base_offset));
2719 la(a2, Address(a2, base_offset));
2720
2721 element_compare(a1, a2, result, cnt1, tmp1, tmp2, v2, v4, v2, elem_size == 1, DONE, Assembler::m2);
2722
2723 bind(DONE);
2724
2725 BLOCK_COMMENT("} arrays_equals_v");
2726 }
2727
2728 void C2_MacroAssembler::string_compare_v(Register str1, Register str2, Register cnt1, Register cnt2,
2729 Register result, Register tmp1, Register tmp2, int encForm) {
2730 Label DIFFERENCE, DONE, L, loop;
2731 bool encLL = encForm == StrIntrinsicNode::LL;
2732 bool encLU = encForm == StrIntrinsicNode::LU;
2733 bool encUL = encForm == StrIntrinsicNode::UL;
2734
2735 bool str1_isL = encLL || encLU;
2736 bool str2_isL = encLL || encUL;
2737
2738 int minCharsInWord = encLL ? wordSize : wordSize / 2;
2739
2740 BLOCK_COMMENT("string_compare_v {");
2741
2742 // for Latin strings, 1 byte for 1 character
2743 // for UTF16 strings, 2 bytes for 1 character
2744 if (!str1_isL)
2745 sraiw(cnt1, cnt1, 1);
2746 if (!str2_isL)
2747 sraiw(cnt2, cnt2, 1);
2748
2749 // if str1 == str2, return the difference
2750 // save the minimum of the string lengths in cnt2.
2751 sub(result, cnt1, cnt2);
2752 bgt(cnt1, cnt2, L);
2753 mv(cnt2, cnt1);
2754 bind(L);
2755
2756 // We focus on the optimization of small sized string.
2757 // Please check below document for string size distribution statistics.
2758 // https://cr.openjdk.org/~shade/density/string-density-report.pdf
2759 if (str1_isL == str2_isL) { // LL or UU
2760 // Below construction of v regs and lmul is based on test on 2 different boards,
2761 // vlen == 128 and vlen == 256 respectively.
2762 if (!encLL && MaxVectorSize == 16) { // UU
2763 element_compare(str1, str2, zr, cnt2, tmp1, tmp2, v4, v8, v4, encLL, DIFFERENCE, Assembler::m4);
2764 } else { // UU + MaxVectorSize or LL
2765 element_compare(str1, str2, zr, cnt2, tmp1, tmp2, v2, v4, v2, encLL, DIFFERENCE, Assembler::m2);
2766 }
2767
2768 j(DONE);
2769 } else { // LU or UL
2770 Register strL = encLU ? str1 : str2;
2771 Register strU = encLU ? str2 : str1;
2772 VectorRegister vstr1 = encLU ? v8 : v4;
2773 VectorRegister vstr2 = encLU ? v4 : v8;
2774
2775 bind(loop);
2776 vsetvli(tmp1, cnt2, Assembler::e8, Assembler::m2);
2777 vle8_v(vstr1, strL);
2778 vsetvli(tmp1, cnt2, Assembler::e16, Assembler::m4);
2779 vzext_vf2(vstr2, vstr1);
2780 vle16_v(vstr1, strU);
2781 vmsne_vv(v4, vstr2, vstr1);
2782 vfirst_m(tmp2, v4);
2783 bgez(tmp2, DIFFERENCE);
2784 sub(cnt2, cnt2, tmp1);
2785 add(strL, strL, tmp1);
2786 shadd(strU, tmp1, strU, tmp1, 1);
2787 bnez(cnt2, loop);
2788 j(DONE);
2789 }
2790
2791 bind(DIFFERENCE);
2792 slli(tmp1, tmp2, 1);
2793 add(str1, str1, str1_isL ? tmp2 : tmp1);
2794 add(str2, str2, str2_isL ? tmp2 : tmp1);
2795 str1_isL ? lbu(tmp1, Address(str1, 0)) : lhu(tmp1, Address(str1, 0));
2796 str2_isL ? lbu(tmp2, Address(str2, 0)) : lhu(tmp2, Address(str2, 0));
2797 sub(result, tmp1, tmp2);
2798
2799 bind(DONE);
2800
2801 BLOCK_COMMENT("} string_compare_v");
2802 }
2803
2804 void C2_MacroAssembler::byte_array_inflate_v(Register src, Register dst, Register len, Register tmp) {
2805 Label loop;
2806 assert_different_registers(src, dst, len, tmp, t0);
2807
2808 BLOCK_COMMENT("byte_array_inflate_v {");
2809 bind(loop);
2810 vsetvli(tmp, len, Assembler::e8, Assembler::m2);
2811 vle8_v(v6, src);
2812 vsetvli(t0, len, Assembler::e16, Assembler::m4);
2813 vzext_vf2(v4, v6);
2814 vse16_v(v4, dst);
2815 sub(len, len, tmp);
2816 add(src, src, tmp);
2817 shadd(dst, tmp, dst, tmp, 1);
2818 bnez(len, loop);
2819 BLOCK_COMMENT("} byte_array_inflate_v");
2820 }
2821
2822 // Compress char[] array to byte[].
2823 // Intrinsic for java.lang.StringUTF16.compress(char[] src, int srcOff, byte[] dst, int dstOff, int len)
2824 // result: the array length if every element in array can be encoded,
2825 // otherwise, the index of first non-latin1 (> 0xff) character.
2826 void C2_MacroAssembler::char_array_compress_v(Register src, Register dst, Register len,
2827 Register result, Register tmp) {
2828 encode_iso_array_v(src, dst, len, result, tmp, false);
2829 }
2830
2831 // Intrinsic for
2832 //
2833 // - sun.nio.cs.ISO_8859_1.Encoder#encodeISOArray0(byte[] sa, int sp, byte[] da, int dp, int len)
2834 // Encodes char[] to byte[] in ISO-8859-1
2835 //
2836 // - java.lang.StringCoding#encodeISOArray0(byte[] sa, int sp, byte[] da, int dp, int len)
2837 // Encodes byte[] (containing UTF-16) to byte[] in ISO-8859-1
2838 //
2839 // - java.lang.StringCoding#encodeAsciiArray0(char[] sa, int sp, byte[] da, int dp, int len)
2840 // Encodes char[] to byte[] in ASCII
2841 //
2842 // This version always returns the number of characters copied. A successful
2843 // copy will complete with the post-condition: 'res' == 'len', while an
2844 // unsuccessful copy will exit with the post-condition: 0 <= 'res' < 'len'.
2845 //
2846 // Clobbers: src, dst, len, result, t0
2847 void C2_MacroAssembler::encode_iso_array_v(Register src, Register dst, Register len,
2848 Register result, Register tmp, bool ascii) {
2849 Label loop, fail, done;
2850
2851 BLOCK_COMMENT("encode_iso_array_v {");
2852 mv(result, 0);
2853
2854 bind(loop);
2855 mv(tmp, ascii ? 0x7f : 0xff);
2856 vsetvli(t0, len, Assembler::e16, Assembler::m2);
2857 vle16_v(v2, src);
2858
2859 vmsgtu_vx(v1, v2, tmp);
2860 vfirst_m(tmp, v1);
2861 vmsbf_m(v0, v1);
2862 // compress char to byte
2863 vsetvli(t0, len, Assembler::e8);
2864 vncvt_x_x_w(v1, v2, Assembler::v0_t);
2865 vse8_v(v1, dst, Assembler::v0_t);
2866
2867 // fail if char > 0x7f/0xff
2868 bgez(tmp, fail);
2869 add(result, result, t0);
2870 add(dst, dst, t0);
2871 sub(len, len, t0);
2872 shadd(src, t0, src, t0, 1);
2873 bnez(len, loop);
2874 j(done);
2875
2876 bind(fail);
2877 add(result, result, tmp);
2878
2879 bind(done);
2880 BLOCK_COMMENT("} encode_iso_array_v");
2881 }
2882
2883 void C2_MacroAssembler::count_positives_v(Register ary, Register len, Register result, Register tmp) {
2884 Label LOOP, SET_RESULT, DONE;
2885
2886 BLOCK_COMMENT("count_positives_v {");
2887 assert_different_registers(ary, len, result, tmp);
2888
2889 mv(result, zr);
2890
2891 bind(LOOP);
2892 vsetvli(t0, len, Assembler::e8, Assembler::m4);
2893 vle8_v(v4, ary);
2894 vmslt_vx(v4, v4, zr);
2895 vfirst_m(tmp, v4);
2896 bgez(tmp, SET_RESULT);
2897 // if tmp == -1, all bytes are positive
2898 add(result, result, t0);
2899
2900 sub(len, len, t0);
2901 add(ary, ary, t0);
2902 bnez(len, LOOP);
2903 j(DONE);
2904
2905 // add remaining positive bytes count
2906 bind(SET_RESULT);
2907 add(result, result, tmp);
2908
2909 bind(DONE);
2910 BLOCK_COMMENT("} count_positives_v");
2911 }
2912
2913 void C2_MacroAssembler::string_indexof_char_v(Register str1, Register cnt1,
2914 Register ch, Register result,
2915 Register tmp1, Register tmp2,
2916 bool isL) {
2917 mv(result, zr);
2918
2919 Label loop, MATCH, DONE;
2920 Assembler::SEW sew = isL ? Assembler::e8 : Assembler::e16;
2921 bind(loop);
2922 vsetvli(tmp1, cnt1, sew, Assembler::m4);
2923 vlex_v(v4, str1, sew);
2924 vmseq_vx(v4, v4, ch);
2925 vfirst_m(tmp2, v4);
2926 bgez(tmp2, MATCH); // if equal, return index
2927
2928 add(result, result, tmp1);
2929 sub(cnt1, cnt1, tmp1);
2930 if (!isL) slli(tmp1, tmp1, 1);
2931 add(str1, str1, tmp1);
2932 bnez(cnt1, loop);
2933
2934 mv(result, -1);
2935 j(DONE);
2936
2937 bind(MATCH);
2938 add(result, result, tmp2);
2939
2940 bind(DONE);
2941 }
2942
2943 // Set dst to NaN if any NaN input.
2944 void C2_MacroAssembler::minmax_fp_v(VectorRegister dst, VectorRegister src1, VectorRegister src2,
2945 BasicType bt, bool is_min, uint vector_length) {
2946 assert_different_registers(dst, src1, src2);
2947
2948 vsetvli_helper(bt, vector_length);
2949
2950 is_min ? vfmin_vv(dst, src1, src2)
2951 : vfmax_vv(dst, src1, src2);
2952
2953 vmfne_vv(v0, src1, src1);
2954 vfadd_vv(dst, src1, src1, Assembler::v0_t);
2955 vmfne_vv(v0, src2, src2);
2956 vfadd_vv(dst, src2, src2, Assembler::v0_t);
2957 }
2958
2959 // Set dst to NaN if any NaN input.
2960 // The destination vector register elements corresponding to masked-off elements
2961 // are handled with a mask-undisturbed policy.
2962 void C2_MacroAssembler::minmax_fp_masked_v(VectorRegister dst, VectorRegister src1, VectorRegister src2,
2963 VectorRegister vmask, VectorRegister tmp1, VectorRegister tmp2,
2964 BasicType bt, bool is_min, uint vector_length) {
2965 assert_different_registers(src1, src2, tmp1, tmp2);
2966 vsetvli_helper(bt, vector_length);
2967
2968 // Check vector elements of src1 and src2 for NaN.
2969 vmfeq_vv(tmp1, src1, src1);
2970 vmfeq_vv(tmp2, src2, src2);
2971
2972 vmandn_mm(v0, vmask, tmp1);
2973 vfadd_vv(dst, src1, src1, Assembler::v0_t);
2974 vmandn_mm(v0, vmask, tmp2);
2975 vfadd_vv(dst, src2, src2, Assembler::v0_t);
2976
2977 vmand_mm(tmp2, tmp1, tmp2);
2978 vmand_mm(v0, vmask, tmp2);
2979 is_min ? vfmin_vv(dst, src1, src2, Assembler::v0_t)
2980 : vfmax_vv(dst, src1, src2, Assembler::v0_t);
2981 }
2982
2983 // Set dst to NaN if any NaN input.
2984 void C2_MacroAssembler::reduce_minmax_fp_v(FloatRegister dst,
2985 FloatRegister src1, VectorRegister src2,
2986 VectorRegister tmp1, VectorRegister tmp2,
2987 bool is_double, bool is_min, uint vector_length, VectorMask vm) {
2988 assert_different_registers(dst, src1);
2989 assert_different_registers(src2, tmp1, tmp2);
2990
2991 Label L_done, L_NaN_1, L_NaN_2;
2992 // Set dst to src1 if src1 is NaN
2993 is_double ? feq_d(t0, src1, src1)
2994 : feq_s(t0, src1, src1);
2995 beqz(t0, L_NaN_2);
2996
2997 vsetvli_helper(is_double ? T_DOUBLE : T_FLOAT, vector_length);
2998 vfmv_s_f(tmp2, src1);
2999
3000 is_min ? vfredmin_vs(tmp1, src2, tmp2, vm)
3001 : vfredmax_vs(tmp1, src2, tmp2, vm);
3002 vfmv_f_s(dst, tmp1);
3003
3004 // Checking NaNs in src2
3005 vmfne_vv(tmp1, src2, src2, vm);
3006 vcpop_m(t0, tmp1, vm);
3007 beqz(t0, L_done);
3008
3009 bind(L_NaN_1);
3010 vfredusum_vs(tmp1, src2, tmp2, vm);
3011 vfmv_f_s(dst, tmp1);
3012 j(L_done);
3013
3014 bind(L_NaN_2);
3015 is_double ? fmv_d(dst, src1)
3016 : fmv_s(dst, src1);
3017 bind(L_done);
3018 }
3019
3020 bool C2_MacroAssembler::in_scratch_emit_size() {
3021 if (ciEnv::current()->task() != nullptr) {
3022 PhaseOutput* phase_output = Compile::current()->output();
3023 if (phase_output != nullptr && phase_output->in_scratch_emit_size()) {
3024 return true;
3025 }
3026 }
3027 return MacroAssembler::in_scratch_emit_size();
3028 }
3029
3030 void C2_MacroAssembler::reduce_integral_v(Register dst, Register src1,
3031 VectorRegister src2, VectorRegister tmp,
3032 int opc, BasicType bt, uint vector_length, VectorMask vm) {
3033 assert(bt == T_BYTE || bt == T_SHORT || bt == T_INT || bt == T_LONG, "unsupported element type");
3034 vsetvli_helper(bt, vector_length);
3035 vmv_s_x(tmp, src1);
3036 switch (opc) {
3037 case Op_AddReductionVI:
3038 case Op_AddReductionVL:
3039 vredsum_vs(tmp, src2, tmp, vm);
3040 break;
3041 case Op_AndReductionV:
3042 vredand_vs(tmp, src2, tmp, vm);
3043 break;
3044 case Op_OrReductionV:
3045 vredor_vs(tmp, src2, tmp, vm);
3046 break;
3047 case Op_XorReductionV:
3048 vredxor_vs(tmp, src2, tmp, vm);
3049 break;
3050 case Op_MaxReductionV:
3051 vredmax_vs(tmp, src2, tmp, vm);
3052 break;
3053 case Op_MinReductionV:
3054 vredmin_vs(tmp, src2, tmp, vm);
3055 break;
3056 default:
3057 ShouldNotReachHere();
3058 }
3059 vmv_x_s(dst, tmp);
3060 }
3061
3062 void C2_MacroAssembler::reduce_mul_integral_v(Register dst, Register src1, VectorRegister src2,
3063 VectorRegister vtmp1, VectorRegister vtmp2,
3064 BasicType bt, uint vector_length, VectorMask vm) {
3065 assert(bt == T_BYTE || bt == T_SHORT || bt == T_INT || bt == T_LONG, "unsupported element type");
3066 vsetvli_helper(bt, vector_length);
3067
3068 vector_length /= 2;
3069 if (vm != Assembler::unmasked) {
3070 // This behaviour is consistent with spec requirements of vector API, for `reduceLanes`:
3071 // If no elements are selected, an operation-specific identity value is returned.
3072 // If the operation is MUL, then the identity value is one.
3073 vmv_v_i(vtmp1, 1);
3074 vmerge_vvm(vtmp2, vtmp1, src2); // vm == v0
3075 slidedown_v(vtmp1, vtmp2, vector_length);
3076
3077 vsetvli_helper(bt, vector_length);
3078 vmul_vv(vtmp1, vtmp1, vtmp2);
3079 } else {
3080 slidedown_v(vtmp1, src2, vector_length);
3081
3082 vsetvli_helper(bt, vector_length);
3083 vmul_vv(vtmp1, vtmp1, src2);
3084 }
3085
3086 while (vector_length > 1) {
3087 vector_length /= 2;
3088 slidedown_v(vtmp2, vtmp1, vector_length);
3089 vsetvli_helper(bt, vector_length);
3090 vmul_vv(vtmp1, vtmp1, vtmp2);
3091 }
3092
3093 vmv_x_s(dst, vtmp1);
3094 if (bt == T_INT) {
3095 mulw(dst, dst, src1);
3096 } else {
3097 mul(dst, dst, src1);
3098 }
3099 }
3100
3101 // Set vl and vtype for full and partial vector operations.
3102 // (vma = mu, vta = tu, vill = false)
3103 void C2_MacroAssembler::vsetvli_helper(BasicType bt, uint vector_length, LMUL vlmul, Register tmp) {
3104 Assembler::SEW sew = Assembler::elemtype_to_sew(bt);
3105 if (vector_length <= 31) {
3106 vsetivli(tmp, vector_length, sew, vlmul);
3107 } else if (vector_length == (MaxVectorSize / type2aelembytes(bt))) {
3108 vsetvli(tmp, x0, sew, vlmul);
3109 } else {
3110 mv(tmp, vector_length);
3111 vsetvli(tmp, tmp, sew, vlmul);
3112 }
3113 }
3114
3115 void C2_MacroAssembler::compare_integral_v(VectorRegister vd, VectorRegister src1, VectorRegister src2,
3116 int cond, BasicType bt, uint vector_length, VectorMask vm) {
3117 assert(is_integral_type(bt), "unsupported element type");
3118 assert(vm == Assembler::v0_t ? vd != v0 : true, "should be different registers");
3119 vsetvli_helper(bt, vector_length);
3120 if (vm == Assembler::v0_t) {
3121 vmclr_m(vd);
3122 }
3123 switch (cond) {
3124 case BoolTest::eq: vmseq_vv(vd, src1, src2, vm); break;
3125 case BoolTest::ne: vmsne_vv(vd, src1, src2, vm); break;
3126 case BoolTest::le: vmsle_vv(vd, src1, src2, vm); break;
3127 case BoolTest::ge: vmsge_vv(vd, src1, src2, vm); break;
3128 case BoolTest::lt: vmslt_vv(vd, src1, src2, vm); break;
3129 case BoolTest::gt: vmsgt_vv(vd, src1, src2, vm); break;
3130 case BoolTest::ule: vmsleu_vv(vd, src1, src2, vm); break;
3131 case BoolTest::uge: vmsgeu_vv(vd, src1, src2, vm); break;
3132 case BoolTest::ult: vmsltu_vv(vd, src1, src2, vm); break;
3133 case BoolTest::ugt: vmsgtu_vv(vd, src1, src2, vm); break;
3134 default:
3135 assert(false, "unsupported compare condition");
3136 ShouldNotReachHere();
3137 }
3138 }
3139
3140 void C2_MacroAssembler::compare_fp_v(VectorRegister vd, VectorRegister src1, VectorRegister src2,
3141 int cond, BasicType bt, uint vector_length, VectorMask vm) {
3142 assert(is_floating_point_type(bt), "unsupported element type");
3143 assert(vm == Assembler::v0_t ? vd != v0 : true, "should be different registers");
3144 vsetvli_helper(bt, vector_length);
3145 if (vm == Assembler::v0_t) {
3146 vmclr_m(vd);
3147 }
3148 switch (cond) {
3149 case BoolTest::eq: vmfeq_vv(vd, src1, src2, vm); break;
3150 case BoolTest::ne: vmfne_vv(vd, src1, src2, vm); break;
3151 case BoolTest::le: vmfle_vv(vd, src1, src2, vm); break;
3152 case BoolTest::ge: vmfge_vv(vd, src1, src2, vm); break;
3153 case BoolTest::lt: vmflt_vv(vd, src1, src2, vm); break;
3154 case BoolTest::gt: vmfgt_vv(vd, src1, src2, vm); break;
3155 default:
3156 assert(false, "unsupported compare condition");
3157 ShouldNotReachHere();
3158 }
3159 }
3160
3161 // In Matcher::scalable_predicate_reg_slots,
3162 // we assume each predicate register is one-eighth of the size of
3163 // scalable vector register, one mask bit per vector byte.
3164 void C2_MacroAssembler::spill_vmask(VectorRegister v, int offset) {
3165 vsetvli_helper(T_BYTE, MaxVectorSize >> 3);
3166 add(t0, sp, offset);
3167 vse8_v(v, t0);
3168 }
3169
3170 void C2_MacroAssembler::unspill_vmask(VectorRegister v, int offset) {
3171 vsetvli_helper(T_BYTE, MaxVectorSize >> 3);
3172 add(t0, sp, offset);
3173 vle8_v(v, t0);
3174 }
3175
3176 void C2_MacroAssembler::integer_extend_v(VectorRegister dst, BasicType dst_bt, uint vector_length,
3177 VectorRegister src, BasicType src_bt, bool is_signed) {
3178 assert(type2aelembytes(dst_bt) > type2aelembytes(src_bt) && type2aelembytes(dst_bt) <= 8 && type2aelembytes(src_bt) <= 4, "invalid element size");
3179 assert(dst_bt != T_FLOAT && dst_bt != T_DOUBLE && src_bt != T_FLOAT && src_bt != T_DOUBLE, "unsupported element type");
3180 // https://github.com/riscv/riscv-v-spec/blob/master/v-spec.adoc#52-vector-operands
3181 // The destination EEW is greater than the source EEW, the source EMUL is at least 1,
3182 // and the overlap is in the highest-numbered part of the destination register group.
3183 // Since LMUL=1, vd and vs cannot be the same.
3184 assert_different_registers(dst, src);
3185
3186 vsetvli_helper(dst_bt, vector_length);
3187 if (is_signed) {
3188 if (src_bt == T_BYTE) {
3189 switch (dst_bt) {
3190 case T_SHORT:
3191 vsext_vf2(dst, src);
3192 break;
3193 case T_INT:
3194 vsext_vf4(dst, src);
3195 break;
3196 case T_LONG:
3197 vsext_vf8(dst, src);
3198 break;
3199 default:
3200 ShouldNotReachHere();
3201 }
3202 } else if (src_bt == T_SHORT) {
3203 if (dst_bt == T_INT) {
3204 vsext_vf2(dst, src);
3205 } else {
3206 vsext_vf4(dst, src);
3207 }
3208 } else if (src_bt == T_INT) {
3209 vsext_vf2(dst, src);
3210 }
3211 } else {
3212 if (src_bt == T_BYTE) {
3213 switch (dst_bt) {
3214 case T_SHORT:
3215 vzext_vf2(dst, src);
3216 break;
3217 case T_INT:
3218 vzext_vf4(dst, src);
3219 break;
3220 case T_LONG:
3221 vzext_vf8(dst, src);
3222 break;
3223 default:
3224 ShouldNotReachHere();
3225 }
3226 } else if (src_bt == T_SHORT) {
3227 if (dst_bt == T_INT) {
3228 vzext_vf2(dst, src);
3229 } else {
3230 vzext_vf4(dst, src);
3231 }
3232 } else if (src_bt == T_INT) {
3233 vzext_vf2(dst, src);
3234 }
3235 }
3236 }
3237
3238 // Vector narrow from src to dst with specified element sizes.
3239 // High part of dst vector will be filled with zero.
3240 void C2_MacroAssembler::integer_narrow_v(VectorRegister dst, BasicType dst_bt, uint vector_length,
3241 VectorRegister src, BasicType src_bt) {
3242 assert(type2aelembytes(dst_bt) < type2aelembytes(src_bt) && type2aelembytes(dst_bt) <= 4 && type2aelembytes(src_bt) <= 8, "invalid element size");
3243 assert(dst_bt != T_FLOAT && dst_bt != T_DOUBLE && src_bt != T_FLOAT && src_bt != T_DOUBLE, "unsupported element type");
3244 mv(t0, vector_length);
3245 if (src_bt == T_LONG) {
3246 // https://github.com/riscv/riscv-v-spec/blob/master/v-spec.adoc#117-vector-narrowing-integer-right-shift-instructions
3247 // Future extensions might add support for versions that narrow to a destination that is 1/4 the width of the source.
3248 // So we can currently only scale down by 1/2 the width at a time.
3249 vsetvli(t0, t0, Assembler::e32, Assembler::mf2);
3250 vncvt_x_x_w(dst, src);
3251 if (dst_bt == T_SHORT || dst_bt == T_BYTE) {
3252 vsetvli(t0, t0, Assembler::e16, Assembler::mf2);
3253 vncvt_x_x_w(dst, dst);
3254 if (dst_bt == T_BYTE) {
3255 vsetvli(t0, t0, Assembler::e8, Assembler::mf2);
3256 vncvt_x_x_w(dst, dst);
3257 }
3258 }
3259 } else if (src_bt == T_INT) {
3260 // T_SHORT
3261 vsetvli(t0, t0, Assembler::e16, Assembler::mf2);
3262 vncvt_x_x_w(dst, src);
3263 if (dst_bt == T_BYTE) {
3264 vsetvli(t0, t0, Assembler::e8, Assembler::mf2);
3265 vncvt_x_x_w(dst, dst);
3266 }
3267 } else if (src_bt == T_SHORT) {
3268 vsetvli(t0, t0, Assembler::e8, Assembler::mf2);
3269 vncvt_x_x_w(dst, src);
3270 }
3271 }
3272
3273 #define VFCVT_SAFE(VFLOATCVT) \
3274 void C2_MacroAssembler::VFLOATCVT##_safe(VectorRegister dst, VectorRegister src) { \
3275 assert_different_registers(dst, src); \
3276 vxor_vv(dst, dst, dst); \
3277 vmfeq_vv(v0, src, src); \
3278 VFLOATCVT(dst, src, Assembler::v0_t); \
3279 }
3280
3281 VFCVT_SAFE(vfcvt_rtz_x_f_v);
3282
3283 #undef VFCVT_SAFE
3284
3285 // Extract a scalar element from an vector at position 'idx'.
3286 // The input elements in src are expected to be of integral type.
3287 void C2_MacroAssembler::extract_v(Register dst, VectorRegister src,
3288 BasicType bt, int idx, VectorRegister vtmp) {
3289 assert(is_integral_type(bt), "unsupported element type");
3290 assert(idx >= 0, "idx cannot be negative");
3291 // Only need the first element after vector slidedown
3292 vsetvli_helper(bt, 1);
3293 if (idx == 0) {
3294 vmv_x_s(dst, src);
3295 } else {
3296 slidedown_v(vtmp, src, idx);
3297 vmv_x_s(dst, vtmp);
3298 }
3299 }
3300
3301 // Extract a scalar element from a vector at position 'idx'.
3302 // The input elements in src are expected to be of integral type.
3303 void C2_MacroAssembler::extract_v(Register dst, VectorRegister src,
3304 BasicType bt, Register idx, VectorRegister vtmp) {
3305 assert(is_integral_type(bt), "unsupported element type");
3306 // Only need the first element after vector slidedown
3307 vsetvli_helper(bt, 1);
3308 vslidedown_vx(vtmp, src, idx);
3309 vmv_x_s(dst, vtmp);
3310 }
3311
3312 // Extract a scalar element from an vector at position 'idx'.
3313 // The input elements in src are expected to be of floating point type.
3314 void C2_MacroAssembler::extract_fp_v(FloatRegister dst, VectorRegister src,
3315 BasicType bt, int idx, VectorRegister vtmp) {
3316 assert(is_floating_point_type(bt), "unsupported element type");
3317 assert(idx >= 0, "idx cannot be negative");
3318 // Only need the first element after vector slidedown
3319 vsetvli_helper(bt, 1);
3320 if (idx == 0) {
3321 vfmv_f_s(dst, src);
3322 } else {
3323 slidedown_v(vtmp, src, idx);
3324 vfmv_f_s(dst, vtmp);
3325 }
3326 }
3327
3328 // Move elements down a vector register group.
3329 // Offset is the start index (offset) for the source.
3330 void C2_MacroAssembler::slidedown_v(VectorRegister dst, VectorRegister src,
3331 uint32_t offset, Register tmp) {
3332 if (is_uimm5(offset)) {
3333 vslidedown_vi(dst, src, offset);
3334 } else {
3335 mv(tmp, offset);
3336 vslidedown_vx(dst, src, tmp);
3337 }
3338 }