1 /*
2 * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "asm/assembler.hpp"
27 #include "asm/assembler.inline.hpp"
28 #include "opto/c2_MacroAssembler.hpp"
29 #include "opto/compile.hpp"
30 #include "opto/intrinsicnode.hpp"
31 #include "opto/output.hpp"
32 #include "opto/subnode.hpp"
33 #include "runtime/objectMonitorTable.hpp"
34 #include "runtime/stubRoutines.hpp"
35 #include "runtime/synchronizer.hpp"
36 #include "utilities/globalDefinitions.hpp"
37
38 #ifdef PRODUCT
39 #define BLOCK_COMMENT(str) /* nothing */
40 #define STOP(error) stop(error)
41 #else
42 #define BLOCK_COMMENT(str) block_comment(str)
43 #define STOP(error) block_comment(error); stop(error)
44 #endif
45
46 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
47
48 void C2_MacroAssembler::entry_barrier() {
49 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
50 // Dummy labels for just measuring the code size
51 Label dummy_slow_path;
52 Label dummy_continuation;
53 Label dummy_guard;
54 Label* slow_path = &dummy_slow_path;
55 Label* continuation = &dummy_continuation;
56 Label* guard = &dummy_guard;
57
58 if (!Compile::current()->output()->in_scratch_emit_size()) {
59 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
60 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
61 Compile::current()->output()->add_stub(stub);
62 slow_path = &stub->entry();
63 continuation = &stub->continuation();
64 guard = &stub->guard();
65 }
66
67 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
68 bs->nmethod_entry_barrier(this, slow_path, continuation, guard);
69 }
70
71 void C2_MacroAssembler::fast_lock(Register obj, Register box,
72 Register tmp1, Register tmp2, Register tmp3, Register tmp4) {
73 // Flag register, zero for success; non-zero for failure.
74 Register flag = t1;
75
76 assert_different_registers(obj, box, tmp1, tmp2, tmp3, tmp4, flag, t0);
77
78 mv(flag, 1);
79
80 // Handle inflated monitor.
81 Label inflated;
82 // Finish fast lock successfully. MUST branch to with flag == 0
83 Label locked;
84 // Finish fast lock unsuccessfully. slow_path MUST branch to with flag != 0
85 Label slow_path;
86
87 if (UseObjectMonitorTable) {
88 // Clear cache in case fast locking succeeds or we need to take the slow-path.
89 sd(zr, Address(box, BasicLock::object_monitor_cache_offset_in_bytes()));
90 }
91
92 if (DiagnoseSyncOnValueBasedClasses != 0) {
93 load_klass(tmp1, obj);
94 lbu(tmp1, Address(tmp1, Klass::misc_flags_offset()));
95 test_bit(tmp1, tmp1, exact_log2(KlassFlags::_misc_is_value_based_class));
96 bnez(tmp1, slow_path);
97 }
98
99 const Register tmp1_mark = tmp1;
100 const Register tmp3_t = tmp3;
101
102 { // Fast locking
103
104 // Push lock to the lock stack and finish successfully. MUST branch to with flag == 0
105 Label push;
106
107 const Register tmp2_top = tmp2;
108
109 // Check if lock-stack is full.
110 lwu(tmp2_top, Address(xthread, JavaThread::lock_stack_top_offset()));
111 mv(tmp3_t, (unsigned)LockStack::end_offset());
112 bge(tmp2_top, tmp3_t, slow_path);
113
114 // Check if recursive.
115 add(tmp3_t, xthread, tmp2_top);
116 ld(tmp3_t, Address(tmp3_t, -oopSize));
117 beq(obj, tmp3_t, push);
118
119 // Relaxed normal load to check for monitor. Optimization for monitor case.
120 ld(tmp1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
121 test_bit(tmp3_t, tmp1_mark, exact_log2(markWord::monitor_value));
122 bnez(tmp3_t, inflated);
123
124 // Not inflated
125 assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid a la");
126
127 // Try to lock. Transition lock-bits 0b01 => 0b00
128 ori(tmp1_mark, tmp1_mark, markWord::unlocked_value);
129 xori(tmp3_t, tmp1_mark, markWord::unlocked_value);
130 cmpxchg(/*addr*/ obj, /*expected*/ tmp1_mark, /*new*/ tmp3_t, Assembler::int64,
131 /*acquire*/ Assembler::aq, /*release*/ Assembler::relaxed, /*result*/ tmp3_t);
132 bne(tmp1_mark, tmp3_t, slow_path);
133
134 bind(push);
135 // After successful lock, push object on lock-stack.
136 add(tmp3_t, xthread, tmp2_top);
137 sd(obj, Address(tmp3_t));
138 addw(tmp2_top, tmp2_top, oopSize);
139 sw(tmp2_top, Address(xthread, JavaThread::lock_stack_top_offset()));
140 j(locked);
141 }
142
143 { // Handle inflated monitor.
144 bind(inflated);
145
146 const Register tmp1_monitor = tmp1;
147 // Offsets into the current thread's object monitor cache (omc).
148 const ByteSize thr_omc_offset = JavaThread::om_cache_offset();
149 const ByteSize omc_monitor_offset = OMCache::monitor_offset();
150 const ByteSize omc_obj_offset = OMCache::obj_offset();
151
152 if (!UseObjectMonitorTable) {
153 assert(tmp1_monitor == tmp1_mark, "should be the same here");
154 } else {
155 const Register tmp2_hash = tmp2;
156 const Register tmp3_bucket = tmp3;
157 Label monitor_found;
158
159 // Save the mark, we might need it to extract the hash.
160 mv(tmp2_hash, tmp1_mark);
161
162 // Look for the monitor in the current thread's object monitor cache (omc).
163
164 ld(tmp1_monitor, Address(xthread, thr_omc_offset + omc_monitor_offset));
165 ld(tmp4, Address(xthread, thr_omc_offset + omc_obj_offset));
166 beq(obj, tmp4, monitor_found);
167
168 // Look for the monitor in the table.
169
170 // Get the hash code.
171 srli(tmp2_hash, tmp2_hash, markWord::hash_shift);
172
173 // Get the table and calculate the bucket's address.
174 la(tmp3_t, ExternalAddress(ObjectMonitorTable::current_table_address()));
175 ld(tmp3_t, Address(tmp3_t));
176 ld(tmp1, Address(tmp3_t, ObjectMonitorTable::table_capacity_mask_offset()));
177 andr(tmp2_hash, tmp2_hash, tmp1);
178 ld(tmp3_t, Address(tmp3_t, ObjectMonitorTable::table_buckets_offset()));
179
180 // Read the monitor from the bucket.
181 shadd(tmp3_bucket, tmp2_hash, tmp3_t, tmp4, LogBytesPerWord);
182 ld(tmp1_monitor, Address(tmp3_bucket));
183
184 // Check if the monitor in the bucket is special (empty, tombstone or removed).
185 mv(tmp2, ObjectMonitorTable::SpecialPointerValues::below_is_special);
186 bltu(tmp1_monitor, tmp2, slow_path);
187
188 // Check if object matches.
189 ld(tmp3, Address(tmp1_monitor, ObjectMonitor::object_offset()));
190 BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
191 bs_asm->try_peek_weak_handle_in_nmethod(this, tmp3, tmp3, tmp2, slow_path);
192 bne(tmp3, obj, slow_path);
193
194 // Store the monitor in the current thread's object monitor cache (omc).
195 sd(tmp1_monitor, Address(xthread, thr_omc_offset + omc_monitor_offset));
196 sd(obj, Address(xthread, thr_omc_offset + omc_obj_offset));
197
198 bind(monitor_found);
199 }
200
201 const Register tmp2_owner_addr = tmp2;
202 const Register tmp3_owner = tmp3;
203
204 const ByteSize monitor_tag = in_ByteSize(UseObjectMonitorTable ? 0 : checked_cast<int>(markWord::monitor_value));
205 const Address owner_address(tmp1_monitor, ObjectMonitor::owner_offset() - monitor_tag);
206 const Address recursions_address(tmp1_monitor, ObjectMonitor::recursions_offset() - monitor_tag);
207
208 Label monitor_locked;
209
210 // Compute owner address.
211 la(tmp2_owner_addr, owner_address);
212
213 // Try to CAS owner (no owner => current thread's _monitor_owner_id).
214 Register tid = tmp4;
215 ld(tid, Address(xthread, JavaThread::monitor_owner_id_offset()));
216 cmpxchg(/*addr*/ tmp2_owner_addr, /*expected*/ zr, /*new*/ tid, Assembler::int64,
217 /*acquire*/ Assembler::aq, /*release*/ Assembler::relaxed, /*result*/ tmp3_owner);
218 beqz(tmp3_owner, monitor_locked);
219
220 // Check if recursive.
221 bne(tmp3_owner, tid, slow_path);
222
223 // Recursive.
224 increment(recursions_address, 1, tmp2, tmp3);
225
226 bind(monitor_locked);
227 if (UseObjectMonitorTable) {
228 // Cache the monitor for unlock.
229 sd(tmp1_monitor, Address(box, BasicLock::object_monitor_cache_offset_in_bytes()));
230 }
231 }
232
233 bind(locked);
234 mv(flag, zr);
235
236 #ifdef ASSERT
237 // Check that locked label is reached with flag == 0.
238 Label flag_correct;
239 beqz(flag, flag_correct);
240 stop("Fast Lock Flag != 0");
241 #endif
242
243 bind(slow_path);
244 #ifdef ASSERT
245 // Check that slow_path label is reached with flag != 0.
246 bnez(flag, flag_correct);
247 stop("Fast Lock Flag == 0");
248 bind(flag_correct);
249 #endif
250 // C2 uses the value of flag (0 vs !0) to determine the continuation.
251 }
252
253 void C2_MacroAssembler::fast_unlock(Register obj, Register box,
254 Register tmp1, Register tmp2, Register tmp3) {
255 // Flag register, zero for success; non-zero for failure.
256 Register flag = t1;
257
258 assert_different_registers(obj, box, tmp1, tmp2, tmp3, flag, t0);
259
260 mv(flag, 1);
261
262 // Handle inflated monitor.
263 Label inflated, inflated_load_mark;
264 // Finish fast unlock successfully. unlocked MUST branch to with flag == 0
265 Label unlocked;
266 // Finish fast unlock unsuccessfully. MUST branch to with flag != 0
267 Label slow_path;
268
269 const Register tmp1_mark = tmp1;
270 const Register tmp2_top = tmp2;
271 const Register tmp3_t = tmp3;
272
273 { // Fast unlock
274 Label push_and_slow_path;
275
276 // Check if obj is top of lock-stack.
277 lwu(tmp2_top, Address(xthread, JavaThread::lock_stack_top_offset()));
278 subw(tmp2_top, tmp2_top, oopSize);
279 add(tmp3_t, xthread, tmp2_top);
280 ld(tmp3_t, Address(tmp3_t));
281 // Top of lock stack was not obj. Must be monitor.
282 bne(obj, tmp3_t, inflated_load_mark);
283
284 // Pop lock-stack.
285 DEBUG_ONLY(add(tmp3_t, xthread, tmp2_top);)
286 DEBUG_ONLY(sd(zr, Address(tmp3_t));)
287 sw(tmp2_top, Address(xthread, JavaThread::lock_stack_top_offset()));
288
289 // Check if recursive.
290 add(tmp3_t, xthread, tmp2_top);
291 ld(tmp3_t, Address(tmp3_t, -oopSize));
292 beq(obj, tmp3_t, unlocked);
293
294 // Not recursive.
295 // Load Mark.
296 ld(tmp1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
297
298 // Check header for monitor (0b10).
299 // Because we got here by popping (meaning we pushed in locked)
300 // there will be no monitor in the box. So we need to push back the obj
301 // so that the runtime can fix any potential anonymous owner.
302 test_bit(tmp3_t, tmp1_mark, exact_log2(markWord::monitor_value));
303 bnez(tmp3_t, UseObjectMonitorTable ? push_and_slow_path : inflated);
304
305 // Try to unlock. Transition lock bits 0b00 => 0b01
306 assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
307 ori(tmp3_t, tmp1_mark, markWord::unlocked_value);
308 cmpxchg(/*addr*/ obj, /*expected*/ tmp1_mark, /*new*/ tmp3_t, Assembler::int64,
309 /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, /*result*/ tmp3_t);
310 beq(tmp1_mark, tmp3_t, unlocked);
311
312 bind(push_and_slow_path);
313 // Compare and exchange failed.
314 // Restore lock-stack and handle the unlock in runtime.
315 DEBUG_ONLY(add(tmp3_t, xthread, tmp2_top);)
316 DEBUG_ONLY(sd(obj, Address(tmp3_t));)
317 addw(tmp2_top, tmp2_top, oopSize);
318 sd(tmp2_top, Address(xthread, JavaThread::lock_stack_top_offset()));
319 j(slow_path);
320 }
321
322 { // Handle inflated monitor.
323 bind(inflated_load_mark);
324 ld(tmp1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
325 #ifdef ASSERT
326 test_bit(tmp3_t, tmp1_mark, exact_log2(markWord::monitor_value));
327 bnez(tmp3_t, inflated);
328 stop("Fast Unlock not monitor");
329 #endif
330
331 bind(inflated);
332
333 #ifdef ASSERT
334 Label check_done;
335 subw(tmp2_top, tmp2_top, oopSize);
336 mv(tmp3_t, in_bytes(JavaThread::lock_stack_base_offset()));
337 blt(tmp2_top, tmp3_t, check_done);
338 add(tmp3_t, xthread, tmp2_top);
339 ld(tmp3_t, Address(tmp3_t));
340 bne(obj, tmp3_t, inflated);
341 stop("Fast Unlock lock on stack");
342 bind(check_done);
343 #endif
344
345 const Register tmp1_monitor = tmp1;
346
347 if (!UseObjectMonitorTable) {
348 assert(tmp1_monitor == tmp1_mark, "should be the same here");
349 // Untag the monitor.
350 subi(tmp1_monitor, tmp1_mark, (int)markWord::monitor_value);
351 } else {
352 ld(tmp1_monitor, Address(box, BasicLock::object_monitor_cache_offset_in_bytes()));
353 // No valid pointer below alignof(ObjectMonitor*). Take the slow path.
354 mv(tmp3_t, alignof(ObjectMonitor*));
355 bltu(tmp1_monitor, tmp3_t, slow_path);
356 }
357
358 const Register tmp2_recursions = tmp2;
359 Label not_recursive;
360
361 // Check if recursive.
362 ld(tmp2_recursions, Address(tmp1_monitor, ObjectMonitor::recursions_offset()));
363 beqz(tmp2_recursions, not_recursive);
364
365 // Recursive unlock.
366 subi(tmp2_recursions, tmp2_recursions, 1);
367 sd(tmp2_recursions, Address(tmp1_monitor, ObjectMonitor::recursions_offset()));
368 j(unlocked);
369
370 bind(not_recursive);
371
372 const Register tmp2_owner_addr = tmp2;
373
374 // Compute owner address.
375 la(tmp2_owner_addr, Address(tmp1_monitor, ObjectMonitor::owner_offset()));
376
377 // Set owner to null.
378 // Release to satisfy the JMM
379 membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore);
380 sd(zr, Address(tmp2_owner_addr));
381 // We need a full fence after clearing owner to avoid stranding.
382 // StoreLoad achieves this.
383 membar(StoreLoad);
384
385 // Check if the entry_list is empty.
386 ld(t0, Address(tmp1_monitor, ObjectMonitor::entry_list_offset()));
387 beqz(t0, unlocked); // If so we are done.
388
389 // Check if there is a successor.
390 ld(tmp3_t, Address(tmp1_monitor, ObjectMonitor::succ_offset()));
391 bnez(tmp3_t, unlocked); // If so we are done.
392
393 // Save the monitor pointer in the current thread, so we can try
394 // to reacquire the lock in SharedRuntime::monitor_exit_helper().
395 sd(tmp1_monitor, Address(xthread, JavaThread::unlocked_inflated_monitor_offset()));
396
397 mv(flag, 1);
398 j(slow_path);
399 }
400
401 bind(unlocked);
402 mv(flag, zr);
403
404 #ifdef ASSERT
405 // Check that unlocked label is reached with flag == 0.
406 Label flag_correct;
407 beqz(flag, flag_correct);
408 stop("Fast Lock Flag != 0");
409 #endif
410
411 bind(slow_path);
412 #ifdef ASSERT
413 // Check that slow_path label is reached with flag != 0.
414 bnez(flag, flag_correct);
415 stop("Fast Lock Flag == 0");
416 bind(flag_correct);
417 #endif
418 // C2 uses the value of flag (0 vs !0) to determine the continuation.
419 }
420
421 // short string
422 // StringUTF16.indexOfChar
423 // StringLatin1.indexOfChar
424 void C2_MacroAssembler::string_indexof_char_short(Register str1, Register cnt1,
425 Register ch, Register result,
426 bool isL)
427 {
428 Register ch1 = t0;
429 Register index = t1;
430
431 BLOCK_COMMENT("string_indexof_char_short {");
432
433 Label LOOP, LOOP1, LOOP4, LOOP8;
434 Label MATCH, MATCH1, MATCH2, MATCH3,
435 MATCH4, MATCH5, MATCH6, MATCH7, NOMATCH;
436
437 mv(result, -1);
438 mv(index, zr);
439
440 bind(LOOP);
441 addi(t0, index, 8);
442 ble(t0, cnt1, LOOP8);
443 addi(t0, index, 4);
444 ble(t0, cnt1, LOOP4);
445 j(LOOP1);
446
447 bind(LOOP8);
448 isL ? lbu(ch1, Address(str1, 0)) : lhu(ch1, Address(str1, 0));
449 beq(ch, ch1, MATCH);
450 isL ? lbu(ch1, Address(str1, 1)) : lhu(ch1, Address(str1, 2));
451 beq(ch, ch1, MATCH1);
452 isL ? lbu(ch1, Address(str1, 2)) : lhu(ch1, Address(str1, 4));
453 beq(ch, ch1, MATCH2);
454 isL ? lbu(ch1, Address(str1, 3)) : lhu(ch1, Address(str1, 6));
455 beq(ch, ch1, MATCH3);
456 isL ? lbu(ch1, Address(str1, 4)) : lhu(ch1, Address(str1, 8));
457 beq(ch, ch1, MATCH4);
458 isL ? lbu(ch1, Address(str1, 5)) : lhu(ch1, Address(str1, 10));
459 beq(ch, ch1, MATCH5);
460 isL ? lbu(ch1, Address(str1, 6)) : lhu(ch1, Address(str1, 12));
461 beq(ch, ch1, MATCH6);
462 isL ? lbu(ch1, Address(str1, 7)) : lhu(ch1, Address(str1, 14));
463 beq(ch, ch1, MATCH7);
464 addi(index, index, 8);
465 addi(str1, str1, isL ? 8 : 16);
466 blt(index, cnt1, LOOP);
467 j(NOMATCH);
468
469 bind(LOOP4);
470 isL ? lbu(ch1, Address(str1, 0)) : lhu(ch1, Address(str1, 0));
471 beq(ch, ch1, MATCH);
472 isL ? lbu(ch1, Address(str1, 1)) : lhu(ch1, Address(str1, 2));
473 beq(ch, ch1, MATCH1);
474 isL ? lbu(ch1, Address(str1, 2)) : lhu(ch1, Address(str1, 4));
475 beq(ch, ch1, MATCH2);
476 isL ? lbu(ch1, Address(str1, 3)) : lhu(ch1, Address(str1, 6));
477 beq(ch, ch1, MATCH3);
478 addi(index, index, 4);
479 addi(str1, str1, isL ? 4 : 8);
480 bge(index, cnt1, NOMATCH);
481
482 bind(LOOP1);
483 isL ? lbu(ch1, Address(str1)) : lhu(ch1, Address(str1));
484 beq(ch, ch1, MATCH);
485 addi(index, index, 1);
486 addi(str1, str1, isL ? 1 : 2);
487 blt(index, cnt1, LOOP1);
488 j(NOMATCH);
489
490 bind(MATCH1);
491 addi(index, index, 1);
492 j(MATCH);
493
494 bind(MATCH2);
495 addi(index, index, 2);
496 j(MATCH);
497
498 bind(MATCH3);
499 addi(index, index, 3);
500 j(MATCH);
501
502 bind(MATCH4);
503 addi(index, index, 4);
504 j(MATCH);
505
506 bind(MATCH5);
507 addi(index, index, 5);
508 j(MATCH);
509
510 bind(MATCH6);
511 addi(index, index, 6);
512 j(MATCH);
513
514 bind(MATCH7);
515 addi(index, index, 7);
516
517 bind(MATCH);
518 mv(result, index);
519 bind(NOMATCH);
520 BLOCK_COMMENT("} string_indexof_char_short");
521 }
522
523 // StringUTF16.indexOfChar
524 // StringLatin1.indexOfChar
525 void C2_MacroAssembler::string_indexof_char(Register str1, Register cnt1,
526 Register ch, Register result,
527 Register tmp1, Register tmp2,
528 Register tmp3, Register tmp4,
529 bool isL)
530 {
531 Label CH1_LOOP, HIT, NOMATCH, DONE, DO_LONG;
532 Register ch1 = t0;
533 Register orig_cnt = t1;
534 Register mask1 = tmp3;
535 Register mask2 = tmp2;
536 Register match_mask = tmp1;
537 Register trailing_char = tmp4;
538 Register unaligned_elems = tmp4;
539
540 BLOCK_COMMENT("string_indexof_char {");
541 beqz(cnt1, NOMATCH);
542
543 subi(t0, cnt1, isL ? 32 : 16);
544 bgtz(t0, DO_LONG);
545 string_indexof_char_short(str1, cnt1, ch, result, isL);
546 j(DONE);
547
548 bind(DO_LONG);
549 mv(orig_cnt, cnt1);
550 if (AvoidUnalignedAccesses) {
551 Label ALIGNED;
552 andi(unaligned_elems, str1, 0x7);
553 beqz(unaligned_elems, ALIGNED);
554 sub(unaligned_elems, unaligned_elems, 8);
555 neg(unaligned_elems, unaligned_elems);
556 if (!isL) {
557 srli(unaligned_elems, unaligned_elems, 1);
558 }
559 // do unaligned part per element
560 string_indexof_char_short(str1, unaligned_elems, ch, result, isL);
561 bgez(result, DONE);
562 mv(orig_cnt, cnt1);
563 sub(cnt1, cnt1, unaligned_elems);
564 bind(ALIGNED);
565 }
566
567 // duplicate ch
568 if (isL) {
569 slli(ch1, ch, 8);
570 orr(ch, ch1, ch);
571 }
572 slli(ch1, ch, 16);
573 orr(ch, ch1, ch);
574 slli(ch1, ch, 32);
575 orr(ch, ch1, ch);
576
577 if (!isL) {
578 slli(cnt1, cnt1, 1);
579 }
580
581 uint64_t mask0101 = UCONST64(0x0101010101010101);
582 uint64_t mask0001 = UCONST64(0x0001000100010001);
583 mv(mask1, isL ? mask0101 : mask0001);
584 uint64_t mask7f7f = UCONST64(0x7f7f7f7f7f7f7f7f);
585 uint64_t mask7fff = UCONST64(0x7fff7fff7fff7fff);
586 mv(mask2, isL ? mask7f7f : mask7fff);
587
588 bind(CH1_LOOP);
589 ld(ch1, Address(str1));
590 addi(str1, str1, 8);
591 subi(cnt1, cnt1, 8);
592 compute_match_mask(ch1, ch, match_mask, mask1, mask2);
593 bnez(match_mask, HIT);
594 bgtz(cnt1, CH1_LOOP);
595 j(NOMATCH);
596
597 bind(HIT);
598 // count bits of trailing zero chars
599 ctzc_bits(trailing_char, match_mask, isL, ch1, result);
600 srli(trailing_char, trailing_char, 3);
601 addi(cnt1, cnt1, 8);
602 ble(cnt1, trailing_char, NOMATCH);
603 // match case
604 if (!isL) {
605 srli(cnt1, cnt1, 1);
606 srli(trailing_char, trailing_char, 1);
607 }
608
609 sub(result, orig_cnt, cnt1);
610 add(result, result, trailing_char);
611 j(DONE);
612
613 bind(NOMATCH);
614 mv(result, -1);
615
616 bind(DONE);
617 BLOCK_COMMENT("} string_indexof_char");
618 }
619
620 typedef void (MacroAssembler::* load_chr_insn)(Register rd, const Address &adr, Register temp);
621
622 // Search for needle in haystack and return index or -1
623 // x10: result
624 // x11: haystack
625 // x12: haystack_len
626 // x13: needle
627 // x14: needle_len
628 void C2_MacroAssembler::string_indexof(Register haystack, Register needle,
629 Register haystack_len, Register needle_len,
630 Register tmp1, Register tmp2,
631 Register tmp3, Register tmp4,
632 Register tmp5, Register tmp6,
633 Register result, int ae)
634 {
635 assert(ae != StrIntrinsicNode::LU, "Invalid encoding");
636
637 Label LINEARSEARCH, LINEARSTUB, DONE, NOMATCH;
638
639 Register ch1 = t0;
640 Register ch2 = t1;
641 Register nlen_tmp = tmp1; // needle len tmp
642 Register hlen_tmp = tmp2; // haystack len tmp
643 Register result_tmp = tmp4;
644
645 bool isLL = ae == StrIntrinsicNode::LL;
646
647 bool needle_isL = ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UL;
648 bool haystack_isL = ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::LU;
649 int needle_chr_shift = needle_isL ? 0 : 1;
650 int haystack_chr_shift = haystack_isL ? 0 : 1;
651 int needle_chr_size = needle_isL ? 1 : 2;
652 int haystack_chr_size = haystack_isL ? 1 : 2;
653 load_chr_insn needle_load_1chr = needle_isL ? (load_chr_insn)&MacroAssembler::lbu :
654 (load_chr_insn)&MacroAssembler::lhu;
655 load_chr_insn haystack_load_1chr = haystack_isL ? (load_chr_insn)&MacroAssembler::lbu :
656 (load_chr_insn)&MacroAssembler::lhu;
657
658 BLOCK_COMMENT("string_indexof {");
659
660 // Note, inline_string_indexOf() generates checks:
661 // if (pattern.count > src.count) return -1;
662 // if (pattern.count == 0) return 0;
663
664 // We have two strings, a source string in haystack, haystack_len and a pattern string
665 // in needle, needle_len. Find the first occurrence of pattern in source or return -1.
666
667 // For larger pattern and source we use a simplified Boyer Moore algorithm.
668 // With a small pattern and source we use linear scan.
669
670 // needle_len >=8 && needle_len < 256 && needle_len < haystack_len/4, use bmh algorithm.
671 sub(result_tmp, haystack_len, needle_len);
672 // needle_len < 8, use linear scan
673 sub(t0, needle_len, 8);
674 bltz(t0, LINEARSEARCH);
675 // needle_len >= 256, use linear scan
676 sub(t0, needle_len, 256);
677 bgez(t0, LINEARSTUB);
678 // needle_len >= haystack_len/4, use linear scan
679 srli(t0, haystack_len, 2);
680 bge(needle_len, t0, LINEARSTUB);
681
682 // Boyer-Moore-Horspool introduction:
683 // The Boyer Moore alogorithm is based on the description here:-
684 //
685 // http://en.wikipedia.org/wiki/Boyer%E2%80%93Moore_string_search_algorithm
686 //
687 // This describes and algorithm with 2 shift rules. The 'Bad Character' rule
688 // and the 'Good Suffix' rule.
689 //
690 // These rules are essentially heuristics for how far we can shift the
691 // pattern along the search string.
692 //
693 // The implementation here uses the 'Bad Character' rule only because of the
694 // complexity of initialisation for the 'Good Suffix' rule.
695 //
696 // This is also known as the Boyer-Moore-Horspool algorithm:
697 //
698 // http://en.wikipedia.org/wiki/Boyer-Moore-Horspool_algorithm
699 //
700 // #define ASIZE 256
701 //
702 // int bm(unsigned char *pattern, int m, unsigned char *src, int n) {
703 // int i, j;
704 // unsigned c;
705 // unsigned char bc[ASIZE];
706 //
707 // /* Preprocessing */
708 // for (i = 0; i < ASIZE; ++i)
709 // bc[i] = m;
710 // for (i = 0; i < m - 1; ) {
711 // c = pattern[i];
712 // ++i;
713 // // c < 256 for Latin1 string, so, no need for branch
714 // #ifdef PATTERN_STRING_IS_LATIN1
715 // bc[c] = m - i;
716 // #else
717 // if (c < ASIZE) bc[c] = m - i;
718 // #endif
719 // }
720 //
721 // /* Searching */
722 // j = 0;
723 // while (j <= n - m) {
724 // c = src[i+j];
725 // if (pattern[m-1] == c)
726 // int k;
727 // for (k = m - 2; k >= 0 && pattern[k] == src[k + j]; --k);
728 // if (k < 0) return j;
729 // // c < 256 for Latin1 string, so, no need for branch
730 // #ifdef SOURCE_STRING_IS_LATIN1_AND_PATTERN_STRING_IS_LATIN1
731 // // LL case: (c< 256) always true. Remove branch
732 // j += bc[pattern[j+m-1]];
733 // #endif
734 // #ifdef SOURCE_STRING_IS_UTF_AND_PATTERN_STRING_IS_UTF
735 // // UU case: need if (c<ASIZE) check. Skip 1 character if not.
736 // if (c < ASIZE)
737 // j += bc[pattern[j+m-1]];
738 // else
739 // j += 1
740 // #endif
741 // #ifdef SOURCE_IS_UTF_AND_PATTERN_IS_LATIN1
742 // // UL case: need if (c<ASIZE) check. Skip <pattern length> if not.
743 // if (c < ASIZE)
744 // j += bc[pattern[j+m-1]];
745 // else
746 // j += m
747 // #endif
748 // }
749 // return -1;
750 // }
751
752 // temp register:t0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, result
753 Label BCLOOP, BCSKIP, BMLOOPSTR2, BMLOOPSTR1, BMSKIP, BMADV, BMMATCH,
754 BMLOOPSTR1_LASTCMP, BMLOOPSTR1_CMP, BMLOOPSTR1_AFTER_LOAD, BM_INIT_LOOP;
755
756 Register haystack_end = haystack_len;
757 Register skipch = tmp2;
758
759 // pattern length is >=8, so, we can read at least 1 register for cases when
760 // UTF->Latin1 conversion is not needed(8 LL or 4UU) and half register for
761 // UL case. We'll re-read last character in inner pre-loop code to have
762 // single outer pre-loop load
763 const int firstStep = isLL ? 7 : 3;
764
765 const int ASIZE = 256;
766 const int STORE_BYTES = 8; // 8 bytes stored per instruction(sd)
767
768 subi(sp, sp, ASIZE);
769
770 // init BC offset table with default value: needle_len
771 slli(t0, needle_len, 8);
772 orr(t0, t0, needle_len); // [63...16][needle_len][needle_len]
773 slli(tmp1, t0, 16);
774 orr(t0, tmp1, t0); // [63...32][needle_len][needle_len][needle_len][needle_len]
775 slli(tmp1, t0, 32);
776 orr(tmp5, tmp1, t0); // tmp5: 8 elements [needle_len]
777
778 mv(ch1, sp); // ch1 is t0
779 mv(tmp6, ASIZE / STORE_BYTES); // loop iterations
780
781 bind(BM_INIT_LOOP);
782 // for (i = 0; i < ASIZE; ++i)
783 // bc[i] = m;
784 for (int i = 0; i < 4; i++) {
785 sd(tmp5, Address(ch1, i * wordSize));
786 }
787 addi(ch1, ch1, 32);
788 subi(tmp6, tmp6, 4);
789 bgtz(tmp6, BM_INIT_LOOP);
790
791 subi(nlen_tmp, needle_len, 1); // m - 1, index of the last element in pattern
792 Register orig_haystack = tmp5;
793 mv(orig_haystack, haystack);
794 // result_tmp = tmp4
795 shadd(haystack_end, result_tmp, haystack, haystack_end, haystack_chr_shift);
796 subi(ch2, needle_len, 1); // bc offset init value, ch2 is t1
797 mv(tmp3, needle);
798
799 // for (i = 0; i < m - 1; ) {
800 // c = pattern[i];
801 // ++i;
802 // // c < 256 for Latin1 string, so, no need for branch
803 // #ifdef PATTERN_STRING_IS_LATIN1
804 // bc[c] = m - i;
805 // #else
806 // if (c < ASIZE) bc[c] = m - i;
807 // #endif
808 // }
809 bind(BCLOOP);
810 (this->*needle_load_1chr)(ch1, Address(tmp3), noreg);
811 addi(tmp3, tmp3, needle_chr_size);
812 if (!needle_isL) {
813 // ae == StrIntrinsicNode::UU
814 mv(tmp6, ASIZE);
815 bgeu(ch1, tmp6, BCSKIP);
816 }
817 add(tmp4, sp, ch1);
818 sb(ch2, Address(tmp4)); // store skip offset to BC offset table
819
820 bind(BCSKIP);
821 subi(ch2, ch2, 1); // for next pattern element, skip distance -1
822 bgtz(ch2, BCLOOP);
823
824 // tmp6: pattern end, address after needle
825 shadd(tmp6, needle_len, needle, tmp6, needle_chr_shift);
826 if (needle_isL == haystack_isL) {
827 // load last 8 bytes (8LL/4UU symbols)
828 ld(tmp6, Address(tmp6, -wordSize));
829 } else {
830 // UL: from UTF-16(source) search Latin1(pattern)
831 lwu(tmp6, Address(tmp6, -wordSize / 2)); // load last 4 bytes(4 symbols)
832 // convert Latin1 to UTF. eg: 0x0000abcd -> 0x0a0b0c0d
833 // We'll have to wait until load completed, but it's still faster than per-character loads+checks
834 srli(tmp3, tmp6, BitsPerByte * (wordSize / 2 - needle_chr_size)); // pattern[m-1], eg:0x0000000a
835 slli(ch2, tmp6, XLEN - 24);
836 srli(ch2, ch2, XLEN - 8); // pattern[m-2], 0x0000000b
837 slli(ch1, tmp6, XLEN - 16);
838 srli(ch1, ch1, XLEN - 8); // pattern[m-3], 0x0000000c
839 zext(tmp6, tmp6, 8); // pattern[m-4], 0x0000000d
840 slli(ch2, ch2, 16);
841 orr(ch2, ch2, ch1); // 0x00000b0c
842 slli(result, tmp3, 48); // use result as temp register
843 orr(tmp6, tmp6, result); // 0x0a00000d
844 slli(result, ch2, 16);
845 orr(tmp6, tmp6, result); // UTF-16:0x0a0b0c0d
846 }
847
848 // i = m - 1;
849 // skipch = j + i;
850 // if (skipch == pattern[m - 1]
851 // for (k = m - 2; k >= 0 && pattern[k] == src[k + j]; --k);
852 // else
853 // move j with bad char offset table
854 bind(BMLOOPSTR2);
855 // compare pattern to source string backward
856 shadd(result, nlen_tmp, haystack, result, haystack_chr_shift);
857 (this->*haystack_load_1chr)(skipch, Address(result), noreg);
858 subi(nlen_tmp, nlen_tmp, firstStep); // nlen_tmp is positive here, because needle_len >= 8
859 if (needle_isL == haystack_isL) {
860 // re-init tmp3. It's for free because it's executed in parallel with
861 // load above. Alternative is to initialize it before loop, but it'll
862 // affect performance on in-order systems with 2 or more ld/st pipelines
863 srli(tmp3, tmp6, BitsPerByte * (wordSize - needle_chr_size)); // UU/LL: pattern[m-1]
864 }
865 if (!isLL) { // UU/UL case
866 slli(ch2, nlen_tmp, 1); // offsets in bytes
867 }
868 bne(tmp3, skipch, BMSKIP); // if not equal, skipch is bad char
869 add(result, haystack, isLL ? nlen_tmp : ch2);
870 // load 8 bytes from source string
871 // if isLL is false then read granularity can be 2
872 load_long_misaligned(ch2, Address(result), ch1, isLL ? 1 : 2); // can use ch1 as temp register here as it will be trashed by next mv anyway
873 mv(ch1, tmp6);
874 if (isLL) {
875 j(BMLOOPSTR1_AFTER_LOAD);
876 } else {
877 subi(nlen_tmp, nlen_tmp, 1); // no need to branch for UU/UL case. cnt1 >= 8
878 j(BMLOOPSTR1_CMP);
879 }
880
881 bind(BMLOOPSTR1);
882 shadd(ch1, nlen_tmp, needle, ch1, needle_chr_shift);
883 (this->*needle_load_1chr)(ch1, Address(ch1), noreg);
884 shadd(ch2, nlen_tmp, haystack, ch2, haystack_chr_shift);
885 (this->*haystack_load_1chr)(ch2, Address(ch2), noreg);
886
887 bind(BMLOOPSTR1_AFTER_LOAD);
888 subi(nlen_tmp, nlen_tmp, 1);
889 bltz(nlen_tmp, BMLOOPSTR1_LASTCMP);
890
891 bind(BMLOOPSTR1_CMP);
892 beq(ch1, ch2, BMLOOPSTR1);
893
894 bind(BMSKIP);
895 if (!isLL) {
896 // if we've met UTF symbol while searching Latin1 pattern, then we can
897 // skip needle_len symbols
898 if (needle_isL != haystack_isL) {
899 mv(result_tmp, needle_len);
900 } else {
901 mv(result_tmp, 1);
902 }
903 mv(t0, ASIZE);
904 bgeu(skipch, t0, BMADV);
905 }
906 add(result_tmp, sp, skipch);
907 lbu(result_tmp, Address(result_tmp)); // load skip offset
908
909 bind(BMADV);
910 subi(nlen_tmp, needle_len, 1);
911 // move haystack after bad char skip offset
912 shadd(haystack, result_tmp, haystack, result, haystack_chr_shift);
913 ble(haystack, haystack_end, BMLOOPSTR2);
914 addi(sp, sp, ASIZE);
915 j(NOMATCH);
916
917 bind(BMLOOPSTR1_LASTCMP);
918 bne(ch1, ch2, BMSKIP);
919
920 bind(BMMATCH);
921 sub(result, haystack, orig_haystack);
922 if (!haystack_isL) {
923 srli(result, result, 1);
924 }
925 addi(sp, sp, ASIZE);
926 j(DONE);
927
928 bind(LINEARSTUB);
929 subi(t0, needle_len, 16); // small patterns still should be handled by simple algorithm
930 bltz(t0, LINEARSEARCH);
931 mv(result, zr);
932 RuntimeAddress stub = nullptr;
933 if (isLL) {
934 stub = RuntimeAddress(StubRoutines::riscv::string_indexof_linear_ll());
935 assert(stub.target() != nullptr, "string_indexof_linear_ll stub has not been generated");
936 } else if (needle_isL) {
937 stub = RuntimeAddress(StubRoutines::riscv::string_indexof_linear_ul());
938 assert(stub.target() != nullptr, "string_indexof_linear_ul stub has not been generated");
939 } else {
940 stub = RuntimeAddress(StubRoutines::riscv::string_indexof_linear_uu());
941 assert(stub.target() != nullptr, "string_indexof_linear_uu stub has not been generated");
942 }
943 address call = reloc_call(stub);
944 if (call == nullptr) {
945 DEBUG_ONLY(reset_labels(LINEARSEARCH, DONE, NOMATCH));
946 ciEnv::current()->record_failure("CodeCache is full");
947 return;
948 }
949 j(DONE);
950
951 bind(NOMATCH);
952 mv(result, -1);
953 j(DONE);
954
955 bind(LINEARSEARCH);
956 string_indexof_linearscan(haystack, needle, haystack_len, needle_len, tmp1, tmp2, tmp3, tmp4, -1, result, ae);
957
958 bind(DONE);
959 BLOCK_COMMENT("} string_indexof");
960 }
961
962 // string_indexof
963 // result: x10
964 // src: x11
965 // src_count: x12
966 // pattern: x13
967 // pattern_count: x14 or 1/2/3/4
968 void C2_MacroAssembler::string_indexof_linearscan(Register haystack, Register needle,
969 Register haystack_len, Register needle_len,
970 Register tmp1, Register tmp2,
971 Register tmp3, Register tmp4,
972 int needle_con_cnt, Register result, int ae)
973 {
974 // Note:
975 // needle_con_cnt > 0 means needle_len register is invalid, needle length is constant
976 // for UU/LL: needle_con_cnt[1, 4], UL: needle_con_cnt = 1
977 assert(needle_con_cnt <= 4, "Invalid needle constant count");
978 assert(ae != StrIntrinsicNode::LU, "Invalid encoding");
979
980 Register ch1 = t0;
981 Register ch2 = t1;
982 Register hlen_neg = haystack_len, nlen_neg = needle_len;
983 Register nlen_tmp = tmp1, hlen_tmp = tmp2, result_tmp = tmp4;
984
985 bool isLL = ae == StrIntrinsicNode::LL;
986
987 bool needle_isL = ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UL;
988 bool haystack_isL = ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::LU;
989 int needle_chr_shift = needle_isL ? 0 : 1;
990 int haystack_chr_shift = haystack_isL ? 0 : 1;
991 int needle_chr_size = needle_isL ? 1 : 2;
992 int haystack_chr_size = haystack_isL ? 1 : 2;
993
994 load_chr_insn needle_load_1chr = needle_isL ? (load_chr_insn)&MacroAssembler::lbu :
995 (load_chr_insn)&MacroAssembler::lhu;
996 load_chr_insn haystack_load_1chr = haystack_isL ? (load_chr_insn)&MacroAssembler::lbu :
997 (load_chr_insn)&MacroAssembler::lhu;
998 load_chr_insn load_2chr = isLL ? (load_chr_insn)&MacroAssembler::lhu : (load_chr_insn)&MacroAssembler::lwu;
999 load_chr_insn load_4chr = isLL ? (load_chr_insn)&MacroAssembler::lwu : (load_chr_insn)&MacroAssembler::ld;
1000
1001 Label DO1, DO2, DO3, MATCH, NOMATCH, DONE;
1002
1003 Register first = tmp3;
1004
1005 if (needle_con_cnt == -1) {
1006 Label DOSHORT, FIRST_LOOP, STR2_NEXT, STR1_LOOP, STR1_NEXT;
1007
1008 subi(t0, needle_len, needle_isL == haystack_isL ? 4 : 2);
1009 bltz(t0, DOSHORT);
1010
1011 (this->*needle_load_1chr)(first, Address(needle), noreg);
1012 slli(t0, needle_len, needle_chr_shift);
1013 add(needle, needle, t0);
1014 neg(nlen_neg, t0);
1015 slli(t0, result_tmp, haystack_chr_shift);
1016 add(haystack, haystack, t0);
1017 neg(hlen_neg, t0);
1018
1019 bind(FIRST_LOOP);
1020 add(t0, haystack, hlen_neg);
1021 (this->*haystack_load_1chr)(ch2, Address(t0), noreg);
1022 beq(first, ch2, STR1_LOOP);
1023
1024 bind(STR2_NEXT);
1025 addi(hlen_neg, hlen_neg, haystack_chr_size);
1026 blez(hlen_neg, FIRST_LOOP);
1027 j(NOMATCH);
1028
1029 bind(STR1_LOOP);
1030 addi(nlen_tmp, nlen_neg, needle_chr_size);
1031 addi(hlen_tmp, hlen_neg, haystack_chr_size);
1032 bgez(nlen_tmp, MATCH);
1033
1034 bind(STR1_NEXT);
1035 add(ch1, needle, nlen_tmp);
1036 (this->*needle_load_1chr)(ch1, Address(ch1), noreg);
1037 add(ch2, haystack, hlen_tmp);
1038 (this->*haystack_load_1chr)(ch2, Address(ch2), noreg);
1039 bne(ch1, ch2, STR2_NEXT);
1040 addi(nlen_tmp, nlen_tmp, needle_chr_size);
1041 addi(hlen_tmp, hlen_tmp, haystack_chr_size);
1042 bltz(nlen_tmp, STR1_NEXT);
1043 j(MATCH);
1044
1045 bind(DOSHORT);
1046 if (needle_isL == haystack_isL) {
1047 subi(t0, needle_len, 2);
1048 bltz(t0, DO1);
1049 bgtz(t0, DO3);
1050 }
1051 }
1052
1053 if (needle_con_cnt == 4) {
1054 Label CH1_LOOP;
1055 (this->*load_4chr)(ch1, Address(needle), noreg);
1056 subi(result_tmp, haystack_len, 4);
1057 slli(tmp3, result_tmp, haystack_chr_shift); // result as tmp
1058 add(haystack, haystack, tmp3);
1059 neg(hlen_neg, tmp3);
1060 if (AvoidUnalignedAccesses) {
1061 // preload first value, then we will read by 1 character per loop, instead of four
1062 // just shifting previous ch2 right by size of character in bits
1063 add(tmp3, haystack, hlen_neg);
1064 (this->*load_4chr)(ch2, Address(tmp3), noreg);
1065 if (isLL) {
1066 // need to erase 1 most significant byte in 32-bit value of ch2
1067 slli(ch2, ch2, 40);
1068 srli(ch2, ch2, 32);
1069 } else {
1070 slli(ch2, ch2, 16); // 2 most significant bytes will be erased by this operation
1071 }
1072 }
1073
1074 bind(CH1_LOOP);
1075 add(tmp3, haystack, hlen_neg);
1076 if (AvoidUnalignedAccesses) {
1077 srli(ch2, ch2, isLL ? 8 : 16);
1078 (this->*haystack_load_1chr)(tmp3, Address(tmp3, isLL ? 3 : 6), noreg);
1079 slli(tmp3, tmp3, isLL ? 24 : 48);
1080 add(ch2, ch2, tmp3);
1081 } else {
1082 (this->*load_4chr)(ch2, Address(tmp3), noreg);
1083 }
1084 beq(ch1, ch2, MATCH);
1085 addi(hlen_neg, hlen_neg, haystack_chr_size);
1086 blez(hlen_neg, CH1_LOOP);
1087 j(NOMATCH);
1088 }
1089
1090 if ((needle_con_cnt == -1 && needle_isL == haystack_isL) || needle_con_cnt == 2) {
1091 Label CH1_LOOP;
1092 BLOCK_COMMENT("string_indexof DO2 {");
1093 bind(DO2);
1094 (this->*load_2chr)(ch1, Address(needle), noreg);
1095 if (needle_con_cnt == 2) {
1096 subi(result_tmp, haystack_len, 2);
1097 }
1098 slli(tmp3, result_tmp, haystack_chr_shift);
1099 add(haystack, haystack, tmp3);
1100 neg(hlen_neg, tmp3);
1101 if (AvoidUnalignedAccesses) {
1102 // preload first value, then we will read by 1 character per loop, instead of two
1103 // just shifting previous ch2 right by size of character in bits
1104 add(tmp3, haystack, hlen_neg);
1105 (this->*haystack_load_1chr)(ch2, Address(tmp3), noreg);
1106 slli(ch2, ch2, isLL ? 8 : 16);
1107 }
1108 bind(CH1_LOOP);
1109 add(tmp3, haystack, hlen_neg);
1110 if (AvoidUnalignedAccesses) {
1111 srli(ch2, ch2, isLL ? 8 : 16);
1112 (this->*haystack_load_1chr)(tmp3, Address(tmp3, isLL ? 1 : 2), noreg);
1113 slli(tmp3, tmp3, isLL ? 8 : 16);
1114 add(ch2, ch2, tmp3);
1115 } else {
1116 (this->*load_2chr)(ch2, Address(tmp3), noreg);
1117 }
1118 beq(ch1, ch2, MATCH);
1119 addi(hlen_neg, hlen_neg, haystack_chr_size);
1120 blez(hlen_neg, CH1_LOOP);
1121 j(NOMATCH);
1122 BLOCK_COMMENT("} string_indexof DO2");
1123 }
1124
1125 if ((needle_con_cnt == -1 && needle_isL == haystack_isL) || needle_con_cnt == 3) {
1126 Label FIRST_LOOP, STR2_NEXT, STR1_LOOP;
1127 BLOCK_COMMENT("string_indexof DO3 {");
1128
1129 bind(DO3);
1130 (this->*load_2chr)(first, Address(needle), noreg);
1131 (this->*needle_load_1chr)(ch1, Address(needle, 2 * needle_chr_size), noreg);
1132 if (needle_con_cnt == 3) {
1133 subi(result_tmp, haystack_len, 3);
1134 }
1135 slli(hlen_tmp, result_tmp, haystack_chr_shift);
1136 add(haystack, haystack, hlen_tmp);
1137 neg(hlen_neg, hlen_tmp);
1138
1139 bind(FIRST_LOOP);
1140 add(ch2, haystack, hlen_neg);
1141 if (AvoidUnalignedAccesses) {
1142 (this->*haystack_load_1chr)(tmp2, Address(ch2, isLL ? 1 : 2), noreg); // we need a temp register, we can safely use hlen_tmp here, which is a synonym for tmp2
1143 (this->*haystack_load_1chr)(ch2, Address(ch2), noreg);
1144 slli(tmp2, tmp2, isLL ? 8 : 16);
1145 add(ch2, ch2, tmp2);
1146 } else {
1147 (this->*load_2chr)(ch2, Address(ch2), noreg);
1148 }
1149 beq(first, ch2, STR1_LOOP);
1150
1151 bind(STR2_NEXT);
1152 addi(hlen_neg, hlen_neg, haystack_chr_size);
1153 blez(hlen_neg, FIRST_LOOP);
1154 j(NOMATCH);
1155
1156 bind(STR1_LOOP);
1157 addi(hlen_tmp, hlen_neg, 2 * haystack_chr_size);
1158 add(ch2, haystack, hlen_tmp);
1159 (this->*haystack_load_1chr)(ch2, Address(ch2), noreg);
1160 bne(ch1, ch2, STR2_NEXT);
1161 j(MATCH);
1162 BLOCK_COMMENT("} string_indexof DO3");
1163 }
1164
1165 if (needle_con_cnt == -1 || needle_con_cnt == 1) {
1166 Label DO1_LOOP;
1167
1168 BLOCK_COMMENT("string_indexof DO1 {");
1169 bind(DO1);
1170 (this->*needle_load_1chr)(ch1, Address(needle), noreg);
1171 subi(result_tmp, haystack_len, 1);
1172 slli(tmp3, result_tmp, haystack_chr_shift);
1173 add(haystack, haystack, tmp3);
1174 neg(hlen_neg, tmp3);
1175
1176 bind(DO1_LOOP);
1177 add(tmp3, haystack, hlen_neg);
1178 (this->*haystack_load_1chr)(ch2, Address(tmp3), noreg);
1179 beq(ch1, ch2, MATCH);
1180 addi(hlen_neg, hlen_neg, haystack_chr_size);
1181 blez(hlen_neg, DO1_LOOP);
1182 BLOCK_COMMENT("} string_indexof DO1");
1183 }
1184
1185 bind(NOMATCH);
1186 mv(result, -1);
1187 j(DONE);
1188
1189 bind(MATCH);
1190 srai(t0, hlen_neg, haystack_chr_shift);
1191 add(result, result_tmp, t0);
1192
1193 bind(DONE);
1194 }
1195
1196 // Compare longwords
1197 void C2_MacroAssembler::string_compare_long_same_encoding(Register result, Register str1, Register str2,
1198 const bool isLL, Register cnt1, Register cnt2,
1199 Register tmp1, Register tmp2, Register tmp3,
1200 const int STUB_THRESHOLD, Label *STUB, Label *SHORT_STRING, Label *DONE) {
1201 Label TAIL_CHECK, TAIL, NEXT_WORD, DIFFERENCE;
1202
1203 const int base_offset = arrayOopDesc::base_offset_in_bytes(T_BYTE);
1204 assert((base_offset % (UseCompactObjectHeaders ? 4 : 8)) == 0, "Must be");
1205
1206 const int minCharsInWord = isLL ? wordSize : wordSize / 2;
1207
1208 // load first parts of strings and finish initialization while loading
1209 beq(str1, str2, *DONE);
1210 // Alignment
1211 if (AvoidUnalignedAccesses && (base_offset % 8) != 0) {
1212 lwu(tmp1, Address(str1));
1213 lwu(tmp2, Address(str2));
1214 bne(tmp1, tmp2, DIFFERENCE);
1215 addi(str1, str1, 4);
1216 addi(str2, str2, 4);
1217 subi(cnt2, cnt2, minCharsInWord / 2);
1218
1219 // A very short string
1220 mv(t0, minCharsInWord);
1221 ble(cnt2, t0, *SHORT_STRING);
1222 }
1223 #ifdef ASSERT
1224 if (AvoidUnalignedAccesses) {
1225 Label align_ok;
1226 orr(t0, str1, str2);
1227 andi(t0, t0, 0x7);
1228 beqz(t0, align_ok);
1229 stop("bad alignment");
1230 bind(align_ok);
1231 }
1232 #endif
1233 // load 8 bytes once to compare
1234 ld(tmp1, Address(str1));
1235 ld(tmp2, Address(str2));
1236 mv(t0, STUB_THRESHOLD);
1237 bge(cnt2, t0, *STUB);
1238 subi(cnt2, cnt2, minCharsInWord);
1239 beqz(cnt2, TAIL_CHECK);
1240 // convert cnt2 from characters to bytes
1241 if (!isLL) {
1242 slli(cnt2, cnt2, 1);
1243 }
1244 add(str2, str2, cnt2);
1245 add(str1, str1, cnt2);
1246 sub(cnt2, zr, cnt2);
1247 addi(cnt2, cnt2, 8);
1248 bne(tmp1, tmp2, DIFFERENCE);
1249 bgez(cnt2, TAIL);
1250
1251 // main loop
1252 bind(NEXT_WORD);
1253 // 8-byte aligned loads when AvoidUnalignedAccesses is enabled
1254 add(t0, str1, cnt2);
1255 ld(tmp1, Address(t0));
1256 add(t0, str2, cnt2);
1257 ld(tmp2, Address(t0));
1258 addi(cnt2, cnt2, 8);
1259 bne(tmp1, tmp2, DIFFERENCE);
1260 bltz(cnt2, NEXT_WORD);
1261
1262 bind(TAIL);
1263 load_long_misaligned(tmp1, Address(str1), tmp3, isLL ? 1 : 2);
1264 load_long_misaligned(tmp2, Address(str2), tmp3, isLL ? 1 : 2);
1265
1266 bind(TAIL_CHECK);
1267 beq(tmp1, tmp2, *DONE);
1268
1269 // Find the first different characters in the longwords and
1270 // compute their difference.
1271 bind(DIFFERENCE);
1272 xorr(tmp3, tmp1, tmp2);
1273 // count bits of trailing zero chars
1274 ctzc_bits(result, tmp3, isLL);
1275 srl(tmp1, tmp1, result);
1276 srl(tmp2, tmp2, result);
1277 if (isLL) {
1278 zext(tmp1, tmp1, 8);
1279 zext(tmp2, tmp2, 8);
1280 } else {
1281 zext(tmp1, tmp1, 16);
1282 zext(tmp2, tmp2, 16);
1283 }
1284 sub(result, tmp1, tmp2);
1285
1286 j(*DONE);
1287 }
1288
1289 // Compare longwords
1290 void C2_MacroAssembler::string_compare_long_different_encoding(Register result, Register str1, Register str2,
1291 bool isLU, Register cnt1, Register cnt2,
1292 Register tmp1, Register tmp2, Register tmp3,
1293 const int STUB_THRESHOLD, Label *STUB, Label *DONE) {
1294 Label TAIL, NEXT_WORD, DIFFERENCE;
1295
1296 const int base_offset = arrayOopDesc::base_offset_in_bytes(T_BYTE);
1297 assert((base_offset % (UseCompactObjectHeaders ? 4 : 8)) == 0, "Must be");
1298
1299 Register strL = isLU ? str1 : str2;
1300 Register strU = isLU ? str2 : str1;
1301 Register tmpL = tmp1, tmpU = tmp2;
1302
1303 // load first parts of strings and finish initialization while loading
1304 mv(t0, STUB_THRESHOLD);
1305 bge(cnt2, t0, *STUB);
1306 lwu(tmpL, Address(strL));
1307 load_long_misaligned(tmpU, Address(strU), tmp3, (base_offset % 8) != 0 ? 4 : 8);
1308 subi(cnt2, cnt2, 4);
1309 add(strL, strL, cnt2);
1310 sub(cnt1, zr, cnt2);
1311 slli(cnt2, cnt2, 1);
1312 add(strU, strU, cnt2);
1313 inflate_lo32(tmp3, tmpL);
1314 mv(tmpL, tmp3);
1315 sub(cnt2, zr, cnt2);
1316 addi(cnt1, cnt1, 4);
1317 addi(cnt2, cnt2, 8);
1318 bne(tmpL, tmpU, DIFFERENCE);
1319 bgez(cnt2, TAIL);
1320
1321 // main loop
1322 bind(NEXT_WORD);
1323 add(t0, strL, cnt1);
1324 lwu(tmpL, Address(t0));
1325 add(t0, strU, cnt2);
1326 load_long_misaligned(tmpU, Address(t0), tmp3, (base_offset % 8) != 0 ? 4 : 8);
1327 addi(cnt1, cnt1, 4);
1328 inflate_lo32(tmp3, tmpL);
1329 mv(tmpL, tmp3);
1330 addi(cnt2, cnt2, 8);
1331 bne(tmpL, tmpU, DIFFERENCE);
1332 bltz(cnt2, NEXT_WORD);
1333
1334 bind(TAIL);
1335 load_int_misaligned(tmpL, Address(strL), tmp3, false);
1336 load_long_misaligned(tmpU, Address(strU), tmp3, 2);
1337 inflate_lo32(tmp3, tmpL);
1338 mv(tmpL, tmp3);
1339
1340 beq(tmpL, tmpU, *DONE);
1341
1342 // Find the first different characters in the longwords and
1343 // compute their difference.
1344 bind(DIFFERENCE);
1345 xorr(tmp3, tmpL, tmpU);
1346 // count bits of trailing zero chars
1347 ctzc_bits(result, tmp3);
1348 srl(tmpL, tmpL, result);
1349 srl(tmpU, tmpU, result);
1350 zext(tmpL, tmpL, 16);
1351 zext(tmpU, tmpU, 16);
1352 if (isLU) {
1353 sub(result, tmpL, tmpU);
1354 } else {
1355 sub(result, tmpU, tmpL);
1356 }
1357
1358 j(*DONE);
1359 }
1360
1361 // Compare strings.
1362 void C2_MacroAssembler::string_compare(Register str1, Register str2,
1363 Register cnt1, Register cnt2, Register result,
1364 Register tmp1, Register tmp2, Register tmp3,
1365 int ae)
1366 {
1367 Label DONE, SHORT_LOOP, SHORT_STRING, SHORT_LAST, STUB,
1368 SHORT_LOOP_TAIL, SHORT_LAST2, SHORT_LAST_INIT,
1369 SHORT_LOOP_START, L;
1370
1371 const int STUB_THRESHOLD = 64 + 8;
1372 bool isLL = ae == StrIntrinsicNode::LL;
1373 bool isLU = ae == StrIntrinsicNode::LU;
1374 bool isUL = ae == StrIntrinsicNode::UL;
1375
1376 bool str1_isL = isLL || isLU;
1377 bool str2_isL = isLL || isUL;
1378
1379 // for L strings, 1 byte for 1 character
1380 // for U strings, 2 bytes for 1 character
1381 int str1_chr_size = str1_isL ? 1 : 2;
1382 int str2_chr_size = str2_isL ? 1 : 2;
1383 int minCharsInWord = isLL ? wordSize : wordSize / 2;
1384
1385 load_chr_insn str1_load_chr = str1_isL ? (load_chr_insn)&MacroAssembler::lbu : (load_chr_insn)&MacroAssembler::lhu;
1386 load_chr_insn str2_load_chr = str2_isL ? (load_chr_insn)&MacroAssembler::lbu : (load_chr_insn)&MacroAssembler::lhu;
1387
1388 BLOCK_COMMENT("string_compare {");
1389
1390 // Bizarrely, the counts are passed in bytes, regardless of whether they
1391 // are L or U strings, however the result is always in characters.
1392 if (!str1_isL) {
1393 sraiw(cnt1, cnt1, 1);
1394 }
1395 if (!str2_isL) {
1396 sraiw(cnt2, cnt2, 1);
1397 }
1398
1399 // Compute the minimum of the string lengths and save the difference in result.
1400 sub(result, cnt1, cnt2);
1401 bgt(cnt1, cnt2, L);
1402 mv(cnt2, cnt1);
1403 bind(L);
1404
1405 // A very short string
1406 mv(t0, minCharsInWord);
1407 ble(cnt2, t0, SHORT_STRING);
1408
1409 // Compare longwords
1410 {
1411 if (str1_isL == str2_isL) { // LL or UU
1412 string_compare_long_same_encoding(result,
1413 str1, str2, isLL,
1414 cnt1, cnt2, tmp1, tmp2, tmp3,
1415 STUB_THRESHOLD, &STUB, &SHORT_STRING, &DONE);
1416 } else { // LU or UL
1417 string_compare_long_different_encoding(result,
1418 str1, str2, isLU,
1419 cnt1, cnt2, tmp1, tmp2, tmp3,
1420 STUB_THRESHOLD, &STUB, &DONE);
1421 }
1422 }
1423
1424 bind(STUB);
1425 RuntimeAddress stub = nullptr;
1426 switch (ae) {
1427 case StrIntrinsicNode::LL:
1428 stub = RuntimeAddress(StubRoutines::riscv::compare_long_string_LL());
1429 break;
1430 case StrIntrinsicNode::UU:
1431 stub = RuntimeAddress(StubRoutines::riscv::compare_long_string_UU());
1432 break;
1433 case StrIntrinsicNode::LU:
1434 stub = RuntimeAddress(StubRoutines::riscv::compare_long_string_LU());
1435 break;
1436 case StrIntrinsicNode::UL:
1437 stub = RuntimeAddress(StubRoutines::riscv::compare_long_string_UL());
1438 break;
1439 default:
1440 ShouldNotReachHere();
1441 }
1442 assert(stub.target() != nullptr, "compare_long_string stub has not been generated");
1443 address call = reloc_call(stub);
1444 if (call == nullptr) {
1445 DEBUG_ONLY(reset_labels(DONE, SHORT_LOOP, SHORT_STRING, SHORT_LAST, SHORT_LOOP_TAIL, SHORT_LAST2, SHORT_LAST_INIT, SHORT_LOOP_START));
1446 ciEnv::current()->record_failure("CodeCache is full");
1447 return;
1448 }
1449 j(DONE);
1450
1451 bind(SHORT_STRING);
1452 // Is the minimum length zero?
1453 beqz(cnt2, DONE);
1454 // arrange code to do most branches while loading and loading next characters
1455 // while comparing previous
1456 (this->*str1_load_chr)(tmp1, Address(str1), t0);
1457 addi(str1, str1, str1_chr_size);
1458 subi(cnt2, cnt2, 1);
1459 beqz(cnt2, SHORT_LAST_INIT);
1460 (this->*str2_load_chr)(cnt1, Address(str2), t0);
1461 addi(str2, str2, str2_chr_size);
1462 j(SHORT_LOOP_START);
1463 bind(SHORT_LOOP);
1464 subi(cnt2, cnt2, 1);
1465 beqz(cnt2, SHORT_LAST);
1466 bind(SHORT_LOOP_START);
1467 (this->*str1_load_chr)(tmp2, Address(str1), t0);
1468 addi(str1, str1, str1_chr_size);
1469 (this->*str2_load_chr)(t0, Address(str2), t0);
1470 addi(str2, str2, str2_chr_size);
1471 bne(tmp1, cnt1, SHORT_LOOP_TAIL);
1472 subi(cnt2, cnt2, 1);
1473 beqz(cnt2, SHORT_LAST2);
1474 (this->*str1_load_chr)(tmp1, Address(str1), t0);
1475 addi(str1, str1, str1_chr_size);
1476 (this->*str2_load_chr)(cnt1, Address(str2), t0);
1477 addi(str2, str2, str2_chr_size);
1478 beq(tmp2, t0, SHORT_LOOP);
1479 sub(result, tmp2, t0);
1480 j(DONE);
1481 bind(SHORT_LOOP_TAIL);
1482 sub(result, tmp1, cnt1);
1483 j(DONE);
1484 bind(SHORT_LAST2);
1485 beq(tmp2, t0, DONE);
1486 sub(result, tmp2, t0);
1487
1488 j(DONE);
1489 bind(SHORT_LAST_INIT);
1490 (this->*str2_load_chr)(cnt1, Address(str2), t0);
1491 addi(str2, str2, str2_chr_size);
1492 bind(SHORT_LAST);
1493 beq(tmp1, cnt1, DONE);
1494 sub(result, tmp1, cnt1);
1495
1496 bind(DONE);
1497
1498 BLOCK_COMMENT("} string_compare");
1499 }
1500
1501 void C2_MacroAssembler::arrays_equals(Register a1, Register a2,
1502 Register tmp1, Register tmp2, Register tmp3,
1503 Register result, int elem_size) {
1504 assert(elem_size == 1 || elem_size == 2, "must be char or byte");
1505 assert_different_registers(a1, a2, result, tmp1, tmp2, tmp3, t0);
1506
1507 int elem_per_word = wordSize / elem_size;
1508 int log_elem_size = exact_log2(elem_size);
1509 int length_offset = arrayOopDesc::length_offset_in_bytes();
1510 int base_offset = arrayOopDesc::base_offset_in_bytes(elem_size == 2 ? T_CHAR : T_BYTE);
1511
1512 assert((base_offset % (UseCompactObjectHeaders ? 4 : 8)) == 0, "Must be");
1513
1514 Register cnt1 = tmp3;
1515 Register cnt2 = tmp1; // cnt2 only used in array length compare
1516 Label DONE, SAME, NEXT_WORD, SHORT, TAIL03, TAIL01;
1517
1518 BLOCK_COMMENT("arrays_equals {");
1519
1520 // if (a1 == a2), return true
1521 beq(a1, a2, SAME);
1522
1523 mv(result, false);
1524 // if (a1 == nullptr || a2 == nullptr)
1525 // return false;
1526 beqz(a1, DONE);
1527 beqz(a2, DONE);
1528
1529 // if (a1.length != a2.length)
1530 // return false;
1531 lwu(cnt1, Address(a1, length_offset));
1532 lwu(cnt2, Address(a2, length_offset));
1533 bne(cnt1, cnt2, DONE);
1534
1535 la(a1, Address(a1, base_offset));
1536 la(a2, Address(a2, base_offset));
1537
1538 // Load 4 bytes once to compare for alignment before main loop.
1539 if (AvoidUnalignedAccesses && (base_offset % 8) != 0) {
1540 subi(cnt1, cnt1, elem_per_word / 2);
1541 bltz(cnt1, TAIL03);
1542 lwu(tmp1, Address(a1));
1543 lwu(tmp2, Address(a2));
1544 addi(a1, a1, 4);
1545 addi(a2, a2, 4);
1546 bne(tmp1, tmp2, DONE);
1547 }
1548
1549 // Check for short strings, i.e. smaller than wordSize.
1550 subi(cnt1, cnt1, elem_per_word);
1551 bltz(cnt1, SHORT);
1552
1553 #ifdef ASSERT
1554 if (AvoidUnalignedAccesses) {
1555 Label align_ok;
1556 orr(t0, a1, a2);
1557 andi(t0, t0, 0x7);
1558 beqz(t0, align_ok);
1559 stop("bad alignment");
1560 bind(align_ok);
1561 }
1562 #endif
1563
1564 // Main 8 byte comparison loop.
1565 bind(NEXT_WORD); {
1566 ld(tmp1, Address(a1));
1567 ld(tmp2, Address(a2));
1568 subi(cnt1, cnt1, elem_per_word);
1569 addi(a1, a1, wordSize);
1570 addi(a2, a2, wordSize);
1571 bne(tmp1, tmp2, DONE);
1572 } bgez(cnt1, NEXT_WORD);
1573
1574 addi(tmp1, cnt1, elem_per_word);
1575 beqz(tmp1, SAME);
1576
1577 bind(SHORT);
1578 test_bit(tmp1, cnt1, 2 - log_elem_size);
1579 beqz(tmp1, TAIL03); // 0-7 bytes left.
1580 {
1581 lwu(tmp1, Address(a1));
1582 lwu(tmp2, Address(a2));
1583 addi(a1, a1, 4);
1584 addi(a2, a2, 4);
1585 bne(tmp1, tmp2, DONE);
1586 }
1587
1588 bind(TAIL03);
1589 test_bit(tmp1, cnt1, 1 - log_elem_size);
1590 beqz(tmp1, TAIL01); // 0-3 bytes left.
1591 {
1592 lhu(tmp1, Address(a1));
1593 lhu(tmp2, Address(a2));
1594 addi(a1, a1, 2);
1595 addi(a2, a2, 2);
1596 bne(tmp1, tmp2, DONE);
1597 }
1598
1599 bind(TAIL01);
1600 if (elem_size == 1) { // Only needed when comparing byte arrays.
1601 test_bit(tmp1, cnt1, 0);
1602 beqz(tmp1, SAME); // 0-1 bytes left.
1603 {
1604 lbu(tmp1, Address(a1));
1605 lbu(tmp2, Address(a2));
1606 bne(tmp1, tmp2, DONE);
1607 }
1608 }
1609
1610 bind(SAME);
1611 mv(result, true);
1612 // That's it.
1613 bind(DONE);
1614
1615 BLOCK_COMMENT("} arrays_equals");
1616 }
1617
1618 // Compare Strings
1619
1620 // For Strings we're passed the address of the first characters in a1 and a2
1621 // and the length in cnt1. There are two implementations.
1622 // For arrays >= 8 bytes, all comparisons (except for the tail) are performed
1623 // 8 bytes at a time. For the tail, we compare a halfword, then a short, and then a byte.
1624 // For strings < 8 bytes, we compare a halfword, then a short, and then a byte.
1625
1626 void C2_MacroAssembler::string_equals(Register a1, Register a2,
1627 Register result, Register cnt1)
1628 {
1629 Label SAME, DONE, SHORT, NEXT_WORD, TAIL03, TAIL01;
1630 Register tmp1 = t0;
1631 Register tmp2 = t1;
1632
1633 assert_different_registers(a1, a2, result, cnt1, tmp1, tmp2);
1634
1635 int base_offset = arrayOopDesc::base_offset_in_bytes(T_BYTE);
1636
1637 assert((base_offset % (UseCompactObjectHeaders ? 4 : 8)) == 0, "Must be");
1638
1639 BLOCK_COMMENT("string_equals {");
1640
1641 mv(result, false);
1642
1643 // Load 4 bytes once to compare for alignment before main loop.
1644 if (AvoidUnalignedAccesses && (base_offset % 8) != 0) {
1645 subi(cnt1, cnt1, 4);
1646 bltz(cnt1, TAIL03);
1647 lwu(tmp1, Address(a1));
1648 lwu(tmp2, Address(a2));
1649 addi(a1, a1, 4);
1650 addi(a2, a2, 4);
1651 bne(tmp1, tmp2, DONE);
1652 }
1653
1654 // Check for short strings, i.e. smaller than wordSize.
1655 subi(cnt1, cnt1, wordSize);
1656 bltz(cnt1, SHORT);
1657
1658 #ifdef ASSERT
1659 if (AvoidUnalignedAccesses) {
1660 Label align_ok;
1661 orr(t0, a1, a2);
1662 andi(t0, t0, 0x7);
1663 beqz(t0, align_ok);
1664 stop("bad alignment");
1665 bind(align_ok);
1666 }
1667 #endif
1668
1669 // Main 8 byte comparison loop.
1670 bind(NEXT_WORD); {
1671 ld(tmp1, Address(a1));
1672 ld(tmp2, Address(a2));
1673 subi(cnt1, cnt1, wordSize);
1674 addi(a1, a1, wordSize);
1675 addi(a2, a2, wordSize);
1676 bne(tmp1, tmp2, DONE);
1677 } bgez(cnt1, NEXT_WORD);
1678
1679 addi(tmp1, cnt1, wordSize);
1680 beqz(tmp1, SAME);
1681
1682 bind(SHORT);
1683 // 0-7 bytes left.
1684 test_bit(tmp1, cnt1, 2);
1685 beqz(tmp1, TAIL03);
1686 {
1687 lwu(tmp1, Address(a1));
1688 lwu(tmp2, Address(a2));
1689 addi(a1, a1, 4);
1690 addi(a2, a2, 4);
1691 bne(tmp1, tmp2, DONE);
1692 }
1693
1694 bind(TAIL03);
1695 // 0-3 bytes left.
1696 test_bit(tmp1, cnt1, 1);
1697 beqz(tmp1, TAIL01);
1698 {
1699 lhu(tmp1, Address(a1));
1700 lhu(tmp2, Address(a2));
1701 addi(a1, a1, 2);
1702 addi(a2, a2, 2);
1703 bne(tmp1, tmp2, DONE);
1704 }
1705
1706 bind(TAIL01);
1707 // 0-1 bytes left.
1708 test_bit(tmp1, cnt1, 0);
1709 beqz(tmp1, SAME);
1710 {
1711 lbu(tmp1, Address(a1));
1712 lbu(tmp2, Address(a2));
1713 bne(tmp1, tmp2, DONE);
1714 }
1715
1716 // Arrays are equal.
1717 bind(SAME);
1718 mv(result, true);
1719
1720 // That's it.
1721 bind(DONE);
1722 BLOCK_COMMENT("} string_equals");
1723 }
1724
1725 // jdk.internal.util.ArraysSupport.vectorizedHashCode
1726 void C2_MacroAssembler::arrays_hashcode(Register ary, Register cnt, Register result,
1727 Register tmp1, Register tmp2, Register tmp3,
1728 Register tmp4, Register tmp5, Register tmp6,
1729 BasicType eltype)
1730 {
1731 assert(!UseRVV, "sanity");
1732 assert_different_registers(ary, cnt, result, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, t0, t1);
1733
1734 const int elsize = arrays_hashcode_elsize(eltype);
1735 const int chunks_end_shift = exact_log2(elsize);
1736
1737 switch (eltype) {
1738 case T_BOOLEAN: BLOCK_COMMENT("arrays_hashcode(unsigned byte) {"); break;
1739 case T_CHAR: BLOCK_COMMENT("arrays_hashcode(char) {"); break;
1740 case T_BYTE: BLOCK_COMMENT("arrays_hashcode(byte) {"); break;
1741 case T_SHORT: BLOCK_COMMENT("arrays_hashcode(short) {"); break;
1742 case T_INT: BLOCK_COMMENT("arrays_hashcode(int) {"); break;
1743 default:
1744 ShouldNotReachHere();
1745 }
1746
1747 const int stride = 4;
1748 const Register pow31_4 = tmp1;
1749 const Register pow31_3 = tmp2;
1750 const Register pow31_2 = tmp3;
1751 const Register chunks = tmp4;
1752 const Register chunks_end = chunks;
1753
1754 Label DONE, TAIL, TAIL_LOOP, WIDE_LOOP;
1755
1756 // result has a value initially
1757
1758 beqz(cnt, DONE);
1759
1760 andi(chunks, cnt, ~(stride - 1));
1761 beqz(chunks, TAIL);
1762
1763 mv(pow31_4, 923521); // [31^^4]
1764 mv(pow31_3, 29791); // [31^^3]
1765 mv(pow31_2, 961); // [31^^2]
1766
1767 shadd(chunks_end, chunks, ary, t0, chunks_end_shift);
1768 andi(cnt, cnt, stride - 1); // don't forget about tail!
1769
1770 bind(WIDE_LOOP);
1771 arrays_hashcode_elload(t0, Address(ary, 0 * elsize), eltype);
1772 arrays_hashcode_elload(t1, Address(ary, 1 * elsize), eltype);
1773 arrays_hashcode_elload(tmp5, Address(ary, 2 * elsize), eltype);
1774 arrays_hashcode_elload(tmp6, Address(ary, 3 * elsize), eltype);
1775 mulw(result, result, pow31_4); // 31^^4 * h
1776 mulw(t0, t0, pow31_3); // 31^^3 * ary[i+0]
1777 addw(result, result, t0);
1778 mulw(t1, t1, pow31_2); // 31^^2 * ary[i+1]
1779 addw(result, result, t1);
1780 slli(t0, tmp5, 5); // optimize 31^^1 * ary[i+2]
1781 subw(tmp5, t0, tmp5); // with ary[i+2]<<5 - ary[i+2]
1782 addw(result, result, tmp5);
1783 addw(result, result, tmp6); // 31^^4 * h + 31^^3 * ary[i+0] + 31^^2 * ary[i+1]
1784 // + 31^^1 * ary[i+2] + 31^^0 * ary[i+3]
1785 addi(ary, ary, elsize * stride);
1786 bne(ary, chunks_end, WIDE_LOOP);
1787 beqz(cnt, DONE);
1788
1789 bind(TAIL);
1790 shadd(chunks_end, cnt, ary, t0, chunks_end_shift);
1791
1792 bind(TAIL_LOOP);
1793 arrays_hashcode_elload(t0, Address(ary), eltype);
1794 slli(t1, result, 5); // optimize 31 * result
1795 subw(result, t1, result); // with result<<5 - result
1796 addw(result, result, t0);
1797 addi(ary, ary, elsize);
1798 bne(ary, chunks_end, TAIL_LOOP);
1799
1800 bind(DONE);
1801 BLOCK_COMMENT("} // arrays_hashcode");
1802 }
1803
1804 void C2_MacroAssembler::arrays_hashcode_v(Register ary, Register cnt, Register result,
1805 Register tmp1, Register tmp2, Register tmp3,
1806 BasicType eltype)
1807 {
1808 assert(UseRVV, "sanity");
1809 assert(StubRoutines::riscv::arrays_hashcode_powers_of_31() != nullptr, "sanity");
1810 assert_different_registers(ary, cnt, result, tmp1, tmp2, tmp3, t0, t1);
1811
1812 // The MaxVectorSize should have been set by detecting RVV max vector register
1813 // size when check UseRVV (i.e. MaxVectorSize == VM_Version::_initial_vector_length).
1814 // Let's use T_INT as all hashCode calculations eventually deal with ints.
1815 const int lmul = 2;
1816 const int stride = MaxVectorSize / sizeof(jint) * lmul;
1817
1818 const int elsize_bytes = arrays_hashcode_elsize(eltype);
1819 const int elsize_shift = exact_log2(elsize_bytes);
1820
1821 switch (eltype) {
1822 case T_BOOLEAN: BLOCK_COMMENT("arrays_hashcode_v(unsigned byte) {"); break;
1823 case T_CHAR: BLOCK_COMMENT("arrays_hashcode_v(char) {"); break;
1824 case T_BYTE: BLOCK_COMMENT("arrays_hashcode_v(byte) {"); break;
1825 case T_SHORT: BLOCK_COMMENT("arrays_hashcode_v(short) {"); break;
1826 case T_INT: BLOCK_COMMENT("arrays_hashcode_v(int) {"); break;
1827 default:
1828 ShouldNotReachHere();
1829 }
1830
1831 const Register pow31_highest = tmp1;
1832 const Register ary_end = tmp2;
1833 const Register consumed = tmp3;
1834
1835 const VectorRegister v_sum = v2;
1836 const VectorRegister v_src = v4;
1837 const VectorRegister v_coeffs = v6;
1838 const VectorRegister v_tmp = v8;
1839
1840 const address adr_pows31 = StubRoutines::riscv::arrays_hashcode_powers_of_31()
1841 + sizeof(jint);
1842 Label VEC_LOOP, DONE, SCALAR_TAIL, SCALAR_TAIL_LOOP;
1843
1844 // NB: at this point (a) 'result' already has some value,
1845 // (b) 'cnt' is not 0 or 1, see java code for details.
1846
1847 andi(t0, cnt, ~(stride - 1));
1848 beqz(t0, SCALAR_TAIL);
1849
1850 la(t1, ExternalAddress(adr_pows31));
1851 lw(pow31_highest, Address(t1, -1 * sizeof(jint)));
1852
1853 vsetvli(consumed, cnt, Assembler::e32, Assembler::m2);
1854 vle32_v(v_coeffs, t1); // 31^^(stride - 1) ... 31^^0
1855 vmv_v_x(v_sum, x0);
1856
1857 bind(VEC_LOOP);
1858 arrays_hashcode_elload_v(v_src, v_tmp, ary, eltype);
1859 vmul_vv(v_src, v_src, v_coeffs);
1860 vmadd_vx(v_sum, pow31_highest, v_src);
1861 mulw(result, result, pow31_highest);
1862 shadd(ary, consumed, ary, t0, elsize_shift);
1863 subw(cnt, cnt, consumed);
1864 andi(t1, cnt, ~(stride - 1));
1865 bnez(t1, VEC_LOOP);
1866
1867 vmv_s_x(v_tmp, x0);
1868 vredsum_vs(v_sum, v_sum, v_tmp);
1869 vmv_x_s(t0, v_sum);
1870 addw(result, result, t0);
1871 beqz(cnt, DONE);
1872
1873 bind(SCALAR_TAIL);
1874 shadd(ary_end, cnt, ary, t0, elsize_shift);
1875
1876 bind(SCALAR_TAIL_LOOP);
1877 arrays_hashcode_elload(t0, Address(ary), eltype);
1878 slli(t1, result, 5); // optimize 31 * result
1879 subw(result, t1, result); // with result<<5 - result
1880 addw(result, result, t0);
1881 addi(ary, ary, elsize_bytes);
1882 bne(ary, ary_end, SCALAR_TAIL_LOOP);
1883
1884 bind(DONE);
1885 BLOCK_COMMENT("} // arrays_hashcode_v");
1886 }
1887
1888 int C2_MacroAssembler::arrays_hashcode_elsize(BasicType eltype) {
1889 switch (eltype) {
1890 case T_BOOLEAN: return sizeof(jboolean);
1891 case T_BYTE: return sizeof(jbyte);
1892 case T_SHORT: return sizeof(jshort);
1893 case T_CHAR: return sizeof(jchar);
1894 case T_INT: return sizeof(jint);
1895 default:
1896 ShouldNotReachHere();
1897 return -1;
1898 }
1899 }
1900
1901 void C2_MacroAssembler::arrays_hashcode_elload(Register dst, Address src, BasicType eltype) {
1902 switch (eltype) {
1903 // T_BOOLEAN used as surrogate for unsigned byte
1904 case T_BOOLEAN: lbu(dst, src); break;
1905 case T_BYTE: lb(dst, src); break;
1906 case T_SHORT: lh(dst, src); break;
1907 case T_CHAR: lhu(dst, src); break;
1908 case T_INT: lw(dst, src); break;
1909 default:
1910 ShouldNotReachHere();
1911 }
1912 }
1913
1914 void C2_MacroAssembler::arrays_hashcode_elload_v(VectorRegister vdst,
1915 VectorRegister vtmp,
1916 Register src,
1917 BasicType eltype) {
1918 assert_different_registers(vdst, vtmp);
1919 switch (eltype) {
1920 case T_BOOLEAN:
1921 vle8_v(vtmp, src);
1922 vzext_vf4(vdst, vtmp);
1923 break;
1924 case T_BYTE:
1925 vle8_v(vtmp, src);
1926 vsext_vf4(vdst, vtmp);
1927 break;
1928 case T_CHAR:
1929 vle16_v(vtmp, src);
1930 vzext_vf2(vdst, vtmp);
1931 break;
1932 case T_SHORT:
1933 vle16_v(vtmp, src);
1934 vsext_vf2(vdst, vtmp);
1935 break;
1936 case T_INT:
1937 vle32_v(vdst, src);
1938 break;
1939 default:
1940 ShouldNotReachHere();
1941 }
1942 }
1943
1944 typedef void (Assembler::*conditional_branch_insn)(Register op1, Register op2, Label& label, bool is_far);
1945 typedef void (MacroAssembler::*float_conditional_branch_insn)(FloatRegister op1, FloatRegister op2, Label& label,
1946 bool is_far, bool is_unordered);
1947
1948 static conditional_branch_insn conditional_branches[] =
1949 {
1950 /* SHORT branches */
1951 (conditional_branch_insn)&MacroAssembler::beq,
1952 (conditional_branch_insn)&MacroAssembler::bgt,
1953 nullptr, // BoolTest::overflow
1954 (conditional_branch_insn)&MacroAssembler::blt,
1955 (conditional_branch_insn)&MacroAssembler::bne,
1956 (conditional_branch_insn)&MacroAssembler::ble,
1957 nullptr, // BoolTest::no_overflow
1958 (conditional_branch_insn)&MacroAssembler::bge,
1959
1960 /* UNSIGNED branches */
1961 (conditional_branch_insn)&MacroAssembler::beq,
1962 (conditional_branch_insn)&MacroAssembler::bgtu,
1963 nullptr,
1964 (conditional_branch_insn)&MacroAssembler::bltu,
1965 (conditional_branch_insn)&MacroAssembler::bne,
1966 (conditional_branch_insn)&MacroAssembler::bleu,
1967 nullptr,
1968 (conditional_branch_insn)&MacroAssembler::bgeu
1969 };
1970
1971 static float_conditional_branch_insn float_conditional_branches[] =
1972 {
1973 /* FLOAT SHORT branches */
1974 (float_conditional_branch_insn)&MacroAssembler::float_beq,
1975 (float_conditional_branch_insn)&MacroAssembler::float_bgt,
1976 nullptr, // BoolTest::overflow
1977 (float_conditional_branch_insn)&MacroAssembler::float_blt,
1978 (float_conditional_branch_insn)&MacroAssembler::float_bne,
1979 (float_conditional_branch_insn)&MacroAssembler::float_ble,
1980 nullptr, // BoolTest::no_overflow
1981 (float_conditional_branch_insn)&MacroAssembler::float_bge,
1982
1983 /* DOUBLE SHORT branches */
1984 (float_conditional_branch_insn)&MacroAssembler::double_beq,
1985 (float_conditional_branch_insn)&MacroAssembler::double_bgt,
1986 nullptr,
1987 (float_conditional_branch_insn)&MacroAssembler::double_blt,
1988 (float_conditional_branch_insn)&MacroAssembler::double_bne,
1989 (float_conditional_branch_insn)&MacroAssembler::double_ble,
1990 nullptr,
1991 (float_conditional_branch_insn)&MacroAssembler::double_bge
1992 };
1993
1994 void C2_MacroAssembler::cmp_branch(int cmpFlag, Register op1, Register op2, Label& label, bool is_far) {
1995 assert(cmpFlag >= 0 && cmpFlag < (int)(sizeof(conditional_branches) / sizeof(conditional_branches[0])),
1996 "invalid conditional branch index");
1997 (this->*conditional_branches[cmpFlag])(op1, op2, label, is_far);
1998 }
1999
2000 // This is a function should only be used by C2. Flip the unordered when unordered-greater, C2 would use
2001 // unordered-lesser instead of unordered-greater. Finally, commute the result bits at function do_one_bytecode().
2002 void C2_MacroAssembler::float_cmp_branch(int cmpFlag, FloatRegister op1, FloatRegister op2, Label& label, bool is_far) {
2003 assert(cmpFlag >= 0 && cmpFlag < (int)(sizeof(float_conditional_branches) / sizeof(float_conditional_branches[0])),
2004 "invalid float conditional branch index");
2005 int booltest_flag = cmpFlag & ~(C2_MacroAssembler::double_branch_mask);
2006 (this->*float_conditional_branches[cmpFlag])(op1, op2, label, is_far,
2007 (booltest_flag == (BoolTest::ge) || booltest_flag == (BoolTest::gt)) ? false : true);
2008 }
2009
2010 void C2_MacroAssembler::enc_cmpUEqNeLeGt_imm0_branch(int cmpFlag, Register op1, Label& L, bool is_far) {
2011 switch (cmpFlag) {
2012 case BoolTest::eq:
2013 case BoolTest::le:
2014 beqz(op1, L, is_far);
2015 break;
2016 case BoolTest::ne:
2017 case BoolTest::gt:
2018 bnez(op1, L, is_far);
2019 break;
2020 default:
2021 ShouldNotReachHere();
2022 }
2023 }
2024
2025 void C2_MacroAssembler::enc_cmpEqNe_imm0_branch(int cmpFlag, Register op1, Label& L, bool is_far) {
2026 switch (cmpFlag) {
2027 case BoolTest::eq:
2028 beqz(op1, L, is_far);
2029 break;
2030 case BoolTest::ne:
2031 bnez(op1, L, is_far);
2032 break;
2033 default:
2034 ShouldNotReachHere();
2035 }
2036 }
2037
2038 void C2_MacroAssembler::enc_cmove(int cmpFlag, Register op1, Register op2, Register dst, Register src) {
2039 bool is_unsigned = (cmpFlag & unsigned_branch_mask) == unsigned_branch_mask;
2040 int op_select = cmpFlag & (~unsigned_branch_mask);
2041
2042 switch (op_select) {
2043 case BoolTest::eq:
2044 cmov_eq(op1, op2, dst, src);
2045 break;
2046 case BoolTest::ne:
2047 cmov_ne(op1, op2, dst, src);
2048 break;
2049 case BoolTest::le:
2050 if (is_unsigned) {
2051 cmov_leu(op1, op2, dst, src);
2052 } else {
2053 cmov_le(op1, op2, dst, src);
2054 }
2055 break;
2056 case BoolTest::ge:
2057 if (is_unsigned) {
2058 cmov_geu(op1, op2, dst, src);
2059 } else {
2060 cmov_ge(op1, op2, dst, src);
2061 }
2062 break;
2063 case BoolTest::lt:
2064 if (is_unsigned) {
2065 cmov_ltu(op1, op2, dst, src);
2066 } else {
2067 cmov_lt(op1, op2, dst, src);
2068 }
2069 break;
2070 case BoolTest::gt:
2071 if (is_unsigned) {
2072 cmov_gtu(op1, op2, dst, src);
2073 } else {
2074 cmov_gt(op1, op2, dst, src);
2075 }
2076 break;
2077 default:
2078 assert(false, "unsupported compare condition");
2079 ShouldNotReachHere();
2080 }
2081 }
2082
2083 void C2_MacroAssembler::enc_cmove_cmp_fp(int cmpFlag, FloatRegister op1, FloatRegister op2, Register dst, Register src, bool is_single) {
2084 int op_select = cmpFlag & (~unsigned_branch_mask);
2085
2086 switch (op_select) {
2087 case BoolTest::eq:
2088 cmov_cmp_fp_eq(op1, op2, dst, src, is_single);
2089 break;
2090 case BoolTest::ne:
2091 cmov_cmp_fp_ne(op1, op2, dst, src, is_single);
2092 break;
2093 case BoolTest::le:
2094 cmov_cmp_fp_le(op1, op2, dst, src, is_single);
2095 break;
2096 case BoolTest::ge:
2097 cmov_cmp_fp_ge(op1, op2, dst, src, is_single);
2098 break;
2099 case BoolTest::lt:
2100 cmov_cmp_fp_lt(op1, op2, dst, src, is_single);
2101 break;
2102 case BoolTest::gt:
2103 cmov_cmp_fp_gt(op1, op2, dst, src, is_single);
2104 break;
2105 default:
2106 assert(false, "unsupported compare condition");
2107 ShouldNotReachHere();
2108 }
2109 }
2110
2111 void C2_MacroAssembler::enc_cmove_fp_cmp(int cmpFlag, Register op1, Register op2,
2112 FloatRegister dst, FloatRegister src, bool is_single) {
2113 bool is_unsigned = (cmpFlag & unsigned_branch_mask) == unsigned_branch_mask;
2114 int op_select = cmpFlag & (~unsigned_branch_mask);
2115
2116 switch (op_select) {
2117 case BoolTest::eq:
2118 cmov_fp_eq(op1, op2, dst, src, is_single);
2119 break;
2120 case BoolTest::ne:
2121 cmov_fp_ne(op1, op2, dst, src, is_single);
2122 break;
2123 case BoolTest::le:
2124 if (is_unsigned) {
2125 cmov_fp_leu(op1, op2, dst, src, is_single);
2126 } else {
2127 cmov_fp_le(op1, op2, dst, src, is_single);
2128 }
2129 break;
2130 case BoolTest::ge:
2131 if (is_unsigned) {
2132 cmov_fp_geu(op1, op2, dst, src, is_single);
2133 } else {
2134 cmov_fp_ge(op1, op2, dst, src, is_single);
2135 }
2136 break;
2137 case BoolTest::lt:
2138 if (is_unsigned) {
2139 cmov_fp_ltu(op1, op2, dst, src, is_single);
2140 } else {
2141 cmov_fp_lt(op1, op2, dst, src, is_single);
2142 }
2143 break;
2144 case BoolTest::gt:
2145 if (is_unsigned) {
2146 cmov_fp_gtu(op1, op2, dst, src, is_single);
2147 } else {
2148 cmov_fp_gt(op1, op2, dst, src, is_single);
2149 }
2150 break;
2151 default:
2152 assert(false, "unsupported compare condition");
2153 ShouldNotReachHere();
2154 }
2155 }
2156
2157 void C2_MacroAssembler::enc_cmove_fp_cmp_fp(int cmpFlag,
2158 FloatRegister op1, FloatRegister op2,
2159 FloatRegister dst, FloatRegister src,
2160 bool cmp_single, bool cmov_single) {
2161 int op_select = cmpFlag & (~unsigned_branch_mask);
2162
2163 switch (op_select) {
2164 case BoolTest::eq:
2165 cmov_fp_cmp_fp_eq(op1, op2, dst, src, cmp_single, cmov_single);
2166 break;
2167 case BoolTest::ne:
2168 cmov_fp_cmp_fp_ne(op1, op2, dst, src, cmp_single, cmov_single);
2169 break;
2170 case BoolTest::le:
2171 cmov_fp_cmp_fp_le(op1, op2, dst, src, cmp_single, cmov_single);
2172 break;
2173 case BoolTest::ge:
2174 cmov_fp_cmp_fp_ge(op1, op2, dst, src, cmp_single, cmov_single);
2175 break;
2176 case BoolTest::lt:
2177 cmov_fp_cmp_fp_lt(op1, op2, dst, src, cmp_single, cmov_single);
2178 break;
2179 case BoolTest::gt:
2180 cmov_fp_cmp_fp_gt(op1, op2, dst, src, cmp_single, cmov_single);
2181 break;
2182 default:
2183 assert(false, "unsupported compare condition");
2184 ShouldNotReachHere();
2185 }
2186 }
2187
2188 // Set dst to NaN if any NaN input.
2189 void C2_MacroAssembler::minmax_fp(FloatRegister dst, FloatRegister src1, FloatRegister src2,
2190 FLOAT_TYPE ft, bool is_min) {
2191 assert_cond((ft != FLOAT_TYPE::half_precision) || UseZfh);
2192
2193 Label Done, Compare;
2194
2195 switch (ft) {
2196 case FLOAT_TYPE::half_precision:
2197 fclass_h(t0, src1);
2198 fclass_h(t1, src2);
2199
2200 orr(t0, t0, t1);
2201 andi(t0, t0, FClassBits::nan); // if src1 or src2 is quiet or signaling NaN then return NaN
2202 beqz(t0, Compare);
2203
2204 fadd_h(dst, src1, src2);
2205 j(Done);
2206
2207 bind(Compare);
2208 if (is_min) {
2209 fmin_h(dst, src1, src2);
2210 } else {
2211 fmax_h(dst, src1, src2);
2212 }
2213 break;
2214 case FLOAT_TYPE::single_precision:
2215 fclass_s(t0, src1);
2216 fclass_s(t1, src2);
2217
2218 orr(t0, t0, t1);
2219 andi(t0, t0, FClassBits::nan); // if src1 or src2 is quiet or signaling NaN then return NaN
2220 beqz(t0, Compare);
2221
2222 fadd_s(dst, src1, src2);
2223 j(Done);
2224
2225 bind(Compare);
2226 if (is_min) {
2227 fmin_s(dst, src1, src2);
2228 } else {
2229 fmax_s(dst, src1, src2);
2230 }
2231 break;
2232 case FLOAT_TYPE::double_precision:
2233 fclass_d(t0, src1);
2234 fclass_d(t1, src2);
2235
2236 orr(t0, t0, t1);
2237 andi(t0, t0, FClassBits::nan); // if src1 or src2 is quiet or signaling NaN then return NaN
2238 beqz(t0, Compare);
2239
2240 fadd_d(dst, src1, src2);
2241 j(Done);
2242
2243 bind(Compare);
2244 if (is_min) {
2245 fmin_d(dst, src1, src2);
2246 } else {
2247 fmax_d(dst, src1, src2);
2248 }
2249 break;
2250 default:
2251 ShouldNotReachHere();
2252 }
2253
2254 bind(Done);
2255 }
2256
2257 // According to Java SE specification, for floating-point round operations, if
2258 // the input is NaN, +/-infinity, or +/-0, the same input is returned as the
2259 // rounded result; this differs from behavior of RISC-V fcvt instructions (which
2260 // round out-of-range values to the nearest max or min value), therefore special
2261 // handling is needed by NaN, +/-Infinity, +/-0.
2262 void C2_MacroAssembler::round_double_mode(FloatRegister dst, FloatRegister src, int round_mode,
2263 Register tmp1, Register tmp2, Register tmp3) {
2264
2265 assert_different_registers(dst, src);
2266 assert_different_registers(tmp1, tmp2, tmp3);
2267
2268 // Set rounding mode for conversions
2269 // Here we use similar modes to double->long and long->double conversions
2270 // Different mode for long->double conversion matter only if long value was not representable as double,
2271 // we got long value as a result of double->long conversion so, it is definitely representable
2272 RoundingMode rm;
2273 switch (round_mode) {
2274 case RoundDoubleModeNode::rmode_ceil:
2275 rm = RoundingMode::rup;
2276 break;
2277 case RoundDoubleModeNode::rmode_floor:
2278 rm = RoundingMode::rdn;
2279 break;
2280 case RoundDoubleModeNode::rmode_rint:
2281 rm = RoundingMode::rne;
2282 break;
2283 default:
2284 ShouldNotReachHere();
2285 }
2286
2287 // tmp1 - is a register to store double converted to long int
2288 // tmp2 - is a register to create constant for comparison
2289 // tmp3 - is a register where we store modified result of double->long conversion
2290 Label done, bad_val;
2291
2292 // Conversion from double to long
2293 fcvt_l_d(tmp1, src, rm);
2294
2295 // Generate constant (tmp2)
2296 // tmp2 = 100...0000
2297 addi(tmp2, zr, 1);
2298 slli(tmp2, tmp2, 63);
2299
2300 // Prepare converted long (tmp1)
2301 // as a result when conversion overflow we got:
2302 // tmp1 = 011...1111 or 100...0000
2303 // Convert it to: tmp3 = 100...0000
2304 addi(tmp3, tmp1, 1);
2305 andi(tmp3, tmp3, -2);
2306 beq(tmp3, tmp2, bad_val);
2307
2308 // Conversion from long to double
2309 fcvt_d_l(dst, tmp1, rm);
2310 // Add sign of input value to result for +/- 0 cases
2311 fsgnj_d(dst, dst, src);
2312 j(done);
2313
2314 // If got conversion overflow return src
2315 bind(bad_val);
2316 fmv_d(dst, src);
2317
2318 bind(done);
2319 }
2320
2321 // According to Java SE specification, for floating-point signum operations, if
2322 // on input we have NaN or +/-0.0 value we should return it,
2323 // otherwise return +/- 1.0 using sign of input.
2324 // one - gives us a floating-point 1.0 (got from matching rule)
2325 // bool is_double - specifies single or double precision operations will be used.
2326 void C2_MacroAssembler::signum_fp(FloatRegister dst, FloatRegister one, bool is_double) {
2327 Label done;
2328
2329 is_double ? fclass_d(t0, dst)
2330 : fclass_s(t0, dst);
2331
2332 // check if input is -0, +0, signaling NaN or quiet NaN
2333 andi(t0, t0, FClassBits::zero | FClassBits::nan);
2334
2335 bnez(t0, done);
2336
2337 // use floating-point 1.0 with a sign of input
2338 is_double ? fsgnj_d(dst, one, dst)
2339 : fsgnj_s(dst, one, dst);
2340
2341 bind(done);
2342 }
2343
2344 static void float16_to_float_slow_path(C2_MacroAssembler& masm, C2GeneralStub<FloatRegister, Register, Register>& stub) {
2345 #define __ masm.
2346 FloatRegister dst = stub.data<0>();
2347 Register src = stub.data<1>();
2348 Register tmp = stub.data<2>();
2349 __ bind(stub.entry());
2350
2351 // following instructions mainly focus on NaN, as riscv does not handle
2352 // NaN well with fcvt, but the code also works for Inf at the same time.
2353
2354 // construct a NaN in 32 bits from the NaN in 16 bits,
2355 // we need the payloads of non-canonical NaNs to be preserved.
2356 __ mv(tmp, 0x7f800000);
2357 // sign-bit was already set via sign-extension if necessary.
2358 __ slli(t0, src, 13);
2359 __ orr(tmp, t0, tmp);
2360 __ fmv_w_x(dst, tmp);
2361
2362 __ j(stub.continuation());
2363 #undef __
2364 }
2365
2366 // j.l.Float.float16ToFloat
2367 void C2_MacroAssembler::float16_to_float(FloatRegister dst, Register src, Register tmp) {
2368 auto stub = C2CodeStub::make<FloatRegister, Register, Register>(dst, src, tmp, 20, float16_to_float_slow_path);
2369
2370 // On riscv, NaN needs a special process as fcvt does not work in that case.
2371 // On riscv, Inf does not need a special process as fcvt can handle it correctly.
2372 // but we consider to get the slow path to process NaN and Inf at the same time,
2373 // as both of them are rare cases, and if we try to get the slow path to handle
2374 // only NaN case it would sacrifise the performance for normal cases,
2375 // i.e. non-NaN and non-Inf cases.
2376
2377 // check whether it's a NaN or +/- Inf.
2378 mv(t0, 0x7c00);
2379 andr(tmp, src, t0);
2380 // jump to stub processing NaN and Inf cases.
2381 beq(t0, tmp, stub->entry(), /* is_far */ true);
2382
2383 // non-NaN or non-Inf cases, just use built-in instructions.
2384 fmv_h_x(dst, src);
2385 fcvt_s_h(dst, dst);
2386
2387 bind(stub->continuation());
2388 }
2389
2390 static void float_to_float16_slow_path(C2_MacroAssembler& masm, C2GeneralStub<Register, FloatRegister, Register>& stub) {
2391 #define __ masm.
2392 Register dst = stub.data<0>();
2393 FloatRegister src = stub.data<1>();
2394 Register tmp = stub.data<2>();
2395 __ bind(stub.entry());
2396
2397 __ float_to_float16_NaN(dst, src, t0, tmp);
2398
2399 __ j(stub.continuation());
2400 #undef __
2401 }
2402
2403 // j.l.Float.floatToFloat16
2404 void C2_MacroAssembler::float_to_float16(Register dst, FloatRegister src, FloatRegister ftmp, Register xtmp) {
2405 auto stub = C2CodeStub::make<Register, FloatRegister, Register>(dst, src, xtmp, 64, float_to_float16_slow_path);
2406
2407 // On riscv, NaN needs a special process as fcvt does not work in that case.
2408
2409 // check whether it's a NaN.
2410 // replace fclass with feq as performance optimization.
2411 feq_s(t0, src, src);
2412 // jump to stub processing NaN cases.
2413 beqz(t0, stub->entry(), /* is_far */ true);
2414
2415 // non-NaN cases, just use built-in instructions.
2416 fcvt_h_s(ftmp, src);
2417 fmv_x_h(dst, ftmp);
2418
2419 bind(stub->continuation());
2420 }
2421
2422 static void float16_to_float_v_slow_path(C2_MacroAssembler& masm, C2GeneralStub<VectorRegister, VectorRegister, uint>& stub) {
2423 #define __ masm.
2424 VectorRegister dst = stub.data<0>();
2425 VectorRegister src = stub.data<1>();
2426 uint vector_length = stub.data<2>();
2427 __ bind(stub.entry());
2428
2429 // following instructions mainly focus on NaN, as riscv does not handle
2430 // NaN well with vfwcvt_f_f_v, but the code also works for Inf at the same time.
2431 //
2432 // construct NaN's in 32 bits from the NaN's in 16 bits,
2433 // we need the payloads of non-canonical NaNs to be preserved.
2434
2435 // adjust vector type to 2 * SEW.
2436 __ vsetvli_helper(T_FLOAT, vector_length, Assembler::m1);
2437 // widen and sign-extend src data.
2438 __ vsext_vf2(dst, src, Assembler::v0_t);
2439 __ mv(t0, 0x7f800000);
2440 // sign-bit was already set via sign-extension if necessary.
2441 __ vsll_vi(dst, dst, 13, Assembler::v0_t);
2442 __ vor_vx(dst, dst, t0, Assembler::v0_t);
2443
2444 __ j(stub.continuation());
2445 #undef __
2446 }
2447
2448 // j.l.Float.float16ToFloat
2449 void C2_MacroAssembler::float16_to_float_v(VectorRegister dst, VectorRegister src, uint vector_length) {
2450 auto stub = C2CodeStub::make<VectorRegister, VectorRegister, uint>
2451 (dst, src, vector_length, 24, float16_to_float_v_slow_path);
2452 assert_different_registers(dst, src);
2453
2454 // On riscv, NaN needs a special process as vfwcvt_f_f_v does not work in that case.
2455 // On riscv, Inf does not need a special process as vfwcvt_f_f_v can handle it correctly.
2456 // but we consider to get the slow path to process NaN and Inf at the same time,
2457 // as both of them are rare cases, and if we try to get the slow path to handle
2458 // only NaN case it would sacrifise the performance for normal cases,
2459 // i.e. non-NaN and non-Inf cases.
2460
2461 vsetvli_helper(BasicType::T_SHORT, vector_length, Assembler::mf2);
2462
2463 // check whether there is a NaN or +/- Inf.
2464 mv(t0, 0x7c00);
2465 vand_vx(v0, src, t0);
2466 // v0 will be used as mask in slow path.
2467 vmseq_vx(v0, v0, t0);
2468 vcpop_m(t0, v0);
2469
2470 // For non-NaN or non-Inf cases, just use built-in instructions.
2471 vfwcvt_f_f_v(dst, src);
2472
2473 // jump to stub processing NaN and Inf cases if there is any of them in the vector-wide.
2474 bnez(t0, stub->entry(), /* is_far */ true);
2475
2476 bind(stub->continuation());
2477 }
2478
2479 static void float_to_float16_v_slow_path(C2_MacroAssembler& masm,
2480 C2GeneralStub<VectorRegister, VectorRegister, VectorRegister>& stub) {
2481 #define __ masm.
2482 VectorRegister dst = stub.data<0>();
2483 VectorRegister src = stub.data<1>();
2484 VectorRegister vtmp = stub.data<2>();
2485 assert_different_registers(dst, src, vtmp);
2486
2487 __ bind(stub.entry());
2488
2489 // Active elements (NaNs) are marked in v0 mask register.
2490 // mul is already set to mf2 in float_to_float16_v.
2491
2492 // Float (32 bits)
2493 // Bit: 31 30 to 23 22 to 0
2494 // +---+------------------+-----------------------------+
2495 // | S | Exponent | Mantissa (Fraction) |
2496 // +---+------------------+-----------------------------+
2497 // 1 bit 8 bits 23 bits
2498 //
2499 // Float (16 bits)
2500 // Bit: 15 14 to 10 9 to 0
2501 // +---+----------------+------------------+
2502 // | S | Exponent | Mantissa |
2503 // +---+----------------+------------------+
2504 // 1 bit 5 bits 10 bits
2505 const int fp_sign_bits = 1;
2506 const int fp32_bits = 32;
2507 const int fp32_mantissa_2nd_part_bits = 9;
2508 const int fp32_mantissa_3rd_part_bits = 4;
2509 const int fp16_exponent_bits = 5;
2510 const int fp16_mantissa_bits = 10;
2511
2512 // preserve the sign bit and exponent, clear mantissa.
2513 __ vnsra_wi(dst, src, fp32_bits - fp_sign_bits - fp16_exponent_bits, Assembler::v0_t);
2514 __ vsll_vi(dst, dst, fp16_mantissa_bits, Assembler::v0_t);
2515
2516 // Preserve high order bit of float NaN in the
2517 // binary16 result NaN (tenth bit); OR in remaining
2518 // bits into lower 9 bits of binary 16 significand.
2519 // | (doppel & 0x007f_e000) >> 13 // 10 bits
2520 // | (doppel & 0x0000_1ff0) >> 4 // 9 bits
2521 // | (doppel & 0x0000_000f)); // 4 bits
2522 //
2523 // Check j.l.Float.floatToFloat16 for more information.
2524 // 10 bits
2525 __ vnsrl_wi(vtmp, src, fp32_mantissa_2nd_part_bits + fp32_mantissa_3rd_part_bits, Assembler::v0_t);
2526 __ mv(t0, 0x3ff); // retain first part of mantissa in a float 32
2527 __ vand_vx(vtmp, vtmp, t0, Assembler::v0_t);
2528 __ vor_vv(dst, dst, vtmp, Assembler::v0_t);
2529 // 9 bits
2530 __ vnsrl_wi(vtmp, src, fp32_mantissa_3rd_part_bits, Assembler::v0_t);
2531 __ mv(t0, 0x1ff); // retain second part of mantissa in a float 32
2532 __ vand_vx(vtmp, vtmp, t0, Assembler::v0_t);
2533 __ vor_vv(dst, dst, vtmp, Assembler::v0_t);
2534 // 4 bits
2535 // Narrow shift is necessary to move data from 32 bits element to 16 bits element in vector register.
2536 __ vnsrl_wi(vtmp, src, 0, Assembler::v0_t);
2537 __ vand_vi(vtmp, vtmp, 0xf, Assembler::v0_t);
2538 __ vor_vv(dst, dst, vtmp, Assembler::v0_t);
2539
2540 __ j(stub.continuation());
2541 #undef __
2542 }
2543
2544 // j.l.Float.float16ToFloat
2545 void C2_MacroAssembler::float_to_float16_v(VectorRegister dst, VectorRegister src,
2546 VectorRegister vtmp, Register tmp, uint vector_length) {
2547 assert_different_registers(dst, src, vtmp);
2548
2549 auto stub = C2CodeStub::make<VectorRegister, VectorRegister, VectorRegister>
2550 (dst, src, vtmp, 56, float_to_float16_v_slow_path);
2551
2552 // On riscv, NaN needs a special process as vfncvt_f_f_w does not work in that case.
2553
2554 vsetvli_helper(BasicType::T_FLOAT, vector_length, Assembler::m1);
2555
2556 // check whether there is a NaN.
2557 // replace v_fclass with vmfne_vv as performance optimization.
2558 vmfne_vv(v0, src, src);
2559 vcpop_m(t0, v0);
2560
2561 vsetvli_helper(BasicType::T_SHORT, vector_length, Assembler::mf2, tmp);
2562
2563 // For non-NaN cases, just use built-in instructions.
2564 vfncvt_f_f_w(dst, src);
2565
2566 // jump to stub processing NaN cases.
2567 bnez(t0, stub->entry(), /* is_far */ true);
2568
2569 bind(stub->continuation());
2570 }
2571
2572 void C2_MacroAssembler::signum_fp_v(VectorRegister dst, VectorRegister one, BasicType bt, int vlen) {
2573 vsetvli_helper(bt, vlen);
2574
2575 // check if input is -0, +0, signaling NaN or quiet NaN
2576 vfclass_v(v0, dst);
2577 mv(t0, FClassBits::zero | FClassBits::nan);
2578 vand_vx(v0, v0, t0);
2579 vmseq_vi(v0, v0, 0);
2580
2581 // use floating-point 1.0 with a sign of input
2582 vfsgnj_vv(dst, one, dst, v0_t);
2583 }
2584
2585 // j.l.Math.round(float)
2586 // Returns the closest int to the argument, with ties rounding to positive infinity.
2587 // We need to handle 3 special cases defined by java api spec:
2588 // NaN,
2589 // float >= Integer.MAX_VALUE,
2590 // float <= Integer.MIN_VALUE.
2591 void C2_MacroAssembler::java_round_float_v(VectorRegister dst, VectorRegister src, FloatRegister ftmp,
2592 BasicType bt, uint vector_length) {
2593 // In riscv, there is no straight corresponding rounding mode to satisfy the behaviour defined,
2594 // in java api spec, i.e. any rounding mode can not handle some corner cases, e.g.
2595 // RNE is the closest one, but it ties to "even", which means 1.5/2.5 both will be converted
2596 // to 2, instead of 2 and 3 respectively.
2597 // RUP does not work either, although java api requires "rounding to positive infinity",
2598 // but both 1.3/1.8 will be converted to 2, instead of 1 and 2 respectively.
2599 //
2600 // The optimal solution for non-NaN cases is:
2601 // src+0.5 => dst, with rdn rounding mode,
2602 // convert dst from float to int, with rnd rounding mode.
2603 // and, this solution works as expected for float >= Integer.MAX_VALUE and float <= Integer.MIN_VALUE.
2604 //
2605 // But, we still need to handle NaN explicilty with vector mask instructions.
2606 //
2607 // Check MacroAssembler::java_round_float and C2_MacroAssembler::vector_round_sve in aarch64 for more details.
2608
2609 csrwi(CSR_FRM, C2_MacroAssembler::rdn);
2610 vsetvli_helper(bt, vector_length);
2611
2612 // don't rearrage the instructions sequence order without performance testing.
2613 // check MacroAssembler::java_round_float in riscv64 for more details.
2614 mv(t0, jint_cast(0.5f));
2615 fmv_w_x(ftmp, t0);
2616
2617 // replacing vfclass with feq as performance optimization
2618 vmfeq_vv(v0, src, src);
2619 // set dst = 0 in cases of NaN
2620 vmv_v_x(dst, zr);
2621
2622 // dst = (src + 0.5) rounded down towards negative infinity
2623 vfadd_vf(dst, src, ftmp, Assembler::v0_t);
2624 vfcvt_x_f_v(dst, dst, Assembler::v0_t); // in RoundingMode::rdn
2625
2626 csrwi(CSR_FRM, C2_MacroAssembler::rne);
2627 }
2628
2629 // java.lang.Math.round(double a)
2630 // Returns the closest long to the argument, with ties rounding to positive infinity.
2631 void C2_MacroAssembler::java_round_double_v(VectorRegister dst, VectorRegister src, FloatRegister ftmp,
2632 BasicType bt, uint vector_length) {
2633 // check C2_MacroAssembler::java_round_float_v above for more details.
2634
2635 csrwi(CSR_FRM, C2_MacroAssembler::rdn);
2636 vsetvli_helper(bt, vector_length);
2637
2638 mv(t0, julong_cast(0.5));
2639 fmv_d_x(ftmp, t0);
2640
2641 // replacing vfclass with feq as performance optimization
2642 vmfeq_vv(v0, src, src);
2643 // set dst = 0 in cases of NaN
2644 vmv_v_x(dst, zr);
2645
2646 // dst = (src + 0.5) rounded down towards negative infinity
2647 vfadd_vf(dst, src, ftmp, Assembler::v0_t);
2648 vfcvt_x_f_v(dst, dst, Assembler::v0_t); // in RoundingMode::rdn
2649
2650 csrwi(CSR_FRM, C2_MacroAssembler::rne);
2651 }
2652
2653 void C2_MacroAssembler::element_compare(Register a1, Register a2, Register result, Register cnt, Register tmp1, Register tmp2,
2654 VectorRegister vr1, VectorRegister vr2, VectorRegister vrs, bool islatin, Label &DONE,
2655 Assembler::LMUL lmul) {
2656 Label loop;
2657 Assembler::SEW sew = islatin ? Assembler::e8 : Assembler::e16;
2658
2659 bind(loop);
2660 vsetvli(tmp1, cnt, sew, lmul);
2661 vlex_v(vr1, a1, sew);
2662 vlex_v(vr2, a2, sew);
2663 vmsne_vv(vrs, vr1, vr2);
2664 vfirst_m(tmp2, vrs);
2665 bgez(tmp2, DONE);
2666 sub(cnt, cnt, tmp1);
2667 if (!islatin) {
2668 slli(tmp1, tmp1, 1); // get byte counts
2669 }
2670 add(a1, a1, tmp1);
2671 add(a2, a2, tmp1);
2672 bnez(cnt, loop);
2673
2674 mv(result, true);
2675 }
2676
2677 void C2_MacroAssembler::string_equals_v(Register a1, Register a2, Register result, Register cnt) {
2678 Label DONE;
2679 Register tmp1 = t0;
2680 Register tmp2 = t1;
2681
2682 BLOCK_COMMENT("string_equals_v {");
2683
2684 mv(result, false);
2685
2686 element_compare(a1, a2, result, cnt, tmp1, tmp2, v2, v4, v2, true, DONE, Assembler::m2);
2687
2688 bind(DONE);
2689 BLOCK_COMMENT("} string_equals_v");
2690 }
2691
2692 // used by C2 ClearArray patterns.
2693 // base: Address of a buffer to be zeroed
2694 // cnt: Count in HeapWords
2695 //
2696 // base, cnt, v4, v5, v6, v7 and t0 are clobbered.
2697 void C2_MacroAssembler::clear_array_v(Register base, Register cnt) {
2698 Label loop;
2699
2700 // making zero words
2701 vsetvli(t0, cnt, Assembler::e64, Assembler::m4);
2702 vxor_vv(v4, v4, v4);
2703
2704 bind(loop);
2705 vsetvli(t0, cnt, Assembler::e64, Assembler::m4);
2706 vse64_v(v4, base);
2707 sub(cnt, cnt, t0);
2708 shadd(base, t0, base, t0, 3);
2709 bnez(cnt, loop);
2710 }
2711
2712 void C2_MacroAssembler::arrays_equals_v(Register a1, Register a2, Register result,
2713 Register cnt1, int elem_size) {
2714 assert(elem_size == 1 || elem_size == 2, "must be char or byte");
2715 assert_different_registers(a1, a2, result, cnt1, t0, t1);
2716
2717 Label DONE;
2718 Register tmp1 = t0;
2719 Register tmp2 = t1;
2720 Register cnt2 = tmp2;
2721 int length_offset = arrayOopDesc::length_offset_in_bytes();
2722 int base_offset = arrayOopDesc::base_offset_in_bytes(elem_size == 2 ? T_CHAR : T_BYTE);
2723
2724 assert((base_offset % (UseCompactObjectHeaders ? 4 : 8)) == 0, "Must be");
2725
2726 BLOCK_COMMENT("arrays_equals_v {");
2727
2728 // if (a1 == a2), return true
2729 mv(result, true);
2730 beq(a1, a2, DONE);
2731
2732 mv(result, false);
2733 // if a1 == null or a2 == null, return false
2734 beqz(a1, DONE);
2735 beqz(a2, DONE);
2736 // if (a1.length != a2.length), return false
2737 lwu(cnt1, Address(a1, length_offset));
2738 lwu(cnt2, Address(a2, length_offset));
2739 bne(cnt1, cnt2, DONE);
2740
2741 la(a1, Address(a1, base_offset));
2742 la(a2, Address(a2, base_offset));
2743
2744 element_compare(a1, a2, result, cnt1, tmp1, tmp2, v2, v4, v2, elem_size == 1, DONE, Assembler::m2);
2745
2746 bind(DONE);
2747
2748 BLOCK_COMMENT("} arrays_equals_v");
2749 }
2750
2751 void C2_MacroAssembler::string_compare_v(Register str1, Register str2, Register cnt1, Register cnt2,
2752 Register result, Register tmp1, Register tmp2, int encForm) {
2753 Label DIFFERENCE, DONE, L, loop;
2754 bool encLL = encForm == StrIntrinsicNode::LL;
2755 bool encLU = encForm == StrIntrinsicNode::LU;
2756 bool encUL = encForm == StrIntrinsicNode::UL;
2757
2758 bool str1_isL = encLL || encLU;
2759 bool str2_isL = encLL || encUL;
2760
2761 int minCharsInWord = encLL ? wordSize : wordSize / 2;
2762
2763 BLOCK_COMMENT("string_compare_v {");
2764
2765 // for Latin strings, 1 byte for 1 character
2766 // for UTF16 strings, 2 bytes for 1 character
2767 if (!str1_isL)
2768 sraiw(cnt1, cnt1, 1);
2769 if (!str2_isL)
2770 sraiw(cnt2, cnt2, 1);
2771
2772 // if str1 == str2, return the difference
2773 // save the minimum of the string lengths in cnt2.
2774 sub(result, cnt1, cnt2);
2775 bgt(cnt1, cnt2, L);
2776 mv(cnt2, cnt1);
2777 bind(L);
2778
2779 // We focus on the optimization of small sized string.
2780 // Please check below document for string size distribution statistics.
2781 // https://cr.openjdk.org/~shade/density/string-density-report.pdf
2782 if (str1_isL == str2_isL) { // LL or UU
2783 // Below construction of v regs and lmul is based on test on 2 different boards,
2784 // vlen == 128 and vlen == 256 respectively.
2785 if (!encLL && MaxVectorSize == 16) { // UU
2786 element_compare(str1, str2, zr, cnt2, tmp1, tmp2, v4, v8, v4, encLL, DIFFERENCE, Assembler::m4);
2787 } else { // UU + MaxVectorSize or LL
2788 element_compare(str1, str2, zr, cnt2, tmp1, tmp2, v2, v4, v2, encLL, DIFFERENCE, Assembler::m2);
2789 }
2790
2791 j(DONE);
2792 } else { // LU or UL
2793 Register strL = encLU ? str1 : str2;
2794 Register strU = encLU ? str2 : str1;
2795 VectorRegister vstr1 = encLU ? v8 : v4;
2796 VectorRegister vstr2 = encLU ? v4 : v8;
2797
2798 bind(loop);
2799 vsetvli(tmp1, cnt2, Assembler::e8, Assembler::m2);
2800 vle8_v(vstr1, strL);
2801 vsetvli(tmp1, cnt2, Assembler::e16, Assembler::m4);
2802 vzext_vf2(vstr2, vstr1);
2803 vle16_v(vstr1, strU);
2804 vmsne_vv(v4, vstr2, vstr1);
2805 vfirst_m(tmp2, v4);
2806 bgez(tmp2, DIFFERENCE);
2807 sub(cnt2, cnt2, tmp1);
2808 add(strL, strL, tmp1);
2809 shadd(strU, tmp1, strU, tmp1, 1);
2810 bnez(cnt2, loop);
2811 j(DONE);
2812 }
2813
2814 bind(DIFFERENCE);
2815 slli(tmp1, tmp2, 1);
2816 add(str1, str1, str1_isL ? tmp2 : tmp1);
2817 add(str2, str2, str2_isL ? tmp2 : tmp1);
2818 str1_isL ? lbu(tmp1, Address(str1, 0)) : lhu(tmp1, Address(str1, 0));
2819 str2_isL ? lbu(tmp2, Address(str2, 0)) : lhu(tmp2, Address(str2, 0));
2820 sub(result, tmp1, tmp2);
2821
2822 bind(DONE);
2823
2824 BLOCK_COMMENT("} string_compare_v");
2825 }
2826
2827 void C2_MacroAssembler::byte_array_inflate_v(Register src, Register dst, Register len, Register tmp) {
2828 Label loop;
2829 assert_different_registers(src, dst, len, tmp, t0);
2830
2831 BLOCK_COMMENT("byte_array_inflate_v {");
2832 bind(loop);
2833 vsetvli(tmp, len, Assembler::e8, Assembler::m2);
2834 vle8_v(v6, src);
2835 vsetvli(t0, len, Assembler::e16, Assembler::m4);
2836 vzext_vf2(v4, v6);
2837 vse16_v(v4, dst);
2838 sub(len, len, tmp);
2839 add(src, src, tmp);
2840 shadd(dst, tmp, dst, tmp, 1);
2841 bnez(len, loop);
2842 BLOCK_COMMENT("} byte_array_inflate_v");
2843 }
2844
2845 // Compress char[] array to byte[].
2846 // Intrinsic for java.lang.StringUTF16.compress(char[] src, int srcOff, byte[] dst, int dstOff, int len)
2847 // result: the array length if every element in array can be encoded,
2848 // otherwise, the index of first non-latin1 (> 0xff) character.
2849 void C2_MacroAssembler::char_array_compress_v(Register src, Register dst, Register len,
2850 Register result, Register tmp) {
2851 encode_iso_array_v(src, dst, len, result, tmp, false);
2852 }
2853
2854 // Intrinsic for
2855 //
2856 // - sun.nio.cs.ISO_8859_1.Encoder#encodeISOArray0(byte[] sa, int sp, byte[] da, int dp, int len)
2857 // Encodes char[] to byte[] in ISO-8859-1
2858 //
2859 // - java.lang.StringCoding#encodeISOArray0(byte[] sa, int sp, byte[] da, int dp, int len)
2860 // Encodes byte[] (containing UTF-16) to byte[] in ISO-8859-1
2861 //
2862 // - java.lang.StringCoding#encodeAsciiArray0(char[] sa, int sp, byte[] da, int dp, int len)
2863 // Encodes char[] to byte[] in ASCII
2864 //
2865 // This version always returns the number of characters copied. A successful
2866 // copy will complete with the post-condition: 'res' == 'len', while an
2867 // unsuccessful copy will exit with the post-condition: 0 <= 'res' < 'len'.
2868 //
2869 // Clobbers: src, dst, len, result, t0
2870 void C2_MacroAssembler::encode_iso_array_v(Register src, Register dst, Register len,
2871 Register result, Register tmp, bool ascii) {
2872 Label loop, fail, done;
2873
2874 BLOCK_COMMENT("encode_iso_array_v {");
2875 mv(result, 0);
2876
2877 bind(loop);
2878 mv(tmp, ascii ? 0x7f : 0xff);
2879 vsetvli(t0, len, Assembler::e16, Assembler::m2);
2880 vle16_v(v2, src);
2881
2882 vmsgtu_vx(v1, v2, tmp);
2883 vfirst_m(tmp, v1);
2884 vmsbf_m(v0, v1);
2885 // compress char to byte
2886 vsetvli(t0, len, Assembler::e8);
2887 vncvt_x_x_w(v1, v2, Assembler::v0_t);
2888 vse8_v(v1, dst, Assembler::v0_t);
2889
2890 // fail if char > 0x7f/0xff
2891 bgez(tmp, fail);
2892 add(result, result, t0);
2893 add(dst, dst, t0);
2894 sub(len, len, t0);
2895 shadd(src, t0, src, t0, 1);
2896 bnez(len, loop);
2897 j(done);
2898
2899 bind(fail);
2900 add(result, result, tmp);
2901
2902 bind(done);
2903 BLOCK_COMMENT("} encode_iso_array_v");
2904 }
2905
2906 void C2_MacroAssembler::count_positives_v(Register ary, Register len, Register result, Register tmp) {
2907 Label LOOP, SET_RESULT, DONE;
2908
2909 BLOCK_COMMENT("count_positives_v {");
2910 assert_different_registers(ary, len, result, tmp);
2911
2912 mv(result, zr);
2913
2914 bind(LOOP);
2915 vsetvli(t0, len, Assembler::e8, Assembler::m4);
2916 vle8_v(v4, ary);
2917 vmslt_vx(v4, v4, zr);
2918 vfirst_m(tmp, v4);
2919 bgez(tmp, SET_RESULT);
2920 // if tmp == -1, all bytes are positive
2921 add(result, result, t0);
2922
2923 sub(len, len, t0);
2924 add(ary, ary, t0);
2925 bnez(len, LOOP);
2926 j(DONE);
2927
2928 // add remaining positive bytes count
2929 bind(SET_RESULT);
2930 add(result, result, tmp);
2931
2932 bind(DONE);
2933 BLOCK_COMMENT("} count_positives_v");
2934 }
2935
2936 void C2_MacroAssembler::string_indexof_char_v(Register str1, Register cnt1,
2937 Register ch, Register result,
2938 Register tmp1, Register tmp2,
2939 bool isL) {
2940 mv(result, zr);
2941
2942 Label loop, MATCH, DONE;
2943 Assembler::SEW sew = isL ? Assembler::e8 : Assembler::e16;
2944 bind(loop);
2945 vsetvli(tmp1, cnt1, sew, Assembler::m4);
2946 vlex_v(v4, str1, sew);
2947 vmseq_vx(v4, v4, ch);
2948 vfirst_m(tmp2, v4);
2949 bgez(tmp2, MATCH); // if equal, return index
2950
2951 add(result, result, tmp1);
2952 sub(cnt1, cnt1, tmp1);
2953 if (!isL) slli(tmp1, tmp1, 1);
2954 add(str1, str1, tmp1);
2955 bnez(cnt1, loop);
2956
2957 mv(result, -1);
2958 j(DONE);
2959
2960 bind(MATCH);
2961 add(result, result, tmp2);
2962
2963 bind(DONE);
2964 }
2965
2966 // Set dst to NaN if any NaN input.
2967 void C2_MacroAssembler::minmax_fp_v(VectorRegister dst, VectorRegister src1, VectorRegister src2,
2968 BasicType bt, bool is_min, uint vector_length) {
2969 assert_different_registers(dst, src1, src2);
2970
2971 vsetvli_helper(bt, vector_length);
2972
2973 is_min ? vfmin_vv(dst, src1, src2)
2974 : vfmax_vv(dst, src1, src2);
2975
2976 vmfne_vv(v0, src1, src1);
2977 vfadd_vv(dst, src1, src1, Assembler::v0_t);
2978 vmfne_vv(v0, src2, src2);
2979 vfadd_vv(dst, src2, src2, Assembler::v0_t);
2980 }
2981
2982 // Set dst to NaN if any NaN input.
2983 // The destination vector register elements corresponding to masked-off elements
2984 // are handled with a mask-undisturbed policy.
2985 void C2_MacroAssembler::minmax_fp_masked_v(VectorRegister dst, VectorRegister src1, VectorRegister src2,
2986 VectorRegister vmask, VectorRegister tmp1, VectorRegister tmp2,
2987 BasicType bt, bool is_min, uint vector_length) {
2988 assert_different_registers(src1, src2, tmp1, tmp2);
2989 vsetvli_helper(bt, vector_length);
2990
2991 // Check vector elements of src1 and src2 for NaN.
2992 vmfeq_vv(tmp1, src1, src1);
2993 vmfeq_vv(tmp2, src2, src2);
2994
2995 vmandn_mm(v0, vmask, tmp1);
2996 vfadd_vv(dst, src1, src1, Assembler::v0_t);
2997 vmandn_mm(v0, vmask, tmp2);
2998 vfadd_vv(dst, src2, src2, Assembler::v0_t);
2999
3000 vmand_mm(tmp2, tmp1, tmp2);
3001 vmand_mm(v0, vmask, tmp2);
3002 is_min ? vfmin_vv(dst, src1, src2, Assembler::v0_t)
3003 : vfmax_vv(dst, src1, src2, Assembler::v0_t);
3004 }
3005
3006 // Set dst to NaN if any NaN input.
3007 void C2_MacroAssembler::reduce_minmax_fp_v(FloatRegister dst,
3008 FloatRegister src1, VectorRegister src2,
3009 VectorRegister tmp1, VectorRegister tmp2,
3010 bool is_double, bool is_min, uint vector_length, VectorMask vm) {
3011 assert_different_registers(dst, src1);
3012 assert_different_registers(src2, tmp1, tmp2);
3013
3014 Label L_done, L_NaN_1, L_NaN_2;
3015 // Set dst to src1 if src1 is NaN
3016 is_double ? feq_d(t0, src1, src1)
3017 : feq_s(t0, src1, src1);
3018 beqz(t0, L_NaN_2);
3019
3020 vsetvli_helper(is_double ? T_DOUBLE : T_FLOAT, vector_length);
3021 vfmv_s_f(tmp2, src1);
3022
3023 is_min ? vfredmin_vs(tmp1, src2, tmp2, vm)
3024 : vfredmax_vs(tmp1, src2, tmp2, vm);
3025 vfmv_f_s(dst, tmp1);
3026
3027 // Checking NaNs in src2
3028 vmfne_vv(tmp1, src2, src2, vm);
3029 vcpop_m(t0, tmp1, vm);
3030 beqz(t0, L_done);
3031
3032 bind(L_NaN_1);
3033 vfredusum_vs(tmp1, src2, tmp2, vm);
3034 vfmv_f_s(dst, tmp1);
3035 j(L_done);
3036
3037 bind(L_NaN_2);
3038 is_double ? fmv_d(dst, src1)
3039 : fmv_s(dst, src1);
3040 bind(L_done);
3041 }
3042
3043 bool C2_MacroAssembler::in_scratch_emit_size() {
3044 if (ciEnv::current()->task() != nullptr) {
3045 PhaseOutput* phase_output = Compile::current()->output();
3046 if (phase_output != nullptr && phase_output->in_scratch_emit_size()) {
3047 return true;
3048 }
3049 }
3050 return MacroAssembler::in_scratch_emit_size();
3051 }
3052
3053 void C2_MacroAssembler::reduce_integral_v(Register dst, Register src1,
3054 VectorRegister src2, VectorRegister tmp,
3055 int opc, BasicType bt, uint vector_length, VectorMask vm) {
3056 assert(bt == T_BYTE || bt == T_SHORT || bt == T_INT || bt == T_LONG, "unsupported element type");
3057 vsetvli_helper(bt, vector_length);
3058 vmv_s_x(tmp, src1);
3059 switch (opc) {
3060 case Op_AddReductionVI:
3061 case Op_AddReductionVL:
3062 vredsum_vs(tmp, src2, tmp, vm);
3063 break;
3064 case Op_AndReductionV:
3065 vredand_vs(tmp, src2, tmp, vm);
3066 break;
3067 case Op_OrReductionV:
3068 vredor_vs(tmp, src2, tmp, vm);
3069 break;
3070 case Op_XorReductionV:
3071 vredxor_vs(tmp, src2, tmp, vm);
3072 break;
3073 case Op_MaxReductionV:
3074 vredmax_vs(tmp, src2, tmp, vm);
3075 break;
3076 case Op_MinReductionV:
3077 vredmin_vs(tmp, src2, tmp, vm);
3078 break;
3079 default:
3080 ShouldNotReachHere();
3081 }
3082 vmv_x_s(dst, tmp);
3083 }
3084
3085 void C2_MacroAssembler::reduce_mul_integral_v(Register dst, Register src1, VectorRegister src2,
3086 VectorRegister vtmp1, VectorRegister vtmp2,
3087 BasicType bt, uint vector_length, VectorMask vm) {
3088 assert(bt == T_BYTE || bt == T_SHORT || bt == T_INT || bt == T_LONG, "unsupported element type");
3089 vsetvli_helper(bt, vector_length);
3090
3091 vector_length /= 2;
3092 if (vm != Assembler::unmasked) {
3093 // This behaviour is consistent with spec requirements of vector API, for `reduceLanes`:
3094 // If no elements are selected, an operation-specific identity value is returned.
3095 // If the operation is MUL, then the identity value is one.
3096 vmv_v_i(vtmp1, 1);
3097 vmerge_vvm(vtmp2, vtmp1, src2); // vm == v0
3098 slidedown_v(vtmp1, vtmp2, vector_length);
3099
3100 vsetvli_helper(bt, vector_length);
3101 vmul_vv(vtmp1, vtmp1, vtmp2);
3102 } else {
3103 slidedown_v(vtmp1, src2, vector_length);
3104
3105 vsetvli_helper(bt, vector_length);
3106 vmul_vv(vtmp1, vtmp1, src2);
3107 }
3108
3109 while (vector_length > 1) {
3110 vector_length /= 2;
3111 slidedown_v(vtmp2, vtmp1, vector_length);
3112 vsetvli_helper(bt, vector_length);
3113 vmul_vv(vtmp1, vtmp1, vtmp2);
3114 }
3115
3116 vmv_x_s(dst, vtmp1);
3117 if (bt == T_INT) {
3118 mulw(dst, dst, src1);
3119 } else {
3120 mul(dst, dst, src1);
3121 }
3122 }
3123
3124 // Set vl and vtype for full and partial vector operations.
3125 // (vma = mu, vta = tu, vill = false)
3126 void C2_MacroAssembler::vsetvli_helper(BasicType bt, uint vector_length, LMUL vlmul, Register tmp) {
3127 Assembler::SEW sew = Assembler::elemtype_to_sew(bt);
3128 if (vector_length <= 31) {
3129 vsetivli(tmp, vector_length, sew, vlmul);
3130 } else if (vector_length == (MaxVectorSize / type2aelembytes(bt))) {
3131 vsetvli(tmp, x0, sew, vlmul);
3132 } else {
3133 mv(tmp, vector_length);
3134 vsetvli(tmp, tmp, sew, vlmul);
3135 }
3136 }
3137
3138 void C2_MacroAssembler::compare_integral_v(VectorRegister vd, VectorRegister src1, VectorRegister src2,
3139 int cond, BasicType bt, uint vector_length, VectorMask vm) {
3140 assert(is_integral_type(bt), "unsupported element type");
3141 assert(vm == Assembler::v0_t ? vd != v0 : true, "should be different registers");
3142 vsetvli_helper(bt, vector_length);
3143 if (vm == Assembler::v0_t) {
3144 vmclr_m(vd);
3145 }
3146 switch (cond) {
3147 case BoolTest::eq: vmseq_vv(vd, src1, src2, vm); break;
3148 case BoolTest::ne: vmsne_vv(vd, src1, src2, vm); break;
3149 case BoolTest::le: vmsle_vv(vd, src1, src2, vm); break;
3150 case BoolTest::ge: vmsge_vv(vd, src1, src2, vm); break;
3151 case BoolTest::lt: vmslt_vv(vd, src1, src2, vm); break;
3152 case BoolTest::gt: vmsgt_vv(vd, src1, src2, vm); break;
3153 case BoolTest::ule: vmsleu_vv(vd, src1, src2, vm); break;
3154 case BoolTest::uge: vmsgeu_vv(vd, src1, src2, vm); break;
3155 case BoolTest::ult: vmsltu_vv(vd, src1, src2, vm); break;
3156 case BoolTest::ugt: vmsgtu_vv(vd, src1, src2, vm); break;
3157 default:
3158 assert(false, "unsupported compare condition");
3159 ShouldNotReachHere();
3160 }
3161 }
3162
3163 void C2_MacroAssembler::compare_fp_v(VectorRegister vd, VectorRegister src1, VectorRegister src2,
3164 int cond, BasicType bt, uint vector_length, VectorMask vm) {
3165 assert(is_floating_point_type(bt), "unsupported element type");
3166 assert(vm == Assembler::v0_t ? vd != v0 : true, "should be different registers");
3167 vsetvli_helper(bt, vector_length);
3168 if (vm == Assembler::v0_t) {
3169 vmclr_m(vd);
3170 }
3171 switch (cond) {
3172 case BoolTest::eq: vmfeq_vv(vd, src1, src2, vm); break;
3173 case BoolTest::ne: vmfne_vv(vd, src1, src2, vm); break;
3174 case BoolTest::le: vmfle_vv(vd, src1, src2, vm); break;
3175 case BoolTest::ge: vmfge_vv(vd, src1, src2, vm); break;
3176 case BoolTest::lt: vmflt_vv(vd, src1, src2, vm); break;
3177 case BoolTest::gt: vmfgt_vv(vd, src1, src2, vm); break;
3178 default:
3179 assert(false, "unsupported compare condition");
3180 ShouldNotReachHere();
3181 }
3182 }
3183
3184 // In Matcher::scalable_predicate_reg_slots,
3185 // we assume each predicate register is one-eighth of the size of
3186 // scalable vector register, one mask bit per vector byte.
3187 void C2_MacroAssembler::spill_vmask(VectorRegister v, int offset) {
3188 vsetvli_helper(T_BYTE, MaxVectorSize >> 3);
3189 add(t0, sp, offset);
3190 vse8_v(v, t0);
3191 }
3192
3193 void C2_MacroAssembler::unspill_vmask(VectorRegister v, int offset) {
3194 vsetvli_helper(T_BYTE, MaxVectorSize >> 3);
3195 add(t0, sp, offset);
3196 vle8_v(v, t0);
3197 }
3198
3199 void C2_MacroAssembler::integer_extend_v(VectorRegister dst, BasicType dst_bt, uint vector_length,
3200 VectorRegister src, BasicType src_bt, bool is_signed) {
3201 assert(type2aelembytes(dst_bt) > type2aelembytes(src_bt) && type2aelembytes(dst_bt) <= 8 && type2aelembytes(src_bt) <= 4, "invalid element size");
3202 assert(dst_bt != T_FLOAT && dst_bt != T_DOUBLE && src_bt != T_FLOAT && src_bt != T_DOUBLE, "unsupported element type");
3203 // https://github.com/riscv/riscv-v-spec/blob/master/v-spec.adoc#52-vector-operands
3204 // The destination EEW is greater than the source EEW, the source EMUL is at least 1,
3205 // and the overlap is in the highest-numbered part of the destination register group.
3206 // Since LMUL=1, vd and vs cannot be the same.
3207 assert_different_registers(dst, src);
3208
3209 vsetvli_helper(dst_bt, vector_length);
3210 if (is_signed) {
3211 if (src_bt == T_BYTE) {
3212 switch (dst_bt) {
3213 case T_SHORT:
3214 vsext_vf2(dst, src);
3215 break;
3216 case T_INT:
3217 vsext_vf4(dst, src);
3218 break;
3219 case T_LONG:
3220 vsext_vf8(dst, src);
3221 break;
3222 default:
3223 ShouldNotReachHere();
3224 }
3225 } else if (src_bt == T_SHORT) {
3226 if (dst_bt == T_INT) {
3227 vsext_vf2(dst, src);
3228 } else {
3229 vsext_vf4(dst, src);
3230 }
3231 } else if (src_bt == T_INT) {
3232 vsext_vf2(dst, src);
3233 }
3234 } else {
3235 if (src_bt == T_BYTE) {
3236 switch (dst_bt) {
3237 case T_SHORT:
3238 vzext_vf2(dst, src);
3239 break;
3240 case T_INT:
3241 vzext_vf4(dst, src);
3242 break;
3243 case T_LONG:
3244 vzext_vf8(dst, src);
3245 break;
3246 default:
3247 ShouldNotReachHere();
3248 }
3249 } else if (src_bt == T_SHORT) {
3250 if (dst_bt == T_INT) {
3251 vzext_vf2(dst, src);
3252 } else {
3253 vzext_vf4(dst, src);
3254 }
3255 } else if (src_bt == T_INT) {
3256 vzext_vf2(dst, src);
3257 }
3258 }
3259 }
3260
3261 // Vector narrow from src to dst with specified element sizes.
3262 // High part of dst vector will be filled with zero.
3263 void C2_MacroAssembler::integer_narrow_v(VectorRegister dst, BasicType dst_bt, uint vector_length,
3264 VectorRegister src, BasicType src_bt) {
3265 assert(type2aelembytes(dst_bt) < type2aelembytes(src_bt) && type2aelembytes(dst_bt) <= 4 && type2aelembytes(src_bt) <= 8, "invalid element size");
3266 assert(dst_bt != T_FLOAT && dst_bt != T_DOUBLE && src_bt != T_FLOAT && src_bt != T_DOUBLE, "unsupported element type");
3267 mv(t0, vector_length);
3268 if (src_bt == T_LONG) {
3269 // https://github.com/riscv/riscv-v-spec/blob/master/v-spec.adoc#117-vector-narrowing-integer-right-shift-instructions
3270 // Future extensions might add support for versions that narrow to a destination that is 1/4 the width of the source.
3271 // So we can currently only scale down by 1/2 the width at a time.
3272 vsetvli(t0, t0, Assembler::e32, Assembler::mf2);
3273 vncvt_x_x_w(dst, src);
3274 if (dst_bt == T_SHORT || dst_bt == T_BYTE) {
3275 vsetvli(t0, t0, Assembler::e16, Assembler::mf2);
3276 vncvt_x_x_w(dst, dst);
3277 if (dst_bt == T_BYTE) {
3278 vsetvli(t0, t0, Assembler::e8, Assembler::mf2);
3279 vncvt_x_x_w(dst, dst);
3280 }
3281 }
3282 } else if (src_bt == T_INT) {
3283 // T_SHORT
3284 vsetvli(t0, t0, Assembler::e16, Assembler::mf2);
3285 vncvt_x_x_w(dst, src);
3286 if (dst_bt == T_BYTE) {
3287 vsetvli(t0, t0, Assembler::e8, Assembler::mf2);
3288 vncvt_x_x_w(dst, dst);
3289 }
3290 } else if (src_bt == T_SHORT) {
3291 vsetvli(t0, t0, Assembler::e8, Assembler::mf2);
3292 vncvt_x_x_w(dst, src);
3293 }
3294 }
3295
3296 #define VFCVT_SAFE(VFLOATCVT) \
3297 void C2_MacroAssembler::VFLOATCVT##_safe(VectorRegister dst, VectorRegister src) { \
3298 assert_different_registers(dst, src); \
3299 vxor_vv(dst, dst, dst); \
3300 vmfeq_vv(v0, src, src); \
3301 VFLOATCVT(dst, src, Assembler::v0_t); \
3302 }
3303
3304 VFCVT_SAFE(vfcvt_rtz_x_f_v);
3305
3306 #undef VFCVT_SAFE
3307
3308 // Extract a scalar element from an vector at position 'idx'.
3309 // The input elements in src are expected to be of integral type.
3310 void C2_MacroAssembler::extract_v(Register dst, VectorRegister src,
3311 BasicType bt, int idx, VectorRegister vtmp) {
3312 assert(is_integral_type(bt), "unsupported element type");
3313 assert(idx >= 0, "idx cannot be negative");
3314 // Only need the first element after vector slidedown
3315 vsetvli_helper(bt, 1);
3316 if (idx == 0) {
3317 vmv_x_s(dst, src);
3318 } else {
3319 slidedown_v(vtmp, src, idx);
3320 vmv_x_s(dst, vtmp);
3321 }
3322 }
3323
3324 // Extract a scalar element from a vector at position 'idx'.
3325 // The input elements in src are expected to be of integral type.
3326 void C2_MacroAssembler::extract_v(Register dst, VectorRegister src,
3327 BasicType bt, Register idx, VectorRegister vtmp) {
3328 assert(is_integral_type(bt), "unsupported element type");
3329 // Only need the first element after vector slidedown
3330 vsetvli_helper(bt, 1);
3331 vslidedown_vx(vtmp, src, idx);
3332 vmv_x_s(dst, vtmp);
3333 }
3334
3335 // Extract a scalar element from an vector at position 'idx'.
3336 // The input elements in src are expected to be of floating point type.
3337 void C2_MacroAssembler::extract_fp_v(FloatRegister dst, VectorRegister src,
3338 BasicType bt, int idx, VectorRegister vtmp) {
3339 assert(is_floating_point_type(bt), "unsupported element type");
3340 assert(idx >= 0, "idx cannot be negative");
3341 // Only need the first element after vector slidedown
3342 vsetvli_helper(bt, 1);
3343 if (idx == 0) {
3344 vfmv_f_s(dst, src);
3345 } else {
3346 slidedown_v(vtmp, src, idx);
3347 vfmv_f_s(dst, vtmp);
3348 }
3349 }
3350
3351 // Move elements down a vector register group.
3352 // Offset is the start index (offset) for the source.
3353 void C2_MacroAssembler::slidedown_v(VectorRegister dst, VectorRegister src,
3354 uint32_t offset, Register tmp) {
3355 if (is_uimm5(offset)) {
3356 vslidedown_vi(dst, src, offset);
3357 } else {
3358 mv(tmp, offset);
3359 vslidedown_vx(dst, src, tmp);
3360 }
3361 }