1 /*
2 * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/assembler.hpp"
26 #include "asm/assembler.inline.hpp"
27 #include "gc/shared/barrierSet.hpp"
28 #include "gc/shared/barrierSetAssembler.hpp"
29 #include "oops/methodData.hpp"
30 #include "opto/c2_MacroAssembler.hpp"
31 #include "opto/intrinsicnode.hpp"
32 #include "opto/output.hpp"
33 #include "opto/opcodes.hpp"
34 #include "opto/subnode.hpp"
35 #include "runtime/globals.hpp"
36 #include "runtime/objectMonitor.hpp"
37 #include "runtime/objectMonitorTable.hpp"
38 #include "runtime/stubRoutines.hpp"
39 #include "runtime/synchronizer.hpp"
40 #include "utilities/checkedCast.hpp"
41 #include "utilities/globalDefinitions.hpp"
42 #include "utilities/powerOfTwo.hpp"
43 #include "utilities/sizes.hpp"
44
45 #ifdef PRODUCT
46 #define BLOCK_COMMENT(str) /* nothing */
47 #define STOP(error) stop(error)
48 #else
49 #define BLOCK_COMMENT(str) block_comment(str)
50 #define STOP(error) block_comment(error); stop(error)
51 #endif
52
53 // C2 compiled method's prolog code.
54 void C2_MacroAssembler::verified_entry(int framesize, int stack_bang_size, bool fp_mode_24b, bool is_stub) {
55 assert(stack_bang_size >= framesize || stack_bang_size <= 0, "stack bang size incorrect");
56
57 assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
58 // Remove word for return addr
59 framesize -= wordSize;
60 stack_bang_size -= wordSize;
61
62 // Calls to C2R adapters often do not accept exceptional returns.
63 // We require that their callers must bang for them. But be careful, because
64 // some VM calls (such as call site linkage) can use several kilobytes of
65 // stack. But the stack safety zone should account for that.
66 // See bugs 4446381, 4468289, 4497237.
67 if (stack_bang_size > 0) {
68 generate_stack_overflow_check(stack_bang_size);
69
70 // We always push rbp, so that on return to interpreter rbp, will be
71 // restored correctly and we can correct the stack.
72 push(rbp);
73 // Save caller's stack pointer into RBP if the frame pointer is preserved.
74 if (PreserveFramePointer) {
75 mov(rbp, rsp);
76 }
77 // Remove word for ebp
78 framesize -= wordSize;
79
80 // Create frame
81 if (framesize) {
82 subptr(rsp, framesize);
83 }
84 } else {
85 subptr(rsp, framesize);
86
87 // Save RBP register now.
88 framesize -= wordSize;
89 movptr(Address(rsp, framesize), rbp);
90 // Save caller's stack pointer into RBP if the frame pointer is preserved.
91 if (PreserveFramePointer) {
92 movptr(rbp, rsp);
93 if (framesize > 0) {
94 addptr(rbp, framesize);
95 }
96 }
97 }
98
99 if (VerifyStackAtCalls) { // Majik cookie to verify stack depth
100 framesize -= wordSize;
101 movptr(Address(rsp, framesize), (int32_t)0xbadb100d);
102 }
103
104 #ifdef ASSERT
105 if (VerifyStackAtCalls) {
106 Label L;
107 push(rax);
108 mov(rax, rsp);
109 andptr(rax, StackAlignmentInBytes-1);
110 cmpptr(rax, StackAlignmentInBytes-wordSize);
111 pop(rax);
112 jcc(Assembler::equal, L);
113 STOP("Stack is not properly aligned!");
114 bind(L);
115 }
116 #endif
117
118 if (!is_stub) {
119 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
120 // We put the non-hot code of the nmethod entry barrier out-of-line in a stub.
121 Label dummy_slow_path;
122 Label dummy_continuation;
123 Label* slow_path = &dummy_slow_path;
124 Label* continuation = &dummy_continuation;
125 if (!Compile::current()->output()->in_scratch_emit_size()) {
126 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
127 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
128 Compile::current()->output()->add_stub(stub);
129 slow_path = &stub->entry();
130 continuation = &stub->continuation();
131 }
132 bs->nmethod_entry_barrier(this, slow_path, continuation);
133 }
134 }
135
136 inline Assembler::AvxVectorLen C2_MacroAssembler::vector_length_encoding(int vlen_in_bytes) {
137 switch (vlen_in_bytes) {
138 case 4: // fall-through
139 case 8: // fall-through
140 case 16: return Assembler::AVX_128bit;
141 case 32: return Assembler::AVX_256bit;
142 case 64: return Assembler::AVX_512bit;
143
144 default: {
145 ShouldNotReachHere();
146 return Assembler::AVX_NoVec;
147 }
148 }
149 }
150
151 // fast_lock and fast_unlock used by C2
152
153 // Because the transitions from emitted code to the runtime
154 // monitorenter/exit helper stubs are so slow it's critical that
155 // we inline both the lock-stack fast path and the inflated fast path.
156 //
157 // See also: cmpFastLock and cmpFastUnlock.
158 //
159 // What follows is a specialized inline transliteration of the code
160 // in enter() and exit(). If we're concerned about I$ bloat another
161 // option would be to emit TrySlowEnter and TrySlowExit methods
162 // at startup-time. These methods would accept arguments as
163 // (rax,=Obj, rbx=Self, rcx=box, rdx=Scratch) and return success-failure
164 // indications in the icc.ZFlag. fast_lock and fast_unlock would simply
165 // marshal the arguments and emit calls to TrySlowEnter and TrySlowExit.
166 // In practice, however, the # of lock sites is bounded and is usually small.
167 // Besides the call overhead, TrySlowEnter and TrySlowExit might suffer
168 // if the processor uses simple bimodal branch predictors keyed by EIP
169 // Since the helper routines would be called from multiple synchronization
170 // sites.
171 //
172 // An even better approach would be write "MonitorEnter()" and "MonitorExit()"
173 // in java - using j.u.c and unsafe - and just bind the lock and unlock sites
174 // to those specialized methods. That'd give us a mostly platform-independent
175 // implementation that the JITs could optimize and inline at their pleasure.
176 // Done correctly, the only time we'd need to cross to native could would be
177 // to park() or unpark() threads. We'd also need a few more unsafe operators
178 // to (a) prevent compiler-JIT reordering of non-volatile accesses, and
179 // (b) explicit barriers or fence operations.
180 //
181 // TODO:
182 //
183 // * Arrange for C2 to pass "Self" into fast_lock and fast_unlock in one of the registers (scr).
184 // This avoids manifesting the Self pointer in the fast_lock and fast_unlock terminals.
185 // Given TLAB allocation, Self is usually manifested in a register, so passing it into
186 // the lock operators would typically be faster than reifying Self.
187 //
188 // * Ideally I'd define the primitives as:
189 // fast_lock (nax Obj, nax box, EAX tmp, nax scr) where box, tmp and scr are KILLED.
190 // fast_unlock (nax Obj, EAX box, nax tmp) where box and tmp are KILLED
191 // Unfortunately ADLC bugs prevent us from expressing the ideal form.
192 // Instead, we're stuck with a rather awkward and brittle register assignments below.
193 // Furthermore the register assignments are overconstrained, possibly resulting in
194 // sub-optimal code near the synchronization site.
195 //
196 // * Eliminate the sp-proximity tests and just use "== Self" tests instead.
197 // Alternately, use a better sp-proximity test.
198 //
199 // * Currently ObjectMonitor._Owner can hold either an sp value or a (THREAD *) value.
200 // Either one is sufficient to uniquely identify a thread.
201 // TODO: eliminate use of sp in _owner and use get_thread(tr) instead.
202 //
203 // * Intrinsify notify() and notifyAll() for the common cases where the
204 // object is locked by the calling thread but the waitlist is empty.
205 // avoid the expensive JNI call to JVM_Notify() and JVM_NotifyAll().
206 //
207 // * use jccb and jmpb instead of jcc and jmp to improve code density.
208 // But beware of excessive branch density on AMD Opterons.
209 //
210 // * Both fast_lock and fast_unlock set the ICC.ZF to indicate success
211 // or failure of the fast path. If the fast path fails then we pass
212 // control to the slow path, typically in C. In fast_lock and
213 // fast_unlock we often branch to DONE_LABEL, just to find that C2
214 // will emit a conditional branch immediately after the node.
215 // So we have branches to branches and lots of ICC.ZF games.
216 // Instead, it might be better to have C2 pass a "FailureLabel"
217 // into fast_lock and fast_unlock. In the case of success, control
218 // will drop through the node. ICC.ZF is undefined at exit.
219 // In the case of failure, the node will branch directly to the
220 // FailureLabel
221
222 // obj: object to lock
223 // box: on-stack box address -- KILLED
224 // rax: tmp -- KILLED
225 // t : tmp -- KILLED
226 void C2_MacroAssembler::fast_lock(Register obj, Register box, Register rax_reg,
227 Register t, Register thread) {
228 assert(rax_reg == rax, "Used for CAS");
229 assert_different_registers(obj, box, rax_reg, t, thread);
230
231 // Handle inflated monitor.
232 Label inflated;
233 // Finish fast lock successfully. ZF value is irrelevant.
234 Label locked;
235 // Finish fast lock unsuccessfully. MUST jump with ZF == 0
236 Label slow_path;
237
238 if (UseObjectMonitorTable) {
239 // Clear cache in case fast locking succeeds or we need to take the slow-path.
240 movptr(Address(box, BasicLock::object_monitor_cache_offset_in_bytes()), 0);
241 }
242
243 if (DiagnoseSyncOnValueBasedClasses != 0) {
244 load_klass(rax_reg, obj, t);
245 testb(Address(rax_reg, Klass::misc_flags_offset()), KlassFlags::_misc_is_value_based_class);
246 jcc(Assembler::notZero, slow_path);
247 }
248
249 const Register mark = t;
250
251 { // Fast Lock
252
253 Label push;
254
255 const Register top = UseObjectMonitorTable ? rax_reg : box;
256
257 // Load the mark.
258 movptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
259
260 // Prefetch top.
261 movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
262
263 // Check for monitor (0b10).
264 testptr(mark, markWord::monitor_value);
265 jcc(Assembler::notZero, inflated);
266
267 // Check if lock-stack is full.
268 cmpl(top, LockStack::end_offset() - 1);
269 jcc(Assembler::greater, slow_path);
270
271 // Check if recursive.
272 cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
273 jccb(Assembler::equal, push);
274
275 // Try to lock. Transition lock bits 0b01 => 0b00
276 movptr(rax_reg, mark);
277 orptr(rax_reg, markWord::unlocked_value);
278 andptr(mark, ~(int32_t)markWord::unlocked_value);
279 lock(); cmpxchgptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
280 jcc(Assembler::notEqual, slow_path);
281
282 if (UseObjectMonitorTable) {
283 // Need to reload top, clobbered by CAS.
284 movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
285 }
286 bind(push);
287 // After successful lock, push object on lock-stack.
288 movptr(Address(thread, top), obj);
289 addl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
290 jmp(locked);
291 }
292
293 { // Handle inflated monitor.
294 bind(inflated);
295
296 const Register monitor = t;
297 // Offsets into the current thread's object monitor cache (omc).
298 const ByteSize thr_omc_offset = JavaThread::om_cache_offset();
299 const ByteSize omc_monitor_offset = OMCache::monitor_offset();
300 const ByteSize omc_obj_offset = OMCache::obj_offset();
301
302 if (!UseObjectMonitorTable) {
303 assert(mark == monitor, "should be the same here");
304 } else {
305 const Register hash = t;
306 Label monitor_found;
307
308 // Look for the monitor in the current thread's object monitor cache (omc).
309
310 movptr(monitor, Address(thread, thr_omc_offset + omc_monitor_offset));
311 cmpptr(obj, Address(thread, thr_omc_offset + omc_obj_offset));
312 jccb(Assembler::equal, monitor_found);
313
314 // Look for the monitor in the table.
315
316 // Get the hash code.
317 movptr(hash, Address(obj, oopDesc::mark_offset_in_bytes()));
318 shrq(hash, markWord::hash_shift);
319 andq(hash, markWord::hash_mask);
320
321 // Get the table and calculate the bucket's address.
322 lea(rax_reg, ExternalAddress(ObjectMonitorTable::current_table_address()));
323 movptr(rax_reg, Address(rax_reg));
324 andq(hash, Address(rax_reg, ObjectMonitorTable::table_capacity_mask_offset()));
325 movptr(rax_reg, Address(rax_reg, ObjectMonitorTable::table_buckets_offset()));
326
327 // Read the monitor from the bucket.
328 movptr(monitor, Address(rax_reg, hash, Address::times_ptr));
329
330 // Check if the monitor in the bucket is special (empty, tombstone or removed)
331 cmpptr(monitor, ObjectMonitorTable::SpecialPointerValues::below_is_special);
332 jcc(Assembler::below, slow_path);
333
334 // Check if object matches.
335 movptr(rax_reg, Address(monitor, ObjectMonitor::object_offset()));
336 BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
337 bs_asm->try_peek_weak_handle_in_nmethod(this, rax_reg, rax_reg, slow_path);
338 cmpptr(rax_reg, obj);
339 jcc(Assembler::notEqual, slow_path);
340
341 // Store the monitor in the current thread's object monitor cache (omc).
342 movptr(Address(thread, thr_omc_offset + omc_monitor_offset), monitor);
343 movptr(Address(thread, thr_omc_offset + omc_obj_offset), obj);
344
345 bind(monitor_found);
346 }
347 const ByteSize monitor_tag = in_ByteSize(UseObjectMonitorTable ? 0 : checked_cast<int>(markWord::monitor_value));
348 const Address recursions_address(monitor, ObjectMonitor::recursions_offset() - monitor_tag);
349 const Address owner_address(monitor, ObjectMonitor::owner_offset() - monitor_tag);
350
351 Label monitor_locked;
352 // Lock the monitor.
353
354 if (UseObjectMonitorTable) {
355 // Cache the monitor for unlock before trashing box. On failure to acquire
356 // the lock, the slow path will reset the entry accordingly (see CacheSetter).
357 movptr(Address(box, BasicLock::object_monitor_cache_offset_in_bytes()), monitor);
358 }
359
360 // Try to CAS owner (no owner => current thread's _monitor_owner_id).
361 xorptr(rax_reg, rax_reg);
362 movptr(box, Address(thread, JavaThread::monitor_owner_id_offset()));
363 lock(); cmpxchgptr(box, owner_address);
364 jccb(Assembler::equal, monitor_locked);
365
366 // Check if recursive.
367 cmpptr(box, rax_reg);
368 jccb(Assembler::notEqual, slow_path);
369
370 // Recursive.
371 increment(recursions_address);
372
373 bind(monitor_locked);
374 }
375
376 bind(locked);
377 // Set ZF = 1
378 xorl(rax_reg, rax_reg);
379
380 #ifdef ASSERT
381 // Check that locked label is reached with ZF set.
382 Label zf_correct;
383 Label zf_bad_zero;
384 jcc(Assembler::zero, zf_correct);
385 jmp(zf_bad_zero);
386 #endif
387
388 bind(slow_path);
389 #ifdef ASSERT
390 // Check that slow_path label is reached with ZF not set.
391 jcc(Assembler::notZero, zf_correct);
392 stop("Fast Lock ZF != 0");
393 bind(zf_bad_zero);
394 stop("Fast Lock ZF != 1");
395 bind(zf_correct);
396 #endif
397 // C2 uses the value of ZF to determine the continuation.
398 }
399
400 // obj: object to lock
401 // rax: tmp -- KILLED
402 // t : tmp - cannot be obj nor rax -- KILLED
403 //
404 // Some commentary on balanced locking:
405 //
406 // fast_lock and fast_unlock are emitted only for provably balanced lock sites.
407 // Methods that don't have provably balanced locking are forced to run in the
408 // interpreter - such methods won't be compiled to use fast_lock and fast_unlock.
409 // The interpreter provides two properties:
410 // I1: At return-time the interpreter automatically and quietly unlocks any
411 // objects acquired in the current activation (frame). Recall that the
412 // interpreter maintains an on-stack list of locks currently held by
413 // a frame.
414 // I2: If a method attempts to unlock an object that is not held by the
415 // frame the interpreter throws IMSX.
416 //
417 // Lets say A(), which has provably balanced locking, acquires O and then calls B().
418 // B() doesn't have provably balanced locking so it runs in the interpreter.
419 // Control returns to A() and A() unlocks O. By I1 and I2, above, we know that O
420 // is still locked by A().
421 //
422 // The only other source of unbalanced locking would be JNI. The "Java Native Interface
423 // Specification" states that an object locked by JNI's MonitorEnter should not be
424 // unlocked by "normal" java-level locking and vice-versa. The specification doesn't
425 // specify what will occur if a program engages in such mixed-mode locking, however.
426 // Arguably given that the spec legislates the JNI case as undefined our implementation
427 // could reasonably *avoid* checking owner in fast_unlock().
428 // In the interest of performance we elide m->Owner==Self check in unlock.
429 // A perfectly viable alternative is to elide the owner check except when
430 // Xcheck:jni is enabled.
431
432 void C2_MacroAssembler::fast_unlock(Register obj, Register reg_rax, Register t, Register thread) {
433 assert(reg_rax == rax, "Used for CAS");
434 assert_different_registers(obj, reg_rax, t);
435
436 // Handle inflated monitor.
437 Label inflated, inflated_check_lock_stack;
438 // Finish fast unlock successfully. MUST jump with ZF == 1
439 Label unlocked, slow_path;
440
441 const Register mark = t;
442 const Register monitor = t;
443 const Register top = UseObjectMonitorTable ? t : reg_rax;
444 const Register box = reg_rax;
445
446 Label dummy;
447 C2FastUnlockStub* stub = nullptr;
448
449 if (!Compile::current()->output()->in_scratch_emit_size()) {
450 stub = new (Compile::current()->comp_arena()) C2FastUnlockStub(obj, mark, reg_rax, thread);
451 Compile::current()->output()->add_stub(stub);
452 }
453
454 Label& push_and_slow_path = stub == nullptr ? dummy : stub->push_and_slow_path();
455
456 { // Fast Unlock
457
458 // Load top.
459 movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
460
461 if (!UseObjectMonitorTable) {
462 // Prefetch mark.
463 movptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
464 }
465
466 // Check if obj is top of lock-stack.
467 cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
468 // Top of lock stack was not obj. Must be monitor.
469 jcc(Assembler::notEqual, inflated_check_lock_stack);
470
471 // Pop lock-stack.
472 DEBUG_ONLY(movptr(Address(thread, top, Address::times_1, -oopSize), 0);)
473 subl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
474
475 // Check if recursive.
476 cmpptr(obj, Address(thread, top, Address::times_1, -2 * oopSize));
477 jcc(Assembler::equal, unlocked);
478
479 // We elide the monitor check, let the CAS fail instead.
480
481 if (UseObjectMonitorTable) {
482 // Load mark.
483 movptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
484 }
485
486 // Try to unlock. Transition lock bits 0b00 => 0b01
487 movptr(reg_rax, mark);
488 andptr(reg_rax, ~(int32_t)markWord::lock_mask_in_place);
489 orptr(mark, markWord::unlocked_value);
490 lock(); cmpxchgptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
491 jcc(Assembler::notEqual, push_and_slow_path);
492 jmp(unlocked);
493 }
494
495
496 { // Handle inflated monitor.
497 bind(inflated_check_lock_stack);
498 #ifdef ASSERT
499 Label check_done;
500 subl(top, oopSize);
501 cmpl(top, in_bytes(JavaThread::lock_stack_base_offset()));
502 jcc(Assembler::below, check_done);
503 cmpptr(obj, Address(thread, top));
504 jcc(Assembler::notEqual, inflated_check_lock_stack);
505 stop("Fast Unlock lock on stack");
506 bind(check_done);
507 if (UseObjectMonitorTable) {
508 movptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
509 }
510 testptr(mark, markWord::monitor_value);
511 jcc(Assembler::notZero, inflated);
512 stop("Fast Unlock not monitor");
513 #endif
514
515 bind(inflated);
516
517 if (!UseObjectMonitorTable) {
518 assert(mark == monitor, "should be the same here");
519 } else {
520 // Uses ObjectMonitorTable. Look for the monitor in our BasicLock on the stack.
521 movptr(monitor, Address(box, BasicLock::object_monitor_cache_offset_in_bytes()));
522 // null check with ZF == 0, no valid pointer below alignof(ObjectMonitor*)
523 cmpptr(monitor, alignof(ObjectMonitor*));
524 jcc(Assembler::below, slow_path);
525 }
526 const ByteSize monitor_tag = in_ByteSize(UseObjectMonitorTable ? 0 : checked_cast<int>(markWord::monitor_value));
527 const Address recursions_address{monitor, ObjectMonitor::recursions_offset() - monitor_tag};
528 const Address succ_address{monitor, ObjectMonitor::succ_offset() - monitor_tag};
529 const Address entry_list_address{monitor, ObjectMonitor::entry_list_offset() - monitor_tag};
530 const Address owner_address{monitor, ObjectMonitor::owner_offset() - monitor_tag};
531
532 Label recursive;
533
534 // Check if recursive.
535 cmpptr(recursions_address, 0);
536 jcc(Assembler::notZero, recursive);
537
538 // Set owner to null.
539 // Release to satisfy the JMM
540 movptr(owner_address, NULL_WORD);
541 // We need a full fence after clearing owner to avoid stranding.
542 // StoreLoad achieves this.
543 membar(StoreLoad);
544
545 // Check if the entry_list is empty.
546 cmpptr(entry_list_address, NULL_WORD);
547 jcc(Assembler::zero, unlocked); // If so we are done.
548
549 // Check if there is a successor.
550 cmpptr(succ_address, NULL_WORD);
551 jcc(Assembler::notZero, unlocked); // If so we are done.
552
553 // Save the monitor pointer in the current thread, so we can try to
554 // reacquire the lock in SharedRuntime::monitor_exit_helper().
555 if (!UseObjectMonitorTable) {
556 andptr(monitor, ~(int32_t)markWord::monitor_value);
557 }
558 movptr(Address(thread, JavaThread::unlocked_inflated_monitor_offset()), monitor);
559
560 orl(t, 1); // Fast Unlock ZF = 0
561 jmpb(slow_path);
562
563 // Recursive unlock.
564 bind(recursive);
565 decrement(recursions_address);
566 }
567
568 bind(unlocked);
569 xorl(t, t); // Fast Unlock ZF = 1
570
571 #ifdef ASSERT
572 // Check that unlocked label is reached with ZF set.
573 Label zf_correct;
574 Label zf_bad_zero;
575 jcc(Assembler::zero, zf_correct);
576 jmp(zf_bad_zero);
577 #endif
578
579 bind(slow_path);
580 if (stub != nullptr) {
581 bind(stub->slow_path_continuation());
582 }
583 #ifdef ASSERT
584 // Check that stub->continuation() label is reached with ZF not set.
585 jcc(Assembler::notZero, zf_correct);
586 stop("Fast Unlock ZF != 0");
587 bind(zf_bad_zero);
588 stop("Fast Unlock ZF != 1");
589 bind(zf_correct);
590 #endif
591 // C2 uses the value of ZF to determine the continuation.
592 }
593
594 static void abort_verify_int_in_range(uint idx, jint val, jint lo, jint hi) {
595 fatal("Invalid CastII, idx: %u, val: %d, lo: %d, hi: %d", idx, val, lo, hi);
596 }
597
598 static void reconstruct_frame_pointer_helper(MacroAssembler* masm, Register dst) {
599 const int framesize = Compile::current()->output()->frame_size_in_bytes();
600 masm->movptr(dst, rsp);
601 if (framesize > 2 * wordSize) {
602 masm->addptr(dst, framesize - 2 * wordSize);
603 }
604 }
605
606 void C2_MacroAssembler::reconstruct_frame_pointer(Register rtmp) {
607 if (PreserveFramePointer) {
608 // frame pointer is valid
609 #ifdef ASSERT
610 // Verify frame pointer value in rbp.
611 reconstruct_frame_pointer_helper(this, rtmp);
612 Label L_success;
613 cmpq(rbp, rtmp);
614 jccb(Assembler::equal, L_success);
615 STOP("frame pointer mismatch");
616 bind(L_success);
617 #endif // ASSERT
618 } else {
619 reconstruct_frame_pointer_helper(this, rbp);
620 }
621 }
622
623 void C2_MacroAssembler::verify_int_in_range(uint idx, const TypeInt* t, Register val) {
624 jint lo = t->_lo;
625 jint hi = t->_hi;
626 assert(lo < hi, "type should not be empty or constant, idx: %u, lo: %d, hi: %d", idx, lo, hi);
627 if (t == TypeInt::INT) {
628 return;
629 }
630
631 BLOCK_COMMENT("CastII {");
632 Label fail;
633 Label succeed;
634
635 if (lo != min_jint) {
636 cmpl(val, lo);
637 jccb(Assembler::less, fail);
638 }
639 if (hi != max_jint) {
640 cmpl(val, hi);
641 jccb(Assembler::greater, fail);
642 }
643 jmpb(succeed);
644
645 bind(fail);
646 movl(c_rarg0, idx);
647 movl(c_rarg1, val);
648 movl(c_rarg2, lo);
649 movl(c_rarg3, hi);
650 reconstruct_frame_pointer(rscratch1);
651 call(RuntimeAddress(CAST_FROM_FN_PTR(address, abort_verify_int_in_range)));
652 hlt();
653 bind(succeed);
654 BLOCK_COMMENT("} // CastII");
655 }
656
657 static void abort_verify_long_in_range(uint idx, jlong val, jlong lo, jlong hi) {
658 fatal("Invalid CastLL, idx: %u, val: " JLONG_FORMAT ", lo: " JLONG_FORMAT ", hi: " JLONG_FORMAT, idx, val, lo, hi);
659 }
660
661 void C2_MacroAssembler::verify_long_in_range(uint idx, const TypeLong* t, Register val, Register tmp) {
662 jlong lo = t->_lo;
663 jlong hi = t->_hi;
664 assert(lo < hi, "type should not be empty or constant, idx: %u, lo: " JLONG_FORMAT ", hi: " JLONG_FORMAT, idx, lo, hi);
665 if (t == TypeLong::LONG) {
666 return;
667 }
668
669 BLOCK_COMMENT("CastLL {");
670 Label fail;
671 Label succeed;
672
673 auto cmp_val = [&](jlong bound) {
674 if (is_simm32(bound)) {
675 cmpq(val, checked_cast<int>(bound));
676 } else {
677 mov64(tmp, bound);
678 cmpq(val, tmp);
679 }
680 };
681
682 if (lo != min_jlong) {
683 cmp_val(lo);
684 jccb(Assembler::less, fail);
685 }
686 if (hi != max_jlong) {
687 cmp_val(hi);
688 jccb(Assembler::greater, fail);
689 }
690 jmpb(succeed);
691
692 bind(fail);
693 movl(c_rarg0, idx);
694 movq(c_rarg1, val);
695 mov64(c_rarg2, lo);
696 mov64(c_rarg3, hi);
697 reconstruct_frame_pointer(rscratch1);
698 call(RuntimeAddress(CAST_FROM_FN_PTR(address, abort_verify_long_in_range)));
699 hlt();
700 bind(succeed);
701 BLOCK_COMMENT("} // CastLL");
702 }
703
704 //-------------------------------------------------------------------------------------------
705 // Generic instructions support for use in .ad files C2 code generation
706
707 void C2_MacroAssembler::vabsnegd(int opcode, XMMRegister dst, XMMRegister src) {
708 if (dst != src) {
709 movdqu(dst, src);
710 }
711 if (opcode == Op_AbsVD) {
712 andpd(dst, ExternalAddress(StubRoutines::x86::vector_double_sign_mask()), noreg);
713 } else {
714 assert((opcode == Op_NegVD),"opcode should be Op_NegD");
715 xorpd(dst, ExternalAddress(StubRoutines::x86::vector_double_sign_flip()), noreg);
716 }
717 }
718
719 void C2_MacroAssembler::vabsnegd(int opcode, XMMRegister dst, XMMRegister src, int vector_len) {
720 if (opcode == Op_AbsVD) {
721 vandpd(dst, src, ExternalAddress(StubRoutines::x86::vector_double_sign_mask()), vector_len, noreg);
722 } else {
723 assert((opcode == Op_NegVD),"opcode should be Op_NegD");
724 vxorpd(dst, src, ExternalAddress(StubRoutines::x86::vector_double_sign_flip()), vector_len, noreg);
725 }
726 }
727
728 void C2_MacroAssembler::vabsnegf(int opcode, XMMRegister dst, XMMRegister src) {
729 if (dst != src) {
730 movdqu(dst, src);
731 }
732 if (opcode == Op_AbsVF) {
733 andps(dst, ExternalAddress(StubRoutines::x86::vector_float_sign_mask()), noreg);
734 } else {
735 assert((opcode == Op_NegVF),"opcode should be Op_NegF");
736 xorps(dst, ExternalAddress(StubRoutines::x86::vector_float_sign_flip()), noreg);
737 }
738 }
739
740 void C2_MacroAssembler::vabsnegf(int opcode, XMMRegister dst, XMMRegister src, int vector_len) {
741 if (opcode == Op_AbsVF) {
742 vandps(dst, src, ExternalAddress(StubRoutines::x86::vector_float_sign_mask()), vector_len, noreg);
743 } else {
744 assert((opcode == Op_NegVF),"opcode should be Op_NegF");
745 vxorps(dst, src, ExternalAddress(StubRoutines::x86::vector_float_sign_flip()), vector_len, noreg);
746 }
747 }
748
749 void C2_MacroAssembler::pminmax(int opcode, BasicType elem_bt, XMMRegister dst, XMMRegister src, XMMRegister tmp) {
750 assert(opcode == Op_MinV || opcode == Op_MaxV, "sanity");
751 assert(tmp == xnoreg || elem_bt == T_LONG, "unused");
752
753 if (opcode == Op_MinV) {
754 if (elem_bt == T_BYTE) {
755 pminsb(dst, src);
756 } else if (elem_bt == T_SHORT) {
757 pminsw(dst, src);
758 } else if (elem_bt == T_INT) {
759 pminsd(dst, src);
760 } else {
761 assert(elem_bt == T_LONG, "required");
762 assert(tmp == xmm0, "required");
763 assert_different_registers(dst, src, tmp);
764 movdqu(xmm0, dst);
765 pcmpgtq(xmm0, src);
766 blendvpd(dst, src); // xmm0 as mask
767 }
768 } else { // opcode == Op_MaxV
769 if (elem_bt == T_BYTE) {
770 pmaxsb(dst, src);
771 } else if (elem_bt == T_SHORT) {
772 pmaxsw(dst, src);
773 } else if (elem_bt == T_INT) {
774 pmaxsd(dst, src);
775 } else {
776 assert(elem_bt == T_LONG, "required");
777 assert(tmp == xmm0, "required");
778 assert_different_registers(dst, src, tmp);
779 movdqu(xmm0, src);
780 pcmpgtq(xmm0, dst);
781 blendvpd(dst, src); // xmm0 as mask
782 }
783 }
784 }
785
786 void C2_MacroAssembler::vpuminmax(int opcode, BasicType elem_bt, XMMRegister dst,
787 XMMRegister src1, Address src2, int vlen_enc) {
788 assert(opcode == Op_UMinV || opcode == Op_UMaxV, "sanity");
789 if (opcode == Op_UMinV) {
790 switch(elem_bt) {
791 case T_BYTE: vpminub(dst, src1, src2, vlen_enc); break;
792 case T_SHORT: vpminuw(dst, src1, src2, vlen_enc); break;
793 case T_INT: vpminud(dst, src1, src2, vlen_enc); break;
794 case T_LONG: evpminuq(dst, k0, src1, src2, false, vlen_enc); break;
795 default: fatal("Unsupported type %s", type2name(elem_bt)); break;
796 }
797 } else {
798 assert(opcode == Op_UMaxV, "required");
799 switch(elem_bt) {
800 case T_BYTE: vpmaxub(dst, src1, src2, vlen_enc); break;
801 case T_SHORT: vpmaxuw(dst, src1, src2, vlen_enc); break;
802 case T_INT: vpmaxud(dst, src1, src2, vlen_enc); break;
803 case T_LONG: evpmaxuq(dst, k0, src1, src2, false, vlen_enc); break;
804 default: fatal("Unsupported type %s", type2name(elem_bt)); break;
805 }
806 }
807 }
808
809 void C2_MacroAssembler::vpuminmaxq(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister xtmp1, XMMRegister xtmp2, int vlen_enc) {
810 // For optimality, leverage a full vector width of 512 bits
811 // for operations over smaller vector sizes on AVX512 targets.
812 if (VM_Version::supports_evex() && !VM_Version::supports_avx512vl()) {
813 if (opcode == Op_UMaxV) {
814 evpmaxuq(dst, k0, src1, src2, false, Assembler::AVX_512bit);
815 } else {
816 assert(opcode == Op_UMinV, "required");
817 evpminuq(dst, k0, src1, src2, false, Assembler::AVX_512bit);
818 }
819 } else {
820 // T1 = -1
821 vpcmpeqq(xtmp1, xtmp1, xtmp1, vlen_enc);
822 // T1 = -1 << 63
823 vpsllq(xtmp1, xtmp1, 63, vlen_enc);
824 // Convert SRC2 to signed value i.e. T2 = T1 + SRC2
825 vpaddq(xtmp2, xtmp1, src2, vlen_enc);
826 // Convert SRC1 to signed value i.e. T1 = T1 + SRC1
827 vpaddq(xtmp1, xtmp1, src1, vlen_enc);
828 // Mask = T2 > T1
829 vpcmpgtq(xtmp1, xtmp2, xtmp1, vlen_enc);
830 if (opcode == Op_UMaxV) {
831 // Res = Mask ? Src2 : Src1
832 vpblendvb(dst, src1, src2, xtmp1, vlen_enc);
833 } else {
834 // Res = Mask ? Src1 : Src2
835 vpblendvb(dst, src2, src1, xtmp1, vlen_enc);
836 }
837 }
838 }
839
840 void C2_MacroAssembler::vpuminmax(int opcode, BasicType elem_bt, XMMRegister dst,
841 XMMRegister src1, XMMRegister src2, int vlen_enc) {
842 assert(opcode == Op_UMinV || opcode == Op_UMaxV, "sanity");
843 if (opcode == Op_UMinV) {
844 switch(elem_bt) {
845 case T_BYTE: vpminub(dst, src1, src2, vlen_enc); break;
846 case T_SHORT: vpminuw(dst, src1, src2, vlen_enc); break;
847 case T_INT: vpminud(dst, src1, src2, vlen_enc); break;
848 case T_LONG: evpminuq(dst, k0, src1, src2, false, vlen_enc); break;
849 default: fatal("Unsupported type %s", type2name(elem_bt)); break;
850 }
851 } else {
852 assert(opcode == Op_UMaxV, "required");
853 switch(elem_bt) {
854 case T_BYTE: vpmaxub(dst, src1, src2, vlen_enc); break;
855 case T_SHORT: vpmaxuw(dst, src1, src2, vlen_enc); break;
856 case T_INT: vpmaxud(dst, src1, src2, vlen_enc); break;
857 case T_LONG: evpmaxuq(dst, k0, src1, src2, false, vlen_enc); break;
858 default: fatal("Unsupported type %s", type2name(elem_bt)); break;
859 }
860 }
861 }
862
863 void C2_MacroAssembler::vpminmax(int opcode, BasicType elem_bt,
864 XMMRegister dst, XMMRegister src1, XMMRegister src2,
865 int vlen_enc) {
866 assert(opcode == Op_MinV || opcode == Op_MaxV, "sanity");
867
868 if (opcode == Op_MinV) {
869 if (elem_bt == T_BYTE) {
870 vpminsb(dst, src1, src2, vlen_enc);
871 } else if (elem_bt == T_SHORT) {
872 vpminsw(dst, src1, src2, vlen_enc);
873 } else if (elem_bt == T_INT) {
874 vpminsd(dst, src1, src2, vlen_enc);
875 } else {
876 assert(elem_bt == T_LONG, "required");
877 if (UseAVX > 2 && (vlen_enc == Assembler::AVX_512bit || VM_Version::supports_avx512vl())) {
878 vpminsq(dst, src1, src2, vlen_enc);
879 } else {
880 assert_different_registers(dst, src1, src2);
881 vpcmpgtq(dst, src1, src2, vlen_enc);
882 vblendvpd(dst, src1, src2, dst, vlen_enc);
883 }
884 }
885 } else { // opcode == Op_MaxV
886 if (elem_bt == T_BYTE) {
887 vpmaxsb(dst, src1, src2, vlen_enc);
888 } else if (elem_bt == T_SHORT) {
889 vpmaxsw(dst, src1, src2, vlen_enc);
890 } else if (elem_bt == T_INT) {
891 vpmaxsd(dst, src1, src2, vlen_enc);
892 } else {
893 assert(elem_bt == T_LONG, "required");
894 if (UseAVX > 2 && (vlen_enc == Assembler::AVX_512bit || VM_Version::supports_avx512vl())) {
895 vpmaxsq(dst, src1, src2, vlen_enc);
896 } else {
897 assert_different_registers(dst, src1, src2);
898 vpcmpgtq(dst, src1, src2, vlen_enc);
899 vblendvpd(dst, src2, src1, dst, vlen_enc);
900 }
901 }
902 }
903 }
904
905 // Float/Double min max
906
907 void C2_MacroAssembler::vminmax_fp(int opcode, BasicType elem_bt,
908 XMMRegister dst, XMMRegister a, XMMRegister b,
909 XMMRegister tmp, XMMRegister atmp, XMMRegister btmp,
910 int vlen_enc) {
911 assert(UseAVX > 0, "required");
912 assert(opcode == Op_MinV || opcode == Op_MinReductionV ||
913 opcode == Op_MaxV || opcode == Op_MaxReductionV, "sanity");
914 assert(elem_bt == T_FLOAT || elem_bt == T_DOUBLE, "sanity");
915 assert_different_registers(a, tmp, atmp, btmp);
916 assert_different_registers(b, tmp, atmp, btmp);
917
918 bool is_min = (opcode == Op_MinV || opcode == Op_MinReductionV);
919 bool is_double_word = is_double_word_type(elem_bt);
920
921 /* Note on 'non-obvious' assembly sequence:
922 *
923 * While there are vminps/vmaxps instructions, there are two important differences between hardware
924 * and Java on how they handle floats:
925 * a. -0.0 and +0.0 are considered equal (vminps/vmaxps will return second parameter when inputs are equal)
926 * b. NaN is not necesarily propagated (vminps/vmaxps will return second parameter when either input is NaN)
927 *
928 * It is still more efficient to use vminps/vmaxps, but with some pre/post-processing:
929 * a. -0.0/+0.0: Bias negative (positive) numbers to second parameter before vminps (vmaxps)
930 * (only useful when signs differ, noop otherwise)
931 * b. NaN: Check if it was the first parameter that had the NaN (with vcmp[UNORD_Q])
932
933 * Following pseudo code describes the algorithm for max[FD] (Min algorithm is on similar lines):
934 * btmp = (b < +0.0) ? a : b
935 * atmp = (b < +0.0) ? b : a
936 * Tmp = Max_Float(atmp , btmp)
937 * Res = (atmp == NaN) ? atmp : Tmp
938 */
939
940 void (MacroAssembler::*vblend)(XMMRegister, XMMRegister, XMMRegister, XMMRegister, int, bool, XMMRegister);
941 void (MacroAssembler::*vmaxmin)(XMMRegister, XMMRegister, XMMRegister, int);
942 void (MacroAssembler::*vcmp)(XMMRegister, XMMRegister, XMMRegister, int, int);
943 XMMRegister mask;
944
945 if (!is_double_word && is_min) {
946 mask = a;
947 vblend = &MacroAssembler::vblendvps;
948 vmaxmin = &MacroAssembler::vminps;
949 vcmp = &MacroAssembler::vcmpps;
950 } else if (!is_double_word && !is_min) {
951 mask = b;
952 vblend = &MacroAssembler::vblendvps;
953 vmaxmin = &MacroAssembler::vmaxps;
954 vcmp = &MacroAssembler::vcmpps;
955 } else if (is_double_word && is_min) {
956 mask = a;
957 vblend = &MacroAssembler::vblendvpd;
958 vmaxmin = &MacroAssembler::vminpd;
959 vcmp = &MacroAssembler::vcmppd;
960 } else {
961 assert(is_double_word && !is_min, "sanity");
962 mask = b;
963 vblend = &MacroAssembler::vblendvpd;
964 vmaxmin = &MacroAssembler::vmaxpd;
965 vcmp = &MacroAssembler::vcmppd;
966 }
967
968 // Make sure EnableX86ECoreOpts isn't disabled on register overlaps
969 XMMRegister maxmin, scratch;
970 if (dst == btmp) {
971 maxmin = btmp;
972 scratch = tmp;
973 } else {
974 maxmin = tmp;
975 scratch = btmp;
976 }
977
978 bool precompute_mask = EnableX86ECoreOpts && UseAVX>1;
979 if (precompute_mask && !is_double_word) {
980 vpsrad(tmp, mask, 32, vlen_enc);
981 mask = tmp;
982 } else if (precompute_mask && is_double_word) {
983 vpxor(tmp, tmp, tmp, vlen_enc);
984 vpcmpgtq(tmp, tmp, mask, vlen_enc);
985 mask = tmp;
986 }
987
988 (this->*vblend)(atmp, a, b, mask, vlen_enc, !precompute_mask, btmp);
989 (this->*vblend)(btmp, b, a, mask, vlen_enc, !precompute_mask, tmp);
990 (this->*vmaxmin)(maxmin, atmp, btmp, vlen_enc);
991 (this->*vcmp)(scratch, atmp, atmp, Assembler::UNORD_Q, vlen_enc);
992 (this->*vblend)(dst, maxmin, atmp, scratch, vlen_enc, false, scratch);
993 }
994
995 void C2_MacroAssembler::evminmax_fp(int opcode, BasicType elem_bt,
996 XMMRegister dst, XMMRegister a, XMMRegister b,
997 KRegister ktmp, XMMRegister atmp, XMMRegister btmp,
998 int vlen_enc) {
999 assert(UseAVX > 2, "required");
1000 assert(opcode == Op_MinV || opcode == Op_MinReductionV ||
1001 opcode == Op_MaxV || opcode == Op_MaxReductionV, "sanity");
1002 assert(elem_bt == T_FLOAT || elem_bt == T_DOUBLE, "sanity");
1003 assert_different_registers(dst, a, atmp, btmp);
1004 assert_different_registers(dst, b, atmp, btmp);
1005
1006 bool is_min = (opcode == Op_MinV || opcode == Op_MinReductionV);
1007 bool is_double_word = is_double_word_type(elem_bt);
1008 bool merge = true;
1009
1010 if (!is_double_word && is_min) {
1011 evpmovd2m(ktmp, a, vlen_enc);
1012 evblendmps(atmp, ktmp, a, b, merge, vlen_enc);
1013 evblendmps(btmp, ktmp, b, a, merge, vlen_enc);
1014 vminps(dst, atmp, btmp, vlen_enc);
1015 evcmpps(ktmp, k0, atmp, atmp, Assembler::UNORD_Q, vlen_enc);
1016 evmovdqul(dst, ktmp, atmp, merge, vlen_enc);
1017 } else if (!is_double_word && !is_min) {
1018 evpmovd2m(ktmp, b, vlen_enc);
1019 evblendmps(atmp, ktmp, a, b, merge, vlen_enc);
1020 evblendmps(btmp, ktmp, b, a, merge, vlen_enc);
1021 vmaxps(dst, atmp, btmp, vlen_enc);
1022 evcmpps(ktmp, k0, atmp, atmp, Assembler::UNORD_Q, vlen_enc);
1023 evmovdqul(dst, ktmp, atmp, merge, vlen_enc);
1024 } else if (is_double_word && is_min) {
1025 evpmovq2m(ktmp, a, vlen_enc);
1026 evblendmpd(atmp, ktmp, a, b, merge, vlen_enc);
1027 evblendmpd(btmp, ktmp, b, a, merge, vlen_enc);
1028 vminpd(dst, atmp, btmp, vlen_enc);
1029 evcmppd(ktmp, k0, atmp, atmp, Assembler::UNORD_Q, vlen_enc);
1030 evmovdquq(dst, ktmp, atmp, merge, vlen_enc);
1031 } else {
1032 assert(is_double_word && !is_min, "sanity");
1033 evpmovq2m(ktmp, b, vlen_enc);
1034 evblendmpd(atmp, ktmp, a, b, merge, vlen_enc);
1035 evblendmpd(btmp, ktmp, b, a, merge, vlen_enc);
1036 vmaxpd(dst, atmp, btmp, vlen_enc);
1037 evcmppd(ktmp, k0, atmp, atmp, Assembler::UNORD_Q, vlen_enc);
1038 evmovdquq(dst, ktmp, atmp, merge, vlen_enc);
1039 }
1040 }
1041
1042 void C2_MacroAssembler::vminmax_fp_avx10_2(int opc, BasicType elem_bt, XMMRegister dst, KRegister mask,
1043 XMMRegister src1, XMMRegister src2, int vlen_enc) {
1044 assert(opc == Op_MinV || opc == Op_MinReductionV ||
1045 opc == Op_MaxV || opc == Op_MaxReductionV, "sanity");
1046
1047 int imm8 = (opc == Op_MinV || opc == Op_MinReductionV) ? AVX10_2_MINMAX_MIN_COMPARE_SIGN
1048 : AVX10_2_MINMAX_MAX_COMPARE_SIGN;
1049 if (elem_bt == T_FLOAT) {
1050 evminmaxps(dst, mask, src1, src2, true, imm8, vlen_enc);
1051 } else {
1052 assert(elem_bt == T_DOUBLE, "");
1053 evminmaxpd(dst, mask, src1, src2, true, imm8, vlen_enc);
1054 }
1055 }
1056
1057 void C2_MacroAssembler::sminmax_fp_avx10_2(int opc, BasicType elem_bt, XMMRegister dst, KRegister mask,
1058 XMMRegister src1, XMMRegister src2) {
1059 assert(opc == Op_MinF || opc == Op_MaxF ||
1060 opc == Op_MinD || opc == Op_MaxD, "sanity");
1061
1062 int imm8 = (opc == Op_MinF || opc == Op_MinD) ? AVX10_2_MINMAX_MIN_COMPARE_SIGN
1063 : AVX10_2_MINMAX_MAX_COMPARE_SIGN;
1064 if (elem_bt == T_FLOAT) {
1065 evminmaxss(dst, mask, src1, src2, true, imm8);
1066 } else {
1067 assert(elem_bt == T_DOUBLE, "");
1068 evminmaxsd(dst, mask, src1, src2, true, imm8);
1069 }
1070 }
1071
1072 // Float/Double signum
1073 void C2_MacroAssembler::signum_fp(int opcode, XMMRegister dst, XMMRegister zero, XMMRegister one) {
1074 assert(opcode == Op_SignumF || opcode == Op_SignumD, "sanity");
1075
1076 Label DONE_LABEL;
1077
1078 // Handle special cases +0.0/-0.0 and NaN, if argument is +0.0/-0.0 or NaN, return argument
1079 // If AVX10.2 (or newer) floating point comparison instructions used, SF=1 for equal and unordered cases
1080 // If other floating point comparison instructions used, ZF=1 for equal and unordered cases
1081 if (opcode == Op_SignumF) {
1082 if (VM_Version::supports_avx10_2()) {
1083 evucomxss(dst, zero);
1084 jcc(Assembler::negative, DONE_LABEL);
1085 } else {
1086 ucomiss(dst, zero);
1087 jcc(Assembler::equal, DONE_LABEL);
1088 }
1089 movflt(dst, one);
1090 jcc(Assembler::above, DONE_LABEL);
1091 xorps(dst, ExternalAddress(StubRoutines::x86::vector_float_sign_flip()), noreg);
1092 } else if (opcode == Op_SignumD) {
1093 if (VM_Version::supports_avx10_2()) {
1094 evucomxsd(dst, zero);
1095 jcc(Assembler::negative, DONE_LABEL);
1096 } else {
1097 ucomisd(dst, zero);
1098 jcc(Assembler::equal, DONE_LABEL);
1099 }
1100 movdbl(dst, one);
1101 jcc(Assembler::above, DONE_LABEL);
1102 xorpd(dst, ExternalAddress(StubRoutines::x86::vector_double_sign_flip()), noreg);
1103 }
1104
1105 bind(DONE_LABEL);
1106 }
1107
1108 void C2_MacroAssembler::vextendbw(bool sign, XMMRegister dst, XMMRegister src) {
1109 if (sign) {
1110 pmovsxbw(dst, src);
1111 } else {
1112 pmovzxbw(dst, src);
1113 }
1114 }
1115
1116 void C2_MacroAssembler::vextendbw(bool sign, XMMRegister dst, XMMRegister src, int vector_len) {
1117 if (sign) {
1118 vpmovsxbw(dst, src, vector_len);
1119 } else {
1120 vpmovzxbw(dst, src, vector_len);
1121 }
1122 }
1123
1124 void C2_MacroAssembler::vextendbd(bool sign, XMMRegister dst, XMMRegister src, int vector_len) {
1125 if (sign) {
1126 vpmovsxbd(dst, src, vector_len);
1127 } else {
1128 vpmovzxbd(dst, src, vector_len);
1129 }
1130 }
1131
1132 void C2_MacroAssembler::vextendwd(bool sign, XMMRegister dst, XMMRegister src, int vector_len) {
1133 if (sign) {
1134 vpmovsxwd(dst, src, vector_len);
1135 } else {
1136 vpmovzxwd(dst, src, vector_len);
1137 }
1138 }
1139
1140 void C2_MacroAssembler::vprotate_imm(int opcode, BasicType etype, XMMRegister dst, XMMRegister src,
1141 int shift, int vector_len) {
1142 if (opcode == Op_RotateLeftV) {
1143 if (etype == T_INT) {
1144 evprold(dst, src, shift, vector_len);
1145 } else {
1146 assert(etype == T_LONG, "expected type T_LONG");
1147 evprolq(dst, src, shift, vector_len);
1148 }
1149 } else {
1150 assert(opcode == Op_RotateRightV, "opcode should be Op_RotateRightV");
1151 if (etype == T_INT) {
1152 evprord(dst, src, shift, vector_len);
1153 } else {
1154 assert(etype == T_LONG, "expected type T_LONG");
1155 evprorq(dst, src, shift, vector_len);
1156 }
1157 }
1158 }
1159
1160 void C2_MacroAssembler::vprotate_var(int opcode, BasicType etype, XMMRegister dst, XMMRegister src,
1161 XMMRegister shift, int vector_len) {
1162 if (opcode == Op_RotateLeftV) {
1163 if (etype == T_INT) {
1164 evprolvd(dst, src, shift, vector_len);
1165 } else {
1166 assert(etype == T_LONG, "expected type T_LONG");
1167 evprolvq(dst, src, shift, vector_len);
1168 }
1169 } else {
1170 assert(opcode == Op_RotateRightV, "opcode should be Op_RotateRightV");
1171 if (etype == T_INT) {
1172 evprorvd(dst, src, shift, vector_len);
1173 } else {
1174 assert(etype == T_LONG, "expected type T_LONG");
1175 evprorvq(dst, src, shift, vector_len);
1176 }
1177 }
1178 }
1179
1180 void C2_MacroAssembler::vshiftd_imm(int opcode, XMMRegister dst, int shift) {
1181 if (opcode == Op_RShiftVI) {
1182 psrad(dst, shift);
1183 } else if (opcode == Op_LShiftVI) {
1184 pslld(dst, shift);
1185 } else {
1186 assert((opcode == Op_URShiftVI),"opcode should be Op_URShiftVI");
1187 psrld(dst, shift);
1188 }
1189 }
1190
1191 void C2_MacroAssembler::vshiftd(int opcode, XMMRegister dst, XMMRegister shift) {
1192 switch (opcode) {
1193 case Op_RShiftVI: psrad(dst, shift); break;
1194 case Op_LShiftVI: pslld(dst, shift); break;
1195 case Op_URShiftVI: psrld(dst, shift); break;
1196
1197 default: assert(false, "%s", NodeClassNames[opcode]);
1198 }
1199 }
1200
1201 void C2_MacroAssembler::vshiftd_imm(int opcode, XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
1202 if (opcode == Op_RShiftVI) {
1203 vpsrad(dst, nds, shift, vector_len);
1204 } else if (opcode == Op_LShiftVI) {
1205 vpslld(dst, nds, shift, vector_len);
1206 } else {
1207 assert((opcode == Op_URShiftVI),"opcode should be Op_URShiftVI");
1208 vpsrld(dst, nds, shift, vector_len);
1209 }
1210 }
1211
1212 void C2_MacroAssembler::vshiftd(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc) {
1213 switch (opcode) {
1214 case Op_RShiftVI: vpsrad(dst, src, shift, vlen_enc); break;
1215 case Op_LShiftVI: vpslld(dst, src, shift, vlen_enc); break;
1216 case Op_URShiftVI: vpsrld(dst, src, shift, vlen_enc); break;
1217
1218 default: assert(false, "%s", NodeClassNames[opcode]);
1219 }
1220 }
1221
1222 void C2_MacroAssembler::vshiftw(int opcode, XMMRegister dst, XMMRegister shift) {
1223 switch (opcode) {
1224 case Op_RShiftVB: // fall-through
1225 case Op_RShiftVS: psraw(dst, shift); break;
1226
1227 case Op_LShiftVB: // fall-through
1228 case Op_LShiftVS: psllw(dst, shift); break;
1229
1230 case Op_URShiftVS: // fall-through
1231 case Op_URShiftVB: psrlw(dst, shift); break;
1232
1233 default: assert(false, "%s", NodeClassNames[opcode]);
1234 }
1235 }
1236
1237 void C2_MacroAssembler::vshiftw(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc) {
1238 switch (opcode) {
1239 case Op_RShiftVB: // fall-through
1240 case Op_RShiftVS: vpsraw(dst, src, shift, vlen_enc); break;
1241
1242 case Op_LShiftVB: // fall-through
1243 case Op_LShiftVS: vpsllw(dst, src, shift, vlen_enc); break;
1244
1245 case Op_URShiftVS: // fall-through
1246 case Op_URShiftVB: vpsrlw(dst, src, shift, vlen_enc); break;
1247
1248 default: assert(false, "%s", NodeClassNames[opcode]);
1249 }
1250 }
1251
1252 void C2_MacroAssembler::vshiftq(int opcode, XMMRegister dst, XMMRegister shift) {
1253 switch (opcode) {
1254 case Op_RShiftVL: psrlq(dst, shift); break; // using srl to implement sra on pre-avs512 systems
1255 case Op_LShiftVL: psllq(dst, shift); break;
1256 case Op_URShiftVL: psrlq(dst, shift); break;
1257
1258 default: assert(false, "%s", NodeClassNames[opcode]);
1259 }
1260 }
1261
1262 void C2_MacroAssembler::vshiftq_imm(int opcode, XMMRegister dst, int shift) {
1263 if (opcode == Op_RShiftVL) {
1264 psrlq(dst, shift); // using srl to implement sra on pre-avs512 systems
1265 } else if (opcode == Op_LShiftVL) {
1266 psllq(dst, shift);
1267 } else {
1268 assert((opcode == Op_URShiftVL),"opcode should be Op_URShiftVL");
1269 psrlq(dst, shift);
1270 }
1271 }
1272
1273 void C2_MacroAssembler::vshiftq(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc) {
1274 switch (opcode) {
1275 case Op_RShiftVL: evpsraq(dst, src, shift, vlen_enc); break;
1276 case Op_LShiftVL: vpsllq(dst, src, shift, vlen_enc); break;
1277 case Op_URShiftVL: vpsrlq(dst, src, shift, vlen_enc); break;
1278
1279 default: assert(false, "%s", NodeClassNames[opcode]);
1280 }
1281 }
1282
1283 void C2_MacroAssembler::vshiftq_imm(int opcode, XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
1284 if (opcode == Op_RShiftVL) {
1285 evpsraq(dst, nds, shift, vector_len);
1286 } else if (opcode == Op_LShiftVL) {
1287 vpsllq(dst, nds, shift, vector_len);
1288 } else {
1289 assert((opcode == Op_URShiftVL),"opcode should be Op_URShiftVL");
1290 vpsrlq(dst, nds, shift, vector_len);
1291 }
1292 }
1293
1294 void C2_MacroAssembler::varshiftd(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc) {
1295 switch (opcode) {
1296 case Op_RShiftVB: // fall-through
1297 case Op_RShiftVS: // fall-through
1298 case Op_RShiftVI: vpsravd(dst, src, shift, vlen_enc); break;
1299
1300 case Op_LShiftVB: // fall-through
1301 case Op_LShiftVS: // fall-through
1302 case Op_LShiftVI: vpsllvd(dst, src, shift, vlen_enc); break;
1303
1304 case Op_URShiftVB: // fall-through
1305 case Op_URShiftVS: // fall-through
1306 case Op_URShiftVI: vpsrlvd(dst, src, shift, vlen_enc); break;
1307
1308 default: assert(false, "%s", NodeClassNames[opcode]);
1309 }
1310 }
1311
1312 void C2_MacroAssembler::varshiftw(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc) {
1313 switch (opcode) {
1314 case Op_RShiftVB: // fall-through
1315 case Op_RShiftVS: evpsravw(dst, src, shift, vlen_enc); break;
1316
1317 case Op_LShiftVB: // fall-through
1318 case Op_LShiftVS: evpsllvw(dst, src, shift, vlen_enc); break;
1319
1320 case Op_URShiftVB: // fall-through
1321 case Op_URShiftVS: evpsrlvw(dst, src, shift, vlen_enc); break;
1322
1323 default: assert(false, "%s", NodeClassNames[opcode]);
1324 }
1325 }
1326
1327 void C2_MacroAssembler::varshiftq(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vlen_enc, XMMRegister tmp) {
1328 assert(UseAVX >= 2, "required");
1329 switch (opcode) {
1330 case Op_RShiftVL: {
1331 if (UseAVX > 2) {
1332 assert(tmp == xnoreg, "not used");
1333 if (!VM_Version::supports_avx512vl()) {
1334 vlen_enc = Assembler::AVX_512bit;
1335 }
1336 evpsravq(dst, src, shift, vlen_enc);
1337 } else {
1338 vmovdqu(tmp, ExternalAddress(StubRoutines::x86::vector_long_sign_mask()));
1339 vpsrlvq(dst, src, shift, vlen_enc);
1340 vpsrlvq(tmp, tmp, shift, vlen_enc);
1341 vpxor(dst, dst, tmp, vlen_enc);
1342 vpsubq(dst, dst, tmp, vlen_enc);
1343 }
1344 break;
1345 }
1346 case Op_LShiftVL: {
1347 assert(tmp == xnoreg, "not used");
1348 vpsllvq(dst, src, shift, vlen_enc);
1349 break;
1350 }
1351 case Op_URShiftVL: {
1352 assert(tmp == xnoreg, "not used");
1353 vpsrlvq(dst, src, shift, vlen_enc);
1354 break;
1355 }
1356 default: assert(false, "%s", NodeClassNames[opcode]);
1357 }
1358 }
1359
1360 // Variable shift src by shift using vtmp and scratch as TEMPs giving word result in dst
1361 void C2_MacroAssembler::varshiftbw(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len, XMMRegister vtmp) {
1362 assert(opcode == Op_LShiftVB ||
1363 opcode == Op_RShiftVB ||
1364 opcode == Op_URShiftVB, "%s", NodeClassNames[opcode]);
1365 bool sign = (opcode != Op_URShiftVB);
1366 assert(vector_len == 0, "required");
1367 vextendbd(sign, dst, src, 1);
1368 vpmovzxbd(vtmp, shift, 1);
1369 varshiftd(opcode, dst, dst, vtmp, 1);
1370 vpand(dst, dst, ExternalAddress(StubRoutines::x86::vector_int_to_byte_mask()), 1, noreg);
1371 vextracti128_high(vtmp, dst);
1372 vpackusdw(dst, dst, vtmp, 0);
1373 }
1374
1375 // Variable shift src by shift using vtmp and scratch as TEMPs giving byte result in dst
1376 void C2_MacroAssembler::evarshiftb(int opcode, XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len, XMMRegister vtmp) {
1377 assert(opcode == Op_LShiftVB ||
1378 opcode == Op_RShiftVB ||
1379 opcode == Op_URShiftVB, "%s", NodeClassNames[opcode]);
1380 bool sign = (opcode != Op_URShiftVB);
1381 int ext_vector_len = vector_len + 1;
1382 vextendbw(sign, dst, src, ext_vector_len);
1383 vpmovzxbw(vtmp, shift, ext_vector_len);
1384 varshiftw(opcode, dst, dst, vtmp, ext_vector_len);
1385 vpand(dst, dst, ExternalAddress(StubRoutines::x86::vector_short_to_byte_mask()), ext_vector_len, noreg);
1386 if (vector_len == 0) {
1387 vextracti128_high(vtmp, dst);
1388 vpackuswb(dst, dst, vtmp, vector_len);
1389 } else {
1390 vextracti64x4_high(vtmp, dst);
1391 vpackuswb(dst, dst, vtmp, vector_len);
1392 vpermq(dst, dst, 0xD8, vector_len);
1393 }
1394 }
1395
1396 void C2_MacroAssembler::insert(BasicType typ, XMMRegister dst, Register val, int idx) {
1397 switch(typ) {
1398 case T_BYTE:
1399 pinsrb(dst, val, idx);
1400 break;
1401 case T_SHORT:
1402 pinsrw(dst, val, idx);
1403 break;
1404 case T_INT:
1405 pinsrd(dst, val, idx);
1406 break;
1407 case T_LONG:
1408 pinsrq(dst, val, idx);
1409 break;
1410 default:
1411 assert(false,"Should not reach here.");
1412 break;
1413 }
1414 }
1415
1416 void C2_MacroAssembler::vinsert(BasicType typ, XMMRegister dst, XMMRegister src, Register val, int idx) {
1417 switch(typ) {
1418 case T_BYTE:
1419 vpinsrb(dst, src, val, idx);
1420 break;
1421 case T_SHORT:
1422 vpinsrw(dst, src, val, idx);
1423 break;
1424 case T_INT:
1425 vpinsrd(dst, src, val, idx);
1426 break;
1427 case T_LONG:
1428 vpinsrq(dst, src, val, idx);
1429 break;
1430 default:
1431 assert(false,"Should not reach here.");
1432 break;
1433 }
1434 }
1435
1436 void C2_MacroAssembler::vgather8b_masked(BasicType elem_bt, XMMRegister dst,
1437 Register base, Register idx_base,
1438 Register mask, Register mask_idx,
1439 Register rtmp, int vlen_enc) {
1440 vpxor(dst, dst, dst, vlen_enc);
1441 if (elem_bt == T_SHORT) {
1442 for (int i = 0; i < 4; i++) {
1443 // dst[i] = mask[i] ? src[idx_base[i]] : 0
1444 Label skip_load;
1445 btq(mask, mask_idx);
1446 jccb(Assembler::carryClear, skip_load);
1447 movl(rtmp, Address(idx_base, i * 4));
1448 pinsrw(dst, Address(base, rtmp, Address::times_2), i);
1449 bind(skip_load);
1450 incq(mask_idx);
1451 }
1452 } else {
1453 assert(elem_bt == T_BYTE, "");
1454 for (int i = 0; i < 8; i++) {
1455 // dst[i] = mask[i] ? src[idx_base[i]] : 0
1456 Label skip_load;
1457 btq(mask, mask_idx);
1458 jccb(Assembler::carryClear, skip_load);
1459 movl(rtmp, Address(idx_base, i * 4));
1460 pinsrb(dst, Address(base, rtmp), i);
1461 bind(skip_load);
1462 incq(mask_idx);
1463 }
1464 }
1465 }
1466
1467 void C2_MacroAssembler::vgather8b(BasicType elem_bt, XMMRegister dst,
1468 Register base, Register idx_base,
1469 Register rtmp, int vlen_enc) {
1470 vpxor(dst, dst, dst, vlen_enc);
1471 if (elem_bt == T_SHORT) {
1472 for (int i = 0; i < 4; i++) {
1473 // dst[i] = src[idx_base[i]]
1474 movl(rtmp, Address(idx_base, i * 4));
1475 pinsrw(dst, Address(base, rtmp, Address::times_2), i);
1476 }
1477 } else {
1478 assert(elem_bt == T_BYTE, "");
1479 for (int i = 0; i < 8; i++) {
1480 // dst[i] = src[idx_base[i]]
1481 movl(rtmp, Address(idx_base, i * 4));
1482 pinsrb(dst, Address(base, rtmp), i);
1483 }
1484 }
1485 }
1486
1487 /*
1488 * Gather using hybrid algorithm, first partially unroll scalar loop
1489 * to accumulate values from gather indices into a quad-word(64bit) slice.
1490 * A slice may hold 8 bytes or 4 short values. This is followed by a vector
1491 * permutation to place the slice into appropriate vector lane
1492 * locations in destination vector. Following pseudo code describes the
1493 * algorithm in detail:
1494 *
1495 * DST_VEC = ZERO_VEC
1496 * PERM_INDEX = {0, 1, 2, 3, 4, 5, 6, 7, 8..}
1497 * TWO_VEC = {2, 2, 2, 2, 2, 2, 2, 2, 2..}
1498 * FOREACH_ITER:
1499 * TMP_VEC_64 = PICK_SUB_WORDS_FROM_GATHER_INDICES
1500 * TEMP_PERM_VEC = PERMUTE TMP_VEC_64 PERM_INDEX
1501 * DST_VEC = DST_VEC OR TEMP_PERM_VEC
1502 * PERM_INDEX = PERM_INDEX - TWO_VEC
1503 *
1504 * With each iteration, doubleword permute indices (0,1) corresponding
1505 * to gathered quadword gets right shifted by two lane positions.
1506 *
1507 */
1508 void C2_MacroAssembler::vgather_subword(BasicType elem_ty, XMMRegister dst,
1509 Register base, Register idx_base,
1510 Register mask, XMMRegister xtmp1,
1511 XMMRegister xtmp2, XMMRegister temp_dst,
1512 Register rtmp, Register mask_idx,
1513 Register length, int vector_len, int vlen_enc) {
1514 Label GATHER8_LOOP;
1515 assert(is_subword_type(elem_ty), "");
1516 movl(length, vector_len);
1517 vpxor(xtmp1, xtmp1, xtmp1, vlen_enc); // xtmp1 = {0, ...}
1518 vpxor(dst, dst, dst, vlen_enc); // dst = {0, ...}
1519 vallones(xtmp2, vlen_enc);
1520 vpsubd(xtmp2, xtmp1, xtmp2, vlen_enc);
1521 vpslld(xtmp2, xtmp2, 1, vlen_enc); // xtmp2 = {2, 2, ...}
1522 load_iota_indices(xtmp1, vector_len * type2aelembytes(elem_ty), T_INT); // xtmp1 = {0, 1, 2, ...}
1523
1524 bind(GATHER8_LOOP);
1525 // TMP_VEC_64(temp_dst) = PICK_SUB_WORDS_FROM_GATHER_INDICES
1526 if (mask == noreg) {
1527 vgather8b(elem_ty, temp_dst, base, idx_base, rtmp, vlen_enc);
1528 } else {
1529 vgather8b_masked(elem_ty, temp_dst, base, idx_base, mask, mask_idx, rtmp, vlen_enc);
1530 }
1531 // TEMP_PERM_VEC(temp_dst) = PERMUTE TMP_VEC_64(temp_dst) PERM_INDEX(xtmp1)
1532 vpermd(temp_dst, xtmp1, temp_dst, vlen_enc == Assembler::AVX_512bit ? vlen_enc : Assembler::AVX_256bit);
1533 // PERM_INDEX(xtmp1) = PERM_INDEX(xtmp1) - TWO_VEC(xtmp2)
1534 vpsubd(xtmp1, xtmp1, xtmp2, vlen_enc);
1535 // DST_VEC = DST_VEC OR TEMP_PERM_VEC
1536 vpor(dst, dst, temp_dst, vlen_enc);
1537 addptr(idx_base, 32 >> (type2aelembytes(elem_ty) - 1));
1538 subl(length, 8 >> (type2aelembytes(elem_ty) - 1));
1539 jcc(Assembler::notEqual, GATHER8_LOOP);
1540 }
1541
1542 void C2_MacroAssembler::vgather(BasicType typ, XMMRegister dst, Register base, XMMRegister idx, XMMRegister mask, int vector_len) {
1543 switch(typ) {
1544 case T_INT:
1545 vpgatherdd(dst, Address(base, idx, Address::times_4), mask, vector_len);
1546 break;
1547 case T_FLOAT:
1548 vgatherdps(dst, Address(base, idx, Address::times_4), mask, vector_len);
1549 break;
1550 case T_LONG:
1551 vpgatherdq(dst, Address(base, idx, Address::times_8), mask, vector_len);
1552 break;
1553 case T_DOUBLE:
1554 vgatherdpd(dst, Address(base, idx, Address::times_8), mask, vector_len);
1555 break;
1556 default:
1557 assert(false,"Should not reach here.");
1558 break;
1559 }
1560 }
1561
1562 void C2_MacroAssembler::evgather(BasicType typ, XMMRegister dst, KRegister mask, Register base, XMMRegister idx, int vector_len) {
1563 switch(typ) {
1564 case T_INT:
1565 evpgatherdd(dst, mask, Address(base, idx, Address::times_4), vector_len);
1566 break;
1567 case T_FLOAT:
1568 evgatherdps(dst, mask, Address(base, idx, Address::times_4), vector_len);
1569 break;
1570 case T_LONG:
1571 evpgatherdq(dst, mask, Address(base, idx, Address::times_8), vector_len);
1572 break;
1573 case T_DOUBLE:
1574 evgatherdpd(dst, mask, Address(base, idx, Address::times_8), vector_len);
1575 break;
1576 default:
1577 assert(false,"Should not reach here.");
1578 break;
1579 }
1580 }
1581
1582 void C2_MacroAssembler::evscatter(BasicType typ, Register base, XMMRegister idx, KRegister mask, XMMRegister src, int vector_len) {
1583 switch(typ) {
1584 case T_INT:
1585 evpscatterdd(Address(base, idx, Address::times_4), mask, src, vector_len);
1586 break;
1587 case T_FLOAT:
1588 evscatterdps(Address(base, idx, Address::times_4), mask, src, vector_len);
1589 break;
1590 case T_LONG:
1591 evpscatterdq(Address(base, idx, Address::times_8), mask, src, vector_len);
1592 break;
1593 case T_DOUBLE:
1594 evscatterdpd(Address(base, idx, Address::times_8), mask, src, vector_len);
1595 break;
1596 default:
1597 assert(false,"Should not reach here.");
1598 break;
1599 }
1600 }
1601
1602 void C2_MacroAssembler::load_vector_mask(XMMRegister dst, XMMRegister src, int vlen_in_bytes, BasicType elem_bt, bool is_legacy) {
1603 if (vlen_in_bytes <= 16) {
1604 pxor (dst, dst);
1605 psubb(dst, src);
1606 switch (elem_bt) {
1607 case T_BYTE: /* nothing to do */ break;
1608 case T_SHORT: pmovsxbw(dst, dst); break;
1609 case T_INT: pmovsxbd(dst, dst); break;
1610 case T_FLOAT: pmovsxbd(dst, dst); break;
1611 case T_LONG: pmovsxbq(dst, dst); break;
1612 case T_DOUBLE: pmovsxbq(dst, dst); break;
1613
1614 default: assert(false, "%s", type2name(elem_bt));
1615 }
1616 } else {
1617 assert(!is_legacy || !is_subword_type(elem_bt) || vlen_in_bytes < 64, "");
1618 int vlen_enc = vector_length_encoding(vlen_in_bytes);
1619
1620 vpxor (dst, dst, dst, vlen_enc);
1621 vpsubb(dst, dst, src, is_legacy ? AVX_256bit : vlen_enc);
1622
1623 switch (elem_bt) {
1624 case T_BYTE: /* nothing to do */ break;
1625 case T_SHORT: vpmovsxbw(dst, dst, vlen_enc); break;
1626 case T_INT: vpmovsxbd(dst, dst, vlen_enc); break;
1627 case T_FLOAT: vpmovsxbd(dst, dst, vlen_enc); break;
1628 case T_LONG: vpmovsxbq(dst, dst, vlen_enc); break;
1629 case T_DOUBLE: vpmovsxbq(dst, dst, vlen_enc); break;
1630
1631 default: assert(false, "%s", type2name(elem_bt));
1632 }
1633 }
1634 }
1635
1636 void C2_MacroAssembler::load_vector_mask(KRegister dst, XMMRegister src, XMMRegister xtmp, bool novlbwdq, int vlen_enc) {
1637 if (novlbwdq) {
1638 vpmovsxbd(xtmp, src, vlen_enc);
1639 evpcmpd(dst, k0, xtmp, ExternalAddress(StubRoutines::x86::vector_int_mask_cmp_bits()),
1640 Assembler::eq, true, vlen_enc, noreg);
1641 } else {
1642 vpxor(xtmp, xtmp, xtmp, vlen_enc);
1643 vpsubb(xtmp, xtmp, src, vlen_enc);
1644 evpmovb2m(dst, xtmp, vlen_enc);
1645 }
1646 }
1647
1648 void C2_MacroAssembler::load_vector(BasicType bt, XMMRegister dst, Address src, int vlen_in_bytes) {
1649 if (is_integral_type(bt)) {
1650 switch (vlen_in_bytes) {
1651 case 4: movdl(dst, src); break;
1652 case 8: movq(dst, src); break;
1653 case 16: movdqu(dst, src); break;
1654 case 32: vmovdqu(dst, src); break;
1655 case 64: evmovdqul(dst, src, Assembler::AVX_512bit); break;
1656 default: ShouldNotReachHere();
1657 }
1658 } else {
1659 switch (vlen_in_bytes) {
1660 case 4: movflt(dst, src); break;
1661 case 8: movdbl(dst, src); break;
1662 case 16: movups(dst, src); break;
1663 case 32: vmovups(dst, src, Assembler::AVX_256bit); break;
1664 case 64: vmovups(dst, src, Assembler::AVX_512bit); break;
1665 default: ShouldNotReachHere();
1666 }
1667 }
1668 }
1669
1670 void C2_MacroAssembler::load_vector(BasicType bt, XMMRegister dst, AddressLiteral src, int vlen_in_bytes, Register rscratch) {
1671 assert(rscratch != noreg || always_reachable(src), "missing");
1672
1673 if (reachable(src)) {
1674 load_vector(bt, dst, as_Address(src), vlen_in_bytes);
1675 } else {
1676 lea(rscratch, src);
1677 load_vector(bt, dst, Address(rscratch, 0), vlen_in_bytes);
1678 }
1679 }
1680
1681 void C2_MacroAssembler::load_constant_vector(BasicType bt, XMMRegister dst, InternalAddress src, int vlen) {
1682 int vlen_enc = vector_length_encoding(vlen);
1683 if (VM_Version::supports_avx()) {
1684 if (bt == T_LONG) {
1685 if (VM_Version::supports_avx2()) {
1686 vpbroadcastq(dst, src, vlen_enc);
1687 } else {
1688 vmovddup(dst, src, vlen_enc);
1689 }
1690 } else if (bt == T_DOUBLE) {
1691 if (vlen_enc != Assembler::AVX_128bit) {
1692 vbroadcastsd(dst, src, vlen_enc, noreg);
1693 } else {
1694 vmovddup(dst, src, vlen_enc);
1695 }
1696 } else {
1697 if (VM_Version::supports_avx2() && is_integral_type(bt)) {
1698 vpbroadcastd(dst, src, vlen_enc);
1699 } else {
1700 vbroadcastss(dst, src, vlen_enc);
1701 }
1702 }
1703 } else if (VM_Version::supports_sse3()) {
1704 movddup(dst, src);
1705 } else {
1706 load_vector(bt, dst, src, vlen);
1707 }
1708 }
1709
1710 void C2_MacroAssembler::load_iota_indices(XMMRegister dst, int vlen_in_bytes, BasicType bt) {
1711 int entry_idx = vector_iota_entry_index(bt);
1712 ExternalAddress addr(StubRoutines::x86::vector_iota_indices(entry_idx));
1713 load_vector(T_BYTE, dst, addr, vlen_in_bytes);
1714 }
1715
1716 // Reductions for vectors of bytes, shorts, ints, longs, floats, and doubles.
1717
1718 void C2_MacroAssembler::reduce_operation_128(BasicType typ, int opcode, XMMRegister dst, XMMRegister src) {
1719 int vector_len = Assembler::AVX_128bit;
1720
1721 switch (opcode) {
1722 case Op_AndReductionV: pand(dst, src); break;
1723 case Op_OrReductionV: por (dst, src); break;
1724 case Op_XorReductionV: pxor(dst, src); break;
1725 case Op_MinReductionV:
1726 switch (typ) {
1727 case T_BYTE: pminsb(dst, src); break;
1728 case T_SHORT: pminsw(dst, src); break;
1729 case T_INT: pminsd(dst, src); break;
1730 case T_LONG: assert(UseAVX > 2, "required");
1731 vpminsq(dst, dst, src, Assembler::AVX_128bit); break;
1732 default: assert(false, "wrong type");
1733 }
1734 break;
1735 case Op_MaxReductionV:
1736 switch (typ) {
1737 case T_BYTE: pmaxsb(dst, src); break;
1738 case T_SHORT: pmaxsw(dst, src); break;
1739 case T_INT: pmaxsd(dst, src); break;
1740 case T_LONG: assert(UseAVX > 2, "required");
1741 vpmaxsq(dst, dst, src, Assembler::AVX_128bit); break;
1742 default: assert(false, "wrong type");
1743 }
1744 break;
1745 case Op_UMinReductionV:
1746 switch (typ) {
1747 case T_BYTE: vpminub(dst, dst, src, Assembler::AVX_128bit); break;
1748 case T_SHORT: vpminuw(dst, dst, src, Assembler::AVX_128bit); break;
1749 case T_INT: vpminud(dst, dst, src, Assembler::AVX_128bit); break;
1750 case T_LONG: evpminuq(dst, k0, dst, src, true, Assembler::AVX_128bit); break;
1751 default: assert(false, "wrong type");
1752 }
1753 break;
1754 case Op_UMaxReductionV:
1755 switch (typ) {
1756 case T_BYTE: vpmaxub(dst, dst, src, Assembler::AVX_128bit); break;
1757 case T_SHORT: vpmaxuw(dst, dst, src, Assembler::AVX_128bit); break;
1758 case T_INT: vpmaxud(dst, dst, src, Assembler::AVX_128bit); break;
1759 case T_LONG: evpmaxuq(dst, k0, dst, src, true, Assembler::AVX_128bit); break;
1760 default: assert(false, "wrong type");
1761 }
1762 break;
1763 case Op_AddReductionVF: addss(dst, src); break;
1764 case Op_AddReductionVD: addsd(dst, src); break;
1765 case Op_AddReductionVI:
1766 switch (typ) {
1767 case T_BYTE: paddb(dst, src); break;
1768 case T_SHORT: paddw(dst, src); break;
1769 case T_INT: paddd(dst, src); break;
1770 default: assert(false, "wrong type");
1771 }
1772 break;
1773 case Op_AddReductionVL: paddq(dst, src); break;
1774 case Op_MulReductionVF: mulss(dst, src); break;
1775 case Op_MulReductionVD: mulsd(dst, src); break;
1776 case Op_MulReductionVI:
1777 switch (typ) {
1778 case T_SHORT: pmullw(dst, src); break;
1779 case T_INT: pmulld(dst, src); break;
1780 default: assert(false, "wrong type");
1781 }
1782 break;
1783 case Op_MulReductionVL: assert(UseAVX > 2, "required");
1784 evpmullq(dst, dst, src, vector_len); break;
1785 default: assert(false, "wrong opcode");
1786 }
1787 }
1788
1789 void C2_MacroAssembler::unordered_reduce_operation_128(BasicType typ, int opcode, XMMRegister dst, XMMRegister src) {
1790 switch (opcode) {
1791 case Op_AddReductionVF: addps(dst, src); break;
1792 case Op_AddReductionVD: addpd(dst, src); break;
1793 case Op_MulReductionVF: mulps(dst, src); break;
1794 case Op_MulReductionVD: mulpd(dst, src); break;
1795 default: assert(false, "%s", NodeClassNames[opcode]);
1796 }
1797 }
1798
1799 void C2_MacroAssembler::reduce_operation_256(BasicType typ, int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2) {
1800 int vector_len = Assembler::AVX_256bit;
1801
1802 switch (opcode) {
1803 case Op_AndReductionV: vpand(dst, src1, src2, vector_len); break;
1804 case Op_OrReductionV: vpor (dst, src1, src2, vector_len); break;
1805 case Op_XorReductionV: vpxor(dst, src1, src2, vector_len); break;
1806 case Op_MinReductionV:
1807 switch (typ) {
1808 case T_BYTE: vpminsb(dst, src1, src2, vector_len); break;
1809 case T_SHORT: vpminsw(dst, src1, src2, vector_len); break;
1810 case T_INT: vpminsd(dst, src1, src2, vector_len); break;
1811 case T_LONG: assert(UseAVX > 2, "required");
1812 vpminsq(dst, src1, src2, vector_len); break;
1813 default: assert(false, "wrong type");
1814 }
1815 break;
1816 case Op_MaxReductionV:
1817 switch (typ) {
1818 case T_BYTE: vpmaxsb(dst, src1, src2, vector_len); break;
1819 case T_SHORT: vpmaxsw(dst, src1, src2, vector_len); break;
1820 case T_INT: vpmaxsd(dst, src1, src2, vector_len); break;
1821 case T_LONG: assert(UseAVX > 2, "required");
1822 vpmaxsq(dst, src1, src2, vector_len); break;
1823 default: assert(false, "wrong type");
1824 }
1825 break;
1826 case Op_UMinReductionV:
1827 switch (typ) {
1828 case T_BYTE: vpminub(dst, src1, src2, vector_len); break;
1829 case T_SHORT: vpminuw(dst, src1, src2, vector_len); break;
1830 case T_INT: vpminud(dst, src1, src2, vector_len); break;
1831 case T_LONG: evpminuq(dst, k0, src1, src2, true, vector_len); break;
1832 default: assert(false, "wrong type");
1833 }
1834 break;
1835 case Op_UMaxReductionV:
1836 switch (typ) {
1837 case T_BYTE: vpmaxub(dst, src1, src2, vector_len); break;
1838 case T_SHORT: vpmaxuw(dst, src1, src2, vector_len); break;
1839 case T_INT: vpmaxud(dst, src1, src2, vector_len); break;
1840 case T_LONG: evpmaxuq(dst, k0, src1, src2, true, vector_len); break;
1841 default: assert(false, "wrong type");
1842 }
1843 break;
1844 case Op_AddReductionVI:
1845 switch (typ) {
1846 case T_BYTE: vpaddb(dst, src1, src2, vector_len); break;
1847 case T_SHORT: vpaddw(dst, src1, src2, vector_len); break;
1848 case T_INT: vpaddd(dst, src1, src2, vector_len); break;
1849 default: assert(false, "wrong type");
1850 }
1851 break;
1852 case Op_AddReductionVL: vpaddq(dst, src1, src2, vector_len); break;
1853 case Op_MulReductionVI:
1854 switch (typ) {
1855 case T_SHORT: vpmullw(dst, src1, src2, vector_len); break;
1856 case T_INT: vpmulld(dst, src1, src2, vector_len); break;
1857 default: assert(false, "wrong type");
1858 }
1859 break;
1860 case Op_MulReductionVL: evpmullq(dst, src1, src2, vector_len); break;
1861 default: assert(false, "wrong opcode");
1862 }
1863 }
1864
1865 void C2_MacroAssembler::unordered_reduce_operation_256(BasicType typ, int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2) {
1866 int vector_len = Assembler::AVX_256bit;
1867
1868 switch (opcode) {
1869 case Op_AddReductionVF: vaddps(dst, src1, src2, vector_len); break;
1870 case Op_AddReductionVD: vaddpd(dst, src1, src2, vector_len); break;
1871 case Op_MulReductionVF: vmulps(dst, src1, src2, vector_len); break;
1872 case Op_MulReductionVD: vmulpd(dst, src1, src2, vector_len); break;
1873 default: assert(false, "%s", NodeClassNames[opcode]);
1874 }
1875 }
1876
1877 void C2_MacroAssembler::reduce_fp(int opcode, int vlen,
1878 XMMRegister dst, XMMRegister src,
1879 XMMRegister vtmp1, XMMRegister vtmp2) {
1880 switch (opcode) {
1881 case Op_AddReductionVF:
1882 case Op_MulReductionVF:
1883 reduceF(opcode, vlen, dst, src, vtmp1, vtmp2);
1884 break;
1885
1886 case Op_AddReductionVD:
1887 case Op_MulReductionVD:
1888 reduceD(opcode, vlen, dst, src, vtmp1, vtmp2);
1889 break;
1890
1891 default: assert(false, "wrong opcode");
1892 }
1893 }
1894
1895 void C2_MacroAssembler::unordered_reduce_fp(int opcode, int vlen,
1896 XMMRegister dst, XMMRegister src,
1897 XMMRegister vtmp1, XMMRegister vtmp2) {
1898 switch (opcode) {
1899 case Op_AddReductionVF:
1900 case Op_MulReductionVF:
1901 unorderedReduceF(opcode, vlen, dst, src, vtmp1, vtmp2);
1902 break;
1903
1904 case Op_AddReductionVD:
1905 case Op_MulReductionVD:
1906 unorderedReduceD(opcode, vlen, dst, src, vtmp1, vtmp2);
1907 break;
1908
1909 default: assert(false, "%s", NodeClassNames[opcode]);
1910 }
1911 }
1912
1913 void C2_MacroAssembler::reduceB(int opcode, int vlen,
1914 Register dst, Register src1, XMMRegister src2,
1915 XMMRegister vtmp1, XMMRegister vtmp2) {
1916 switch (vlen) {
1917 case 8: reduce8B (opcode, dst, src1, src2, vtmp1, vtmp2); break;
1918 case 16: reduce16B(opcode, dst, src1, src2, vtmp1, vtmp2); break;
1919 case 32: reduce32B(opcode, dst, src1, src2, vtmp1, vtmp2); break;
1920 case 64: reduce64B(opcode, dst, src1, src2, vtmp1, vtmp2); break;
1921
1922 default: assert(false, "wrong vector length");
1923 }
1924 }
1925
1926 void C2_MacroAssembler::mulreduceB(int opcode, int vlen,
1927 Register dst, Register src1, XMMRegister src2,
1928 XMMRegister vtmp1, XMMRegister vtmp2) {
1929 switch (vlen) {
1930 case 8: mulreduce8B (opcode, dst, src1, src2, vtmp1, vtmp2); break;
1931 case 16: mulreduce16B(opcode, dst, src1, src2, vtmp1, vtmp2); break;
1932 case 32: mulreduce32B(opcode, dst, src1, src2, vtmp1, vtmp2); break;
1933 case 64: mulreduce64B(opcode, dst, src1, src2, vtmp1, vtmp2); break;
1934
1935 default: assert(false, "wrong vector length");
1936 }
1937 }
1938
1939 void C2_MacroAssembler::reduceS(int opcode, int vlen,
1940 Register dst, Register src1, XMMRegister src2,
1941 XMMRegister vtmp1, XMMRegister vtmp2) {
1942 switch (vlen) {
1943 case 4: reduce4S (opcode, dst, src1, src2, vtmp1, vtmp2); break;
1944 case 8: reduce8S (opcode, dst, src1, src2, vtmp1, vtmp2); break;
1945 case 16: reduce16S(opcode, dst, src1, src2, vtmp1, vtmp2); break;
1946 case 32: reduce32S(opcode, dst, src1, src2, vtmp1, vtmp2); break;
1947
1948 default: assert(false, "wrong vector length");
1949 }
1950 }
1951
1952 void C2_MacroAssembler::reduceI(int opcode, int vlen,
1953 Register dst, Register src1, XMMRegister src2,
1954 XMMRegister vtmp1, XMMRegister vtmp2) {
1955 switch (vlen) {
1956 case 2: reduce2I (opcode, dst, src1, src2, vtmp1, vtmp2); break;
1957 case 4: reduce4I (opcode, dst, src1, src2, vtmp1, vtmp2); break;
1958 case 8: reduce8I (opcode, dst, src1, src2, vtmp1, vtmp2); break;
1959 case 16: reduce16I(opcode, dst, src1, src2, vtmp1, vtmp2); break;
1960
1961 default: assert(false, "wrong vector length");
1962 }
1963 }
1964
1965 void C2_MacroAssembler::reduceL(int opcode, int vlen,
1966 Register dst, Register src1, XMMRegister src2,
1967 XMMRegister vtmp1, XMMRegister vtmp2) {
1968 switch (vlen) {
1969 case 2: reduce2L(opcode, dst, src1, src2, vtmp1, vtmp2); break;
1970 case 4: reduce4L(opcode, dst, src1, src2, vtmp1, vtmp2); break;
1971 case 8: reduce8L(opcode, dst, src1, src2, vtmp1, vtmp2); break;
1972
1973 default: assert(false, "wrong vector length");
1974 }
1975 }
1976
1977 void C2_MacroAssembler::reduceF(int opcode, int vlen, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2) {
1978 switch (vlen) {
1979 case 2:
1980 assert(vtmp2 == xnoreg, "");
1981 reduce2F(opcode, dst, src, vtmp1);
1982 break;
1983 case 4:
1984 assert(vtmp2 == xnoreg, "");
1985 reduce4F(opcode, dst, src, vtmp1);
1986 break;
1987 case 8:
1988 reduce8F(opcode, dst, src, vtmp1, vtmp2);
1989 break;
1990 case 16:
1991 reduce16F(opcode, dst, src, vtmp1, vtmp2);
1992 break;
1993 default: assert(false, "wrong vector length");
1994 }
1995 }
1996
1997 void C2_MacroAssembler::reduceD(int opcode, int vlen, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2) {
1998 switch (vlen) {
1999 case 2:
2000 assert(vtmp2 == xnoreg, "");
2001 reduce2D(opcode, dst, src, vtmp1);
2002 break;
2003 case 4:
2004 reduce4D(opcode, dst, src, vtmp1, vtmp2);
2005 break;
2006 case 8:
2007 reduce8D(opcode, dst, src, vtmp1, vtmp2);
2008 break;
2009 default: assert(false, "wrong vector length");
2010 }
2011 }
2012
2013 void C2_MacroAssembler::unorderedReduceF(int opcode, int vlen, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2) {
2014 switch (vlen) {
2015 case 2:
2016 assert(vtmp1 == xnoreg, "");
2017 assert(vtmp2 == xnoreg, "");
2018 unorderedReduce2F(opcode, dst, src);
2019 break;
2020 case 4:
2021 assert(vtmp2 == xnoreg, "");
2022 unorderedReduce4F(opcode, dst, src, vtmp1);
2023 break;
2024 case 8:
2025 unorderedReduce8F(opcode, dst, src, vtmp1, vtmp2);
2026 break;
2027 case 16:
2028 unorderedReduce16F(opcode, dst, src, vtmp1, vtmp2);
2029 break;
2030 default: assert(false, "wrong vector length");
2031 }
2032 }
2033
2034 void C2_MacroAssembler::unorderedReduceD(int opcode, int vlen, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2) {
2035 switch (vlen) {
2036 case 2:
2037 assert(vtmp1 == xnoreg, "");
2038 assert(vtmp2 == xnoreg, "");
2039 unorderedReduce2D(opcode, dst, src);
2040 break;
2041 case 4:
2042 assert(vtmp2 == xnoreg, "");
2043 unorderedReduce4D(opcode, dst, src, vtmp1);
2044 break;
2045 case 8:
2046 unorderedReduce8D(opcode, dst, src, vtmp1, vtmp2);
2047 break;
2048 default: assert(false, "wrong vector length");
2049 }
2050 }
2051
2052 void C2_MacroAssembler::reduce2I(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2) {
2053 if (opcode == Op_AddReductionVI) {
2054 if (vtmp1 != src2) {
2055 movdqu(vtmp1, src2);
2056 }
2057 phaddd(vtmp1, vtmp1);
2058 } else {
2059 pshufd(vtmp1, src2, 0x1);
2060 reduce_operation_128(T_INT, opcode, vtmp1, src2);
2061 }
2062 movdl(vtmp2, src1);
2063 reduce_operation_128(T_INT, opcode, vtmp1, vtmp2);
2064 movdl(dst, vtmp1);
2065 }
2066
2067 void C2_MacroAssembler::reduce4I(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2) {
2068 if (opcode == Op_AddReductionVI) {
2069 if (vtmp1 != src2) {
2070 movdqu(vtmp1, src2);
2071 }
2072 phaddd(vtmp1, src2);
2073 reduce2I(opcode, dst, src1, vtmp1, vtmp1, vtmp2);
2074 } else {
2075 pshufd(vtmp2, src2, 0xE);
2076 reduce_operation_128(T_INT, opcode, vtmp2, src2);
2077 reduce2I(opcode, dst, src1, vtmp2, vtmp1, vtmp2);
2078 }
2079 }
2080
2081 void C2_MacroAssembler::reduce8I(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2) {
2082 if (opcode == Op_AddReductionVI) {
2083 vphaddd(vtmp1, src2, src2, Assembler::AVX_256bit);
2084 vextracti128_high(vtmp2, vtmp1);
2085 vpaddd(vtmp1, vtmp1, vtmp2, Assembler::AVX_128bit);
2086 reduce2I(opcode, dst, src1, vtmp1, vtmp1, vtmp2);
2087 } else {
2088 vextracti128_high(vtmp1, src2);
2089 reduce_operation_128(T_INT, opcode, vtmp1, src2);
2090 reduce4I(opcode, dst, src1, vtmp1, vtmp1, vtmp2);
2091 }
2092 }
2093
2094 void C2_MacroAssembler::reduce16I(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2) {
2095 vextracti64x4_high(vtmp2, src2);
2096 reduce_operation_256(T_INT, opcode, vtmp2, vtmp2, src2);
2097 reduce8I(opcode, dst, src1, vtmp2, vtmp1, vtmp2);
2098 }
2099
2100 void C2_MacroAssembler::reduce8B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2) {
2101 pshufd(vtmp2, src2, 0x1);
2102 reduce_operation_128(T_BYTE, opcode, vtmp2, src2);
2103 movdqu(vtmp1, vtmp2);
2104 psrldq(vtmp1, 2);
2105 reduce_operation_128(T_BYTE, opcode, vtmp1, vtmp2);
2106 movdqu(vtmp2, vtmp1);
2107 psrldq(vtmp2, 1);
2108 reduce_operation_128(T_BYTE, opcode, vtmp1, vtmp2);
2109 movdl(vtmp2, src1);
2110 if (opcode == Op_UMinReductionV || opcode == Op_UMaxReductionV) {
2111 pmovzxbd(vtmp1, vtmp1);
2112 } else {
2113 pmovsxbd(vtmp1, vtmp1);
2114 }
2115 reduce_operation_128(T_INT, opcode, vtmp1, vtmp2);
2116 pextrb(dst, vtmp1, 0x0);
2117 movsbl(dst, dst);
2118 }
2119
2120 void C2_MacroAssembler::reduce16B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2) {
2121 pshufd(vtmp1, src2, 0xE);
2122 reduce_operation_128(T_BYTE, opcode, vtmp1, src2);
2123 reduce8B(opcode, dst, src1, vtmp1, vtmp1, vtmp2);
2124 }
2125
2126 void C2_MacroAssembler::reduce32B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2) {
2127 vextracti128_high(vtmp2, src2);
2128 reduce_operation_128(T_BYTE, opcode, vtmp2, src2);
2129 reduce16B(opcode, dst, src1, vtmp2, vtmp1, vtmp2);
2130 }
2131
2132 void C2_MacroAssembler::reduce64B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2) {
2133 vextracti64x4_high(vtmp1, src2);
2134 reduce_operation_256(T_BYTE, opcode, vtmp1, vtmp1, src2);
2135 reduce32B(opcode, dst, src1, vtmp1, vtmp1, vtmp2);
2136 }
2137
2138 void C2_MacroAssembler::mulreduce8B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2) {
2139 pmovsxbw(vtmp2, src2);
2140 reduce8S(opcode, dst, src1, vtmp2, vtmp1, vtmp2);
2141 }
2142
2143 void C2_MacroAssembler::mulreduce16B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2) {
2144 if (UseAVX > 1) {
2145 int vector_len = Assembler::AVX_256bit;
2146 vpmovsxbw(vtmp1, src2, vector_len);
2147 reduce16S(opcode, dst, src1, vtmp1, vtmp1, vtmp2);
2148 } else {
2149 pmovsxbw(vtmp2, src2);
2150 reduce8S(opcode, dst, src1, vtmp2, vtmp1, vtmp2);
2151 pshufd(vtmp2, src2, 0xe);
2152 pmovsxbw(vtmp2, vtmp2);
2153 reduce8S(opcode, dst, dst, vtmp2, vtmp1, vtmp2);
2154 }
2155 }
2156
2157 void C2_MacroAssembler::mulreduce32B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2) {
2158 if (UseAVX > 2 && VM_Version::supports_avx512bw()) {
2159 int vector_len = Assembler::AVX_512bit;
2160 vpmovsxbw(vtmp1, src2, vector_len);
2161 reduce32S(opcode, dst, src1, vtmp1, vtmp2, vtmp1);
2162 } else {
2163 assert(UseAVX >= 2,"Should not reach here.");
2164 mulreduce16B(opcode, dst, src1, src2, vtmp1, vtmp2);
2165 vextracti128_high(vtmp2, src2);
2166 mulreduce16B(opcode, dst, dst, vtmp2, vtmp1, vtmp2);
2167 }
2168 }
2169
2170 void C2_MacroAssembler::mulreduce64B(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2) {
2171 mulreduce32B(opcode, dst, src1, src2, vtmp1, vtmp2);
2172 vextracti64x4_high(vtmp2, src2);
2173 mulreduce32B(opcode, dst, dst, vtmp2, vtmp1, vtmp2);
2174 }
2175
2176 void C2_MacroAssembler::reduce4S(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2) {
2177 if (opcode == Op_AddReductionVI) {
2178 if (vtmp1 != src2) {
2179 movdqu(vtmp1, src2);
2180 }
2181 phaddw(vtmp1, vtmp1);
2182 phaddw(vtmp1, vtmp1);
2183 } else {
2184 pshufd(vtmp2, src2, 0x1);
2185 reduce_operation_128(T_SHORT, opcode, vtmp2, src2);
2186 movdqu(vtmp1, vtmp2);
2187 psrldq(vtmp1, 2);
2188 reduce_operation_128(T_SHORT, opcode, vtmp1, vtmp2);
2189 }
2190 movdl(vtmp2, src1);
2191 if (opcode == Op_UMinReductionV || opcode == Op_UMaxReductionV) {
2192 pmovzxwd(vtmp1, vtmp1);
2193 } else {
2194 pmovsxwd(vtmp1, vtmp1);
2195 }
2196 reduce_operation_128(T_INT, opcode, vtmp1, vtmp2);
2197 pextrw(dst, vtmp1, 0x0);
2198 movswl(dst, dst);
2199 }
2200
2201 void C2_MacroAssembler::reduce8S(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2) {
2202 if (opcode == Op_AddReductionVI) {
2203 if (vtmp1 != src2) {
2204 movdqu(vtmp1, src2);
2205 }
2206 phaddw(vtmp1, src2);
2207 } else {
2208 assert_different_registers(src2, vtmp1);
2209 pshufd(vtmp1, src2, 0xE);
2210 reduce_operation_128(T_SHORT, opcode, vtmp1, src2);
2211 }
2212 reduce4S(opcode, dst, src1, vtmp1, vtmp1, vtmp2);
2213 }
2214
2215 void C2_MacroAssembler::reduce16S(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2) {
2216 if (opcode == Op_AddReductionVI) {
2217 int vector_len = Assembler::AVX_256bit;
2218 vphaddw(vtmp2, src2, src2, vector_len);
2219 vpermq(vtmp2, vtmp2, 0xD8, vector_len);
2220 } else {
2221 assert_different_registers(src2, vtmp2);
2222 vextracti128_high(vtmp2, src2);
2223 reduce_operation_128(T_SHORT, opcode, vtmp2, src2);
2224 }
2225 reduce8S(opcode, dst, src1, vtmp2, vtmp1, vtmp2);
2226 }
2227
2228 void C2_MacroAssembler::reduce32S(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2) {
2229 assert_different_registers(src2, vtmp1);
2230 int vector_len = Assembler::AVX_256bit;
2231 vextracti64x4_high(vtmp1, src2);
2232 reduce_operation_256(T_SHORT, opcode, vtmp1, vtmp1, src2);
2233 reduce16S(opcode, dst, src1, vtmp1, vtmp1, vtmp2);
2234 }
2235
2236 void C2_MacroAssembler::reduce2L(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2) {
2237 pshufd(vtmp2, src2, 0xE);
2238 reduce_operation_128(T_LONG, opcode, vtmp2, src2);
2239 movdq(vtmp1, src1);
2240 reduce_operation_128(T_LONG, opcode, vtmp1, vtmp2);
2241 movdq(dst, vtmp1);
2242 }
2243
2244 void C2_MacroAssembler::reduce4L(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2) {
2245 vextracti128_high(vtmp1, src2);
2246 reduce_operation_128(T_LONG, opcode, vtmp1, src2);
2247 reduce2L(opcode, dst, src1, vtmp1, vtmp1, vtmp2);
2248 }
2249
2250 void C2_MacroAssembler::reduce8L(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2) {
2251 vextracti64x4_high(vtmp2, src2);
2252 reduce_operation_256(T_LONG, opcode, vtmp2, vtmp2, src2);
2253 reduce4L(opcode, dst, src1, vtmp2, vtmp1, vtmp2);
2254 }
2255
2256 void C2_MacroAssembler::genmask(KRegister dst, Register len, Register temp) {
2257 mov64(temp, -1L);
2258 bzhiq(temp, temp, len);
2259 kmovql(dst, temp);
2260 }
2261
2262 void C2_MacroAssembler::reduce2F(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp) {
2263 reduce_operation_128(T_FLOAT, opcode, dst, src);
2264 pshufd(vtmp, src, 0x1);
2265 reduce_operation_128(T_FLOAT, opcode, dst, vtmp);
2266 }
2267
2268 void C2_MacroAssembler::reduce4F(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp) {
2269 reduce2F(opcode, dst, src, vtmp);
2270 pshufd(vtmp, src, 0x2);
2271 reduce_operation_128(T_FLOAT, opcode, dst, vtmp);
2272 pshufd(vtmp, src, 0x3);
2273 reduce_operation_128(T_FLOAT, opcode, dst, vtmp);
2274 }
2275
2276 void C2_MacroAssembler::reduce8F(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2) {
2277 reduce4F(opcode, dst, src, vtmp2);
2278 vextractf128_high(vtmp2, src);
2279 reduce4F(opcode, dst, vtmp2, vtmp1);
2280 }
2281
2282 void C2_MacroAssembler::reduce16F(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2) {
2283 reduce8F(opcode, dst, src, vtmp1, vtmp2);
2284 vextracti64x4_high(vtmp1, src);
2285 reduce8F(opcode, dst, vtmp1, vtmp1, vtmp2);
2286 }
2287
2288 void C2_MacroAssembler::unorderedReduce2F(int opcode, XMMRegister dst, XMMRegister src) {
2289 pshufd(dst, src, 0x1);
2290 reduce_operation_128(T_FLOAT, opcode, dst, src);
2291 }
2292
2293 void C2_MacroAssembler::unorderedReduce4F(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp) {
2294 pshufd(vtmp, src, 0xE);
2295 unordered_reduce_operation_128(T_FLOAT, opcode, vtmp, src);
2296 unorderedReduce2F(opcode, dst, vtmp);
2297 }
2298
2299 void C2_MacroAssembler::unorderedReduce8F(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2) {
2300 vextractf128_high(vtmp1, src);
2301 unordered_reduce_operation_128(T_FLOAT, opcode, vtmp1, src);
2302 unorderedReduce4F(opcode, dst, vtmp1, vtmp2);
2303 }
2304
2305 void C2_MacroAssembler::unorderedReduce16F(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2) {
2306 vextractf64x4_high(vtmp2, src);
2307 unordered_reduce_operation_256(T_FLOAT, opcode, vtmp2, vtmp2, src);
2308 unorderedReduce8F(opcode, dst, vtmp2, vtmp1, vtmp2);
2309 }
2310
2311 void C2_MacroAssembler::reduce2D(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp) {
2312 reduce_operation_128(T_DOUBLE, opcode, dst, src);
2313 pshufd(vtmp, src, 0xE);
2314 reduce_operation_128(T_DOUBLE, opcode, dst, vtmp);
2315 }
2316
2317 void C2_MacroAssembler::reduce4D(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2) {
2318 reduce2D(opcode, dst, src, vtmp2);
2319 vextractf128_high(vtmp2, src);
2320 reduce2D(opcode, dst, vtmp2, vtmp1);
2321 }
2322
2323 void C2_MacroAssembler::reduce8D(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2) {
2324 reduce4D(opcode, dst, src, vtmp1, vtmp2);
2325 vextracti64x4_high(vtmp1, src);
2326 reduce4D(opcode, dst, vtmp1, vtmp1, vtmp2);
2327 }
2328
2329 void C2_MacroAssembler::unorderedReduce2D(int opcode, XMMRegister dst, XMMRegister src) {
2330 pshufd(dst, src, 0xE);
2331 reduce_operation_128(T_DOUBLE, opcode, dst, src);
2332 }
2333
2334 void C2_MacroAssembler::unorderedReduce4D(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp) {
2335 vextractf128_high(vtmp, src);
2336 unordered_reduce_operation_128(T_DOUBLE, opcode, vtmp, src);
2337 unorderedReduce2D(opcode, dst, vtmp);
2338 }
2339
2340 void C2_MacroAssembler::unorderedReduce8D(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp1, XMMRegister vtmp2) {
2341 vextractf64x4_high(vtmp2, src);
2342 unordered_reduce_operation_256(T_DOUBLE, opcode, vtmp2, vtmp2, src);
2343 unorderedReduce4D(opcode, dst, vtmp2, vtmp1);
2344 }
2345
2346 void C2_MacroAssembler::evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, Address src, bool merge, int vector_len) {
2347 MacroAssembler::evmovdqu(type, kmask, dst, src, merge, vector_len);
2348 }
2349
2350 void C2_MacroAssembler::evmovdqu(BasicType type, KRegister kmask, Address dst, XMMRegister src, bool merge, int vector_len) {
2351 MacroAssembler::evmovdqu(type, kmask, dst, src, merge, vector_len);
2352 }
2353
2354 void C2_MacroAssembler::evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, XMMRegister src, bool merge, int vector_len) {
2355 MacroAssembler::evmovdqu(type, kmask, dst, src, merge, vector_len);
2356 }
2357
2358 void C2_MacroAssembler::vmovmask(BasicType elem_bt, XMMRegister dst, Address src, XMMRegister mask,
2359 int vec_enc) {
2360 switch(elem_bt) {
2361 case T_INT:
2362 case T_FLOAT:
2363 vmaskmovps(dst, src, mask, vec_enc);
2364 break;
2365 case T_LONG:
2366 case T_DOUBLE:
2367 vmaskmovpd(dst, src, mask, vec_enc);
2368 break;
2369 default:
2370 fatal("Unsupported type %s", type2name(elem_bt));
2371 break;
2372 }
2373 }
2374
2375 void C2_MacroAssembler::vmovmask(BasicType elem_bt, Address dst, XMMRegister src, XMMRegister mask,
2376 int vec_enc) {
2377 switch(elem_bt) {
2378 case T_INT:
2379 case T_FLOAT:
2380 vmaskmovps(dst, src, mask, vec_enc);
2381 break;
2382 case T_LONG:
2383 case T_DOUBLE:
2384 vmaskmovpd(dst, src, mask, vec_enc);
2385 break;
2386 default:
2387 fatal("Unsupported type %s", type2name(elem_bt));
2388 break;
2389 }
2390 }
2391
2392 void C2_MacroAssembler::reduceFloatMinMax(int opcode, int vlen, bool is_dst_valid,
2393 XMMRegister dst, XMMRegister src,
2394 XMMRegister tmp, XMMRegister atmp, XMMRegister btmp,
2395 XMMRegister xmm_0, XMMRegister xmm_1) {
2396 const int permconst[] = {1, 14};
2397 XMMRegister wsrc = src;
2398 XMMRegister wdst = xmm_0;
2399 XMMRegister wtmp = (xmm_1 == xnoreg) ? xmm_0: xmm_1;
2400
2401 int vlen_enc = Assembler::AVX_128bit;
2402 if (vlen == 16) {
2403 vlen_enc = Assembler::AVX_256bit;
2404 }
2405
2406 for (int i = log2(vlen) - 1; i >=0; i--) {
2407 if (i == 0 && !is_dst_valid) {
2408 wdst = dst;
2409 }
2410 if (i == 3) {
2411 vextracti64x4_high(wtmp, wsrc);
2412 } else if (i == 2) {
2413 vextracti128_high(wtmp, wsrc);
2414 } else { // i = [0,1]
2415 vpermilps(wtmp, wsrc, permconst[i], vlen_enc);
2416 }
2417
2418 if (VM_Version::supports_avx10_2()) {
2419 vminmax_fp_avx10_2(opcode, T_FLOAT, wdst, k0, wtmp, wsrc, vlen_enc);
2420 } else {
2421 vminmax_fp(opcode, T_FLOAT, wdst, wtmp, wsrc, tmp, atmp, btmp, vlen_enc);
2422 }
2423 wsrc = wdst;
2424 vlen_enc = Assembler::AVX_128bit;
2425 }
2426 if (is_dst_valid) {
2427 if (VM_Version::supports_avx10_2()) {
2428 vminmax_fp_avx10_2(opcode, T_FLOAT, dst, k0, wdst, dst, Assembler::AVX_128bit);
2429 } else {
2430 vminmax_fp(opcode, T_FLOAT, dst, wdst, dst, tmp, atmp, btmp, Assembler::AVX_128bit);
2431 }
2432 }
2433 }
2434
2435 void C2_MacroAssembler::reduceDoubleMinMax(int opcode, int vlen, bool is_dst_valid, XMMRegister dst, XMMRegister src,
2436 XMMRegister tmp, XMMRegister atmp, XMMRegister btmp,
2437 XMMRegister xmm_0, XMMRegister xmm_1) {
2438 XMMRegister wsrc = src;
2439 XMMRegister wdst = xmm_0;
2440 XMMRegister wtmp = (xmm_1 == xnoreg) ? xmm_0: xmm_1;
2441 int vlen_enc = Assembler::AVX_128bit;
2442 if (vlen == 8) {
2443 vlen_enc = Assembler::AVX_256bit;
2444 }
2445 for (int i = log2(vlen) - 1; i >=0; i--) {
2446 if (i == 0 && !is_dst_valid) {
2447 wdst = dst;
2448 }
2449 if (i == 1) {
2450 vextracti128_high(wtmp, wsrc);
2451 } else if (i == 2) {
2452 vextracti64x4_high(wtmp, wsrc);
2453 } else {
2454 assert(i == 0, "%d", i);
2455 vpermilpd(wtmp, wsrc, 1, vlen_enc);
2456 }
2457
2458 if (VM_Version::supports_avx10_2()) {
2459 vminmax_fp_avx10_2(opcode, T_DOUBLE, wdst, k0, wtmp, wsrc, vlen_enc);
2460 } else {
2461 vminmax_fp(opcode, T_DOUBLE, wdst, wtmp, wsrc, tmp, atmp, btmp, vlen_enc);
2462 }
2463
2464 wsrc = wdst;
2465 vlen_enc = Assembler::AVX_128bit;
2466 }
2467
2468 if (is_dst_valid) {
2469 if (VM_Version::supports_avx10_2()) {
2470 vminmax_fp_avx10_2(opcode, T_DOUBLE, dst, k0, wdst, dst, Assembler::AVX_128bit);
2471 } else {
2472 vminmax_fp(opcode, T_DOUBLE, dst, wdst, dst, tmp, atmp, btmp, Assembler::AVX_128bit);
2473 }
2474 }
2475 }
2476
2477 void C2_MacroAssembler::extract(BasicType bt, Register dst, XMMRegister src, int idx) {
2478 switch (bt) {
2479 case T_BYTE: pextrb(dst, src, idx); break;
2480 case T_SHORT: pextrw(dst, src, idx); break;
2481 case T_INT: pextrd(dst, src, idx); break;
2482 case T_LONG: pextrq(dst, src, idx); break;
2483
2484 default:
2485 assert(false,"Should not reach here.");
2486 break;
2487 }
2488 }
2489
2490 XMMRegister C2_MacroAssembler::get_lane(BasicType typ, XMMRegister dst, XMMRegister src, int elemindex) {
2491 int esize = type2aelembytes(typ);
2492 int elem_per_lane = 16/esize;
2493 int lane = elemindex / elem_per_lane;
2494 int eindex = elemindex % elem_per_lane;
2495
2496 if (lane >= 2) {
2497 assert(UseAVX > 2, "required");
2498 vextractf32x4(dst, src, lane & 3);
2499 return dst;
2500 } else if (lane > 0) {
2501 assert(UseAVX > 0, "required");
2502 vextractf128(dst, src, lane);
2503 return dst;
2504 } else {
2505 return src;
2506 }
2507 }
2508
2509 void C2_MacroAssembler::movsxl(BasicType typ, Register dst) {
2510 if (typ == T_BYTE) {
2511 movsbl(dst, dst);
2512 } else if (typ == T_SHORT) {
2513 movswl(dst, dst);
2514 }
2515 }
2516
2517 void C2_MacroAssembler::get_elem(BasicType typ, Register dst, XMMRegister src, int elemindex) {
2518 int esize = type2aelembytes(typ);
2519 int elem_per_lane = 16/esize;
2520 int eindex = elemindex % elem_per_lane;
2521 assert(is_integral_type(typ),"required");
2522
2523 if (eindex == 0) {
2524 if (typ == T_LONG) {
2525 movq(dst, src);
2526 } else {
2527 movdl(dst, src);
2528 movsxl(typ, dst);
2529 }
2530 } else {
2531 extract(typ, dst, src, eindex);
2532 movsxl(typ, dst);
2533 }
2534 }
2535
2536 void C2_MacroAssembler::get_elem(BasicType typ, XMMRegister dst, XMMRegister src, int elemindex, XMMRegister vtmp) {
2537 int esize = type2aelembytes(typ);
2538 int elem_per_lane = 16/esize;
2539 int eindex = elemindex % elem_per_lane;
2540 assert((typ == T_FLOAT || typ == T_DOUBLE),"required");
2541
2542 if (eindex == 0) {
2543 movq(dst, src);
2544 } else {
2545 if (typ == T_FLOAT) {
2546 if (UseAVX == 0) {
2547 movdqu(dst, src);
2548 shufps(dst, dst, eindex);
2549 } else {
2550 vshufps(dst, src, src, eindex, Assembler::AVX_128bit);
2551 }
2552 } else {
2553 if (UseAVX == 0) {
2554 movdqu(dst, src);
2555 psrldq(dst, eindex*esize);
2556 } else {
2557 vpsrldq(dst, src, eindex*esize, Assembler::AVX_128bit);
2558 }
2559 movq(dst, dst);
2560 }
2561 }
2562 // Zero upper bits
2563 if (typ == T_FLOAT) {
2564 if (UseAVX == 0) {
2565 assert(vtmp != xnoreg, "required.");
2566 movdqu(vtmp, ExternalAddress(StubRoutines::x86::vector_32_bit_mask()), noreg);
2567 pand(dst, vtmp);
2568 } else {
2569 vpand(dst, dst, ExternalAddress(StubRoutines::x86::vector_32_bit_mask()), Assembler::AVX_128bit, noreg);
2570 }
2571 }
2572 }
2573
2574 void C2_MacroAssembler::evpcmp(BasicType typ, KRegister kdmask, KRegister ksmask, XMMRegister src1, XMMRegister src2, int comparison, int vector_len) {
2575 switch(typ) {
2576 case T_BYTE:
2577 case T_BOOLEAN:
2578 evpcmpb(kdmask, ksmask, src1, src2, comparison, /*signed*/ true, vector_len);
2579 break;
2580 case T_SHORT:
2581 case T_CHAR:
2582 evpcmpw(kdmask, ksmask, src1, src2, comparison, /*signed*/ true, vector_len);
2583 break;
2584 case T_INT:
2585 case T_FLOAT:
2586 evpcmpd(kdmask, ksmask, src1, src2, comparison, /*signed*/ true, vector_len);
2587 break;
2588 case T_LONG:
2589 case T_DOUBLE:
2590 evpcmpq(kdmask, ksmask, src1, src2, comparison, /*signed*/ true, vector_len);
2591 break;
2592 default:
2593 assert(false,"Should not reach here.");
2594 break;
2595 }
2596 }
2597
2598 void C2_MacroAssembler::evpcmp(BasicType typ, KRegister kdmask, KRegister ksmask, XMMRegister src1, AddressLiteral src2, int comparison, int vector_len, Register rscratch) {
2599 assert(rscratch != noreg || always_reachable(src2), "missing");
2600
2601 switch(typ) {
2602 case T_BOOLEAN:
2603 case T_BYTE:
2604 evpcmpb(kdmask, ksmask, src1, src2, comparison, /*signed*/ true, vector_len, rscratch);
2605 break;
2606 case T_CHAR:
2607 case T_SHORT:
2608 evpcmpw(kdmask, ksmask, src1, src2, comparison, /*signed*/ true, vector_len, rscratch);
2609 break;
2610 case T_INT:
2611 case T_FLOAT:
2612 evpcmpd(kdmask, ksmask, src1, src2, comparison, /*signed*/ true, vector_len, rscratch);
2613 break;
2614 case T_LONG:
2615 case T_DOUBLE:
2616 evpcmpq(kdmask, ksmask, src1, src2, comparison, /*signed*/ true, vector_len, rscratch);
2617 break;
2618 default:
2619 assert(false,"Should not reach here.");
2620 break;
2621 }
2622 }
2623
2624 void C2_MacroAssembler::evpblend(BasicType typ, XMMRegister dst, KRegister kmask, XMMRegister src1, XMMRegister src2, bool merge, int vector_len) {
2625 switch(typ) {
2626 case T_BYTE:
2627 evpblendmb(dst, kmask, src1, src2, merge, vector_len);
2628 break;
2629 case T_SHORT:
2630 evpblendmw(dst, kmask, src1, src2, merge, vector_len);
2631 break;
2632 case T_INT:
2633 case T_FLOAT:
2634 evpblendmd(dst, kmask, src1, src2, merge, vector_len);
2635 break;
2636 case T_LONG:
2637 case T_DOUBLE:
2638 evpblendmq(dst, kmask, src1, src2, merge, vector_len);
2639 break;
2640 default:
2641 assert(false,"Should not reach here.");
2642 break;
2643 }
2644 }
2645
2646 void C2_MacroAssembler::vectortest(BasicType bt, XMMRegister src1, XMMRegister src2, XMMRegister vtmp, int vlen_in_bytes) {
2647 assert(vlen_in_bytes <= 32, "");
2648 int esize = type2aelembytes(bt);
2649 if (vlen_in_bytes == 32) {
2650 assert(vtmp == xnoreg, "required.");
2651 if (esize >= 4) {
2652 vtestps(src1, src2, AVX_256bit);
2653 } else {
2654 vptest(src1, src2, AVX_256bit);
2655 }
2656 return;
2657 }
2658 if (vlen_in_bytes < 16) {
2659 // Duplicate the lower part to fill the whole register,
2660 // Don't need to do so for src2
2661 assert(vtmp != xnoreg, "required");
2662 int shuffle_imm = (vlen_in_bytes == 4) ? 0x00 : 0x04;
2663 pshufd(vtmp, src1, shuffle_imm);
2664 } else {
2665 assert(vtmp == xnoreg, "required");
2666 vtmp = src1;
2667 }
2668 if (esize >= 4 && VM_Version::supports_avx()) {
2669 vtestps(vtmp, src2, AVX_128bit);
2670 } else {
2671 ptest(vtmp, src2);
2672 }
2673 }
2674
2675 void C2_MacroAssembler::vpadd(BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2, int vlen_enc) {
2676 #ifdef ASSERT
2677 bool is_bw = ((elem_bt == T_BYTE) || (elem_bt == T_SHORT));
2678 bool is_bw_supported = VM_Version::supports_avx512bw();
2679 if (is_bw && !is_bw_supported) {
2680 assert(vlen_enc != Assembler::AVX_512bit, "required");
2681 assert((dst->encoding() < 16) && (src1->encoding() < 16) && (src2->encoding() < 16),
2682 "XMM register should be 0-15");
2683 }
2684 #endif // ASSERT
2685 switch (elem_bt) {
2686 case T_BYTE: vpaddb(dst, src1, src2, vlen_enc); return;
2687 case T_SHORT: vpaddw(dst, src1, src2, vlen_enc); return;
2688 case T_INT: vpaddd(dst, src1, src2, vlen_enc); return;
2689 case T_FLOAT: vaddps(dst, src1, src2, vlen_enc); return;
2690 case T_LONG: vpaddq(dst, src1, src2, vlen_enc); return;
2691 case T_DOUBLE: vaddpd(dst, src1, src2, vlen_enc); return;
2692 default: fatal("Unsupported type %s", type2name(elem_bt)); return;
2693 }
2694 }
2695
2696 void C2_MacroAssembler::vpbroadcast(BasicType elem_bt, XMMRegister dst, Register src, int vlen_enc) {
2697 assert(UseAVX >= 2, "required");
2698 bool is_bw = ((elem_bt == T_BYTE) || (elem_bt == T_SHORT));
2699 bool is_vl = vlen_enc != Assembler::AVX_512bit;
2700 if ((UseAVX > 2) &&
2701 (!is_bw || VM_Version::supports_avx512bw()) &&
2702 (!is_vl || VM_Version::supports_avx512vl())) {
2703 switch (elem_bt) {
2704 case T_BYTE: evpbroadcastb(dst, src, vlen_enc); return;
2705 case T_SHORT: evpbroadcastw(dst, src, vlen_enc); return;
2706 case T_FLOAT: case T_INT: evpbroadcastd(dst, src, vlen_enc); return;
2707 case T_DOUBLE: case T_LONG: evpbroadcastq(dst, src, vlen_enc); return;
2708 default: fatal("Unsupported type %s", type2name(elem_bt)); return;
2709 }
2710 } else {
2711 assert(vlen_enc != Assembler::AVX_512bit, "required");
2712 assert((dst->encoding() < 16),"XMM register should be 0-15");
2713 switch (elem_bt) {
2714 case T_BYTE: movdl(dst, src); vpbroadcastb(dst, dst, vlen_enc); return;
2715 case T_SHORT: movdl(dst, src); vpbroadcastw(dst, dst, vlen_enc); return;
2716 case T_INT: movdl(dst, src); vpbroadcastd(dst, dst, vlen_enc); return;
2717 case T_FLOAT: movdl(dst, src); vbroadcastss(dst, dst, vlen_enc); return;
2718 case T_LONG: movdq(dst, src); vpbroadcastq(dst, dst, vlen_enc); return;
2719 case T_DOUBLE: movdq(dst, src); vbroadcastsd(dst, dst, vlen_enc); return;
2720 default: fatal("Unsupported type %s", type2name(elem_bt)); return;
2721 }
2722 }
2723 }
2724
2725 void C2_MacroAssembler::vconvert_b2x(BasicType to_elem_bt, XMMRegister dst, XMMRegister src, int vlen_enc) {
2726 switch (to_elem_bt) {
2727 case T_SHORT:
2728 vpmovsxbw(dst, src, vlen_enc);
2729 break;
2730 case T_INT:
2731 vpmovsxbd(dst, src, vlen_enc);
2732 break;
2733 case T_FLOAT:
2734 vpmovsxbd(dst, src, vlen_enc);
2735 vcvtdq2ps(dst, dst, vlen_enc);
2736 break;
2737 case T_LONG:
2738 vpmovsxbq(dst, src, vlen_enc);
2739 break;
2740 case T_DOUBLE: {
2741 int mid_vlen_enc = (vlen_enc == Assembler::AVX_512bit) ? Assembler::AVX_256bit : Assembler::AVX_128bit;
2742 vpmovsxbd(dst, src, mid_vlen_enc);
2743 vcvtdq2pd(dst, dst, vlen_enc);
2744 break;
2745 }
2746 default:
2747 fatal("Unsupported type %s", type2name(to_elem_bt));
2748 break;
2749 }
2750 }
2751
2752 //-------------------------------------------------------------------------------------------
2753
2754 // IndexOf for constant substrings with size >= 8 chars
2755 // which don't need to be loaded through stack.
2756 void C2_MacroAssembler::string_indexofC8(Register str1, Register str2,
2757 Register cnt1, Register cnt2,
2758 int int_cnt2, Register result,
2759 XMMRegister vec, Register tmp,
2760 int ae) {
2761 ShortBranchVerifier sbv(this);
2762 assert(UseSSE42Intrinsics, "SSE4.2 intrinsics are required");
2763 assert(ae != StrIntrinsicNode::LU, "Invalid encoding");
2764
2765 // This method uses the pcmpestri instruction with bound registers
2766 // inputs:
2767 // xmm - substring
2768 // rax - substring length (elements count)
2769 // mem - scanned string
2770 // rdx - string length (elements count)
2771 // 0xd - mode: 1100 (substring search) + 01 (unsigned shorts)
2772 // 0xc - mode: 1100 (substring search) + 00 (unsigned bytes)
2773 // outputs:
2774 // rcx - matched index in string
2775 assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri");
2776 int mode = (ae == StrIntrinsicNode::LL) ? 0x0c : 0x0d; // bytes or shorts
2777 int stride = (ae == StrIntrinsicNode::LL) ? 16 : 8; //UU, UL -> 8
2778 Address::ScaleFactor scale1 = (ae == StrIntrinsicNode::LL) ? Address::times_1 : Address::times_2;
2779 Address::ScaleFactor scale2 = (ae == StrIntrinsicNode::UL) ? Address::times_1 : scale1;
2780
2781 Label RELOAD_SUBSTR, SCAN_TO_SUBSTR, SCAN_SUBSTR,
2782 RET_FOUND, RET_NOT_FOUND, EXIT, FOUND_SUBSTR,
2783 MATCH_SUBSTR_HEAD, RELOAD_STR, FOUND_CANDIDATE;
2784
2785 // Note, inline_string_indexOf() generates checks:
2786 // if (substr.count > string.count) return -1;
2787 // if (substr.count == 0) return 0;
2788 assert(int_cnt2 >= stride, "this code is used only for cnt2 >= 8 chars");
2789
2790 // Load substring.
2791 if (ae == StrIntrinsicNode::UL) {
2792 pmovzxbw(vec, Address(str2, 0));
2793 } else {
2794 movdqu(vec, Address(str2, 0));
2795 }
2796 movl(cnt2, int_cnt2);
2797 movptr(result, str1); // string addr
2798
2799 if (int_cnt2 > stride) {
2800 jmpb(SCAN_TO_SUBSTR);
2801
2802 // Reload substr for rescan, this code
2803 // is executed only for large substrings (> 8 chars)
2804 bind(RELOAD_SUBSTR);
2805 if (ae == StrIntrinsicNode::UL) {
2806 pmovzxbw(vec, Address(str2, 0));
2807 } else {
2808 movdqu(vec, Address(str2, 0));
2809 }
2810 negptr(cnt2); // Jumped here with negative cnt2, convert to positive
2811
2812 bind(RELOAD_STR);
2813 // We came here after the beginning of the substring was
2814 // matched but the rest of it was not so we need to search
2815 // again. Start from the next element after the previous match.
2816
2817 // cnt2 is number of substring reminding elements and
2818 // cnt1 is number of string reminding elements when cmp failed.
2819 // Restored cnt1 = cnt1 - cnt2 + int_cnt2
2820 subl(cnt1, cnt2);
2821 addl(cnt1, int_cnt2);
2822 movl(cnt2, int_cnt2); // Now restore cnt2
2823
2824 decrementl(cnt1); // Shift to next element
2825 cmpl(cnt1, cnt2);
2826 jcc(Assembler::negative, RET_NOT_FOUND); // Left less then substring
2827
2828 addptr(result, (1<<scale1));
2829
2830 } // (int_cnt2 > 8)
2831
2832 // Scan string for start of substr in 16-byte vectors
2833 bind(SCAN_TO_SUBSTR);
2834 pcmpestri(vec, Address(result, 0), mode);
2835 jccb(Assembler::below, FOUND_CANDIDATE); // CF == 1
2836 subl(cnt1, stride);
2837 jccb(Assembler::lessEqual, RET_NOT_FOUND); // Scanned full string
2838 cmpl(cnt1, cnt2);
2839 jccb(Assembler::negative, RET_NOT_FOUND); // Left less then substring
2840 addptr(result, 16);
2841 jmpb(SCAN_TO_SUBSTR);
2842
2843 // Found a potential substr
2844 bind(FOUND_CANDIDATE);
2845 // Matched whole vector if first element matched (tmp(rcx) == 0).
2846 if (int_cnt2 == stride) {
2847 jccb(Assembler::overflow, RET_FOUND); // OF == 1
2848 } else { // int_cnt2 > 8
2849 jccb(Assembler::overflow, FOUND_SUBSTR);
2850 }
2851 // After pcmpestri tmp(rcx) contains matched element index
2852 // Compute start addr of substr
2853 lea(result, Address(result, tmp, scale1));
2854
2855 // Make sure string is still long enough
2856 subl(cnt1, tmp);
2857 cmpl(cnt1, cnt2);
2858 if (int_cnt2 == stride) {
2859 jccb(Assembler::greaterEqual, SCAN_TO_SUBSTR);
2860 } else { // int_cnt2 > 8
2861 jccb(Assembler::greaterEqual, MATCH_SUBSTR_HEAD);
2862 }
2863 // Left less then substring.
2864
2865 bind(RET_NOT_FOUND);
2866 movl(result, -1);
2867 jmp(EXIT);
2868
2869 if (int_cnt2 > stride) {
2870 // This code is optimized for the case when whole substring
2871 // is matched if its head is matched.
2872 bind(MATCH_SUBSTR_HEAD);
2873 pcmpestri(vec, Address(result, 0), mode);
2874 // Reload only string if does not match
2875 jcc(Assembler::noOverflow, RELOAD_STR); // OF == 0
2876
2877 Label CONT_SCAN_SUBSTR;
2878 // Compare the rest of substring (> 8 chars).
2879 bind(FOUND_SUBSTR);
2880 // First 8 chars are already matched.
2881 negptr(cnt2);
2882 addptr(cnt2, stride);
2883
2884 bind(SCAN_SUBSTR);
2885 subl(cnt1, stride);
2886 cmpl(cnt2, -stride); // Do not read beyond substring
2887 jccb(Assembler::lessEqual, CONT_SCAN_SUBSTR);
2888 // Back-up strings to avoid reading beyond substring:
2889 // cnt1 = cnt1 - cnt2 + 8
2890 addl(cnt1, cnt2); // cnt2 is negative
2891 addl(cnt1, stride);
2892 movl(cnt2, stride); negptr(cnt2);
2893 bind(CONT_SCAN_SUBSTR);
2894 if (int_cnt2 < (int)G) {
2895 int tail_off1 = int_cnt2<<scale1;
2896 int tail_off2 = int_cnt2<<scale2;
2897 if (ae == StrIntrinsicNode::UL) {
2898 pmovzxbw(vec, Address(str2, cnt2, scale2, tail_off2));
2899 } else {
2900 movdqu(vec, Address(str2, cnt2, scale2, tail_off2));
2901 }
2902 pcmpestri(vec, Address(result, cnt2, scale1, tail_off1), mode);
2903 } else {
2904 // calculate index in register to avoid integer overflow (int_cnt2*2)
2905 movl(tmp, int_cnt2);
2906 addptr(tmp, cnt2);
2907 if (ae == StrIntrinsicNode::UL) {
2908 pmovzxbw(vec, Address(str2, tmp, scale2, 0));
2909 } else {
2910 movdqu(vec, Address(str2, tmp, scale2, 0));
2911 }
2912 pcmpestri(vec, Address(result, tmp, scale1, 0), mode);
2913 }
2914 // Need to reload strings pointers if not matched whole vector
2915 jcc(Assembler::noOverflow, RELOAD_SUBSTR); // OF == 0
2916 addptr(cnt2, stride);
2917 jcc(Assembler::negative, SCAN_SUBSTR);
2918 // Fall through if found full substring
2919
2920 } // (int_cnt2 > 8)
2921
2922 bind(RET_FOUND);
2923 // Found result if we matched full small substring.
2924 // Compute substr offset
2925 subptr(result, str1);
2926 if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) {
2927 shrl(result, 1); // index
2928 }
2929 bind(EXIT);
2930
2931 } // string_indexofC8
2932
2933 // Small strings are loaded through stack if they cross page boundary.
2934 void C2_MacroAssembler::string_indexof(Register str1, Register str2,
2935 Register cnt1, Register cnt2,
2936 int int_cnt2, Register result,
2937 XMMRegister vec, Register tmp,
2938 int ae) {
2939 ShortBranchVerifier sbv(this);
2940 assert(UseSSE42Intrinsics, "SSE4.2 intrinsics are required");
2941 assert(ae != StrIntrinsicNode::LU, "Invalid encoding");
2942
2943 //
2944 // int_cnt2 is length of small (< 8 chars) constant substring
2945 // or (-1) for non constant substring in which case its length
2946 // is in cnt2 register.
2947 //
2948 // Note, inline_string_indexOf() generates checks:
2949 // if (substr.count > string.count) return -1;
2950 // if (substr.count == 0) return 0;
2951 //
2952 int stride = (ae == StrIntrinsicNode::LL) ? 16 : 8; //UU, UL -> 8
2953 assert(int_cnt2 == -1 || (0 < int_cnt2 && int_cnt2 < stride), "should be != 0");
2954 // This method uses the pcmpestri instruction with bound registers
2955 // inputs:
2956 // xmm - substring
2957 // rax - substring length (elements count)
2958 // mem - scanned string
2959 // rdx - string length (elements count)
2960 // 0xd - mode: 1100 (substring search) + 01 (unsigned shorts)
2961 // 0xc - mode: 1100 (substring search) + 00 (unsigned bytes)
2962 // outputs:
2963 // rcx - matched index in string
2964 assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri");
2965 int mode = (ae == StrIntrinsicNode::LL) ? 0x0c : 0x0d; // bytes or shorts
2966 Address::ScaleFactor scale1 = (ae == StrIntrinsicNode::LL) ? Address::times_1 : Address::times_2;
2967 Address::ScaleFactor scale2 = (ae == StrIntrinsicNode::UL) ? Address::times_1 : scale1;
2968
2969 Label RELOAD_SUBSTR, SCAN_TO_SUBSTR, SCAN_SUBSTR, ADJUST_STR,
2970 RET_FOUND, RET_NOT_FOUND, CLEANUP, FOUND_SUBSTR,
2971 FOUND_CANDIDATE;
2972
2973 { //========================================================
2974 // We don't know where these strings are located
2975 // and we can't read beyond them. Load them through stack.
2976 Label BIG_STRINGS, CHECK_STR, COPY_SUBSTR, COPY_STR;
2977
2978 movptr(tmp, rsp); // save old SP
2979
2980 if (int_cnt2 > 0) { // small (< 8 chars) constant substring
2981 if (int_cnt2 == (1>>scale2)) { // One byte
2982 assert((ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UL), "Only possible for latin1 encoding");
2983 load_unsigned_byte(result, Address(str2, 0));
2984 movdl(vec, result); // move 32 bits
2985 } else if (ae == StrIntrinsicNode::LL && int_cnt2 == 3) { // Three bytes
2986 // Not enough header space in 32-bit VM: 12+3 = 15.
2987 movl(result, Address(str2, -1));
2988 shrl(result, 8);
2989 movdl(vec, result); // move 32 bits
2990 } else if (ae != StrIntrinsicNode::UL && int_cnt2 == (2>>scale2)) { // One char
2991 load_unsigned_short(result, Address(str2, 0));
2992 movdl(vec, result); // move 32 bits
2993 } else if (ae != StrIntrinsicNode::UL && int_cnt2 == (4>>scale2)) { // Two chars
2994 movdl(vec, Address(str2, 0)); // move 32 bits
2995 } else if (ae != StrIntrinsicNode::UL && int_cnt2 == (8>>scale2)) { // Four chars
2996 movq(vec, Address(str2, 0)); // move 64 bits
2997 } else { // cnt2 = { 3, 5, 6, 7 } || (ae == StrIntrinsicNode::UL && cnt2 ={2, ..., 7})
2998 // Array header size is 12 bytes in 32-bit VM
2999 // + 6 bytes for 3 chars == 18 bytes,
3000 // enough space to load vec and shift.
3001 assert(HeapWordSize*TypeArrayKlass::header_size() >= 12,"sanity");
3002 if (ae == StrIntrinsicNode::UL) {
3003 int tail_off = int_cnt2-8;
3004 pmovzxbw(vec, Address(str2, tail_off));
3005 psrldq(vec, -2*tail_off);
3006 }
3007 else {
3008 int tail_off = int_cnt2*(1<<scale2);
3009 movdqu(vec, Address(str2, tail_off-16));
3010 psrldq(vec, 16-tail_off);
3011 }
3012 }
3013 } else { // not constant substring
3014 cmpl(cnt2, stride);
3015 jccb(Assembler::aboveEqual, BIG_STRINGS); // Both strings are big enough
3016
3017 // We can read beyond string if srt+16 does not cross page boundary
3018 // since heaps are aligned and mapped by pages.
3019 assert(os::vm_page_size() < (int)G, "default page should be small");
3020 movl(result, str2); // We need only low 32 bits
3021 andl(result, ((int)os::vm_page_size()-1));
3022 cmpl(result, ((int)os::vm_page_size()-16));
3023 jccb(Assembler::belowEqual, CHECK_STR);
3024
3025 // Move small strings to stack to allow load 16 bytes into vec.
3026 subptr(rsp, 16);
3027 int stk_offset = wordSize-(1<<scale2);
3028 push(cnt2);
3029
3030 bind(COPY_SUBSTR);
3031 if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UL) {
3032 load_unsigned_byte(result, Address(str2, cnt2, scale2, -1));
3033 movb(Address(rsp, cnt2, scale2, stk_offset), result);
3034 } else if (ae == StrIntrinsicNode::UU) {
3035 load_unsigned_short(result, Address(str2, cnt2, scale2, -2));
3036 movw(Address(rsp, cnt2, scale2, stk_offset), result);
3037 }
3038 decrement(cnt2);
3039 jccb(Assembler::notZero, COPY_SUBSTR);
3040
3041 pop(cnt2);
3042 movptr(str2, rsp); // New substring address
3043 } // non constant
3044
3045 bind(CHECK_STR);
3046 cmpl(cnt1, stride);
3047 jccb(Assembler::aboveEqual, BIG_STRINGS);
3048
3049 // Check cross page boundary.
3050 movl(result, str1); // We need only low 32 bits
3051 andl(result, ((int)os::vm_page_size()-1));
3052 cmpl(result, ((int)os::vm_page_size()-16));
3053 jccb(Assembler::belowEqual, BIG_STRINGS);
3054
3055 subptr(rsp, 16);
3056 int stk_offset = -(1<<scale1);
3057 if (int_cnt2 < 0) { // not constant
3058 push(cnt2);
3059 stk_offset += wordSize;
3060 }
3061 movl(cnt2, cnt1);
3062
3063 bind(COPY_STR);
3064 if (ae == StrIntrinsicNode::LL) {
3065 load_unsigned_byte(result, Address(str1, cnt2, scale1, -1));
3066 movb(Address(rsp, cnt2, scale1, stk_offset), result);
3067 } else {
3068 load_unsigned_short(result, Address(str1, cnt2, scale1, -2));
3069 movw(Address(rsp, cnt2, scale1, stk_offset), result);
3070 }
3071 decrement(cnt2);
3072 jccb(Assembler::notZero, COPY_STR);
3073
3074 if (int_cnt2 < 0) { // not constant
3075 pop(cnt2);
3076 }
3077 movptr(str1, rsp); // New string address
3078
3079 bind(BIG_STRINGS);
3080 // Load substring.
3081 if (int_cnt2 < 0) { // -1
3082 if (ae == StrIntrinsicNode::UL) {
3083 pmovzxbw(vec, Address(str2, 0));
3084 } else {
3085 movdqu(vec, Address(str2, 0));
3086 }
3087 push(cnt2); // substr count
3088 push(str2); // substr addr
3089 push(str1); // string addr
3090 } else {
3091 // Small (< 8 chars) constant substrings are loaded already.
3092 movl(cnt2, int_cnt2);
3093 }
3094 push(tmp); // original SP
3095
3096 } // Finished loading
3097
3098 //========================================================
3099 // Start search
3100 //
3101
3102 movptr(result, str1); // string addr
3103
3104 if (int_cnt2 < 0) { // Only for non constant substring
3105 jmpb(SCAN_TO_SUBSTR);
3106
3107 // SP saved at sp+0
3108 // String saved at sp+1*wordSize
3109 // Substr saved at sp+2*wordSize
3110 // Substr count saved at sp+3*wordSize
3111
3112 // Reload substr for rescan, this code
3113 // is executed only for large substrings (> 8 chars)
3114 bind(RELOAD_SUBSTR);
3115 movptr(str2, Address(rsp, 2*wordSize));
3116 movl(cnt2, Address(rsp, 3*wordSize));
3117 if (ae == StrIntrinsicNode::UL) {
3118 pmovzxbw(vec, Address(str2, 0));
3119 } else {
3120 movdqu(vec, Address(str2, 0));
3121 }
3122 // We came here after the beginning of the substring was
3123 // matched but the rest of it was not so we need to search
3124 // again. Start from the next element after the previous match.
3125 subptr(str1, result); // Restore counter
3126 if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) {
3127 shrl(str1, 1);
3128 }
3129 addl(cnt1, str1);
3130 decrementl(cnt1); // Shift to next element
3131 cmpl(cnt1, cnt2);
3132 jcc(Assembler::negative, RET_NOT_FOUND); // Left less then substring
3133
3134 addptr(result, (1<<scale1));
3135 } // non constant
3136
3137 // Scan string for start of substr in 16-byte vectors
3138 bind(SCAN_TO_SUBSTR);
3139 assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri");
3140 pcmpestri(vec, Address(result, 0), mode);
3141 jccb(Assembler::below, FOUND_CANDIDATE); // CF == 1
3142 subl(cnt1, stride);
3143 jccb(Assembler::lessEqual, RET_NOT_FOUND); // Scanned full string
3144 cmpl(cnt1, cnt2);
3145 jccb(Assembler::negative, RET_NOT_FOUND); // Left less then substring
3146 addptr(result, 16);
3147
3148 bind(ADJUST_STR);
3149 cmpl(cnt1, stride); // Do not read beyond string
3150 jccb(Assembler::greaterEqual, SCAN_TO_SUBSTR);
3151 // Back-up string to avoid reading beyond string.
3152 lea(result, Address(result, cnt1, scale1, -16));
3153 movl(cnt1, stride);
3154 jmpb(SCAN_TO_SUBSTR);
3155
3156 // Found a potential substr
3157 bind(FOUND_CANDIDATE);
3158 // After pcmpestri tmp(rcx) contains matched element index
3159
3160 // Make sure string is still long enough
3161 subl(cnt1, tmp);
3162 cmpl(cnt1, cnt2);
3163 jccb(Assembler::greaterEqual, FOUND_SUBSTR);
3164 // Left less then substring.
3165
3166 bind(RET_NOT_FOUND);
3167 movl(result, -1);
3168 jmp(CLEANUP);
3169
3170 bind(FOUND_SUBSTR);
3171 // Compute start addr of substr
3172 lea(result, Address(result, tmp, scale1));
3173 if (int_cnt2 > 0) { // Constant substring
3174 // Repeat search for small substring (< 8 chars)
3175 // from new point without reloading substring.
3176 // Have to check that we don't read beyond string.
3177 cmpl(tmp, stride-int_cnt2);
3178 jccb(Assembler::greater, ADJUST_STR);
3179 // Fall through if matched whole substring.
3180 } else { // non constant
3181 assert(int_cnt2 == -1, "should be != 0");
3182
3183 addl(tmp, cnt2);
3184 // Found result if we matched whole substring.
3185 cmpl(tmp, stride);
3186 jcc(Assembler::lessEqual, RET_FOUND);
3187
3188 // Repeat search for small substring (<= 8 chars)
3189 // from new point 'str1' without reloading substring.
3190 cmpl(cnt2, stride);
3191 // Have to check that we don't read beyond string.
3192 jccb(Assembler::lessEqual, ADJUST_STR);
3193
3194 Label CHECK_NEXT, CONT_SCAN_SUBSTR, RET_FOUND_LONG;
3195 // Compare the rest of substring (> 8 chars).
3196 movptr(str1, result);
3197
3198 cmpl(tmp, cnt2);
3199 // First 8 chars are already matched.
3200 jccb(Assembler::equal, CHECK_NEXT);
3201
3202 bind(SCAN_SUBSTR);
3203 pcmpestri(vec, Address(str1, 0), mode);
3204 // Need to reload strings pointers if not matched whole vector
3205 jcc(Assembler::noOverflow, RELOAD_SUBSTR); // OF == 0
3206
3207 bind(CHECK_NEXT);
3208 subl(cnt2, stride);
3209 jccb(Assembler::lessEqual, RET_FOUND_LONG); // Found full substring
3210 addptr(str1, 16);
3211 if (ae == StrIntrinsicNode::UL) {
3212 addptr(str2, 8);
3213 } else {
3214 addptr(str2, 16);
3215 }
3216 subl(cnt1, stride);
3217 cmpl(cnt2, stride); // Do not read beyond substring
3218 jccb(Assembler::greaterEqual, CONT_SCAN_SUBSTR);
3219 // Back-up strings to avoid reading beyond substring.
3220
3221 if (ae == StrIntrinsicNode::UL) {
3222 lea(str2, Address(str2, cnt2, scale2, -8));
3223 lea(str1, Address(str1, cnt2, scale1, -16));
3224 } else {
3225 lea(str2, Address(str2, cnt2, scale2, -16));
3226 lea(str1, Address(str1, cnt2, scale1, -16));
3227 }
3228 subl(cnt1, cnt2);
3229 movl(cnt2, stride);
3230 addl(cnt1, stride);
3231 bind(CONT_SCAN_SUBSTR);
3232 if (ae == StrIntrinsicNode::UL) {
3233 pmovzxbw(vec, Address(str2, 0));
3234 } else {
3235 movdqu(vec, Address(str2, 0));
3236 }
3237 jmp(SCAN_SUBSTR);
3238
3239 bind(RET_FOUND_LONG);
3240 movptr(str1, Address(rsp, wordSize));
3241 } // non constant
3242
3243 bind(RET_FOUND);
3244 // Compute substr offset
3245 subptr(result, str1);
3246 if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) {
3247 shrl(result, 1); // index
3248 }
3249 bind(CLEANUP);
3250 pop(rsp); // restore SP
3251
3252 } // string_indexof
3253
3254 void C2_MacroAssembler::string_indexof_char(Register str1, Register cnt1, Register ch, Register result,
3255 XMMRegister vec1, XMMRegister vec2, XMMRegister vec3, Register tmp) {
3256 ShortBranchVerifier sbv(this);
3257 assert(UseSSE42Intrinsics, "SSE4.2 intrinsics are required");
3258
3259 int stride = 8;
3260
3261 Label FOUND_CHAR, SCAN_TO_CHAR, SCAN_TO_CHAR_LOOP,
3262 SCAN_TO_8_CHAR, SCAN_TO_8_CHAR_LOOP, SCAN_TO_16_CHAR_LOOP,
3263 RET_NOT_FOUND, SCAN_TO_8_CHAR_INIT,
3264 FOUND_SEQ_CHAR, DONE_LABEL;
3265
3266 movptr(result, str1);
3267 if (UseAVX >= 2) {
3268 cmpl(cnt1, stride);
3269 jcc(Assembler::less, SCAN_TO_CHAR);
3270 cmpl(cnt1, 2*stride);
3271 jcc(Assembler::less, SCAN_TO_8_CHAR_INIT);
3272 movdl(vec1, ch);
3273 vpbroadcastw(vec1, vec1, Assembler::AVX_256bit);
3274 vpxor(vec2, vec2);
3275 movl(tmp, cnt1);
3276 andl(tmp, 0xFFFFFFF0); //vector count (in chars)
3277 andl(cnt1,0x0000000F); //tail count (in chars)
3278
3279 bind(SCAN_TO_16_CHAR_LOOP);
3280 vmovdqu(vec3, Address(result, 0));
3281 vpcmpeqw(vec3, vec3, vec1, 1);
3282 vptest(vec2, vec3);
3283 jcc(Assembler::carryClear, FOUND_CHAR);
3284 addptr(result, 32);
3285 subl(tmp, 2*stride);
3286 jcc(Assembler::notZero, SCAN_TO_16_CHAR_LOOP);
3287 jmp(SCAN_TO_8_CHAR);
3288 bind(SCAN_TO_8_CHAR_INIT);
3289 movdl(vec1, ch);
3290 pshuflw(vec1, vec1, 0x00);
3291 pshufd(vec1, vec1, 0);
3292 pxor(vec2, vec2);
3293 }
3294 bind(SCAN_TO_8_CHAR);
3295 cmpl(cnt1, stride);
3296 jcc(Assembler::less, SCAN_TO_CHAR);
3297 if (UseAVX < 2) {
3298 movdl(vec1, ch);
3299 pshuflw(vec1, vec1, 0x00);
3300 pshufd(vec1, vec1, 0);
3301 pxor(vec2, vec2);
3302 }
3303 movl(tmp, cnt1);
3304 andl(tmp, 0xFFFFFFF8); //vector count (in chars)
3305 andl(cnt1,0x00000007); //tail count (in chars)
3306
3307 bind(SCAN_TO_8_CHAR_LOOP);
3308 movdqu(vec3, Address(result, 0));
3309 pcmpeqw(vec3, vec1);
3310 ptest(vec2, vec3);
3311 jcc(Assembler::carryClear, FOUND_CHAR);
3312 addptr(result, 16);
3313 subl(tmp, stride);
3314 jcc(Assembler::notZero, SCAN_TO_8_CHAR_LOOP);
3315 bind(SCAN_TO_CHAR);
3316 testl(cnt1, cnt1);
3317 jcc(Assembler::zero, RET_NOT_FOUND);
3318 bind(SCAN_TO_CHAR_LOOP);
3319 load_unsigned_short(tmp, Address(result, 0));
3320 cmpl(ch, tmp);
3321 jccb(Assembler::equal, FOUND_SEQ_CHAR);
3322 addptr(result, 2);
3323 subl(cnt1, 1);
3324 jccb(Assembler::zero, RET_NOT_FOUND);
3325 jmp(SCAN_TO_CHAR_LOOP);
3326
3327 bind(RET_NOT_FOUND);
3328 movl(result, -1);
3329 jmpb(DONE_LABEL);
3330
3331 bind(FOUND_CHAR);
3332 if (UseAVX >= 2) {
3333 vpmovmskb(tmp, vec3);
3334 } else {
3335 pmovmskb(tmp, vec3);
3336 }
3337 bsfl(ch, tmp);
3338 addptr(result, ch);
3339
3340 bind(FOUND_SEQ_CHAR);
3341 subptr(result, str1);
3342 shrl(result, 1);
3343
3344 bind(DONE_LABEL);
3345 } // string_indexof_char
3346
3347 void C2_MacroAssembler::stringL_indexof_char(Register str1, Register cnt1, Register ch, Register result,
3348 XMMRegister vec1, XMMRegister vec2, XMMRegister vec3, Register tmp) {
3349 ShortBranchVerifier sbv(this);
3350 assert(UseSSE42Intrinsics, "SSE4.2 intrinsics are required");
3351
3352 int stride = 16;
3353
3354 Label FOUND_CHAR, SCAN_TO_CHAR_INIT, SCAN_TO_CHAR_LOOP,
3355 SCAN_TO_16_CHAR, SCAN_TO_16_CHAR_LOOP, SCAN_TO_32_CHAR_LOOP,
3356 RET_NOT_FOUND, SCAN_TO_16_CHAR_INIT,
3357 FOUND_SEQ_CHAR, DONE_LABEL;
3358
3359 movptr(result, str1);
3360 if (UseAVX >= 2) {
3361 cmpl(cnt1, stride);
3362 jcc(Assembler::less, SCAN_TO_CHAR_INIT);
3363 cmpl(cnt1, stride*2);
3364 jcc(Assembler::less, SCAN_TO_16_CHAR_INIT);
3365 movdl(vec1, ch);
3366 vpbroadcastb(vec1, vec1, Assembler::AVX_256bit);
3367 vpxor(vec2, vec2);
3368 movl(tmp, cnt1);
3369 andl(tmp, 0xFFFFFFE0); //vector count (in chars)
3370 andl(cnt1,0x0000001F); //tail count (in chars)
3371
3372 bind(SCAN_TO_32_CHAR_LOOP);
3373 vmovdqu(vec3, Address(result, 0));
3374 vpcmpeqb(vec3, vec3, vec1, Assembler::AVX_256bit);
3375 vptest(vec2, vec3);
3376 jcc(Assembler::carryClear, FOUND_CHAR);
3377 addptr(result, 32);
3378 subl(tmp, stride*2);
3379 jcc(Assembler::notZero, SCAN_TO_32_CHAR_LOOP);
3380 jmp(SCAN_TO_16_CHAR);
3381
3382 bind(SCAN_TO_16_CHAR_INIT);
3383 movdl(vec1, ch);
3384 pxor(vec2, vec2);
3385 pshufb(vec1, vec2);
3386 }
3387
3388 bind(SCAN_TO_16_CHAR);
3389 cmpl(cnt1, stride);
3390 jcc(Assembler::less, SCAN_TO_CHAR_INIT);//less than 16 entries left
3391 if (UseAVX < 2) {
3392 movdl(vec1, ch);
3393 pxor(vec2, vec2);
3394 pshufb(vec1, vec2);
3395 }
3396 movl(tmp, cnt1);
3397 andl(tmp, 0xFFFFFFF0); //vector count (in bytes)
3398 andl(cnt1,0x0000000F); //tail count (in bytes)
3399
3400 bind(SCAN_TO_16_CHAR_LOOP);
3401 movdqu(vec3, Address(result, 0));
3402 pcmpeqb(vec3, vec1);
3403 ptest(vec2, vec3);
3404 jcc(Assembler::carryClear, FOUND_CHAR);
3405 addptr(result, 16);
3406 subl(tmp, stride);
3407 jcc(Assembler::notZero, SCAN_TO_16_CHAR_LOOP);//last 16 items...
3408
3409 bind(SCAN_TO_CHAR_INIT);
3410 testl(cnt1, cnt1);
3411 jcc(Assembler::zero, RET_NOT_FOUND);
3412 bind(SCAN_TO_CHAR_LOOP);
3413 load_unsigned_byte(tmp, Address(result, 0));
3414 cmpl(ch, tmp);
3415 jccb(Assembler::equal, FOUND_SEQ_CHAR);
3416 addptr(result, 1);
3417 subl(cnt1, 1);
3418 jccb(Assembler::zero, RET_NOT_FOUND);
3419 jmp(SCAN_TO_CHAR_LOOP);
3420
3421 bind(RET_NOT_FOUND);
3422 movl(result, -1);
3423 jmpb(DONE_LABEL);
3424
3425 bind(FOUND_CHAR);
3426 if (UseAVX >= 2) {
3427 vpmovmskb(tmp, vec3);
3428 } else {
3429 pmovmskb(tmp, vec3);
3430 }
3431 bsfl(ch, tmp);
3432 addptr(result, ch);
3433
3434 bind(FOUND_SEQ_CHAR);
3435 subptr(result, str1);
3436
3437 bind(DONE_LABEL);
3438 } // stringL_indexof_char
3439
3440 int C2_MacroAssembler::arrays_hashcode_elsize(BasicType eltype) {
3441 switch (eltype) {
3442 case T_BOOLEAN: return sizeof(jboolean);
3443 case T_BYTE: return sizeof(jbyte);
3444 case T_SHORT: return sizeof(jshort);
3445 case T_CHAR: return sizeof(jchar);
3446 case T_INT: return sizeof(jint);
3447 default:
3448 ShouldNotReachHere();
3449 return -1;
3450 }
3451 }
3452
3453 void C2_MacroAssembler::arrays_hashcode_elload(Register dst, Address src, BasicType eltype) {
3454 switch (eltype) {
3455 // T_BOOLEAN used as surrogate for unsigned byte
3456 case T_BOOLEAN: movzbl(dst, src); break;
3457 case T_BYTE: movsbl(dst, src); break;
3458 case T_SHORT: movswl(dst, src); break;
3459 case T_CHAR: movzwl(dst, src); break;
3460 case T_INT: movl(dst, src); break;
3461 default:
3462 ShouldNotReachHere();
3463 }
3464 }
3465
3466 void C2_MacroAssembler::arrays_hashcode_elvload(XMMRegister dst, Address src, BasicType eltype) {
3467 load_vector(eltype, dst, src, arrays_hashcode_elsize(eltype) * 8);
3468 }
3469
3470 void C2_MacroAssembler::arrays_hashcode_elvload(XMMRegister dst, AddressLiteral src, BasicType eltype) {
3471 load_vector(eltype, dst, src, arrays_hashcode_elsize(eltype) * 8);
3472 }
3473
3474 void C2_MacroAssembler::arrays_hashcode_elvcast(XMMRegister dst, BasicType eltype) {
3475 const int vlen = Assembler::AVX_256bit;
3476 switch (eltype) {
3477 case T_BOOLEAN: vector_unsigned_cast(dst, dst, vlen, T_BYTE, T_INT); break;
3478 case T_BYTE: vector_signed_cast(dst, dst, vlen, T_BYTE, T_INT); break;
3479 case T_SHORT: vector_signed_cast(dst, dst, vlen, T_SHORT, T_INT); break;
3480 case T_CHAR: vector_unsigned_cast(dst, dst, vlen, T_SHORT, T_INT); break;
3481 case T_INT:
3482 // do nothing
3483 break;
3484 default:
3485 ShouldNotReachHere();
3486 }
3487 }
3488
3489 void C2_MacroAssembler::arrays_hashcode(Register ary1, Register cnt1, Register result,
3490 Register index, Register tmp2, Register tmp3, XMMRegister vnext,
3491 XMMRegister vcoef0, XMMRegister vcoef1, XMMRegister vcoef2, XMMRegister vcoef3,
3492 XMMRegister vresult0, XMMRegister vresult1, XMMRegister vresult2, XMMRegister vresult3,
3493 XMMRegister vtmp0, XMMRegister vtmp1, XMMRegister vtmp2, XMMRegister vtmp3,
3494 BasicType eltype) {
3495 ShortBranchVerifier sbv(this);
3496 assert(UseAVX >= 2, "AVX2 intrinsics are required");
3497 assert_different_registers(ary1, cnt1, result, index, tmp2, tmp3);
3498 assert_different_registers(vnext, vcoef0, vcoef1, vcoef2, vcoef3, vresult0, vresult1, vresult2, vresult3, vtmp0, vtmp1, vtmp2, vtmp3);
3499
3500 Label SHORT_UNROLLED_BEGIN, SHORT_UNROLLED_LOOP_BEGIN,
3501 SHORT_UNROLLED_LOOP_EXIT,
3502 UNROLLED_SCALAR_LOOP_BEGIN, UNROLLED_SCALAR_SKIP, UNROLLED_SCALAR_RESUME,
3503 UNROLLED_VECTOR_LOOP_BEGIN,
3504 END;
3505 switch (eltype) {
3506 case T_BOOLEAN: BLOCK_COMMENT("arrays_hashcode(unsigned byte) {"); break;
3507 case T_CHAR: BLOCK_COMMENT("arrays_hashcode(char) {"); break;
3508 case T_BYTE: BLOCK_COMMENT("arrays_hashcode(byte) {"); break;
3509 case T_SHORT: BLOCK_COMMENT("arrays_hashcode(short) {"); break;
3510 case T_INT: BLOCK_COMMENT("arrays_hashcode(int) {"); break;
3511 default: BLOCK_COMMENT("arrays_hashcode {"); break;
3512 }
3513
3514 // For "renaming" for readibility of the code
3515 const XMMRegister vcoef[] = { vcoef0, vcoef1, vcoef2, vcoef3 },
3516 vresult[] = { vresult0, vresult1, vresult2, vresult3 },
3517 vtmp[] = { vtmp0, vtmp1, vtmp2, vtmp3 };
3518
3519 const int elsize = arrays_hashcode_elsize(eltype);
3520
3521 /*
3522 if (cnt1 >= 2) {
3523 if (cnt1 >= 32) {
3524 UNROLLED VECTOR LOOP
3525 }
3526 UNROLLED SCALAR LOOP
3527 }
3528 SINGLE SCALAR
3529 */
3530
3531 cmpl(cnt1, 32);
3532 jcc(Assembler::less, SHORT_UNROLLED_BEGIN);
3533
3534 // cnt1 >= 32 && generate_vectorized_loop
3535 xorl(index, index);
3536
3537 // vresult = IntVector.zero(I256);
3538 for (int idx = 0; idx < 4; idx++) {
3539 vpxor(vresult[idx], vresult[idx]);
3540 }
3541 // vnext = IntVector.broadcast(I256, power_of_31_backwards[0]);
3542 Register bound = tmp2;
3543 Register next = tmp3;
3544 lea(tmp2, ExternalAddress(StubRoutines::x86::arrays_hashcode_powers_of_31() + (0 * sizeof(jint))));
3545 movl(next, Address(tmp2, 0));
3546 movdl(vnext, next);
3547 vpbroadcastd(vnext, vnext, Assembler::AVX_256bit);
3548
3549 // index = 0;
3550 // bound = cnt1 & ~(32 - 1);
3551 movl(bound, cnt1);
3552 andl(bound, ~(32 - 1));
3553 // for (; index < bound; index += 32) {
3554 bind(UNROLLED_VECTOR_LOOP_BEGIN);
3555 // result *= next;
3556 imull(result, next);
3557 // loop fission to upfront the cost of fetching from memory, OOO execution
3558 // can then hopefully do a better job of prefetching
3559 for (int idx = 0; idx < 4; idx++) {
3560 arrays_hashcode_elvload(vtmp[idx], Address(ary1, index, Address::times(elsize), 8 * idx * elsize), eltype);
3561 }
3562 // vresult = vresult * vnext + ary1[index+8*idx:index+8*idx+7];
3563 for (int idx = 0; idx < 4; idx++) {
3564 vpmulld(vresult[idx], vresult[idx], vnext, Assembler::AVX_256bit);
3565 arrays_hashcode_elvcast(vtmp[idx], eltype);
3566 vpaddd(vresult[idx], vresult[idx], vtmp[idx], Assembler::AVX_256bit);
3567 }
3568 // index += 32;
3569 addl(index, 32);
3570 // index < bound;
3571 cmpl(index, bound);
3572 jcc(Assembler::less, UNROLLED_VECTOR_LOOP_BEGIN);
3573 // }
3574
3575 lea(ary1, Address(ary1, bound, Address::times(elsize)));
3576 subl(cnt1, bound);
3577 // release bound
3578
3579 // vresult *= IntVector.fromArray(I256, power_of_31_backwards, 1);
3580 for (int idx = 0; idx < 4; idx++) {
3581 lea(tmp2, ExternalAddress(StubRoutines::x86::arrays_hashcode_powers_of_31() + ((8 * idx + 1) * sizeof(jint))));
3582 arrays_hashcode_elvload(vcoef[idx], Address(tmp2, 0), T_INT);
3583 vpmulld(vresult[idx], vresult[idx], vcoef[idx], Assembler::AVX_256bit);
3584 }
3585 // result += vresult.reduceLanes(ADD);
3586 for (int idx = 0; idx < 4; idx++) {
3587 reduceI(Op_AddReductionVI, 256/(sizeof(jint) * 8), result, result, vresult[idx], vtmp[(idx * 2 + 0) % 4], vtmp[(idx * 2 + 1) % 4]);
3588 }
3589
3590 // } else if (cnt1 < 32) {
3591
3592 bind(SHORT_UNROLLED_BEGIN);
3593 // int i = 1;
3594 movl(index, 1);
3595 cmpl(index, cnt1);
3596 jcc(Assembler::greaterEqual, SHORT_UNROLLED_LOOP_EXIT);
3597
3598 // for (; i < cnt1 ; i += 2) {
3599 bind(SHORT_UNROLLED_LOOP_BEGIN);
3600 movl(tmp3, 961);
3601 imull(result, tmp3);
3602 arrays_hashcode_elload(tmp2, Address(ary1, index, Address::times(elsize), -elsize), eltype);
3603 movl(tmp3, tmp2);
3604 shll(tmp3, 5);
3605 subl(tmp3, tmp2);
3606 addl(result, tmp3);
3607 arrays_hashcode_elload(tmp3, Address(ary1, index, Address::times(elsize)), eltype);
3608 addl(result, tmp3);
3609 addl(index, 2);
3610 cmpl(index, cnt1);
3611 jccb(Assembler::less, SHORT_UNROLLED_LOOP_BEGIN);
3612
3613 // }
3614 // if (i >= cnt1) {
3615 bind(SHORT_UNROLLED_LOOP_EXIT);
3616 jccb(Assembler::greater, END);
3617 movl(tmp2, result);
3618 shll(result, 5);
3619 subl(result, tmp2);
3620 arrays_hashcode_elload(tmp3, Address(ary1, index, Address::times(elsize), -elsize), eltype);
3621 addl(result, tmp3);
3622 // }
3623 bind(END);
3624
3625 BLOCK_COMMENT("} // arrays_hashcode");
3626
3627 } // arrays_hashcode
3628
3629 // helper function for string_compare
3630 void C2_MacroAssembler::load_next_elements(Register elem1, Register elem2, Register str1, Register str2,
3631 Address::ScaleFactor scale, Address::ScaleFactor scale1,
3632 Address::ScaleFactor scale2, Register index, int ae) {
3633 if (ae == StrIntrinsicNode::LL) {
3634 load_unsigned_byte(elem1, Address(str1, index, scale, 0));
3635 load_unsigned_byte(elem2, Address(str2, index, scale, 0));
3636 } else if (ae == StrIntrinsicNode::UU) {
3637 load_unsigned_short(elem1, Address(str1, index, scale, 0));
3638 load_unsigned_short(elem2, Address(str2, index, scale, 0));
3639 } else {
3640 load_unsigned_byte(elem1, Address(str1, index, scale1, 0));
3641 load_unsigned_short(elem2, Address(str2, index, scale2, 0));
3642 }
3643 }
3644
3645 // Compare strings, used for char[] and byte[].
3646 void C2_MacroAssembler::string_compare(Register str1, Register str2,
3647 Register cnt1, Register cnt2, Register result,
3648 XMMRegister vec1, int ae, KRegister mask) {
3649 ShortBranchVerifier sbv(this);
3650 Label LENGTH_DIFF_LABEL, POP_LABEL, DONE_LABEL, WHILE_HEAD_LABEL;
3651 Label COMPARE_WIDE_VECTORS_LOOP_FAILED; // used only AVX3
3652 int stride, stride2, adr_stride, adr_stride1, adr_stride2;
3653 int stride2x2 = 0x40;
3654 Address::ScaleFactor scale = Address::no_scale;
3655 Address::ScaleFactor scale1 = Address::no_scale;
3656 Address::ScaleFactor scale2 = Address::no_scale;
3657
3658 if (ae != StrIntrinsicNode::LL) {
3659 stride2x2 = 0x20;
3660 }
3661
3662 if (ae == StrIntrinsicNode::LU || ae == StrIntrinsicNode::UL) {
3663 shrl(cnt2, 1);
3664 }
3665 // Compute the minimum of the string lengths and the
3666 // difference of the string lengths (stack).
3667 // Do the conditional move stuff
3668 movl(result, cnt1);
3669 subl(cnt1, cnt2);
3670 push(cnt1);
3671 cmov32(Assembler::lessEqual, cnt2, result); // cnt2 = min(cnt1, cnt2)
3672
3673 // Is the minimum length zero?
3674 testl(cnt2, cnt2);
3675 jcc(Assembler::zero, LENGTH_DIFF_LABEL);
3676 if (ae == StrIntrinsicNode::LL) {
3677 // Load first bytes
3678 load_unsigned_byte(result, Address(str1, 0)); // result = str1[0]
3679 load_unsigned_byte(cnt1, Address(str2, 0)); // cnt1 = str2[0]
3680 } else if (ae == StrIntrinsicNode::UU) {
3681 // Load first characters
3682 load_unsigned_short(result, Address(str1, 0));
3683 load_unsigned_short(cnt1, Address(str2, 0));
3684 } else {
3685 load_unsigned_byte(result, Address(str1, 0));
3686 load_unsigned_short(cnt1, Address(str2, 0));
3687 }
3688 subl(result, cnt1);
3689 jcc(Assembler::notZero, POP_LABEL);
3690
3691 if (ae == StrIntrinsicNode::UU) {
3692 // Divide length by 2 to get number of chars
3693 shrl(cnt2, 1);
3694 }
3695 cmpl(cnt2, 1);
3696 jcc(Assembler::equal, LENGTH_DIFF_LABEL);
3697
3698 // Check if the strings start at the same location and setup scale and stride
3699 if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
3700 cmpptr(str1, str2);
3701 jcc(Assembler::equal, LENGTH_DIFF_LABEL);
3702 if (ae == StrIntrinsicNode::LL) {
3703 scale = Address::times_1;
3704 stride = 16;
3705 } else {
3706 scale = Address::times_2;
3707 stride = 8;
3708 }
3709 } else {
3710 scale1 = Address::times_1;
3711 scale2 = Address::times_2;
3712 // scale not used
3713 stride = 8;
3714 }
3715
3716 if (UseAVX >= 2 && UseSSE42Intrinsics) {
3717 Label COMPARE_WIDE_VECTORS, VECTOR_NOT_EQUAL, COMPARE_WIDE_TAIL, COMPARE_SMALL_STR;
3718 Label COMPARE_WIDE_VECTORS_LOOP, COMPARE_16_CHARS, COMPARE_INDEX_CHAR;
3719 Label COMPARE_WIDE_VECTORS_LOOP_AVX2;
3720 Label COMPARE_TAIL_LONG;
3721 Label COMPARE_WIDE_VECTORS_LOOP_AVX3; // used only AVX3
3722
3723 int pcmpmask = 0x19;
3724 if (ae == StrIntrinsicNode::LL) {
3725 pcmpmask &= ~0x01;
3726 }
3727
3728 // Setup to compare 16-chars (32-bytes) vectors,
3729 // start from first character again because it has aligned address.
3730 if (ae == StrIntrinsicNode::LL) {
3731 stride2 = 32;
3732 } else {
3733 stride2 = 16;
3734 }
3735 if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
3736 adr_stride = stride << scale;
3737 } else {
3738 adr_stride1 = 8; //stride << scale1;
3739 adr_stride2 = 16; //stride << scale2;
3740 }
3741
3742 assert(result == rax && cnt2 == rdx && cnt1 == rcx, "pcmpestri");
3743 // rax and rdx are used by pcmpestri as elements counters
3744 movl(result, cnt2);
3745 andl(cnt2, ~(stride2-1)); // cnt2 holds the vector count
3746 jcc(Assembler::zero, COMPARE_TAIL_LONG);
3747
3748 // fast path : compare first 2 8-char vectors.
3749 bind(COMPARE_16_CHARS);
3750 if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
3751 movdqu(vec1, Address(str1, 0));
3752 } else {
3753 pmovzxbw(vec1, Address(str1, 0));
3754 }
3755 pcmpestri(vec1, Address(str2, 0), pcmpmask);
3756 jccb(Assembler::below, COMPARE_INDEX_CHAR);
3757
3758 if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
3759 movdqu(vec1, Address(str1, adr_stride));
3760 pcmpestri(vec1, Address(str2, adr_stride), pcmpmask);
3761 } else {
3762 pmovzxbw(vec1, Address(str1, adr_stride1));
3763 pcmpestri(vec1, Address(str2, adr_stride2), pcmpmask);
3764 }
3765 jccb(Assembler::aboveEqual, COMPARE_WIDE_VECTORS);
3766 addl(cnt1, stride);
3767
3768 // Compare the characters at index in cnt1
3769 bind(COMPARE_INDEX_CHAR); // cnt1 has the offset of the mismatching character
3770 load_next_elements(result, cnt2, str1, str2, scale, scale1, scale2, cnt1, ae);
3771 subl(result, cnt2);
3772 jmp(POP_LABEL);
3773
3774 // Setup the registers to start vector comparison loop
3775 bind(COMPARE_WIDE_VECTORS);
3776 if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
3777 lea(str1, Address(str1, result, scale));
3778 lea(str2, Address(str2, result, scale));
3779 } else {
3780 lea(str1, Address(str1, result, scale1));
3781 lea(str2, Address(str2, result, scale2));
3782 }
3783 subl(result, stride2);
3784 subl(cnt2, stride2);
3785 jcc(Assembler::zero, COMPARE_WIDE_TAIL);
3786 negptr(result);
3787
3788 // In a loop, compare 16-chars (32-bytes) at once using (vpxor+vptest)
3789 bind(COMPARE_WIDE_VECTORS_LOOP);
3790
3791 if ((AVX3Threshold == 0) && VM_Version::supports_avx512vlbw()) { // trying 64 bytes fast loop
3792 cmpl(cnt2, stride2x2);
3793 jccb(Assembler::below, COMPARE_WIDE_VECTORS_LOOP_AVX2);
3794 testl(cnt2, stride2x2-1); // cnt2 holds the vector count
3795 jccb(Assembler::notZero, COMPARE_WIDE_VECTORS_LOOP_AVX2); // means we cannot subtract by 0x40
3796
3797 bind(COMPARE_WIDE_VECTORS_LOOP_AVX3); // the hottest loop
3798 if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
3799 evmovdquq(vec1, Address(str1, result, scale), Assembler::AVX_512bit);
3800 evpcmpeqb(mask, vec1, Address(str2, result, scale), Assembler::AVX_512bit); // k7 == 11..11, if operands equal, otherwise k7 has some 0
3801 } else {
3802 vpmovzxbw(vec1, Address(str1, result, scale1), Assembler::AVX_512bit);
3803 evpcmpeqb(mask, vec1, Address(str2, result, scale2), Assembler::AVX_512bit); // k7 == 11..11, if operands equal, otherwise k7 has some 0
3804 }
3805 kortestql(mask, mask);
3806 jcc(Assembler::aboveEqual, COMPARE_WIDE_VECTORS_LOOP_FAILED); // miscompare
3807 addptr(result, stride2x2); // update since we already compared at this addr
3808 subl(cnt2, stride2x2); // and sub the size too
3809 jccb(Assembler::notZero, COMPARE_WIDE_VECTORS_LOOP_AVX3);
3810
3811 vpxor(vec1, vec1);
3812 jmpb(COMPARE_WIDE_TAIL);
3813 }//if (VM_Version::supports_avx512vlbw())
3814
3815 bind(COMPARE_WIDE_VECTORS_LOOP_AVX2);
3816 if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
3817 vmovdqu(vec1, Address(str1, result, scale));
3818 vpxor(vec1, Address(str2, result, scale));
3819 } else {
3820 vpmovzxbw(vec1, Address(str1, result, scale1), Assembler::AVX_256bit);
3821 vpxor(vec1, Address(str2, result, scale2));
3822 }
3823 vptest(vec1, vec1);
3824 jcc(Assembler::notZero, VECTOR_NOT_EQUAL);
3825 addptr(result, stride2);
3826 subl(cnt2, stride2);
3827 jcc(Assembler::notZero, COMPARE_WIDE_VECTORS_LOOP);
3828 // clean upper bits of YMM registers
3829 vpxor(vec1, vec1);
3830
3831 // compare wide vectors tail
3832 bind(COMPARE_WIDE_TAIL);
3833 testptr(result, result);
3834 jcc(Assembler::zero, LENGTH_DIFF_LABEL);
3835
3836 movl(result, stride2);
3837 movl(cnt2, result);
3838 negptr(result);
3839 jmp(COMPARE_WIDE_VECTORS_LOOP_AVX2);
3840
3841 // Identifies the mismatching (higher or lower)16-bytes in the 32-byte vectors.
3842 bind(VECTOR_NOT_EQUAL);
3843 // clean upper bits of YMM registers
3844 vpxor(vec1, vec1);
3845 if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
3846 lea(str1, Address(str1, result, scale));
3847 lea(str2, Address(str2, result, scale));
3848 } else {
3849 lea(str1, Address(str1, result, scale1));
3850 lea(str2, Address(str2, result, scale2));
3851 }
3852 jmp(COMPARE_16_CHARS);
3853
3854 // Compare tail chars, length between 1 to 15 chars
3855 bind(COMPARE_TAIL_LONG);
3856 movl(cnt2, result);
3857 cmpl(cnt2, stride);
3858 jcc(Assembler::less, COMPARE_SMALL_STR);
3859
3860 if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
3861 movdqu(vec1, Address(str1, 0));
3862 } else {
3863 pmovzxbw(vec1, Address(str1, 0));
3864 }
3865 pcmpestri(vec1, Address(str2, 0), pcmpmask);
3866 jcc(Assembler::below, COMPARE_INDEX_CHAR);
3867 subptr(cnt2, stride);
3868 jcc(Assembler::zero, LENGTH_DIFF_LABEL);
3869 if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
3870 lea(str1, Address(str1, result, scale));
3871 lea(str2, Address(str2, result, scale));
3872 } else {
3873 lea(str1, Address(str1, result, scale1));
3874 lea(str2, Address(str2, result, scale2));
3875 }
3876 negptr(cnt2);
3877 jmpb(WHILE_HEAD_LABEL);
3878
3879 bind(COMPARE_SMALL_STR);
3880 } else if (UseSSE42Intrinsics) {
3881 Label COMPARE_WIDE_VECTORS, VECTOR_NOT_EQUAL, COMPARE_TAIL;
3882 int pcmpmask = 0x19;
3883 // Setup to compare 8-char (16-byte) vectors,
3884 // start from first character again because it has aligned address.
3885 movl(result, cnt2);
3886 andl(cnt2, ~(stride - 1)); // cnt2 holds the vector count
3887 if (ae == StrIntrinsicNode::LL) {
3888 pcmpmask &= ~0x01;
3889 }
3890 jcc(Assembler::zero, COMPARE_TAIL);
3891 if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
3892 lea(str1, Address(str1, result, scale));
3893 lea(str2, Address(str2, result, scale));
3894 } else {
3895 lea(str1, Address(str1, result, scale1));
3896 lea(str2, Address(str2, result, scale2));
3897 }
3898 negptr(result);
3899
3900 // pcmpestri
3901 // inputs:
3902 // vec1- substring
3903 // rax - negative string length (elements count)
3904 // mem - scanned string
3905 // rdx - string length (elements count)
3906 // pcmpmask - cmp mode: 11000 (string compare with negated result)
3907 // + 00 (unsigned bytes) or + 01 (unsigned shorts)
3908 // outputs:
3909 // rcx - first mismatched element index
3910 assert(result == rax && cnt2 == rdx && cnt1 == rcx, "pcmpestri");
3911
3912 bind(COMPARE_WIDE_VECTORS);
3913 if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
3914 movdqu(vec1, Address(str1, result, scale));
3915 pcmpestri(vec1, Address(str2, result, scale), pcmpmask);
3916 } else {
3917 pmovzxbw(vec1, Address(str1, result, scale1));
3918 pcmpestri(vec1, Address(str2, result, scale2), pcmpmask);
3919 }
3920 // After pcmpestri cnt1(rcx) contains mismatched element index
3921
3922 jccb(Assembler::below, VECTOR_NOT_EQUAL); // CF==1
3923 addptr(result, stride);
3924 subptr(cnt2, stride);
3925 jccb(Assembler::notZero, COMPARE_WIDE_VECTORS);
3926
3927 // compare wide vectors tail
3928 testptr(result, result);
3929 jcc(Assembler::zero, LENGTH_DIFF_LABEL);
3930
3931 movl(cnt2, stride);
3932 movl(result, stride);
3933 negptr(result);
3934 if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
3935 movdqu(vec1, Address(str1, result, scale));
3936 pcmpestri(vec1, Address(str2, result, scale), pcmpmask);
3937 } else {
3938 pmovzxbw(vec1, Address(str1, result, scale1));
3939 pcmpestri(vec1, Address(str2, result, scale2), pcmpmask);
3940 }
3941 jccb(Assembler::aboveEqual, LENGTH_DIFF_LABEL);
3942
3943 // Mismatched characters in the vectors
3944 bind(VECTOR_NOT_EQUAL);
3945 addptr(cnt1, result);
3946 load_next_elements(result, cnt2, str1, str2, scale, scale1, scale2, cnt1, ae);
3947 subl(result, cnt2);
3948 jmpb(POP_LABEL);
3949
3950 bind(COMPARE_TAIL); // limit is zero
3951 movl(cnt2, result);
3952 // Fallthru to tail compare
3953 }
3954 // Shift str2 and str1 to the end of the arrays, negate min
3955 if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
3956 lea(str1, Address(str1, cnt2, scale));
3957 lea(str2, Address(str2, cnt2, scale));
3958 } else {
3959 lea(str1, Address(str1, cnt2, scale1));
3960 lea(str2, Address(str2, cnt2, scale2));
3961 }
3962 decrementl(cnt2); // first character was compared already
3963 negptr(cnt2);
3964
3965 // Compare the rest of the elements
3966 bind(WHILE_HEAD_LABEL);
3967 load_next_elements(result, cnt1, str1, str2, scale, scale1, scale2, cnt2, ae);
3968 subl(result, cnt1);
3969 jccb(Assembler::notZero, POP_LABEL);
3970 increment(cnt2);
3971 jccb(Assembler::notZero, WHILE_HEAD_LABEL);
3972
3973 // Strings are equal up to min length. Return the length difference.
3974 bind(LENGTH_DIFF_LABEL);
3975 pop(result);
3976 if (ae == StrIntrinsicNode::UU) {
3977 // Divide diff by 2 to get number of chars
3978 sarl(result, 1);
3979 }
3980 jmpb(DONE_LABEL);
3981
3982 if (VM_Version::supports_avx512vlbw()) {
3983
3984 bind(COMPARE_WIDE_VECTORS_LOOP_FAILED);
3985
3986 kmovql(cnt1, mask);
3987 notq(cnt1);
3988 bsfq(cnt2, cnt1);
3989 if (ae != StrIntrinsicNode::LL) {
3990 // Divide diff by 2 to get number of chars
3991 sarl(cnt2, 1);
3992 }
3993 addq(result, cnt2);
3994 if (ae == StrIntrinsicNode::LL) {
3995 load_unsigned_byte(cnt1, Address(str2, result));
3996 load_unsigned_byte(result, Address(str1, result));
3997 } else if (ae == StrIntrinsicNode::UU) {
3998 load_unsigned_short(cnt1, Address(str2, result, scale));
3999 load_unsigned_short(result, Address(str1, result, scale));
4000 } else {
4001 load_unsigned_short(cnt1, Address(str2, result, scale2));
4002 load_unsigned_byte(result, Address(str1, result, scale1));
4003 }
4004 subl(result, cnt1);
4005 jmpb(POP_LABEL);
4006 }//if (VM_Version::supports_avx512vlbw())
4007
4008 // Discard the stored length difference
4009 bind(POP_LABEL);
4010 pop(cnt1);
4011
4012 // That's it
4013 bind(DONE_LABEL);
4014 if(ae == StrIntrinsicNode::UL) {
4015 negl(result);
4016 }
4017
4018 }
4019
4020 // Search for Non-ASCII character (Negative byte value) in a byte array,
4021 // return the index of the first such character, otherwise the length
4022 // of the array segment searched.
4023 // ..\jdk\src\java.base\share\classes\java\lang\StringCoding.java
4024 // @IntrinsicCandidate
4025 // public static int countPositives(byte[] ba, int off, int len) {
4026 // for (int i = off; i < off + len; i++) {
4027 // if (ba[i] < 0) {
4028 // return i - off;
4029 // }
4030 // }
4031 // return len;
4032 // }
4033 void C2_MacroAssembler::count_positives(Register ary1, Register len,
4034 Register result, Register tmp1,
4035 XMMRegister vec1, XMMRegister vec2, KRegister mask1, KRegister mask2) {
4036 // rsi: byte array
4037 // rcx: len
4038 // rax: result
4039 ShortBranchVerifier sbv(this);
4040 assert_different_registers(ary1, len, result, tmp1);
4041 assert_different_registers(vec1, vec2);
4042 Label ADJUST, TAIL_ADJUST, DONE, TAIL_START, CHAR_ADJUST, COMPARE_CHAR, COMPARE_VECTORS, COMPARE_BYTE;
4043
4044 movl(result, len); // copy
4045 // len == 0
4046 testl(len, len);
4047 jcc(Assembler::zero, DONE);
4048
4049 if ((AVX3Threshold == 0) && (UseAVX > 2) && // AVX512
4050 VM_Version::supports_avx512vlbw() &&
4051 VM_Version::supports_bmi2()) {
4052
4053 Label test_64_loop, test_tail, BREAK_LOOP;
4054 movl(tmp1, len);
4055 vpxor(vec2, vec2, vec2, Assembler::AVX_512bit);
4056
4057 andl(tmp1, 0x0000003f); // tail count (in chars) 0x3F
4058 andl(len, 0xffffffc0); // vector count (in chars)
4059 jccb(Assembler::zero, test_tail);
4060
4061 lea(ary1, Address(ary1, len, Address::times_1));
4062 negptr(len);
4063
4064 bind(test_64_loop);
4065 // Check whether our 64 elements of size byte contain negatives
4066 evpcmpgtb(mask1, vec2, Address(ary1, len, Address::times_1), Assembler::AVX_512bit);
4067 kortestql(mask1, mask1);
4068 jcc(Assembler::notZero, BREAK_LOOP);
4069
4070 addptr(len, 64);
4071 jccb(Assembler::notZero, test_64_loop);
4072
4073 bind(test_tail);
4074 // bail out when there is nothing to be done
4075 testl(tmp1, -1);
4076 jcc(Assembler::zero, DONE);
4077
4078
4079 // check the tail for absense of negatives
4080 // ~(~0 << len) applied up to two times (for 32-bit scenario)
4081 {
4082 Register tmp3_aliased = len;
4083 mov64(tmp3_aliased, 0xFFFFFFFFFFFFFFFF);
4084 shlxq(tmp3_aliased, tmp3_aliased, tmp1);
4085 notq(tmp3_aliased);
4086 kmovql(mask2, tmp3_aliased);
4087 }
4088
4089 evpcmpgtb(mask1, mask2, vec2, Address(ary1, 0), Assembler::AVX_512bit);
4090 ktestq(mask1, mask2);
4091 jcc(Assembler::zero, DONE);
4092
4093 // do a full check for negative registers in the tail
4094 movl(len, tmp1); // tmp1 holds low 6-bit from original len;
4095 // ary1 already pointing to the right place
4096 jmpb(TAIL_START);
4097
4098 bind(BREAK_LOOP);
4099 // At least one byte in the last 64 byte block was negative.
4100 // Set up to look at the last 64 bytes as if they were a tail
4101 lea(ary1, Address(ary1, len, Address::times_1));
4102 addptr(result, len);
4103 // Ignore the very last byte: if all others are positive,
4104 // it must be negative, so we can skip right to the 2+1 byte
4105 // end comparison at this point
4106 orl(result, 63);
4107 movl(len, 63);
4108 // Fallthru to tail compare
4109 } else {
4110
4111 if (UseAVX >= 2) {
4112 // With AVX2, use 32-byte vector compare
4113 Label COMPARE_WIDE_VECTORS, BREAK_LOOP;
4114
4115 // Compare 32-byte vectors
4116 testl(len, 0xffffffe0); // vector count (in bytes)
4117 jccb(Assembler::zero, TAIL_START);
4118
4119 andl(len, 0xffffffe0);
4120 lea(ary1, Address(ary1, len, Address::times_1));
4121 negptr(len);
4122
4123 movl(tmp1, 0x80808080); // create mask to test for Unicode chars in vector
4124 movdl(vec2, tmp1);
4125 vpbroadcastd(vec2, vec2, Assembler::AVX_256bit);
4126
4127 bind(COMPARE_WIDE_VECTORS);
4128 vmovdqu(vec1, Address(ary1, len, Address::times_1));
4129 vptest(vec1, vec2);
4130 jccb(Assembler::notZero, BREAK_LOOP);
4131 addptr(len, 32);
4132 jccb(Assembler::notZero, COMPARE_WIDE_VECTORS);
4133
4134 testl(result, 0x0000001f); // any bytes remaining?
4135 jcc(Assembler::zero, DONE);
4136
4137 // Quick test using the already prepared vector mask
4138 movl(len, result);
4139 andl(len, 0x0000001f);
4140 vmovdqu(vec1, Address(ary1, len, Address::times_1, -32));
4141 vptest(vec1, vec2);
4142 jcc(Assembler::zero, DONE);
4143 // There are zeros, jump to the tail to determine exactly where
4144 jmpb(TAIL_START);
4145
4146 bind(BREAK_LOOP);
4147 // At least one byte in the last 32-byte vector is negative.
4148 // Set up to look at the last 32 bytes as if they were a tail
4149 lea(ary1, Address(ary1, len, Address::times_1));
4150 addptr(result, len);
4151 // Ignore the very last byte: if all others are positive,
4152 // it must be negative, so we can skip right to the 2+1 byte
4153 // end comparison at this point
4154 orl(result, 31);
4155 movl(len, 31);
4156 // Fallthru to tail compare
4157 } else if (UseSSE42Intrinsics) {
4158 // With SSE4.2, use double quad vector compare
4159 Label COMPARE_WIDE_VECTORS, BREAK_LOOP;
4160
4161 // Compare 16-byte vectors
4162 testl(len, 0xfffffff0); // vector count (in bytes)
4163 jcc(Assembler::zero, TAIL_START);
4164
4165 andl(len, 0xfffffff0);
4166 lea(ary1, Address(ary1, len, Address::times_1));
4167 negptr(len);
4168
4169 movl(tmp1, 0x80808080);
4170 movdl(vec2, tmp1);
4171 pshufd(vec2, vec2, 0);
4172
4173 bind(COMPARE_WIDE_VECTORS);
4174 movdqu(vec1, Address(ary1, len, Address::times_1));
4175 ptest(vec1, vec2);
4176 jccb(Assembler::notZero, BREAK_LOOP);
4177 addptr(len, 16);
4178 jccb(Assembler::notZero, COMPARE_WIDE_VECTORS);
4179
4180 testl(result, 0x0000000f); // len is zero, any bytes remaining?
4181 jcc(Assembler::zero, DONE);
4182
4183 // Quick test using the already prepared vector mask
4184 movl(len, result);
4185 andl(len, 0x0000000f); // tail count (in bytes)
4186 movdqu(vec1, Address(ary1, len, Address::times_1, -16));
4187 ptest(vec1, vec2);
4188 jcc(Assembler::zero, DONE);
4189 jmpb(TAIL_START);
4190
4191 bind(BREAK_LOOP);
4192 // At least one byte in the last 16-byte vector is negative.
4193 // Set up and look at the last 16 bytes as if they were a tail
4194 lea(ary1, Address(ary1, len, Address::times_1));
4195 addptr(result, len);
4196 // Ignore the very last byte: if all others are positive,
4197 // it must be negative, so we can skip right to the 2+1 byte
4198 // end comparison at this point
4199 orl(result, 15);
4200 movl(len, 15);
4201 // Fallthru to tail compare
4202 }
4203 }
4204
4205 bind(TAIL_START);
4206 // Compare 4-byte vectors
4207 andl(len, 0xfffffffc); // vector count (in bytes)
4208 jccb(Assembler::zero, COMPARE_CHAR);
4209
4210 lea(ary1, Address(ary1, len, Address::times_1));
4211 negptr(len);
4212
4213 bind(COMPARE_VECTORS);
4214 movl(tmp1, Address(ary1, len, Address::times_1));
4215 andl(tmp1, 0x80808080);
4216 jccb(Assembler::notZero, TAIL_ADJUST);
4217 addptr(len, 4);
4218 jccb(Assembler::notZero, COMPARE_VECTORS);
4219
4220 // Compare trailing char (final 2-3 bytes), if any
4221 bind(COMPARE_CHAR);
4222
4223 testl(result, 0x2); // tail char
4224 jccb(Assembler::zero, COMPARE_BYTE);
4225 load_unsigned_short(tmp1, Address(ary1, 0));
4226 andl(tmp1, 0x00008080);
4227 jccb(Assembler::notZero, CHAR_ADJUST);
4228 lea(ary1, Address(ary1, 2));
4229
4230 bind(COMPARE_BYTE);
4231 testl(result, 0x1); // tail byte
4232 jccb(Assembler::zero, DONE);
4233 load_unsigned_byte(tmp1, Address(ary1, 0));
4234 testl(tmp1, 0x00000080);
4235 jccb(Assembler::zero, DONE);
4236 subptr(result, 1);
4237 jmpb(DONE);
4238
4239 bind(TAIL_ADJUST);
4240 // there are negative bits in the last 4 byte block.
4241 // Adjust result and check the next three bytes
4242 addptr(result, len);
4243 orl(result, 3);
4244 lea(ary1, Address(ary1, len, Address::times_1));
4245 jmpb(COMPARE_CHAR);
4246
4247 bind(CHAR_ADJUST);
4248 // We are looking at a char + optional byte tail, and found that one
4249 // of the bytes in the char is negative. Adjust the result, check the
4250 // first byte and readjust if needed.
4251 andl(result, 0xfffffffc);
4252 testl(tmp1, 0x00000080); // little-endian, so lowest byte comes first
4253 jccb(Assembler::notZero, DONE);
4254 addptr(result, 1);
4255
4256 // That's it
4257 bind(DONE);
4258 if (UseAVX >= 2) {
4259 // clean upper bits of YMM registers
4260 vpxor(vec1, vec1);
4261 vpxor(vec2, vec2);
4262 }
4263 }
4264
4265 // Compare char[] or byte[] arrays aligned to 4 bytes or substrings.
4266 void C2_MacroAssembler::arrays_equals(bool is_array_equ, Register ary1, Register ary2,
4267 Register limit, Register result, Register chr,
4268 XMMRegister vec1, XMMRegister vec2, bool is_char,
4269 KRegister mask, bool expand_ary2) {
4270 // for expand_ary2, limit is the (smaller) size of the second array.
4271 ShortBranchVerifier sbv(this);
4272 Label TRUE_LABEL, FALSE_LABEL, DONE, COMPARE_VECTORS, COMPARE_CHAR, COMPARE_BYTE;
4273
4274 assert((!expand_ary2) || ((expand_ary2) && (UseAVX == 2)),
4275 "Expansion only implemented for AVX2");
4276
4277 int length_offset = arrayOopDesc::length_offset_in_bytes();
4278 int base_offset = arrayOopDesc::base_offset_in_bytes(is_char ? T_CHAR : T_BYTE);
4279
4280 Address::ScaleFactor scaleFactor = expand_ary2 ? Address::times_2 : Address::times_1;
4281 int scaleIncr = expand_ary2 ? 8 : 16;
4282
4283 if (is_array_equ) {
4284 // Check the input args
4285 cmpoop(ary1, ary2);
4286 jcc(Assembler::equal, TRUE_LABEL);
4287
4288 // Need additional checks for arrays_equals.
4289 testptr(ary1, ary1);
4290 jcc(Assembler::zero, FALSE_LABEL);
4291 testptr(ary2, ary2);
4292 jcc(Assembler::zero, FALSE_LABEL);
4293
4294 // Check the lengths
4295 movl(limit, Address(ary1, length_offset));
4296 cmpl(limit, Address(ary2, length_offset));
4297 jcc(Assembler::notEqual, FALSE_LABEL);
4298 }
4299
4300 // count == 0
4301 testl(limit, limit);
4302 jcc(Assembler::zero, TRUE_LABEL);
4303
4304 if (is_array_equ) {
4305 // Load array address
4306 lea(ary1, Address(ary1, base_offset));
4307 lea(ary2, Address(ary2, base_offset));
4308 }
4309
4310 if (is_array_equ && is_char) {
4311 // arrays_equals when used for char[].
4312 shll(limit, 1); // byte count != 0
4313 }
4314 movl(result, limit); // copy
4315
4316 if (UseAVX >= 2) {
4317 // With AVX2, use 32-byte vector compare
4318 Label COMPARE_WIDE_VECTORS, COMPARE_WIDE_VECTORS_16, COMPARE_TAIL, COMPARE_TAIL_16;
4319
4320 // Compare 32-byte vectors
4321 if (expand_ary2) {
4322 andl(result, 0x0000000f); // tail count (in bytes)
4323 andl(limit, 0xfffffff0); // vector count (in bytes)
4324 jcc(Assembler::zero, COMPARE_TAIL);
4325 } else {
4326 andl(result, 0x0000001f); // tail count (in bytes)
4327 andl(limit, 0xffffffe0); // vector count (in bytes)
4328 jcc(Assembler::zero, COMPARE_TAIL_16);
4329 }
4330
4331 lea(ary1, Address(ary1, limit, scaleFactor));
4332 lea(ary2, Address(ary2, limit, Address::times_1));
4333 negptr(limit);
4334
4335 if ((AVX3Threshold == 0) && VM_Version::supports_avx512vlbw()) { // trying 64 bytes fast loop
4336 Label COMPARE_WIDE_VECTORS_LOOP_AVX2, COMPARE_WIDE_VECTORS_LOOP_AVX3;
4337
4338 cmpl(limit, -64);
4339 jcc(Assembler::greater, COMPARE_WIDE_VECTORS_LOOP_AVX2);
4340
4341 bind(COMPARE_WIDE_VECTORS_LOOP_AVX3); // the hottest loop
4342
4343 evmovdquq(vec1, Address(ary1, limit, Address::times_1), Assembler::AVX_512bit);
4344 evpcmpeqb(mask, vec1, Address(ary2, limit, Address::times_1), Assembler::AVX_512bit);
4345 kortestql(mask, mask);
4346 jcc(Assembler::aboveEqual, FALSE_LABEL); // miscompare
4347 addptr(limit, 64); // update since we already compared at this addr
4348 cmpl(limit, -64);
4349 jccb(Assembler::lessEqual, COMPARE_WIDE_VECTORS_LOOP_AVX3);
4350
4351 // At this point we may still need to compare -limit+result bytes.
4352 // We could execute the next two instruction and just continue via non-wide path:
4353 // cmpl(limit, 0);
4354 // jcc(Assembler::equal, COMPARE_TAIL); // true
4355 // But since we stopped at the points ary{1,2}+limit which are
4356 // not farther than 64 bytes from the ends of arrays ary{1,2}+result
4357 // (|limit| <= 32 and result < 32),
4358 // we may just compare the last 64 bytes.
4359 //
4360 addptr(result, -64); // it is safe, bc we just came from this area
4361 evmovdquq(vec1, Address(ary1, result, Address::times_1), Assembler::AVX_512bit);
4362 evpcmpeqb(mask, vec1, Address(ary2, result, Address::times_1), Assembler::AVX_512bit);
4363 kortestql(mask, mask);
4364 jcc(Assembler::aboveEqual, FALSE_LABEL); // miscompare
4365
4366 jmp(TRUE_LABEL);
4367
4368 bind(COMPARE_WIDE_VECTORS_LOOP_AVX2);
4369
4370 }//if (VM_Version::supports_avx512vlbw())
4371
4372 bind(COMPARE_WIDE_VECTORS);
4373 vmovdqu(vec1, Address(ary1, limit, scaleFactor));
4374 if (expand_ary2) {
4375 vpmovzxbw(vec2, Address(ary2, limit, Address::times_1), Assembler::AVX_256bit);
4376 } else {
4377 vmovdqu(vec2, Address(ary2, limit, Address::times_1));
4378 }
4379 vpxor(vec1, vec2);
4380
4381 vptest(vec1, vec1);
4382 jcc(Assembler::notZero, FALSE_LABEL);
4383 addptr(limit, scaleIncr * 2);
4384 jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
4385
4386 testl(result, result);
4387 jcc(Assembler::zero, TRUE_LABEL);
4388
4389 vmovdqu(vec1, Address(ary1, result, scaleFactor, -32));
4390 if (expand_ary2) {
4391 vpmovzxbw(vec2, Address(ary2, result, Address::times_1, -16), Assembler::AVX_256bit);
4392 } else {
4393 vmovdqu(vec2, Address(ary2, result, Address::times_1, -32));
4394 }
4395 vpxor(vec1, vec2);
4396
4397 vptest(vec1, vec1);
4398 jcc(Assembler::notZero, FALSE_LABEL);
4399 jmp(TRUE_LABEL);
4400
4401 bind(COMPARE_TAIL_16); // limit is zero
4402 movl(limit, result);
4403
4404 // Compare 16-byte chunks
4405 andl(result, 0x0000000f); // tail count (in bytes)
4406 andl(limit, 0xfffffff0); // vector count (in bytes)
4407 jcc(Assembler::zero, COMPARE_TAIL);
4408
4409 lea(ary1, Address(ary1, limit, scaleFactor));
4410 lea(ary2, Address(ary2, limit, Address::times_1));
4411 negptr(limit);
4412
4413 bind(COMPARE_WIDE_VECTORS_16);
4414 movdqu(vec1, Address(ary1, limit, scaleFactor));
4415 if (expand_ary2) {
4416 vpmovzxbw(vec2, Address(ary2, limit, Address::times_1), Assembler::AVX_128bit);
4417 } else {
4418 movdqu(vec2, Address(ary2, limit, Address::times_1));
4419 }
4420 pxor(vec1, vec2);
4421
4422 ptest(vec1, vec1);
4423 jcc(Assembler::notZero, FALSE_LABEL);
4424 addptr(limit, scaleIncr);
4425 jcc(Assembler::notZero, COMPARE_WIDE_VECTORS_16);
4426
4427 bind(COMPARE_TAIL); // limit is zero
4428 movl(limit, result);
4429 // Fallthru to tail compare
4430 } else if (UseSSE42Intrinsics) {
4431 // With SSE4.2, use double quad vector compare
4432 Label COMPARE_WIDE_VECTORS, COMPARE_TAIL;
4433
4434 // Compare 16-byte vectors
4435 andl(result, 0x0000000f); // tail count (in bytes)
4436 andl(limit, 0xfffffff0); // vector count (in bytes)
4437 jcc(Assembler::zero, COMPARE_TAIL);
4438
4439 lea(ary1, Address(ary1, limit, Address::times_1));
4440 lea(ary2, Address(ary2, limit, Address::times_1));
4441 negptr(limit);
4442
4443 bind(COMPARE_WIDE_VECTORS);
4444 movdqu(vec1, Address(ary1, limit, Address::times_1));
4445 movdqu(vec2, Address(ary2, limit, Address::times_1));
4446 pxor(vec1, vec2);
4447
4448 ptest(vec1, vec1);
4449 jcc(Assembler::notZero, FALSE_LABEL);
4450 addptr(limit, 16);
4451 jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
4452
4453 testl(result, result);
4454 jcc(Assembler::zero, TRUE_LABEL);
4455
4456 movdqu(vec1, Address(ary1, result, Address::times_1, -16));
4457 movdqu(vec2, Address(ary2, result, Address::times_1, -16));
4458 pxor(vec1, vec2);
4459
4460 ptest(vec1, vec1);
4461 jccb(Assembler::notZero, FALSE_LABEL);
4462 jmpb(TRUE_LABEL);
4463
4464 bind(COMPARE_TAIL); // limit is zero
4465 movl(limit, result);
4466 // Fallthru to tail compare
4467 }
4468
4469 // Compare 4-byte vectors
4470 if (expand_ary2) {
4471 testl(result, result);
4472 jccb(Assembler::zero, TRUE_LABEL);
4473 } else {
4474 andl(limit, 0xfffffffc); // vector count (in bytes)
4475 jccb(Assembler::zero, COMPARE_CHAR);
4476 }
4477
4478 lea(ary1, Address(ary1, limit, scaleFactor));
4479 lea(ary2, Address(ary2, limit, Address::times_1));
4480 negptr(limit);
4481
4482 bind(COMPARE_VECTORS);
4483 if (expand_ary2) {
4484 // There are no "vector" operations for bytes to shorts
4485 movzbl(chr, Address(ary2, limit, Address::times_1));
4486 cmpw(Address(ary1, limit, Address::times_2), chr);
4487 jccb(Assembler::notEqual, FALSE_LABEL);
4488 addptr(limit, 1);
4489 jcc(Assembler::notZero, COMPARE_VECTORS);
4490 jmp(TRUE_LABEL);
4491 } else {
4492 movl(chr, Address(ary1, limit, Address::times_1));
4493 cmpl(chr, Address(ary2, limit, Address::times_1));
4494 jccb(Assembler::notEqual, FALSE_LABEL);
4495 addptr(limit, 4);
4496 jcc(Assembler::notZero, COMPARE_VECTORS);
4497 }
4498
4499 // Compare trailing char (final 2 bytes), if any
4500 bind(COMPARE_CHAR);
4501 testl(result, 0x2); // tail char
4502 jccb(Assembler::zero, COMPARE_BYTE);
4503 load_unsigned_short(chr, Address(ary1, 0));
4504 load_unsigned_short(limit, Address(ary2, 0));
4505 cmpl(chr, limit);
4506 jccb(Assembler::notEqual, FALSE_LABEL);
4507
4508 if (is_array_equ && is_char) {
4509 bind(COMPARE_BYTE);
4510 } else {
4511 lea(ary1, Address(ary1, 2));
4512 lea(ary2, Address(ary2, 2));
4513
4514 bind(COMPARE_BYTE);
4515 testl(result, 0x1); // tail byte
4516 jccb(Assembler::zero, TRUE_LABEL);
4517 load_unsigned_byte(chr, Address(ary1, 0));
4518 load_unsigned_byte(limit, Address(ary2, 0));
4519 cmpl(chr, limit);
4520 jccb(Assembler::notEqual, FALSE_LABEL);
4521 }
4522 bind(TRUE_LABEL);
4523 movl(result, 1); // return true
4524 jmpb(DONE);
4525
4526 bind(FALSE_LABEL);
4527 xorl(result, result); // return false
4528
4529 // That's it
4530 bind(DONE);
4531 if (UseAVX >= 2) {
4532 // clean upper bits of YMM registers
4533 vpxor(vec1, vec1);
4534 vpxor(vec2, vec2);
4535 }
4536 }
4537
4538 static void convertF2I_slowpath(C2_MacroAssembler& masm, C2GeneralStub<Register, XMMRegister, address>& stub) {
4539 #define __ masm.
4540 Register dst = stub.data<0>();
4541 XMMRegister src = stub.data<1>();
4542 address target = stub.data<2>();
4543 __ bind(stub.entry());
4544 __ subptr(rsp, 8);
4545 __ movdbl(Address(rsp), src);
4546 __ call(RuntimeAddress(target));
4547 // APX REX2 encoding for pop(dst) increases the stub size by 1 byte.
4548 __ pop(dst);
4549 __ jmp(stub.continuation());
4550 #undef __
4551 }
4552
4553 void C2_MacroAssembler::convertF2I(BasicType dst_bt, BasicType src_bt, Register dst, XMMRegister src) {
4554 assert(dst_bt == T_INT || dst_bt == T_LONG, "");
4555 assert(src_bt == T_FLOAT || src_bt == T_DOUBLE, "");
4556
4557 address slowpath_target;
4558 if (dst_bt == T_INT) {
4559 if (src_bt == T_FLOAT) {
4560 cvttss2sil(dst, src);
4561 cmpl(dst, 0x80000000);
4562 slowpath_target = StubRoutines::x86::f2i_fixup();
4563 } else {
4564 cvttsd2sil(dst, src);
4565 cmpl(dst, 0x80000000);
4566 slowpath_target = StubRoutines::x86::d2i_fixup();
4567 }
4568 } else {
4569 if (src_bt == T_FLOAT) {
4570 cvttss2siq(dst, src);
4571 cmp64(dst, ExternalAddress(StubRoutines::x86::double_sign_flip()));
4572 slowpath_target = StubRoutines::x86::f2l_fixup();
4573 } else {
4574 cvttsd2siq(dst, src);
4575 cmp64(dst, ExternalAddress(StubRoutines::x86::double_sign_flip()));
4576 slowpath_target = StubRoutines::x86::d2l_fixup();
4577 }
4578 }
4579
4580 // Using the APX extended general purpose registers increases the instruction encoding size by 1 byte.
4581 int max_size = 23 + (UseAPX ? 1 : 0);
4582 auto stub = C2CodeStub::make<Register, XMMRegister, address>(dst, src, slowpath_target, max_size, convertF2I_slowpath);
4583 jcc(Assembler::equal, stub->entry());
4584 bind(stub->continuation());
4585 }
4586
4587 void C2_MacroAssembler::evmasked_op(int ideal_opc, BasicType eType, KRegister mask, XMMRegister dst,
4588 XMMRegister src1, int imm8, bool merge, int vlen_enc) {
4589 switch(ideal_opc) {
4590 case Op_LShiftVS:
4591 Assembler::evpsllw(dst, mask, src1, imm8, merge, vlen_enc); break;
4592 case Op_LShiftVI:
4593 Assembler::evpslld(dst, mask, src1, imm8, merge, vlen_enc); break;
4594 case Op_LShiftVL:
4595 Assembler::evpsllq(dst, mask, src1, imm8, merge, vlen_enc); break;
4596 case Op_RShiftVS:
4597 Assembler::evpsraw(dst, mask, src1, imm8, merge, vlen_enc); break;
4598 case Op_RShiftVI:
4599 Assembler::evpsrad(dst, mask, src1, imm8, merge, vlen_enc); break;
4600 case Op_RShiftVL:
4601 Assembler::evpsraq(dst, mask, src1, imm8, merge, vlen_enc); break;
4602 case Op_URShiftVS:
4603 Assembler::evpsrlw(dst, mask, src1, imm8, merge, vlen_enc); break;
4604 case Op_URShiftVI:
4605 Assembler::evpsrld(dst, mask, src1, imm8, merge, vlen_enc); break;
4606 case Op_URShiftVL:
4607 Assembler::evpsrlq(dst, mask, src1, imm8, merge, vlen_enc); break;
4608 case Op_RotateRightV:
4609 evrord(eType, dst, mask, src1, imm8, merge, vlen_enc); break;
4610 case Op_RotateLeftV:
4611 evrold(eType, dst, mask, src1, imm8, merge, vlen_enc); break;
4612 default:
4613 fatal("Unsupported operation %s", NodeClassNames[ideal_opc]);
4614 break;
4615 }
4616 }
4617
4618 void C2_MacroAssembler::evmasked_saturating_op(int ideal_opc, BasicType elem_bt, KRegister mask, XMMRegister dst, XMMRegister src1,
4619 XMMRegister src2, bool is_unsigned, bool merge, int vlen_enc) {
4620 if (is_unsigned) {
4621 evmasked_saturating_unsigned_op(ideal_opc, elem_bt, mask, dst, src1, src2, merge, vlen_enc);
4622 } else {
4623 evmasked_saturating_signed_op(ideal_opc, elem_bt, mask, dst, src1, src2, merge, vlen_enc);
4624 }
4625 }
4626
4627 void C2_MacroAssembler::evmasked_saturating_signed_op(int ideal_opc, BasicType elem_bt, KRegister mask, XMMRegister dst,
4628 XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) {
4629 switch (elem_bt) {
4630 case T_BYTE:
4631 if (ideal_opc == Op_SaturatingAddV) {
4632 evpaddsb(dst, mask, src1, src2, merge, vlen_enc);
4633 } else {
4634 assert(ideal_opc == Op_SaturatingSubV, "");
4635 evpsubsb(dst, mask, src1, src2, merge, vlen_enc);
4636 }
4637 break;
4638 case T_SHORT:
4639 if (ideal_opc == Op_SaturatingAddV) {
4640 evpaddsw(dst, mask, src1, src2, merge, vlen_enc);
4641 } else {
4642 assert(ideal_opc == Op_SaturatingSubV, "");
4643 evpsubsw(dst, mask, src1, src2, merge, vlen_enc);
4644 }
4645 break;
4646 default:
4647 fatal("Unsupported type %s", type2name(elem_bt));
4648 break;
4649 }
4650 }
4651
4652 void C2_MacroAssembler::evmasked_saturating_unsigned_op(int ideal_opc, BasicType elem_bt, KRegister mask, XMMRegister dst,
4653 XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) {
4654 switch (elem_bt) {
4655 case T_BYTE:
4656 if (ideal_opc == Op_SaturatingAddV) {
4657 evpaddusb(dst, mask, src1, src2, merge, vlen_enc);
4658 } else {
4659 assert(ideal_opc == Op_SaturatingSubV, "");
4660 evpsubusb(dst, mask, src1, src2, merge, vlen_enc);
4661 }
4662 break;
4663 case T_SHORT:
4664 if (ideal_opc == Op_SaturatingAddV) {
4665 evpaddusw(dst, mask, src1, src2, merge, vlen_enc);
4666 } else {
4667 assert(ideal_opc == Op_SaturatingSubV, "");
4668 evpsubusw(dst, mask, src1, src2, merge, vlen_enc);
4669 }
4670 break;
4671 default:
4672 fatal("Unsupported type %s", type2name(elem_bt));
4673 break;
4674 }
4675 }
4676
4677 void C2_MacroAssembler::evmasked_saturating_op(int ideal_opc, BasicType elem_bt, KRegister mask, XMMRegister dst, XMMRegister src1,
4678 Address src2, bool is_unsigned, bool merge, int vlen_enc) {
4679 if (is_unsigned) {
4680 evmasked_saturating_unsigned_op(ideal_opc, elem_bt, mask, dst, src1, src2, merge, vlen_enc);
4681 } else {
4682 evmasked_saturating_signed_op(ideal_opc, elem_bt, mask, dst, src1, src2, merge, vlen_enc);
4683 }
4684 }
4685
4686 void C2_MacroAssembler::evmasked_saturating_signed_op(int ideal_opc, BasicType elem_bt, KRegister mask, XMMRegister dst,
4687 XMMRegister src1, Address src2, bool merge, int vlen_enc) {
4688 switch (elem_bt) {
4689 case T_BYTE:
4690 if (ideal_opc == Op_SaturatingAddV) {
4691 evpaddsb(dst, mask, src1, src2, merge, vlen_enc);
4692 } else {
4693 assert(ideal_opc == Op_SaturatingSubV, "");
4694 evpsubsb(dst, mask, src1, src2, merge, vlen_enc);
4695 }
4696 break;
4697 case T_SHORT:
4698 if (ideal_opc == Op_SaturatingAddV) {
4699 evpaddsw(dst, mask, src1, src2, merge, vlen_enc);
4700 } else {
4701 assert(ideal_opc == Op_SaturatingSubV, "");
4702 evpsubsw(dst, mask, src1, src2, merge, vlen_enc);
4703 }
4704 break;
4705 default:
4706 fatal("Unsupported type %s", type2name(elem_bt));
4707 break;
4708 }
4709 }
4710
4711 void C2_MacroAssembler::evmasked_saturating_unsigned_op(int ideal_opc, BasicType elem_bt, KRegister mask, XMMRegister dst,
4712 XMMRegister src1, Address src2, bool merge, int vlen_enc) {
4713 switch (elem_bt) {
4714 case T_BYTE:
4715 if (ideal_opc == Op_SaturatingAddV) {
4716 evpaddusb(dst, mask, src1, src2, merge, vlen_enc);
4717 } else {
4718 assert(ideal_opc == Op_SaturatingSubV, "");
4719 evpsubusb(dst, mask, src1, src2, merge, vlen_enc);
4720 }
4721 break;
4722 case T_SHORT:
4723 if (ideal_opc == Op_SaturatingAddV) {
4724 evpaddusw(dst, mask, src1, src2, merge, vlen_enc);
4725 } else {
4726 assert(ideal_opc == Op_SaturatingSubV, "");
4727 evpsubusw(dst, mask, src1, src2, merge, vlen_enc);
4728 }
4729 break;
4730 default:
4731 fatal("Unsupported type %s", type2name(elem_bt));
4732 break;
4733 }
4734 }
4735
4736 void C2_MacroAssembler::evmasked_op(int ideal_opc, BasicType eType, KRegister mask, XMMRegister dst,
4737 XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc,
4738 bool is_varshift) {
4739 switch (ideal_opc) {
4740 case Op_AddVB:
4741 evpaddb(dst, mask, src1, src2, merge, vlen_enc); break;
4742 case Op_AddVS:
4743 evpaddw(dst, mask, src1, src2, merge, vlen_enc); break;
4744 case Op_AddVI:
4745 evpaddd(dst, mask, src1, src2, merge, vlen_enc); break;
4746 case Op_AddVL:
4747 evpaddq(dst, mask, src1, src2, merge, vlen_enc); break;
4748 case Op_AddVF:
4749 evaddps(dst, mask, src1, src2, merge, vlen_enc); break;
4750 case Op_AddVD:
4751 evaddpd(dst, mask, src1, src2, merge, vlen_enc); break;
4752 case Op_SubVB:
4753 evpsubb(dst, mask, src1, src2, merge, vlen_enc); break;
4754 case Op_SubVS:
4755 evpsubw(dst, mask, src1, src2, merge, vlen_enc); break;
4756 case Op_SubVI:
4757 evpsubd(dst, mask, src1, src2, merge, vlen_enc); break;
4758 case Op_SubVL:
4759 evpsubq(dst, mask, src1, src2, merge, vlen_enc); break;
4760 case Op_SubVF:
4761 evsubps(dst, mask, src1, src2, merge, vlen_enc); break;
4762 case Op_SubVD:
4763 evsubpd(dst, mask, src1, src2, merge, vlen_enc); break;
4764 case Op_MulVS:
4765 evpmullw(dst, mask, src1, src2, merge, vlen_enc); break;
4766 case Op_MulVI:
4767 evpmulld(dst, mask, src1, src2, merge, vlen_enc); break;
4768 case Op_MulVL:
4769 evpmullq(dst, mask, src1, src2, merge, vlen_enc); break;
4770 case Op_MulVF:
4771 evmulps(dst, mask, src1, src2, merge, vlen_enc); break;
4772 case Op_MulVD:
4773 evmulpd(dst, mask, src1, src2, merge, vlen_enc); break;
4774 case Op_DivVF:
4775 evdivps(dst, mask, src1, src2, merge, vlen_enc); break;
4776 case Op_DivVD:
4777 evdivpd(dst, mask, src1, src2, merge, vlen_enc); break;
4778 case Op_SqrtVF:
4779 evsqrtps(dst, mask, src1, src2, merge, vlen_enc); break;
4780 case Op_SqrtVD:
4781 evsqrtpd(dst, mask, src1, src2, merge, vlen_enc); break;
4782 case Op_AbsVB:
4783 evpabsb(dst, mask, src2, merge, vlen_enc); break;
4784 case Op_AbsVS:
4785 evpabsw(dst, mask, src2, merge, vlen_enc); break;
4786 case Op_AbsVI:
4787 evpabsd(dst, mask, src2, merge, vlen_enc); break;
4788 case Op_AbsVL:
4789 evpabsq(dst, mask, src2, merge, vlen_enc); break;
4790 case Op_FmaVF:
4791 evpfma213ps(dst, mask, src1, src2, merge, vlen_enc); break;
4792 case Op_FmaVD:
4793 evpfma213pd(dst, mask, src1, src2, merge, vlen_enc); break;
4794 case Op_VectorRearrange:
4795 evperm(eType, dst, mask, src2, src1, merge, vlen_enc); break;
4796 case Op_LShiftVS:
4797 evpsllw(dst, mask, src1, src2, merge, vlen_enc, is_varshift); break;
4798 case Op_LShiftVI:
4799 evpslld(dst, mask, src1, src2, merge, vlen_enc, is_varshift); break;
4800 case Op_LShiftVL:
4801 evpsllq(dst, mask, src1, src2, merge, vlen_enc, is_varshift); break;
4802 case Op_RShiftVS:
4803 evpsraw(dst, mask, src1, src2, merge, vlen_enc, is_varshift); break;
4804 case Op_RShiftVI:
4805 evpsrad(dst, mask, src1, src2, merge, vlen_enc, is_varshift); break;
4806 case Op_RShiftVL:
4807 evpsraq(dst, mask, src1, src2, merge, vlen_enc, is_varshift); break;
4808 case Op_URShiftVS:
4809 evpsrlw(dst, mask, src1, src2, merge, vlen_enc, is_varshift); break;
4810 case Op_URShiftVI:
4811 evpsrld(dst, mask, src1, src2, merge, vlen_enc, is_varshift); break;
4812 case Op_URShiftVL:
4813 evpsrlq(dst, mask, src1, src2, merge, vlen_enc, is_varshift); break;
4814 case Op_RotateLeftV:
4815 evrold(eType, dst, mask, src1, src2, merge, vlen_enc); break;
4816 case Op_RotateRightV:
4817 evrord(eType, dst, mask, src1, src2, merge, vlen_enc); break;
4818 case Op_MaxV:
4819 evpmaxs(eType, dst, mask, src1, src2, merge, vlen_enc); break;
4820 case Op_MinV:
4821 evpmins(eType, dst, mask, src1, src2, merge, vlen_enc); break;
4822 case Op_UMinV:
4823 evpminu(eType, dst, mask, src1, src2, merge, vlen_enc); break;
4824 case Op_UMaxV:
4825 evpmaxu(eType, dst, mask, src1, src2, merge, vlen_enc); break;
4826 case Op_XorV:
4827 evxor(eType, dst, mask, src1, src2, merge, vlen_enc); break;
4828 case Op_OrV:
4829 evor(eType, dst, mask, src1, src2, merge, vlen_enc); break;
4830 case Op_AndV:
4831 evand(eType, dst, mask, src1, src2, merge, vlen_enc); break;
4832 default:
4833 fatal("Unsupported operation %s", NodeClassNames[ideal_opc]);
4834 break;
4835 }
4836 }
4837
4838 void C2_MacroAssembler::evmasked_op(int ideal_opc, BasicType eType, KRegister mask, XMMRegister dst,
4839 XMMRegister src1, Address src2, bool merge, int vlen_enc) {
4840 switch (ideal_opc) {
4841 case Op_AddVB:
4842 evpaddb(dst, mask, src1, src2, merge, vlen_enc); break;
4843 case Op_AddVS:
4844 evpaddw(dst, mask, src1, src2, merge, vlen_enc); break;
4845 case Op_AddVI:
4846 evpaddd(dst, mask, src1, src2, merge, vlen_enc); break;
4847 case Op_AddVL:
4848 evpaddq(dst, mask, src1, src2, merge, vlen_enc); break;
4849 case Op_AddVF:
4850 evaddps(dst, mask, src1, src2, merge, vlen_enc); break;
4851 case Op_AddVD:
4852 evaddpd(dst, mask, src1, src2, merge, vlen_enc); break;
4853 case Op_SubVB:
4854 evpsubb(dst, mask, src1, src2, merge, vlen_enc); break;
4855 case Op_SubVS:
4856 evpsubw(dst, mask, src1, src2, merge, vlen_enc); break;
4857 case Op_SubVI:
4858 evpsubd(dst, mask, src1, src2, merge, vlen_enc); break;
4859 case Op_SubVL:
4860 evpsubq(dst, mask, src1, src2, merge, vlen_enc); break;
4861 case Op_SubVF:
4862 evsubps(dst, mask, src1, src2, merge, vlen_enc); break;
4863 case Op_SubVD:
4864 evsubpd(dst, mask, src1, src2, merge, vlen_enc); break;
4865 case Op_MulVS:
4866 evpmullw(dst, mask, src1, src2, merge, vlen_enc); break;
4867 case Op_MulVI:
4868 evpmulld(dst, mask, src1, src2, merge, vlen_enc); break;
4869 case Op_MulVL:
4870 evpmullq(dst, mask, src1, src2, merge, vlen_enc); break;
4871 case Op_MulVF:
4872 evmulps(dst, mask, src1, src2, merge, vlen_enc); break;
4873 case Op_MulVD:
4874 evmulpd(dst, mask, src1, src2, merge, vlen_enc); break;
4875 case Op_DivVF:
4876 evdivps(dst, mask, src1, src2, merge, vlen_enc); break;
4877 case Op_DivVD:
4878 evdivpd(dst, mask, src1, src2, merge, vlen_enc); break;
4879 case Op_FmaVF:
4880 evpfma213ps(dst, mask, src1, src2, merge, vlen_enc); break;
4881 case Op_FmaVD:
4882 evpfma213pd(dst, mask, src1, src2, merge, vlen_enc); break;
4883 case Op_MaxV:
4884 evpmaxs(eType, dst, mask, src1, src2, merge, vlen_enc); break;
4885 case Op_MinV:
4886 evpmins(eType, dst, mask, src1, src2, merge, vlen_enc); break;
4887 case Op_UMaxV:
4888 evpmaxu(eType, dst, mask, src1, src2, merge, vlen_enc); break;
4889 case Op_UMinV:
4890 evpminu(eType, dst, mask, src1, src2, merge, vlen_enc); break;
4891 case Op_XorV:
4892 evxor(eType, dst, mask, src1, src2, merge, vlen_enc); break;
4893 case Op_OrV:
4894 evor(eType, dst, mask, src1, src2, merge, vlen_enc); break;
4895 case Op_AndV:
4896 evand(eType, dst, mask, src1, src2, merge, vlen_enc); break;
4897 default:
4898 fatal("Unsupported operation %s", NodeClassNames[ideal_opc]);
4899 break;
4900 }
4901 }
4902
4903 void C2_MacroAssembler::masked_op(int ideal_opc, int mask_len, KRegister dst,
4904 KRegister src1, KRegister src2) {
4905 BasicType etype = T_ILLEGAL;
4906 switch(mask_len) {
4907 case 2:
4908 case 4:
4909 case 8: etype = T_BYTE; break;
4910 case 16: etype = T_SHORT; break;
4911 case 32: etype = T_INT; break;
4912 case 64: etype = T_LONG; break;
4913 default: fatal("Unsupported type"); break;
4914 }
4915 assert(etype != T_ILLEGAL, "");
4916 switch(ideal_opc) {
4917 case Op_AndVMask:
4918 kand(etype, dst, src1, src2); break;
4919 case Op_OrVMask:
4920 kor(etype, dst, src1, src2); break;
4921 case Op_XorVMask:
4922 kxor(etype, dst, src1, src2); break;
4923 default:
4924 fatal("Unsupported masked operation"); break;
4925 }
4926 }
4927
4928 /*
4929 * Following routine handles special floating point values(NaN/Inf/-Inf/Max/Min) for casting operation.
4930 * If src is NaN, the result is 0.
4931 * If the src is negative infinity or any value less than or equal to the value of Integer.MIN_VALUE,
4932 * the result is equal to the value of Integer.MIN_VALUE.
4933 * If the src is positive infinity or any value greater than or equal to the value of Integer.MAX_VALUE,
4934 * the result is equal to the value of Integer.MAX_VALUE.
4935 */
4936 void C2_MacroAssembler::vector_cast_float_to_int_special_cases_avx(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
4937 XMMRegister xtmp2, XMMRegister xtmp3, XMMRegister xtmp4,
4938 Register rscratch, AddressLiteral float_sign_flip,
4939 int vec_enc) {
4940 assert(rscratch != noreg || always_reachable(float_sign_flip), "missing");
4941 Label done;
4942 vmovdqu(xtmp1, float_sign_flip, vec_enc, rscratch);
4943 vpcmpeqd(xtmp2, dst, xtmp1, vec_enc);
4944 vptest(xtmp2, xtmp2, vec_enc);
4945 jccb(Assembler::equal, done);
4946
4947 vpcmpeqd(xtmp4, xtmp4, xtmp4, vec_enc);
4948 vpxor(xtmp1, xtmp1, xtmp4, vec_enc);
4949
4950 vpxor(xtmp4, xtmp4, xtmp4, vec_enc);
4951 vcmpps(xtmp3, src, src, Assembler::UNORD_Q, vec_enc);
4952 vblendvps(dst, dst, xtmp4, xtmp3, vec_enc);
4953
4954 // Recompute the mask for remaining special value.
4955 vpxor(xtmp2, xtmp2, xtmp3, vec_enc);
4956 // Extract SRC values corresponding to TRUE mask lanes.
4957 vpand(xtmp4, xtmp2, src, vec_enc);
4958 // Flip mask bits so that MSB bit of MASK lanes corresponding to +ve special
4959 // values are set.
4960 vpxor(xtmp3, xtmp2, xtmp4, vec_enc);
4961
4962 vblendvps(dst, dst, xtmp1, xtmp3, vec_enc);
4963 bind(done);
4964 }
4965
4966 void C2_MacroAssembler::vector_cast_float_to_int_special_cases_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
4967 XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2,
4968 Register rscratch, AddressLiteral float_sign_flip,
4969 int vec_enc) {
4970 assert(rscratch != noreg || always_reachable(float_sign_flip), "missing");
4971 Label done;
4972 evmovdqul(xtmp1, k0, float_sign_flip, false, vec_enc, rscratch);
4973 Assembler::evpcmpeqd(ktmp1, k0, xtmp1, dst, vec_enc);
4974 kortestwl(ktmp1, ktmp1);
4975 jccb(Assembler::equal, done);
4976
4977 vpxor(xtmp2, xtmp2, xtmp2, vec_enc);
4978 evcmpps(ktmp2, k0, src, src, Assembler::UNORD_Q, vec_enc);
4979 evmovdqul(dst, ktmp2, xtmp2, true, vec_enc);
4980
4981 kxorwl(ktmp1, ktmp1, ktmp2);
4982 evcmpps(ktmp1, ktmp1, src, xtmp2, Assembler::NLT_UQ, vec_enc);
4983 vpternlogd(xtmp2, 0x11, xtmp1, xtmp1, vec_enc);
4984 evmovdqul(dst, ktmp1, xtmp2, true, vec_enc);
4985 bind(done);
4986 }
4987
4988 void C2_MacroAssembler::vector_cast_float_to_long_special_cases_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
4989 XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2,
4990 Register rscratch, AddressLiteral double_sign_flip,
4991 int vec_enc) {
4992 assert(rscratch != noreg || always_reachable(double_sign_flip), "missing");
4993
4994 Label done;
4995 evmovdquq(xtmp1, k0, double_sign_flip, false, vec_enc, rscratch);
4996 Assembler::evpcmpeqq(ktmp1, k0, xtmp1, dst, vec_enc);
4997 kortestwl(ktmp1, ktmp1);
4998 jccb(Assembler::equal, done);
4999
5000 vpxor(xtmp2, xtmp2, xtmp2, vec_enc);
5001 evcmpps(ktmp2, k0, src, src, Assembler::UNORD_Q, vec_enc);
5002 evmovdquq(dst, ktmp2, xtmp2, true, vec_enc);
5003
5004 kxorwl(ktmp1, ktmp1, ktmp2);
5005 evcmpps(ktmp1, ktmp1, src, xtmp2, Assembler::NLT_UQ, vec_enc);
5006 vpternlogq(xtmp2, 0x11, xtmp1, xtmp1, vec_enc);
5007 evmovdquq(dst, ktmp1, xtmp2, true, vec_enc);
5008 bind(done);
5009 }
5010
5011 void C2_MacroAssembler::vector_cast_double_to_int_special_cases_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
5012 XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2,
5013 Register rscratch, AddressLiteral float_sign_flip,
5014 int vec_enc) {
5015 assert(rscratch != noreg || always_reachable(float_sign_flip), "missing");
5016 Label done;
5017 evmovdquq(xtmp1, k0, float_sign_flip, false, vec_enc, rscratch);
5018 Assembler::evpcmpeqd(ktmp1, k0, xtmp1, dst, vec_enc);
5019 kortestwl(ktmp1, ktmp1);
5020 jccb(Assembler::equal, done);
5021
5022 vpxor(xtmp2, xtmp2, xtmp2, vec_enc);
5023 evcmppd(ktmp2, k0, src, src, Assembler::UNORD_Q, vec_enc);
5024 evmovdqul(dst, ktmp2, xtmp2, true, vec_enc);
5025
5026 kxorwl(ktmp1, ktmp1, ktmp2);
5027 evcmppd(ktmp1, ktmp1, src, xtmp2, Assembler::NLT_UQ, vec_enc);
5028 vpternlogq(xtmp2, 0x11, xtmp1, xtmp1, vec_enc);
5029 evmovdqul(dst, ktmp1, xtmp2, true, vec_enc);
5030 bind(done);
5031 }
5032
5033 /*
5034 * Following routine handles special floating point values(NaN/Inf/-Inf/Max/Min) for casting operation.
5035 * If src is NaN, the result is 0.
5036 * If the src is negative infinity or any value less than or equal to the value of Long.MIN_VALUE,
5037 * the result is equal to the value of Long.MIN_VALUE.
5038 * If the src is positive infinity or any value greater than or equal to the value of Long.MAX_VALUE,
5039 * the result is equal to the value of Long.MAX_VALUE.
5040 */
5041 void C2_MacroAssembler::vector_cast_double_to_long_special_cases_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
5042 XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2,
5043 Register rscratch, AddressLiteral double_sign_flip,
5044 int vec_enc) {
5045 assert(rscratch != noreg || always_reachable(double_sign_flip), "missing");
5046
5047 Label done;
5048 evmovdqul(xtmp1, k0, double_sign_flip, false, vec_enc, rscratch);
5049 evpcmpeqq(ktmp1, xtmp1, dst, vec_enc);
5050 kortestwl(ktmp1, ktmp1);
5051 jccb(Assembler::equal, done);
5052
5053 vpxor(xtmp2, xtmp2, xtmp2, vec_enc);
5054 evcmppd(ktmp2, k0, src, src, Assembler::UNORD_Q, vec_enc);
5055 evmovdquq(dst, ktmp2, xtmp2, true, vec_enc);
5056
5057 kxorwl(ktmp1, ktmp1, ktmp2);
5058 evcmppd(ktmp1, ktmp1, src, xtmp2, Assembler::NLT_UQ, vec_enc);
5059 vpternlogq(xtmp2, 0x11, xtmp1, xtmp1, vec_enc);
5060 evmovdquq(dst, ktmp1, xtmp2, true, vec_enc);
5061 bind(done);
5062 }
5063
5064 void C2_MacroAssembler::vector_crosslane_doubleword_pack_avx(XMMRegister dst, XMMRegister src, XMMRegister zero,
5065 XMMRegister xtmp, int index, int vec_enc) {
5066 assert(vec_enc < Assembler::AVX_512bit, "");
5067 if (vec_enc == Assembler::AVX_256bit) {
5068 vextractf128_high(xtmp, src);
5069 vshufps(dst, src, xtmp, index, vec_enc);
5070 } else {
5071 vshufps(dst, src, zero, index, vec_enc);
5072 }
5073 }
5074
5075 void C2_MacroAssembler::vector_cast_double_to_int_special_cases_avx(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
5076 XMMRegister xtmp3, XMMRegister xtmp4, XMMRegister xtmp5, Register rscratch,
5077 AddressLiteral float_sign_flip, int src_vec_enc) {
5078 assert(rscratch != noreg || always_reachable(float_sign_flip), "missing");
5079
5080 Label done;
5081 // Compare the destination lanes with float_sign_flip
5082 // value to get mask for all special values.
5083 movdqu(xtmp1, float_sign_flip, rscratch);
5084 vpcmpeqd(xtmp2, dst, xtmp1, Assembler::AVX_128bit);
5085 ptest(xtmp2, xtmp2);
5086 jccb(Assembler::equal, done);
5087
5088 // Flip float_sign_flip to get max integer value.
5089 vpcmpeqd(xtmp4, xtmp4, xtmp4, Assembler::AVX_128bit);
5090 pxor(xtmp1, xtmp4);
5091
5092 // Set detination lanes corresponding to unordered source lanes as zero.
5093 vpxor(xtmp4, xtmp4, xtmp4, src_vec_enc);
5094 vcmppd(xtmp3, src, src, Assembler::UNORD_Q, src_vec_enc);
5095
5096 // Shuffle mask vector and pack lower doubles word from each quadword lane.
5097 vector_crosslane_doubleword_pack_avx(xtmp3, xtmp3, xtmp4, xtmp5, 0x88, src_vec_enc);
5098 vblendvps(dst, dst, xtmp4, xtmp3, Assembler::AVX_128bit);
5099
5100 // Recompute the mask for remaining special value.
5101 pxor(xtmp2, xtmp3);
5102 // Extract mask corresponding to non-negative source lanes.
5103 vcmppd(xtmp3, src, xtmp4, Assembler::NLT_UQ, src_vec_enc);
5104
5105 // Shuffle mask vector and pack lower doubles word from each quadword lane.
5106 vector_crosslane_doubleword_pack_avx(xtmp3, xtmp3, xtmp4, xtmp5, 0x88, src_vec_enc);
5107 pand(xtmp3, xtmp2);
5108
5109 // Replace destination lanes holding special value(0x80000000) with max int
5110 // if corresponding source lane holds a +ve value.
5111 vblendvps(dst, dst, xtmp1, xtmp3, Assembler::AVX_128bit);
5112 bind(done);
5113 }
5114
5115
5116 void C2_MacroAssembler::vector_cast_int_to_subword(BasicType to_elem_bt, XMMRegister dst, XMMRegister zero,
5117 XMMRegister xtmp, Register rscratch, int vec_enc) {
5118 switch(to_elem_bt) {
5119 case T_SHORT:
5120 assert(rscratch != noreg || always_reachable(ExternalAddress(StubRoutines::x86::vector_int_to_short_mask())), "missing");
5121 vpand(dst, dst, ExternalAddress(StubRoutines::x86::vector_int_to_short_mask()), vec_enc, rscratch);
5122 vpackusdw(dst, dst, zero, vec_enc);
5123 if (vec_enc == Assembler::AVX_256bit) {
5124 vector_crosslane_doubleword_pack_avx(dst, dst, zero, xtmp, 0x44, vec_enc);
5125 }
5126 break;
5127 case T_BYTE:
5128 assert(rscratch != noreg || always_reachable(ExternalAddress(StubRoutines::x86::vector_int_to_byte_mask())), "missing");
5129 vpand(dst, dst, ExternalAddress(StubRoutines::x86::vector_int_to_byte_mask()), vec_enc, rscratch);
5130 vpackusdw(dst, dst, zero, vec_enc);
5131 if (vec_enc == Assembler::AVX_256bit) {
5132 vector_crosslane_doubleword_pack_avx(dst, dst, zero, xtmp, 0x44, vec_enc);
5133 }
5134 vpackuswb(dst, dst, zero, vec_enc);
5135 break;
5136 default: assert(false, "Unexpected basic type for target of vector cast int to subword: %s", type2name(to_elem_bt));
5137 }
5138 }
5139
5140 /*
5141 * Algorithm for vector D2L and F2I conversions (AVX 10.2 unsupported):-
5142 * a) Perform vector D2L/F2I cast.
5143 * b) Choose fast path if none of the result vector lane contains 0x80000000 value.
5144 * It signifies that source value could be any of the special floating point
5145 * values(NaN,-Inf,Inf,Max,-Min).
5146 * c) Set destination to zero if source is NaN value.
5147 * d) Replace 0x80000000 with MaxInt if source lane contains a +ve value.
5148 */
5149
5150 void C2_MacroAssembler::vector_castF2X_avx(BasicType to_elem_bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
5151 XMMRegister xtmp2, XMMRegister xtmp3, XMMRegister xtmp4,
5152 AddressLiteral float_sign_flip, Register rscratch, int vec_enc) {
5153 int to_elem_sz = type2aelembytes(to_elem_bt);
5154 assert(to_elem_sz <= 4, "");
5155 vcvttps2dq(dst, src, vec_enc);
5156 vector_cast_float_to_int_special_cases_avx(dst, src, xtmp1, xtmp2, xtmp3, xtmp4, rscratch, float_sign_flip, vec_enc);
5157 if (to_elem_sz < 4) {
5158 vpxor(xtmp4, xtmp4, xtmp4, vec_enc);
5159 vector_cast_int_to_subword(to_elem_bt, dst, xtmp4, xtmp3, rscratch, vec_enc);
5160 }
5161 }
5162
5163 void C2_MacroAssembler::vector_castF2X_evex(BasicType to_elem_bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
5164 XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2, AddressLiteral float_sign_flip,
5165 Register rscratch, int vec_enc) {
5166 int to_elem_sz = type2aelembytes(to_elem_bt);
5167 assert(to_elem_sz <= 4, "");
5168 vcvttps2dq(dst, src, vec_enc);
5169 vector_cast_float_to_int_special_cases_evex(dst, src, xtmp1, xtmp2, ktmp1, ktmp2, rscratch, float_sign_flip, vec_enc);
5170 switch(to_elem_bt) {
5171 case T_INT:
5172 break;
5173 case T_SHORT:
5174 evpmovdw(dst, dst, vec_enc);
5175 break;
5176 case T_BYTE:
5177 evpmovdb(dst, dst, vec_enc);
5178 break;
5179 default: assert(false, "Unexpected basic type for target of vector castF2X EVEX: %s", type2name(to_elem_bt));
5180 }
5181 }
5182
5183 void C2_MacroAssembler::vector_castF2L_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
5184 KRegister ktmp1, KRegister ktmp2, AddressLiteral double_sign_flip,
5185 Register rscratch, int vec_enc) {
5186 evcvttps2qq(dst, src, vec_enc);
5187 vector_cast_float_to_long_special_cases_evex(dst, src, xtmp1, xtmp2, ktmp1, ktmp2, rscratch, double_sign_flip, vec_enc);
5188 }
5189
5190 // Handling for downcasting from double to integer or sub-word types on AVX2.
5191 void C2_MacroAssembler::vector_castD2X_avx(BasicType to_elem_bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
5192 XMMRegister xtmp2, XMMRegister xtmp3, XMMRegister xtmp4, XMMRegister xtmp5,
5193 AddressLiteral float_sign_flip, Register rscratch, int vec_enc) {
5194 int to_elem_sz = type2aelembytes(to_elem_bt);
5195 assert(to_elem_sz < 8, "");
5196 vcvttpd2dq(dst, src, vec_enc);
5197 vector_cast_double_to_int_special_cases_avx(dst, src, xtmp1, xtmp2, xtmp3, xtmp4, xtmp5, rscratch,
5198 float_sign_flip, vec_enc);
5199 if (to_elem_sz < 4) {
5200 // xtmp4 holds all zero lanes.
5201 vector_cast_int_to_subword(to_elem_bt, dst, xtmp4, xtmp5, rscratch, Assembler::AVX_128bit);
5202 }
5203 }
5204
5205 void C2_MacroAssembler::vector_castD2X_evex(BasicType to_elem_bt, XMMRegister dst, XMMRegister src,
5206 XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp1,
5207 KRegister ktmp2, AddressLiteral sign_flip,
5208 Register rscratch, int vec_enc) {
5209 if (VM_Version::supports_avx512dq()) {
5210 evcvttpd2qq(dst, src, vec_enc);
5211 vector_cast_double_to_long_special_cases_evex(dst, src, xtmp1, xtmp2, ktmp1, ktmp2, rscratch, sign_flip, vec_enc);
5212 switch(to_elem_bt) {
5213 case T_LONG:
5214 break;
5215 case T_INT:
5216 evpmovsqd(dst, dst, vec_enc);
5217 break;
5218 case T_SHORT:
5219 evpmovsqd(dst, dst, vec_enc);
5220 evpmovdw(dst, dst, vec_enc);
5221 break;
5222 case T_BYTE:
5223 evpmovsqd(dst, dst, vec_enc);
5224 evpmovdb(dst, dst, vec_enc);
5225 break;
5226 default: assert(false, "Unexpected basic type for target of vector castD2X AVX512DQ EVEX: %s", type2name(to_elem_bt));
5227 }
5228 } else {
5229 assert(type2aelembytes(to_elem_bt) <= 4, "");
5230 vcvttpd2dq(dst, src, vec_enc);
5231 vector_cast_double_to_int_special_cases_evex(dst, src, xtmp1, xtmp2, ktmp1, ktmp2, rscratch, sign_flip, vec_enc);
5232 switch(to_elem_bt) {
5233 case T_INT:
5234 break;
5235 case T_SHORT:
5236 evpmovdw(dst, dst, vec_enc);
5237 break;
5238 case T_BYTE:
5239 evpmovdb(dst, dst, vec_enc);
5240 break;
5241 default: assert(false, "Unexpected basic type for target of vector castD2X EVEX: %s", type2name(to_elem_bt));
5242 }
5243 }
5244 }
5245
5246 void C2_MacroAssembler::vector_castF2X_avx10_2(BasicType to_elem_bt, XMMRegister dst, XMMRegister src, int vec_enc) {
5247 switch(to_elem_bt) {
5248 case T_LONG:
5249 evcvttps2qqs(dst, src, vec_enc);
5250 break;
5251 case T_INT:
5252 evcvttps2dqs(dst, src, vec_enc);
5253 break;
5254 case T_SHORT:
5255 evcvttps2dqs(dst, src, vec_enc);
5256 evpmovdw(dst, dst, vec_enc);
5257 break;
5258 case T_BYTE:
5259 evcvttps2dqs(dst, src, vec_enc);
5260 evpmovdb(dst, dst, vec_enc);
5261 break;
5262 default: assert(false, "Unexpected basic type for target of vector castF2X AVX10 (reg src): %s", type2name(to_elem_bt));
5263 }
5264 }
5265
5266 void C2_MacroAssembler::vector_castF2X_avx10_2(BasicType to_elem_bt, XMMRegister dst, Address src, int vec_enc) {
5267 switch(to_elem_bt) {
5268 case T_LONG:
5269 evcvttps2qqs(dst, src, vec_enc);
5270 break;
5271 case T_INT:
5272 evcvttps2dqs(dst, src, vec_enc);
5273 break;
5274 case T_SHORT:
5275 evcvttps2dqs(dst, src, vec_enc);
5276 evpmovdw(dst, dst, vec_enc);
5277 break;
5278 case T_BYTE:
5279 evcvttps2dqs(dst, src, vec_enc);
5280 evpmovdb(dst, dst, vec_enc);
5281 break;
5282 default: assert(false, "Unexpected basic type for target of vector castF2X AVX10 (mem src): %s", type2name(to_elem_bt));
5283 }
5284 }
5285
5286 void C2_MacroAssembler::vector_castD2X_avx10_2(BasicType to_elem_bt, XMMRegister dst, XMMRegister src, int vec_enc) {
5287 switch(to_elem_bt) {
5288 case T_LONG:
5289 evcvttpd2qqs(dst, src, vec_enc);
5290 break;
5291 case T_INT:
5292 evcvttpd2dqs(dst, src, vec_enc);
5293 break;
5294 case T_SHORT:
5295 evcvttpd2dqs(dst, src, vec_enc);
5296 evpmovdw(dst, dst, vec_enc);
5297 break;
5298 case T_BYTE:
5299 evcvttpd2dqs(dst, src, vec_enc);
5300 evpmovdb(dst, dst, vec_enc);
5301 break;
5302 default: assert(false, "Unexpected basic type for target of vector castD2X AVX10 (reg src): %s", type2name(to_elem_bt));
5303 }
5304 }
5305
5306 void C2_MacroAssembler::vector_castD2X_avx10_2(BasicType to_elem_bt, XMMRegister dst, Address src, int vec_enc) {
5307 switch(to_elem_bt) {
5308 case T_LONG:
5309 evcvttpd2qqs(dst, src, vec_enc);
5310 break;
5311 case T_INT:
5312 evcvttpd2dqs(dst, src, vec_enc);
5313 break;
5314 case T_SHORT:
5315 evcvttpd2dqs(dst, src, vec_enc);
5316 evpmovdw(dst, dst, vec_enc);
5317 break;
5318 case T_BYTE:
5319 evcvttpd2dqs(dst, src, vec_enc);
5320 evpmovdb(dst, dst, vec_enc);
5321 break;
5322 default: assert(false, "Unexpected basic type for target of vector castD2X AVX10 (mem src): %s", type2name(to_elem_bt));
5323 }
5324 }
5325
5326 void C2_MacroAssembler::vector_round_double_evex(XMMRegister dst, XMMRegister src,
5327 AddressLiteral double_sign_flip, AddressLiteral new_mxcsr, int vec_enc,
5328 Register tmp, XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2) {
5329 // Perform floor(val+0.5) operation under the influence of MXCSR.RC mode roundTowards -inf.
5330 // and re-instantiate original MXCSR.RC mode after that.
5331 ldmxcsr(new_mxcsr, tmp /*rscratch*/);
5332
5333 mov64(tmp, julong_cast(0.5L));
5334 evpbroadcastq(xtmp1, tmp, vec_enc);
5335 vaddpd(xtmp1, src , xtmp1, vec_enc);
5336 evcvtpd2qq(dst, xtmp1, vec_enc);
5337 vector_cast_double_to_long_special_cases_evex(dst, src, xtmp1, xtmp2, ktmp1, ktmp2, tmp /*rscratch*/,
5338 double_sign_flip, vec_enc);;
5339
5340 ldmxcsr(ExternalAddress(StubRoutines::x86::addr_mxcsr_std()), tmp /*rscratch*/);
5341 }
5342
5343 void C2_MacroAssembler::vector_round_float_evex(XMMRegister dst, XMMRegister src,
5344 AddressLiteral float_sign_flip, AddressLiteral new_mxcsr, int vec_enc,
5345 Register tmp, XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2) {
5346 // Perform floor(val+0.5) operation under the influence of MXCSR.RC mode roundTowards -inf.
5347 // and re-instantiate original MXCSR.RC mode after that.
5348 ldmxcsr(new_mxcsr, tmp /*rscratch*/);
5349
5350 movl(tmp, jint_cast(0.5));
5351 movq(xtmp1, tmp);
5352 vbroadcastss(xtmp1, xtmp1, vec_enc);
5353 vaddps(xtmp1, src , xtmp1, vec_enc);
5354 vcvtps2dq(dst, xtmp1, vec_enc);
5355 vector_cast_float_to_int_special_cases_evex(dst, src, xtmp1, xtmp2, ktmp1, ktmp2, tmp /*rscratch*/,
5356 float_sign_flip, vec_enc);
5357
5358 ldmxcsr(ExternalAddress(StubRoutines::x86::addr_mxcsr_std()), tmp /*rscratch*/);
5359 }
5360
5361 void C2_MacroAssembler::vector_round_float_avx(XMMRegister dst, XMMRegister src,
5362 AddressLiteral float_sign_flip, AddressLiteral new_mxcsr, int vec_enc,
5363 Register tmp, XMMRegister xtmp1, XMMRegister xtmp2, XMMRegister xtmp3, XMMRegister xtmp4) {
5364 // Perform floor(val+0.5) operation under the influence of MXCSR.RC mode roundTowards -inf.
5365 // and re-instantiate original MXCSR.RC mode after that.
5366 ldmxcsr(new_mxcsr, tmp /*rscratch*/);
5367
5368 movl(tmp, jint_cast(0.5));
5369 movq(xtmp1, tmp);
5370 vbroadcastss(xtmp1, xtmp1, vec_enc);
5371 vaddps(xtmp1, src , xtmp1, vec_enc);
5372 vcvtps2dq(dst, xtmp1, vec_enc);
5373 vector_cast_float_to_int_special_cases_avx(dst, src, xtmp1, xtmp2, xtmp3, xtmp4, tmp /*rscratch*/, float_sign_flip, vec_enc);
5374
5375 ldmxcsr(ExternalAddress(StubRoutines::x86::addr_mxcsr_std()), tmp /*rscratch*/);
5376 }
5377
5378 void C2_MacroAssembler::vector_unsigned_cast(XMMRegister dst, XMMRegister src, int vlen_enc,
5379 BasicType from_elem_bt, BasicType to_elem_bt) {
5380 switch (from_elem_bt) {
5381 case T_BYTE:
5382 switch (to_elem_bt) {
5383 case T_SHORT: vpmovzxbw(dst, src, vlen_enc); break;
5384 case T_INT: vpmovzxbd(dst, src, vlen_enc); break;
5385 case T_LONG: vpmovzxbq(dst, src, vlen_enc); break;
5386 default: ShouldNotReachHere();
5387 }
5388 break;
5389 case T_SHORT:
5390 switch (to_elem_bt) {
5391 case T_INT: vpmovzxwd(dst, src, vlen_enc); break;
5392 case T_LONG: vpmovzxwq(dst, src, vlen_enc); break;
5393 default: ShouldNotReachHere();
5394 }
5395 break;
5396 case T_INT:
5397 assert(to_elem_bt == T_LONG, "");
5398 vpmovzxdq(dst, src, vlen_enc);
5399 break;
5400 default:
5401 ShouldNotReachHere();
5402 }
5403 }
5404
5405 void C2_MacroAssembler::vector_signed_cast(XMMRegister dst, XMMRegister src, int vlen_enc,
5406 BasicType from_elem_bt, BasicType to_elem_bt) {
5407 switch (from_elem_bt) {
5408 case T_BYTE:
5409 switch (to_elem_bt) {
5410 case T_SHORT: vpmovsxbw(dst, src, vlen_enc); break;
5411 case T_INT: vpmovsxbd(dst, src, vlen_enc); break;
5412 case T_LONG: vpmovsxbq(dst, src, vlen_enc); break;
5413 default: ShouldNotReachHere();
5414 }
5415 break;
5416 case T_SHORT:
5417 switch (to_elem_bt) {
5418 case T_INT: vpmovsxwd(dst, src, vlen_enc); break;
5419 case T_LONG: vpmovsxwq(dst, src, vlen_enc); break;
5420 default: ShouldNotReachHere();
5421 }
5422 break;
5423 case T_INT:
5424 assert(to_elem_bt == T_LONG, "");
5425 vpmovsxdq(dst, src, vlen_enc);
5426 break;
5427 default:
5428 ShouldNotReachHere();
5429 }
5430 }
5431
5432 void C2_MacroAssembler::vector_mask_cast(XMMRegister dst, XMMRegister src,
5433 BasicType dst_bt, BasicType src_bt, int vlen) {
5434 int vlen_enc = vector_length_encoding(MAX2(type2aelembytes(src_bt), type2aelembytes(dst_bt)) * vlen);
5435 assert(vlen_enc != AVX_512bit, "");
5436
5437 int dst_bt_size = type2aelembytes(dst_bt);
5438 int src_bt_size = type2aelembytes(src_bt);
5439 if (dst_bt_size > src_bt_size) {
5440 switch (dst_bt_size / src_bt_size) {
5441 case 2: vpmovsxbw(dst, src, vlen_enc); break;
5442 case 4: vpmovsxbd(dst, src, vlen_enc); break;
5443 case 8: vpmovsxbq(dst, src, vlen_enc); break;
5444 default: ShouldNotReachHere();
5445 }
5446 } else {
5447 assert(dst_bt_size < src_bt_size, "");
5448 switch (src_bt_size / dst_bt_size) {
5449 case 2: {
5450 if (vlen_enc == AVX_128bit) {
5451 vpacksswb(dst, src, src, vlen_enc);
5452 } else {
5453 vpacksswb(dst, src, src, vlen_enc);
5454 vpermq(dst, dst, 0x08, vlen_enc);
5455 }
5456 break;
5457 }
5458 case 4: {
5459 if (vlen_enc == AVX_128bit) {
5460 vpackssdw(dst, src, src, vlen_enc);
5461 vpacksswb(dst, dst, dst, vlen_enc);
5462 } else {
5463 vpackssdw(dst, src, src, vlen_enc);
5464 vpermq(dst, dst, 0x08, vlen_enc);
5465 vpacksswb(dst, dst, dst, AVX_128bit);
5466 }
5467 break;
5468 }
5469 case 8: {
5470 if (vlen_enc == AVX_128bit) {
5471 vpshufd(dst, src, 0x08, vlen_enc);
5472 vpackssdw(dst, dst, dst, vlen_enc);
5473 vpacksswb(dst, dst, dst, vlen_enc);
5474 } else {
5475 vpshufd(dst, src, 0x08, vlen_enc);
5476 vpermq(dst, dst, 0x08, vlen_enc);
5477 vpackssdw(dst, dst, dst, AVX_128bit);
5478 vpacksswb(dst, dst, dst, AVX_128bit);
5479 }
5480 break;
5481 }
5482 default: ShouldNotReachHere();
5483 }
5484 }
5485 }
5486
5487 void C2_MacroAssembler::evpternlog(XMMRegister dst, int func, KRegister mask, XMMRegister src2, XMMRegister src3,
5488 bool merge, BasicType bt, int vlen_enc) {
5489 if (bt == T_INT) {
5490 evpternlogd(dst, func, mask, src2, src3, merge, vlen_enc);
5491 } else {
5492 assert(bt == T_LONG, "");
5493 evpternlogq(dst, func, mask, src2, src3, merge, vlen_enc);
5494 }
5495 }
5496
5497 void C2_MacroAssembler::evpternlog(XMMRegister dst, int func, KRegister mask, XMMRegister src2, Address src3,
5498 bool merge, BasicType bt, int vlen_enc) {
5499 if (bt == T_INT) {
5500 evpternlogd(dst, func, mask, src2, src3, merge, vlen_enc);
5501 } else {
5502 assert(bt == T_LONG, "");
5503 evpternlogq(dst, func, mask, src2, src3, merge, vlen_enc);
5504 }
5505 }
5506
5507 void C2_MacroAssembler::vector_long_to_maskvec(XMMRegister dst, Register src, Register rtmp1,
5508 Register rtmp2, XMMRegister xtmp, int mask_len,
5509 int vec_enc) {
5510 int index = 0;
5511 int vindex = 0;
5512 mov64(rtmp1, 0x0101010101010101L);
5513 pdepq(rtmp1, src, rtmp1);
5514 if (mask_len > 8) {
5515 movq(rtmp2, src);
5516 vpxor(xtmp, xtmp, xtmp, vec_enc);
5517 movq(xtmp, rtmp1);
5518 }
5519 movq(dst, rtmp1);
5520
5521 mask_len -= 8;
5522 while (mask_len > 0) {
5523 assert ((mask_len & 0x7) == 0, "mask must be multiple of 8");
5524 index++;
5525 if ((index % 2) == 0) {
5526 pxor(xtmp, xtmp);
5527 }
5528 mov64(rtmp1, 0x0101010101010101L);
5529 shrq(rtmp2, 8);
5530 pdepq(rtmp1, rtmp2, rtmp1);
5531 pinsrq(xtmp, rtmp1, index % 2);
5532 vindex = index / 2;
5533 if (vindex) {
5534 // Write entire 16 byte vector when both 64 bit
5535 // lanes are update to save redundant instructions.
5536 if (index % 2) {
5537 vinsertf128(dst, dst, xtmp, vindex);
5538 }
5539 } else {
5540 vmovdqu(dst, xtmp);
5541 }
5542 mask_len -= 8;
5543 }
5544 }
5545
5546 void C2_MacroAssembler::vector_mask_operation_helper(int opc, Register dst, Register tmp, int masklen) {
5547 switch(opc) {
5548 case Op_VectorMaskTrueCount:
5549 popcntq(dst, tmp);
5550 break;
5551 case Op_VectorMaskLastTrue:
5552 if (VM_Version::supports_lzcnt()) {
5553 lzcntq(tmp, tmp);
5554 movl(dst, 63);
5555 subl(dst, tmp);
5556 } else {
5557 movl(dst, -1);
5558 bsrq(tmp, tmp);
5559 cmov32(Assembler::notZero, dst, tmp);
5560 }
5561 break;
5562 case Op_VectorMaskFirstTrue:
5563 if (UseCountTrailingZerosInstruction) {
5564 if (masklen < 32) {
5565 orl(tmp, 1 << masklen);
5566 tzcntl(dst, tmp);
5567 } else if (masklen == 32) {
5568 tzcntl(dst, tmp);
5569 } else {
5570 assert(masklen == 64, "");
5571 tzcntq(dst, tmp);
5572 }
5573 } else {
5574 if (masklen < 32) {
5575 orl(tmp, 1 << masklen);
5576 bsfl(dst, tmp);
5577 } else {
5578 assert(masklen == 32 || masklen == 64, "");
5579 movl(dst, masklen);
5580 if (masklen == 32) {
5581 bsfl(tmp, tmp);
5582 } else {
5583 bsfq(tmp, tmp);
5584 }
5585 cmov32(Assembler::notZero, dst, tmp);
5586 }
5587 }
5588 break;
5589 case Op_VectorMaskToLong:
5590 assert(dst == tmp, "Dst and tmp should be the same for toLong operations");
5591 break;
5592 default: assert(false, "Unhandled mask operation");
5593 }
5594 }
5595
5596 void C2_MacroAssembler::vector_mask_operation(int opc, Register dst, KRegister mask, Register tmp,
5597 int masklen, int masksize, int vec_enc) {
5598 assert(VM_Version::supports_popcnt(), "");
5599
5600 if(VM_Version::supports_avx512bw()) {
5601 kmovql(tmp, mask);
5602 } else {
5603 assert(masklen <= 16, "");
5604 kmovwl(tmp, mask);
5605 }
5606
5607 // Mask generated out of partial vector comparisons/replicate/mask manipulation
5608 // operations needs to be clipped.
5609 if (masksize < 16 && opc != Op_VectorMaskFirstTrue) {
5610 andq(tmp, (1 << masklen) - 1);
5611 }
5612
5613 vector_mask_operation_helper(opc, dst, tmp, masklen);
5614 }
5615
5616 void C2_MacroAssembler::vector_mask_operation(int opc, Register dst, XMMRegister mask, XMMRegister xtmp,
5617 Register tmp, int masklen, BasicType bt, int vec_enc) {
5618 assert((vec_enc == AVX_128bit && VM_Version::supports_avx()) ||
5619 (vec_enc == AVX_256bit && (VM_Version::supports_avx2() || type2aelembytes(bt) >= 4)), "");
5620 assert(VM_Version::supports_popcnt(), "");
5621
5622 bool need_clip = false;
5623 switch(bt) {
5624 case T_BOOLEAN:
5625 // While masks of other types contain 0, -1; boolean masks contain lane values of 0, 1
5626 vpxor(xtmp, xtmp, xtmp, vec_enc);
5627 vpsubb(xtmp, xtmp, mask, vec_enc);
5628 vpmovmskb(tmp, xtmp, vec_enc);
5629 need_clip = masklen < 16;
5630 break;
5631 case T_BYTE:
5632 vpmovmskb(tmp, mask, vec_enc);
5633 need_clip = masklen < 16;
5634 break;
5635 case T_SHORT:
5636 vpacksswb(xtmp, mask, mask, vec_enc);
5637 if (masklen >= 16) {
5638 vpermpd(xtmp, xtmp, 8, vec_enc);
5639 }
5640 vpmovmskb(tmp, xtmp, Assembler::AVX_128bit);
5641 need_clip = masklen < 16;
5642 break;
5643 case T_INT:
5644 case T_FLOAT:
5645 vmovmskps(tmp, mask, vec_enc);
5646 need_clip = masklen < 4;
5647 break;
5648 case T_LONG:
5649 case T_DOUBLE:
5650 vmovmskpd(tmp, mask, vec_enc);
5651 need_clip = masklen < 2;
5652 break;
5653 default: assert(false, "Unhandled type, %s", type2name(bt));
5654 }
5655
5656 // Mask generated out of partial vector comparisons/replicate/mask manipulation
5657 // operations needs to be clipped.
5658 if (need_clip && opc != Op_VectorMaskFirstTrue) {
5659 // need_clip implies masklen < 32
5660 andq(tmp, (1 << masklen) - 1);
5661 }
5662
5663 vector_mask_operation_helper(opc, dst, tmp, masklen);
5664 }
5665
5666 void C2_MacroAssembler::vector_mask_compress(KRegister dst, KRegister src, Register rtmp1,
5667 Register rtmp2, int mask_len) {
5668 kmov(rtmp1, src);
5669 andq(rtmp1, (0xFFFFFFFFFFFFFFFFUL >> (64 - mask_len)));
5670 mov64(rtmp2, -1L);
5671 pextq(rtmp2, rtmp2, rtmp1);
5672 kmov(dst, rtmp2);
5673 }
5674
5675 void C2_MacroAssembler::vector_compress_expand_avx2(int opcode, XMMRegister dst, XMMRegister src,
5676 XMMRegister mask, Register rtmp, Register rscratch,
5677 XMMRegister permv, XMMRegister xtmp, BasicType bt,
5678 int vec_enc) {
5679 assert(type2aelembytes(bt) >= 4, "");
5680 assert(opcode == Op_CompressV || opcode == Op_ExpandV, "");
5681 address compress_perm_table = nullptr;
5682 address expand_perm_table = nullptr;
5683 if (type2aelembytes(bt) == 8) {
5684 compress_perm_table = StubRoutines::x86::compress_perm_table64();
5685 expand_perm_table = StubRoutines::x86::expand_perm_table64();
5686 vmovmskpd(rtmp, mask, vec_enc);
5687 } else {
5688 compress_perm_table = StubRoutines::x86::compress_perm_table32();
5689 expand_perm_table = StubRoutines::x86::expand_perm_table32();
5690 vmovmskps(rtmp, mask, vec_enc);
5691 }
5692 shlq(rtmp, 5); // for 32 byte permute row.
5693 if (opcode == Op_CompressV) {
5694 lea(rscratch, ExternalAddress(compress_perm_table));
5695 } else {
5696 lea(rscratch, ExternalAddress(expand_perm_table));
5697 }
5698 addptr(rtmp, rscratch);
5699 vmovdqu(permv, Address(rtmp));
5700 vpermps(dst, permv, src, Assembler::AVX_256bit);
5701 vpxor(xtmp, xtmp, xtmp, vec_enc);
5702 // Blend the result with zero vector using permute mask, each column entry
5703 // in a permute table row contains either a valid permute index or a -1 (default)
5704 // value, this can potentially be used as a blending mask after
5705 // compressing/expanding the source vector lanes.
5706 vblendvps(dst, dst, xtmp, permv, vec_enc, true, permv);
5707 }
5708
5709 void C2_MacroAssembler::vector_compress_expand(int opcode, XMMRegister dst, XMMRegister src, KRegister mask,
5710 bool merge, BasicType bt, int vec_enc) {
5711 if (opcode == Op_CompressV) {
5712 switch(bt) {
5713 case T_BYTE:
5714 evpcompressb(dst, mask, src, merge, vec_enc);
5715 break;
5716 case T_CHAR:
5717 case T_SHORT:
5718 evpcompressw(dst, mask, src, merge, vec_enc);
5719 break;
5720 case T_INT:
5721 evpcompressd(dst, mask, src, merge, vec_enc);
5722 break;
5723 case T_FLOAT:
5724 evcompressps(dst, mask, src, merge, vec_enc);
5725 break;
5726 case T_LONG:
5727 evpcompressq(dst, mask, src, merge, vec_enc);
5728 break;
5729 case T_DOUBLE:
5730 evcompresspd(dst, mask, src, merge, vec_enc);
5731 break;
5732 default:
5733 fatal("Unsupported type %s", type2name(bt));
5734 break;
5735 }
5736 } else {
5737 assert(opcode == Op_ExpandV, "");
5738 switch(bt) {
5739 case T_BYTE:
5740 evpexpandb(dst, mask, src, merge, vec_enc);
5741 break;
5742 case T_CHAR:
5743 case T_SHORT:
5744 evpexpandw(dst, mask, src, merge, vec_enc);
5745 break;
5746 case T_INT:
5747 evpexpandd(dst, mask, src, merge, vec_enc);
5748 break;
5749 case T_FLOAT:
5750 evexpandps(dst, mask, src, merge, vec_enc);
5751 break;
5752 case T_LONG:
5753 evpexpandq(dst, mask, src, merge, vec_enc);
5754 break;
5755 case T_DOUBLE:
5756 evexpandpd(dst, mask, src, merge, vec_enc);
5757 break;
5758 default:
5759 fatal("Unsupported type %s", type2name(bt));
5760 break;
5761 }
5762 }
5763 }
5764
5765 void C2_MacroAssembler::vector_signum_evex(int opcode, XMMRegister dst, XMMRegister src, XMMRegister zero, XMMRegister one,
5766 KRegister ktmp1, int vec_enc) {
5767 if (opcode == Op_SignumVD) {
5768 vsubpd(dst, zero, one, vec_enc);
5769 // if src < 0 ? -1 : 1
5770 evcmppd(ktmp1, k0, src, zero, Assembler::LT_OQ, vec_enc);
5771 evblendmpd(dst, ktmp1, one, dst, true, vec_enc);
5772 // if src == NaN, -0.0 or 0.0 return src.
5773 evcmppd(ktmp1, k0, src, zero, Assembler::EQ_UQ, vec_enc);
5774 evblendmpd(dst, ktmp1, dst, src, true, vec_enc);
5775 } else {
5776 assert(opcode == Op_SignumVF, "");
5777 vsubps(dst, zero, one, vec_enc);
5778 // if src < 0 ? -1 : 1
5779 evcmpps(ktmp1, k0, src, zero, Assembler::LT_OQ, vec_enc);
5780 evblendmps(dst, ktmp1, one, dst, true, vec_enc);
5781 // if src == NaN, -0.0 or 0.0 return src.
5782 evcmpps(ktmp1, k0, src, zero, Assembler::EQ_UQ, vec_enc);
5783 evblendmps(dst, ktmp1, dst, src, true, vec_enc);
5784 }
5785 }
5786
5787 void C2_MacroAssembler::vector_signum_avx(int opcode, XMMRegister dst, XMMRegister src, XMMRegister zero, XMMRegister one,
5788 XMMRegister xtmp1, int vec_enc) {
5789 if (opcode == Op_SignumVD) {
5790 vsubpd(dst, zero, one, vec_enc);
5791 // if src < 0 ? -1 : 1
5792 vblendvpd(dst, one, dst, src, vec_enc, true, xtmp1);
5793 // if src == NaN, -0.0 or 0.0 return src.
5794 vcmppd(xtmp1, src, zero, Assembler::EQ_UQ, vec_enc);
5795 vblendvpd(dst, dst, src, xtmp1, vec_enc, false, xtmp1);
5796 } else {
5797 assert(opcode == Op_SignumVF, "");
5798 vsubps(dst, zero, one, vec_enc);
5799 // if src < 0 ? -1 : 1
5800 vblendvps(dst, one, dst, src, vec_enc, true, xtmp1);
5801 // if src == NaN, -0.0 or 0.0 return src.
5802 vcmpps(xtmp1, src, zero, Assembler::EQ_UQ, vec_enc);
5803 vblendvps(dst, dst, src, xtmp1, vec_enc, false, xtmp1);
5804 }
5805 }
5806
5807 void C2_MacroAssembler::vector_maskall_operation(KRegister dst, Register src, int mask_len) {
5808 if (VM_Version::supports_avx512bw()) {
5809 if (mask_len > 32) {
5810 kmovql(dst, src);
5811 } else {
5812 kmovdl(dst, src);
5813 if (mask_len != 32) {
5814 kshiftrdl(dst, dst, 32 - mask_len);
5815 }
5816 }
5817 } else {
5818 assert(mask_len <= 16, "");
5819 kmovwl(dst, src);
5820 if (mask_len != 16) {
5821 kshiftrwl(dst, dst, 16 - mask_len);
5822 }
5823 }
5824 }
5825
5826 void C2_MacroAssembler::vbroadcast(BasicType bt, XMMRegister dst, int imm32, Register rtmp, int vec_enc) {
5827 int lane_size = type2aelembytes(bt);
5828 if ((is_non_subword_integral_type(bt) && VM_Version::supports_avx512vl()) ||
5829 (is_subword_type(bt) && VM_Version::supports_avx512vlbw())) {
5830 movptr(rtmp, imm32);
5831 switch(lane_size) {
5832 case 1 : evpbroadcastb(dst, rtmp, vec_enc); break;
5833 case 2 : evpbroadcastw(dst, rtmp, vec_enc); break;
5834 case 4 : evpbroadcastd(dst, rtmp, vec_enc); break;
5835 case 8 : evpbroadcastq(dst, rtmp, vec_enc); break;
5836 fatal("Unsupported lane size %d", lane_size);
5837 break;
5838 }
5839 } else {
5840 movptr(rtmp, imm32);
5841 movq(dst, rtmp);
5842 switch(lane_size) {
5843 case 1 : vpbroadcastb(dst, dst, vec_enc); break;
5844 case 2 : vpbroadcastw(dst, dst, vec_enc); break;
5845 case 4 : vpbroadcastd(dst, dst, vec_enc); break;
5846 case 8 : vpbroadcastq(dst, dst, vec_enc); break;
5847 fatal("Unsupported lane size %d", lane_size);
5848 break;
5849 }
5850 }
5851 }
5852
5853 //
5854 // Following is lookup table based popcount computation algorithm:-
5855 // Index Bit set count
5856 // [ 0000 -> 0,
5857 // 0001 -> 1,
5858 // 0010 -> 1,
5859 // 0011 -> 2,
5860 // 0100 -> 1,
5861 // 0101 -> 2,
5862 // 0110 -> 2,
5863 // 0111 -> 3,
5864 // 1000 -> 1,
5865 // 1001 -> 2,
5866 // 1010 -> 3,
5867 // 1011 -> 3,
5868 // 1100 -> 2,
5869 // 1101 -> 3,
5870 // 1111 -> 4 ]
5871 // a. Count the number of 1s in 4 LSB bits of each byte. These bits are used as
5872 // shuffle indices for lookup table access.
5873 // b. Right shift each byte of vector lane by 4 positions.
5874 // c. Count the number of 1s in 4 MSB bits each byte. These bits are used as
5875 // shuffle indices for lookup table access.
5876 // d. Add the bitset count of upper and lower 4 bits of each byte.
5877 // e. Unpack double words to quad words and compute sum of absolute difference of bitset
5878 // count of all the bytes of a quadword.
5879 // f. Perform step e. for upper 128bit vector lane.
5880 // g. Pack the bitset count of quadwords back to double word.
5881 // h. Unpacking and packing operations are not needed for 64bit vector lane.
5882
5883 void C2_MacroAssembler::vector_popcount_byte(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
5884 XMMRegister xtmp2, Register rtmp, int vec_enc) {
5885 assert((vec_enc == Assembler::AVX_512bit && VM_Version::supports_avx512bw()) || VM_Version::supports_avx2(), "");
5886 vbroadcast(T_INT, xtmp1, 0x0F0F0F0F, rtmp, vec_enc);
5887 vpsrlw(dst, src, 4, vec_enc);
5888 vpand(dst, dst, xtmp1, vec_enc);
5889 vpand(xtmp1, src, xtmp1, vec_enc);
5890 vmovdqu(xtmp2, ExternalAddress(StubRoutines::x86::vector_popcount_lut()), vec_enc, noreg);
5891 vpshufb(xtmp1, xtmp2, xtmp1, vec_enc);
5892 vpshufb(dst, xtmp2, dst, vec_enc);
5893 vpaddb(dst, dst, xtmp1, vec_enc);
5894 }
5895
5896 void C2_MacroAssembler::vector_popcount_int(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
5897 XMMRegister xtmp2, Register rtmp, int vec_enc) {
5898 vector_popcount_byte(xtmp1, src, dst, xtmp2, rtmp, vec_enc);
5899 // Following code is as per steps e,f,g and h of above algorithm.
5900 vpxor(xtmp2, xtmp2, xtmp2, vec_enc);
5901 vpunpckhdq(dst, xtmp1, xtmp2, vec_enc);
5902 vpsadbw(dst, dst, xtmp2, vec_enc);
5903 vpunpckldq(xtmp1, xtmp1, xtmp2, vec_enc);
5904 vpsadbw(xtmp1, xtmp1, xtmp2, vec_enc);
5905 vpackuswb(dst, xtmp1, dst, vec_enc);
5906 }
5907
5908 void C2_MacroAssembler::vector_popcount_short(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
5909 XMMRegister xtmp2, Register rtmp, int vec_enc) {
5910 vector_popcount_byte(xtmp1, src, dst, xtmp2, rtmp, vec_enc);
5911 // Add the popcount of upper and lower bytes of word.
5912 vbroadcast(T_INT, xtmp2, 0x00FF00FF, rtmp, vec_enc);
5913 vpsrlw(dst, xtmp1, 8, vec_enc);
5914 vpand(xtmp1, xtmp1, xtmp2, vec_enc);
5915 vpaddw(dst, dst, xtmp1, vec_enc);
5916 }
5917
5918 void C2_MacroAssembler::vector_popcount_long(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
5919 XMMRegister xtmp2, Register rtmp, int vec_enc) {
5920 vector_popcount_byte(xtmp1, src, dst, xtmp2, rtmp, vec_enc);
5921 vpxor(xtmp2, xtmp2, xtmp2, vec_enc);
5922 vpsadbw(dst, xtmp1, xtmp2, vec_enc);
5923 }
5924
5925 void C2_MacroAssembler::vector_popcount_integral(BasicType bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
5926 XMMRegister xtmp2, Register rtmp, int vec_enc) {
5927 switch(bt) {
5928 case T_LONG:
5929 vector_popcount_long(dst, src, xtmp1, xtmp2, rtmp, vec_enc);
5930 break;
5931 case T_INT:
5932 vector_popcount_int(dst, src, xtmp1, xtmp2, rtmp, vec_enc);
5933 break;
5934 case T_CHAR:
5935 case T_SHORT:
5936 vector_popcount_short(dst, src, xtmp1, xtmp2, rtmp, vec_enc);
5937 break;
5938 case T_BYTE:
5939 case T_BOOLEAN:
5940 vector_popcount_byte(dst, src, xtmp1, xtmp2, rtmp, vec_enc);
5941 break;
5942 default:
5943 fatal("Unsupported type %s", type2name(bt));
5944 break;
5945 }
5946 }
5947
5948 void C2_MacroAssembler::vector_popcount_integral_evex(BasicType bt, XMMRegister dst, XMMRegister src,
5949 KRegister mask, bool merge, int vec_enc) {
5950 assert(VM_Version::supports_avx512vl() || vec_enc == Assembler::AVX_512bit, "");
5951 switch(bt) {
5952 case T_LONG:
5953 assert(VM_Version::supports_avx512_vpopcntdq(), "");
5954 evpopcntq(dst, mask, src, merge, vec_enc);
5955 break;
5956 case T_INT:
5957 assert(VM_Version::supports_avx512_vpopcntdq(), "");
5958 evpopcntd(dst, mask, src, merge, vec_enc);
5959 break;
5960 case T_CHAR:
5961 case T_SHORT:
5962 assert(VM_Version::supports_avx512_bitalg(), "");
5963 evpopcntw(dst, mask, src, merge, vec_enc);
5964 break;
5965 case T_BYTE:
5966 case T_BOOLEAN:
5967 assert(VM_Version::supports_avx512_bitalg(), "");
5968 evpopcntb(dst, mask, src, merge, vec_enc);
5969 break;
5970 default:
5971 fatal("Unsupported type %s", type2name(bt));
5972 break;
5973 }
5974 }
5975
5976 // Bit reversal algorithm first reverses the bits of each byte followed by
5977 // a byte level reversal for multi-byte primitive types (short/int/long).
5978 // Algorithm performs a lookup table access to get reverse bit sequence
5979 // corresponding to a 4 bit value. Thus a reverse bit sequence for a byte
5980 // is obtained by swapping the reverse bit sequences of upper and lower
5981 // nibble of a byte.
5982 void C2_MacroAssembler::vector_reverse_bit(BasicType bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
5983 XMMRegister xtmp2, Register rtmp, int vec_enc) {
5984 if (VM_Version::supports_avx512vlbw()) {
5985
5986 // Get the reverse bit sequence of lower nibble of each byte.
5987 vmovdqu(xtmp1, ExternalAddress(StubRoutines::x86::vector_reverse_bit_lut()), vec_enc, noreg);
5988 vbroadcast(T_INT, xtmp2, 0x0F0F0F0F, rtmp, vec_enc);
5989 evpandq(dst, xtmp2, src, vec_enc);
5990 vpshufb(dst, xtmp1, dst, vec_enc);
5991 vpsllq(dst, dst, 4, vec_enc);
5992
5993 // Get the reverse bit sequence of upper nibble of each byte.
5994 vpandn(xtmp2, xtmp2, src, vec_enc);
5995 vpsrlq(xtmp2, xtmp2, 4, vec_enc);
5996 vpshufb(xtmp2, xtmp1, xtmp2, vec_enc);
5997
5998 // Perform logical OR operation b/w left shifted reverse bit sequence of lower nibble and
5999 // right shifted reverse bit sequence of upper nibble to obtain the reverse bit sequence of each byte.
6000 evporq(xtmp2, dst, xtmp2, vec_enc);
6001 vector_reverse_byte(bt, dst, xtmp2, vec_enc);
6002
6003 } else if(vec_enc == Assembler::AVX_512bit) {
6004 // Shift based bit reversal.
6005 assert(bt == T_LONG || bt == T_INT, "");
6006
6007 // Swap lower and upper nibble of each byte.
6008 vector_swap_nbits(4, 0x0F0F0F0F, xtmp1, src, xtmp2, rtmp, vec_enc);
6009
6010 // Swap two least and most significant bits of each nibble.
6011 vector_swap_nbits(2, 0x33333333, dst, xtmp1, xtmp2, rtmp, vec_enc);
6012
6013 // Swap adjacent pair of bits.
6014 evmovdqul(xtmp1, k0, dst, true, vec_enc);
6015 vector_swap_nbits(1, 0x55555555, dst, xtmp1, xtmp2, rtmp, vec_enc);
6016
6017 evmovdqul(xtmp1, k0, dst, true, vec_enc);
6018 vector_reverse_byte64(bt, dst, xtmp1, xtmp1, xtmp2, rtmp, vec_enc);
6019 } else {
6020 vmovdqu(xtmp1, ExternalAddress(StubRoutines::x86::vector_reverse_bit_lut()), vec_enc, rtmp);
6021 vbroadcast(T_INT, xtmp2, 0x0F0F0F0F, rtmp, vec_enc);
6022
6023 // Get the reverse bit sequence of lower nibble of each byte.
6024 vpand(dst, xtmp2, src, vec_enc);
6025 vpshufb(dst, xtmp1, dst, vec_enc);
6026 vpsllq(dst, dst, 4, vec_enc);
6027
6028 // Get the reverse bit sequence of upper nibble of each byte.
6029 vpandn(xtmp2, xtmp2, src, vec_enc);
6030 vpsrlq(xtmp2, xtmp2, 4, vec_enc);
6031 vpshufb(xtmp2, xtmp1, xtmp2, vec_enc);
6032
6033 // Perform logical OR operation b/w left shifted reverse bit sequence of lower nibble and
6034 // right shifted reverse bit sequence of upper nibble to obtain the reverse bit sequence of each byte.
6035 vpor(xtmp2, dst, xtmp2, vec_enc);
6036 vector_reverse_byte(bt, dst, xtmp2, vec_enc);
6037 }
6038 }
6039
6040 void C2_MacroAssembler::vector_reverse_bit_gfni(BasicType bt, XMMRegister dst, XMMRegister src, AddressLiteral mask, int vec_enc,
6041 XMMRegister xtmp, Register rscratch) {
6042 assert(VM_Version::supports_gfni(), "");
6043 assert(rscratch != noreg || always_reachable(mask), "missing");
6044
6045 // Galois field instruction based bit reversal based on following algorithm.
6046 // http://0x80.pl/articles/avx512-galois-field-for-bit-shuffling.html
6047 vpbroadcastq(xtmp, mask, vec_enc, rscratch);
6048 vgf2p8affineqb(xtmp, src, xtmp, 0, vec_enc);
6049 vector_reverse_byte(bt, dst, xtmp, vec_enc);
6050 }
6051
6052 void C2_MacroAssembler::vector_swap_nbits(int nbits, int bitmask, XMMRegister dst, XMMRegister src,
6053 XMMRegister xtmp1, Register rtmp, int vec_enc) {
6054 vbroadcast(T_INT, xtmp1, bitmask, rtmp, vec_enc);
6055 evpandq(dst, xtmp1, src, vec_enc);
6056 vpsllq(dst, dst, nbits, vec_enc);
6057 vpandn(xtmp1, xtmp1, src, vec_enc);
6058 vpsrlq(xtmp1, xtmp1, nbits, vec_enc);
6059 evporq(dst, dst, xtmp1, vec_enc);
6060 }
6061
6062 void C2_MacroAssembler::vector_reverse_byte64(BasicType bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
6063 XMMRegister xtmp2, Register rtmp, int vec_enc) {
6064 // Shift based bit reversal.
6065 assert(VM_Version::supports_evex(), "");
6066 switch(bt) {
6067 case T_LONG:
6068 // Swap upper and lower double word of each quad word.
6069 evprorq(xtmp1, k0, src, 32, true, vec_enc);
6070 evprord(xtmp1, k0, xtmp1, 16, true, vec_enc);
6071 vector_swap_nbits(8, 0x00FF00FF, dst, xtmp1, xtmp2, rtmp, vec_enc);
6072 break;
6073 case T_INT:
6074 // Swap upper and lower word of each double word.
6075 evprord(xtmp1, k0, src, 16, true, vec_enc);
6076 vector_swap_nbits(8, 0x00FF00FF, dst, xtmp1, xtmp2, rtmp, vec_enc);
6077 break;
6078 case T_CHAR:
6079 case T_SHORT:
6080 // Swap upper and lower byte of each word.
6081 vector_swap_nbits(8, 0x00FF00FF, dst, src, xtmp2, rtmp, vec_enc);
6082 break;
6083 case T_BYTE:
6084 evmovdquq(dst, k0, src, true, vec_enc);
6085 break;
6086 default:
6087 fatal("Unsupported type %s", type2name(bt));
6088 break;
6089 }
6090 }
6091
6092 void C2_MacroAssembler::vector_reverse_byte(BasicType bt, XMMRegister dst, XMMRegister src, int vec_enc) {
6093 if (bt == T_BYTE) {
6094 if (VM_Version::supports_avx512vl() || vec_enc == Assembler::AVX_512bit) {
6095 evmovdquq(dst, k0, src, true, vec_enc);
6096 } else {
6097 vmovdqu(dst, src);
6098 }
6099 return;
6100 }
6101 // Perform byte reversal by shuffling the bytes of a multi-byte primitive type using
6102 // pre-computed shuffle indices.
6103 switch(bt) {
6104 case T_LONG:
6105 vmovdqu(dst, ExternalAddress(StubRoutines::x86::vector_reverse_byte_perm_mask_long()), vec_enc, noreg);
6106 break;
6107 case T_INT:
6108 vmovdqu(dst, ExternalAddress(StubRoutines::x86::vector_reverse_byte_perm_mask_int()), vec_enc, noreg);
6109 break;
6110 case T_CHAR:
6111 case T_SHORT:
6112 vmovdqu(dst, ExternalAddress(StubRoutines::x86::vector_reverse_byte_perm_mask_short()), vec_enc, noreg);
6113 break;
6114 default:
6115 fatal("Unsupported type %s", type2name(bt));
6116 break;
6117 }
6118 vpshufb(dst, src, dst, vec_enc);
6119 }
6120
6121 void C2_MacroAssembler::vector_count_leading_zeros_evex(BasicType bt, XMMRegister dst, XMMRegister src,
6122 XMMRegister xtmp1, XMMRegister xtmp2, XMMRegister xtmp3,
6123 KRegister ktmp, Register rtmp, bool merge, int vec_enc) {
6124 assert(is_integral_type(bt), "");
6125 assert(VM_Version::supports_avx512vl() || vec_enc == Assembler::AVX_512bit, "");
6126 assert(VM_Version::supports_avx512cd(), "");
6127 switch(bt) {
6128 case T_LONG:
6129 evplzcntq(dst, ktmp, src, merge, vec_enc);
6130 break;
6131 case T_INT:
6132 evplzcntd(dst, ktmp, src, merge, vec_enc);
6133 break;
6134 case T_SHORT:
6135 vpternlogd(xtmp1, 0xff, xtmp1, xtmp1, vec_enc);
6136 vpunpcklwd(xtmp2, xtmp1, src, vec_enc);
6137 evplzcntd(xtmp2, ktmp, xtmp2, merge, vec_enc);
6138 vpunpckhwd(dst, xtmp1, src, vec_enc);
6139 evplzcntd(dst, ktmp, dst, merge, vec_enc);
6140 vpackusdw(dst, xtmp2, dst, vec_enc);
6141 break;
6142 case T_BYTE:
6143 // T1 = Compute leading zero counts of 4 LSB bits of each byte by
6144 // accessing the lookup table.
6145 // T2 = Compute leading zero counts of 4 MSB bits of each byte by
6146 // accessing the lookup table.
6147 // Add T1 to T2 if 4 MSB bits of byte are all zeros.
6148 assert(VM_Version::supports_avx512bw(), "");
6149 evmovdquq(xtmp1, ExternalAddress(StubRoutines::x86::vector_count_leading_zeros_lut()), vec_enc, rtmp);
6150 vbroadcast(T_INT, dst, 0x0F0F0F0F, rtmp, vec_enc);
6151 vpand(xtmp2, dst, src, vec_enc);
6152 vpshufb(xtmp2, xtmp1, xtmp2, vec_enc);
6153 vpsrlw(xtmp3, src, 4, vec_enc);
6154 vpand(xtmp3, dst, xtmp3, vec_enc);
6155 vpshufb(dst, xtmp1, xtmp3, vec_enc);
6156 vpxor(xtmp1, xtmp1, xtmp1, vec_enc);
6157 evpcmpeqb(ktmp, xtmp1, xtmp3, vec_enc);
6158 evpaddb(dst, ktmp, dst, xtmp2, true, vec_enc);
6159 break;
6160 default:
6161 fatal("Unsupported type %s", type2name(bt));
6162 break;
6163 }
6164 }
6165
6166 void C2_MacroAssembler::vector_count_leading_zeros_byte_avx(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
6167 XMMRegister xtmp2, XMMRegister xtmp3, Register rtmp, int vec_enc) {
6168 vmovdqu(xtmp1, ExternalAddress(StubRoutines::x86::vector_count_leading_zeros_lut()), rtmp);
6169 vbroadcast(T_INT, xtmp2, 0x0F0F0F0F, rtmp, vec_enc);
6170 // T1 = Compute leading zero counts of 4 LSB bits of each byte by
6171 // accessing the lookup table.
6172 vpand(dst, xtmp2, src, vec_enc);
6173 vpshufb(dst, xtmp1, dst, vec_enc);
6174 // T2 = Compute leading zero counts of 4 MSB bits of each byte by
6175 // accessing the lookup table.
6176 vpsrlw(xtmp3, src, 4, vec_enc);
6177 vpand(xtmp3, xtmp2, xtmp3, vec_enc);
6178 vpshufb(xtmp2, xtmp1, xtmp3, vec_enc);
6179 // Add T1 to T2 if 4 MSB bits of byte are all zeros.
6180 vpxor(xtmp1, xtmp1, xtmp1, vec_enc);
6181 vpcmpeqb(xtmp3, xtmp1, xtmp3, vec_enc);
6182 vpaddb(dst, dst, xtmp2, vec_enc);
6183 vpblendvb(dst, xtmp2, dst, xtmp3, vec_enc);
6184 }
6185
6186 void C2_MacroAssembler::vector_count_leading_zeros_short_avx(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
6187 XMMRegister xtmp2, XMMRegister xtmp3, Register rtmp, int vec_enc) {
6188 vector_count_leading_zeros_byte_avx(dst, src, xtmp1, xtmp2, xtmp3, rtmp, vec_enc);
6189 // Add zero counts of lower byte and upper byte of a word if
6190 // upper byte holds a zero value.
6191 vpsrlw(xtmp3, src, 8, vec_enc);
6192 // xtmp1 is set to all zeros by vector_count_leading_zeros_byte_avx.
6193 vpcmpeqw(xtmp3, xtmp1, xtmp3, vec_enc);
6194 vpsllw(xtmp2, dst, 8, vec_enc);
6195 vpaddw(xtmp2, xtmp2, dst, vec_enc);
6196 vpblendvb(dst, dst, xtmp2, xtmp3, vec_enc);
6197 vpsrlw(dst, dst, 8, vec_enc);
6198 }
6199
6200 void C2_MacroAssembler::vector_count_leading_zeros_int_avx(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
6201 XMMRegister xtmp2, XMMRegister xtmp3, int vec_enc) {
6202 // By converting the integer to a float, we can obtain the number of leading zeros based on the exponent of the float.
6203 // As the float exponent contains a bias of 127 for nonzero values, the bias must be removed before interpreting the
6204 // exponent as the leading zero count.
6205
6206 // Remove the bit to the right of the highest set bit ensuring that the conversion to float cannot round up to a higher
6207 // power of 2, which has a higher exponent than the input. This transformation is valid as only the highest set bit
6208 // contributes to the leading number of zeros.
6209 vpsrld(dst, src, 1, vec_enc);
6210 vpandn(dst, dst, src, vec_enc);
6211
6212 vcvtdq2ps(dst, dst, vec_enc);
6213
6214 // By comparing the register to itself, all the bits in the destination are set.
6215 vpcmpeqd(xtmp1, xtmp1, xtmp1, vec_enc);
6216
6217 // Move the biased exponent to the low end of the lane and mask with 0xFF to discard the sign bit.
6218 vpsrld(xtmp2, xtmp1, 24, vec_enc);
6219 vpsrld(dst, dst, 23, vec_enc);
6220 vpand(dst, xtmp2, dst, vec_enc);
6221
6222 // Subtract 127 from the exponent, which removes the bias from the exponent.
6223 vpsrld(xtmp2, xtmp1, 25, vec_enc);
6224 vpsubd(dst, dst, xtmp2, vec_enc);
6225
6226 vpsrld(xtmp2, xtmp1, 27, vec_enc);
6227
6228 // If the original value is 0 the exponent would not have bias, so the subtraction creates a negative number. If this
6229 // is found in any of the lanes, replace the lane with -1 from xtmp1.
6230 vblendvps(dst, dst, xtmp1, dst, vec_enc, true, xtmp3);
6231
6232 // If the original value is negative, replace the lane with 31.
6233 vblendvps(dst, dst, xtmp2, src, vec_enc, true, xtmp3);
6234
6235 // Subtract the exponent from 31, giving the final result. For 0, the result is 32 as the exponent was replaced with -1,
6236 // and for negative numbers the result is 0 as the exponent was replaced with 31.
6237 vpsubd(dst, xtmp2, dst, vec_enc);
6238 }
6239
6240 void C2_MacroAssembler::vector_count_leading_zeros_long_avx(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
6241 XMMRegister xtmp2, XMMRegister xtmp3, Register rtmp, int vec_enc) {
6242 // Find the leading zeros of the top and bottom halves of the long individually.
6243 vector_count_leading_zeros_int_avx(dst, src, xtmp1, xtmp2, xtmp3, vec_enc);
6244
6245 // Move the top half result to the bottom half of xtmp1, setting the top half to 0.
6246 vpsrlq(xtmp1, dst, 32, vec_enc);
6247 // By moving the top half result to the right by 6 bits, if the top half was empty (i.e. 32 is returned) the result bit will
6248 // be in the most significant position of the bottom half.
6249 vpsrlq(xtmp2, dst, 6, vec_enc);
6250
6251 // In the bottom half, add the top half and bottom half results.
6252 vpaddq(dst, xtmp1, dst, vec_enc);
6253
6254 // For the bottom half, choose between the values using the most significant bit of xtmp2.
6255 // If the MSB is set, then bottom+top in dst is the resulting value. If the top half is less than 32 xtmp1 is chosen,
6256 // which contains only the top half result.
6257 // In the top half the MSB is always zero, so the value in xtmp1 is always chosen. This value is always 0, which clears
6258 // the lane as required.
6259 vblendvps(dst, xtmp1, dst, xtmp2, vec_enc, true, xtmp3);
6260 }
6261
6262 void C2_MacroAssembler::vector_count_leading_zeros_avx(BasicType bt, XMMRegister dst, XMMRegister src,
6263 XMMRegister xtmp1, XMMRegister xtmp2, XMMRegister xtmp3,
6264 Register rtmp, int vec_enc) {
6265 assert(is_integral_type(bt), "unexpected type");
6266 assert(vec_enc < Assembler::AVX_512bit, "");
6267 switch(bt) {
6268 case T_LONG:
6269 vector_count_leading_zeros_long_avx(dst, src, xtmp1, xtmp2, xtmp3, rtmp, vec_enc);
6270 break;
6271 case T_INT:
6272 vector_count_leading_zeros_int_avx(dst, src, xtmp1, xtmp2, xtmp3, vec_enc);
6273 break;
6274 case T_SHORT:
6275 vector_count_leading_zeros_short_avx(dst, src, xtmp1, xtmp2, xtmp3, rtmp, vec_enc);
6276 break;
6277 case T_BYTE:
6278 vector_count_leading_zeros_byte_avx(dst, src, xtmp1, xtmp2, xtmp3, rtmp, vec_enc);
6279 break;
6280 default:
6281 fatal("Unsupported type %s", type2name(bt));
6282 break;
6283 }
6284 }
6285
6286 void C2_MacroAssembler::vpsub(BasicType bt, XMMRegister dst, XMMRegister src1, XMMRegister src2, int vec_enc) {
6287 switch(bt) {
6288 case T_BYTE:
6289 vpsubb(dst, src1, src2, vec_enc);
6290 break;
6291 case T_SHORT:
6292 vpsubw(dst, src1, src2, vec_enc);
6293 break;
6294 case T_INT:
6295 vpsubd(dst, src1, src2, vec_enc);
6296 break;
6297 case T_LONG:
6298 vpsubq(dst, src1, src2, vec_enc);
6299 break;
6300 default:
6301 fatal("Unsupported type %s", type2name(bt));
6302 break;
6303 }
6304 }
6305
6306 // Trailing zero count computation is based on leading zero count operation as per
6307 // following equation. All AVX3 targets support AVX512CD feature which offers
6308 // direct vector instruction to compute leading zero count.
6309 // CTZ = PRIM_TYPE_WIDHT - CLZ((x - 1) & ~x)
6310 void C2_MacroAssembler::vector_count_trailing_zeros_evex(BasicType bt, XMMRegister dst, XMMRegister src,
6311 XMMRegister xtmp1, XMMRegister xtmp2, XMMRegister xtmp3,
6312 XMMRegister xtmp4, KRegister ktmp, Register rtmp, int vec_enc) {
6313 assert(is_integral_type(bt), "");
6314 // xtmp = -1
6315 vpternlogd(xtmp4, 0xff, xtmp4, xtmp4, vec_enc);
6316 // xtmp = xtmp + src
6317 vpadd(bt, xtmp4, xtmp4, src, vec_enc);
6318 // xtmp = xtmp & ~src
6319 vpternlogd(xtmp4, 0x40, xtmp4, src, vec_enc);
6320 vector_count_leading_zeros_evex(bt, dst, xtmp4, xtmp1, xtmp2, xtmp3, ktmp, rtmp, true, vec_enc);
6321 vbroadcast(bt, xtmp4, 8 * type2aelembytes(bt), rtmp, vec_enc);
6322 vpsub(bt, dst, xtmp4, dst, vec_enc);
6323 }
6324
6325 // Trailing zero count computation for AVX2 targets is based on popcount operation as per following equation
6326 // CTZ = PRIM_TYPE_WIDHT - POPC(x | -x)
6327 void C2_MacroAssembler::vector_count_trailing_zeros_avx(BasicType bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
6328 XMMRegister xtmp2, XMMRegister xtmp3, Register rtmp, int vec_enc) {
6329 assert(is_integral_type(bt), "");
6330 // xtmp = 0
6331 vpxor(xtmp3 , xtmp3, xtmp3, vec_enc);
6332 // xtmp = 0 - src
6333 vpsub(bt, xtmp3, xtmp3, src, vec_enc);
6334 // xtmp = xtmp | src
6335 vpor(xtmp3, xtmp3, src, vec_enc);
6336 vector_popcount_integral(bt, dst, xtmp3, xtmp1, xtmp2, rtmp, vec_enc);
6337 vbroadcast(bt, xtmp1, 8 * type2aelembytes(bt), rtmp, vec_enc);
6338 vpsub(bt, dst, xtmp1, dst, vec_enc);
6339 }
6340
6341 void C2_MacroAssembler::udivI(Register rax, Register divisor, Register rdx) {
6342 Label done;
6343 Label neg_divisor_fastpath;
6344 cmpl(divisor, 0);
6345 jccb(Assembler::less, neg_divisor_fastpath);
6346 xorl(rdx, rdx);
6347 divl(divisor);
6348 jmpb(done);
6349 bind(neg_divisor_fastpath);
6350 // Fastpath for divisor < 0:
6351 // quotient = (dividend & ~(dividend - divisor)) >>> (Integer.SIZE - 1)
6352 // See Hacker's Delight (2nd ed), section 9.3 which is implemented in java.lang.Long.divideUnsigned()
6353 movl(rdx, rax);
6354 subl(rdx, divisor);
6355 if (VM_Version::supports_bmi1() && VM_Version::supports_avx()) {
6356 andnl(rax, rdx, rax);
6357 } else {
6358 notl(rdx);
6359 andl(rax, rdx);
6360 }
6361 shrl(rax, 31);
6362 bind(done);
6363 }
6364
6365 void C2_MacroAssembler::umodI(Register rax, Register divisor, Register rdx) {
6366 Label done;
6367 Label neg_divisor_fastpath;
6368 cmpl(divisor, 0);
6369 jccb(Assembler::less, neg_divisor_fastpath);
6370 xorl(rdx, rdx);
6371 divl(divisor);
6372 jmpb(done);
6373 bind(neg_divisor_fastpath);
6374 // Fastpath when divisor < 0:
6375 // remainder = dividend - (((dividend & ~(dividend - divisor)) >> (Integer.SIZE - 1)) & divisor)
6376 // See Hacker's Delight (2nd ed), section 9.3 which is implemented in java.lang.Long.remainderUnsigned()
6377 movl(rdx, rax);
6378 subl(rax, divisor);
6379 if (VM_Version::supports_bmi1() && VM_Version::supports_avx()) {
6380 andnl(rax, rax, rdx);
6381 } else {
6382 notl(rax);
6383 andl(rax, rdx);
6384 }
6385 sarl(rax, 31);
6386 andl(rax, divisor);
6387 subl(rdx, rax);
6388 bind(done);
6389 }
6390
6391 void C2_MacroAssembler::udivmodI(Register rax, Register divisor, Register rdx, Register tmp) {
6392 Label done;
6393 Label neg_divisor_fastpath;
6394
6395 cmpl(divisor, 0);
6396 jccb(Assembler::less, neg_divisor_fastpath);
6397 xorl(rdx, rdx);
6398 divl(divisor);
6399 jmpb(done);
6400 bind(neg_divisor_fastpath);
6401 // Fastpath for divisor < 0:
6402 // quotient = (dividend & ~(dividend - divisor)) >>> (Integer.SIZE - 1)
6403 // remainder = dividend - (((dividend & ~(dividend - divisor)) >> (Integer.SIZE - 1)) & divisor)
6404 // See Hacker's Delight (2nd ed), section 9.3 which is implemented in
6405 // java.lang.Long.divideUnsigned() and java.lang.Long.remainderUnsigned()
6406 movl(rdx, rax);
6407 subl(rax, divisor);
6408 if (VM_Version::supports_bmi1() && VM_Version::supports_avx()) {
6409 andnl(rax, rax, rdx);
6410 } else {
6411 notl(rax);
6412 andl(rax, rdx);
6413 }
6414 movl(tmp, rax);
6415 shrl(rax, 31); // quotient
6416 sarl(tmp, 31);
6417 andl(tmp, divisor);
6418 subl(rdx, tmp); // remainder
6419 bind(done);
6420 }
6421
6422 void C2_MacroAssembler::reverseI(Register dst, Register src, XMMRegister xtmp1,
6423 XMMRegister xtmp2, Register rtmp) {
6424 if(VM_Version::supports_gfni()) {
6425 // Galois field instruction based bit reversal based on following algorithm.
6426 // http://0x80.pl/articles/avx512-galois-field-for-bit-shuffling.html
6427 mov64(rtmp, 0x8040201008040201L);
6428 movq(xtmp1, src);
6429 movq(xtmp2, rtmp);
6430 gf2p8affineqb(xtmp1, xtmp2, 0);
6431 movq(dst, xtmp1);
6432 } else {
6433 // Swap even and odd numbered bits.
6434 movl(rtmp, src);
6435 andl(rtmp, 0x55555555);
6436 shll(rtmp, 1);
6437 movl(dst, src);
6438 andl(dst, 0xAAAAAAAA);
6439 shrl(dst, 1);
6440 orl(dst, rtmp);
6441
6442 // Swap LSB and MSB 2 bits of each nibble.
6443 movl(rtmp, dst);
6444 andl(rtmp, 0x33333333);
6445 shll(rtmp, 2);
6446 andl(dst, 0xCCCCCCCC);
6447 shrl(dst, 2);
6448 orl(dst, rtmp);
6449
6450 // Swap LSB and MSB 4 bits of each byte.
6451 movl(rtmp, dst);
6452 andl(rtmp, 0x0F0F0F0F);
6453 shll(rtmp, 4);
6454 andl(dst, 0xF0F0F0F0);
6455 shrl(dst, 4);
6456 orl(dst, rtmp);
6457 }
6458 bswapl(dst);
6459 }
6460
6461 void C2_MacroAssembler::reverseL(Register dst, Register src, XMMRegister xtmp1,
6462 XMMRegister xtmp2, Register rtmp1, Register rtmp2) {
6463 if(VM_Version::supports_gfni()) {
6464 // Galois field instruction based bit reversal based on following algorithm.
6465 // http://0x80.pl/articles/avx512-galois-field-for-bit-shuffling.html
6466 mov64(rtmp1, 0x8040201008040201L);
6467 movq(xtmp1, src);
6468 movq(xtmp2, rtmp1);
6469 gf2p8affineqb(xtmp1, xtmp2, 0);
6470 movq(dst, xtmp1);
6471 } else {
6472 // Swap even and odd numbered bits.
6473 movq(rtmp1, src);
6474 mov64(rtmp2, 0x5555555555555555L);
6475 andq(rtmp1, rtmp2);
6476 shlq(rtmp1, 1);
6477 movq(dst, src);
6478 notq(rtmp2);
6479 andq(dst, rtmp2);
6480 shrq(dst, 1);
6481 orq(dst, rtmp1);
6482
6483 // Swap LSB and MSB 2 bits of each nibble.
6484 movq(rtmp1, dst);
6485 mov64(rtmp2, 0x3333333333333333L);
6486 andq(rtmp1, rtmp2);
6487 shlq(rtmp1, 2);
6488 notq(rtmp2);
6489 andq(dst, rtmp2);
6490 shrq(dst, 2);
6491 orq(dst, rtmp1);
6492
6493 // Swap LSB and MSB 4 bits of each byte.
6494 movq(rtmp1, dst);
6495 mov64(rtmp2, 0x0F0F0F0F0F0F0F0FL);
6496 andq(rtmp1, rtmp2);
6497 shlq(rtmp1, 4);
6498 notq(rtmp2);
6499 andq(dst, rtmp2);
6500 shrq(dst, 4);
6501 orq(dst, rtmp1);
6502 }
6503 bswapq(dst);
6504 }
6505
6506 void C2_MacroAssembler::udivL(Register rax, Register divisor, Register rdx) {
6507 Label done;
6508 Label neg_divisor_fastpath;
6509 cmpq(divisor, 0);
6510 jccb(Assembler::less, neg_divisor_fastpath);
6511 xorl(rdx, rdx);
6512 divq(divisor);
6513 jmpb(done);
6514 bind(neg_divisor_fastpath);
6515 // Fastpath for divisor < 0:
6516 // quotient = (dividend & ~(dividend - divisor)) >>> (Long.SIZE - 1)
6517 // See Hacker's Delight (2nd ed), section 9.3 which is implemented in java.lang.Long.divideUnsigned()
6518 movq(rdx, rax);
6519 subq(rdx, divisor);
6520 if (VM_Version::supports_bmi1() && VM_Version::supports_avx()) {
6521 andnq(rax, rdx, rax);
6522 } else {
6523 notq(rdx);
6524 andq(rax, rdx);
6525 }
6526 shrq(rax, 63);
6527 bind(done);
6528 }
6529
6530 void C2_MacroAssembler::umodL(Register rax, Register divisor, Register rdx) {
6531 Label done;
6532 Label neg_divisor_fastpath;
6533 cmpq(divisor, 0);
6534 jccb(Assembler::less, neg_divisor_fastpath);
6535 xorq(rdx, rdx);
6536 divq(divisor);
6537 jmp(done);
6538 bind(neg_divisor_fastpath);
6539 // Fastpath when divisor < 0:
6540 // remainder = dividend - (((dividend & ~(dividend - divisor)) >> (Long.SIZE - 1)) & divisor)
6541 // See Hacker's Delight (2nd ed), section 9.3 which is implemented in java.lang.Long.remainderUnsigned()
6542 movq(rdx, rax);
6543 subq(rax, divisor);
6544 if (VM_Version::supports_bmi1() && VM_Version::supports_avx()) {
6545 andnq(rax, rax, rdx);
6546 } else {
6547 notq(rax);
6548 andq(rax, rdx);
6549 }
6550 sarq(rax, 63);
6551 andq(rax, divisor);
6552 subq(rdx, rax);
6553 bind(done);
6554 }
6555
6556 void C2_MacroAssembler::udivmodL(Register rax, Register divisor, Register rdx, Register tmp) {
6557 Label done;
6558 Label neg_divisor_fastpath;
6559 cmpq(divisor, 0);
6560 jccb(Assembler::less, neg_divisor_fastpath);
6561 xorq(rdx, rdx);
6562 divq(divisor);
6563 jmp(done);
6564 bind(neg_divisor_fastpath);
6565 // Fastpath for divisor < 0:
6566 // quotient = (dividend & ~(dividend - divisor)) >>> (Long.SIZE - 1)
6567 // remainder = dividend - (((dividend & ~(dividend - divisor)) >> (Long.SIZE - 1)) & divisor)
6568 // See Hacker's Delight (2nd ed), section 9.3 which is implemented in
6569 // java.lang.Long.divideUnsigned() and java.lang.Long.remainderUnsigned()
6570 movq(rdx, rax);
6571 subq(rax, divisor);
6572 if (VM_Version::supports_bmi1() && VM_Version::supports_avx()) {
6573 andnq(rax, rax, rdx);
6574 } else {
6575 notq(rax);
6576 andq(rax, rdx);
6577 }
6578 movq(tmp, rax);
6579 shrq(rax, 63); // quotient
6580 sarq(tmp, 63);
6581 andq(tmp, divisor);
6582 subq(rdx, tmp); // remainder
6583 bind(done);
6584 }
6585
6586 void C2_MacroAssembler::rearrange_bytes(XMMRegister dst, XMMRegister shuffle, XMMRegister src, XMMRegister xtmp1,
6587 XMMRegister xtmp2, XMMRegister xtmp3, Register rtmp, KRegister ktmp,
6588 int vlen_enc) {
6589 assert(VM_Version::supports_avx512bw(), "");
6590 // Byte shuffles are inlane operations and indices are determined using
6591 // lower 4 bit of each shuffle lane, thus all shuffle indices are
6592 // normalized to index range 0-15. This makes sure that all the multiples
6593 // of an index value are placed at same relative position in 128 bit
6594 // lane i.e. elements corresponding to shuffle indices 16, 32 and 64
6595 // will be 16th element in their respective 128 bit lanes.
6596 movl(rtmp, 16);
6597 evpbroadcastb(xtmp1, rtmp, vlen_enc);
6598
6599 // Compute a mask for shuffle vector by comparing indices with expression INDEX < 16,
6600 // Broadcast first 128 bit lane across entire vector, shuffle the vector lanes using
6601 // original shuffle indices and move the shuffled lanes corresponding to true
6602 // mask to destination vector.
6603 evpcmpb(ktmp, k0, shuffle, xtmp1, Assembler::lt, true, vlen_enc);
6604 evshufi64x2(xtmp2, src, src, 0x0, vlen_enc);
6605 evpshufb(dst, ktmp, xtmp2, shuffle, false, vlen_enc);
6606
6607 // Perform above steps with lane comparison expression as INDEX >= 16 && INDEX < 32
6608 // and broadcasting second 128 bit lane.
6609 evpcmpb(ktmp, k0, shuffle, xtmp1, Assembler::nlt, true, vlen_enc);
6610 vpsllq(xtmp2, xtmp1, 0x1, vlen_enc);
6611 evpcmpb(ktmp, ktmp, shuffle, xtmp2, Assembler::lt, true, vlen_enc);
6612 evshufi64x2(xtmp3, src, src, 0x55, vlen_enc);
6613 evpshufb(dst, ktmp, xtmp3, shuffle, true, vlen_enc);
6614
6615 // Perform above steps with lane comparison expression as INDEX >= 32 && INDEX < 48
6616 // and broadcasting third 128 bit lane.
6617 evpcmpb(ktmp, k0, shuffle, xtmp2, Assembler::nlt, true, vlen_enc);
6618 vpaddb(xtmp1, xtmp1, xtmp2, vlen_enc);
6619 evpcmpb(ktmp, ktmp, shuffle, xtmp1, Assembler::lt, true, vlen_enc);
6620 evshufi64x2(xtmp3, src, src, 0xAA, vlen_enc);
6621 evpshufb(dst, ktmp, xtmp3, shuffle, true, vlen_enc);
6622
6623 // Perform above steps with lane comparison expression as INDEX >= 48 && INDEX < 64
6624 // and broadcasting third 128 bit lane.
6625 evpcmpb(ktmp, k0, shuffle, xtmp1, Assembler::nlt, true, vlen_enc);
6626 vpsllq(xtmp2, xtmp2, 0x1, vlen_enc);
6627 evpcmpb(ktmp, ktmp, shuffle, xtmp2, Assembler::lt, true, vlen_enc);
6628 evshufi64x2(xtmp3, src, src, 0xFF, vlen_enc);
6629 evpshufb(dst, ktmp, xtmp3, shuffle, true, vlen_enc);
6630 }
6631
6632 void C2_MacroAssembler::vector_rearrange_int_float(BasicType bt, XMMRegister dst,
6633 XMMRegister shuffle, XMMRegister src, int vlen_enc) {
6634 if (vlen_enc == AVX_128bit) {
6635 vpermilps(dst, src, shuffle, vlen_enc);
6636 } else if (bt == T_INT) {
6637 vpermd(dst, shuffle, src, vlen_enc);
6638 } else {
6639 assert(bt == T_FLOAT, "");
6640 vpermps(dst, shuffle, src, vlen_enc);
6641 }
6642 }
6643
6644 void C2_MacroAssembler::efp16sh(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2) {
6645 switch(opcode) {
6646 case Op_AddHF: vaddsh(dst, src1, src2); break;
6647 case Op_SubHF: vsubsh(dst, src1, src2); break;
6648 case Op_MulHF: vmulsh(dst, src1, src2); break;
6649 case Op_DivHF: vdivsh(dst, src1, src2); break;
6650 default: assert(false, "%s", NodeClassNames[opcode]); break;
6651 }
6652 }
6653
6654 void C2_MacroAssembler::vector_saturating_op(int ideal_opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2, int vlen_enc) {
6655 switch(elem_bt) {
6656 case T_BYTE:
6657 if (ideal_opc == Op_SaturatingAddV) {
6658 vpaddsb(dst, src1, src2, vlen_enc);
6659 } else {
6660 assert(ideal_opc == Op_SaturatingSubV, "");
6661 vpsubsb(dst, src1, src2, vlen_enc);
6662 }
6663 break;
6664 case T_SHORT:
6665 if (ideal_opc == Op_SaturatingAddV) {
6666 vpaddsw(dst, src1, src2, vlen_enc);
6667 } else {
6668 assert(ideal_opc == Op_SaturatingSubV, "");
6669 vpsubsw(dst, src1, src2, vlen_enc);
6670 }
6671 break;
6672 default:
6673 fatal("Unsupported type %s", type2name(elem_bt));
6674 break;
6675 }
6676 }
6677
6678 void C2_MacroAssembler::vector_saturating_unsigned_op(int ideal_opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2, int vlen_enc) {
6679 switch(elem_bt) {
6680 case T_BYTE:
6681 if (ideal_opc == Op_SaturatingAddV) {
6682 vpaddusb(dst, src1, src2, vlen_enc);
6683 } else {
6684 assert(ideal_opc == Op_SaturatingSubV, "");
6685 vpsubusb(dst, src1, src2, vlen_enc);
6686 }
6687 break;
6688 case T_SHORT:
6689 if (ideal_opc == Op_SaturatingAddV) {
6690 vpaddusw(dst, src1, src2, vlen_enc);
6691 } else {
6692 assert(ideal_opc == Op_SaturatingSubV, "");
6693 vpsubusw(dst, src1, src2, vlen_enc);
6694 }
6695 break;
6696 default:
6697 fatal("Unsupported type %s", type2name(elem_bt));
6698 break;
6699 }
6700 }
6701
6702 void C2_MacroAssembler::vector_sub_dq_saturating_unsigned_evex(BasicType elem_bt, XMMRegister dst, XMMRegister src1,
6703 XMMRegister src2, KRegister ktmp, int vlen_enc) {
6704 // For unsigned subtraction, overflow happens when magnitude of second input is greater than first input.
6705 // overflow_mask = Inp1 <u Inp2
6706 evpcmpu(elem_bt, ktmp, src2, src1, Assembler::lt, vlen_enc);
6707 // Res = overflow_mask ? Zero : INP1 - INP2 (non-commutative and non-associative)
6708 evmasked_op(elem_bt == T_INT ? Op_SubVI : Op_SubVL, elem_bt, ktmp, dst, src1, src2, false, vlen_enc, false);
6709 }
6710
6711 void C2_MacroAssembler::vector_sub_dq_saturating_unsigned_avx(BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2,
6712 XMMRegister xtmp1, XMMRegister xtmp2, int vlen_enc) {
6713 // Emulate unsigned comparison using signed comparison
6714 // Mask = Inp1 <u Inp2 => Inp1 + MIN_VALUE < Inp2 + MIN_VALUE
6715 vpgenmin_value(elem_bt, xtmp1, xtmp1, vlen_enc, true);
6716 vpadd(elem_bt, xtmp2, src1, xtmp1, vlen_enc);
6717 vpadd(elem_bt, xtmp1, src2, xtmp1, vlen_enc);
6718
6719 vpcmpgt(elem_bt, xtmp2, xtmp1, xtmp2, vlen_enc);
6720
6721 // Res = INP1 - INP2 (non-commutative and non-associative)
6722 vpsub(elem_bt, dst, src1, src2, vlen_enc);
6723 // Res = Mask ? Zero : Res
6724 vpxor(xtmp1, xtmp1, xtmp1, vlen_enc);
6725 vpblendvb(dst, dst, xtmp1, xtmp2, vlen_enc);
6726 }
6727
6728 void C2_MacroAssembler::vector_add_dq_saturating_unsigned_evex(BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2,
6729 XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp, int vlen_enc) {
6730 // Unsigned values ranges comprise of only +ve numbers, thus there exist only an upper bound saturation.
6731 // overflow_mask = (SRC1 + SRC2) <u (SRC1 | SRC2)
6732 // Res = Signed Add INP1, INP2
6733 vpadd(elem_bt, dst, src1, src2, vlen_enc);
6734 // T1 = SRC1 | SRC2
6735 vpor(xtmp1, src1, src2, vlen_enc);
6736 // Max_Unsigned = -1
6737 vpternlogd(xtmp2, 0xff, xtmp2, xtmp2, vlen_enc);
6738 // Unsigned compare: Mask = Res <u T1
6739 evpcmpu(elem_bt, ktmp, dst, xtmp1, Assembler::lt, vlen_enc);
6740 // res = Mask ? Max_Unsigned : Res
6741 evpblend(elem_bt, dst, ktmp, dst, xtmp2, true, vlen_enc);
6742 }
6743
6744 //
6745 // Section 2-13 Hacker's Delight list following overflow detection check for saturating
6746 // unsigned addition operation.
6747 // overflow_mask = ((a & b) | ((a | b) & ~( a + b))) >>> 31 == 1
6748 //
6749 // We empirically determined its semantic equivalence to following reduced expression
6750 // overflow_mask = (a + b) <u (a | b)
6751 //
6752 // and also verified it though Alive2 solver.
6753 // (https://alive2.llvm.org/ce/z/XDQ7dY)
6754 //
6755
6756 void C2_MacroAssembler::vector_add_dq_saturating_unsigned_avx(BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2,
6757 XMMRegister xtmp1, XMMRegister xtmp2, XMMRegister xtmp3, int vlen_enc) {
6758 // Res = Signed Add INP1, INP2
6759 vpadd(elem_bt, dst, src1, src2, vlen_enc);
6760 // Compute T1 = INP1 | INP2
6761 vpor(xtmp3, src1, src2, vlen_enc);
6762 // T1 = Minimum signed value.
6763 vpgenmin_value(elem_bt, xtmp2, xtmp1, vlen_enc, true);
6764 // Convert T1 to signed value, T1 = T1 + MIN_VALUE
6765 vpadd(elem_bt, xtmp3, xtmp3, xtmp2, vlen_enc);
6766 // Convert Res to signed value, Res<s> = Res + MIN_VALUE
6767 vpadd(elem_bt, xtmp2, xtmp2, dst, vlen_enc);
6768 // Compute overflow detection mask = Res<1> <s T1
6769 if (elem_bt == T_INT) {
6770 vpcmpgtd(xtmp3, xtmp3, xtmp2, vlen_enc);
6771 } else {
6772 assert(elem_bt == T_LONG, "");
6773 vpcmpgtq(xtmp3, xtmp3, xtmp2, vlen_enc);
6774 }
6775 vpblendvb(dst, dst, xtmp1, xtmp3, vlen_enc);
6776 }
6777
6778 void C2_MacroAssembler::evpmovq2m_emu(KRegister ktmp, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
6779 int vlen_enc, bool xtmp2_hold_M1) {
6780 if (VM_Version::supports_avx512dq()) {
6781 evpmovq2m(ktmp, src, vlen_enc);
6782 } else {
6783 assert(VM_Version::supports_evex(), "");
6784 if (!xtmp2_hold_M1) {
6785 vpternlogq(xtmp2, 0xff, xtmp2, xtmp2, vlen_enc);
6786 }
6787 evpsraq(xtmp1, src, 63, vlen_enc);
6788 evpcmpeqq(ktmp, k0, xtmp1, xtmp2, vlen_enc);
6789 }
6790 }
6791
6792 void C2_MacroAssembler::evpmovd2m_emu(KRegister ktmp, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
6793 int vlen_enc, bool xtmp2_hold_M1) {
6794 if (VM_Version::supports_avx512dq()) {
6795 evpmovd2m(ktmp, src, vlen_enc);
6796 } else {
6797 assert(VM_Version::supports_evex(), "");
6798 if (!xtmp2_hold_M1) {
6799 vpternlogd(xtmp2, 0xff, xtmp2, xtmp2, vlen_enc);
6800 }
6801 vpsrad(xtmp1, src, 31, vlen_enc);
6802 Assembler::evpcmpeqd(ktmp, k0, xtmp1, xtmp2, vlen_enc);
6803 }
6804 }
6805
6806
6807 void C2_MacroAssembler::vpsign_extend_dq(BasicType elem_bt, XMMRegister dst, XMMRegister src, int vlen_enc) {
6808 if (elem_bt == T_LONG) {
6809 if (VM_Version::supports_evex()) {
6810 evpsraq(dst, src, 63, vlen_enc);
6811 } else {
6812 vpsrad(dst, src, 31, vlen_enc);
6813 vpshufd(dst, dst, 0xF5, vlen_enc);
6814 }
6815 } else {
6816 assert(elem_bt == T_INT, "");
6817 vpsrad(dst, src, 31, vlen_enc);
6818 }
6819 }
6820
6821 void C2_MacroAssembler::vpgenmax_value(BasicType elem_bt, XMMRegister dst, XMMRegister allones, int vlen_enc, bool compute_allones) {
6822 if (compute_allones) {
6823 if (VM_Version::supports_avx512vl() || vlen_enc == Assembler::AVX_512bit) {
6824 vpternlogd(allones, 0xff, allones, allones, vlen_enc);
6825 } else {
6826 vpcmpeqq(allones, allones, allones, vlen_enc);
6827 }
6828 }
6829 if (elem_bt == T_LONG) {
6830 vpsrlq(dst, allones, 1, vlen_enc);
6831 } else {
6832 assert(elem_bt == T_INT, "");
6833 vpsrld(dst, allones, 1, vlen_enc);
6834 }
6835 }
6836
6837 void C2_MacroAssembler::vpgenmin_value(BasicType elem_bt, XMMRegister dst, XMMRegister allones, int vlen_enc, bool compute_allones) {
6838 if (compute_allones) {
6839 if (VM_Version::supports_avx512vl() || vlen_enc == Assembler::AVX_512bit) {
6840 vpternlogd(allones, 0xff, allones, allones, vlen_enc);
6841 } else {
6842 vpcmpeqq(allones, allones, allones, vlen_enc);
6843 }
6844 }
6845 if (elem_bt == T_LONG) {
6846 vpsllq(dst, allones, 63, vlen_enc);
6847 } else {
6848 assert(elem_bt == T_INT, "");
6849 vpslld(dst, allones, 31, vlen_enc);
6850 }
6851 }
6852
6853 void C2_MacroAssembler::evpcmpu(BasicType elem_bt, KRegister kmask, XMMRegister src1, XMMRegister src2,
6854 Assembler::ComparisonPredicate cond, int vlen_enc) {
6855 switch(elem_bt) {
6856 case T_LONG: evpcmpuq(kmask, src1, src2, cond, vlen_enc); break;
6857 case T_INT: evpcmpud(kmask, src1, src2, cond, vlen_enc); break;
6858 case T_SHORT: evpcmpuw(kmask, src1, src2, cond, vlen_enc); break;
6859 case T_BYTE: evpcmpub(kmask, src1, src2, cond, vlen_enc); break;
6860 default: fatal("Unsupported type %s", type2name(elem_bt)); break;
6861 }
6862 }
6863
6864 void C2_MacroAssembler::vpcmpgt(BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2, int vlen_enc) {
6865 switch(elem_bt) {
6866 case T_LONG: vpcmpgtq(dst, src1, src2, vlen_enc); break;
6867 case T_INT: vpcmpgtd(dst, src1, src2, vlen_enc); break;
6868 case T_SHORT: vpcmpgtw(dst, src1, src2, vlen_enc); break;
6869 case T_BYTE: vpcmpgtb(dst, src1, src2, vlen_enc); break;
6870 default: fatal("Unsupported type %s", type2name(elem_bt)); break;
6871 }
6872 }
6873
6874 void C2_MacroAssembler::evpmov_vec_to_mask(BasicType elem_bt, KRegister ktmp, XMMRegister src, XMMRegister xtmp1,
6875 XMMRegister xtmp2, int vlen_enc, bool xtmp2_hold_M1) {
6876 if (elem_bt == T_LONG) {
6877 evpmovq2m_emu(ktmp, src, xtmp1, xtmp2, vlen_enc, xtmp2_hold_M1);
6878 } else {
6879 assert(elem_bt == T_INT, "");
6880 evpmovd2m_emu(ktmp, src, xtmp1, xtmp2, vlen_enc, xtmp2_hold_M1);
6881 }
6882 }
6883
6884 void C2_MacroAssembler::vector_addsub_dq_saturating_evex(int ideal_opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1,
6885 XMMRegister src2, XMMRegister xtmp1, XMMRegister xtmp2,
6886 KRegister ktmp1, KRegister ktmp2, int vlen_enc) {
6887 assert(elem_bt == T_INT || elem_bt == T_LONG, "");
6888 // Addition/Subtraction happens over two's compliment representation of numbers and is agnostic to signed'ness.
6889 // Overflow detection based on Hacker's delight section 2-13.
6890 if (ideal_opc == Op_SaturatingAddV) {
6891 // res = src1 + src2
6892 vpadd(elem_bt, dst, src1, src2, vlen_enc);
6893 // Overflow occurs if result polarity does not comply with equivalent polarity inputs.
6894 // overflow = (((res ^ src1) & (res ^ src2)) >>> 31(I)/63(L)) == 1
6895 vpxor(xtmp1, dst, src1, vlen_enc);
6896 vpxor(xtmp2, dst, src2, vlen_enc);
6897 vpand(xtmp2, xtmp1, xtmp2, vlen_enc);
6898 } else {
6899 assert(ideal_opc == Op_SaturatingSubV, "");
6900 // res = src1 - src2
6901 vpsub(elem_bt, dst, src1, src2, vlen_enc);
6902 // Overflow occurs when both inputs have opposite polarity and
6903 // result polarity does not comply with first input polarity.
6904 // overflow = ((src1 ^ src2) & (res ^ src1) >>> 31(I)/63(L)) == 1;
6905 vpxor(xtmp1, src1, src2, vlen_enc);
6906 vpxor(xtmp2, dst, src1, vlen_enc);
6907 vpand(xtmp2, xtmp1, xtmp2, vlen_enc);
6908 }
6909
6910 // Compute overflow detection mask.
6911 evpmov_vec_to_mask(elem_bt, ktmp1, xtmp2, xtmp2, xtmp1, vlen_enc);
6912 // Note: xtmp1 hold -1 in all its lanes after above call.
6913
6914 // Compute mask based on first input polarity.
6915 evpmov_vec_to_mask(elem_bt, ktmp2, src1, xtmp2, xtmp1, vlen_enc, true);
6916
6917 vpgenmax_value(elem_bt, xtmp2, xtmp1, vlen_enc, true);
6918 vpgenmin_value(elem_bt, xtmp1, xtmp1, vlen_enc);
6919
6920 // Compose a vector of saturating (MAX/MIN) values, where lanes corresponding to
6921 // set bits in first input polarity mask holds a min value.
6922 evpblend(elem_bt, xtmp2, ktmp2, xtmp2, xtmp1, true, vlen_enc);
6923 // Blend destination lanes with saturated values using overflow detection mask.
6924 evpblend(elem_bt, dst, ktmp1, dst, xtmp2, true, vlen_enc);
6925 }
6926
6927
6928 void C2_MacroAssembler::vector_addsub_dq_saturating_avx(int ideal_opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1,
6929 XMMRegister src2, XMMRegister xtmp1, XMMRegister xtmp2,
6930 XMMRegister xtmp3, XMMRegister xtmp4, int vlen_enc) {
6931 assert(elem_bt == T_INT || elem_bt == T_LONG, "");
6932 // Addition/Subtraction happens over two's compliment representation of numbers and is agnostic to signed'ness.
6933 // Overflow detection based on Hacker's delight section 2-13.
6934 if (ideal_opc == Op_SaturatingAddV) {
6935 // res = src1 + src2
6936 vpadd(elem_bt, dst, src1, src2, vlen_enc);
6937 // Overflow occurs if result polarity does not comply with equivalent polarity inputs.
6938 // overflow = (((res ^ src1) & (res ^ src2)) >>> 31(I)/63(L)) == 1
6939 vpxor(xtmp1, dst, src1, vlen_enc);
6940 vpxor(xtmp2, dst, src2, vlen_enc);
6941 vpand(xtmp2, xtmp1, xtmp2, vlen_enc);
6942 } else {
6943 assert(ideal_opc == Op_SaturatingSubV, "");
6944 // res = src1 - src2
6945 vpsub(elem_bt, dst, src1, src2, vlen_enc);
6946 // Overflow occurs when both inputs have opposite polarity and
6947 // result polarity does not comply with first input polarity.
6948 // overflow = ((src1 ^ src2) & (res ^ src1) >>> 31(I)/63(L)) == 1;
6949 vpxor(xtmp1, src1, src2, vlen_enc);
6950 vpxor(xtmp2, dst, src1, vlen_enc);
6951 vpand(xtmp2, xtmp1, xtmp2, vlen_enc);
6952 }
6953
6954 // Sign-extend to compute overflow detection mask.
6955 vpsign_extend_dq(elem_bt, xtmp3, xtmp2, vlen_enc);
6956
6957 vpcmpeqd(xtmp1, xtmp1, xtmp1, vlen_enc);
6958 vpgenmax_value(elem_bt, xtmp2, xtmp1, vlen_enc);
6959 vpgenmin_value(elem_bt, xtmp1, xtmp1, vlen_enc);
6960
6961 // Compose saturating min/max vector using first input polarity mask.
6962 vpsign_extend_dq(elem_bt, xtmp4, src1, vlen_enc);
6963 vpblendvb(xtmp1, xtmp2, xtmp1, xtmp4, vlen_enc);
6964
6965 // Blend result with saturating vector using overflow detection mask.
6966 vpblendvb(dst, dst, xtmp1, xtmp3, vlen_enc);
6967 }
6968
6969 void C2_MacroAssembler::vector_saturating_op(int ideal_opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, Address src2, int vlen_enc) {
6970 switch(elem_bt) {
6971 case T_BYTE:
6972 if (ideal_opc == Op_SaturatingAddV) {
6973 vpaddsb(dst, src1, src2, vlen_enc);
6974 } else {
6975 assert(ideal_opc == Op_SaturatingSubV, "");
6976 vpsubsb(dst, src1, src2, vlen_enc);
6977 }
6978 break;
6979 case T_SHORT:
6980 if (ideal_opc == Op_SaturatingAddV) {
6981 vpaddsw(dst, src1, src2, vlen_enc);
6982 } else {
6983 assert(ideal_opc == Op_SaturatingSubV, "");
6984 vpsubsw(dst, src1, src2, vlen_enc);
6985 }
6986 break;
6987 default:
6988 fatal("Unsupported type %s", type2name(elem_bt));
6989 break;
6990 }
6991 }
6992
6993 void C2_MacroAssembler::vector_saturating_unsigned_op(int ideal_opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, Address src2, int vlen_enc) {
6994 switch(elem_bt) {
6995 case T_BYTE:
6996 if (ideal_opc == Op_SaturatingAddV) {
6997 vpaddusb(dst, src1, src2, vlen_enc);
6998 } else {
6999 assert(ideal_opc == Op_SaturatingSubV, "");
7000 vpsubusb(dst, src1, src2, vlen_enc);
7001 }
7002 break;
7003 case T_SHORT:
7004 if (ideal_opc == Op_SaturatingAddV) {
7005 vpaddusw(dst, src1, src2, vlen_enc);
7006 } else {
7007 assert(ideal_opc == Op_SaturatingSubV, "");
7008 vpsubusw(dst, src1, src2, vlen_enc);
7009 }
7010 break;
7011 default:
7012 fatal("Unsupported type %s", type2name(elem_bt));
7013 break;
7014 }
7015 }
7016
7017 void C2_MacroAssembler::select_from_two_vectors_evex(BasicType elem_bt, XMMRegister dst, XMMRegister src1,
7018 XMMRegister src2, int vlen_enc) {
7019 switch(elem_bt) {
7020 case T_BYTE:
7021 evpermi2b(dst, src1, src2, vlen_enc);
7022 break;
7023 case T_SHORT:
7024 evpermi2w(dst, src1, src2, vlen_enc);
7025 break;
7026 case T_INT:
7027 evpermi2d(dst, src1, src2, vlen_enc);
7028 break;
7029 case T_LONG:
7030 evpermi2q(dst, src1, src2, vlen_enc);
7031 break;
7032 case T_FLOAT:
7033 evpermi2ps(dst, src1, src2, vlen_enc);
7034 break;
7035 case T_DOUBLE:
7036 evpermi2pd(dst, src1, src2, vlen_enc);
7037 break;
7038 default:
7039 fatal("Unsupported type %s", type2name(elem_bt));
7040 break;
7041 }
7042 }
7043
7044 void C2_MacroAssembler::vector_saturating_op(int ideal_opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2, bool is_unsigned, int vlen_enc) {
7045 if (is_unsigned) {
7046 vector_saturating_unsigned_op(ideal_opc, elem_bt, dst, src1, src2, vlen_enc);
7047 } else {
7048 vector_saturating_op(ideal_opc, elem_bt, dst, src1, src2, vlen_enc);
7049 }
7050 }
7051
7052 void C2_MacroAssembler::vector_saturating_op(int ideal_opc, BasicType elem_bt, XMMRegister dst, XMMRegister src1, Address src2, bool is_unsigned, int vlen_enc) {
7053 if (is_unsigned) {
7054 vector_saturating_unsigned_op(ideal_opc, elem_bt, dst, src1, src2, vlen_enc);
7055 } else {
7056 vector_saturating_op(ideal_opc, elem_bt, dst, src1, src2, vlen_enc);
7057 }
7058 }
7059
7060 void C2_MacroAssembler::evfp16ph(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2, int vlen_enc) {
7061 switch(opcode) {
7062 case Op_AddVHF: evaddph(dst, src1, src2, vlen_enc); break;
7063 case Op_SubVHF: evsubph(dst, src1, src2, vlen_enc); break;
7064 case Op_MulVHF: evmulph(dst, src1, src2, vlen_enc); break;
7065 case Op_DivVHF: evdivph(dst, src1, src2, vlen_enc); break;
7066 default: assert(false, "%s", NodeClassNames[opcode]); break;
7067 }
7068 }
7069
7070 void C2_MacroAssembler::evfp16ph(int opcode, XMMRegister dst, XMMRegister src1, Address src2, int vlen_enc) {
7071 switch(opcode) {
7072 case Op_AddVHF: evaddph(dst, src1, src2, vlen_enc); break;
7073 case Op_SubVHF: evsubph(dst, src1, src2, vlen_enc); break;
7074 case Op_MulVHF: evmulph(dst, src1, src2, vlen_enc); break;
7075 case Op_DivVHF: evdivph(dst, src1, src2, vlen_enc); break;
7076 default: assert(false, "%s", NodeClassNames[opcode]); break;
7077 }
7078 }
7079
7080 void C2_MacroAssembler::sminmax_fp16(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
7081 KRegister ktmp, XMMRegister xtmp1, XMMRegister xtmp2) {
7082 vminmax_fp16(opcode, dst, src1, src2, ktmp, xtmp1, xtmp2, Assembler::AVX_128bit);
7083 }
7084
7085 void C2_MacroAssembler::sminmax_fp16_avx10_2(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
7086 KRegister ktmp) {
7087 if (opcode == Op_MaxHF) {
7088 // dst = max(src1, src2)
7089 evminmaxsh(dst, ktmp, src1, src2, true, AVX10_2_MINMAX_MAX_COMPARE_SIGN);
7090 } else {
7091 assert(opcode == Op_MinHF, "");
7092 // dst = min(src1, src2)
7093 evminmaxsh(dst, ktmp, src1, src2, true, AVX10_2_MINMAX_MIN_COMPARE_SIGN);
7094 }
7095 }
7096
7097 void C2_MacroAssembler::vminmax_fp16(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
7098 KRegister ktmp, XMMRegister xtmp1, XMMRegister xtmp2, int vlen_enc) {
7099 if (opcode == Op_MaxVHF || opcode == Op_MaxHF) {
7100 // Move sign bits of src2 to mask register.
7101 evpmovw2m(ktmp, src2, vlen_enc);
7102 // xtmp1 = src2 < 0 ? src2 : src1
7103 evpblendmw(xtmp1, ktmp, src1, src2, true, vlen_enc);
7104 // xtmp2 = src2 < 0 ? ? src1 : src2
7105 evpblendmw(xtmp2, ktmp, src2, src1, true, vlen_enc);
7106 // Idea behind above swapping is to make seconds source operand a +ve value.
7107 // As per instruction semantic, if the values being compared are both 0.0s (of either sign), the value in
7108 // the second source operand is returned. If only one value is a NaN (SNaN or QNaN) for this instruction,
7109 // the second source operand, either a NaN or a valid floating-point value, is returned
7110 // dst = max(xtmp1, xtmp2)
7111 evmaxph(dst, xtmp1, xtmp2, vlen_enc);
7112 // isNaN = is_unordered_quiet(xtmp1)
7113 evcmpph(ktmp, k0, xtmp1, xtmp1, Assembler::UNORD_Q, vlen_enc);
7114 // Final result is same as first source if its a NaN value,
7115 // in case second operand holds a NaN value then as per above semantics
7116 // result is same as second operand.
7117 Assembler::evmovdquw(dst, ktmp, xtmp1, true, vlen_enc);
7118 } else {
7119 assert(opcode == Op_MinVHF || opcode == Op_MinHF, "");
7120 // Move sign bits of src1 to mask register.
7121 evpmovw2m(ktmp, src1, vlen_enc);
7122 // xtmp1 = src1 < 0 ? src2 : src1
7123 evpblendmw(xtmp1, ktmp, src1, src2, true, vlen_enc);
7124 // xtmp2 = src1 < 0 ? src1 : src2
7125 evpblendmw(xtmp2, ktmp, src2, src1, true, vlen_enc);
7126 // Idea behind above swapping is to make seconds source operand a -ve value.
7127 // As per instruction semantics, if the values being compared are both 0.0s (of either sign), the value in
7128 // the second source operand is returned.
7129 // If only one value is a NaN (SNaN or QNaN) for this instruction, the second source operand, either a NaN
7130 // or a valid floating-point value, is written to the result.
7131 // dst = min(xtmp1, xtmp2)
7132 evminph(dst, xtmp1, xtmp2, vlen_enc);
7133 // isNaN = is_unordered_quiet(xtmp1)
7134 evcmpph(ktmp, k0, xtmp1, xtmp1, Assembler::UNORD_Q, vlen_enc);
7135 // Final result is same as first source if its a NaN value,
7136 // in case second operand holds a NaN value then as per above semantics
7137 // result is same as second operand.
7138 Assembler::evmovdquw(dst, ktmp, xtmp1, true, vlen_enc);
7139 }
7140 }
7141
7142 void C2_MacroAssembler::vminmax_fp16_avx10_2(int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2,
7143 KRegister ktmp, int vlen_enc) {
7144 if (opcode == Op_MaxVHF) {
7145 // dst = max(src1, src2)
7146 evminmaxph(dst, ktmp, src1, src2, true, AVX10_2_MINMAX_MAX_COMPARE_SIGN, vlen_enc);
7147 } else {
7148 assert(opcode == Op_MinVHF, "");
7149 // dst = min(src1, src2)
7150 evminmaxph(dst, ktmp, src1, src2, true, AVX10_2_MINMAX_MIN_COMPARE_SIGN, vlen_enc);
7151 }
7152 }
7153
7154 void C2_MacroAssembler::vminmax_fp16_avx10_2(int opcode, XMMRegister dst, XMMRegister src1, Address src2,
7155 KRegister ktmp, int vlen_enc) {
7156 if (opcode == Op_MaxVHF) {
7157 // dst = max(src1, src2)
7158 evminmaxph(dst, ktmp, src1, src2, true, AVX10_2_MINMAX_MAX_COMPARE_SIGN, vlen_enc);
7159 } else {
7160 assert(opcode == Op_MinVHF, "");
7161 // dst = min(src1, src2)
7162 evminmaxph(dst, ktmp, src1, src2, true, AVX10_2_MINMAX_MIN_COMPARE_SIGN, vlen_enc);
7163 }
7164 }
7165
7166 int C2_MacroAssembler::vector_iota_entry_index(BasicType bt) {
7167 // The vector iota entries array is ordered by type B/S/I/L/F/D, and
7168 // the offset between two types is 16.
7169 switch(bt) {
7170 case T_BYTE:
7171 return 0;
7172 case T_SHORT:
7173 return 1;
7174 case T_INT:
7175 return 2;
7176 case T_LONG:
7177 return 3;
7178 case T_FLOAT:
7179 return 4;
7180 case T_DOUBLE:
7181 return 5;
7182 default:
7183 ShouldNotReachHere();
7184 }
7185 }