1 //
2 // Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 //
5 // This code is free software; you can redistribute it and/or modify it
6 // under the terms of the GNU General Public License version 2 only, as
7 // published by the Free Software Foundation.
8 //
9 // This code is distributed in the hope that it will be useful, but WITHOUT
10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 // version 2 for more details (a copy is included in the LICENSE file that
13 // accompanied this code).
14 //
15 // You should have received a copy of the GNU General Public License version
16 // 2 along with this work; if not, write to the Free Software Foundation,
17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 //
19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 // or visit www.oracle.com if you need additional information or have any
21 // questions.
22 //
23 //
24
25 // AMD64 Architecture Description File
26
27 //----------REGISTER DEFINITION BLOCK------------------------------------------
28 // This information is used by the matcher and the register allocator to
29 // describe individual registers and classes of registers within the target
30 // architecture.
31
32 register %{
33 //----------Architecture Description Register Definitions----------------------
34 // General Registers
35 // "reg_def" name ( register save type, C convention save type,
36 // ideal register type, encoding );
37 // Register Save Types:
38 //
39 // NS = No-Save: The register allocator assumes that these registers
40 // can be used without saving upon entry to the method, &
41 // that they do not need to be saved at call sites.
42 //
43 // SOC = Save-On-Call: The register allocator assumes that these registers
44 // can be used without saving upon entry to the method,
45 // but that they must be saved at call sites.
46 //
47 // SOE = Save-On-Entry: The register allocator assumes that these registers
48 // must be saved before using them upon entry to the
49 // method, but they do not need to be saved at call
50 // sites.
51 //
52 // AS = Always-Save: The register allocator assumes that these registers
53 // must be saved before using them upon entry to the
54 // method, & that they must be saved at call sites.
55 //
56 // Ideal Register Type is used to determine how to save & restore a
57 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
58 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
59 //
60 // The encoding number is the actual bit-pattern placed into the opcodes.
61
62 // General Registers
63 // R8-R15 must be encoded with REX. (RSP, RBP, RSI, RDI need REX when
64 // used as byte registers)
65
66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
69
70 reg_def RAX (SOC, SOC, Op_RegI, 0, rax->as_VMReg());
71 reg_def RAX_H(SOC, SOC, Op_RegI, 0, rax->as_VMReg()->next());
72
73 reg_def RCX (SOC, SOC, Op_RegI, 1, rcx->as_VMReg());
74 reg_def RCX_H(SOC, SOC, Op_RegI, 1, rcx->as_VMReg()->next());
75
76 reg_def RDX (SOC, SOC, Op_RegI, 2, rdx->as_VMReg());
77 reg_def RDX_H(SOC, SOC, Op_RegI, 2, rdx->as_VMReg()->next());
78
79 reg_def RBX (SOC, SOE, Op_RegI, 3, rbx->as_VMReg());
80 reg_def RBX_H(SOC, SOE, Op_RegI, 3, rbx->as_VMReg()->next());
81
82 reg_def RSP (NS, NS, Op_RegI, 4, rsp->as_VMReg());
83 reg_def RSP_H(NS, NS, Op_RegI, 4, rsp->as_VMReg()->next());
84
85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
86 reg_def RBP (NS, SOE, Op_RegI, 5, rbp->as_VMReg());
87 reg_def RBP_H(NS, SOE, Op_RegI, 5, rbp->as_VMReg()->next());
88
89 #ifdef _WIN64
90
91 reg_def RSI (SOC, SOE, Op_RegI, 6, rsi->as_VMReg());
92 reg_def RSI_H(SOC, SOE, Op_RegI, 6, rsi->as_VMReg()->next());
93
94 reg_def RDI (SOC, SOE, Op_RegI, 7, rdi->as_VMReg());
95 reg_def RDI_H(SOC, SOE, Op_RegI, 7, rdi->as_VMReg()->next());
96
97 #else
98
99 reg_def RSI (SOC, SOC, Op_RegI, 6, rsi->as_VMReg());
100 reg_def RSI_H(SOC, SOC, Op_RegI, 6, rsi->as_VMReg()->next());
101
102 reg_def RDI (SOC, SOC, Op_RegI, 7, rdi->as_VMReg());
103 reg_def RDI_H(SOC, SOC, Op_RegI, 7, rdi->as_VMReg()->next());
104
105 #endif
106
107 reg_def R8 (SOC, SOC, Op_RegI, 8, r8->as_VMReg());
108 reg_def R8_H (SOC, SOC, Op_RegI, 8, r8->as_VMReg()->next());
109
110 reg_def R9 (SOC, SOC, Op_RegI, 9, r9->as_VMReg());
111 reg_def R9_H (SOC, SOC, Op_RegI, 9, r9->as_VMReg()->next());
112
113 reg_def R10 (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
115
116 reg_def R11 (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
118
119 reg_def R12 (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
121
122 reg_def R13 (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
124
125 reg_def R14 (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
127
128 reg_def R15 (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
130
131 reg_def R16 (SOC, SOC, Op_RegI, 16, r16->as_VMReg());
132 reg_def R16_H(SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
133
134 reg_def R17 (SOC, SOC, Op_RegI, 17, r17->as_VMReg());
135 reg_def R17_H(SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
136
137 reg_def R18 (SOC, SOC, Op_RegI, 18, r18->as_VMReg());
138 reg_def R18_H(SOC, SOC, Op_RegI, 18, r18->as_VMReg()->next());
139
140 reg_def R19 (SOC, SOC, Op_RegI, 19, r19->as_VMReg());
141 reg_def R19_H(SOC, SOC, Op_RegI, 19, r19->as_VMReg()->next());
142
143 reg_def R20 (SOC, SOC, Op_RegI, 20, r20->as_VMReg());
144 reg_def R20_H(SOC, SOC, Op_RegI, 20, r20->as_VMReg()->next());
145
146 reg_def R21 (SOC, SOC, Op_RegI, 21, r21->as_VMReg());
147 reg_def R21_H(SOC, SOC, Op_RegI, 21, r21->as_VMReg()->next());
148
149 reg_def R22 (SOC, SOC, Op_RegI, 22, r22->as_VMReg());
150 reg_def R22_H(SOC, SOC, Op_RegI, 22, r22->as_VMReg()->next());
151
152 reg_def R23 (SOC, SOC, Op_RegI, 23, r23->as_VMReg());
153 reg_def R23_H(SOC, SOC, Op_RegI, 23, r23->as_VMReg()->next());
154
155 reg_def R24 (SOC, SOC, Op_RegI, 24, r24->as_VMReg());
156 reg_def R24_H(SOC, SOC, Op_RegI, 24, r24->as_VMReg()->next());
157
158 reg_def R25 (SOC, SOC, Op_RegI, 25, r25->as_VMReg());
159 reg_def R25_H(SOC, SOC, Op_RegI, 25, r25->as_VMReg()->next());
160
161 reg_def R26 (SOC, SOC, Op_RegI, 26, r26->as_VMReg());
162 reg_def R26_H(SOC, SOC, Op_RegI, 26, r26->as_VMReg()->next());
163
164 reg_def R27 (SOC, SOC, Op_RegI, 27, r27->as_VMReg());
165 reg_def R27_H(SOC, SOC, Op_RegI, 27, r27->as_VMReg()->next());
166
167 reg_def R28 (SOC, SOC, Op_RegI, 28, r28->as_VMReg());
168 reg_def R28_H(SOC, SOC, Op_RegI, 28, r28->as_VMReg()->next());
169
170 reg_def R29 (SOC, SOC, Op_RegI, 29, r29->as_VMReg());
171 reg_def R29_H(SOC, SOC, Op_RegI, 29, r29->as_VMReg()->next());
172
173 reg_def R30 (SOC, SOC, Op_RegI, 30, r30->as_VMReg());
174 reg_def R30_H(SOC, SOC, Op_RegI, 30, r30->as_VMReg()->next());
175
176 reg_def R31 (SOC, SOC, Op_RegI, 31, r31->as_VMReg());
177 reg_def R31_H(SOC, SOC, Op_RegI, 31, r31->as_VMReg()->next());
178
179 // Floating Point Registers
180
181 // Specify priority of register selection within phases of register
182 // allocation. Highest priority is first. A useful heuristic is to
183 // give registers a low priority when they are required by machine
184 // instructions, like EAX and EDX on I486, and choose no-save registers
185 // before save-on-call, & save-on-call before save-on-entry. Registers
186 // which participate in fixed calling sequences should come last.
187 // Registers which are used as pairs must fall on an even boundary.
188
189 alloc_class chunk0(R10, R10_H,
190 R11, R11_H,
191 R8, R8_H,
192 R9, R9_H,
193 R12, R12_H,
194 RCX, RCX_H,
195 RBX, RBX_H,
196 RDI, RDI_H,
197 RDX, RDX_H,
198 RSI, RSI_H,
199 RAX, RAX_H,
200 RBP, RBP_H,
201 R13, R13_H,
202 R14, R14_H,
203 R15, R15_H,
204 R16, R16_H,
205 R17, R17_H,
206 R18, R18_H,
207 R19, R19_H,
208 R20, R20_H,
209 R21, R21_H,
210 R22, R22_H,
211 R23, R23_H,
212 R24, R24_H,
213 R25, R25_H,
214 R26, R26_H,
215 R27, R27_H,
216 R28, R28_H,
217 R29, R29_H,
218 R30, R30_H,
219 R31, R31_H,
220 RSP, RSP_H);
221
222
223 //----------Architecture Description Register Classes--------------------------
224 // Several register classes are automatically defined based upon information in
225 // this architecture description.
226 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
227 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
228 //
229
230 // Empty register class.
231 reg_class no_reg();
232
233 // Class for all pointer/long registers including APX extended GPRs.
234 reg_class all_reg(RAX, RAX_H,
235 RDX, RDX_H,
236 RBP, RBP_H,
237 RDI, RDI_H,
238 RSI, RSI_H,
239 RCX, RCX_H,
240 RBX, RBX_H,
241 RSP, RSP_H,
242 R8, R8_H,
243 R9, R9_H,
244 R10, R10_H,
245 R11, R11_H,
246 R12, R12_H,
247 R13, R13_H,
248 R14, R14_H,
249 R15, R15_H,
250 R16, R16_H,
251 R17, R17_H,
252 R18, R18_H,
253 R19, R19_H,
254 R20, R20_H,
255 R21, R21_H,
256 R22, R22_H,
257 R23, R23_H,
258 R24, R24_H,
259 R25, R25_H,
260 R26, R26_H,
261 R27, R27_H,
262 R28, R28_H,
263 R29, R29_H,
264 R30, R30_H,
265 R31, R31_H);
266
267 // Class for all int registers including APX extended GPRs.
268 reg_class all_int_reg(RAX
269 RDX,
270 RBP,
271 RDI,
272 RSI,
273 RCX,
274 RBX,
275 R8,
276 R9,
277 R10,
278 R11,
279 R12,
280 R13,
281 R14,
282 R16,
283 R17,
284 R18,
285 R19,
286 R20,
287 R21,
288 R22,
289 R23,
290 R24,
291 R25,
292 R26,
293 R27,
294 R28,
295 R29,
296 R30,
297 R31);
298
299 // Class for all pointer registers
300 reg_class any_reg %{
301 return _ANY_REG_mask;
302 %}
303
304 // Class for all pointer registers (excluding RSP)
305 reg_class ptr_reg %{
306 return _PTR_REG_mask;
307 %}
308
309 // Class for all pointer registers (excluding RSP and RBP)
310 reg_class ptr_reg_no_rbp %{
311 return _PTR_REG_NO_RBP_mask;
312 %}
313
314 // Class for all pointer registers (excluding RAX and RSP)
315 reg_class ptr_no_rax_reg %{
316 return _PTR_NO_RAX_REG_mask;
317 %}
318
319 // Class for all pointer registers (excluding RAX, RBX, and RSP)
320 reg_class ptr_no_rax_rbx_reg %{
321 return _PTR_NO_RAX_RBX_REG_mask;
322 %}
323
324 // Class for all long registers (excluding RSP)
325 reg_class long_reg %{
326 return _LONG_REG_mask;
327 %}
328
329 // Class for all long registers (excluding RAX, RDX and RSP)
330 reg_class long_no_rax_rdx_reg %{
331 return _LONG_NO_RAX_RDX_REG_mask;
332 %}
333
334 // Class for all long registers (excluding RCX and RSP)
335 reg_class long_no_rcx_reg %{
336 return _LONG_NO_RCX_REG_mask;
337 %}
338
339 // Class for all long registers (excluding RBP and R13)
340 reg_class long_no_rbp_r13_reg %{
341 return _LONG_NO_RBP_R13_REG_mask;
342 %}
343
344 // Class for all int registers (excluding RSP)
345 reg_class int_reg %{
346 return _INT_REG_mask;
347 %}
348
349 // Class for all int registers (excluding RAX, RDX, and RSP)
350 reg_class int_no_rax_rdx_reg %{
351 return _INT_NO_RAX_RDX_REG_mask;
352 %}
353
354 // Class for all int registers (excluding RCX and RSP)
355 reg_class int_no_rcx_reg %{
356 return _INT_NO_RCX_REG_mask;
357 %}
358
359 // Class for all int registers (excluding RBP and R13)
360 reg_class int_no_rbp_r13_reg %{
361 return _INT_NO_RBP_R13_REG_mask;
362 %}
363
364 // Singleton class for RAX pointer register
365 reg_class ptr_rax_reg(RAX, RAX_H);
366
367 // Singleton class for RBX pointer register
368 reg_class ptr_rbx_reg(RBX, RBX_H);
369
370 // Singleton class for RSI pointer register
371 reg_class ptr_rsi_reg(RSI, RSI_H);
372
373 // Singleton class for RBP pointer register
374 reg_class ptr_rbp_reg(RBP, RBP_H);
375
376 // Singleton class for RDI pointer register
377 reg_class ptr_rdi_reg(RDI, RDI_H);
378
379 // Singleton class for stack pointer
380 reg_class ptr_rsp_reg(RSP, RSP_H);
381
382 // Singleton class for TLS pointer
383 reg_class ptr_r15_reg(R15, R15_H);
384
385 // Singleton class for RAX long register
386 reg_class long_rax_reg(RAX, RAX_H);
387
388 // Singleton class for RCX long register
389 reg_class long_rcx_reg(RCX, RCX_H);
390
391 // Singleton class for RDX long register
392 reg_class long_rdx_reg(RDX, RDX_H);
393
394 // Singleton class for R11 long register
395 reg_class long_r11_reg(R11, R11_H);
396
397 // Singleton class for RAX int register
398 reg_class int_rax_reg(RAX);
399
400 // Singleton class for RBX int register
401 reg_class int_rbx_reg(RBX);
402
403 // Singleton class for RCX int register
404 reg_class int_rcx_reg(RCX);
405
406 // Singleton class for RDX int register
407 reg_class int_rdx_reg(RDX);
408
409 // Singleton class for RDI int register
410 reg_class int_rdi_reg(RDI);
411
412 // Singleton class for instruction pointer
413 // reg_class ip_reg(RIP);
414
415 %}
416
417 //----------SOURCE BLOCK-------------------------------------------------------
418 // This is a block of C++ code which provides values, functions, and
419 // definitions necessary in the rest of the architecture description
420
421 source_hpp %{
422
423 #include "peephole_x86_64.hpp"
424
425 %}
426
427 // Register masks
428 source_hpp %{
429 extern RegMask _ANY_REG_mask;
430 extern RegMask _PTR_REG_mask;
431 extern RegMask _PTR_REG_NO_RBP_mask;
432 extern RegMask _PTR_NO_RAX_REG_mask;
433 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
434 extern RegMask _LONG_REG_mask;
435 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
436 extern RegMask _LONG_NO_RCX_REG_mask;
437 extern RegMask _LONG_NO_RBP_R13_REG_mask;
438 extern RegMask _INT_REG_mask;
439 extern RegMask _INT_NO_RAX_RDX_REG_mask;
440 extern RegMask _INT_NO_RCX_REG_mask;
441 extern RegMask _INT_NO_RBP_R13_REG_mask;
442 extern RegMask _FLOAT_REG_mask;
443
444 extern RegMask _STACK_OR_PTR_REG_mask;
445 extern RegMask _STACK_OR_LONG_REG_mask;
446 extern RegMask _STACK_OR_INT_REG_mask;
447
448 inline const RegMask& STACK_OR_PTR_REG_mask() { return _STACK_OR_PTR_REG_mask; }
449 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
450 inline const RegMask& STACK_OR_INT_REG_mask() { return _STACK_OR_INT_REG_mask; }
451
452 %}
453
454 source %{
455 #define RELOC_IMM64 Assembler::imm_operand
456 #define RELOC_DISP32 Assembler::disp32_operand
457
458 #define __ masm->
459
460 RegMask _ANY_REG_mask;
461 RegMask _PTR_REG_mask;
462 RegMask _PTR_REG_NO_RBP_mask;
463 RegMask _PTR_NO_RAX_REG_mask;
464 RegMask _PTR_NO_RAX_RBX_REG_mask;
465 RegMask _LONG_REG_mask;
466 RegMask _LONG_NO_RAX_RDX_REG_mask;
467 RegMask _LONG_NO_RCX_REG_mask;
468 RegMask _LONG_NO_RBP_R13_REG_mask;
469 RegMask _INT_REG_mask;
470 RegMask _INT_NO_RAX_RDX_REG_mask;
471 RegMask _INT_NO_RCX_REG_mask;
472 RegMask _INT_NO_RBP_R13_REG_mask;
473 RegMask _FLOAT_REG_mask;
474 RegMask _STACK_OR_PTR_REG_mask;
475 RegMask _STACK_OR_LONG_REG_mask;
476 RegMask _STACK_OR_INT_REG_mask;
477
478 static bool need_r12_heapbase() {
479 return UseCompressedOops;
480 }
481
482 void reg_mask_init() {
483 constexpr Register egprs[] = {r16, r17, r18, r19, r20, r21, r22, r23, r24, r25, r26, r27, r28, r29, r30, r31};
484
485 // _ALL_REG_mask is generated by adlc from the all_reg register class below.
486 // We derive a number of subsets from it.
487 _ANY_REG_mask = _ALL_REG_mask;
488
489 if (PreserveFramePointer) {
490 _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
491 _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
492 }
493 if (need_r12_heapbase()) {
494 _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
495 _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
496 }
497
498 _PTR_REG_mask = _ANY_REG_mask;
499 _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
500 _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
501 _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
502 _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
503 if (!UseAPX) {
504 for (uint i = 0; i < sizeof(egprs)/sizeof(Register); i++) {
505 _PTR_REG_mask.Remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()));
506 _PTR_REG_mask.Remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()->next()));
507 }
508 }
509
510 _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
511 _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
512
513 _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
514 _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
515 _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
516
517 _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
518 _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
519 _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
520
521 _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
522 _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
523 _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
524
525
526 _LONG_REG_mask = _PTR_REG_mask;
527 _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
528 _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
529
530 _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
531 _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
532 _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
533 _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
534 _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
535
536 _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
537 _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
538 _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
539
540 _LONG_NO_RBP_R13_REG_mask = _LONG_REG_mask;
541 _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
542 _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
543 _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
544 _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()->next()));
545
546 _INT_REG_mask = _ALL_INT_REG_mask;
547 if (!UseAPX) {
548 for (uint i = 0; i < sizeof(egprs)/sizeof(Register); i++) {
549 _INT_REG_mask.Remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()));
550 }
551 }
552
553 if (PreserveFramePointer) {
554 _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
555 }
556 if (need_r12_heapbase()) {
557 _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
558 }
559
560 _STACK_OR_INT_REG_mask = _INT_REG_mask;
561 _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
562
563 _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
564 _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
565 _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
566
567 _INT_NO_RCX_REG_mask = _INT_REG_mask;
568 _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
569
570 _INT_NO_RBP_R13_REG_mask = _INT_REG_mask;
571 _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
572 _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
573
574 // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
575 // from the float_reg_legacy/float_reg_evex register class.
576 _FLOAT_REG_mask = VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask;
577 }
578
579 static bool generate_vzeroupper(Compile* C) {
580 return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false; // Generate vzeroupper
581 }
582
583 static int clear_avx_size() {
584 return generate_vzeroupper(Compile::current()) ? 3: 0; // vzeroupper
585 }
586
587 // !!!!! Special hack to get all types of calls to specify the byte offset
588 // from the start of the call to the point where the return address
589 // will point.
590 int MachCallStaticJavaNode::ret_addr_offset()
591 {
592 int offset = 5; // 5 bytes from start of call to where return address points
593 offset += clear_avx_size();
594 return offset;
595 }
596
597 int MachCallDynamicJavaNode::ret_addr_offset()
598 {
599 int offset = 15; // 15 bytes from start of call to where return address points
600 offset += clear_avx_size();
601 return offset;
602 }
603
604 int MachCallRuntimeNode::ret_addr_offset() {
605 int offset = 13; // movq r10,#addr; callq (r10)
606 if (this->ideal_Opcode() != Op_CallLeafVector) {
607 offset += clear_avx_size();
608 }
609 return offset;
610 }
611 //
612 // Compute padding required for nodes which need alignment
613 //
614
615 // The address of the call instruction needs to be 4-byte aligned to
616 // ensure that it does not span a cache line so that it can be patched.
617 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
618 {
619 current_offset += clear_avx_size(); // skip vzeroupper
620 current_offset += 1; // skip call opcode byte
621 return align_up(current_offset, alignment_required()) - current_offset;
622 }
623
624 // The address of the call instruction needs to be 4-byte aligned to
625 // ensure that it does not span a cache line so that it can be patched.
626 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
627 {
628 current_offset += clear_avx_size(); // skip vzeroupper
629 current_offset += 11; // skip movq instruction + call opcode byte
630 return align_up(current_offset, alignment_required()) - current_offset;
631 }
632
633 // This could be in MacroAssembler but it's fairly C2 specific
634 static void emit_cmpfp_fixup(MacroAssembler* masm) {
635 Label exit;
636 __ jccb(Assembler::noParity, exit);
637 __ pushf();
638 //
639 // comiss/ucomiss instructions set ZF,PF,CF flags and
640 // zero OF,AF,SF for NaN values.
641 // Fixup flags by zeroing ZF,PF so that compare of NaN
642 // values returns 'less than' result (CF is set).
643 // Leave the rest of flags unchanged.
644 //
645 // 7 6 5 4 3 2 1 0
646 // |S|Z|r|A|r|P|r|C| (r - reserved bit)
647 // 0 0 1 0 1 0 1 1 (0x2B)
648 //
649 __ andq(Address(rsp, 0), 0xffffff2b);
650 __ popf();
651 __ bind(exit);
652 }
653
654 static void emit_cmpfp3(MacroAssembler* masm, Register dst) {
655 Label done;
656 __ movl(dst, -1);
657 __ jcc(Assembler::parity, done);
658 __ jcc(Assembler::below, done);
659 __ setcc(Assembler::notEqual, dst);
660 __ bind(done);
661 }
662
663 // Math.min() # Math.max()
664 // --------------------------
665 // ucomis[s/d] #
666 // ja -> b # a
667 // jp -> NaN # NaN
668 // jb -> a # b
669 // je #
670 // |-jz -> a | b # a & b
671 // | -> a #
672 static void emit_fp_min_max(MacroAssembler* masm, XMMRegister dst,
673 XMMRegister a, XMMRegister b,
674 XMMRegister xmmt, Register rt,
675 bool min, bool single) {
676
677 Label nan, zero, below, above, done;
678
679 if (single)
680 __ ucomiss(a, b);
681 else
682 __ ucomisd(a, b);
683
684 if (dst->encoding() != (min ? b : a)->encoding())
685 __ jccb(Assembler::above, above); // CF=0 & ZF=0
686 else
687 __ jccb(Assembler::above, done);
688
689 __ jccb(Assembler::parity, nan); // PF=1
690 __ jccb(Assembler::below, below); // CF=1
691
692 // equal
693 __ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
694 if (single) {
695 __ ucomiss(a, xmmt);
696 __ jccb(Assembler::equal, zero);
697
698 __ movflt(dst, a);
699 __ jmp(done);
700 }
701 else {
702 __ ucomisd(a, xmmt);
703 __ jccb(Assembler::equal, zero);
704
705 __ movdbl(dst, a);
706 __ jmp(done);
707 }
708
709 __ bind(zero);
710 if (min)
711 __ vpor(dst, a, b, Assembler::AVX_128bit);
712 else
713 __ vpand(dst, a, b, Assembler::AVX_128bit);
714
715 __ jmp(done);
716
717 __ bind(above);
718 if (single)
719 __ movflt(dst, min ? b : a);
720 else
721 __ movdbl(dst, min ? b : a);
722
723 __ jmp(done);
724
725 __ bind(nan);
726 if (single) {
727 __ movl(rt, 0x7fc00000); // Float.NaN
728 __ movdl(dst, rt);
729 }
730 else {
731 __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
732 __ movdq(dst, rt);
733 }
734 __ jmp(done);
735
736 __ bind(below);
737 if (single)
738 __ movflt(dst, min ? a : b);
739 else
740 __ movdbl(dst, min ? a : b);
741
742 __ bind(done);
743 }
744
745 //=============================================================================
746 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
747
748 int ConstantTable::calculate_table_base_offset() const {
749 return 0; // absolute addressing, no offset
750 }
751
752 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
753 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
754 ShouldNotReachHere();
755 }
756
757 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
758 // Empty encoding
759 }
760
761 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
762 return 0;
763 }
764
765 #ifndef PRODUCT
766 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
767 st->print("# MachConstantBaseNode (empty encoding)");
768 }
769 #endif
770
771
772 //=============================================================================
773 #ifndef PRODUCT
774 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
775 Compile* C = ra_->C;
776
777 int framesize = C->output()->frame_size_in_bytes();
778 int bangsize = C->output()->bang_size_in_bytes();
779 assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
780 // Remove wordSize for return addr which is already pushed.
781 framesize -= wordSize;
782
783 if (C->output()->need_stack_bang(bangsize)) {
784 framesize -= wordSize;
785 st->print("# stack bang (%d bytes)", bangsize);
786 st->print("\n\t");
787 st->print("pushq rbp\t# Save rbp");
788 if (PreserveFramePointer) {
789 st->print("\n\t");
790 st->print("movq rbp, rsp\t# Save the caller's SP into rbp");
791 }
792 if (framesize) {
793 st->print("\n\t");
794 st->print("subq rsp, #%d\t# Create frame",framesize);
795 }
796 } else {
797 st->print("subq rsp, #%d\t# Create frame",framesize);
798 st->print("\n\t");
799 framesize -= wordSize;
800 st->print("movq [rsp + #%d], rbp\t# Save rbp",framesize);
801 if (PreserveFramePointer) {
802 st->print("\n\t");
803 st->print("movq rbp, rsp\t# Save the caller's SP into rbp");
804 if (framesize > 0) {
805 st->print("\n\t");
806 st->print("addq rbp, #%d", framesize);
807 }
808 }
809 }
810
811 if (VerifyStackAtCalls) {
812 st->print("\n\t");
813 framesize -= wordSize;
814 st->print("movq [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
815 #ifdef ASSERT
816 st->print("\n\t");
817 st->print("# stack alignment check");
818 #endif
819 }
820 if (C->stub_function() != nullptr && BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
821 st->print("\n\t");
822 st->print("cmpl [r15_thread + #disarmed_guard_value_offset], #disarmed_guard_value\t");
823 st->print("\n\t");
824 st->print("je fast_entry\t");
825 st->print("\n\t");
826 st->print("call #nmethod_entry_barrier_stub\t");
827 st->print("\n\tfast_entry:");
828 }
829 st->cr();
830 }
831 #endif
832
833 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
834 Compile* C = ra_->C;
835
836 int framesize = C->output()->frame_size_in_bytes();
837 int bangsize = C->output()->bang_size_in_bytes();
838
839 if (C->clinit_barrier_on_entry()) {
840 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
841 assert(!C->method()->holder()->is_not_initialized() || C->do_clinit_barriers(), "initialization should have been started");
842
843 Label L_skip_barrier;
844 Register klass = rscratch1;
845
846 __ mov_metadata(klass, C->method()->holder()->constant_encoding());
847 __ clinit_barrier(klass, r15_thread, &L_skip_barrier /*L_fast_path*/);
848
849 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
850
851 __ bind(L_skip_barrier);
852 }
853
854 __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != nullptr);
855
856 C->output()->set_frame_complete(__ offset());
857
858 if (C->has_mach_constant_base_node()) {
859 // NOTE: We set the table base offset here because users might be
860 // emitted before MachConstantBaseNode.
861 ConstantTable& constant_table = C->output()->constant_table();
862 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
863 }
864 }
865
866 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
867 {
868 return MachNode::size(ra_); // too many variables; just compute it
869 // the hard way
870 }
871
872 int MachPrologNode::reloc() const
873 {
874 return 0; // a large enough number
875 }
876
877 //=============================================================================
878 #ifndef PRODUCT
879 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
880 {
881 Compile* C = ra_->C;
882 if (generate_vzeroupper(C)) {
883 st->print("vzeroupper");
884 st->cr(); st->print("\t");
885 }
886
887 int framesize = C->output()->frame_size_in_bytes();
888 assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
889 // Remove word for return adr already pushed
890 // and RBP
891 framesize -= 2*wordSize;
892
893 if (framesize) {
894 st->print_cr("addq rsp, %d\t# Destroy frame", framesize);
895 st->print("\t");
896 }
897
898 st->print_cr("popq rbp");
899 if (do_polling() && C->is_method_compilation()) {
900 st->print("\t");
901 st->print_cr("cmpq rsp, poll_offset[r15_thread] \n\t"
902 "ja #safepoint_stub\t"
903 "# Safepoint: poll for GC");
904 }
905 }
906 #endif
907
908 void MachEpilogNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
909 {
910 Compile* C = ra_->C;
911
912 if (generate_vzeroupper(C)) {
913 // Clear upper bits of YMM registers when current compiled code uses
914 // wide vectors to avoid AVX <-> SSE transition penalty during call.
915 __ vzeroupper();
916 }
917
918 int framesize = C->output()->frame_size_in_bytes();
919 assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
920 // Remove word for return adr already pushed
921 // and RBP
922 framesize -= 2*wordSize;
923
924 // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
925
926 if (framesize) {
927 __ addq(rsp, framesize);
928 }
929
930 __ popq(rbp);
931
932 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
933 __ reserved_stack_check();
934 }
935
936 if (do_polling() && C->is_method_compilation()) {
937 Label dummy_label;
938 Label* code_stub = &dummy_label;
939 if (!C->output()->in_scratch_emit_size()) {
940 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
941 C->output()->add_stub(stub);
942 code_stub = &stub->entry();
943 }
944 __ relocate(relocInfo::poll_return_type);
945 __ safepoint_poll(*code_stub, r15_thread, true /* at_return */, true /* in_nmethod */);
946 }
947 }
948
949 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
950 {
951 return MachNode::size(ra_); // too many variables; just compute it
952 // the hard way
953 }
954
955 int MachEpilogNode::reloc() const
956 {
957 return 2; // a large enough number
958 }
959
960 const Pipeline* MachEpilogNode::pipeline() const
961 {
962 return MachNode::pipeline_class();
963 }
964
965 //=============================================================================
966
967 enum RC {
968 rc_bad,
969 rc_int,
970 rc_kreg,
971 rc_float,
972 rc_stack
973 };
974
975 static enum RC rc_class(OptoReg::Name reg)
976 {
977 if( !OptoReg::is_valid(reg) ) return rc_bad;
978
979 if (OptoReg::is_stack(reg)) return rc_stack;
980
981 VMReg r = OptoReg::as_VMReg(reg);
982
983 if (r->is_Register()) return rc_int;
984
985 if (r->is_KRegister()) return rc_kreg;
986
987 assert(r->is_XMMRegister(), "must be");
988 return rc_float;
989 }
990
991 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
992 static void vec_mov_helper(C2_MacroAssembler *masm, int src_lo, int dst_lo,
993 int src_hi, int dst_hi, uint ireg, outputStream* st);
994
995 void vec_spill_helper(C2_MacroAssembler *masm, bool is_load,
996 int stack_offset, int reg, uint ireg, outputStream* st);
997
998 static void vec_stack_to_stack_helper(C2_MacroAssembler *masm, int src_offset,
999 int dst_offset, uint ireg, outputStream* st) {
1000 if (masm) {
1001 switch (ireg) {
1002 case Op_VecS:
1003 __ movq(Address(rsp, -8), rax);
1004 __ movl(rax, Address(rsp, src_offset));
1005 __ movl(Address(rsp, dst_offset), rax);
1006 __ movq(rax, Address(rsp, -8));
1007 break;
1008 case Op_VecD:
1009 __ pushq(Address(rsp, src_offset));
1010 __ popq (Address(rsp, dst_offset));
1011 break;
1012 case Op_VecX:
1013 __ pushq(Address(rsp, src_offset));
1014 __ popq (Address(rsp, dst_offset));
1015 __ pushq(Address(rsp, src_offset+8));
1016 __ popq (Address(rsp, dst_offset+8));
1017 break;
1018 case Op_VecY:
1019 __ vmovdqu(Address(rsp, -32), xmm0);
1020 __ vmovdqu(xmm0, Address(rsp, src_offset));
1021 __ vmovdqu(Address(rsp, dst_offset), xmm0);
1022 __ vmovdqu(xmm0, Address(rsp, -32));
1023 break;
1024 case Op_VecZ:
1025 __ evmovdquq(Address(rsp, -64), xmm0, 2);
1026 __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
1027 __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
1028 __ evmovdquq(xmm0, Address(rsp, -64), 2);
1029 break;
1030 default:
1031 ShouldNotReachHere();
1032 }
1033 #ifndef PRODUCT
1034 } else {
1035 switch (ireg) {
1036 case Op_VecS:
1037 st->print("movq [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1038 "movl rax, [rsp + #%d]\n\t"
1039 "movl [rsp + #%d], rax\n\t"
1040 "movq rax, [rsp - #8]",
1041 src_offset, dst_offset);
1042 break;
1043 case Op_VecD:
1044 st->print("pushq [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1045 "popq [rsp + #%d]",
1046 src_offset, dst_offset);
1047 break;
1048 case Op_VecX:
1049 st->print("pushq [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
1050 "popq [rsp + #%d]\n\t"
1051 "pushq [rsp + #%d]\n\t"
1052 "popq [rsp + #%d]",
1053 src_offset, dst_offset, src_offset+8, dst_offset+8);
1054 break;
1055 case Op_VecY:
1056 st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
1057 "vmovdqu xmm0, [rsp + #%d]\n\t"
1058 "vmovdqu [rsp + #%d], xmm0\n\t"
1059 "vmovdqu xmm0, [rsp - #32]",
1060 src_offset, dst_offset);
1061 break;
1062 case Op_VecZ:
1063 st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
1064 "vmovdqu xmm0, [rsp + #%d]\n\t"
1065 "vmovdqu [rsp + #%d], xmm0\n\t"
1066 "vmovdqu xmm0, [rsp - #64]",
1067 src_offset, dst_offset);
1068 break;
1069 default:
1070 ShouldNotReachHere();
1071 }
1072 #endif
1073 }
1074 }
1075
1076 uint MachSpillCopyNode::implementation(C2_MacroAssembler* masm,
1077 PhaseRegAlloc* ra_,
1078 bool do_size,
1079 outputStream* st) const {
1080 assert(masm != nullptr || st != nullptr, "sanity");
1081 // Get registers to move
1082 OptoReg::Name src_second = ra_->get_reg_second(in(1));
1083 OptoReg::Name src_first = ra_->get_reg_first(in(1));
1084 OptoReg::Name dst_second = ra_->get_reg_second(this);
1085 OptoReg::Name dst_first = ra_->get_reg_first(this);
1086
1087 enum RC src_second_rc = rc_class(src_second);
1088 enum RC src_first_rc = rc_class(src_first);
1089 enum RC dst_second_rc = rc_class(dst_second);
1090 enum RC dst_first_rc = rc_class(dst_first);
1091
1092 assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
1093 "must move at least 1 register" );
1094
1095 if (src_first == dst_first && src_second == dst_second) {
1096 // Self copy, no move
1097 return 0;
1098 }
1099 if (bottom_type()->isa_vect() != nullptr && bottom_type()->isa_vectmask() == nullptr) {
1100 uint ireg = ideal_reg();
1101 assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
1102 assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
1103 if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
1104 // mem -> mem
1105 int src_offset = ra_->reg2offset(src_first);
1106 int dst_offset = ra_->reg2offset(dst_first);
1107 vec_stack_to_stack_helper(masm, src_offset, dst_offset, ireg, st);
1108 } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
1109 vec_mov_helper(masm, src_first, dst_first, src_second, dst_second, ireg, st);
1110 } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
1111 int stack_offset = ra_->reg2offset(dst_first);
1112 vec_spill_helper(masm, false, stack_offset, src_first, ireg, st);
1113 } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
1114 int stack_offset = ra_->reg2offset(src_first);
1115 vec_spill_helper(masm, true, stack_offset, dst_first, ireg, st);
1116 } else {
1117 ShouldNotReachHere();
1118 }
1119 return 0;
1120 }
1121 if (src_first_rc == rc_stack) {
1122 // mem ->
1123 if (dst_first_rc == rc_stack) {
1124 // mem -> mem
1125 assert(src_second != dst_first, "overlap");
1126 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1127 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1128 // 64-bit
1129 int src_offset = ra_->reg2offset(src_first);
1130 int dst_offset = ra_->reg2offset(dst_first);
1131 if (masm) {
1132 __ pushq(Address(rsp, src_offset));
1133 __ popq (Address(rsp, dst_offset));
1134 #ifndef PRODUCT
1135 } else {
1136 st->print("pushq [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1137 "popq [rsp + #%d]",
1138 src_offset, dst_offset);
1139 #endif
1140 }
1141 } else {
1142 // 32-bit
1143 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1144 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1145 // No pushl/popl, so:
1146 int src_offset = ra_->reg2offset(src_first);
1147 int dst_offset = ra_->reg2offset(dst_first);
1148 if (masm) {
1149 __ movq(Address(rsp, -8), rax);
1150 __ movl(rax, Address(rsp, src_offset));
1151 __ movl(Address(rsp, dst_offset), rax);
1152 __ movq(rax, Address(rsp, -8));
1153 #ifndef PRODUCT
1154 } else {
1155 st->print("movq [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1156 "movl rax, [rsp + #%d]\n\t"
1157 "movl [rsp + #%d], rax\n\t"
1158 "movq rax, [rsp - #8]",
1159 src_offset, dst_offset);
1160 #endif
1161 }
1162 }
1163 return 0;
1164 } else if (dst_first_rc == rc_int) {
1165 // mem -> gpr
1166 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1167 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1168 // 64-bit
1169 int offset = ra_->reg2offset(src_first);
1170 if (masm) {
1171 __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1172 #ifndef PRODUCT
1173 } else {
1174 st->print("movq %s, [rsp + #%d]\t# spill",
1175 Matcher::regName[dst_first],
1176 offset);
1177 #endif
1178 }
1179 } else {
1180 // 32-bit
1181 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1182 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1183 int offset = ra_->reg2offset(src_first);
1184 if (masm) {
1185 __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1186 #ifndef PRODUCT
1187 } else {
1188 st->print("movl %s, [rsp + #%d]\t# spill",
1189 Matcher::regName[dst_first],
1190 offset);
1191 #endif
1192 }
1193 }
1194 return 0;
1195 } else if (dst_first_rc == rc_float) {
1196 // mem-> xmm
1197 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1198 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1199 // 64-bit
1200 int offset = ra_->reg2offset(src_first);
1201 if (masm) {
1202 __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1203 #ifndef PRODUCT
1204 } else {
1205 st->print("%s %s, [rsp + #%d]\t# spill",
1206 UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
1207 Matcher::regName[dst_first],
1208 offset);
1209 #endif
1210 }
1211 } else {
1212 // 32-bit
1213 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1214 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1215 int offset = ra_->reg2offset(src_first);
1216 if (masm) {
1217 __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1218 #ifndef PRODUCT
1219 } else {
1220 st->print("movss %s, [rsp + #%d]\t# spill",
1221 Matcher::regName[dst_first],
1222 offset);
1223 #endif
1224 }
1225 }
1226 return 0;
1227 } else if (dst_first_rc == rc_kreg) {
1228 // mem -> kreg
1229 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1230 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1231 // 64-bit
1232 int offset = ra_->reg2offset(src_first);
1233 if (masm) {
1234 __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1235 #ifndef PRODUCT
1236 } else {
1237 st->print("kmovq %s, [rsp + #%d]\t# spill",
1238 Matcher::regName[dst_first],
1239 offset);
1240 #endif
1241 }
1242 }
1243 return 0;
1244 }
1245 } else if (src_first_rc == rc_int) {
1246 // gpr ->
1247 if (dst_first_rc == rc_stack) {
1248 // gpr -> mem
1249 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1250 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1251 // 64-bit
1252 int offset = ra_->reg2offset(dst_first);
1253 if (masm) {
1254 __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1255 #ifndef PRODUCT
1256 } else {
1257 st->print("movq [rsp + #%d], %s\t# spill",
1258 offset,
1259 Matcher::regName[src_first]);
1260 #endif
1261 }
1262 } else {
1263 // 32-bit
1264 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1265 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1266 int offset = ra_->reg2offset(dst_first);
1267 if (masm) {
1268 __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1269 #ifndef PRODUCT
1270 } else {
1271 st->print("movl [rsp + #%d], %s\t# spill",
1272 offset,
1273 Matcher::regName[src_first]);
1274 #endif
1275 }
1276 }
1277 return 0;
1278 } else if (dst_first_rc == rc_int) {
1279 // gpr -> gpr
1280 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1281 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1282 // 64-bit
1283 if (masm) {
1284 __ movq(as_Register(Matcher::_regEncode[dst_first]),
1285 as_Register(Matcher::_regEncode[src_first]));
1286 #ifndef PRODUCT
1287 } else {
1288 st->print("movq %s, %s\t# spill",
1289 Matcher::regName[dst_first],
1290 Matcher::regName[src_first]);
1291 #endif
1292 }
1293 return 0;
1294 } else {
1295 // 32-bit
1296 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1297 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1298 if (masm) {
1299 __ movl(as_Register(Matcher::_regEncode[dst_first]),
1300 as_Register(Matcher::_regEncode[src_first]));
1301 #ifndef PRODUCT
1302 } else {
1303 st->print("movl %s, %s\t# spill",
1304 Matcher::regName[dst_first],
1305 Matcher::regName[src_first]);
1306 #endif
1307 }
1308 return 0;
1309 }
1310 } else if (dst_first_rc == rc_float) {
1311 // gpr -> xmm
1312 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1313 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1314 // 64-bit
1315 if (masm) {
1316 __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1317 #ifndef PRODUCT
1318 } else {
1319 st->print("movdq %s, %s\t# spill",
1320 Matcher::regName[dst_first],
1321 Matcher::regName[src_first]);
1322 #endif
1323 }
1324 } else {
1325 // 32-bit
1326 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1327 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1328 if (masm) {
1329 __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1330 #ifndef PRODUCT
1331 } else {
1332 st->print("movdl %s, %s\t# spill",
1333 Matcher::regName[dst_first],
1334 Matcher::regName[src_first]);
1335 #endif
1336 }
1337 }
1338 return 0;
1339 } else if (dst_first_rc == rc_kreg) {
1340 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1341 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1342 // 64-bit
1343 if (masm) {
1344 __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1345 #ifndef PRODUCT
1346 } else {
1347 st->print("kmovq %s, %s\t# spill",
1348 Matcher::regName[dst_first],
1349 Matcher::regName[src_first]);
1350 #endif
1351 }
1352 }
1353 Unimplemented();
1354 return 0;
1355 }
1356 } else if (src_first_rc == rc_float) {
1357 // xmm ->
1358 if (dst_first_rc == rc_stack) {
1359 // xmm -> mem
1360 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1361 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1362 // 64-bit
1363 int offset = ra_->reg2offset(dst_first);
1364 if (masm) {
1365 __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1366 #ifndef PRODUCT
1367 } else {
1368 st->print("movsd [rsp + #%d], %s\t# spill",
1369 offset,
1370 Matcher::regName[src_first]);
1371 #endif
1372 }
1373 } else {
1374 // 32-bit
1375 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1376 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1377 int offset = ra_->reg2offset(dst_first);
1378 if (masm) {
1379 __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1380 #ifndef PRODUCT
1381 } else {
1382 st->print("movss [rsp + #%d], %s\t# spill",
1383 offset,
1384 Matcher::regName[src_first]);
1385 #endif
1386 }
1387 }
1388 return 0;
1389 } else if (dst_first_rc == rc_int) {
1390 // xmm -> gpr
1391 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1392 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1393 // 64-bit
1394 if (masm) {
1395 __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1396 #ifndef PRODUCT
1397 } else {
1398 st->print("movdq %s, %s\t# spill",
1399 Matcher::regName[dst_first],
1400 Matcher::regName[src_first]);
1401 #endif
1402 }
1403 } else {
1404 // 32-bit
1405 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1406 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1407 if (masm) {
1408 __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1409 #ifndef PRODUCT
1410 } else {
1411 st->print("movdl %s, %s\t# spill",
1412 Matcher::regName[dst_first],
1413 Matcher::regName[src_first]);
1414 #endif
1415 }
1416 }
1417 return 0;
1418 } else if (dst_first_rc == rc_float) {
1419 // xmm -> xmm
1420 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1421 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1422 // 64-bit
1423 if (masm) {
1424 __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1425 #ifndef PRODUCT
1426 } else {
1427 st->print("%s %s, %s\t# spill",
1428 UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
1429 Matcher::regName[dst_first],
1430 Matcher::regName[src_first]);
1431 #endif
1432 }
1433 } else {
1434 // 32-bit
1435 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1436 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1437 if (masm) {
1438 __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1439 #ifndef PRODUCT
1440 } else {
1441 st->print("%s %s, %s\t# spill",
1442 UseXmmRegToRegMoveAll ? "movaps" : "movss ",
1443 Matcher::regName[dst_first],
1444 Matcher::regName[src_first]);
1445 #endif
1446 }
1447 }
1448 return 0;
1449 } else if (dst_first_rc == rc_kreg) {
1450 assert(false, "Illegal spilling");
1451 return 0;
1452 }
1453 } else if (src_first_rc == rc_kreg) {
1454 if (dst_first_rc == rc_stack) {
1455 // mem -> kreg
1456 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1457 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1458 // 64-bit
1459 int offset = ra_->reg2offset(dst_first);
1460 if (masm) {
1461 __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
1462 #ifndef PRODUCT
1463 } else {
1464 st->print("kmovq [rsp + #%d] , %s\t# spill",
1465 offset,
1466 Matcher::regName[src_first]);
1467 #endif
1468 }
1469 }
1470 return 0;
1471 } else if (dst_first_rc == rc_int) {
1472 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1473 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1474 // 64-bit
1475 if (masm) {
1476 __ kmov(as_Register(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
1477 #ifndef PRODUCT
1478 } else {
1479 st->print("kmovq %s, %s\t# spill",
1480 Matcher::regName[dst_first],
1481 Matcher::regName[src_first]);
1482 #endif
1483 }
1484 }
1485 Unimplemented();
1486 return 0;
1487 } else if (dst_first_rc == rc_kreg) {
1488 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1489 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1490 // 64-bit
1491 if (masm) {
1492 __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
1493 #ifndef PRODUCT
1494 } else {
1495 st->print("kmovq %s, %s\t# spill",
1496 Matcher::regName[dst_first],
1497 Matcher::regName[src_first]);
1498 #endif
1499 }
1500 }
1501 return 0;
1502 } else if (dst_first_rc == rc_float) {
1503 assert(false, "Illegal spill");
1504 return 0;
1505 }
1506 }
1507
1508 assert(0," foo ");
1509 Unimplemented();
1510 return 0;
1511 }
1512
1513 #ifndef PRODUCT
1514 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
1515 implementation(nullptr, ra_, false, st);
1516 }
1517 #endif
1518
1519 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1520 implementation(masm, ra_, false, nullptr);
1521 }
1522
1523 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1524 return MachNode::size(ra_);
1525 }
1526
1527 //=============================================================================
1528 #ifndef PRODUCT
1529 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1530 {
1531 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1532 int reg = ra_->get_reg_first(this);
1533 st->print("leaq %s, [rsp + #%d]\t# box lock",
1534 Matcher::regName[reg], offset);
1535 }
1536 #endif
1537
1538 void BoxLockNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
1539 {
1540 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1541 int reg = ra_->get_encode(this);
1542
1543 __ lea(as_Register(reg), Address(rsp, offset));
1544 }
1545
1546 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
1547 {
1548 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1549 return (offset < 0x80) ? 5 : 8; // REX
1550 }
1551
1552 //=============================================================================
1553 #ifndef PRODUCT
1554 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1555 {
1556 if (UseCompressedClassPointers) {
1557 st->print_cr("movl rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1558 st->print_cr("\tcmpl rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
1559 } else {
1560 st->print_cr("movq rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1561 st->print_cr("\tcmpq rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
1562 }
1563 st->print_cr("\tjne SharedRuntime::_ic_miss_stub");
1564 }
1565 #endif
1566
1567 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
1568 {
1569 __ ic_check(InteriorEntryAlignment);
1570 }
1571
1572 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
1573 {
1574 return MachNode::size(ra_); // too many variables; just compute it
1575 // the hard way
1576 }
1577
1578
1579 //=============================================================================
1580
1581 bool Matcher::supports_vector_calling_convention(void) {
1582 if (EnableVectorSupport && UseVectorStubs) {
1583 return true;
1584 }
1585 return false;
1586 }
1587
1588 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
1589 assert(EnableVectorSupport && UseVectorStubs, "sanity");
1590 int lo = XMM0_num;
1591 int hi = XMM0b_num;
1592 if (ideal_reg == Op_VecX) hi = XMM0d_num;
1593 else if (ideal_reg == Op_VecY) hi = XMM0h_num;
1594 else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
1595 return OptoRegPair(hi, lo);
1596 }
1597
1598 // Is this branch offset short enough that a short branch can be used?
1599 //
1600 // NOTE: If the platform does not provide any short branch variants, then
1601 // this method should return false for offset 0.
1602 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1603 // The passed offset is relative to address of the branch.
1604 // On 86 a branch displacement is calculated relative to address
1605 // of a next instruction.
1606 offset -= br_size;
1607
1608 // the short version of jmpConUCF2 contains multiple branches,
1609 // making the reach slightly less
1610 if (rule == jmpConUCF2_rule)
1611 return (-126 <= offset && offset <= 125);
1612 return (-128 <= offset && offset <= 127);
1613 }
1614
1615 // Return whether or not this register is ever used as an argument.
1616 // This function is used on startup to build the trampoline stubs in
1617 // generateOptoStub. Registers not mentioned will be killed by the VM
1618 // call in the trampoline, and arguments in those registers not be
1619 // available to the callee.
1620 bool Matcher::can_be_java_arg(int reg)
1621 {
1622 return
1623 reg == RDI_num || reg == RDI_H_num ||
1624 reg == RSI_num || reg == RSI_H_num ||
1625 reg == RDX_num || reg == RDX_H_num ||
1626 reg == RCX_num || reg == RCX_H_num ||
1627 reg == R8_num || reg == R8_H_num ||
1628 reg == R9_num || reg == R9_H_num ||
1629 reg == R12_num || reg == R12_H_num ||
1630 reg == XMM0_num || reg == XMM0b_num ||
1631 reg == XMM1_num || reg == XMM1b_num ||
1632 reg == XMM2_num || reg == XMM2b_num ||
1633 reg == XMM3_num || reg == XMM3b_num ||
1634 reg == XMM4_num || reg == XMM4b_num ||
1635 reg == XMM5_num || reg == XMM5b_num ||
1636 reg == XMM6_num || reg == XMM6b_num ||
1637 reg == XMM7_num || reg == XMM7b_num;
1638 }
1639
1640 bool Matcher::is_spillable_arg(int reg)
1641 {
1642 return can_be_java_arg(reg);
1643 }
1644
1645 uint Matcher::int_pressure_limit()
1646 {
1647 return (INTPRESSURE == -1) ? _INT_REG_mask.Size() : INTPRESSURE;
1648 }
1649
1650 uint Matcher::float_pressure_limit()
1651 {
1652 // After experiment around with different values, the following default threshold
1653 // works best for LCM's register pressure scheduling on x64.
1654 uint dec_count = VM_Version::supports_evex() ? 4 : 2;
1655 uint default_float_pressure_threshold = _FLOAT_REG_mask.Size() - dec_count;
1656 return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
1657 }
1658
1659 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
1660 // In 64 bit mode a code which use multiply when
1661 // devisor is constant is faster than hardware
1662 // DIV instruction (it uses MulHiL).
1663 return false;
1664 }
1665
1666 // Register for DIVI projection of divmodI
1667 RegMask Matcher::divI_proj_mask() {
1668 return INT_RAX_REG_mask();
1669 }
1670
1671 // Register for MODI projection of divmodI
1672 RegMask Matcher::modI_proj_mask() {
1673 return INT_RDX_REG_mask();
1674 }
1675
1676 // Register for DIVL projection of divmodL
1677 RegMask Matcher::divL_proj_mask() {
1678 return LONG_RAX_REG_mask();
1679 }
1680
1681 // Register for MODL projection of divmodL
1682 RegMask Matcher::modL_proj_mask() {
1683 return LONG_RDX_REG_mask();
1684 }
1685
1686 // Register for saving SP into on method handle invokes. Not used on x86_64.
1687 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1688 return NO_REG_mask();
1689 }
1690
1691 %}
1692
1693 //----------ENCODING BLOCK-----------------------------------------------------
1694 // This block specifies the encoding classes used by the compiler to
1695 // output byte streams. Encoding classes are parameterized macros
1696 // used by Machine Instruction Nodes in order to generate the bit
1697 // encoding of the instruction. Operands specify their base encoding
1698 // interface with the interface keyword. There are currently
1699 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
1700 // COND_INTER. REG_INTER causes an operand to generate a function
1701 // which returns its register number when queried. CONST_INTER causes
1702 // an operand to generate a function which returns the value of the
1703 // constant when queried. MEMORY_INTER causes an operand to generate
1704 // four functions which return the Base Register, the Index Register,
1705 // the Scale Value, and the Offset Value of the operand when queried.
1706 // COND_INTER causes an operand to generate six functions which return
1707 // the encoding code (ie - encoding bits for the instruction)
1708 // associated with each basic boolean condition for a conditional
1709 // instruction.
1710 //
1711 // Instructions specify two basic values for encoding. Again, a
1712 // function is available to check if the constant displacement is an
1713 // oop. They use the ins_encode keyword to specify their encoding
1714 // classes (which must be a sequence of enc_class names, and their
1715 // parameters, specified in the encoding block), and they use the
1716 // opcode keyword to specify, in order, their primary, secondary, and
1717 // tertiary opcode. Only the opcode sections which a particular
1718 // instruction needs for encoding need to be specified.
1719 encode %{
1720 enc_class cdql_enc(no_rax_rdx_RegI div)
1721 %{
1722 // Full implementation of Java idiv and irem; checks for
1723 // special case as described in JVM spec., p.243 & p.271.
1724 //
1725 // normal case special case
1726 //
1727 // input : rax: dividend min_int
1728 // reg: divisor -1
1729 //
1730 // output: rax: quotient (= rax idiv reg) min_int
1731 // rdx: remainder (= rax irem reg) 0
1732 //
1733 // Code sequnce:
1734 //
1735 // 0: 3d 00 00 00 80 cmp $0x80000000,%eax
1736 // 5: 75 07/08 jne e <normal>
1737 // 7: 33 d2 xor %edx,%edx
1738 // [div >= 8 -> offset + 1]
1739 // [REX_B]
1740 // 9: 83 f9 ff cmp $0xffffffffffffffff,$div
1741 // c: 74 03/04 je 11 <done>
1742 // 000000000000000e <normal>:
1743 // e: 99 cltd
1744 // [div >= 8 -> offset + 1]
1745 // [REX_B]
1746 // f: f7 f9 idiv $div
1747 // 0000000000000011 <done>:
1748 Label normal;
1749 Label done;
1750
1751 // cmp $0x80000000,%eax
1752 __ cmpl(as_Register(RAX_enc), 0x80000000);
1753
1754 // jne e <normal>
1755 __ jccb(Assembler::notEqual, normal);
1756
1757 // xor %edx,%edx
1758 __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
1759
1760 // cmp $0xffffffffffffffff,%ecx
1761 __ cmpl($div$$Register, -1);
1762
1763 // je 11 <done>
1764 __ jccb(Assembler::equal, done);
1765
1766 // <normal>
1767 // cltd
1768 __ bind(normal);
1769 __ cdql();
1770
1771 // idivl
1772 // <done>
1773 __ idivl($div$$Register);
1774 __ bind(done);
1775 %}
1776
1777 enc_class cdqq_enc(no_rax_rdx_RegL div)
1778 %{
1779 // Full implementation of Java ldiv and lrem; checks for
1780 // special case as described in JVM spec., p.243 & p.271.
1781 //
1782 // normal case special case
1783 //
1784 // input : rax: dividend min_long
1785 // reg: divisor -1
1786 //
1787 // output: rax: quotient (= rax idiv reg) min_long
1788 // rdx: remainder (= rax irem reg) 0
1789 //
1790 // Code sequnce:
1791 //
1792 // 0: 48 ba 00 00 00 00 00 mov $0x8000000000000000,%rdx
1793 // 7: 00 00 80
1794 // a: 48 39 d0 cmp %rdx,%rax
1795 // d: 75 08 jne 17 <normal>
1796 // f: 33 d2 xor %edx,%edx
1797 // 11: 48 83 f9 ff cmp $0xffffffffffffffff,$div
1798 // 15: 74 05 je 1c <done>
1799 // 0000000000000017 <normal>:
1800 // 17: 48 99 cqto
1801 // 19: 48 f7 f9 idiv $div
1802 // 000000000000001c <done>:
1803 Label normal;
1804 Label done;
1805
1806 // mov $0x8000000000000000,%rdx
1807 __ mov64(as_Register(RDX_enc), 0x8000000000000000);
1808
1809 // cmp %rdx,%rax
1810 __ cmpq(as_Register(RAX_enc), as_Register(RDX_enc));
1811
1812 // jne 17 <normal>
1813 __ jccb(Assembler::notEqual, normal);
1814
1815 // xor %edx,%edx
1816 __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
1817
1818 // cmp $0xffffffffffffffff,$div
1819 __ cmpq($div$$Register, -1);
1820
1821 // je 1e <done>
1822 __ jccb(Assembler::equal, done);
1823
1824 // <normal>
1825 // cqto
1826 __ bind(normal);
1827 __ cdqq();
1828
1829 // idivq (note: must be emitted by the user of this rule)
1830 // <done>
1831 __ idivq($div$$Register);
1832 __ bind(done);
1833 %}
1834
1835 enc_class clear_avx %{
1836 debug_only(int off0 = __ offset());
1837 if (generate_vzeroupper(Compile::current())) {
1838 // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
1839 // Clear upper bits of YMM registers when current compiled code uses
1840 // wide vectors to avoid AVX <-> SSE transition penalty during call.
1841 __ vzeroupper();
1842 }
1843 debug_only(int off1 = __ offset());
1844 assert(off1 - off0 == clear_avx_size(), "correct size prediction");
1845 %}
1846
1847 enc_class Java_To_Runtime(method meth) %{
1848 // No relocation needed
1849 if (SCCache::is_on_for_write()) {
1850 // Created runtime_call_type relocation when caching code
1851 __ lea(r10, RuntimeAddress((address)$meth$$method));
1852 } else {
1853 __ mov64(r10, (int64_t) $meth$$method);
1854 }
1855 __ call(r10);
1856 __ post_call_nop();
1857 %}
1858
1859 enc_class Java_Static_Call(method meth)
1860 %{
1861 // JAVA STATIC CALL
1862 // CALL to fixup routine. Fixup routine uses ScopeDesc info to
1863 // determine who we intended to call.
1864 if (!_method) {
1865 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, $meth$$method)));
1866 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
1867 // The NOP here is purely to ensure that eliding a call to
1868 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
1869 __ addr_nop_5();
1870 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
1871 } else {
1872 int method_index = resolved_method_index(masm);
1873 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
1874 : static_call_Relocation::spec(method_index);
1875 address mark = __ pc();
1876 int call_offset = __ offset();
1877 __ call(AddressLiteral(CAST_FROM_FN_PTR(address, $meth$$method), rspec));
1878 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
1879 // Calls of the same statically bound method can share
1880 // a stub to the interpreter.
1881 __ code()->shared_stub_to_interp_for(_method, call_offset);
1882 } else {
1883 // Emit stubs for static call.
1884 address stub = CompiledDirectCall::emit_to_interp_stub(masm, mark);
1885 __ clear_inst_mark();
1886 if (stub == nullptr) {
1887 ciEnv::current()->record_failure("CodeCache is full");
1888 return;
1889 }
1890 }
1891 }
1892 __ post_call_nop();
1893 %}
1894
1895 enc_class Java_Dynamic_Call(method meth) %{
1896 __ ic_call((address)$meth$$method, resolved_method_index(masm));
1897 __ post_call_nop();
1898 %}
1899
1900 %}
1901
1902
1903
1904 //----------FRAME--------------------------------------------------------------
1905 // Definition of frame structure and management information.
1906 //
1907 // S T A C K L A Y O U T Allocators stack-slot number
1908 // | (to get allocators register number
1909 // G Owned by | | v add OptoReg::stack0())
1910 // r CALLER | |
1911 // o | +--------+ pad to even-align allocators stack-slot
1912 // w V | pad0 | numbers; owned by CALLER
1913 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
1914 // h ^ | in | 5
1915 // | | args | 4 Holes in incoming args owned by SELF
1916 // | | | | 3
1917 // | | +--------+
1918 // V | | old out| Empty on Intel, window on Sparc
1919 // | old |preserve| Must be even aligned.
1920 // | SP-+--------+----> Matcher::_old_SP, even aligned
1921 // | | in | 3 area for Intel ret address
1922 // Owned by |preserve| Empty on Sparc.
1923 // SELF +--------+
1924 // | | pad2 | 2 pad to align old SP
1925 // | +--------+ 1
1926 // | | locks | 0
1927 // | +--------+----> OptoReg::stack0(), even aligned
1928 // | | pad1 | 11 pad to align new SP
1929 // | +--------+
1930 // | | | 10
1931 // | | spills | 9 spills
1932 // V | | 8 (pad0 slot for callee)
1933 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
1934 // ^ | out | 7
1935 // | | args | 6 Holes in outgoing args owned by CALLEE
1936 // Owned by +--------+
1937 // CALLEE | new out| 6 Empty on Intel, window on Sparc
1938 // | new |preserve| Must be even-aligned.
1939 // | SP-+--------+----> Matcher::_new_SP, even aligned
1940 // | | |
1941 //
1942 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
1943 // known from SELF's arguments and the Java calling convention.
1944 // Region 6-7 is determined per call site.
1945 // Note 2: If the calling convention leaves holes in the incoming argument
1946 // area, those holes are owned by SELF. Holes in the outgoing area
1947 // are owned by the CALLEE. Holes should not be necessary in the
1948 // incoming area, as the Java calling convention is completely under
1949 // the control of the AD file. Doubles can be sorted and packed to
1950 // avoid holes. Holes in the outgoing arguments may be necessary for
1951 // varargs C calling conventions.
1952 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
1953 // even aligned with pad0 as needed.
1954 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
1955 // region 6-11 is even aligned; it may be padded out more so that
1956 // the region from SP to FP meets the minimum stack alignment.
1957 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
1958 // alignment. Region 11, pad1, may be dynamically extended so that
1959 // SP meets the minimum alignment.
1960
1961 frame
1962 %{
1963 // These three registers define part of the calling convention
1964 // between compiled code and the interpreter.
1965 inline_cache_reg(RAX); // Inline Cache Register
1966
1967 // Optional: name the operand used by cisc-spilling to access
1968 // [stack_pointer + offset]
1969 cisc_spilling_operand_name(indOffset32);
1970
1971 // Number of stack slots consumed by locking an object
1972 sync_stack_slots(2);
1973
1974 // Compiled code's Frame Pointer
1975 frame_pointer(RSP);
1976
1977 // Interpreter stores its frame pointer in a register which is
1978 // stored to the stack by I2CAdaptors.
1979 // I2CAdaptors convert from interpreted java to compiled java.
1980 interpreter_frame_pointer(RBP);
1981
1982 // Stack alignment requirement
1983 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
1984
1985 // Number of outgoing stack slots killed above the out_preserve_stack_slots
1986 // for calls to C. Supports the var-args backing area for register parms.
1987 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
1988
1989 // The after-PROLOG location of the return address. Location of
1990 // return address specifies a type (REG or STACK) and a number
1991 // representing the register number (i.e. - use a register name) or
1992 // stack slot.
1993 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
1994 // Otherwise, it is above the locks and verification slot and alignment word
1995 return_addr(STACK - 2 +
1996 align_up((Compile::current()->in_preserve_stack_slots() +
1997 Compile::current()->fixed_slots()),
1998 stack_alignment_in_slots()));
1999
2000 // Location of compiled Java return values. Same as C for now.
2001 return_value
2002 %{
2003 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
2004 "only return normal values");
2005
2006 static const int lo[Op_RegL + 1] = {
2007 0,
2008 0,
2009 RAX_num, // Op_RegN
2010 RAX_num, // Op_RegI
2011 RAX_num, // Op_RegP
2012 XMM0_num, // Op_RegF
2013 XMM0_num, // Op_RegD
2014 RAX_num // Op_RegL
2015 };
2016 static const int hi[Op_RegL + 1] = {
2017 0,
2018 0,
2019 OptoReg::Bad, // Op_RegN
2020 OptoReg::Bad, // Op_RegI
2021 RAX_H_num, // Op_RegP
2022 OptoReg::Bad, // Op_RegF
2023 XMM0b_num, // Op_RegD
2024 RAX_H_num // Op_RegL
2025 };
2026 // Excluded flags and vector registers.
2027 assert(ARRAY_SIZE(hi) == _last_machine_leaf - 8, "missing type");
2028 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
2029 %}
2030 %}
2031
2032 //----------ATTRIBUTES---------------------------------------------------------
2033 //----------Operand Attributes-------------------------------------------------
2034 op_attrib op_cost(0); // Required cost attribute
2035
2036 //----------Instruction Attributes---------------------------------------------
2037 ins_attrib ins_cost(100); // Required cost attribute
2038 ins_attrib ins_size(8); // Required size attribute (in bits)
2039 ins_attrib ins_short_branch(0); // Required flag: is this instruction
2040 // a non-matching short branch variant
2041 // of some long branch?
2042 ins_attrib ins_alignment(1); // Required alignment attribute (must
2043 // be a power of 2) specifies the
2044 // alignment that some part of the
2045 // instruction (not necessarily the
2046 // start) requires. If > 1, a
2047 // compute_padding() function must be
2048 // provided for the instruction
2049
2050 //----------OPERANDS-----------------------------------------------------------
2051 // Operand definitions must precede instruction definitions for correct parsing
2052 // in the ADLC because operands constitute user defined types which are used in
2053 // instruction definitions.
2054
2055 //----------Simple Operands----------------------------------------------------
2056 // Immediate Operands
2057 // Integer Immediate
2058 operand immI()
2059 %{
2060 match(ConI);
2061
2062 op_cost(10);
2063 format %{ %}
2064 interface(CONST_INTER);
2065 %}
2066
2067 // Constant for test vs zero
2068 operand immI_0()
2069 %{
2070 predicate(n->get_int() == 0);
2071 match(ConI);
2072
2073 op_cost(0);
2074 format %{ %}
2075 interface(CONST_INTER);
2076 %}
2077
2078 // Constant for increment
2079 operand immI_1()
2080 %{
2081 predicate(n->get_int() == 1);
2082 match(ConI);
2083
2084 op_cost(0);
2085 format %{ %}
2086 interface(CONST_INTER);
2087 %}
2088
2089 // Constant for decrement
2090 operand immI_M1()
2091 %{
2092 predicate(n->get_int() == -1);
2093 match(ConI);
2094
2095 op_cost(0);
2096 format %{ %}
2097 interface(CONST_INTER);
2098 %}
2099
2100 operand immI_2()
2101 %{
2102 predicate(n->get_int() == 2);
2103 match(ConI);
2104
2105 op_cost(0);
2106 format %{ %}
2107 interface(CONST_INTER);
2108 %}
2109
2110 operand immI_4()
2111 %{
2112 predicate(n->get_int() == 4);
2113 match(ConI);
2114
2115 op_cost(0);
2116 format %{ %}
2117 interface(CONST_INTER);
2118 %}
2119
2120 operand immI_8()
2121 %{
2122 predicate(n->get_int() == 8);
2123 match(ConI);
2124
2125 op_cost(0);
2126 format %{ %}
2127 interface(CONST_INTER);
2128 %}
2129
2130 // Valid scale values for addressing modes
2131 operand immI2()
2132 %{
2133 predicate(0 <= n->get_int() && (n->get_int() <= 3));
2134 match(ConI);
2135
2136 format %{ %}
2137 interface(CONST_INTER);
2138 %}
2139
2140 operand immU7()
2141 %{
2142 predicate((0 <= n->get_int()) && (n->get_int() <= 0x7F));
2143 match(ConI);
2144
2145 op_cost(5);
2146 format %{ %}
2147 interface(CONST_INTER);
2148 %}
2149
2150 operand immI8()
2151 %{
2152 predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
2153 match(ConI);
2154
2155 op_cost(5);
2156 format %{ %}
2157 interface(CONST_INTER);
2158 %}
2159
2160 operand immU8()
2161 %{
2162 predicate((0 <= n->get_int()) && (n->get_int() <= 255));
2163 match(ConI);
2164
2165 op_cost(5);
2166 format %{ %}
2167 interface(CONST_INTER);
2168 %}
2169
2170 operand immI16()
2171 %{
2172 predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
2173 match(ConI);
2174
2175 op_cost(10);
2176 format %{ %}
2177 interface(CONST_INTER);
2178 %}
2179
2180 // Int Immediate non-negative
2181 operand immU31()
2182 %{
2183 predicate(n->get_int() >= 0);
2184 match(ConI);
2185
2186 op_cost(0);
2187 format %{ %}
2188 interface(CONST_INTER);
2189 %}
2190
2191 // Pointer Immediate
2192 operand immP()
2193 %{
2194 match(ConP);
2195
2196 op_cost(10);
2197 format %{ %}
2198 interface(CONST_INTER);
2199 %}
2200
2201 // Null Pointer Immediate
2202 operand immP0()
2203 %{
2204 predicate(n->get_ptr() == 0);
2205 match(ConP);
2206
2207 op_cost(5);
2208 format %{ %}
2209 interface(CONST_INTER);
2210 %}
2211
2212 // Pointer Immediate
2213 operand immN() %{
2214 match(ConN);
2215
2216 op_cost(10);
2217 format %{ %}
2218 interface(CONST_INTER);
2219 %}
2220
2221 operand immNKlass() %{
2222 match(ConNKlass);
2223
2224 op_cost(10);
2225 format %{ %}
2226 interface(CONST_INTER);
2227 %}
2228
2229 // Null Pointer Immediate
2230 operand immN0() %{
2231 predicate(n->get_narrowcon() == 0);
2232 match(ConN);
2233
2234 op_cost(5);
2235 format %{ %}
2236 interface(CONST_INTER);
2237 %}
2238
2239 operand immP31()
2240 %{
2241 predicate(n->as_Type()->type()->reloc() == relocInfo::none
2242 && (n->get_ptr() >> 31) == 0);
2243 match(ConP);
2244
2245 op_cost(5);
2246 format %{ %}
2247 interface(CONST_INTER);
2248 %}
2249
2250
2251 // Long Immediate
2252 operand immL()
2253 %{
2254 match(ConL);
2255
2256 op_cost(20);
2257 format %{ %}
2258 interface(CONST_INTER);
2259 %}
2260
2261 // Long Immediate 8-bit
2262 operand immL8()
2263 %{
2264 predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
2265 match(ConL);
2266
2267 op_cost(5);
2268 format %{ %}
2269 interface(CONST_INTER);
2270 %}
2271
2272 // Long Immediate 32-bit unsigned
2273 operand immUL32()
2274 %{
2275 predicate(n->get_long() == (unsigned int) (n->get_long()));
2276 match(ConL);
2277
2278 op_cost(10);
2279 format %{ %}
2280 interface(CONST_INTER);
2281 %}
2282
2283 // Long Immediate 32-bit signed
2284 operand immL32()
2285 %{
2286 predicate(n->get_long() == (int) (n->get_long()));
2287 match(ConL);
2288
2289 op_cost(15);
2290 format %{ %}
2291 interface(CONST_INTER);
2292 %}
2293
2294 operand immL_Pow2()
2295 %{
2296 predicate(is_power_of_2((julong)n->get_long()));
2297 match(ConL);
2298
2299 op_cost(15);
2300 format %{ %}
2301 interface(CONST_INTER);
2302 %}
2303
2304 operand immL_NotPow2()
2305 %{
2306 predicate(is_power_of_2((julong)~n->get_long()));
2307 match(ConL);
2308
2309 op_cost(15);
2310 format %{ %}
2311 interface(CONST_INTER);
2312 %}
2313
2314 // Long Immediate zero
2315 operand immL0()
2316 %{
2317 predicate(n->get_long() == 0L);
2318 match(ConL);
2319
2320 op_cost(10);
2321 format %{ %}
2322 interface(CONST_INTER);
2323 %}
2324
2325 // Constant for increment
2326 operand immL1()
2327 %{
2328 predicate(n->get_long() == 1);
2329 match(ConL);
2330
2331 format %{ %}
2332 interface(CONST_INTER);
2333 %}
2334
2335 // Constant for decrement
2336 operand immL_M1()
2337 %{
2338 predicate(n->get_long() == -1);
2339 match(ConL);
2340
2341 format %{ %}
2342 interface(CONST_INTER);
2343 %}
2344
2345 // Long Immediate: low 32-bit mask
2346 operand immL_32bits()
2347 %{
2348 predicate(n->get_long() == 0xFFFFFFFFL);
2349 match(ConL);
2350 op_cost(20);
2351
2352 format %{ %}
2353 interface(CONST_INTER);
2354 %}
2355
2356 // Int Immediate: 2^n-1, positive
2357 operand immI_Pow2M1()
2358 %{
2359 predicate((n->get_int() > 0)
2360 && is_power_of_2((juint)n->get_int() + 1));
2361 match(ConI);
2362
2363 op_cost(20);
2364 format %{ %}
2365 interface(CONST_INTER);
2366 %}
2367
2368 // Float Immediate zero
2369 operand immF0()
2370 %{
2371 predicate(jint_cast(n->getf()) == 0);
2372 match(ConF);
2373
2374 op_cost(5);
2375 format %{ %}
2376 interface(CONST_INTER);
2377 %}
2378
2379 // Float Immediate
2380 operand immF()
2381 %{
2382 match(ConF);
2383
2384 op_cost(15);
2385 format %{ %}
2386 interface(CONST_INTER);
2387 %}
2388
2389 // Double Immediate zero
2390 operand immD0()
2391 %{
2392 predicate(jlong_cast(n->getd()) == 0);
2393 match(ConD);
2394
2395 op_cost(5);
2396 format %{ %}
2397 interface(CONST_INTER);
2398 %}
2399
2400 // Double Immediate
2401 operand immD()
2402 %{
2403 match(ConD);
2404
2405 op_cost(15);
2406 format %{ %}
2407 interface(CONST_INTER);
2408 %}
2409
2410 // Immediates for special shifts (sign extend)
2411
2412 // Constants for increment
2413 operand immI_16()
2414 %{
2415 predicate(n->get_int() == 16);
2416 match(ConI);
2417
2418 format %{ %}
2419 interface(CONST_INTER);
2420 %}
2421
2422 operand immI_24()
2423 %{
2424 predicate(n->get_int() == 24);
2425 match(ConI);
2426
2427 format %{ %}
2428 interface(CONST_INTER);
2429 %}
2430
2431 // Constant for byte-wide masking
2432 operand immI_255()
2433 %{
2434 predicate(n->get_int() == 255);
2435 match(ConI);
2436
2437 format %{ %}
2438 interface(CONST_INTER);
2439 %}
2440
2441 // Constant for short-wide masking
2442 operand immI_65535()
2443 %{
2444 predicate(n->get_int() == 65535);
2445 match(ConI);
2446
2447 format %{ %}
2448 interface(CONST_INTER);
2449 %}
2450
2451 // Constant for byte-wide masking
2452 operand immL_255()
2453 %{
2454 predicate(n->get_long() == 255);
2455 match(ConL);
2456
2457 format %{ %}
2458 interface(CONST_INTER);
2459 %}
2460
2461 // Constant for short-wide masking
2462 operand immL_65535()
2463 %{
2464 predicate(n->get_long() == 65535);
2465 match(ConL);
2466
2467 format %{ %}
2468 interface(CONST_INTER);
2469 %}
2470
2471 // AOT Runtime Constants Address
2472 operand immAOTRuntimeConstantsAddress()
2473 %{
2474 // Check if the address is in the range of AOT Runtime Constants
2475 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
2476 match(ConP);
2477
2478 op_cost(0);
2479 format %{ %}
2480 interface(CONST_INTER);
2481 %}
2482
2483 operand kReg()
2484 %{
2485 constraint(ALLOC_IN_RC(vectmask_reg));
2486 match(RegVectMask);
2487 format %{%}
2488 interface(REG_INTER);
2489 %}
2490
2491 // Register Operands
2492 // Integer Register
2493 operand rRegI()
2494 %{
2495 constraint(ALLOC_IN_RC(int_reg));
2496 match(RegI);
2497
2498 match(rax_RegI);
2499 match(rbx_RegI);
2500 match(rcx_RegI);
2501 match(rdx_RegI);
2502 match(rdi_RegI);
2503
2504 format %{ %}
2505 interface(REG_INTER);
2506 %}
2507
2508 // Special Registers
2509 operand rax_RegI()
2510 %{
2511 constraint(ALLOC_IN_RC(int_rax_reg));
2512 match(RegI);
2513 match(rRegI);
2514
2515 format %{ "RAX" %}
2516 interface(REG_INTER);
2517 %}
2518
2519 // Special Registers
2520 operand rbx_RegI()
2521 %{
2522 constraint(ALLOC_IN_RC(int_rbx_reg));
2523 match(RegI);
2524 match(rRegI);
2525
2526 format %{ "RBX" %}
2527 interface(REG_INTER);
2528 %}
2529
2530 operand rcx_RegI()
2531 %{
2532 constraint(ALLOC_IN_RC(int_rcx_reg));
2533 match(RegI);
2534 match(rRegI);
2535
2536 format %{ "RCX" %}
2537 interface(REG_INTER);
2538 %}
2539
2540 operand rdx_RegI()
2541 %{
2542 constraint(ALLOC_IN_RC(int_rdx_reg));
2543 match(RegI);
2544 match(rRegI);
2545
2546 format %{ "RDX" %}
2547 interface(REG_INTER);
2548 %}
2549
2550 operand rdi_RegI()
2551 %{
2552 constraint(ALLOC_IN_RC(int_rdi_reg));
2553 match(RegI);
2554 match(rRegI);
2555
2556 format %{ "RDI" %}
2557 interface(REG_INTER);
2558 %}
2559
2560 operand no_rax_rdx_RegI()
2561 %{
2562 constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
2563 match(RegI);
2564 match(rbx_RegI);
2565 match(rcx_RegI);
2566 match(rdi_RegI);
2567
2568 format %{ %}
2569 interface(REG_INTER);
2570 %}
2571
2572 operand no_rbp_r13_RegI()
2573 %{
2574 constraint(ALLOC_IN_RC(int_no_rbp_r13_reg));
2575 match(RegI);
2576 match(rRegI);
2577 match(rax_RegI);
2578 match(rbx_RegI);
2579 match(rcx_RegI);
2580 match(rdx_RegI);
2581 match(rdi_RegI);
2582
2583 format %{ %}
2584 interface(REG_INTER);
2585 %}
2586
2587 // Pointer Register
2588 operand any_RegP()
2589 %{
2590 constraint(ALLOC_IN_RC(any_reg));
2591 match(RegP);
2592 match(rax_RegP);
2593 match(rbx_RegP);
2594 match(rdi_RegP);
2595 match(rsi_RegP);
2596 match(rbp_RegP);
2597 match(r15_RegP);
2598 match(rRegP);
2599
2600 format %{ %}
2601 interface(REG_INTER);
2602 %}
2603
2604 operand rRegP()
2605 %{
2606 constraint(ALLOC_IN_RC(ptr_reg));
2607 match(RegP);
2608 match(rax_RegP);
2609 match(rbx_RegP);
2610 match(rdi_RegP);
2611 match(rsi_RegP);
2612 match(rbp_RegP); // See Q&A below about
2613 match(r15_RegP); // r15_RegP and rbp_RegP.
2614
2615 format %{ %}
2616 interface(REG_INTER);
2617 %}
2618
2619 operand rRegN() %{
2620 constraint(ALLOC_IN_RC(int_reg));
2621 match(RegN);
2622
2623 format %{ %}
2624 interface(REG_INTER);
2625 %}
2626
2627 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
2628 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
2629 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
2630 // The output of an instruction is controlled by the allocator, which respects
2631 // register class masks, not match rules. Unless an instruction mentions
2632 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
2633 // by the allocator as an input.
2634 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
2635 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
2636 // result, RBP is not included in the output of the instruction either.
2637
2638 // This operand is not allowed to use RBP even if
2639 // RBP is not used to hold the frame pointer.
2640 operand no_rbp_RegP()
2641 %{
2642 constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
2643 match(RegP);
2644 match(rbx_RegP);
2645 match(rsi_RegP);
2646 match(rdi_RegP);
2647
2648 format %{ %}
2649 interface(REG_INTER);
2650 %}
2651
2652 // Special Registers
2653 // Return a pointer value
2654 operand rax_RegP()
2655 %{
2656 constraint(ALLOC_IN_RC(ptr_rax_reg));
2657 match(RegP);
2658 match(rRegP);
2659
2660 format %{ %}
2661 interface(REG_INTER);
2662 %}
2663
2664 // Special Registers
2665 // Return a compressed pointer value
2666 operand rax_RegN()
2667 %{
2668 constraint(ALLOC_IN_RC(int_rax_reg));
2669 match(RegN);
2670 match(rRegN);
2671
2672 format %{ %}
2673 interface(REG_INTER);
2674 %}
2675
2676 // Used in AtomicAdd
2677 operand rbx_RegP()
2678 %{
2679 constraint(ALLOC_IN_RC(ptr_rbx_reg));
2680 match(RegP);
2681 match(rRegP);
2682
2683 format %{ %}
2684 interface(REG_INTER);
2685 %}
2686
2687 operand rsi_RegP()
2688 %{
2689 constraint(ALLOC_IN_RC(ptr_rsi_reg));
2690 match(RegP);
2691 match(rRegP);
2692
2693 format %{ %}
2694 interface(REG_INTER);
2695 %}
2696
2697 operand rbp_RegP()
2698 %{
2699 constraint(ALLOC_IN_RC(ptr_rbp_reg));
2700 match(RegP);
2701 match(rRegP);
2702
2703 format %{ %}
2704 interface(REG_INTER);
2705 %}
2706
2707 // Used in rep stosq
2708 operand rdi_RegP()
2709 %{
2710 constraint(ALLOC_IN_RC(ptr_rdi_reg));
2711 match(RegP);
2712 match(rRegP);
2713
2714 format %{ %}
2715 interface(REG_INTER);
2716 %}
2717
2718 operand r15_RegP()
2719 %{
2720 constraint(ALLOC_IN_RC(ptr_r15_reg));
2721 match(RegP);
2722 match(rRegP);
2723
2724 format %{ %}
2725 interface(REG_INTER);
2726 %}
2727
2728 operand rRegL()
2729 %{
2730 constraint(ALLOC_IN_RC(long_reg));
2731 match(RegL);
2732 match(rax_RegL);
2733 match(rdx_RegL);
2734
2735 format %{ %}
2736 interface(REG_INTER);
2737 %}
2738
2739 // Special Registers
2740 operand no_rax_rdx_RegL()
2741 %{
2742 constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
2743 match(RegL);
2744 match(rRegL);
2745
2746 format %{ %}
2747 interface(REG_INTER);
2748 %}
2749
2750 operand rax_RegL()
2751 %{
2752 constraint(ALLOC_IN_RC(long_rax_reg));
2753 match(RegL);
2754 match(rRegL);
2755
2756 format %{ "RAX" %}
2757 interface(REG_INTER);
2758 %}
2759
2760 operand rcx_RegL()
2761 %{
2762 constraint(ALLOC_IN_RC(long_rcx_reg));
2763 match(RegL);
2764 match(rRegL);
2765
2766 format %{ %}
2767 interface(REG_INTER);
2768 %}
2769
2770 operand rdx_RegL()
2771 %{
2772 constraint(ALLOC_IN_RC(long_rdx_reg));
2773 match(RegL);
2774 match(rRegL);
2775
2776 format %{ %}
2777 interface(REG_INTER);
2778 %}
2779
2780 operand r11_RegL()
2781 %{
2782 constraint(ALLOC_IN_RC(long_r11_reg));
2783 match(RegL);
2784 match(rRegL);
2785
2786 format %{ %}
2787 interface(REG_INTER);
2788 %}
2789
2790 operand no_rbp_r13_RegL()
2791 %{
2792 constraint(ALLOC_IN_RC(long_no_rbp_r13_reg));
2793 match(RegL);
2794 match(rRegL);
2795 match(rax_RegL);
2796 match(rcx_RegL);
2797 match(rdx_RegL);
2798
2799 format %{ %}
2800 interface(REG_INTER);
2801 %}
2802
2803 // Flags register, used as output of compare instructions
2804 operand rFlagsReg()
2805 %{
2806 constraint(ALLOC_IN_RC(int_flags));
2807 match(RegFlags);
2808
2809 format %{ "RFLAGS" %}
2810 interface(REG_INTER);
2811 %}
2812
2813 // Flags register, used as output of FLOATING POINT compare instructions
2814 operand rFlagsRegU()
2815 %{
2816 constraint(ALLOC_IN_RC(int_flags));
2817 match(RegFlags);
2818
2819 format %{ "RFLAGS_U" %}
2820 interface(REG_INTER);
2821 %}
2822
2823 operand rFlagsRegUCF() %{
2824 constraint(ALLOC_IN_RC(int_flags));
2825 match(RegFlags);
2826 predicate(false);
2827
2828 format %{ "RFLAGS_U_CF" %}
2829 interface(REG_INTER);
2830 %}
2831
2832 // Float register operands
2833 operand regF() %{
2834 constraint(ALLOC_IN_RC(float_reg));
2835 match(RegF);
2836
2837 format %{ %}
2838 interface(REG_INTER);
2839 %}
2840
2841 // Float register operands
2842 operand legRegF() %{
2843 constraint(ALLOC_IN_RC(float_reg_legacy));
2844 match(RegF);
2845
2846 format %{ %}
2847 interface(REG_INTER);
2848 %}
2849
2850 // Float register operands
2851 operand vlRegF() %{
2852 constraint(ALLOC_IN_RC(float_reg_vl));
2853 match(RegF);
2854
2855 format %{ %}
2856 interface(REG_INTER);
2857 %}
2858
2859 // Double register operands
2860 operand regD() %{
2861 constraint(ALLOC_IN_RC(double_reg));
2862 match(RegD);
2863
2864 format %{ %}
2865 interface(REG_INTER);
2866 %}
2867
2868 // Double register operands
2869 operand legRegD() %{
2870 constraint(ALLOC_IN_RC(double_reg_legacy));
2871 match(RegD);
2872
2873 format %{ %}
2874 interface(REG_INTER);
2875 %}
2876
2877 // Double register operands
2878 operand vlRegD() %{
2879 constraint(ALLOC_IN_RC(double_reg_vl));
2880 match(RegD);
2881
2882 format %{ %}
2883 interface(REG_INTER);
2884 %}
2885
2886 //----------Memory Operands----------------------------------------------------
2887 // Direct Memory Operand
2888 // operand direct(immP addr)
2889 // %{
2890 // match(addr);
2891
2892 // format %{ "[$addr]" %}
2893 // interface(MEMORY_INTER) %{
2894 // base(0xFFFFFFFF);
2895 // index(0x4);
2896 // scale(0x0);
2897 // disp($addr);
2898 // %}
2899 // %}
2900
2901 // Indirect Memory Operand
2902 operand indirect(any_RegP reg)
2903 %{
2904 constraint(ALLOC_IN_RC(ptr_reg));
2905 match(reg);
2906
2907 format %{ "[$reg]" %}
2908 interface(MEMORY_INTER) %{
2909 base($reg);
2910 index(0x4);
2911 scale(0x0);
2912 disp(0x0);
2913 %}
2914 %}
2915
2916 // Indirect Memory Plus Short Offset Operand
2917 operand indOffset8(any_RegP reg, immL8 off)
2918 %{
2919 constraint(ALLOC_IN_RC(ptr_reg));
2920 match(AddP reg off);
2921
2922 format %{ "[$reg + $off (8-bit)]" %}
2923 interface(MEMORY_INTER) %{
2924 base($reg);
2925 index(0x4);
2926 scale(0x0);
2927 disp($off);
2928 %}
2929 %}
2930
2931 // Indirect Memory Plus Long Offset Operand
2932 operand indOffset32(any_RegP reg, immL32 off)
2933 %{
2934 constraint(ALLOC_IN_RC(ptr_reg));
2935 match(AddP reg off);
2936
2937 format %{ "[$reg + $off (32-bit)]" %}
2938 interface(MEMORY_INTER) %{
2939 base($reg);
2940 index(0x4);
2941 scale(0x0);
2942 disp($off);
2943 %}
2944 %}
2945
2946 // Indirect Memory Plus Index Register Plus Offset Operand
2947 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
2948 %{
2949 constraint(ALLOC_IN_RC(ptr_reg));
2950 match(AddP (AddP reg lreg) off);
2951
2952 op_cost(10);
2953 format %{"[$reg + $off + $lreg]" %}
2954 interface(MEMORY_INTER) %{
2955 base($reg);
2956 index($lreg);
2957 scale(0x0);
2958 disp($off);
2959 %}
2960 %}
2961
2962 // Indirect Memory Plus Index Register Plus Offset Operand
2963 operand indIndex(any_RegP reg, rRegL lreg)
2964 %{
2965 constraint(ALLOC_IN_RC(ptr_reg));
2966 match(AddP reg lreg);
2967
2968 op_cost(10);
2969 format %{"[$reg + $lreg]" %}
2970 interface(MEMORY_INTER) %{
2971 base($reg);
2972 index($lreg);
2973 scale(0x0);
2974 disp(0x0);
2975 %}
2976 %}
2977
2978 // Indirect Memory Times Scale Plus Index Register
2979 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
2980 %{
2981 constraint(ALLOC_IN_RC(ptr_reg));
2982 match(AddP reg (LShiftL lreg scale));
2983
2984 op_cost(10);
2985 format %{"[$reg + $lreg << $scale]" %}
2986 interface(MEMORY_INTER) %{
2987 base($reg);
2988 index($lreg);
2989 scale($scale);
2990 disp(0x0);
2991 %}
2992 %}
2993
2994 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
2995 %{
2996 constraint(ALLOC_IN_RC(ptr_reg));
2997 predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
2998 match(AddP reg (LShiftL (ConvI2L idx) scale));
2999
3000 op_cost(10);
3001 format %{"[$reg + pos $idx << $scale]" %}
3002 interface(MEMORY_INTER) %{
3003 base($reg);
3004 index($idx);
3005 scale($scale);
3006 disp(0x0);
3007 %}
3008 %}
3009
3010 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3011 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
3012 %{
3013 constraint(ALLOC_IN_RC(ptr_reg));
3014 match(AddP (AddP reg (LShiftL lreg scale)) off);
3015
3016 op_cost(10);
3017 format %{"[$reg + $off + $lreg << $scale]" %}
3018 interface(MEMORY_INTER) %{
3019 base($reg);
3020 index($lreg);
3021 scale($scale);
3022 disp($off);
3023 %}
3024 %}
3025
3026 // Indirect Memory Plus Positive Index Register Plus Offset Operand
3027 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
3028 %{
3029 constraint(ALLOC_IN_RC(ptr_reg));
3030 predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
3031 match(AddP (AddP reg (ConvI2L idx)) off);
3032
3033 op_cost(10);
3034 format %{"[$reg + $off + $idx]" %}
3035 interface(MEMORY_INTER) %{
3036 base($reg);
3037 index($idx);
3038 scale(0x0);
3039 disp($off);
3040 %}
3041 %}
3042
3043 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3044 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
3045 %{
3046 constraint(ALLOC_IN_RC(ptr_reg));
3047 predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3048 match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
3049
3050 op_cost(10);
3051 format %{"[$reg + $off + $idx << $scale]" %}
3052 interface(MEMORY_INTER) %{
3053 base($reg);
3054 index($idx);
3055 scale($scale);
3056 disp($off);
3057 %}
3058 %}
3059
3060 // Indirect Narrow Oop Plus Offset Operand
3061 // Note: x86 architecture doesn't support "scale * index + offset" without a base
3062 // we can't free r12 even with CompressedOops::base() == nullptr.
3063 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
3064 predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
3065 constraint(ALLOC_IN_RC(ptr_reg));
3066 match(AddP (DecodeN reg) off);
3067
3068 op_cost(10);
3069 format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
3070 interface(MEMORY_INTER) %{
3071 base(0xc); // R12
3072 index($reg);
3073 scale(0x3);
3074 disp($off);
3075 %}
3076 %}
3077
3078 // Indirect Memory Operand
3079 operand indirectNarrow(rRegN reg)
3080 %{
3081 predicate(CompressedOops::shift() == 0);
3082 constraint(ALLOC_IN_RC(ptr_reg));
3083 match(DecodeN reg);
3084
3085 format %{ "[$reg]" %}
3086 interface(MEMORY_INTER) %{
3087 base($reg);
3088 index(0x4);
3089 scale(0x0);
3090 disp(0x0);
3091 %}
3092 %}
3093
3094 // Indirect Memory Plus Short Offset Operand
3095 operand indOffset8Narrow(rRegN reg, immL8 off)
3096 %{
3097 predicate(CompressedOops::shift() == 0);
3098 constraint(ALLOC_IN_RC(ptr_reg));
3099 match(AddP (DecodeN reg) off);
3100
3101 format %{ "[$reg + $off (8-bit)]" %}
3102 interface(MEMORY_INTER) %{
3103 base($reg);
3104 index(0x4);
3105 scale(0x0);
3106 disp($off);
3107 %}
3108 %}
3109
3110 // Indirect Memory Plus Long Offset Operand
3111 operand indOffset32Narrow(rRegN reg, immL32 off)
3112 %{
3113 predicate(CompressedOops::shift() == 0);
3114 constraint(ALLOC_IN_RC(ptr_reg));
3115 match(AddP (DecodeN reg) off);
3116
3117 format %{ "[$reg + $off (32-bit)]" %}
3118 interface(MEMORY_INTER) %{
3119 base($reg);
3120 index(0x4);
3121 scale(0x0);
3122 disp($off);
3123 %}
3124 %}
3125
3126 // Indirect Memory Plus Index Register Plus Offset Operand
3127 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
3128 %{
3129 predicate(CompressedOops::shift() == 0);
3130 constraint(ALLOC_IN_RC(ptr_reg));
3131 match(AddP (AddP (DecodeN reg) lreg) off);
3132
3133 op_cost(10);
3134 format %{"[$reg + $off + $lreg]" %}
3135 interface(MEMORY_INTER) %{
3136 base($reg);
3137 index($lreg);
3138 scale(0x0);
3139 disp($off);
3140 %}
3141 %}
3142
3143 // Indirect Memory Plus Index Register Plus Offset Operand
3144 operand indIndexNarrow(rRegN reg, rRegL lreg)
3145 %{
3146 predicate(CompressedOops::shift() == 0);
3147 constraint(ALLOC_IN_RC(ptr_reg));
3148 match(AddP (DecodeN reg) lreg);
3149
3150 op_cost(10);
3151 format %{"[$reg + $lreg]" %}
3152 interface(MEMORY_INTER) %{
3153 base($reg);
3154 index($lreg);
3155 scale(0x0);
3156 disp(0x0);
3157 %}
3158 %}
3159
3160 // Indirect Memory Times Scale Plus Index Register
3161 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
3162 %{
3163 predicate(CompressedOops::shift() == 0);
3164 constraint(ALLOC_IN_RC(ptr_reg));
3165 match(AddP (DecodeN reg) (LShiftL lreg scale));
3166
3167 op_cost(10);
3168 format %{"[$reg + $lreg << $scale]" %}
3169 interface(MEMORY_INTER) %{
3170 base($reg);
3171 index($lreg);
3172 scale($scale);
3173 disp(0x0);
3174 %}
3175 %}
3176
3177 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3178 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
3179 %{
3180 predicate(CompressedOops::shift() == 0);
3181 constraint(ALLOC_IN_RC(ptr_reg));
3182 match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
3183
3184 op_cost(10);
3185 format %{"[$reg + $off + $lreg << $scale]" %}
3186 interface(MEMORY_INTER) %{
3187 base($reg);
3188 index($lreg);
3189 scale($scale);
3190 disp($off);
3191 %}
3192 %}
3193
3194 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
3195 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
3196 %{
3197 constraint(ALLOC_IN_RC(ptr_reg));
3198 predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
3199 match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
3200
3201 op_cost(10);
3202 format %{"[$reg + $off + $idx]" %}
3203 interface(MEMORY_INTER) %{
3204 base($reg);
3205 index($idx);
3206 scale(0x0);
3207 disp($off);
3208 %}
3209 %}
3210
3211 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3212 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
3213 %{
3214 constraint(ALLOC_IN_RC(ptr_reg));
3215 predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3216 match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
3217
3218 op_cost(10);
3219 format %{"[$reg + $off + $idx << $scale]" %}
3220 interface(MEMORY_INTER) %{
3221 base($reg);
3222 index($idx);
3223 scale($scale);
3224 disp($off);
3225 %}
3226 %}
3227
3228 //----------Special Memory Operands--------------------------------------------
3229 // Stack Slot Operand - This operand is used for loading and storing temporary
3230 // values on the stack where a match requires a value to
3231 // flow through memory.
3232 operand stackSlotP(sRegP reg)
3233 %{
3234 constraint(ALLOC_IN_RC(stack_slots));
3235 // No match rule because this operand is only generated in matching
3236
3237 format %{ "[$reg]" %}
3238 interface(MEMORY_INTER) %{
3239 base(0x4); // RSP
3240 index(0x4); // No Index
3241 scale(0x0); // No Scale
3242 disp($reg); // Stack Offset
3243 %}
3244 %}
3245
3246 operand stackSlotI(sRegI reg)
3247 %{
3248 constraint(ALLOC_IN_RC(stack_slots));
3249 // No match rule because this operand is only generated in matching
3250
3251 format %{ "[$reg]" %}
3252 interface(MEMORY_INTER) %{
3253 base(0x4); // RSP
3254 index(0x4); // No Index
3255 scale(0x0); // No Scale
3256 disp($reg); // Stack Offset
3257 %}
3258 %}
3259
3260 operand stackSlotF(sRegF reg)
3261 %{
3262 constraint(ALLOC_IN_RC(stack_slots));
3263 // No match rule because this operand is only generated in matching
3264
3265 format %{ "[$reg]" %}
3266 interface(MEMORY_INTER) %{
3267 base(0x4); // RSP
3268 index(0x4); // No Index
3269 scale(0x0); // No Scale
3270 disp($reg); // Stack Offset
3271 %}
3272 %}
3273
3274 operand stackSlotD(sRegD reg)
3275 %{
3276 constraint(ALLOC_IN_RC(stack_slots));
3277 // No match rule because this operand is only generated in matching
3278
3279 format %{ "[$reg]" %}
3280 interface(MEMORY_INTER) %{
3281 base(0x4); // RSP
3282 index(0x4); // No Index
3283 scale(0x0); // No Scale
3284 disp($reg); // Stack Offset
3285 %}
3286 %}
3287 operand stackSlotL(sRegL reg)
3288 %{
3289 constraint(ALLOC_IN_RC(stack_slots));
3290 // No match rule because this operand is only generated in matching
3291
3292 format %{ "[$reg]" %}
3293 interface(MEMORY_INTER) %{
3294 base(0x4); // RSP
3295 index(0x4); // No Index
3296 scale(0x0); // No Scale
3297 disp($reg); // Stack Offset
3298 %}
3299 %}
3300
3301 //----------Conditional Branch Operands----------------------------------------
3302 // Comparison Op - This is the operation of the comparison, and is limited to
3303 // the following set of codes:
3304 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
3305 //
3306 // Other attributes of the comparison, such as unsignedness, are specified
3307 // by the comparison instruction that sets a condition code flags register.
3308 // That result is represented by a flags operand whose subtype is appropriate
3309 // to the unsignedness (etc.) of the comparison.
3310 //
3311 // Later, the instruction which matches both the Comparison Op (a Bool) and
3312 // the flags (produced by the Cmp) specifies the coding of the comparison op
3313 // by matching a specific subtype of Bool operand below, such as cmpOpU.
3314
3315 // Comparison Code
3316 operand cmpOp()
3317 %{
3318 match(Bool);
3319
3320 format %{ "" %}
3321 interface(COND_INTER) %{
3322 equal(0x4, "e");
3323 not_equal(0x5, "ne");
3324 less(0xC, "l");
3325 greater_equal(0xD, "ge");
3326 less_equal(0xE, "le");
3327 greater(0xF, "g");
3328 overflow(0x0, "o");
3329 no_overflow(0x1, "no");
3330 %}
3331 %}
3332
3333 // Comparison Code, unsigned compare. Used by FP also, with
3334 // C2 (unordered) turned into GT or LT already. The other bits
3335 // C0 and C3 are turned into Carry & Zero flags.
3336 operand cmpOpU()
3337 %{
3338 match(Bool);
3339
3340 format %{ "" %}
3341 interface(COND_INTER) %{
3342 equal(0x4, "e");
3343 not_equal(0x5, "ne");
3344 less(0x2, "b");
3345 greater_equal(0x3, "ae");
3346 less_equal(0x6, "be");
3347 greater(0x7, "a");
3348 overflow(0x0, "o");
3349 no_overflow(0x1, "no");
3350 %}
3351 %}
3352
3353
3354 // Floating comparisons that don't require any fixup for the unordered case,
3355 // If both inputs of the comparison are the same, ZF is always set so we
3356 // don't need to use cmpOpUCF2 for eq/ne
3357 operand cmpOpUCF() %{
3358 match(Bool);
3359 predicate(n->as_Bool()->_test._test == BoolTest::lt ||
3360 n->as_Bool()->_test._test == BoolTest::ge ||
3361 n->as_Bool()->_test._test == BoolTest::le ||
3362 n->as_Bool()->_test._test == BoolTest::gt ||
3363 n->in(1)->in(1) == n->in(1)->in(2));
3364 format %{ "" %}
3365 interface(COND_INTER) %{
3366 equal(0xb, "np");
3367 not_equal(0xa, "p");
3368 less(0x2, "b");
3369 greater_equal(0x3, "ae");
3370 less_equal(0x6, "be");
3371 greater(0x7, "a");
3372 overflow(0x0, "o");
3373 no_overflow(0x1, "no");
3374 %}
3375 %}
3376
3377
3378 // Floating comparisons that can be fixed up with extra conditional jumps
3379 operand cmpOpUCF2() %{
3380 match(Bool);
3381 predicate((n->as_Bool()->_test._test == BoolTest::ne ||
3382 n->as_Bool()->_test._test == BoolTest::eq) &&
3383 n->in(1)->in(1) != n->in(1)->in(2));
3384 format %{ "" %}
3385 interface(COND_INTER) %{
3386 equal(0x4, "e");
3387 not_equal(0x5, "ne");
3388 less(0x2, "b");
3389 greater_equal(0x3, "ae");
3390 less_equal(0x6, "be");
3391 greater(0x7, "a");
3392 overflow(0x0, "o");
3393 no_overflow(0x1, "no");
3394 %}
3395 %}
3396
3397 //----------OPERAND CLASSES----------------------------------------------------
3398 // Operand Classes are groups of operands that are used as to simplify
3399 // instruction definitions by not requiring the AD writer to specify separate
3400 // instructions for every form of operand when the instruction accepts
3401 // multiple operand types with the same basic encoding and format. The classic
3402 // case of this is memory operands.
3403
3404 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
3405 indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
3406 indCompressedOopOffset,
3407 indirectNarrow, indOffset8Narrow, indOffset32Narrow,
3408 indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
3409 indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
3410
3411 //----------PIPELINE-----------------------------------------------------------
3412 // Rules which define the behavior of the target architectures pipeline.
3413 pipeline %{
3414
3415 //----------ATTRIBUTES---------------------------------------------------------
3416 attributes %{
3417 variable_size_instructions; // Fixed size instructions
3418 max_instructions_per_bundle = 3; // Up to 3 instructions per bundle
3419 instruction_unit_size = 1; // An instruction is 1 bytes long
3420 instruction_fetch_unit_size = 16; // The processor fetches one line
3421 instruction_fetch_units = 1; // of 16 bytes
3422
3423 // List of nop instructions
3424 nops( MachNop );
3425 %}
3426
3427 //----------RESOURCES----------------------------------------------------------
3428 // Resources are the functional units available to the machine
3429
3430 // Generic P2/P3 pipeline
3431 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
3432 // 3 instructions decoded per cycle.
3433 // 2 load/store ops per cycle, 1 branch, 1 FPU,
3434 // 3 ALU op, only ALU0 handles mul instructions.
3435 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
3436 MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
3437 BR, FPU,
3438 ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
3439
3440 //----------PIPELINE DESCRIPTION-----------------------------------------------
3441 // Pipeline Description specifies the stages in the machine's pipeline
3442
3443 // Generic P2/P3 pipeline
3444 pipe_desc(S0, S1, S2, S3, S4, S5);
3445
3446 //----------PIPELINE CLASSES---------------------------------------------------
3447 // Pipeline Classes describe the stages in which input and output are
3448 // referenced by the hardware pipeline.
3449
3450 // Naming convention: ialu or fpu
3451 // Then: _reg
3452 // Then: _reg if there is a 2nd register
3453 // Then: _long if it's a pair of instructions implementing a long
3454 // Then: _fat if it requires the big decoder
3455 // Or: _mem if it requires the big decoder and a memory unit.
3456
3457 // Integer ALU reg operation
3458 pipe_class ialu_reg(rRegI dst)
3459 %{
3460 single_instruction;
3461 dst : S4(write);
3462 dst : S3(read);
3463 DECODE : S0; // any decoder
3464 ALU : S3; // any alu
3465 %}
3466
3467 // Long ALU reg operation
3468 pipe_class ialu_reg_long(rRegL dst)
3469 %{
3470 instruction_count(2);
3471 dst : S4(write);
3472 dst : S3(read);
3473 DECODE : S0(2); // any 2 decoders
3474 ALU : S3(2); // both alus
3475 %}
3476
3477 // Integer ALU reg operation using big decoder
3478 pipe_class ialu_reg_fat(rRegI dst)
3479 %{
3480 single_instruction;
3481 dst : S4(write);
3482 dst : S3(read);
3483 D0 : S0; // big decoder only
3484 ALU : S3; // any alu
3485 %}
3486
3487 // Integer ALU reg-reg operation
3488 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
3489 %{
3490 single_instruction;
3491 dst : S4(write);
3492 src : S3(read);
3493 DECODE : S0; // any decoder
3494 ALU : S3; // any alu
3495 %}
3496
3497 // Integer ALU reg-reg operation
3498 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
3499 %{
3500 single_instruction;
3501 dst : S4(write);
3502 src : S3(read);
3503 D0 : S0; // big decoder only
3504 ALU : S3; // any alu
3505 %}
3506
3507 // Integer ALU reg-mem operation
3508 pipe_class ialu_reg_mem(rRegI dst, memory mem)
3509 %{
3510 single_instruction;
3511 dst : S5(write);
3512 mem : S3(read);
3513 D0 : S0; // big decoder only
3514 ALU : S4; // any alu
3515 MEM : S3; // any mem
3516 %}
3517
3518 // Integer mem operation (prefetch)
3519 pipe_class ialu_mem(memory mem)
3520 %{
3521 single_instruction;
3522 mem : S3(read);
3523 D0 : S0; // big decoder only
3524 MEM : S3; // any mem
3525 %}
3526
3527 // Integer Store to Memory
3528 pipe_class ialu_mem_reg(memory mem, rRegI src)
3529 %{
3530 single_instruction;
3531 mem : S3(read);
3532 src : S5(read);
3533 D0 : S0; // big decoder only
3534 ALU : S4; // any alu
3535 MEM : S3;
3536 %}
3537
3538 // // Long Store to Memory
3539 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
3540 // %{
3541 // instruction_count(2);
3542 // mem : S3(read);
3543 // src : S5(read);
3544 // D0 : S0(2); // big decoder only; twice
3545 // ALU : S4(2); // any 2 alus
3546 // MEM : S3(2); // Both mems
3547 // %}
3548
3549 // Integer Store to Memory
3550 pipe_class ialu_mem_imm(memory mem)
3551 %{
3552 single_instruction;
3553 mem : S3(read);
3554 D0 : S0; // big decoder only
3555 ALU : S4; // any alu
3556 MEM : S3;
3557 %}
3558
3559 // Integer ALU0 reg-reg operation
3560 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
3561 %{
3562 single_instruction;
3563 dst : S4(write);
3564 src : S3(read);
3565 D0 : S0; // Big decoder only
3566 ALU0 : S3; // only alu0
3567 %}
3568
3569 // Integer ALU0 reg-mem operation
3570 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
3571 %{
3572 single_instruction;
3573 dst : S5(write);
3574 mem : S3(read);
3575 D0 : S0; // big decoder only
3576 ALU0 : S4; // ALU0 only
3577 MEM : S3; // any mem
3578 %}
3579
3580 // Integer ALU reg-reg operation
3581 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
3582 %{
3583 single_instruction;
3584 cr : S4(write);
3585 src1 : S3(read);
3586 src2 : S3(read);
3587 DECODE : S0; // any decoder
3588 ALU : S3; // any alu
3589 %}
3590
3591 // Integer ALU reg-imm operation
3592 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
3593 %{
3594 single_instruction;
3595 cr : S4(write);
3596 src1 : S3(read);
3597 DECODE : S0; // any decoder
3598 ALU : S3; // any alu
3599 %}
3600
3601 // Integer ALU reg-mem operation
3602 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
3603 %{
3604 single_instruction;
3605 cr : S4(write);
3606 src1 : S3(read);
3607 src2 : S3(read);
3608 D0 : S0; // big decoder only
3609 ALU : S4; // any alu
3610 MEM : S3;
3611 %}
3612
3613 // Conditional move reg-reg
3614 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
3615 %{
3616 instruction_count(4);
3617 y : S4(read);
3618 q : S3(read);
3619 p : S3(read);
3620 DECODE : S0(4); // any decoder
3621 %}
3622
3623 // Conditional move reg-reg
3624 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
3625 %{
3626 single_instruction;
3627 dst : S4(write);
3628 src : S3(read);
3629 cr : S3(read);
3630 DECODE : S0; // any decoder
3631 %}
3632
3633 // Conditional move reg-mem
3634 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
3635 %{
3636 single_instruction;
3637 dst : S4(write);
3638 src : S3(read);
3639 cr : S3(read);
3640 DECODE : S0; // any decoder
3641 MEM : S3;
3642 %}
3643
3644 // Conditional move reg-reg long
3645 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
3646 %{
3647 single_instruction;
3648 dst : S4(write);
3649 src : S3(read);
3650 cr : S3(read);
3651 DECODE : S0(2); // any 2 decoders
3652 %}
3653
3654 // Float reg-reg operation
3655 pipe_class fpu_reg(regD dst)
3656 %{
3657 instruction_count(2);
3658 dst : S3(read);
3659 DECODE : S0(2); // any 2 decoders
3660 FPU : S3;
3661 %}
3662
3663 // Float reg-reg operation
3664 pipe_class fpu_reg_reg(regD dst, regD src)
3665 %{
3666 instruction_count(2);
3667 dst : S4(write);
3668 src : S3(read);
3669 DECODE : S0(2); // any 2 decoders
3670 FPU : S3;
3671 %}
3672
3673 // Float reg-reg operation
3674 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
3675 %{
3676 instruction_count(3);
3677 dst : S4(write);
3678 src1 : S3(read);
3679 src2 : S3(read);
3680 DECODE : S0(3); // any 3 decoders
3681 FPU : S3(2);
3682 %}
3683
3684 // Float reg-reg operation
3685 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
3686 %{
3687 instruction_count(4);
3688 dst : S4(write);
3689 src1 : S3(read);
3690 src2 : S3(read);
3691 src3 : S3(read);
3692 DECODE : S0(4); // any 3 decoders
3693 FPU : S3(2);
3694 %}
3695
3696 // Float reg-reg operation
3697 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
3698 %{
3699 instruction_count(4);
3700 dst : S4(write);
3701 src1 : S3(read);
3702 src2 : S3(read);
3703 src3 : S3(read);
3704 DECODE : S1(3); // any 3 decoders
3705 D0 : S0; // Big decoder only
3706 FPU : S3(2);
3707 MEM : S3;
3708 %}
3709
3710 // Float reg-mem operation
3711 pipe_class fpu_reg_mem(regD dst, memory mem)
3712 %{
3713 instruction_count(2);
3714 dst : S5(write);
3715 mem : S3(read);
3716 D0 : S0; // big decoder only
3717 DECODE : S1; // any decoder for FPU POP
3718 FPU : S4;
3719 MEM : S3; // any mem
3720 %}
3721
3722 // Float reg-mem operation
3723 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
3724 %{
3725 instruction_count(3);
3726 dst : S5(write);
3727 src1 : S3(read);
3728 mem : S3(read);
3729 D0 : S0; // big decoder only
3730 DECODE : S1(2); // any decoder for FPU POP
3731 FPU : S4;
3732 MEM : S3; // any mem
3733 %}
3734
3735 // Float mem-reg operation
3736 pipe_class fpu_mem_reg(memory mem, regD src)
3737 %{
3738 instruction_count(2);
3739 src : S5(read);
3740 mem : S3(read);
3741 DECODE : S0; // any decoder for FPU PUSH
3742 D0 : S1; // big decoder only
3743 FPU : S4;
3744 MEM : S3; // any mem
3745 %}
3746
3747 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
3748 %{
3749 instruction_count(3);
3750 src1 : S3(read);
3751 src2 : S3(read);
3752 mem : S3(read);
3753 DECODE : S0(2); // any decoder for FPU PUSH
3754 D0 : S1; // big decoder only
3755 FPU : S4;
3756 MEM : S3; // any mem
3757 %}
3758
3759 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
3760 %{
3761 instruction_count(3);
3762 src1 : S3(read);
3763 src2 : S3(read);
3764 mem : S4(read);
3765 DECODE : S0; // any decoder for FPU PUSH
3766 D0 : S0(2); // big decoder only
3767 FPU : S4;
3768 MEM : S3(2); // any mem
3769 %}
3770
3771 pipe_class fpu_mem_mem(memory dst, memory src1)
3772 %{
3773 instruction_count(2);
3774 src1 : S3(read);
3775 dst : S4(read);
3776 D0 : S0(2); // big decoder only
3777 MEM : S3(2); // any mem
3778 %}
3779
3780 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
3781 %{
3782 instruction_count(3);
3783 src1 : S3(read);
3784 src2 : S3(read);
3785 dst : S4(read);
3786 D0 : S0(3); // big decoder only
3787 FPU : S4;
3788 MEM : S3(3); // any mem
3789 %}
3790
3791 pipe_class fpu_mem_reg_con(memory mem, regD src1)
3792 %{
3793 instruction_count(3);
3794 src1 : S4(read);
3795 mem : S4(read);
3796 DECODE : S0; // any decoder for FPU PUSH
3797 D0 : S0(2); // big decoder only
3798 FPU : S4;
3799 MEM : S3(2); // any mem
3800 %}
3801
3802 // Float load constant
3803 pipe_class fpu_reg_con(regD dst)
3804 %{
3805 instruction_count(2);
3806 dst : S5(write);
3807 D0 : S0; // big decoder only for the load
3808 DECODE : S1; // any decoder for FPU POP
3809 FPU : S4;
3810 MEM : S3; // any mem
3811 %}
3812
3813 // Float load constant
3814 pipe_class fpu_reg_reg_con(regD dst, regD src)
3815 %{
3816 instruction_count(3);
3817 dst : S5(write);
3818 src : S3(read);
3819 D0 : S0; // big decoder only for the load
3820 DECODE : S1(2); // any decoder for FPU POP
3821 FPU : S4;
3822 MEM : S3; // any mem
3823 %}
3824
3825 // UnConditional branch
3826 pipe_class pipe_jmp(label labl)
3827 %{
3828 single_instruction;
3829 BR : S3;
3830 %}
3831
3832 // Conditional branch
3833 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
3834 %{
3835 single_instruction;
3836 cr : S1(read);
3837 BR : S3;
3838 %}
3839
3840 // Allocation idiom
3841 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
3842 %{
3843 instruction_count(1); force_serialization;
3844 fixed_latency(6);
3845 heap_ptr : S3(read);
3846 DECODE : S0(3);
3847 D0 : S2;
3848 MEM : S3;
3849 ALU : S3(2);
3850 dst : S5(write);
3851 BR : S5;
3852 %}
3853
3854 // Generic big/slow expanded idiom
3855 pipe_class pipe_slow()
3856 %{
3857 instruction_count(10); multiple_bundles; force_serialization;
3858 fixed_latency(100);
3859 D0 : S0(2);
3860 MEM : S3(2);
3861 %}
3862
3863 // The real do-nothing guy
3864 pipe_class empty()
3865 %{
3866 instruction_count(0);
3867 %}
3868
3869 // Define the class for the Nop node
3870 define
3871 %{
3872 MachNop = empty;
3873 %}
3874
3875 %}
3876
3877 //----------INSTRUCTIONS-------------------------------------------------------
3878 //
3879 // match -- States which machine-independent subtree may be replaced
3880 // by this instruction.
3881 // ins_cost -- The estimated cost of this instruction is used by instruction
3882 // selection to identify a minimum cost tree of machine
3883 // instructions that matches a tree of machine-independent
3884 // instructions.
3885 // format -- A string providing the disassembly for this instruction.
3886 // The value of an instruction's operand may be inserted
3887 // by referring to it with a '$' prefix.
3888 // opcode -- Three instruction opcodes may be provided. These are referred
3889 // to within an encode class as $primary, $secondary, and $tertiary
3890 // rrspectively. The primary opcode is commonly used to
3891 // indicate the type of machine instruction, while secondary
3892 // and tertiary are often used for prefix options or addressing
3893 // modes.
3894 // ins_encode -- A list of encode classes with parameters. The encode class
3895 // name must have been defined in an 'enc_class' specification
3896 // in the encode section of the architecture description.
3897
3898 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
3899 // Load Float
3900 instruct MoveF2VL(vlRegF dst, regF src) %{
3901 match(Set dst src);
3902 format %{ "movss $dst,$src\t! load float (4 bytes)" %}
3903 ins_encode %{
3904 ShouldNotReachHere();
3905 %}
3906 ins_pipe( fpu_reg_reg );
3907 %}
3908
3909 // Load Float
3910 instruct MoveF2LEG(legRegF dst, regF src) %{
3911 match(Set dst src);
3912 format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
3913 ins_encode %{
3914 ShouldNotReachHere();
3915 %}
3916 ins_pipe( fpu_reg_reg );
3917 %}
3918
3919 // Load Float
3920 instruct MoveVL2F(regF dst, vlRegF src) %{
3921 match(Set dst src);
3922 format %{ "movss $dst,$src\t! load float (4 bytes)" %}
3923 ins_encode %{
3924 ShouldNotReachHere();
3925 %}
3926 ins_pipe( fpu_reg_reg );
3927 %}
3928
3929 // Load Float
3930 instruct MoveLEG2F(regF dst, legRegF src) %{
3931 match(Set dst src);
3932 format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
3933 ins_encode %{
3934 ShouldNotReachHere();
3935 %}
3936 ins_pipe( fpu_reg_reg );
3937 %}
3938
3939 // Load Double
3940 instruct MoveD2VL(vlRegD dst, regD src) %{
3941 match(Set dst src);
3942 format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
3943 ins_encode %{
3944 ShouldNotReachHere();
3945 %}
3946 ins_pipe( fpu_reg_reg );
3947 %}
3948
3949 // Load Double
3950 instruct MoveD2LEG(legRegD dst, regD src) %{
3951 match(Set dst src);
3952 format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
3953 ins_encode %{
3954 ShouldNotReachHere();
3955 %}
3956 ins_pipe( fpu_reg_reg );
3957 %}
3958
3959 // Load Double
3960 instruct MoveVL2D(regD dst, vlRegD src) %{
3961 match(Set dst src);
3962 format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
3963 ins_encode %{
3964 ShouldNotReachHere();
3965 %}
3966 ins_pipe( fpu_reg_reg );
3967 %}
3968
3969 // Load Double
3970 instruct MoveLEG2D(regD dst, legRegD src) %{
3971 match(Set dst src);
3972 format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
3973 ins_encode %{
3974 ShouldNotReachHere();
3975 %}
3976 ins_pipe( fpu_reg_reg );
3977 %}
3978
3979 //----------Load/Store/Move Instructions---------------------------------------
3980 //----------Load Instructions--------------------------------------------------
3981
3982 // Load Byte (8 bit signed)
3983 instruct loadB(rRegI dst, memory mem)
3984 %{
3985 match(Set dst (LoadB mem));
3986
3987 ins_cost(125);
3988 format %{ "movsbl $dst, $mem\t# byte" %}
3989
3990 ins_encode %{
3991 __ movsbl($dst$$Register, $mem$$Address);
3992 %}
3993
3994 ins_pipe(ialu_reg_mem);
3995 %}
3996
3997 // Load Byte (8 bit signed) into Long Register
3998 instruct loadB2L(rRegL dst, memory mem)
3999 %{
4000 match(Set dst (ConvI2L (LoadB mem)));
4001
4002 ins_cost(125);
4003 format %{ "movsbq $dst, $mem\t# byte -> long" %}
4004
4005 ins_encode %{
4006 __ movsbq($dst$$Register, $mem$$Address);
4007 %}
4008
4009 ins_pipe(ialu_reg_mem);
4010 %}
4011
4012 // Load Unsigned Byte (8 bit UNsigned)
4013 instruct loadUB(rRegI dst, memory mem)
4014 %{
4015 match(Set dst (LoadUB mem));
4016
4017 ins_cost(125);
4018 format %{ "movzbl $dst, $mem\t# ubyte" %}
4019
4020 ins_encode %{
4021 __ movzbl($dst$$Register, $mem$$Address);
4022 %}
4023
4024 ins_pipe(ialu_reg_mem);
4025 %}
4026
4027 // Load Unsigned Byte (8 bit UNsigned) into Long Register
4028 instruct loadUB2L(rRegL dst, memory mem)
4029 %{
4030 match(Set dst (ConvI2L (LoadUB mem)));
4031
4032 ins_cost(125);
4033 format %{ "movzbq $dst, $mem\t# ubyte -> long" %}
4034
4035 ins_encode %{
4036 __ movzbq($dst$$Register, $mem$$Address);
4037 %}
4038
4039 ins_pipe(ialu_reg_mem);
4040 %}
4041
4042 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
4043 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
4044 match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
4045 effect(KILL cr);
4046
4047 format %{ "movzbq $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
4048 "andl $dst, right_n_bits($mask, 8)" %}
4049 ins_encode %{
4050 Register Rdst = $dst$$Register;
4051 __ movzbq(Rdst, $mem$$Address);
4052 __ andl(Rdst, $mask$$constant & right_n_bits(8));
4053 %}
4054 ins_pipe(ialu_reg_mem);
4055 %}
4056
4057 // Load Short (16 bit signed)
4058 instruct loadS(rRegI dst, memory mem)
4059 %{
4060 match(Set dst (LoadS mem));
4061
4062 ins_cost(125);
4063 format %{ "movswl $dst, $mem\t# short" %}
4064
4065 ins_encode %{
4066 __ movswl($dst$$Register, $mem$$Address);
4067 %}
4068
4069 ins_pipe(ialu_reg_mem);
4070 %}
4071
4072 // Load Short (16 bit signed) to Byte (8 bit signed)
4073 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4074 match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
4075
4076 ins_cost(125);
4077 format %{ "movsbl $dst, $mem\t# short -> byte" %}
4078 ins_encode %{
4079 __ movsbl($dst$$Register, $mem$$Address);
4080 %}
4081 ins_pipe(ialu_reg_mem);
4082 %}
4083
4084 // Load Short (16 bit signed) into Long Register
4085 instruct loadS2L(rRegL dst, memory mem)
4086 %{
4087 match(Set dst (ConvI2L (LoadS mem)));
4088
4089 ins_cost(125);
4090 format %{ "movswq $dst, $mem\t# short -> long" %}
4091
4092 ins_encode %{
4093 __ movswq($dst$$Register, $mem$$Address);
4094 %}
4095
4096 ins_pipe(ialu_reg_mem);
4097 %}
4098
4099 // Load Unsigned Short/Char (16 bit UNsigned)
4100 instruct loadUS(rRegI dst, memory mem)
4101 %{
4102 match(Set dst (LoadUS mem));
4103
4104 ins_cost(125);
4105 format %{ "movzwl $dst, $mem\t# ushort/char" %}
4106
4107 ins_encode %{
4108 __ movzwl($dst$$Register, $mem$$Address);
4109 %}
4110
4111 ins_pipe(ialu_reg_mem);
4112 %}
4113
4114 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
4115 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4116 match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
4117
4118 ins_cost(125);
4119 format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
4120 ins_encode %{
4121 __ movsbl($dst$$Register, $mem$$Address);
4122 %}
4123 ins_pipe(ialu_reg_mem);
4124 %}
4125
4126 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
4127 instruct loadUS2L(rRegL dst, memory mem)
4128 %{
4129 match(Set dst (ConvI2L (LoadUS mem)));
4130
4131 ins_cost(125);
4132 format %{ "movzwq $dst, $mem\t# ushort/char -> long" %}
4133
4134 ins_encode %{
4135 __ movzwq($dst$$Register, $mem$$Address);
4136 %}
4137
4138 ins_pipe(ialu_reg_mem);
4139 %}
4140
4141 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
4142 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
4143 match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4144
4145 format %{ "movzbq $dst, $mem\t# ushort/char & 0xFF -> long" %}
4146 ins_encode %{
4147 __ movzbq($dst$$Register, $mem$$Address);
4148 %}
4149 ins_pipe(ialu_reg_mem);
4150 %}
4151
4152 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
4153 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
4154 match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4155 effect(KILL cr);
4156
4157 format %{ "movzwq $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
4158 "andl $dst, right_n_bits($mask, 16)" %}
4159 ins_encode %{
4160 Register Rdst = $dst$$Register;
4161 __ movzwq(Rdst, $mem$$Address);
4162 __ andl(Rdst, $mask$$constant & right_n_bits(16));
4163 %}
4164 ins_pipe(ialu_reg_mem);
4165 %}
4166
4167 // Load Integer
4168 instruct loadI(rRegI dst, memory mem)
4169 %{
4170 match(Set dst (LoadI mem));
4171
4172 ins_cost(125);
4173 format %{ "movl $dst, $mem\t# int" %}
4174
4175 ins_encode %{
4176 __ movl($dst$$Register, $mem$$Address);
4177 %}
4178
4179 ins_pipe(ialu_reg_mem);
4180 %}
4181
4182 // Load Integer (32 bit signed) to Byte (8 bit signed)
4183 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4184 match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
4185
4186 ins_cost(125);
4187 format %{ "movsbl $dst, $mem\t# int -> byte" %}
4188 ins_encode %{
4189 __ movsbl($dst$$Register, $mem$$Address);
4190 %}
4191 ins_pipe(ialu_reg_mem);
4192 %}
4193
4194 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
4195 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
4196 match(Set dst (AndI (LoadI mem) mask));
4197
4198 ins_cost(125);
4199 format %{ "movzbl $dst, $mem\t# int -> ubyte" %}
4200 ins_encode %{
4201 __ movzbl($dst$$Register, $mem$$Address);
4202 %}
4203 ins_pipe(ialu_reg_mem);
4204 %}
4205
4206 // Load Integer (32 bit signed) to Short (16 bit signed)
4207 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
4208 match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
4209
4210 ins_cost(125);
4211 format %{ "movswl $dst, $mem\t# int -> short" %}
4212 ins_encode %{
4213 __ movswl($dst$$Register, $mem$$Address);
4214 %}
4215 ins_pipe(ialu_reg_mem);
4216 %}
4217
4218 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
4219 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
4220 match(Set dst (AndI (LoadI mem) mask));
4221
4222 ins_cost(125);
4223 format %{ "movzwl $dst, $mem\t# int -> ushort/char" %}
4224 ins_encode %{
4225 __ movzwl($dst$$Register, $mem$$Address);
4226 %}
4227 ins_pipe(ialu_reg_mem);
4228 %}
4229
4230 // Load Integer into Long Register
4231 instruct loadI2L(rRegL dst, memory mem)
4232 %{
4233 match(Set dst (ConvI2L (LoadI mem)));
4234
4235 ins_cost(125);
4236 format %{ "movslq $dst, $mem\t# int -> long" %}
4237
4238 ins_encode %{
4239 __ movslq($dst$$Register, $mem$$Address);
4240 %}
4241
4242 ins_pipe(ialu_reg_mem);
4243 %}
4244
4245 // Load Integer with mask 0xFF into Long Register
4246 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
4247 match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4248
4249 format %{ "movzbq $dst, $mem\t# int & 0xFF -> long" %}
4250 ins_encode %{
4251 __ movzbq($dst$$Register, $mem$$Address);
4252 %}
4253 ins_pipe(ialu_reg_mem);
4254 %}
4255
4256 // Load Integer with mask 0xFFFF into Long Register
4257 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
4258 match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4259
4260 format %{ "movzwq $dst, $mem\t# int & 0xFFFF -> long" %}
4261 ins_encode %{
4262 __ movzwq($dst$$Register, $mem$$Address);
4263 %}
4264 ins_pipe(ialu_reg_mem);
4265 %}
4266
4267 // Load Integer with a 31-bit mask into Long Register
4268 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
4269 match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4270 effect(KILL cr);
4271
4272 format %{ "movl $dst, $mem\t# int & 31-bit mask -> long\n\t"
4273 "andl $dst, $mask" %}
4274 ins_encode %{
4275 Register Rdst = $dst$$Register;
4276 __ movl(Rdst, $mem$$Address);
4277 __ andl(Rdst, $mask$$constant);
4278 %}
4279 ins_pipe(ialu_reg_mem);
4280 %}
4281
4282 // Load Unsigned Integer into Long Register
4283 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
4284 %{
4285 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
4286
4287 ins_cost(125);
4288 format %{ "movl $dst, $mem\t# uint -> long" %}
4289
4290 ins_encode %{
4291 __ movl($dst$$Register, $mem$$Address);
4292 %}
4293
4294 ins_pipe(ialu_reg_mem);
4295 %}
4296
4297 // Load Long
4298 instruct loadL(rRegL dst, memory mem)
4299 %{
4300 match(Set dst (LoadL mem));
4301
4302 ins_cost(125);
4303 format %{ "movq $dst, $mem\t# long" %}
4304
4305 ins_encode %{
4306 __ movq($dst$$Register, $mem$$Address);
4307 %}
4308
4309 ins_pipe(ialu_reg_mem); // XXX
4310 %}
4311
4312 // Load Range
4313 instruct loadRange(rRegI dst, memory mem)
4314 %{
4315 match(Set dst (LoadRange mem));
4316
4317 ins_cost(125); // XXX
4318 format %{ "movl $dst, $mem\t# range" %}
4319 ins_encode %{
4320 __ movl($dst$$Register, $mem$$Address);
4321 %}
4322 ins_pipe(ialu_reg_mem);
4323 %}
4324
4325 // Load Pointer
4326 instruct loadP(rRegP dst, memory mem)
4327 %{
4328 match(Set dst (LoadP mem));
4329 predicate(n->as_Load()->barrier_data() == 0);
4330
4331 ins_cost(125); // XXX
4332 format %{ "movq $dst, $mem\t# ptr" %}
4333 ins_encode %{
4334 __ movq($dst$$Register, $mem$$Address);
4335 %}
4336 ins_pipe(ialu_reg_mem); // XXX
4337 %}
4338
4339 // Load Compressed Pointer
4340 instruct loadN(rRegN dst, memory mem)
4341 %{
4342 predicate(n->as_Load()->barrier_data() == 0);
4343 match(Set dst (LoadN mem));
4344
4345 ins_cost(125); // XXX
4346 format %{ "movl $dst, $mem\t# compressed ptr" %}
4347 ins_encode %{
4348 __ movl($dst$$Register, $mem$$Address);
4349 %}
4350 ins_pipe(ialu_reg_mem); // XXX
4351 %}
4352
4353
4354 // Load Klass Pointer
4355 instruct loadKlass(rRegP dst, memory mem)
4356 %{
4357 match(Set dst (LoadKlass mem));
4358
4359 ins_cost(125); // XXX
4360 format %{ "movq $dst, $mem\t# class" %}
4361 ins_encode %{
4362 __ movq($dst$$Register, $mem$$Address);
4363 %}
4364 ins_pipe(ialu_reg_mem); // XXX
4365 %}
4366
4367 // Load narrow Klass Pointer
4368 instruct loadNKlass(rRegN dst, memory mem)
4369 %{
4370 predicate(!UseCompactObjectHeaders);
4371 match(Set dst (LoadNKlass mem));
4372
4373 ins_cost(125); // XXX
4374 format %{ "movl $dst, $mem\t# compressed klass ptr" %}
4375 ins_encode %{
4376 __ movl($dst$$Register, $mem$$Address);
4377 %}
4378 ins_pipe(ialu_reg_mem); // XXX
4379 %}
4380
4381 instruct loadNKlassCompactHeaders(rRegN dst, memory mem, rFlagsReg cr)
4382 %{
4383 predicate(UseCompactObjectHeaders);
4384 match(Set dst (LoadNKlass mem));
4385 effect(KILL cr);
4386 ins_cost(125); // XXX
4387 format %{
4388 "movl $dst, $mem\t# compressed klass ptr, shifted\n\t"
4389 "shrl $dst, markWord::klass_shift_at_offset"
4390 %}
4391 ins_encode %{
4392 __ movl($dst$$Register, $mem$$Address);
4393 __ shrl($dst$$Register, markWord::klass_shift_at_offset);
4394 %}
4395 ins_pipe(ialu_reg_mem); // XXX
4396 %}
4397
4398 // Load Float
4399 instruct loadF(regF dst, memory mem)
4400 %{
4401 match(Set dst (LoadF mem));
4402
4403 ins_cost(145); // XXX
4404 format %{ "movss $dst, $mem\t# float" %}
4405 ins_encode %{
4406 __ movflt($dst$$XMMRegister, $mem$$Address);
4407 %}
4408 ins_pipe(pipe_slow); // XXX
4409 %}
4410
4411 // Load Double
4412 instruct loadD_partial(regD dst, memory mem)
4413 %{
4414 predicate(!UseXmmLoadAndClearUpper);
4415 match(Set dst (LoadD mem));
4416
4417 ins_cost(145); // XXX
4418 format %{ "movlpd $dst, $mem\t# double" %}
4419 ins_encode %{
4420 __ movdbl($dst$$XMMRegister, $mem$$Address);
4421 %}
4422 ins_pipe(pipe_slow); // XXX
4423 %}
4424
4425 instruct loadD(regD dst, memory mem)
4426 %{
4427 predicate(UseXmmLoadAndClearUpper);
4428 match(Set dst (LoadD mem));
4429
4430 ins_cost(145); // XXX
4431 format %{ "movsd $dst, $mem\t# double" %}
4432 ins_encode %{
4433 __ movdbl($dst$$XMMRegister, $mem$$Address);
4434 %}
4435 ins_pipe(pipe_slow); // XXX
4436 %}
4437
4438 instruct loadAOTRCAddress(rRegP dst, immAOTRuntimeConstantsAddress con)
4439 %{
4440 match(Set dst con);
4441
4442 format %{ "leaq $dst, $con\t# AOT Runtime Constants Address" %}
4443
4444 ins_encode %{
4445 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
4446 %}
4447
4448 ins_pipe(ialu_reg_fat);
4449 %}
4450
4451 // max = java.lang.Math.max(float a, float b)
4452 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
4453 predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
4454 match(Set dst (MaxF a b));
4455 effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
4456 format %{ "maxF $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
4457 ins_encode %{
4458 __ vminmax_fp(Op_MaxV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
4459 %}
4460 ins_pipe( pipe_slow );
4461 %}
4462
4463 instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
4464 predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
4465 match(Set dst (MaxF a b));
4466 effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
4467
4468 format %{ "$dst = max($a, $b)\t# intrinsic (float)" %}
4469 ins_encode %{
4470 emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
4471 false /*min*/, true /*single*/);
4472 %}
4473 ins_pipe( pipe_slow );
4474 %}
4475
4476 // max = java.lang.Math.max(double a, double b)
4477 instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
4478 predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
4479 match(Set dst (MaxD a b));
4480 effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
4481 format %{ "maxD $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
4482 ins_encode %{
4483 __ vminmax_fp(Op_MaxV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
4484 %}
4485 ins_pipe( pipe_slow );
4486 %}
4487
4488 instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
4489 predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
4490 match(Set dst (MaxD a b));
4491 effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
4492
4493 format %{ "$dst = max($a, $b)\t# intrinsic (double)" %}
4494 ins_encode %{
4495 emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
4496 false /*min*/, false /*single*/);
4497 %}
4498 ins_pipe( pipe_slow );
4499 %}
4500
4501 // min = java.lang.Math.min(float a, float b)
4502 instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
4503 predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
4504 match(Set dst (MinF a b));
4505 effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
4506 format %{ "minF $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
4507 ins_encode %{
4508 __ vminmax_fp(Op_MinV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
4509 %}
4510 ins_pipe( pipe_slow );
4511 %}
4512
4513 instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
4514 predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
4515 match(Set dst (MinF a b));
4516 effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
4517
4518 format %{ "$dst = min($a, $b)\t# intrinsic (float)" %}
4519 ins_encode %{
4520 emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
4521 true /*min*/, true /*single*/);
4522 %}
4523 ins_pipe( pipe_slow );
4524 %}
4525
4526 // min = java.lang.Math.min(double a, double b)
4527 instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
4528 predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
4529 match(Set dst (MinD a b));
4530 effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
4531 format %{ "minD $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
4532 ins_encode %{
4533 __ vminmax_fp(Op_MinV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
4534 %}
4535 ins_pipe( pipe_slow );
4536 %}
4537
4538 instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
4539 predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
4540 match(Set dst (MinD a b));
4541 effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
4542
4543 format %{ "$dst = min($a, $b)\t# intrinsic (double)" %}
4544 ins_encode %{
4545 emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
4546 true /*min*/, false /*single*/);
4547 %}
4548 ins_pipe( pipe_slow );
4549 %}
4550
4551 // Load Effective Address
4552 instruct leaP8(rRegP dst, indOffset8 mem)
4553 %{
4554 match(Set dst mem);
4555
4556 ins_cost(110); // XXX
4557 format %{ "leaq $dst, $mem\t# ptr 8" %}
4558 ins_encode %{
4559 __ leaq($dst$$Register, $mem$$Address);
4560 %}
4561 ins_pipe(ialu_reg_reg_fat);
4562 %}
4563
4564 instruct leaP32(rRegP dst, indOffset32 mem)
4565 %{
4566 match(Set dst mem);
4567
4568 ins_cost(110);
4569 format %{ "leaq $dst, $mem\t# ptr 32" %}
4570 ins_encode %{
4571 __ leaq($dst$$Register, $mem$$Address);
4572 %}
4573 ins_pipe(ialu_reg_reg_fat);
4574 %}
4575
4576 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
4577 %{
4578 match(Set dst mem);
4579
4580 ins_cost(110);
4581 format %{ "leaq $dst, $mem\t# ptr idxoff" %}
4582 ins_encode %{
4583 __ leaq($dst$$Register, $mem$$Address);
4584 %}
4585 ins_pipe(ialu_reg_reg_fat);
4586 %}
4587
4588 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
4589 %{
4590 match(Set dst mem);
4591
4592 ins_cost(110);
4593 format %{ "leaq $dst, $mem\t# ptr idxscale" %}
4594 ins_encode %{
4595 __ leaq($dst$$Register, $mem$$Address);
4596 %}
4597 ins_pipe(ialu_reg_reg_fat);
4598 %}
4599
4600 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
4601 %{
4602 match(Set dst mem);
4603
4604 ins_cost(110);
4605 format %{ "leaq $dst, $mem\t# ptr idxscale" %}
4606 ins_encode %{
4607 __ leaq($dst$$Register, $mem$$Address);
4608 %}
4609 ins_pipe(ialu_reg_reg_fat);
4610 %}
4611
4612 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
4613 %{
4614 match(Set dst mem);
4615
4616 ins_cost(110);
4617 format %{ "leaq $dst, $mem\t# ptr idxscaleoff" %}
4618 ins_encode %{
4619 __ leaq($dst$$Register, $mem$$Address);
4620 %}
4621 ins_pipe(ialu_reg_reg_fat);
4622 %}
4623
4624 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
4625 %{
4626 match(Set dst mem);
4627
4628 ins_cost(110);
4629 format %{ "leaq $dst, $mem\t# ptr posidxoff" %}
4630 ins_encode %{
4631 __ leaq($dst$$Register, $mem$$Address);
4632 %}
4633 ins_pipe(ialu_reg_reg_fat);
4634 %}
4635
4636 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
4637 %{
4638 match(Set dst mem);
4639
4640 ins_cost(110);
4641 format %{ "leaq $dst, $mem\t# ptr posidxscaleoff" %}
4642 ins_encode %{
4643 __ leaq($dst$$Register, $mem$$Address);
4644 %}
4645 ins_pipe(ialu_reg_reg_fat);
4646 %}
4647
4648 // Load Effective Address which uses Narrow (32-bits) oop
4649 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
4650 %{
4651 predicate(UseCompressedOops && (CompressedOops::shift() != 0));
4652 match(Set dst mem);
4653
4654 ins_cost(110);
4655 format %{ "leaq $dst, $mem\t# ptr compressedoopoff32" %}
4656 ins_encode %{
4657 __ leaq($dst$$Register, $mem$$Address);
4658 %}
4659 ins_pipe(ialu_reg_reg_fat);
4660 %}
4661
4662 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
4663 %{
4664 predicate(CompressedOops::shift() == 0);
4665 match(Set dst mem);
4666
4667 ins_cost(110); // XXX
4668 format %{ "leaq $dst, $mem\t# ptr off8narrow" %}
4669 ins_encode %{
4670 __ leaq($dst$$Register, $mem$$Address);
4671 %}
4672 ins_pipe(ialu_reg_reg_fat);
4673 %}
4674
4675 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
4676 %{
4677 predicate(CompressedOops::shift() == 0);
4678 match(Set dst mem);
4679
4680 ins_cost(110);
4681 format %{ "leaq $dst, $mem\t# ptr off32narrow" %}
4682 ins_encode %{
4683 __ leaq($dst$$Register, $mem$$Address);
4684 %}
4685 ins_pipe(ialu_reg_reg_fat);
4686 %}
4687
4688 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
4689 %{
4690 predicate(CompressedOops::shift() == 0);
4691 match(Set dst mem);
4692
4693 ins_cost(110);
4694 format %{ "leaq $dst, $mem\t# ptr idxoffnarrow" %}
4695 ins_encode %{
4696 __ leaq($dst$$Register, $mem$$Address);
4697 %}
4698 ins_pipe(ialu_reg_reg_fat);
4699 %}
4700
4701 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
4702 %{
4703 predicate(CompressedOops::shift() == 0);
4704 match(Set dst mem);
4705
4706 ins_cost(110);
4707 format %{ "leaq $dst, $mem\t# ptr idxscalenarrow" %}
4708 ins_encode %{
4709 __ leaq($dst$$Register, $mem$$Address);
4710 %}
4711 ins_pipe(ialu_reg_reg_fat);
4712 %}
4713
4714 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
4715 %{
4716 predicate(CompressedOops::shift() == 0);
4717 match(Set dst mem);
4718
4719 ins_cost(110);
4720 format %{ "leaq $dst, $mem\t# ptr idxscaleoffnarrow" %}
4721 ins_encode %{
4722 __ leaq($dst$$Register, $mem$$Address);
4723 %}
4724 ins_pipe(ialu_reg_reg_fat);
4725 %}
4726
4727 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
4728 %{
4729 predicate(CompressedOops::shift() == 0);
4730 match(Set dst mem);
4731
4732 ins_cost(110);
4733 format %{ "leaq $dst, $mem\t# ptr posidxoffnarrow" %}
4734 ins_encode %{
4735 __ leaq($dst$$Register, $mem$$Address);
4736 %}
4737 ins_pipe(ialu_reg_reg_fat);
4738 %}
4739
4740 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
4741 %{
4742 predicate(CompressedOops::shift() == 0);
4743 match(Set dst mem);
4744
4745 ins_cost(110);
4746 format %{ "leaq $dst, $mem\t# ptr posidxscaleoffnarrow" %}
4747 ins_encode %{
4748 __ leaq($dst$$Register, $mem$$Address);
4749 %}
4750 ins_pipe(ialu_reg_reg_fat);
4751 %}
4752
4753 instruct loadConI(rRegI dst, immI src)
4754 %{
4755 match(Set dst src);
4756
4757 format %{ "movl $dst, $src\t# int" %}
4758 ins_encode %{
4759 __ movl($dst$$Register, $src$$constant);
4760 %}
4761 ins_pipe(ialu_reg_fat); // XXX
4762 %}
4763
4764 instruct loadConI0(rRegI dst, immI_0 src, rFlagsReg cr)
4765 %{
4766 match(Set dst src);
4767 effect(KILL cr);
4768
4769 ins_cost(50);
4770 format %{ "xorl $dst, $dst\t# int" %}
4771 ins_encode %{
4772 __ xorl($dst$$Register, $dst$$Register);
4773 %}
4774 ins_pipe(ialu_reg);
4775 %}
4776
4777 instruct loadConL(rRegL dst, immL src)
4778 %{
4779 match(Set dst src);
4780
4781 ins_cost(150);
4782 format %{ "movq $dst, $src\t# long" %}
4783 ins_encode %{
4784 __ mov64($dst$$Register, $src$$constant);
4785 %}
4786 ins_pipe(ialu_reg);
4787 %}
4788
4789 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
4790 %{
4791 match(Set dst src);
4792 effect(KILL cr);
4793
4794 ins_cost(50);
4795 format %{ "xorl $dst, $dst\t# long" %}
4796 ins_encode %{
4797 __ xorl($dst$$Register, $dst$$Register);
4798 %}
4799 ins_pipe(ialu_reg); // XXX
4800 %}
4801
4802 instruct loadConUL32(rRegL dst, immUL32 src)
4803 %{
4804 match(Set dst src);
4805
4806 ins_cost(60);
4807 format %{ "movl $dst, $src\t# long (unsigned 32-bit)" %}
4808 ins_encode %{
4809 __ movl($dst$$Register, $src$$constant);
4810 %}
4811 ins_pipe(ialu_reg);
4812 %}
4813
4814 instruct loadConL32(rRegL dst, immL32 src)
4815 %{
4816 match(Set dst src);
4817
4818 ins_cost(70);
4819 format %{ "movq $dst, $src\t# long (32-bit)" %}
4820 ins_encode %{
4821 __ movq($dst$$Register, $src$$constant);
4822 %}
4823 ins_pipe(ialu_reg);
4824 %}
4825
4826 instruct loadConP(rRegP dst, immP con) %{
4827 match(Set dst con);
4828
4829 format %{ "movq $dst, $con\t# ptr" %}
4830 ins_encode %{
4831 __ mov64($dst$$Register, $con$$constant, $con->constant_reloc(), RELOC_IMM64);
4832 %}
4833 ins_pipe(ialu_reg_fat); // XXX
4834 %}
4835
4836 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
4837 %{
4838 match(Set dst src);
4839 effect(KILL cr);
4840
4841 ins_cost(50);
4842 format %{ "xorl $dst, $dst\t# ptr" %}
4843 ins_encode %{
4844 __ xorl($dst$$Register, $dst$$Register);
4845 %}
4846 ins_pipe(ialu_reg);
4847 %}
4848
4849 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
4850 %{
4851 match(Set dst src);
4852 effect(KILL cr);
4853
4854 ins_cost(60);
4855 format %{ "movl $dst, $src\t# ptr (positive 32-bit)" %}
4856 ins_encode %{
4857 __ movl($dst$$Register, $src$$constant);
4858 %}
4859 ins_pipe(ialu_reg);
4860 %}
4861
4862 instruct loadConF(regF dst, immF con) %{
4863 match(Set dst con);
4864 ins_cost(125);
4865 format %{ "movss $dst, [$constantaddress]\t# load from constant table: float=$con" %}
4866 ins_encode %{
4867 __ movflt($dst$$XMMRegister, $constantaddress($con));
4868 %}
4869 ins_pipe(pipe_slow);
4870 %}
4871
4872 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
4873 match(Set dst src);
4874 effect(KILL cr);
4875 format %{ "xorq $dst, $src\t# compressed null pointer" %}
4876 ins_encode %{
4877 __ xorq($dst$$Register, $dst$$Register);
4878 %}
4879 ins_pipe(ialu_reg);
4880 %}
4881
4882 instruct loadConN(rRegN dst, immN src) %{
4883 match(Set dst src);
4884
4885 ins_cost(125);
4886 format %{ "movl $dst, $src\t# compressed ptr" %}
4887 ins_encode %{
4888 address con = (address)$src$$constant;
4889 if (con == nullptr) {
4890 ShouldNotReachHere();
4891 } else {
4892 __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
4893 }
4894 %}
4895 ins_pipe(ialu_reg_fat); // XXX
4896 %}
4897
4898 instruct loadConNKlass(rRegN dst, immNKlass src) %{
4899 match(Set dst src);
4900
4901 ins_cost(125);
4902 format %{ "movl $dst, $src\t# compressed klass ptr" %}
4903 ins_encode %{
4904 address con = (address)$src$$constant;
4905 if (con == nullptr) {
4906 ShouldNotReachHere();
4907 } else {
4908 __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
4909 }
4910 %}
4911 ins_pipe(ialu_reg_fat); // XXX
4912 %}
4913
4914 instruct loadConF0(regF dst, immF0 src)
4915 %{
4916 match(Set dst src);
4917 ins_cost(100);
4918
4919 format %{ "xorps $dst, $dst\t# float 0.0" %}
4920 ins_encode %{
4921 __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
4922 %}
4923 ins_pipe(pipe_slow);
4924 %}
4925
4926 // Use the same format since predicate() can not be used here.
4927 instruct loadConD(regD dst, immD con) %{
4928 match(Set dst con);
4929 ins_cost(125);
4930 format %{ "movsd $dst, [$constantaddress]\t# load from constant table: double=$con" %}
4931 ins_encode %{
4932 __ movdbl($dst$$XMMRegister, $constantaddress($con));
4933 %}
4934 ins_pipe(pipe_slow);
4935 %}
4936
4937 instruct loadConD0(regD dst, immD0 src)
4938 %{
4939 match(Set dst src);
4940 ins_cost(100);
4941
4942 format %{ "xorpd $dst, $dst\t# double 0.0" %}
4943 ins_encode %{
4944 __ xorpd($dst$$XMMRegister, $dst$$XMMRegister);
4945 %}
4946 ins_pipe(pipe_slow);
4947 %}
4948
4949 instruct loadSSI(rRegI dst, stackSlotI src)
4950 %{
4951 match(Set dst src);
4952
4953 ins_cost(125);
4954 format %{ "movl $dst, $src\t# int stk" %}
4955 ins_encode %{
4956 __ movl($dst$$Register, $src$$Address);
4957 %}
4958 ins_pipe(ialu_reg_mem);
4959 %}
4960
4961 instruct loadSSL(rRegL dst, stackSlotL src)
4962 %{
4963 match(Set dst src);
4964
4965 ins_cost(125);
4966 format %{ "movq $dst, $src\t# long stk" %}
4967 ins_encode %{
4968 __ movq($dst$$Register, $src$$Address);
4969 %}
4970 ins_pipe(ialu_reg_mem);
4971 %}
4972
4973 instruct loadSSP(rRegP dst, stackSlotP src)
4974 %{
4975 match(Set dst src);
4976
4977 ins_cost(125);
4978 format %{ "movq $dst, $src\t# ptr stk" %}
4979 ins_encode %{
4980 __ movq($dst$$Register, $src$$Address);
4981 %}
4982 ins_pipe(ialu_reg_mem);
4983 %}
4984
4985 instruct loadSSF(regF dst, stackSlotF src)
4986 %{
4987 match(Set dst src);
4988
4989 ins_cost(125);
4990 format %{ "movss $dst, $src\t# float stk" %}
4991 ins_encode %{
4992 __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
4993 %}
4994 ins_pipe(pipe_slow); // XXX
4995 %}
4996
4997 // Use the same format since predicate() can not be used here.
4998 instruct loadSSD(regD dst, stackSlotD src)
4999 %{
5000 match(Set dst src);
5001
5002 ins_cost(125);
5003 format %{ "movsd $dst, $src\t# double stk" %}
5004 ins_encode %{
5005 __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
5006 %}
5007 ins_pipe(pipe_slow); // XXX
5008 %}
5009
5010 // Prefetch instructions for allocation.
5011 // Must be safe to execute with invalid address (cannot fault).
5012
5013 instruct prefetchAlloc( memory mem ) %{
5014 predicate(AllocatePrefetchInstr==3);
5015 match(PrefetchAllocation mem);
5016 ins_cost(125);
5017
5018 format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
5019 ins_encode %{
5020 __ prefetchw($mem$$Address);
5021 %}
5022 ins_pipe(ialu_mem);
5023 %}
5024
5025 instruct prefetchAllocNTA( memory mem ) %{
5026 predicate(AllocatePrefetchInstr==0);
5027 match(PrefetchAllocation mem);
5028 ins_cost(125);
5029
5030 format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
5031 ins_encode %{
5032 __ prefetchnta($mem$$Address);
5033 %}
5034 ins_pipe(ialu_mem);
5035 %}
5036
5037 instruct prefetchAllocT0( memory mem ) %{
5038 predicate(AllocatePrefetchInstr==1);
5039 match(PrefetchAllocation mem);
5040 ins_cost(125);
5041
5042 format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
5043 ins_encode %{
5044 __ prefetcht0($mem$$Address);
5045 %}
5046 ins_pipe(ialu_mem);
5047 %}
5048
5049 instruct prefetchAllocT2( memory mem ) %{
5050 predicate(AllocatePrefetchInstr==2);
5051 match(PrefetchAllocation mem);
5052 ins_cost(125);
5053
5054 format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
5055 ins_encode %{
5056 __ prefetcht2($mem$$Address);
5057 %}
5058 ins_pipe(ialu_mem);
5059 %}
5060
5061 //----------Store Instructions-------------------------------------------------
5062
5063 // Store Byte
5064 instruct storeB(memory mem, rRegI src)
5065 %{
5066 match(Set mem (StoreB mem src));
5067
5068 ins_cost(125); // XXX
5069 format %{ "movb $mem, $src\t# byte" %}
5070 ins_encode %{
5071 __ movb($mem$$Address, $src$$Register);
5072 %}
5073 ins_pipe(ialu_mem_reg);
5074 %}
5075
5076 // Store Char/Short
5077 instruct storeC(memory mem, rRegI src)
5078 %{
5079 match(Set mem (StoreC mem src));
5080
5081 ins_cost(125); // XXX
5082 format %{ "movw $mem, $src\t# char/short" %}
5083 ins_encode %{
5084 __ movw($mem$$Address, $src$$Register);
5085 %}
5086 ins_pipe(ialu_mem_reg);
5087 %}
5088
5089 // Store Integer
5090 instruct storeI(memory mem, rRegI src)
5091 %{
5092 match(Set mem (StoreI mem src));
5093
5094 ins_cost(125); // XXX
5095 format %{ "movl $mem, $src\t# int" %}
5096 ins_encode %{
5097 __ movl($mem$$Address, $src$$Register);
5098 %}
5099 ins_pipe(ialu_mem_reg);
5100 %}
5101
5102 // Store Long
5103 instruct storeL(memory mem, rRegL src)
5104 %{
5105 match(Set mem (StoreL mem src));
5106
5107 ins_cost(125); // XXX
5108 format %{ "movq $mem, $src\t# long" %}
5109 ins_encode %{
5110 __ movq($mem$$Address, $src$$Register);
5111 %}
5112 ins_pipe(ialu_mem_reg); // XXX
5113 %}
5114
5115 // Store Pointer
5116 instruct storeP(memory mem, any_RegP src)
5117 %{
5118 predicate(n->as_Store()->barrier_data() == 0);
5119 match(Set mem (StoreP mem src));
5120
5121 ins_cost(125); // XXX
5122 format %{ "movq $mem, $src\t# ptr" %}
5123 ins_encode %{
5124 __ movq($mem$$Address, $src$$Register);
5125 %}
5126 ins_pipe(ialu_mem_reg);
5127 %}
5128
5129 instruct storeImmP0(memory mem, immP0 zero)
5130 %{
5131 predicate(UseCompressedOops && (CompressedOops::base() == nullptr) && n->as_Store()->barrier_data() == 0);
5132 match(Set mem (StoreP mem zero));
5133
5134 ins_cost(125); // XXX
5135 format %{ "movq $mem, R12\t# ptr (R12_heapbase==0)" %}
5136 ins_encode %{
5137 __ movq($mem$$Address, r12);
5138 %}
5139 ins_pipe(ialu_mem_reg);
5140 %}
5141
5142 // Store Null Pointer, mark word, or other simple pointer constant.
5143 instruct storeImmP(memory mem, immP31 src)
5144 %{
5145 predicate(n->as_Store()->barrier_data() == 0);
5146 match(Set mem (StoreP mem src));
5147
5148 ins_cost(150); // XXX
5149 format %{ "movq $mem, $src\t# ptr" %}
5150 ins_encode %{
5151 __ movq($mem$$Address, $src$$constant);
5152 %}
5153 ins_pipe(ialu_mem_imm);
5154 %}
5155
5156 // Store Compressed Pointer
5157 instruct storeN(memory mem, rRegN src)
5158 %{
5159 predicate(n->as_Store()->barrier_data() == 0);
5160 match(Set mem (StoreN mem src));
5161
5162 ins_cost(125); // XXX
5163 format %{ "movl $mem, $src\t# compressed ptr" %}
5164 ins_encode %{
5165 __ movl($mem$$Address, $src$$Register);
5166 %}
5167 ins_pipe(ialu_mem_reg);
5168 %}
5169
5170 instruct storeNKlass(memory mem, rRegN src)
5171 %{
5172 match(Set mem (StoreNKlass mem src));
5173
5174 ins_cost(125); // XXX
5175 format %{ "movl $mem, $src\t# compressed klass ptr" %}
5176 ins_encode %{
5177 __ movl($mem$$Address, $src$$Register);
5178 %}
5179 ins_pipe(ialu_mem_reg);
5180 %}
5181
5182 instruct storeImmN0(memory mem, immN0 zero)
5183 %{
5184 predicate(CompressedOops::base() == nullptr && n->as_Store()->barrier_data() == 0);
5185 match(Set mem (StoreN mem zero));
5186
5187 ins_cost(125); // XXX
5188 format %{ "movl $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
5189 ins_encode %{
5190 __ movl($mem$$Address, r12);
5191 %}
5192 ins_pipe(ialu_mem_reg);
5193 %}
5194
5195 instruct storeImmN(memory mem, immN src)
5196 %{
5197 predicate(n->as_Store()->barrier_data() == 0);
5198 match(Set mem (StoreN mem src));
5199
5200 ins_cost(150); // XXX
5201 format %{ "movl $mem, $src\t# compressed ptr" %}
5202 ins_encode %{
5203 address con = (address)$src$$constant;
5204 if (con == nullptr) {
5205 __ movl($mem$$Address, 0);
5206 } else {
5207 __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
5208 }
5209 %}
5210 ins_pipe(ialu_mem_imm);
5211 %}
5212
5213 instruct storeImmNKlass(memory mem, immNKlass src)
5214 %{
5215 match(Set mem (StoreNKlass mem src));
5216
5217 ins_cost(150); // XXX
5218 format %{ "movl $mem, $src\t# compressed klass ptr" %}
5219 ins_encode %{
5220 __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
5221 %}
5222 ins_pipe(ialu_mem_imm);
5223 %}
5224
5225 // Store Integer Immediate
5226 instruct storeImmI0(memory mem, immI_0 zero)
5227 %{
5228 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
5229 match(Set mem (StoreI mem zero));
5230
5231 ins_cost(125); // XXX
5232 format %{ "movl $mem, R12\t# int (R12_heapbase==0)" %}
5233 ins_encode %{
5234 __ movl($mem$$Address, r12);
5235 %}
5236 ins_pipe(ialu_mem_reg);
5237 %}
5238
5239 instruct storeImmI(memory mem, immI src)
5240 %{
5241 match(Set mem (StoreI mem src));
5242
5243 ins_cost(150);
5244 format %{ "movl $mem, $src\t# int" %}
5245 ins_encode %{
5246 __ movl($mem$$Address, $src$$constant);
5247 %}
5248 ins_pipe(ialu_mem_imm);
5249 %}
5250
5251 // Store Long Immediate
5252 instruct storeImmL0(memory mem, immL0 zero)
5253 %{
5254 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
5255 match(Set mem (StoreL mem zero));
5256
5257 ins_cost(125); // XXX
5258 format %{ "movq $mem, R12\t# long (R12_heapbase==0)" %}
5259 ins_encode %{
5260 __ movq($mem$$Address, r12);
5261 %}
5262 ins_pipe(ialu_mem_reg);
5263 %}
5264
5265 instruct storeImmL(memory mem, immL32 src)
5266 %{
5267 match(Set mem (StoreL mem src));
5268
5269 ins_cost(150);
5270 format %{ "movq $mem, $src\t# long" %}
5271 ins_encode %{
5272 __ movq($mem$$Address, $src$$constant);
5273 %}
5274 ins_pipe(ialu_mem_imm);
5275 %}
5276
5277 // Store Short/Char Immediate
5278 instruct storeImmC0(memory mem, immI_0 zero)
5279 %{
5280 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
5281 match(Set mem (StoreC mem zero));
5282
5283 ins_cost(125); // XXX
5284 format %{ "movw $mem, R12\t# short/char (R12_heapbase==0)" %}
5285 ins_encode %{
5286 __ movw($mem$$Address, r12);
5287 %}
5288 ins_pipe(ialu_mem_reg);
5289 %}
5290
5291 instruct storeImmI16(memory mem, immI16 src)
5292 %{
5293 predicate(UseStoreImmI16);
5294 match(Set mem (StoreC mem src));
5295
5296 ins_cost(150);
5297 format %{ "movw $mem, $src\t# short/char" %}
5298 ins_encode %{
5299 __ movw($mem$$Address, $src$$constant);
5300 %}
5301 ins_pipe(ialu_mem_imm);
5302 %}
5303
5304 // Store Byte Immediate
5305 instruct storeImmB0(memory mem, immI_0 zero)
5306 %{
5307 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
5308 match(Set mem (StoreB mem zero));
5309
5310 ins_cost(125); // XXX
5311 format %{ "movb $mem, R12\t# short/char (R12_heapbase==0)" %}
5312 ins_encode %{
5313 __ movb($mem$$Address, r12);
5314 %}
5315 ins_pipe(ialu_mem_reg);
5316 %}
5317
5318 instruct storeImmB(memory mem, immI8 src)
5319 %{
5320 match(Set mem (StoreB mem src));
5321
5322 ins_cost(150); // XXX
5323 format %{ "movb $mem, $src\t# byte" %}
5324 ins_encode %{
5325 __ movb($mem$$Address, $src$$constant);
5326 %}
5327 ins_pipe(ialu_mem_imm);
5328 %}
5329
5330 // Store Float
5331 instruct storeF(memory mem, regF src)
5332 %{
5333 match(Set mem (StoreF mem src));
5334
5335 ins_cost(95); // XXX
5336 format %{ "movss $mem, $src\t# float" %}
5337 ins_encode %{
5338 __ movflt($mem$$Address, $src$$XMMRegister);
5339 %}
5340 ins_pipe(pipe_slow); // XXX
5341 %}
5342
5343 // Store immediate Float value (it is faster than store from XMM register)
5344 instruct storeF0(memory mem, immF0 zero)
5345 %{
5346 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
5347 match(Set mem (StoreF mem zero));
5348
5349 ins_cost(25); // XXX
5350 format %{ "movl $mem, R12\t# float 0. (R12_heapbase==0)" %}
5351 ins_encode %{
5352 __ movl($mem$$Address, r12);
5353 %}
5354 ins_pipe(ialu_mem_reg);
5355 %}
5356
5357 instruct storeF_imm(memory mem, immF src)
5358 %{
5359 match(Set mem (StoreF mem src));
5360
5361 ins_cost(50);
5362 format %{ "movl $mem, $src\t# float" %}
5363 ins_encode %{
5364 __ movl($mem$$Address, jint_cast($src$$constant));
5365 %}
5366 ins_pipe(ialu_mem_imm);
5367 %}
5368
5369 // Store Double
5370 instruct storeD(memory mem, regD src)
5371 %{
5372 match(Set mem (StoreD mem src));
5373
5374 ins_cost(95); // XXX
5375 format %{ "movsd $mem, $src\t# double" %}
5376 ins_encode %{
5377 __ movdbl($mem$$Address, $src$$XMMRegister);
5378 %}
5379 ins_pipe(pipe_slow); // XXX
5380 %}
5381
5382 // Store immediate double 0.0 (it is faster than store from XMM register)
5383 instruct storeD0_imm(memory mem, immD0 src)
5384 %{
5385 predicate(!UseCompressedOops || (CompressedOops::base() != nullptr));
5386 match(Set mem (StoreD mem src));
5387
5388 ins_cost(50);
5389 format %{ "movq $mem, $src\t# double 0." %}
5390 ins_encode %{
5391 __ movq($mem$$Address, $src$$constant);
5392 %}
5393 ins_pipe(ialu_mem_imm);
5394 %}
5395
5396 instruct storeD0(memory mem, immD0 zero)
5397 %{
5398 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
5399 match(Set mem (StoreD mem zero));
5400
5401 ins_cost(25); // XXX
5402 format %{ "movq $mem, R12\t# double 0. (R12_heapbase==0)" %}
5403 ins_encode %{
5404 __ movq($mem$$Address, r12);
5405 %}
5406 ins_pipe(ialu_mem_reg);
5407 %}
5408
5409 instruct storeSSI(stackSlotI dst, rRegI src)
5410 %{
5411 match(Set dst src);
5412
5413 ins_cost(100);
5414 format %{ "movl $dst, $src\t# int stk" %}
5415 ins_encode %{
5416 __ movl($dst$$Address, $src$$Register);
5417 %}
5418 ins_pipe( ialu_mem_reg );
5419 %}
5420
5421 instruct storeSSL(stackSlotL dst, rRegL src)
5422 %{
5423 match(Set dst src);
5424
5425 ins_cost(100);
5426 format %{ "movq $dst, $src\t# long stk" %}
5427 ins_encode %{
5428 __ movq($dst$$Address, $src$$Register);
5429 %}
5430 ins_pipe(ialu_mem_reg);
5431 %}
5432
5433 instruct storeSSP(stackSlotP dst, rRegP src)
5434 %{
5435 match(Set dst src);
5436
5437 ins_cost(100);
5438 format %{ "movq $dst, $src\t# ptr stk" %}
5439 ins_encode %{
5440 __ movq($dst$$Address, $src$$Register);
5441 %}
5442 ins_pipe(ialu_mem_reg);
5443 %}
5444
5445 instruct storeSSF(stackSlotF dst, regF src)
5446 %{
5447 match(Set dst src);
5448
5449 ins_cost(95); // XXX
5450 format %{ "movss $dst, $src\t# float stk" %}
5451 ins_encode %{
5452 __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
5453 %}
5454 ins_pipe(pipe_slow); // XXX
5455 %}
5456
5457 instruct storeSSD(stackSlotD dst, regD src)
5458 %{
5459 match(Set dst src);
5460
5461 ins_cost(95); // XXX
5462 format %{ "movsd $dst, $src\t# double stk" %}
5463 ins_encode %{
5464 __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
5465 %}
5466 ins_pipe(pipe_slow); // XXX
5467 %}
5468
5469 instruct cacheWB(indirect addr)
5470 %{
5471 predicate(VM_Version::supports_data_cache_line_flush());
5472 match(CacheWB addr);
5473
5474 ins_cost(100);
5475 format %{"cache wb $addr" %}
5476 ins_encode %{
5477 assert($addr->index_position() < 0, "should be");
5478 assert($addr$$disp == 0, "should be");
5479 __ cache_wb(Address($addr$$base$$Register, 0));
5480 %}
5481 ins_pipe(pipe_slow); // XXX
5482 %}
5483
5484 instruct cacheWBPreSync()
5485 %{
5486 predicate(VM_Version::supports_data_cache_line_flush());
5487 match(CacheWBPreSync);
5488
5489 ins_cost(100);
5490 format %{"cache wb presync" %}
5491 ins_encode %{
5492 __ cache_wbsync(true);
5493 %}
5494 ins_pipe(pipe_slow); // XXX
5495 %}
5496
5497 instruct cacheWBPostSync()
5498 %{
5499 predicate(VM_Version::supports_data_cache_line_flush());
5500 match(CacheWBPostSync);
5501
5502 ins_cost(100);
5503 format %{"cache wb postsync" %}
5504 ins_encode %{
5505 __ cache_wbsync(false);
5506 %}
5507 ins_pipe(pipe_slow); // XXX
5508 %}
5509
5510 //----------BSWAP Instructions-------------------------------------------------
5511 instruct bytes_reverse_int(rRegI dst) %{
5512 match(Set dst (ReverseBytesI dst));
5513
5514 format %{ "bswapl $dst" %}
5515 ins_encode %{
5516 __ bswapl($dst$$Register);
5517 %}
5518 ins_pipe( ialu_reg );
5519 %}
5520
5521 instruct bytes_reverse_long(rRegL dst) %{
5522 match(Set dst (ReverseBytesL dst));
5523
5524 format %{ "bswapq $dst" %}
5525 ins_encode %{
5526 __ bswapq($dst$$Register);
5527 %}
5528 ins_pipe( ialu_reg);
5529 %}
5530
5531 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
5532 match(Set dst (ReverseBytesUS dst));
5533 effect(KILL cr);
5534
5535 format %{ "bswapl $dst\n\t"
5536 "shrl $dst,16\n\t" %}
5537 ins_encode %{
5538 __ bswapl($dst$$Register);
5539 __ shrl($dst$$Register, 16);
5540 %}
5541 ins_pipe( ialu_reg );
5542 %}
5543
5544 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
5545 match(Set dst (ReverseBytesS dst));
5546 effect(KILL cr);
5547
5548 format %{ "bswapl $dst\n\t"
5549 "sar $dst,16\n\t" %}
5550 ins_encode %{
5551 __ bswapl($dst$$Register);
5552 __ sarl($dst$$Register, 16);
5553 %}
5554 ins_pipe( ialu_reg );
5555 %}
5556
5557 //---------- Zeros Count Instructions ------------------------------------------
5558
5559 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
5560 predicate(UseCountLeadingZerosInstruction);
5561 match(Set dst (CountLeadingZerosI src));
5562 effect(KILL cr);
5563
5564 format %{ "lzcntl $dst, $src\t# count leading zeros (int)" %}
5565 ins_encode %{
5566 __ lzcntl($dst$$Register, $src$$Register);
5567 %}
5568 ins_pipe(ialu_reg);
5569 %}
5570
5571 instruct countLeadingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
5572 predicate(UseCountLeadingZerosInstruction);
5573 match(Set dst (CountLeadingZerosI (LoadI src)));
5574 effect(KILL cr);
5575 ins_cost(175);
5576 format %{ "lzcntl $dst, $src\t# count leading zeros (int)" %}
5577 ins_encode %{
5578 __ lzcntl($dst$$Register, $src$$Address);
5579 %}
5580 ins_pipe(ialu_reg_mem);
5581 %}
5582
5583 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
5584 predicate(!UseCountLeadingZerosInstruction);
5585 match(Set dst (CountLeadingZerosI src));
5586 effect(KILL cr);
5587
5588 format %{ "bsrl $dst, $src\t# count leading zeros (int)\n\t"
5589 "jnz skip\n\t"
5590 "movl $dst, -1\n"
5591 "skip:\n\t"
5592 "negl $dst\n\t"
5593 "addl $dst, 31" %}
5594 ins_encode %{
5595 Register Rdst = $dst$$Register;
5596 Register Rsrc = $src$$Register;
5597 Label skip;
5598 __ bsrl(Rdst, Rsrc);
5599 __ jccb(Assembler::notZero, skip);
5600 __ movl(Rdst, -1);
5601 __ bind(skip);
5602 __ negl(Rdst);
5603 __ addl(Rdst, BitsPerInt - 1);
5604 %}
5605 ins_pipe(ialu_reg);
5606 %}
5607
5608 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
5609 predicate(UseCountLeadingZerosInstruction);
5610 match(Set dst (CountLeadingZerosL src));
5611 effect(KILL cr);
5612
5613 format %{ "lzcntq $dst, $src\t# count leading zeros (long)" %}
5614 ins_encode %{
5615 __ lzcntq($dst$$Register, $src$$Register);
5616 %}
5617 ins_pipe(ialu_reg);
5618 %}
5619
5620 instruct countLeadingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
5621 predicate(UseCountLeadingZerosInstruction);
5622 match(Set dst (CountLeadingZerosL (LoadL src)));
5623 effect(KILL cr);
5624 ins_cost(175);
5625 format %{ "lzcntq $dst, $src\t# count leading zeros (long)" %}
5626 ins_encode %{
5627 __ lzcntq($dst$$Register, $src$$Address);
5628 %}
5629 ins_pipe(ialu_reg_mem);
5630 %}
5631
5632 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
5633 predicate(!UseCountLeadingZerosInstruction);
5634 match(Set dst (CountLeadingZerosL src));
5635 effect(KILL cr);
5636
5637 format %{ "bsrq $dst, $src\t# count leading zeros (long)\n\t"
5638 "jnz skip\n\t"
5639 "movl $dst, -1\n"
5640 "skip:\n\t"
5641 "negl $dst\n\t"
5642 "addl $dst, 63" %}
5643 ins_encode %{
5644 Register Rdst = $dst$$Register;
5645 Register Rsrc = $src$$Register;
5646 Label skip;
5647 __ bsrq(Rdst, Rsrc);
5648 __ jccb(Assembler::notZero, skip);
5649 __ movl(Rdst, -1);
5650 __ bind(skip);
5651 __ negl(Rdst);
5652 __ addl(Rdst, BitsPerLong - 1);
5653 %}
5654 ins_pipe(ialu_reg);
5655 %}
5656
5657 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
5658 predicate(UseCountTrailingZerosInstruction);
5659 match(Set dst (CountTrailingZerosI src));
5660 effect(KILL cr);
5661
5662 format %{ "tzcntl $dst, $src\t# count trailing zeros (int)" %}
5663 ins_encode %{
5664 __ tzcntl($dst$$Register, $src$$Register);
5665 %}
5666 ins_pipe(ialu_reg);
5667 %}
5668
5669 instruct countTrailingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
5670 predicate(UseCountTrailingZerosInstruction);
5671 match(Set dst (CountTrailingZerosI (LoadI src)));
5672 effect(KILL cr);
5673 ins_cost(175);
5674 format %{ "tzcntl $dst, $src\t# count trailing zeros (int)" %}
5675 ins_encode %{
5676 __ tzcntl($dst$$Register, $src$$Address);
5677 %}
5678 ins_pipe(ialu_reg_mem);
5679 %}
5680
5681 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
5682 predicate(!UseCountTrailingZerosInstruction);
5683 match(Set dst (CountTrailingZerosI src));
5684 effect(KILL cr);
5685
5686 format %{ "bsfl $dst, $src\t# count trailing zeros (int)\n\t"
5687 "jnz done\n\t"
5688 "movl $dst, 32\n"
5689 "done:" %}
5690 ins_encode %{
5691 Register Rdst = $dst$$Register;
5692 Label done;
5693 __ bsfl(Rdst, $src$$Register);
5694 __ jccb(Assembler::notZero, done);
5695 __ movl(Rdst, BitsPerInt);
5696 __ bind(done);
5697 %}
5698 ins_pipe(ialu_reg);
5699 %}
5700
5701 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
5702 predicate(UseCountTrailingZerosInstruction);
5703 match(Set dst (CountTrailingZerosL src));
5704 effect(KILL cr);
5705
5706 format %{ "tzcntq $dst, $src\t# count trailing zeros (long)" %}
5707 ins_encode %{
5708 __ tzcntq($dst$$Register, $src$$Register);
5709 %}
5710 ins_pipe(ialu_reg);
5711 %}
5712
5713 instruct countTrailingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
5714 predicate(UseCountTrailingZerosInstruction);
5715 match(Set dst (CountTrailingZerosL (LoadL src)));
5716 effect(KILL cr);
5717 ins_cost(175);
5718 format %{ "tzcntq $dst, $src\t# count trailing zeros (long)" %}
5719 ins_encode %{
5720 __ tzcntq($dst$$Register, $src$$Address);
5721 %}
5722 ins_pipe(ialu_reg_mem);
5723 %}
5724
5725 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
5726 predicate(!UseCountTrailingZerosInstruction);
5727 match(Set dst (CountTrailingZerosL src));
5728 effect(KILL cr);
5729
5730 format %{ "bsfq $dst, $src\t# count trailing zeros (long)\n\t"
5731 "jnz done\n\t"
5732 "movl $dst, 64\n"
5733 "done:" %}
5734 ins_encode %{
5735 Register Rdst = $dst$$Register;
5736 Label done;
5737 __ bsfq(Rdst, $src$$Register);
5738 __ jccb(Assembler::notZero, done);
5739 __ movl(Rdst, BitsPerLong);
5740 __ bind(done);
5741 %}
5742 ins_pipe(ialu_reg);
5743 %}
5744
5745 //--------------- Reverse Operation Instructions ----------------
5746 instruct bytes_reversebit_int(rRegI dst, rRegI src, rRegI rtmp, rFlagsReg cr) %{
5747 predicate(!VM_Version::supports_gfni());
5748 match(Set dst (ReverseI src));
5749 effect(TEMP dst, TEMP rtmp, KILL cr);
5750 format %{ "reverse_int $dst $src\t! using $rtmp as TEMP" %}
5751 ins_encode %{
5752 __ reverseI($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp$$Register);
5753 %}
5754 ins_pipe( ialu_reg );
5755 %}
5756
5757 instruct bytes_reversebit_int_gfni(rRegI dst, rRegI src, vlRegF xtmp1, vlRegF xtmp2, rRegL rtmp, rFlagsReg cr) %{
5758 predicate(VM_Version::supports_gfni());
5759 match(Set dst (ReverseI src));
5760 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
5761 format %{ "reverse_int $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
5762 ins_encode %{
5763 __ reverseI($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register);
5764 %}
5765 ins_pipe( ialu_reg );
5766 %}
5767
5768 instruct bytes_reversebit_long(rRegL dst, rRegL src, rRegL rtmp1, rRegL rtmp2, rFlagsReg cr) %{
5769 predicate(!VM_Version::supports_gfni());
5770 match(Set dst (ReverseL src));
5771 effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, KILL cr);
5772 format %{ "reverse_long $dst $src\t! using $rtmp1 and $rtmp2 as TEMP" %}
5773 ins_encode %{
5774 __ reverseL($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp1$$Register, $rtmp2$$Register);
5775 %}
5776 ins_pipe( ialu_reg );
5777 %}
5778
5779 instruct bytes_reversebit_long_gfni(rRegL dst, rRegL src, vlRegD xtmp1, vlRegD xtmp2, rRegL rtmp, rFlagsReg cr) %{
5780 predicate(VM_Version::supports_gfni());
5781 match(Set dst (ReverseL src));
5782 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
5783 format %{ "reverse_long $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
5784 ins_encode %{
5785 __ reverseL($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register, noreg);
5786 %}
5787 ins_pipe( ialu_reg );
5788 %}
5789
5790 //---------- Population Count Instructions -------------------------------------
5791
5792 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
5793 predicate(UsePopCountInstruction);
5794 match(Set dst (PopCountI src));
5795 effect(KILL cr);
5796
5797 format %{ "popcnt $dst, $src" %}
5798 ins_encode %{
5799 __ popcntl($dst$$Register, $src$$Register);
5800 %}
5801 ins_pipe(ialu_reg);
5802 %}
5803
5804 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
5805 predicate(UsePopCountInstruction);
5806 match(Set dst (PopCountI (LoadI mem)));
5807 effect(KILL cr);
5808
5809 format %{ "popcnt $dst, $mem" %}
5810 ins_encode %{
5811 __ popcntl($dst$$Register, $mem$$Address);
5812 %}
5813 ins_pipe(ialu_reg);
5814 %}
5815
5816 // Note: Long.bitCount(long) returns an int.
5817 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
5818 predicate(UsePopCountInstruction);
5819 match(Set dst (PopCountL src));
5820 effect(KILL cr);
5821
5822 format %{ "popcnt $dst, $src" %}
5823 ins_encode %{
5824 __ popcntq($dst$$Register, $src$$Register);
5825 %}
5826 ins_pipe(ialu_reg);
5827 %}
5828
5829 // Note: Long.bitCount(long) returns an int.
5830 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
5831 predicate(UsePopCountInstruction);
5832 match(Set dst (PopCountL (LoadL mem)));
5833 effect(KILL cr);
5834
5835 format %{ "popcnt $dst, $mem" %}
5836 ins_encode %{
5837 __ popcntq($dst$$Register, $mem$$Address);
5838 %}
5839 ins_pipe(ialu_reg);
5840 %}
5841
5842
5843 //----------MemBar Instructions-----------------------------------------------
5844 // Memory barrier flavors
5845
5846 instruct membar_acquire()
5847 %{
5848 match(MemBarAcquire);
5849 match(LoadFence);
5850 ins_cost(0);
5851
5852 size(0);
5853 format %{ "MEMBAR-acquire ! (empty encoding)" %}
5854 ins_encode();
5855 ins_pipe(empty);
5856 %}
5857
5858 instruct membar_acquire_lock()
5859 %{
5860 match(MemBarAcquireLock);
5861 ins_cost(0);
5862
5863 size(0);
5864 format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
5865 ins_encode();
5866 ins_pipe(empty);
5867 %}
5868
5869 instruct membar_release()
5870 %{
5871 match(MemBarRelease);
5872 match(StoreFence);
5873 ins_cost(0);
5874
5875 size(0);
5876 format %{ "MEMBAR-release ! (empty encoding)" %}
5877 ins_encode();
5878 ins_pipe(empty);
5879 %}
5880
5881 instruct membar_release_lock()
5882 %{
5883 match(MemBarReleaseLock);
5884 ins_cost(0);
5885
5886 size(0);
5887 format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
5888 ins_encode();
5889 ins_pipe(empty);
5890 %}
5891
5892 instruct membar_volatile(rFlagsReg cr) %{
5893 match(MemBarVolatile);
5894 effect(KILL cr);
5895 ins_cost(400);
5896
5897 format %{
5898 $$template
5899 $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
5900 %}
5901 ins_encode %{
5902 __ membar(Assembler::StoreLoad);
5903 %}
5904 ins_pipe(pipe_slow);
5905 %}
5906
5907 instruct unnecessary_membar_volatile()
5908 %{
5909 match(MemBarVolatile);
5910 predicate(Matcher::post_store_load_barrier(n));
5911 ins_cost(0);
5912
5913 size(0);
5914 format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
5915 ins_encode();
5916 ins_pipe(empty);
5917 %}
5918
5919 instruct membar_storestore() %{
5920 match(MemBarStoreStore);
5921 match(StoreStoreFence);
5922 ins_cost(0);
5923
5924 size(0);
5925 format %{ "MEMBAR-storestore (empty encoding)" %}
5926 ins_encode( );
5927 ins_pipe(empty);
5928 %}
5929
5930 //----------Move Instructions--------------------------------------------------
5931
5932 instruct castX2P(rRegP dst, rRegL src)
5933 %{
5934 match(Set dst (CastX2P src));
5935
5936 format %{ "movq $dst, $src\t# long->ptr" %}
5937 ins_encode %{
5938 if ($dst$$reg != $src$$reg) {
5939 __ movptr($dst$$Register, $src$$Register);
5940 }
5941 %}
5942 ins_pipe(ialu_reg_reg); // XXX
5943 %}
5944
5945 instruct castP2X(rRegL dst, rRegP src)
5946 %{
5947 match(Set dst (CastP2X src));
5948
5949 format %{ "movq $dst, $src\t# ptr -> long" %}
5950 ins_encode %{
5951 if ($dst$$reg != $src$$reg) {
5952 __ movptr($dst$$Register, $src$$Register);
5953 }
5954 %}
5955 ins_pipe(ialu_reg_reg); // XXX
5956 %}
5957
5958 // Convert oop into int for vectors alignment masking
5959 instruct convP2I(rRegI dst, rRegP src)
5960 %{
5961 match(Set dst (ConvL2I (CastP2X src)));
5962
5963 format %{ "movl $dst, $src\t# ptr -> int" %}
5964 ins_encode %{
5965 __ movl($dst$$Register, $src$$Register);
5966 %}
5967 ins_pipe(ialu_reg_reg); // XXX
5968 %}
5969
5970 // Convert compressed oop into int for vectors alignment masking
5971 // in case of 32bit oops (heap < 4Gb).
5972 instruct convN2I(rRegI dst, rRegN src)
5973 %{
5974 predicate(CompressedOops::shift() == 0);
5975 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
5976
5977 format %{ "movl $dst, $src\t# compressed ptr -> int" %}
5978 ins_encode %{
5979 __ movl($dst$$Register, $src$$Register);
5980 %}
5981 ins_pipe(ialu_reg_reg); // XXX
5982 %}
5983
5984 // Convert oop pointer into compressed form
5985 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
5986 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
5987 match(Set dst (EncodeP src));
5988 effect(KILL cr);
5989 format %{ "encode_heap_oop $dst,$src" %}
5990 ins_encode %{
5991 Register s = $src$$Register;
5992 Register d = $dst$$Register;
5993 if (s != d) {
5994 __ movq(d, s);
5995 }
5996 __ encode_heap_oop(d);
5997 %}
5998 ins_pipe(ialu_reg_long);
5999 %}
6000
6001 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6002 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
6003 match(Set dst (EncodeP src));
6004 effect(KILL cr);
6005 format %{ "encode_heap_oop_not_null $dst,$src" %}
6006 ins_encode %{
6007 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
6008 %}
6009 ins_pipe(ialu_reg_long);
6010 %}
6011
6012 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
6013 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
6014 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
6015 match(Set dst (DecodeN src));
6016 effect(KILL cr);
6017 format %{ "decode_heap_oop $dst,$src" %}
6018 ins_encode %{
6019 Register s = $src$$Register;
6020 Register d = $dst$$Register;
6021 if (s != d) {
6022 __ movq(d, s);
6023 }
6024 __ decode_heap_oop(d);
6025 %}
6026 ins_pipe(ialu_reg_long);
6027 %}
6028
6029 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6030 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
6031 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
6032 match(Set dst (DecodeN src));
6033 effect(KILL cr);
6034 format %{ "decode_heap_oop_not_null $dst,$src" %}
6035 ins_encode %{
6036 Register s = $src$$Register;
6037 Register d = $dst$$Register;
6038 if (s != d) {
6039 __ decode_heap_oop_not_null(d, s);
6040 } else {
6041 __ decode_heap_oop_not_null(d);
6042 }
6043 %}
6044 ins_pipe(ialu_reg_long);
6045 %}
6046
6047 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6048 match(Set dst (EncodePKlass src));
6049 effect(TEMP dst, KILL cr);
6050 format %{ "encode_and_move_klass_not_null $dst,$src" %}
6051 ins_encode %{
6052 __ encode_and_move_klass_not_null($dst$$Register, $src$$Register);
6053 %}
6054 ins_pipe(ialu_reg_long);
6055 %}
6056
6057 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6058 match(Set dst (DecodeNKlass src));
6059 effect(TEMP dst, KILL cr);
6060 format %{ "decode_and_move_klass_not_null $dst,$src" %}
6061 ins_encode %{
6062 __ decode_and_move_klass_not_null($dst$$Register, $src$$Register);
6063 %}
6064 ins_pipe(ialu_reg_long);
6065 %}
6066
6067 //----------Conditional Move---------------------------------------------------
6068 // Jump
6069 // dummy instruction for generating temp registers
6070 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
6071 match(Jump (LShiftL switch_val shift));
6072 ins_cost(350);
6073 predicate(false);
6074 effect(TEMP dest);
6075
6076 format %{ "leaq $dest, [$constantaddress]\n\t"
6077 "jmp [$dest + $switch_val << $shift]\n\t" %}
6078 ins_encode %{
6079 // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6080 // to do that and the compiler is using that register as one it can allocate.
6081 // So we build it all by hand.
6082 // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
6083 // ArrayAddress dispatch(table, index);
6084 Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
6085 __ lea($dest$$Register, $constantaddress);
6086 __ jmp(dispatch);
6087 %}
6088 ins_pipe(pipe_jmp);
6089 %}
6090
6091 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
6092 match(Jump (AddL (LShiftL switch_val shift) offset));
6093 ins_cost(350);
6094 effect(TEMP dest);
6095
6096 format %{ "leaq $dest, [$constantaddress]\n\t"
6097 "jmp [$dest + $switch_val << $shift + $offset]\n\t" %}
6098 ins_encode %{
6099 // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6100 // to do that and the compiler is using that register as one it can allocate.
6101 // So we build it all by hand.
6102 // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6103 // ArrayAddress dispatch(table, index);
6104 Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6105 __ lea($dest$$Register, $constantaddress);
6106 __ jmp(dispatch);
6107 %}
6108 ins_pipe(pipe_jmp);
6109 %}
6110
6111 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
6112 match(Jump switch_val);
6113 ins_cost(350);
6114 effect(TEMP dest);
6115
6116 format %{ "leaq $dest, [$constantaddress]\n\t"
6117 "jmp [$dest + $switch_val]\n\t" %}
6118 ins_encode %{
6119 // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6120 // to do that and the compiler is using that register as one it can allocate.
6121 // So we build it all by hand.
6122 // Address index(noreg, switch_reg, Address::times_1);
6123 // ArrayAddress dispatch(table, index);
6124 Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
6125 __ lea($dest$$Register, $constantaddress);
6126 __ jmp(dispatch);
6127 %}
6128 ins_pipe(pipe_jmp);
6129 %}
6130
6131 // Conditional move
6132 instruct cmovI_imm_01(rRegI dst, immI_1 src, rFlagsReg cr, cmpOp cop)
6133 %{
6134 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
6135 match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
6136
6137 ins_cost(100); // XXX
6138 format %{ "setbn$cop $dst\t# signed, int" %}
6139 ins_encode %{
6140 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
6141 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
6142 %}
6143 ins_pipe(ialu_reg);
6144 %}
6145
6146 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
6147 %{
6148 match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6149
6150 ins_cost(200); // XXX
6151 format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6152 ins_encode %{
6153 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
6154 %}
6155 ins_pipe(pipe_cmov_reg);
6156 %}
6157
6158 instruct cmovI_imm_01U(rRegI dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
6159 %{
6160 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
6161 match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
6162
6163 ins_cost(100); // XXX
6164 format %{ "setbn$cop $dst\t# unsigned, int" %}
6165 ins_encode %{
6166 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
6167 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
6168 %}
6169 ins_pipe(ialu_reg);
6170 %}
6171
6172 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
6173 match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6174
6175 ins_cost(200); // XXX
6176 format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6177 ins_encode %{
6178 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
6179 %}
6180 ins_pipe(pipe_cmov_reg);
6181 %}
6182
6183 instruct cmovI_imm_01UCF(rRegI dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
6184 %{
6185 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
6186 match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
6187
6188 ins_cost(100); // XXX
6189 format %{ "setbn$cop $dst\t# unsigned, int" %}
6190 ins_encode %{
6191 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
6192 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
6193 %}
6194 ins_pipe(ialu_reg);
6195 %}
6196
6197 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
6198 match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6199 ins_cost(200);
6200 expand %{
6201 cmovI_regU(cop, cr, dst, src);
6202 %}
6203 %}
6204
6205 instruct cmovI_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
6206 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
6207 match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6208
6209 ins_cost(200); // XXX
6210 format %{ "cmovpl $dst, $src\n\t"
6211 "cmovnel $dst, $src" %}
6212 ins_encode %{
6213 __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
6214 __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
6215 %}
6216 ins_pipe(pipe_cmov_reg);
6217 %}
6218
6219 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
6220 // inputs of the CMove
6221 instruct cmovI_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
6222 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
6223 match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
6224
6225 ins_cost(200); // XXX
6226 format %{ "cmovpl $dst, $src\n\t"
6227 "cmovnel $dst, $src" %}
6228 ins_encode %{
6229 __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
6230 __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
6231 %}
6232 ins_pipe(pipe_cmov_reg);
6233 %}
6234
6235 // Conditional move
6236 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
6237 match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6238
6239 ins_cost(250); // XXX
6240 format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6241 ins_encode %{
6242 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
6243 %}
6244 ins_pipe(pipe_cmov_mem);
6245 %}
6246
6247 // Conditional move
6248 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
6249 %{
6250 match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6251
6252 ins_cost(250); // XXX
6253 format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6254 ins_encode %{
6255 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
6256 %}
6257 ins_pipe(pipe_cmov_mem);
6258 %}
6259
6260 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
6261 match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6262 ins_cost(250);
6263 expand %{
6264 cmovI_memU(cop, cr, dst, src);
6265 %}
6266 %}
6267
6268 // Conditional move
6269 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
6270 %{
6271 match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6272
6273 ins_cost(200); // XXX
6274 format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
6275 ins_encode %{
6276 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
6277 %}
6278 ins_pipe(pipe_cmov_reg);
6279 %}
6280
6281 // Conditional move
6282 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
6283 %{
6284 match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6285
6286 ins_cost(200); // XXX
6287 format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
6288 ins_encode %{
6289 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
6290 %}
6291 ins_pipe(pipe_cmov_reg);
6292 %}
6293
6294 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
6295 match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6296 ins_cost(200);
6297 expand %{
6298 cmovN_regU(cop, cr, dst, src);
6299 %}
6300 %}
6301
6302 instruct cmovN_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
6303 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
6304 match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6305
6306 ins_cost(200); // XXX
6307 format %{ "cmovpl $dst, $src\n\t"
6308 "cmovnel $dst, $src" %}
6309 ins_encode %{
6310 __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
6311 __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
6312 %}
6313 ins_pipe(pipe_cmov_reg);
6314 %}
6315
6316 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
6317 // inputs of the CMove
6318 instruct cmovN_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
6319 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
6320 match(Set dst (CMoveN (Binary cop cr) (Binary src dst)));
6321
6322 ins_cost(200); // XXX
6323 format %{ "cmovpl $dst, $src\n\t"
6324 "cmovnel $dst, $src" %}
6325 ins_encode %{
6326 __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
6327 __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
6328 %}
6329 ins_pipe(pipe_cmov_reg);
6330 %}
6331
6332 // Conditional move
6333 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
6334 %{
6335 match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6336
6337 ins_cost(200); // XXX
6338 format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
6339 ins_encode %{
6340 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
6341 %}
6342 ins_pipe(pipe_cmov_reg); // XXX
6343 %}
6344
6345 // Conditional move
6346 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
6347 %{
6348 match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6349
6350 ins_cost(200); // XXX
6351 format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
6352 ins_encode %{
6353 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
6354 %}
6355 ins_pipe(pipe_cmov_reg); // XXX
6356 %}
6357
6358 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
6359 match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6360 ins_cost(200);
6361 expand %{
6362 cmovP_regU(cop, cr, dst, src);
6363 %}
6364 %}
6365
6366 instruct cmovP_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
6367 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
6368 match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6369
6370 ins_cost(200); // XXX
6371 format %{ "cmovpq $dst, $src\n\t"
6372 "cmovneq $dst, $src" %}
6373 ins_encode %{
6374 __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
6375 __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
6376 %}
6377 ins_pipe(pipe_cmov_reg);
6378 %}
6379
6380 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
6381 // inputs of the CMove
6382 instruct cmovP_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
6383 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
6384 match(Set dst (CMoveP (Binary cop cr) (Binary src dst)));
6385
6386 ins_cost(200); // XXX
6387 format %{ "cmovpq $dst, $src\n\t"
6388 "cmovneq $dst, $src" %}
6389 ins_encode %{
6390 __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
6391 __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
6392 %}
6393 ins_pipe(pipe_cmov_reg);
6394 %}
6395
6396 instruct cmovL_imm_01(rRegL dst, immL1 src, rFlagsReg cr, cmpOp cop)
6397 %{
6398 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
6399 match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
6400
6401 ins_cost(100); // XXX
6402 format %{ "setbn$cop $dst\t# signed, long" %}
6403 ins_encode %{
6404 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
6405 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
6406 %}
6407 ins_pipe(ialu_reg);
6408 %}
6409
6410 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
6411 %{
6412 match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6413
6414 ins_cost(200); // XXX
6415 format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6416 ins_encode %{
6417 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
6418 %}
6419 ins_pipe(pipe_cmov_reg); // XXX
6420 %}
6421
6422 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
6423 %{
6424 match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6425
6426 ins_cost(200); // XXX
6427 format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6428 ins_encode %{
6429 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
6430 %}
6431 ins_pipe(pipe_cmov_mem); // XXX
6432 %}
6433
6434 instruct cmovL_imm_01U(rRegL dst, immL1 src, rFlagsRegU cr, cmpOpU cop)
6435 %{
6436 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
6437 match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
6438
6439 ins_cost(100); // XXX
6440 format %{ "setbn$cop $dst\t# unsigned, long" %}
6441 ins_encode %{
6442 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
6443 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
6444 %}
6445 ins_pipe(ialu_reg);
6446 %}
6447
6448 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
6449 %{
6450 match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6451
6452 ins_cost(200); // XXX
6453 format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6454 ins_encode %{
6455 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
6456 %}
6457 ins_pipe(pipe_cmov_reg); // XXX
6458 %}
6459
6460 instruct cmovL_imm_01UCF(rRegL dst, immL1 src, rFlagsRegUCF cr, cmpOpUCF cop)
6461 %{
6462 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
6463 match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
6464
6465 ins_cost(100); // XXX
6466 format %{ "setbn$cop $dst\t# unsigned, long" %}
6467 ins_encode %{
6468 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
6469 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
6470 %}
6471 ins_pipe(ialu_reg);
6472 %}
6473
6474 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
6475 match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6476 ins_cost(200);
6477 expand %{
6478 cmovL_regU(cop, cr, dst, src);
6479 %}
6480 %}
6481
6482 instruct cmovL_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
6483 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
6484 match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6485
6486 ins_cost(200); // XXX
6487 format %{ "cmovpq $dst, $src\n\t"
6488 "cmovneq $dst, $src" %}
6489 ins_encode %{
6490 __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
6491 __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
6492 %}
6493 ins_pipe(pipe_cmov_reg);
6494 %}
6495
6496 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
6497 // inputs of the CMove
6498 instruct cmovL_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
6499 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
6500 match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
6501
6502 ins_cost(200); // XXX
6503 format %{ "cmovpq $dst, $src\n\t"
6504 "cmovneq $dst, $src" %}
6505 ins_encode %{
6506 __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
6507 __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
6508 %}
6509 ins_pipe(pipe_cmov_reg);
6510 %}
6511
6512 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
6513 %{
6514 match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6515
6516 ins_cost(200); // XXX
6517 format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6518 ins_encode %{
6519 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
6520 %}
6521 ins_pipe(pipe_cmov_mem); // XXX
6522 %}
6523
6524 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
6525 match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6526 ins_cost(200);
6527 expand %{
6528 cmovL_memU(cop, cr, dst, src);
6529 %}
6530 %}
6531
6532 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
6533 %{
6534 match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6535
6536 ins_cost(200); // XXX
6537 format %{ "jn$cop skip\t# signed cmove float\n\t"
6538 "movss $dst, $src\n"
6539 "skip:" %}
6540 ins_encode %{
6541 Label Lskip;
6542 // Invert sense of branch from sense of CMOV
6543 __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6544 __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6545 __ bind(Lskip);
6546 %}
6547 ins_pipe(pipe_slow);
6548 %}
6549
6550 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
6551 %{
6552 match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6553
6554 ins_cost(200); // XXX
6555 format %{ "jn$cop skip\t# unsigned cmove float\n\t"
6556 "movss $dst, $src\n"
6557 "skip:" %}
6558 ins_encode %{
6559 Label Lskip;
6560 // Invert sense of branch from sense of CMOV
6561 __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6562 __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6563 __ bind(Lskip);
6564 %}
6565 ins_pipe(pipe_slow);
6566 %}
6567
6568 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
6569 match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6570 ins_cost(200);
6571 expand %{
6572 cmovF_regU(cop, cr, dst, src);
6573 %}
6574 %}
6575
6576 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
6577 %{
6578 match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6579
6580 ins_cost(200); // XXX
6581 format %{ "jn$cop skip\t# signed cmove double\n\t"
6582 "movsd $dst, $src\n"
6583 "skip:" %}
6584 ins_encode %{
6585 Label Lskip;
6586 // Invert sense of branch from sense of CMOV
6587 __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6588 __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
6589 __ bind(Lskip);
6590 %}
6591 ins_pipe(pipe_slow);
6592 %}
6593
6594 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
6595 %{
6596 match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6597
6598 ins_cost(200); // XXX
6599 format %{ "jn$cop skip\t# unsigned cmove double\n\t"
6600 "movsd $dst, $src\n"
6601 "skip:" %}
6602 ins_encode %{
6603 Label Lskip;
6604 // Invert sense of branch from sense of CMOV
6605 __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6606 __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
6607 __ bind(Lskip);
6608 %}
6609 ins_pipe(pipe_slow);
6610 %}
6611
6612 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
6613 match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6614 ins_cost(200);
6615 expand %{
6616 cmovD_regU(cop, cr, dst, src);
6617 %}
6618 %}
6619
6620 //----------Arithmetic Instructions--------------------------------------------
6621 //----------Addition Instructions----------------------------------------------
6622
6623 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
6624 %{
6625 match(Set dst (AddI dst src));
6626 effect(KILL cr);
6627 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
6628 format %{ "addl $dst, $src\t# int" %}
6629 ins_encode %{
6630 __ addl($dst$$Register, $src$$Register);
6631 %}
6632 ins_pipe(ialu_reg_reg);
6633 %}
6634
6635 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
6636 %{
6637 match(Set dst (AddI dst src));
6638 effect(KILL cr);
6639 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
6640
6641 format %{ "addl $dst, $src\t# int" %}
6642 ins_encode %{
6643 __ addl($dst$$Register, $src$$constant);
6644 %}
6645 ins_pipe( ialu_reg );
6646 %}
6647
6648 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
6649 %{
6650 match(Set dst (AddI dst (LoadI src)));
6651 effect(KILL cr);
6652 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
6653
6654 ins_cost(150); // XXX
6655 format %{ "addl $dst, $src\t# int" %}
6656 ins_encode %{
6657 __ addl($dst$$Register, $src$$Address);
6658 %}
6659 ins_pipe(ialu_reg_mem);
6660 %}
6661
6662 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
6663 %{
6664 match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6665 effect(KILL cr);
6666 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
6667
6668 ins_cost(150); // XXX
6669 format %{ "addl $dst, $src\t# int" %}
6670 ins_encode %{
6671 __ addl($dst$$Address, $src$$Register);
6672 %}
6673 ins_pipe(ialu_mem_reg);
6674 %}
6675
6676 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
6677 %{
6678 match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6679 effect(KILL cr);
6680 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
6681
6682
6683 ins_cost(125); // XXX
6684 format %{ "addl $dst, $src\t# int" %}
6685 ins_encode %{
6686 __ addl($dst$$Address, $src$$constant);
6687 %}
6688 ins_pipe(ialu_mem_imm);
6689 %}
6690
6691 instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
6692 %{
6693 predicate(UseIncDec);
6694 match(Set dst (AddI dst src));
6695 effect(KILL cr);
6696
6697 format %{ "incl $dst\t# int" %}
6698 ins_encode %{
6699 __ incrementl($dst$$Register);
6700 %}
6701 ins_pipe(ialu_reg);
6702 %}
6703
6704 instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
6705 %{
6706 predicate(UseIncDec);
6707 match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6708 effect(KILL cr);
6709
6710 ins_cost(125); // XXX
6711 format %{ "incl $dst\t# int" %}
6712 ins_encode %{
6713 __ incrementl($dst$$Address);
6714 %}
6715 ins_pipe(ialu_mem_imm);
6716 %}
6717
6718 // XXX why does that use AddI
6719 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
6720 %{
6721 predicate(UseIncDec);
6722 match(Set dst (AddI dst src));
6723 effect(KILL cr);
6724
6725 format %{ "decl $dst\t# int" %}
6726 ins_encode %{
6727 __ decrementl($dst$$Register);
6728 %}
6729 ins_pipe(ialu_reg);
6730 %}
6731
6732 // XXX why does that use AddI
6733 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
6734 %{
6735 predicate(UseIncDec);
6736 match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6737 effect(KILL cr);
6738
6739 ins_cost(125); // XXX
6740 format %{ "decl $dst\t# int" %}
6741 ins_encode %{
6742 __ decrementl($dst$$Address);
6743 %}
6744 ins_pipe(ialu_mem_imm);
6745 %}
6746
6747 instruct leaI_rReg_immI2_immI(rRegI dst, rRegI index, immI2 scale, immI disp)
6748 %{
6749 predicate(VM_Version::supports_fast_2op_lea());
6750 match(Set dst (AddI (LShiftI index scale) disp));
6751
6752 format %{ "leal $dst, [$index << $scale + $disp]\t# int" %}
6753 ins_encode %{
6754 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
6755 __ leal($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
6756 %}
6757 ins_pipe(ialu_reg_reg);
6758 %}
6759
6760 instruct leaI_rReg_rReg_immI(rRegI dst, rRegI base, rRegI index, immI disp)
6761 %{
6762 predicate(VM_Version::supports_fast_3op_lea());
6763 match(Set dst (AddI (AddI base index) disp));
6764
6765 format %{ "leal $dst, [$base + $index + $disp]\t# int" %}
6766 ins_encode %{
6767 __ leal($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
6768 %}
6769 ins_pipe(ialu_reg_reg);
6770 %}
6771
6772 instruct leaI_rReg_rReg_immI2(rRegI dst, no_rbp_r13_RegI base, rRegI index, immI2 scale)
6773 %{
6774 predicate(VM_Version::supports_fast_2op_lea());
6775 match(Set dst (AddI base (LShiftI index scale)));
6776
6777 format %{ "leal $dst, [$base + $index << $scale]\t# int" %}
6778 ins_encode %{
6779 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
6780 __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale));
6781 %}
6782 ins_pipe(ialu_reg_reg);
6783 %}
6784
6785 instruct leaI_rReg_rReg_immI2_immI(rRegI dst, rRegI base, rRegI index, immI2 scale, immI disp)
6786 %{
6787 predicate(VM_Version::supports_fast_3op_lea());
6788 match(Set dst (AddI (AddI base (LShiftI index scale)) disp));
6789
6790 format %{ "leal $dst, [$base + $index << $scale + $disp]\t# int" %}
6791 ins_encode %{
6792 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
6793 __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
6794 %}
6795 ins_pipe(ialu_reg_reg);
6796 %}
6797
6798 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
6799 %{
6800 match(Set dst (AddL dst src));
6801 effect(KILL cr);
6802 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
6803
6804 format %{ "addq $dst, $src\t# long" %}
6805 ins_encode %{
6806 __ addq($dst$$Register, $src$$Register);
6807 %}
6808 ins_pipe(ialu_reg_reg);
6809 %}
6810
6811 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
6812 %{
6813 match(Set dst (AddL dst src));
6814 effect(KILL cr);
6815 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
6816
6817 format %{ "addq $dst, $src\t# long" %}
6818 ins_encode %{
6819 __ addq($dst$$Register, $src$$constant);
6820 %}
6821 ins_pipe( ialu_reg );
6822 %}
6823
6824 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
6825 %{
6826 match(Set dst (AddL dst (LoadL src)));
6827 effect(KILL cr);
6828 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
6829
6830 ins_cost(150); // XXX
6831 format %{ "addq $dst, $src\t# long" %}
6832 ins_encode %{
6833 __ addq($dst$$Register, $src$$Address);
6834 %}
6835 ins_pipe(ialu_reg_mem);
6836 %}
6837
6838 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
6839 %{
6840 match(Set dst (StoreL dst (AddL (LoadL dst) src)));
6841 effect(KILL cr);
6842 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
6843
6844 ins_cost(150); // XXX
6845 format %{ "addq $dst, $src\t# long" %}
6846 ins_encode %{
6847 __ addq($dst$$Address, $src$$Register);
6848 %}
6849 ins_pipe(ialu_mem_reg);
6850 %}
6851
6852 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
6853 %{
6854 match(Set dst (StoreL dst (AddL (LoadL dst) src)));
6855 effect(KILL cr);
6856 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
6857
6858 ins_cost(125); // XXX
6859 format %{ "addq $dst, $src\t# long" %}
6860 ins_encode %{
6861 __ addq($dst$$Address, $src$$constant);
6862 %}
6863 ins_pipe(ialu_mem_imm);
6864 %}
6865
6866 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
6867 %{
6868 predicate(UseIncDec);
6869 match(Set dst (AddL dst src));
6870 effect(KILL cr);
6871
6872 format %{ "incq $dst\t# long" %}
6873 ins_encode %{
6874 __ incrementq($dst$$Register);
6875 %}
6876 ins_pipe(ialu_reg);
6877 %}
6878
6879 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
6880 %{
6881 predicate(UseIncDec);
6882 match(Set dst (StoreL dst (AddL (LoadL dst) src)));
6883 effect(KILL cr);
6884
6885 ins_cost(125); // XXX
6886 format %{ "incq $dst\t# long" %}
6887 ins_encode %{
6888 __ incrementq($dst$$Address);
6889 %}
6890 ins_pipe(ialu_mem_imm);
6891 %}
6892
6893 // XXX why does that use AddL
6894 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
6895 %{
6896 predicate(UseIncDec);
6897 match(Set dst (AddL dst src));
6898 effect(KILL cr);
6899
6900 format %{ "decq $dst\t# long" %}
6901 ins_encode %{
6902 __ decrementq($dst$$Register);
6903 %}
6904 ins_pipe(ialu_reg);
6905 %}
6906
6907 // XXX why does that use AddL
6908 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
6909 %{
6910 predicate(UseIncDec);
6911 match(Set dst (StoreL dst (AddL (LoadL dst) src)));
6912 effect(KILL cr);
6913
6914 ins_cost(125); // XXX
6915 format %{ "decq $dst\t# long" %}
6916 ins_encode %{
6917 __ decrementq($dst$$Address);
6918 %}
6919 ins_pipe(ialu_mem_imm);
6920 %}
6921
6922 instruct leaL_rReg_immI2_immL32(rRegL dst, rRegL index, immI2 scale, immL32 disp)
6923 %{
6924 predicate(VM_Version::supports_fast_2op_lea());
6925 match(Set dst (AddL (LShiftL index scale) disp));
6926
6927 format %{ "leaq $dst, [$index << $scale + $disp]\t# long" %}
6928 ins_encode %{
6929 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
6930 __ leaq($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
6931 %}
6932 ins_pipe(ialu_reg_reg);
6933 %}
6934
6935 instruct leaL_rReg_rReg_immL32(rRegL dst, rRegL base, rRegL index, immL32 disp)
6936 %{
6937 predicate(VM_Version::supports_fast_3op_lea());
6938 match(Set dst (AddL (AddL base index) disp));
6939
6940 format %{ "leaq $dst, [$base + $index + $disp]\t# long" %}
6941 ins_encode %{
6942 __ leaq($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
6943 %}
6944 ins_pipe(ialu_reg_reg);
6945 %}
6946
6947 instruct leaL_rReg_rReg_immI2(rRegL dst, no_rbp_r13_RegL base, rRegL index, immI2 scale)
6948 %{
6949 predicate(VM_Version::supports_fast_2op_lea());
6950 match(Set dst (AddL base (LShiftL index scale)));
6951
6952 format %{ "leaq $dst, [$base + $index << $scale]\t# long" %}
6953 ins_encode %{
6954 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
6955 __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale));
6956 %}
6957 ins_pipe(ialu_reg_reg);
6958 %}
6959
6960 instruct leaL_rReg_rReg_immI2_immL32(rRegL dst, rRegL base, rRegL index, immI2 scale, immL32 disp)
6961 %{
6962 predicate(VM_Version::supports_fast_3op_lea());
6963 match(Set dst (AddL (AddL base (LShiftL index scale)) disp));
6964
6965 format %{ "leaq $dst, [$base + $index << $scale + $disp]\t# long" %}
6966 ins_encode %{
6967 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
6968 __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
6969 %}
6970 ins_pipe(ialu_reg_reg);
6971 %}
6972
6973 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
6974 %{
6975 match(Set dst (AddP dst src));
6976 effect(KILL cr);
6977 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
6978
6979 format %{ "addq $dst, $src\t# ptr" %}
6980 ins_encode %{
6981 __ addq($dst$$Register, $src$$Register);
6982 %}
6983 ins_pipe(ialu_reg_reg);
6984 %}
6985
6986 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
6987 %{
6988 match(Set dst (AddP dst src));
6989 effect(KILL cr);
6990 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
6991
6992 format %{ "addq $dst, $src\t# ptr" %}
6993 ins_encode %{
6994 __ addq($dst$$Register, $src$$constant);
6995 %}
6996 ins_pipe( ialu_reg );
6997 %}
6998
6999 // XXX addP mem ops ????
7000
7001 instruct checkCastPP(rRegP dst)
7002 %{
7003 match(Set dst (CheckCastPP dst));
7004
7005 size(0);
7006 format %{ "# checkcastPP of $dst" %}
7007 ins_encode(/* empty encoding */);
7008 ins_pipe(empty);
7009 %}
7010
7011 instruct castPP(rRegP dst)
7012 %{
7013 match(Set dst (CastPP dst));
7014
7015 size(0);
7016 format %{ "# castPP of $dst" %}
7017 ins_encode(/* empty encoding */);
7018 ins_pipe(empty);
7019 %}
7020
7021 instruct castII(rRegI dst)
7022 %{
7023 match(Set dst (CastII dst));
7024
7025 size(0);
7026 format %{ "# castII of $dst" %}
7027 ins_encode(/* empty encoding */);
7028 ins_cost(0);
7029 ins_pipe(empty);
7030 %}
7031
7032 instruct castLL(rRegL dst)
7033 %{
7034 match(Set dst (CastLL dst));
7035
7036 size(0);
7037 format %{ "# castLL of $dst" %}
7038 ins_encode(/* empty encoding */);
7039 ins_cost(0);
7040 ins_pipe(empty);
7041 %}
7042
7043 instruct castFF(regF dst)
7044 %{
7045 match(Set dst (CastFF dst));
7046
7047 size(0);
7048 format %{ "# castFF of $dst" %}
7049 ins_encode(/* empty encoding */);
7050 ins_cost(0);
7051 ins_pipe(empty);
7052 %}
7053
7054 instruct castDD(regD dst)
7055 %{
7056 match(Set dst (CastDD dst));
7057
7058 size(0);
7059 format %{ "# castDD of $dst" %}
7060 ins_encode(/* empty encoding */);
7061 ins_cost(0);
7062 ins_pipe(empty);
7063 %}
7064
7065 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
7066 instruct compareAndSwapP(rRegI res,
7067 memory mem_ptr,
7068 rax_RegP oldval, rRegP newval,
7069 rFlagsReg cr)
7070 %{
7071 predicate(n->as_LoadStore()->barrier_data() == 0);
7072 match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
7073 match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
7074 effect(KILL cr, KILL oldval);
7075
7076 format %{ "cmpxchgq $mem_ptr,$newval\t# "
7077 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7078 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
7079 ins_encode %{
7080 __ lock();
7081 __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
7082 __ setcc(Assembler::equal, $res$$Register);
7083 %}
7084 ins_pipe( pipe_cmpxchg );
7085 %}
7086
7087 instruct compareAndSwapL(rRegI res,
7088 memory mem_ptr,
7089 rax_RegL oldval, rRegL newval,
7090 rFlagsReg cr)
7091 %{
7092 match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
7093 match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
7094 effect(KILL cr, KILL oldval);
7095
7096 format %{ "cmpxchgq $mem_ptr,$newval\t# "
7097 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7098 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
7099 ins_encode %{
7100 __ lock();
7101 __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
7102 __ setcc(Assembler::equal, $res$$Register);
7103 %}
7104 ins_pipe( pipe_cmpxchg );
7105 %}
7106
7107 instruct compareAndSwapI(rRegI res,
7108 memory mem_ptr,
7109 rax_RegI oldval, rRegI newval,
7110 rFlagsReg cr)
7111 %{
7112 match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
7113 match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
7114 effect(KILL cr, KILL oldval);
7115
7116 format %{ "cmpxchgl $mem_ptr,$newval\t# "
7117 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7118 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
7119 ins_encode %{
7120 __ lock();
7121 __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
7122 __ setcc(Assembler::equal, $res$$Register);
7123 %}
7124 ins_pipe( pipe_cmpxchg );
7125 %}
7126
7127 instruct compareAndSwapB(rRegI res,
7128 memory mem_ptr,
7129 rax_RegI oldval, rRegI newval,
7130 rFlagsReg cr)
7131 %{
7132 match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
7133 match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
7134 effect(KILL cr, KILL oldval);
7135
7136 format %{ "cmpxchgb $mem_ptr,$newval\t# "
7137 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7138 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
7139 ins_encode %{
7140 __ lock();
7141 __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
7142 __ setcc(Assembler::equal, $res$$Register);
7143 %}
7144 ins_pipe( pipe_cmpxchg );
7145 %}
7146
7147 instruct compareAndSwapS(rRegI res,
7148 memory mem_ptr,
7149 rax_RegI oldval, rRegI newval,
7150 rFlagsReg cr)
7151 %{
7152 match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
7153 match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
7154 effect(KILL cr, KILL oldval);
7155
7156 format %{ "cmpxchgw $mem_ptr,$newval\t# "
7157 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7158 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
7159 ins_encode %{
7160 __ lock();
7161 __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
7162 __ setcc(Assembler::equal, $res$$Register);
7163 %}
7164 ins_pipe( pipe_cmpxchg );
7165 %}
7166
7167 instruct compareAndSwapN(rRegI res,
7168 memory mem_ptr,
7169 rax_RegN oldval, rRegN newval,
7170 rFlagsReg cr) %{
7171 predicate(n->as_LoadStore()->barrier_data() == 0);
7172 match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
7173 match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
7174 effect(KILL cr, KILL oldval);
7175
7176 format %{ "cmpxchgl $mem_ptr,$newval\t# "
7177 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7178 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
7179 ins_encode %{
7180 __ lock();
7181 __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
7182 __ setcc(Assembler::equal, $res$$Register);
7183 %}
7184 ins_pipe( pipe_cmpxchg );
7185 %}
7186
7187 instruct compareAndExchangeB(
7188 memory mem_ptr,
7189 rax_RegI oldval, rRegI newval,
7190 rFlagsReg cr)
7191 %{
7192 match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
7193 effect(KILL cr);
7194
7195 format %{ "cmpxchgb $mem_ptr,$newval\t# "
7196 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
7197 ins_encode %{
7198 __ lock();
7199 __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
7200 %}
7201 ins_pipe( pipe_cmpxchg );
7202 %}
7203
7204 instruct compareAndExchangeS(
7205 memory mem_ptr,
7206 rax_RegI oldval, rRegI newval,
7207 rFlagsReg cr)
7208 %{
7209 match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
7210 effect(KILL cr);
7211
7212 format %{ "cmpxchgw $mem_ptr,$newval\t# "
7213 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
7214 ins_encode %{
7215 __ lock();
7216 __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
7217 %}
7218 ins_pipe( pipe_cmpxchg );
7219 %}
7220
7221 instruct compareAndExchangeI(
7222 memory mem_ptr,
7223 rax_RegI oldval, rRegI newval,
7224 rFlagsReg cr)
7225 %{
7226 match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
7227 effect(KILL cr);
7228
7229 format %{ "cmpxchgl $mem_ptr,$newval\t# "
7230 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
7231 ins_encode %{
7232 __ lock();
7233 __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
7234 %}
7235 ins_pipe( pipe_cmpxchg );
7236 %}
7237
7238 instruct compareAndExchangeL(
7239 memory mem_ptr,
7240 rax_RegL oldval, rRegL newval,
7241 rFlagsReg cr)
7242 %{
7243 match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
7244 effect(KILL cr);
7245
7246 format %{ "cmpxchgq $mem_ptr,$newval\t# "
7247 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
7248 ins_encode %{
7249 __ lock();
7250 __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
7251 %}
7252 ins_pipe( pipe_cmpxchg );
7253 %}
7254
7255 instruct compareAndExchangeN(
7256 memory mem_ptr,
7257 rax_RegN oldval, rRegN newval,
7258 rFlagsReg cr) %{
7259 predicate(n->as_LoadStore()->barrier_data() == 0);
7260 match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
7261 effect(KILL cr);
7262
7263 format %{ "cmpxchgl $mem_ptr,$newval\t# "
7264 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
7265 ins_encode %{
7266 __ lock();
7267 __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
7268 %}
7269 ins_pipe( pipe_cmpxchg );
7270 %}
7271
7272 instruct compareAndExchangeP(
7273 memory mem_ptr,
7274 rax_RegP oldval, rRegP newval,
7275 rFlagsReg cr)
7276 %{
7277 predicate(n->as_LoadStore()->barrier_data() == 0);
7278 match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
7279 effect(KILL cr);
7280
7281 format %{ "cmpxchgq $mem_ptr,$newval\t# "
7282 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
7283 ins_encode %{
7284 __ lock();
7285 __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
7286 %}
7287 ins_pipe( pipe_cmpxchg );
7288 %}
7289
7290 instruct xaddB_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
7291 predicate(n->as_LoadStore()->result_not_used());
7292 match(Set dummy (GetAndAddB mem add));
7293 effect(KILL cr);
7294 format %{ "addb_lock $mem, $add" %}
7295 ins_encode %{
7296 __ lock();
7297 __ addb($mem$$Address, $add$$Register);
7298 %}
7299 ins_pipe(pipe_cmpxchg);
7300 %}
7301
7302 instruct xaddB_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7303 predicate(n->as_LoadStore()->result_not_used());
7304 match(Set dummy (GetAndAddB mem add));
7305 effect(KILL cr);
7306 format %{ "addb_lock $mem, $add" %}
7307 ins_encode %{
7308 __ lock();
7309 __ addb($mem$$Address, $add$$constant);
7310 %}
7311 ins_pipe(pipe_cmpxchg);
7312 %}
7313
7314 instruct xaddB(memory mem, rRegI newval, rFlagsReg cr) %{
7315 predicate(!n->as_LoadStore()->result_not_used());
7316 match(Set newval (GetAndAddB mem newval));
7317 effect(KILL cr);
7318 format %{ "xaddb_lock $mem, $newval" %}
7319 ins_encode %{
7320 __ lock();
7321 __ xaddb($mem$$Address, $newval$$Register);
7322 %}
7323 ins_pipe(pipe_cmpxchg);
7324 %}
7325
7326 instruct xaddS_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
7327 predicate(n->as_LoadStore()->result_not_used());
7328 match(Set dummy (GetAndAddS mem add));
7329 effect(KILL cr);
7330 format %{ "addw_lock $mem, $add" %}
7331 ins_encode %{
7332 __ lock();
7333 __ addw($mem$$Address, $add$$Register);
7334 %}
7335 ins_pipe(pipe_cmpxchg);
7336 %}
7337
7338 instruct xaddS_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7339 predicate(UseStoreImmI16 && n->as_LoadStore()->result_not_used());
7340 match(Set dummy (GetAndAddS mem add));
7341 effect(KILL cr);
7342 format %{ "addw_lock $mem, $add" %}
7343 ins_encode %{
7344 __ lock();
7345 __ addw($mem$$Address, $add$$constant);
7346 %}
7347 ins_pipe(pipe_cmpxchg);
7348 %}
7349
7350 instruct xaddS(memory mem, rRegI newval, rFlagsReg cr) %{
7351 predicate(!n->as_LoadStore()->result_not_used());
7352 match(Set newval (GetAndAddS mem newval));
7353 effect(KILL cr);
7354 format %{ "xaddw_lock $mem, $newval" %}
7355 ins_encode %{
7356 __ lock();
7357 __ xaddw($mem$$Address, $newval$$Register);
7358 %}
7359 ins_pipe(pipe_cmpxchg);
7360 %}
7361
7362 instruct xaddI_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
7363 predicate(n->as_LoadStore()->result_not_used());
7364 match(Set dummy (GetAndAddI mem add));
7365 effect(KILL cr);
7366 format %{ "addl_lock $mem, $add" %}
7367 ins_encode %{
7368 __ lock();
7369 __ addl($mem$$Address, $add$$Register);
7370 %}
7371 ins_pipe(pipe_cmpxchg);
7372 %}
7373
7374 instruct xaddI_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7375 predicate(n->as_LoadStore()->result_not_used());
7376 match(Set dummy (GetAndAddI mem add));
7377 effect(KILL cr);
7378 format %{ "addl_lock $mem, $add" %}
7379 ins_encode %{
7380 __ lock();
7381 __ addl($mem$$Address, $add$$constant);
7382 %}
7383 ins_pipe(pipe_cmpxchg);
7384 %}
7385
7386 instruct xaddI(memory mem, rRegI newval, rFlagsReg cr) %{
7387 predicate(!n->as_LoadStore()->result_not_used());
7388 match(Set newval (GetAndAddI mem newval));
7389 effect(KILL cr);
7390 format %{ "xaddl_lock $mem, $newval" %}
7391 ins_encode %{
7392 __ lock();
7393 __ xaddl($mem$$Address, $newval$$Register);
7394 %}
7395 ins_pipe(pipe_cmpxchg);
7396 %}
7397
7398 instruct xaddL_reg_no_res(memory mem, Universe dummy, rRegL add, rFlagsReg cr) %{
7399 predicate(n->as_LoadStore()->result_not_used());
7400 match(Set dummy (GetAndAddL mem add));
7401 effect(KILL cr);
7402 format %{ "addq_lock $mem, $add" %}
7403 ins_encode %{
7404 __ lock();
7405 __ addq($mem$$Address, $add$$Register);
7406 %}
7407 ins_pipe(pipe_cmpxchg);
7408 %}
7409
7410 instruct xaddL_imm_no_res(memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
7411 predicate(n->as_LoadStore()->result_not_used());
7412 match(Set dummy (GetAndAddL mem add));
7413 effect(KILL cr);
7414 format %{ "addq_lock $mem, $add" %}
7415 ins_encode %{
7416 __ lock();
7417 __ addq($mem$$Address, $add$$constant);
7418 %}
7419 ins_pipe(pipe_cmpxchg);
7420 %}
7421
7422 instruct xaddL(memory mem, rRegL newval, rFlagsReg cr) %{
7423 predicate(!n->as_LoadStore()->result_not_used());
7424 match(Set newval (GetAndAddL mem newval));
7425 effect(KILL cr);
7426 format %{ "xaddq_lock $mem, $newval" %}
7427 ins_encode %{
7428 __ lock();
7429 __ xaddq($mem$$Address, $newval$$Register);
7430 %}
7431 ins_pipe(pipe_cmpxchg);
7432 %}
7433
7434 instruct xchgB( memory mem, rRegI newval) %{
7435 match(Set newval (GetAndSetB mem newval));
7436 format %{ "XCHGB $newval,[$mem]" %}
7437 ins_encode %{
7438 __ xchgb($newval$$Register, $mem$$Address);
7439 %}
7440 ins_pipe( pipe_cmpxchg );
7441 %}
7442
7443 instruct xchgS( memory mem, rRegI newval) %{
7444 match(Set newval (GetAndSetS mem newval));
7445 format %{ "XCHGW $newval,[$mem]" %}
7446 ins_encode %{
7447 __ xchgw($newval$$Register, $mem$$Address);
7448 %}
7449 ins_pipe( pipe_cmpxchg );
7450 %}
7451
7452 instruct xchgI( memory mem, rRegI newval) %{
7453 match(Set newval (GetAndSetI mem newval));
7454 format %{ "XCHGL $newval,[$mem]" %}
7455 ins_encode %{
7456 __ xchgl($newval$$Register, $mem$$Address);
7457 %}
7458 ins_pipe( pipe_cmpxchg );
7459 %}
7460
7461 instruct xchgL( memory mem, rRegL newval) %{
7462 match(Set newval (GetAndSetL mem newval));
7463 format %{ "XCHGL $newval,[$mem]" %}
7464 ins_encode %{
7465 __ xchgq($newval$$Register, $mem$$Address);
7466 %}
7467 ins_pipe( pipe_cmpxchg );
7468 %}
7469
7470 instruct xchgP( memory mem, rRegP newval) %{
7471 match(Set newval (GetAndSetP mem newval));
7472 predicate(n->as_LoadStore()->barrier_data() == 0);
7473 format %{ "XCHGQ $newval,[$mem]" %}
7474 ins_encode %{
7475 __ xchgq($newval$$Register, $mem$$Address);
7476 %}
7477 ins_pipe( pipe_cmpxchg );
7478 %}
7479
7480 instruct xchgN( memory mem, rRegN newval) %{
7481 predicate(n->as_LoadStore()->barrier_data() == 0);
7482 match(Set newval (GetAndSetN mem newval));
7483 format %{ "XCHGL $newval,$mem]" %}
7484 ins_encode %{
7485 __ xchgl($newval$$Register, $mem$$Address);
7486 %}
7487 ins_pipe( pipe_cmpxchg );
7488 %}
7489
7490 //----------Abs Instructions-------------------------------------------
7491
7492 // Integer Absolute Instructions
7493 instruct absI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7494 %{
7495 match(Set dst (AbsI src));
7496 effect(TEMP dst, KILL cr);
7497 format %{ "xorl $dst, $dst\t# abs int\n\t"
7498 "subl $dst, $src\n\t"
7499 "cmovll $dst, $src" %}
7500 ins_encode %{
7501 __ xorl($dst$$Register, $dst$$Register);
7502 __ subl($dst$$Register, $src$$Register);
7503 __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
7504 %}
7505
7506 ins_pipe(ialu_reg_reg);
7507 %}
7508
7509 // Long Absolute Instructions
7510 instruct absL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7511 %{
7512 match(Set dst (AbsL src));
7513 effect(TEMP dst, KILL cr);
7514 format %{ "xorl $dst, $dst\t# abs long\n\t"
7515 "subq $dst, $src\n\t"
7516 "cmovlq $dst, $src" %}
7517 ins_encode %{
7518 __ xorl($dst$$Register, $dst$$Register);
7519 __ subq($dst$$Register, $src$$Register);
7520 __ cmovq(Assembler::less, $dst$$Register, $src$$Register);
7521 %}
7522
7523 ins_pipe(ialu_reg_reg);
7524 %}
7525
7526 //----------Subtraction Instructions-------------------------------------------
7527
7528 // Integer Subtraction Instructions
7529 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7530 %{
7531 match(Set dst (SubI dst src));
7532 effect(KILL cr);
7533 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
7534
7535 format %{ "subl $dst, $src\t# int" %}
7536 ins_encode %{
7537 __ subl($dst$$Register, $src$$Register);
7538 %}
7539 ins_pipe(ialu_reg_reg);
7540 %}
7541
7542 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
7543 %{
7544 match(Set dst (SubI dst (LoadI src)));
7545 effect(KILL cr);
7546 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
7547
7548 ins_cost(150);
7549 format %{ "subl $dst, $src\t# int" %}
7550 ins_encode %{
7551 __ subl($dst$$Register, $src$$Address);
7552 %}
7553 ins_pipe(ialu_reg_mem);
7554 %}
7555
7556 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
7557 %{
7558 match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7559 effect(KILL cr);
7560 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
7561
7562 ins_cost(150);
7563 format %{ "subl $dst, $src\t# int" %}
7564 ins_encode %{
7565 __ subl($dst$$Address, $src$$Register);
7566 %}
7567 ins_pipe(ialu_mem_reg);
7568 %}
7569
7570 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7571 %{
7572 match(Set dst (SubL dst src));
7573 effect(KILL cr);
7574 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
7575
7576 format %{ "subq $dst, $src\t# long" %}
7577 ins_encode %{
7578 __ subq($dst$$Register, $src$$Register);
7579 %}
7580 ins_pipe(ialu_reg_reg);
7581 %}
7582
7583 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7584 %{
7585 match(Set dst (SubL dst (LoadL src)));
7586 effect(KILL cr);
7587 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
7588
7589 ins_cost(150);
7590 format %{ "subq $dst, $src\t# long" %}
7591 ins_encode %{
7592 __ subq($dst$$Register, $src$$Address);
7593 %}
7594 ins_pipe(ialu_reg_mem);
7595 %}
7596
7597 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7598 %{
7599 match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7600 effect(KILL cr);
7601 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
7602
7603 ins_cost(150);
7604 format %{ "subq $dst, $src\t# long" %}
7605 ins_encode %{
7606 __ subq($dst$$Address, $src$$Register);
7607 %}
7608 ins_pipe(ialu_mem_reg);
7609 %}
7610
7611 // Subtract from a pointer
7612 // XXX hmpf???
7613 instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
7614 %{
7615 match(Set dst (AddP dst (SubI zero src)));
7616 effect(KILL cr);
7617
7618 format %{ "subq $dst, $src\t# ptr - int" %}
7619 ins_encode %{
7620 __ subq($dst$$Register, $src$$Register);
7621 %}
7622 ins_pipe(ialu_reg_reg);
7623 %}
7624
7625 instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
7626 %{
7627 match(Set dst (SubI zero dst));
7628 effect(KILL cr);
7629 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
7630
7631 format %{ "negl $dst\t# int" %}
7632 ins_encode %{
7633 __ negl($dst$$Register);
7634 %}
7635 ins_pipe(ialu_reg);
7636 %}
7637
7638 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
7639 %{
7640 match(Set dst (NegI dst));
7641 effect(KILL cr);
7642 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
7643
7644 format %{ "negl $dst\t# int" %}
7645 ins_encode %{
7646 __ negl($dst$$Register);
7647 %}
7648 ins_pipe(ialu_reg);
7649 %}
7650
7651 instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
7652 %{
7653 match(Set dst (StoreI dst (SubI zero (LoadI dst))));
7654 effect(KILL cr);
7655 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
7656
7657 format %{ "negl $dst\t# int" %}
7658 ins_encode %{
7659 __ negl($dst$$Address);
7660 %}
7661 ins_pipe(ialu_reg);
7662 %}
7663
7664 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
7665 %{
7666 match(Set dst (SubL zero dst));
7667 effect(KILL cr);
7668 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
7669
7670 format %{ "negq $dst\t# long" %}
7671 ins_encode %{
7672 __ negq($dst$$Register);
7673 %}
7674 ins_pipe(ialu_reg);
7675 %}
7676
7677 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
7678 %{
7679 match(Set dst (NegL dst));
7680 effect(KILL cr);
7681 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
7682
7683 format %{ "negq $dst\t# int" %}
7684 ins_encode %{
7685 __ negq($dst$$Register);
7686 %}
7687 ins_pipe(ialu_reg);
7688 %}
7689
7690 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
7691 %{
7692 match(Set dst (StoreL dst (SubL zero (LoadL dst))));
7693 effect(KILL cr);
7694 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
7695
7696 format %{ "negq $dst\t# long" %}
7697 ins_encode %{
7698 __ negq($dst$$Address);
7699 %}
7700 ins_pipe(ialu_reg);
7701 %}
7702
7703 //----------Multiplication/Division Instructions-------------------------------
7704 // Integer Multiplication Instructions
7705 // Multiply Register
7706
7707 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7708 %{
7709 match(Set dst (MulI dst src));
7710 effect(KILL cr);
7711
7712 ins_cost(300);
7713 format %{ "imull $dst, $src\t# int" %}
7714 ins_encode %{
7715 __ imull($dst$$Register, $src$$Register);
7716 %}
7717 ins_pipe(ialu_reg_reg_alu0);
7718 %}
7719
7720 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
7721 %{
7722 match(Set dst (MulI src imm));
7723 effect(KILL cr);
7724
7725 ins_cost(300);
7726 format %{ "imull $dst, $src, $imm\t# int" %}
7727 ins_encode %{
7728 __ imull($dst$$Register, $src$$Register, $imm$$constant);
7729 %}
7730 ins_pipe(ialu_reg_reg_alu0);
7731 %}
7732
7733 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
7734 %{
7735 match(Set dst (MulI dst (LoadI src)));
7736 effect(KILL cr);
7737
7738 ins_cost(350);
7739 format %{ "imull $dst, $src\t# int" %}
7740 ins_encode %{
7741 __ imull($dst$$Register, $src$$Address);
7742 %}
7743 ins_pipe(ialu_reg_mem_alu0);
7744 %}
7745
7746 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
7747 %{
7748 match(Set dst (MulI (LoadI src) imm));
7749 effect(KILL cr);
7750
7751 ins_cost(300);
7752 format %{ "imull $dst, $src, $imm\t# int" %}
7753 ins_encode %{
7754 __ imull($dst$$Register, $src$$Address, $imm$$constant);
7755 %}
7756 ins_pipe(ialu_reg_mem_alu0);
7757 %}
7758
7759 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
7760 %{
7761 match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
7762 effect(KILL cr, KILL src2);
7763
7764 expand %{ mulI_rReg(dst, src1, cr);
7765 mulI_rReg(src2, src3, cr);
7766 addI_rReg(dst, src2, cr); %}
7767 %}
7768
7769 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7770 %{
7771 match(Set dst (MulL dst src));
7772 effect(KILL cr);
7773
7774 ins_cost(300);
7775 format %{ "imulq $dst, $src\t# long" %}
7776 ins_encode %{
7777 __ imulq($dst$$Register, $src$$Register);
7778 %}
7779 ins_pipe(ialu_reg_reg_alu0);
7780 %}
7781
7782 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
7783 %{
7784 match(Set dst (MulL src imm));
7785 effect(KILL cr);
7786
7787 ins_cost(300);
7788 format %{ "imulq $dst, $src, $imm\t# long" %}
7789 ins_encode %{
7790 __ imulq($dst$$Register, $src$$Register, $imm$$constant);
7791 %}
7792 ins_pipe(ialu_reg_reg_alu0);
7793 %}
7794
7795 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
7796 %{
7797 match(Set dst (MulL dst (LoadL src)));
7798 effect(KILL cr);
7799
7800 ins_cost(350);
7801 format %{ "imulq $dst, $src\t# long" %}
7802 ins_encode %{
7803 __ imulq($dst$$Register, $src$$Address);
7804 %}
7805 ins_pipe(ialu_reg_mem_alu0);
7806 %}
7807
7808 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
7809 %{
7810 match(Set dst (MulL (LoadL src) imm));
7811 effect(KILL cr);
7812
7813 ins_cost(300);
7814 format %{ "imulq $dst, $src, $imm\t# long" %}
7815 ins_encode %{
7816 __ imulq($dst$$Register, $src$$Address, $imm$$constant);
7817 %}
7818 ins_pipe(ialu_reg_mem_alu0);
7819 %}
7820
7821 instruct mulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
7822 %{
7823 match(Set dst (MulHiL src rax));
7824 effect(USE_KILL rax, KILL cr);
7825
7826 ins_cost(300);
7827 format %{ "imulq RDX:RAX, RAX, $src\t# mulhi" %}
7828 ins_encode %{
7829 __ imulq($src$$Register);
7830 %}
7831 ins_pipe(ialu_reg_reg_alu0);
7832 %}
7833
7834 instruct umulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
7835 %{
7836 match(Set dst (UMulHiL src rax));
7837 effect(USE_KILL rax, KILL cr);
7838
7839 ins_cost(300);
7840 format %{ "mulq RDX:RAX, RAX, $src\t# umulhi" %}
7841 ins_encode %{
7842 __ mulq($src$$Register);
7843 %}
7844 ins_pipe(ialu_reg_reg_alu0);
7845 %}
7846
7847 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
7848 rFlagsReg cr)
7849 %{
7850 match(Set rax (DivI rax div));
7851 effect(KILL rdx, KILL cr);
7852
7853 ins_cost(30*100+10*100); // XXX
7854 format %{ "cmpl rax, 0x80000000\t# idiv\n\t"
7855 "jne,s normal\n\t"
7856 "xorl rdx, rdx\n\t"
7857 "cmpl $div, -1\n\t"
7858 "je,s done\n"
7859 "normal: cdql\n\t"
7860 "idivl $div\n"
7861 "done:" %}
7862 ins_encode(cdql_enc(div));
7863 ins_pipe(ialu_reg_reg_alu0);
7864 %}
7865
7866 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
7867 rFlagsReg cr)
7868 %{
7869 match(Set rax (DivL rax div));
7870 effect(KILL rdx, KILL cr);
7871
7872 ins_cost(30*100+10*100); // XXX
7873 format %{ "movq rdx, 0x8000000000000000\t# ldiv\n\t"
7874 "cmpq rax, rdx\n\t"
7875 "jne,s normal\n\t"
7876 "xorl rdx, rdx\n\t"
7877 "cmpq $div, -1\n\t"
7878 "je,s done\n"
7879 "normal: cdqq\n\t"
7880 "idivq $div\n"
7881 "done:" %}
7882 ins_encode(cdqq_enc(div));
7883 ins_pipe(ialu_reg_reg_alu0);
7884 %}
7885
7886 instruct udivI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div, rFlagsReg cr)
7887 %{
7888 match(Set rax (UDivI rax div));
7889 effect(KILL rdx, KILL cr);
7890
7891 ins_cost(300);
7892 format %{ "udivl $rax,$rax,$div\t# UDivI\n" %}
7893 ins_encode %{
7894 __ udivI($rax$$Register, $div$$Register, $rdx$$Register);
7895 %}
7896 ins_pipe(ialu_reg_reg_alu0);
7897 %}
7898
7899 instruct udivL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div, rFlagsReg cr)
7900 %{
7901 match(Set rax (UDivL rax div));
7902 effect(KILL rdx, KILL cr);
7903
7904 ins_cost(300);
7905 format %{ "udivq $rax,$rax,$div\t# UDivL\n" %}
7906 ins_encode %{
7907 __ udivL($rax$$Register, $div$$Register, $rdx$$Register);
7908 %}
7909 ins_pipe(ialu_reg_reg_alu0);
7910 %}
7911
7912 // Integer DIVMOD with Register, both quotient and mod results
7913 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
7914 rFlagsReg cr)
7915 %{
7916 match(DivModI rax div);
7917 effect(KILL cr);
7918
7919 ins_cost(30*100+10*100); // XXX
7920 format %{ "cmpl rax, 0x80000000\t# idiv\n\t"
7921 "jne,s normal\n\t"
7922 "xorl rdx, rdx\n\t"
7923 "cmpl $div, -1\n\t"
7924 "je,s done\n"
7925 "normal: cdql\n\t"
7926 "idivl $div\n"
7927 "done:" %}
7928 ins_encode(cdql_enc(div));
7929 ins_pipe(pipe_slow);
7930 %}
7931
7932 // Long DIVMOD with Register, both quotient and mod results
7933 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
7934 rFlagsReg cr)
7935 %{
7936 match(DivModL rax div);
7937 effect(KILL cr);
7938
7939 ins_cost(30*100+10*100); // XXX
7940 format %{ "movq rdx, 0x8000000000000000\t# ldiv\n\t"
7941 "cmpq rax, rdx\n\t"
7942 "jne,s normal\n\t"
7943 "xorl rdx, rdx\n\t"
7944 "cmpq $div, -1\n\t"
7945 "je,s done\n"
7946 "normal: cdqq\n\t"
7947 "idivq $div\n"
7948 "done:" %}
7949 ins_encode(cdqq_enc(div));
7950 ins_pipe(pipe_slow);
7951 %}
7952
7953 // Unsigned integer DIVMOD with Register, both quotient and mod results
7954 instruct udivModI_rReg_divmod(rax_RegI rax, no_rax_rdx_RegI tmp, rdx_RegI rdx,
7955 no_rax_rdx_RegI div, rFlagsReg cr)
7956 %{
7957 match(UDivModI rax div);
7958 effect(TEMP tmp, KILL cr);
7959
7960 ins_cost(300);
7961 format %{ "udivl $rax,$rax,$div\t# begin UDivModI\n\t"
7962 "umodl $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModI\n"
7963 %}
7964 ins_encode %{
7965 __ udivmodI($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
7966 %}
7967 ins_pipe(pipe_slow);
7968 %}
7969
7970 // Unsigned long DIVMOD with Register, both quotient and mod results
7971 instruct udivModL_rReg_divmod(rax_RegL rax, no_rax_rdx_RegL tmp, rdx_RegL rdx,
7972 no_rax_rdx_RegL div, rFlagsReg cr)
7973 %{
7974 match(UDivModL rax div);
7975 effect(TEMP tmp, KILL cr);
7976
7977 ins_cost(300);
7978 format %{ "udivq $rax,$rax,$div\t# begin UDivModL\n\t"
7979 "umodq $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModL\n"
7980 %}
7981 ins_encode %{
7982 __ udivmodL($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
7983 %}
7984 ins_pipe(pipe_slow);
7985 %}
7986
7987 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
7988 rFlagsReg cr)
7989 %{
7990 match(Set rdx (ModI rax div));
7991 effect(KILL rax, KILL cr);
7992
7993 ins_cost(300); // XXX
7994 format %{ "cmpl rax, 0x80000000\t# irem\n\t"
7995 "jne,s normal\n\t"
7996 "xorl rdx, rdx\n\t"
7997 "cmpl $div, -1\n\t"
7998 "je,s done\n"
7999 "normal: cdql\n\t"
8000 "idivl $div\n"
8001 "done:" %}
8002 ins_encode(cdql_enc(div));
8003 ins_pipe(ialu_reg_reg_alu0);
8004 %}
8005
8006 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
8007 rFlagsReg cr)
8008 %{
8009 match(Set rdx (ModL rax div));
8010 effect(KILL rax, KILL cr);
8011
8012 ins_cost(300); // XXX
8013 format %{ "movq rdx, 0x8000000000000000\t# lrem\n\t"
8014 "cmpq rax, rdx\n\t"
8015 "jne,s normal\n\t"
8016 "xorl rdx, rdx\n\t"
8017 "cmpq $div, -1\n\t"
8018 "je,s done\n"
8019 "normal: cdqq\n\t"
8020 "idivq $div\n"
8021 "done:" %}
8022 ins_encode(cdqq_enc(div));
8023 ins_pipe(ialu_reg_reg_alu0);
8024 %}
8025
8026 instruct umodI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div, rFlagsReg cr)
8027 %{
8028 match(Set rdx (UModI rax div));
8029 effect(KILL rax, KILL cr);
8030
8031 ins_cost(300);
8032 format %{ "umodl $rdx,$rax,$div\t# UModI\n" %}
8033 ins_encode %{
8034 __ umodI($rax$$Register, $div$$Register, $rdx$$Register);
8035 %}
8036 ins_pipe(ialu_reg_reg_alu0);
8037 %}
8038
8039 instruct umodL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div, rFlagsReg cr)
8040 %{
8041 match(Set rdx (UModL rax div));
8042 effect(KILL rax, KILL cr);
8043
8044 ins_cost(300);
8045 format %{ "umodq $rdx,$rax,$div\t# UModL\n" %}
8046 ins_encode %{
8047 __ umodL($rax$$Register, $div$$Register, $rdx$$Register);
8048 %}
8049 ins_pipe(ialu_reg_reg_alu0);
8050 %}
8051
8052 // Integer Shift Instructions
8053 // Shift Left by one, two, three
8054 instruct salI_rReg_immI2(rRegI dst, immI2 shift, rFlagsReg cr)
8055 %{
8056 match(Set dst (LShiftI dst shift));
8057 effect(KILL cr);
8058
8059 format %{ "sall $dst, $shift" %}
8060 ins_encode %{
8061 __ sall($dst$$Register, $shift$$constant);
8062 %}
8063 ins_pipe(ialu_reg);
8064 %}
8065
8066 // Shift Left by 8-bit immediate
8067 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8068 %{
8069 match(Set dst (LShiftI dst shift));
8070 effect(KILL cr);
8071
8072 format %{ "sall $dst, $shift" %}
8073 ins_encode %{
8074 __ sall($dst$$Register, $shift$$constant);
8075 %}
8076 ins_pipe(ialu_reg);
8077 %}
8078
8079 // Shift Left by 8-bit immediate
8080 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8081 %{
8082 match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8083 effect(KILL cr);
8084
8085 format %{ "sall $dst, $shift" %}
8086 ins_encode %{
8087 __ sall($dst$$Address, $shift$$constant);
8088 %}
8089 ins_pipe(ialu_mem_imm);
8090 %}
8091
8092 // Shift Left by variable
8093 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8094 %{
8095 predicate(!VM_Version::supports_bmi2());
8096 match(Set dst (LShiftI dst shift));
8097 effect(KILL cr);
8098
8099 format %{ "sall $dst, $shift" %}
8100 ins_encode %{
8101 __ sall($dst$$Register);
8102 %}
8103 ins_pipe(ialu_reg_reg);
8104 %}
8105
8106 // Shift Left by variable
8107 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8108 %{
8109 predicate(!VM_Version::supports_bmi2());
8110 match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8111 effect(KILL cr);
8112
8113 format %{ "sall $dst, $shift" %}
8114 ins_encode %{
8115 __ sall($dst$$Address);
8116 %}
8117 ins_pipe(ialu_mem_reg);
8118 %}
8119
8120 instruct salI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
8121 %{
8122 predicate(VM_Version::supports_bmi2());
8123 match(Set dst (LShiftI src shift));
8124
8125 format %{ "shlxl $dst, $src, $shift" %}
8126 ins_encode %{
8127 __ shlxl($dst$$Register, $src$$Register, $shift$$Register);
8128 %}
8129 ins_pipe(ialu_reg_reg);
8130 %}
8131
8132 instruct salI_mem_rReg(rRegI dst, memory src, rRegI shift)
8133 %{
8134 predicate(VM_Version::supports_bmi2());
8135 match(Set dst (LShiftI (LoadI src) shift));
8136 ins_cost(175);
8137 format %{ "shlxl $dst, $src, $shift" %}
8138 ins_encode %{
8139 __ shlxl($dst$$Register, $src$$Address, $shift$$Register);
8140 %}
8141 ins_pipe(ialu_reg_mem);
8142 %}
8143
8144 // Arithmetic Shift Right by 8-bit immediate
8145 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8146 %{
8147 match(Set dst (RShiftI dst shift));
8148 effect(KILL cr);
8149
8150 format %{ "sarl $dst, $shift" %}
8151 ins_encode %{
8152 __ sarl($dst$$Register, $shift$$constant);
8153 %}
8154 ins_pipe(ialu_mem_imm);
8155 %}
8156
8157 // Arithmetic Shift Right by 8-bit immediate
8158 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8159 %{
8160 match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8161 effect(KILL cr);
8162
8163 format %{ "sarl $dst, $shift" %}
8164 ins_encode %{
8165 __ sarl($dst$$Address, $shift$$constant);
8166 %}
8167 ins_pipe(ialu_mem_imm);
8168 %}
8169
8170 // Arithmetic Shift Right by variable
8171 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8172 %{
8173 predicate(!VM_Version::supports_bmi2());
8174 match(Set dst (RShiftI dst shift));
8175 effect(KILL cr);
8176
8177 format %{ "sarl $dst, $shift" %}
8178 ins_encode %{
8179 __ sarl($dst$$Register);
8180 %}
8181 ins_pipe(ialu_reg_reg);
8182 %}
8183
8184 // Arithmetic Shift Right by variable
8185 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8186 %{
8187 predicate(!VM_Version::supports_bmi2());
8188 match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8189 effect(KILL cr);
8190
8191 format %{ "sarl $dst, $shift" %}
8192 ins_encode %{
8193 __ sarl($dst$$Address);
8194 %}
8195 ins_pipe(ialu_mem_reg);
8196 %}
8197
8198 instruct sarI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
8199 %{
8200 predicate(VM_Version::supports_bmi2());
8201 match(Set dst (RShiftI src shift));
8202
8203 format %{ "sarxl $dst, $src, $shift" %}
8204 ins_encode %{
8205 __ sarxl($dst$$Register, $src$$Register, $shift$$Register);
8206 %}
8207 ins_pipe(ialu_reg_reg);
8208 %}
8209
8210 instruct sarI_mem_rReg(rRegI dst, memory src, rRegI shift)
8211 %{
8212 predicate(VM_Version::supports_bmi2());
8213 match(Set dst (RShiftI (LoadI src) shift));
8214 ins_cost(175);
8215 format %{ "sarxl $dst, $src, $shift" %}
8216 ins_encode %{
8217 __ sarxl($dst$$Register, $src$$Address, $shift$$Register);
8218 %}
8219 ins_pipe(ialu_reg_mem);
8220 %}
8221
8222 // Logical Shift Right by 8-bit immediate
8223 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8224 %{
8225 match(Set dst (URShiftI dst shift));
8226 effect(KILL cr);
8227
8228 format %{ "shrl $dst, $shift" %}
8229 ins_encode %{
8230 __ shrl($dst$$Register, $shift$$constant);
8231 %}
8232 ins_pipe(ialu_reg);
8233 %}
8234
8235 // Logical Shift Right by 8-bit immediate
8236 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8237 %{
8238 match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8239 effect(KILL cr);
8240
8241 format %{ "shrl $dst, $shift" %}
8242 ins_encode %{
8243 __ shrl($dst$$Address, $shift$$constant);
8244 %}
8245 ins_pipe(ialu_mem_imm);
8246 %}
8247
8248 // Logical Shift Right by variable
8249 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8250 %{
8251 predicate(!VM_Version::supports_bmi2());
8252 match(Set dst (URShiftI dst shift));
8253 effect(KILL cr);
8254
8255 format %{ "shrl $dst, $shift" %}
8256 ins_encode %{
8257 __ shrl($dst$$Register);
8258 %}
8259 ins_pipe(ialu_reg_reg);
8260 %}
8261
8262 // Logical Shift Right by variable
8263 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8264 %{
8265 predicate(!VM_Version::supports_bmi2());
8266 match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8267 effect(KILL cr);
8268
8269 format %{ "shrl $dst, $shift" %}
8270 ins_encode %{
8271 __ shrl($dst$$Address);
8272 %}
8273 ins_pipe(ialu_mem_reg);
8274 %}
8275
8276 instruct shrI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
8277 %{
8278 predicate(VM_Version::supports_bmi2());
8279 match(Set dst (URShiftI src shift));
8280
8281 format %{ "shrxl $dst, $src, $shift" %}
8282 ins_encode %{
8283 __ shrxl($dst$$Register, $src$$Register, $shift$$Register);
8284 %}
8285 ins_pipe(ialu_reg_reg);
8286 %}
8287
8288 instruct shrI_mem_rReg(rRegI dst, memory src, rRegI shift)
8289 %{
8290 predicate(VM_Version::supports_bmi2());
8291 match(Set dst (URShiftI (LoadI src) shift));
8292 ins_cost(175);
8293 format %{ "shrxl $dst, $src, $shift" %}
8294 ins_encode %{
8295 __ shrxl($dst$$Register, $src$$Address, $shift$$Register);
8296 %}
8297 ins_pipe(ialu_reg_mem);
8298 %}
8299
8300 // Long Shift Instructions
8301 // Shift Left by one, two, three
8302 instruct salL_rReg_immI2(rRegL dst, immI2 shift, rFlagsReg cr)
8303 %{
8304 match(Set dst (LShiftL dst shift));
8305 effect(KILL cr);
8306
8307 format %{ "salq $dst, $shift" %}
8308 ins_encode %{
8309 __ salq($dst$$Register, $shift$$constant);
8310 %}
8311 ins_pipe(ialu_reg);
8312 %}
8313
8314 // Shift Left by 8-bit immediate
8315 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8316 %{
8317 match(Set dst (LShiftL dst shift));
8318 effect(KILL cr);
8319
8320 format %{ "salq $dst, $shift" %}
8321 ins_encode %{
8322 __ salq($dst$$Register, $shift$$constant);
8323 %}
8324 ins_pipe(ialu_reg);
8325 %}
8326
8327 // Shift Left by 8-bit immediate
8328 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8329 %{
8330 match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8331 effect(KILL cr);
8332
8333 format %{ "salq $dst, $shift" %}
8334 ins_encode %{
8335 __ salq($dst$$Address, $shift$$constant);
8336 %}
8337 ins_pipe(ialu_mem_imm);
8338 %}
8339
8340 // Shift Left by variable
8341 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8342 %{
8343 predicate(!VM_Version::supports_bmi2());
8344 match(Set dst (LShiftL dst shift));
8345 effect(KILL cr);
8346
8347 format %{ "salq $dst, $shift" %}
8348 ins_encode %{
8349 __ salq($dst$$Register);
8350 %}
8351 ins_pipe(ialu_reg_reg);
8352 %}
8353
8354 // Shift Left by variable
8355 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8356 %{
8357 predicate(!VM_Version::supports_bmi2());
8358 match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8359 effect(KILL cr);
8360
8361 format %{ "salq $dst, $shift" %}
8362 ins_encode %{
8363 __ salq($dst$$Address);
8364 %}
8365 ins_pipe(ialu_mem_reg);
8366 %}
8367
8368 instruct salL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
8369 %{
8370 predicate(VM_Version::supports_bmi2());
8371 match(Set dst (LShiftL src shift));
8372
8373 format %{ "shlxq $dst, $src, $shift" %}
8374 ins_encode %{
8375 __ shlxq($dst$$Register, $src$$Register, $shift$$Register);
8376 %}
8377 ins_pipe(ialu_reg_reg);
8378 %}
8379
8380 instruct salL_mem_rReg(rRegL dst, memory src, rRegI shift)
8381 %{
8382 predicate(VM_Version::supports_bmi2());
8383 match(Set dst (LShiftL (LoadL src) shift));
8384 ins_cost(175);
8385 format %{ "shlxq $dst, $src, $shift" %}
8386 ins_encode %{
8387 __ shlxq($dst$$Register, $src$$Address, $shift$$Register);
8388 %}
8389 ins_pipe(ialu_reg_mem);
8390 %}
8391
8392 // Arithmetic Shift Right by 8-bit immediate
8393 instruct sarL_rReg_imm(rRegL dst, immI shift, rFlagsReg cr)
8394 %{
8395 match(Set dst (RShiftL dst shift));
8396 effect(KILL cr);
8397
8398 format %{ "sarq $dst, $shift" %}
8399 ins_encode %{
8400 __ sarq($dst$$Register, (unsigned char)($shift$$constant & 0x3F));
8401 %}
8402 ins_pipe(ialu_mem_imm);
8403 %}
8404
8405 // Arithmetic Shift Right by 8-bit immediate
8406 instruct sarL_mem_imm(memory dst, immI shift, rFlagsReg cr)
8407 %{
8408 match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8409 effect(KILL cr);
8410
8411 format %{ "sarq $dst, $shift" %}
8412 ins_encode %{
8413 __ sarq($dst$$Address, (unsigned char)($shift$$constant & 0x3F));
8414 %}
8415 ins_pipe(ialu_mem_imm);
8416 %}
8417
8418 // Arithmetic Shift Right by variable
8419 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8420 %{
8421 predicate(!VM_Version::supports_bmi2());
8422 match(Set dst (RShiftL dst shift));
8423 effect(KILL cr);
8424
8425 format %{ "sarq $dst, $shift" %}
8426 ins_encode %{
8427 __ sarq($dst$$Register);
8428 %}
8429 ins_pipe(ialu_reg_reg);
8430 %}
8431
8432 // Arithmetic Shift Right by variable
8433 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8434 %{
8435 predicate(!VM_Version::supports_bmi2());
8436 match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8437 effect(KILL cr);
8438
8439 format %{ "sarq $dst, $shift" %}
8440 ins_encode %{
8441 __ sarq($dst$$Address);
8442 %}
8443 ins_pipe(ialu_mem_reg);
8444 %}
8445
8446 instruct sarL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
8447 %{
8448 predicate(VM_Version::supports_bmi2());
8449 match(Set dst (RShiftL src shift));
8450
8451 format %{ "sarxq $dst, $src, $shift" %}
8452 ins_encode %{
8453 __ sarxq($dst$$Register, $src$$Register, $shift$$Register);
8454 %}
8455 ins_pipe(ialu_reg_reg);
8456 %}
8457
8458 instruct sarL_mem_rReg(rRegL dst, memory src, rRegI shift)
8459 %{
8460 predicate(VM_Version::supports_bmi2());
8461 match(Set dst (RShiftL (LoadL src) shift));
8462 ins_cost(175);
8463 format %{ "sarxq $dst, $src, $shift" %}
8464 ins_encode %{
8465 __ sarxq($dst$$Register, $src$$Address, $shift$$Register);
8466 %}
8467 ins_pipe(ialu_reg_mem);
8468 %}
8469
8470 // Logical Shift Right by 8-bit immediate
8471 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8472 %{
8473 match(Set dst (URShiftL dst shift));
8474 effect(KILL cr);
8475
8476 format %{ "shrq $dst, $shift" %}
8477 ins_encode %{
8478 __ shrq($dst$$Register, $shift$$constant);
8479 %}
8480 ins_pipe(ialu_reg);
8481 %}
8482
8483 // Logical Shift Right by 8-bit immediate
8484 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8485 %{
8486 match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8487 effect(KILL cr);
8488
8489 format %{ "shrq $dst, $shift" %}
8490 ins_encode %{
8491 __ shrq($dst$$Address, $shift$$constant);
8492 %}
8493 ins_pipe(ialu_mem_imm);
8494 %}
8495
8496 // Logical Shift Right by variable
8497 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8498 %{
8499 predicate(!VM_Version::supports_bmi2());
8500 match(Set dst (URShiftL dst shift));
8501 effect(KILL cr);
8502
8503 format %{ "shrq $dst, $shift" %}
8504 ins_encode %{
8505 __ shrq($dst$$Register);
8506 %}
8507 ins_pipe(ialu_reg_reg);
8508 %}
8509
8510 // Logical Shift Right by variable
8511 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8512 %{
8513 predicate(!VM_Version::supports_bmi2());
8514 match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8515 effect(KILL cr);
8516
8517 format %{ "shrq $dst, $shift" %}
8518 ins_encode %{
8519 __ shrq($dst$$Address);
8520 %}
8521 ins_pipe(ialu_mem_reg);
8522 %}
8523
8524 instruct shrL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
8525 %{
8526 predicate(VM_Version::supports_bmi2());
8527 match(Set dst (URShiftL src shift));
8528
8529 format %{ "shrxq $dst, $src, $shift" %}
8530 ins_encode %{
8531 __ shrxq($dst$$Register, $src$$Register, $shift$$Register);
8532 %}
8533 ins_pipe(ialu_reg_reg);
8534 %}
8535
8536 instruct shrL_mem_rReg(rRegL dst, memory src, rRegI shift)
8537 %{
8538 predicate(VM_Version::supports_bmi2());
8539 match(Set dst (URShiftL (LoadL src) shift));
8540 ins_cost(175);
8541 format %{ "shrxq $dst, $src, $shift" %}
8542 ins_encode %{
8543 __ shrxq($dst$$Register, $src$$Address, $shift$$Register);
8544 %}
8545 ins_pipe(ialu_reg_mem);
8546 %}
8547
8548 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
8549 // This idiom is used by the compiler for the i2b bytecode.
8550 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
8551 %{
8552 match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
8553
8554 format %{ "movsbl $dst, $src\t# i2b" %}
8555 ins_encode %{
8556 __ movsbl($dst$$Register, $src$$Register);
8557 %}
8558 ins_pipe(ialu_reg_reg);
8559 %}
8560
8561 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
8562 // This idiom is used by the compiler the i2s bytecode.
8563 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
8564 %{
8565 match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
8566
8567 format %{ "movswl $dst, $src\t# i2s" %}
8568 ins_encode %{
8569 __ movswl($dst$$Register, $src$$Register);
8570 %}
8571 ins_pipe(ialu_reg_reg);
8572 %}
8573
8574 // ROL/ROR instructions
8575
8576 // Rotate left by constant.
8577 instruct rolI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
8578 %{
8579 predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
8580 match(Set dst (RotateLeft dst shift));
8581 effect(KILL cr);
8582 format %{ "roll $dst, $shift" %}
8583 ins_encode %{
8584 __ roll($dst$$Register, $shift$$constant);
8585 %}
8586 ins_pipe(ialu_reg);
8587 %}
8588
8589 instruct rolI_immI8(rRegI dst, rRegI src, immI8 shift)
8590 %{
8591 predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
8592 match(Set dst (RotateLeft src shift));
8593 format %{ "rolxl $dst, $src, $shift" %}
8594 ins_encode %{
8595 int shift = 32 - ($shift$$constant & 31);
8596 __ rorxl($dst$$Register, $src$$Register, shift);
8597 %}
8598 ins_pipe(ialu_reg_reg);
8599 %}
8600
8601 instruct rolI_mem_immI8(rRegI dst, memory src, immI8 shift)
8602 %{
8603 predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
8604 match(Set dst (RotateLeft (LoadI src) shift));
8605 ins_cost(175);
8606 format %{ "rolxl $dst, $src, $shift" %}
8607 ins_encode %{
8608 int shift = 32 - ($shift$$constant & 31);
8609 __ rorxl($dst$$Register, $src$$Address, shift);
8610 %}
8611 ins_pipe(ialu_reg_mem);
8612 %}
8613
8614 // Rotate Left by variable
8615 instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8616 %{
8617 predicate(n->bottom_type()->basic_type() == T_INT);
8618 match(Set dst (RotateLeft dst shift));
8619 effect(KILL cr);
8620 format %{ "roll $dst, $shift" %}
8621 ins_encode %{
8622 __ roll($dst$$Register);
8623 %}
8624 ins_pipe(ialu_reg_reg);
8625 %}
8626
8627 // Rotate Right by constant.
8628 instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
8629 %{
8630 predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
8631 match(Set dst (RotateRight dst shift));
8632 effect(KILL cr);
8633 format %{ "rorl $dst, $shift" %}
8634 ins_encode %{
8635 __ rorl($dst$$Register, $shift$$constant);
8636 %}
8637 ins_pipe(ialu_reg);
8638 %}
8639
8640 // Rotate Right by constant.
8641 instruct rorI_immI8(rRegI dst, rRegI src, immI8 shift)
8642 %{
8643 predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
8644 match(Set dst (RotateRight src shift));
8645 format %{ "rorxl $dst, $src, $shift" %}
8646 ins_encode %{
8647 __ rorxl($dst$$Register, $src$$Register, $shift$$constant);
8648 %}
8649 ins_pipe(ialu_reg_reg);
8650 %}
8651
8652 instruct rorI_mem_immI8(rRegI dst, memory src, immI8 shift)
8653 %{
8654 predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
8655 match(Set dst (RotateRight (LoadI src) shift));
8656 ins_cost(175);
8657 format %{ "rorxl $dst, $src, $shift" %}
8658 ins_encode %{
8659 __ rorxl($dst$$Register, $src$$Address, $shift$$constant);
8660 %}
8661 ins_pipe(ialu_reg_mem);
8662 %}
8663
8664 // Rotate Right by variable
8665 instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8666 %{
8667 predicate(n->bottom_type()->basic_type() == T_INT);
8668 match(Set dst (RotateRight dst shift));
8669 effect(KILL cr);
8670 format %{ "rorl $dst, $shift" %}
8671 ins_encode %{
8672 __ rorl($dst$$Register);
8673 %}
8674 ins_pipe(ialu_reg_reg);
8675 %}
8676
8677 // Rotate Left by constant.
8678 instruct rolL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
8679 %{
8680 predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
8681 match(Set dst (RotateLeft dst shift));
8682 effect(KILL cr);
8683 format %{ "rolq $dst, $shift" %}
8684 ins_encode %{
8685 __ rolq($dst$$Register, $shift$$constant);
8686 %}
8687 ins_pipe(ialu_reg);
8688 %}
8689
8690 instruct rolL_immI8(rRegL dst, rRegL src, immI8 shift)
8691 %{
8692 predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
8693 match(Set dst (RotateLeft src shift));
8694 format %{ "rolxq $dst, $src, $shift" %}
8695 ins_encode %{
8696 int shift = 64 - ($shift$$constant & 63);
8697 __ rorxq($dst$$Register, $src$$Register, shift);
8698 %}
8699 ins_pipe(ialu_reg_reg);
8700 %}
8701
8702 instruct rolL_mem_immI8(rRegL dst, memory src, immI8 shift)
8703 %{
8704 predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
8705 match(Set dst (RotateLeft (LoadL src) shift));
8706 ins_cost(175);
8707 format %{ "rolxq $dst, $src, $shift" %}
8708 ins_encode %{
8709 int shift = 64 - ($shift$$constant & 63);
8710 __ rorxq($dst$$Register, $src$$Address, shift);
8711 %}
8712 ins_pipe(ialu_reg_mem);
8713 %}
8714
8715 // Rotate Left by variable
8716 instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8717 %{
8718 predicate(n->bottom_type()->basic_type() == T_LONG);
8719 match(Set dst (RotateLeft dst shift));
8720 effect(KILL cr);
8721 format %{ "rolq $dst, $shift" %}
8722 ins_encode %{
8723 __ rolq($dst$$Register);
8724 %}
8725 ins_pipe(ialu_reg_reg);
8726 %}
8727
8728 // Rotate Right by constant.
8729 instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
8730 %{
8731 predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
8732 match(Set dst (RotateRight dst shift));
8733 effect(KILL cr);
8734 format %{ "rorq $dst, $shift" %}
8735 ins_encode %{
8736 __ rorq($dst$$Register, $shift$$constant);
8737 %}
8738 ins_pipe(ialu_reg);
8739 %}
8740
8741 // Rotate Right by constant
8742 instruct rorL_immI8(rRegL dst, rRegL src, immI8 shift)
8743 %{
8744 predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
8745 match(Set dst (RotateRight src shift));
8746 format %{ "rorxq $dst, $src, $shift" %}
8747 ins_encode %{
8748 __ rorxq($dst$$Register, $src$$Register, $shift$$constant);
8749 %}
8750 ins_pipe(ialu_reg_reg);
8751 %}
8752
8753 instruct rorL_mem_immI8(rRegL dst, memory src, immI8 shift)
8754 %{
8755 predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
8756 match(Set dst (RotateRight (LoadL src) shift));
8757 ins_cost(175);
8758 format %{ "rorxq $dst, $src, $shift" %}
8759 ins_encode %{
8760 __ rorxq($dst$$Register, $src$$Address, $shift$$constant);
8761 %}
8762 ins_pipe(ialu_reg_mem);
8763 %}
8764
8765 // Rotate Right by variable
8766 instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8767 %{
8768 predicate(n->bottom_type()->basic_type() == T_LONG);
8769 match(Set dst (RotateRight dst shift));
8770 effect(KILL cr);
8771 format %{ "rorq $dst, $shift" %}
8772 ins_encode %{
8773 __ rorq($dst$$Register);
8774 %}
8775 ins_pipe(ialu_reg_reg);
8776 %}
8777
8778 //----------------------------- CompressBits/ExpandBits ------------------------
8779
8780 instruct compressBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
8781 predicate(n->bottom_type()->isa_long());
8782 match(Set dst (CompressBits src mask));
8783 format %{ "pextq $dst, $src, $mask\t! parallel bit extract" %}
8784 ins_encode %{
8785 __ pextq($dst$$Register, $src$$Register, $mask$$Register);
8786 %}
8787 ins_pipe( pipe_slow );
8788 %}
8789
8790 instruct expandBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
8791 predicate(n->bottom_type()->isa_long());
8792 match(Set dst (ExpandBits src mask));
8793 format %{ "pdepq $dst, $src, $mask\t! parallel bit deposit" %}
8794 ins_encode %{
8795 __ pdepq($dst$$Register, $src$$Register, $mask$$Register);
8796 %}
8797 ins_pipe( pipe_slow );
8798 %}
8799
8800 instruct compressBitsL_mem(rRegL dst, rRegL src, memory mask) %{
8801 predicate(n->bottom_type()->isa_long());
8802 match(Set dst (CompressBits src (LoadL mask)));
8803 format %{ "pextq $dst, $src, $mask\t! parallel bit extract" %}
8804 ins_encode %{
8805 __ pextq($dst$$Register, $src$$Register, $mask$$Address);
8806 %}
8807 ins_pipe( pipe_slow );
8808 %}
8809
8810 instruct expandBitsL_mem(rRegL dst, rRegL src, memory mask) %{
8811 predicate(n->bottom_type()->isa_long());
8812 match(Set dst (ExpandBits src (LoadL mask)));
8813 format %{ "pdepq $dst, $src, $mask\t! parallel bit deposit" %}
8814 ins_encode %{
8815 __ pdepq($dst$$Register, $src$$Register, $mask$$Address);
8816 %}
8817 ins_pipe( pipe_slow );
8818 %}
8819
8820
8821 // Logical Instructions
8822
8823 // Integer Logical Instructions
8824
8825 // And Instructions
8826 // And Register with Register
8827 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8828 %{
8829 match(Set dst (AndI dst src));
8830 effect(KILL cr);
8831 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
8832
8833 format %{ "andl $dst, $src\t# int" %}
8834 ins_encode %{
8835 __ andl($dst$$Register, $src$$Register);
8836 %}
8837 ins_pipe(ialu_reg_reg);
8838 %}
8839
8840 // And Register with Immediate 255
8841 instruct andI_rReg_imm255(rRegI dst, rRegI src, immI_255 mask)
8842 %{
8843 match(Set dst (AndI src mask));
8844
8845 format %{ "movzbl $dst, $src\t# int & 0xFF" %}
8846 ins_encode %{
8847 __ movzbl($dst$$Register, $src$$Register);
8848 %}
8849 ins_pipe(ialu_reg);
8850 %}
8851
8852 // And Register with Immediate 255 and promote to long
8853 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
8854 %{
8855 match(Set dst (ConvI2L (AndI src mask)));
8856
8857 format %{ "movzbl $dst, $src\t# int & 0xFF -> long" %}
8858 ins_encode %{
8859 __ movzbl($dst$$Register, $src$$Register);
8860 %}
8861 ins_pipe(ialu_reg);
8862 %}
8863
8864 // And Register with Immediate 65535
8865 instruct andI_rReg_imm65535(rRegI dst, rRegI src, immI_65535 mask)
8866 %{
8867 match(Set dst (AndI src mask));
8868
8869 format %{ "movzwl $dst, $src\t# int & 0xFFFF" %}
8870 ins_encode %{
8871 __ movzwl($dst$$Register, $src$$Register);
8872 %}
8873 ins_pipe(ialu_reg);
8874 %}
8875
8876 // And Register with Immediate 65535 and promote to long
8877 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
8878 %{
8879 match(Set dst (ConvI2L (AndI src mask)));
8880
8881 format %{ "movzwl $dst, $src\t# int & 0xFFFF -> long" %}
8882 ins_encode %{
8883 __ movzwl($dst$$Register, $src$$Register);
8884 %}
8885 ins_pipe(ialu_reg);
8886 %}
8887
8888 // Can skip int2long conversions after AND with small bitmask
8889 instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src, immI_Pow2M1 mask, rRegI tmp, rFlagsReg cr)
8890 %{
8891 predicate(VM_Version::supports_bmi2());
8892 ins_cost(125);
8893 effect(TEMP tmp, KILL cr);
8894 match(Set dst (ConvI2L (AndI src mask)));
8895 format %{ "bzhiq $dst, $src, $mask \t# using $tmp as TEMP, int & immI_Pow2M1 -> long" %}
8896 ins_encode %{
8897 __ movl($tmp$$Register, exact_log2($mask$$constant + 1));
8898 __ bzhiq($dst$$Register, $src$$Register, $tmp$$Register);
8899 %}
8900 ins_pipe(ialu_reg_reg);
8901 %}
8902
8903 // And Register with Immediate
8904 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8905 %{
8906 match(Set dst (AndI dst src));
8907 effect(KILL cr);
8908 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
8909
8910 format %{ "andl $dst, $src\t# int" %}
8911 ins_encode %{
8912 __ andl($dst$$Register, $src$$constant);
8913 %}
8914 ins_pipe(ialu_reg);
8915 %}
8916
8917 // And Register with Memory
8918 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8919 %{
8920 match(Set dst (AndI dst (LoadI src)));
8921 effect(KILL cr);
8922 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
8923
8924 ins_cost(150);
8925 format %{ "andl $dst, $src\t# int" %}
8926 ins_encode %{
8927 __ andl($dst$$Register, $src$$Address);
8928 %}
8929 ins_pipe(ialu_reg_mem);
8930 %}
8931
8932 // And Memory with Register
8933 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8934 %{
8935 match(Set dst (StoreB dst (AndI (LoadB dst) src)));
8936 effect(KILL cr);
8937 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
8938
8939 ins_cost(150);
8940 format %{ "andb $dst, $src\t# byte" %}
8941 ins_encode %{
8942 __ andb($dst$$Address, $src$$Register);
8943 %}
8944 ins_pipe(ialu_mem_reg);
8945 %}
8946
8947 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8948 %{
8949 match(Set dst (StoreI dst (AndI (LoadI dst) src)));
8950 effect(KILL cr);
8951 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
8952
8953 ins_cost(150);
8954 format %{ "andl $dst, $src\t# int" %}
8955 ins_encode %{
8956 __ andl($dst$$Address, $src$$Register);
8957 %}
8958 ins_pipe(ialu_mem_reg);
8959 %}
8960
8961 // And Memory with Immediate
8962 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
8963 %{
8964 match(Set dst (StoreI dst (AndI (LoadI dst) src)));
8965 effect(KILL cr);
8966 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
8967
8968 ins_cost(125);
8969 format %{ "andl $dst, $src\t# int" %}
8970 ins_encode %{
8971 __ andl($dst$$Address, $src$$constant);
8972 %}
8973 ins_pipe(ialu_mem_imm);
8974 %}
8975
8976 // BMI1 instructions
8977 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
8978 match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
8979 predicate(UseBMI1Instructions);
8980 effect(KILL cr);
8981 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
8982
8983 ins_cost(125);
8984 format %{ "andnl $dst, $src1, $src2" %}
8985
8986 ins_encode %{
8987 __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
8988 %}
8989 ins_pipe(ialu_reg_mem);
8990 %}
8991
8992 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
8993 match(Set dst (AndI (XorI src1 minus_1) src2));
8994 predicate(UseBMI1Instructions);
8995 effect(KILL cr);
8996 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
8997
8998 format %{ "andnl $dst, $src1, $src2" %}
8999
9000 ins_encode %{
9001 __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
9002 %}
9003 ins_pipe(ialu_reg);
9004 %}
9005
9006 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
9007 match(Set dst (AndI (SubI imm_zero src) src));
9008 predicate(UseBMI1Instructions);
9009 effect(KILL cr);
9010 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
9011
9012 format %{ "blsil $dst, $src" %}
9013
9014 ins_encode %{
9015 __ blsil($dst$$Register, $src$$Register);
9016 %}
9017 ins_pipe(ialu_reg);
9018 %}
9019
9020 instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
9021 match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
9022 predicate(UseBMI1Instructions);
9023 effect(KILL cr);
9024 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
9025
9026 ins_cost(125);
9027 format %{ "blsil $dst, $src" %}
9028
9029 ins_encode %{
9030 __ blsil($dst$$Register, $src$$Address);
9031 %}
9032 ins_pipe(ialu_reg_mem);
9033 %}
9034
9035 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
9036 %{
9037 match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
9038 predicate(UseBMI1Instructions);
9039 effect(KILL cr);
9040 flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
9041
9042 ins_cost(125);
9043 format %{ "blsmskl $dst, $src" %}
9044
9045 ins_encode %{
9046 __ blsmskl($dst$$Register, $src$$Address);
9047 %}
9048 ins_pipe(ialu_reg_mem);
9049 %}
9050
9051 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
9052 %{
9053 match(Set dst (XorI (AddI src minus_1) src));
9054 predicate(UseBMI1Instructions);
9055 effect(KILL cr);
9056 flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
9057
9058 format %{ "blsmskl $dst, $src" %}
9059
9060 ins_encode %{
9061 __ blsmskl($dst$$Register, $src$$Register);
9062 %}
9063
9064 ins_pipe(ialu_reg);
9065 %}
9066
9067 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
9068 %{
9069 match(Set dst (AndI (AddI src minus_1) src) );
9070 predicate(UseBMI1Instructions);
9071 effect(KILL cr);
9072 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
9073
9074 format %{ "blsrl $dst, $src" %}
9075
9076 ins_encode %{
9077 __ blsrl($dst$$Register, $src$$Register);
9078 %}
9079
9080 ins_pipe(ialu_reg_mem);
9081 %}
9082
9083 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
9084 %{
9085 match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
9086 predicate(UseBMI1Instructions);
9087 effect(KILL cr);
9088 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
9089
9090 ins_cost(125);
9091 format %{ "blsrl $dst, $src" %}
9092
9093 ins_encode %{
9094 __ blsrl($dst$$Register, $src$$Address);
9095 %}
9096
9097 ins_pipe(ialu_reg);
9098 %}
9099
9100 // Or Instructions
9101 // Or Register with Register
9102 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9103 %{
9104 match(Set dst (OrI dst src));
9105 effect(KILL cr);
9106 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
9107
9108 format %{ "orl $dst, $src\t# int" %}
9109 ins_encode %{
9110 __ orl($dst$$Register, $src$$Register);
9111 %}
9112 ins_pipe(ialu_reg_reg);
9113 %}
9114
9115 // Or Register with Immediate
9116 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9117 %{
9118 match(Set dst (OrI dst src));
9119 effect(KILL cr);
9120 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
9121
9122 format %{ "orl $dst, $src\t# int" %}
9123 ins_encode %{
9124 __ orl($dst$$Register, $src$$constant);
9125 %}
9126 ins_pipe(ialu_reg);
9127 %}
9128
9129 // Or Register with Memory
9130 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9131 %{
9132 match(Set dst (OrI dst (LoadI src)));
9133 effect(KILL cr);
9134 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
9135
9136 ins_cost(150);
9137 format %{ "orl $dst, $src\t# int" %}
9138 ins_encode %{
9139 __ orl($dst$$Register, $src$$Address);
9140 %}
9141 ins_pipe(ialu_reg_mem);
9142 %}
9143
9144 // Or Memory with Register
9145 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9146 %{
9147 match(Set dst (StoreB dst (OrI (LoadB dst) src)));
9148 effect(KILL cr);
9149 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
9150
9151 ins_cost(150);
9152 format %{ "orb $dst, $src\t# byte" %}
9153 ins_encode %{
9154 __ orb($dst$$Address, $src$$Register);
9155 %}
9156 ins_pipe(ialu_mem_reg);
9157 %}
9158
9159 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9160 %{
9161 match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9162 effect(KILL cr);
9163 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
9164
9165 ins_cost(150);
9166 format %{ "orl $dst, $src\t# int" %}
9167 ins_encode %{
9168 __ orl($dst$$Address, $src$$Register);
9169 %}
9170 ins_pipe(ialu_mem_reg);
9171 %}
9172
9173 // Or Memory with Immediate
9174 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
9175 %{
9176 match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9177 effect(KILL cr);
9178 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
9179
9180 ins_cost(125);
9181 format %{ "orl $dst, $src\t# int" %}
9182 ins_encode %{
9183 __ orl($dst$$Address, $src$$constant);
9184 %}
9185 ins_pipe(ialu_mem_imm);
9186 %}
9187
9188 // Xor Instructions
9189 // Xor Register with Register
9190 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9191 %{
9192 match(Set dst (XorI dst src));
9193 effect(KILL cr);
9194 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
9195
9196 format %{ "xorl $dst, $src\t# int" %}
9197 ins_encode %{
9198 __ xorl($dst$$Register, $src$$Register);
9199 %}
9200 ins_pipe(ialu_reg_reg);
9201 %}
9202
9203 // Xor Register with Immediate -1
9204 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
9205 match(Set dst (XorI dst imm));
9206
9207 format %{ "not $dst" %}
9208 ins_encode %{
9209 __ notl($dst$$Register);
9210 %}
9211 ins_pipe(ialu_reg);
9212 %}
9213
9214 // Xor Register with Immediate
9215 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9216 %{
9217 match(Set dst (XorI dst src));
9218 effect(KILL cr);
9219 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
9220
9221 format %{ "xorl $dst, $src\t# int" %}
9222 ins_encode %{
9223 __ xorl($dst$$Register, $src$$constant);
9224 %}
9225 ins_pipe(ialu_reg);
9226 %}
9227
9228 // Xor Register with Memory
9229 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9230 %{
9231 match(Set dst (XorI dst (LoadI src)));
9232 effect(KILL cr);
9233 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
9234
9235 ins_cost(150);
9236 format %{ "xorl $dst, $src\t# int" %}
9237 ins_encode %{
9238 __ xorl($dst$$Register, $src$$Address);
9239 %}
9240 ins_pipe(ialu_reg_mem);
9241 %}
9242
9243 // Xor Memory with Register
9244 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9245 %{
9246 match(Set dst (StoreB dst (XorI (LoadB dst) src)));
9247 effect(KILL cr);
9248 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
9249
9250 ins_cost(150);
9251 format %{ "xorb $dst, $src\t# byte" %}
9252 ins_encode %{
9253 __ xorb($dst$$Address, $src$$Register);
9254 %}
9255 ins_pipe(ialu_mem_reg);
9256 %}
9257
9258 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9259 %{
9260 match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9261 effect(KILL cr);
9262 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
9263
9264 ins_cost(150);
9265 format %{ "xorl $dst, $src\t# int" %}
9266 ins_encode %{
9267 __ xorl($dst$$Address, $src$$Register);
9268 %}
9269 ins_pipe(ialu_mem_reg);
9270 %}
9271
9272 // Xor Memory with Immediate
9273 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
9274 %{
9275 match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9276 effect(KILL cr);
9277 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
9278
9279 ins_cost(125);
9280 format %{ "xorl $dst, $src\t# int" %}
9281 ins_encode %{
9282 __ xorl($dst$$Address, $src$$constant);
9283 %}
9284 ins_pipe(ialu_mem_imm);
9285 %}
9286
9287
9288 // Long Logical Instructions
9289
9290 // And Instructions
9291 // And Register with Register
9292 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9293 %{
9294 match(Set dst (AndL dst src));
9295 effect(KILL cr);
9296 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
9297
9298 format %{ "andq $dst, $src\t# long" %}
9299 ins_encode %{
9300 __ andq($dst$$Register, $src$$Register);
9301 %}
9302 ins_pipe(ialu_reg_reg);
9303 %}
9304
9305 // And Register with Immediate 255
9306 instruct andL_rReg_imm255(rRegL dst, rRegL src, immL_255 mask)
9307 %{
9308 match(Set dst (AndL src mask));
9309
9310 format %{ "movzbl $dst, $src\t# long & 0xFF" %}
9311 ins_encode %{
9312 // movzbl zeroes out the upper 32-bit and does not need REX.W
9313 __ movzbl($dst$$Register, $src$$Register);
9314 %}
9315 ins_pipe(ialu_reg);
9316 %}
9317
9318 // And Register with Immediate 65535
9319 instruct andL_rReg_imm65535(rRegL dst, rRegL src, immL_65535 mask)
9320 %{
9321 match(Set dst (AndL src mask));
9322
9323 format %{ "movzwl $dst, $src\t# long & 0xFFFF" %}
9324 ins_encode %{
9325 // movzwl zeroes out the upper 32-bit and does not need REX.W
9326 __ movzwl($dst$$Register, $src$$Register);
9327 %}
9328 ins_pipe(ialu_reg);
9329 %}
9330
9331 // And Register with Immediate
9332 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9333 %{
9334 match(Set dst (AndL dst src));
9335 effect(KILL cr);
9336 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
9337
9338 format %{ "andq $dst, $src\t# long" %}
9339 ins_encode %{
9340 __ andq($dst$$Register, $src$$constant);
9341 %}
9342 ins_pipe(ialu_reg);
9343 %}
9344
9345 // And Register with Memory
9346 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9347 %{
9348 match(Set dst (AndL dst (LoadL src)));
9349 effect(KILL cr);
9350 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
9351
9352 ins_cost(150);
9353 format %{ "andq $dst, $src\t# long" %}
9354 ins_encode %{
9355 __ andq($dst$$Register, $src$$Address);
9356 %}
9357 ins_pipe(ialu_reg_mem);
9358 %}
9359
9360 // And Memory with Register
9361 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9362 %{
9363 match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9364 effect(KILL cr);
9365 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
9366
9367 ins_cost(150);
9368 format %{ "andq $dst, $src\t# long" %}
9369 ins_encode %{
9370 __ andq($dst$$Address, $src$$Register);
9371 %}
9372 ins_pipe(ialu_mem_reg);
9373 %}
9374
9375 // And Memory with Immediate
9376 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9377 %{
9378 match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9379 effect(KILL cr);
9380 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
9381
9382 ins_cost(125);
9383 format %{ "andq $dst, $src\t# long" %}
9384 ins_encode %{
9385 __ andq($dst$$Address, $src$$constant);
9386 %}
9387 ins_pipe(ialu_mem_imm);
9388 %}
9389
9390 instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
9391 %{
9392 // con should be a pure 64-bit immediate given that not(con) is a power of 2
9393 // because AND/OR works well enough for 8/32-bit values.
9394 predicate(log2i_graceful(~n->in(3)->in(2)->get_long()) > 30);
9395
9396 match(Set dst (StoreL dst (AndL (LoadL dst) con)));
9397 effect(KILL cr);
9398
9399 ins_cost(125);
9400 format %{ "btrq $dst, log2(not($con))\t# long" %}
9401 ins_encode %{
9402 __ btrq($dst$$Address, log2i_exact((julong)~$con$$constant));
9403 %}
9404 ins_pipe(ialu_mem_imm);
9405 %}
9406
9407 // BMI1 instructions
9408 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
9409 match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
9410 predicate(UseBMI1Instructions);
9411 effect(KILL cr);
9412 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
9413
9414 ins_cost(125);
9415 format %{ "andnq $dst, $src1, $src2" %}
9416
9417 ins_encode %{
9418 __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
9419 %}
9420 ins_pipe(ialu_reg_mem);
9421 %}
9422
9423 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
9424 match(Set dst (AndL (XorL src1 minus_1) src2));
9425 predicate(UseBMI1Instructions);
9426 effect(KILL cr);
9427 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
9428
9429 format %{ "andnq $dst, $src1, $src2" %}
9430
9431 ins_encode %{
9432 __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
9433 %}
9434 ins_pipe(ialu_reg_mem);
9435 %}
9436
9437 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
9438 match(Set dst (AndL (SubL imm_zero src) src));
9439 predicate(UseBMI1Instructions);
9440 effect(KILL cr);
9441 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
9442
9443 format %{ "blsiq $dst, $src" %}
9444
9445 ins_encode %{
9446 __ blsiq($dst$$Register, $src$$Register);
9447 %}
9448 ins_pipe(ialu_reg);
9449 %}
9450
9451 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
9452 match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
9453 predicate(UseBMI1Instructions);
9454 effect(KILL cr);
9455 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
9456
9457 ins_cost(125);
9458 format %{ "blsiq $dst, $src" %}
9459
9460 ins_encode %{
9461 __ blsiq($dst$$Register, $src$$Address);
9462 %}
9463 ins_pipe(ialu_reg_mem);
9464 %}
9465
9466 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9467 %{
9468 match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
9469 predicate(UseBMI1Instructions);
9470 effect(KILL cr);
9471 flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
9472
9473 ins_cost(125);
9474 format %{ "blsmskq $dst, $src" %}
9475
9476 ins_encode %{
9477 __ blsmskq($dst$$Register, $src$$Address);
9478 %}
9479 ins_pipe(ialu_reg_mem);
9480 %}
9481
9482 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9483 %{
9484 match(Set dst (XorL (AddL src minus_1) src));
9485 predicate(UseBMI1Instructions);
9486 effect(KILL cr);
9487 flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
9488
9489 format %{ "blsmskq $dst, $src" %}
9490
9491 ins_encode %{
9492 __ blsmskq($dst$$Register, $src$$Register);
9493 %}
9494
9495 ins_pipe(ialu_reg);
9496 %}
9497
9498 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9499 %{
9500 match(Set dst (AndL (AddL src minus_1) src) );
9501 predicate(UseBMI1Instructions);
9502 effect(KILL cr);
9503 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
9504
9505 format %{ "blsrq $dst, $src" %}
9506
9507 ins_encode %{
9508 __ blsrq($dst$$Register, $src$$Register);
9509 %}
9510
9511 ins_pipe(ialu_reg);
9512 %}
9513
9514 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9515 %{
9516 match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
9517 predicate(UseBMI1Instructions);
9518 effect(KILL cr);
9519 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
9520
9521 ins_cost(125);
9522 format %{ "blsrq $dst, $src" %}
9523
9524 ins_encode %{
9525 __ blsrq($dst$$Register, $src$$Address);
9526 %}
9527
9528 ins_pipe(ialu_reg);
9529 %}
9530
9531 // Or Instructions
9532 // Or Register with Register
9533 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9534 %{
9535 match(Set dst (OrL dst src));
9536 effect(KILL cr);
9537 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
9538
9539 format %{ "orq $dst, $src\t# long" %}
9540 ins_encode %{
9541 __ orq($dst$$Register, $src$$Register);
9542 %}
9543 ins_pipe(ialu_reg_reg);
9544 %}
9545
9546 // Use any_RegP to match R15 (TLS register) without spilling.
9547 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
9548 match(Set dst (OrL dst (CastP2X src)));
9549 effect(KILL cr);
9550 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
9551
9552 format %{ "orq $dst, $src\t# long" %}
9553 ins_encode %{
9554 __ orq($dst$$Register, $src$$Register);
9555 %}
9556 ins_pipe(ialu_reg_reg);
9557 %}
9558
9559
9560 // Or Register with Immediate
9561 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9562 %{
9563 match(Set dst (OrL dst src));
9564 effect(KILL cr);
9565 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
9566
9567 format %{ "orq $dst, $src\t# long" %}
9568 ins_encode %{
9569 __ orq($dst$$Register, $src$$constant);
9570 %}
9571 ins_pipe(ialu_reg);
9572 %}
9573
9574 // Or Register with Memory
9575 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9576 %{
9577 match(Set dst (OrL dst (LoadL src)));
9578 effect(KILL cr);
9579 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
9580
9581 ins_cost(150);
9582 format %{ "orq $dst, $src\t# long" %}
9583 ins_encode %{
9584 __ orq($dst$$Register, $src$$Address);
9585 %}
9586 ins_pipe(ialu_reg_mem);
9587 %}
9588
9589 // Or Memory with Register
9590 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9591 %{
9592 match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9593 effect(KILL cr);
9594 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
9595
9596 ins_cost(150);
9597 format %{ "orq $dst, $src\t# long" %}
9598 ins_encode %{
9599 __ orq($dst$$Address, $src$$Register);
9600 %}
9601 ins_pipe(ialu_mem_reg);
9602 %}
9603
9604 // Or Memory with Immediate
9605 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9606 %{
9607 match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9608 effect(KILL cr);
9609 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
9610
9611 ins_cost(125);
9612 format %{ "orq $dst, $src\t# long" %}
9613 ins_encode %{
9614 __ orq($dst$$Address, $src$$constant);
9615 %}
9616 ins_pipe(ialu_mem_imm);
9617 %}
9618
9619 instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
9620 %{
9621 // con should be a pure 64-bit power of 2 immediate
9622 // because AND/OR works well enough for 8/32-bit values.
9623 predicate(log2i_graceful(n->in(3)->in(2)->get_long()) > 31);
9624
9625 match(Set dst (StoreL dst (OrL (LoadL dst) con)));
9626 effect(KILL cr);
9627
9628 ins_cost(125);
9629 format %{ "btsq $dst, log2($con)\t# long" %}
9630 ins_encode %{
9631 __ btsq($dst$$Address, log2i_exact((julong)$con$$constant));
9632 %}
9633 ins_pipe(ialu_mem_imm);
9634 %}
9635
9636 // Xor Instructions
9637 // Xor Register with Register
9638 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9639 %{
9640 match(Set dst (XorL dst src));
9641 effect(KILL cr);
9642 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
9643
9644 format %{ "xorq $dst, $src\t# long" %}
9645 ins_encode %{
9646 __ xorq($dst$$Register, $src$$Register);
9647 %}
9648 ins_pipe(ialu_reg_reg);
9649 %}
9650
9651 // Xor Register with Immediate -1
9652 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
9653 match(Set dst (XorL dst imm));
9654
9655 format %{ "notq $dst" %}
9656 ins_encode %{
9657 __ notq($dst$$Register);
9658 %}
9659 ins_pipe(ialu_reg);
9660 %}
9661
9662 // Xor Register with Immediate
9663 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9664 %{
9665 match(Set dst (XorL dst src));
9666 effect(KILL cr);
9667 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
9668
9669 format %{ "xorq $dst, $src\t# long" %}
9670 ins_encode %{
9671 __ xorq($dst$$Register, $src$$constant);
9672 %}
9673 ins_pipe(ialu_reg);
9674 %}
9675
9676 // Xor Register with Memory
9677 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9678 %{
9679 match(Set dst (XorL dst (LoadL src)));
9680 effect(KILL cr);
9681 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
9682
9683 ins_cost(150);
9684 format %{ "xorq $dst, $src\t# long" %}
9685 ins_encode %{
9686 __ xorq($dst$$Register, $src$$Address);
9687 %}
9688 ins_pipe(ialu_reg_mem);
9689 %}
9690
9691 // Xor Memory with Register
9692 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9693 %{
9694 match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9695 effect(KILL cr);
9696 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
9697
9698 ins_cost(150);
9699 format %{ "xorq $dst, $src\t# long" %}
9700 ins_encode %{
9701 __ xorq($dst$$Address, $src$$Register);
9702 %}
9703 ins_pipe(ialu_mem_reg);
9704 %}
9705
9706 // Xor Memory with Immediate
9707 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9708 %{
9709 match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9710 effect(KILL cr);
9711 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
9712
9713 ins_cost(125);
9714 format %{ "xorq $dst, $src\t# long" %}
9715 ins_encode %{
9716 __ xorq($dst$$Address, $src$$constant);
9717 %}
9718 ins_pipe(ialu_mem_imm);
9719 %}
9720
9721 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
9722 %{
9723 match(Set dst (CmpLTMask p q));
9724 effect(KILL cr);
9725
9726 ins_cost(400);
9727 format %{ "cmpl $p, $q\t# cmpLTMask\n\t"
9728 "setcc $dst \t# emits setlt + movzbl or setzul for APX"
9729 "negl $dst" %}
9730 ins_encode %{
9731 __ cmpl($p$$Register, $q$$Register);
9732 __ setcc(Assembler::less, $dst$$Register);
9733 __ negl($dst$$Register);
9734 %}
9735 ins_pipe(pipe_slow);
9736 %}
9737
9738 instruct cmpLTMask0(rRegI dst, immI_0 zero, rFlagsReg cr)
9739 %{
9740 match(Set dst (CmpLTMask dst zero));
9741 effect(KILL cr);
9742
9743 ins_cost(100);
9744 format %{ "sarl $dst, #31\t# cmpLTMask0" %}
9745 ins_encode %{
9746 __ sarl($dst$$Register, 31);
9747 %}
9748 ins_pipe(ialu_reg);
9749 %}
9750
9751 /* Better to save a register than avoid a branch */
9752 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9753 %{
9754 match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
9755 effect(KILL cr);
9756 ins_cost(300);
9757 format %{ "subl $p,$q\t# cadd_cmpLTMask\n\t"
9758 "jge done\n\t"
9759 "addl $p,$y\n"
9760 "done: " %}
9761 ins_encode %{
9762 Register Rp = $p$$Register;
9763 Register Rq = $q$$Register;
9764 Register Ry = $y$$Register;
9765 Label done;
9766 __ subl(Rp, Rq);
9767 __ jccb(Assembler::greaterEqual, done);
9768 __ addl(Rp, Ry);
9769 __ bind(done);
9770 %}
9771 ins_pipe(pipe_cmplt);
9772 %}
9773
9774 /* Better to save a register than avoid a branch */
9775 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9776 %{
9777 match(Set y (AndI (CmpLTMask p q) y));
9778 effect(KILL cr);
9779
9780 ins_cost(300);
9781
9782 format %{ "cmpl $p, $q\t# and_cmpLTMask\n\t"
9783 "jlt done\n\t"
9784 "xorl $y, $y\n"
9785 "done: " %}
9786 ins_encode %{
9787 Register Rp = $p$$Register;
9788 Register Rq = $q$$Register;
9789 Register Ry = $y$$Register;
9790 Label done;
9791 __ cmpl(Rp, Rq);
9792 __ jccb(Assembler::less, done);
9793 __ xorl(Ry, Ry);
9794 __ bind(done);
9795 %}
9796 ins_pipe(pipe_cmplt);
9797 %}
9798
9799
9800 //---------- FP Instructions------------------------------------------------
9801
9802 // Really expensive, avoid
9803 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
9804 %{
9805 match(Set cr (CmpF src1 src2));
9806
9807 ins_cost(500);
9808 format %{ "ucomiss $src1, $src2\n\t"
9809 "jnp,s exit\n\t"
9810 "pushfq\t# saw NaN, set CF\n\t"
9811 "andq [rsp], #0xffffff2b\n\t"
9812 "popfq\n"
9813 "exit:" %}
9814 ins_encode %{
9815 __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9816 emit_cmpfp_fixup(masm);
9817 %}
9818 ins_pipe(pipe_slow);
9819 %}
9820
9821 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
9822 match(Set cr (CmpF src1 src2));
9823
9824 ins_cost(100);
9825 format %{ "ucomiss $src1, $src2" %}
9826 ins_encode %{
9827 __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9828 %}
9829 ins_pipe(pipe_slow);
9830 %}
9831
9832 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
9833 match(Set cr (CmpF src1 (LoadF src2)));
9834
9835 ins_cost(100);
9836 format %{ "ucomiss $src1, $src2" %}
9837 ins_encode %{
9838 __ ucomiss($src1$$XMMRegister, $src2$$Address);
9839 %}
9840 ins_pipe(pipe_slow);
9841 %}
9842
9843 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
9844 match(Set cr (CmpF src con));
9845 ins_cost(100);
9846 format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
9847 ins_encode %{
9848 __ ucomiss($src$$XMMRegister, $constantaddress($con));
9849 %}
9850 ins_pipe(pipe_slow);
9851 %}
9852
9853 // Really expensive, avoid
9854 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
9855 %{
9856 match(Set cr (CmpD src1 src2));
9857
9858 ins_cost(500);
9859 format %{ "ucomisd $src1, $src2\n\t"
9860 "jnp,s exit\n\t"
9861 "pushfq\t# saw NaN, set CF\n\t"
9862 "andq [rsp], #0xffffff2b\n\t"
9863 "popfq\n"
9864 "exit:" %}
9865 ins_encode %{
9866 __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9867 emit_cmpfp_fixup(masm);
9868 %}
9869 ins_pipe(pipe_slow);
9870 %}
9871
9872 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
9873 match(Set cr (CmpD src1 src2));
9874
9875 ins_cost(100);
9876 format %{ "ucomisd $src1, $src2 test" %}
9877 ins_encode %{
9878 __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9879 %}
9880 ins_pipe(pipe_slow);
9881 %}
9882
9883 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
9884 match(Set cr (CmpD src1 (LoadD src2)));
9885
9886 ins_cost(100);
9887 format %{ "ucomisd $src1, $src2" %}
9888 ins_encode %{
9889 __ ucomisd($src1$$XMMRegister, $src2$$Address);
9890 %}
9891 ins_pipe(pipe_slow);
9892 %}
9893
9894 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
9895 match(Set cr (CmpD src con));
9896 ins_cost(100);
9897 format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
9898 ins_encode %{
9899 __ ucomisd($src$$XMMRegister, $constantaddress($con));
9900 %}
9901 ins_pipe(pipe_slow);
9902 %}
9903
9904 // Compare into -1,0,1
9905 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
9906 %{
9907 match(Set dst (CmpF3 src1 src2));
9908 effect(KILL cr);
9909
9910 ins_cost(275);
9911 format %{ "ucomiss $src1, $src2\n\t"
9912 "movl $dst, #-1\n\t"
9913 "jp,s done\n\t"
9914 "jb,s done\n\t"
9915 "setne $dst\n\t"
9916 "movzbl $dst, $dst\n"
9917 "done:" %}
9918 ins_encode %{
9919 __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9920 emit_cmpfp3(masm, $dst$$Register);
9921 %}
9922 ins_pipe(pipe_slow);
9923 %}
9924
9925 // Compare into -1,0,1
9926 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
9927 %{
9928 match(Set dst (CmpF3 src1 (LoadF src2)));
9929 effect(KILL cr);
9930
9931 ins_cost(275);
9932 format %{ "ucomiss $src1, $src2\n\t"
9933 "movl $dst, #-1\n\t"
9934 "jp,s done\n\t"
9935 "jb,s done\n\t"
9936 "setne $dst\n\t"
9937 "movzbl $dst, $dst\n"
9938 "done:" %}
9939 ins_encode %{
9940 __ ucomiss($src1$$XMMRegister, $src2$$Address);
9941 emit_cmpfp3(masm, $dst$$Register);
9942 %}
9943 ins_pipe(pipe_slow);
9944 %}
9945
9946 // Compare into -1,0,1
9947 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
9948 match(Set dst (CmpF3 src con));
9949 effect(KILL cr);
9950
9951 ins_cost(275);
9952 format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
9953 "movl $dst, #-1\n\t"
9954 "jp,s done\n\t"
9955 "jb,s done\n\t"
9956 "setne $dst\n\t"
9957 "movzbl $dst, $dst\n"
9958 "done:" %}
9959 ins_encode %{
9960 __ ucomiss($src$$XMMRegister, $constantaddress($con));
9961 emit_cmpfp3(masm, $dst$$Register);
9962 %}
9963 ins_pipe(pipe_slow);
9964 %}
9965
9966 // Compare into -1,0,1
9967 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
9968 %{
9969 match(Set dst (CmpD3 src1 src2));
9970 effect(KILL cr);
9971
9972 ins_cost(275);
9973 format %{ "ucomisd $src1, $src2\n\t"
9974 "movl $dst, #-1\n\t"
9975 "jp,s done\n\t"
9976 "jb,s done\n\t"
9977 "setne $dst\n\t"
9978 "movzbl $dst, $dst\n"
9979 "done:" %}
9980 ins_encode %{
9981 __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9982 emit_cmpfp3(masm, $dst$$Register);
9983 %}
9984 ins_pipe(pipe_slow);
9985 %}
9986
9987 // Compare into -1,0,1
9988 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
9989 %{
9990 match(Set dst (CmpD3 src1 (LoadD src2)));
9991 effect(KILL cr);
9992
9993 ins_cost(275);
9994 format %{ "ucomisd $src1, $src2\n\t"
9995 "movl $dst, #-1\n\t"
9996 "jp,s done\n\t"
9997 "jb,s done\n\t"
9998 "setne $dst\n\t"
9999 "movzbl $dst, $dst\n"
10000 "done:" %}
10001 ins_encode %{
10002 __ ucomisd($src1$$XMMRegister, $src2$$Address);
10003 emit_cmpfp3(masm, $dst$$Register);
10004 %}
10005 ins_pipe(pipe_slow);
10006 %}
10007
10008 // Compare into -1,0,1
10009 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
10010 match(Set dst (CmpD3 src con));
10011 effect(KILL cr);
10012
10013 ins_cost(275);
10014 format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10015 "movl $dst, #-1\n\t"
10016 "jp,s done\n\t"
10017 "jb,s done\n\t"
10018 "setne $dst\n\t"
10019 "movzbl $dst, $dst\n"
10020 "done:" %}
10021 ins_encode %{
10022 __ ucomisd($src$$XMMRegister, $constantaddress($con));
10023 emit_cmpfp3(masm, $dst$$Register);
10024 %}
10025 ins_pipe(pipe_slow);
10026 %}
10027
10028 //----------Arithmetic Conversion Instructions---------------------------------
10029
10030 instruct convF2D_reg_reg(regD dst, regF src)
10031 %{
10032 match(Set dst (ConvF2D src));
10033
10034 format %{ "cvtss2sd $dst, $src" %}
10035 ins_encode %{
10036 __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
10037 %}
10038 ins_pipe(pipe_slow); // XXX
10039 %}
10040
10041 instruct convF2D_reg_mem(regD dst, memory src)
10042 %{
10043 predicate(UseAVX == 0);
10044 match(Set dst (ConvF2D (LoadF src)));
10045
10046 format %{ "cvtss2sd $dst, $src" %}
10047 ins_encode %{
10048 __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
10049 %}
10050 ins_pipe(pipe_slow); // XXX
10051 %}
10052
10053 instruct convD2F_reg_reg(regF dst, regD src)
10054 %{
10055 match(Set dst (ConvD2F src));
10056
10057 format %{ "cvtsd2ss $dst, $src" %}
10058 ins_encode %{
10059 __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
10060 %}
10061 ins_pipe(pipe_slow); // XXX
10062 %}
10063
10064 instruct convD2F_reg_mem(regF dst, memory src)
10065 %{
10066 predicate(UseAVX == 0);
10067 match(Set dst (ConvD2F (LoadD src)));
10068
10069 format %{ "cvtsd2ss $dst, $src" %}
10070 ins_encode %{
10071 __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
10072 %}
10073 ins_pipe(pipe_slow); // XXX
10074 %}
10075
10076 // XXX do mem variants
10077 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
10078 %{
10079 match(Set dst (ConvF2I src));
10080 effect(KILL cr);
10081 format %{ "convert_f2i $dst, $src" %}
10082 ins_encode %{
10083 __ convertF2I(T_INT, T_FLOAT, $dst$$Register, $src$$XMMRegister);
10084 %}
10085 ins_pipe(pipe_slow);
10086 %}
10087
10088 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
10089 %{
10090 match(Set dst (ConvF2L src));
10091 effect(KILL cr);
10092 format %{ "convert_f2l $dst, $src"%}
10093 ins_encode %{
10094 __ convertF2I(T_LONG, T_FLOAT, $dst$$Register, $src$$XMMRegister);
10095 %}
10096 ins_pipe(pipe_slow);
10097 %}
10098
10099 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
10100 %{
10101 match(Set dst (ConvD2I src));
10102 effect(KILL cr);
10103 format %{ "convert_d2i $dst, $src"%}
10104 ins_encode %{
10105 __ convertF2I(T_INT, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
10106 %}
10107 ins_pipe(pipe_slow);
10108 %}
10109
10110 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10111 %{
10112 match(Set dst (ConvD2L src));
10113 effect(KILL cr);
10114 format %{ "convert_d2l $dst, $src"%}
10115 ins_encode %{
10116 __ convertF2I(T_LONG, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
10117 %}
10118 ins_pipe(pipe_slow);
10119 %}
10120
10121 instruct round_double_reg(rRegL dst, regD src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
10122 %{
10123 match(Set dst (RoundD src));
10124 effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
10125 format %{ "round_double $dst,$src \t! using $rtmp and $rcx as TEMP"%}
10126 ins_encode %{
10127 __ round_double($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
10128 %}
10129 ins_pipe(pipe_slow);
10130 %}
10131
10132 instruct round_float_reg(rRegI dst, regF src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
10133 %{
10134 match(Set dst (RoundF src));
10135 effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
10136 format %{ "round_float $dst,$src" %}
10137 ins_encode %{
10138 __ round_float($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
10139 %}
10140 ins_pipe(pipe_slow);
10141 %}
10142
10143 instruct convI2F_reg_reg(vlRegF dst, rRegI src)
10144 %{
10145 predicate(!UseXmmI2F);
10146 match(Set dst (ConvI2F src));
10147
10148 format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10149 ins_encode %{
10150 if (UseAVX > 0) {
10151 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
10152 }
10153 __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10154 %}
10155 ins_pipe(pipe_slow); // XXX
10156 %}
10157
10158 instruct convI2F_reg_mem(regF dst, memory src)
10159 %{
10160 predicate(UseAVX == 0);
10161 match(Set dst (ConvI2F (LoadI src)));
10162
10163 format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10164 ins_encode %{
10165 __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10166 %}
10167 ins_pipe(pipe_slow); // XXX
10168 %}
10169
10170 instruct convI2D_reg_reg(vlRegD dst, rRegI src)
10171 %{
10172 predicate(!UseXmmI2D);
10173 match(Set dst (ConvI2D src));
10174
10175 format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10176 ins_encode %{
10177 if (UseAVX > 0) {
10178 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
10179 }
10180 __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10181 %}
10182 ins_pipe(pipe_slow); // XXX
10183 %}
10184
10185 instruct convI2D_reg_mem(regD dst, memory src)
10186 %{
10187 predicate(UseAVX == 0);
10188 match(Set dst (ConvI2D (LoadI src)));
10189
10190 format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10191 ins_encode %{
10192 __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10193 %}
10194 ins_pipe(pipe_slow); // XXX
10195 %}
10196
10197 instruct convXI2F_reg(regF dst, rRegI src)
10198 %{
10199 predicate(UseXmmI2F);
10200 match(Set dst (ConvI2F src));
10201
10202 format %{ "movdl $dst, $src\n\t"
10203 "cvtdq2psl $dst, $dst\t# i2f" %}
10204 ins_encode %{
10205 __ movdl($dst$$XMMRegister, $src$$Register);
10206 __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10207 %}
10208 ins_pipe(pipe_slow); // XXX
10209 %}
10210
10211 instruct convXI2D_reg(regD dst, rRegI src)
10212 %{
10213 predicate(UseXmmI2D);
10214 match(Set dst (ConvI2D src));
10215
10216 format %{ "movdl $dst, $src\n\t"
10217 "cvtdq2pdl $dst, $dst\t# i2d" %}
10218 ins_encode %{
10219 __ movdl($dst$$XMMRegister, $src$$Register);
10220 __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10221 %}
10222 ins_pipe(pipe_slow); // XXX
10223 %}
10224
10225 instruct convL2F_reg_reg(vlRegF dst, rRegL src)
10226 %{
10227 match(Set dst (ConvL2F src));
10228
10229 format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10230 ins_encode %{
10231 if (UseAVX > 0) {
10232 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
10233 }
10234 __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10235 %}
10236 ins_pipe(pipe_slow); // XXX
10237 %}
10238
10239 instruct convL2F_reg_mem(regF dst, memory src)
10240 %{
10241 predicate(UseAVX == 0);
10242 match(Set dst (ConvL2F (LoadL src)));
10243
10244 format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10245 ins_encode %{
10246 __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10247 %}
10248 ins_pipe(pipe_slow); // XXX
10249 %}
10250
10251 instruct convL2D_reg_reg(vlRegD dst, rRegL src)
10252 %{
10253 match(Set dst (ConvL2D src));
10254
10255 format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10256 ins_encode %{
10257 if (UseAVX > 0) {
10258 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
10259 }
10260 __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10261 %}
10262 ins_pipe(pipe_slow); // XXX
10263 %}
10264
10265 instruct convL2D_reg_mem(regD dst, memory src)
10266 %{
10267 predicate(UseAVX == 0);
10268 match(Set dst (ConvL2D (LoadL src)));
10269
10270 format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10271 ins_encode %{
10272 __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10273 %}
10274 ins_pipe(pipe_slow); // XXX
10275 %}
10276
10277 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10278 %{
10279 match(Set dst (ConvI2L src));
10280
10281 ins_cost(125);
10282 format %{ "movslq $dst, $src\t# i2l" %}
10283 ins_encode %{
10284 __ movslq($dst$$Register, $src$$Register);
10285 %}
10286 ins_pipe(ialu_reg_reg);
10287 %}
10288
10289 // Zero-extend convert int to long
10290 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10291 %{
10292 match(Set dst (AndL (ConvI2L src) mask));
10293
10294 format %{ "movl $dst, $src\t# i2l zero-extend\n\t" %}
10295 ins_encode %{
10296 if ($dst$$reg != $src$$reg) {
10297 __ movl($dst$$Register, $src$$Register);
10298 }
10299 %}
10300 ins_pipe(ialu_reg_reg);
10301 %}
10302
10303 // Zero-extend convert int to long
10304 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10305 %{
10306 match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10307
10308 format %{ "movl $dst, $src\t# i2l zero-extend\n\t" %}
10309 ins_encode %{
10310 __ movl($dst$$Register, $src$$Address);
10311 %}
10312 ins_pipe(ialu_reg_mem);
10313 %}
10314
10315 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10316 %{
10317 match(Set dst (AndL src mask));
10318
10319 format %{ "movl $dst, $src\t# zero-extend long" %}
10320 ins_encode %{
10321 __ movl($dst$$Register, $src$$Register);
10322 %}
10323 ins_pipe(ialu_reg_reg);
10324 %}
10325
10326 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10327 %{
10328 match(Set dst (ConvL2I src));
10329
10330 format %{ "movl $dst, $src\t# l2i" %}
10331 ins_encode %{
10332 __ movl($dst$$Register, $src$$Register);
10333 %}
10334 ins_pipe(ialu_reg_reg);
10335 %}
10336
10337
10338 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10339 match(Set dst (MoveF2I src));
10340 effect(DEF dst, USE src);
10341
10342 ins_cost(125);
10343 format %{ "movl $dst, $src\t# MoveF2I_stack_reg" %}
10344 ins_encode %{
10345 __ movl($dst$$Register, Address(rsp, $src$$disp));
10346 %}
10347 ins_pipe(ialu_reg_mem);
10348 %}
10349
10350 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
10351 match(Set dst (MoveI2F src));
10352 effect(DEF dst, USE src);
10353
10354 ins_cost(125);
10355 format %{ "movss $dst, $src\t# MoveI2F_stack_reg" %}
10356 ins_encode %{
10357 __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
10358 %}
10359 ins_pipe(pipe_slow);
10360 %}
10361
10362 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
10363 match(Set dst (MoveD2L src));
10364 effect(DEF dst, USE src);
10365
10366 ins_cost(125);
10367 format %{ "movq $dst, $src\t# MoveD2L_stack_reg" %}
10368 ins_encode %{
10369 __ movq($dst$$Register, Address(rsp, $src$$disp));
10370 %}
10371 ins_pipe(ialu_reg_mem);
10372 %}
10373
10374 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
10375 predicate(!UseXmmLoadAndClearUpper);
10376 match(Set dst (MoveL2D src));
10377 effect(DEF dst, USE src);
10378
10379 ins_cost(125);
10380 format %{ "movlpd $dst, $src\t# MoveL2D_stack_reg" %}
10381 ins_encode %{
10382 __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10383 %}
10384 ins_pipe(pipe_slow);
10385 %}
10386
10387 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
10388 predicate(UseXmmLoadAndClearUpper);
10389 match(Set dst (MoveL2D src));
10390 effect(DEF dst, USE src);
10391
10392 ins_cost(125);
10393 format %{ "movsd $dst, $src\t# MoveL2D_stack_reg" %}
10394 ins_encode %{
10395 __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10396 %}
10397 ins_pipe(pipe_slow);
10398 %}
10399
10400
10401 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
10402 match(Set dst (MoveF2I src));
10403 effect(DEF dst, USE src);
10404
10405 ins_cost(95); // XXX
10406 format %{ "movss $dst, $src\t# MoveF2I_reg_stack" %}
10407 ins_encode %{
10408 __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
10409 %}
10410 ins_pipe(pipe_slow);
10411 %}
10412
10413 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
10414 match(Set dst (MoveI2F src));
10415 effect(DEF dst, USE src);
10416
10417 ins_cost(100);
10418 format %{ "movl $dst, $src\t# MoveI2F_reg_stack" %}
10419 ins_encode %{
10420 __ movl(Address(rsp, $dst$$disp), $src$$Register);
10421 %}
10422 ins_pipe( ialu_mem_reg );
10423 %}
10424
10425 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
10426 match(Set dst (MoveD2L src));
10427 effect(DEF dst, USE src);
10428
10429 ins_cost(95); // XXX
10430 format %{ "movsd $dst, $src\t# MoveL2D_reg_stack" %}
10431 ins_encode %{
10432 __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
10433 %}
10434 ins_pipe(pipe_slow);
10435 %}
10436
10437 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
10438 match(Set dst (MoveL2D src));
10439 effect(DEF dst, USE src);
10440
10441 ins_cost(100);
10442 format %{ "movq $dst, $src\t# MoveL2D_reg_stack" %}
10443 ins_encode %{
10444 __ movq(Address(rsp, $dst$$disp), $src$$Register);
10445 %}
10446 ins_pipe(ialu_mem_reg);
10447 %}
10448
10449 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
10450 match(Set dst (MoveF2I src));
10451 effect(DEF dst, USE src);
10452 ins_cost(85);
10453 format %{ "movd $dst,$src\t# MoveF2I" %}
10454 ins_encode %{
10455 __ movdl($dst$$Register, $src$$XMMRegister);
10456 %}
10457 ins_pipe( pipe_slow );
10458 %}
10459
10460 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
10461 match(Set dst (MoveD2L src));
10462 effect(DEF dst, USE src);
10463 ins_cost(85);
10464 format %{ "movd $dst,$src\t# MoveD2L" %}
10465 ins_encode %{
10466 __ movdq($dst$$Register, $src$$XMMRegister);
10467 %}
10468 ins_pipe( pipe_slow );
10469 %}
10470
10471 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
10472 match(Set dst (MoveI2F src));
10473 effect(DEF dst, USE src);
10474 ins_cost(100);
10475 format %{ "movd $dst,$src\t# MoveI2F" %}
10476 ins_encode %{
10477 __ movdl($dst$$XMMRegister, $src$$Register);
10478 %}
10479 ins_pipe( pipe_slow );
10480 %}
10481
10482 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
10483 match(Set dst (MoveL2D src));
10484 effect(DEF dst, USE src);
10485 ins_cost(100);
10486 format %{ "movd $dst,$src\t# MoveL2D" %}
10487 ins_encode %{
10488 __ movdq($dst$$XMMRegister, $src$$Register);
10489 %}
10490 ins_pipe( pipe_slow );
10491 %}
10492
10493 // Fast clearing of an array
10494 // Small non-constant lenght ClearArray for non-AVX512 targets.
10495 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
10496 Universe dummy, rFlagsReg cr)
10497 %{
10498 predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX <= 2));
10499 match(Set dummy (ClearArray cnt base));
10500 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
10501
10502 format %{ $$template
10503 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
10504 $$emit$$"cmp InitArrayShortSize,rcx\n\t"
10505 $$emit$$"jg LARGE\n\t"
10506 $$emit$$"dec rcx\n\t"
10507 $$emit$$"js DONE\t# Zero length\n\t"
10508 $$emit$$"mov rax,(rdi,rcx,8)\t# LOOP\n\t"
10509 $$emit$$"dec rcx\n\t"
10510 $$emit$$"jge LOOP\n\t"
10511 $$emit$$"jmp DONE\n\t"
10512 $$emit$$"# LARGE:\n\t"
10513 if (UseFastStosb) {
10514 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
10515 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--\n\t"
10516 } else if (UseXMMForObjInit) {
10517 $$emit$$"mov rdi,rax\n\t"
10518 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
10519 $$emit$$"jmpq L_zero_64_bytes\n\t"
10520 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10521 $$emit$$"vmovdqu ymm0,(rax)\n\t"
10522 $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10523 $$emit$$"add 0x40,rax\n\t"
10524 $$emit$$"# L_zero_64_bytes:\n\t"
10525 $$emit$$"sub 0x8,rcx\n\t"
10526 $$emit$$"jge L_loop\n\t"
10527 $$emit$$"add 0x4,rcx\n\t"
10528 $$emit$$"jl L_tail\n\t"
10529 $$emit$$"vmovdqu ymm0,(rax)\n\t"
10530 $$emit$$"add 0x20,rax\n\t"
10531 $$emit$$"sub 0x4,rcx\n\t"
10532 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10533 $$emit$$"add 0x4,rcx\n\t"
10534 $$emit$$"jle L_end\n\t"
10535 $$emit$$"dec rcx\n\t"
10536 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10537 $$emit$$"vmovq xmm0,(rax)\n\t"
10538 $$emit$$"add 0x8,rax\n\t"
10539 $$emit$$"dec rcx\n\t"
10540 $$emit$$"jge L_sloop\n\t"
10541 $$emit$$"# L_end:\n\t"
10542 } else {
10543 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--\n\t"
10544 }
10545 $$emit$$"# DONE"
10546 %}
10547 ins_encode %{
10548 __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10549 $tmp$$XMMRegister, false, knoreg);
10550 %}
10551 ins_pipe(pipe_slow);
10552 %}
10553
10554 // Small non-constant length ClearArray for AVX512 targets.
10555 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
10556 Universe dummy, rFlagsReg cr)
10557 %{
10558 predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX > 2));
10559 match(Set dummy (ClearArray cnt base));
10560 ins_cost(125);
10561 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
10562
10563 format %{ $$template
10564 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
10565 $$emit$$"cmp InitArrayShortSize,rcx\n\t"
10566 $$emit$$"jg LARGE\n\t"
10567 $$emit$$"dec rcx\n\t"
10568 $$emit$$"js DONE\t# Zero length\n\t"
10569 $$emit$$"mov rax,(rdi,rcx,8)\t# LOOP\n\t"
10570 $$emit$$"dec rcx\n\t"
10571 $$emit$$"jge LOOP\n\t"
10572 $$emit$$"jmp DONE\n\t"
10573 $$emit$$"# LARGE:\n\t"
10574 if (UseFastStosb) {
10575 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
10576 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--\n\t"
10577 } else if (UseXMMForObjInit) {
10578 $$emit$$"mov rdi,rax\n\t"
10579 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
10580 $$emit$$"jmpq L_zero_64_bytes\n\t"
10581 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10582 $$emit$$"vmovdqu ymm0,(rax)\n\t"
10583 $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10584 $$emit$$"add 0x40,rax\n\t"
10585 $$emit$$"# L_zero_64_bytes:\n\t"
10586 $$emit$$"sub 0x8,rcx\n\t"
10587 $$emit$$"jge L_loop\n\t"
10588 $$emit$$"add 0x4,rcx\n\t"
10589 $$emit$$"jl L_tail\n\t"
10590 $$emit$$"vmovdqu ymm0,(rax)\n\t"
10591 $$emit$$"add 0x20,rax\n\t"
10592 $$emit$$"sub 0x4,rcx\n\t"
10593 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10594 $$emit$$"add 0x4,rcx\n\t"
10595 $$emit$$"jle L_end\n\t"
10596 $$emit$$"dec rcx\n\t"
10597 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10598 $$emit$$"vmovq xmm0,(rax)\n\t"
10599 $$emit$$"add 0x8,rax\n\t"
10600 $$emit$$"dec rcx\n\t"
10601 $$emit$$"jge L_sloop\n\t"
10602 $$emit$$"# L_end:\n\t"
10603 } else {
10604 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--\n\t"
10605 }
10606 $$emit$$"# DONE"
10607 %}
10608 ins_encode %{
10609 __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10610 $tmp$$XMMRegister, false, $ktmp$$KRegister);
10611 %}
10612 ins_pipe(pipe_slow);
10613 %}
10614
10615 // Large non-constant length ClearArray for non-AVX512 targets.
10616 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
10617 Universe dummy, rFlagsReg cr)
10618 %{
10619 predicate((UseAVX <=2) && ((ClearArrayNode*)n)->is_large());
10620 match(Set dummy (ClearArray cnt base));
10621 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
10622
10623 format %{ $$template
10624 if (UseFastStosb) {
10625 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
10626 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
10627 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--"
10628 } else if (UseXMMForObjInit) {
10629 $$emit$$"mov rdi,rax\t# ClearArray:\n\t"
10630 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
10631 $$emit$$"jmpq L_zero_64_bytes\n\t"
10632 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10633 $$emit$$"vmovdqu ymm0,(rax)\n\t"
10634 $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10635 $$emit$$"add 0x40,rax\n\t"
10636 $$emit$$"# L_zero_64_bytes:\n\t"
10637 $$emit$$"sub 0x8,rcx\n\t"
10638 $$emit$$"jge L_loop\n\t"
10639 $$emit$$"add 0x4,rcx\n\t"
10640 $$emit$$"jl L_tail\n\t"
10641 $$emit$$"vmovdqu ymm0,(rax)\n\t"
10642 $$emit$$"add 0x20,rax\n\t"
10643 $$emit$$"sub 0x4,rcx\n\t"
10644 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10645 $$emit$$"add 0x4,rcx\n\t"
10646 $$emit$$"jle L_end\n\t"
10647 $$emit$$"dec rcx\n\t"
10648 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10649 $$emit$$"vmovq xmm0,(rax)\n\t"
10650 $$emit$$"add 0x8,rax\n\t"
10651 $$emit$$"dec rcx\n\t"
10652 $$emit$$"jge L_sloop\n\t"
10653 $$emit$$"# L_end:\n\t"
10654 } else {
10655 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
10656 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--"
10657 }
10658 %}
10659 ins_encode %{
10660 __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10661 $tmp$$XMMRegister, true, knoreg);
10662 %}
10663 ins_pipe(pipe_slow);
10664 %}
10665
10666 // Large non-constant length ClearArray for AVX512 targets.
10667 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
10668 Universe dummy, rFlagsReg cr)
10669 %{
10670 predicate((UseAVX > 2) && ((ClearArrayNode*)n)->is_large());
10671 match(Set dummy (ClearArray cnt base));
10672 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
10673
10674 format %{ $$template
10675 if (UseFastStosb) {
10676 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
10677 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
10678 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--"
10679 } else if (UseXMMForObjInit) {
10680 $$emit$$"mov rdi,rax\t# ClearArray:\n\t"
10681 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
10682 $$emit$$"jmpq L_zero_64_bytes\n\t"
10683 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10684 $$emit$$"vmovdqu ymm0,(rax)\n\t"
10685 $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10686 $$emit$$"add 0x40,rax\n\t"
10687 $$emit$$"# L_zero_64_bytes:\n\t"
10688 $$emit$$"sub 0x8,rcx\n\t"
10689 $$emit$$"jge L_loop\n\t"
10690 $$emit$$"add 0x4,rcx\n\t"
10691 $$emit$$"jl L_tail\n\t"
10692 $$emit$$"vmovdqu ymm0,(rax)\n\t"
10693 $$emit$$"add 0x20,rax\n\t"
10694 $$emit$$"sub 0x4,rcx\n\t"
10695 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10696 $$emit$$"add 0x4,rcx\n\t"
10697 $$emit$$"jle L_end\n\t"
10698 $$emit$$"dec rcx\n\t"
10699 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10700 $$emit$$"vmovq xmm0,(rax)\n\t"
10701 $$emit$$"add 0x8,rax\n\t"
10702 $$emit$$"dec rcx\n\t"
10703 $$emit$$"jge L_sloop\n\t"
10704 $$emit$$"# L_end:\n\t"
10705 } else {
10706 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
10707 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--"
10708 }
10709 %}
10710 ins_encode %{
10711 __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10712 $tmp$$XMMRegister, true, $ktmp$$KRegister);
10713 %}
10714 ins_pipe(pipe_slow);
10715 %}
10716
10717 // Small constant length ClearArray for AVX512 targets.
10718 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rRegI zero, kReg ktmp, Universe dummy, rFlagsReg cr)
10719 %{
10720 predicate(!((ClearArrayNode*)n)->is_large() && (MaxVectorSize >= 32) && VM_Version::supports_avx512vl());
10721 match(Set dummy (ClearArray cnt base));
10722 ins_cost(100);
10723 effect(TEMP tmp, TEMP zero, TEMP ktmp, KILL cr);
10724 format %{ "clear_mem_imm $base , $cnt \n\t" %}
10725 ins_encode %{
10726 __ clear_mem($base$$Register, $cnt$$constant, $zero$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
10727 %}
10728 ins_pipe(pipe_slow);
10729 %}
10730
10731 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10732 rax_RegI result, legRegD tmp1, rFlagsReg cr)
10733 %{
10734 predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
10735 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10736 effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10737
10738 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
10739 ins_encode %{
10740 __ string_compare($str1$$Register, $str2$$Register,
10741 $cnt1$$Register, $cnt2$$Register, $result$$Register,
10742 $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
10743 %}
10744 ins_pipe( pipe_slow );
10745 %}
10746
10747 instruct string_compareL_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10748 rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
10749 %{
10750 predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
10751 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10752 effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10753
10754 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
10755 ins_encode %{
10756 __ string_compare($str1$$Register, $str2$$Register,
10757 $cnt1$$Register, $cnt2$$Register, $result$$Register,
10758 $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
10759 %}
10760 ins_pipe( pipe_slow );
10761 %}
10762
10763 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10764 rax_RegI result, legRegD tmp1, rFlagsReg cr)
10765 %{
10766 predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
10767 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10768 effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10769
10770 format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
10771 ins_encode %{
10772 __ string_compare($str1$$Register, $str2$$Register,
10773 $cnt1$$Register, $cnt2$$Register, $result$$Register,
10774 $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
10775 %}
10776 ins_pipe( pipe_slow );
10777 %}
10778
10779 instruct string_compareU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10780 rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
10781 %{
10782 predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
10783 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10784 effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10785
10786 format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
10787 ins_encode %{
10788 __ string_compare($str1$$Register, $str2$$Register,
10789 $cnt1$$Register, $cnt2$$Register, $result$$Register,
10790 $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
10791 %}
10792 ins_pipe( pipe_slow );
10793 %}
10794
10795 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10796 rax_RegI result, legRegD tmp1, rFlagsReg cr)
10797 %{
10798 predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
10799 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10800 effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10801
10802 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
10803 ins_encode %{
10804 __ string_compare($str1$$Register, $str2$$Register,
10805 $cnt1$$Register, $cnt2$$Register, $result$$Register,
10806 $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
10807 %}
10808 ins_pipe( pipe_slow );
10809 %}
10810
10811 instruct string_compareLU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10812 rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
10813 %{
10814 predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
10815 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10816 effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10817
10818 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
10819 ins_encode %{
10820 __ string_compare($str1$$Register, $str2$$Register,
10821 $cnt1$$Register, $cnt2$$Register, $result$$Register,
10822 $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
10823 %}
10824 ins_pipe( pipe_slow );
10825 %}
10826
10827 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
10828 rax_RegI result, legRegD tmp1, rFlagsReg cr)
10829 %{
10830 predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
10831 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10832 effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10833
10834 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
10835 ins_encode %{
10836 __ string_compare($str2$$Register, $str1$$Register,
10837 $cnt2$$Register, $cnt1$$Register, $result$$Register,
10838 $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
10839 %}
10840 ins_pipe( pipe_slow );
10841 %}
10842
10843 instruct string_compareUL_evex(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
10844 rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
10845 %{
10846 predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
10847 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10848 effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10849
10850 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
10851 ins_encode %{
10852 __ string_compare($str2$$Register, $str1$$Register,
10853 $cnt2$$Register, $cnt1$$Register, $result$$Register,
10854 $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
10855 %}
10856 ins_pipe( pipe_slow );
10857 %}
10858
10859 // fast search of substring with known size.
10860 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10861 rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10862 %{
10863 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
10864 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10865 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10866
10867 format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
10868 ins_encode %{
10869 int icnt2 = (int)$int_cnt2$$constant;
10870 if (icnt2 >= 16) {
10871 // IndexOf for constant substrings with size >= 16 elements
10872 // which don't need to be loaded through stack.
10873 __ string_indexofC8($str1$$Register, $str2$$Register,
10874 $cnt1$$Register, $cnt2$$Register,
10875 icnt2, $result$$Register,
10876 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10877 } else {
10878 // Small strings are loaded through stack if they cross page boundary.
10879 __ string_indexof($str1$$Register, $str2$$Register,
10880 $cnt1$$Register, $cnt2$$Register,
10881 icnt2, $result$$Register,
10882 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10883 }
10884 %}
10885 ins_pipe( pipe_slow );
10886 %}
10887
10888 // fast search of substring with known size.
10889 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10890 rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10891 %{
10892 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
10893 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10894 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10895
10896 format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
10897 ins_encode %{
10898 int icnt2 = (int)$int_cnt2$$constant;
10899 if (icnt2 >= 8) {
10900 // IndexOf for constant substrings with size >= 8 elements
10901 // which don't need to be loaded through stack.
10902 __ string_indexofC8($str1$$Register, $str2$$Register,
10903 $cnt1$$Register, $cnt2$$Register,
10904 icnt2, $result$$Register,
10905 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10906 } else {
10907 // Small strings are loaded through stack if they cross page boundary.
10908 __ string_indexof($str1$$Register, $str2$$Register,
10909 $cnt1$$Register, $cnt2$$Register,
10910 icnt2, $result$$Register,
10911 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10912 }
10913 %}
10914 ins_pipe( pipe_slow );
10915 %}
10916
10917 // fast search of substring with known size.
10918 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10919 rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10920 %{
10921 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
10922 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10923 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10924
10925 format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
10926 ins_encode %{
10927 int icnt2 = (int)$int_cnt2$$constant;
10928 if (icnt2 >= 8) {
10929 // IndexOf for constant substrings with size >= 8 elements
10930 // which don't need to be loaded through stack.
10931 __ string_indexofC8($str1$$Register, $str2$$Register,
10932 $cnt1$$Register, $cnt2$$Register,
10933 icnt2, $result$$Register,
10934 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10935 } else {
10936 // Small strings are loaded through stack if they cross page boundary.
10937 __ string_indexof($str1$$Register, $str2$$Register,
10938 $cnt1$$Register, $cnt2$$Register,
10939 icnt2, $result$$Register,
10940 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10941 }
10942 %}
10943 ins_pipe( pipe_slow );
10944 %}
10945
10946 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10947 rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
10948 %{
10949 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
10950 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10951 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10952
10953 format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL all" %}
10954 ins_encode %{
10955 __ string_indexof($str1$$Register, $str2$$Register,
10956 $cnt1$$Register, $cnt2$$Register,
10957 (-1), $result$$Register,
10958 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10959 %}
10960 ins_pipe( pipe_slow );
10961 %}
10962
10963 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10964 rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
10965 %{
10966 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
10967 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10968 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10969
10970 format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL all" %}
10971 ins_encode %{
10972 __ string_indexof($str1$$Register, $str2$$Register,
10973 $cnt1$$Register, $cnt2$$Register,
10974 (-1), $result$$Register,
10975 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10976 %}
10977 ins_pipe( pipe_slow );
10978 %}
10979
10980 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10981 rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
10982 %{
10983 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
10984 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10985 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10986
10987 format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL all" %}
10988 ins_encode %{
10989 __ string_indexof($str1$$Register, $str2$$Register,
10990 $cnt1$$Register, $cnt2$$Register,
10991 (-1), $result$$Register,
10992 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10993 %}
10994 ins_pipe( pipe_slow );
10995 %}
10996
10997 instruct string_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
10998 rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
10999 %{
11000 predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
11001 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11002 effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11003 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result // KILL all" %}
11004 ins_encode %{
11005 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11006 $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
11007 %}
11008 ins_pipe( pipe_slow );
11009 %}
11010
11011 instruct stringL_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11012 rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
11013 %{
11014 predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
11015 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11016 effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11017 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result // KILL all" %}
11018 ins_encode %{
11019 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11020 $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
11021 %}
11022 ins_pipe( pipe_slow );
11023 %}
11024
11025 // fast string equals
11026 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11027 legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr)
11028 %{
11029 predicate(!VM_Version::supports_avx512vlbw());
11030 match(Set result (StrEquals (Binary str1 str2) cnt));
11031 effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11032
11033 format %{ "String Equals $str1,$str2,$cnt -> $result // KILL $tmp1, $tmp2, $tmp3" %}
11034 ins_encode %{
11035 __ arrays_equals(false, $str1$$Register, $str2$$Register,
11036 $cnt$$Register, $result$$Register, $tmp3$$Register,
11037 $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
11038 %}
11039 ins_pipe( pipe_slow );
11040 %}
11041
11042 instruct string_equals_evex(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11043 legRegD tmp1, legRegD tmp2, kReg ktmp, rbx_RegI tmp3, rFlagsReg cr)
11044 %{
11045 predicate(VM_Version::supports_avx512vlbw());
11046 match(Set result (StrEquals (Binary str1 str2) cnt));
11047 effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11048
11049 format %{ "String Equals $str1,$str2,$cnt -> $result // KILL $tmp1, $tmp2, $tmp3" %}
11050 ins_encode %{
11051 __ arrays_equals(false, $str1$$Register, $str2$$Register,
11052 $cnt$$Register, $result$$Register, $tmp3$$Register,
11053 $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
11054 %}
11055 ins_pipe( pipe_slow );
11056 %}
11057
11058 // fast array equals
11059 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11060 legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11061 %{
11062 predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11063 match(Set result (AryEq ary1 ary2));
11064 effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11065
11066 format %{ "Array Equals byte[] $ary1,$ary2 -> $result // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11067 ins_encode %{
11068 __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11069 $tmp3$$Register, $result$$Register, $tmp4$$Register,
11070 $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
11071 %}
11072 ins_pipe( pipe_slow );
11073 %}
11074
11075 instruct array_equalsB_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11076 legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11077 %{
11078 predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11079 match(Set result (AryEq ary1 ary2));
11080 effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11081
11082 format %{ "Array Equals byte[] $ary1,$ary2 -> $result // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11083 ins_encode %{
11084 __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11085 $tmp3$$Register, $result$$Register, $tmp4$$Register,
11086 $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
11087 %}
11088 ins_pipe( pipe_slow );
11089 %}
11090
11091 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11092 legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11093 %{
11094 predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11095 match(Set result (AryEq ary1 ary2));
11096 effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11097
11098 format %{ "Array Equals char[] $ary1,$ary2 -> $result // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11099 ins_encode %{
11100 __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11101 $tmp3$$Register, $result$$Register, $tmp4$$Register,
11102 $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
11103 %}
11104 ins_pipe( pipe_slow );
11105 %}
11106
11107 instruct array_equalsC_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11108 legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11109 %{
11110 predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11111 match(Set result (AryEq ary1 ary2));
11112 effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11113
11114 format %{ "Array Equals char[] $ary1,$ary2 -> $result // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11115 ins_encode %{
11116 __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11117 $tmp3$$Register, $result$$Register, $tmp4$$Register,
11118 $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
11119 %}
11120 ins_pipe( pipe_slow );
11121 %}
11122
11123 instruct arrays_hashcode(rdi_RegP ary1, rdx_RegI cnt1, rbx_RegI result, immU8 basic_type,
11124 legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, legRegD tmp_vec4,
11125 legRegD tmp_vec5, legRegD tmp_vec6, legRegD tmp_vec7, legRegD tmp_vec8,
11126 legRegD tmp_vec9, legRegD tmp_vec10, legRegD tmp_vec11, legRegD tmp_vec12,
11127 legRegD tmp_vec13, rRegI tmp1, rRegI tmp2, rRegI tmp3, rFlagsReg cr)
11128 %{
11129 predicate(UseAVX >= 2);
11130 match(Set result (VectorizedHashCode (Binary ary1 cnt1) (Binary result basic_type)));
11131 effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, TEMP tmp_vec4, TEMP tmp_vec5, TEMP tmp_vec6,
11132 TEMP tmp_vec7, TEMP tmp_vec8, TEMP tmp_vec9, TEMP tmp_vec10, TEMP tmp_vec11, TEMP tmp_vec12,
11133 TEMP tmp_vec13, TEMP tmp1, TEMP tmp2, TEMP tmp3, USE_KILL ary1, USE_KILL cnt1,
11134 USE basic_type, KILL cr);
11135
11136 format %{ "Array HashCode array[] $ary1,$cnt1,$result,$basic_type -> $result // KILL all" %}
11137 ins_encode %{
11138 __ arrays_hashcode($ary1$$Register, $cnt1$$Register, $result$$Register,
11139 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
11140 $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister,
11141 $tmp_vec4$$XMMRegister, $tmp_vec5$$XMMRegister, $tmp_vec6$$XMMRegister,
11142 $tmp_vec7$$XMMRegister, $tmp_vec8$$XMMRegister, $tmp_vec9$$XMMRegister,
11143 $tmp_vec10$$XMMRegister, $tmp_vec11$$XMMRegister, $tmp_vec12$$XMMRegister,
11144 $tmp_vec13$$XMMRegister, (BasicType)$basic_type$$constant);
11145 %}
11146 ins_pipe( pipe_slow );
11147 %}
11148
11149 instruct count_positives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11150 legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr,)
11151 %{
11152 predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11153 match(Set result (CountPositives ary1 len));
11154 effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11155
11156 format %{ "countPositives byte[] $ary1,$len -> $result // KILL $tmp1, $tmp2, $tmp3" %}
11157 ins_encode %{
11158 __ count_positives($ary1$$Register, $len$$Register,
11159 $result$$Register, $tmp3$$Register,
11160 $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
11161 %}
11162 ins_pipe( pipe_slow );
11163 %}
11164
11165 instruct count_positives_evex(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11166 legRegD tmp1, legRegD tmp2, kReg ktmp1, kReg ktmp2, rbx_RegI tmp3, rFlagsReg cr,)
11167 %{
11168 predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11169 match(Set result (CountPositives ary1 len));
11170 effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11171
11172 format %{ "countPositives byte[] $ary1,$len -> $result // KILL $tmp1, $tmp2, $tmp3" %}
11173 ins_encode %{
11174 __ count_positives($ary1$$Register, $len$$Register,
11175 $result$$Register, $tmp3$$Register,
11176 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
11177 %}
11178 ins_pipe( pipe_slow );
11179 %}
11180
11181 // fast char[] to byte[] compression
11182 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
11183 legRegD tmp4, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11184 predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11185 match(Set result (StrCompressedCopy src (Binary dst len)));
11186 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst,
11187 USE_KILL len, KILL tmp5, KILL cr);
11188
11189 format %{ "String Compress $src,$dst -> $result // KILL RAX, RCX, RDX" %}
11190 ins_encode %{
11191 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11192 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11193 $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
11194 knoreg, knoreg);
11195 %}
11196 ins_pipe( pipe_slow );
11197 %}
11198
11199 instruct string_compress_evex(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
11200 legRegD tmp4, kReg ktmp1, kReg ktmp2, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11201 predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11202 match(Set result (StrCompressedCopy src (Binary dst len)));
11203 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst,
11204 USE_KILL len, KILL tmp5, KILL cr);
11205
11206 format %{ "String Compress $src,$dst -> $result // KILL RAX, RCX, RDX" %}
11207 ins_encode %{
11208 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11209 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11210 $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
11211 $ktmp1$$KRegister, $ktmp2$$KRegister);
11212 %}
11213 ins_pipe( pipe_slow );
11214 %}
11215 // fast byte[] to char[] inflation
11216 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11217 legRegD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
11218 predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11219 match(Set dummy (StrInflatedCopy src (Binary dst len)));
11220 effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11221
11222 format %{ "String Inflate $src,$dst // KILL $tmp1, $tmp2" %}
11223 ins_encode %{
11224 __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11225 $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
11226 %}
11227 ins_pipe( pipe_slow );
11228 %}
11229
11230 instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11231 legRegD tmp1, kReg ktmp, rcx_RegI tmp2, rFlagsReg cr) %{
11232 predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11233 match(Set dummy (StrInflatedCopy src (Binary dst len)));
11234 effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11235
11236 format %{ "String Inflate $src,$dst // KILL $tmp1, $tmp2" %}
11237 ins_encode %{
11238 __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11239 $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
11240 %}
11241 ins_pipe( pipe_slow );
11242 %}
11243
11244 // encode char[] to byte[] in ISO_8859_1
11245 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11246 legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
11247 rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11248 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
11249 match(Set result (EncodeISOArray src (Binary dst len)));
11250 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11251
11252 format %{ "Encode iso array $src,$dst,$len -> $result // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11253 ins_encode %{
11254 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11255 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11256 $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
11257 %}
11258 ins_pipe( pipe_slow );
11259 %}
11260
11261 // encode char[] to byte[] in ASCII
11262 instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11263 legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
11264 rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11265 predicate(((EncodeISOArrayNode*)n)->is_ascii());
11266 match(Set result (EncodeISOArray src (Binary dst len)));
11267 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11268
11269 format %{ "Encode ascii array $src,$dst,$len -> $result // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11270 ins_encode %{
11271 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11272 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11273 $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
11274 %}
11275 ins_pipe( pipe_slow );
11276 %}
11277
11278 //----------Overflow Math Instructions-----------------------------------------
11279
11280 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11281 %{
11282 match(Set cr (OverflowAddI op1 op2));
11283 effect(DEF cr, USE_KILL op1, USE op2);
11284
11285 format %{ "addl $op1, $op2\t# overflow check int" %}
11286
11287 ins_encode %{
11288 __ addl($op1$$Register, $op2$$Register);
11289 %}
11290 ins_pipe(ialu_reg_reg);
11291 %}
11292
11293 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
11294 %{
11295 match(Set cr (OverflowAddI op1 op2));
11296 effect(DEF cr, USE_KILL op1, USE op2);
11297
11298 format %{ "addl $op1, $op2\t# overflow check int" %}
11299
11300 ins_encode %{
11301 __ addl($op1$$Register, $op2$$constant);
11302 %}
11303 ins_pipe(ialu_reg_reg);
11304 %}
11305
11306 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11307 %{
11308 match(Set cr (OverflowAddL op1 op2));
11309 effect(DEF cr, USE_KILL op1, USE op2);
11310
11311 format %{ "addq $op1, $op2\t# overflow check long" %}
11312 ins_encode %{
11313 __ addq($op1$$Register, $op2$$Register);
11314 %}
11315 ins_pipe(ialu_reg_reg);
11316 %}
11317
11318 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
11319 %{
11320 match(Set cr (OverflowAddL op1 op2));
11321 effect(DEF cr, USE_KILL op1, USE op2);
11322
11323 format %{ "addq $op1, $op2\t# overflow check long" %}
11324 ins_encode %{
11325 __ addq($op1$$Register, $op2$$constant);
11326 %}
11327 ins_pipe(ialu_reg_reg);
11328 %}
11329
11330 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11331 %{
11332 match(Set cr (OverflowSubI op1 op2));
11333
11334 format %{ "cmpl $op1, $op2\t# overflow check int" %}
11335 ins_encode %{
11336 __ cmpl($op1$$Register, $op2$$Register);
11337 %}
11338 ins_pipe(ialu_reg_reg);
11339 %}
11340
11341 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11342 %{
11343 match(Set cr (OverflowSubI op1 op2));
11344
11345 format %{ "cmpl $op1, $op2\t# overflow check int" %}
11346 ins_encode %{
11347 __ cmpl($op1$$Register, $op2$$constant);
11348 %}
11349 ins_pipe(ialu_reg_reg);
11350 %}
11351
11352 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11353 %{
11354 match(Set cr (OverflowSubL op1 op2));
11355
11356 format %{ "cmpq $op1, $op2\t# overflow check long" %}
11357 ins_encode %{
11358 __ cmpq($op1$$Register, $op2$$Register);
11359 %}
11360 ins_pipe(ialu_reg_reg);
11361 %}
11362
11363 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11364 %{
11365 match(Set cr (OverflowSubL op1 op2));
11366
11367 format %{ "cmpq $op1, $op2\t# overflow check long" %}
11368 ins_encode %{
11369 __ cmpq($op1$$Register, $op2$$constant);
11370 %}
11371 ins_pipe(ialu_reg_reg);
11372 %}
11373
11374 instruct overflowNegI_rReg(rFlagsReg cr, immI_0 zero, rax_RegI op2)
11375 %{
11376 match(Set cr (OverflowSubI zero op2));
11377 effect(DEF cr, USE_KILL op2);
11378
11379 format %{ "negl $op2\t# overflow check int" %}
11380 ins_encode %{
11381 __ negl($op2$$Register);
11382 %}
11383 ins_pipe(ialu_reg_reg);
11384 %}
11385
11386 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
11387 %{
11388 match(Set cr (OverflowSubL zero op2));
11389 effect(DEF cr, USE_KILL op2);
11390
11391 format %{ "negq $op2\t# overflow check long" %}
11392 ins_encode %{
11393 __ negq($op2$$Register);
11394 %}
11395 ins_pipe(ialu_reg_reg);
11396 %}
11397
11398 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11399 %{
11400 match(Set cr (OverflowMulI op1 op2));
11401 effect(DEF cr, USE_KILL op1, USE op2);
11402
11403 format %{ "imull $op1, $op2\t# overflow check int" %}
11404 ins_encode %{
11405 __ imull($op1$$Register, $op2$$Register);
11406 %}
11407 ins_pipe(ialu_reg_reg_alu0);
11408 %}
11409
11410 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
11411 %{
11412 match(Set cr (OverflowMulI op1 op2));
11413 effect(DEF cr, TEMP tmp, USE op1, USE op2);
11414
11415 format %{ "imull $tmp, $op1, $op2\t# overflow check int" %}
11416 ins_encode %{
11417 __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
11418 %}
11419 ins_pipe(ialu_reg_reg_alu0);
11420 %}
11421
11422 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11423 %{
11424 match(Set cr (OverflowMulL op1 op2));
11425 effect(DEF cr, USE_KILL op1, USE op2);
11426
11427 format %{ "imulq $op1, $op2\t# overflow check long" %}
11428 ins_encode %{
11429 __ imulq($op1$$Register, $op2$$Register);
11430 %}
11431 ins_pipe(ialu_reg_reg_alu0);
11432 %}
11433
11434 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
11435 %{
11436 match(Set cr (OverflowMulL op1 op2));
11437 effect(DEF cr, TEMP tmp, USE op1, USE op2);
11438
11439 format %{ "imulq $tmp, $op1, $op2\t# overflow check long" %}
11440 ins_encode %{
11441 __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
11442 %}
11443 ins_pipe(ialu_reg_reg_alu0);
11444 %}
11445
11446
11447 //----------Control Flow Instructions------------------------------------------
11448 // Signed compare Instructions
11449
11450 // XXX more variants!!
11451 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11452 %{
11453 match(Set cr (CmpI op1 op2));
11454 effect(DEF cr, USE op1, USE op2);
11455
11456 format %{ "cmpl $op1, $op2" %}
11457 ins_encode %{
11458 __ cmpl($op1$$Register, $op2$$Register);
11459 %}
11460 ins_pipe(ialu_cr_reg_reg);
11461 %}
11462
11463 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11464 %{
11465 match(Set cr (CmpI op1 op2));
11466
11467 format %{ "cmpl $op1, $op2" %}
11468 ins_encode %{
11469 __ cmpl($op1$$Register, $op2$$constant);
11470 %}
11471 ins_pipe(ialu_cr_reg_imm);
11472 %}
11473
11474 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
11475 %{
11476 match(Set cr (CmpI op1 (LoadI op2)));
11477
11478 ins_cost(500); // XXX
11479 format %{ "cmpl $op1, $op2" %}
11480 ins_encode %{
11481 __ cmpl($op1$$Register, $op2$$Address);
11482 %}
11483 ins_pipe(ialu_cr_reg_mem);
11484 %}
11485
11486 instruct testI_reg(rFlagsReg cr, rRegI src, immI_0 zero)
11487 %{
11488 match(Set cr (CmpI src zero));
11489
11490 format %{ "testl $src, $src" %}
11491 ins_encode %{
11492 __ testl($src$$Register, $src$$Register);
11493 %}
11494 ins_pipe(ialu_cr_reg_imm);
11495 %}
11496
11497 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI_0 zero)
11498 %{
11499 match(Set cr (CmpI (AndI src con) zero));
11500
11501 format %{ "testl $src, $con" %}
11502 ins_encode %{
11503 __ testl($src$$Register, $con$$constant);
11504 %}
11505 ins_pipe(ialu_cr_reg_imm);
11506 %}
11507
11508 instruct testI_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2, immI_0 zero)
11509 %{
11510 match(Set cr (CmpI (AndI src1 src2) zero));
11511
11512 format %{ "testl $src1, $src2" %}
11513 ins_encode %{
11514 __ testl($src1$$Register, $src2$$Register);
11515 %}
11516 ins_pipe(ialu_cr_reg_imm);
11517 %}
11518
11519 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI_0 zero)
11520 %{
11521 match(Set cr (CmpI (AndI src (LoadI mem)) zero));
11522
11523 format %{ "testl $src, $mem" %}
11524 ins_encode %{
11525 __ testl($src$$Register, $mem$$Address);
11526 %}
11527 ins_pipe(ialu_cr_reg_mem);
11528 %}
11529
11530 // Unsigned compare Instructions; really, same as signed except they
11531 // produce an rFlagsRegU instead of rFlagsReg.
11532 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
11533 %{
11534 match(Set cr (CmpU op1 op2));
11535
11536 format %{ "cmpl $op1, $op2\t# unsigned" %}
11537 ins_encode %{
11538 __ cmpl($op1$$Register, $op2$$Register);
11539 %}
11540 ins_pipe(ialu_cr_reg_reg);
11541 %}
11542
11543 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
11544 %{
11545 match(Set cr (CmpU op1 op2));
11546
11547 format %{ "cmpl $op1, $op2\t# unsigned" %}
11548 ins_encode %{
11549 __ cmpl($op1$$Register, $op2$$constant);
11550 %}
11551 ins_pipe(ialu_cr_reg_imm);
11552 %}
11553
11554 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
11555 %{
11556 match(Set cr (CmpU op1 (LoadI op2)));
11557
11558 ins_cost(500); // XXX
11559 format %{ "cmpl $op1, $op2\t# unsigned" %}
11560 ins_encode %{
11561 __ cmpl($op1$$Register, $op2$$Address);
11562 %}
11563 ins_pipe(ialu_cr_reg_mem);
11564 %}
11565
11566 instruct testU_reg(rFlagsRegU cr, rRegI src, immI_0 zero)
11567 %{
11568 match(Set cr (CmpU src zero));
11569
11570 format %{ "testl $src, $src\t# unsigned" %}
11571 ins_encode %{
11572 __ testl($src$$Register, $src$$Register);
11573 %}
11574 ins_pipe(ialu_cr_reg_imm);
11575 %}
11576
11577 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
11578 %{
11579 match(Set cr (CmpP op1 op2));
11580
11581 format %{ "cmpq $op1, $op2\t# ptr" %}
11582 ins_encode %{
11583 __ cmpq($op1$$Register, $op2$$Register);
11584 %}
11585 ins_pipe(ialu_cr_reg_reg);
11586 %}
11587
11588 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
11589 %{
11590 match(Set cr (CmpP op1 (LoadP op2)));
11591 predicate(n->in(2)->as_Load()->barrier_data() == 0);
11592
11593 ins_cost(500); // XXX
11594 format %{ "cmpq $op1, $op2\t# ptr" %}
11595 ins_encode %{
11596 __ cmpq($op1$$Register, $op2$$Address);
11597 %}
11598 ins_pipe(ialu_cr_reg_mem);
11599 %}
11600
11601 // XXX this is generalized by compP_rReg_mem???
11602 // Compare raw pointer (used in out-of-heap check).
11603 // Only works because non-oop pointers must be raw pointers
11604 // and raw pointers have no anti-dependencies.
11605 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
11606 %{
11607 predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none &&
11608 n->in(2)->as_Load()->barrier_data() == 0);
11609 match(Set cr (CmpP op1 (LoadP op2)));
11610
11611 format %{ "cmpq $op1, $op2\t# raw ptr" %}
11612 ins_encode %{
11613 __ cmpq($op1$$Register, $op2$$Address);
11614 %}
11615 ins_pipe(ialu_cr_reg_mem);
11616 %}
11617
11618 // This will generate a signed flags result. This should be OK since
11619 // any compare to a zero should be eq/neq.
11620 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
11621 %{
11622 match(Set cr (CmpP src zero));
11623
11624 format %{ "testq $src, $src\t# ptr" %}
11625 ins_encode %{
11626 __ testq($src$$Register, $src$$Register);
11627 %}
11628 ins_pipe(ialu_cr_reg_imm);
11629 %}
11630
11631 // This will generate a signed flags result. This should be OK since
11632 // any compare to a zero should be eq/neq.
11633 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
11634 %{
11635 predicate((!UseCompressedOops || (CompressedOops::base() != nullptr)) &&
11636 n->in(1)->as_Load()->barrier_data() == 0);
11637 match(Set cr (CmpP (LoadP op) zero));
11638
11639 ins_cost(500); // XXX
11640 format %{ "testq $op, 0xffffffffffffffff\t# ptr" %}
11641 ins_encode %{
11642 __ testq($op$$Address, 0xFFFFFFFF);
11643 %}
11644 ins_pipe(ialu_cr_reg_imm);
11645 %}
11646
11647 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
11648 %{
11649 predicate(UseCompressedOops && (CompressedOops::base() == nullptr) &&
11650 n->in(1)->as_Load()->barrier_data() == 0);
11651 match(Set cr (CmpP (LoadP mem) zero));
11652
11653 format %{ "cmpq R12, $mem\t# ptr (R12_heapbase==0)" %}
11654 ins_encode %{
11655 __ cmpq(r12, $mem$$Address);
11656 %}
11657 ins_pipe(ialu_cr_reg_mem);
11658 %}
11659
11660 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
11661 %{
11662 match(Set cr (CmpN op1 op2));
11663
11664 format %{ "cmpl $op1, $op2\t# compressed ptr" %}
11665 ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
11666 ins_pipe(ialu_cr_reg_reg);
11667 %}
11668
11669 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
11670 %{
11671 predicate(n->in(2)->as_Load()->barrier_data() == 0);
11672 match(Set cr (CmpN src (LoadN mem)));
11673
11674 format %{ "cmpl $src, $mem\t# compressed ptr" %}
11675 ins_encode %{
11676 __ cmpl($src$$Register, $mem$$Address);
11677 %}
11678 ins_pipe(ialu_cr_reg_mem);
11679 %}
11680
11681 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
11682 match(Set cr (CmpN op1 op2));
11683
11684 format %{ "cmpl $op1, $op2\t# compressed ptr" %}
11685 ins_encode %{
11686 __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
11687 %}
11688 ins_pipe(ialu_cr_reg_imm);
11689 %}
11690
11691 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
11692 %{
11693 predicate(n->in(2)->as_Load()->barrier_data() == 0);
11694 match(Set cr (CmpN src (LoadN mem)));
11695
11696 format %{ "cmpl $mem, $src\t# compressed ptr" %}
11697 ins_encode %{
11698 __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
11699 %}
11700 ins_pipe(ialu_cr_reg_mem);
11701 %}
11702
11703 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
11704 match(Set cr (CmpN op1 op2));
11705
11706 format %{ "cmpl $op1, $op2\t# compressed klass ptr" %}
11707 ins_encode %{
11708 __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
11709 %}
11710 ins_pipe(ialu_cr_reg_imm);
11711 %}
11712
11713 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
11714 %{
11715 predicate(!UseCompactObjectHeaders);
11716 match(Set cr (CmpN src (LoadNKlass mem)));
11717
11718 format %{ "cmpl $mem, $src\t# compressed klass ptr" %}
11719 ins_encode %{
11720 __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
11721 %}
11722 ins_pipe(ialu_cr_reg_mem);
11723 %}
11724
11725 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
11726 match(Set cr (CmpN src zero));
11727
11728 format %{ "testl $src, $src\t# compressed ptr" %}
11729 ins_encode %{ __ testl($src$$Register, $src$$Register); %}
11730 ins_pipe(ialu_cr_reg_imm);
11731 %}
11732
11733 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
11734 %{
11735 predicate(CompressedOops::base() != nullptr &&
11736 n->in(1)->as_Load()->barrier_data() == 0);
11737 match(Set cr (CmpN (LoadN mem) zero));
11738
11739 ins_cost(500); // XXX
11740 format %{ "testl $mem, 0xffffffff\t# compressed ptr" %}
11741 ins_encode %{
11742 __ cmpl($mem$$Address, (int)0xFFFFFFFF);
11743 %}
11744 ins_pipe(ialu_cr_reg_mem);
11745 %}
11746
11747 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
11748 %{
11749 predicate(CompressedOops::base() == nullptr &&
11750 n->in(1)->as_Load()->barrier_data() == 0);
11751 match(Set cr (CmpN (LoadN mem) zero));
11752
11753 format %{ "cmpl R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
11754 ins_encode %{
11755 __ cmpl(r12, $mem$$Address);
11756 %}
11757 ins_pipe(ialu_cr_reg_mem);
11758 %}
11759
11760 // Yanked all unsigned pointer compare operations.
11761 // Pointer compares are done with CmpP which is already unsigned.
11762
11763 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11764 %{
11765 match(Set cr (CmpL op1 op2));
11766
11767 format %{ "cmpq $op1, $op2" %}
11768 ins_encode %{
11769 __ cmpq($op1$$Register, $op2$$Register);
11770 %}
11771 ins_pipe(ialu_cr_reg_reg);
11772 %}
11773
11774 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11775 %{
11776 match(Set cr (CmpL op1 op2));
11777
11778 format %{ "cmpq $op1, $op2" %}
11779 ins_encode %{
11780 __ cmpq($op1$$Register, $op2$$constant);
11781 %}
11782 ins_pipe(ialu_cr_reg_imm);
11783 %}
11784
11785 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
11786 %{
11787 match(Set cr (CmpL op1 (LoadL op2)));
11788
11789 format %{ "cmpq $op1, $op2" %}
11790 ins_encode %{
11791 __ cmpq($op1$$Register, $op2$$Address);
11792 %}
11793 ins_pipe(ialu_cr_reg_mem);
11794 %}
11795
11796 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
11797 %{
11798 match(Set cr (CmpL src zero));
11799
11800 format %{ "testq $src, $src" %}
11801 ins_encode %{
11802 __ testq($src$$Register, $src$$Register);
11803 %}
11804 ins_pipe(ialu_cr_reg_imm);
11805 %}
11806
11807 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
11808 %{
11809 match(Set cr (CmpL (AndL src con) zero));
11810
11811 format %{ "testq $src, $con\t# long" %}
11812 ins_encode %{
11813 __ testq($src$$Register, $con$$constant);
11814 %}
11815 ins_pipe(ialu_cr_reg_imm);
11816 %}
11817
11818 instruct testL_reg_reg(rFlagsReg cr, rRegL src1, rRegL src2, immL0 zero)
11819 %{
11820 match(Set cr (CmpL (AndL src1 src2) zero));
11821
11822 format %{ "testq $src1, $src2\t# long" %}
11823 ins_encode %{
11824 __ testq($src1$$Register, $src2$$Register);
11825 %}
11826 ins_pipe(ialu_cr_reg_imm);
11827 %}
11828
11829 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
11830 %{
11831 match(Set cr (CmpL (AndL src (LoadL mem)) zero));
11832
11833 format %{ "testq $src, $mem" %}
11834 ins_encode %{
11835 __ testq($src$$Register, $mem$$Address);
11836 %}
11837 ins_pipe(ialu_cr_reg_mem);
11838 %}
11839
11840 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
11841 %{
11842 match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
11843
11844 format %{ "testq $src, $mem" %}
11845 ins_encode %{
11846 __ testq($src$$Register, $mem$$Address);
11847 %}
11848 ins_pipe(ialu_cr_reg_mem);
11849 %}
11850
11851 // Manifest a CmpU result in an integer register. Very painful.
11852 // This is the test to avoid.
11853 instruct cmpU3_reg_reg(rRegI dst, rRegI src1, rRegI src2, rFlagsReg flags)
11854 %{
11855 match(Set dst (CmpU3 src1 src2));
11856 effect(KILL flags);
11857
11858 ins_cost(275); // XXX
11859 format %{ "cmpl $src1, $src2\t# CmpL3\n\t"
11860 "movl $dst, -1\n\t"
11861 "jb,u done\n\t"
11862 "setcc $dst \t# emits setne + movzbl or setzune for APX"
11863 "done:" %}
11864 ins_encode %{
11865 Label done;
11866 __ cmpl($src1$$Register, $src2$$Register);
11867 __ movl($dst$$Register, -1);
11868 __ jccb(Assembler::below, done);
11869 __ setcc(Assembler::notZero, $dst$$Register);
11870 __ bind(done);
11871 %}
11872 ins_pipe(pipe_slow);
11873 %}
11874
11875 // Manifest a CmpL result in an integer register. Very painful.
11876 // This is the test to avoid.
11877 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
11878 %{
11879 match(Set dst (CmpL3 src1 src2));
11880 effect(KILL flags);
11881
11882 ins_cost(275); // XXX
11883 format %{ "cmpq $src1, $src2\t# CmpL3\n\t"
11884 "movl $dst, -1\n\t"
11885 "jl,s done\n\t"
11886 "setcc $dst \t# emits setne + movzbl or setzune for APX"
11887 "done:" %}
11888 ins_encode %{
11889 Label done;
11890 __ cmpq($src1$$Register, $src2$$Register);
11891 __ movl($dst$$Register, -1);
11892 __ jccb(Assembler::less, done);
11893 __ setcc(Assembler::notZero, $dst$$Register);
11894 __ bind(done);
11895 %}
11896 ins_pipe(pipe_slow);
11897 %}
11898
11899 // Manifest a CmpUL result in an integer register. Very painful.
11900 // This is the test to avoid.
11901 instruct cmpUL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
11902 %{
11903 match(Set dst (CmpUL3 src1 src2));
11904 effect(KILL flags);
11905
11906 ins_cost(275); // XXX
11907 format %{ "cmpq $src1, $src2\t# CmpL3\n\t"
11908 "movl $dst, -1\n\t"
11909 "jb,u done\n\t"
11910 "setcc $dst \t# emits setne + movzbl or setzune for APX"
11911 "done:" %}
11912 ins_encode %{
11913 Label done;
11914 __ cmpq($src1$$Register, $src2$$Register);
11915 __ movl($dst$$Register, -1);
11916 __ jccb(Assembler::below, done);
11917 __ setcc(Assembler::notZero, $dst$$Register);
11918 __ bind(done);
11919 %}
11920 ins_pipe(pipe_slow);
11921 %}
11922
11923 // Unsigned long compare Instructions; really, same as signed long except they
11924 // produce an rFlagsRegU instead of rFlagsReg.
11925 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
11926 %{
11927 match(Set cr (CmpUL op1 op2));
11928
11929 format %{ "cmpq $op1, $op2\t# unsigned" %}
11930 ins_encode %{
11931 __ cmpq($op1$$Register, $op2$$Register);
11932 %}
11933 ins_pipe(ialu_cr_reg_reg);
11934 %}
11935
11936 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
11937 %{
11938 match(Set cr (CmpUL op1 op2));
11939
11940 format %{ "cmpq $op1, $op2\t# unsigned" %}
11941 ins_encode %{
11942 __ cmpq($op1$$Register, $op2$$constant);
11943 %}
11944 ins_pipe(ialu_cr_reg_imm);
11945 %}
11946
11947 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
11948 %{
11949 match(Set cr (CmpUL op1 (LoadL op2)));
11950
11951 format %{ "cmpq $op1, $op2\t# unsigned" %}
11952 ins_encode %{
11953 __ cmpq($op1$$Register, $op2$$Address);
11954 %}
11955 ins_pipe(ialu_cr_reg_mem);
11956 %}
11957
11958 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
11959 %{
11960 match(Set cr (CmpUL src zero));
11961
11962 format %{ "testq $src, $src\t# unsigned" %}
11963 ins_encode %{
11964 __ testq($src$$Register, $src$$Register);
11965 %}
11966 ins_pipe(ialu_cr_reg_imm);
11967 %}
11968
11969 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
11970 %{
11971 match(Set cr (CmpI (LoadB mem) imm));
11972
11973 ins_cost(125);
11974 format %{ "cmpb $mem, $imm" %}
11975 ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
11976 ins_pipe(ialu_cr_reg_mem);
11977 %}
11978
11979 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU7 imm, immI_0 zero)
11980 %{
11981 match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
11982
11983 ins_cost(125);
11984 format %{ "testb $mem, $imm\t# ubyte" %}
11985 ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
11986 ins_pipe(ialu_cr_reg_mem);
11987 %}
11988
11989 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
11990 %{
11991 match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
11992
11993 ins_cost(125);
11994 format %{ "testb $mem, $imm\t# byte" %}
11995 ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
11996 ins_pipe(ialu_cr_reg_mem);
11997 %}
11998
11999 //----------Max and Min--------------------------------------------------------
12000 // Min Instructions
12001
12002 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
12003 %{
12004 effect(USE_DEF dst, USE src, USE cr);
12005
12006 format %{ "cmovlgt $dst, $src\t# min" %}
12007 ins_encode %{
12008 __ cmovl(Assembler::greater, $dst$$Register, $src$$Register);
12009 %}
12010 ins_pipe(pipe_cmov_reg);
12011 %}
12012
12013
12014 instruct minI_rReg(rRegI dst, rRegI src)
12015 %{
12016 match(Set dst (MinI dst src));
12017
12018 ins_cost(200);
12019 expand %{
12020 rFlagsReg cr;
12021 compI_rReg(cr, dst, src);
12022 cmovI_reg_g(dst, src, cr);
12023 %}
12024 %}
12025
12026 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
12027 %{
12028 effect(USE_DEF dst, USE src, USE cr);
12029
12030 format %{ "cmovllt $dst, $src\t# max" %}
12031 ins_encode %{
12032 __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
12033 %}
12034 ins_pipe(pipe_cmov_reg);
12035 %}
12036
12037
12038 instruct maxI_rReg(rRegI dst, rRegI src)
12039 %{
12040 match(Set dst (MaxI dst src));
12041
12042 ins_cost(200);
12043 expand %{
12044 rFlagsReg cr;
12045 compI_rReg(cr, dst, src);
12046 cmovI_reg_l(dst, src, cr);
12047 %}
12048 %}
12049
12050 // ============================================================================
12051 // Branch Instructions
12052
12053 // Jump Direct - Label defines a relative address from JMP+1
12054 instruct jmpDir(label labl)
12055 %{
12056 match(Goto);
12057 effect(USE labl);
12058
12059 ins_cost(300);
12060 format %{ "jmp $labl" %}
12061 size(5);
12062 ins_encode %{
12063 Label* L = $labl$$label;
12064 __ jmp(*L, false); // Always long jump
12065 %}
12066 ins_pipe(pipe_jmp);
12067 %}
12068
12069 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12070 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
12071 %{
12072 match(If cop cr);
12073 effect(USE labl);
12074
12075 ins_cost(300);
12076 format %{ "j$cop $labl" %}
12077 size(6);
12078 ins_encode %{
12079 Label* L = $labl$$label;
12080 __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12081 %}
12082 ins_pipe(pipe_jcc);
12083 %}
12084
12085 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12086 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
12087 %{
12088 match(CountedLoopEnd cop cr);
12089 effect(USE labl);
12090
12091 ins_cost(300);
12092 format %{ "j$cop $labl\t# loop end" %}
12093 size(6);
12094 ins_encode %{
12095 Label* L = $labl$$label;
12096 __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12097 %}
12098 ins_pipe(pipe_jcc);
12099 %}
12100
12101 // Jump Direct Conditional - using unsigned comparison
12102 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12103 match(If cop cmp);
12104 effect(USE labl);
12105
12106 ins_cost(300);
12107 format %{ "j$cop,u $labl" %}
12108 size(6);
12109 ins_encode %{
12110 Label* L = $labl$$label;
12111 __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12112 %}
12113 ins_pipe(pipe_jcc);
12114 %}
12115
12116 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12117 match(If cop cmp);
12118 effect(USE labl);
12119
12120 ins_cost(200);
12121 format %{ "j$cop,u $labl" %}
12122 size(6);
12123 ins_encode %{
12124 Label* L = $labl$$label;
12125 __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12126 %}
12127 ins_pipe(pipe_jcc);
12128 %}
12129
12130 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12131 match(If cop cmp);
12132 effect(USE labl);
12133
12134 ins_cost(200);
12135 format %{ $$template
12136 if ($cop$$cmpcode == Assembler::notEqual) {
12137 $$emit$$"jp,u $labl\n\t"
12138 $$emit$$"j$cop,u $labl"
12139 } else {
12140 $$emit$$"jp,u done\n\t"
12141 $$emit$$"j$cop,u $labl\n\t"
12142 $$emit$$"done:"
12143 }
12144 %}
12145 ins_encode %{
12146 Label* l = $labl$$label;
12147 if ($cop$$cmpcode == Assembler::notEqual) {
12148 __ jcc(Assembler::parity, *l, false);
12149 __ jcc(Assembler::notEqual, *l, false);
12150 } else if ($cop$$cmpcode == Assembler::equal) {
12151 Label done;
12152 __ jccb(Assembler::parity, done);
12153 __ jcc(Assembler::equal, *l, false);
12154 __ bind(done);
12155 } else {
12156 ShouldNotReachHere();
12157 }
12158 %}
12159 ins_pipe(pipe_jcc);
12160 %}
12161
12162 // ============================================================================
12163 // The 2nd slow-half of a subtype check. Scan the subklass's 2ndary
12164 // superklass array for an instance of the superklass. Set a hidden
12165 // internal cache on a hit (cache is checked with exposed code in
12166 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
12167 // encoding ALSO sets flags.
12168
12169 instruct partialSubtypeCheck(rdi_RegP result,
12170 rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12171 rFlagsReg cr)
12172 %{
12173 match(Set result (PartialSubtypeCheck sub super));
12174 predicate(!UseSecondarySupersTable);
12175 effect(KILL rcx, KILL cr);
12176
12177 ins_cost(1100); // slightly larger than the next version
12178 format %{ "movq rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12179 "movl rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12180 "addq rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12181 "repne scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
12182 "jne,s miss\t\t# Missed: rdi not-zero\n\t"
12183 "movq [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12184 "xorq $result, $result\t\t Hit: rdi zero\n\t"
12185 "miss:\t" %}
12186
12187 ins_encode %{
12188 Label miss;
12189 // NB: Callers may assume that, when $result is a valid register,
12190 // check_klass_subtype_slow_path_linear sets it to a nonzero
12191 // value.
12192 __ check_klass_subtype_slow_path_linear($sub$$Register, $super$$Register,
12193 $rcx$$Register, $result$$Register,
12194 nullptr, &miss,
12195 /*set_cond_codes:*/ true);
12196 __ xorptr($result$$Register, $result$$Register);
12197 __ bind(miss);
12198 %}
12199
12200 ins_pipe(pipe_slow);
12201 %}
12202
12203 // ============================================================================
12204 // Two versions of hashtable-based partialSubtypeCheck, both used when
12205 // we need to search for a super class in the secondary supers array.
12206 // The first is used when we don't know _a priori_ the class being
12207 // searched for. The second, far more common, is used when we do know:
12208 // this is used for instanceof, checkcast, and any case where C2 can
12209 // determine it by constant propagation.
12210
12211 instruct partialSubtypeCheckVarSuper(rsi_RegP sub, rax_RegP super, rdi_RegP result,
12212 rdx_RegL temp1, rcx_RegL temp2, rbx_RegP temp3, r11_RegL temp4,
12213 rFlagsReg cr)
12214 %{
12215 match(Set result (PartialSubtypeCheck sub super));
12216 predicate(UseSecondarySupersTable);
12217 effect(KILL cr, TEMP temp1, TEMP temp2, TEMP temp3, TEMP temp4);
12218
12219 ins_cost(1000);
12220 format %{ "partialSubtypeCheck $result, $sub, $super" %}
12221
12222 ins_encode %{
12223 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register, $temp1$$Register, $temp2$$Register,
12224 $temp3$$Register, $temp4$$Register, $result$$Register);
12225 %}
12226
12227 ins_pipe(pipe_slow);
12228 %}
12229
12230 instruct partialSubtypeCheckConstSuper(rsi_RegP sub, rax_RegP super_reg, immP super_con, rdi_RegP result,
12231 rdx_RegL temp1, rcx_RegL temp2, rbx_RegP temp3, r11_RegL temp4,
12232 rFlagsReg cr)
12233 %{
12234 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
12235 predicate(UseSecondarySupersTable);
12236 effect(KILL cr, TEMP temp1, TEMP temp2, TEMP temp3, TEMP temp4);
12237
12238 ins_cost(700); // smaller than the next version
12239 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
12240
12241 ins_encode %{
12242 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
12243 if (InlineSecondarySupersTest) {
12244 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register, $temp1$$Register, $temp2$$Register,
12245 $temp3$$Register, $temp4$$Register, $result$$Register,
12246 super_klass_slot);
12247 } else {
12248 __ call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
12249 }
12250 %}
12251
12252 ins_pipe(pipe_slow);
12253 %}
12254
12255 // ============================================================================
12256 // Branch Instructions -- short offset versions
12257 //
12258 // These instructions are used to replace jumps of a long offset (the default
12259 // match) with jumps of a shorter offset. These instructions are all tagged
12260 // with the ins_short_branch attribute, which causes the ADLC to suppress the
12261 // match rules in general matching. Instead, the ADLC generates a conversion
12262 // method in the MachNode which can be used to do in-place replacement of the
12263 // long variant with the shorter variant. The compiler will determine if a
12264 // branch can be taken by the is_short_branch_offset() predicate in the machine
12265 // specific code section of the file.
12266
12267 // Jump Direct - Label defines a relative address from JMP+1
12268 instruct jmpDir_short(label labl) %{
12269 match(Goto);
12270 effect(USE labl);
12271
12272 ins_cost(300);
12273 format %{ "jmp,s $labl" %}
12274 size(2);
12275 ins_encode %{
12276 Label* L = $labl$$label;
12277 __ jmpb(*L);
12278 %}
12279 ins_pipe(pipe_jmp);
12280 ins_short_branch(1);
12281 %}
12282
12283 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12284 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
12285 match(If cop cr);
12286 effect(USE labl);
12287
12288 ins_cost(300);
12289 format %{ "j$cop,s $labl" %}
12290 size(2);
12291 ins_encode %{
12292 Label* L = $labl$$label;
12293 __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12294 %}
12295 ins_pipe(pipe_jcc);
12296 ins_short_branch(1);
12297 %}
12298
12299 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12300 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
12301 match(CountedLoopEnd cop cr);
12302 effect(USE labl);
12303
12304 ins_cost(300);
12305 format %{ "j$cop,s $labl\t# loop end" %}
12306 size(2);
12307 ins_encode %{
12308 Label* L = $labl$$label;
12309 __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12310 %}
12311 ins_pipe(pipe_jcc);
12312 ins_short_branch(1);
12313 %}
12314
12315 // Jump Direct Conditional - using unsigned comparison
12316 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12317 match(If cop cmp);
12318 effect(USE labl);
12319
12320 ins_cost(300);
12321 format %{ "j$cop,us $labl" %}
12322 size(2);
12323 ins_encode %{
12324 Label* L = $labl$$label;
12325 __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12326 %}
12327 ins_pipe(pipe_jcc);
12328 ins_short_branch(1);
12329 %}
12330
12331 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12332 match(If cop cmp);
12333 effect(USE labl);
12334
12335 ins_cost(300);
12336 format %{ "j$cop,us $labl" %}
12337 size(2);
12338 ins_encode %{
12339 Label* L = $labl$$label;
12340 __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12341 %}
12342 ins_pipe(pipe_jcc);
12343 ins_short_branch(1);
12344 %}
12345
12346 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12347 match(If cop cmp);
12348 effect(USE labl);
12349
12350 ins_cost(300);
12351 format %{ $$template
12352 if ($cop$$cmpcode == Assembler::notEqual) {
12353 $$emit$$"jp,u,s $labl\n\t"
12354 $$emit$$"j$cop,u,s $labl"
12355 } else {
12356 $$emit$$"jp,u,s done\n\t"
12357 $$emit$$"j$cop,u,s $labl\n\t"
12358 $$emit$$"done:"
12359 }
12360 %}
12361 size(4);
12362 ins_encode %{
12363 Label* l = $labl$$label;
12364 if ($cop$$cmpcode == Assembler::notEqual) {
12365 __ jccb(Assembler::parity, *l);
12366 __ jccb(Assembler::notEqual, *l);
12367 } else if ($cop$$cmpcode == Assembler::equal) {
12368 Label done;
12369 __ jccb(Assembler::parity, done);
12370 __ jccb(Assembler::equal, *l);
12371 __ bind(done);
12372 } else {
12373 ShouldNotReachHere();
12374 }
12375 %}
12376 ins_pipe(pipe_jcc);
12377 ins_short_branch(1);
12378 %}
12379
12380 // ============================================================================
12381 // inlined locking and unlocking
12382
12383 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
12384 predicate(LockingMode != LM_LIGHTWEIGHT);
12385 match(Set cr (FastLock object box));
12386 effect(TEMP tmp, TEMP scr, USE_KILL box);
12387 ins_cost(300);
12388 format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
12389 ins_encode %{
12390 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12391 $scr$$Register, noreg, noreg, r15_thread, nullptr);
12392 %}
12393 ins_pipe(pipe_slow);
12394 %}
12395
12396 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
12397 predicate(LockingMode != LM_LIGHTWEIGHT);
12398 match(Set cr (FastUnlock object box));
12399 effect(TEMP tmp, USE_KILL box);
12400 ins_cost(300);
12401 format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
12402 ins_encode %{
12403 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register);
12404 %}
12405 ins_pipe(pipe_slow);
12406 %}
12407
12408 instruct cmpFastLockLightweight(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI rax_reg, rRegP tmp) %{
12409 predicate(LockingMode == LM_LIGHTWEIGHT);
12410 match(Set cr (FastLock object box));
12411 effect(TEMP rax_reg, TEMP tmp, USE_KILL box);
12412 ins_cost(300);
12413 format %{ "fastlock $object,$box\t! kills $box,$rax_reg,$tmp" %}
12414 ins_encode %{
12415 __ fast_lock_lightweight($object$$Register, $box$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
12416 %}
12417 ins_pipe(pipe_slow);
12418 %}
12419
12420 instruct cmpFastUnlockLightweight(rFlagsReg cr, rRegP object, rax_RegP rax_reg, rRegP tmp) %{
12421 predicate(LockingMode == LM_LIGHTWEIGHT);
12422 match(Set cr (FastUnlock object rax_reg));
12423 effect(TEMP tmp, USE_KILL rax_reg);
12424 ins_cost(300);
12425 format %{ "fastunlock $object,$rax_reg\t! kills $rax_reg,$tmp" %}
12426 ins_encode %{
12427 __ fast_unlock_lightweight($object$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
12428 %}
12429 ins_pipe(pipe_slow);
12430 %}
12431
12432
12433 // ============================================================================
12434 // Safepoint Instructions
12435 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
12436 %{
12437 match(SafePoint poll);
12438 effect(KILL cr, USE poll);
12439
12440 format %{ "testl rax, [$poll]\t"
12441 "# Safepoint: poll for GC" %}
12442 ins_cost(125);
12443 ins_encode %{
12444 __ relocate(relocInfo::poll_type);
12445 address pre_pc = __ pc();
12446 __ testl(rax, Address($poll$$Register, 0));
12447 assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
12448 %}
12449 ins_pipe(ialu_reg_mem);
12450 %}
12451
12452 instruct mask_all_evexL(kReg dst, rRegL src) %{
12453 match(Set dst (MaskAll src));
12454 format %{ "mask_all_evexL $dst, $src \t! mask all operation" %}
12455 ins_encode %{
12456 int mask_len = Matcher::vector_length(this);
12457 __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
12458 %}
12459 ins_pipe( pipe_slow );
12460 %}
12461
12462 instruct mask_all_evexI_GT32(kReg dst, rRegI src, rRegL tmp) %{
12463 predicate(Matcher::vector_length(n) > 32);
12464 match(Set dst (MaskAll src));
12465 effect(TEMP tmp);
12466 format %{ "mask_all_evexI_GT32 $dst, $src \t! using $tmp as TEMP" %}
12467 ins_encode %{
12468 int mask_len = Matcher::vector_length(this);
12469 __ movslq($tmp$$Register, $src$$Register);
12470 __ vector_maskall_operation($dst$$KRegister, $tmp$$Register, mask_len);
12471 %}
12472 ins_pipe( pipe_slow );
12473 %}
12474
12475 // ============================================================================
12476 // Procedure Call/Return Instructions
12477 // Call Java Static Instruction
12478 // Note: If this code changes, the corresponding ret_addr_offset() and
12479 // compute_padding() functions will have to be adjusted.
12480 instruct CallStaticJavaDirect(method meth) %{
12481 match(CallStaticJava);
12482 effect(USE meth);
12483
12484 ins_cost(300);
12485 format %{ "call,static " %}
12486 opcode(0xE8); /* E8 cd */
12487 ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
12488 ins_pipe(pipe_slow);
12489 ins_alignment(4);
12490 %}
12491
12492 // Call Java Dynamic Instruction
12493 // Note: If this code changes, the corresponding ret_addr_offset() and
12494 // compute_padding() functions will have to be adjusted.
12495 instruct CallDynamicJavaDirect(method meth)
12496 %{
12497 match(CallDynamicJava);
12498 effect(USE meth);
12499
12500 ins_cost(300);
12501 format %{ "movq rax, #Universe::non_oop_word()\n\t"
12502 "call,dynamic " %}
12503 ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
12504 ins_pipe(pipe_slow);
12505 ins_alignment(4);
12506 %}
12507
12508 // Call Runtime Instruction
12509 instruct CallRuntimeDirect(method meth)
12510 %{
12511 match(CallRuntime);
12512 effect(USE meth);
12513
12514 ins_cost(300);
12515 format %{ "call,runtime " %}
12516 ins_encode(clear_avx, Java_To_Runtime(meth));
12517 ins_pipe(pipe_slow);
12518 %}
12519
12520 // Call runtime without safepoint
12521 instruct CallLeafDirect(method meth)
12522 %{
12523 match(CallLeaf);
12524 effect(USE meth);
12525
12526 ins_cost(300);
12527 format %{ "call_leaf,runtime " %}
12528 ins_encode(clear_avx, Java_To_Runtime(meth));
12529 ins_pipe(pipe_slow);
12530 %}
12531
12532 // Call runtime without safepoint and with vector arguments
12533 instruct CallLeafDirectVector(method meth)
12534 %{
12535 match(CallLeafVector);
12536 effect(USE meth);
12537
12538 ins_cost(300);
12539 format %{ "call_leaf,vector " %}
12540 ins_encode(Java_To_Runtime(meth));
12541 ins_pipe(pipe_slow);
12542 %}
12543
12544 // Call runtime without safepoint
12545 instruct CallLeafNoFPDirect(method meth)
12546 %{
12547 match(CallLeafNoFP);
12548 effect(USE meth);
12549
12550 ins_cost(300);
12551 format %{ "call_leaf_nofp,runtime " %}
12552 ins_encode(clear_avx, Java_To_Runtime(meth));
12553 ins_pipe(pipe_slow);
12554 %}
12555
12556 // Return Instruction
12557 // Remove the return address & jump to it.
12558 // Notice: We always emit a nop after a ret to make sure there is room
12559 // for safepoint patching
12560 instruct Ret()
12561 %{
12562 match(Return);
12563
12564 format %{ "ret" %}
12565 ins_encode %{
12566 __ ret(0);
12567 %}
12568 ins_pipe(pipe_jmp);
12569 %}
12570
12571 // Tail Call; Jump from runtime stub to Java code.
12572 // Also known as an 'interprocedural jump'.
12573 // Target of jump will eventually return to caller.
12574 // TailJump below removes the return address.
12575 // Don't use rbp for 'jump_target' because a MachEpilogNode has already been
12576 // emitted just above the TailCall which has reset rbp to the caller state.
12577 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_ptr)
12578 %{
12579 match(TailCall jump_target method_ptr);
12580
12581 ins_cost(300);
12582 format %{ "jmp $jump_target\t# rbx holds method" %}
12583 ins_encode %{
12584 __ jmp($jump_target$$Register);
12585 %}
12586 ins_pipe(pipe_jmp);
12587 %}
12588
12589 // Tail Jump; remove the return address; jump to target.
12590 // TailCall above leaves the return address around.
12591 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
12592 %{
12593 match(TailJump jump_target ex_oop);
12594
12595 ins_cost(300);
12596 format %{ "popq rdx\t# pop return address\n\t"
12597 "jmp $jump_target" %}
12598 ins_encode %{
12599 __ popq(as_Register(RDX_enc));
12600 __ jmp($jump_target$$Register);
12601 %}
12602 ins_pipe(pipe_jmp);
12603 %}
12604
12605 // Forward exception.
12606 instruct ForwardExceptionjmp()
12607 %{
12608 match(ForwardException);
12609
12610 format %{ "jmp forward_exception_stub" %}
12611 ins_encode %{
12612 __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()), noreg);
12613 %}
12614 ins_pipe(pipe_jmp);
12615 %}
12616
12617 // Create exception oop: created by stack-crawling runtime code.
12618 // Created exception is now available to this handler, and is setup
12619 // just prior to jumping to this handler. No code emitted.
12620 instruct CreateException(rax_RegP ex_oop)
12621 %{
12622 match(Set ex_oop (CreateEx));
12623
12624 size(0);
12625 // use the following format syntax
12626 format %{ "# exception oop is in rax; no code emitted" %}
12627 ins_encode();
12628 ins_pipe(empty);
12629 %}
12630
12631 // Rethrow exception:
12632 // The exception oop will come in the first argument position.
12633 // Then JUMP (not call) to the rethrow stub code.
12634 instruct RethrowException()
12635 %{
12636 match(Rethrow);
12637
12638 // use the following format syntax
12639 format %{ "jmp rethrow_stub" %}
12640 ins_encode %{
12641 __ jump(RuntimeAddress(OptoRuntime::rethrow_stub()), noreg);
12642 %}
12643 ins_pipe(pipe_jmp);
12644 %}
12645
12646 // ============================================================================
12647 // This name is KNOWN by the ADLC and cannot be changed.
12648 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
12649 // for this guy.
12650 instruct tlsLoadP(r15_RegP dst) %{
12651 match(Set dst (ThreadLocal));
12652 effect(DEF dst);
12653
12654 size(0);
12655 format %{ "# TLS is in R15" %}
12656 ins_encode( /*empty encoding*/ );
12657 ins_pipe(ialu_reg_reg);
12658 %}
12659
12660
12661 //----------PEEPHOLE RULES-----------------------------------------------------
12662 // These must follow all instruction definitions as they use the names
12663 // defined in the instructions definitions.
12664 //
12665 // peeppredicate ( rule_predicate );
12666 // // the predicate unless which the peephole rule will be ignored
12667 //
12668 // peepmatch ( root_instr_name [preceding_instruction]* );
12669 //
12670 // peepprocedure ( procedure_name );
12671 // // provide a procedure name to perform the optimization, the procedure should
12672 // // reside in the architecture dependent peephole file, the method has the
12673 // // signature of MachNode* (Block*, int, PhaseRegAlloc*, (MachNode*)(*)(), int...)
12674 // // with the arguments being the basic block, the current node index inside the
12675 // // block, the register allocator, the functions upon invoked return a new node
12676 // // defined in peepreplace, and the rules of the nodes appearing in the
12677 // // corresponding peepmatch, the function return true if successful, else
12678 // // return false
12679 //
12680 // peepconstraint %{
12681 // (instruction_number.operand_name relational_op instruction_number.operand_name
12682 // [, ...] );
12683 // // instruction numbers are zero-based using left to right order in peepmatch
12684 //
12685 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
12686 // // provide an instruction_number.operand_name for each operand that appears
12687 // // in the replacement instruction's match rule
12688 //
12689 // ---------VM FLAGS---------------------------------------------------------
12690 //
12691 // All peephole optimizations can be turned off using -XX:-OptoPeephole
12692 //
12693 // Each peephole rule is given an identifying number starting with zero and
12694 // increasing by one in the order seen by the parser. An individual peephole
12695 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
12696 // on the command-line.
12697 //
12698 // ---------CURRENT LIMITATIONS----------------------------------------------
12699 //
12700 // Only transformations inside a basic block (do we need more for peephole)
12701 //
12702 // ---------EXAMPLE----------------------------------------------------------
12703 //
12704 // // pertinent parts of existing instructions in architecture description
12705 // instruct movI(rRegI dst, rRegI src)
12706 // %{
12707 // match(Set dst (CopyI src));
12708 // %}
12709 //
12710 // instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
12711 // %{
12712 // match(Set dst (AddI dst src));
12713 // effect(KILL cr);
12714 // %}
12715 //
12716 // instruct leaI_rReg_immI(rRegI dst, immI_1 src)
12717 // %{
12718 // match(Set dst (AddI dst src));
12719 // %}
12720 //
12721 // 1. Simple replacement
12722 // - Only match adjacent instructions in same basic block
12723 // - Only equality constraints
12724 // - Only constraints between operands, not (0.dest_reg == RAX_enc)
12725 // - Only one replacement instruction
12726 //
12727 // // Change (inc mov) to lea
12728 // peephole %{
12729 // // lea should only be emitted when beneficial
12730 // peeppredicate( VM_Version::supports_fast_2op_lea() );
12731 // // increment preceded by register-register move
12732 // peepmatch ( incI_rReg movI );
12733 // // require that the destination register of the increment
12734 // // match the destination register of the move
12735 // peepconstraint ( 0.dst == 1.dst );
12736 // // construct a replacement instruction that sets
12737 // // the destination to ( move's source register + one )
12738 // peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
12739 // %}
12740 //
12741 // 2. Procedural replacement
12742 // - More flexible finding relevent nodes
12743 // - More flexible constraints
12744 // - More flexible transformations
12745 // - May utilise architecture-dependent API more effectively
12746 // - Currently only one replacement instruction due to adlc parsing capabilities
12747 //
12748 // // Change (inc mov) to lea
12749 // peephole %{
12750 // // lea should only be emitted when beneficial
12751 // peeppredicate( VM_Version::supports_fast_2op_lea() );
12752 // // the rule numbers of these nodes inside are passed into the function below
12753 // peepmatch ( incI_rReg movI );
12754 // // the method that takes the responsibility of transformation
12755 // peepprocedure ( inc_mov_to_lea );
12756 // // the replacement is a leaI_rReg_immI, a lambda upon invoked creating this
12757 // // node is passed into the function above
12758 // peepreplace ( leaI_rReg_immI() );
12759 // %}
12760
12761 // These instructions is not matched by the matcher but used by the peephole
12762 instruct leaI_rReg_rReg_peep(rRegI dst, rRegI src1, rRegI src2)
12763 %{
12764 predicate(false);
12765 match(Set dst (AddI src1 src2));
12766 format %{ "leal $dst, [$src1 + $src2]" %}
12767 ins_encode %{
12768 Register dst = $dst$$Register;
12769 Register src1 = $src1$$Register;
12770 Register src2 = $src2$$Register;
12771 if (src1 != rbp && src1 != r13) {
12772 __ leal(dst, Address(src1, src2, Address::times_1));
12773 } else {
12774 assert(src2 != rbp && src2 != r13, "");
12775 __ leal(dst, Address(src2, src1, Address::times_1));
12776 }
12777 %}
12778 ins_pipe(ialu_reg_reg);
12779 %}
12780
12781 instruct leaI_rReg_immI_peep(rRegI dst, rRegI src1, immI src2)
12782 %{
12783 predicate(false);
12784 match(Set dst (AddI src1 src2));
12785 format %{ "leal $dst, [$src1 + $src2]" %}
12786 ins_encode %{
12787 __ leal($dst$$Register, Address($src1$$Register, $src2$$constant));
12788 %}
12789 ins_pipe(ialu_reg_reg);
12790 %}
12791
12792 instruct leaI_rReg_immI2_peep(rRegI dst, rRegI src, immI2 shift)
12793 %{
12794 predicate(false);
12795 match(Set dst (LShiftI src shift));
12796 format %{ "leal $dst, [$src << $shift]" %}
12797 ins_encode %{
12798 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
12799 Register src = $src$$Register;
12800 if (scale == Address::times_2 && src != rbp && src != r13) {
12801 __ leal($dst$$Register, Address(src, src, Address::times_1));
12802 } else {
12803 __ leal($dst$$Register, Address(noreg, src, scale));
12804 }
12805 %}
12806 ins_pipe(ialu_reg_reg);
12807 %}
12808
12809 instruct leaL_rReg_rReg_peep(rRegL dst, rRegL src1, rRegL src2)
12810 %{
12811 predicate(false);
12812 match(Set dst (AddL src1 src2));
12813 format %{ "leaq $dst, [$src1 + $src2]" %}
12814 ins_encode %{
12815 Register dst = $dst$$Register;
12816 Register src1 = $src1$$Register;
12817 Register src2 = $src2$$Register;
12818 if (src1 != rbp && src1 != r13) {
12819 __ leaq(dst, Address(src1, src2, Address::times_1));
12820 } else {
12821 assert(src2 != rbp && src2 != r13, "");
12822 __ leaq(dst, Address(src2, src1, Address::times_1));
12823 }
12824 %}
12825 ins_pipe(ialu_reg_reg);
12826 %}
12827
12828 instruct leaL_rReg_immL32_peep(rRegL dst, rRegL src1, immL32 src2)
12829 %{
12830 predicate(false);
12831 match(Set dst (AddL src1 src2));
12832 format %{ "leaq $dst, [$src1 + $src2]" %}
12833 ins_encode %{
12834 __ leaq($dst$$Register, Address($src1$$Register, $src2$$constant));
12835 %}
12836 ins_pipe(ialu_reg_reg);
12837 %}
12838
12839 instruct leaL_rReg_immI2_peep(rRegL dst, rRegL src, immI2 shift)
12840 %{
12841 predicate(false);
12842 match(Set dst (LShiftL src shift));
12843 format %{ "leaq $dst, [$src << $shift]" %}
12844 ins_encode %{
12845 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
12846 Register src = $src$$Register;
12847 if (scale == Address::times_2 && src != rbp && src != r13) {
12848 __ leaq($dst$$Register, Address(src, src, Address::times_1));
12849 } else {
12850 __ leaq($dst$$Register, Address(noreg, src, scale));
12851 }
12852 %}
12853 ins_pipe(ialu_reg_reg);
12854 %}
12855
12856 // These peephole rules replace mov + I pairs (where I is one of {add, inc, dec,
12857 // sal}) with lea instructions. The {add, sal} rules are beneficial in
12858 // processors with at least partial ALU support for lea
12859 // (supports_fast_2op_lea()), whereas the {inc, dec} rules are only generally
12860 // beneficial for processors with full ALU support
12861 // (VM_Version::supports_fast_3op_lea()) and Intel Cascade Lake.
12862
12863 peephole
12864 %{
12865 peeppredicate(VM_Version::supports_fast_2op_lea());
12866 peepmatch (addI_rReg);
12867 peepprocedure (lea_coalesce_reg);
12868 peepreplace (leaI_rReg_rReg_peep());
12869 %}
12870
12871 peephole
12872 %{
12873 peeppredicate(VM_Version::supports_fast_2op_lea());
12874 peepmatch (addI_rReg_imm);
12875 peepprocedure (lea_coalesce_imm);
12876 peepreplace (leaI_rReg_immI_peep());
12877 %}
12878
12879 peephole
12880 %{
12881 peeppredicate(VM_Version::supports_fast_3op_lea() ||
12882 VM_Version::is_intel_cascade_lake());
12883 peepmatch (incI_rReg);
12884 peepprocedure (lea_coalesce_imm);
12885 peepreplace (leaI_rReg_immI_peep());
12886 %}
12887
12888 peephole
12889 %{
12890 peeppredicate(VM_Version::supports_fast_3op_lea() ||
12891 VM_Version::is_intel_cascade_lake());
12892 peepmatch (decI_rReg);
12893 peepprocedure (lea_coalesce_imm);
12894 peepreplace (leaI_rReg_immI_peep());
12895 %}
12896
12897 peephole
12898 %{
12899 peeppredicate(VM_Version::supports_fast_2op_lea());
12900 peepmatch (salI_rReg_immI2);
12901 peepprocedure (lea_coalesce_imm);
12902 peepreplace (leaI_rReg_immI2_peep());
12903 %}
12904
12905 peephole
12906 %{
12907 peeppredicate(VM_Version::supports_fast_2op_lea());
12908 peepmatch (addL_rReg);
12909 peepprocedure (lea_coalesce_reg);
12910 peepreplace (leaL_rReg_rReg_peep());
12911 %}
12912
12913 peephole
12914 %{
12915 peeppredicate(VM_Version::supports_fast_2op_lea());
12916 peepmatch (addL_rReg_imm);
12917 peepprocedure (lea_coalesce_imm);
12918 peepreplace (leaL_rReg_immL32_peep());
12919 %}
12920
12921 peephole
12922 %{
12923 peeppredicate(VM_Version::supports_fast_3op_lea() ||
12924 VM_Version::is_intel_cascade_lake());
12925 peepmatch (incL_rReg);
12926 peepprocedure (lea_coalesce_imm);
12927 peepreplace (leaL_rReg_immL32_peep());
12928 %}
12929
12930 peephole
12931 %{
12932 peeppredicate(VM_Version::supports_fast_3op_lea() ||
12933 VM_Version::is_intel_cascade_lake());
12934 peepmatch (decL_rReg);
12935 peepprocedure (lea_coalesce_imm);
12936 peepreplace (leaL_rReg_immL32_peep());
12937 %}
12938
12939 peephole
12940 %{
12941 peeppredicate(VM_Version::supports_fast_2op_lea());
12942 peepmatch (salL_rReg_immI2);
12943 peepprocedure (lea_coalesce_imm);
12944 peepreplace (leaL_rReg_immI2_peep());
12945 %}
12946
12947 // These peephole rules matches instructions which set flags and are followed by a testI/L_reg
12948 // The test instruction is redudanent in case the downstream instuctions (like JCC or CMOV) only use flags that are already set by the previous instruction
12949
12950 //int variant
12951 peephole
12952 %{
12953 peepmatch (testI_reg);
12954 peepprocedure (test_may_remove);
12955 %}
12956
12957 //long variant
12958 peephole
12959 %{
12960 peepmatch (testL_reg);
12961 peepprocedure (test_may_remove);
12962 %}
12963
12964
12965 //----------SMARTSPILL RULES---------------------------------------------------
12966 // These must follow all instruction definitions as they use the names
12967 // defined in the instructions definitions.