1 /*
2 * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #ifndef CPU_AARCH64_MACROASSEMBLER_AARCH64_HPP
27 #define CPU_AARCH64_MACROASSEMBLER_AARCH64_HPP
28
29 #include "asm/assembler.inline.hpp"
30 #include "oops/compressedOops.hpp"
31 #include "runtime/vm_version.hpp"
32 #include "utilities/powerOfTwo.hpp"
33
34 // MacroAssembler extends Assembler by frequently used macros.
35 //
36 // Instructions for which a 'better' code sequence exists depending
37 // on arguments should also go in here.
38
39 class MacroAssembler: public Assembler {
40 friend class LIR_Assembler;
41
42 public:
43 using Assembler::mov;
44 using Assembler::movi;
45
46 protected:
47
48 // Support for VM calls
49 //
50 // This is the base routine called by the different versions of call_VM_leaf. The interpreter
51 // may customize this version by overriding it for its purposes (e.g., to save/restore
52 // additional registers when doing a VM call).
53 virtual void call_VM_leaf_base(
54 address entry_point, // the entry point
55 int number_of_arguments, // the number of arguments to pop after the call
56 Label *retaddr = NULL
57 );
58
59 virtual void call_VM_leaf_base(
60 address entry_point, // the entry point
61 int number_of_arguments, // the number of arguments to pop after the call
62 Label &retaddr) {
63 call_VM_leaf_base(entry_point, number_of_arguments, &retaddr);
64 }
65
66 // This is the base routine called by the different versions of call_VM. The interpreter
67 // may customize this version by overriding it for its purposes (e.g., to save/restore
68 // additional registers when doing a VM call).
69 //
70 // If no java_thread register is specified (noreg) than rthread will be used instead. call_VM_base
71 // returns the register which contains the thread upon return. If a thread register has been
72 // specified, the return value will correspond to that register. If no last_java_sp is specified
73 // (noreg) than rsp will be used instead.
74 virtual void call_VM_base( // returns the register containing the thread upon return
75 Register oop_result, // where an oop-result ends up if any; use noreg otherwise
76 Register java_thread, // the thread if computed before ; use noreg otherwise
77 Register last_java_sp, // to set up last_Java_frame in stubs; use noreg otherwise
78 address entry_point, // the entry point
79 int number_of_arguments, // the number of arguments (w/o thread) to pop after the call
80 bool check_exceptions // whether to check for pending exceptions after return
81 );
82
83 void call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions = true);
84
85 enum KlassDecodeMode {
86 KlassDecodeNone,
87 KlassDecodeZero,
88 KlassDecodeXor,
89 KlassDecodeMovk
90 };
91
92 KlassDecodeMode klass_decode_mode();
93
94 private:
95 static KlassDecodeMode _klass_decode_mode;
96
97 public:
98 MacroAssembler(CodeBuffer* code) : Assembler(code) {}
99
100 // These routines should emit JVMTI PopFrame and ForceEarlyReturn handling code.
101 // The implementation is only non-empty for the InterpreterMacroAssembler,
102 // as only the interpreter handles PopFrame and ForceEarlyReturn requests.
103 virtual void check_and_handle_popframe(Register java_thread);
104 virtual void check_and_handle_earlyret(Register java_thread);
105
106 void safepoint_poll(Label& slow_path, bool at_return, bool acquire, bool in_nmethod);
107
108 // Biased locking support
109 // lock_reg and obj_reg must be loaded up with the appropriate values.
110 // swap_reg is killed.
111 // tmp_reg must be supplied and must not be rscratch1 or rscratch2
112 // Optional slow case is for implementations (interpreter and C1) which branch to
113 // slow case directly. Leaves condition codes set for C2's Fast_Lock node.
114 void biased_locking_enter(Register lock_reg, Register obj_reg,
115 Register swap_reg, Register tmp_reg,
116 bool swap_reg_contains_mark,
117 Label& done, Label* slow_case = NULL,
118 BiasedLockingCounters* counters = NULL);
119 void biased_locking_exit (Register obj_reg, Register temp_reg, Label& done);
120
121
122 // Helper functions for statistics gathering.
123 // Unconditional atomic increment.
124 void atomic_incw(Register counter_addr, Register tmp, Register tmp2);
125 void atomic_incw(Address counter_addr, Register tmp1, Register tmp2, Register tmp3) {
126 lea(tmp1, counter_addr);
127 atomic_incw(tmp1, tmp2, tmp3);
128 }
129 // Load Effective Address
130 void lea(Register r, const Address &a) {
131 InstructionMark im(this);
132 code_section()->relocate(inst_mark(), a.rspec());
133 a.lea(this, r);
134 }
135
136 /* Sometimes we get misaligned loads and stores, usually from Unsafe
137 accesses, and these can exceed the offset range. */
138 Address legitimize_address(const Address &a, int size, Register scratch) {
139 if (a.getMode() == Address::base_plus_offset) {
140 if (! Address::offset_ok_for_immed(a.offset(), exact_log2(size))) {
141 block_comment("legitimize_address {");
142 lea(scratch, a);
143 block_comment("} legitimize_address");
144 return Address(scratch);
145 }
146 }
147 return a;
148 }
149
150 void addmw(Address a, Register incr, Register scratch) {
151 ldrw(scratch, a);
152 addw(scratch, scratch, incr);
153 strw(scratch, a);
154 }
155
156 // Add constant to memory word
157 void addmw(Address a, int imm, Register scratch) {
158 ldrw(scratch, a);
159 if (imm > 0)
160 addw(scratch, scratch, (unsigned)imm);
161 else
162 subw(scratch, scratch, (unsigned)-imm);
163 strw(scratch, a);
164 }
165
166 void bind(Label& L) {
167 Assembler::bind(L);
168 code()->clear_last_insn();
169 }
170
171 void membar(Membar_mask_bits order_constraint);
172
173 using Assembler::ldr;
174 using Assembler::str;
175 using Assembler::ldrw;
176 using Assembler::strw;
177
178 void ldr(Register Rx, const Address &adr);
179 void ldrw(Register Rw, const Address &adr);
180 void str(Register Rx, const Address &adr);
181 void strw(Register Rx, const Address &adr);
182
183 // Frame creation and destruction shared between JITs.
184 void build_frame(int framesize);
185 void remove_frame(int framesize);
186
187 virtual void _call_Unimplemented(address call_site) {
188 mov(rscratch2, call_site);
189 }
190
191 // Microsoft's MSVC team thinks that the __FUNCSIG__ is approximately (sympathy for calling conventions) equivalent to __PRETTY_FUNCTION__
192 // Also, from Clang patch: "It is very similar to GCC's PRETTY_FUNCTION, except it prints the calling convention."
193 // https://reviews.llvm.org/D3311
194
195 #ifdef _WIN64
196 #define call_Unimplemented() _call_Unimplemented((address)__FUNCSIG__)
197 #else
198 #define call_Unimplemented() _call_Unimplemented((address)__PRETTY_FUNCTION__)
199 #endif
200
201 // aliases defined in AARCH64 spec
202
203 template<class T>
204 inline void cmpw(Register Rd, T imm) { subsw(zr, Rd, imm); }
205
206 inline void cmp(Register Rd, unsigned char imm8) { subs(zr, Rd, imm8); }
207 inline void cmp(Register Rd, unsigned imm) = delete;
208
209 template<class T>
210 inline void cmnw(Register Rd, T imm) { addsw(zr, Rd, imm); }
211
212 inline void cmn(Register Rd, unsigned char imm8) { adds(zr, Rd, imm8); }
213 inline void cmn(Register Rd, unsigned imm) = delete;
214
215 void cset(Register Rd, Assembler::Condition cond) {
216 csinc(Rd, zr, zr, ~cond);
217 }
218 void csetw(Register Rd, Assembler::Condition cond) {
219 csincw(Rd, zr, zr, ~cond);
220 }
221
222 void cneg(Register Rd, Register Rn, Assembler::Condition cond) {
223 csneg(Rd, Rn, Rn, ~cond);
224 }
225 void cnegw(Register Rd, Register Rn, Assembler::Condition cond) {
226 csnegw(Rd, Rn, Rn, ~cond);
227 }
228
229 inline void movw(Register Rd, Register Rn) {
230 if (Rd == sp || Rn == sp) {
231 Assembler::addw(Rd, Rn, 0U);
232 } else {
233 orrw(Rd, zr, Rn);
234 }
235 }
236 inline void mov(Register Rd, Register Rn) {
237 assert(Rd != r31_sp && Rn != r31_sp, "should be");
238 if (Rd == Rn) {
239 } else if (Rd == sp || Rn == sp) {
240 Assembler::add(Rd, Rn, 0U);
241 } else {
242 orr(Rd, zr, Rn);
243 }
244 }
245
246 inline void moviw(Register Rd, unsigned imm) { orrw(Rd, zr, imm); }
247 inline void movi(Register Rd, unsigned imm) { orr(Rd, zr, imm); }
248
249 inline void tstw(Register Rd, Register Rn) { andsw(zr, Rd, Rn); }
250 inline void tst(Register Rd, Register Rn) { ands(zr, Rd, Rn); }
251
252 inline void tstw(Register Rd, uint64_t imm) { andsw(zr, Rd, imm); }
253 inline void tst(Register Rd, uint64_t imm) { ands(zr, Rd, imm); }
254
255 inline void bfiw(Register Rd, Register Rn, unsigned lsb, unsigned width) {
256 bfmw(Rd, Rn, ((32 - lsb) & 31), (width - 1));
257 }
258 inline void bfi(Register Rd, Register Rn, unsigned lsb, unsigned width) {
259 bfm(Rd, Rn, ((64 - lsb) & 63), (width - 1));
260 }
261
262 inline void bfxilw(Register Rd, Register Rn, unsigned lsb, unsigned width) {
263 bfmw(Rd, Rn, lsb, (lsb + width - 1));
264 }
265 inline void bfxil(Register Rd, Register Rn, unsigned lsb, unsigned width) {
266 bfm(Rd, Rn, lsb , (lsb + width - 1));
267 }
268
269 inline void sbfizw(Register Rd, Register Rn, unsigned lsb, unsigned width) {
270 sbfmw(Rd, Rn, ((32 - lsb) & 31), (width - 1));
271 }
272 inline void sbfiz(Register Rd, Register Rn, unsigned lsb, unsigned width) {
273 sbfm(Rd, Rn, ((64 - lsb) & 63), (width - 1));
274 }
275
276 inline void sbfxw(Register Rd, Register Rn, unsigned lsb, unsigned width) {
277 sbfmw(Rd, Rn, lsb, (lsb + width - 1));
278 }
279 inline void sbfx(Register Rd, Register Rn, unsigned lsb, unsigned width) {
280 sbfm(Rd, Rn, lsb , (lsb + width - 1));
281 }
282
283 inline void ubfizw(Register Rd, Register Rn, unsigned lsb, unsigned width) {
284 ubfmw(Rd, Rn, ((32 - lsb) & 31), (width - 1));
285 }
286 inline void ubfiz(Register Rd, Register Rn, unsigned lsb, unsigned width) {
287 ubfm(Rd, Rn, ((64 - lsb) & 63), (width - 1));
288 }
289
290 inline void ubfxw(Register Rd, Register Rn, unsigned lsb, unsigned width) {
291 ubfmw(Rd, Rn, lsb, (lsb + width - 1));
292 }
293 inline void ubfx(Register Rd, Register Rn, unsigned lsb, unsigned width) {
294 ubfm(Rd, Rn, lsb , (lsb + width - 1));
295 }
296
297 inline void asrw(Register Rd, Register Rn, unsigned imm) {
298 sbfmw(Rd, Rn, imm, 31);
299 }
300
301 inline void asr(Register Rd, Register Rn, unsigned imm) {
302 sbfm(Rd, Rn, imm, 63);
303 }
304
305 inline void lslw(Register Rd, Register Rn, unsigned imm) {
306 ubfmw(Rd, Rn, ((32 - imm) & 31), (31 - imm));
307 }
308
309 inline void lsl(Register Rd, Register Rn, unsigned imm) {
310 ubfm(Rd, Rn, ((64 - imm) & 63), (63 - imm));
311 }
312
313 inline void lsrw(Register Rd, Register Rn, unsigned imm) {
314 ubfmw(Rd, Rn, imm, 31);
315 }
316
317 inline void lsr(Register Rd, Register Rn, unsigned imm) {
318 ubfm(Rd, Rn, imm, 63);
319 }
320
321 inline void rorw(Register Rd, Register Rn, unsigned imm) {
322 extrw(Rd, Rn, Rn, imm);
323 }
324
325 inline void ror(Register Rd, Register Rn, unsigned imm) {
326 extr(Rd, Rn, Rn, imm);
327 }
328
329 inline void sxtbw(Register Rd, Register Rn) {
330 sbfmw(Rd, Rn, 0, 7);
331 }
332 inline void sxthw(Register Rd, Register Rn) {
333 sbfmw(Rd, Rn, 0, 15);
334 }
335 inline void sxtb(Register Rd, Register Rn) {
336 sbfm(Rd, Rn, 0, 7);
337 }
338 inline void sxth(Register Rd, Register Rn) {
339 sbfm(Rd, Rn, 0, 15);
340 }
341 inline void sxtw(Register Rd, Register Rn) {
342 sbfm(Rd, Rn, 0, 31);
343 }
344
345 inline void uxtbw(Register Rd, Register Rn) {
346 ubfmw(Rd, Rn, 0, 7);
347 }
348 inline void uxthw(Register Rd, Register Rn) {
349 ubfmw(Rd, Rn, 0, 15);
350 }
351 inline void uxtb(Register Rd, Register Rn) {
352 ubfm(Rd, Rn, 0, 7);
353 }
354 inline void uxth(Register Rd, Register Rn) {
355 ubfm(Rd, Rn, 0, 15);
356 }
357 inline void uxtw(Register Rd, Register Rn) {
358 ubfm(Rd, Rn, 0, 31);
359 }
360
361 inline void cmnw(Register Rn, Register Rm) {
362 addsw(zr, Rn, Rm);
363 }
364 inline void cmn(Register Rn, Register Rm) {
365 adds(zr, Rn, Rm);
366 }
367
368 inline void cmpw(Register Rn, Register Rm) {
369 subsw(zr, Rn, Rm);
370 }
371 inline void cmp(Register Rn, Register Rm) {
372 subs(zr, Rn, Rm);
373 }
374
375 inline void negw(Register Rd, Register Rn) {
376 subw(Rd, zr, Rn);
377 }
378
379 inline void neg(Register Rd, Register Rn) {
380 sub(Rd, zr, Rn);
381 }
382
383 inline void negsw(Register Rd, Register Rn) {
384 subsw(Rd, zr, Rn);
385 }
386
387 inline void negs(Register Rd, Register Rn) {
388 subs(Rd, zr, Rn);
389 }
390
391 inline void cmnw(Register Rn, Register Rm, enum shift_kind kind, unsigned shift = 0) {
392 addsw(zr, Rn, Rm, kind, shift);
393 }
394 inline void cmn(Register Rn, Register Rm, enum shift_kind kind, unsigned shift = 0) {
395 adds(zr, Rn, Rm, kind, shift);
396 }
397
398 inline void cmpw(Register Rn, Register Rm, enum shift_kind kind, unsigned shift = 0) {
399 subsw(zr, Rn, Rm, kind, shift);
400 }
401 inline void cmp(Register Rn, Register Rm, enum shift_kind kind, unsigned shift = 0) {
402 subs(zr, Rn, Rm, kind, shift);
403 }
404
405 inline void negw(Register Rd, Register Rn, enum shift_kind kind, unsigned shift = 0) {
406 subw(Rd, zr, Rn, kind, shift);
407 }
408
409 inline void neg(Register Rd, Register Rn, enum shift_kind kind, unsigned shift = 0) {
410 sub(Rd, zr, Rn, kind, shift);
411 }
412
413 inline void negsw(Register Rd, Register Rn, enum shift_kind kind, unsigned shift = 0) {
414 subsw(Rd, zr, Rn, kind, shift);
415 }
416
417 inline void negs(Register Rd, Register Rn, enum shift_kind kind, unsigned shift = 0) {
418 subs(Rd, zr, Rn, kind, shift);
419 }
420
421 inline void mnegw(Register Rd, Register Rn, Register Rm) {
422 msubw(Rd, Rn, Rm, zr);
423 }
424 inline void mneg(Register Rd, Register Rn, Register Rm) {
425 msub(Rd, Rn, Rm, zr);
426 }
427
428 inline void mulw(Register Rd, Register Rn, Register Rm) {
429 maddw(Rd, Rn, Rm, zr);
430 }
431 inline void mul(Register Rd, Register Rn, Register Rm) {
432 madd(Rd, Rn, Rm, zr);
433 }
434
435 inline void smnegl(Register Rd, Register Rn, Register Rm) {
436 smsubl(Rd, Rn, Rm, zr);
437 }
438 inline void smull(Register Rd, Register Rn, Register Rm) {
439 smaddl(Rd, Rn, Rm, zr);
440 }
441
442 inline void umnegl(Register Rd, Register Rn, Register Rm) {
443 umsubl(Rd, Rn, Rm, zr);
444 }
445 inline void umull(Register Rd, Register Rn, Register Rm) {
446 umaddl(Rd, Rn, Rm, zr);
447 }
448
449 #define WRAP(INSN) \
450 void INSN(Register Rd, Register Rn, Register Rm, Register Ra) { \
451 if ((VM_Version::features() & VM_Version::CPU_A53MAC) && Ra != zr) \
452 nop(); \
453 Assembler::INSN(Rd, Rn, Rm, Ra); \
454 }
455
456 WRAP(madd) WRAP(msub) WRAP(maddw) WRAP(msubw)
457 WRAP(smaddl) WRAP(smsubl) WRAP(umaddl) WRAP(umsubl)
458 #undef WRAP
459
460
461 // macro assembly operations needed for aarch64
462
463 // first two private routines for loading 32 bit or 64 bit constants
464 private:
465
466 void mov_immediate64(Register dst, uint64_t imm64);
467 void mov_immediate32(Register dst, uint32_t imm32);
468
469 int push(unsigned int bitset, Register stack);
470 int pop(unsigned int bitset, Register stack);
471
472 int push_fp(unsigned int bitset, Register stack);
473 int pop_fp(unsigned int bitset, Register stack);
474
475 void mov(Register dst, Address a);
476
477 public:
478 void push(RegSet regs, Register stack) { if (regs.bits()) push(regs.bits(), stack); }
479 void pop(RegSet regs, Register stack) { if (regs.bits()) pop(regs.bits(), stack); }
480
481 void push_fp(FloatRegSet regs, Register stack) { if (regs.bits()) push_fp(regs.bits(), stack); }
482 void pop_fp(FloatRegSet regs, Register stack) { if (regs.bits()) pop_fp(regs.bits(), stack); }
483
484 static RegSet call_clobbered_registers();
485
486 // Push and pop everything that might be clobbered by a native
487 // runtime call except rscratch1 and rscratch2. (They are always
488 // scratch, so we don't have to protect them.) Only save the lower
489 // 64 bits of each vector register. Additonal registers can be excluded
490 // in a passed RegSet.
491 void push_call_clobbered_registers_except(RegSet exclude);
492 void pop_call_clobbered_registers_except(RegSet exclude);
493
494 void push_call_clobbered_registers() {
495 push_call_clobbered_registers_except(RegSet());
496 }
497 void pop_call_clobbered_registers() {
498 pop_call_clobbered_registers_except(RegSet());
499 }
500
501
502 // now mov instructions for loading absolute addresses and 32 or
503 // 64 bit integers
504
505 inline void mov(Register dst, address addr) { mov_immediate64(dst, (uint64_t)addr); }
506
507 inline void mov(Register dst, int imm64) { mov_immediate64(dst, (uint64_t)imm64); }
508 inline void mov(Register dst, long imm64) { mov_immediate64(dst, (uint64_t)imm64); }
509 inline void mov(Register dst, long long imm64) { mov_immediate64(dst, (uint64_t)imm64); }
510 inline void mov(Register dst, unsigned int imm64) { mov_immediate64(dst, (uint64_t)imm64); }
511 inline void mov(Register dst, unsigned long imm64) { mov_immediate64(dst, (uint64_t)imm64); }
512 inline void mov(Register dst, unsigned long long imm64) { mov_immediate64(dst, (uint64_t)imm64); }
513
514 inline void movw(Register dst, uint32_t imm32)
515 {
516 mov_immediate32(dst, imm32);
517 }
518
519 void mov(Register dst, RegisterOrConstant src) {
520 if (src.is_register())
521 mov(dst, src.as_register());
522 else
523 mov(dst, src.as_constant());
524 }
525
526 void movptr(Register r, uintptr_t imm64);
527
528 void mov(FloatRegister Vd, SIMD_Arrangement T, uint64_t imm64);
529
530 void mov(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn) {
531 orr(Vd, T, Vn, Vn);
532 }
533
534
535 public:
536
537 // Generalized Test Bit And Branch, including a "far" variety which
538 // spans more than 32KiB.
539 void tbr(Condition cond, Register Rt, int bitpos, Label &dest, bool isfar = false) {
540 assert(cond == EQ || cond == NE, "must be");
541
542 if (isfar)
543 cond = ~cond;
544
545 void (Assembler::* branch)(Register Rt, int bitpos, Label &L);
546 if (cond == Assembler::EQ)
547 branch = &Assembler::tbz;
548 else
549 branch = &Assembler::tbnz;
550
551 if (isfar) {
552 Label L;
553 (this->*branch)(Rt, bitpos, L);
554 b(dest);
555 bind(L);
556 } else {
557 (this->*branch)(Rt, bitpos, dest);
558 }
559 }
560
561 // macro instructions for accessing and updating floating point
562 // status register
563 //
564 // FPSR : op1 == 011
565 // CRn == 0100
566 // CRm == 0100
567 // op2 == 001
568
569 inline void get_fpsr(Register reg)
570 {
571 mrs(0b11, 0b0100, 0b0100, 0b001, reg);
572 }
573
574 inline void set_fpsr(Register reg)
575 {
576 msr(0b011, 0b0100, 0b0100, 0b001, reg);
577 }
578
579 inline void clear_fpsr()
580 {
581 msr(0b011, 0b0100, 0b0100, 0b001, zr);
582 }
583
584 // DCZID_EL0: op1 == 011
585 // CRn == 0000
586 // CRm == 0000
587 // op2 == 111
588 inline void get_dczid_el0(Register reg)
589 {
590 mrs(0b011, 0b0000, 0b0000, 0b111, reg);
591 }
592
593 // CTR_EL0: op1 == 011
594 // CRn == 0000
595 // CRm == 0000
596 // op2 == 001
597 inline void get_ctr_el0(Register reg)
598 {
599 mrs(0b011, 0b0000, 0b0000, 0b001, reg);
600 }
601
602 // idiv variant which deals with MINLONG as dividend and -1 as divisor
603 int corrected_idivl(Register result, Register ra, Register rb,
604 bool want_remainder, Register tmp = rscratch1);
605 int corrected_idivq(Register result, Register ra, Register rb,
606 bool want_remainder, Register tmp = rscratch1);
607
608 // Support for NULL-checks
609 //
610 // Generates code that causes a NULL OS exception if the content of reg is NULL.
611 // If the accessed location is M[reg + offset] and the offset is known, provide the
612 // offset. No explicit code generation is needed if the offset is within a certain
613 // range (0 <= offset <= page_size).
614
615 virtual void null_check(Register reg, int offset = -1);
616 static bool needs_explicit_null_check(intptr_t offset);
617 static bool uses_implicit_null_check(void* address);
618
619 static address target_addr_for_insn(address insn_addr, unsigned insn);
620 static address target_addr_for_insn(address insn_addr) {
621 unsigned insn = *(unsigned*)insn_addr;
622 return target_addr_for_insn(insn_addr, insn);
623 }
624
625 // Required platform-specific helpers for Label::patch_instructions.
626 // They _shadow_ the declarations in AbstractAssembler, which are undefined.
627 static int pd_patch_instruction_size(address branch, address target);
628 static void pd_patch_instruction(address branch, address target, const char* file = NULL, int line = 0) {
629 pd_patch_instruction_size(branch, target);
630 }
631 static address pd_call_destination(address branch) {
632 return target_addr_for_insn(branch);
633 }
634 #ifndef PRODUCT
635 static void pd_print_patched_instruction(address branch);
636 #endif
637
638 static int patch_oop(address insn_addr, address o);
639 static int patch_narrow_klass(address insn_addr, narrowKlass n);
640
641 address emit_trampoline_stub(int insts_call_instruction_offset, address target);
642 void emit_static_call_stub();
643
644 // The following 4 methods return the offset of the appropriate move instruction
645
646 // Support for fast byte/short loading with zero extension (depending on particular CPU)
647 int load_unsigned_byte(Register dst, Address src);
648 int load_unsigned_short(Register dst, Address src);
649
650 // Support for fast byte/short loading with sign extension (depending on particular CPU)
651 int load_signed_byte(Register dst, Address src);
652 int load_signed_short(Register dst, Address src);
653
654 int load_signed_byte32(Register dst, Address src);
655 int load_signed_short32(Register dst, Address src);
656
657 // Support for sign-extension (hi:lo = extend_sign(lo))
658 void extend_sign(Register hi, Register lo);
659
660 // Load and store values by size and signed-ness
661 void load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2 = noreg);
662 void store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2 = noreg);
663
664 // Support for inc/dec with optimal instruction selection depending on value
665
666 // x86_64 aliases an unqualified register/address increment and
667 // decrement to call incrementq and decrementq but also supports
668 // explicitly sized calls to incrementq/decrementq or
669 // incrementl/decrementl
670
671 // for aarch64 the proper convention would be to use
672 // increment/decrement for 64 bit operatons and
673 // incrementw/decrementw for 32 bit operations. so when porting
674 // x86_64 code we can leave calls to increment/decrement as is,
675 // replace incrementq/decrementq with increment/decrement and
676 // replace incrementl/decrementl with incrementw/decrementw.
677
678 // n.b. increment/decrement calls with an Address destination will
679 // need to use a scratch register to load the value to be
680 // incremented. increment/decrement calls which add or subtract a
681 // constant value greater than 2^12 will need to use a 2nd scratch
682 // register to hold the constant. so, a register increment/decrement
683 // may trash rscratch2 and an address increment/decrement trash
684 // rscratch and rscratch2
685
686 void decrementw(Address dst, int value = 1);
687 void decrementw(Register reg, int value = 1);
688
689 void decrement(Register reg, int value = 1);
690 void decrement(Address dst, int value = 1);
691
692 void incrementw(Address dst, int value = 1);
693 void incrementw(Register reg, int value = 1);
694
695 void increment(Register reg, int value = 1);
696 void increment(Address dst, int value = 1);
697
698
699 // Alignment
700 void align(int modulus);
701
702 // Stack frame creation/removal
703 void enter()
704 {
705 stp(rfp, lr, Address(pre(sp, -2 * wordSize)));
706 mov(rfp, sp);
707 }
708 void leave()
709 {
710 mov(sp, rfp);
711 ldp(rfp, lr, Address(post(sp, 2 * wordSize)));
712 }
713
714 // Support for getting the JavaThread pointer (i.e.; a reference to thread-local information)
715 // The pointer will be loaded into the thread register.
716 void get_thread(Register thread);
717
718
719 // Support for VM calls
720 //
721 // It is imperative that all calls into the VM are handled via the call_VM macros.
722 // They make sure that the stack linkage is setup correctly. call_VM's correspond
723 // to ENTRY/ENTRY_X entry points while call_VM_leaf's correspond to LEAF entry points.
724
725
726 void call_VM(Register oop_result,
727 address entry_point,
728 bool check_exceptions = true);
729 void call_VM(Register oop_result,
730 address entry_point,
731 Register arg_1,
732 bool check_exceptions = true);
733 void call_VM(Register oop_result,
734 address entry_point,
735 Register arg_1, Register arg_2,
736 bool check_exceptions = true);
737 void call_VM(Register oop_result,
738 address entry_point,
739 Register arg_1, Register arg_2, Register arg_3,
740 bool check_exceptions = true);
741
742 // Overloadings with last_Java_sp
743 void call_VM(Register oop_result,
744 Register last_java_sp,
745 address entry_point,
746 int number_of_arguments = 0,
747 bool check_exceptions = true);
748 void call_VM(Register oop_result,
749 Register last_java_sp,
750 address entry_point,
751 Register arg_1, bool
752 check_exceptions = true);
753 void call_VM(Register oop_result,
754 Register last_java_sp,
755 address entry_point,
756 Register arg_1, Register arg_2,
757 bool check_exceptions = true);
758 void call_VM(Register oop_result,
759 Register last_java_sp,
760 address entry_point,
761 Register arg_1, Register arg_2, Register arg_3,
762 bool check_exceptions = true);
763
764 void get_vm_result (Register oop_result, Register thread);
765 void get_vm_result_2(Register metadata_result, Register thread);
766
767 // These always tightly bind to MacroAssembler::call_VM_base
768 // bypassing the virtual implementation
769 void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, int number_of_arguments = 0, bool check_exceptions = true);
770 void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, bool check_exceptions = true);
771 void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, Register arg_2, bool check_exceptions = true);
772 void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, Register arg_2, Register arg_3, bool check_exceptions = true);
773 void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, Register arg_2, Register arg_3, Register arg_4, bool check_exceptions = true);
774
775 void call_VM_leaf(address entry_point,
776 int number_of_arguments = 0);
777 void call_VM_leaf(address entry_point,
778 Register arg_1);
779 void call_VM_leaf(address entry_point,
780 Register arg_1, Register arg_2);
781 void call_VM_leaf(address entry_point,
782 Register arg_1, Register arg_2, Register arg_3);
783
784 // These always tightly bind to MacroAssembler::call_VM_leaf_base
785 // bypassing the virtual implementation
786 void super_call_VM_leaf(address entry_point);
787 void super_call_VM_leaf(address entry_point, Register arg_1);
788 void super_call_VM_leaf(address entry_point, Register arg_1, Register arg_2);
789 void super_call_VM_leaf(address entry_point, Register arg_1, Register arg_2, Register arg_3);
790 void super_call_VM_leaf(address entry_point, Register arg_1, Register arg_2, Register arg_3, Register arg_4);
791
792 // last Java Frame (fills frame anchor)
793 void set_last_Java_frame(Register last_java_sp,
794 Register last_java_fp,
795 address last_java_pc,
796 Register scratch);
797
798 void set_last_Java_frame(Register last_java_sp,
799 Register last_java_fp,
800 Label &last_java_pc,
801 Register scratch);
802
803 void set_last_Java_frame(Register last_java_sp,
804 Register last_java_fp,
805 Register last_java_pc,
806 Register scratch);
807
808 void reset_last_Java_frame(Register thread);
809
810 // thread in the default location (rthread)
811 void reset_last_Java_frame(bool clear_fp);
812
813 // Stores
814 void store_check(Register obj); // store check for obj - register is destroyed afterwards
815 void store_check(Register obj, Address dst); // same as above, dst is exact store location (reg. is destroyed)
816
817 void resolve_jobject(Register value, Register thread, Register tmp);
818
819 // C 'boolean' to Java boolean: x == 0 ? 0 : 1
820 void c2bool(Register x);
821
822 void load_method_holder_cld(Register rresult, Register rmethod);
823 void load_method_holder(Register holder, Register method);
824
825 // oop manipulations
826 void load_klass(Register dst, Register src);
827 void store_klass(Register dst, Register src);
828 void cmp_klass(Register oop, Register trial_klass, Register tmp);
829
830 void resolve_weak_handle(Register result, Register tmp);
831 void resolve_oop_handle(Register result, Register tmp = r5);
832 void load_mirror(Register dst, Register method, Register tmp = r5);
833
834 void access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
835 Register tmp1, Register tmp_thread);
836
837 void access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register src,
838 Register tmp1, Register tmp_thread);
839
840 void load_heap_oop(Register dst, Address src, Register tmp1 = noreg,
841 Register thread_tmp = noreg, DecoratorSet decorators = 0);
842
843 void load_heap_oop_not_null(Register dst, Address src, Register tmp1 = noreg,
844 Register thread_tmp = noreg, DecoratorSet decorators = 0);
845 void store_heap_oop(Address dst, Register src, Register tmp1 = noreg,
846 Register tmp_thread = noreg, DecoratorSet decorators = 0);
847
848 // currently unimplemented
849 // Used for storing NULL. All other oop constants should be
850 // stored using routines that take a jobject.
851 void store_heap_oop_null(Address dst);
852
853 void load_prototype_header(Register dst, Register src);
854
855 void store_klass_gap(Register dst, Register src);
856
857 // This dummy is to prevent a call to store_heap_oop from
858 // converting a zero (like NULL) into a Register by giving
859 // the compiler two choices it can't resolve
860
861 void store_heap_oop(Address dst, void* dummy);
862
863 void encode_heap_oop(Register d, Register s);
864 void encode_heap_oop(Register r) { encode_heap_oop(r, r); }
865 void decode_heap_oop(Register d, Register s);
866 void decode_heap_oop(Register r) { decode_heap_oop(r, r); }
867 void encode_heap_oop_not_null(Register r);
868 void decode_heap_oop_not_null(Register r);
869 void encode_heap_oop_not_null(Register dst, Register src);
870 void decode_heap_oop_not_null(Register dst, Register src);
871
872 void set_narrow_oop(Register dst, jobject obj);
873
874 void encode_klass_not_null(Register r);
875 void decode_klass_not_null(Register r);
876 void encode_klass_not_null(Register dst, Register src);
877 void decode_klass_not_null(Register dst, Register src);
878
879 void set_narrow_klass(Register dst, Klass* k);
880
881 // if heap base register is used - reinit it with the correct value
882 void reinit_heapbase();
883
884 DEBUG_ONLY(void verify_heapbase(const char* msg);)
885
886 void push_CPU_state(bool save_vectors = false, bool use_sve = false,
887 int sve_vector_size_in_bytes = 0);
888 void pop_CPU_state(bool restore_vectors = false, bool use_sve = false,
889 int sve_vector_size_in_bytes = 0);
890
891 // Round up to a power of two
892 void round_to(Register reg, int modulus);
893
894 // allocation
895 void eden_allocate(
896 Register obj, // result: pointer to object after successful allocation
897 Register var_size_in_bytes, // object size in bytes if unknown at compile time; invalid otherwise
898 int con_size_in_bytes, // object size in bytes if known at compile time
899 Register t1, // temp register
900 Label& slow_case // continuation point if fast allocation fails
901 );
902 void tlab_allocate(
903 Register obj, // result: pointer to object after successful allocation
904 Register var_size_in_bytes, // object size in bytes if unknown at compile time; invalid otherwise
905 int con_size_in_bytes, // object size in bytes if known at compile time
906 Register t1, // temp register
907 Register t2, // temp register
908 Label& slow_case // continuation point if fast allocation fails
909 );
910 void verify_tlab();
911
912 // interface method calling
913 void lookup_interface_method(Register recv_klass,
914 Register intf_klass,
915 RegisterOrConstant itable_index,
916 Register method_result,
917 Register scan_temp,
918 Label& no_such_interface,
919 bool return_method = true);
920
921 // virtual method calling
922 // n.b. x86 allows RegisterOrConstant for vtable_index
923 void lookup_virtual_method(Register recv_klass,
924 RegisterOrConstant vtable_index,
925 Register method_result);
926
927 // Test sub_klass against super_klass, with fast and slow paths.
928
929 // The fast path produces a tri-state answer: yes / no / maybe-slow.
930 // One of the three labels can be NULL, meaning take the fall-through.
931 // If super_check_offset is -1, the value is loaded up from super_klass.
932 // No registers are killed, except temp_reg.
933 void check_klass_subtype_fast_path(Register sub_klass,
934 Register super_klass,
935 Register temp_reg,
936 Label* L_success,
937 Label* L_failure,
938 Label* L_slow_path,
939 RegisterOrConstant super_check_offset = RegisterOrConstant(-1));
940
941 // The rest of the type check; must be wired to a corresponding fast path.
942 // It does not repeat the fast path logic, so don't use it standalone.
943 // The temp_reg and temp2_reg can be noreg, if no temps are available.
944 // Updates the sub's secondary super cache as necessary.
945 // If set_cond_codes, condition codes will be Z on success, NZ on failure.
946 void check_klass_subtype_slow_path(Register sub_klass,
947 Register super_klass,
948 Register temp_reg,
949 Register temp2_reg,
950 Label* L_success,
951 Label* L_failure,
952 bool set_cond_codes = false);
953
954 // Simplified, combined version, good for typical uses.
955 // Falls through on failure.
956 void check_klass_subtype(Register sub_klass,
957 Register super_klass,
958 Register temp_reg,
959 Label& L_success);
960
961 void clinit_barrier(Register klass,
962 Register thread,
963 Label* L_fast_path = NULL,
964 Label* L_slow_path = NULL);
965
966 Address argument_address(RegisterOrConstant arg_slot, int extra_slot_offset = 0);
967
968 void verify_sve_vector_length();
969 void reinitialize_ptrue() {
970 if (UseSVE > 0) {
971 sve_ptrue(ptrue, B);
972 }
973 }
974 void verify_ptrue();
975
976 // Debugging
977
978 // only if +VerifyOops
979 void verify_oop(Register reg, const char* s = "broken oop");
980 void verify_oop_addr(Address addr, const char * s = "broken oop addr");
981
982 // TODO: verify method and klass metadata (compare against vptr?)
983 void _verify_method_ptr(Register reg, const char * msg, const char * file, int line) {}
984 void _verify_klass_ptr(Register reg, const char * msg, const char * file, int line){}
985
986 #define verify_method_ptr(reg) _verify_method_ptr(reg, "broken method " #reg, __FILE__, __LINE__)
987 #define verify_klass_ptr(reg) _verify_klass_ptr(reg, "broken klass " #reg, __FILE__, __LINE__)
988
989 // only if +VerifyFPU
990 void verify_FPU(int stack_depth, const char* s = "illegal FPU state");
991
992 // prints msg, dumps registers and stops execution
993 void stop(const char* msg);
994
995 static void debug64(char* msg, int64_t pc, int64_t regs[]);
996
997 void untested() { stop("untested"); }
998
999 void unimplemented(const char* what = "");
1000
1001 void should_not_reach_here() { stop("should not reach here"); }
1002
1003 // Stack overflow checking
1004 void bang_stack_with_offset(int offset) {
1005 // stack grows down, caller passes positive offset
1006 assert(offset > 0, "must bang with negative offset");
1007 sub(rscratch2, sp, offset);
1008 str(zr, Address(rscratch2));
1009 }
1010
1011 // Writes to stack successive pages until offset reached to check for
1012 // stack overflow + shadow pages. Also, clobbers tmp
1013 void bang_stack_size(Register size, Register tmp);
1014
1015 // Check for reserved stack access in method being exited (for JIT)
1016 void reserved_stack_check();
1017
1018 // Arithmetics
1019
1020 void addptr(const Address &dst, int32_t src);
1021 void cmpptr(Register src1, Address src2);
1022
1023 void cmpoop(Register obj1, Register obj2);
1024
1025 // Various forms of CAS
1026
1027 void cmpxchg_obj_header(Register oldv, Register newv, Register obj, Register tmp,
1028 Label &suceed, Label *fail);
1029 void cmpxchgptr(Register oldv, Register newv, Register addr, Register tmp,
1030 Label &suceed, Label *fail);
1031
1032 void cmpxchgw(Register oldv, Register newv, Register addr, Register tmp,
1033 Label &suceed, Label *fail);
1034
1035 void atomic_add(Register prev, RegisterOrConstant incr, Register addr);
1036 void atomic_addw(Register prev, RegisterOrConstant incr, Register addr);
1037 void atomic_addal(Register prev, RegisterOrConstant incr, Register addr);
1038 void atomic_addalw(Register prev, RegisterOrConstant incr, Register addr);
1039
1040 void atomic_xchg(Register prev, Register newv, Register addr);
1041 void atomic_xchgw(Register prev, Register newv, Register addr);
1042 void atomic_xchgl(Register prev, Register newv, Register addr);
1043 void atomic_xchglw(Register prev, Register newv, Register addr);
1044 void atomic_xchgal(Register prev, Register newv, Register addr);
1045 void atomic_xchgalw(Register prev, Register newv, Register addr);
1046
1047 void orptr(Address adr, RegisterOrConstant src) {
1048 ldr(rscratch1, adr);
1049 if (src.is_register())
1050 orr(rscratch1, rscratch1, src.as_register());
1051 else
1052 orr(rscratch1, rscratch1, src.as_constant());
1053 str(rscratch1, adr);
1054 }
1055
1056 // A generic CAS; success or failure is in the EQ flag.
1057 // Clobbers rscratch1
1058 void cmpxchg(Register addr, Register expected, Register new_val,
1059 enum operand_size size,
1060 bool acquire, bool release, bool weak,
1061 Register result);
1062
1063 private:
1064 void compare_eq(Register rn, Register rm, enum operand_size size);
1065
1066 #ifdef ASSERT
1067 // Template short-hand support to clean-up after a failed call to trampoline
1068 // call generation (see trampoline_call() below), when a set of Labels must
1069 // be reset (before returning).
1070 template<typename Label, typename... More>
1071 void reset_labels(Label &lbl, More&... more) {
1072 lbl.reset(); reset_labels(more...);
1073 }
1074 template<typename Label>
1075 void reset_labels(Label &lbl) {
1076 lbl.reset();
1077 }
1078 #endif
1079
1080 public:
1081 // Calls
1082
1083 address trampoline_call(Address entry, CodeBuffer* cbuf = NULL);
1084
1085 static bool far_branches() {
1086 return ReservedCodeCacheSize > branch_range;
1087 }
1088
1089 // Check if branches to the the non nmethod section require a far jump
1090 static bool codestub_branch_needs_far_jump() {
1091 return CodeCache::max_distance_to_non_nmethod() > branch_range;
1092 }
1093
1094 // Jumps that can reach anywhere in the code cache.
1095 // Trashes tmp.
1096 void far_call(Address entry, CodeBuffer *cbuf = NULL, Register tmp = rscratch1);
1097 int far_jump(Address entry, CodeBuffer *cbuf = NULL, Register tmp = rscratch1);
1098
1099 static int far_codestub_branch_size() {
1100 if (codestub_branch_needs_far_jump()) {
1101 return 3 * 4; // adrp, add, br
1102 } else {
1103 return 4;
1104 }
1105 }
1106
1107 // Emit the CompiledIC call idiom
1108 address ic_call(address entry, jint method_index = 0);
1109
1110 public:
1111
1112 // Data
1113
1114 void mov_metadata(Register dst, Metadata* obj);
1115 Address allocate_metadata_address(Metadata* obj);
1116 Address constant_oop_address(jobject obj);
1117
1118 void movoop(Register dst, jobject obj, bool immediate = false);
1119
1120 // CRC32 code for java.util.zip.CRC32::updateBytes() instrinsic.
1121 void kernel_crc32(Register crc, Register buf, Register len,
1122 Register table0, Register table1, Register table2, Register table3,
1123 Register tmp, Register tmp2, Register tmp3);
1124 // CRC32 code for java.util.zip.CRC32C::updateBytes() instrinsic.
1125 void kernel_crc32c(Register crc, Register buf, Register len,
1126 Register table0, Register table1, Register table2, Register table3,
1127 Register tmp, Register tmp2, Register tmp3);
1128
1129 // Stack push and pop individual 64 bit registers
1130 void push(Register src);
1131 void pop(Register dst);
1132
1133 void repne_scan(Register addr, Register value, Register count,
1134 Register scratch);
1135 void repne_scanw(Register addr, Register value, Register count,
1136 Register scratch);
1137
1138 typedef void (MacroAssembler::* add_sub_imm_insn)(Register Rd, Register Rn, unsigned imm);
1139 typedef void (MacroAssembler::* add_sub_reg_insn)(Register Rd, Register Rn, Register Rm, enum shift_kind kind, unsigned shift);
1140
1141 // If a constant does not fit in an immediate field, generate some
1142 // number of MOV instructions and then perform the operation
1143 void wrap_add_sub_imm_insn(Register Rd, Register Rn, uint64_t imm,
1144 add_sub_imm_insn insn1,
1145 add_sub_reg_insn insn2, bool is32);
1146 // Seperate vsn which sets the flags
1147 void wrap_adds_subs_imm_insn(Register Rd, Register Rn, uint64_t imm,
1148 add_sub_imm_insn insn1,
1149 add_sub_reg_insn insn2, bool is32);
1150
1151 #define WRAP(INSN, is32) \
1152 void INSN(Register Rd, Register Rn, uint64_t imm) { \
1153 wrap_add_sub_imm_insn(Rd, Rn, imm, &Assembler::INSN, &Assembler::INSN, is32); \
1154 } \
1155 \
1156 void INSN(Register Rd, Register Rn, Register Rm, \
1157 enum shift_kind kind, unsigned shift = 0) { \
1158 Assembler::INSN(Rd, Rn, Rm, kind, shift); \
1159 } \
1160 \
1161 void INSN(Register Rd, Register Rn, Register Rm) { \
1162 Assembler::INSN(Rd, Rn, Rm); \
1163 } \
1164 \
1165 void INSN(Register Rd, Register Rn, Register Rm, \
1166 ext::operation option, int amount = 0) { \
1167 Assembler::INSN(Rd, Rn, Rm, option, amount); \
1168 }
1169
1170 WRAP(add, false) WRAP(addw, true) WRAP(sub, false) WRAP(subw, true)
1171
1172 #undef WRAP
1173 #define WRAP(INSN, is32) \
1174 void INSN(Register Rd, Register Rn, uint64_t imm) { \
1175 wrap_adds_subs_imm_insn(Rd, Rn, imm, &Assembler::INSN, &Assembler::INSN, is32); \
1176 } \
1177 \
1178 void INSN(Register Rd, Register Rn, Register Rm, \
1179 enum shift_kind kind, unsigned shift = 0) { \
1180 Assembler::INSN(Rd, Rn, Rm, kind, shift); \
1181 } \
1182 \
1183 void INSN(Register Rd, Register Rn, Register Rm) { \
1184 Assembler::INSN(Rd, Rn, Rm); \
1185 } \
1186 \
1187 void INSN(Register Rd, Register Rn, Register Rm, \
1188 ext::operation option, int amount = 0) { \
1189 Assembler::INSN(Rd, Rn, Rm, option, amount); \
1190 }
1191
1192 WRAP(adds, false) WRAP(addsw, true) WRAP(subs, false) WRAP(subsw, true)
1193
1194 void add(Register Rd, Register Rn, RegisterOrConstant increment);
1195 void addw(Register Rd, Register Rn, RegisterOrConstant increment);
1196 void sub(Register Rd, Register Rn, RegisterOrConstant decrement);
1197 void subw(Register Rd, Register Rn, RegisterOrConstant decrement);
1198
1199 void adrp(Register reg1, const Address &dest, uint64_t &byte_offset);
1200
1201 void tableswitch(Register index, jint lowbound, jint highbound,
1202 Label &jumptable, Label &jumptable_end, int stride = 1) {
1203 adr(rscratch1, jumptable);
1204 subsw(rscratch2, index, lowbound);
1205 subsw(zr, rscratch2, highbound - lowbound);
1206 br(Assembler::HS, jumptable_end);
1207 add(rscratch1, rscratch1, rscratch2,
1208 ext::sxtw, exact_log2(stride * Assembler::instruction_size));
1209 br(rscratch1);
1210 }
1211
1212 // Form an address from base + offset in Rd. Rd may or may not
1213 // actually be used: you must use the Address that is returned. It
1214 // is up to you to ensure that the shift provided matches the size
1215 // of your data.
1216 Address form_address(Register Rd, Register base, int64_t byte_offset, int shift);
1217
1218 // Return true iff an address is within the 48-bit AArch64 address
1219 // space.
1220 bool is_valid_AArch64_address(address a) {
1221 return ((uint64_t)a >> 48) == 0;
1222 }
1223
1224 // Load the base of the cardtable byte map into reg.
1225 void load_byte_map_base(Register reg);
1226
1227 // Prolog generator routines to support switch between x86 code and
1228 // generated ARM code
1229
1230 // routine to generate an x86 prolog for a stub function which
1231 // bootstraps into the generated ARM code which directly follows the
1232 // stub
1233 //
1234
1235 public:
1236
1237 void ldr_constant(Register dest, const Address &const_addr) {
1238 if (NearCpool) {
1239 ldr(dest, const_addr);
1240 } else {
1241 uint64_t offset;
1242 adrp(dest, InternalAddress(const_addr.target()), offset);
1243 ldr(dest, Address(dest, offset));
1244 }
1245 }
1246
1247 address read_polling_page(Register r, relocInfo::relocType rtype);
1248 void get_polling_page(Register dest, relocInfo::relocType rtype);
1249
1250 // CRC32 code for java.util.zip.CRC32::updateBytes() instrinsic.
1251 void update_byte_crc32(Register crc, Register val, Register table);
1252 void update_word_crc32(Register crc, Register v, Register tmp,
1253 Register table0, Register table1, Register table2, Register table3,
1254 bool upper = false);
1255
1256 address has_negatives(Register ary1, Register len, Register result);
1257
1258 address arrays_equals(Register a1, Register a2, Register result, Register cnt1,
1259 Register tmp1, Register tmp2, Register tmp3, int elem_size);
1260
1261 void string_equals(Register a1, Register a2, Register result, Register cnt1,
1262 int elem_size);
1263
1264 void fill_words(Register base, Register cnt, Register value);
1265 address zero_words(Register base, uint64_t cnt);
1266 address zero_words(Register ptr, Register cnt);
1267 void zero_dcache_blocks(Register base, Register cnt);
1268
1269 static const int zero_words_block_size;
1270
1271 address byte_array_inflate(Register src, Register dst, Register len,
1272 FloatRegister vtmp1, FloatRegister vtmp2,
1273 FloatRegister vtmp3, Register tmp4);
1274
1275 void char_array_compress(Register src, Register dst, Register len,
1276 Register res,
1277 FloatRegister vtmp0, FloatRegister vtmp1,
1278 FloatRegister vtmp2, FloatRegister vtmp3,
1279 FloatRegister vtmp4, FloatRegister vtmp5);
1280
1281 void encode_iso_array(Register src, Register dst,
1282 Register len, Register res, bool ascii,
1283 FloatRegister vtmp0, FloatRegister vtmp1,
1284 FloatRegister vtmp2, FloatRegister vtmp3,
1285 FloatRegister vtmp4, FloatRegister vtmp5);
1286
1287 void fast_log(FloatRegister vtmp0, FloatRegister vtmp1, FloatRegister vtmp2,
1288 FloatRegister vtmp3, FloatRegister vtmp4, FloatRegister vtmp5,
1289 FloatRegister tmpC1, FloatRegister tmpC2, FloatRegister tmpC3,
1290 FloatRegister tmpC4, Register tmp1, Register tmp2,
1291 Register tmp3, Register tmp4, Register tmp5);
1292 void generate_dsin_dcos(bool isCos, address npio2_hw, address two_over_pi,
1293 address pio2, address dsin_coef, address dcos_coef);
1294 private:
1295 // begin trigonometric functions support block
1296 void generate__ieee754_rem_pio2(address npio2_hw, address two_over_pi, address pio2);
1297 void generate__kernel_rem_pio2(address two_over_pi, address pio2);
1298 void generate_kernel_sin(FloatRegister x, bool iyIsOne, address dsin_coef);
1299 void generate_kernel_cos(FloatRegister x, address dcos_coef);
1300 // end trigonometric functions support block
1301 void add2_with_carry(Register final_dest_hi, Register dest_hi, Register dest_lo,
1302 Register src1, Register src2);
1303 void add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2) {
1304 add2_with_carry(dest_hi, dest_hi, dest_lo, src1, src2);
1305 }
1306 void multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart,
1307 Register y, Register y_idx, Register z,
1308 Register carry, Register product,
1309 Register idx, Register kdx);
1310 void multiply_128_x_128_loop(Register y, Register z,
1311 Register carry, Register carry2,
1312 Register idx, Register jdx,
1313 Register yz_idx1, Register yz_idx2,
1314 Register tmp, Register tmp3, Register tmp4,
1315 Register tmp7, Register product_hi);
1316 void kernel_crc32_using_crc32(Register crc, Register buf,
1317 Register len, Register tmp0, Register tmp1, Register tmp2,
1318 Register tmp3);
1319 void kernel_crc32c_using_crc32c(Register crc, Register buf,
1320 Register len, Register tmp0, Register tmp1, Register tmp2,
1321 Register tmp3);
1322 public:
1323 void multiply_to_len(Register x, Register xlen, Register y, Register ylen, Register z,
1324 Register zlen, Register tmp1, Register tmp2, Register tmp3,
1325 Register tmp4, Register tmp5, Register tmp6, Register tmp7);
1326 void mul_add(Register out, Register in, Register offs, Register len, Register k);
1327 void ghash_multiply(FloatRegister result_lo, FloatRegister result_hi,
1328 FloatRegister a, FloatRegister b, FloatRegister a1_xor_a0,
1329 FloatRegister tmp1, FloatRegister tmp2, FloatRegister tmp3);
1330 void ghash_reduce(FloatRegister result, FloatRegister lo, FloatRegister hi,
1331 FloatRegister p, FloatRegister z, FloatRegister t1);
1332 void ghash_processBlocks_wide(address p, Register state, Register subkeyH,
1333 Register data, Register blocks, int unrolls);
1334 void ghash_modmul (FloatRegister result,
1335 FloatRegister result_lo, FloatRegister result_hi, FloatRegister b,
1336 FloatRegister a, FloatRegister vzr, FloatRegister a1_xor_a0, FloatRegister p,
1337 FloatRegister t1, FloatRegister t2, FloatRegister t3);
1338
1339 void aesenc_loadkeys(Register key, Register keylen);
1340 void aesecb_encrypt(Register from, Register to, Register keylen,
1341 FloatRegister data = v0, int unrolls = 1);
1342 void aesecb_decrypt(Register from, Register to, Register key, Register keylen);
1343 void aes_round(FloatRegister input, FloatRegister subkey);
1344
1345 // Place an ISB after code may have been modified due to a safepoint.
1346 void safepoint_isb();
1347
1348 private:
1349 // Return the effective address r + (r1 << ext) + offset.
1350 // Uses rscratch2.
1351 Address offsetted_address(Register r, Register r1, Address::extend ext,
1352 int offset, int size);
1353
1354 private:
1355 // Returns an address on the stack which is reachable with a ldr/str of size
1356 // Uses rscratch2 if the address is not directly reachable
1357 Address spill_address(int size, int offset, Register tmp=rscratch2);
1358 Address sve_spill_address(int sve_reg_size_in_bytes, int offset, Register tmp=rscratch2);
1359
1360 bool merge_alignment_check(Register base, size_t size, int64_t cur_offset, int64_t prev_offset) const;
1361
1362 // Check whether two loads/stores can be merged into ldp/stp.
1363 bool ldst_can_merge(Register rx, const Address &adr, size_t cur_size_in_bytes, bool is_store) const;
1364
1365 // Merge current load/store with previous load/store into ldp/stp.
1366 void merge_ldst(Register rx, const Address &adr, size_t cur_size_in_bytes, bool is_store);
1367
1368 // Try to merge two loads/stores into ldp/stp. If success, returns true else false.
1369 bool try_merge_ldst(Register rt, const Address &adr, size_t cur_size_in_bytes, bool is_store);
1370
1371 public:
1372 void spill(Register Rx, bool is64, int offset) {
1373 if (is64) {
1374 str(Rx, spill_address(8, offset));
1375 } else {
1376 strw(Rx, spill_address(4, offset));
1377 }
1378 }
1379 void spill(FloatRegister Vx, SIMD_RegVariant T, int offset) {
1380 str(Vx, T, spill_address(1 << (int)T, offset));
1381 }
1382 void spill_sve_vector(FloatRegister Zx, int offset, int vector_reg_size_in_bytes) {
1383 sve_str(Zx, sve_spill_address(vector_reg_size_in_bytes, offset));
1384 }
1385 void unspill(Register Rx, bool is64, int offset) {
1386 if (is64) {
1387 ldr(Rx, spill_address(8, offset));
1388 } else {
1389 ldrw(Rx, spill_address(4, offset));
1390 }
1391 }
1392 void unspill(FloatRegister Vx, SIMD_RegVariant T, int offset) {
1393 ldr(Vx, T, spill_address(1 << (int)T, offset));
1394 }
1395 void unspill_sve_vector(FloatRegister Zx, int offset, int vector_reg_size_in_bytes) {
1396 sve_ldr(Zx, sve_spill_address(vector_reg_size_in_bytes, offset));
1397 }
1398 void spill_copy128(int src_offset, int dst_offset,
1399 Register tmp1=rscratch1, Register tmp2=rscratch2) {
1400 if (src_offset < 512 && (src_offset & 7) == 0 &&
1401 dst_offset < 512 && (dst_offset & 7) == 0) {
1402 ldp(tmp1, tmp2, Address(sp, src_offset));
1403 stp(tmp1, tmp2, Address(sp, dst_offset));
1404 } else {
1405 unspill(tmp1, true, src_offset);
1406 spill(tmp1, true, dst_offset);
1407 unspill(tmp1, true, src_offset+8);
1408 spill(tmp1, true, dst_offset+8);
1409 }
1410 }
1411 void spill_copy_sve_vector_stack_to_stack(int src_offset, int dst_offset,
1412 int sve_vec_reg_size_in_bytes) {
1413 assert(sve_vec_reg_size_in_bytes % 16 == 0, "unexpected sve vector reg size");
1414 for (int i = 0; i < sve_vec_reg_size_in_bytes / 16; i++) {
1415 spill_copy128(src_offset, dst_offset);
1416 src_offset += 16;
1417 dst_offset += 16;
1418 }
1419 }
1420 void cache_wb(Address line);
1421 void cache_wbsync(bool is_pre);
1422
1423 // Code for java.lang.Thread::onSpinWait() intrinsic.
1424 void spin_wait();
1425
1426 private:
1427 // Check the current thread doesn't need a cross modify fence.
1428 void verify_cross_modify_fence_not_required() PRODUCT_RETURN;
1429
1430 };
1431
1432 #ifdef ASSERT
1433 inline bool AbstractAssembler::pd_check_instruction_mark() { return false; }
1434 #endif
1435
1436 /**
1437 * class SkipIfEqual:
1438 *
1439 * Instantiating this class will result in assembly code being output that will
1440 * jump around any code emitted between the creation of the instance and it's
1441 * automatic destruction at the end of a scope block, depending on the value of
1442 * the flag passed to the constructor, which will be checked at run-time.
1443 */
1444 class SkipIfEqual {
1445 private:
1446 MacroAssembler* _masm;
1447 Label _label;
1448
1449 public:
1450 SkipIfEqual(MacroAssembler*, const bool* flag_addr, bool value);
1451 ~SkipIfEqual();
1452 };
1453
1454 struct tableswitch {
1455 Register _reg;
1456 int _insn_index; jint _first_key; jint _last_key;
1457 Label _after;
1458 Label _branches;
1459 };
1460
1461 #endif // CPU_AARCH64_MACROASSEMBLER_AARCH64_HPP