1 /*
2 * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #ifndef CPU_AARCH64_MACROASSEMBLER_AARCH64_HPP
27 #define CPU_AARCH64_MACROASSEMBLER_AARCH64_HPP
28
29 #include "asm/assembler.inline.hpp"
30 #include "oops/compressedOops.hpp"
31 #include "runtime/vm_version.hpp"
32 #include "utilities/powerOfTwo.hpp"
33
34 // MacroAssembler extends Assembler by frequently used macros.
35 //
36 // Instructions for which a 'better' code sequence exists depending
37 // on arguments should also go in here.
38
39 class MacroAssembler: public Assembler {
40 friend class LIR_Assembler;
41
42 public:
43 using Assembler::mov;
44 using Assembler::movi;
45
46 protected:
47
48 // Support for VM calls
49 //
50 // This is the base routine called by the different versions of call_VM_leaf. The interpreter
51 // may customize this version by overriding it for its purposes (e.g., to save/restore
52 // additional registers when doing a VM call).
53 virtual void call_VM_leaf_base(
54 address entry_point, // the entry point
55 int number_of_arguments, // the number of arguments to pop after the call
56 Label *retaddr = NULL
57 );
58
59 virtual void call_VM_leaf_base(
60 address entry_point, // the entry point
61 int number_of_arguments, // the number of arguments to pop after the call
62 Label &retaddr) {
63 call_VM_leaf_base(entry_point, number_of_arguments, &retaddr);
64 }
65
66 // This is the base routine called by the different versions of call_VM. The interpreter
67 // may customize this version by overriding it for its purposes (e.g., to save/restore
68 // additional registers when doing a VM call).
69 //
70 // If no java_thread register is specified (noreg) than rthread will be used instead. call_VM_base
71 // returns the register which contains the thread upon return. If a thread register has been
72 // specified, the return value will correspond to that register. If no last_java_sp is specified
73 // (noreg) than rsp will be used instead.
74 virtual void call_VM_base( // returns the register containing the thread upon return
75 Register oop_result, // where an oop-result ends up if any; use noreg otherwise
76 Register java_thread, // the thread if computed before ; use noreg otherwise
77 Register last_java_sp, // to set up last_Java_frame in stubs; use noreg otherwise
78 address entry_point, // the entry point
79 int number_of_arguments, // the number of arguments (w/o thread) to pop after the call
80 bool check_exceptions // whether to check for pending exceptions after return
81 );
82
83 void call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions = true);
84
85 enum KlassDecodeMode {
86 KlassDecodeNone,
87 KlassDecodeZero,
88 KlassDecodeXor,
89 KlassDecodeMovk
90 };
91
92 KlassDecodeMode klass_decode_mode();
93
94 private:
95 static KlassDecodeMode _klass_decode_mode;
96
97 public:
98 MacroAssembler(CodeBuffer* code) : Assembler(code) {}
99
100 // These routines should emit JVMTI PopFrame and ForceEarlyReturn handling code.
101 // The implementation is only non-empty for the InterpreterMacroAssembler,
102 // as only the interpreter handles PopFrame and ForceEarlyReturn requests.
103 virtual void check_and_handle_popframe(Register java_thread);
104 virtual void check_and_handle_earlyret(Register java_thread);
105
106 void safepoint_poll(Label& slow_path, bool at_return, bool acquire, bool in_nmethod);
107
108 // Biased locking support
109 // lock_reg and obj_reg must be loaded up with the appropriate values.
110 // swap_reg is killed.
111 // tmp_reg must be supplied and must not be rscratch1 or rscratch2
112 // Optional slow case is for implementations (interpreter and C1) which branch to
113 // slow case directly. Leaves condition codes set for C2's Fast_Lock node.
114 void biased_locking_enter(Register lock_reg, Register obj_reg,
115 Register swap_reg, Register tmp_reg,
116 bool swap_reg_contains_mark,
117 Label& done, Label* slow_case = NULL,
118 BiasedLockingCounters* counters = NULL);
119 void biased_locking_exit (Register obj_reg, Register temp_reg, Label& done);
120
121
122 // Helper functions for statistics gathering.
123 // Unconditional atomic increment.
124 void atomic_incw(Register counter_addr, Register tmp, Register tmp2);
125 void atomic_incw(Address counter_addr, Register tmp1, Register tmp2, Register tmp3) {
126 lea(tmp1, counter_addr);
127 atomic_incw(tmp1, tmp2, tmp3);
128 }
129 // Load Effective Address
130 void lea(Register r, const Address &a) {
131 InstructionMark im(this);
132 code_section()->relocate(inst_mark(), a.rspec());
133 a.lea(this, r);
134 }
135
136 /* Sometimes we get misaligned loads and stores, usually from Unsafe
137 accesses, and these can exceed the offset range. */
138 Address legitimize_address(const Address &a, int size, Register scratch) {
139 if (a.getMode() == Address::base_plus_offset) {
140 if (! Address::offset_ok_for_immed(a.offset(), exact_log2(size))) {
141 block_comment("legitimize_address {");
142 lea(scratch, a);
143 block_comment("} legitimize_address");
144 return Address(scratch);
145 }
146 }
147 return a;
148 }
149
150 void addmw(Address a, Register incr, Register scratch) {
151 ldrw(scratch, a);
152 addw(scratch, scratch, incr);
153 strw(scratch, a);
154 }
155
156 // Add constant to memory word
157 void addmw(Address a, int imm, Register scratch) {
158 ldrw(scratch, a);
159 if (imm > 0)
160 addw(scratch, scratch, (unsigned)imm);
161 else
162 subw(scratch, scratch, (unsigned)-imm);
163 strw(scratch, a);
164 }
165
166 void bind(Label& L) {
167 Assembler::bind(L);
168 code()->clear_last_insn();
169 }
170
171 void membar(Membar_mask_bits order_constraint);
172
173 using Assembler::ldr;
174 using Assembler::str;
175 using Assembler::ldrw;
176 using Assembler::strw;
177
178 void ldr(Register Rx, const Address &adr);
179 void ldrw(Register Rw, const Address &adr);
180 void str(Register Rx, const Address &adr);
181 void strw(Register Rx, const Address &adr);
182
183 // Frame creation and destruction shared between JITs.
184 void build_frame(int framesize);
185 void remove_frame(int framesize);
186
187 virtual void _call_Unimplemented(address call_site) {
188 mov(rscratch2, call_site);
189 }
190
191 // Microsoft's MSVC team thinks that the __FUNCSIG__ is approximately (sympathy for calling conventions) equivalent to __PRETTY_FUNCTION__
192 // Also, from Clang patch: "It is very similar to GCC's PRETTY_FUNCTION, except it prints the calling convention."
193 // https://reviews.llvm.org/D3311
194
195 #ifdef _WIN64
196 #define call_Unimplemented() _call_Unimplemented((address)__FUNCSIG__)
197 #else
198 #define call_Unimplemented() _call_Unimplemented((address)__PRETTY_FUNCTION__)
199 #endif
200
201 // aliases defined in AARCH64 spec
202
203 template<class T>
204 inline void cmpw(Register Rd, T imm) { subsw(zr, Rd, imm); }
205
206 inline void cmp(Register Rd, unsigned char imm8) { subs(zr, Rd, imm8); }
207 inline void cmp(Register Rd, unsigned imm) = delete;
208
209 template<class T>
210 inline void cmnw(Register Rd, T imm) { addsw(zr, Rd, imm); }
211
212 inline void cmn(Register Rd, unsigned char imm8) { adds(zr, Rd, imm8); }
213 inline void cmn(Register Rd, unsigned imm) = delete;
214
215 void cset(Register Rd, Assembler::Condition cond) {
216 csinc(Rd, zr, zr, ~cond);
217 }
218 void csetw(Register Rd, Assembler::Condition cond) {
219 csincw(Rd, zr, zr, ~cond);
220 }
221
222 void cneg(Register Rd, Register Rn, Assembler::Condition cond) {
223 csneg(Rd, Rn, Rn, ~cond);
224 }
225 void cnegw(Register Rd, Register Rn, Assembler::Condition cond) {
226 csnegw(Rd, Rn, Rn, ~cond);
227 }
228
229 inline void movw(Register Rd, Register Rn) {
230 if (Rd == sp || Rn == sp) {
231 Assembler::addw(Rd, Rn, 0U);
232 } else {
233 orrw(Rd, zr, Rn);
234 }
235 }
236 inline void mov(Register Rd, Register Rn) {
237 assert(Rd != r31_sp && Rn != r31_sp, "should be");
238 if (Rd == Rn) {
239 } else if (Rd == sp || Rn == sp) {
240 Assembler::add(Rd, Rn, 0U);
241 } else {
242 orr(Rd, zr, Rn);
243 }
244 }
245
246 inline void moviw(Register Rd, unsigned imm) { orrw(Rd, zr, imm); }
247 inline void movi(Register Rd, unsigned imm) { orr(Rd, zr, imm); }
248
249 inline void tstw(Register Rd, Register Rn) { andsw(zr, Rd, Rn); }
250 inline void tst(Register Rd, Register Rn) { ands(zr, Rd, Rn); }
251
252 inline void tstw(Register Rd, uint64_t imm) { andsw(zr, Rd, imm); }
253 inline void tst(Register Rd, uint64_t imm) { ands(zr, Rd, imm); }
254
255 inline void bfiw(Register Rd, Register Rn, unsigned lsb, unsigned width) {
256 bfmw(Rd, Rn, ((32 - lsb) & 31), (width - 1));
257 }
258 inline void bfi(Register Rd, Register Rn, unsigned lsb, unsigned width) {
259 bfm(Rd, Rn, ((64 - lsb) & 63), (width - 1));
260 }
261
262 inline void bfxilw(Register Rd, Register Rn, unsigned lsb, unsigned width) {
263 bfmw(Rd, Rn, lsb, (lsb + width - 1));
264 }
265 inline void bfxil(Register Rd, Register Rn, unsigned lsb, unsigned width) {
266 bfm(Rd, Rn, lsb , (lsb + width - 1));
267 }
268
269 inline void sbfizw(Register Rd, Register Rn, unsigned lsb, unsigned width) {
270 sbfmw(Rd, Rn, ((32 - lsb) & 31), (width - 1));
271 }
272 inline void sbfiz(Register Rd, Register Rn, unsigned lsb, unsigned width) {
273 sbfm(Rd, Rn, ((64 - lsb) & 63), (width - 1));
274 }
275
276 inline void sbfxw(Register Rd, Register Rn, unsigned lsb, unsigned width) {
277 sbfmw(Rd, Rn, lsb, (lsb + width - 1));
278 }
279 inline void sbfx(Register Rd, Register Rn, unsigned lsb, unsigned width) {
280 sbfm(Rd, Rn, lsb , (lsb + width - 1));
281 }
282
283 inline void ubfizw(Register Rd, Register Rn, unsigned lsb, unsigned width) {
284 ubfmw(Rd, Rn, ((32 - lsb) & 31), (width - 1));
285 }
286 inline void ubfiz(Register Rd, Register Rn, unsigned lsb, unsigned width) {
287 ubfm(Rd, Rn, ((64 - lsb) & 63), (width - 1));
288 }
289
290 inline void ubfxw(Register Rd, Register Rn, unsigned lsb, unsigned width) {
291 ubfmw(Rd, Rn, lsb, (lsb + width - 1));
292 }
293 inline void ubfx(Register Rd, Register Rn, unsigned lsb, unsigned width) {
294 ubfm(Rd, Rn, lsb , (lsb + width - 1));
295 }
296
297 inline void asrw(Register Rd, Register Rn, unsigned imm) {
298 sbfmw(Rd, Rn, imm, 31);
299 }
300
301 inline void asr(Register Rd, Register Rn, unsigned imm) {
302 sbfm(Rd, Rn, imm, 63);
303 }
304
305 inline void lslw(Register Rd, Register Rn, unsigned imm) {
306 ubfmw(Rd, Rn, ((32 - imm) & 31), (31 - imm));
307 }
308
309 inline void lsl(Register Rd, Register Rn, unsigned imm) {
310 ubfm(Rd, Rn, ((64 - imm) & 63), (63 - imm));
311 }
312
313 inline void lsrw(Register Rd, Register Rn, unsigned imm) {
314 ubfmw(Rd, Rn, imm, 31);
315 }
316
317 inline void lsr(Register Rd, Register Rn, unsigned imm) {
318 ubfm(Rd, Rn, imm, 63);
319 }
320
321 inline void rorw(Register Rd, Register Rn, unsigned imm) {
322 extrw(Rd, Rn, Rn, imm);
323 }
324
325 inline void ror(Register Rd, Register Rn, unsigned imm) {
326 extr(Rd, Rn, Rn, imm);
327 }
328
329 inline void sxtbw(Register Rd, Register Rn) {
330 sbfmw(Rd, Rn, 0, 7);
331 }
332 inline void sxthw(Register Rd, Register Rn) {
333 sbfmw(Rd, Rn, 0, 15);
334 }
335 inline void sxtb(Register Rd, Register Rn) {
336 sbfm(Rd, Rn, 0, 7);
337 }
338 inline void sxth(Register Rd, Register Rn) {
339 sbfm(Rd, Rn, 0, 15);
340 }
341 inline void sxtw(Register Rd, Register Rn) {
342 sbfm(Rd, Rn, 0, 31);
343 }
344
345 inline void uxtbw(Register Rd, Register Rn) {
346 ubfmw(Rd, Rn, 0, 7);
347 }
348 inline void uxthw(Register Rd, Register Rn) {
349 ubfmw(Rd, Rn, 0, 15);
350 }
351 inline void uxtb(Register Rd, Register Rn) {
352 ubfm(Rd, Rn, 0, 7);
353 }
354 inline void uxth(Register Rd, Register Rn) {
355 ubfm(Rd, Rn, 0, 15);
356 }
357 inline void uxtw(Register Rd, Register Rn) {
358 ubfm(Rd, Rn, 0, 31);
359 }
360
361 inline void cmnw(Register Rn, Register Rm) {
362 addsw(zr, Rn, Rm);
363 }
364 inline void cmn(Register Rn, Register Rm) {
365 adds(zr, Rn, Rm);
366 }
367
368 inline void cmpw(Register Rn, Register Rm) {
369 subsw(zr, Rn, Rm);
370 }
371 inline void cmp(Register Rn, Register Rm) {
372 subs(zr, Rn, Rm);
373 }
374
375 inline void negw(Register Rd, Register Rn) {
376 subw(Rd, zr, Rn);
377 }
378
379 inline void neg(Register Rd, Register Rn) {
380 sub(Rd, zr, Rn);
381 }
382
383 inline void negsw(Register Rd, Register Rn) {
384 subsw(Rd, zr, Rn);
385 }
386
387 inline void negs(Register Rd, Register Rn) {
388 subs(Rd, zr, Rn);
389 }
390
391 inline void cmnw(Register Rn, Register Rm, enum shift_kind kind, unsigned shift = 0) {
392 addsw(zr, Rn, Rm, kind, shift);
393 }
394 inline void cmn(Register Rn, Register Rm, enum shift_kind kind, unsigned shift = 0) {
395 adds(zr, Rn, Rm, kind, shift);
396 }
397
398 inline void cmpw(Register Rn, Register Rm, enum shift_kind kind, unsigned shift = 0) {
399 subsw(zr, Rn, Rm, kind, shift);
400 }
401 inline void cmp(Register Rn, Register Rm, enum shift_kind kind, unsigned shift = 0) {
402 subs(zr, Rn, Rm, kind, shift);
403 }
404
405 inline void negw(Register Rd, Register Rn, enum shift_kind kind, unsigned shift = 0) {
406 subw(Rd, zr, Rn, kind, shift);
407 }
408
409 inline void neg(Register Rd, Register Rn, enum shift_kind kind, unsigned shift = 0) {
410 sub(Rd, zr, Rn, kind, shift);
411 }
412
413 inline void negsw(Register Rd, Register Rn, enum shift_kind kind, unsigned shift = 0) {
414 subsw(Rd, zr, Rn, kind, shift);
415 }
416
417 inline void negs(Register Rd, Register Rn, enum shift_kind kind, unsigned shift = 0) {
418 subs(Rd, zr, Rn, kind, shift);
419 }
420
421 inline void mnegw(Register Rd, Register Rn, Register Rm) {
422 msubw(Rd, Rn, Rm, zr);
423 }
424 inline void mneg(Register Rd, Register Rn, Register Rm) {
425 msub(Rd, Rn, Rm, zr);
426 }
427
428 inline void mulw(Register Rd, Register Rn, Register Rm) {
429 maddw(Rd, Rn, Rm, zr);
430 }
431 inline void mul(Register Rd, Register Rn, Register Rm) {
432 madd(Rd, Rn, Rm, zr);
433 }
434
435 inline void smnegl(Register Rd, Register Rn, Register Rm) {
436 smsubl(Rd, Rn, Rm, zr);
437 }
438 inline void smull(Register Rd, Register Rn, Register Rm) {
439 smaddl(Rd, Rn, Rm, zr);
440 }
441
442 inline void umnegl(Register Rd, Register Rn, Register Rm) {
443 umsubl(Rd, Rn, Rm, zr);
444 }
445 inline void umull(Register Rd, Register Rn, Register Rm) {
446 umaddl(Rd, Rn, Rm, zr);
447 }
448
449 #define WRAP(INSN) \
450 void INSN(Register Rd, Register Rn, Register Rm, Register Ra) { \
451 if ((VM_Version::features() & VM_Version::CPU_A53MAC) && Ra != zr) \
452 nop(); \
453 Assembler::INSN(Rd, Rn, Rm, Ra); \
454 }
455
456 WRAP(madd) WRAP(msub) WRAP(maddw) WRAP(msubw)
457 WRAP(smaddl) WRAP(smsubl) WRAP(umaddl) WRAP(umsubl)
458 #undef WRAP
459
460
461 // macro assembly operations needed for aarch64
462
463 // first two private routines for loading 32 bit or 64 bit constants
464 private:
465
466 void mov_immediate64(Register dst, uint64_t imm64);
467 void mov_immediate32(Register dst, uint32_t imm32);
468
469 int push(unsigned int bitset, Register stack);
470 int pop(unsigned int bitset, Register stack);
471
472 int push_fp(unsigned int bitset, Register stack);
473 int pop_fp(unsigned int bitset, Register stack);
474
475 void mov(Register dst, Address a);
476
477 public:
478 void push(RegSet regs, Register stack) { if (regs.bits()) push(regs.bits(), stack); }
479 void pop(RegSet regs, Register stack) { if (regs.bits()) pop(regs.bits(), stack); }
480
481 void push_fp(FloatRegSet regs, Register stack) { if (regs.bits()) push_fp(regs.bits(), stack); }
482 void pop_fp(FloatRegSet regs, Register stack) { if (regs.bits()) pop_fp(regs.bits(), stack); }
483
484 static RegSet call_clobbered_registers();
485
486 // Push and pop everything that might be clobbered by a native
487 // runtime call except rscratch1 and rscratch2. (They are always
488 // scratch, so we don't have to protect them.) Only save the lower
489 // 64 bits of each vector register. Additonal registers can be excluded
490 // in a passed RegSet.
491 void push_call_clobbered_registers_except(RegSet exclude);
492 void pop_call_clobbered_registers_except(RegSet exclude);
493
494 void push_call_clobbered_registers() {
495 push_call_clobbered_registers_except(RegSet());
496 }
497 void pop_call_clobbered_registers() {
498 pop_call_clobbered_registers_except(RegSet());
499 }
500
501
502 // now mov instructions for loading absolute addresses and 32 or
503 // 64 bit integers
504
505 inline void mov(Register dst, address addr) { mov_immediate64(dst, (uint64_t)addr); }
506
507 inline void mov(Register dst, int imm64) { mov_immediate64(dst, (uint64_t)imm64); }
508 inline void mov(Register dst, long imm64) { mov_immediate64(dst, (uint64_t)imm64); }
509 inline void mov(Register dst, long long imm64) { mov_immediate64(dst, (uint64_t)imm64); }
510 inline void mov(Register dst, unsigned int imm64) { mov_immediate64(dst, (uint64_t)imm64); }
511 inline void mov(Register dst, unsigned long imm64) { mov_immediate64(dst, (uint64_t)imm64); }
512 inline void mov(Register dst, unsigned long long imm64) { mov_immediate64(dst, (uint64_t)imm64); }
513
514 inline void movw(Register dst, uint32_t imm32)
515 {
516 mov_immediate32(dst, imm32);
517 }
518
519 void mov(Register dst, RegisterOrConstant src) {
520 if (src.is_register())
521 mov(dst, src.as_register());
522 else
523 mov(dst, src.as_constant());
524 }
525
526 void movptr(Register r, uintptr_t imm64);
527
528 void mov(FloatRegister Vd, SIMD_Arrangement T, uint64_t imm64);
529
530 void mov(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn) {
531 orr(Vd, T, Vn, Vn);
532 }
533
534
535 public:
536
537 // Generalized Test Bit And Branch, including a "far" variety which
538 // spans more than 32KiB.
539 void tbr(Condition cond, Register Rt, int bitpos, Label &dest, bool isfar = false) {
540 assert(cond == EQ || cond == NE, "must be");
541
542 if (isfar)
543 cond = ~cond;
544
545 void (Assembler::* branch)(Register Rt, int bitpos, Label &L);
546 if (cond == Assembler::EQ)
547 branch = &Assembler::tbz;
548 else
549 branch = &Assembler::tbnz;
550
551 if (isfar) {
552 Label L;
553 (this->*branch)(Rt, bitpos, L);
554 b(dest);
555 bind(L);
556 } else {
557 (this->*branch)(Rt, bitpos, dest);
558 }
559 }
560
561 // macro instructions for accessing and updating floating point
562 // status register
563 //
564 // FPSR : op1 == 011
565 // CRn == 0100
566 // CRm == 0100
567 // op2 == 001
568
569 inline void get_fpsr(Register reg)
570 {
571 mrs(0b11, 0b0100, 0b0100, 0b001, reg);
572 }
573
574 inline void set_fpsr(Register reg)
575 {
576 msr(0b011, 0b0100, 0b0100, 0b001, reg);
577 }
578
579 inline void clear_fpsr()
580 {
581 msr(0b011, 0b0100, 0b0100, 0b001, zr);
582 }
583
584 // DCZID_EL0: op1 == 011
585 // CRn == 0000
586 // CRm == 0000
587 // op2 == 111
588 inline void get_dczid_el0(Register reg)
589 {
590 mrs(0b011, 0b0000, 0b0000, 0b111, reg);
591 }
592
593 // CTR_EL0: op1 == 011
594 // CRn == 0000
595 // CRm == 0000
596 // op2 == 001
597 inline void get_ctr_el0(Register reg)
598 {
599 mrs(0b011, 0b0000, 0b0000, 0b001, reg);
600 }
601
602 // idiv variant which deals with MINLONG as dividend and -1 as divisor
603 int corrected_idivl(Register result, Register ra, Register rb,
604 bool want_remainder, Register tmp = rscratch1);
605 int corrected_idivq(Register result, Register ra, Register rb,
606 bool want_remainder, Register tmp = rscratch1);
607
608 // Support for NULL-checks
609 //
610 // Generates code that causes a NULL OS exception if the content of reg is NULL.
611 // If the accessed location is M[reg + offset] and the offset is known, provide the
612 // offset. No explicit code generation is needed if the offset is within a certain
613 // range (0 <= offset <= page_size).
614
615 virtual void null_check(Register reg, int offset = -1);
616 static bool needs_explicit_null_check(intptr_t offset);
617 static bool uses_implicit_null_check(void* address);
618
619 static address target_addr_for_insn(address insn_addr, unsigned insn);
620 static address target_addr_for_insn(address insn_addr) {
621 unsigned insn = *(unsigned*)insn_addr;
622 return target_addr_for_insn(insn_addr, insn);
623 }
624
625 // Required platform-specific helpers for Label::patch_instructions.
626 // They _shadow_ the declarations in AbstractAssembler, which are undefined.
627 static int pd_patch_instruction_size(address branch, address target);
628 static void pd_patch_instruction(address branch, address target, const char* file = NULL, int line = 0) {
629 pd_patch_instruction_size(branch, target);
630 }
631 static address pd_call_destination(address branch) {
632 return target_addr_for_insn(branch);
633 }
634 #ifndef PRODUCT
635 static void pd_print_patched_instruction(address branch);
636 #endif
637
638 static int patch_oop(address insn_addr, address o);
639 static int patch_narrow_klass(address insn_addr, narrowKlass n);
640
641 address emit_trampoline_stub(int insts_call_instruction_offset, address target);
642 void emit_static_call_stub();
643
644 // The following 4 methods return the offset of the appropriate move instruction
645
646 // Support for fast byte/short loading with zero extension (depending on particular CPU)
647 int load_unsigned_byte(Register dst, Address src);
648 int load_unsigned_short(Register dst, Address src);
649
650 // Support for fast byte/short loading with sign extension (depending on particular CPU)
651 int load_signed_byte(Register dst, Address src);
652 int load_signed_short(Register dst, Address src);
653
654 int load_signed_byte32(Register dst, Address src);
655 int load_signed_short32(Register dst, Address src);
656
657 // Support for sign-extension (hi:lo = extend_sign(lo))
658 void extend_sign(Register hi, Register lo);
659
660 // Load and store values by size and signed-ness
661 void load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2 = noreg);
662 void store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2 = noreg);
663
664 // Support for inc/dec with optimal instruction selection depending on value
665
666 // x86_64 aliases an unqualified register/address increment and
667 // decrement to call incrementq and decrementq but also supports
668 // explicitly sized calls to incrementq/decrementq or
669 // incrementl/decrementl
670
671 // for aarch64 the proper convention would be to use
672 // increment/decrement for 64 bit operatons and
673 // incrementw/decrementw for 32 bit operations. so when porting
674 // x86_64 code we can leave calls to increment/decrement as is,
675 // replace incrementq/decrementq with increment/decrement and
676 // replace incrementl/decrementl with incrementw/decrementw.
677
678 // n.b. increment/decrement calls with an Address destination will
679 // need to use a scratch register to load the value to be
680 // incremented. increment/decrement calls which add or subtract a
681 // constant value greater than 2^12 will need to use a 2nd scratch
682 // register to hold the constant. so, a register increment/decrement
683 // may trash rscratch2 and an address increment/decrement trash
684 // rscratch and rscratch2
685
686 void decrementw(Address dst, int value = 1);
687 void decrementw(Register reg, int value = 1);
688
689 void decrement(Register reg, int value = 1);
690 void decrement(Address dst, int value = 1);
691
692 void incrementw(Address dst, int value = 1);
693 void incrementw(Register reg, int value = 1);
694
695 void increment(Register reg, int value = 1);
696 void increment(Address dst, int value = 1);
697
698
699 // Alignment
700 void align(int modulus);
701
702 // Stack frame creation/removal
703 void enter()
704 {
705 stp(rfp, lr, Address(pre(sp, -2 * wordSize)));
706 mov(rfp, sp);
707 }
708 void leave()
709 {
710 mov(sp, rfp);
711 ldp(rfp, lr, Address(post(sp, 2 * wordSize)));
712 }
713
714 // Support for getting the JavaThread pointer (i.e.; a reference to thread-local information)
715 // The pointer will be loaded into the thread register.
716 void get_thread(Register thread);
717
718
719 // Support for VM calls
720 //
721 // It is imperative that all calls into the VM are handled via the call_VM macros.
722 // They make sure that the stack linkage is setup correctly. call_VM's correspond
723 // to ENTRY/ENTRY_X entry points while call_VM_leaf's correspond to LEAF entry points.
724
725
726 void call_VM(Register oop_result,
727 address entry_point,
728 bool check_exceptions = true);
729 void call_VM(Register oop_result,
730 address entry_point,
731 Register arg_1,
732 bool check_exceptions = true);
733 void call_VM(Register oop_result,
734 address entry_point,
735 Register arg_1, Register arg_2,
736 bool check_exceptions = true);
737 void call_VM(Register oop_result,
738 address entry_point,
739 Register arg_1, Register arg_2, Register arg_3,
740 bool check_exceptions = true);
741
742 // Overloadings with last_Java_sp
743 void call_VM(Register oop_result,
744 Register last_java_sp,
745 address entry_point,
746 int number_of_arguments = 0,
747 bool check_exceptions = true);
748 void call_VM(Register oop_result,
749 Register last_java_sp,
750 address entry_point,
751 Register arg_1, bool
752 check_exceptions = true);
753 void call_VM(Register oop_result,
754 Register last_java_sp,
755 address entry_point,
756 Register arg_1, Register arg_2,
757 bool check_exceptions = true);
758 void call_VM(Register oop_result,
759 Register last_java_sp,
760 address entry_point,
761 Register arg_1, Register arg_2, Register arg_3,
762 bool check_exceptions = true);
763
764 void get_vm_result (Register oop_result, Register thread);
765 void get_vm_result_2(Register metadata_result, Register thread);
766
767 // These always tightly bind to MacroAssembler::call_VM_base
768 // bypassing the virtual implementation
769 void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, int number_of_arguments = 0, bool check_exceptions = true);
770 void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, bool check_exceptions = true);
771 void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, Register arg_2, bool check_exceptions = true);
772 void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, Register arg_2, Register arg_3, bool check_exceptions = true);
773 void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, Register arg_2, Register arg_3, Register arg_4, bool check_exceptions = true);
774
775 void call_VM_leaf(address entry_point,
776 int number_of_arguments = 0);
777 void call_VM_leaf(address entry_point,
778 Register arg_1);
779 void call_VM_leaf(address entry_point,
780 Register arg_1, Register arg_2);
781 void call_VM_leaf(address entry_point,
782 Register arg_1, Register arg_2, Register arg_3);
783
784 // These always tightly bind to MacroAssembler::call_VM_leaf_base
785 // bypassing the virtual implementation
786 void super_call_VM_leaf(address entry_point);
787 void super_call_VM_leaf(address entry_point, Register arg_1);
788 void super_call_VM_leaf(address entry_point, Register arg_1, Register arg_2);
789 void super_call_VM_leaf(address entry_point, Register arg_1, Register arg_2, Register arg_3);
790 void super_call_VM_leaf(address entry_point, Register arg_1, Register arg_2, Register arg_3, Register arg_4);
791
792 // last Java Frame (fills frame anchor)
793 void set_last_Java_frame(Register last_java_sp,
794 Register last_java_fp,
795 address last_java_pc,
796 Register scratch);
797
798 void set_last_Java_frame(Register last_java_sp,
799 Register last_java_fp,
800 Label &last_java_pc,
801 Register scratch);
802
803 void set_last_Java_frame(Register last_java_sp,
804 Register last_java_fp,
805 Register last_java_pc,
806 Register scratch);
807
808 void reset_last_Java_frame(Register thread);
809
810 // thread in the default location (rthread)
811 void reset_last_Java_frame(bool clear_fp);
812
813 // Stores
814 void store_check(Register obj); // store check for obj - register is destroyed afterwards
815 void store_check(Register obj, Address dst); // same as above, dst is exact store location (reg. is destroyed)
816
817 void resolve_jobject(Register value, Register thread, Register tmp);
818
819 // C 'boolean' to Java boolean: x == 0 ? 0 : 1
820 void c2bool(Register x);
821
822 void load_method_holder_cld(Register rresult, Register rmethod);
823 void load_method_holder(Register holder, Register method);
824
825 // oop manipulations
826 void load_nklass(Register dst, Register src);
827 void load_klass(Register dst, Register src, bool null_check = false);
828 void store_klass(Register dst, Register src);
829 void cmp_klass(Register oop, Register trial_klass, Register tmp);
830
831 void resolve_weak_handle(Register result, Register tmp);
832 void resolve_oop_handle(Register result, Register tmp = r5);
833 void load_mirror(Register dst, Register method, Register tmp = r5);
834
835 void access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
836 Register tmp1, Register tmp_thread);
837
838 void access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register src,
839 Register tmp1, Register tmp_thread);
840
841 void load_heap_oop(Register dst, Address src, Register tmp1 = noreg,
842 Register thread_tmp = noreg, DecoratorSet decorators = 0);
843
844 void load_heap_oop_not_null(Register dst, Address src, Register tmp1 = noreg,
845 Register thread_tmp = noreg, DecoratorSet decorators = 0);
846 void store_heap_oop(Address dst, Register src, Register tmp1 = noreg,
847 Register tmp_thread = noreg, DecoratorSet decorators = 0);
848
849 // currently unimplemented
850 // Used for storing NULL. All other oop constants should be
851 // stored using routines that take a jobject.
852 void store_heap_oop_null(Address dst);
853
854 void store_klass_gap(Register dst, Register src);
855
856 void load_prototype_header(Register dst, Register src);
857
858 // This dummy is to prevent a call to store_heap_oop from
859 // converting a zero (like NULL) into a Register by giving
860 // the compiler two choices it can't resolve
861
862 void store_heap_oop(Address dst, void* dummy);
863
864 void encode_heap_oop(Register d, Register s);
865 void encode_heap_oop(Register r) { encode_heap_oop(r, r); }
866 void decode_heap_oop(Register d, Register s);
867 void decode_heap_oop(Register r) { decode_heap_oop(r, r); }
868 void encode_heap_oop_not_null(Register r);
869 void decode_heap_oop_not_null(Register r);
870 void encode_heap_oop_not_null(Register dst, Register src);
871 void decode_heap_oop_not_null(Register dst, Register src);
872
873 void set_narrow_oop(Register dst, jobject obj);
874
875 void encode_klass_not_null(Register r);
876 void decode_klass_not_null(Register r);
877 void encode_klass_not_null(Register dst, Register src);
878 void decode_klass_not_null(Register dst, Register src);
879
880 void set_narrow_klass(Register dst, Klass* k);
881
882 // if heap base register is used - reinit it with the correct value
883 void reinit_heapbase();
884
885 DEBUG_ONLY(void verify_heapbase(const char* msg);)
886
887 void push_CPU_state(bool save_vectors = false, bool use_sve = false,
888 int sve_vector_size_in_bytes = 0);
889 void pop_CPU_state(bool restore_vectors = false, bool use_sve = false,
890 int sve_vector_size_in_bytes = 0);
891
892 // Round up to a power of two
893 void round_to(Register reg, int modulus);
894
895 // allocation
896 void eden_allocate(
897 Register obj, // result: pointer to object after successful allocation
898 Register var_size_in_bytes, // object size in bytes if unknown at compile time; invalid otherwise
899 int con_size_in_bytes, // object size in bytes if known at compile time
900 Register t1, // temp register
901 Label& slow_case // continuation point if fast allocation fails
902 );
903 void tlab_allocate(
904 Register obj, // result: pointer to object after successful allocation
905 Register var_size_in_bytes, // object size in bytes if unknown at compile time; invalid otherwise
906 int con_size_in_bytes, // object size in bytes if known at compile time
907 Register t1, // temp register
908 Register t2, // temp register
909 Label& slow_case // continuation point if fast allocation fails
910 );
911 void verify_tlab();
912
913 // interface method calling
914 void lookup_interface_method(Register recv_klass,
915 Register intf_klass,
916 RegisterOrConstant itable_index,
917 Register method_result,
918 Register scan_temp,
919 Label& no_such_interface,
920 bool return_method = true);
921
922 // virtual method calling
923 // n.b. x86 allows RegisterOrConstant for vtable_index
924 void lookup_virtual_method(Register recv_klass,
925 RegisterOrConstant vtable_index,
926 Register method_result);
927
928 // Test sub_klass against super_klass, with fast and slow paths.
929
930 // The fast path produces a tri-state answer: yes / no / maybe-slow.
931 // One of the three labels can be NULL, meaning take the fall-through.
932 // If super_check_offset is -1, the value is loaded up from super_klass.
933 // No registers are killed, except temp_reg.
934 void check_klass_subtype_fast_path(Register sub_klass,
935 Register super_klass,
936 Register temp_reg,
937 Label* L_success,
938 Label* L_failure,
939 Label* L_slow_path,
940 RegisterOrConstant super_check_offset = RegisterOrConstant(-1));
941
942 // The rest of the type check; must be wired to a corresponding fast path.
943 // It does not repeat the fast path logic, so don't use it standalone.
944 // The temp_reg and temp2_reg can be noreg, if no temps are available.
945 // Updates the sub's secondary super cache as necessary.
946 // If set_cond_codes, condition codes will be Z on success, NZ on failure.
947 void check_klass_subtype_slow_path(Register sub_klass,
948 Register super_klass,
949 Register temp_reg,
950 Register temp2_reg,
951 Label* L_success,
952 Label* L_failure,
953 bool set_cond_codes = false);
954
955 // Simplified, combined version, good for typical uses.
956 // Falls through on failure.
957 void check_klass_subtype(Register sub_klass,
958 Register super_klass,
959 Register temp_reg,
960 Label& L_success);
961
962 void clinit_barrier(Register klass,
963 Register thread,
964 Label* L_fast_path = NULL,
965 Label* L_slow_path = NULL);
966
967 Address argument_address(RegisterOrConstant arg_slot, int extra_slot_offset = 0);
968
969 void verify_sve_vector_length();
970 void reinitialize_ptrue() {
971 if (UseSVE > 0) {
972 sve_ptrue(ptrue, B);
973 }
974 }
975 void verify_ptrue();
976
977 // Debugging
978
979 // only if +VerifyOops
980 void verify_oop(Register reg, const char* s = "broken oop");
981 void verify_oop_addr(Address addr, const char * s = "broken oop addr");
982
983 // TODO: verify method and klass metadata (compare against vptr?)
984 void _verify_method_ptr(Register reg, const char * msg, const char * file, int line) {}
985 void _verify_klass_ptr(Register reg, const char * msg, const char * file, int line){}
986
987 #define verify_method_ptr(reg) _verify_method_ptr(reg, "broken method " #reg, __FILE__, __LINE__)
988 #define verify_klass_ptr(reg) _verify_klass_ptr(reg, "broken klass " #reg, __FILE__, __LINE__)
989
990 // only if +VerifyFPU
991 void verify_FPU(int stack_depth, const char* s = "illegal FPU state");
992
993 // prints msg, dumps registers and stops execution
994 void stop(const char* msg);
995
996 static void debug64(char* msg, int64_t pc, int64_t regs[]);
997
998 void untested() { stop("untested"); }
999
1000 void unimplemented(const char* what = "");
1001
1002 void should_not_reach_here() { stop("should not reach here"); }
1003
1004 // Stack overflow checking
1005 void bang_stack_with_offset(int offset) {
1006 // stack grows down, caller passes positive offset
1007 assert(offset > 0, "must bang with negative offset");
1008 sub(rscratch2, sp, offset);
1009 str(zr, Address(rscratch2));
1010 }
1011
1012 // Writes to stack successive pages until offset reached to check for
1013 // stack overflow + shadow pages. Also, clobbers tmp
1014 void bang_stack_size(Register size, Register tmp);
1015
1016 // Check for reserved stack access in method being exited (for JIT)
1017 void reserved_stack_check();
1018
1019 // Arithmetics
1020
1021 void addptr(const Address &dst, int32_t src);
1022 void cmpptr(Register src1, Address src2);
1023
1024 void cmpoop(Register obj1, Register obj2);
1025
1026 // Various forms of CAS
1027
1028 void cmpxchg_obj_header(Register oldv, Register newv, Register obj, Register tmp,
1029 Label &suceed, Label *fail);
1030 void cmpxchgptr(Register oldv, Register newv, Register addr, Register tmp,
1031 Label &suceed, Label *fail);
1032
1033 void cmpxchgw(Register oldv, Register newv, Register addr, Register tmp,
1034 Label &suceed, Label *fail);
1035
1036 void atomic_add(Register prev, RegisterOrConstant incr, Register addr);
1037 void atomic_addw(Register prev, RegisterOrConstant incr, Register addr);
1038 void atomic_addal(Register prev, RegisterOrConstant incr, Register addr);
1039 void atomic_addalw(Register prev, RegisterOrConstant incr, Register addr);
1040
1041 void atomic_xchg(Register prev, Register newv, Register addr);
1042 void atomic_xchgw(Register prev, Register newv, Register addr);
1043 void atomic_xchgl(Register prev, Register newv, Register addr);
1044 void atomic_xchglw(Register prev, Register newv, Register addr);
1045 void atomic_xchgal(Register prev, Register newv, Register addr);
1046 void atomic_xchgalw(Register prev, Register newv, Register addr);
1047
1048 void orptr(Address adr, RegisterOrConstant src) {
1049 ldr(rscratch1, adr);
1050 if (src.is_register())
1051 orr(rscratch1, rscratch1, src.as_register());
1052 else
1053 orr(rscratch1, rscratch1, src.as_constant());
1054 str(rscratch1, adr);
1055 }
1056
1057 // A generic CAS; success or failure is in the EQ flag.
1058 // Clobbers rscratch1
1059 void cmpxchg(Register addr, Register expected, Register new_val,
1060 enum operand_size size,
1061 bool acquire, bool release, bool weak,
1062 Register result);
1063
1064 private:
1065 void compare_eq(Register rn, Register rm, enum operand_size size);
1066
1067 #ifdef ASSERT
1068 // Template short-hand support to clean-up after a failed call to trampoline
1069 // call generation (see trampoline_call() below), when a set of Labels must
1070 // be reset (before returning).
1071 template<typename Label, typename... More>
1072 void reset_labels(Label &lbl, More&... more) {
1073 lbl.reset(); reset_labels(more...);
1074 }
1075 template<typename Label>
1076 void reset_labels(Label &lbl) {
1077 lbl.reset();
1078 }
1079 #endif
1080
1081 public:
1082 // Calls
1083
1084 address trampoline_call(Address entry, CodeBuffer* cbuf = NULL);
1085
1086 static bool far_branches() {
1087 return ReservedCodeCacheSize > branch_range;
1088 }
1089
1090 // Check if branches to the the non nmethod section require a far jump
1091 static bool codestub_branch_needs_far_jump() {
1092 return CodeCache::max_distance_to_non_nmethod() > branch_range;
1093 }
1094
1095 // Jumps that can reach anywhere in the code cache.
1096 // Trashes tmp.
1097 void far_call(Address entry, CodeBuffer *cbuf = NULL, Register tmp = rscratch1);
1098 int far_jump(Address entry, CodeBuffer *cbuf = NULL, Register tmp = rscratch1);
1099
1100 static int far_codestub_branch_size() {
1101 if (codestub_branch_needs_far_jump()) {
1102 return 3 * 4; // adrp, add, br
1103 } else {
1104 return 4;
1105 }
1106 }
1107
1108 // Emit the CompiledIC call idiom
1109 address ic_call(address entry, jint method_index = 0);
1110
1111 public:
1112
1113 // Data
1114
1115 void mov_metadata(Register dst, Metadata* obj);
1116 Address allocate_metadata_address(Metadata* obj);
1117 Address constant_oop_address(jobject obj);
1118
1119 void movoop(Register dst, jobject obj, bool immediate = false);
1120
1121 // CRC32 code for java.util.zip.CRC32::updateBytes() instrinsic.
1122 void kernel_crc32(Register crc, Register buf, Register len,
1123 Register table0, Register table1, Register table2, Register table3,
1124 Register tmp, Register tmp2, Register tmp3);
1125 // CRC32 code for java.util.zip.CRC32C::updateBytes() instrinsic.
1126 void kernel_crc32c(Register crc, Register buf, Register len,
1127 Register table0, Register table1, Register table2, Register table3,
1128 Register tmp, Register tmp2, Register tmp3);
1129
1130 // Stack push and pop individual 64 bit registers
1131 void push(Register src);
1132 void pop(Register dst);
1133
1134 void repne_scan(Register addr, Register value, Register count,
1135 Register scratch);
1136 void repne_scanw(Register addr, Register value, Register count,
1137 Register scratch);
1138
1139 typedef void (MacroAssembler::* add_sub_imm_insn)(Register Rd, Register Rn, unsigned imm);
1140 typedef void (MacroAssembler::* add_sub_reg_insn)(Register Rd, Register Rn, Register Rm, enum shift_kind kind, unsigned shift);
1141
1142 // If a constant does not fit in an immediate field, generate some
1143 // number of MOV instructions and then perform the operation
1144 void wrap_add_sub_imm_insn(Register Rd, Register Rn, uint64_t imm,
1145 add_sub_imm_insn insn1,
1146 add_sub_reg_insn insn2, bool is32);
1147 // Seperate vsn which sets the flags
1148 void wrap_adds_subs_imm_insn(Register Rd, Register Rn, uint64_t imm,
1149 add_sub_imm_insn insn1,
1150 add_sub_reg_insn insn2, bool is32);
1151
1152 #define WRAP(INSN, is32) \
1153 void INSN(Register Rd, Register Rn, uint64_t imm) { \
1154 wrap_add_sub_imm_insn(Rd, Rn, imm, &Assembler::INSN, &Assembler::INSN, is32); \
1155 } \
1156 \
1157 void INSN(Register Rd, Register Rn, Register Rm, \
1158 enum shift_kind kind, unsigned shift = 0) { \
1159 Assembler::INSN(Rd, Rn, Rm, kind, shift); \
1160 } \
1161 \
1162 void INSN(Register Rd, Register Rn, Register Rm) { \
1163 Assembler::INSN(Rd, Rn, Rm); \
1164 } \
1165 \
1166 void INSN(Register Rd, Register Rn, Register Rm, \
1167 ext::operation option, int amount = 0) { \
1168 Assembler::INSN(Rd, Rn, Rm, option, amount); \
1169 }
1170
1171 WRAP(add, false) WRAP(addw, true) WRAP(sub, false) WRAP(subw, true)
1172
1173 #undef WRAP
1174 #define WRAP(INSN, is32) \
1175 void INSN(Register Rd, Register Rn, uint64_t imm) { \
1176 wrap_adds_subs_imm_insn(Rd, Rn, imm, &Assembler::INSN, &Assembler::INSN, is32); \
1177 } \
1178 \
1179 void INSN(Register Rd, Register Rn, Register Rm, \
1180 enum shift_kind kind, unsigned shift = 0) { \
1181 Assembler::INSN(Rd, Rn, Rm, kind, shift); \
1182 } \
1183 \
1184 void INSN(Register Rd, Register Rn, Register Rm) { \
1185 Assembler::INSN(Rd, Rn, Rm); \
1186 } \
1187 \
1188 void INSN(Register Rd, Register Rn, Register Rm, \
1189 ext::operation option, int amount = 0) { \
1190 Assembler::INSN(Rd, Rn, Rm, option, amount); \
1191 }
1192
1193 WRAP(adds, false) WRAP(addsw, true) WRAP(subs, false) WRAP(subsw, true)
1194
1195 void add(Register Rd, Register Rn, RegisterOrConstant increment);
1196 void addw(Register Rd, Register Rn, RegisterOrConstant increment);
1197 void sub(Register Rd, Register Rn, RegisterOrConstant decrement);
1198 void subw(Register Rd, Register Rn, RegisterOrConstant decrement);
1199
1200 void adrp(Register reg1, const Address &dest, uint64_t &byte_offset);
1201
1202 void tableswitch(Register index, jint lowbound, jint highbound,
1203 Label &jumptable, Label &jumptable_end, int stride = 1) {
1204 adr(rscratch1, jumptable);
1205 subsw(rscratch2, index, lowbound);
1206 subsw(zr, rscratch2, highbound - lowbound);
1207 br(Assembler::HS, jumptable_end);
1208 add(rscratch1, rscratch1, rscratch2,
1209 ext::sxtw, exact_log2(stride * Assembler::instruction_size));
1210 br(rscratch1);
1211 }
1212
1213 // Form an address from base + offset in Rd. Rd may or may not
1214 // actually be used: you must use the Address that is returned. It
1215 // is up to you to ensure that the shift provided matches the size
1216 // of your data.
1217 Address form_address(Register Rd, Register base, int64_t byte_offset, int shift);
1218
1219 // Return true iff an address is within the 48-bit AArch64 address
1220 // space.
1221 bool is_valid_AArch64_address(address a) {
1222 return ((uint64_t)a >> 48) == 0;
1223 }
1224
1225 // Load the base of the cardtable byte map into reg.
1226 void load_byte_map_base(Register reg);
1227
1228 // Prolog generator routines to support switch between x86 code and
1229 // generated ARM code
1230
1231 // routine to generate an x86 prolog for a stub function which
1232 // bootstraps into the generated ARM code which directly follows the
1233 // stub
1234 //
1235
1236 public:
1237
1238 void ldr_constant(Register dest, const Address &const_addr) {
1239 if (NearCpool) {
1240 ldr(dest, const_addr);
1241 } else {
1242 uint64_t offset;
1243 adrp(dest, InternalAddress(const_addr.target()), offset);
1244 ldr(dest, Address(dest, offset));
1245 }
1246 }
1247
1248 address read_polling_page(Register r, relocInfo::relocType rtype);
1249 void get_polling_page(Register dest, relocInfo::relocType rtype);
1250
1251 // CRC32 code for java.util.zip.CRC32::updateBytes() instrinsic.
1252 void update_byte_crc32(Register crc, Register val, Register table);
1253 void update_word_crc32(Register crc, Register v, Register tmp,
1254 Register table0, Register table1, Register table2, Register table3,
1255 bool upper = false);
1256
1257 address has_negatives(Register ary1, Register len, Register result);
1258
1259 address arrays_equals(Register a1, Register a2, Register result, Register cnt1,
1260 Register tmp1, Register tmp2, Register tmp3, int elem_size);
1261
1262 void string_equals(Register a1, Register a2, Register result, Register cnt1,
1263 int elem_size);
1264
1265 void fill_words(Register base, Register cnt, Register value);
1266 address zero_words(Register base, uint64_t cnt);
1267 address zero_words(Register ptr, Register cnt);
1268 void zero_dcache_blocks(Register base, Register cnt);
1269
1270 static const int zero_words_block_size;
1271
1272 address byte_array_inflate(Register src, Register dst, Register len,
1273 FloatRegister vtmp1, FloatRegister vtmp2,
1274 FloatRegister vtmp3, Register tmp4);
1275
1276 void char_array_compress(Register src, Register dst, Register len,
1277 Register res,
1278 FloatRegister vtmp0, FloatRegister vtmp1,
1279 FloatRegister vtmp2, FloatRegister vtmp3,
1280 FloatRegister vtmp4, FloatRegister vtmp5);
1281
1282 void encode_iso_array(Register src, Register dst,
1283 Register len, Register res, bool ascii,
1284 FloatRegister vtmp0, FloatRegister vtmp1,
1285 FloatRegister vtmp2, FloatRegister vtmp3,
1286 FloatRegister vtmp4, FloatRegister vtmp5);
1287
1288 void fast_log(FloatRegister vtmp0, FloatRegister vtmp1, FloatRegister vtmp2,
1289 FloatRegister vtmp3, FloatRegister vtmp4, FloatRegister vtmp5,
1290 FloatRegister tmpC1, FloatRegister tmpC2, FloatRegister tmpC3,
1291 FloatRegister tmpC4, Register tmp1, Register tmp2,
1292 Register tmp3, Register tmp4, Register tmp5);
1293 void generate_dsin_dcos(bool isCos, address npio2_hw, address two_over_pi,
1294 address pio2, address dsin_coef, address dcos_coef);
1295 private:
1296 // begin trigonometric functions support block
1297 void generate__ieee754_rem_pio2(address npio2_hw, address two_over_pi, address pio2);
1298 void generate__kernel_rem_pio2(address two_over_pi, address pio2);
1299 void generate_kernel_sin(FloatRegister x, bool iyIsOne, address dsin_coef);
1300 void generate_kernel_cos(FloatRegister x, address dcos_coef);
1301 // end trigonometric functions support block
1302 void add2_with_carry(Register final_dest_hi, Register dest_hi, Register dest_lo,
1303 Register src1, Register src2);
1304 void add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2) {
1305 add2_with_carry(dest_hi, dest_hi, dest_lo, src1, src2);
1306 }
1307 void multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart,
1308 Register y, Register y_idx, Register z,
1309 Register carry, Register product,
1310 Register idx, Register kdx);
1311 void multiply_128_x_128_loop(Register y, Register z,
1312 Register carry, Register carry2,
1313 Register idx, Register jdx,
1314 Register yz_idx1, Register yz_idx2,
1315 Register tmp, Register tmp3, Register tmp4,
1316 Register tmp7, Register product_hi);
1317 void kernel_crc32_using_crc32(Register crc, Register buf,
1318 Register len, Register tmp0, Register tmp1, Register tmp2,
1319 Register tmp3);
1320 void kernel_crc32c_using_crc32c(Register crc, Register buf,
1321 Register len, Register tmp0, Register tmp1, Register tmp2,
1322 Register tmp3);
1323 public:
1324 void multiply_to_len(Register x, Register xlen, Register y, Register ylen, Register z,
1325 Register zlen, Register tmp1, Register tmp2, Register tmp3,
1326 Register tmp4, Register tmp5, Register tmp6, Register tmp7);
1327 void mul_add(Register out, Register in, Register offs, Register len, Register k);
1328 void ghash_multiply(FloatRegister result_lo, FloatRegister result_hi,
1329 FloatRegister a, FloatRegister b, FloatRegister a1_xor_a0,
1330 FloatRegister tmp1, FloatRegister tmp2, FloatRegister tmp3);
1331 void ghash_reduce(FloatRegister result, FloatRegister lo, FloatRegister hi,
1332 FloatRegister p, FloatRegister z, FloatRegister t1);
1333 void ghash_processBlocks_wide(address p, Register state, Register subkeyH,
1334 Register data, Register blocks, int unrolls);
1335 void ghash_modmul (FloatRegister result,
1336 FloatRegister result_lo, FloatRegister result_hi, FloatRegister b,
1337 FloatRegister a, FloatRegister vzr, FloatRegister a1_xor_a0, FloatRegister p,
1338 FloatRegister t1, FloatRegister t2, FloatRegister t3);
1339
1340 void aesenc_loadkeys(Register key, Register keylen);
1341 void aesecb_encrypt(Register from, Register to, Register keylen,
1342 FloatRegister data = v0, int unrolls = 1);
1343 void aesecb_decrypt(Register from, Register to, Register key, Register keylen);
1344 void aes_round(FloatRegister input, FloatRegister subkey);
1345
1346 // Place an ISB after code may have been modified due to a safepoint.
1347 void safepoint_isb();
1348
1349 private:
1350 // Return the effective address r + (r1 << ext) + offset.
1351 // Uses rscratch2.
1352 Address offsetted_address(Register r, Register r1, Address::extend ext,
1353 int offset, int size);
1354
1355 private:
1356 // Returns an address on the stack which is reachable with a ldr/str of size
1357 // Uses rscratch2 if the address is not directly reachable
1358 Address spill_address(int size, int offset, Register tmp=rscratch2);
1359 Address sve_spill_address(int sve_reg_size_in_bytes, int offset, Register tmp=rscratch2);
1360
1361 bool merge_alignment_check(Register base, size_t size, int64_t cur_offset, int64_t prev_offset) const;
1362
1363 // Check whether two loads/stores can be merged into ldp/stp.
1364 bool ldst_can_merge(Register rx, const Address &adr, size_t cur_size_in_bytes, bool is_store) const;
1365
1366 // Merge current load/store with previous load/store into ldp/stp.
1367 void merge_ldst(Register rx, const Address &adr, size_t cur_size_in_bytes, bool is_store);
1368
1369 // Try to merge two loads/stores into ldp/stp. If success, returns true else false.
1370 bool try_merge_ldst(Register rt, const Address &adr, size_t cur_size_in_bytes, bool is_store);
1371
1372 public:
1373 void spill(Register Rx, bool is64, int offset) {
1374 if (is64) {
1375 str(Rx, spill_address(8, offset));
1376 } else {
1377 strw(Rx, spill_address(4, offset));
1378 }
1379 }
1380 void spill(FloatRegister Vx, SIMD_RegVariant T, int offset) {
1381 str(Vx, T, spill_address(1 << (int)T, offset));
1382 }
1383 void spill_sve_vector(FloatRegister Zx, int offset, int vector_reg_size_in_bytes) {
1384 sve_str(Zx, sve_spill_address(vector_reg_size_in_bytes, offset));
1385 }
1386 void unspill(Register Rx, bool is64, int offset) {
1387 if (is64) {
1388 ldr(Rx, spill_address(8, offset));
1389 } else {
1390 ldrw(Rx, spill_address(4, offset));
1391 }
1392 }
1393 void unspill(FloatRegister Vx, SIMD_RegVariant T, int offset) {
1394 ldr(Vx, T, spill_address(1 << (int)T, offset));
1395 }
1396 void unspill_sve_vector(FloatRegister Zx, int offset, int vector_reg_size_in_bytes) {
1397 sve_ldr(Zx, sve_spill_address(vector_reg_size_in_bytes, offset));
1398 }
1399 void spill_copy128(int src_offset, int dst_offset,
1400 Register tmp1=rscratch1, Register tmp2=rscratch2) {
1401 if (src_offset < 512 && (src_offset & 7) == 0 &&
1402 dst_offset < 512 && (dst_offset & 7) == 0) {
1403 ldp(tmp1, tmp2, Address(sp, src_offset));
1404 stp(tmp1, tmp2, Address(sp, dst_offset));
1405 } else {
1406 unspill(tmp1, true, src_offset);
1407 spill(tmp1, true, dst_offset);
1408 unspill(tmp1, true, src_offset+8);
1409 spill(tmp1, true, dst_offset+8);
1410 }
1411 }
1412 void spill_copy_sve_vector_stack_to_stack(int src_offset, int dst_offset,
1413 int sve_vec_reg_size_in_bytes) {
1414 assert(sve_vec_reg_size_in_bytes % 16 == 0, "unexpected sve vector reg size");
1415 for (int i = 0; i < sve_vec_reg_size_in_bytes / 16; i++) {
1416 spill_copy128(src_offset, dst_offset);
1417 src_offset += 16;
1418 dst_offset += 16;
1419 }
1420 }
1421 void cache_wb(Address line);
1422 void cache_wbsync(bool is_pre);
1423
1424 // Code for java.lang.Thread::onSpinWait() intrinsic.
1425 void spin_wait();
1426
1427 void lightweight_lock(Register obj, Register t1, Register t2, Register t3, Label& slow);
1428 void lightweight_unlock(Register obj, Register t1, Register t2, Register t3, Label& slow);
1429
1430 private:
1431 // Check the current thread doesn't need a cross modify fence.
1432 void verify_cross_modify_fence_not_required() PRODUCT_RETURN;
1433
1434 };
1435
1436 #ifdef ASSERT
1437 inline bool AbstractAssembler::pd_check_instruction_mark() { return false; }
1438 #endif
1439
1440 /**
1441 * class SkipIfEqual:
1442 *
1443 * Instantiating this class will result in assembly code being output that will
1444 * jump around any code emitted between the creation of the instance and it's
1445 * automatic destruction at the end of a scope block, depending on the value of
1446 * the flag passed to the constructor, which will be checked at run-time.
1447 */
1448 class SkipIfEqual {
1449 private:
1450 MacroAssembler* _masm;
1451 Label _label;
1452
1453 public:
1454 SkipIfEqual(MacroAssembler*, const bool* flag_addr, bool value);
1455 ~SkipIfEqual();
1456 };
1457
1458 struct tableswitch {
1459 Register _reg;
1460 int _insn_index; jint _first_key; jint _last_key;
1461 Label _after;
1462 Label _branches;
1463 };
1464
1465 #endif // CPU_AARCH64_MACROASSEMBLER_AARCH64_HPP