1 /*
2 * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
4 * Copyright (c) 2020, 2024, Huawei Technologies Co., Ltd. All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #ifndef CPU_RISCV_MACROASSEMBLER_RISCV_HPP
28 #define CPU_RISCV_MACROASSEMBLER_RISCV_HPP
29
30 #include "asm/assembler.inline.hpp"
31 #include "code/vmreg.hpp"
32 #include "metaprogramming/enableIf.hpp"
33 #include "oops/compressedOops.hpp"
34 #include "utilities/powerOfTwo.hpp"
35 #include "runtime/signature.hpp"
36
37 class ciInlineKlass;
38
39 // MacroAssembler extends Assembler by frequently used macros.
40 //
41 // Instructions for which a 'better' code sequence exists depending
42 // on arguments should also go in here.
43
44 class MacroAssembler: public Assembler {
45
46 public:
47
48 MacroAssembler(CodeBuffer* code) : Assembler(code) {}
49
50 void safepoint_poll(Label& slow_path, bool at_return, bool in_nmethod, Register tmp_reg = t0);
51
52 // Alignment
53 int align(int modulus, int extra_offset = 0);
54
55 static inline void assert_alignment(address pc, int alignment = MacroAssembler::instruction_size) {
56 assert(is_aligned(pc, alignment), "bad alignment");
57 }
58
59 // nop
60 void post_call_nop();
61
62 // Stack frame creation/removal
63 // Note that SP must be updated to the right place before saving/restoring RA and FP
64 // because signal based thread suspend/resume could happen asynchronously.
65 void enter() {
66 subi(sp, sp, 2 * wordSize);
67 sd(ra, Address(sp, wordSize));
68 sd(fp, Address(sp));
69 addi(fp, sp, 2 * wordSize);
70 }
71
72 void leave() {
73 subi(sp, fp, 2 * wordSize);
74 ld(fp, Address(sp));
75 ld(ra, Address(sp, wordSize));
76 addi(sp, sp, 2 * wordSize);
77 }
78
79
80 // Support for getting the JavaThread pointer (i.e.; a reference to thread-local information)
81 // The pointer will be loaded into the thread register.
82 void get_thread(Register thread);
83
84 // Support for VM calls
85 //
86 // It is imperative that all calls into the VM are handled via the call_VM macros.
87 // They make sure that the stack linkage is setup correctly. call_VM's correspond
88 // to ENTRY/ENTRY_X entry points while call_VM_leaf's correspond to LEAF entry points.
89
90 void call_VM(Register oop_result,
91 address entry_point,
92 bool check_exceptions = true);
93 void call_VM(Register oop_result,
94 address entry_point,
95 Register arg_1,
96 bool check_exceptions = true);
97 void call_VM(Register oop_result,
98 address entry_point,
99 Register arg_1, Register arg_2,
100 bool check_exceptions = true);
101 void call_VM(Register oop_result,
102 address entry_point,
103 Register arg_1, Register arg_2, Register arg_3,
104 bool check_exceptions = true);
105
106 // Overloadings with last_Java_sp
107 void call_VM(Register oop_result,
108 Register last_java_sp,
109 address entry_point,
110 int number_of_arguments = 0,
111 bool check_exceptions = true);
112 void call_VM(Register oop_result,
113 Register last_java_sp,
114 address entry_point,
115 Register arg_1,
116 bool check_exceptions = true);
117 void call_VM(Register oop_result,
118 Register last_java_sp,
119 address entry_point,
120 Register arg_1, Register arg_2,
121 bool check_exceptions = true);
122 void call_VM(Register oop_result,
123 Register last_java_sp,
124 address entry_point,
125 Register arg_1, Register arg_2, Register arg_3,
126 bool check_exceptions = true);
127
128 void get_vm_result_oop(Register oop_result, Register java_thread);
129 void get_vm_result_metadata(Register metadata_result, Register java_thread);
130
131 // These always tightly bind to MacroAssembler::call_VM_leaf_base
132 // bypassing the virtual implementation
133 void call_VM_leaf(address entry_point,
134 int number_of_arguments = 0);
135 void call_VM_leaf(address entry_point,
136 Register arg_0);
137 void call_VM_leaf(address entry_point,
138 Register arg_0, Register arg_1);
139 void call_VM_leaf(address entry_point,
140 Register arg_0, Register arg_1, Register arg_2);
141
142 // These always tightly bind to MacroAssembler::call_VM_base
143 // bypassing the virtual implementation
144 void super_call_VM_leaf(address entry_point);
145 void super_call_VM_leaf(address entry_point, Register arg_0);
146 void super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1);
147 void super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2);
148 void super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3);
149
150 // last Java Frame (fills frame anchor)
151 void set_last_Java_frame(Register last_java_sp, Register last_java_fp, address last_java_pc, Register tmp);
152 void set_last_Java_frame(Register last_java_sp, Register last_java_fp, Label &last_java_pc, Register tmp);
153 void set_last_Java_frame(Register last_java_sp, Register last_java_fp, Register last_java_pc);
154
155 // thread in the default location (xthread)
156 void reset_last_Java_frame(bool clear_fp);
157
158 virtual void call_VM_leaf_base(
159 address entry_point, // the entry point
160 int number_of_arguments, // the number of arguments to pop after the call
161 Label* retaddr = nullptr
162 );
163
164 virtual void call_VM_leaf_base(
165 address entry_point, // the entry point
166 int number_of_arguments, // the number of arguments to pop after the call
167 Label& retaddr) {
168 call_VM_leaf_base(entry_point, number_of_arguments, &retaddr);
169 }
170
171 virtual void call_VM_base( // returns the register containing the thread upon return
172 Register oop_result, // where an oop-result ends up if any; use noreg otherwise
173 Register java_thread, // the thread if computed before ; use noreg otherwise
174 Register last_java_sp, // to set up last_Java_frame in stubs; use noreg otherwise
175 Label* return_pc, // to set up last_Java_frame; use nullptr otherwise
176 address entry_point, // the entry point
177 int number_of_arguments, // the number of arguments (w/o thread) to pop after the call
178 bool check_exceptions // whether to check for pending exceptions after return
179 );
180
181 void call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions);
182
183 virtual void check_and_handle_earlyret(Register java_thread);
184 virtual void check_and_handle_popframe(Register java_thread);
185
186 void resolve_weak_handle(Register result, Register tmp1, Register tmp2);
187 void resolve_oop_handle(Register result, Register tmp1, Register tmp2);
188 void resolve_jobject(Register value, Register tmp1, Register tmp2);
189 void resolve_global_jobject(Register value, Register tmp1, Register tmp2);
190
191 void movoop(Register dst, jobject obj);
192 void mov_metadata(Register dst, Metadata* obj);
193 void bang_stack_size(Register size, Register tmp);
194 void set_narrow_oop(Register dst, jobject obj);
195 void set_narrow_klass(Register dst, Klass* k);
196
197 void load_mirror(Register dst, Register method, Register tmp1, Register tmp2);
198 void access_load_at(BasicType type, DecoratorSet decorators, Register dst,
199 Address src, Register tmp1, Register tmp2);
200 void access_store_at(BasicType type, DecoratorSet decorators, Address dst,
201 Register val, Register tmp1, Register tmp2, Register tmp3);
202 void load_klass(Register dst, Register src, Register tmp = t0);
203 void load_prototype_header(Register dst, Register src, Register tmp = t0);
204 void load_narrow_klass_compact(Register dst, Register src);
205 void load_narrow_klass(Register dst, Register src);
206 void store_klass(Register dst, Register src, Register tmp = t0);
207 void cmp_klass_beq(Register obj, Register klass,
208 Register tmp1, Register tmp2,
209 Label &L, bool is_far = false);
210 void cmp_klass_bne(Register obj, Register klass,
211 Register tmp1, Register tmp2,
212 Label &L, bool is_far = false);
213
214 void encode_klass_not_null(Register r, Register tmp = t0);
215 void decode_klass_not_null(Register r, Register tmp = t0);
216 void encode_klass_not_null(Register dst, Register src, Register tmp);
217 void decode_klass_not_null(Register dst, Register src, Register tmp);
218 void decode_heap_oop_not_null(Register r);
219 void decode_heap_oop_not_null(Register dst, Register src);
220 void decode_heap_oop(Register d, Register s);
221 void decode_heap_oop(Register r) { decode_heap_oop(r, r); }
222 void encode_heap_oop_not_null(Register r);
223 void encode_heap_oop_not_null(Register dst, Register src);
224 void encode_heap_oop(Register d, Register s);
225 void encode_heap_oop(Register r) { encode_heap_oop(r, r); };
226 void load_heap_oop(Register dst, Address src, Register tmp1,
227 Register tmp2, DecoratorSet decorators = 0);
228 void load_heap_oop_not_null(Register dst, Address src, Register tmp1,
229 Register tmp2, DecoratorSet decorators = 0);
230 void store_heap_oop(Address dst, Register val, Register tmp1,
231 Register tmp2, Register tmp3, DecoratorSet decorators = 0);
232
233 void store_klass_gap(Register dst, Register src);
234
235 // currently unimplemented
236 // Used for storing null. All other oop constants should be
237 // stored using routines that take a jobject.
238 void store_heap_oop_null(Address dst);
239
240 // This dummy is to prevent a call to store_heap_oop from
241 // converting a zero (linked null) into a Register by giving
242 // the compiler two choices it can't resolve
243
244 void store_heap_oop(Address dst, void* dummy);
245
246 // Support for null-checks
247 //
248 // Generates code that causes a null OS exception if the content of reg is null.
249 // If the accessed location is M[reg + offset] and the offset is known, provide the
250 // offset. No explicit code generateion is needed if the offset is within a certain
251 // range (0 <= offset <= page_size).
252
253 virtual void null_check(Register reg, int offset = -1);
254 static bool needs_explicit_null_check(intptr_t offset);
255 static bool uses_implicit_null_check(void* address);
256
257 void test_field_is_null_free_inline_type(Register flags, Register temp_reg, Label& is_null_free);
258 void test_field_is_not_null_free_inline_type(Register flags, Register temp_reg, Label& not_null_free_inline_type);
259 void test_field_is_flat(Register flags, Register temp_reg, Label& is_flat);
260
261 void test_markword_is_inline_type(Register markword, Label& is_inline_type);
262 void test_oop_is_not_inline_type(Register object, Register tmp, Label& not_inline_type, bool can_be_null = true);
263 void test_oop_prototype_bit(Register oop, Register temp_reg, int32_t tst_bit, bool jmp_set, Label& jmp_label);
264 void test_flat_array_oop(Register klass, Register temp_reg, Label& is_flat_array);
265 void test_null_free_array_oop(Register oop, Register temp_reg, Label& is_null_free_array);
266 void test_non_flat_array_oop(Register oop, Register temp_reg, Label&is_non_flat_array);
267 void test_non_null_free_array_oop(Register oop, Register temp_reg, Label&is_non_null_free_array);
268
269 // Check array klass layout helper for flat or null-free arrays...
270 void test_flat_array_layout(Register lh, Label& is_flat_array);
271
272 void inline_layout_info(Register holder_klass, Register index, Register layout_info);
273
274 void flat_field_copy(DecoratorSet decorators, Register src, Register dst, Register inline_layout_info);
275
276 // inline type data payload offsets...
277 void payload_offset(Register inline_klass, Register offset);
278 void payload_address(Register oop, Register data, Register inline_klass);
279
280 // interface method calling
281 void lookup_interface_method(Register recv_klass,
282 Register intf_klass,
283 RegisterOrConstant itable_index,
284 Register method_result,
285 Register scan_tmp,
286 Label& no_such_interface,
287 bool return_method = true);
288
289 void lookup_interface_method_stub(Register recv_klass,
290 Register holder_klass,
291 Register resolved_klass,
292 Register method_result,
293 Register temp_reg,
294 Register temp_reg2,
295 int itable_index,
296 Label& L_no_such_interface);
297
298 // virtual method calling
299 // n.n. x86 allows RegisterOrConstant for vtable_index
300 void lookup_virtual_method(Register recv_klass,
301 RegisterOrConstant vtable_index,
302 Register method_result);
303
304 // Form an address from base + offset in Rd. Rd my or may not
305 // actually be used: you must use the Address that is returned. It
306 // is up to you to ensure that the shift provided matches the size
307 // of your data.
308 Address form_address(Register Rd, Register base, int64_t byte_offset);
309
310 // Sometimes we get misaligned loads and stores, usually from Unsafe
311 // accesses, and these can exceed the offset range.
312 Address legitimize_address(Register Rd, const Address &adr) {
313 if (adr.getMode() == Address::base_plus_offset) {
314 if (!is_simm12(adr.offset())) {
315 return form_address(Rd, adr.base(), adr.offset());
316 }
317 }
318 return adr;
319 }
320
321 // allocation
322
323 void tlab_allocate(
324 Register obj, // result: pointer to object after successful allocation
325 Register var_size_in_bytes, // object size in bytes if unknown at compile time; invalid otherwise
326 int con_size_in_bytes, // object size in bytes if known at compile time
327 Register tmp1, // temp register
328 Register tmp2, // temp register
329 Label& slow_case, // continuation point of fast allocation fails
330 bool is_far = false
331 );
332
333 // Test sub_klass against super_klass, with fast and slow paths.
334
335 // The fast path produces a tri-state answer: yes / no / maybe-slow.
336 // One of the three labels can be null, meaning take the fall-through.
337 // If super_check_offset is -1, the value is loaded up from super_klass.
338 // No registers are killed, except tmp_reg
339 void check_klass_subtype_fast_path(Register sub_klass,
340 Register super_klass,
341 Register tmp_reg,
342 Label* L_success,
343 Label* L_failure,
344 Label* L_slow_path,
345 Register super_check_offset = noreg);
346
347 // The reset of the type check; must be wired to a corresponding fast path.
348 // It does not repeat the fast path logic, so don't use it standalone.
349 // The tmp1_reg and tmp2_reg can be noreg, if no temps are available.
350 // Updates the sub's secondary super cache as necessary.
351 void check_klass_subtype_slow_path(Register sub_klass,
352 Register super_klass,
353 Register tmp1_reg,
354 Register tmp2_reg,
355 Label* L_success,
356 Label* L_failure,
357 bool set_cond_codes = false);
358
359 void check_klass_subtype_slow_path_linear(Register sub_klass,
360 Register super_klass,
361 Register tmp1_reg,
362 Register tmp2_reg,
363 Label* L_success,
364 Label* L_failure,
365 bool set_cond_codes = false);
366
367 void check_klass_subtype_slow_path_table(Register sub_klass,
368 Register super_klass,
369 Register tmp1_reg,
370 Register tmp2_reg,
371 Label* L_success,
372 Label* L_failure,
373 bool set_cond_codes = false);
374
375 // If r is valid, return r.
376 // If r is invalid, remove a register r2 from available_regs, add r2
377 // to regs_to_push, then return r2.
378 Register allocate_if_noreg(const Register r,
379 RegSetIterator<Register> &available_regs,
380 RegSet ®s_to_push);
381
382 // Secondary subtype checking
383 void lookup_secondary_supers_table_var(Register sub_klass,
384 Register r_super_klass,
385 Register result,
386 Register tmp1,
387 Register tmp2,
388 Register tmp3,
389 Register tmp4,
390 Label *L_success);
391
392 void population_count(Register dst, Register src, Register tmp1, Register tmp2);
393
394 // As above, but with a constant super_klass.
395 // The result is in Register result, not the condition codes.
396 bool lookup_secondary_supers_table_const(Register r_sub_klass,
397 Register r_super_klass,
398 Register result,
399 Register tmp1,
400 Register tmp2,
401 Register tmp3,
402 Register tmp4,
403 u1 super_klass_slot,
404 bool stub_is_near = false);
405
406 void verify_secondary_supers_table(Register r_sub_klass,
407 Register r_super_klass,
408 Register result,
409 Register tmp1,
410 Register tmp2,
411 Register tmp3);
412
413 void lookup_secondary_supers_table_slow_path(Register r_super_klass,
414 Register r_array_base,
415 Register r_array_index,
416 Register r_bitmap,
417 Register result,
418 Register tmp,
419 bool is_stub = true);
420
421 void check_klass_subtype(Register sub_klass,
422 Register super_klass,
423 Register tmp_reg,
424 Label& L_success);
425
426 Address argument_address(RegisterOrConstant arg_slot, int extra_slot_offset = 0);
427
428 void profile_receiver_type(Register recv, Register mdp, int mdp_offset);
429
430 // only if +VerifyOops
431 void _verify_oop(Register reg, const char* s, const char* file, int line);
432 void _verify_oop_addr(Address addr, const char* s, const char* file, int line);
433
434 void _verify_oop_checked(Register reg, const char* s, const char* file, int line) {
435 if (VerifyOops) {
436 _verify_oop(reg, s, file, line);
437 }
438 }
439 void _verify_oop_addr_checked(Address reg, const char* s, const char* file, int line) {
440 if (VerifyOops) {
441 _verify_oop_addr(reg, s, file, line);
442 }
443 }
444
445 void _verify_method_ptr(Register reg, const char* msg, const char* file, int line) {}
446 void _verify_klass_ptr(Register reg, const char* msg, const char* file, int line) {}
447
448 #define verify_oop(reg) _verify_oop_checked(reg, "broken oop " #reg, __FILE__, __LINE__)
449 #define verify_oop_msg(reg, msg) _verify_oop_checked(reg, "broken oop " #reg ", " #msg, __FILE__, __LINE__)
450 #define verify_oop_addr(addr) _verify_oop_addr_checked(addr, "broken oop addr " #addr, __FILE__, __LINE__)
451 #define verify_method_ptr(reg) _verify_method_ptr(reg, "broken method " #reg, __FILE__, __LINE__)
452 #define verify_klass_ptr(reg) _verify_method_ptr(reg, "broken klass " #reg, __FILE__, __LINE__)
453
454 // A more convenient access to fence for our purposes
455 // We used four bit to indicate the read and write bits in the predecessors and successors,
456 // and extended i for r, o for w if UseConservativeFence enabled.
457 enum Membar_mask_bits {
458 StoreStore = 0b0101, // (pred = w + succ = w)
459 LoadStore = 0b1001, // (pred = r + succ = w)
460 StoreLoad = 0b0110, // (pred = w + succ = r)
461 LoadLoad = 0b1010, // (pred = r + succ = r)
462 AnyAny = LoadStore | StoreLoad // (pred = rw + succ = rw)
463 };
464
465 void membar(uint32_t order_constraint);
466
467 private:
468
469 static void membar_mask_to_pred_succ(uint32_t order_constraint,
470 uint32_t& predecessor, uint32_t& successor) {
471 predecessor = (order_constraint >> 2) & 0x3;
472 successor = order_constraint & 0x3;
473
474 // extend rw -> iorw:
475 // 01(w) -> 0101(ow)
476 // 10(r) -> 1010(ir)
477 // 11(rw)-> 1111(iorw)
478 if (UseConservativeFence) {
479 predecessor |= predecessor << 2;
480 successor |= successor << 2;
481 }
482 }
483
484 static int pred_succ_to_membar_mask(uint32_t predecessor, uint32_t successor) {
485 return ((predecessor & 0x3) << 2) | (successor & 0x3);
486 }
487
488 public:
489
490 void cmodx_fence();
491
492 void pause() {
493 // Zihintpause
494 // PAUSE is encoded as a FENCE instruction with pred=W, succ=0, fm=0, rd=x0, and rs1=x0.
495 Assembler::fence(w, 0);
496 }
497
498 // prints msg, dumps registers and stops execution
499 void stop(const char* msg);
500
501 static void debug64(char* msg, int64_t pc, int64_t regs[]);
502
503 void unimplemented(const char* what = "");
504
505 void should_not_reach_here() { stop("should not reach here"); }
506
507 static address target_addr_for_insn(address insn_addr);
508
509 // Required platform-specific helpers for Label::patch_instructions.
510 // They _shadow_ the declarations in AbstractAssembler, which are undefined.
511 static int pd_patch_instruction_size(address branch, address target);
512 static void pd_patch_instruction(address branch, address target, const char* file = nullptr, int line = 0) {
513 pd_patch_instruction_size(branch, target);
514 }
515 static address pd_call_destination(address branch) {
516 return target_addr_for_insn(branch);
517 }
518
519 static int patch_oop(address insn_addr, address o);
520
521 static address get_target_of_li32(address insn_addr);
522 static int patch_imm_in_li32(address branch, int32_t target);
523
524 // Return whether code is emitted to a scratch blob.
525 virtual bool in_scratch_emit_size() {
526 return false;
527 }
528
529 address emit_reloc_call_address_stub(int insts_call_instruction_offset, address target);
530 static int max_reloc_call_address_stub_size();
531
532 void emit_static_call_stub();
533 static int static_call_stub_size();
534
535 // The following 4 methods return the offset of the appropriate move instruction
536
537 // Support for fast byte/short loading with zero extension (depending on particular CPU)
538 int load_unsigned_byte(Register dst, Address src);
539 int load_unsigned_short(Register dst, Address src);
540
541 // Support for fast byte/short loading with sign extension (depending on particular CPU)
542 int load_signed_byte(Register dst, Address src);
543 int load_signed_short(Register dst, Address src);
544
545 // Load and store values by size and signed-ness
546 void load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed);
547 void store_sized_value(Address dst, Register src, size_t size_in_bytes);
548
549 // Misaligned loads, will use the best way, according to the AvoidUnalignedAccess flag
550 void load_short_misaligned(Register dst, Address src, Register tmp, bool is_signed, int granularity = 1);
551 void load_int_misaligned(Register dst, Address src, Register tmp, bool is_signed, int granularity = 1);
552 void load_long_misaligned(Register dst, Address src, Register tmp, int granularity = 1);
553
554 public:
555 // Standard pseudo instructions
556 inline void nop() {
557 addi(x0, x0, 0);
558 }
559
560 inline void mv(Register Rd, Register Rs) {
561 if (Rd != Rs) {
562 addi(Rd, Rs, 0);
563 }
564 }
565
566 inline void notr(Register Rd, Register Rs) {
567 if (do_compress_zcb(Rd, Rs) && (Rd == Rs)) {
568 c_not(Rd);
569 } else {
570 xori(Rd, Rs, -1);
571 }
572 }
573
574 inline void neg(Register Rd, Register Rs) {
575 sub(Rd, x0, Rs);
576 }
577
578 inline void negw(Register Rd, Register Rs) {
579 subw(Rd, x0, Rs);
580 }
581
582 inline void sext_w(Register Rd, Register Rs) {
583 addiw(Rd, Rs, 0);
584 }
585
586 inline void zext_b(Register Rd, Register Rs) {
587 if (do_compress_zcb(Rd, Rs) && (Rd == Rs)) {
588 c_zext_b(Rd);
589 } else {
590 andi(Rd, Rs, 0xFF);
591 }
592 }
593
594 inline void seqz(Register Rd, Register Rs) {
595 sltiu(Rd, Rs, 1);
596 }
597
598 inline void snez(Register Rd, Register Rs) {
599 sltu(Rd, x0, Rs);
600 }
601
602 inline void sltz(Register Rd, Register Rs) {
603 slt(Rd, Rs, x0);
604 }
605
606 inline void sgtz(Register Rd, Register Rs) {
607 slt(Rd, x0, Rs);
608 }
609
610 // Bit-manipulation extension pseudo instructions
611 // zero extend word
612 inline void zext_w(Register Rd, Register Rs) {
613 assert(UseZba, "must be");
614 if (do_compress_zcb(Rd, Rs) && (Rd == Rs)) {
615 c_zext_w(Rd);
616 } else {
617 add_uw(Rd, Rs, zr);
618 }
619 }
620
621 // Floating-point data-processing pseudo instructions
622 inline void fmv_s(FloatRegister Rd, FloatRegister Rs) {
623 if (Rd != Rs) {
624 fsgnj_s(Rd, Rs, Rs);
625 }
626 }
627
628 inline void fabs_s(FloatRegister Rd, FloatRegister Rs) {
629 fsgnjx_s(Rd, Rs, Rs);
630 }
631
632 inline void fneg_s(FloatRegister Rd, FloatRegister Rs) {
633 fsgnjn_s(Rd, Rs, Rs);
634 }
635
636 inline void fmv_d(FloatRegister Rd, FloatRegister Rs) {
637 if (Rd != Rs) {
638 fsgnj_d(Rd, Rs, Rs);
639 }
640 }
641
642 inline void fabs_d(FloatRegister Rd, FloatRegister Rs) {
643 fsgnjx_d(Rd, Rs, Rs);
644 }
645
646 inline void fneg_d(FloatRegister Rd, FloatRegister Rs) {
647 fsgnjn_d(Rd, Rs, Rs);
648 }
649
650 // Control and status pseudo instructions
651 void csrr(Register Rd, unsigned csr); // read csr
652 void csrw(unsigned csr, Register Rs); // write csr
653 void csrs(unsigned csr, Register Rs); // set bits in csr
654 void csrc(unsigned csr, Register Rs); // clear bits in csr
655 void csrwi(unsigned csr, unsigned imm);
656 void csrsi(unsigned csr, unsigned imm);
657 void csrci(unsigned csr, unsigned imm);
658 void frcsr(Register Rd) { csrr(Rd, CSR_FCSR); }; // read float-point csr
659 void fscsr(Register Rd, Register Rs); // swap float-point csr
660 void fscsr(Register Rs); // write float-point csr
661 void frrm(Register Rd) { csrr(Rd, CSR_FRM); }; // read float-point rounding mode
662 void fsrm(Register Rd, Register Rs); // swap float-point rounding mode
663 void fsrm(Register Rs); // write float-point rounding mode
664 void fsrmi(Register Rd, unsigned imm);
665 void fsrmi(unsigned imm);
666 void frflags(Register Rd) { csrr(Rd, CSR_FFLAGS); }; // read float-point exception flags
667 void fsflags(Register Rd, Register Rs); // swap float-point exception flags
668 void fsflags(Register Rs); // write float-point exception flags
669 void fsflagsi(Register Rd, unsigned imm);
670 void fsflagsi(unsigned imm);
671 // Requires Zicntr
672 void rdinstret(Register Rd) { csrr(Rd, CSR_INSTRET); }; // read instruction-retired counter
673 void rdcycle(Register Rd) { csrr(Rd, CSR_CYCLE); }; // read cycle counter
674 void rdtime(Register Rd) { csrr(Rd, CSR_TIME); }; // read time
675
676 // Restore cpu control state after JNI call
677 void restore_cpu_control_state_after_jni(Register tmp);
678
679 // Control transfer pseudo instructions
680 void beqz(Register Rs, const address dest);
681 void bnez(Register Rs, const address dest);
682 void blez(Register Rs, const address dest);
683 void bgez(Register Rs, const address dest);
684 void bltz(Register Rs, const address dest);
685 void bgtz(Register Rs, const address dest);
686
687 void cmov_eq(Register cmp1, Register cmp2, Register dst, Register src);
688 void cmov_ne(Register cmp1, Register cmp2, Register dst, Register src);
689 void cmov_le(Register cmp1, Register cmp2, Register dst, Register src);
690 void cmov_leu(Register cmp1, Register cmp2, Register dst, Register src);
691 void cmov_ge(Register cmp1, Register cmp2, Register dst, Register src);
692 void cmov_geu(Register cmp1, Register cmp2, Register dst, Register src);
693 void cmov_lt(Register cmp1, Register cmp2, Register dst, Register src);
694 void cmov_ltu(Register cmp1, Register cmp2, Register dst, Register src);
695 void cmov_gt(Register cmp1, Register cmp2, Register dst, Register src);
696 void cmov_gtu(Register cmp1, Register cmp2, Register dst, Register src);
697
698 void cmov_cmp_fp_eq(FloatRegister cmp1, FloatRegister cmp2, Register dst, Register src, bool is_single);
699 void cmov_cmp_fp_ne(FloatRegister cmp1, FloatRegister cmp2, Register dst, Register src, bool is_single);
700 void cmov_cmp_fp_le(FloatRegister cmp1, FloatRegister cmp2, Register dst, Register src, bool is_single);
701 void cmov_cmp_fp_ge(FloatRegister cmp1, FloatRegister cmp2, Register dst, Register src, bool is_single);
702 void cmov_cmp_fp_lt(FloatRegister cmp1, FloatRegister cmp2, Register dst, Register src, bool is_single);
703 void cmov_cmp_fp_gt(FloatRegister cmp1, FloatRegister cmp2, Register dst, Register src, bool is_single);
704
705 void cmov_fp_eq(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single);
706 void cmov_fp_ne(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single);
707 void cmov_fp_le(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single);
708 void cmov_fp_leu(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single);
709 void cmov_fp_ge(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single);
710 void cmov_fp_geu(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single);
711 void cmov_fp_lt(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single);
712 void cmov_fp_ltu(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single);
713 void cmov_fp_gt(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single);
714 void cmov_fp_gtu(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single);
715
716 void cmov_fp_cmp_fp_eq(FloatRegister cmp1, FloatRegister cmp2, FloatRegister dst, FloatRegister src, bool cmp_single, bool cmov_single);
717 void cmov_fp_cmp_fp_ne(FloatRegister cmp1, FloatRegister cmp2, FloatRegister dst, FloatRegister src, bool cmp_single, bool cmov_single);
718 void cmov_fp_cmp_fp_le(FloatRegister cmp1, FloatRegister cmp2, FloatRegister dst, FloatRegister src, bool cmp_single, bool cmov_single);
719 void cmov_fp_cmp_fp_ge(FloatRegister cmp1, FloatRegister cmp2, FloatRegister dst, FloatRegister src, bool cmp_single, bool cmov_single);
720 void cmov_fp_cmp_fp_lt(FloatRegister cmp1, FloatRegister cmp2, FloatRegister dst, FloatRegister src, bool cmp_single, bool cmov_single);
721 void cmov_fp_cmp_fp_gt(FloatRegister cmp1, FloatRegister cmp2, FloatRegister dst, FloatRegister src, bool cmp_single, bool cmov_single);
722
723 public:
724 // We try to follow risc-v asm menomics.
725 // But as we don't layout a reachable GOT,
726 // we often need to resort to movptr, li <48imm>.
727 // https://github.com/riscv-non-isa/riscv-asm-manual/blob/main/src/asm-manual.adoc
728
729 // Hotspot only use the standard calling convention using x1/ra.
730 // The alternative calling convection using x5/t0 is not used.
731 // Using x5 as a temp causes the CPU to mispredict returns.
732
733 // JALR, return address stack updates:
734 // | rd is x1/x5 | rs1 is x1/x5 | rd=rs1 | RAS action
735 // | ----------- | ------------ | ------ |-------------
736 // | No | No | - | None
737 // | No | Yes | - | Pop
738 // | Yes | No | - | Push
739 // | Yes | Yes | No | Pop, then push
740 // | Yes | Yes | Yes | Push
741 //
742 // JAL, return address stack updates:
743 // | rd is x1/x5 | RAS action
744 // | ----------- | ----------
745 // | Yes | Push
746 // | No | None
747 //
748 // JUMPs uses Rd = x0/zero and Rs = x6/t1 or imm
749 // CALLS uses Rd = x1/ra and Rs = x6/t1 or imm (or x1/ra*)
750 // RETURNS uses Rd = x0/zero and Rs = x1/ra
751 // *use of x1/ra should not normally be used, special case only.
752
753 // jump: jal x0, offset
754 // For long reach uses temp register for:
755 // la + jr
756 void j(const address dest, Register temp = t1);
757 void j(const Address &dest, Register temp = t1);
758 void j(Label &l, Register temp = noreg);
759
760 // jump register: jalr x0, offset(rs)
761 void jr(Register Rd, int32_t offset = 0);
762
763 // call: la + jalr x1
764 void call(const address dest, Register temp = t1);
765
766 // jalr: jalr x1, offset(rs)
767 void jalr(Register Rs, int32_t offset = 0);
768
769 // Emit a runtime call. Only invalidates the tmp register which
770 // is used to keep the entry address for jalr/movptr.
771 // Uses call() for intra code cache, else movptr + jalr.
772 // Clobebrs t1
773 void rt_call(address dest, Register tmp = t1);
774
775 // ret: jalr x0, 0(x1)
776 inline void ret() {
777 Assembler::jalr(x0, x1, 0);
778 }
779
780 //label
781 void beqz(Register Rs, Label &l, bool is_far = false);
782 void bnez(Register Rs, Label &l, bool is_far = false);
783 void blez(Register Rs, Label &l, bool is_far = false);
784 void bgez(Register Rs, Label &l, bool is_far = false);
785 void bltz(Register Rs, Label &l, bool is_far = false);
786 void bgtz(Register Rs, Label &l, bool is_far = false);
787
788 void beq (Register Rs1, Register Rs2, Label &L, bool is_far = false);
789 void bne (Register Rs1, Register Rs2, Label &L, bool is_far = false);
790 void blt (Register Rs1, Register Rs2, Label &L, bool is_far = false);
791 void bge (Register Rs1, Register Rs2, Label &L, bool is_far = false);
792 void bltu(Register Rs1, Register Rs2, Label &L, bool is_far = false);
793 void bgeu(Register Rs1, Register Rs2, Label &L, bool is_far = false);
794
795 void bgt (Register Rs, Register Rt, const address dest);
796 void ble (Register Rs, Register Rt, const address dest);
797 void bgtu(Register Rs, Register Rt, const address dest);
798 void bleu(Register Rs, Register Rt, const address dest);
799
800 void bgt (Register Rs, Register Rt, Label &l, bool is_far = false);
801 void ble (Register Rs, Register Rt, Label &l, bool is_far = false);
802 void bgtu(Register Rs, Register Rt, Label &l, bool is_far = false);
803 void bleu(Register Rs, Register Rt, Label &l, bool is_far = false);
804
805 #define INSN_ENTRY_RELOC(result_type, header) \
806 result_type header { \
807 guarantee(rtype == relocInfo::internal_word_type, \
808 "only internal_word_type relocs make sense here"); \
809 relocate(InternalAddress(dest).rspec()); \
810 IncompressibleScope scope(this); /* relocations */
811
812 #define INSN(NAME) \
813 void NAME(Register Rs1, Register Rs2, const address dest) { \
814 assert_cond(dest != nullptr); \
815 int64_t offset = dest - pc(); \
816 guarantee(is_simm13(offset) && is_even(offset), \
817 "offset is invalid: is_simm_13: %s offset: " INT64_FORMAT, \
818 BOOL_TO_STR(is_simm13(offset)), offset); \
819 Assembler::NAME(Rs1, Rs2, offset); \
820 } \
821 INSN_ENTRY_RELOC(void, NAME(Register Rs1, Register Rs2, address dest, relocInfo::relocType rtype)) \
822 NAME(Rs1, Rs2, dest); \
823 }
824
825 INSN(beq);
826 INSN(bne);
827 INSN(bge);
828 INSN(bgeu);
829 INSN(blt);
830 INSN(bltu);
831
832 #undef INSN
833
834 #undef INSN_ENTRY_RELOC
835
836 void float_beq(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far = false, bool is_unordered = false);
837 void float_bne(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far = false, bool is_unordered = false);
838 void float_ble(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far = false, bool is_unordered = false);
839 void float_bge(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far = false, bool is_unordered = false);
840 void float_blt(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far = false, bool is_unordered = false);
841 void float_bgt(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far = false, bool is_unordered = false);
842
843 void double_beq(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far = false, bool is_unordered = false);
844 void double_bne(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far = false, bool is_unordered = false);
845 void double_ble(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far = false, bool is_unordered = false);
846 void double_bge(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far = false, bool is_unordered = false);
847 void double_blt(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far = false, bool is_unordered = false);
848 void double_bgt(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far = false, bool is_unordered = false);
849
850 private:
851 // The signed 20-bit upper imm can materialize at most negative 0xF...F80000000, two G.
852 // The following signed 12-bit imm can at max subtract 0x800, two K, from that previously loaded two G.
853 bool is_valid_32bit_offset(int64_t x) {
854 constexpr int64_t twoG = (2 * G);
855 constexpr int64_t twoK = (2 * K);
856 return x < (twoG - twoK) && x >= (-twoG - twoK);
857 }
858
859 // Ensure that the auipc can reach the destination at x from anywhere within
860 // the code cache so that if it is relocated we know it will still reach.
861 bool is_32bit_offset_from_codecache(int64_t x) {
862 int64_t low = (int64_t)CodeCache::low_bound();
863 int64_t high = (int64_t)CodeCache::high_bound();
864 return is_valid_32bit_offset(x - low) && is_valid_32bit_offset(x - high);
865 }
866
867 public:
868 // Stack push and pop individual 64 bit registers
869 void push_reg(Register Rs);
870 void pop_reg(Register Rd);
871
872 int push_reg(RegSet regset, Register stack);
873 int pop_reg(RegSet regset, Register stack);
874
875 int push_fp(FloatRegSet regset, Register stack);
876 int pop_fp(FloatRegSet regset, Register stack);
877
878 #ifdef COMPILER2
879 int push_v(VectorRegSet regset, Register stack);
880 int pop_v(VectorRegSet regset, Register stack);
881 #endif // COMPILER2
882
883 // Push and pop everything that might be clobbered by a native
884 // runtime call except t0 and t1. (They are always
885 // temporary registers, so we don't have to protect them.)
886 // Additional registers can be excluded in a passed RegSet.
887 void push_call_clobbered_registers_except(RegSet exclude);
888 void pop_call_clobbered_registers_except(RegSet exclude);
889
890 void push_call_clobbered_registers() {
891 push_call_clobbered_registers_except(RegSet());
892 }
893 void pop_call_clobbered_registers() {
894 pop_call_clobbered_registers_except(RegSet());
895 }
896
897 void push_CPU_state(bool save_vectors = false, int vector_size_in_bytes = 0);
898 void pop_CPU_state(bool restore_vectors = false, int vector_size_in_bytes = 0);
899
900 void push_cont_fastpath(Register java_thread = xthread);
901 void pop_cont_fastpath(Register java_thread = xthread);
902
903 // if heap base register is used - reinit it with the correct value
904 void reinit_heapbase();
905
906 void bind(Label& L) {
907 Assembler::bind(L);
908 // fences across basic blocks should not be merged
909 code()->clear_last_merge_candidate();
910 }
911
912 typedef void (MacroAssembler::* compare_and_branch_insn)(Register Rs1, Register Rs2, const address dest);
913 typedef void (MacroAssembler::* compare_and_branch_label_insn)(Register Rs1, Register Rs2, Label &L, bool is_far);
914 typedef void (MacroAssembler::* jal_jalr_insn)(Register Rt, address dest);
915
916 void wrap_label(Register r, Label &L, jal_jalr_insn insn);
917 void wrap_label(Register r1, Register r2, Label &L,
918 compare_and_branch_insn insn,
919 compare_and_branch_label_insn neg_insn, bool is_far = false);
920
921 void la(Register Rd, Label &label);
922 void la(Register Rd, const address addr);
923 void la(Register Rd, const address addr, int32_t &offset);
924 void la(Register Rd, const Address &adr);
925
926 void li16u(Register Rd, uint16_t imm);
927 void li32(Register Rd, int32_t imm);
928 void li (Register Rd, int64_t imm); // optimized load immediate
929
930 // mv
931 void mv(Register Rd, address addr) { li(Rd, (int64_t)addr); }
932 void mv(Register Rd, address addr, int32_t &offset) {
933 // Split address into a lower 12-bit sign-extended offset and the remainder,
934 // so that the offset could be encoded in jalr or load/store instruction.
935 offset = ((int32_t)(int64_t)addr << 20) >> 20;
936 li(Rd, (int64_t)addr - offset);
937 }
938
939 template<typename T, ENABLE_IF(std::is_integral<T>::value)>
940 inline void mv(Register Rd, T o) { li(Rd, (int64_t)o); }
941
942 void mv(Register Rd, RegisterOrConstant src) {
943 if (src.is_register()) {
944 mv(Rd, src.as_register());
945 } else {
946 mv(Rd, src.as_constant());
947 }
948 }
949
950 // Generates a load of a 48-bit constant which can be
951 // patched to any 48-bit constant, i.e. address.
952 // If common case supply additional temp register
953 // to shorten the instruction sequence.
954 void movptr(Register Rd, const Address &addr, Register tmp = noreg);
955 void movptr(Register Rd, address addr, Register tmp = noreg);
956 void movptr(Register Rd, address addr, int32_t &offset, Register tmp = noreg);
957
958 private:
959 void movptr1(Register Rd, uintptr_t addr, int32_t &offset);
960 void movptr2(Register Rd, uintptr_t addr, int32_t &offset, Register tmp);
961 public:
962 // float imm move
963 static bool can_hf_imm_load(short imm);
964 static bool can_fp_imm_load(float imm);
965 static bool can_dp_imm_load(double imm);
966 void fli_h(FloatRegister Rd, short imm);
967 void fli_s(FloatRegister Rd, float imm);
968 void fli_d(FloatRegister Rd, double imm);
969
970 // arith
971 void add (Register Rd, Register Rn, int64_t increment, Register tmp = t0);
972 void sub (Register Rd, Register Rn, int64_t decrement, Register tmp = t0);
973 void addw(Register Rd, Register Rn, int64_t increment, Register tmp = t0);
974 void subw(Register Rd, Register Rn, int64_t decrement, Register tmp = t0);
975
976 void subi(Register Rd, Register Rn, int64_t decrement) {
977 assert(is_simm12(-decrement), "Must be");
978 addi(Rd, Rn, -decrement);
979 }
980
981 void subiw(Register Rd, Register Rn, int64_t decrement) {
982 assert(is_simm12(-decrement), "Must be");
983 addiw(Rd, Rn, -decrement);
984 }
985
986 #define INSN(NAME) \
987 inline void NAME(Register Rd, Register Rs1, Register Rs2) { \
988 Assembler::NAME(Rd, Rs1, Rs2); \
989 }
990
991 INSN(add);
992 INSN(addw);
993 INSN(sub);
994 INSN(subw);
995
996 #undef INSN
997
998 // logic
999 void andrw(Register Rd, Register Rs1, Register Rs2);
1000 void orrw(Register Rd, Register Rs1, Register Rs2);
1001 void xorrw(Register Rd, Register Rs1, Register Rs2);
1002
1003 // logic with negate
1004 void andn(Register Rd, Register Rs1, Register Rs2);
1005 void orn(Register Rd, Register Rs1, Register Rs2);
1006
1007 // reverse bytes
1008 void revbw(Register Rd, Register Rs, Register tmp1 = t0, Register tmp2= t1); // reverse bytes in lower word, sign-extend
1009 void revb(Register Rd, Register Rs, Register tmp1 = t0, Register tmp2 = t1); // reverse bytes in doubleword
1010
1011 void ror(Register dst, Register src, Register shift, Register tmp = t0);
1012 void ror(Register dst, Register src, uint32_t shift, Register tmp = t0);
1013 void rolw(Register dst, Register src, uint32_t shift, Register tmp = t0);
1014
1015 void orptr(Address adr, RegisterOrConstant src, Register tmp1 = t0, Register tmp2 = t1);
1016
1017 // Load and Store Instructions
1018 #define INSN_ENTRY_RELOC(result_type, header) \
1019 result_type header { \
1020 guarantee(rtype == relocInfo::internal_word_type, \
1021 "only internal_word_type relocs make sense here"); \
1022 relocate(InternalAddress(dest).rspec()); \
1023 IncompressibleScope scope(this); /* relocations */
1024
1025 #define INSN(NAME) \
1026 void NAME(Register Rd, address dest) { \
1027 assert_cond(dest != nullptr); \
1028 if (CodeCache::contains(dest)) { \
1029 int64_t distance = dest - pc(); \
1030 assert(is_valid_32bit_offset(distance), "Must be"); \
1031 auipc(Rd, (int32_t)distance + 0x800); \
1032 Assembler::NAME(Rd, Rd, ((int32_t)distance << 20) >> 20); \
1033 } else { \
1034 int32_t offset = 0; \
1035 movptr(Rd, dest, offset); \
1036 Assembler::NAME(Rd, Rd, offset); \
1037 } \
1038 } \
1039 INSN_ENTRY_RELOC(void, NAME(Register Rd, address dest, relocInfo::relocType rtype)) \
1040 NAME(Rd, dest); \
1041 } \
1042 void NAME(Register Rd, const Address &adr, Register temp = t0) { \
1043 switch (adr.getMode()) { \
1044 case Address::literal: { \
1045 relocate(adr.rspec(), [&] { \
1046 NAME(Rd, adr.target()); \
1047 }); \
1048 break; \
1049 } \
1050 case Address::base_plus_offset: { \
1051 if (is_simm12(adr.offset())) { \
1052 Assembler::NAME(Rd, adr.base(), adr.offset()); \
1053 } else { \
1054 int32_t offset = ((int32_t)adr.offset() << 20) >> 20; \
1055 if (Rd == adr.base()) { \
1056 la(temp, Address(adr.base(), adr.offset() - offset)); \
1057 Assembler::NAME(Rd, temp, offset); \
1058 } else { \
1059 la(Rd, Address(adr.base(), adr.offset() - offset)); \
1060 Assembler::NAME(Rd, Rd, offset); \
1061 } \
1062 } \
1063 break; \
1064 } \
1065 default: \
1066 ShouldNotReachHere(); \
1067 } \
1068 } \
1069 void NAME(Register Rd, Label &L) { \
1070 wrap_label(Rd, L, &MacroAssembler::NAME); \
1071 }
1072
1073 INSN(lb);
1074 INSN(lbu);
1075 INSN(lh);
1076 INSN(lhu);
1077 INSN(lw);
1078 INSN(lwu);
1079 INSN(ld);
1080
1081 #undef INSN
1082
1083 #define INSN(NAME) \
1084 void NAME(FloatRegister Rd, address dest, Register temp = t0) { \
1085 assert_cond(dest != nullptr); \
1086 if (CodeCache::contains(dest)) { \
1087 int64_t distance = dest - pc(); \
1088 assert(is_valid_32bit_offset(distance), "Must be"); \
1089 auipc(temp, (int32_t)distance + 0x800); \
1090 Assembler::NAME(Rd, temp, ((int32_t)distance << 20) >> 20); \
1091 } else { \
1092 int32_t offset = 0; \
1093 movptr(temp, dest, offset); \
1094 Assembler::NAME(Rd, temp, offset); \
1095 } \
1096 } \
1097 INSN_ENTRY_RELOC(void, NAME(FloatRegister Rd, address dest, \
1098 relocInfo::relocType rtype, Register temp = t0)) \
1099 NAME(Rd, dest, temp); \
1100 } \
1101 void NAME(FloatRegister Rd, const Address &adr, Register temp = t0) { \
1102 switch (adr.getMode()) { \
1103 case Address::literal: { \
1104 relocate(adr.rspec(), [&] { \
1105 NAME(Rd, adr.target(), temp); \
1106 }); \
1107 break; \
1108 } \
1109 case Address::base_plus_offset: { \
1110 if (is_simm12(adr.offset())) { \
1111 Assembler::NAME(Rd, adr.base(), adr.offset()); \
1112 } else { \
1113 int32_t offset = ((int32_t)adr.offset() << 20) >> 20; \
1114 la(temp, Address(adr.base(), adr.offset() - offset)); \
1115 Assembler::NAME(Rd, temp, offset); \
1116 } \
1117 break; \
1118 } \
1119 default: \
1120 ShouldNotReachHere(); \
1121 } \
1122 }
1123
1124 INSN(flh);
1125 INSN(flw);
1126 INSN(fld);
1127
1128 #undef INSN
1129
1130 #define INSN(NAME, REGISTER) \
1131 INSN_ENTRY_RELOC(void, NAME(REGISTER Rs, address dest, \
1132 relocInfo::relocType rtype, Register temp = t0)) \
1133 NAME(Rs, dest, temp); \
1134 }
1135
1136 INSN(sb, Register);
1137 INSN(sh, Register);
1138 INSN(sw, Register);
1139 INSN(sd, Register);
1140 INSN(fsw, FloatRegister);
1141 INSN(fsd, FloatRegister);
1142
1143 #undef INSN
1144
1145 #define INSN(NAME) \
1146 void NAME(Register Rs, address dest, Register temp = t0) { \
1147 assert_cond(dest != nullptr); \
1148 assert_different_registers(Rs, temp); \
1149 if (CodeCache::contains(dest)) { \
1150 int64_t distance = dest - pc(); \
1151 assert(is_valid_32bit_offset(distance), "Must be"); \
1152 auipc(temp, (int32_t)distance + 0x800); \
1153 Assembler::NAME(Rs, temp, ((int32_t)distance << 20) >> 20); \
1154 } else { \
1155 int32_t offset = 0; \
1156 movptr(temp, dest, offset); \
1157 Assembler::NAME(Rs, temp, offset); \
1158 } \
1159 } \
1160 void NAME(Register Rs, const Address &adr, Register temp = t0) { \
1161 switch (adr.getMode()) { \
1162 case Address::literal: { \
1163 assert_different_registers(Rs, temp); \
1164 relocate(adr.rspec(), [&] { \
1165 NAME(Rs, adr.target(), temp); \
1166 }); \
1167 break; \
1168 } \
1169 case Address::base_plus_offset: { \
1170 if (is_simm12(adr.offset())) { \
1171 Assembler::NAME(Rs, adr.base(), adr.offset()); \
1172 } else { \
1173 assert_different_registers(Rs, temp); \
1174 int32_t offset = ((int32_t)adr.offset() << 20) >> 20; \
1175 la(temp, Address(adr.base(), adr.offset() - offset)); \
1176 Assembler::NAME(Rs, temp, offset); \
1177 } \
1178 break; \
1179 } \
1180 default: \
1181 ShouldNotReachHere(); \
1182 } \
1183 }
1184
1185 INSN(sb);
1186 INSN(sh);
1187 INSN(sw);
1188 INSN(sd);
1189
1190 #undef INSN
1191
1192 #define INSN(NAME) \
1193 void NAME(FloatRegister Rs, address dest, Register temp = t0) { \
1194 assert_cond(dest != nullptr); \
1195 if (CodeCache::contains(dest)) { \
1196 int64_t distance = dest - pc(); \
1197 assert(is_valid_32bit_offset(distance), "Must be"); \
1198 auipc(temp, (int32_t)distance + 0x800); \
1199 Assembler::NAME(Rs, temp, ((int32_t)distance << 20) >> 20); \
1200 } else { \
1201 int32_t offset = 0; \
1202 movptr(temp, dest, offset); \
1203 Assembler::NAME(Rs, temp, offset); \
1204 } \
1205 } \
1206 void NAME(FloatRegister Rs, const Address &adr, Register temp = t0) { \
1207 switch (adr.getMode()) { \
1208 case Address::literal: { \
1209 relocate(adr.rspec(), [&] { \
1210 NAME(Rs, adr.target(), temp); \
1211 }); \
1212 break; \
1213 } \
1214 case Address::base_plus_offset: { \
1215 if (is_simm12(adr.offset())) { \
1216 Assembler::NAME(Rs, adr.base(), adr.offset()); \
1217 } else { \
1218 int32_t offset = ((int32_t)adr.offset() << 20) >> 20; \
1219 la(temp, Address(adr.base(), adr.offset() - offset)); \
1220 Assembler::NAME(Rs, temp, offset); \
1221 } \
1222 break; \
1223 } \
1224 default: \
1225 ShouldNotReachHere(); \
1226 } \
1227 }
1228
1229 INSN(fsw);
1230 INSN(fsd);
1231
1232 #undef INSN
1233
1234 #undef INSN_ENTRY_RELOC
1235
1236 void cmpxchg(Register addr, Register expected,
1237 Register new_val,
1238 Assembler::operand_size size,
1239 Assembler::Aqrl acquire, Assembler::Aqrl release,
1240 Register result, bool result_as_bool = false);
1241 void weak_cmpxchg(Register addr, Register expected,
1242 Register new_val,
1243 Assembler::operand_size size,
1244 Assembler::Aqrl acquire, Assembler::Aqrl release,
1245 Register result);
1246 void cmpxchg_narrow_value_helper(Register addr, Register expected, Register new_val,
1247 Assembler::operand_size size,
1248 Register shift, Register mask, Register aligned_addr);
1249 void cmpxchg_narrow_value(Register addr, Register expected,
1250 Register new_val,
1251 Assembler::operand_size size,
1252 Assembler::Aqrl acquire, Assembler::Aqrl release,
1253 Register result, bool result_as_bool,
1254 Register tmp1, Register tmp2, Register tmp3);
1255 void weak_cmpxchg_narrow_value(Register addr, Register expected,
1256 Register new_val,
1257 Assembler::operand_size size,
1258 Assembler::Aqrl acquire, Assembler::Aqrl release,
1259 Register result,
1260 Register tmp1, Register tmp2, Register tmp3);
1261
1262 void atomic_add(Register prev, RegisterOrConstant incr, Register addr);
1263 void atomic_addw(Register prev, RegisterOrConstant incr, Register addr);
1264 void atomic_addal(Register prev, RegisterOrConstant incr, Register addr);
1265 void atomic_addalw(Register prev, RegisterOrConstant incr, Register addr);
1266
1267 void atomic_xchg(Register prev, Register newv, Register addr);
1268 void atomic_xchgw(Register prev, Register newv, Register addr);
1269 void atomic_xchgal(Register prev, Register newv, Register addr);
1270 void atomic_xchgalw(Register prev, Register newv, Register addr);
1271 void atomic_xchgwu(Register prev, Register newv, Register addr);
1272 void atomic_xchgalwu(Register prev, Register newv, Register addr);
1273
1274 void atomic_cas(Register prev, Register newv, Register addr, Assembler::operand_size size,
1275 Assembler::Aqrl acquire = Assembler::relaxed, Assembler::Aqrl release = Assembler::relaxed);
1276
1277 // Emit a far call/jump. Only invalidates the tmp register which
1278 // is used to keep the entry address for jalr.
1279 // The address must be inside the code cache.
1280 // Supported entry.rspec():
1281 // - relocInfo::external_word_type
1282 // - relocInfo::runtime_call_type
1283 // - relocInfo::none
1284 // Clobbers t1 default.
1285 void far_call(const Address &entry, Register tmp = t1);
1286 void far_jump(const Address &entry, Register tmp = t1);
1287
1288 static int far_branch_size() {
1289 return 2 * MacroAssembler::instruction_size; // auipc + jalr, see far_call() & far_jump()
1290 }
1291
1292 void load_byte_map_base(Register reg);
1293
1294 void bang_stack_with_offset(int offset) {
1295 // stack grows down, caller passes positive offset
1296 assert(offset > 0, "must bang with negative offset");
1297 sub(t0, sp, offset);
1298 sd(zr, Address(t0));
1299 }
1300
1301 virtual void _call_Unimplemented(address call_site) {
1302 mv(t1, call_site);
1303 }
1304
1305 #define call_Unimplemented() _call_Unimplemented((address)__PRETTY_FUNCTION__)
1306
1307 // Frame creation and destruction shared between JITs.
1308 void build_frame(int framesize);
1309 void remove_frame(int framesize);
1310
1311 void verified_entry(Compile* C, int sp_inc);
1312
1313 void reserved_stack_check();
1314
1315 void get_polling_page(Register dest, relocInfo::relocType rtype);
1316 void read_polling_page(Register r, int32_t offset, relocInfo::relocType rtype);
1317
1318 // RISCV64 OpenJDK uses three different types of calls:
1319 //
1320 // - far call: auipc reg, pc_relative_offset; jalr ra, reg, offset
1321 // The offset has the range [-(2G + 2K), 2G - 2K). Addresses out of the
1322 // range in the code cache requires indirect call.
1323 // If a jump is needed rather than a call, a far jump 'jalr x0, reg, offset'
1324 // can be used instead.
1325 // All instructions are embedded at a call site.
1326 //
1327 // - indirect call: movptr + jalr
1328 // This can reach anywhere in the address space, but it cannot be patched
1329 // while code is running, so it must only be modified at a safepoint.
1330 // This form of call is most suitable for targets at fixed addresses,
1331 // which will never be patched.
1332 //
1333 // - reloc call:
1334 // This too can reach anywhere in the address space but is only available
1335 // in C1/C2-generated code (nmethod).
1336 //
1337 // [Main code section]
1338 // auipc
1339 // ld <address_from_stub_section>
1340 // jalr
1341 //
1342 // [Stub section]
1343 // address stub:
1344 // <64-bit destination address>
1345 //
1346 // To change the destination we simply atomically store the new
1347 // address in the stub section.
1348 // There is a benign race in that the other thread might observe the old
1349 // 64-bit destination address before it observes the new address. That does
1350 // not matter because the destination method has been invalidated, so there
1351 // will be a trap at its start.
1352
1353 // Emit a reloc call and create a stub to hold the entry point address.
1354 // Supported entry.rspec():
1355 // - relocInfo::runtime_call_type
1356 // - relocInfo::opt_virtual_call_type
1357 // - relocInfo::static_call_type
1358 // - relocInfo::virtual_call_type
1359 //
1360 // Return: the call PC or nullptr if CodeCache is full.
1361 address reloc_call(Address entry, Register tmp = t1);
1362
1363 address ic_call(address entry, jint method_index = 0);
1364 static int ic_check_size();
1365 int ic_check(int end_alignment = MacroAssembler::instruction_size);
1366
1367 // Support for memory inc/dec
1368 // n.b. increment/decrement calls with an Address destination will
1369 // need to use a scratch register to load the value to be
1370 // incremented. increment/decrement calls which add or subtract a
1371 // constant value other than sign-extended 12-bit immediate will need
1372 // to use a 2nd scratch register to hold the constant. so, an address
1373 // increment/decrement may trash both t0 and t1.
1374
1375 void increment(const Address dst, int64_t value = 1, Register tmp1 = t0, Register tmp2 = t1);
1376 void incrementw(const Address dst, int32_t value = 1, Register tmp1 = t0, Register tmp2 = t1);
1377
1378 void decrement(const Address dst, int64_t value = 1, Register tmp1 = t0, Register tmp2 = t1);
1379 void decrementw(const Address dst, int32_t value = 1, Register tmp1 = t0, Register tmp2 = t1);
1380
1381 void clinit_barrier(Register klass, Register tmp, Label* L_fast_path = nullptr, Label* L_slow_path = nullptr);
1382
1383 void load_method_holder_cld(Register result, Register method);
1384 void load_method_holder(Register holder, Register method);
1385 void load_metadata(Register dst, Register src);
1386
1387 void compute_index(Register str1, Register trailing_zeros, Register match_mask,
1388 Register result, Register char_tmp, Register tmp,
1389 bool haystack_isL);
1390 void compute_match_mask(Register src, Register pattern, Register match_mask,
1391 Register mask1, Register mask2);
1392
1393 // CRC32 code for java.util.zip.CRC32::updateBytes() intrinsic.
1394 void kernel_crc32(Register crc, Register buf, Register len,
1395 Register table0, Register table1, Register table2, Register table3,
1396 Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register tmp6);
1397 void update_word_crc32(Register crc, Register v, Register tmp1, Register tmp2, Register tmp3,
1398 Register table0, Register table1, Register table2, Register table3,
1399 bool upper);
1400 void update_byte_crc32(Register crc, Register val, Register table);
1401
1402 #ifdef COMPILER2
1403 void vector_update_crc32(Register crc, Register buf, Register len,
1404 Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5,
1405 Register table0, Register table3);
1406 void kernel_crc32_vclmul_fold(Register crc, Register buf, Register len,
1407 Register table0, Register table1, Register table2, Register table3,
1408 Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5);
1409 void crc32_vclmul_fold_to_16_bytes_vectorsize_32(VectorRegister vx, VectorRegister vy, VectorRegister vt,
1410 VectorRegister vtmp1, VectorRegister vtmp2, VectorRegister vtmp3, VectorRegister vtmp4);
1411 void kernel_crc32_vclmul_fold_vectorsize_32(Register crc, Register buf, Register len,
1412 Register vclmul_table, Register tmp1, Register tmp2);
1413 void crc32_vclmul_fold_16_bytes_vectorsize_16(VectorRegister vx, VectorRegister vt,
1414 VectorRegister vtmp1, VectorRegister vtmp2, VectorRegister vtmp3, VectorRegister vtmp4,
1415 Register buf, Register tmp, const int STEP);
1416 void crc32_vclmul_fold_16_bytes_vectorsize_16_2(VectorRegister vx, VectorRegister vy, VectorRegister vt,
1417 VectorRegister vtmp1, VectorRegister vtmp2, VectorRegister vtmp3, VectorRegister vtmp4,
1418 Register tmp);
1419 void crc32_vclmul_fold_16_bytes_vectorsize_16_3(VectorRegister vx, VectorRegister vy, VectorRegister vt,
1420 VectorRegister vtmp1, VectorRegister vtmp2, VectorRegister vtmp3, VectorRegister vtmp4,
1421 Register tmp);
1422 void kernel_crc32_vclmul_fold_vectorsize_16(Register crc, Register buf, Register len,
1423 Register vclmul_table, Register tmp1, Register tmp2);
1424
1425 void mul_add(Register out, Register in, Register offset,
1426 Register len, Register k, Register tmp);
1427 void wide_mul(Register prod_lo, Register prod_hi, Register n, Register m);
1428 void wide_madd(Register sum_lo, Register sum_hi, Register n,
1429 Register m, Register tmp1, Register tmp2);
1430 void cad(Register dst, Register src1, Register src2, Register carry);
1431 void cadc(Register dst, Register src1, Register src2, Register carry);
1432 void adc(Register dst, Register src1, Register src2, Register carry);
1433 void add2_with_carry(Register final_dest_hi, Register dest_hi, Register dest_lo,
1434 Register src1, Register src2, Register carry);
1435 void multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart,
1436 Register y, Register y_idx, Register z,
1437 Register carry, Register product,
1438 Register idx, Register kdx);
1439 void multiply_128_x_128_loop(Register y, Register z,
1440 Register carry, Register carry2,
1441 Register idx, Register jdx,
1442 Register yz_idx1, Register yz_idx2,
1443 Register tmp, Register tmp3, Register tmp4,
1444 Register tmp6, Register product_hi);
1445 void multiply_to_len(Register x, Register xlen, Register y, Register ylen,
1446 Register z, Register tmp0,
1447 Register tmp1, Register tmp2, Register tmp3, Register tmp4,
1448 Register tmp5, Register tmp6, Register product_hi);
1449
1450 #endif // COMPILER2
1451
1452 void inflate_lo32(Register Rd, Register Rs, Register tmp1 = t0, Register tmp2 = t1);
1453 void inflate_hi32(Register Rd, Register Rs, Register tmp1 = t0, Register tmp2 = t1);
1454
1455 void ctzc_bits(Register Rd, Register Rs, bool isLL = false,
1456 Register tmp1 = t0, Register tmp2 = t1);
1457
1458 void zero_words(Register base, uint64_t cnt);
1459 address zero_words(Register ptr, Register cnt);
1460 void fill_words(Register base, Register cnt, Register value);
1461 void zero_memory(Register addr, Register len, Register tmp);
1462 void zero_dcache_blocks(Register base, Register cnt, Register tmp1, Register tmp2);
1463
1464 // shift left by shamt and add
1465 void shadd(Register Rd, Register Rs1, Register Rs2, Register tmp, int shamt);
1466
1467 // test single bit in Rs, result is set to Rd
1468 void test_bit(Register Rd, Register Rs, uint32_t bit_pos);
1469
1470 // Here the float instructions with safe deal with some exceptions.
1471 // e.g. convert from NaN, +Inf, -Inf to int, float, double
1472 // will trigger exception, we need to deal with these situations
1473 // to get correct results.
1474 void fcvt_w_s_safe(Register dst, FloatRegister src, Register tmp = t0);
1475 void fcvt_l_s_safe(Register dst, FloatRegister src, Register tmp = t0);
1476 void fcvt_w_d_safe(Register dst, FloatRegister src, Register tmp = t0);
1477 void fcvt_l_d_safe(Register dst, FloatRegister src, Register tmp = t0);
1478
1479 void java_round_float(Register dst, FloatRegister src, FloatRegister ftmp);
1480 void java_round_double(Register dst, FloatRegister src, FloatRegister ftmp);
1481
1482 // Helper routine processing the slow path of NaN when converting float to float16
1483 void float_to_float16_NaN(Register dst, FloatRegister src, Register tmp1, Register tmp2);
1484
1485 // vector load/store unit-stride instructions
1486 void vlex_v(VectorRegister vd, Register base, Assembler::SEW sew, VectorMask vm = unmasked) {
1487 switch (sew) {
1488 case Assembler::e64:
1489 vle64_v(vd, base, vm);
1490 break;
1491 case Assembler::e32:
1492 vle32_v(vd, base, vm);
1493 break;
1494 case Assembler::e16:
1495 vle16_v(vd, base, vm);
1496 break;
1497 case Assembler::e8: // fall through
1498 default:
1499 vle8_v(vd, base, vm);
1500 break;
1501 }
1502 }
1503
1504 void vsex_v(VectorRegister store_data, Register base, Assembler::SEW sew, VectorMask vm = unmasked) {
1505 switch (sew) {
1506 case Assembler::e64:
1507 vse64_v(store_data, base, vm);
1508 break;
1509 case Assembler::e32:
1510 vse32_v(store_data, base, vm);
1511 break;
1512 case Assembler::e16:
1513 vse16_v(store_data, base, vm);
1514 break;
1515 case Assembler::e8: // fall through
1516 default:
1517 vse8_v(store_data, base, vm);
1518 break;
1519 }
1520 }
1521
1522 // vector pseudo instructions
1523 // rotate vector register left with shift bits, 32-bit version
1524 inline void vrole32_vi(VectorRegister vd, uint32_t shift, VectorRegister tmp_vr) {
1525 vsrl_vi(tmp_vr, vd, 32 - shift);
1526 vsll_vi(vd, vd, shift);
1527 vor_vv(vd, vd, tmp_vr);
1528 }
1529
1530 inline void vl1r_v(VectorRegister vd, Register rs) {
1531 vl1re8_v(vd, rs);
1532 }
1533
1534 inline void vmnot_m(VectorRegister vd, VectorRegister vs) {
1535 vmnand_mm(vd, vs, vs);
1536 }
1537
1538 inline void vncvt_x_x_w(VectorRegister vd, VectorRegister vs, VectorMask vm = unmasked) {
1539 vnsrl_wx(vd, vs, x0, vm);
1540 }
1541
1542 inline void vneg_v(VectorRegister vd, VectorRegister vs, VectorMask vm = unmasked) {
1543 vrsub_vx(vd, vs, x0, vm);
1544 }
1545
1546 inline void vfneg_v(VectorRegister vd, VectorRegister vs, VectorMask vm = unmasked) {
1547 vfsgnjn_vv(vd, vs, vs, vm);
1548 }
1549
1550 inline void vfabs_v(VectorRegister vd, VectorRegister vs, VectorMask vm = unmasked) {
1551 vfsgnjx_vv(vd, vs, vs, vm);
1552 }
1553
1554 inline void vmsgt_vv(VectorRegister vd, VectorRegister vs2, VectorRegister vs1, VectorMask vm = unmasked) {
1555 vmslt_vv(vd, vs1, vs2, vm);
1556 }
1557
1558 inline void vmsgtu_vv(VectorRegister vd, VectorRegister vs2, VectorRegister vs1, VectorMask vm = unmasked) {
1559 vmsltu_vv(vd, vs1, vs2, vm);
1560 }
1561
1562 inline void vmsge_vv(VectorRegister vd, VectorRegister vs2, VectorRegister vs1, VectorMask vm = unmasked) {
1563 vmsle_vv(vd, vs1, vs2, vm);
1564 }
1565
1566 inline void vmsgeu_vv(VectorRegister vd, VectorRegister vs2, VectorRegister vs1, VectorMask vm = unmasked) {
1567 vmsleu_vv(vd, vs1, vs2, vm);
1568 }
1569
1570 inline void vmfgt_vv(VectorRegister vd, VectorRegister vs2, VectorRegister vs1, VectorMask vm = unmasked) {
1571 vmflt_vv(vd, vs1, vs2, vm);
1572 }
1573
1574 inline void vmfge_vv(VectorRegister vd, VectorRegister vs2, VectorRegister vs1, VectorMask vm = unmasked) {
1575 vmfle_vv(vd, vs1, vs2, vm);
1576 }
1577
1578 inline void vmsltu_vi(VectorRegister Vd, VectorRegister Vs2, uint32_t imm, VectorMask vm = unmasked) {
1579 guarantee(imm >= 1 && imm <= 16, "imm is invalid");
1580 vmsleu_vi(Vd, Vs2, imm-1, vm);
1581 }
1582
1583 inline void vmsgeu_vi(VectorRegister Vd, VectorRegister Vs2, uint32_t imm, VectorMask vm = unmasked) {
1584 guarantee(imm >= 1 && imm <= 16, "imm is invalid");
1585 vmsgtu_vi(Vd, Vs2, imm-1, vm);
1586 }
1587
1588 // Copy mask register
1589 inline void vmmv_m(VectorRegister vd, VectorRegister vs) {
1590 vmand_mm(vd, vs, vs);
1591 }
1592
1593 // Clear mask register
1594 inline void vmclr_m(VectorRegister vd) {
1595 vmxor_mm(vd, vd, vd);
1596 }
1597
1598 // Set mask register
1599 inline void vmset_m(VectorRegister vd) {
1600 vmxnor_mm(vd, vd, vd);
1601 }
1602
1603 inline void vnot_v(VectorRegister Vd, VectorRegister Vs, VectorMask vm = unmasked) {
1604 vxor_vi(Vd, Vs, -1, vm);
1605 }
1606
1607 static const int zero_words_block_size;
1608
1609 void cast_primitive_type(BasicType type, Register Rt) {
1610 switch (type) {
1611 case T_BOOLEAN:
1612 sltu(Rt, zr, Rt);
1613 break;
1614 case T_CHAR :
1615 zext(Rt, Rt, 16);
1616 break;
1617 case T_BYTE :
1618 sext(Rt, Rt, 8);
1619 break;
1620 case T_SHORT :
1621 sext(Rt, Rt, 16);
1622 break;
1623 case T_INT :
1624 sext(Rt, Rt, 32);
1625 break;
1626 case T_LONG : /* nothing to do */ break;
1627 case T_VOID : /* nothing to do */ break;
1628 case T_FLOAT : /* nothing to do */ break;
1629 case T_DOUBLE : /* nothing to do */ break;
1630 default: ShouldNotReachHere();
1631 }
1632 }
1633
1634 // float cmp with unordered_result
1635 void float_compare(Register result, FloatRegister Rs1, FloatRegister Rs2, int unordered_result);
1636 void double_compare(Register result, FloatRegister Rs1, FloatRegister Rs2, int unordered_result);
1637
1638 // Zero/Sign-extend
1639 void zext(Register dst, Register src, int bits);
1640 void sext(Register dst, Register src, int bits);
1641
1642 private:
1643 void cmp_x2i(Register dst, Register src1, Register src2, Register tmp, bool is_signed = true);
1644
1645 public:
1646 // compare src1 and src2 and get -1/0/1 in dst.
1647 // if [src1 > src2], dst = 1;
1648 // if [src1 == src2], dst = 0;
1649 // if [src1 < src2], dst = -1;
1650 void cmp_l2i(Register dst, Register src1, Register src2, Register tmp = t0);
1651 void cmp_ul2i(Register dst, Register src1, Register src2, Register tmp = t0);
1652 void cmp_uw2i(Register dst, Register src1, Register src2, Register tmp = t0);
1653
1654 // support for argument shuffling
1655 void move32_64(VMRegPair src, VMRegPair dst, Register tmp = t0);
1656 void float_move(VMRegPair src, VMRegPair dst, Register tmp = t0);
1657 void long_move(VMRegPair src, VMRegPair dst, Register tmp = t0);
1658 void double_move(VMRegPair src, VMRegPair dst, Register tmp = t0);
1659 void object_move(OopMap* map,
1660 int oop_handle_offset,
1661 int framesize_in_slots,
1662 VMRegPair src,
1663 VMRegPair dst,
1664 bool is_receiver,
1665 int* receiver_offset);
1666
1667 #ifdef ASSERT
1668 // Template short-hand support to clean-up after a failed call to trampoline
1669 // call generation (see trampoline_call() below), when a set of Labels must
1670 // be reset (before returning).
1671 template<typename Label, typename... More>
1672 void reset_labels(Label& lbl, More&... more) {
1673 lbl.reset(); reset_labels(more...);
1674 }
1675 template<typename Label>
1676 void reset_labels(Label& lbl) {
1677 lbl.reset();
1678 }
1679 #endif
1680
1681 private:
1682
1683 void repne_scan(Register addr, Register value, Register count, Register tmp);
1684
1685 int bitset_to_regs(unsigned int bitset, unsigned char* regs);
1686 Address add_memory_helper(const Address dst, Register tmp);
1687
1688 void load_reserved(Register dst, Register addr, Assembler::operand_size size, Assembler::Aqrl acquire);
1689 void store_conditional(Register dst, Register new_val, Register addr, Assembler::operand_size size, Assembler::Aqrl release);
1690
1691 public:
1692 void fast_lock(Register basic_lock, Register obj, Register tmp1, Register tmp2, Register tmp3, Label& slow);
1693 void fast_unlock(Register obj, Register tmp1, Register tmp2, Register tmp3, Label& slow);
1694
1695 public:
1696 enum {
1697 // movptr
1698 movptr1_instruction_size = 6 * MacroAssembler::instruction_size, // lui, addi, slli, addi, slli, addi. See movptr1().
1699 movptr2_instruction_size = 5 * MacroAssembler::instruction_size, // lui, lui, slli, add, addi. See movptr2().
1700 load_pc_relative_instruction_size = 2 * MacroAssembler::instruction_size // auipc, ld
1701 };
1702
1703 static bool is_load_pc_relative_at(address branch);
1704 static bool is_li16u_at(address instr);
1705
1706 static bool is_jal_at(address instr) { assert_cond(instr != nullptr); return extract_opcode(instr) == 0b1101111; }
1707 static bool is_jalr_at(address instr) { assert_cond(instr != nullptr); return extract_opcode(instr) == 0b1100111 && extract_funct3(instr) == 0b000; }
1708 static bool is_branch_at(address instr) { assert_cond(instr != nullptr); return extract_opcode(instr) == 0b1100011; }
1709 static bool is_ld_at(address instr) { assert_cond(instr != nullptr); return is_load_at(instr) && extract_funct3(instr) == 0b011; }
1710 static bool is_load_at(address instr) { assert_cond(instr != nullptr); return extract_opcode(instr) == 0b0000011; }
1711 static bool is_float_load_at(address instr) { assert_cond(instr != nullptr); return extract_opcode(instr) == 0b0000111; }
1712 static bool is_auipc_at(address instr) { assert_cond(instr != nullptr); return extract_opcode(instr) == 0b0010111; }
1713 static bool is_jump_at(address instr) { assert_cond(instr != nullptr); return is_branch_at(instr) || is_jal_at(instr) || is_jalr_at(instr); }
1714 static bool is_add_at(address instr) { assert_cond(instr != nullptr); return extract_opcode(instr) == 0b0110011 && extract_funct3(instr) == 0b000; }
1715 static bool is_addi_at(address instr) { assert_cond(instr != nullptr); return extract_opcode(instr) == 0b0010011 && extract_funct3(instr) == 0b000; }
1716 static bool is_addiw_at(address instr) { assert_cond(instr != nullptr); return extract_opcode(instr) == 0b0011011 && extract_funct3(instr) == 0b000; }
1717 static bool is_addiw_to_zr_at(address instr){ assert_cond(instr != nullptr); return is_addiw_at(instr) && extract_rd(instr) == zr; }
1718 static bool is_lui_at(address instr) { assert_cond(instr != nullptr); return extract_opcode(instr) == 0b0110111; }
1719 static bool is_lui_to_zr_at(address instr) { assert_cond(instr != nullptr); return is_lui_at(instr) && extract_rd(instr) == zr; }
1720
1721 static bool is_srli_at(address instr) {
1722 assert_cond(instr != nullptr);
1723 return extract_opcode(instr) == 0b0010011 &&
1724 extract_funct3(instr) == 0b101 &&
1725 Assembler::extract(((unsigned*)instr)[0], 31, 26) == 0b000000;
1726 }
1727
1728 static bool is_slli_shift_at(address instr, uint32_t shift) {
1729 assert_cond(instr != nullptr);
1730 return (extract_opcode(instr) == 0b0010011 && // opcode field
1731 extract_funct3(instr) == 0b001 && // funct3 field, select the type of operation
1732 Assembler::extract(Assembler::ld_instr(instr), 25, 20) == shift); // shamt field
1733 }
1734
1735 static bool is_movptr1_at(address instr);
1736 static bool is_movptr2_at(address instr);
1737
1738 static bool is_lwu_to_zr(address instr);
1739
1740 static Register extract_rs1(address instr);
1741 static Register extract_rs2(address instr);
1742 static Register extract_rd(address instr);
1743 static uint32_t extract_opcode(address instr);
1744 static uint32_t extract_funct3(address instr);
1745
1746 // the instruction sequence of movptr is as below:
1747 // lui
1748 // addi
1749 // slli
1750 // addi
1751 // slli
1752 // addi/jalr/load
1753 static bool check_movptr1_data_dependency(address instr) {
1754 address lui = instr;
1755 address addi1 = lui + MacroAssembler::instruction_size;
1756 address slli1 = addi1 + MacroAssembler::instruction_size;
1757 address addi2 = slli1 + MacroAssembler::instruction_size;
1758 address slli2 = addi2 + MacroAssembler::instruction_size;
1759 address last_instr = slli2 + MacroAssembler::instruction_size;
1760 return extract_rs1(addi1) == extract_rd(lui) &&
1761 extract_rs1(addi1) == extract_rd(addi1) &&
1762 extract_rs1(slli1) == extract_rd(addi1) &&
1763 extract_rs1(slli1) == extract_rd(slli1) &&
1764 extract_rs1(addi2) == extract_rd(slli1) &&
1765 extract_rs1(addi2) == extract_rd(addi2) &&
1766 extract_rs1(slli2) == extract_rd(addi2) &&
1767 extract_rs1(slli2) == extract_rd(slli2) &&
1768 extract_rs1(last_instr) == extract_rd(slli2);
1769 }
1770
1771 // the instruction sequence of movptr2 is as below:
1772 // lui
1773 // lui
1774 // slli
1775 // add
1776 // addi/jalr/load
1777 static bool check_movptr2_data_dependency(address instr) {
1778 address lui1 = instr;
1779 address lui2 = lui1 + MacroAssembler::instruction_size;
1780 address slli = lui2 + MacroAssembler::instruction_size;
1781 address add = slli + MacroAssembler::instruction_size;
1782 address last_instr = add + MacroAssembler::instruction_size;
1783 return extract_rd(add) == extract_rd(lui2) &&
1784 extract_rs1(add) == extract_rd(lui2) &&
1785 extract_rs2(add) == extract_rd(slli) &&
1786 extract_rs1(slli) == extract_rd(lui1) &&
1787 extract_rd(slli) == extract_rd(lui1) &&
1788 extract_rs1(last_instr) == extract_rd(add);
1789 }
1790
1791 // the instruction sequence of li16u is as below:
1792 // lui
1793 // srli
1794 static bool check_li16u_data_dependency(address instr) {
1795 address lui = instr;
1796 address srli = lui + MacroAssembler::instruction_size;
1797
1798 return extract_rs1(srli) == extract_rd(lui) &&
1799 extract_rs1(srli) == extract_rd(srli);
1800 }
1801
1802 // the instruction sequence of li32 is as below:
1803 // lui
1804 // addiw
1805 static bool check_li32_data_dependency(address instr) {
1806 address lui = instr;
1807 address addiw = lui + MacroAssembler::instruction_size;
1808
1809 return extract_rs1(addiw) == extract_rd(lui) &&
1810 extract_rs1(addiw) == extract_rd(addiw);
1811 }
1812
1813 // the instruction sequence of pc-relative is as below:
1814 // auipc
1815 // jalr/addi/load/float_load
1816 static bool check_pc_relative_data_dependency(address instr) {
1817 address auipc = instr;
1818 address last_instr = auipc + MacroAssembler::instruction_size;
1819
1820 return extract_rs1(last_instr) == extract_rd(auipc);
1821 }
1822
1823 // the instruction sequence of load_label is as below:
1824 // auipc
1825 // load
1826 static bool check_load_pc_relative_data_dependency(address instr) {
1827 address auipc = instr;
1828 address load = auipc + MacroAssembler::instruction_size;
1829
1830 return extract_rd(load) == extract_rd(auipc) &&
1831 extract_rs1(load) == extract_rd(load);
1832 }
1833
1834 static bool is_li32_at(address instr);
1835 static bool is_pc_relative_at(address branch);
1836
1837 static bool is_membar(address addr) {
1838 return (Bytes::get_native_u4(addr) & 0x7f) == 0b1111 && extract_funct3(addr) == 0;
1839 }
1840 static uint32_t get_membar_kind(address addr);
1841 static void set_membar_kind(address addr, uint32_t order_kind);
1842
1843 public:
1844 // Inline type specific methods
1845 #include "asm/macroAssembler_common.hpp"
1846 };
1847
1848 #ifdef ASSERT
1849 inline bool AbstractAssembler::pd_check_instruction_mark() { return false; }
1850 #endif
1851
1852 #endif // CPU_RISCV_MACROASSEMBLER_RISCV_HPP