1 /*
2 * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
4 * Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "asm/macroAssembler.hpp"
28 #include "code/compiledIC.hpp"
29 #include "nativeInst_riscv.hpp"
30 #include "oops/oop.inline.hpp"
31 #include "runtime/atomicAccess.hpp"
32 #include "runtime/handles.hpp"
33 #include "runtime/orderAccess.hpp"
34 #include "runtime/safepoint.hpp"
35 #include "runtime/sharedRuntime.hpp"
36 #include "runtime/stubRoutines.hpp"
37 #include "utilities/align.hpp"
38 #include "utilities/ostream.hpp"
39 #ifdef COMPILER1
40 #include "c1/c1_Runtime1.hpp"
41 #endif
42
43 //-----------------------------------------------------------------------------
44 // NativeInstruction
45
46 bool NativeInstruction::is_call_at(address addr) {
47 return NativeCall::is_at(addr);
48 }
49
50 //-----------------------------------------------------------------------------
51 // NativeCall
52
53 address NativeCall::destination() const {
54 address addr = instruction_address();
55 assert(NativeCall::is_at(addr), "unexpected code at call site");
56
57 address stub_addr = MacroAssembler::target_addr_for_insn(addr);
58
59 CodeBlob* cb = CodeCache::find_blob(addr);
60 assert(cb != nullptr && cb->is_nmethod(), "nmethod expected");
61 nmethod *nm = (nmethod *)cb;
62 assert(nm->stub_contains(stub_addr), "Sanity");
63 assert(stub_addr != nullptr, "Sanity");
64
65 return stub_address_destination_at(stub_addr);
66 }
67
68 address NativeCall::reloc_destination() {
69 address call_addr = instruction_address();
70 assert(NativeCall::is_at(call_addr), "unexpected code at call site");
71
72 CodeBlob *code = CodeCache::find_blob(call_addr);
73 assert(code != nullptr, "Could not find the containing code blob");
74
75 address stub_addr = nullptr;
76 if (code->is_nmethod()) {
77 // TODO: Need to revisit this when porting the AOT features.
78 stub_addr = trampoline_stub_Relocation::get_trampoline_for(call_addr, code->as_nmethod());
79 assert(stub_addr != nullptr, "Sanity");
80 }
81
82 return stub_addr;
83 }
84
85 void NativeCall::verify() {
86 assert(NativeCall::is_at(instruction_address()), "unexpected code at call site");
87 }
88
89 void NativeCall::print() {
90 assert(NativeCall::is_at(instruction_address()), "unexpected code at call site");
91 tty->print_cr(PTR_FORMAT ": auipc,ld,jalr x1, offset/reg, ", p2i(instruction_address()));
92 }
93
94 void NativeCall::optimize_call(address dest, bool mt_safe) {
95 // Skip over auipc + ld
96 address jmp_ins_pc = instruction_address() + 2 * NativeInstruction::instruction_size;
97 // Rutime calls may be unaligned, but they are never changed after relocation.
98 assert(!mt_safe || is_aligned(jmp_ins_pc, NativeInstruction::instruction_size), "Must be naturally aligned: %p", jmp_ins_pc);
99 // If reachable use JAL
100 if (Assembler::reachable_from_branch_at(jmp_ins_pc, dest)) {
101 int64_t distance = dest - jmp_ins_pc;
102 uint32_t new_jal = Assembler::encode_jal(ra, distance);
103 AtomicAccess::store((uint32_t *)jmp_ins_pc, new_jal);
104 } else if (!MacroAssembler::is_jalr_at(jmp_ins_pc)) { // The jalr is always identical: jalr ra, 0(t1)
105 uint32_t new_jalr = Assembler::encode_jalr(ra, t1, 0);
106 AtomicAccess::store((uint32_t *)jmp_ins_pc, new_jalr);
107 } else {
108 // No change to instruction stream
109 return;
110 }
111 // We changed instruction stream
112 if (mt_safe) {
113 // IC invalidate provides a leading full fence, it thus happens after we changed the instruction stream.
114 ICache::invalidate_range(jmp_ins_pc, NativeInstruction::instruction_size);
115 }
116 }
117
118 bool NativeCall::set_destination_mt_safe(address dest) {
119 assert(NativeCall::is_at(instruction_address()), "unexpected code at call site");
120 assert((CodeCache_lock->is_locked() || SafepointSynchronize::is_at_safepoint()) ||
121 CompiledICLocker::is_safe(instruction_address()),
122 "concurrent code patching");
123
124 address stub_addr = stub_address();
125 assert(stub_addr != nullptr, "No stub?");
126 set_stub_address_destination_at(stub_addr, dest); // release
127 // optimize_call happens after we stored new address in addr stub.
128 // patches jalr -> jal/jal -> jalr depending on dest
129 optimize_call(dest, true);
130
131 return true;
132 }
133
134 // The argument passed in is the address to the stub containing the destination
135 bool NativeCall::reloc_set_destination(address stub_addr) {
136 address call_addr = instruction_address();
137 assert(NativeCall::is_at(call_addr), "unexpected code at call site");
138
139 CodeBlob *code = CodeCache::find_blob(call_addr);
140 assert(code != nullptr, "Could not find the containing code blob");
141
142 if (code->is_nmethod()) {
143 // TODO: Need to revisit this when porting the AOT features.
144 assert(stub_addr != nullptr, "Sanity");
145 assert(stub_addr == trampoline_stub_Relocation::get_trampoline_for(call_addr, code->as_nmethod()), "Sanity");
146 MacroAssembler::pd_patch_instruction_size(call_addr, stub_addr); // patches auipc + ld to stub_addr
147
148 address dest = stub_address_destination_at(stub_addr);
149 optimize_call(dest, false); // patches jalr -> jal/jal -> jalr depending on dest
150 }
151
152 return true;
153 }
154
155 void NativeCall::set_stub_address_destination_at(address dest, address value) {
156 assert_cond(dest != nullptr);
157 assert_cond(value != nullptr);
158
159 set_data64_at(dest, (uint64_t)value);
160 OrderAccess::release();
161 }
162
163 address NativeCall::stub_address_destination_at(address src) {
164 assert_cond(src != nullptr);
165 address dest = (address)get_data64_at(src);
166 return dest;
167 }
168
169 address NativeCall::stub_address() {
170 address call_addr = instruction_address();
171
172 CodeBlob *code = CodeCache::find_blob(call_addr);
173 assert(code != nullptr, "Could not find the containing code blob");
174
175 address stub_addr = MacroAssembler::target_addr_for_insn(call_addr);
176 assert(code->contains(stub_addr), "Sanity");
177 return stub_addr;
178 }
179
180 bool NativeCall::is_at(address addr) {
181 assert_cond(addr != nullptr);
182 const int instr_size = NativeInstruction::instruction_size;
183 if (MacroAssembler::is_auipc_at(addr) &&
184 MacroAssembler::is_ld_at(addr + instr_size) &&
185 MacroAssembler::is_jalr_at(addr + 2 * instr_size) &&
186 (MacroAssembler::extract_rd(addr) == x6) &&
187 (MacroAssembler::extract_rd(addr + instr_size) == x6) &&
188 (MacroAssembler::extract_rs1(addr + instr_size) == x6) &&
189 (MacroAssembler::extract_rs1(addr + 2 * instr_size) == x6) &&
190 (MacroAssembler::extract_rd(addr + 2 * instr_size) == x1)) {
191 return true;
192 }
193 if (MacroAssembler::is_auipc_at(addr) &&
194 MacroAssembler::is_ld_at(addr + instr_size) &&
195 MacroAssembler::is_jal_at(addr + 2 * instr_size) &&
196 (MacroAssembler::extract_rd(addr) == x6) &&
197 (MacroAssembler::extract_rd(addr + instr_size) == x6) &&
198 (MacroAssembler::extract_rs1(addr + instr_size) == x6) &&
199 (MacroAssembler::extract_rd(addr + 2 * instr_size) == x1)) {
200 return true;
201 }
202 return false;
203 }
204
205 bool NativeCall::is_call_before(address return_address) {
206 return NativeCall::is_at(return_address - NativeCall::instruction_size);
207 }
208
209 NativeCall* nativeCall_at(address addr) {
210 assert_cond(addr != nullptr);
211 NativeCall* call = (NativeCall*)(addr);
212 DEBUG_ONLY(call->verify());
213 return call;
214 }
215
216 NativeCall* nativeCall_before(address return_address) {
217 assert_cond(return_address != nullptr);
218 NativeCall* call = nullptr;
219 call = (NativeCall*)(return_address - NativeCall::instruction_size);
220 DEBUG_ONLY(call->verify());
221 return call;
222 }
223
224 //-------------------------------------------------------------------
225
226 void NativeMovConstReg::verify() {
227 NativeInstruction* ni = nativeInstruction_at(instruction_address());
228 if (ni->is_movptr() || ni->is_auipc()) {
229 return;
230 }
231 fatal("should be MOVPTR or AUIPC");
232 }
233
234 intptr_t NativeMovConstReg::data() const {
235 address addr = MacroAssembler::target_addr_for_insn(instruction_address());
236 if (maybe_cpool_ref(instruction_address())) {
237 return Bytes::get_native_u8(addr);
238 } else {
239 return (intptr_t)addr;
240 }
241 }
242
243 void NativeMovConstReg::set_data(intptr_t x) {
244 if (maybe_cpool_ref(instruction_address())) {
245 address addr = MacroAssembler::target_addr_for_insn(instruction_address());
246 Bytes::put_native_u8(addr, x);
247 } else {
248 // Store x into the instruction stream.
249 MacroAssembler::pd_patch_instruction_size(instruction_address(), (address)x);
250 ICache::invalidate_range(instruction_address(), movptr1_instruction_size /* > movptr2_instruction_size */ );
251 }
252
253 // Find and replace the oop/metadata corresponding to this
254 // instruction in oops section.
255 CodeBlob* cb = CodeCache::find_blob(instruction_address());
256 nmethod* nm = cb->as_nmethod_or_null();
257 if (nm != nullptr) {
258 RelocIterator iter(nm, instruction_address(), next_instruction_address());
259 while (iter.next()) {
260 if (iter.type() == relocInfo::oop_type) {
261 oop* oop_addr = iter.oop_reloc()->oop_addr();
262 Bytes::put_native_u8((address)oop_addr, x);
263 break;
264 } else if (iter.type() == relocInfo::metadata_type) {
265 Metadata** metadata_addr = iter.metadata_reloc()->metadata_addr();
266 Bytes::put_native_u8((address)metadata_addr, x);
267 break;
268 }
269 }
270 }
271 }
272
273 void NativeMovConstReg::print() {
274 tty->print_cr(PTR_FORMAT ": mov reg, " INTPTR_FORMAT,
275 p2i(instruction_address()), data());
276 }
277
278 //--------------------------------------------------------------------------------
279
280 void NativeJump::verify() { }
281
282 address NativeJump::jump_destination() const {
283 address dest = MacroAssembler::target_addr_for_insn(instruction_address());
284
285 // We use jump to self as the unresolved address which the inline
286 // cache code (and relocs) know about
287 // As a special case we also use sequence movptr(r,0), jalr(r,0)
288 // i.e. jump to 0 when we need leave space for a wide immediate
289 // load
290
291 // return -1 if jump to self or to 0
292 if ((dest == (address) this) || dest == nullptr) {
293 dest = (address) -1;
294 }
295
296 return dest;
297 };
298
299 void NativeJump::set_jump_destination(address dest) {
300 // We use jump to self as the unresolved address which the inline
301 // cache code (and relocs) know about
302 if (dest == (address) -1)
303 dest = instruction_address();
304
305 MacroAssembler::pd_patch_instruction(instruction_address(), dest);
306 ICache::invalidate_range(instruction_address(), instruction_size);
307 }
308
309 //-------------------------------------------------------------------
310
311 address NativeGeneralJump::jump_destination() const {
312 NativeMovConstReg* move = nativeMovConstReg_at(instruction_address());
313 address dest = (address) move->data();
314
315 // We use jump to self as the unresolved address which the inline
316 // cache code (and relocs) know about
317 // As a special case we also use jump to 0 when first generating
318 // a general jump
319
320 // return -1 if jump to self or to 0
321 if ((dest == (address) this) || dest == nullptr) {
322 dest = (address) -1;
323 }
324
325 return dest;
326 }
327
328 //-------------------------------------------------------------------
329
330 bool NativeInstruction::is_safepoint_poll() {
331 return MacroAssembler::is_lwu_to_zr(address(this));
332 }
333
334 bool NativeInstruction::is_stop() {
335 // an illegal instruction, 'csrrw x0, time, x0'
336 uint32_t encoded = Assembler::encode_csrrw(x0, Assembler::time, x0);
337 return uint_at(0) == encoded;
338 }
339
340 //-------------------------------------------------------------------
341
342 void NativeGeneralJump::insert_unconditional(address code_pos, address entry) {
343 CodeBuffer cb(code_pos, instruction_size);
344 MacroAssembler a(&cb);
345 Assembler::IncompressibleScope scope(&a); // Fixed length: see NativeGeneralJump::get_instruction_size()
346
347 MacroAssembler::assert_alignment(code_pos);
348
349 int32_t offset = 0;
350 a.movptr(t1, entry, offset, t0); // lui, lui, slli, add
351 a.jr(t1, offset); // jalr
352
353 ICache::invalidate_range(code_pos, instruction_size);
354 }
355
356 // MT-safe patching of a long jump instruction.
357 void NativeGeneralJump::replace_mt_safe(address instr_addr, address code_buffer) {
358 ShouldNotCallThis();
359 }
360
361 //-------------------------------------------------------------------
362
363 void NativePostCallNop::make_deopt() {
364 MacroAssembler::assert_alignment(addr_at(0));
365 NativeDeoptInstruction::insert(addr_at(0));
366 }
367
368 bool NativePostCallNop::decode(int32_t& oopmap_slot, int32_t& cb_offset) const {
369 // Discard the high 32 bits
370 int32_t data = (int32_t)(intptr_t)MacroAssembler::get_target_of_li32(addr_at(4));
371 if (data == 0) {
372 return false; // no information encoded
373 }
374 cb_offset = (data & 0xffffff);
375 oopmap_slot = (data >> 24) & 0xff;
376 return true; // decoding succeeded
377 }
378
379 bool NativePostCallNop::patch(int32_t oopmap_slot, int32_t cb_offset) {
380 MacroAssembler::assert_alignment(addr_at(4));
381 if (((oopmap_slot & 0xff) != oopmap_slot) || ((cb_offset & 0xffffff) != cb_offset)) {
382 return false; // cannot encode
383 }
384 int32_t data = (oopmap_slot << 24) | cb_offset;
385 assert(data != 0, "must be");
386 assert(MacroAssembler::is_lui_to_zr_at(addr_at(4)) && MacroAssembler::is_addiw_to_zr_at(addr_at(8)), "must be");
387
388 MacroAssembler::patch_imm_in_li32(addr_at(4), data);
389 return true; // successfully encoded
390 }
391
392 bool NativeDeoptInstruction::is_deopt_at(address instr) {
393 assert(instr != nullptr, "Must be");
394 uint32_t value = Assembler::ld_instr(instr);
395 uint32_t encoded = Assembler::encode_csrrw(x0, Assembler::instret, x0);
396 return value == encoded;
397 }
398
399 // Inserts an undefined instruction at a given pc
400 void NativeDeoptInstruction::insert(address code_pos) {
401 MacroAssembler::assert_alignment(code_pos);
402 uint32_t encoded = Assembler::encode_csrrw(x0, Assembler::instret, x0);
403 Assembler::sd_instr(code_pos, encoded);
404 ICache::invalidate_range(code_pos, 4);
405 }