1 /*
2 * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "asm/macroAssembler.hpp"
27 #include "code/codeCache.hpp"
28 #include "code/compiledIC.hpp"
29 #include "gc/shared/collectedHeap.hpp"
30 #include "nativeInst_aarch64.hpp"
31 #include "oops/oop.inline.hpp"
32 #include "runtime/handles.hpp"
33 #include "runtime/orderAccess.hpp"
34 #include "runtime/sharedRuntime.hpp"
35 #include "runtime/stubRoutines.hpp"
36 #include "utilities/ostream.hpp"
37 #ifdef COMPILER1
38 #include "c1/c1_Runtime1.hpp"
39 #endif
40 #if INCLUDE_JVMCI
41 #include "jvmci/jvmciEnv.hpp"
42 #endif
43
44 void NativeCall::verify() {
45 assert(NativeCall::is_call_at((address)this), "unexpected code at call site");
46 }
47
48 void NativeInstruction::wrote(int offset) {
49 ICache::invalidate_word(addr_at(offset));
50 }
51
52 address NativeCall::destination() const {
53 address addr = instruction_address();
54 address destination = addr + displacement();
55
56 // Performance optimization: no need to call find_blob() if it is a self-call
57 if (destination == addr) {
58 return destination;
59 }
60
61 // Do we use a trampoline stub for this call?
62 CodeBlob* cb = CodeCache::find_blob(addr);
63 assert(cb != nullptr && cb->is_nmethod(), "nmethod expected");
64 nmethod *nm = cb->as_nmethod();
65 if (nm->stub_contains(destination) && is_NativeCallTrampolineStub_at(destination)) {
66 // Yes we do, so get the destination from the trampoline stub.
67 const address trampoline_stub_addr = destination;
68 destination = nativeCallTrampolineStub_at(trampoline_stub_addr)->destination();
69 }
70
71 return destination;
72 }
73
74 // Similar to replace_mt_safe, but just changes the destination. The
75 // important thing is that free-running threads are able to execute this
76 // call instruction at all times.
77 //
78 // Used in the runtime linkage of calls; see class CompiledIC.
79 void NativeCall::set_destination_mt_safe(address dest) {
80 assert((CodeCache_lock->is_locked() || SafepointSynchronize::is_at_safepoint()) ||
81 CompiledICLocker::is_safe(addr_at(0)),
82 "concurrent code patching");
83
84 address addr_call = addr_at(0);
85 bool reachable = Assembler::reachable_from_branch_at(addr_call, dest);
86 assert(NativeCall::is_call_at(addr_call), "unexpected code at call site");
87
88 // Patch the constant in the call's trampoline stub.
89 address trampoline_stub_addr = get_trampoline();
90 if (trampoline_stub_addr != nullptr) {
91 assert (! is_NativeCallTrampolineStub_at(dest), "chained trampolines");
92 nativeCallTrampolineStub_at(trampoline_stub_addr)->set_destination(dest);
93 }
94
95 // Patch the call.
96 if (reachable) {
97 set_destination(dest);
98 } else {
99 assert (trampoline_stub_addr != nullptr, "we need a trampoline");
100 set_destination(trampoline_stub_addr);
101 }
102
103 ICache::invalidate_range(addr_call, instruction_size);
104 }
105
106 address NativeCall::get_trampoline() {
107 address call_addr = instruction_address();
108
109 CodeBlob *code = CodeCache::find_blob(call_addr);
110 assert(code != nullptr && code->is_nmethod(), "nmethod expected");
111 nmethod* nm = code->as_nmethod();
112
113 address bl_destination = call_addr + displacement();
114 if (nm->stub_contains(bl_destination) &&
115 is_NativeCallTrampolineStub_at(bl_destination))
116 return bl_destination;
117
118 return trampoline_stub_Relocation::get_trampoline_for(call_addr, nm);
119 }
120
121 // Inserts a native call instruction at a given pc
122 void NativeCall::insert(address code_pos, address entry) { Unimplemented(); }
123
124 //-------------------------------------------------------------------
125
126 void NativeMovConstReg::verify() {
127 if (! (nativeInstruction_at(instruction_address())->is_movz() ||
128 is_adrp_at(instruction_address()) ||
129 is_ldr_literal_at(instruction_address())) ) {
130 fatal("should be MOVZ or ADRP or LDR (literal)");
131 }
132 }
133
134
135 intptr_t NativeMovConstReg::data() const {
136 address addr = MacroAssembler::target_addr_for_insn(instruction_address());
137 if (maybe_cpool_ref(instruction_address())) {
138 return *(intptr_t*)addr;
139 } else {
140 return (intptr_t)addr;
141 }
142 }
143
144 void NativeMovConstReg::set_data(intptr_t x) {
145 if (maybe_cpool_ref(instruction_address())) {
146 MACOS_AARCH64_ONLY(os::thread_wx_enable_write());
147 address addr = MacroAssembler::target_addr_for_insn(instruction_address());
148 *(intptr_t*)addr = x;
149 } else {
150 // Store x into the instruction stream.
151 MacroAssembler::pd_patch_instruction(instruction_address(), (address)x);
152 ICache::invalidate_range(instruction_address(), instruction_size);
153 }
154
155 // Find and replace the oop/metadata corresponding to this
156 // instruction in oops section.
157 CodeBlob* cb = CodeCache::find_blob(instruction_address());
158 nmethod* nm = cb->as_nmethod_or_null();
159 if (nm != nullptr) {
160 RelocIterator iter(nm, instruction_address(), next_instruction_address());
161 while (iter.next()) {
162 if (iter.type() == relocInfo::oop_type) {
163 oop* oop_addr = iter.oop_reloc()->oop_addr();
164 *oop_addr = cast_to_oop(x);
165 break;
166 } else if (iter.type() == relocInfo::metadata_type) {
167 Metadata** metadata_addr = iter.metadata_reloc()->metadata_addr();
168 *metadata_addr = (Metadata*)x;
169 break;
170 }
171 }
172 }
173 }
174
175 void NativeMovConstReg::print() {
176 tty->print_cr(PTR_FORMAT ": mov reg, " INTPTR_FORMAT,
177 p2i(instruction_address()), data());
178 }
179
180 //-------------------------------------------------------------------
181
182 int NativeMovRegMem::offset() const {
183 address pc = instruction_address();
184 unsigned insn = *(unsigned*)pc;
185 if (Instruction_aarch64::extract(insn, 28, 24) == 0b10000) {
186 address addr = MacroAssembler::target_addr_for_insn(pc);
187 return *addr;
188 } else {
189 return (int)(intptr_t)MacroAssembler::target_addr_for_insn(instruction_address());
190 }
191 }
192
193 void NativeMovRegMem::set_offset(int x) {
194 address pc = instruction_address();
195 if (maybe_cpool_ref(pc)) {
196 address addr = MacroAssembler::target_addr_for_insn(pc);
197 *(int64_t*)addr = x;
198 } else {
199 MacroAssembler::pd_patch_instruction(pc, (address)intptr_t(x));
200 ICache::invalidate_range(instruction_address(), instruction_size);
201 }
202 }
203
204 void NativeMovRegMem::verify() {
205 #ifdef ASSERT
206 MacroAssembler::target_addr_for_insn(instruction_address());
207 #endif
208 }
209
210 //--------------------------------------------------------------------------------
211
212 void NativeJump::verify() { ; }
213
214
215 address NativeJump::jump_destination() const {
216 address dest = MacroAssembler::target_addr_for_insn(instruction_address());
217
218 // We use jump to self as the unresolved address which the inline
219 // cache code (and relocs) know about
220 // As a special case we also use sequence movptr(r,0); br(r);
221 // i.e. jump to 0 when we need leave space for a wide immediate
222 // load
223
224 // return -1 if jump to self or to 0
225 if ((dest == (address)this) || dest == nullptr) {
226 dest = (address) -1;
227 }
228 return dest;
229 }
230
231 void NativeJump::set_jump_destination(address dest) {
232 // We use jump to self as the unresolved address which the inline
233 // cache code (and relocs) know about
234 if (dest == (address) -1)
235 dest = instruction_address();
236
237 MacroAssembler::pd_patch_instruction(instruction_address(), dest);
238 ICache::invalidate_range(instruction_address(), instruction_size);
239 };
240
241 //-------------------------------------------------------------------
242
243 address NativeGeneralJump::jump_destination() const {
244 NativeMovConstReg* move = nativeMovConstReg_at(instruction_address());
245 address dest = (address) move->data();
246
247 // We use jump to self as the unresolved address which the inline
248 // cache code (and relocs) know about
249 // As a special case we also use jump to 0 when first generating
250 // a general jump
251
252 // return -1 if jump to self or to 0
253 if ((dest == (address)this) || dest == nullptr) {
254 dest = (address) -1;
255 }
256 return dest;
257 }
258
259 void NativeGeneralJump::set_jump_destination(address dest) {
260 NativeMovConstReg* move = nativeMovConstReg_at(instruction_address());
261
262 // We use jump to self as the unresolved address which the inline
263 // cache code (and relocs) know about
264 if (dest == (address) -1) {
265 dest = instruction_address();
266 }
267
268 move->set_data((uintptr_t) dest);
269 };
270
271 //-------------------------------------------------------------------
272
273 bool NativeInstruction::is_safepoint_poll() {
274 // a safepoint_poll is implemented in two steps as either
275 //
276 // adrp(reg, polling_page);
277 // ldr(zr, [reg, #offset]);
278 //
279 // or
280 //
281 // mov(reg, polling_page);
282 // ldr(zr, [reg, #offset]);
283 //
284 // or
285 //
286 // ldr(reg, [rthread, #offset]);
287 // ldr(zr, [reg, #offset]);
288 //
289 // however, we cannot rely on the polling page address load always
290 // directly preceding the read from the page. C1 does that but C2
291 // has to do the load and read as two independent instruction
292 // generation steps. that's because with a single macro sequence the
293 // generic C2 code can only add the oop map before the mov/adrp and
294 // the trap handler expects an oop map to be associated with the
295 // load. with the load scheuled as a prior step the oop map goes
296 // where it is needed.
297 //
298 // so all we can do here is check that marked instruction is a load
299 // word to zr
300 return is_ldrw_to_zr(address(this));
301 }
302
303 bool NativeInstruction::is_adrp_at(address instr) {
304 unsigned insn = *(unsigned*)instr;
305 return (Instruction_aarch64::extract(insn, 31, 24) & 0b10011111) == 0b10010000;
306 }
307
308 bool NativeInstruction::is_ldr_literal_at(address instr) {
309 unsigned insn = *(unsigned*)instr;
310 return (Instruction_aarch64::extract(insn, 29, 24) & 0b011011) == 0b00011000;
311 }
312
313 bool NativeInstruction::is_ldrw_to_zr(address instr) {
314 unsigned insn = *(unsigned*)instr;
315 return (Instruction_aarch64::extract(insn, 31, 22) == 0b1011100101 &&
316 Instruction_aarch64::extract(insn, 4, 0) == 0b11111);
317 }
318
319 bool NativeInstruction::is_general_jump() {
320 if (is_movz()) {
321 NativeInstruction* inst1 = nativeInstruction_at(addr_at(instruction_size * 1));
322 if (inst1->is_movk()) {
323 NativeInstruction* inst2 = nativeInstruction_at(addr_at(instruction_size * 2));
324 if (inst2->is_movk()) {
325 NativeInstruction* inst3 = nativeInstruction_at(addr_at(instruction_size * 3));
326 if (inst3->is_blr()) {
327 return true;
328 }
329 }
330 }
331 }
332 return false;
333 }
334
335 bool NativeInstruction::is_movz() {
336 return Instruction_aarch64::extract(int_at(0), 30, 23) == 0b10100101;
337 }
338
339 bool NativeInstruction::is_movk() {
340 return Instruction_aarch64::extract(int_at(0), 30, 23) == 0b11100101;
341 }
342
343 void NativeIllegalInstruction::insert(address code_pos) {
344 *(juint*)code_pos = 0xd4bbd5a1; // dcps1 #0xdead
345 }
346
347 bool NativeInstruction::is_stop() {
348 return uint_at(0) == 0xd4bbd5c1; // dcps1 #0xdeae
349 }
350
351 //-------------------------------------------------------------------
352
353 // MT-safe patching of a long jump instruction.
354 void NativeGeneralJump::replace_mt_safe(address instr_addr, address code_buffer) {
355 ShouldNotCallThis();
356 }
357
358 address NativeCallTrampolineStub::destination(nmethod *nm) const {
359 return ptr_at(data_offset);
360 }
361
362 void NativeCallTrampolineStub::set_destination(address new_destination) {
363 set_ptr_at(data_offset, new_destination);
364 OrderAccess::release();
365 }
366
367 #if INCLUDE_JVMCI
368 // Generate a trampoline for a branch to dest. If there's no need for a
369 // trampoline, simply patch the call directly to dest.
370 void NativeCall::trampoline_jump(CodeBuffer &cbuf, address dest, JVMCI_TRAPS) {
371 MacroAssembler a(&cbuf);
372
373 if (!a.far_branches()) {
374 // If not using far branches, patch this call directly to dest.
375 set_destination(dest);
376 } else if (!is_NativeCallTrampolineStub_at(instruction_address() + displacement())) {
377 // If we want far branches and there isn't a trampoline stub, emit one.
378 address stub = a.emit_trampoline_stub(instruction_address() - cbuf.insts()->start(), dest);
379 if (stub == nullptr) {
380 JVMCI_ERROR("could not emit trampoline stub - code cache is full");
381 }
382 // The relocation created while emitting the stub will ensure this
383 // call instruction is subsequently patched to call the stub.
384 } else {
385 // Not sure how this can be happen but be defensive
386 JVMCI_ERROR("single-use stub should not exist");
387 }
388 }
389 #endif
390
391 void NativePostCallNop::make_deopt() {
392 NativeDeoptInstruction::insert(addr_at(0));
393 }
394
395 bool NativePostCallNop::patch(int32_t oopmap_slot, int32_t cb_offset) {
396 if (((oopmap_slot & 0xff) != oopmap_slot) || ((cb_offset & 0xffffff) != cb_offset)) {
397 return false; // cannot encode
398 }
399 uint32_t data = ((uint32_t)oopmap_slot << 24) | cb_offset;
400 #ifdef ASSERT
401 assert(data != 0, "must be");
402 uint32_t insn1 = uint_at(4);
403 uint32_t insn2 = uint_at(8);
404 assert (is_movk_to_zr(insn1) && is_movk_to_zr(insn2), "must be");
405 #endif
406
407 uint32_t lo = data & 0xffff;
408 uint32_t hi = data >> 16;
409 Instruction_aarch64::patch(addr_at(4), 20, 5, lo);
410 Instruction_aarch64::patch(addr_at(8), 20, 5, hi);
411 return true; // successfully encoded
412 }
413
414 void NativeDeoptInstruction::verify() {
415 }
416
417 // Inserts an undefined instruction at a given pc
418 void NativeDeoptInstruction::insert(address code_pos) {
419 // 1 1 0 1 | 0 1 0 0 | 1 0 1 imm16 0 0 0 0 1
420 // d | 4 | a | de | 0 | 0 |
421 // 0xd4, 0x20, 0x00, 0x00
422 uint32_t insn = 0xd4ade001;
423 uint32_t *pos = (uint32_t *) code_pos;
424 *pos = insn;
425 /**code_pos = 0xd4;
426 *(code_pos+1) = 0x60;
427 *(code_pos+2) = 0x00;
428 *(code_pos+3) = 0x00;*/
429 ICache::invalidate_range(code_pos, 4);
430 }