1 /*
2 * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "asm/macroAssembler.hpp"
27 #include "code/codeCache.hpp"
28 #include "code/compiledIC.hpp"
29 #include "gc/shared/collectedHeap.hpp"
30 #include "nativeInst_aarch64.hpp"
31 #include "oops/oop.inline.hpp"
32 #include "runtime/handles.hpp"
33 #include "runtime/orderAccess.hpp"
34 #include "runtime/sharedRuntime.hpp"
35 #include "runtime/stubRoutines.hpp"
36 #include "utilities/ostream.hpp"
37 #ifdef COMPILER1
38 #include "c1/c1_Runtime1.hpp"
39 #endif
40
41 void NativeCall::verify() {
42 assert(NativeCall::is_call_at((address)this), "unexpected code at call site");
43 }
44
45 void NativeInstruction::wrote(int offset) {
46 ICache::invalidate_word(addr_at(offset));
47 }
48
49 address NativeCall::destination() const {
50 address addr = instruction_address();
51 address destination = addr + displacement();
52
53 // Performance optimization: no need to call find_blob() if it is a self-call
54 if (destination == addr) {
55 return destination;
56 }
57
58 // Do we use a trampoline stub for this call?
59 CodeBlob* cb = CodeCache::find_blob(addr);
60 assert(cb != nullptr && cb->is_nmethod(), "nmethod expected");
61 nmethod *nm = cb->as_nmethod();
62 if (nm->stub_contains(destination) && is_NativeCallTrampolineStub_at(destination)) {
63 // Yes we do, so get the destination from the trampoline stub.
64 const address trampoline_stub_addr = destination;
65 destination = nativeCallTrampolineStub_at(trampoline_stub_addr)->destination();
66 }
67
68 return destination;
69 }
70
71 // Similar to replace_mt_safe, but just changes the destination. The
72 // important thing is that free-running threads are able to execute this
73 // call instruction at all times.
74 //
75 // Used in the runtime linkage of calls; see class CompiledIC.
76 void NativeCall::set_destination_mt_safe(address dest) {
77 assert((CodeCache_lock->is_locked() || SafepointSynchronize::is_at_safepoint()) ||
78 CompiledICLocker::is_safe(addr_at(0)),
79 "concurrent code patching");
80
81 address addr_call = addr_at(0);
82 bool reachable = Assembler::reachable_from_branch_at(addr_call, dest);
83 assert(NativeCall::is_call_at(addr_call), "unexpected code at call site");
84
85 // Patch the constant in the call's trampoline stub.
86 address trampoline_stub_addr = get_trampoline();
87 if (trampoline_stub_addr != nullptr) {
88 assert (! is_NativeCallTrampolineStub_at(dest), "chained trampolines");
89 nativeCallTrampolineStub_at(trampoline_stub_addr)->set_destination(dest);
90 }
91
92 // Patch the call.
93 if (reachable) {
94 set_destination(dest);
95 } else {
96 assert (trampoline_stub_addr != nullptr, "we need a trampoline");
97 set_destination(trampoline_stub_addr);
98 }
99
100 ICache::invalidate_range(addr_call, instruction_size);
101 }
102
103 address NativeCall::get_trampoline() {
104 address call_addr = instruction_address();
105
106 CodeBlob *code = CodeCache::find_blob(call_addr);
107 assert(code != nullptr && code->is_nmethod(), "nmethod expected");
108 nmethod* nm = code->as_nmethod();
109
110 address bl_destination = call_addr + displacement();
111 if (nm->stub_contains(bl_destination) &&
112 is_NativeCallTrampolineStub_at(bl_destination))
113 return bl_destination;
114
115 return trampoline_stub_Relocation::get_trampoline_for(call_addr, nm);
116 }
117
118 // Inserts a native call instruction at a given pc
119 void NativeCall::insert(address code_pos, address entry) { Unimplemented(); }
120
121 //-------------------------------------------------------------------
122
123 void NativeMovConstReg::verify() {
124 if (! (nativeInstruction_at(instruction_address())->is_movz() ||
125 is_adrp_at(instruction_address()) ||
126 is_ldr_literal_at(instruction_address())) ) {
127 fatal("should be MOVZ or ADRP or LDR (literal)");
128 }
129 }
130
131
132 intptr_t NativeMovConstReg::data() const {
133 address addr = MacroAssembler::target_addr_for_insn(instruction_address());
134 if (maybe_cpool_ref(instruction_address())) {
135 return *(intptr_t*)addr;
136 } else {
137 return (intptr_t)addr;
138 }
139 }
140
141 void NativeMovConstReg::set_data(intptr_t x) {
142 if (maybe_cpool_ref(instruction_address())) {
143 MACOS_AARCH64_ONLY(os::thread_wx_enable_write());
144 address addr = MacroAssembler::target_addr_for_insn(instruction_address());
145 *(intptr_t*)addr = x;
146 } else {
147 // Store x into the instruction stream.
148 MacroAssembler::pd_patch_instruction(instruction_address(), (address)x);
149 ICache::invalidate_range(instruction_address(), instruction_size);
150 }
151
152 // Find and replace the oop/metadata corresponding to this
153 // instruction in oops section.
154 CodeBlob* cb = CodeCache::find_blob(instruction_address());
155 nmethod* nm = cb->as_nmethod_or_null();
156 if (nm != nullptr) {
157 RelocIterator iter(nm, instruction_address(), next_instruction_address());
158 while (iter.next()) {
159 if (iter.type() == relocInfo::oop_type) {
160 oop* oop_addr = iter.oop_reloc()->oop_addr();
161 *oop_addr = cast_to_oop(x);
162 break;
163 } else if (iter.type() == relocInfo::metadata_type) {
164 Metadata** metadata_addr = iter.metadata_reloc()->metadata_addr();
165 *metadata_addr = (Metadata*)x;
166 break;
167 }
168 }
169 }
170 }
171
172 void NativeMovConstReg::print() {
173 tty->print_cr(PTR_FORMAT ": mov reg, " INTPTR_FORMAT,
174 p2i(instruction_address()), data());
175 }
176
177 //-------------------------------------------------------------------
178
179 int NativeMovRegMem::offset() const {
180 address pc = instruction_address();
181 unsigned insn = *(unsigned*)pc;
182 if (Instruction_aarch64::extract(insn, 28, 24) == 0b10000) {
183 address addr = MacroAssembler::target_addr_for_insn(pc);
184 return *addr;
185 } else {
186 return (int)(intptr_t)MacroAssembler::target_addr_for_insn(instruction_address());
187 }
188 }
189
190 void NativeMovRegMem::set_offset(int x) {
191 address pc = instruction_address();
192 if (maybe_cpool_ref(pc)) {
193 address addr = MacroAssembler::target_addr_for_insn(pc);
194 *(int64_t*)addr = x;
195 } else {
196 MacroAssembler::pd_patch_instruction(pc, (address)intptr_t(x));
197 ICache::invalidate_range(instruction_address(), instruction_size);
198 }
199 }
200
201 void NativeMovRegMem::verify() {
202 #ifdef ASSERT
203 MacroAssembler::target_addr_for_insn(instruction_address());
204 #endif
205 }
206
207 //--------------------------------------------------------------------------------
208
209 void NativeJump::verify() { ; }
210
211 void NativeJump::insert(address code_pos, address entry) {
212 // Dispacement is relative to the jump instruction PC
213 intptr_t disp = (intptr_t)entry - ((intptr_t)code_pos);
214
215 // The jump immediate is 26 bits and it will at execution time be scaled by 4
216 int64_t imm26 = disp >> 2;
217
218 // The farthest that we can jump is +/- 128MiB
219 guarantee(Assembler::is_simm(imm26, 26), "maximum offset is 128MiB, you asking for %ld", imm26);
220
221 // Patch with opcode | offset
222 *((int32_t*)code_pos) = 0x14000000 | imm26;
223
224 // Tell hardware to invalidate icache line containing code_pos
225 ICache::invalidate_range(code_pos, instruction_size);
226 }
227
228 address NativeJump::jump_destination() const {
229 address dest = MacroAssembler::target_addr_for_insn(instruction_address());
230
231 // We use jump to self as the unresolved address which the inline
232 // cache code (and relocs) know about
233 // As a special case we also use sequence movptr(r,0); br(r);
234 // i.e. jump to 0 when we need leave space for a wide immediate
235 // load
236
237 // return -1 if jump to self or to 0
238 if ((dest == (address)this) || dest == nullptr) {
239 dest = (address) -1;
240 }
241 return dest;
242 }
243
244 void NativeJump::set_jump_destination(address dest) {
245 // We use jump to self as the unresolved address which the inline
246 // cache code (and relocs) know about
247 if (dest == (address) -1)
248 dest = instruction_address();
249
250 MacroAssembler::pd_patch_instruction(instruction_address(), dest);
251 ICache::invalidate_range(instruction_address(), instruction_size);
252 };
253
254 //-------------------------------------------------------------------
255
256 address NativeGeneralJump::jump_destination() const {
257 NativeMovConstReg* move = nativeMovConstReg_at(instruction_address());
258 address dest = (address) move->data();
259
260 // We use jump to self as the unresolved address which the inline
261 // cache code (and relocs) know about
262 // As a special case we also use jump to 0 when first generating
263 // a general jump
264
265 // return -1 if jump to self or to 0
266 if ((dest == (address)this) || dest == nullptr) {
267 dest = (address) -1;
268 }
269 return dest;
270 }
271
272 void NativeGeneralJump::set_jump_destination(address dest) {
273 NativeMovConstReg* move = nativeMovConstReg_at(instruction_address());
274
275 // We use jump to self as the unresolved address which the inline
276 // cache code (and relocs) know about
277 if (dest == (address) -1) {
278 dest = instruction_address();
279 }
280
281 move->set_data((uintptr_t) dest);
282 };
283
284 //-------------------------------------------------------------------
285
286 bool NativeInstruction::is_safepoint_poll() {
287 // a safepoint_poll is implemented in two steps as either
288 //
289 // adrp(reg, polling_page);
290 // ldr(zr, [reg, #offset]);
291 //
292 // or
293 //
294 // mov(reg, polling_page);
295 // ldr(zr, [reg, #offset]);
296 //
297 // or
298 //
299 // ldr(reg, [rthread, #offset]);
300 // ldr(zr, [reg, #offset]);
301 //
302 // however, we cannot rely on the polling page address load always
303 // directly preceding the read from the page. C1 does that but C2
304 // has to do the load and read as two independent instruction
305 // generation steps. that's because with a single macro sequence the
306 // generic C2 code can only add the oop map before the mov/adrp and
307 // the trap handler expects an oop map to be associated with the
308 // load. with the load scheuled as a prior step the oop map goes
309 // where it is needed.
310 //
311 // so all we can do here is check that marked instruction is a load
312 // word to zr
313 return is_ldrw_to_zr(address(this));
314 }
315
316 bool NativeInstruction::is_adrp_at(address instr) {
317 unsigned insn = *(unsigned*)instr;
318 return (Instruction_aarch64::extract(insn, 31, 24) & 0b10011111) == 0b10010000;
319 }
320
321 bool NativeInstruction::is_ldr_literal_at(address instr) {
322 unsigned insn = *(unsigned*)instr;
323 return (Instruction_aarch64::extract(insn, 29, 24) & 0b011011) == 0b00011000;
324 }
325
326 bool NativeInstruction::is_ldrw_to_zr(address instr) {
327 unsigned insn = *(unsigned*)instr;
328 return (Instruction_aarch64::extract(insn, 31, 22) == 0b1011100101 &&
329 Instruction_aarch64::extract(insn, 4, 0) == 0b11111);
330 }
331
332 bool NativeInstruction::is_general_jump() {
333 if (is_movz()) {
334 NativeInstruction* inst1 = nativeInstruction_at(addr_at(instruction_size * 1));
335 if (inst1->is_movk()) {
336 NativeInstruction* inst2 = nativeInstruction_at(addr_at(instruction_size * 2));
337 if (inst2->is_movk()) {
338 NativeInstruction* inst3 = nativeInstruction_at(addr_at(instruction_size * 3));
339 if (inst3->is_blr()) {
340 return true;
341 }
342 }
343 }
344 }
345 return false;
346 }
347
348 bool NativeInstruction::is_movz() {
349 return Instruction_aarch64::extract(int_at(0), 30, 23) == 0b10100101;
350 }
351
352 bool NativeInstruction::is_movk() {
353 return Instruction_aarch64::extract(int_at(0), 30, 23) == 0b11100101;
354 }
355
356 void NativeIllegalInstruction::insert(address code_pos) {
357 *(juint*)code_pos = 0xd4bbd5a1; // dcps1 #0xdead
358 }
359
360 bool NativeInstruction::is_stop() {
361 return uint_at(0) == 0xd4bbd5c1; // dcps1 #0xdeae
362 }
363
364 //-------------------------------------------------------------------
365
366 // MT-safe patching of a long jump instruction.
367 void NativeGeneralJump::replace_mt_safe(address instr_addr, address code_buffer) {
368 ShouldNotCallThis();
369 }
370
371 address NativeCallTrampolineStub::destination(nmethod *nm) const {
372 return ptr_at(data_offset);
373 }
374
375 void NativeCallTrampolineStub::set_destination(address new_destination) {
376 set_ptr_at(data_offset, new_destination);
377 OrderAccess::release();
378 }
379
380 void NativePostCallNop::make_deopt() {
381 NativeDeoptInstruction::insert(addr_at(0));
382 }
383
384 bool NativePostCallNop::patch(int32_t oopmap_slot, int32_t cb_offset) {
385 if (((oopmap_slot & 0xff) != oopmap_slot) || ((cb_offset & 0xffffff) != cb_offset)) {
386 return false; // cannot encode
387 }
388 uint32_t data = ((uint32_t)oopmap_slot << 24) | cb_offset;
389 #ifdef ASSERT
390 assert(data != 0, "must be");
391 uint32_t insn1 = uint_at(4);
392 uint32_t insn2 = uint_at(8);
393 assert (is_movk_to_zr(insn1) && is_movk_to_zr(insn2), "must be");
394 #endif
395
396 uint32_t lo = data & 0xffff;
397 uint32_t hi = data >> 16;
398 Instruction_aarch64::patch(addr_at(4), 20, 5, lo);
399 Instruction_aarch64::patch(addr_at(8), 20, 5, hi);
400 return true; // successfully encoded
401 }
402
403 void NativeDeoptInstruction::verify() {
404 }
405
406 // Inserts an undefined instruction at a given pc
407 void NativeDeoptInstruction::insert(address code_pos) {
408 // 1 1 0 1 | 0 1 0 0 | 1 0 1 imm16 0 0 0 0 1
409 // d | 4 | a | de | 0 | 0 |
410 // 0xd4, 0x20, 0x00, 0x00
411 uint32_t insn = 0xd4ade001;
412 uint32_t *pos = (uint32_t *) code_pos;
413 *pos = insn;
414 /**code_pos = 0xd4;
415 *(code_pos+1) = 0x60;
416 *(code_pos+2) = 0x00;
417 *(code_pos+3) = 0x00;*/
418 ICache::invalidate_range(code_pos, 4);
419 }