1 /*
2 * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2012, 2020 SAP SE. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "asm/macroAssembler.inline.hpp"
28 #include "code/compiledIC.hpp"
29 #include "memory/resourceArea.hpp"
30 #include "nativeInst_ppc.hpp"
31 #include "oops/compressedOops.inline.hpp"
32 #include "oops/oop.hpp"
33 #include "runtime/handles.hpp"
34 #include "runtime/orderAccess.hpp"
35 #include "runtime/safepoint.hpp"
36 #include "runtime/sharedRuntime.hpp"
37 #include "runtime/stubRoutines.hpp"
38 #include "utilities/ostream.hpp"
39 #ifdef COMPILER1
40 #include "c1/c1_Runtime1.hpp"
41 #endif
42
43 // We use an illtrap for marking a method as not_entrant
44 // Work around a C++ compiler bug which changes 'this'
45 bool NativeInstruction::is_sigill_not_entrant_at(address addr) {
46 if (!Assembler::is_illtrap(addr)) return false;
47 CodeBlob* cb = CodeCache::find_blob(addr);
48 if (cb == nullptr || !cb->is_nmethod()) return false;
49 nmethod *nm = (nmethod *)cb;
50 // This method is not_entrant iff the illtrap instruction is
51 // located at the verified entry point.
52 return nm->verified_entry_point() == addr;
53 }
54
55 #ifdef ASSERT
56 void NativeInstruction::verify() {
57 // Make sure code pattern is actually an instruction address.
58 address addr = addr_at(0);
59 if (addr == 0 || ((intptr_t)addr & 3) != 0) {
60 fatal("not an instruction address");
61 }
62 }
63 #endif // ASSERT
64
65 // Extract call destination from a NativeCall. The call might use a trampoline stub.
66 address NativeCall::destination() const {
67 address addr = (address)this;
68 address destination = Assembler::bxx_destination(addr);
69
70 // Do we use a trampoline stub for this call?
71 // Trampoline stubs are located behind the main code.
72 if (destination > addr) {
73 // Filter out recursive method invocation (call to verified/unverified entry point).
74 CodeBlob* cb = CodeCache::find_blob(addr);
75 assert(cb && cb->is_nmethod(), "sanity");
76 nmethod *nm = (nmethod *)cb;
77 if (nm->stub_contains(destination) && is_NativeCallTrampolineStub_at(destination)) {
78 // Yes we do, so get the destination from the trampoline stub.
79 const address trampoline_stub_addr = destination;
80 destination = NativeCallTrampolineStub_at(trampoline_stub_addr)->destination(nm);
81 }
82 }
83
84 return destination;
85 }
86
87 // Similar to replace_mt_safe, but just changes the destination. The
88 // important thing is that free-running threads are able to execute this
89 // call instruction at all times. Thus, the displacement field must be
90 // instruction-word-aligned.
91 //
92 // Used in the runtime linkage of calls; see class CompiledIC.
93 //
94 // Add parameter assert_lock to switch off assertion
95 // during code generation, where no lock is needed.
96 void NativeCall::set_destination_mt_safe(address dest, bool assert_lock) {
97 assert(!assert_lock ||
98 (CodeCache_lock->is_locked() || SafepointSynchronize::is_at_safepoint()) ||
99 CompiledICLocker::is_safe(addr_at(0)),
100 "concurrent code patching");
101
102 ResourceMark rm;
103 int code_size = 1 * BytesPerInstWord;
104 address addr_call = addr_at(0);
105 assert(MacroAssembler::is_bl(*(int*)addr_call), "unexpected code at call-site");
106
107 CodeBuffer cb(addr_call, code_size + 1);
108 MacroAssembler* a = new MacroAssembler(&cb);
109
110 // Patch the call.
111 if (!ReoptimizeCallSequences || !a->is_within_range_of_b(dest, addr_call)) {
112 address trampoline_stub_addr = get_trampoline();
113
114 // We did not find a trampoline stub because the current codeblob
115 // does not provide this information. The branch will be patched
116 // later during a final fixup, when all necessary information is
117 // available.
118 if (trampoline_stub_addr == 0)
119 return;
120
121 // Patch the constant in the call's trampoline stub.
122 NativeCallTrampolineStub_at(trampoline_stub_addr)->set_destination(dest);
123 dest = trampoline_stub_addr;
124 }
125
126 OrderAccess::release();
127 a->bl(dest);
128
129 ICache::ppc64_flush_icache_bytes(addr_call, code_size);
130 }
131
132 address NativeCall::get_trampoline() {
133 address call_addr = addr_at(0);
134
135 CodeBlob *code = CodeCache::find_blob(call_addr);
136 assert(code != nullptr, "Could not find the containing code blob");
137
138 // There are no relocations available when the code gets relocated
139 // because of CodeBuffer expansion.
140 if (code->relocation_size() == 0)
141 return nullptr;
142
143 address bl_destination = Assembler::bxx_destination(call_addr);
144 if (code->contains(bl_destination) &&
145 is_NativeCallTrampolineStub_at(bl_destination))
146 return bl_destination;
147
148 // If the codeBlob is not a nmethod, this is because we get here from the
149 // CodeBlob constructor, which is called within the nmethod constructor.
150 return trampoline_stub_Relocation::get_trampoline_for(call_addr, (nmethod*)code);
151 }
152
153 #ifdef ASSERT
154 void NativeCall::verify() {
155 address addr = addr_at(0);
156
157 if (!NativeCall::is_call_at(addr)) {
158 tty->print_cr("not a NativeCall at " PTR_FORMAT, p2i(addr));
159 // TODO: PPC port: Disassembler::decode(addr - 20, addr + 20, tty);
160 fatal("not a NativeCall at " PTR_FORMAT, p2i(addr));
161 }
162 }
163 #endif // ASSERT
164
165 #ifdef ASSERT
166 void NativeFarCall::verify() {
167 address addr = addr_at(0);
168
169 NativeInstruction::verify();
170 if (!NativeFarCall::is_far_call_at(addr)) {
171 tty->print_cr("not a NativeFarCall at " PTR_FORMAT, p2i(addr));
172 // TODO: PPC port: Disassembler::decode(addr, 20, 20, tty);
173 fatal("not a NativeFarCall at " PTR_FORMAT, p2i(addr));
174 }
175 }
176 #endif // ASSERT
177
178 address NativeMovConstReg::next_instruction_address() const {
179 #ifdef ASSERT
180 CodeBlob* nm = CodeCache::find_blob(instruction_address());
181 assert(nm != nullptr, "Could not find code blob");
182 assert(!MacroAssembler::is_set_narrow_oop(addr_at(0), nm->content_begin()), "Should not patch narrow oop here");
183 #endif
184
185 if (MacroAssembler::is_load_const_from_method_toc_at(addr_at(0))) {
186 return addr_at(load_const_from_method_toc_instruction_size);
187 } else {
188 return addr_at(load_const_instruction_size);
189 }
190 }
191
192 intptr_t NativeMovConstReg::data() const {
193 address addr = addr_at(0);
194
195 if (MacroAssembler::is_load_const_at(addr)) {
196 return MacroAssembler::get_const(addr);
197 }
198
199 CodeBlob* cb = CodeCache::find_blob(addr);
200 assert(cb != nullptr, "Could not find code blob");
201 if (MacroAssembler::is_set_narrow_oop(addr, cb->content_begin())) {
202 narrowOop no = MacroAssembler::get_narrow_oop(addr, cb->content_begin());
203 // We can reach here during GC with 'no' pointing to new object location
204 // while 'heap()->is_in' still reports false (e.g. with SerialGC).
205 // Therefore we use raw decoding.
206 if (CompressedOops::is_null(no)) return 0;
207 return cast_from_oop<intptr_t>(CompressedOops::decode_raw(no));
208 } else {
209 assert(MacroAssembler::is_load_const_from_method_toc_at(addr), "must be load_const_from_pool");
210
211 address ctable = cb->content_begin();
212 int offset = MacroAssembler::get_offset_of_load_const_from_method_toc_at(addr);
213 return *(intptr_t *)(ctable + offset);
214 }
215 }
216
217 address NativeMovConstReg::set_data_plain(intptr_t data, CodeBlob *cb) {
218 address addr = instruction_address();
219 address next_address = nullptr;
220 if (!cb) cb = CodeCache::find_blob(addr);
221
222 if (cb != nullptr && MacroAssembler::is_load_const_from_method_toc_at(addr)) {
223 // A load from the method's TOC (ctable).
224 assert(cb->is_nmethod(), "must be nmethod");
225 const address ctable = cb->content_begin();
226 const int toc_offset = MacroAssembler::get_offset_of_load_const_from_method_toc_at(addr);
227 *(intptr_t *)(ctable + toc_offset) = data;
228 next_address = addr + BytesPerInstWord;
229 } else if (cb != nullptr &&
230 MacroAssembler::is_calculate_address_from_global_toc_at(addr, cb->content_begin())) {
231 // A calculation relative to the global TOC.
232 if (MacroAssembler::get_address_of_calculate_address_from_global_toc_at(addr, cb->content_begin()) !=
233 (address)data) {
234 const address inst2_addr = addr;
235 const address inst1_addr =
236 MacroAssembler::patch_calculate_address_from_global_toc_at(inst2_addr, cb->content_begin(),
237 (address)data);
238 assert(inst1_addr != nullptr && inst1_addr < inst2_addr, "first instruction must be found");
239 const int range = inst2_addr - inst1_addr + BytesPerInstWord;
240 ICache::ppc64_flush_icache_bytes(inst1_addr, range);
241 }
242 next_address = addr + 1 * BytesPerInstWord;
243 } else if (MacroAssembler::is_load_const_at(addr)) {
244 // A normal 5 instruction load_const code sequence.
245 if (MacroAssembler::get_const(addr) != (long)data) {
246 // This is not mt safe, ok in methods like CodeBuffer::copy_code().
247 MacroAssembler::patch_const(addr, (long)data);
248 ICache::ppc64_flush_icache_bytes(addr, load_const_instruction_size);
249 }
250 next_address = addr + 5 * BytesPerInstWord;
251 } else if (MacroAssembler::is_bl(* (int*) addr)) {
252 // A single branch-and-link instruction.
253 ResourceMark rm;
254 const int code_size = 1 * BytesPerInstWord;
255 CodeBuffer cb(addr, code_size + 1);
256 MacroAssembler* a = new MacroAssembler(&cb);
257 a->bl((address) data);
258 ICache::ppc64_flush_icache_bytes(addr, code_size);
259 next_address = addr + code_size;
260 } else {
261 ShouldNotReachHere();
262 }
263
264 return next_address;
265 }
266
267 void NativeMovConstReg::set_data(intptr_t data) {
268 // Store the value into the instruction stream.
269 CodeBlob *cb = CodeCache::find_blob(instruction_address());
270 address next_address = set_data_plain(data, cb);
271
272 // Also store the value into an oop_Relocation cell, if any.
273 if (cb && cb->is_nmethod()) {
274 RelocIterator iter((nmethod *) cb, instruction_address(), next_address);
275 oop* oop_addr = nullptr;
276 Metadata** metadata_addr = nullptr;
277 while (iter.next()) {
278 if (iter.type() == relocInfo::oop_type) {
279 oop_Relocation *r = iter.oop_reloc();
280 if (oop_addr == nullptr) {
281 oop_addr = r->oop_addr();
282 *oop_addr = cast_to_oop(data);
283 } else {
284 assert(oop_addr == r->oop_addr(), "must be only one set-oop here");
285 }
286 }
287 if (iter.type() == relocInfo::metadata_type) {
288 metadata_Relocation *r = iter.metadata_reloc();
289 if (metadata_addr == nullptr) {
290 metadata_addr = r->metadata_addr();
291 *metadata_addr = (Metadata*)data;
292 } else {
293 assert(metadata_addr == r->metadata_addr(), "must be only one set-metadata here");
294 }
295 }
296 }
297 }
298 }
299
300 void NativeMovConstReg::set_narrow_oop(narrowOop data, CodeBlob *code /* = nullptr */) {
301 address inst2_addr = addr_at(0);
302 CodeBlob* cb = (code) ? code : CodeCache::find_blob(instruction_address());
303 assert(cb != nullptr, "Could not find code blob");
304 if (MacroAssembler::get_narrow_oop(inst2_addr, cb->content_begin()) == data) {
305 return;
306 }
307 const address inst1_addr =
308 MacroAssembler::patch_set_narrow_oop(inst2_addr, cb->content_begin(), data);
309 assert(inst1_addr != nullptr && inst1_addr < inst2_addr, "first instruction must be found");
310 const int range = inst2_addr - inst1_addr + BytesPerInstWord;
311 ICache::ppc64_flush_icache_bytes(inst1_addr, range);
312 }
313
314 // Do not use an assertion here. Let clients decide whether they only
315 // want this when assertions are enabled.
316 #ifdef ASSERT
317 void NativeMovConstReg::verify() {
318 address addr = addr_at(0);
319 if (! MacroAssembler::is_load_const_at(addr) &&
320 ! MacroAssembler::is_load_const_from_method_toc_at(addr)) {
321 CodeBlob* cb = CodeCache::find_blob(addr);
322 if (! (cb != nullptr && MacroAssembler::is_calculate_address_from_global_toc_at(addr, cb->content_begin())) &&
323 ! (cb != nullptr && MacroAssembler::is_set_narrow_oop(addr, cb->content_begin())) &&
324 ! MacroAssembler::is_bl(*((int*) addr))) {
325 tty->print_cr("not a NativeMovConstReg at " PTR_FORMAT, p2i(addr));
326 // TODO: PPC port: Disassembler::decode(addr, 20, 20, tty);
327 fatal("not a NativeMovConstReg at " PTR_FORMAT, p2i(addr));
328 }
329 }
330 }
331 #endif // ASSERT
332
333 void NativeJump::patch_verified_entry(address entry, address verified_entry, address dest) {
334 ResourceMark rm;
335 int code_size = 1 * BytesPerInstWord;
336 CodeBuffer cb(verified_entry, code_size + 1);
337 MacroAssembler* a = new MacroAssembler(&cb);
338 #ifdef COMPILER2
339 assert(dest == SharedRuntime::get_handle_wrong_method_stub(), "expected fixed destination of patch");
340 #endif
341 // Patch this nmethod atomically. Always use illtrap/trap in debug build.
342 if (DEBUG_ONLY(false &&) a->is_within_range_of_b(dest, a->pc())) {
343 a->b(dest);
344 } else {
345 // The signal handler will continue at dest=OptoRuntime::handle_wrong_method_stub().
346 // We use an illtrap for marking a method as not_entrant.
347 a->illtrap();
348 }
349 ICache::ppc64_flush_icache_bytes(verified_entry, code_size);
350 }
351
352 #ifdef ASSERT
353 void NativeJump::verify() {
354 address addr = addr_at(0);
355
356 NativeInstruction::verify();
357 if (!NativeJump::is_jump_at(addr)) {
358 tty->print_cr("not a NativeJump at " PTR_FORMAT, p2i(addr));
359 // TODO: PPC port: Disassembler::decode(addr, 20, 20, tty);
360 fatal("not a NativeJump at " PTR_FORMAT, p2i(addr));
361 }
362 }
363 #endif // ASSERT
364
365
366 void NativeGeneralJump::insert_unconditional(address code_pos, address entry) {
367 CodeBuffer cb(code_pos, BytesPerInstWord + 1);
368 MacroAssembler a(&cb);
369 a.b(entry);
370 ICache::ppc64_flush_icache_bytes(code_pos, NativeGeneralJump::instruction_size);
371 }
372
373 // MT-safe patching of a jmp instruction.
374 void NativeGeneralJump::replace_mt_safe(address instr_addr, address code_buffer) {
375 // Bytes beyond offset NativeGeneralJump::instruction_size are copied by caller.
376
377 // Finally patch out the jump.
378 volatile juint *jump_addr = (volatile juint*)instr_addr;
379 // Release not needed because caller uses invalidate_range after copying the remaining bytes.
380 //Atomic::release_store(jump_addr, *((juint*)code_buffer));
381 *jump_addr = *((juint*)code_buffer); // atomically store code over branch instruction
382 ICache::ppc64_flush_icache_bytes(instr_addr, NativeGeneralJump::instruction_size);
383 }
384
385
386 //-------------------------------------------------------------------
387
388 // Call trampoline stubs.
389 //
390 // Layout and instructions of a call trampoline stub:
391 // 0: load the TOC (part 1)
392 // 4: load the TOC (part 2)
393 // 8: load the call target from the constant pool (part 1)
394 // [12: load the call target from the constant pool (part 2, optional)]
395 // ..: branch via CTR
396 //
397
398 address NativeCallTrampolineStub::encoded_destination_addr() const {
399 address instruction_addr = addr_at(0 * BytesPerInstWord);
400 if (!MacroAssembler::is_ld_largeoffset(instruction_addr)) {
401 instruction_addr = addr_at(2 * BytesPerInstWord);
402 assert(MacroAssembler::is_ld_largeoffset(instruction_addr),
403 "must be a ld with large offset (from the constant pool)");
404 }
405 return instruction_addr;
406 }
407
408 address NativeCallTrampolineStub::destination(nmethod *nm) const {
409 CodeBlob* cb = nm ? nm : CodeCache::find_blob(addr_at(0));
410 assert(cb != nullptr, "Could not find code blob");
411 address ctable = cb->content_begin();
412
413 return *(address*)(ctable + destination_toc_offset());
414 }
415
416 int NativeCallTrampolineStub::destination_toc_offset() const {
417 return MacroAssembler::get_ld_largeoffset_offset(encoded_destination_addr());
418 }
419
420 void NativeCallTrampolineStub::set_destination(address new_destination) {
421 CodeBlob* cb = CodeCache::find_blob(addr_at(0));
422 assert(cb != nullptr, "Could not find code blob");
423 address ctable = cb->content_begin();
424
425 *(address*)(ctable + destination_toc_offset()) = new_destination;
426 }
427
428 void NativePostCallNop::make_deopt() {
429 NativeDeoptInstruction::insert(addr_at(0));
430 }
431
432 bool NativePostCallNop::patch(int32_t oopmap_slot, int32_t cb_offset) {
433 int32_t i2, i1;
434 assert(is_aligned(cb_offset, 4), "cb offset alignment does not match instruction alignment");
435 assert(!decode(i1, i2), "already patched");
436
437 cb_offset = cb_offset >> 2;
438 if (((oopmap_slot & ppc_oopmap_slot_mask) != oopmap_slot) || ((cb_offset & ppc_cb_offset_mask) != cb_offset)) {
439 return false; // cannot encode
440 }
441 const uint32_t data = oopmap_slot << ppc_cb_offset_bits | cb_offset;
442 const uint32_t lo_data = data & ppc_data_lo_mask;
443 const uint32_t hi_data = data >> ppc_data_lo_bits;
444 const uint32_t nineth_bit = 1 << (31 - 9);
445 uint32_t instr = Assembler::CMPLI_OPCODE | hi_data << ppc_data_hi_shift | nineth_bit | lo_data;
446 *(uint32_t*)addr_at(0) = instr;
447
448 int32_t oopmap_slot_dec, cb_offset_dec;
449 assert(is_post_call_nop(), "pcn not recognized");
450 assert(decode(oopmap_slot_dec, cb_offset_dec), "encoding failed");
451 assert(oopmap_slot == oopmap_slot_dec, "oopmap slot encoding is wrong");
452 assert((cb_offset << 2) == cb_offset_dec, "cb offset encoding is wrong");
453
454 return true; // encoding succeeded
455 }
456
457 void NativeDeoptInstruction::verify() {
458 }
459
460 bool NativeDeoptInstruction::is_deopt_at(address code_pos) {
461 if (!Assembler::is_illtrap(code_pos)) return false;
462 CodeBlob* cb = CodeCache::find_blob(code_pos);
463 if (cb == nullptr || !cb->is_nmethod()) return false;
464 nmethod *nm = (nmethod *)cb;
465 // see NativeInstruction::is_sigill_not_entrant_at()
466 return nm->verified_entry_point() != code_pos;
467 }
468
469 // Inserts an instruction which is specified to cause a SIGILL at a given pc
470 void NativeDeoptInstruction::insert(address code_pos) {
471 ResourceMark rm;
472 int code_size = 1 * BytesPerInstWord;
473 CodeBuffer cb(code_pos, code_size + 1);
474 MacroAssembler* a = new MacroAssembler(&cb);
475 a->illtrap();
476 ICache::ppc64_flush_icache_bytes(code_pos, code_size);
477 }