1 /*
2 * Copyright (c) 2002, 2024, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2012, 2024 SAP SE. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #ifndef CPU_PPC_MACROASSEMBLER_PPC_INLINE_HPP
27 #define CPU_PPC_MACROASSEMBLER_PPC_INLINE_HPP
28
29 #include "asm/assembler.inline.hpp"
30 #include "asm/macroAssembler.hpp"
31 #include "asm/codeBuffer.hpp"
32 #include "code/codeCache.hpp"
33 #include "gc/shared/barrierSet.hpp"
34 #include "gc/shared/barrierSetAssembler.hpp"
35 #include "oops/accessDecorators.hpp"
36 #include "oops/compressedOops.hpp"
37 #include "runtime/os.inline.hpp"
38 #include "runtime/safepointMechanism.hpp"
39 #include "runtime/vm_version.hpp"
40 #include "utilities/powerOfTwo.hpp"
41
42 inline bool MacroAssembler::is_ld_largeoffset(address a) {
43 const int inst1 = *(int *)a;
44 const int inst2 = *(int *)(a+4);
45 return (is_ld(inst1)) ||
46 (is_addis(inst1) && is_ld(inst2) && inv_ra_field(inst2) == inv_rt_field(inst1));
47 }
48
49 inline int MacroAssembler::get_ld_largeoffset_offset(address a) {
50 assert(MacroAssembler::is_ld_largeoffset(a), "must be ld with large offset");
51
52 const int inst1 = *(int *)a;
53 if (is_ld(inst1)) {
54 return inv_d1_field(inst1);
55 } else {
56 const int inst2 = *(int *)(a+4);
57 return (inv_d1_field(inst1) << 16) + inv_d1_field(inst2);
58 }
59 }
60
61 inline void MacroAssembler::round_to(Register r, int modulus) {
62 assert(is_power_of_2((jlong)modulus), "must be power of 2");
63 addi(r, r, modulus-1);
64 clrrdi(r, r, log2i_exact((jlong)modulus));
65 }
66
67 // Move register if destination register and target register are different.
68 inline void MacroAssembler::mr_if_needed(Register rd, Register rs) {
69 if (rs != rd) mr(rd, rs);
70 }
71 inline void MacroAssembler::fmr_if_needed(FloatRegister rd, FloatRegister rs) {
72 if (rs != rd) fmr(rd, rs);
73 }
74 inline void MacroAssembler::endgroup_if_needed(bool needed) {
75 if (needed) {
76 endgroup();
77 }
78 }
79
80 inline void MacroAssembler::membar(int bits) {
81 // Comment: Usage of elemental_membar(bits) is not recommended for Power 8.
82 // If elemental_membar(bits) is used, disable optimization of acquire-release
83 // (Matcher::post_membar_release where we use PPC64_ONLY(xop == Op_MemBarRelease ||))!
84 if (bits & StoreLoad) { sync(); }
85 else if (bits) { lwsync(); }
86 }
87 inline void MacroAssembler::release() { membar(LoadStore | StoreStore); }
88 inline void MacroAssembler::acquire() { membar(LoadLoad | LoadStore); }
89 inline void MacroAssembler::fence() { membar(LoadLoad | LoadStore | StoreLoad | StoreStore); }
90
91 // Address of the global TOC.
92 inline address MacroAssembler::global_toc() {
93 return CodeCache::low_bound();
94 }
95
96 // Offset of given address to the global TOC.
97 inline int MacroAssembler::offset_to_global_toc(const address addr) {
98 intptr_t offset = (intptr_t)addr - (intptr_t)MacroAssembler::global_toc();
99 assert(Assembler::is_uimm((long)offset, 31), "must be in range");
100 return (int)offset;
101 }
102
103 // Address of current method's TOC.
104 inline address MacroAssembler::method_toc() {
105 return code()->consts()->start();
106 }
107
108 // Offset of given address to current method's TOC.
109 inline int MacroAssembler::offset_to_method_toc(address addr) {
110 intptr_t offset = (intptr_t)addr - (intptr_t)method_toc();
111 assert(Assembler::is_uimm((long)offset, 31), "must be in range");
112 return (int)offset;
113 }
114
115 inline bool MacroAssembler::is_calculate_address_from_global_toc_at(address a, address bound) {
116 const address inst2_addr = a;
117 const int inst2 = *(int *) a;
118
119 // The relocation points to the second instruction, the addi.
120 if (!is_addi(inst2)) return false;
121
122 // The addi reads and writes the same register dst.
123 const int dst = inv_rt_field(inst2);
124 if (inv_ra_field(inst2) != dst) return false;
125
126 // Now, find the preceding addis which writes to dst.
127 int inst1 = 0;
128 address inst1_addr = inst2_addr - BytesPerInstWord;
129 while (inst1_addr >= bound) {
130 inst1 = *(int *) inst1_addr;
131 if (is_addis(inst1) && inv_rt_field(inst1) == dst) {
132 // stop, found the addis which writes dst
133 break;
134 }
135 inst1_addr -= BytesPerInstWord;
136 }
137
138 if (!(inst1 == 0 || inv_ra_field(inst1) == 29 /* R29 */)) return false;
139 return is_addis(inst1);
140 }
141
142 #ifdef _LP64
143 // Detect narrow oop constants.
144 inline bool MacroAssembler::is_set_narrow_oop(address a, address bound) {
145 const address inst2_addr = a;
146 const int inst2 = *(int *)a;
147 // The relocation points to the second instruction, the ori.
148 if (!is_ori(inst2)) return false;
149
150 // The ori reads and writes the same register dst.
151 const int dst = inv_rta_field(inst2);
152 if (inv_rs_field(inst2) != dst) return false;
153
154 // Now, find the preceding addis which writes to dst.
155 int inst1 = 0;
156 address inst1_addr = inst2_addr - BytesPerInstWord;
157 while (inst1_addr >= bound) {
158 inst1 = *(int *) inst1_addr;
159 if (is_lis(inst1) && inv_rs_field(inst1) == dst) return true;
160 inst1_addr -= BytesPerInstWord;
161 }
162 return false;
163 }
164 #endif
165
166
167 inline bool MacroAssembler::is_load_const_at(address a) {
168 const int* p_inst = (int *) a;
169 bool b = is_lis(*p_inst++);
170 if (is_ori(*p_inst)) {
171 p_inst++;
172 b = b && is_rldicr(*p_inst++); // TODO: could be made more precise: `sldi'!
173 b = b && is_oris(*p_inst++);
174 b = b && is_ori(*p_inst);
175 } else if (is_lis(*p_inst)) {
176 p_inst++;
177 b = b && is_ori(*p_inst++);
178 b = b && is_ori(*p_inst);
179 // TODO: could enhance reliability by adding is_insrdi
180 } else return false;
181 return b;
182 }
183
184 inline void MacroAssembler::set_oop_constant(jobject obj, Register d) {
185 set_oop(constant_oop_address(obj), d);
186 }
187
188 inline void MacroAssembler::set_oop(AddressLiteral obj_addr, Register d) {
189 assert(obj_addr.rspec().type() == relocInfo::oop_type, "must be an oop reloc");
190 load_const(d, obj_addr);
191 }
192
193 inline void MacroAssembler::pd_patch_instruction(address branch, address target, const char* file, int line) {
194 if (is_branch(branch)) {
195 jint& stub_inst = *(jint*) branch;
196 stub_inst = patched_branch(target - branch, stub_inst, 0);
197 } else if (is_calculate_address_from_global_toc_at(branch + BytesPerInstWord, branch)) {
198 const address inst1_addr = branch;
199 const address inst2_addr = branch + BytesPerInstWord;
200 patch_calculate_address_from_global_toc_at(inst2_addr, inst1_addr, target);
201 } else if (is_load_const_at(branch)) {
202 patch_const(branch, (long)target);
203 } else {
204 assert(false, "instruction at " PTR_FORMAT " not recognized", p2i(branch));
205 }
206 }
207
208 // Relocation of conditional far branches.
209 inline bool MacroAssembler::is_bc_far_variant1_at(address instruction_addr) {
210 // Variant 1, the 1st instruction contains the destination address:
211 //
212 // bcxx DEST
213 // nop
214 //
215 const int instruction_1 = *(int*)(instruction_addr);
216 const int instruction_2 = *(int*)(instruction_addr + 4);
217 return is_bcxx(instruction_1) &&
218 (inv_bd_field(instruction_1, (intptr_t)instruction_addr) != (intptr_t)(instruction_addr + 2*4)) &&
219 is_nop(instruction_2);
220 }
221
222 // Relocation of conditional far branches.
223 inline bool MacroAssembler::is_bc_far_variant2_at(address instruction_addr) {
224 // Variant 2, the 2nd instruction contains the destination address:
225 //
226 // b!cxx SKIP
227 // bxx DEST
228 // SKIP:
229 //
230 const int instruction_1 = *(int*)(instruction_addr);
231 const int instruction_2 = *(int*)(instruction_addr + 4);
232 return is_bcxx(instruction_1) &&
233 (inv_bd_field(instruction_1, (intptr_t)instruction_addr) == (intptr_t)(instruction_addr + 2*4)) &&
234 is_bxx(instruction_2);
235 }
236
237 // Relocation for conditional branches
238 inline bool MacroAssembler::is_bc_far_variant3_at(address instruction_addr) {
239 // Variant 3, far cond branch to the next instruction, already patched to nops:
240 //
241 // nop
242 // endgroup
243 // SKIP/DEST:
244 //
245 const int instruction_1 = *(int*)(instruction_addr);
246 const int instruction_2 = *(int*)(instruction_addr + 4);
247 return is_nop(instruction_1) &&
248 is_endgroup(instruction_2);
249 }
250
251 // set dst to -1, 0, +1 as follows: if CCR0bi is "greater than", dst is set to 1,
252 // if CCR0bi is "equal", dst is set to 0, otherwise it's set to -1.
253 inline void MacroAssembler::set_cmp3(Register dst) {
254 assert_different_registers(dst, R0);
255 // P10, prefer using setbc instructions
256 if (VM_Version::has_brw()) {
257 setbc(R0, CCR0, Assembler::greater); // Set 1 to R0 if CCR0bi is "greater than", otherwise 0
258 setnbc(dst, CCR0, Assembler::less); // Set -1 to dst if CCR0bi is "less than", otherwise 0
259 } else {
260 mfcr(R0); // copy CR register to R0
261 srwi(dst, R0, 30); // copy the first two bits to dst
262 srawi(R0, R0, 31); // move the first bit to last position - sign extended
263 }
264 orr(dst, dst, R0); // dst | R0 will be -1, 0, or +1
265 }
266
267 // set dst to (treat_unordered_like_less ? -1 : +1)
268 inline void MacroAssembler::set_cmpu3(Register dst, bool treat_unordered_like_less) {
269 if (treat_unordered_like_less) {
270 cror(CCR0, Assembler::less, CCR0, Assembler::summary_overflow); // treat unordered like less
271 } else {
272 cror(CCR0, Assembler::greater, CCR0, Assembler::summary_overflow); // treat unordered like greater
273 }
274 set_cmp3(dst);
275 }
276
277 // Branch-free implementation to convert !=0 to 1
278 // Set register dst to 1 if dst is non-zero. Uses setbcr instruction on Power10.
279 inline void MacroAssembler::normalize_bool(Register dst, Register temp, bool is_64bit) {
280
281 if (VM_Version::has_brw()) {
282 if (is_64bit) {
283 cmpdi(CCR0, dst, 0);
284 } else {
285 cmpwi(CCR0, dst, 0);
286 }
287 setbcr(dst, CCR0, Assembler::equal);
288 } else {
289 assert_different_registers(temp, dst);
290 neg(temp, dst);
291 orr(temp, dst, temp);
292 if (is_64bit) {
293 srdi(dst, temp, 63);
294 } else {
295 srwi(dst, temp, 31);
296 }
297 }
298 }
299
300 // Convenience bc_far versions
301 inline void MacroAssembler::blt_far(ConditionRegister crx, Label& L, int optimize) { MacroAssembler::bc_far(bcondCRbiIs1, bi0(crx, less), L, optimize); }
302 inline void MacroAssembler::bgt_far(ConditionRegister crx, Label& L, int optimize) { MacroAssembler::bc_far(bcondCRbiIs1, bi0(crx, greater), L, optimize); }
303 inline void MacroAssembler::beq_far(ConditionRegister crx, Label& L, int optimize) { MacroAssembler::bc_far(bcondCRbiIs1, bi0(crx, equal), L, optimize); }
304 inline void MacroAssembler::bso_far(ConditionRegister crx, Label& L, int optimize) { MacroAssembler::bc_far(bcondCRbiIs1, bi0(crx, summary_overflow), L, optimize); }
305 inline void MacroAssembler::bge_far(ConditionRegister crx, Label& L, int optimize) { MacroAssembler::bc_far(bcondCRbiIs0, bi0(crx, less), L, optimize); }
306 inline void MacroAssembler::ble_far(ConditionRegister crx, Label& L, int optimize) { MacroAssembler::bc_far(bcondCRbiIs0, bi0(crx, greater), L, optimize); }
307 inline void MacroAssembler::bne_far(ConditionRegister crx, Label& L, int optimize) { MacroAssembler::bc_far(bcondCRbiIs0, bi0(crx, equal), L, optimize); }
308 inline void MacroAssembler::bns_far(ConditionRegister crx, Label& L, int optimize) { MacroAssembler::bc_far(bcondCRbiIs0, bi0(crx, summary_overflow), L, optimize); }
309
310 inline address MacroAssembler::call_stub(Register function_entry) {
311 mtctr(function_entry);
312 bctrl();
313 return pc();
314 }
315
316 inline void MacroAssembler::call_stub_and_return_to(Register function_entry, Register return_pc) {
317 assert_different_registers(function_entry, return_pc);
318 mtlr(return_pc);
319 mtctr(function_entry);
320 bctr();
321 }
322
323 // Get the pc where the last emitted call will return to.
324 inline address MacroAssembler::last_calls_return_pc() {
325 return _last_calls_return_pc;
326 }
327
328 // Read from the polling page, its address is already in a register.
329 inline void MacroAssembler::load_from_polling_page(Register polling_page_address, int offset) {
330 if (USE_POLL_BIT_ONLY) {
331 int encoding = SafepointMechanism::poll_bit();
332 tdi(traptoGreaterThanUnsigned | traptoEqual, polling_page_address, encoding);
333 } else {
334 ld(R0, offset, polling_page_address);
335 }
336 }
337
338 // Trap-instruction-based checks.
339
340 inline void MacroAssembler::trap_null_check(Register a, trap_to_bits cmp) {
341 assert(TrapBasedNullChecks, "sanity");
342 tdi(cmp, a/*reg a*/, 0);
343 }
344
345 inline void MacroAssembler::trap_ic_miss_check(Register a, Register b) {
346 td(traptoGreaterThanUnsigned | traptoLessThanUnsigned, a, b);
347 }
348
349 // Do an explicit null check if access to a+offset will not raise a SIGSEGV.
350 // Either issue a trap instruction that raises SIGTRAP, or do a compare that
351 // branches to exception_entry.
352 // No support for compressed oops (base page of heap). Does not distinguish
353 // loads and stores.
354 inline void MacroAssembler::null_check_throw(Register a, int offset, Register temp_reg,
355 address exception_entry) {
356 if (!ImplicitNullChecks || needs_explicit_null_check(offset) || !os::zero_page_read_protected()) {
357 if (TrapBasedNullChecks) {
358 assert(UseSIGTRAP, "sanity");
359 trap_null_check(a);
360 } else {
361 Label ok;
362 cmpdi(CCR0, a, 0);
363 bne(CCR0, ok);
364 load_const_optimized(temp_reg, exception_entry);
365 mtctr(temp_reg);
366 bctr();
367 bind(ok);
368 }
369 }
370 }
371
372 inline void MacroAssembler::null_check(Register a, int offset, Label *Lis_null) {
373 if (!ImplicitNullChecks || needs_explicit_null_check(offset) || !os::zero_page_read_protected()) {
374 if (TrapBasedNullChecks) {
375 assert(UseSIGTRAP, "sanity");
376 trap_null_check(a);
377 } else if (Lis_null){
378 Label ok;
379 cmpdi(CCR0, a, 0);
380 beq(CCR0, *Lis_null);
381 }
382 }
383 }
384
385 inline void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators,
386 Register base, RegisterOrConstant ind_or_offs, Register val,
387 Register tmp1, Register tmp2, Register tmp3,
388 MacroAssembler::PreservationLevel preservation_level) {
389 assert((decorators & ~(AS_RAW | IN_HEAP | IN_NATIVE | IS_ARRAY | IS_NOT_NULL |
390 ON_UNKNOWN_OOP_REF | IS_DEST_UNINITIALIZED)) == 0, "unsupported decorator");
391 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
392 bool as_raw = (decorators & AS_RAW) != 0;
393 decorators = AccessInternal::decorator_fixup(decorators, type);
394 if (as_raw) {
395 bs->BarrierSetAssembler::store_at(this, decorators, type,
396 base, ind_or_offs, val,
397 tmp1, tmp2, tmp3, preservation_level);
398 } else {
399 bs->store_at(this, decorators, type,
400 base, ind_or_offs, val,
401 tmp1, tmp2, tmp3, preservation_level);
402 }
403 }
404
405 inline void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators,
406 Register base, RegisterOrConstant ind_or_offs, Register dst,
407 Register tmp1, Register tmp2,
408 MacroAssembler::PreservationLevel preservation_level,
409 Label *L_handle_null) {
410 assert((decorators & ~(AS_RAW | IN_HEAP | IN_NATIVE | IS_ARRAY | IS_NOT_NULL |
411 ON_PHANTOM_OOP_REF | ON_WEAK_OOP_REF)) == 0, "unsupported decorator");
412 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
413 decorators = AccessInternal::decorator_fixup(decorators, type);
414 bool as_raw = (decorators & AS_RAW) != 0;
415 if (as_raw) {
416 bs->BarrierSetAssembler::load_at(this, decorators, type,
417 base, ind_or_offs, dst,
418 tmp1, tmp2, preservation_level, L_handle_null);
419 } else {
420 bs->load_at(this, decorators, type,
421 base, ind_or_offs, dst,
422 tmp1, tmp2, preservation_level, L_handle_null);
423 }
424 }
425
426 inline void MacroAssembler::load_heap_oop(Register d, RegisterOrConstant offs, Register s1,
427 Register tmp1, Register tmp2,
428 MacroAssembler::PreservationLevel preservation_level,
429 DecoratorSet decorators, Label *L_handle_null) {
430 access_load_at(T_OBJECT, decorators | IN_HEAP, s1, offs, d, tmp1, tmp2,
431 preservation_level, L_handle_null);
432 }
433
434 inline void MacroAssembler::store_heap_oop(Register val, RegisterOrConstant offs, Register base,
435 Register tmp1, Register tmp2, Register tmp3,
436 MacroAssembler::PreservationLevel preservation_level,
437 DecoratorSet decorators) {
438 access_store_at(T_OBJECT, decorators | IN_HEAP, base, offs, val, tmp1, tmp2, tmp3, preservation_level);
439 }
440
441 inline Register MacroAssembler::encode_heap_oop_not_null(Register d, Register src) {
442 Register current = (src != noreg) ? src : d; // Oop to be compressed is in d if no src provided.
443 if (CompressedOops::base_overlaps()) {
444 sub_const_optimized(d, current, CompressedOops::base(), R0);
445 current = d;
446 }
447 if (CompressedOops::shift() != 0) {
448 rldicl(d, current, 64-CompressedOops::shift(), 32); // Clears the upper bits.
449 current = d;
450 }
451 return current; // Encoded oop is in this register.
452 }
453
454 inline Register MacroAssembler::encode_heap_oop(Register d, Register src) {
455 if (CompressedOops::base() != nullptr) {
456 if (VM_Version::has_isel()) {
457 cmpdi(CCR0, src, 0);
458 Register co = encode_heap_oop_not_null(d, src);
459 assert(co == d, "sanity");
460 isel_0(d, CCR0, Assembler::equal);
461 } else {
462 Label isNull;
463 or_(d, src, src); // move and compare 0
464 beq(CCR0, isNull);
465 encode_heap_oop_not_null(d, src);
466 bind(isNull);
467 }
468 return d;
469 } else {
470 return encode_heap_oop_not_null(d, src);
471 }
472 }
473
474 inline Register MacroAssembler::decode_heap_oop_not_null(Register d, Register src) {
475 if (CompressedOops::base_disjoint() && src != noreg && src != d &&
476 CompressedOops::shift() != 0) {
477 load_const_optimized(d, CompressedOops::base(), R0);
478 rldimi(d, src, CompressedOops::shift(), 32-CompressedOops::shift());
479 return d;
480 }
481
482 Register current = (src != noreg) ? src : d; // Compressed oop is in d if no src provided.
483 if (CompressedOops::shift() != 0) {
484 sldi(d, current, CompressedOops::shift());
485 current = d;
486 }
487 if (CompressedOops::base() != nullptr) {
488 add_const_optimized(d, current, CompressedOops::base(), R0);
489 current = d;
490 }
491 return current; // Decoded oop is in this register.
492 }
493
494 inline void MacroAssembler::decode_heap_oop(Register d) {
495 Label isNull;
496 bool use_isel = false;
497 if (CompressedOops::base() != nullptr) {
498 cmpwi(CCR0, d, 0);
499 if (VM_Version::has_isel()) {
500 use_isel = true;
501 } else {
502 beq(CCR0, isNull);
503 }
504 }
505 decode_heap_oop_not_null(d);
506 if (use_isel) {
507 isel_0(d, CCR0, Assembler::equal);
508 }
509 bind(isNull);
510 }
511
512 // SIGTRAP-based range checks for arrays.
513 inline void MacroAssembler::trap_range_check_l(Register a, Register b) {
514 tw (traptoLessThanUnsigned, a/*reg a*/, b/*reg b*/);
515 }
516 inline void MacroAssembler::trap_range_check_l(Register a, int si16) {
517 twi(traptoLessThanUnsigned, a/*reg a*/, si16);
518 }
519 inline void MacroAssembler::trap_range_check_le(Register a, int si16) {
520 twi(traptoEqual | traptoLessThanUnsigned, a/*reg a*/, si16);
521 }
522 inline void MacroAssembler::trap_range_check_g(Register a, int si16) {
523 twi(traptoGreaterThanUnsigned, a/*reg a*/, si16);
524 }
525 inline void MacroAssembler::trap_range_check_ge(Register a, Register b) {
526 tw (traptoEqual | traptoGreaterThanUnsigned, a/*reg a*/, b/*reg b*/);
527 }
528 inline void MacroAssembler::trap_range_check_ge(Register a, int si16) {
529 twi(traptoEqual | traptoGreaterThanUnsigned, a/*reg a*/, si16);
530 }
531
532 // unsigned integer multiplication 64*64 -> 128 bits
533 inline void MacroAssembler::multiply64(Register dest_hi, Register dest_lo,
534 Register x, Register y) {
535 mulld(dest_lo, x, y);
536 mulhdu(dest_hi, x, y);
537 }
538
539 #if defined(ABI_ELFv2)
540 inline address MacroAssembler::function_entry() { return pc(); }
541 #else
542 inline address MacroAssembler::function_entry() { return emit_fd(); }
543 #endif
544
545 #endif // CPU_PPC_MACROASSEMBLER_PPC_INLINE_HPP