1 /* 2 * Copyright (c) 2002, 2024, Oracle and/or its affiliates. All rights reserved. 3 * Copyright (c) 2012, 2024 SAP SE. All rights reserved. 4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 * 6 * This code is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 only, as 8 * published by the Free Software Foundation. 9 * 10 * This code is distributed in the hope that it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * version 2 for more details (a copy is included in the LICENSE file that 14 * accompanied this code). 15 * 16 * You should have received a copy of the GNU General Public License version 17 * 2 along with this work; if not, write to the Free Software Foundation, 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 * or visit www.oracle.com if you need additional information or have any 22 * questions. 23 * 24 */ 25 26 #ifndef CPU_PPC_MACROASSEMBLER_PPC_INLINE_HPP 27 #define CPU_PPC_MACROASSEMBLER_PPC_INLINE_HPP 28 29 #include "asm/assembler.inline.hpp" 30 #include "asm/macroAssembler.hpp" 31 #include "asm/codeBuffer.hpp" 32 #include "code/codeCache.hpp" 33 #include "gc/shared/barrierSet.hpp" 34 #include "gc/shared/barrierSetAssembler.hpp" 35 #include "oops/accessDecorators.hpp" 36 #include "oops/compressedOops.hpp" 37 #include "runtime/os.inline.hpp" 38 #include "runtime/safepointMechanism.hpp" 39 #include "runtime/vm_version.hpp" 40 #include "utilities/powerOfTwo.hpp" 41 42 inline bool MacroAssembler::is_ld_largeoffset(address a) { 43 const int inst1 = *(int *)a; 44 const int inst2 = *(int *)(a+4); 45 return (is_ld(inst1)) || 46 (is_addis(inst1) && is_ld(inst2) && inv_ra_field(inst2) == inv_rt_field(inst1)); 47 } 48 49 inline int MacroAssembler::get_ld_largeoffset_offset(address a) { 50 assert(MacroAssembler::is_ld_largeoffset(a), "must be ld with large offset"); 51 52 const int inst1 = *(int *)a; 53 if (is_ld(inst1)) { 54 return inv_d1_field(inst1); 55 } else { 56 const int inst2 = *(int *)(a+4); 57 return (inv_d1_field(inst1) << 16) + inv_d1_field(inst2); 58 } 59 } 60 61 inline void MacroAssembler::round_to(Register r, int modulus) { 62 assert(is_power_of_2((jlong)modulus), "must be power of 2"); 63 addi(r, r, modulus-1); 64 clrrdi(r, r, log2i_exact((jlong)modulus)); 65 } 66 67 // Move register if destination register and target register are different. 68 inline void MacroAssembler::mr_if_needed(Register rd, Register rs) { 69 if (rs != rd) mr(rd, rs); 70 } 71 inline void MacroAssembler::fmr_if_needed(FloatRegister rd, FloatRegister rs) { 72 if (rs != rd) fmr(rd, rs); 73 } 74 inline void MacroAssembler::endgroup_if_needed(bool needed) { 75 if (needed) { 76 endgroup(); 77 } 78 } 79 80 inline void MacroAssembler::membar(int bits) { 81 // Comment: Usage of elemental_membar(bits) is not recommended for Power 8. 82 // If elemental_membar(bits) is used, disable optimization of acquire-release 83 // (Matcher::post_membar_release where we use PPC64_ONLY(xop == Op_MemBarRelease ||))! 84 if (bits & StoreLoad) { sync(); } 85 else if (bits) { lwsync(); } 86 } 87 inline void MacroAssembler::release() { membar(LoadStore | StoreStore); } 88 inline void MacroAssembler::acquire() { membar(LoadLoad | LoadStore); } 89 inline void MacroAssembler::fence() { membar(LoadLoad | LoadStore | StoreLoad | StoreStore); } 90 91 // Address of the global TOC. 92 inline address MacroAssembler::global_toc() { 93 return CodeCache::low_bound(); 94 } 95 96 // Offset of given address to the global TOC. 97 inline int MacroAssembler::offset_to_global_toc(const address addr) { 98 intptr_t offset = (intptr_t)addr - (intptr_t)MacroAssembler::global_toc(); 99 assert(Assembler::is_uimm((long)offset, 31), "must be in range"); 100 return (int)offset; 101 } 102 103 // Address of current method's TOC. 104 inline address MacroAssembler::method_toc() { 105 return code()->consts()->start(); 106 } 107 108 // Offset of given address to current method's TOC. 109 inline int MacroAssembler::offset_to_method_toc(address addr) { 110 intptr_t offset = (intptr_t)addr - (intptr_t)method_toc(); 111 assert(Assembler::is_uimm((long)offset, 31), "must be in range"); 112 return (int)offset; 113 } 114 115 inline bool MacroAssembler::is_calculate_address_from_global_toc_at(address a, address bound) { 116 const address inst2_addr = a; 117 const int inst2 = *(int *) a; 118 119 // The relocation points to the second instruction, the addi. 120 if (!is_addi(inst2)) return false; 121 122 // The addi reads and writes the same register dst. 123 const int dst = inv_rt_field(inst2); 124 if (inv_ra_field(inst2) != dst) return false; 125 126 // Now, find the preceding addis which writes to dst. 127 int inst1 = 0; 128 address inst1_addr = inst2_addr - BytesPerInstWord; 129 while (inst1_addr >= bound) { 130 inst1 = *(int *) inst1_addr; 131 if (is_addis(inst1) && inv_rt_field(inst1) == dst) { 132 // stop, found the addis which writes dst 133 break; 134 } 135 inst1_addr -= BytesPerInstWord; 136 } 137 138 if (!(inst1 == 0 || inv_ra_field(inst1) == 29 /* R29 */)) return false; 139 return is_addis(inst1); 140 } 141 142 #ifdef _LP64 143 // Detect narrow oop constants. 144 inline bool MacroAssembler::is_set_narrow_oop(address a, address bound) { 145 const address inst2_addr = a; 146 const int inst2 = *(int *)a; 147 // The relocation points to the second instruction, the ori. 148 if (!is_ori(inst2)) return false; 149 150 // The ori reads and writes the same register dst. 151 const int dst = inv_rta_field(inst2); 152 if (inv_rs_field(inst2) != dst) return false; 153 154 // Now, find the preceding addis which writes to dst. 155 int inst1 = 0; 156 address inst1_addr = inst2_addr - BytesPerInstWord; 157 while (inst1_addr >= bound) { 158 inst1 = *(int *) inst1_addr; 159 if (is_lis(inst1) && inv_rs_field(inst1) == dst) return true; 160 inst1_addr -= BytesPerInstWord; 161 } 162 return false; 163 } 164 #endif 165 166 167 inline bool MacroAssembler::is_load_const_at(address a) { 168 const int* p_inst = (int *) a; 169 bool b = is_lis(*p_inst++); 170 if (is_ori(*p_inst)) { 171 p_inst++; 172 b = b && is_rldicr(*p_inst++); // TODO: could be made more precise: `sldi'! 173 b = b && is_oris(*p_inst++); 174 b = b && is_ori(*p_inst); 175 } else if (is_lis(*p_inst)) { 176 p_inst++; 177 b = b && is_ori(*p_inst++); 178 b = b && is_ori(*p_inst); 179 // TODO: could enhance reliability by adding is_insrdi 180 } else return false; 181 return b; 182 } 183 184 inline void MacroAssembler::set_oop_constant(jobject obj, Register d) { 185 set_oop(constant_oop_address(obj), d); 186 } 187 188 inline void MacroAssembler::set_oop(AddressLiteral obj_addr, Register d) { 189 assert(obj_addr.rspec().type() == relocInfo::oop_type, "must be an oop reloc"); 190 load_const(d, obj_addr); 191 } 192 193 inline void MacroAssembler::pd_patch_instruction(address branch, address target, const char* file, int line) { 194 if (is_branch(branch)) { 195 jint& stub_inst = *(jint*) branch; 196 stub_inst = patched_branch(target - branch, stub_inst, 0); 197 } else if (is_calculate_address_from_global_toc_at(branch + BytesPerInstWord, branch)) { 198 const address inst1_addr = branch; 199 const address inst2_addr = branch + BytesPerInstWord; 200 patch_calculate_address_from_global_toc_at(inst2_addr, inst1_addr, target); 201 } else if (is_load_const_at(branch)) { 202 patch_const(branch, (long)target); 203 } else { 204 assert(false, "instruction at " PTR_FORMAT " not recognized", p2i(branch)); 205 } 206 } 207 208 // Relocation of conditional far branches. 209 inline bool MacroAssembler::is_bc_far_variant1_at(address instruction_addr) { 210 // Variant 1, the 1st instruction contains the destination address: 211 // 212 // bcxx DEST 213 // nop 214 // 215 const int instruction_1 = *(int*)(instruction_addr); 216 const int instruction_2 = *(int*)(instruction_addr + 4); 217 return is_bcxx(instruction_1) && 218 (inv_bd_field(instruction_1, (intptr_t)instruction_addr) != (intptr_t)(instruction_addr + 2*4)) && 219 is_nop(instruction_2); 220 } 221 222 // Relocation of conditional far branches. 223 inline bool MacroAssembler::is_bc_far_variant2_at(address instruction_addr) { 224 // Variant 2, the 2nd instruction contains the destination address: 225 // 226 // b!cxx SKIP 227 // bxx DEST 228 // SKIP: 229 // 230 const int instruction_1 = *(int*)(instruction_addr); 231 const int instruction_2 = *(int*)(instruction_addr + 4); 232 return is_bcxx(instruction_1) && 233 (inv_bd_field(instruction_1, (intptr_t)instruction_addr) == (intptr_t)(instruction_addr + 2*4)) && 234 is_bxx(instruction_2); 235 } 236 237 // Relocation for conditional branches 238 inline bool MacroAssembler::is_bc_far_variant3_at(address instruction_addr) { 239 // Variant 3, far cond branch to the next instruction, already patched to nops: 240 // 241 // nop 242 // endgroup 243 // SKIP/DEST: 244 // 245 const int instruction_1 = *(int*)(instruction_addr); 246 const int instruction_2 = *(int*)(instruction_addr + 4); 247 return is_nop(instruction_1) && 248 is_endgroup(instruction_2); 249 } 250 251 // set dst to -1, 0, +1 as follows: if CCR0bi is "greater than", dst is set to 1, 252 // if CCR0bi is "equal", dst is set to 0, otherwise it's set to -1. 253 inline void MacroAssembler::set_cmp3(Register dst) { 254 assert_different_registers(dst, R0); 255 // P10, prefer using setbc instructions 256 if (VM_Version::has_brw()) { 257 setbc(R0, CCR0, Assembler::greater); // Set 1 to R0 if CCR0bi is "greater than", otherwise 0 258 setnbc(dst, CCR0, Assembler::less); // Set -1 to dst if CCR0bi is "less than", otherwise 0 259 } else { 260 mfcr(R0); // copy CR register to R0 261 srwi(dst, R0, 30); // copy the first two bits to dst 262 srawi(R0, R0, 31); // move the first bit to last position - sign extended 263 } 264 orr(dst, dst, R0); // dst | R0 will be -1, 0, or +1 265 } 266 267 // set dst to (treat_unordered_like_less ? -1 : +1) 268 inline void MacroAssembler::set_cmpu3(Register dst, bool treat_unordered_like_less) { 269 if (treat_unordered_like_less) { 270 cror(CCR0, Assembler::less, CCR0, Assembler::summary_overflow); // treat unordered like less 271 } else { 272 cror(CCR0, Assembler::greater, CCR0, Assembler::summary_overflow); // treat unordered like greater 273 } 274 set_cmp3(dst); 275 } 276 277 // Branch-free implementation to convert !=0 to 1 278 // Set register dst to 1 if dst is non-zero. Uses setbcr instruction on Power10. 279 inline void MacroAssembler::normalize_bool(Register dst, Register temp, bool is_64bit) { 280 281 if (VM_Version::has_brw()) { 282 if (is_64bit) { 283 cmpdi(CCR0, dst, 0); 284 } else { 285 cmpwi(CCR0, dst, 0); 286 } 287 setbcr(dst, CCR0, Assembler::equal); 288 } else { 289 assert_different_registers(temp, dst); 290 neg(temp, dst); 291 orr(temp, dst, temp); 292 if (is_64bit) { 293 srdi(dst, temp, 63); 294 } else { 295 srwi(dst, temp, 31); 296 } 297 } 298 } 299 300 // Convenience bc_far versions 301 inline void MacroAssembler::blt_far(ConditionRegister crx, Label& L, int optimize) { MacroAssembler::bc_far(bcondCRbiIs1, bi0(crx, less), L, optimize); } 302 inline void MacroAssembler::bgt_far(ConditionRegister crx, Label& L, int optimize) { MacroAssembler::bc_far(bcondCRbiIs1, bi0(crx, greater), L, optimize); } 303 inline void MacroAssembler::beq_far(ConditionRegister crx, Label& L, int optimize) { MacroAssembler::bc_far(bcondCRbiIs1, bi0(crx, equal), L, optimize); } 304 inline void MacroAssembler::bso_far(ConditionRegister crx, Label& L, int optimize) { MacroAssembler::bc_far(bcondCRbiIs1, bi0(crx, summary_overflow), L, optimize); } 305 inline void MacroAssembler::bge_far(ConditionRegister crx, Label& L, int optimize) { MacroAssembler::bc_far(bcondCRbiIs0, bi0(crx, less), L, optimize); } 306 inline void MacroAssembler::ble_far(ConditionRegister crx, Label& L, int optimize) { MacroAssembler::bc_far(bcondCRbiIs0, bi0(crx, greater), L, optimize); } 307 inline void MacroAssembler::bne_far(ConditionRegister crx, Label& L, int optimize) { MacroAssembler::bc_far(bcondCRbiIs0, bi0(crx, equal), L, optimize); } 308 inline void MacroAssembler::bns_far(ConditionRegister crx, Label& L, int optimize) { MacroAssembler::bc_far(bcondCRbiIs0, bi0(crx, summary_overflow), L, optimize); } 309 310 inline address MacroAssembler::call_stub(Register function_entry) { 311 mtctr(function_entry); 312 bctrl(); 313 return pc(); 314 } 315 316 inline void MacroAssembler::call_stub_and_return_to(Register function_entry, Register return_pc) { 317 assert_different_registers(function_entry, return_pc); 318 mtlr(return_pc); 319 mtctr(function_entry); 320 bctr(); 321 } 322 323 // Get the pc where the last emitted call will return to. 324 inline address MacroAssembler::last_calls_return_pc() { 325 return _last_calls_return_pc; 326 } 327 328 // Read from the polling page, its address is already in a register. 329 inline void MacroAssembler::load_from_polling_page(Register polling_page_address, int offset) { 330 if (USE_POLL_BIT_ONLY) { 331 int encoding = SafepointMechanism::poll_bit(); 332 tdi(traptoGreaterThanUnsigned | traptoEqual, polling_page_address, encoding); 333 } else { 334 ld(R0, offset, polling_page_address); 335 } 336 } 337 338 // Trap-instruction-based checks. 339 340 inline void MacroAssembler::trap_null_check(Register a, trap_to_bits cmp) { 341 assert(TrapBasedNullChecks, "sanity"); 342 tdi(cmp, a/*reg a*/, 0); 343 } 344 345 inline void MacroAssembler::trap_ic_miss_check(Register a, Register b) { 346 td(traptoGreaterThanUnsigned | traptoLessThanUnsigned, a, b); 347 } 348 349 // Do an explicit null check if access to a+offset will not raise a SIGSEGV. 350 // Either issue a trap instruction that raises SIGTRAP, or do a compare that 351 // branches to exception_entry. 352 // No support for compressed oops (base page of heap). Does not distinguish 353 // loads and stores. 354 inline void MacroAssembler::null_check_throw(Register a, int offset, Register temp_reg, 355 address exception_entry) { 356 if (!ImplicitNullChecks || needs_explicit_null_check(offset) || !os::zero_page_read_protected()) { 357 if (TrapBasedNullChecks) { 358 assert(UseSIGTRAP, "sanity"); 359 trap_null_check(a); 360 } else { 361 Label ok; 362 cmpdi(CCR0, a, 0); 363 bne(CCR0, ok); 364 load_const_optimized(temp_reg, exception_entry); 365 mtctr(temp_reg); 366 bctr(); 367 bind(ok); 368 } 369 } 370 } 371 372 inline void MacroAssembler::null_check(Register a, int offset, Label *Lis_null) { 373 if (!ImplicitNullChecks || needs_explicit_null_check(offset) || !os::zero_page_read_protected()) { 374 if (TrapBasedNullChecks) { 375 assert(UseSIGTRAP, "sanity"); 376 trap_null_check(a); 377 } else if (Lis_null){ 378 Label ok; 379 cmpdi(CCR0, a, 0); 380 beq(CCR0, *Lis_null); 381 } 382 } 383 } 384 385 inline void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, 386 Register base, RegisterOrConstant ind_or_offs, Register val, 387 Register tmp1, Register tmp2, Register tmp3, 388 MacroAssembler::PreservationLevel preservation_level) { 389 assert((decorators & ~(AS_RAW | IN_HEAP | IN_NATIVE | IS_ARRAY | IS_NOT_NULL | 390 ON_UNKNOWN_OOP_REF | IS_DEST_UNINITIALIZED)) == 0, "unsupported decorator"); 391 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); 392 bool as_raw = (decorators & AS_RAW) != 0; 393 decorators = AccessInternal::decorator_fixup(decorators, type); 394 if (as_raw) { 395 bs->BarrierSetAssembler::store_at(this, decorators, type, 396 base, ind_or_offs, val, 397 tmp1, tmp2, tmp3, preservation_level); 398 } else { 399 bs->store_at(this, decorators, type, 400 base, ind_or_offs, val, 401 tmp1, tmp2, tmp3, preservation_level); 402 } 403 } 404 405 inline void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, 406 Register base, RegisterOrConstant ind_or_offs, Register dst, 407 Register tmp1, Register tmp2, 408 MacroAssembler::PreservationLevel preservation_level, 409 Label *L_handle_null) { 410 assert((decorators & ~(AS_RAW | IN_HEAP | IN_NATIVE | IS_ARRAY | IS_NOT_NULL | 411 ON_PHANTOM_OOP_REF | ON_WEAK_OOP_REF)) == 0, "unsupported decorator"); 412 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); 413 decorators = AccessInternal::decorator_fixup(decorators, type); 414 bool as_raw = (decorators & AS_RAW) != 0; 415 if (as_raw) { 416 bs->BarrierSetAssembler::load_at(this, decorators, type, 417 base, ind_or_offs, dst, 418 tmp1, tmp2, preservation_level, L_handle_null); 419 } else { 420 bs->load_at(this, decorators, type, 421 base, ind_or_offs, dst, 422 tmp1, tmp2, preservation_level, L_handle_null); 423 } 424 } 425 426 inline void MacroAssembler::load_heap_oop(Register d, RegisterOrConstant offs, Register s1, 427 Register tmp1, Register tmp2, 428 MacroAssembler::PreservationLevel preservation_level, 429 DecoratorSet decorators, Label *L_handle_null) { 430 access_load_at(T_OBJECT, decorators | IN_HEAP, s1, offs, d, tmp1, tmp2, 431 preservation_level, L_handle_null); 432 } 433 434 inline void MacroAssembler::store_heap_oop(Register val, RegisterOrConstant offs, Register base, 435 Register tmp1, Register tmp2, Register tmp3, 436 MacroAssembler::PreservationLevel preservation_level, 437 DecoratorSet decorators) { 438 access_store_at(T_OBJECT, decorators | IN_HEAP, base, offs, val, tmp1, tmp2, tmp3, preservation_level); 439 } 440 441 inline Register MacroAssembler::encode_heap_oop_not_null(Register d, Register src) { 442 Register current = (src != noreg) ? src : d; // Oop to be compressed is in d if no src provided. 443 if (CompressedOops::base_overlaps()) { 444 sub_const_optimized(d, current, CompressedOops::base(), R0); 445 current = d; 446 } 447 if (CompressedOops::shift() != 0) { 448 rldicl(d, current, 64-CompressedOops::shift(), 32); // Clears the upper bits. 449 current = d; 450 } 451 return current; // Encoded oop is in this register. 452 } 453 454 inline Register MacroAssembler::encode_heap_oop(Register d, Register src) { 455 if (CompressedOops::base() != nullptr) { 456 if (VM_Version::has_isel()) { 457 cmpdi(CCR0, src, 0); 458 Register co = encode_heap_oop_not_null(d, src); 459 assert(co == d, "sanity"); 460 isel_0(d, CCR0, Assembler::equal); 461 } else { 462 Label isNull; 463 or_(d, src, src); // move and compare 0 464 beq(CCR0, isNull); 465 encode_heap_oop_not_null(d, src); 466 bind(isNull); 467 } 468 return d; 469 } else { 470 return encode_heap_oop_not_null(d, src); 471 } 472 } 473 474 inline Register MacroAssembler::decode_heap_oop_not_null(Register d, Register src) { 475 if (CompressedOops::base_disjoint() && src != noreg && src != d && 476 CompressedOops::shift() != 0) { 477 load_const_optimized(d, CompressedOops::base(), R0); 478 rldimi(d, src, CompressedOops::shift(), 32-CompressedOops::shift()); 479 return d; 480 } 481 482 Register current = (src != noreg) ? src : d; // Compressed oop is in d if no src provided. 483 if (CompressedOops::shift() != 0) { 484 sldi(d, current, CompressedOops::shift()); 485 current = d; 486 } 487 if (CompressedOops::base() != nullptr) { 488 add_const_optimized(d, current, CompressedOops::base(), R0); 489 current = d; 490 } 491 return current; // Decoded oop is in this register. 492 } 493 494 inline void MacroAssembler::decode_heap_oop(Register d) { 495 Label isNull; 496 bool use_isel = false; 497 if (CompressedOops::base() != nullptr) { 498 cmpwi(CCR0, d, 0); 499 if (VM_Version::has_isel()) { 500 use_isel = true; 501 } else { 502 beq(CCR0, isNull); 503 } 504 } 505 decode_heap_oop_not_null(d); 506 if (use_isel) { 507 isel_0(d, CCR0, Assembler::equal); 508 } 509 bind(isNull); 510 } 511 512 // SIGTRAP-based range checks for arrays. 513 inline void MacroAssembler::trap_range_check_l(Register a, Register b) { 514 tw (traptoLessThanUnsigned, a/*reg a*/, b/*reg b*/); 515 } 516 inline void MacroAssembler::trap_range_check_l(Register a, int si16) { 517 twi(traptoLessThanUnsigned, a/*reg a*/, si16); 518 } 519 inline void MacroAssembler::trap_range_check_le(Register a, int si16) { 520 twi(traptoEqual | traptoLessThanUnsigned, a/*reg a*/, si16); 521 } 522 inline void MacroAssembler::trap_range_check_g(Register a, int si16) { 523 twi(traptoGreaterThanUnsigned, a/*reg a*/, si16); 524 } 525 inline void MacroAssembler::trap_range_check_ge(Register a, Register b) { 526 tw (traptoEqual | traptoGreaterThanUnsigned, a/*reg a*/, b/*reg b*/); 527 } 528 inline void MacroAssembler::trap_range_check_ge(Register a, int si16) { 529 twi(traptoEqual | traptoGreaterThanUnsigned, a/*reg a*/, si16); 530 } 531 532 // unsigned integer multiplication 64*64 -> 128 bits 533 inline void MacroAssembler::multiply64(Register dest_hi, Register dest_lo, 534 Register x, Register y) { 535 mulld(dest_lo, x, y); 536 mulhdu(dest_hi, x, y); 537 } 538 539 #if defined(ABI_ELFv2) 540 inline address MacroAssembler::function_entry() { return pc(); } 541 #else 542 inline address MacroAssembler::function_entry() { return emit_fd(); } 543 #endif 544 545 #endif // CPU_PPC_MACROASSEMBLER_PPC_INLINE_HPP