1 /* 2 * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved. 3 * Copyright (c) 2014, 2024, Red Hat Inc. All rights reserved. 4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 * 6 * This code is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 only, as 8 * published by the Free Software Foundation. 9 * 10 * This code is distributed in the hope that it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * version 2 for more details (a copy is included in the LICENSE file that 14 * accompanied this code). 15 * 16 * You should have received a copy of the GNU General Public License version 17 * 2 along with this work; if not, write to the Free Software Foundation, 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 * or visit www.oracle.com if you need additional information or have any 22 * questions. 23 * 24 */ 25 26 #include "asm/assembler.hpp" 27 #include "asm/assembler.inline.hpp" 28 #include "ci/ciEnv.hpp" 29 #include "ci/ciInlineKlass.hpp" 30 #include "code/compiledIC.hpp" 31 #include "compiler/compileTask.hpp" 32 #include "compiler/disassembler.hpp" 33 #include "compiler/oopMap.hpp" 34 #include "gc/shared/barrierSet.hpp" 35 #include "gc/shared/barrierSetAssembler.hpp" 36 #include "gc/shared/cardTableBarrierSet.hpp" 37 #include "gc/shared/cardTable.hpp" 38 #include "gc/shared/collectedHeap.hpp" 39 #include "gc/shared/tlab_globals.hpp" 40 #include "interpreter/bytecodeHistogram.hpp" 41 #include "interpreter/interpreter.hpp" 42 #include "interpreter/interpreterRuntime.hpp" 43 #include "jvm.h" 44 #include "memory/resourceArea.hpp" 45 #include "memory/universe.hpp" 46 #include "nativeInst_aarch64.hpp" 47 #include "oops/accessDecorators.hpp" 48 #include "oops/compressedKlass.inline.hpp" 49 #include "oops/compressedOops.inline.hpp" 50 #include "oops/klass.inline.hpp" 51 #include "oops/resolvedFieldEntry.hpp" 52 #include "runtime/continuation.hpp" 53 #include "runtime/globals.hpp" 54 #include "runtime/icache.hpp" 55 #include "runtime/interfaceSupport.inline.hpp" 56 #include "runtime/javaThread.hpp" 57 #include "runtime/jniHandles.inline.hpp" 58 #include "runtime/sharedRuntime.hpp" 59 #include "runtime/signature_cc.hpp" 60 #include "runtime/stubRoutines.hpp" 61 #include "utilities/globalDefinitions.hpp" 62 #include "utilities/powerOfTwo.hpp" 63 #include "vmreg_aarch64.inline.hpp" 64 #ifdef COMPILER1 65 #include "c1/c1_LIRAssembler.hpp" 66 #endif 67 #ifdef COMPILER2 68 #include "oops/oop.hpp" 69 #include "opto/compile.hpp" 70 #include "opto/node.hpp" 71 #include "opto/output.hpp" 72 #endif 73 74 #include <sys/types.h> 75 76 #ifdef PRODUCT 77 #define BLOCK_COMMENT(str) /* nothing */ 78 #else 79 #define BLOCK_COMMENT(str) block_comment(str) 80 #endif 81 #define STOP(str) stop(str); 82 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":") 83 84 #ifdef ASSERT 85 extern "C" void disnm(intptr_t p); 86 #endif 87 // Target-dependent relocation processing 88 // 89 // Instruction sequences whose target may need to be retrieved or 90 // patched are distinguished by their leading instruction, sorting 91 // them into three main instruction groups and related subgroups. 92 // 93 // 1) Branch, Exception and System (insn count = 1) 94 // 1a) Unconditional branch (immediate): 95 // b/bl imm19 96 // 1b) Compare & branch (immediate): 97 // cbz/cbnz Rt imm19 98 // 1c) Test & branch (immediate): 99 // tbz/tbnz Rt imm14 100 // 1d) Conditional branch (immediate): 101 // b.cond imm19 102 // 103 // 2) Loads and Stores (insn count = 1) 104 // 2a) Load register literal: 105 // ldr Rt imm19 106 // 107 // 3) Data Processing Immediate (insn count = 2 or 3) 108 // 3a) PC-rel. addressing 109 // adr/adrp Rx imm21; ldr/str Ry Rx #imm12 110 // adr/adrp Rx imm21; add Ry Rx #imm12 111 // adr/adrp Rx imm21; movk Rx #imm16<<32; ldr/str Ry, [Rx, #offset_in_page] 112 // adr/adrp Rx imm21 113 // adr/adrp Rx imm21; movk Rx #imm16<<32 114 // adr/adrp Rx imm21; movk Rx #imm16<<32; add Ry, Rx, #offset_in_page 115 // The latter form can only happen when the target is an 116 // ExternalAddress, and (by definition) ExternalAddresses don't 117 // move. Because of that property, there is never any need to 118 // patch the last of the three instructions. However, 119 // MacroAssembler::target_addr_for_insn takes all three 120 // instructions into account and returns the correct address. 121 // 3b) Move wide (immediate) 122 // movz Rx #imm16; movk Rx #imm16 << 16; movk Rx #imm16 << 32; 123 // 124 // A switch on a subset of the instruction's bits provides an 125 // efficient dispatch to these subcases. 126 // 127 // insn[28:26] -> main group ('x' == don't care) 128 // 00x -> UNALLOCATED 129 // 100 -> Data Processing Immediate 130 // 101 -> Branch, Exception and System 131 // x1x -> Loads and Stores 132 // 133 // insn[30:25] -> subgroup ('_' == group, 'x' == don't care). 134 // n.b. in some cases extra bits need to be checked to verify the 135 // instruction is as expected 136 // 137 // 1) ... xx101x Branch, Exception and System 138 // 1a) 00___x Unconditional branch (immediate) 139 // 1b) 01___0 Compare & branch (immediate) 140 // 1c) 01___1 Test & branch (immediate) 141 // 1d) 10___0 Conditional branch (immediate) 142 // other Should not happen 143 // 144 // 2) ... xxx1x0 Loads and Stores 145 // 2a) xx1__00 Load/Store register (insn[28] == 1 && insn[24] == 0) 146 // 2aa) x01__00 Load register literal (i.e. requires insn[29] == 0) 147 // strictly should be 64 bit non-FP/SIMD i.e. 148 // 0101_000 (i.e. requires insn[31:24] == 01011000) 149 // 150 // 3) ... xx100x Data Processing Immediate 151 // 3a) xx___00 PC-rel. addressing (n.b. requires insn[24] == 0) 152 // 3b) xx___101 Move wide (immediate) (n.b. requires insn[24:23] == 01) 153 // strictly should be 64 bit movz #imm16<<0 154 // 110___10100 (i.e. requires insn[31:21] == 11010010100) 155 // 156 class RelocActions { 157 protected: 158 typedef int (*reloc_insn)(address insn_addr, address &target); 159 160 virtual reloc_insn adrpMem() = 0; 161 virtual reloc_insn adrpAdd() = 0; 162 virtual reloc_insn adrpMovk() = 0; 163 164 const address _insn_addr; 165 const uint32_t _insn; 166 167 static uint32_t insn_at(address insn_addr, int n) { 168 return ((uint32_t*)insn_addr)[n]; 169 } 170 uint32_t insn_at(int n) const { 171 return insn_at(_insn_addr, n); 172 } 173 174 public: 175 176 RelocActions(address insn_addr) : _insn_addr(insn_addr), _insn(insn_at(insn_addr, 0)) {} 177 RelocActions(address insn_addr, uint32_t insn) 178 : _insn_addr(insn_addr), _insn(insn) {} 179 180 virtual int unconditionalBranch(address insn_addr, address &target) = 0; 181 virtual int conditionalBranch(address insn_addr, address &target) = 0; 182 virtual int testAndBranch(address insn_addr, address &target) = 0; 183 virtual int loadStore(address insn_addr, address &target) = 0; 184 virtual int adr(address insn_addr, address &target) = 0; 185 virtual int adrp(address insn_addr, address &target, reloc_insn inner) = 0; 186 virtual int immediate(address insn_addr, address &target) = 0; 187 virtual void verify(address insn_addr, address &target) = 0; 188 189 int ALWAYSINLINE run(address insn_addr, address &target) { 190 int instructions = 1; 191 192 uint32_t dispatch = Instruction_aarch64::extract(_insn, 30, 25); 193 switch(dispatch) { 194 case 0b001010: 195 case 0b001011: { 196 instructions = unconditionalBranch(insn_addr, target); 197 break; 198 } 199 case 0b101010: // Conditional branch (immediate) 200 case 0b011010: { // Compare & branch (immediate) 201 instructions = conditionalBranch(insn_addr, target); 202 break; 203 } 204 case 0b011011: { 205 instructions = testAndBranch(insn_addr, target); 206 break; 207 } 208 case 0b001100: 209 case 0b001110: 210 case 0b011100: 211 case 0b011110: 212 case 0b101100: 213 case 0b101110: 214 case 0b111100: 215 case 0b111110: { 216 // load/store 217 if ((Instruction_aarch64::extract(_insn, 29, 24) & 0b111011) == 0b011000) { 218 // Load register (literal) 219 instructions = loadStore(insn_addr, target); 220 break; 221 } else { 222 // nothing to do 223 assert(target == nullptr, "did not expect to relocate target for polling page load"); 224 } 225 break; 226 } 227 case 0b001000: 228 case 0b011000: 229 case 0b101000: 230 case 0b111000: { 231 // adr/adrp 232 assert(Instruction_aarch64::extract(_insn, 28, 24) == 0b10000, "must be"); 233 int shift = Instruction_aarch64::extract(_insn, 31, 31); 234 if (shift) { 235 uint32_t insn2 = insn_at(1); 236 if (Instruction_aarch64::extract(insn2, 29, 24) == 0b111001 && 237 Instruction_aarch64::extract(_insn, 4, 0) == 238 Instruction_aarch64::extract(insn2, 9, 5)) { 239 instructions = adrp(insn_addr, target, adrpMem()); 240 } else if (Instruction_aarch64::extract(insn2, 31, 22) == 0b1001000100 && 241 Instruction_aarch64::extract(_insn, 4, 0) == 242 Instruction_aarch64::extract(insn2, 4, 0)) { 243 instructions = adrp(insn_addr, target, adrpAdd()); 244 } else if (Instruction_aarch64::extract(insn2, 31, 21) == 0b11110010110 && 245 Instruction_aarch64::extract(_insn, 4, 0) == 246 Instruction_aarch64::extract(insn2, 4, 0)) { 247 instructions = adrp(insn_addr, target, adrpMovk()); 248 } else { 249 ShouldNotReachHere(); 250 } 251 } else { 252 instructions = adr(insn_addr, target); 253 } 254 break; 255 } 256 case 0b001001: 257 case 0b011001: 258 case 0b101001: 259 case 0b111001: { 260 instructions = immediate(insn_addr, target); 261 break; 262 } 263 default: { 264 ShouldNotReachHere(); 265 } 266 } 267 268 verify(insn_addr, target); 269 return instructions * NativeInstruction::instruction_size; 270 } 271 }; 272 273 class Patcher : public RelocActions { 274 virtual reloc_insn adrpMem() { return &Patcher::adrpMem_impl; } 275 virtual reloc_insn adrpAdd() { return &Patcher::adrpAdd_impl; } 276 virtual reloc_insn adrpMovk() { return &Patcher::adrpMovk_impl; } 277 278 public: 279 Patcher(address insn_addr) : RelocActions(insn_addr) {} 280 281 virtual int unconditionalBranch(address insn_addr, address &target) { 282 intptr_t offset = (target - insn_addr) >> 2; 283 Instruction_aarch64::spatch(insn_addr, 25, 0, offset); 284 return 1; 285 } 286 virtual int conditionalBranch(address insn_addr, address &target) { 287 intptr_t offset = (target - insn_addr) >> 2; 288 Instruction_aarch64::spatch(insn_addr, 23, 5, offset); 289 return 1; 290 } 291 virtual int testAndBranch(address insn_addr, address &target) { 292 intptr_t offset = (target - insn_addr) >> 2; 293 Instruction_aarch64::spatch(insn_addr, 18, 5, offset); 294 return 1; 295 } 296 virtual int loadStore(address insn_addr, address &target) { 297 intptr_t offset = (target - insn_addr) >> 2; 298 Instruction_aarch64::spatch(insn_addr, 23, 5, offset); 299 return 1; 300 } 301 virtual int adr(address insn_addr, address &target) { 302 #ifdef ASSERT 303 assert(Instruction_aarch64::extract(_insn, 28, 24) == 0b10000, "must be"); 304 #endif 305 // PC-rel. addressing 306 ptrdiff_t offset = target - insn_addr; 307 int offset_lo = offset & 3; 308 offset >>= 2; 309 Instruction_aarch64::spatch(insn_addr, 23, 5, offset); 310 Instruction_aarch64::patch(insn_addr, 30, 29, offset_lo); 311 return 1; 312 } 313 virtual int adrp(address insn_addr, address &target, reloc_insn inner) { 314 int instructions = 1; 315 #ifdef ASSERT 316 assert(Instruction_aarch64::extract(_insn, 28, 24) == 0b10000, "must be"); 317 #endif 318 ptrdiff_t offset = target - insn_addr; 319 instructions = 2; 320 precond(inner != nullptr); 321 // Give the inner reloc a chance to modify the target. 322 address adjusted_target = target; 323 instructions = (*inner)(insn_addr, adjusted_target); 324 uintptr_t pc_page = (uintptr_t)insn_addr >> 12; 325 uintptr_t adr_page = (uintptr_t)adjusted_target >> 12; 326 offset = adr_page - pc_page; 327 int offset_lo = offset & 3; 328 offset >>= 2; 329 Instruction_aarch64::spatch(insn_addr, 23, 5, offset); 330 Instruction_aarch64::patch(insn_addr, 30, 29, offset_lo); 331 return instructions; 332 } 333 static int adrpMem_impl(address insn_addr, address &target) { 334 uintptr_t dest = (uintptr_t)target; 335 int offset_lo = dest & 0xfff; 336 uint32_t insn2 = insn_at(insn_addr, 1); 337 uint32_t size = Instruction_aarch64::extract(insn2, 31, 30); 338 Instruction_aarch64::patch(insn_addr + sizeof (uint32_t), 21, 10, offset_lo >> size); 339 guarantee(((dest >> size) << size) == dest, "misaligned target"); 340 return 2; 341 } 342 static int adrpAdd_impl(address insn_addr, address &target) { 343 uintptr_t dest = (uintptr_t)target; 344 int offset_lo = dest & 0xfff; 345 Instruction_aarch64::patch(insn_addr + sizeof (uint32_t), 21, 10, offset_lo); 346 return 2; 347 } 348 static int adrpMovk_impl(address insn_addr, address &target) { 349 uintptr_t dest = uintptr_t(target); 350 Instruction_aarch64::patch(insn_addr + sizeof (uint32_t), 20, 5, (uintptr_t)target >> 32); 351 dest = (dest & 0xffffffffULL) | (uintptr_t(insn_addr) & 0xffff00000000ULL); 352 target = address(dest); 353 return 2; 354 } 355 virtual int immediate(address insn_addr, address &target) { 356 assert(Instruction_aarch64::extract(_insn, 31, 21) == 0b11010010100, "must be"); 357 uint64_t dest = (uint64_t)target; 358 // Move wide constant 359 assert(nativeInstruction_at(insn_addr+4)->is_movk(), "wrong insns in patch"); 360 assert(nativeInstruction_at(insn_addr+8)->is_movk(), "wrong insns in patch"); 361 Instruction_aarch64::patch(insn_addr, 20, 5, dest & 0xffff); 362 Instruction_aarch64::patch(insn_addr+4, 20, 5, (dest >>= 16) & 0xffff); 363 Instruction_aarch64::patch(insn_addr+8, 20, 5, (dest >>= 16) & 0xffff); 364 return 3; 365 } 366 virtual void verify(address insn_addr, address &target) { 367 #ifdef ASSERT 368 address address_is = MacroAssembler::target_addr_for_insn(insn_addr); 369 if (!(address_is == target)) { 370 tty->print_cr("%p at %p should be %p", address_is, insn_addr, target); 371 disnm((intptr_t)insn_addr); 372 assert(address_is == target, "should be"); 373 } 374 #endif 375 } 376 }; 377 378 // If insn1 and insn2 use the same register to form an address, either 379 // by an offsetted LDR or a simple ADD, return the offset. If the 380 // second instruction is an LDR, the offset may be scaled. 381 static bool offset_for(uint32_t insn1, uint32_t insn2, ptrdiff_t &byte_offset) { 382 if (Instruction_aarch64::extract(insn2, 29, 24) == 0b111001 && 383 Instruction_aarch64::extract(insn1, 4, 0) == 384 Instruction_aarch64::extract(insn2, 9, 5)) { 385 // Load/store register (unsigned immediate) 386 byte_offset = Instruction_aarch64::extract(insn2, 21, 10); 387 uint32_t size = Instruction_aarch64::extract(insn2, 31, 30); 388 byte_offset <<= size; 389 return true; 390 } else if (Instruction_aarch64::extract(insn2, 31, 22) == 0b1001000100 && 391 Instruction_aarch64::extract(insn1, 4, 0) == 392 Instruction_aarch64::extract(insn2, 4, 0)) { 393 // add (immediate) 394 byte_offset = Instruction_aarch64::extract(insn2, 21, 10); 395 return true; 396 } 397 return false; 398 } 399 400 class AArch64Decoder : public RelocActions { 401 virtual reloc_insn adrpMem() { return &AArch64Decoder::adrpMem_impl; } 402 virtual reloc_insn adrpAdd() { return &AArch64Decoder::adrpAdd_impl; } 403 virtual reloc_insn adrpMovk() { return &AArch64Decoder::adrpMovk_impl; } 404 405 public: 406 AArch64Decoder(address insn_addr, uint32_t insn) : RelocActions(insn_addr, insn) {} 407 408 virtual int loadStore(address insn_addr, address &target) { 409 intptr_t offset = Instruction_aarch64::sextract(_insn, 23, 5); 410 target = insn_addr + (offset << 2); 411 return 1; 412 } 413 virtual int unconditionalBranch(address insn_addr, address &target) { 414 intptr_t offset = Instruction_aarch64::sextract(_insn, 25, 0); 415 target = insn_addr + (offset << 2); 416 return 1; 417 } 418 virtual int conditionalBranch(address insn_addr, address &target) { 419 intptr_t offset = Instruction_aarch64::sextract(_insn, 23, 5); 420 target = address(((uint64_t)insn_addr + (offset << 2))); 421 return 1; 422 } 423 virtual int testAndBranch(address insn_addr, address &target) { 424 intptr_t offset = Instruction_aarch64::sextract(_insn, 18, 5); 425 target = address(((uint64_t)insn_addr + (offset << 2))); 426 return 1; 427 } 428 virtual int adr(address insn_addr, address &target) { 429 // PC-rel. addressing 430 intptr_t offset = Instruction_aarch64::extract(_insn, 30, 29); 431 offset |= Instruction_aarch64::sextract(_insn, 23, 5) << 2; 432 target = address((uint64_t)insn_addr + offset); 433 return 1; 434 } 435 virtual int adrp(address insn_addr, address &target, reloc_insn inner) { 436 assert(Instruction_aarch64::extract(_insn, 28, 24) == 0b10000, "must be"); 437 intptr_t offset = Instruction_aarch64::extract(_insn, 30, 29); 438 offset |= Instruction_aarch64::sextract(_insn, 23, 5) << 2; 439 int shift = 12; 440 offset <<= shift; 441 uint64_t target_page = ((uint64_t)insn_addr) + offset; 442 target_page &= ((uint64_t)-1) << shift; 443 uint32_t insn2 = insn_at(1); 444 target = address(target_page); 445 precond(inner != nullptr); 446 (*inner)(insn_addr, target); 447 return 2; 448 } 449 static int adrpMem_impl(address insn_addr, address &target) { 450 uint32_t insn2 = insn_at(insn_addr, 1); 451 // Load/store register (unsigned immediate) 452 ptrdiff_t byte_offset = Instruction_aarch64::extract(insn2, 21, 10); 453 uint32_t size = Instruction_aarch64::extract(insn2, 31, 30); 454 byte_offset <<= size; 455 target += byte_offset; 456 return 2; 457 } 458 static int adrpAdd_impl(address insn_addr, address &target) { 459 uint32_t insn2 = insn_at(insn_addr, 1); 460 // add (immediate) 461 ptrdiff_t byte_offset = Instruction_aarch64::extract(insn2, 21, 10); 462 target += byte_offset; 463 return 2; 464 } 465 static int adrpMovk_impl(address insn_addr, address &target) { 466 uint32_t insn2 = insn_at(insn_addr, 1); 467 uint64_t dest = uint64_t(target); 468 dest = (dest & 0xffff0000ffffffff) | 469 ((uint64_t)Instruction_aarch64::extract(insn2, 20, 5) << 32); 470 target = address(dest); 471 472 // We know the destination 4k page. Maybe we have a third 473 // instruction. 474 uint32_t insn = insn_at(insn_addr, 0); 475 uint32_t insn3 = insn_at(insn_addr, 2); 476 ptrdiff_t byte_offset; 477 if (offset_for(insn, insn3, byte_offset)) { 478 target += byte_offset; 479 return 3; 480 } else { 481 return 2; 482 } 483 } 484 virtual int immediate(address insn_addr, address &target) { 485 uint32_t *insns = (uint32_t *)insn_addr; 486 assert(Instruction_aarch64::extract(_insn, 31, 21) == 0b11010010100, "must be"); 487 // Move wide constant: movz, movk, movk. See movptr(). 488 assert(nativeInstruction_at(insns+1)->is_movk(), "wrong insns in patch"); 489 assert(nativeInstruction_at(insns+2)->is_movk(), "wrong insns in patch"); 490 target = address(uint64_t(Instruction_aarch64::extract(_insn, 20, 5)) 491 + (uint64_t(Instruction_aarch64::extract(insns[1], 20, 5)) << 16) 492 + (uint64_t(Instruction_aarch64::extract(insns[2], 20, 5)) << 32)); 493 assert(nativeInstruction_at(insn_addr+4)->is_movk(), "wrong insns in patch"); 494 assert(nativeInstruction_at(insn_addr+8)->is_movk(), "wrong insns in patch"); 495 return 3; 496 } 497 virtual void verify(address insn_addr, address &target) { 498 } 499 }; 500 501 address MacroAssembler::target_addr_for_insn(address insn_addr, uint32_t insn) { 502 AArch64Decoder decoder(insn_addr, insn); 503 address target; 504 decoder.run(insn_addr, target); 505 return target; 506 } 507 508 // Patch any kind of instruction; there may be several instructions. 509 // Return the total length (in bytes) of the instructions. 510 int MacroAssembler::pd_patch_instruction_size(address insn_addr, address target) { 511 Patcher patcher(insn_addr); 512 return patcher.run(insn_addr, target); 513 } 514 515 int MacroAssembler::patch_oop(address insn_addr, address o) { 516 int instructions; 517 unsigned insn = *(unsigned*)insn_addr; 518 assert(nativeInstruction_at(insn_addr+4)->is_movk(), "wrong insns in patch"); 519 520 // OOPs are either narrow (32 bits) or wide (48 bits). We encode 521 // narrow OOPs by setting the upper 16 bits in the first 522 // instruction. 523 if (Instruction_aarch64::extract(insn, 31, 21) == 0b11010010101) { 524 // Move narrow OOP 525 uint32_t n = CompressedOops::narrow_oop_value(cast_to_oop(o)); 526 Instruction_aarch64::patch(insn_addr, 20, 5, n >> 16); 527 Instruction_aarch64::patch(insn_addr+4, 20, 5, n & 0xffff); 528 instructions = 2; 529 } else { 530 // Move wide OOP 531 assert(nativeInstruction_at(insn_addr+8)->is_movk(), "wrong insns in patch"); 532 uintptr_t dest = (uintptr_t)o; 533 Instruction_aarch64::patch(insn_addr, 20, 5, dest & 0xffff); 534 Instruction_aarch64::patch(insn_addr+4, 20, 5, (dest >>= 16) & 0xffff); 535 Instruction_aarch64::patch(insn_addr+8, 20, 5, (dest >>= 16) & 0xffff); 536 instructions = 3; 537 } 538 return instructions * NativeInstruction::instruction_size; 539 } 540 541 int MacroAssembler::patch_narrow_klass(address insn_addr, narrowKlass n) { 542 // Metadata pointers are either narrow (32 bits) or wide (48 bits). 543 // We encode narrow ones by setting the upper 16 bits in the first 544 // instruction. 545 NativeInstruction *insn = nativeInstruction_at(insn_addr); 546 assert(Instruction_aarch64::extract(insn->encoding(), 31, 21) == 0b11010010101 && 547 nativeInstruction_at(insn_addr+4)->is_movk(), "wrong insns in patch"); 548 549 Instruction_aarch64::patch(insn_addr, 20, 5, n >> 16); 550 Instruction_aarch64::patch(insn_addr+4, 20, 5, n & 0xffff); 551 return 2 * NativeInstruction::instruction_size; 552 } 553 554 address MacroAssembler::target_addr_for_insn_or_null(address insn_addr, unsigned insn) { 555 if (NativeInstruction::is_ldrw_to_zr(address(&insn))) { 556 return nullptr; 557 } 558 return MacroAssembler::target_addr_for_insn(insn_addr, insn); 559 } 560 561 void MacroAssembler::safepoint_poll(Label& slow_path, bool at_return, bool acquire, bool in_nmethod, Register tmp) { 562 if (acquire) { 563 lea(tmp, Address(rthread, JavaThread::polling_word_offset())); 564 ldar(tmp, tmp); 565 } else { 566 ldr(tmp, Address(rthread, JavaThread::polling_word_offset())); 567 } 568 if (at_return) { 569 // Note that when in_nmethod is set, the stack pointer is incremented before the poll. Therefore, 570 // we may safely use the sp instead to perform the stack watermark check. 571 cmp(in_nmethod ? sp : rfp, tmp); 572 br(Assembler::HI, slow_path); 573 } else { 574 tbnz(tmp, log2i_exact(SafepointMechanism::poll_bit()), slow_path); 575 } 576 } 577 578 void MacroAssembler::rt_call(address dest, Register tmp) { 579 CodeBlob *cb = CodeCache::find_blob(dest); 580 if (cb) { 581 far_call(RuntimeAddress(dest)); 582 } else { 583 lea(tmp, RuntimeAddress(dest)); 584 blr(tmp); 585 } 586 } 587 588 void MacroAssembler::push_cont_fastpath(Register java_thread) { 589 if (!Continuations::enabled()) return; 590 Label done; 591 ldr(rscratch1, Address(java_thread, JavaThread::cont_fastpath_offset())); 592 cmp(sp, rscratch1); 593 br(Assembler::LS, done); 594 mov(rscratch1, sp); // we can't use sp as the source in str 595 str(rscratch1, Address(java_thread, JavaThread::cont_fastpath_offset())); 596 bind(done); 597 } 598 599 void MacroAssembler::pop_cont_fastpath(Register java_thread) { 600 if (!Continuations::enabled()) return; 601 Label done; 602 ldr(rscratch1, Address(java_thread, JavaThread::cont_fastpath_offset())); 603 cmp(sp, rscratch1); 604 br(Assembler::LO, done); 605 str(zr, Address(java_thread, JavaThread::cont_fastpath_offset())); 606 bind(done); 607 } 608 609 void MacroAssembler::reset_last_Java_frame(bool clear_fp) { 610 // we must set sp to zero to clear frame 611 str(zr, Address(rthread, JavaThread::last_Java_sp_offset())); 612 613 // must clear fp, so that compiled frames are not confused; it is 614 // possible that we need it only for debugging 615 if (clear_fp) { 616 str(zr, Address(rthread, JavaThread::last_Java_fp_offset())); 617 } 618 619 // Always clear the pc because it could have been set by make_walkable() 620 str(zr, Address(rthread, JavaThread::last_Java_pc_offset())); 621 } 622 623 // Calls to C land 624 // 625 // When entering C land, the rfp, & resp of the last Java frame have to be recorded 626 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp 627 // has to be reset to 0. This is required to allow proper stack traversal. 628 void MacroAssembler::set_last_Java_frame(Register last_java_sp, 629 Register last_java_fp, 630 Register last_java_pc, 631 Register scratch) { 632 633 if (last_java_pc->is_valid()) { 634 str(last_java_pc, Address(rthread, 635 JavaThread::frame_anchor_offset() 636 + JavaFrameAnchor::last_Java_pc_offset())); 637 } 638 639 // determine last_java_sp register 640 if (last_java_sp == sp) { 641 mov(scratch, sp); 642 last_java_sp = scratch; 643 } else if (!last_java_sp->is_valid()) { 644 last_java_sp = esp; 645 } 646 647 str(last_java_sp, Address(rthread, JavaThread::last_Java_sp_offset())); 648 649 // last_java_fp is optional 650 if (last_java_fp->is_valid()) { 651 str(last_java_fp, Address(rthread, JavaThread::last_Java_fp_offset())); 652 } 653 } 654 655 void MacroAssembler::set_last_Java_frame(Register last_java_sp, 656 Register last_java_fp, 657 address last_java_pc, 658 Register scratch) { 659 assert(last_java_pc != nullptr, "must provide a valid PC"); 660 661 adr(scratch, last_java_pc); 662 str(scratch, Address(rthread, 663 JavaThread::frame_anchor_offset() 664 + JavaFrameAnchor::last_Java_pc_offset())); 665 666 set_last_Java_frame(last_java_sp, last_java_fp, noreg, scratch); 667 } 668 669 void MacroAssembler::set_last_Java_frame(Register last_java_sp, 670 Register last_java_fp, 671 Label &L, 672 Register scratch) { 673 if (L.is_bound()) { 674 set_last_Java_frame(last_java_sp, last_java_fp, target(L), scratch); 675 } else { 676 InstructionMark im(this); 677 L.add_patch_at(code(), locator()); 678 set_last_Java_frame(last_java_sp, last_java_fp, pc() /* Patched later */, scratch); 679 } 680 } 681 682 static inline bool target_needs_far_branch(address addr) { 683 // codecache size <= 128M 684 if (!MacroAssembler::far_branches()) { 685 return false; 686 } 687 // codecache size > 240M 688 if (MacroAssembler::codestub_branch_needs_far_jump()) { 689 return true; 690 } 691 // codecache size: 128M..240M 692 return !CodeCache::is_non_nmethod(addr); 693 } 694 695 void MacroAssembler::far_call(Address entry, Register tmp) { 696 assert(ReservedCodeCacheSize < 4*G, "branch out of range"); 697 assert(CodeCache::find_blob(entry.target()) != nullptr, 698 "destination of far call not found in code cache"); 699 assert(entry.rspec().type() == relocInfo::external_word_type 700 || entry.rspec().type() == relocInfo::runtime_call_type 701 || entry.rspec().type() == relocInfo::none, "wrong entry relocInfo type"); 702 if (target_needs_far_branch(entry.target())) { 703 uint64_t offset; 704 // We can use ADRP here because we know that the total size of 705 // the code cache cannot exceed 2Gb (ADRP limit is 4GB). 706 adrp(tmp, entry, offset); 707 add(tmp, tmp, offset); 708 blr(tmp); 709 } else { 710 bl(entry); 711 } 712 } 713 714 int MacroAssembler::far_jump(Address entry, Register tmp) { 715 assert(ReservedCodeCacheSize < 4*G, "branch out of range"); 716 assert(CodeCache::find_blob(entry.target()) != nullptr, 717 "destination of far call not found in code cache"); 718 assert(entry.rspec().type() == relocInfo::external_word_type 719 || entry.rspec().type() == relocInfo::runtime_call_type 720 || entry.rspec().type() == relocInfo::none, "wrong entry relocInfo type"); 721 address start = pc(); 722 if (target_needs_far_branch(entry.target())) { 723 uint64_t offset; 724 // We can use ADRP here because we know that the total size of 725 // the code cache cannot exceed 2Gb (ADRP limit is 4GB). 726 adrp(tmp, entry, offset); 727 add(tmp, tmp, offset); 728 br(tmp); 729 } else { 730 b(entry); 731 } 732 return pc() - start; 733 } 734 735 void MacroAssembler::reserved_stack_check() { 736 // testing if reserved zone needs to be enabled 737 Label no_reserved_zone_enabling; 738 739 ldr(rscratch1, Address(rthread, JavaThread::reserved_stack_activation_offset())); 740 cmp(sp, rscratch1); 741 br(Assembler::LO, no_reserved_zone_enabling); 742 743 enter(); // LR and FP are live. 744 lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone))); 745 mov(c_rarg0, rthread); 746 blr(rscratch1); 747 leave(); 748 749 // We have already removed our own frame. 750 // throw_delayed_StackOverflowError will think that it's been 751 // called by our caller. 752 lea(rscratch1, RuntimeAddress(SharedRuntime::throw_delayed_StackOverflowError_entry())); 753 br(rscratch1); 754 should_not_reach_here(); 755 756 bind(no_reserved_zone_enabling); 757 } 758 759 static void pass_arg0(MacroAssembler* masm, Register arg) { 760 if (c_rarg0 != arg ) { 761 masm->mov(c_rarg0, arg); 762 } 763 } 764 765 static void pass_arg1(MacroAssembler* masm, Register arg) { 766 if (c_rarg1 != arg ) { 767 masm->mov(c_rarg1, arg); 768 } 769 } 770 771 static void pass_arg2(MacroAssembler* masm, Register arg) { 772 if (c_rarg2 != arg ) { 773 masm->mov(c_rarg2, arg); 774 } 775 } 776 777 static void pass_arg3(MacroAssembler* masm, Register arg) { 778 if (c_rarg3 != arg ) { 779 masm->mov(c_rarg3, arg); 780 } 781 } 782 783 static bool is_preemptable(address entry_point) { 784 return entry_point == CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter); 785 } 786 787 void MacroAssembler::call_VM_base(Register oop_result, 788 Register java_thread, 789 Register last_java_sp, 790 address entry_point, 791 int number_of_arguments, 792 bool check_exceptions) { 793 // determine java_thread register 794 if (!java_thread->is_valid()) { 795 java_thread = rthread; 796 } 797 798 // determine last_java_sp register 799 if (!last_java_sp->is_valid()) { 800 last_java_sp = esp; 801 } 802 803 // debugging support 804 assert(number_of_arguments >= 0 , "cannot have negative number of arguments"); 805 assert(java_thread == rthread, "unexpected register"); 806 #ifdef ASSERT 807 // TraceBytecodes does not use r12 but saves it over the call, so don't verify 808 // if ((UseCompressedOops || UseCompressedClassPointers) && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?"); 809 #endif // ASSERT 810 811 assert(java_thread != oop_result , "cannot use the same register for java_thread & oop_result"); 812 assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp"); 813 814 // push java thread (becomes first argument of C function) 815 816 mov(c_rarg0, java_thread); 817 818 // set last Java frame before call 819 assert(last_java_sp != rfp, "can't use rfp"); 820 821 Label l; 822 if (is_preemptable(entry_point)) { 823 // skip setting last_pc since we already set it to desired value. 824 set_last_Java_frame(last_java_sp, rfp, noreg, rscratch1); 825 } else { 826 set_last_Java_frame(last_java_sp, rfp, l, rscratch1); 827 } 828 829 // do the call, remove parameters 830 MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments, &l); 831 832 // lr could be poisoned with PAC signature during throw_pending_exception 833 // if it was tail-call optimized by compiler, since lr is not callee-saved 834 // reload it with proper value 835 adr(lr, l); 836 837 // reset last Java frame 838 // Only interpreter should have to clear fp 839 reset_last_Java_frame(true); 840 841 // C++ interp handles this in the interpreter 842 check_and_handle_popframe(java_thread); 843 check_and_handle_earlyret(java_thread); 844 845 if (check_exceptions) { 846 // check for pending exceptions (java_thread is set upon return) 847 ldr(rscratch1, Address(java_thread, in_bytes(Thread::pending_exception_offset()))); 848 Label ok; 849 cbz(rscratch1, ok); 850 lea(rscratch1, RuntimeAddress(StubRoutines::forward_exception_entry())); 851 br(rscratch1); 852 bind(ok); 853 } 854 855 // get oop result if there is one and reset the value in the thread 856 if (oop_result->is_valid()) { 857 get_vm_result(oop_result, java_thread); 858 } 859 } 860 861 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) { 862 call_VM_base(oop_result, noreg, noreg, entry_point, number_of_arguments, check_exceptions); 863 } 864 865 // Check the entry target is always reachable from any branch. 866 static bool is_always_within_branch_range(Address entry) { 867 const address target = entry.target(); 868 869 if (!CodeCache::contains(target)) { 870 // We always use trampolines for callees outside CodeCache. 871 assert(entry.rspec().type() == relocInfo::runtime_call_type, "non-runtime call of an external target"); 872 return false; 873 } 874 875 if (!MacroAssembler::far_branches()) { 876 return true; 877 } 878 879 if (entry.rspec().type() == relocInfo::runtime_call_type) { 880 // Runtime calls are calls of a non-compiled method (stubs, adapters). 881 // Non-compiled methods stay forever in CodeCache. 882 // We check whether the longest possible branch is within the branch range. 883 assert(CodeCache::find_blob(target) != nullptr && 884 !CodeCache::find_blob(target)->is_nmethod(), 885 "runtime call of compiled method"); 886 const address right_longest_branch_start = CodeCache::high_bound() - NativeInstruction::instruction_size; 887 const address left_longest_branch_start = CodeCache::low_bound(); 888 const bool is_reachable = Assembler::reachable_from_branch_at(left_longest_branch_start, target) && 889 Assembler::reachable_from_branch_at(right_longest_branch_start, target); 890 return is_reachable; 891 } 892 893 return false; 894 } 895 896 // Maybe emit a call via a trampoline. If the code cache is small 897 // trampolines won't be emitted. 898 address MacroAssembler::trampoline_call(Address entry) { 899 assert(entry.rspec().type() == relocInfo::runtime_call_type 900 || entry.rspec().type() == relocInfo::opt_virtual_call_type 901 || entry.rspec().type() == relocInfo::static_call_type 902 || entry.rspec().type() == relocInfo::virtual_call_type, "wrong reloc type"); 903 904 address target = entry.target(); 905 906 if (!is_always_within_branch_range(entry)) { 907 if (!in_scratch_emit_size()) { 908 // We don't want to emit a trampoline if C2 is generating dummy 909 // code during its branch shortening phase. 910 if (entry.rspec().type() == relocInfo::runtime_call_type) { 911 assert(CodeBuffer::supports_shared_stubs(), "must support shared stubs"); 912 code()->share_trampoline_for(entry.target(), offset()); 913 } else { 914 address stub = emit_trampoline_stub(offset(), target); 915 if (stub == nullptr) { 916 postcond(pc() == badAddress); 917 return nullptr; // CodeCache is full 918 } 919 } 920 } 921 target = pc(); 922 } 923 924 address call_pc = pc(); 925 relocate(entry.rspec()); 926 bl(target); 927 928 postcond(pc() != badAddress); 929 return call_pc; 930 } 931 932 // Emit a trampoline stub for a call to a target which is too far away. 933 // 934 // code sequences: 935 // 936 // call-site: 937 // branch-and-link to <destination> or <trampoline stub> 938 // 939 // Related trampoline stub for this call site in the stub section: 940 // load the call target from the constant pool 941 // branch (LR still points to the call site above) 942 943 address MacroAssembler::emit_trampoline_stub(int insts_call_instruction_offset, 944 address dest) { 945 // Max stub size: alignment nop, TrampolineStub. 946 address stub = start_a_stub(max_trampoline_stub_size()); 947 if (stub == nullptr) { 948 return nullptr; // CodeBuffer::expand failed 949 } 950 951 // Create a trampoline stub relocation which relates this trampoline stub 952 // with the call instruction at insts_call_instruction_offset in the 953 // instructions code-section. 954 align(wordSize); 955 relocate(trampoline_stub_Relocation::spec(code()->insts()->start() 956 + insts_call_instruction_offset)); 957 const int stub_start_offset = offset(); 958 959 // Now, create the trampoline stub's code: 960 // - load the call 961 // - call 962 Label target; 963 ldr(rscratch1, target); 964 br(rscratch1); 965 bind(target); 966 assert(offset() - stub_start_offset == NativeCallTrampolineStub::data_offset, 967 "should be"); 968 emit_int64((int64_t)dest); 969 970 const address stub_start_addr = addr_at(stub_start_offset); 971 972 assert(is_NativeCallTrampolineStub_at(stub_start_addr), "doesn't look like a trampoline"); 973 974 end_a_stub(); 975 return stub_start_addr; 976 } 977 978 int MacroAssembler::max_trampoline_stub_size() { 979 // Max stub size: alignment nop, TrampolineStub. 980 return NativeInstruction::instruction_size + NativeCallTrampolineStub::instruction_size; 981 } 982 983 void MacroAssembler::emit_static_call_stub() { 984 // CompiledDirectCall::set_to_interpreted knows the 985 // exact layout of this stub. 986 987 isb(); 988 mov_metadata(rmethod, nullptr); 989 990 // Jump to the entry point of the c2i stub. 991 movptr(rscratch1, 0); 992 br(rscratch1); 993 } 994 995 int MacroAssembler::static_call_stub_size() { 996 // isb; movk; movz; movz; movk; movz; movz; br 997 return 8 * NativeInstruction::instruction_size; 998 } 999 1000 void MacroAssembler::c2bool(Register x) { 1001 // implements x == 0 ? 0 : 1 1002 // note: must only look at least-significant byte of x 1003 // since C-style booleans are stored in one byte 1004 // only! (was bug) 1005 tst(x, 0xff); 1006 cset(x, Assembler::NE); 1007 } 1008 1009 address MacroAssembler::ic_call(address entry, jint method_index) { 1010 RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index); 1011 // address const_ptr = long_constant((jlong)Universe::non_oop_word()); 1012 // uintptr_t offset; 1013 // ldr_constant(rscratch2, const_ptr); 1014 movptr(rscratch2, (intptr_t)Universe::non_oop_word()); 1015 return trampoline_call(Address(entry, rh)); 1016 } 1017 1018 int MacroAssembler::ic_check_size() { 1019 int extra_instructions = UseCompactObjectHeaders ? 1 : 0; 1020 if (target_needs_far_branch(CAST_FROM_FN_PTR(address, SharedRuntime::get_ic_miss_stub()))) { 1021 return NativeInstruction::instruction_size * (7 + extra_instructions); 1022 } else { 1023 return NativeInstruction::instruction_size * (5 + extra_instructions); 1024 } 1025 } 1026 1027 int MacroAssembler::ic_check(int end_alignment) { 1028 Register receiver = j_rarg0; 1029 Register data = rscratch2; 1030 Register tmp1 = rscratch1; 1031 Register tmp2 = r10; 1032 1033 // The UEP of a code blob ensures that the VEP is padded. However, the padding of the UEP is placed 1034 // before the inline cache check, so we don't have to execute any nop instructions when dispatching 1035 // through the UEP, yet we can ensure that the VEP is aligned appropriately. That's why we align 1036 // before the inline cache check here, and not after 1037 align(end_alignment, offset() + ic_check_size()); 1038 1039 int uep_offset = offset(); 1040 1041 if (UseCompactObjectHeaders) { 1042 load_narrow_klass_compact(tmp1, receiver); 1043 ldrw(tmp2, Address(data, CompiledICData::speculated_klass_offset())); 1044 cmpw(tmp1, tmp2); 1045 } else if (UseCompressedClassPointers) { 1046 ldrw(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes())); 1047 ldrw(tmp2, Address(data, CompiledICData::speculated_klass_offset())); 1048 cmpw(tmp1, tmp2); 1049 } else { 1050 ldr(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes())); 1051 ldr(tmp2, Address(data, CompiledICData::speculated_klass_offset())); 1052 cmp(tmp1, tmp2); 1053 } 1054 1055 Label dont; 1056 br(Assembler::EQ, dont); 1057 far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub())); 1058 bind(dont); 1059 assert((offset() % end_alignment) == 0, "Misaligned verified entry point"); 1060 1061 return uep_offset; 1062 } 1063 1064 // Implementation of call_VM versions 1065 1066 void MacroAssembler::call_VM(Register oop_result, 1067 address entry_point, 1068 bool check_exceptions) { 1069 call_VM_helper(oop_result, entry_point, 0, check_exceptions); 1070 } 1071 1072 void MacroAssembler::call_VM(Register oop_result, 1073 address entry_point, 1074 Register arg_1, 1075 bool check_exceptions) { 1076 pass_arg1(this, arg_1); 1077 call_VM_helper(oop_result, entry_point, 1, check_exceptions); 1078 } 1079 1080 void MacroAssembler::call_VM(Register oop_result, 1081 address entry_point, 1082 Register arg_1, 1083 Register arg_2, 1084 bool check_exceptions) { 1085 assert_different_registers(arg_1, c_rarg2); 1086 pass_arg2(this, arg_2); 1087 pass_arg1(this, arg_1); 1088 call_VM_helper(oop_result, entry_point, 2, check_exceptions); 1089 } 1090 1091 void MacroAssembler::call_VM(Register oop_result, 1092 address entry_point, 1093 Register arg_1, 1094 Register arg_2, 1095 Register arg_3, 1096 bool check_exceptions) { 1097 assert_different_registers(arg_1, c_rarg2, c_rarg3); 1098 assert_different_registers(arg_2, c_rarg3); 1099 pass_arg3(this, arg_3); 1100 1101 pass_arg2(this, arg_2); 1102 1103 pass_arg1(this, arg_1); 1104 call_VM_helper(oop_result, entry_point, 3, check_exceptions); 1105 } 1106 1107 void MacroAssembler::call_VM(Register oop_result, 1108 Register last_java_sp, 1109 address entry_point, 1110 int number_of_arguments, 1111 bool check_exceptions) { 1112 call_VM_base(oop_result, rthread, last_java_sp, entry_point, number_of_arguments, check_exceptions); 1113 } 1114 1115 void MacroAssembler::call_VM(Register oop_result, 1116 Register last_java_sp, 1117 address entry_point, 1118 Register arg_1, 1119 bool check_exceptions) { 1120 pass_arg1(this, arg_1); 1121 call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions); 1122 } 1123 1124 void MacroAssembler::call_VM(Register oop_result, 1125 Register last_java_sp, 1126 address entry_point, 1127 Register arg_1, 1128 Register arg_2, 1129 bool check_exceptions) { 1130 1131 assert_different_registers(arg_1, c_rarg2); 1132 pass_arg2(this, arg_2); 1133 pass_arg1(this, arg_1); 1134 call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions); 1135 } 1136 1137 void MacroAssembler::call_VM(Register oop_result, 1138 Register last_java_sp, 1139 address entry_point, 1140 Register arg_1, 1141 Register arg_2, 1142 Register arg_3, 1143 bool check_exceptions) { 1144 assert_different_registers(arg_1, c_rarg2, c_rarg3); 1145 assert_different_registers(arg_2, c_rarg3); 1146 pass_arg3(this, arg_3); 1147 pass_arg2(this, arg_2); 1148 pass_arg1(this, arg_1); 1149 call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions); 1150 } 1151 1152 1153 void MacroAssembler::get_vm_result(Register oop_result, Register java_thread) { 1154 ldr(oop_result, Address(java_thread, JavaThread::vm_result_offset())); 1155 str(zr, Address(java_thread, JavaThread::vm_result_offset())); 1156 verify_oop_msg(oop_result, "broken oop in call_VM_base"); 1157 } 1158 1159 void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) { 1160 ldr(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset())); 1161 str(zr, Address(java_thread, JavaThread::vm_result_2_offset())); 1162 } 1163 1164 void MacroAssembler::align(int modulus) { 1165 align(modulus, offset()); 1166 } 1167 1168 // Ensure that the code at target bytes offset from the current offset() is aligned 1169 // according to modulus. 1170 void MacroAssembler::align(int modulus, int target) { 1171 int delta = target - offset(); 1172 while ((offset() + delta) % modulus != 0) nop(); 1173 } 1174 1175 void MacroAssembler::post_call_nop() { 1176 if (!Continuations::enabled()) { 1177 return; 1178 } 1179 InstructionMark im(this); 1180 relocate(post_call_nop_Relocation::spec()); 1181 InlineSkippedInstructionsCounter skipCounter(this); 1182 nop(); 1183 movk(zr, 0); 1184 movk(zr, 0); 1185 } 1186 1187 // these are no-ops overridden by InterpreterMacroAssembler 1188 1189 void MacroAssembler::check_and_handle_earlyret(Register java_thread) { } 1190 1191 void MacroAssembler::check_and_handle_popframe(Register java_thread) { } 1192 1193 void MacroAssembler::get_default_value_oop(Register inline_klass, Register temp_reg, Register obj) { 1194 assert_different_registers(inline_klass, temp_reg, obj, rscratch2); 1195 #ifdef ASSERT 1196 { 1197 Label done_check; 1198 test_klass_is_inline_type(inline_klass, temp_reg, done_check); 1199 stop("get_default_value_oop from non inline type klass"); 1200 bind(done_check); 1201 } 1202 #endif 1203 Register offset = temp_reg; 1204 // Getting the offset of the pre-allocated default value 1205 ldr(offset, Address(inline_klass, in_bytes(InstanceKlass::adr_inlineklass_fixed_block_offset()))); 1206 load_sized_value(offset, Address(offset, in_bytes(InlineKlass::default_value_offset_offset())), sizeof(int), true /*is_signed*/); 1207 1208 // Getting the mirror 1209 ldr(obj, Address(inline_klass, in_bytes(Klass::java_mirror_offset()))); 1210 resolve_oop_handle(obj, inline_klass, rscratch2); 1211 1212 // Getting the pre-allocated default value from the mirror 1213 Address field(obj, offset); 1214 load_heap_oop(obj, field, inline_klass, rscratch2); 1215 } 1216 1217 void MacroAssembler::get_empty_inline_type_oop(Register inline_klass, Register temp_reg, Register obj) { 1218 #ifdef ASSERT 1219 { 1220 Label done_check; 1221 test_klass_is_empty_inline_type(inline_klass, temp_reg, done_check); 1222 stop("get_empty_value from non-empty inline klass"); 1223 bind(done_check); 1224 } 1225 #endif 1226 get_default_value_oop(inline_klass, temp_reg, obj); 1227 } 1228 1229 // Look up the method for a megamorphic invokeinterface call. 1230 // The target method is determined by <intf_klass, itable_index>. 1231 // The receiver klass is in recv_klass. 1232 // On success, the result will be in method_result, and execution falls through. 1233 // On failure, execution transfers to the given label. 1234 void MacroAssembler::lookup_interface_method(Register recv_klass, 1235 Register intf_klass, 1236 RegisterOrConstant itable_index, 1237 Register method_result, 1238 Register scan_temp, 1239 Label& L_no_such_interface, 1240 bool return_method) { 1241 assert_different_registers(recv_klass, intf_klass, scan_temp); 1242 assert_different_registers(method_result, intf_klass, scan_temp); 1243 assert(recv_klass != method_result || !return_method, 1244 "recv_klass can be destroyed when method isn't needed"); 1245 assert(itable_index.is_constant() || itable_index.as_register() == method_result, 1246 "caller must use same register for non-constant itable index as for method"); 1247 1248 // Compute start of first itableOffsetEntry (which is at the end of the vtable) 1249 int vtable_base = in_bytes(Klass::vtable_start_offset()); 1250 int itentry_off = in_bytes(itableMethodEntry::method_offset()); 1251 int scan_step = itableOffsetEntry::size() * wordSize; 1252 int vte_size = vtableEntry::size_in_bytes(); 1253 assert(vte_size == wordSize, "else adjust times_vte_scale"); 1254 1255 ldrw(scan_temp, Address(recv_klass, Klass::vtable_length_offset())); 1256 1257 // Could store the aligned, prescaled offset in the klass. 1258 // lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base)); 1259 lea(scan_temp, Address(recv_klass, scan_temp, Address::lsl(3))); 1260 add(scan_temp, scan_temp, vtable_base); 1261 1262 if (return_method) { 1263 // Adjust recv_klass by scaled itable_index, so we can free itable_index. 1264 assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below"); 1265 // lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off)); 1266 lea(recv_klass, Address(recv_klass, itable_index, Address::lsl(3))); 1267 if (itentry_off) 1268 add(recv_klass, recv_klass, itentry_off); 1269 } 1270 1271 // for (scan = klass->itable(); scan->interface() != nullptr; scan += scan_step) { 1272 // if (scan->interface() == intf) { 1273 // result = (klass + scan->offset() + itable_index); 1274 // } 1275 // } 1276 Label search, found_method; 1277 1278 ldr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset())); 1279 cmp(intf_klass, method_result); 1280 br(Assembler::EQ, found_method); 1281 bind(search); 1282 // Check that the previous entry is non-null. A null entry means that 1283 // the receiver class doesn't implement the interface, and wasn't the 1284 // same as when the caller was compiled. 1285 cbz(method_result, L_no_such_interface); 1286 if (itableOffsetEntry::interface_offset() != 0) { 1287 add(scan_temp, scan_temp, scan_step); 1288 ldr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset())); 1289 } else { 1290 ldr(method_result, Address(pre(scan_temp, scan_step))); 1291 } 1292 cmp(intf_klass, method_result); 1293 br(Assembler::NE, search); 1294 1295 bind(found_method); 1296 1297 // Got a hit. 1298 if (return_method) { 1299 ldrw(scan_temp, Address(scan_temp, itableOffsetEntry::offset_offset())); 1300 ldr(method_result, Address(recv_klass, scan_temp, Address::uxtw(0))); 1301 } 1302 } 1303 1304 // Look up the method for a megamorphic invokeinterface call in a single pass over itable: 1305 // - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICData 1306 // - find a holder_klass (class that implements the method) vtable offset and get the method from vtable by index 1307 // The target method is determined by <holder_klass, itable_index>. 1308 // The receiver klass is in recv_klass. 1309 // On success, the result will be in method_result, and execution falls through. 1310 // On failure, execution transfers to the given label. 1311 void MacroAssembler::lookup_interface_method_stub(Register recv_klass, 1312 Register holder_klass, 1313 Register resolved_klass, 1314 Register method_result, 1315 Register temp_itbl_klass, 1316 Register scan_temp, 1317 int itable_index, 1318 Label& L_no_such_interface) { 1319 // 'method_result' is only used as output register at the very end of this method. 1320 // Until then we can reuse it as 'holder_offset'. 1321 Register holder_offset = method_result; 1322 assert_different_registers(resolved_klass, recv_klass, holder_klass, temp_itbl_klass, scan_temp, holder_offset); 1323 1324 int vtable_start_offset = in_bytes(Klass::vtable_start_offset()); 1325 int itable_offset_entry_size = itableOffsetEntry::size() * wordSize; 1326 int ioffset = in_bytes(itableOffsetEntry::interface_offset()); 1327 int ooffset = in_bytes(itableOffsetEntry::offset_offset()); 1328 1329 Label L_loop_search_resolved_entry, L_resolved_found, L_holder_found; 1330 1331 ldrw(scan_temp, Address(recv_klass, Klass::vtable_length_offset())); 1332 add(recv_klass, recv_klass, vtable_start_offset + ioffset); 1333 // itableOffsetEntry[] itable = recv_klass + Klass::vtable_start_offset() + sizeof(vtableEntry) * recv_klass->_vtable_len; 1334 // temp_itbl_klass = itable[0]._interface; 1335 int vtblEntrySize = vtableEntry::size_in_bytes(); 1336 assert(vtblEntrySize == wordSize, "ldr lsl shift amount must be 3"); 1337 ldr(temp_itbl_klass, Address(recv_klass, scan_temp, Address::lsl(exact_log2(vtblEntrySize)))); 1338 mov(holder_offset, zr); 1339 // scan_temp = &(itable[0]._interface) 1340 lea(scan_temp, Address(recv_klass, scan_temp, Address::lsl(exact_log2(vtblEntrySize)))); 1341 1342 // Initial checks: 1343 // - if (holder_klass != resolved_klass), go to "scan for resolved" 1344 // - if (itable[0] == holder_klass), shortcut to "holder found" 1345 // - if (itable[0] == 0), no such interface 1346 cmp(resolved_klass, holder_klass); 1347 br(Assembler::NE, L_loop_search_resolved_entry); 1348 cmp(holder_klass, temp_itbl_klass); 1349 br(Assembler::EQ, L_holder_found); 1350 cbz(temp_itbl_klass, L_no_such_interface); 1351 1352 // Loop: Look for holder_klass record in itable 1353 // do { 1354 // temp_itbl_klass = *(scan_temp += itable_offset_entry_size); 1355 // if (temp_itbl_klass == holder_klass) { 1356 // goto L_holder_found; // Found! 1357 // } 1358 // } while (temp_itbl_klass != 0); 1359 // goto L_no_such_interface // Not found. 1360 Label L_search_holder; 1361 bind(L_search_holder); 1362 ldr(temp_itbl_klass, Address(pre(scan_temp, itable_offset_entry_size))); 1363 cmp(holder_klass, temp_itbl_klass); 1364 br(Assembler::EQ, L_holder_found); 1365 cbnz(temp_itbl_klass, L_search_holder); 1366 1367 b(L_no_such_interface); 1368 1369 // Loop: Look for resolved_class record in itable 1370 // while (true) { 1371 // temp_itbl_klass = *(scan_temp += itable_offset_entry_size); 1372 // if (temp_itbl_klass == 0) { 1373 // goto L_no_such_interface; 1374 // } 1375 // if (temp_itbl_klass == resolved_klass) { 1376 // goto L_resolved_found; // Found! 1377 // } 1378 // if (temp_itbl_klass == holder_klass) { 1379 // holder_offset = scan_temp; 1380 // } 1381 // } 1382 // 1383 Label L_loop_search_resolved; 1384 bind(L_loop_search_resolved); 1385 ldr(temp_itbl_klass, Address(pre(scan_temp, itable_offset_entry_size))); 1386 bind(L_loop_search_resolved_entry); 1387 cbz(temp_itbl_klass, L_no_such_interface); 1388 cmp(resolved_klass, temp_itbl_klass); 1389 br(Assembler::EQ, L_resolved_found); 1390 cmp(holder_klass, temp_itbl_klass); 1391 br(Assembler::NE, L_loop_search_resolved); 1392 mov(holder_offset, scan_temp); 1393 b(L_loop_search_resolved); 1394 1395 // See if we already have a holder klass. If not, go and scan for it. 1396 bind(L_resolved_found); 1397 cbz(holder_offset, L_search_holder); 1398 mov(scan_temp, holder_offset); 1399 1400 // Finally, scan_temp contains holder_klass vtable offset 1401 bind(L_holder_found); 1402 ldrw(method_result, Address(scan_temp, ooffset - ioffset)); 1403 add(recv_klass, recv_klass, itable_index * wordSize + in_bytes(itableMethodEntry::method_offset()) 1404 - vtable_start_offset - ioffset); // substract offsets to restore the original value of recv_klass 1405 ldr(method_result, Address(recv_klass, method_result, Address::uxtw(0))); 1406 } 1407 1408 // virtual method calling 1409 void MacroAssembler::lookup_virtual_method(Register recv_klass, 1410 RegisterOrConstant vtable_index, 1411 Register method_result) { 1412 assert(vtableEntry::size() * wordSize == 8, 1413 "adjust the scaling in the code below"); 1414 int64_t vtable_offset_in_bytes = in_bytes(Klass::vtable_start_offset() + vtableEntry::method_offset()); 1415 1416 if (vtable_index.is_register()) { 1417 lea(method_result, Address(recv_klass, 1418 vtable_index.as_register(), 1419 Address::lsl(LogBytesPerWord))); 1420 ldr(method_result, Address(method_result, vtable_offset_in_bytes)); 1421 } else { 1422 vtable_offset_in_bytes += vtable_index.as_constant() * wordSize; 1423 ldr(method_result, 1424 form_address(rscratch1, recv_klass, vtable_offset_in_bytes, 0)); 1425 } 1426 } 1427 1428 void MacroAssembler::check_klass_subtype(Register sub_klass, 1429 Register super_klass, 1430 Register temp_reg, 1431 Label& L_success) { 1432 Label L_failure; 1433 check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg, &L_success, &L_failure, nullptr); 1434 check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, noreg, &L_success, nullptr); 1435 bind(L_failure); 1436 } 1437 1438 1439 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass, 1440 Register super_klass, 1441 Register temp_reg, 1442 Label* L_success, 1443 Label* L_failure, 1444 Label* L_slow_path, 1445 Register super_check_offset) { 1446 assert_different_registers(sub_klass, super_klass, temp_reg, super_check_offset); 1447 bool must_load_sco = ! super_check_offset->is_valid(); 1448 if (must_load_sco) { 1449 assert(temp_reg != noreg, "supply either a temp or a register offset"); 1450 } 1451 1452 Label L_fallthrough; 1453 int label_nulls = 0; 1454 if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; } 1455 if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; } 1456 if (L_slow_path == nullptr) { L_slow_path = &L_fallthrough; label_nulls++; } 1457 assert(label_nulls <= 1, "at most one null in the batch"); 1458 1459 int sco_offset = in_bytes(Klass::super_check_offset_offset()); 1460 Address super_check_offset_addr(super_klass, sco_offset); 1461 1462 // Hacked jmp, which may only be used just before L_fallthrough. 1463 #define final_jmp(label) \ 1464 if (&(label) == &L_fallthrough) { /*do nothing*/ } \ 1465 else b(label) /*omit semi*/ 1466 1467 // If the pointers are equal, we are done (e.g., String[] elements). 1468 // This self-check enables sharing of secondary supertype arrays among 1469 // non-primary types such as array-of-interface. Otherwise, each such 1470 // type would need its own customized SSA. 1471 // We move this check to the front of the fast path because many 1472 // type checks are in fact trivially successful in this manner, 1473 // so we get a nicely predicted branch right at the start of the check. 1474 cmp(sub_klass, super_klass); 1475 br(Assembler::EQ, *L_success); 1476 1477 // Check the supertype display: 1478 if (must_load_sco) { 1479 ldrw(temp_reg, super_check_offset_addr); 1480 super_check_offset = temp_reg; 1481 } 1482 1483 Address super_check_addr(sub_klass, super_check_offset); 1484 ldr(rscratch1, super_check_addr); 1485 cmp(super_klass, rscratch1); // load displayed supertype 1486 br(Assembler::EQ, *L_success); 1487 1488 // This check has worked decisively for primary supers. 1489 // Secondary supers are sought in the super_cache ('super_cache_addr'). 1490 // (Secondary supers are interfaces and very deeply nested subtypes.) 1491 // This works in the same check above because of a tricky aliasing 1492 // between the super_cache and the primary super display elements. 1493 // (The 'super_check_addr' can address either, as the case requires.) 1494 // Note that the cache is updated below if it does not help us find 1495 // what we need immediately. 1496 // So if it was a primary super, we can just fail immediately. 1497 // Otherwise, it's the slow path for us (no success at this point). 1498 1499 sub(rscratch1, super_check_offset, in_bytes(Klass::secondary_super_cache_offset())); 1500 if (L_failure == &L_fallthrough) { 1501 cbz(rscratch1, *L_slow_path); 1502 } else { 1503 cbnz(rscratch1, *L_failure); 1504 final_jmp(*L_slow_path); 1505 } 1506 1507 bind(L_fallthrough); 1508 1509 #undef final_jmp 1510 } 1511 1512 // These two are taken from x86, but they look generally useful 1513 1514 // scans count pointer sized words at [addr] for occurrence of value, 1515 // generic 1516 void MacroAssembler::repne_scan(Register addr, Register value, Register count, 1517 Register scratch) { 1518 Label Lloop, Lexit; 1519 cbz(count, Lexit); 1520 bind(Lloop); 1521 ldr(scratch, post(addr, wordSize)); 1522 cmp(value, scratch); 1523 br(EQ, Lexit); 1524 sub(count, count, 1); 1525 cbnz(count, Lloop); 1526 bind(Lexit); 1527 } 1528 1529 // scans count 4 byte words at [addr] for occurrence of value, 1530 // generic 1531 void MacroAssembler::repne_scanw(Register addr, Register value, Register count, 1532 Register scratch) { 1533 Label Lloop, Lexit; 1534 cbz(count, Lexit); 1535 bind(Lloop); 1536 ldrw(scratch, post(addr, wordSize)); 1537 cmpw(value, scratch); 1538 br(EQ, Lexit); 1539 sub(count, count, 1); 1540 cbnz(count, Lloop); 1541 bind(Lexit); 1542 } 1543 1544 void MacroAssembler::check_klass_subtype_slow_path_linear(Register sub_klass, 1545 Register super_klass, 1546 Register temp_reg, 1547 Register temp2_reg, 1548 Label* L_success, 1549 Label* L_failure, 1550 bool set_cond_codes) { 1551 // NB! Callers may assume that, when temp2_reg is a valid register, 1552 // this code sets it to a nonzero value. 1553 1554 assert_different_registers(sub_klass, super_klass, temp_reg); 1555 if (temp2_reg != noreg) 1556 assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg, rscratch1); 1557 #define IS_A_TEMP(reg) ((reg) == temp_reg || (reg) == temp2_reg) 1558 1559 Label L_fallthrough; 1560 int label_nulls = 0; 1561 if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; } 1562 if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; } 1563 assert(label_nulls <= 1, "at most one null in the batch"); 1564 1565 // a couple of useful fields in sub_klass: 1566 int ss_offset = in_bytes(Klass::secondary_supers_offset()); 1567 int sc_offset = in_bytes(Klass::secondary_super_cache_offset()); 1568 Address secondary_supers_addr(sub_klass, ss_offset); 1569 Address super_cache_addr( sub_klass, sc_offset); 1570 1571 BLOCK_COMMENT("check_klass_subtype_slow_path"); 1572 1573 // Do a linear scan of the secondary super-klass chain. 1574 // This code is rarely used, so simplicity is a virtue here. 1575 // The repne_scan instruction uses fixed registers, which we must spill. 1576 // Don't worry too much about pre-existing connections with the input regs. 1577 1578 assert(sub_klass != r0, "killed reg"); // killed by mov(r0, super) 1579 assert(sub_klass != r2, "killed reg"); // killed by lea(r2, &pst_counter) 1580 1581 RegSet pushed_registers; 1582 if (!IS_A_TEMP(r2)) pushed_registers += r2; 1583 if (!IS_A_TEMP(r5)) pushed_registers += r5; 1584 1585 if (super_klass != r0) { 1586 if (!IS_A_TEMP(r0)) pushed_registers += r0; 1587 } 1588 1589 push(pushed_registers, sp); 1590 1591 // Get super_klass value into r0 (even if it was in r5 or r2). 1592 if (super_klass != r0) { 1593 mov(r0, super_klass); 1594 } 1595 1596 #ifndef PRODUCT 1597 incrementw(ExternalAddress((address)&SharedRuntime::_partial_subtype_ctr)); 1598 #endif //PRODUCT 1599 1600 // We will consult the secondary-super array. 1601 ldr(r5, secondary_supers_addr); 1602 // Load the array length. 1603 ldrw(r2, Address(r5, Array<Klass*>::length_offset_in_bytes())); 1604 // Skip to start of data. 1605 add(r5, r5, Array<Klass*>::base_offset_in_bytes()); 1606 1607 cmp(sp, zr); // Clear Z flag; SP is never zero 1608 // Scan R2 words at [R5] for an occurrence of R0. 1609 // Set NZ/Z based on last compare. 1610 repne_scan(r5, r0, r2, rscratch1); 1611 1612 // Unspill the temp. registers: 1613 pop(pushed_registers, sp); 1614 1615 br(Assembler::NE, *L_failure); 1616 1617 // Success. Cache the super we found and proceed in triumph. 1618 1619 if (UseSecondarySupersCache) { 1620 str(super_klass, super_cache_addr); 1621 } 1622 1623 if (L_success != &L_fallthrough) { 1624 b(*L_success); 1625 } 1626 1627 #undef IS_A_TEMP 1628 1629 bind(L_fallthrough); 1630 } 1631 1632 // If Register r is invalid, remove a new register from 1633 // available_regs, and add new register to regs_to_push. 1634 Register MacroAssembler::allocate_if_noreg(Register r, 1635 RegSetIterator<Register> &available_regs, 1636 RegSet ®s_to_push) { 1637 if (!r->is_valid()) { 1638 r = *available_regs++; 1639 regs_to_push += r; 1640 } 1641 return r; 1642 } 1643 1644 // check_klass_subtype_slow_path_table() looks for super_klass in the 1645 // hash table belonging to super_klass, branching to L_success or 1646 // L_failure as appropriate. This is essentially a shim which 1647 // allocates registers as necessary then calls 1648 // lookup_secondary_supers_table() to do the work. Any of the temp 1649 // regs may be noreg, in which case this logic will chooses some 1650 // registers push and pop them from the stack. 1651 void MacroAssembler::check_klass_subtype_slow_path_table(Register sub_klass, 1652 Register super_klass, 1653 Register temp_reg, 1654 Register temp2_reg, 1655 Register temp3_reg, 1656 Register result_reg, 1657 FloatRegister vtemp, 1658 Label* L_success, 1659 Label* L_failure, 1660 bool set_cond_codes) { 1661 RegSet temps = RegSet::of(temp_reg, temp2_reg, temp3_reg); 1662 1663 assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg, rscratch1); 1664 1665 Label L_fallthrough; 1666 int label_nulls = 0; 1667 if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; } 1668 if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; } 1669 assert(label_nulls <= 1, "at most one null in the batch"); 1670 1671 BLOCK_COMMENT("check_klass_subtype_slow_path"); 1672 1673 RegSetIterator<Register> available_regs 1674 = (RegSet::range(r0, r15) - temps - sub_klass - super_klass).begin(); 1675 1676 RegSet pushed_regs; 1677 1678 temp_reg = allocate_if_noreg(temp_reg, available_regs, pushed_regs); 1679 temp2_reg = allocate_if_noreg(temp2_reg, available_regs, pushed_regs); 1680 temp3_reg = allocate_if_noreg(temp3_reg, available_regs, pushed_regs); 1681 result_reg = allocate_if_noreg(result_reg, available_regs, pushed_regs); 1682 1683 push(pushed_regs, sp); 1684 1685 lookup_secondary_supers_table_var(sub_klass, 1686 super_klass, 1687 temp_reg, temp2_reg, temp3_reg, vtemp, result_reg, 1688 nullptr); 1689 cmp(result_reg, zr); 1690 1691 // Unspill the temp. registers: 1692 pop(pushed_regs, sp); 1693 1694 // NB! Callers may assume that, when set_cond_codes is true, this 1695 // code sets temp2_reg to a nonzero value. 1696 if (set_cond_codes) { 1697 mov(temp2_reg, 1); 1698 } 1699 1700 br(Assembler::NE, *L_failure); 1701 1702 if (L_success != &L_fallthrough) { 1703 b(*L_success); 1704 } 1705 1706 bind(L_fallthrough); 1707 } 1708 1709 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass, 1710 Register super_klass, 1711 Register temp_reg, 1712 Register temp2_reg, 1713 Label* L_success, 1714 Label* L_failure, 1715 bool set_cond_codes) { 1716 if (UseSecondarySupersTable) { 1717 check_klass_subtype_slow_path_table 1718 (sub_klass, super_klass, temp_reg, temp2_reg, /*temp3*/noreg, /*result*/noreg, 1719 /*vtemp*/fnoreg, 1720 L_success, L_failure, set_cond_codes); 1721 } else { 1722 check_klass_subtype_slow_path_linear 1723 (sub_klass, super_klass, temp_reg, temp2_reg, L_success, L_failure, set_cond_codes); 1724 } 1725 } 1726 1727 1728 // Ensure that the inline code and the stub are using the same registers. 1729 #define LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS \ 1730 do { \ 1731 assert(r_super_klass == r0 && \ 1732 r_array_base == r1 && \ 1733 r_array_length == r2 && \ 1734 (r_array_index == r3 || r_array_index == noreg) && \ 1735 (r_sub_klass == r4 || r_sub_klass == noreg) && \ 1736 (r_bitmap == rscratch2 || r_bitmap == noreg) && \ 1737 (result == r5 || result == noreg), "registers must match aarch64.ad"); \ 1738 } while(0) 1739 1740 bool MacroAssembler::lookup_secondary_supers_table_const(Register r_sub_klass, 1741 Register r_super_klass, 1742 Register temp1, 1743 Register temp2, 1744 Register temp3, 1745 FloatRegister vtemp, 1746 Register result, 1747 u1 super_klass_slot, 1748 bool stub_is_near) { 1749 assert_different_registers(r_sub_klass, temp1, temp2, temp3, result, rscratch1, rscratch2); 1750 1751 Label L_fallthrough; 1752 1753 BLOCK_COMMENT("lookup_secondary_supers_table {"); 1754 1755 const Register 1756 r_array_base = temp1, // r1 1757 r_array_length = temp2, // r2 1758 r_array_index = temp3, // r3 1759 r_bitmap = rscratch2; 1760 1761 LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS; 1762 1763 u1 bit = super_klass_slot; 1764 1765 // Make sure that result is nonzero if the TBZ below misses. 1766 mov(result, 1); 1767 1768 // We're going to need the bitmap in a vector reg and in a core reg, 1769 // so load both now. 1770 ldr(r_bitmap, Address(r_sub_klass, Klass::secondary_supers_bitmap_offset())); 1771 if (bit != 0) { 1772 ldrd(vtemp, Address(r_sub_klass, Klass::secondary_supers_bitmap_offset())); 1773 } 1774 // First check the bitmap to see if super_klass might be present. If 1775 // the bit is zero, we are certain that super_klass is not one of 1776 // the secondary supers. 1777 tbz(r_bitmap, bit, L_fallthrough); 1778 1779 // Get the first array index that can contain super_klass into r_array_index. 1780 if (bit != 0) { 1781 shld(vtemp, vtemp, Klass::SECONDARY_SUPERS_TABLE_MASK - bit); 1782 cnt(vtemp, T8B, vtemp); 1783 addv(vtemp, T8B, vtemp); 1784 fmovd(r_array_index, vtemp); 1785 } else { 1786 mov(r_array_index, (u1)1); 1787 } 1788 // NB! r_array_index is off by 1. It is compensated by keeping r_array_base off by 1 word. 1789 1790 // We will consult the secondary-super array. 1791 ldr(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset()))); 1792 1793 // The value i in r_array_index is >= 1, so even though r_array_base 1794 // points to the length, we don't need to adjust it to point to the 1795 // data. 1796 assert(Array<Klass*>::base_offset_in_bytes() == wordSize, "Adjust this code"); 1797 assert(Array<Klass*>::length_offset_in_bytes() == 0, "Adjust this code"); 1798 1799 ldr(result, Address(r_array_base, r_array_index, Address::lsl(LogBytesPerWord))); 1800 eor(result, result, r_super_klass); 1801 cbz(result, L_fallthrough); // Found a match 1802 1803 // Is there another entry to check? Consult the bitmap. 1804 tbz(r_bitmap, (bit + 1) & Klass::SECONDARY_SUPERS_TABLE_MASK, L_fallthrough); 1805 1806 // Linear probe. 1807 if (bit != 0) { 1808 ror(r_bitmap, r_bitmap, bit); 1809 } 1810 1811 // The slot we just inspected is at secondary_supers[r_array_index - 1]. 1812 // The next slot to be inspected, by the stub we're about to call, 1813 // is secondary_supers[r_array_index]. Bits 0 and 1 in the bitmap 1814 // have been checked. 1815 Address stub = RuntimeAddress(StubRoutines::lookup_secondary_supers_table_slow_path_stub()); 1816 if (stub_is_near) { 1817 bl(stub); 1818 } else { 1819 address call = trampoline_call(stub); 1820 if (call == nullptr) { 1821 return false; // trampoline allocation failed 1822 } 1823 } 1824 1825 BLOCK_COMMENT("} lookup_secondary_supers_table"); 1826 1827 bind(L_fallthrough); 1828 1829 if (VerifySecondarySupers) { 1830 verify_secondary_supers_table(r_sub_klass, r_super_klass, // r4, r0 1831 temp1, temp2, result); // r1, r2, r5 1832 } 1833 return true; 1834 } 1835 1836 // At runtime, return 0 in result if r_super_klass is a superclass of 1837 // r_sub_klass, otherwise return nonzero. Use this version of 1838 // lookup_secondary_supers_table() if you don't know ahead of time 1839 // which superclass will be searched for. Used by interpreter and 1840 // runtime stubs. It is larger and has somewhat greater latency than 1841 // the version above, which takes a constant super_klass_slot. 1842 void MacroAssembler::lookup_secondary_supers_table_var(Register r_sub_klass, 1843 Register r_super_klass, 1844 Register temp1, 1845 Register temp2, 1846 Register temp3, 1847 FloatRegister vtemp, 1848 Register result, 1849 Label *L_success) { 1850 assert_different_registers(r_sub_klass, temp1, temp2, temp3, result, rscratch1, rscratch2); 1851 1852 Label L_fallthrough; 1853 1854 BLOCK_COMMENT("lookup_secondary_supers_table {"); 1855 1856 const Register 1857 r_array_index = temp3, 1858 slot = rscratch1, 1859 r_bitmap = rscratch2; 1860 1861 ldrb(slot, Address(r_super_klass, Klass::hash_slot_offset())); 1862 1863 // Make sure that result is nonzero if the test below misses. 1864 mov(result, 1); 1865 1866 ldr(r_bitmap, Address(r_sub_klass, Klass::secondary_supers_bitmap_offset())); 1867 1868 // First check the bitmap to see if super_klass might be present. If 1869 // the bit is zero, we are certain that super_klass is not one of 1870 // the secondary supers. 1871 1872 // This next instruction is equivalent to: 1873 // mov(tmp_reg, (u1)(Klass::SECONDARY_SUPERS_TABLE_SIZE - 1)); 1874 // sub(temp2, tmp_reg, slot); 1875 eor(temp2, slot, (u1)(Klass::SECONDARY_SUPERS_TABLE_SIZE - 1)); 1876 lslv(temp2, r_bitmap, temp2); 1877 tbz(temp2, Klass::SECONDARY_SUPERS_TABLE_SIZE - 1, L_fallthrough); 1878 1879 bool must_save_v0 = (vtemp == fnoreg); 1880 if (must_save_v0) { 1881 // temp1 and result are free, so use them to preserve vtemp 1882 vtemp = v0; 1883 mov(temp1, vtemp, D, 0); 1884 mov(result, vtemp, D, 1); 1885 } 1886 1887 // Get the first array index that can contain super_klass into r_array_index. 1888 mov(vtemp, D, 0, temp2); 1889 cnt(vtemp, T8B, vtemp); 1890 addv(vtemp, T8B, vtemp); 1891 mov(r_array_index, vtemp, D, 0); 1892 1893 if (must_save_v0) { 1894 mov(vtemp, D, 0, temp1 ); 1895 mov(vtemp, D, 1, result); 1896 } 1897 1898 // NB! r_array_index is off by 1. It is compensated by keeping r_array_base off by 1 word. 1899 1900 const Register 1901 r_array_base = temp1, 1902 r_array_length = temp2; 1903 1904 // The value i in r_array_index is >= 1, so even though r_array_base 1905 // points to the length, we don't need to adjust it to point to the 1906 // data. 1907 assert(Array<Klass*>::base_offset_in_bytes() == wordSize, "Adjust this code"); 1908 assert(Array<Klass*>::length_offset_in_bytes() == 0, "Adjust this code"); 1909 1910 // We will consult the secondary-super array. 1911 ldr(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset()))); 1912 1913 ldr(result, Address(r_array_base, r_array_index, Address::lsl(LogBytesPerWord))); 1914 eor(result, result, r_super_klass); 1915 cbz(result, L_success ? *L_success : L_fallthrough); // Found a match 1916 1917 // Is there another entry to check? Consult the bitmap. 1918 rorv(r_bitmap, r_bitmap, slot); 1919 // rol(r_bitmap, r_bitmap, 1); 1920 tbz(r_bitmap, 1, L_fallthrough); 1921 1922 // The slot we just inspected is at secondary_supers[r_array_index - 1]. 1923 // The next slot to be inspected, by the logic we're about to call, 1924 // is secondary_supers[r_array_index]. Bits 0 and 1 in the bitmap 1925 // have been checked. 1926 lookup_secondary_supers_table_slow_path(r_super_klass, r_array_base, r_array_index, 1927 r_bitmap, r_array_length, result, /*is_stub*/false); 1928 1929 BLOCK_COMMENT("} lookup_secondary_supers_table"); 1930 1931 bind(L_fallthrough); 1932 1933 if (VerifySecondarySupers) { 1934 verify_secondary_supers_table(r_sub_klass, r_super_klass, // r4, r0 1935 temp1, temp2, result); // r1, r2, r5 1936 } 1937 1938 if (L_success) { 1939 cbz(result, *L_success); 1940 } 1941 } 1942 1943 // Called by code generated by check_klass_subtype_slow_path 1944 // above. This is called when there is a collision in the hashed 1945 // lookup in the secondary supers array. 1946 void MacroAssembler::lookup_secondary_supers_table_slow_path(Register r_super_klass, 1947 Register r_array_base, 1948 Register r_array_index, 1949 Register r_bitmap, 1950 Register temp1, 1951 Register result, 1952 bool is_stub) { 1953 assert_different_registers(r_super_klass, r_array_base, r_array_index, r_bitmap, temp1, result, rscratch1); 1954 1955 const Register 1956 r_array_length = temp1, 1957 r_sub_klass = noreg; // unused 1958 1959 if (is_stub) { 1960 LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS; 1961 } 1962 1963 Label L_fallthrough, L_huge; 1964 1965 // Load the array length. 1966 ldrw(r_array_length, Address(r_array_base, Array<Klass*>::length_offset_in_bytes())); 1967 // And adjust the array base to point to the data. 1968 // NB! Effectively increments current slot index by 1. 1969 assert(Array<Klass*>::base_offset_in_bytes() == wordSize, ""); 1970 add(r_array_base, r_array_base, Array<Klass*>::base_offset_in_bytes()); 1971 1972 // The bitmap is full to bursting. 1973 // Implicit invariant: BITMAP_FULL implies (length > 0) 1974 assert(Klass::SECONDARY_SUPERS_BITMAP_FULL == ~uintx(0), ""); 1975 cmpw(r_array_length, (u1)(Klass::SECONDARY_SUPERS_TABLE_SIZE - 2)); 1976 br(GT, L_huge); 1977 1978 // NB! Our caller has checked bits 0 and 1 in the bitmap. The 1979 // current slot (at secondary_supers[r_array_index]) has not yet 1980 // been inspected, and r_array_index may be out of bounds if we 1981 // wrapped around the end of the array. 1982 1983 { // This is conventional linear probing, but instead of terminating 1984 // when a null entry is found in the table, we maintain a bitmap 1985 // in which a 0 indicates missing entries. 1986 // As long as the bitmap is not completely full, 1987 // array_length == popcount(bitmap). The array_length check above 1988 // guarantees there are 0s in the bitmap, so the loop eventually 1989 // terminates. 1990 Label L_loop; 1991 bind(L_loop); 1992 1993 // Check for wraparound. 1994 cmp(r_array_index, r_array_length); 1995 csel(r_array_index, zr, r_array_index, GE); 1996 1997 ldr(rscratch1, Address(r_array_base, r_array_index, Address::lsl(LogBytesPerWord))); 1998 eor(result, rscratch1, r_super_klass); 1999 cbz(result, L_fallthrough); 2000 2001 tbz(r_bitmap, 2, L_fallthrough); // look-ahead check (Bit 2); result is non-zero 2002 2003 ror(r_bitmap, r_bitmap, 1); 2004 add(r_array_index, r_array_index, 1); 2005 b(L_loop); 2006 } 2007 2008 { // Degenerate case: more than 64 secondary supers. 2009 // FIXME: We could do something smarter here, maybe a vectorized 2010 // comparison or a binary search, but is that worth any added 2011 // complexity? 2012 bind(L_huge); 2013 cmp(sp, zr); // Clear Z flag; SP is never zero 2014 repne_scan(r_array_base, r_super_klass, r_array_length, rscratch1); 2015 cset(result, NE); // result == 0 iff we got a match. 2016 } 2017 2018 bind(L_fallthrough); 2019 } 2020 2021 // Make sure that the hashed lookup and a linear scan agree. 2022 void MacroAssembler::verify_secondary_supers_table(Register r_sub_klass, 2023 Register r_super_klass, 2024 Register temp1, 2025 Register temp2, 2026 Register result) { 2027 assert_different_registers(r_sub_klass, r_super_klass, temp1, temp2, result, rscratch1); 2028 2029 const Register 2030 r_array_base = temp1, 2031 r_array_length = temp2, 2032 r_array_index = noreg, // unused 2033 r_bitmap = noreg; // unused 2034 2035 BLOCK_COMMENT("verify_secondary_supers_table {"); 2036 2037 // We will consult the secondary-super array. 2038 ldr(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset()))); 2039 2040 // Load the array length. 2041 ldrw(r_array_length, Address(r_array_base, Array<Klass*>::length_offset_in_bytes())); 2042 // And adjust the array base to point to the data. 2043 add(r_array_base, r_array_base, Array<Klass*>::base_offset_in_bytes()); 2044 2045 cmp(sp, zr); // Clear Z flag; SP is never zero 2046 // Scan R2 words at [R5] for an occurrence of R0. 2047 // Set NZ/Z based on last compare. 2048 repne_scan(/*addr*/r_array_base, /*value*/r_super_klass, /*count*/r_array_length, rscratch2); 2049 // rscratch1 == 0 iff we got a match. 2050 cset(rscratch1, NE); 2051 2052 Label passed; 2053 cmp(result, zr); 2054 cset(result, NE); // normalize result to 0/1 for comparison 2055 2056 cmp(rscratch1, result); 2057 br(EQ, passed); 2058 { 2059 mov(r0, r_super_klass); // r0 <- r0 2060 mov(r1, r_sub_klass); // r1 <- r4 2061 mov(r2, /*expected*/rscratch1); // r2 <- r8 2062 mov(r3, result); // r3 <- r5 2063 mov(r4, (address)("mismatch")); // r4 <- const 2064 rt_call(CAST_FROM_FN_PTR(address, Klass::on_secondary_supers_verification_failure), rscratch2); 2065 should_not_reach_here(); 2066 } 2067 bind(passed); 2068 2069 BLOCK_COMMENT("} verify_secondary_supers_table"); 2070 } 2071 2072 void MacroAssembler::clinit_barrier(Register klass, Register scratch, Label* L_fast_path, Label* L_slow_path) { 2073 assert(L_fast_path != nullptr || L_slow_path != nullptr, "at least one is required"); 2074 assert_different_registers(klass, rthread, scratch); 2075 2076 Label L_fallthrough, L_tmp; 2077 if (L_fast_path == nullptr) { 2078 L_fast_path = &L_fallthrough; 2079 } else if (L_slow_path == nullptr) { 2080 L_slow_path = &L_fallthrough; 2081 } 2082 // Fast path check: class is fully initialized 2083 lea(scratch, Address(klass, InstanceKlass::init_state_offset())); 2084 ldarb(scratch, scratch); 2085 subs(zr, scratch, InstanceKlass::fully_initialized); 2086 br(Assembler::EQ, *L_fast_path); 2087 2088 // Fast path check: current thread is initializer thread 2089 ldr(scratch, Address(klass, InstanceKlass::init_thread_offset())); 2090 cmp(rthread, scratch); 2091 2092 if (L_slow_path == &L_fallthrough) { 2093 br(Assembler::EQ, *L_fast_path); 2094 bind(*L_slow_path); 2095 } else if (L_fast_path == &L_fallthrough) { 2096 br(Assembler::NE, *L_slow_path); 2097 bind(*L_fast_path); 2098 } else { 2099 Unimplemented(); 2100 } 2101 } 2102 2103 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) { 2104 if (!VerifyOops || VerifyAdapterSharing) { 2105 // Below address of the code string confuses VerifyAdapterSharing 2106 // because it may differ between otherwise equivalent adapters. 2107 return; 2108 } 2109 2110 // Pass register number to verify_oop_subroutine 2111 const char* b = nullptr; 2112 { 2113 ResourceMark rm; 2114 stringStream ss; 2115 ss.print("verify_oop: %s: %s (%s:%d)", reg->name(), s, file, line); 2116 b = code_string(ss.as_string()); 2117 } 2118 BLOCK_COMMENT("verify_oop {"); 2119 2120 strip_return_address(); // This might happen within a stack frame. 2121 protect_return_address(); 2122 stp(r0, rscratch1, Address(pre(sp, -2 * wordSize))); 2123 stp(rscratch2, lr, Address(pre(sp, -2 * wordSize))); 2124 2125 mov(r0, reg); 2126 movptr(rscratch1, (uintptr_t)(address)b); 2127 2128 // call indirectly to solve generation ordering problem 2129 lea(rscratch2, RuntimeAddress(StubRoutines::verify_oop_subroutine_entry_address())); 2130 ldr(rscratch2, Address(rscratch2)); 2131 blr(rscratch2); 2132 2133 ldp(rscratch2, lr, Address(post(sp, 2 * wordSize))); 2134 ldp(r0, rscratch1, Address(post(sp, 2 * wordSize))); 2135 authenticate_return_address(); 2136 2137 BLOCK_COMMENT("} verify_oop"); 2138 } 2139 2140 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) { 2141 if (!VerifyOops || VerifyAdapterSharing) { 2142 // Below address of the code string confuses VerifyAdapterSharing 2143 // because it may differ between otherwise equivalent adapters. 2144 return; 2145 } 2146 2147 const char* b = nullptr; 2148 { 2149 ResourceMark rm; 2150 stringStream ss; 2151 ss.print("verify_oop_addr: %s (%s:%d)", s, file, line); 2152 b = code_string(ss.as_string()); 2153 } 2154 BLOCK_COMMENT("verify_oop_addr {"); 2155 2156 strip_return_address(); // This might happen within a stack frame. 2157 protect_return_address(); 2158 stp(r0, rscratch1, Address(pre(sp, -2 * wordSize))); 2159 stp(rscratch2, lr, Address(pre(sp, -2 * wordSize))); 2160 2161 // addr may contain sp so we will have to adjust it based on the 2162 // pushes that we just did. 2163 if (addr.uses(sp)) { 2164 lea(r0, addr); 2165 ldr(r0, Address(r0, 4 * wordSize)); 2166 } else { 2167 ldr(r0, addr); 2168 } 2169 movptr(rscratch1, (uintptr_t)(address)b); 2170 2171 // call indirectly to solve generation ordering problem 2172 lea(rscratch2, RuntimeAddress(StubRoutines::verify_oop_subroutine_entry_address())); 2173 ldr(rscratch2, Address(rscratch2)); 2174 blr(rscratch2); 2175 2176 ldp(rscratch2, lr, Address(post(sp, 2 * wordSize))); 2177 ldp(r0, rscratch1, Address(post(sp, 2 * wordSize))); 2178 authenticate_return_address(); 2179 2180 BLOCK_COMMENT("} verify_oop_addr"); 2181 } 2182 2183 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot, 2184 int extra_slot_offset) { 2185 // cf. TemplateTable::prepare_invoke(), if (load_receiver). 2186 int stackElementSize = Interpreter::stackElementSize; 2187 int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0); 2188 #ifdef ASSERT 2189 int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1); 2190 assert(offset1 - offset == stackElementSize, "correct arithmetic"); 2191 #endif 2192 if (arg_slot.is_constant()) { 2193 return Address(esp, arg_slot.as_constant() * stackElementSize 2194 + offset); 2195 } else { 2196 add(rscratch1, esp, arg_slot.as_register(), 2197 ext::uxtx, exact_log2(stackElementSize)); 2198 return Address(rscratch1, offset); 2199 } 2200 } 2201 2202 void MacroAssembler::call_VM_leaf_base(address entry_point, 2203 int number_of_arguments, 2204 Label *retaddr) { 2205 Label E, L; 2206 2207 stp(rscratch1, rmethod, Address(pre(sp, -2 * wordSize))); 2208 2209 mov(rscratch1, entry_point); 2210 blr(rscratch1); 2211 if (retaddr) 2212 bind(*retaddr); 2213 2214 ldp(rscratch1, rmethod, Address(post(sp, 2 * wordSize))); 2215 } 2216 2217 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) { 2218 call_VM_leaf_base(entry_point, number_of_arguments); 2219 } 2220 2221 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) { 2222 pass_arg0(this, arg_0); 2223 call_VM_leaf_base(entry_point, 1); 2224 } 2225 2226 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) { 2227 assert_different_registers(arg_1, c_rarg0); 2228 pass_arg0(this, arg_0); 2229 pass_arg1(this, arg_1); 2230 call_VM_leaf_base(entry_point, 2); 2231 } 2232 2233 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, 2234 Register arg_1, Register arg_2) { 2235 assert_different_registers(arg_1, c_rarg0); 2236 assert_different_registers(arg_2, c_rarg0, c_rarg1); 2237 pass_arg0(this, arg_0); 2238 pass_arg1(this, arg_1); 2239 pass_arg2(this, arg_2); 2240 call_VM_leaf_base(entry_point, 3); 2241 } 2242 2243 void MacroAssembler::super_call_VM_leaf(address entry_point) { 2244 MacroAssembler::call_VM_leaf_base(entry_point, 1); 2245 } 2246 2247 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) { 2248 pass_arg0(this, arg_0); 2249 MacroAssembler::call_VM_leaf_base(entry_point, 1); 2250 } 2251 2252 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) { 2253 2254 assert_different_registers(arg_0, c_rarg1); 2255 pass_arg1(this, arg_1); 2256 pass_arg0(this, arg_0); 2257 MacroAssembler::call_VM_leaf_base(entry_point, 2); 2258 } 2259 2260 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) { 2261 assert_different_registers(arg_0, c_rarg1, c_rarg2); 2262 assert_different_registers(arg_1, c_rarg2); 2263 pass_arg2(this, arg_2); 2264 pass_arg1(this, arg_1); 2265 pass_arg0(this, arg_0); 2266 MacroAssembler::call_VM_leaf_base(entry_point, 3); 2267 } 2268 2269 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) { 2270 assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3); 2271 assert_different_registers(arg_1, c_rarg2, c_rarg3); 2272 assert_different_registers(arg_2, c_rarg3); 2273 pass_arg3(this, arg_3); 2274 pass_arg2(this, arg_2); 2275 pass_arg1(this, arg_1); 2276 pass_arg0(this, arg_0); 2277 MacroAssembler::call_VM_leaf_base(entry_point, 4); 2278 } 2279 2280 void MacroAssembler::null_check(Register reg, int offset) { 2281 if (needs_explicit_null_check(offset)) { 2282 // provoke OS null exception if reg is null by 2283 // accessing M[reg] w/o changing any registers 2284 // NOTE: this is plenty to provoke a segv 2285 ldr(zr, Address(reg)); 2286 } else { 2287 // nothing to do, (later) access of M[reg + offset] 2288 // will provoke OS null exception if reg is null 2289 } 2290 } 2291 2292 void MacroAssembler::test_markword_is_inline_type(Register markword, Label& is_inline_type) { 2293 assert_different_registers(markword, rscratch2); 2294 mov(rscratch2, markWord::inline_type_mask_in_place); 2295 andr(markword, markword, rscratch2); 2296 mov(rscratch2, markWord::inline_type_pattern); 2297 cmp(markword, rscratch2); 2298 br(Assembler::EQ, is_inline_type); 2299 } 2300 2301 void MacroAssembler::test_klass_is_inline_type(Register klass, Register temp_reg, Label& is_inline_type) { 2302 ldrh(temp_reg, Address(klass, Klass::access_flags_offset())); 2303 andr(temp_reg, temp_reg, JVM_ACC_IDENTITY); 2304 cbz(temp_reg, is_inline_type); 2305 } 2306 2307 void MacroAssembler::test_oop_is_not_inline_type(Register object, Register tmp, Label& not_inline_type) { 2308 assert_different_registers(tmp, rscratch1); 2309 cbz(object, not_inline_type); 2310 const int is_inline_type_mask = markWord::inline_type_pattern; 2311 ldr(tmp, Address(object, oopDesc::mark_offset_in_bytes())); 2312 mov(rscratch1, is_inline_type_mask); 2313 andr(tmp, tmp, rscratch1); 2314 cmp(tmp, rscratch1); 2315 br(Assembler::NE, not_inline_type); 2316 } 2317 2318 void MacroAssembler::test_klass_is_empty_inline_type(Register klass, Register temp_reg, Label& is_empty_inline_type) { 2319 #ifdef ASSERT 2320 { 2321 Label done_check; 2322 test_klass_is_inline_type(klass, temp_reg, done_check); 2323 stop("test_klass_is_empty_inline_type with non inline type klass"); 2324 bind(done_check); 2325 } 2326 #endif 2327 ldrw(temp_reg, Address(klass, InstanceKlass::misc_flags_offset())); 2328 andr(temp_reg, temp_reg, InstanceKlassFlags::is_empty_inline_type_value()); 2329 cbnz(temp_reg, is_empty_inline_type); 2330 } 2331 2332 void MacroAssembler::test_field_is_null_free_inline_type(Register flags, Register temp_reg, Label& is_null_free_inline_type) { 2333 assert(temp_reg == noreg, "not needed"); // keep signature uniform with x86 2334 tbnz(flags, ResolvedFieldEntry::is_null_free_inline_type_shift, is_null_free_inline_type); 2335 } 2336 2337 void MacroAssembler::test_field_is_not_null_free_inline_type(Register flags, Register temp_reg, Label& not_null_free_inline_type) { 2338 assert(temp_reg == noreg, "not needed"); // keep signature uniform with x86 2339 tbz(flags, ResolvedFieldEntry::is_null_free_inline_type_shift, not_null_free_inline_type); 2340 } 2341 2342 void MacroAssembler::test_field_is_flat(Register flags, Register temp_reg, Label& is_flat) { 2343 assert(temp_reg == noreg, "not needed"); // keep signature uniform with x86 2344 tbnz(flags, ResolvedFieldEntry::is_flat_shift, is_flat); 2345 } 2346 2347 void MacroAssembler::test_field_has_null_marker(Register flags, Register temp_reg, Label& has_null_marker) { 2348 assert(temp_reg == noreg, "not needed"); // keep signature uniform with x86 2349 tbnz(flags, ResolvedFieldEntry::has_null_marker_shift, has_null_marker); 2350 } 2351 2352 void MacroAssembler::test_oop_prototype_bit(Register oop, Register temp_reg, int32_t test_bit, bool jmp_set, Label& jmp_label) { 2353 Label test_mark_word; 2354 // load mark word 2355 ldr(temp_reg, Address(oop, oopDesc::mark_offset_in_bytes())); 2356 // check displaced 2357 tst(temp_reg, markWord::unlocked_value); 2358 br(Assembler::NE, test_mark_word); 2359 // slow path use klass prototype 2360 load_prototype_header(temp_reg, oop); 2361 2362 bind(test_mark_word); 2363 andr(temp_reg, temp_reg, test_bit); 2364 if (jmp_set) { 2365 cbnz(temp_reg, jmp_label); 2366 } else { 2367 cbz(temp_reg, jmp_label); 2368 } 2369 } 2370 2371 void MacroAssembler::test_flat_array_oop(Register oop, Register temp_reg, Label& is_flat_array) { 2372 test_oop_prototype_bit(oop, temp_reg, markWord::flat_array_bit_in_place, true, is_flat_array); 2373 } 2374 2375 void MacroAssembler::test_non_flat_array_oop(Register oop, Register temp_reg, 2376 Label&is_non_flat_array) { 2377 test_oop_prototype_bit(oop, temp_reg, markWord::flat_array_bit_in_place, false, is_non_flat_array); 2378 } 2379 2380 void MacroAssembler::test_null_free_array_oop(Register oop, Register temp_reg, Label& is_null_free_array) { 2381 test_oop_prototype_bit(oop, temp_reg, markWord::null_free_array_bit_in_place, true, is_null_free_array); 2382 } 2383 2384 void MacroAssembler::test_non_null_free_array_oop(Register oop, Register temp_reg, Label&is_non_null_free_array) { 2385 test_oop_prototype_bit(oop, temp_reg, markWord::null_free_array_bit_in_place, false, is_non_null_free_array); 2386 } 2387 2388 void MacroAssembler::test_flat_array_layout(Register lh, Label& is_flat_array) { 2389 tst(lh, Klass::_lh_array_tag_flat_value_bit_inplace); 2390 br(Assembler::NE, is_flat_array); 2391 } 2392 2393 void MacroAssembler::test_non_flat_array_layout(Register lh, Label& is_non_flat_array) { 2394 tst(lh, Klass::_lh_array_tag_flat_value_bit_inplace); 2395 br(Assembler::EQ, is_non_flat_array); 2396 } 2397 2398 // MacroAssembler protected routines needed to implement 2399 // public methods 2400 2401 void MacroAssembler::mov(Register r, Address dest) { 2402 code_section()->relocate(pc(), dest.rspec()); 2403 uint64_t imm64 = (uint64_t)dest.target(); 2404 movptr(r, imm64); 2405 } 2406 2407 // Move a constant pointer into r. In AArch64 mode the virtual 2408 // address space is 48 bits in size, so we only need three 2409 // instructions to create a patchable instruction sequence that can 2410 // reach anywhere. 2411 void MacroAssembler::movptr(Register r, uintptr_t imm64) { 2412 #ifndef PRODUCT 2413 { 2414 char buffer[64]; 2415 snprintf(buffer, sizeof(buffer), "0x%" PRIX64, (uint64_t)imm64); 2416 block_comment(buffer); 2417 } 2418 #endif 2419 assert(imm64 < (1ull << 48), "48-bit overflow in address constant"); 2420 movz(r, imm64 & 0xffff); 2421 imm64 >>= 16; 2422 movk(r, imm64 & 0xffff, 16); 2423 imm64 >>= 16; 2424 movk(r, imm64 & 0xffff, 32); 2425 } 2426 2427 // Macro to mov replicated immediate to vector register. 2428 // imm64: only the lower 8/16/32 bits are considered for B/H/S type. That is, 2429 // the upper 56/48/32 bits must be zeros for B/H/S type. 2430 // Vd will get the following values for different arrangements in T 2431 // imm64 == hex 000000gh T8B: Vd = ghghghghghghghgh 2432 // imm64 == hex 000000gh T16B: Vd = ghghghghghghghghghghghghghghghgh 2433 // imm64 == hex 0000efgh T4H: Vd = efghefghefghefgh 2434 // imm64 == hex 0000efgh T8H: Vd = efghefghefghefghefghefghefghefgh 2435 // imm64 == hex abcdefgh T2S: Vd = abcdefghabcdefgh 2436 // imm64 == hex abcdefgh T4S: Vd = abcdefghabcdefghabcdefghabcdefgh 2437 // imm64 == hex abcdefgh T1D: Vd = 00000000abcdefgh 2438 // imm64 == hex abcdefgh T2D: Vd = 00000000abcdefgh00000000abcdefgh 2439 // Clobbers rscratch1 2440 void MacroAssembler::mov(FloatRegister Vd, SIMD_Arrangement T, uint64_t imm64) { 2441 assert(T != T1Q, "unsupported"); 2442 if (T == T1D || T == T2D) { 2443 int imm = operand_valid_for_movi_immediate(imm64, T); 2444 if (-1 != imm) { 2445 movi(Vd, T, imm); 2446 } else { 2447 mov(rscratch1, imm64); 2448 dup(Vd, T, rscratch1); 2449 } 2450 return; 2451 } 2452 2453 #ifdef ASSERT 2454 if (T == T8B || T == T16B) assert((imm64 & ~0xff) == 0, "extraneous bits (T8B/T16B)"); 2455 if (T == T4H || T == T8H) assert((imm64 & ~0xffff) == 0, "extraneous bits (T4H/T8H)"); 2456 if (T == T2S || T == T4S) assert((imm64 & ~0xffffffff) == 0, "extraneous bits (T2S/T4S)"); 2457 #endif 2458 int shift = operand_valid_for_movi_immediate(imm64, T); 2459 uint32_t imm32 = imm64 & 0xffffffffULL; 2460 if (shift >= 0) { 2461 movi(Vd, T, (imm32 >> shift) & 0xff, shift); 2462 } else { 2463 movw(rscratch1, imm32); 2464 dup(Vd, T, rscratch1); 2465 } 2466 } 2467 2468 void MacroAssembler::mov_immediate64(Register dst, uint64_t imm64) 2469 { 2470 #ifndef PRODUCT 2471 { 2472 char buffer[64]; 2473 snprintf(buffer, sizeof(buffer), "0x%" PRIX64, imm64); 2474 block_comment(buffer); 2475 } 2476 #endif 2477 if (operand_valid_for_logical_immediate(false, imm64)) { 2478 orr(dst, zr, imm64); 2479 } else { 2480 // we can use a combination of MOVZ or MOVN with 2481 // MOVK to build up the constant 2482 uint64_t imm_h[4]; 2483 int zero_count = 0; 2484 int neg_count = 0; 2485 int i; 2486 for (i = 0; i < 4; i++) { 2487 imm_h[i] = ((imm64 >> (i * 16)) & 0xffffL); 2488 if (imm_h[i] == 0) { 2489 zero_count++; 2490 } else if (imm_h[i] == 0xffffL) { 2491 neg_count++; 2492 } 2493 } 2494 if (zero_count == 4) { 2495 // one MOVZ will do 2496 movz(dst, 0); 2497 } else if (neg_count == 4) { 2498 // one MOVN will do 2499 movn(dst, 0); 2500 } else if (zero_count == 3) { 2501 for (i = 0; i < 4; i++) { 2502 if (imm_h[i] != 0L) { 2503 movz(dst, (uint32_t)imm_h[i], (i << 4)); 2504 break; 2505 } 2506 } 2507 } else if (neg_count == 3) { 2508 // one MOVN will do 2509 for (int i = 0; i < 4; i++) { 2510 if (imm_h[i] != 0xffffL) { 2511 movn(dst, (uint32_t)imm_h[i] ^ 0xffffL, (i << 4)); 2512 break; 2513 } 2514 } 2515 } else if (zero_count == 2) { 2516 // one MOVZ and one MOVK will do 2517 for (i = 0; i < 3; i++) { 2518 if (imm_h[i] != 0L) { 2519 movz(dst, (uint32_t)imm_h[i], (i << 4)); 2520 i++; 2521 break; 2522 } 2523 } 2524 for (;i < 4; i++) { 2525 if (imm_h[i] != 0L) { 2526 movk(dst, (uint32_t)imm_h[i], (i << 4)); 2527 } 2528 } 2529 } else if (neg_count == 2) { 2530 // one MOVN and one MOVK will do 2531 for (i = 0; i < 4; i++) { 2532 if (imm_h[i] != 0xffffL) { 2533 movn(dst, (uint32_t)imm_h[i] ^ 0xffffL, (i << 4)); 2534 i++; 2535 break; 2536 } 2537 } 2538 for (;i < 4; i++) { 2539 if (imm_h[i] != 0xffffL) { 2540 movk(dst, (uint32_t)imm_h[i], (i << 4)); 2541 } 2542 } 2543 } else if (zero_count == 1) { 2544 // one MOVZ and two MOVKs will do 2545 for (i = 0; i < 4; i++) { 2546 if (imm_h[i] != 0L) { 2547 movz(dst, (uint32_t)imm_h[i], (i << 4)); 2548 i++; 2549 break; 2550 } 2551 } 2552 for (;i < 4; i++) { 2553 if (imm_h[i] != 0x0L) { 2554 movk(dst, (uint32_t)imm_h[i], (i << 4)); 2555 } 2556 } 2557 } else if (neg_count == 1) { 2558 // one MOVN and two MOVKs will do 2559 for (i = 0; i < 4; i++) { 2560 if (imm_h[i] != 0xffffL) { 2561 movn(dst, (uint32_t)imm_h[i] ^ 0xffffL, (i << 4)); 2562 i++; 2563 break; 2564 } 2565 } 2566 for (;i < 4; i++) { 2567 if (imm_h[i] != 0xffffL) { 2568 movk(dst, (uint32_t)imm_h[i], (i << 4)); 2569 } 2570 } 2571 } else { 2572 // use a MOVZ and 3 MOVKs (makes it easier to debug) 2573 movz(dst, (uint32_t)imm_h[0], 0); 2574 for (i = 1; i < 4; i++) { 2575 movk(dst, (uint32_t)imm_h[i], (i << 4)); 2576 } 2577 } 2578 } 2579 } 2580 2581 void MacroAssembler::mov_immediate32(Register dst, uint32_t imm32) 2582 { 2583 #ifndef PRODUCT 2584 { 2585 char buffer[64]; 2586 snprintf(buffer, sizeof(buffer), "0x%" PRIX32, imm32); 2587 block_comment(buffer); 2588 } 2589 #endif 2590 if (operand_valid_for_logical_immediate(true, imm32)) { 2591 orrw(dst, zr, imm32); 2592 } else { 2593 // we can use MOVZ, MOVN or two calls to MOVK to build up the 2594 // constant 2595 uint32_t imm_h[2]; 2596 imm_h[0] = imm32 & 0xffff; 2597 imm_h[1] = ((imm32 >> 16) & 0xffff); 2598 if (imm_h[0] == 0) { 2599 movzw(dst, imm_h[1], 16); 2600 } else if (imm_h[0] == 0xffff) { 2601 movnw(dst, imm_h[1] ^ 0xffff, 16); 2602 } else if (imm_h[1] == 0) { 2603 movzw(dst, imm_h[0], 0); 2604 } else if (imm_h[1] == 0xffff) { 2605 movnw(dst, imm_h[0] ^ 0xffff, 0); 2606 } else { 2607 // use a MOVZ and MOVK (makes it easier to debug) 2608 movzw(dst, imm_h[0], 0); 2609 movkw(dst, imm_h[1], 16); 2610 } 2611 } 2612 } 2613 2614 // Form an address from base + offset in Rd. Rd may or may 2615 // not actually be used: you must use the Address that is returned. 2616 // It is up to you to ensure that the shift provided matches the size 2617 // of your data. 2618 Address MacroAssembler::form_address(Register Rd, Register base, int64_t byte_offset, int shift) { 2619 if (Address::offset_ok_for_immed(byte_offset, shift)) 2620 // It fits; no need for any heroics 2621 return Address(base, byte_offset); 2622 2623 // Don't do anything clever with negative or misaligned offsets 2624 unsigned mask = (1 << shift) - 1; 2625 if (byte_offset < 0 || byte_offset & mask) { 2626 mov(Rd, byte_offset); 2627 add(Rd, base, Rd); 2628 return Address(Rd); 2629 } 2630 2631 // See if we can do this with two 12-bit offsets 2632 { 2633 uint64_t word_offset = byte_offset >> shift; 2634 uint64_t masked_offset = word_offset & 0xfff000; 2635 if (Address::offset_ok_for_immed(word_offset - masked_offset, 0) 2636 && Assembler::operand_valid_for_add_sub_immediate(masked_offset << shift)) { 2637 add(Rd, base, masked_offset << shift); 2638 word_offset -= masked_offset; 2639 return Address(Rd, word_offset << shift); 2640 } 2641 } 2642 2643 // Do it the hard way 2644 mov(Rd, byte_offset); 2645 add(Rd, base, Rd); 2646 return Address(Rd); 2647 } 2648 2649 int MacroAssembler::corrected_idivl(Register result, Register ra, Register rb, 2650 bool want_remainder, Register scratch) 2651 { 2652 // Full implementation of Java idiv and irem. The function 2653 // returns the (pc) offset of the div instruction - may be needed 2654 // for implicit exceptions. 2655 // 2656 // constraint : ra/rb =/= scratch 2657 // normal case 2658 // 2659 // input : ra: dividend 2660 // rb: divisor 2661 // 2662 // result: either 2663 // quotient (= ra idiv rb) 2664 // remainder (= ra irem rb) 2665 2666 assert(ra != scratch && rb != scratch, "reg cannot be scratch"); 2667 2668 int idivl_offset = offset(); 2669 if (! want_remainder) { 2670 sdivw(result, ra, rb); 2671 } else { 2672 sdivw(scratch, ra, rb); 2673 Assembler::msubw(result, scratch, rb, ra); 2674 } 2675 2676 return idivl_offset; 2677 } 2678 2679 int MacroAssembler::corrected_idivq(Register result, Register ra, Register rb, 2680 bool want_remainder, Register scratch) 2681 { 2682 // Full implementation of Java ldiv and lrem. The function 2683 // returns the (pc) offset of the div instruction - may be needed 2684 // for implicit exceptions. 2685 // 2686 // constraint : ra/rb =/= scratch 2687 // normal case 2688 // 2689 // input : ra: dividend 2690 // rb: divisor 2691 // 2692 // result: either 2693 // quotient (= ra idiv rb) 2694 // remainder (= ra irem rb) 2695 2696 assert(ra != scratch && rb != scratch, "reg cannot be scratch"); 2697 2698 int idivq_offset = offset(); 2699 if (! want_remainder) { 2700 sdiv(result, ra, rb); 2701 } else { 2702 sdiv(scratch, ra, rb); 2703 Assembler::msub(result, scratch, rb, ra); 2704 } 2705 2706 return idivq_offset; 2707 } 2708 2709 void MacroAssembler::membar(Membar_mask_bits order_constraint) { 2710 address prev = pc() - NativeMembar::instruction_size; 2711 address last = code()->last_insn(); 2712 if (last != nullptr && nativeInstruction_at(last)->is_Membar() && prev == last) { 2713 NativeMembar *bar = NativeMembar_at(prev); 2714 if (AlwaysMergeDMB) { 2715 bar->set_kind(bar->get_kind() | order_constraint); 2716 BLOCK_COMMENT("merged membar(always)"); 2717 return; 2718 } 2719 // Don't promote DMB ST|DMB LD to DMB (a full barrier) because 2720 // doing so would introduce a StoreLoad which the caller did not 2721 // intend 2722 if (bar->get_kind() == order_constraint 2723 || bar->get_kind() == AnyAny 2724 || order_constraint == AnyAny) { 2725 // We are merging two memory barrier instructions. On AArch64 we 2726 // can do this simply by ORing them together. 2727 bar->set_kind(bar->get_kind() | order_constraint); 2728 BLOCK_COMMENT("merged membar"); 2729 return; 2730 } else { 2731 // A special case like "DMB ST;DMB LD;DMB ST", the last DMB can be skipped 2732 // We need check the last 2 instructions 2733 address prev2 = prev - NativeMembar::instruction_size; 2734 if (last != code()->last_label() && nativeInstruction_at(prev2)->is_Membar()) { 2735 NativeMembar *bar2 = NativeMembar_at(prev2); 2736 assert(bar2->get_kind() == order_constraint, "it should be merged before"); 2737 BLOCK_COMMENT("merged membar(elided)"); 2738 return; 2739 } 2740 } 2741 } 2742 code()->set_last_insn(pc()); 2743 dmb(Assembler::barrier(order_constraint)); 2744 } 2745 2746 bool MacroAssembler::try_merge_ldst(Register rt, const Address &adr, size_t size_in_bytes, bool is_store) { 2747 if (ldst_can_merge(rt, adr, size_in_bytes, is_store)) { 2748 merge_ldst(rt, adr, size_in_bytes, is_store); 2749 code()->clear_last_insn(); 2750 return true; 2751 } else { 2752 assert(size_in_bytes == 8 || size_in_bytes == 4, "only 8 bytes or 4 bytes load/store is supported."); 2753 const uint64_t mask = size_in_bytes - 1; 2754 if (adr.getMode() == Address::base_plus_offset && 2755 (adr.offset() & mask) == 0) { // only supports base_plus_offset. 2756 code()->set_last_insn(pc()); 2757 } 2758 return false; 2759 } 2760 } 2761 2762 void MacroAssembler::ldr(Register Rx, const Address &adr) { 2763 // We always try to merge two adjacent loads into one ldp. 2764 if (!try_merge_ldst(Rx, adr, 8, false)) { 2765 Assembler::ldr(Rx, adr); 2766 } 2767 } 2768 2769 void MacroAssembler::ldrw(Register Rw, const Address &adr) { 2770 // We always try to merge two adjacent loads into one ldp. 2771 if (!try_merge_ldst(Rw, adr, 4, false)) { 2772 Assembler::ldrw(Rw, adr); 2773 } 2774 } 2775 2776 void MacroAssembler::str(Register Rx, const Address &adr) { 2777 // We always try to merge two adjacent stores into one stp. 2778 if (!try_merge_ldst(Rx, adr, 8, true)) { 2779 Assembler::str(Rx, adr); 2780 } 2781 } 2782 2783 void MacroAssembler::strw(Register Rw, const Address &adr) { 2784 // We always try to merge two adjacent stores into one stp. 2785 if (!try_merge_ldst(Rw, adr, 4, true)) { 2786 Assembler::strw(Rw, adr); 2787 } 2788 } 2789 2790 // MacroAssembler routines found actually to be needed 2791 2792 void MacroAssembler::push(Register src) 2793 { 2794 str(src, Address(pre(esp, -1 * wordSize))); 2795 } 2796 2797 void MacroAssembler::pop(Register dst) 2798 { 2799 ldr(dst, Address(post(esp, 1 * wordSize))); 2800 } 2801 2802 // Note: load_unsigned_short used to be called load_unsigned_word. 2803 int MacroAssembler::load_unsigned_short(Register dst, Address src) { 2804 int off = offset(); 2805 ldrh(dst, src); 2806 return off; 2807 } 2808 2809 int MacroAssembler::load_unsigned_byte(Register dst, Address src) { 2810 int off = offset(); 2811 ldrb(dst, src); 2812 return off; 2813 } 2814 2815 int MacroAssembler::load_signed_short(Register dst, Address src) { 2816 int off = offset(); 2817 ldrsh(dst, src); 2818 return off; 2819 } 2820 2821 int MacroAssembler::load_signed_byte(Register dst, Address src) { 2822 int off = offset(); 2823 ldrsb(dst, src); 2824 return off; 2825 } 2826 2827 int MacroAssembler::load_signed_short32(Register dst, Address src) { 2828 int off = offset(); 2829 ldrshw(dst, src); 2830 return off; 2831 } 2832 2833 int MacroAssembler::load_signed_byte32(Register dst, Address src) { 2834 int off = offset(); 2835 ldrsbw(dst, src); 2836 return off; 2837 } 2838 2839 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed) { 2840 switch (size_in_bytes) { 2841 case 8: ldr(dst, src); break; 2842 case 4: ldrw(dst, src); break; 2843 case 2: is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break; 2844 case 1: is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break; 2845 default: ShouldNotReachHere(); 2846 } 2847 } 2848 2849 void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes) { 2850 switch (size_in_bytes) { 2851 case 8: str(src, dst); break; 2852 case 4: strw(src, dst); break; 2853 case 2: strh(src, dst); break; 2854 case 1: strb(src, dst); break; 2855 default: ShouldNotReachHere(); 2856 } 2857 } 2858 2859 void MacroAssembler::decrementw(Register reg, int value) 2860 { 2861 if (value < 0) { incrementw(reg, -value); return; } 2862 if (value == 0) { return; } 2863 if (value < (1 << 12)) { subw(reg, reg, value); return; } 2864 /* else */ { 2865 guarantee(reg != rscratch2, "invalid dst for register decrement"); 2866 movw(rscratch2, (unsigned)value); 2867 subw(reg, reg, rscratch2); 2868 } 2869 } 2870 2871 void MacroAssembler::decrement(Register reg, int value) 2872 { 2873 if (value < 0) { increment(reg, -value); return; } 2874 if (value == 0) { return; } 2875 if (value < (1 << 12)) { sub(reg, reg, value); return; } 2876 /* else */ { 2877 assert(reg != rscratch2, "invalid dst for register decrement"); 2878 mov(rscratch2, (uint64_t)value); 2879 sub(reg, reg, rscratch2); 2880 } 2881 } 2882 2883 void MacroAssembler::decrementw(Address dst, int value) 2884 { 2885 assert(!dst.uses(rscratch1), "invalid dst for address decrement"); 2886 if (dst.getMode() == Address::literal) { 2887 assert(abs(value) < (1 << 12), "invalid value and address mode combination"); 2888 lea(rscratch2, dst); 2889 dst = Address(rscratch2); 2890 } 2891 ldrw(rscratch1, dst); 2892 decrementw(rscratch1, value); 2893 strw(rscratch1, dst); 2894 } 2895 2896 void MacroAssembler::decrement(Address dst, int value) 2897 { 2898 assert(!dst.uses(rscratch1), "invalid address for decrement"); 2899 if (dst.getMode() == Address::literal) { 2900 assert(abs(value) < (1 << 12), "invalid value and address mode combination"); 2901 lea(rscratch2, dst); 2902 dst = Address(rscratch2); 2903 } 2904 ldr(rscratch1, dst); 2905 decrement(rscratch1, value); 2906 str(rscratch1, dst); 2907 } 2908 2909 void MacroAssembler::incrementw(Register reg, int value) 2910 { 2911 if (value < 0) { decrementw(reg, -value); return; } 2912 if (value == 0) { return; } 2913 if (value < (1 << 12)) { addw(reg, reg, value); return; } 2914 /* else */ { 2915 assert(reg != rscratch2, "invalid dst for register increment"); 2916 movw(rscratch2, (unsigned)value); 2917 addw(reg, reg, rscratch2); 2918 } 2919 } 2920 2921 void MacroAssembler::increment(Register reg, int value) 2922 { 2923 if (value < 0) { decrement(reg, -value); return; } 2924 if (value == 0) { return; } 2925 if (value < (1 << 12)) { add(reg, reg, value); return; } 2926 /* else */ { 2927 assert(reg != rscratch2, "invalid dst for register increment"); 2928 movw(rscratch2, (unsigned)value); 2929 add(reg, reg, rscratch2); 2930 } 2931 } 2932 2933 void MacroAssembler::incrementw(Address dst, int value) 2934 { 2935 assert(!dst.uses(rscratch1), "invalid dst for address increment"); 2936 if (dst.getMode() == Address::literal) { 2937 assert(abs(value) < (1 << 12), "invalid value and address mode combination"); 2938 lea(rscratch2, dst); 2939 dst = Address(rscratch2); 2940 } 2941 ldrw(rscratch1, dst); 2942 incrementw(rscratch1, value); 2943 strw(rscratch1, dst); 2944 } 2945 2946 void MacroAssembler::increment(Address dst, int value) 2947 { 2948 assert(!dst.uses(rscratch1), "invalid dst for address increment"); 2949 if (dst.getMode() == Address::literal) { 2950 assert(abs(value) < (1 << 12), "invalid value and address mode combination"); 2951 lea(rscratch2, dst); 2952 dst = Address(rscratch2); 2953 } 2954 ldr(rscratch1, dst); 2955 increment(rscratch1, value); 2956 str(rscratch1, dst); 2957 } 2958 2959 // Push lots of registers in the bit set supplied. Don't push sp. 2960 // Return the number of words pushed 2961 int MacroAssembler::push(unsigned int bitset, Register stack) { 2962 int words_pushed = 0; 2963 2964 // Scan bitset to accumulate register pairs 2965 unsigned char regs[32]; 2966 int count = 0; 2967 for (int reg = 0; reg <= 30; reg++) { 2968 if (1 & bitset) 2969 regs[count++] = reg; 2970 bitset >>= 1; 2971 } 2972 regs[count++] = zr->raw_encoding(); 2973 count &= ~1; // Only push an even number of regs 2974 2975 if (count) { 2976 stp(as_Register(regs[0]), as_Register(regs[1]), 2977 Address(pre(stack, -count * wordSize))); 2978 words_pushed += 2; 2979 } 2980 for (int i = 2; i < count; i += 2) { 2981 stp(as_Register(regs[i]), as_Register(regs[i+1]), 2982 Address(stack, i * wordSize)); 2983 words_pushed += 2; 2984 } 2985 2986 assert(words_pushed == count, "oops, pushed != count"); 2987 2988 return count; 2989 } 2990 2991 int MacroAssembler::pop(unsigned int bitset, Register stack) { 2992 int words_pushed = 0; 2993 2994 // Scan bitset to accumulate register pairs 2995 unsigned char regs[32]; 2996 int count = 0; 2997 for (int reg = 0; reg <= 30; reg++) { 2998 if (1 & bitset) 2999 regs[count++] = reg; 3000 bitset >>= 1; 3001 } 3002 regs[count++] = zr->raw_encoding(); 3003 count &= ~1; 3004 3005 for (int i = 2; i < count; i += 2) { 3006 ldp(as_Register(regs[i]), as_Register(regs[i+1]), 3007 Address(stack, i * wordSize)); 3008 words_pushed += 2; 3009 } 3010 if (count) { 3011 ldp(as_Register(regs[0]), as_Register(regs[1]), 3012 Address(post(stack, count * wordSize))); 3013 words_pushed += 2; 3014 } 3015 3016 assert(words_pushed == count, "oops, pushed != count"); 3017 3018 return count; 3019 } 3020 3021 // Push lots of registers in the bit set supplied. Don't push sp. 3022 // Return the number of dwords pushed 3023 int MacroAssembler::push_fp(unsigned int bitset, Register stack, FpPushPopMode mode) { 3024 int words_pushed = 0; 3025 bool use_sve = false; 3026 int sve_vector_size_in_bytes = 0; 3027 3028 #ifdef COMPILER2 3029 use_sve = Matcher::supports_scalable_vector(); 3030 sve_vector_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE); 3031 #endif 3032 3033 // Scan bitset to accumulate register pairs 3034 unsigned char regs[32]; 3035 int count = 0; 3036 for (int reg = 0; reg <= 31; reg++) { 3037 if (1 & bitset) 3038 regs[count++] = reg; 3039 bitset >>= 1; 3040 } 3041 3042 if (count == 0) { 3043 return 0; 3044 } 3045 3046 if (mode == PushPopFull) { 3047 if (use_sve && sve_vector_size_in_bytes > 16) { 3048 mode = PushPopSVE; 3049 } else { 3050 mode = PushPopNeon; 3051 } 3052 } 3053 3054 #ifndef PRODUCT 3055 { 3056 char buffer[48]; 3057 if (mode == PushPopSVE) { 3058 snprintf(buffer, sizeof(buffer), "push_fp: %d SVE registers", count); 3059 } else if (mode == PushPopNeon) { 3060 snprintf(buffer, sizeof(buffer), "push_fp: %d Neon registers", count); 3061 } else { 3062 snprintf(buffer, sizeof(buffer), "push_fp: %d fp registers", count); 3063 } 3064 block_comment(buffer); 3065 } 3066 #endif 3067 3068 if (mode == PushPopSVE) { 3069 sub(stack, stack, sve_vector_size_in_bytes * count); 3070 for (int i = 0; i < count; i++) { 3071 sve_str(as_FloatRegister(regs[i]), Address(stack, i)); 3072 } 3073 return count * sve_vector_size_in_bytes / 8; 3074 } 3075 3076 if (mode == PushPopNeon) { 3077 if (count == 1) { 3078 strq(as_FloatRegister(regs[0]), Address(pre(stack, -wordSize * 2))); 3079 return 2; 3080 } 3081 3082 bool odd = (count & 1) == 1; 3083 int push_slots = count + (odd ? 1 : 0); 3084 3085 // Always pushing full 128 bit registers. 3086 stpq(as_FloatRegister(regs[0]), as_FloatRegister(regs[1]), Address(pre(stack, -push_slots * wordSize * 2))); 3087 words_pushed += 2; 3088 3089 for (int i = 2; i + 1 < count; i += 2) { 3090 stpq(as_FloatRegister(regs[i]), as_FloatRegister(regs[i+1]), Address(stack, i * wordSize * 2)); 3091 words_pushed += 2; 3092 } 3093 3094 if (odd) { 3095 strq(as_FloatRegister(regs[count - 1]), Address(stack, (count - 1) * wordSize * 2)); 3096 words_pushed++; 3097 } 3098 3099 assert(words_pushed == count, "oops, pushed(%d) != count(%d)", words_pushed, count); 3100 return count * 2; 3101 } 3102 3103 if (mode == PushPopFp) { 3104 bool odd = (count & 1) == 1; 3105 int push_slots = count + (odd ? 1 : 0); 3106 3107 if (count == 1) { 3108 // Stack pointer must be 16 bytes aligned 3109 strd(as_FloatRegister(regs[0]), Address(pre(stack, -push_slots * wordSize))); 3110 return 1; 3111 } 3112 3113 stpd(as_FloatRegister(regs[0]), as_FloatRegister(regs[1]), Address(pre(stack, -push_slots * wordSize))); 3114 words_pushed += 2; 3115 3116 for (int i = 2; i + 1 < count; i += 2) { 3117 stpd(as_FloatRegister(regs[i]), as_FloatRegister(regs[i+1]), Address(stack, i * wordSize)); 3118 words_pushed += 2; 3119 } 3120 3121 if (odd) { 3122 // Stack pointer must be 16 bytes aligned 3123 strd(as_FloatRegister(regs[count - 1]), Address(stack, (count - 1) * wordSize)); 3124 words_pushed++; 3125 } 3126 3127 assert(words_pushed == count, "oops, pushed != count"); 3128 3129 return count; 3130 } 3131 3132 return 0; 3133 } 3134 3135 // Return the number of dwords popped 3136 int MacroAssembler::pop_fp(unsigned int bitset, Register stack, FpPushPopMode mode) { 3137 int words_pushed = 0; 3138 bool use_sve = false; 3139 int sve_vector_size_in_bytes = 0; 3140 3141 #ifdef COMPILER2 3142 use_sve = Matcher::supports_scalable_vector(); 3143 sve_vector_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE); 3144 #endif 3145 // Scan bitset to accumulate register pairs 3146 unsigned char regs[32]; 3147 int count = 0; 3148 for (int reg = 0; reg <= 31; reg++) { 3149 if (1 & bitset) 3150 regs[count++] = reg; 3151 bitset >>= 1; 3152 } 3153 3154 if (count == 0) { 3155 return 0; 3156 } 3157 3158 if (mode == PushPopFull) { 3159 if (use_sve && sve_vector_size_in_bytes > 16) { 3160 mode = PushPopSVE; 3161 } else { 3162 mode = PushPopNeon; 3163 } 3164 } 3165 3166 #ifndef PRODUCT 3167 { 3168 char buffer[48]; 3169 if (mode == PushPopSVE) { 3170 snprintf(buffer, sizeof(buffer), "pop_fp: %d SVE registers", count); 3171 } else if (mode == PushPopNeon) { 3172 snprintf(buffer, sizeof(buffer), "pop_fp: %d Neon registers", count); 3173 } else { 3174 snprintf(buffer, sizeof(buffer), "pop_fp: %d fp registers", count); 3175 } 3176 block_comment(buffer); 3177 } 3178 #endif 3179 3180 if (mode == PushPopSVE) { 3181 for (int i = count - 1; i >= 0; i--) { 3182 sve_ldr(as_FloatRegister(regs[i]), Address(stack, i)); 3183 } 3184 add(stack, stack, sve_vector_size_in_bytes * count); 3185 return count * sve_vector_size_in_bytes / 8; 3186 } 3187 3188 if (mode == PushPopNeon) { 3189 if (count == 1) { 3190 ldrq(as_FloatRegister(regs[0]), Address(post(stack, wordSize * 2))); 3191 return 2; 3192 } 3193 3194 bool odd = (count & 1) == 1; 3195 int push_slots = count + (odd ? 1 : 0); 3196 3197 if (odd) { 3198 ldrq(as_FloatRegister(regs[count - 1]), Address(stack, (count - 1) * wordSize * 2)); 3199 words_pushed++; 3200 } 3201 3202 for (int i = 2; i + 1 < count; i += 2) { 3203 ldpq(as_FloatRegister(regs[i]), as_FloatRegister(regs[i+1]), Address(stack, i * wordSize * 2)); 3204 words_pushed += 2; 3205 } 3206 3207 ldpq(as_FloatRegister(regs[0]), as_FloatRegister(regs[1]), Address(post(stack, push_slots * wordSize * 2))); 3208 words_pushed += 2; 3209 3210 assert(words_pushed == count, "oops, pushed(%d) != count(%d)", words_pushed, count); 3211 3212 return count * 2; 3213 } 3214 3215 if (mode == PushPopFp) { 3216 bool odd = (count & 1) == 1; 3217 int push_slots = count + (odd ? 1 : 0); 3218 3219 if (count == 1) { 3220 ldrd(as_FloatRegister(regs[0]), Address(post(stack, push_slots * wordSize))); 3221 return 1; 3222 } 3223 3224 if (odd) { 3225 ldrd(as_FloatRegister(regs[count - 1]), Address(stack, (count - 1) * wordSize)); 3226 words_pushed++; 3227 } 3228 3229 for (int i = 2; i + 1 < count; i += 2) { 3230 ldpd(as_FloatRegister(regs[i]), as_FloatRegister(regs[i+1]), Address(stack, i * wordSize)); 3231 words_pushed += 2; 3232 } 3233 3234 ldpd(as_FloatRegister(regs[0]), as_FloatRegister(regs[1]), Address(post(stack, push_slots * wordSize))); 3235 words_pushed += 2; 3236 3237 assert(words_pushed == count, "oops, pushed != count"); 3238 3239 return count; 3240 } 3241 3242 return 0; 3243 } 3244 3245 // Return the number of dwords pushed 3246 int MacroAssembler::push_p(unsigned int bitset, Register stack) { 3247 bool use_sve = false; 3248 int sve_predicate_size_in_slots = 0; 3249 3250 #ifdef COMPILER2 3251 use_sve = Matcher::supports_scalable_vector(); 3252 if (use_sve) { 3253 sve_predicate_size_in_slots = Matcher::scalable_predicate_reg_slots(); 3254 } 3255 #endif 3256 3257 if (!use_sve) { 3258 return 0; 3259 } 3260 3261 unsigned char regs[PRegister::number_of_registers]; 3262 int count = 0; 3263 for (int reg = 0; reg < PRegister::number_of_registers; reg++) { 3264 if (1 & bitset) 3265 regs[count++] = reg; 3266 bitset >>= 1; 3267 } 3268 3269 if (count == 0) { 3270 return 0; 3271 } 3272 3273 int total_push_bytes = align_up(sve_predicate_size_in_slots * 3274 VMRegImpl::stack_slot_size * count, 16); 3275 sub(stack, stack, total_push_bytes); 3276 for (int i = 0; i < count; i++) { 3277 sve_str(as_PRegister(regs[i]), Address(stack, i)); 3278 } 3279 return total_push_bytes / 8; 3280 } 3281 3282 // Return the number of dwords popped 3283 int MacroAssembler::pop_p(unsigned int bitset, Register stack) { 3284 bool use_sve = false; 3285 int sve_predicate_size_in_slots = 0; 3286 3287 #ifdef COMPILER2 3288 use_sve = Matcher::supports_scalable_vector(); 3289 if (use_sve) { 3290 sve_predicate_size_in_slots = Matcher::scalable_predicate_reg_slots(); 3291 } 3292 #endif 3293 3294 if (!use_sve) { 3295 return 0; 3296 } 3297 3298 unsigned char regs[PRegister::number_of_registers]; 3299 int count = 0; 3300 for (int reg = 0; reg < PRegister::number_of_registers; reg++) { 3301 if (1 & bitset) 3302 regs[count++] = reg; 3303 bitset >>= 1; 3304 } 3305 3306 if (count == 0) { 3307 return 0; 3308 } 3309 3310 int total_pop_bytes = align_up(sve_predicate_size_in_slots * 3311 VMRegImpl::stack_slot_size * count, 16); 3312 for (int i = count - 1; i >= 0; i--) { 3313 sve_ldr(as_PRegister(regs[i]), Address(stack, i)); 3314 } 3315 add(stack, stack, total_pop_bytes); 3316 return total_pop_bytes / 8; 3317 } 3318 3319 #ifdef ASSERT 3320 void MacroAssembler::verify_heapbase(const char* msg) { 3321 #if 0 3322 assert (UseCompressedOops || UseCompressedClassPointers, "should be compressed"); 3323 assert (Universe::heap() != nullptr, "java heap should be initialized"); 3324 if (!UseCompressedOops || Universe::ptr_base() == nullptr) { 3325 // rheapbase is allocated as general register 3326 return; 3327 } 3328 if (CheckCompressedOops) { 3329 Label ok; 3330 push(1 << rscratch1->encoding(), sp); // cmpptr trashes rscratch1 3331 cmpptr(rheapbase, ExternalAddress(CompressedOops::base_addr())); 3332 br(Assembler::EQ, ok); 3333 stop(msg); 3334 bind(ok); 3335 pop(1 << rscratch1->encoding(), sp); 3336 } 3337 #endif 3338 } 3339 #endif 3340 3341 void MacroAssembler::resolve_jobject(Register value, Register tmp1, Register tmp2) { 3342 assert_different_registers(value, tmp1, tmp2); 3343 Label done, tagged, weak_tagged; 3344 3345 cbz(value, done); // Use null as-is. 3346 tst(value, JNIHandles::tag_mask); // Test for tag. 3347 br(Assembler::NE, tagged); 3348 3349 // Resolve local handle 3350 access_load_at(T_OBJECT, IN_NATIVE | AS_RAW, value, Address(value, 0), tmp1, tmp2); 3351 verify_oop(value); 3352 b(done); 3353 3354 bind(tagged); 3355 STATIC_ASSERT(JNIHandles::TypeTag::weak_global == 0b1); 3356 tbnz(value, 0, weak_tagged); // Test for weak tag. 3357 3358 // Resolve global handle 3359 access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp1, tmp2); 3360 verify_oop(value); 3361 b(done); 3362 3363 bind(weak_tagged); 3364 // Resolve jweak. 3365 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, 3366 value, Address(value, -JNIHandles::TypeTag::weak_global), tmp1, tmp2); 3367 verify_oop(value); 3368 3369 bind(done); 3370 } 3371 3372 void MacroAssembler::resolve_global_jobject(Register value, Register tmp1, Register tmp2) { 3373 assert_different_registers(value, tmp1, tmp2); 3374 Label done; 3375 3376 cbz(value, done); // Use null as-is. 3377 3378 #ifdef ASSERT 3379 { 3380 STATIC_ASSERT(JNIHandles::TypeTag::global == 0b10); 3381 Label valid_global_tag; 3382 tbnz(value, 1, valid_global_tag); // Test for global tag 3383 stop("non global jobject using resolve_global_jobject"); 3384 bind(valid_global_tag); 3385 } 3386 #endif 3387 3388 // Resolve global handle 3389 access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp1, tmp2); 3390 verify_oop(value); 3391 3392 bind(done); 3393 } 3394 3395 void MacroAssembler::stop(const char* msg) { 3396 BLOCK_COMMENT(msg); 3397 dcps1(0xdeae); 3398 emit_int64((uintptr_t)msg); 3399 } 3400 3401 void MacroAssembler::unimplemented(const char* what) { 3402 const char* buf = nullptr; 3403 { 3404 ResourceMark rm; 3405 stringStream ss; 3406 ss.print("unimplemented: %s", what); 3407 buf = code_string(ss.as_string()); 3408 } 3409 stop(buf); 3410 } 3411 3412 void MacroAssembler::_assert_asm(Assembler::Condition cc, const char* msg) { 3413 #ifdef ASSERT 3414 Label OK; 3415 br(cc, OK); 3416 stop(msg); 3417 bind(OK); 3418 #endif 3419 } 3420 3421 // If a constant does not fit in an immediate field, generate some 3422 // number of MOV instructions and then perform the operation. 3423 void MacroAssembler::wrap_add_sub_imm_insn(Register Rd, Register Rn, uint64_t imm, 3424 add_sub_imm_insn insn1, 3425 add_sub_reg_insn insn2, 3426 bool is32) { 3427 assert(Rd != zr, "Rd = zr and not setting flags?"); 3428 bool fits = operand_valid_for_add_sub_immediate(is32 ? (int32_t)imm : imm); 3429 if (fits) { 3430 (this->*insn1)(Rd, Rn, imm); 3431 } else { 3432 if (uabs(imm) < (1 << 24)) { 3433 (this->*insn1)(Rd, Rn, imm & -(1 << 12)); 3434 (this->*insn1)(Rd, Rd, imm & ((1 << 12)-1)); 3435 } else { 3436 assert_different_registers(Rd, Rn); 3437 mov(Rd, imm); 3438 (this->*insn2)(Rd, Rn, Rd, LSL, 0); 3439 } 3440 } 3441 } 3442 3443 // Separate vsn which sets the flags. Optimisations are more restricted 3444 // because we must set the flags correctly. 3445 void MacroAssembler::wrap_adds_subs_imm_insn(Register Rd, Register Rn, uint64_t imm, 3446 add_sub_imm_insn insn1, 3447 add_sub_reg_insn insn2, 3448 bool is32) { 3449 bool fits = operand_valid_for_add_sub_immediate(is32 ? (int32_t)imm : imm); 3450 if (fits) { 3451 (this->*insn1)(Rd, Rn, imm); 3452 } else { 3453 assert_different_registers(Rd, Rn); 3454 assert(Rd != zr, "overflow in immediate operand"); 3455 mov(Rd, imm); 3456 (this->*insn2)(Rd, Rn, Rd, LSL, 0); 3457 } 3458 } 3459 3460 3461 void MacroAssembler::add(Register Rd, Register Rn, RegisterOrConstant increment) { 3462 if (increment.is_register()) { 3463 add(Rd, Rn, increment.as_register()); 3464 } else { 3465 add(Rd, Rn, increment.as_constant()); 3466 } 3467 } 3468 3469 void MacroAssembler::addw(Register Rd, Register Rn, RegisterOrConstant increment) { 3470 if (increment.is_register()) { 3471 addw(Rd, Rn, increment.as_register()); 3472 } else { 3473 addw(Rd, Rn, increment.as_constant()); 3474 } 3475 } 3476 3477 void MacroAssembler::sub(Register Rd, Register Rn, RegisterOrConstant decrement) { 3478 if (decrement.is_register()) { 3479 sub(Rd, Rn, decrement.as_register()); 3480 } else { 3481 sub(Rd, Rn, decrement.as_constant()); 3482 } 3483 } 3484 3485 void MacroAssembler::subw(Register Rd, Register Rn, RegisterOrConstant decrement) { 3486 if (decrement.is_register()) { 3487 subw(Rd, Rn, decrement.as_register()); 3488 } else { 3489 subw(Rd, Rn, decrement.as_constant()); 3490 } 3491 } 3492 3493 void MacroAssembler::reinit_heapbase() 3494 { 3495 if (UseCompressedOops) { 3496 if (Universe::is_fully_initialized()) { 3497 mov(rheapbase, CompressedOops::base()); 3498 } else { 3499 lea(rheapbase, ExternalAddress(CompressedOops::base_addr())); 3500 ldr(rheapbase, Address(rheapbase)); 3501 } 3502 } 3503 } 3504 3505 // this simulates the behaviour of the x86 cmpxchg instruction using a 3506 // load linked/store conditional pair. we use the acquire/release 3507 // versions of these instructions so that we flush pending writes as 3508 // per Java semantics. 3509 3510 // n.b the x86 version assumes the old value to be compared against is 3511 // in rax and updates rax with the value located in memory if the 3512 // cmpxchg fails. we supply a register for the old value explicitly 3513 3514 // the aarch64 load linked/store conditional instructions do not 3515 // accept an offset. so, unlike x86, we must provide a plain register 3516 // to identify the memory word to be compared/exchanged rather than a 3517 // register+offset Address. 3518 3519 void MacroAssembler::cmpxchgptr(Register oldv, Register newv, Register addr, Register tmp, 3520 Label &succeed, Label *fail) { 3521 // oldv holds comparison value 3522 // newv holds value to write in exchange 3523 // addr identifies memory word to compare against/update 3524 if (UseLSE) { 3525 mov(tmp, oldv); 3526 casal(Assembler::xword, oldv, newv, addr); 3527 cmp(tmp, oldv); 3528 br(Assembler::EQ, succeed); 3529 membar(AnyAny); 3530 } else { 3531 Label retry_load, nope; 3532 prfm(Address(addr), PSTL1STRM); 3533 bind(retry_load); 3534 // flush and load exclusive from the memory location 3535 // and fail if it is not what we expect 3536 ldaxr(tmp, addr); 3537 cmp(tmp, oldv); 3538 br(Assembler::NE, nope); 3539 // if we store+flush with no intervening write tmp will be zero 3540 stlxr(tmp, newv, addr); 3541 cbzw(tmp, succeed); 3542 // retry so we only ever return after a load fails to compare 3543 // ensures we don't return a stale value after a failed write. 3544 b(retry_load); 3545 // if the memory word differs we return it in oldv and signal a fail 3546 bind(nope); 3547 membar(AnyAny); 3548 mov(oldv, tmp); 3549 } 3550 if (fail) 3551 b(*fail); 3552 } 3553 3554 void MacroAssembler::cmpxchg_obj_header(Register oldv, Register newv, Register obj, Register tmp, 3555 Label &succeed, Label *fail) { 3556 assert(oopDesc::mark_offset_in_bytes() == 0, "assumption"); 3557 cmpxchgptr(oldv, newv, obj, tmp, succeed, fail); 3558 } 3559 3560 void MacroAssembler::cmpxchgw(Register oldv, Register newv, Register addr, Register tmp, 3561 Label &succeed, Label *fail) { 3562 // oldv holds comparison value 3563 // newv holds value to write in exchange 3564 // addr identifies memory word to compare against/update 3565 // tmp returns 0/1 for success/failure 3566 if (UseLSE) { 3567 mov(tmp, oldv); 3568 casal(Assembler::word, oldv, newv, addr); 3569 cmp(tmp, oldv); 3570 br(Assembler::EQ, succeed); 3571 membar(AnyAny); 3572 } else { 3573 Label retry_load, nope; 3574 prfm(Address(addr), PSTL1STRM); 3575 bind(retry_load); 3576 // flush and load exclusive from the memory location 3577 // and fail if it is not what we expect 3578 ldaxrw(tmp, addr); 3579 cmp(tmp, oldv); 3580 br(Assembler::NE, nope); 3581 // if we store+flush with no intervening write tmp will be zero 3582 stlxrw(tmp, newv, addr); 3583 cbzw(tmp, succeed); 3584 // retry so we only ever return after a load fails to compare 3585 // ensures we don't return a stale value after a failed write. 3586 b(retry_load); 3587 // if the memory word differs we return it in oldv and signal a fail 3588 bind(nope); 3589 membar(AnyAny); 3590 mov(oldv, tmp); 3591 } 3592 if (fail) 3593 b(*fail); 3594 } 3595 3596 // A generic CAS; success or failure is in the EQ flag. A weak CAS 3597 // doesn't retry and may fail spuriously. If the oldval is wanted, 3598 // Pass a register for the result, otherwise pass noreg. 3599 3600 // Clobbers rscratch1 3601 void MacroAssembler::cmpxchg(Register addr, Register expected, 3602 Register new_val, 3603 enum operand_size size, 3604 bool acquire, bool release, 3605 bool weak, 3606 Register result) { 3607 if (result == noreg) result = rscratch1; 3608 BLOCK_COMMENT("cmpxchg {"); 3609 if (UseLSE) { 3610 mov(result, expected); 3611 lse_cas(result, new_val, addr, size, acquire, release, /*not_pair*/ true); 3612 compare_eq(result, expected, size); 3613 #ifdef ASSERT 3614 // Poison rscratch1 which is written on !UseLSE branch 3615 mov(rscratch1, 0x1f1f1f1f1f1f1f1f); 3616 #endif 3617 } else { 3618 Label retry_load, done; 3619 prfm(Address(addr), PSTL1STRM); 3620 bind(retry_load); 3621 load_exclusive(result, addr, size, acquire); 3622 compare_eq(result, expected, size); 3623 br(Assembler::NE, done); 3624 store_exclusive(rscratch1, new_val, addr, size, release); 3625 if (weak) { 3626 cmpw(rscratch1, 0u); // If the store fails, return NE to our caller. 3627 } else { 3628 cbnzw(rscratch1, retry_load); 3629 } 3630 bind(done); 3631 } 3632 BLOCK_COMMENT("} cmpxchg"); 3633 } 3634 3635 // A generic comparison. Only compares for equality, clobbers rscratch1. 3636 void MacroAssembler::compare_eq(Register rm, Register rn, enum operand_size size) { 3637 if (size == xword) { 3638 cmp(rm, rn); 3639 } else if (size == word) { 3640 cmpw(rm, rn); 3641 } else if (size == halfword) { 3642 eorw(rscratch1, rm, rn); 3643 ands(zr, rscratch1, 0xffff); 3644 } else if (size == byte) { 3645 eorw(rscratch1, rm, rn); 3646 ands(zr, rscratch1, 0xff); 3647 } else { 3648 ShouldNotReachHere(); 3649 } 3650 } 3651 3652 3653 static bool different(Register a, RegisterOrConstant b, Register c) { 3654 if (b.is_constant()) 3655 return a != c; 3656 else 3657 return a != b.as_register() && a != c && b.as_register() != c; 3658 } 3659 3660 #define ATOMIC_OP(NAME, LDXR, OP, IOP, AOP, STXR, sz) \ 3661 void MacroAssembler::atomic_##NAME(Register prev, RegisterOrConstant incr, Register addr) { \ 3662 if (UseLSE) { \ 3663 prev = prev->is_valid() ? prev : zr; \ 3664 if (incr.is_register()) { \ 3665 AOP(sz, incr.as_register(), prev, addr); \ 3666 } else { \ 3667 mov(rscratch2, incr.as_constant()); \ 3668 AOP(sz, rscratch2, prev, addr); \ 3669 } \ 3670 return; \ 3671 } \ 3672 Register result = rscratch2; \ 3673 if (prev->is_valid()) \ 3674 result = different(prev, incr, addr) ? prev : rscratch2; \ 3675 \ 3676 Label retry_load; \ 3677 prfm(Address(addr), PSTL1STRM); \ 3678 bind(retry_load); \ 3679 LDXR(result, addr); \ 3680 OP(rscratch1, result, incr); \ 3681 STXR(rscratch2, rscratch1, addr); \ 3682 cbnzw(rscratch2, retry_load); \ 3683 if (prev->is_valid() && prev != result) { \ 3684 IOP(prev, rscratch1, incr); \ 3685 } \ 3686 } 3687 3688 ATOMIC_OP(add, ldxr, add, sub, ldadd, stxr, Assembler::xword) 3689 ATOMIC_OP(addw, ldxrw, addw, subw, ldadd, stxrw, Assembler::word) 3690 ATOMIC_OP(addal, ldaxr, add, sub, ldaddal, stlxr, Assembler::xword) 3691 ATOMIC_OP(addalw, ldaxrw, addw, subw, ldaddal, stlxrw, Assembler::word) 3692 3693 #undef ATOMIC_OP 3694 3695 #define ATOMIC_XCHG(OP, AOP, LDXR, STXR, sz) \ 3696 void MacroAssembler::atomic_##OP(Register prev, Register newv, Register addr) { \ 3697 if (UseLSE) { \ 3698 prev = prev->is_valid() ? prev : zr; \ 3699 AOP(sz, newv, prev, addr); \ 3700 return; \ 3701 } \ 3702 Register result = rscratch2; \ 3703 if (prev->is_valid()) \ 3704 result = different(prev, newv, addr) ? prev : rscratch2; \ 3705 \ 3706 Label retry_load; \ 3707 prfm(Address(addr), PSTL1STRM); \ 3708 bind(retry_load); \ 3709 LDXR(result, addr); \ 3710 STXR(rscratch1, newv, addr); \ 3711 cbnzw(rscratch1, retry_load); \ 3712 if (prev->is_valid() && prev != result) \ 3713 mov(prev, result); \ 3714 } 3715 3716 ATOMIC_XCHG(xchg, swp, ldxr, stxr, Assembler::xword) 3717 ATOMIC_XCHG(xchgw, swp, ldxrw, stxrw, Assembler::word) 3718 ATOMIC_XCHG(xchgl, swpl, ldxr, stlxr, Assembler::xword) 3719 ATOMIC_XCHG(xchglw, swpl, ldxrw, stlxrw, Assembler::word) 3720 ATOMIC_XCHG(xchgal, swpal, ldaxr, stlxr, Assembler::xword) 3721 ATOMIC_XCHG(xchgalw, swpal, ldaxrw, stlxrw, Assembler::word) 3722 3723 #undef ATOMIC_XCHG 3724 3725 #ifndef PRODUCT 3726 extern "C" void findpc(intptr_t x); 3727 #endif 3728 3729 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) 3730 { 3731 // In order to get locks to work, we need to fake a in_VM state 3732 if (ShowMessageBoxOnError ) { 3733 JavaThread* thread = JavaThread::current(); 3734 JavaThreadState saved_state = thread->thread_state(); 3735 thread->set_thread_state(_thread_in_vm); 3736 #ifndef PRODUCT 3737 if (CountBytecodes || TraceBytecodes || StopInterpreterAt) { 3738 ttyLocker ttyl; 3739 BytecodeCounter::print(); 3740 } 3741 #endif 3742 if (os::message_box(msg, "Execution stopped, print registers?")) { 3743 ttyLocker ttyl; 3744 tty->print_cr(" pc = 0x%016" PRIx64, pc); 3745 #ifndef PRODUCT 3746 tty->cr(); 3747 findpc(pc); 3748 tty->cr(); 3749 #endif 3750 tty->print_cr(" r0 = 0x%016" PRIx64, regs[0]); 3751 tty->print_cr(" r1 = 0x%016" PRIx64, regs[1]); 3752 tty->print_cr(" r2 = 0x%016" PRIx64, regs[2]); 3753 tty->print_cr(" r3 = 0x%016" PRIx64, regs[3]); 3754 tty->print_cr(" r4 = 0x%016" PRIx64, regs[4]); 3755 tty->print_cr(" r5 = 0x%016" PRIx64, regs[5]); 3756 tty->print_cr(" r6 = 0x%016" PRIx64, regs[6]); 3757 tty->print_cr(" r7 = 0x%016" PRIx64, regs[7]); 3758 tty->print_cr(" r8 = 0x%016" PRIx64, regs[8]); 3759 tty->print_cr(" r9 = 0x%016" PRIx64, regs[9]); 3760 tty->print_cr("r10 = 0x%016" PRIx64, regs[10]); 3761 tty->print_cr("r11 = 0x%016" PRIx64, regs[11]); 3762 tty->print_cr("r12 = 0x%016" PRIx64, regs[12]); 3763 tty->print_cr("r13 = 0x%016" PRIx64, regs[13]); 3764 tty->print_cr("r14 = 0x%016" PRIx64, regs[14]); 3765 tty->print_cr("r15 = 0x%016" PRIx64, regs[15]); 3766 tty->print_cr("r16 = 0x%016" PRIx64, regs[16]); 3767 tty->print_cr("r17 = 0x%016" PRIx64, regs[17]); 3768 tty->print_cr("r18 = 0x%016" PRIx64, regs[18]); 3769 tty->print_cr("r19 = 0x%016" PRIx64, regs[19]); 3770 tty->print_cr("r20 = 0x%016" PRIx64, regs[20]); 3771 tty->print_cr("r21 = 0x%016" PRIx64, regs[21]); 3772 tty->print_cr("r22 = 0x%016" PRIx64, regs[22]); 3773 tty->print_cr("r23 = 0x%016" PRIx64, regs[23]); 3774 tty->print_cr("r24 = 0x%016" PRIx64, regs[24]); 3775 tty->print_cr("r25 = 0x%016" PRIx64, regs[25]); 3776 tty->print_cr("r26 = 0x%016" PRIx64, regs[26]); 3777 tty->print_cr("r27 = 0x%016" PRIx64, regs[27]); 3778 tty->print_cr("r28 = 0x%016" PRIx64, regs[28]); 3779 tty->print_cr("r30 = 0x%016" PRIx64, regs[30]); 3780 tty->print_cr("r31 = 0x%016" PRIx64, regs[31]); 3781 BREAKPOINT; 3782 } 3783 } 3784 fatal("DEBUG MESSAGE: %s", msg); 3785 } 3786 3787 RegSet MacroAssembler::call_clobbered_gp_registers() { 3788 RegSet regs = RegSet::range(r0, r17) - RegSet::of(rscratch1, rscratch2); 3789 #ifndef R18_RESERVED 3790 regs += r18_tls; 3791 #endif 3792 return regs; 3793 } 3794 3795 void MacroAssembler::push_call_clobbered_registers_except(RegSet exclude) { 3796 int step = 4 * wordSize; 3797 push(call_clobbered_gp_registers() - exclude, sp); 3798 sub(sp, sp, step); 3799 mov(rscratch1, -step); 3800 // Push v0-v7, v16-v31. 3801 for (int i = 31; i>= 4; i -= 4) { 3802 if (i <= v7->encoding() || i >= v16->encoding()) 3803 st1(as_FloatRegister(i-3), as_FloatRegister(i-2), as_FloatRegister(i-1), 3804 as_FloatRegister(i), T1D, Address(post(sp, rscratch1))); 3805 } 3806 st1(as_FloatRegister(0), as_FloatRegister(1), as_FloatRegister(2), 3807 as_FloatRegister(3), T1D, Address(sp)); 3808 } 3809 3810 void MacroAssembler::pop_call_clobbered_registers_except(RegSet exclude) { 3811 for (int i = 0; i < 32; i += 4) { 3812 if (i <= v7->encoding() || i >= v16->encoding()) 3813 ld1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2), 3814 as_FloatRegister(i+3), T1D, Address(post(sp, 4 * wordSize))); 3815 } 3816 3817 reinitialize_ptrue(); 3818 3819 pop(call_clobbered_gp_registers() - exclude, sp); 3820 } 3821 3822 void MacroAssembler::push_CPU_state(bool save_vectors, bool use_sve, 3823 int sve_vector_size_in_bytes, int total_predicate_in_bytes) { 3824 push(RegSet::range(r0, r29), sp); // integer registers except lr & sp 3825 if (save_vectors && use_sve && sve_vector_size_in_bytes > 16) { 3826 sub(sp, sp, sve_vector_size_in_bytes * FloatRegister::number_of_registers); 3827 for (int i = 0; i < FloatRegister::number_of_registers; i++) { 3828 sve_str(as_FloatRegister(i), Address(sp, i)); 3829 } 3830 } else { 3831 int step = (save_vectors ? 8 : 4) * wordSize; 3832 mov(rscratch1, -step); 3833 sub(sp, sp, step); 3834 for (int i = 28; i >= 4; i -= 4) { 3835 st1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2), 3836 as_FloatRegister(i+3), save_vectors ? T2D : T1D, Address(post(sp, rscratch1))); 3837 } 3838 st1(v0, v1, v2, v3, save_vectors ? T2D : T1D, sp); 3839 } 3840 if (save_vectors && use_sve && total_predicate_in_bytes > 0) { 3841 sub(sp, sp, total_predicate_in_bytes); 3842 for (int i = 0; i < PRegister::number_of_registers; i++) { 3843 sve_str(as_PRegister(i), Address(sp, i)); 3844 } 3845 } 3846 } 3847 3848 void MacroAssembler::pop_CPU_state(bool restore_vectors, bool use_sve, 3849 int sve_vector_size_in_bytes, int total_predicate_in_bytes) { 3850 if (restore_vectors && use_sve && total_predicate_in_bytes > 0) { 3851 for (int i = PRegister::number_of_registers - 1; i >= 0; i--) { 3852 sve_ldr(as_PRegister(i), Address(sp, i)); 3853 } 3854 add(sp, sp, total_predicate_in_bytes); 3855 } 3856 if (restore_vectors && use_sve && sve_vector_size_in_bytes > 16) { 3857 for (int i = FloatRegister::number_of_registers - 1; i >= 0; i--) { 3858 sve_ldr(as_FloatRegister(i), Address(sp, i)); 3859 } 3860 add(sp, sp, sve_vector_size_in_bytes * FloatRegister::number_of_registers); 3861 } else { 3862 int step = (restore_vectors ? 8 : 4) * wordSize; 3863 for (int i = 0; i <= 28; i += 4) 3864 ld1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2), 3865 as_FloatRegister(i+3), restore_vectors ? T2D : T1D, Address(post(sp, step))); 3866 } 3867 3868 // We may use predicate registers and rely on ptrue with SVE, 3869 // regardless of wide vector (> 8 bytes) used or not. 3870 if (use_sve) { 3871 reinitialize_ptrue(); 3872 } 3873 3874 // integer registers except lr & sp 3875 pop(RegSet::range(r0, r17), sp); 3876 #ifdef R18_RESERVED 3877 ldp(zr, r19, Address(post(sp, 2 * wordSize))); 3878 pop(RegSet::range(r20, r29), sp); 3879 #else 3880 pop(RegSet::range(r18_tls, r29), sp); 3881 #endif 3882 } 3883 3884 /** 3885 * Helpers for multiply_to_len(). 3886 */ 3887 void MacroAssembler::add2_with_carry(Register final_dest_hi, Register dest_hi, Register dest_lo, 3888 Register src1, Register src2) { 3889 adds(dest_lo, dest_lo, src1); 3890 adc(dest_hi, dest_hi, zr); 3891 adds(dest_lo, dest_lo, src2); 3892 adc(final_dest_hi, dest_hi, zr); 3893 } 3894 3895 // Generate an address from (r + r1 extend offset). "size" is the 3896 // size of the operand. The result may be in rscratch2. 3897 Address MacroAssembler::offsetted_address(Register r, Register r1, 3898 Address::extend ext, int offset, int size) { 3899 if (offset || (ext.shift() % size != 0)) { 3900 lea(rscratch2, Address(r, r1, ext)); 3901 return Address(rscratch2, offset); 3902 } else { 3903 return Address(r, r1, ext); 3904 } 3905 } 3906 3907 Address MacroAssembler::spill_address(int size, int offset, Register tmp) 3908 { 3909 assert(offset >= 0, "spill to negative address?"); 3910 // Offset reachable ? 3911 // Not aligned - 9 bits signed offset 3912 // Aligned - 12 bits unsigned offset shifted 3913 Register base = sp; 3914 if ((offset & (size-1)) && offset >= (1<<8)) { 3915 add(tmp, base, offset & ((1<<12)-1)); 3916 base = tmp; 3917 offset &= -1u<<12; 3918 } 3919 3920 if (offset >= (1<<12) * size) { 3921 add(tmp, base, offset & (((1<<12)-1)<<12)); 3922 base = tmp; 3923 offset &= ~(((1<<12)-1)<<12); 3924 } 3925 3926 return Address(base, offset); 3927 } 3928 3929 Address MacroAssembler::sve_spill_address(int sve_reg_size_in_bytes, int offset, Register tmp) { 3930 assert(offset >= 0, "spill to negative address?"); 3931 3932 Register base = sp; 3933 3934 // An immediate offset in the range 0 to 255 which is multiplied 3935 // by the current vector or predicate register size in bytes. 3936 if (offset % sve_reg_size_in_bytes == 0 && offset < ((1<<8)*sve_reg_size_in_bytes)) { 3937 return Address(base, offset / sve_reg_size_in_bytes); 3938 } 3939 3940 add(tmp, base, offset); 3941 return Address(tmp); 3942 } 3943 3944 // Checks whether offset is aligned. 3945 // Returns true if it is, else false. 3946 bool MacroAssembler::merge_alignment_check(Register base, 3947 size_t size, 3948 int64_t cur_offset, 3949 int64_t prev_offset) const { 3950 if (AvoidUnalignedAccesses) { 3951 if (base == sp) { 3952 // Checks whether low offset if aligned to pair of registers. 3953 int64_t pair_mask = size * 2 - 1; 3954 int64_t offset = prev_offset > cur_offset ? cur_offset : prev_offset; 3955 return (offset & pair_mask) == 0; 3956 } else { // If base is not sp, we can't guarantee the access is aligned. 3957 return false; 3958 } 3959 } else { 3960 int64_t mask = size - 1; 3961 // Load/store pair instruction only supports element size aligned offset. 3962 return (cur_offset & mask) == 0 && (prev_offset & mask) == 0; 3963 } 3964 } 3965 3966 // Checks whether current and previous loads/stores can be merged. 3967 // Returns true if it can be merged, else false. 3968 bool MacroAssembler::ldst_can_merge(Register rt, 3969 const Address &adr, 3970 size_t cur_size_in_bytes, 3971 bool is_store) const { 3972 address prev = pc() - NativeInstruction::instruction_size; 3973 address last = code()->last_insn(); 3974 3975 if (last == nullptr || !nativeInstruction_at(last)->is_Imm_LdSt()) { 3976 return false; 3977 } 3978 3979 if (adr.getMode() != Address::base_plus_offset || prev != last) { 3980 return false; 3981 } 3982 3983 NativeLdSt* prev_ldst = NativeLdSt_at(prev); 3984 size_t prev_size_in_bytes = prev_ldst->size_in_bytes(); 3985 3986 assert(prev_size_in_bytes == 4 || prev_size_in_bytes == 8, "only supports 64/32bit merging."); 3987 assert(cur_size_in_bytes == 4 || cur_size_in_bytes == 8, "only supports 64/32bit merging."); 3988 3989 if (cur_size_in_bytes != prev_size_in_bytes || is_store != prev_ldst->is_store()) { 3990 return false; 3991 } 3992 3993 int64_t max_offset = 63 * prev_size_in_bytes; 3994 int64_t min_offset = -64 * prev_size_in_bytes; 3995 3996 assert(prev_ldst->is_not_pre_post_index(), "pre-index or post-index is not supported to be merged."); 3997 3998 // Only same base can be merged. 3999 if (adr.base() != prev_ldst->base()) { 4000 return false; 4001 } 4002 4003 int64_t cur_offset = adr.offset(); 4004 int64_t prev_offset = prev_ldst->offset(); 4005 size_t diff = abs(cur_offset - prev_offset); 4006 if (diff != prev_size_in_bytes) { 4007 return false; 4008 } 4009 4010 // Following cases can not be merged: 4011 // ldr x2, [x2, #8] 4012 // ldr x3, [x2, #16] 4013 // or: 4014 // ldr x2, [x3, #8] 4015 // ldr x2, [x3, #16] 4016 // If t1 and t2 is the same in "ldp t1, t2, [xn, #imm]", we'll get SIGILL. 4017 if (!is_store && (adr.base() == prev_ldst->target() || rt == prev_ldst->target())) { 4018 return false; 4019 } 4020 4021 int64_t low_offset = prev_offset > cur_offset ? cur_offset : prev_offset; 4022 // Offset range must be in ldp/stp instruction's range. 4023 if (low_offset > max_offset || low_offset < min_offset) { 4024 return false; 4025 } 4026 4027 if (merge_alignment_check(adr.base(), prev_size_in_bytes, cur_offset, prev_offset)) { 4028 return true; 4029 } 4030 4031 return false; 4032 } 4033 4034 // Merge current load/store with previous load/store into ldp/stp. 4035 void MacroAssembler::merge_ldst(Register rt, 4036 const Address &adr, 4037 size_t cur_size_in_bytes, 4038 bool is_store) { 4039 4040 assert(ldst_can_merge(rt, adr, cur_size_in_bytes, is_store) == true, "cur and prev must be able to be merged."); 4041 4042 Register rt_low, rt_high; 4043 address prev = pc() - NativeInstruction::instruction_size; 4044 NativeLdSt* prev_ldst = NativeLdSt_at(prev); 4045 4046 int64_t offset; 4047 4048 if (adr.offset() < prev_ldst->offset()) { 4049 offset = adr.offset(); 4050 rt_low = rt; 4051 rt_high = prev_ldst->target(); 4052 } else { 4053 offset = prev_ldst->offset(); 4054 rt_low = prev_ldst->target(); 4055 rt_high = rt; 4056 } 4057 4058 Address adr_p = Address(prev_ldst->base(), offset); 4059 // Overwrite previous generated binary. 4060 code_section()->set_end(prev); 4061 4062 const size_t sz = prev_ldst->size_in_bytes(); 4063 assert(sz == 8 || sz == 4, "only supports 64/32bit merging."); 4064 if (!is_store) { 4065 BLOCK_COMMENT("merged ldr pair"); 4066 if (sz == 8) { 4067 ldp(rt_low, rt_high, adr_p); 4068 } else { 4069 ldpw(rt_low, rt_high, adr_p); 4070 } 4071 } else { 4072 BLOCK_COMMENT("merged str pair"); 4073 if (sz == 8) { 4074 stp(rt_low, rt_high, adr_p); 4075 } else { 4076 stpw(rt_low, rt_high, adr_p); 4077 } 4078 } 4079 } 4080 4081 /** 4082 * Multiply 64 bit by 64 bit first loop. 4083 */ 4084 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart, 4085 Register y, Register y_idx, Register z, 4086 Register carry, Register product, 4087 Register idx, Register kdx) { 4088 // 4089 // jlong carry, x[], y[], z[]; 4090 // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) { 4091 // huge_128 product = y[idx] * x[xstart] + carry; 4092 // z[kdx] = (jlong)product; 4093 // carry = (jlong)(product >>> 64); 4094 // } 4095 // z[xstart] = carry; 4096 // 4097 4098 Label L_first_loop, L_first_loop_exit; 4099 Label L_one_x, L_one_y, L_multiply; 4100 4101 subsw(xstart, xstart, 1); 4102 br(Assembler::MI, L_one_x); 4103 4104 lea(rscratch1, Address(x, xstart, Address::lsl(LogBytesPerInt))); 4105 ldr(x_xstart, Address(rscratch1)); 4106 ror(x_xstart, x_xstart, 32); // convert big-endian to little-endian 4107 4108 bind(L_first_loop); 4109 subsw(idx, idx, 1); 4110 br(Assembler::MI, L_first_loop_exit); 4111 subsw(idx, idx, 1); 4112 br(Assembler::MI, L_one_y); 4113 lea(rscratch1, Address(y, idx, Address::uxtw(LogBytesPerInt))); 4114 ldr(y_idx, Address(rscratch1)); 4115 ror(y_idx, y_idx, 32); // convert big-endian to little-endian 4116 bind(L_multiply); 4117 4118 // AArch64 has a multiply-accumulate instruction that we can't use 4119 // here because it has no way to process carries, so we have to use 4120 // separate add and adc instructions. Bah. 4121 umulh(rscratch1, x_xstart, y_idx); // x_xstart * y_idx -> rscratch1:product 4122 mul(product, x_xstart, y_idx); 4123 adds(product, product, carry); 4124 adc(carry, rscratch1, zr); // x_xstart * y_idx + carry -> carry:product 4125 4126 subw(kdx, kdx, 2); 4127 ror(product, product, 32); // back to big-endian 4128 str(product, offsetted_address(z, kdx, Address::uxtw(LogBytesPerInt), 0, BytesPerLong)); 4129 4130 b(L_first_loop); 4131 4132 bind(L_one_y); 4133 ldrw(y_idx, Address(y, 0)); 4134 b(L_multiply); 4135 4136 bind(L_one_x); 4137 ldrw(x_xstart, Address(x, 0)); 4138 b(L_first_loop); 4139 4140 bind(L_first_loop_exit); 4141 } 4142 4143 /** 4144 * Multiply 128 bit by 128. Unrolled inner loop. 4145 * 4146 */ 4147 void MacroAssembler::multiply_128_x_128_loop(Register y, Register z, 4148 Register carry, Register carry2, 4149 Register idx, Register jdx, 4150 Register yz_idx1, Register yz_idx2, 4151 Register tmp, Register tmp3, Register tmp4, 4152 Register tmp6, Register product_hi) { 4153 4154 // jlong carry, x[], y[], z[]; 4155 // int kdx = ystart+1; 4156 // for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop 4157 // huge_128 tmp3 = (y[idx+1] * product_hi) + z[kdx+idx+1] + carry; 4158 // jlong carry2 = (jlong)(tmp3 >>> 64); 4159 // huge_128 tmp4 = (y[idx] * product_hi) + z[kdx+idx] + carry2; 4160 // carry = (jlong)(tmp4 >>> 64); 4161 // z[kdx+idx+1] = (jlong)tmp3; 4162 // z[kdx+idx] = (jlong)tmp4; 4163 // } 4164 // idx += 2; 4165 // if (idx > 0) { 4166 // yz_idx1 = (y[idx] * product_hi) + z[kdx+idx] + carry; 4167 // z[kdx+idx] = (jlong)yz_idx1; 4168 // carry = (jlong)(yz_idx1 >>> 64); 4169 // } 4170 // 4171 4172 Label L_third_loop, L_third_loop_exit, L_post_third_loop_done; 4173 4174 lsrw(jdx, idx, 2); 4175 4176 bind(L_third_loop); 4177 4178 subsw(jdx, jdx, 1); 4179 br(Assembler::MI, L_third_loop_exit); 4180 subw(idx, idx, 4); 4181 4182 lea(rscratch1, Address(y, idx, Address::uxtw(LogBytesPerInt))); 4183 4184 ldp(yz_idx2, yz_idx1, Address(rscratch1, 0)); 4185 4186 lea(tmp6, Address(z, idx, Address::uxtw(LogBytesPerInt))); 4187 4188 ror(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian 4189 ror(yz_idx2, yz_idx2, 32); 4190 4191 ldp(rscratch2, rscratch1, Address(tmp6, 0)); 4192 4193 mul(tmp3, product_hi, yz_idx1); // yz_idx1 * product_hi -> tmp4:tmp3 4194 umulh(tmp4, product_hi, yz_idx1); 4195 4196 ror(rscratch1, rscratch1, 32); // convert big-endian to little-endian 4197 ror(rscratch2, rscratch2, 32); 4198 4199 mul(tmp, product_hi, yz_idx2); // yz_idx2 * product_hi -> carry2:tmp 4200 umulh(carry2, product_hi, yz_idx2); 4201 4202 // propagate sum of both multiplications into carry:tmp4:tmp3 4203 adds(tmp3, tmp3, carry); 4204 adc(tmp4, tmp4, zr); 4205 adds(tmp3, tmp3, rscratch1); 4206 adcs(tmp4, tmp4, tmp); 4207 adc(carry, carry2, zr); 4208 adds(tmp4, tmp4, rscratch2); 4209 adc(carry, carry, zr); 4210 4211 ror(tmp3, tmp3, 32); // convert little-endian to big-endian 4212 ror(tmp4, tmp4, 32); 4213 stp(tmp4, tmp3, Address(tmp6, 0)); 4214 4215 b(L_third_loop); 4216 bind (L_third_loop_exit); 4217 4218 andw (idx, idx, 0x3); 4219 cbz(idx, L_post_third_loop_done); 4220 4221 Label L_check_1; 4222 subsw(idx, idx, 2); 4223 br(Assembler::MI, L_check_1); 4224 4225 lea(rscratch1, Address(y, idx, Address::uxtw(LogBytesPerInt))); 4226 ldr(yz_idx1, Address(rscratch1, 0)); 4227 ror(yz_idx1, yz_idx1, 32); 4228 mul(tmp3, product_hi, yz_idx1); // yz_idx1 * product_hi -> tmp4:tmp3 4229 umulh(tmp4, product_hi, yz_idx1); 4230 lea(rscratch1, Address(z, idx, Address::uxtw(LogBytesPerInt))); 4231 ldr(yz_idx2, Address(rscratch1, 0)); 4232 ror(yz_idx2, yz_idx2, 32); 4233 4234 add2_with_carry(carry, tmp4, tmp3, carry, yz_idx2); 4235 4236 ror(tmp3, tmp3, 32); 4237 str(tmp3, Address(rscratch1, 0)); 4238 4239 bind (L_check_1); 4240 4241 andw (idx, idx, 0x1); 4242 subsw(idx, idx, 1); 4243 br(Assembler::MI, L_post_third_loop_done); 4244 ldrw(tmp4, Address(y, idx, Address::uxtw(LogBytesPerInt))); 4245 mul(tmp3, tmp4, product_hi); // tmp4 * product_hi -> carry2:tmp3 4246 umulh(carry2, tmp4, product_hi); 4247 ldrw(tmp4, Address(z, idx, Address::uxtw(LogBytesPerInt))); 4248 4249 add2_with_carry(carry2, tmp3, tmp4, carry); 4250 4251 strw(tmp3, Address(z, idx, Address::uxtw(LogBytesPerInt))); 4252 extr(carry, carry2, tmp3, 32); 4253 4254 bind(L_post_third_loop_done); 4255 } 4256 4257 /** 4258 * Code for BigInteger::multiplyToLen() intrinsic. 4259 * 4260 * r0: x 4261 * r1: xlen 4262 * r2: y 4263 * r3: ylen 4264 * r4: z 4265 * r5: tmp0 4266 * r10: tmp1 4267 * r11: tmp2 4268 * r12: tmp3 4269 * r13: tmp4 4270 * r14: tmp5 4271 * r15: tmp6 4272 * r16: tmp7 4273 * 4274 */ 4275 void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen, 4276 Register z, Register tmp0, 4277 Register tmp1, Register tmp2, Register tmp3, Register tmp4, 4278 Register tmp5, Register tmp6, Register product_hi) { 4279 4280 assert_different_registers(x, xlen, y, ylen, z, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, product_hi); 4281 4282 const Register idx = tmp1; 4283 const Register kdx = tmp2; 4284 const Register xstart = tmp3; 4285 4286 const Register y_idx = tmp4; 4287 const Register carry = tmp5; 4288 const Register product = xlen; 4289 const Register x_xstart = tmp0; 4290 4291 // First Loop. 4292 // 4293 // final static long LONG_MASK = 0xffffffffL; 4294 // int xstart = xlen - 1; 4295 // int ystart = ylen - 1; 4296 // long carry = 0; 4297 // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) { 4298 // long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry; 4299 // z[kdx] = (int)product; 4300 // carry = product >>> 32; 4301 // } 4302 // z[xstart] = (int)carry; 4303 // 4304 4305 movw(idx, ylen); // idx = ylen; 4306 addw(kdx, xlen, ylen); // kdx = xlen+ylen; 4307 mov(carry, zr); // carry = 0; 4308 4309 Label L_done; 4310 4311 movw(xstart, xlen); 4312 subsw(xstart, xstart, 1); 4313 br(Assembler::MI, L_done); 4314 4315 multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx); 4316 4317 Label L_second_loop; 4318 cbzw(kdx, L_second_loop); 4319 4320 Label L_carry; 4321 subw(kdx, kdx, 1); 4322 cbzw(kdx, L_carry); 4323 4324 strw(carry, Address(z, kdx, Address::uxtw(LogBytesPerInt))); 4325 lsr(carry, carry, 32); 4326 subw(kdx, kdx, 1); 4327 4328 bind(L_carry); 4329 strw(carry, Address(z, kdx, Address::uxtw(LogBytesPerInt))); 4330 4331 // Second and third (nested) loops. 4332 // 4333 // for (int i = xstart-1; i >= 0; i--) { // Second loop 4334 // carry = 0; 4335 // for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop 4336 // long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) + 4337 // (z[k] & LONG_MASK) + carry; 4338 // z[k] = (int)product; 4339 // carry = product >>> 32; 4340 // } 4341 // z[i] = (int)carry; 4342 // } 4343 // 4344 // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = product_hi 4345 4346 const Register jdx = tmp1; 4347 4348 bind(L_second_loop); 4349 mov(carry, zr); // carry = 0; 4350 movw(jdx, ylen); // j = ystart+1 4351 4352 subsw(xstart, xstart, 1); // i = xstart-1; 4353 br(Assembler::MI, L_done); 4354 4355 str(z, Address(pre(sp, -4 * wordSize))); 4356 4357 Label L_last_x; 4358 lea(z, offsetted_address(z, xstart, Address::uxtw(LogBytesPerInt), 4, BytesPerInt)); // z = z + k - j 4359 subsw(xstart, xstart, 1); // i = xstart-1; 4360 br(Assembler::MI, L_last_x); 4361 4362 lea(rscratch1, Address(x, xstart, Address::uxtw(LogBytesPerInt))); 4363 ldr(product_hi, Address(rscratch1)); 4364 ror(product_hi, product_hi, 32); // convert big-endian to little-endian 4365 4366 Label L_third_loop_prologue; 4367 bind(L_third_loop_prologue); 4368 4369 str(ylen, Address(sp, wordSize)); 4370 stp(x, xstart, Address(sp, 2 * wordSize)); 4371 multiply_128_x_128_loop(y, z, carry, x, jdx, ylen, product, 4372 tmp2, x_xstart, tmp3, tmp4, tmp6, product_hi); 4373 ldp(z, ylen, Address(post(sp, 2 * wordSize))); 4374 ldp(x, xlen, Address(post(sp, 2 * wordSize))); // copy old xstart -> xlen 4375 4376 addw(tmp3, xlen, 1); 4377 strw(carry, Address(z, tmp3, Address::uxtw(LogBytesPerInt))); 4378 subsw(tmp3, tmp3, 1); 4379 br(Assembler::MI, L_done); 4380 4381 lsr(carry, carry, 32); 4382 strw(carry, Address(z, tmp3, Address::uxtw(LogBytesPerInt))); 4383 b(L_second_loop); 4384 4385 // Next infrequent code is moved outside loops. 4386 bind(L_last_x); 4387 ldrw(product_hi, Address(x, 0)); 4388 b(L_third_loop_prologue); 4389 4390 bind(L_done); 4391 } 4392 4393 // Code for BigInteger::mulAdd intrinsic 4394 // out = r0 4395 // in = r1 4396 // offset = r2 (already out.length-offset) 4397 // len = r3 4398 // k = r4 4399 // 4400 // pseudo code from java implementation: 4401 // carry = 0; 4402 // offset = out.length-offset - 1; 4403 // for (int j=len-1; j >= 0; j--) { 4404 // product = (in[j] & LONG_MASK) * kLong + (out[offset] & LONG_MASK) + carry; 4405 // out[offset--] = (int)product; 4406 // carry = product >>> 32; 4407 // } 4408 // return (int)carry; 4409 void MacroAssembler::mul_add(Register out, Register in, Register offset, 4410 Register len, Register k) { 4411 Label LOOP, END; 4412 // pre-loop 4413 cmp(len, zr); // cmp, not cbz/cbnz: to use condition twice => less branches 4414 csel(out, zr, out, Assembler::EQ); 4415 br(Assembler::EQ, END); 4416 add(in, in, len, LSL, 2); // in[j+1] address 4417 add(offset, out, offset, LSL, 2); // out[offset + 1] address 4418 mov(out, zr); // used to keep carry now 4419 BIND(LOOP); 4420 ldrw(rscratch1, Address(pre(in, -4))); 4421 madd(rscratch1, rscratch1, k, out); 4422 ldrw(rscratch2, Address(pre(offset, -4))); 4423 add(rscratch1, rscratch1, rscratch2); 4424 strw(rscratch1, Address(offset)); 4425 lsr(out, rscratch1, 32); 4426 subs(len, len, 1); 4427 br(Assembler::NE, LOOP); 4428 BIND(END); 4429 } 4430 4431 /** 4432 * Emits code to update CRC-32 with a byte value according to constants in table 4433 * 4434 * @param [in,out]crc Register containing the crc. 4435 * @param [in]val Register containing the byte to fold into the CRC. 4436 * @param [in]table Register containing the table of crc constants. 4437 * 4438 * uint32_t crc; 4439 * val = crc_table[(val ^ crc) & 0xFF]; 4440 * crc = val ^ (crc >> 8); 4441 * 4442 */ 4443 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) { 4444 eor(val, val, crc); 4445 andr(val, val, 0xff); 4446 ldrw(val, Address(table, val, Address::lsl(2))); 4447 eor(crc, val, crc, Assembler::LSR, 8); 4448 } 4449 4450 /** 4451 * Emits code to update CRC-32 with a 32-bit value according to tables 0 to 3 4452 * 4453 * @param [in,out]crc Register containing the crc. 4454 * @param [in]v Register containing the 32-bit to fold into the CRC. 4455 * @param [in]table0 Register containing table 0 of crc constants. 4456 * @param [in]table1 Register containing table 1 of crc constants. 4457 * @param [in]table2 Register containing table 2 of crc constants. 4458 * @param [in]table3 Register containing table 3 of crc constants. 4459 * 4460 * uint32_t crc; 4461 * v = crc ^ v 4462 * crc = table3[v&0xff]^table2[(v>>8)&0xff]^table1[(v>>16)&0xff]^table0[v>>24] 4463 * 4464 */ 4465 void MacroAssembler::update_word_crc32(Register crc, Register v, Register tmp, 4466 Register table0, Register table1, Register table2, Register table3, 4467 bool upper) { 4468 eor(v, crc, v, upper ? LSR:LSL, upper ? 32:0); 4469 uxtb(tmp, v); 4470 ldrw(crc, Address(table3, tmp, Address::lsl(2))); 4471 ubfx(tmp, v, 8, 8); 4472 ldrw(tmp, Address(table2, tmp, Address::lsl(2))); 4473 eor(crc, crc, tmp); 4474 ubfx(tmp, v, 16, 8); 4475 ldrw(tmp, Address(table1, tmp, Address::lsl(2))); 4476 eor(crc, crc, tmp); 4477 ubfx(tmp, v, 24, 8); 4478 ldrw(tmp, Address(table0, tmp, Address::lsl(2))); 4479 eor(crc, crc, tmp); 4480 } 4481 4482 void MacroAssembler::kernel_crc32_using_crypto_pmull(Register crc, Register buf, 4483 Register len, Register tmp0, Register tmp1, Register tmp2, Register tmp3) { 4484 Label CRC_by4_loop, CRC_by1_loop, CRC_less128, CRC_by128_pre, CRC_by32_loop, CRC_less32, L_exit; 4485 assert_different_registers(crc, buf, len, tmp0, tmp1, tmp2); 4486 4487 subs(tmp0, len, 384); 4488 mvnw(crc, crc); 4489 br(Assembler::GE, CRC_by128_pre); 4490 BIND(CRC_less128); 4491 subs(len, len, 32); 4492 br(Assembler::GE, CRC_by32_loop); 4493 BIND(CRC_less32); 4494 adds(len, len, 32 - 4); 4495 br(Assembler::GE, CRC_by4_loop); 4496 adds(len, len, 4); 4497 br(Assembler::GT, CRC_by1_loop); 4498 b(L_exit); 4499 4500 BIND(CRC_by32_loop); 4501 ldp(tmp0, tmp1, Address(buf)); 4502 crc32x(crc, crc, tmp0); 4503 ldp(tmp2, tmp3, Address(buf, 16)); 4504 crc32x(crc, crc, tmp1); 4505 add(buf, buf, 32); 4506 crc32x(crc, crc, tmp2); 4507 subs(len, len, 32); 4508 crc32x(crc, crc, tmp3); 4509 br(Assembler::GE, CRC_by32_loop); 4510 cmn(len, (u1)32); 4511 br(Assembler::NE, CRC_less32); 4512 b(L_exit); 4513 4514 BIND(CRC_by4_loop); 4515 ldrw(tmp0, Address(post(buf, 4))); 4516 subs(len, len, 4); 4517 crc32w(crc, crc, tmp0); 4518 br(Assembler::GE, CRC_by4_loop); 4519 adds(len, len, 4); 4520 br(Assembler::LE, L_exit); 4521 BIND(CRC_by1_loop); 4522 ldrb(tmp0, Address(post(buf, 1))); 4523 subs(len, len, 1); 4524 crc32b(crc, crc, tmp0); 4525 br(Assembler::GT, CRC_by1_loop); 4526 b(L_exit); 4527 4528 BIND(CRC_by128_pre); 4529 kernel_crc32_common_fold_using_crypto_pmull(crc, buf, len, tmp0, tmp1, tmp2, 4530 4*256*sizeof(juint) + 8*sizeof(juint)); 4531 mov(crc, 0); 4532 crc32x(crc, crc, tmp0); 4533 crc32x(crc, crc, tmp1); 4534 4535 cbnz(len, CRC_less128); 4536 4537 BIND(L_exit); 4538 mvnw(crc, crc); 4539 } 4540 4541 void MacroAssembler::kernel_crc32_using_crc32(Register crc, Register buf, 4542 Register len, Register tmp0, Register tmp1, Register tmp2, 4543 Register tmp3) { 4544 Label CRC_by64_loop, CRC_by4_loop, CRC_by1_loop, CRC_less64, CRC_by64_pre, CRC_by32_loop, CRC_less32, L_exit; 4545 assert_different_registers(crc, buf, len, tmp0, tmp1, tmp2, tmp3); 4546 4547 mvnw(crc, crc); 4548 4549 subs(len, len, 128); 4550 br(Assembler::GE, CRC_by64_pre); 4551 BIND(CRC_less64); 4552 adds(len, len, 128-32); 4553 br(Assembler::GE, CRC_by32_loop); 4554 BIND(CRC_less32); 4555 adds(len, len, 32-4); 4556 br(Assembler::GE, CRC_by4_loop); 4557 adds(len, len, 4); 4558 br(Assembler::GT, CRC_by1_loop); 4559 b(L_exit); 4560 4561 BIND(CRC_by32_loop); 4562 ldp(tmp0, tmp1, Address(post(buf, 16))); 4563 subs(len, len, 32); 4564 crc32x(crc, crc, tmp0); 4565 ldr(tmp2, Address(post(buf, 8))); 4566 crc32x(crc, crc, tmp1); 4567 ldr(tmp3, Address(post(buf, 8))); 4568 crc32x(crc, crc, tmp2); 4569 crc32x(crc, crc, tmp3); 4570 br(Assembler::GE, CRC_by32_loop); 4571 cmn(len, (u1)32); 4572 br(Assembler::NE, CRC_less32); 4573 b(L_exit); 4574 4575 BIND(CRC_by4_loop); 4576 ldrw(tmp0, Address(post(buf, 4))); 4577 subs(len, len, 4); 4578 crc32w(crc, crc, tmp0); 4579 br(Assembler::GE, CRC_by4_loop); 4580 adds(len, len, 4); 4581 br(Assembler::LE, L_exit); 4582 BIND(CRC_by1_loop); 4583 ldrb(tmp0, Address(post(buf, 1))); 4584 subs(len, len, 1); 4585 crc32b(crc, crc, tmp0); 4586 br(Assembler::GT, CRC_by1_loop); 4587 b(L_exit); 4588 4589 BIND(CRC_by64_pre); 4590 sub(buf, buf, 8); 4591 ldp(tmp0, tmp1, Address(buf, 8)); 4592 crc32x(crc, crc, tmp0); 4593 ldr(tmp2, Address(buf, 24)); 4594 crc32x(crc, crc, tmp1); 4595 ldr(tmp3, Address(buf, 32)); 4596 crc32x(crc, crc, tmp2); 4597 ldr(tmp0, Address(buf, 40)); 4598 crc32x(crc, crc, tmp3); 4599 ldr(tmp1, Address(buf, 48)); 4600 crc32x(crc, crc, tmp0); 4601 ldr(tmp2, Address(buf, 56)); 4602 crc32x(crc, crc, tmp1); 4603 ldr(tmp3, Address(pre(buf, 64))); 4604 4605 b(CRC_by64_loop); 4606 4607 align(CodeEntryAlignment); 4608 BIND(CRC_by64_loop); 4609 subs(len, len, 64); 4610 crc32x(crc, crc, tmp2); 4611 ldr(tmp0, Address(buf, 8)); 4612 crc32x(crc, crc, tmp3); 4613 ldr(tmp1, Address(buf, 16)); 4614 crc32x(crc, crc, tmp0); 4615 ldr(tmp2, Address(buf, 24)); 4616 crc32x(crc, crc, tmp1); 4617 ldr(tmp3, Address(buf, 32)); 4618 crc32x(crc, crc, tmp2); 4619 ldr(tmp0, Address(buf, 40)); 4620 crc32x(crc, crc, tmp3); 4621 ldr(tmp1, Address(buf, 48)); 4622 crc32x(crc, crc, tmp0); 4623 ldr(tmp2, Address(buf, 56)); 4624 crc32x(crc, crc, tmp1); 4625 ldr(tmp3, Address(pre(buf, 64))); 4626 br(Assembler::GE, CRC_by64_loop); 4627 4628 // post-loop 4629 crc32x(crc, crc, tmp2); 4630 crc32x(crc, crc, tmp3); 4631 4632 sub(len, len, 64); 4633 add(buf, buf, 8); 4634 cmn(len, (u1)128); 4635 br(Assembler::NE, CRC_less64); 4636 BIND(L_exit); 4637 mvnw(crc, crc); 4638 } 4639 4640 /** 4641 * @param crc register containing existing CRC (32-bit) 4642 * @param buf register pointing to input byte buffer (byte*) 4643 * @param len register containing number of bytes 4644 * @param table register that will contain address of CRC table 4645 * @param tmp scratch register 4646 */ 4647 void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len, 4648 Register table0, Register table1, Register table2, Register table3, 4649 Register tmp, Register tmp2, Register tmp3) { 4650 Label L_by16, L_by16_loop, L_by4, L_by4_loop, L_by1, L_by1_loop, L_exit; 4651 4652 if (UseCryptoPmullForCRC32) { 4653 kernel_crc32_using_crypto_pmull(crc, buf, len, table0, table1, table2, table3); 4654 return; 4655 } 4656 4657 if (UseCRC32) { 4658 kernel_crc32_using_crc32(crc, buf, len, table0, table1, table2, table3); 4659 return; 4660 } 4661 4662 mvnw(crc, crc); 4663 4664 { 4665 uint64_t offset; 4666 adrp(table0, ExternalAddress(StubRoutines::crc_table_addr()), offset); 4667 add(table0, table0, offset); 4668 } 4669 add(table1, table0, 1*256*sizeof(juint)); 4670 add(table2, table0, 2*256*sizeof(juint)); 4671 add(table3, table0, 3*256*sizeof(juint)); 4672 4673 { // Neon code start 4674 cmp(len, (u1)64); 4675 br(Assembler::LT, L_by16); 4676 eor(v16, T16B, v16, v16); 4677 4678 Label L_fold; 4679 4680 add(tmp, table0, 4*256*sizeof(juint)); // Point at the Neon constants 4681 4682 ld1(v0, v1, T2D, post(buf, 32)); 4683 ld1r(v4, T2D, post(tmp, 8)); 4684 ld1r(v5, T2D, post(tmp, 8)); 4685 ld1r(v6, T2D, post(tmp, 8)); 4686 ld1r(v7, T2D, post(tmp, 8)); 4687 mov(v16, S, 0, crc); 4688 4689 eor(v0, T16B, v0, v16); 4690 sub(len, len, 64); 4691 4692 BIND(L_fold); 4693 pmull(v22, T8H, v0, v5, T8B); 4694 pmull(v20, T8H, v0, v7, T8B); 4695 pmull(v23, T8H, v0, v4, T8B); 4696 pmull(v21, T8H, v0, v6, T8B); 4697 4698 pmull2(v18, T8H, v0, v5, T16B); 4699 pmull2(v16, T8H, v0, v7, T16B); 4700 pmull2(v19, T8H, v0, v4, T16B); 4701 pmull2(v17, T8H, v0, v6, T16B); 4702 4703 uzp1(v24, T8H, v20, v22); 4704 uzp2(v25, T8H, v20, v22); 4705 eor(v20, T16B, v24, v25); 4706 4707 uzp1(v26, T8H, v16, v18); 4708 uzp2(v27, T8H, v16, v18); 4709 eor(v16, T16B, v26, v27); 4710 4711 ushll2(v22, T4S, v20, T8H, 8); 4712 ushll(v20, T4S, v20, T4H, 8); 4713 4714 ushll2(v18, T4S, v16, T8H, 8); 4715 ushll(v16, T4S, v16, T4H, 8); 4716 4717 eor(v22, T16B, v23, v22); 4718 eor(v18, T16B, v19, v18); 4719 eor(v20, T16B, v21, v20); 4720 eor(v16, T16B, v17, v16); 4721 4722 uzp1(v17, T2D, v16, v20); 4723 uzp2(v21, T2D, v16, v20); 4724 eor(v17, T16B, v17, v21); 4725 4726 ushll2(v20, T2D, v17, T4S, 16); 4727 ushll(v16, T2D, v17, T2S, 16); 4728 4729 eor(v20, T16B, v20, v22); 4730 eor(v16, T16B, v16, v18); 4731 4732 uzp1(v17, T2D, v20, v16); 4733 uzp2(v21, T2D, v20, v16); 4734 eor(v28, T16B, v17, v21); 4735 4736 pmull(v22, T8H, v1, v5, T8B); 4737 pmull(v20, T8H, v1, v7, T8B); 4738 pmull(v23, T8H, v1, v4, T8B); 4739 pmull(v21, T8H, v1, v6, T8B); 4740 4741 pmull2(v18, T8H, v1, v5, T16B); 4742 pmull2(v16, T8H, v1, v7, T16B); 4743 pmull2(v19, T8H, v1, v4, T16B); 4744 pmull2(v17, T8H, v1, v6, T16B); 4745 4746 ld1(v0, v1, T2D, post(buf, 32)); 4747 4748 uzp1(v24, T8H, v20, v22); 4749 uzp2(v25, T8H, v20, v22); 4750 eor(v20, T16B, v24, v25); 4751 4752 uzp1(v26, T8H, v16, v18); 4753 uzp2(v27, T8H, v16, v18); 4754 eor(v16, T16B, v26, v27); 4755 4756 ushll2(v22, T4S, v20, T8H, 8); 4757 ushll(v20, T4S, v20, T4H, 8); 4758 4759 ushll2(v18, T4S, v16, T8H, 8); 4760 ushll(v16, T4S, v16, T4H, 8); 4761 4762 eor(v22, T16B, v23, v22); 4763 eor(v18, T16B, v19, v18); 4764 eor(v20, T16B, v21, v20); 4765 eor(v16, T16B, v17, v16); 4766 4767 uzp1(v17, T2D, v16, v20); 4768 uzp2(v21, T2D, v16, v20); 4769 eor(v16, T16B, v17, v21); 4770 4771 ushll2(v20, T2D, v16, T4S, 16); 4772 ushll(v16, T2D, v16, T2S, 16); 4773 4774 eor(v20, T16B, v22, v20); 4775 eor(v16, T16B, v16, v18); 4776 4777 uzp1(v17, T2D, v20, v16); 4778 uzp2(v21, T2D, v20, v16); 4779 eor(v20, T16B, v17, v21); 4780 4781 shl(v16, T2D, v28, 1); 4782 shl(v17, T2D, v20, 1); 4783 4784 eor(v0, T16B, v0, v16); 4785 eor(v1, T16B, v1, v17); 4786 4787 subs(len, len, 32); 4788 br(Assembler::GE, L_fold); 4789 4790 mov(crc, 0); 4791 mov(tmp, v0, D, 0); 4792 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, false); 4793 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, true); 4794 mov(tmp, v0, D, 1); 4795 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, false); 4796 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, true); 4797 mov(tmp, v1, D, 0); 4798 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, false); 4799 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, true); 4800 mov(tmp, v1, D, 1); 4801 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, false); 4802 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, true); 4803 4804 add(len, len, 32); 4805 } // Neon code end 4806 4807 BIND(L_by16); 4808 subs(len, len, 16); 4809 br(Assembler::GE, L_by16_loop); 4810 adds(len, len, 16-4); 4811 br(Assembler::GE, L_by4_loop); 4812 adds(len, len, 4); 4813 br(Assembler::GT, L_by1_loop); 4814 b(L_exit); 4815 4816 BIND(L_by4_loop); 4817 ldrw(tmp, Address(post(buf, 4))); 4818 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3); 4819 subs(len, len, 4); 4820 br(Assembler::GE, L_by4_loop); 4821 adds(len, len, 4); 4822 br(Assembler::LE, L_exit); 4823 BIND(L_by1_loop); 4824 subs(len, len, 1); 4825 ldrb(tmp, Address(post(buf, 1))); 4826 update_byte_crc32(crc, tmp, table0); 4827 br(Assembler::GT, L_by1_loop); 4828 b(L_exit); 4829 4830 align(CodeEntryAlignment); 4831 BIND(L_by16_loop); 4832 subs(len, len, 16); 4833 ldp(tmp, tmp3, Address(post(buf, 16))); 4834 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, false); 4835 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, true); 4836 update_word_crc32(crc, tmp3, tmp2, table0, table1, table2, table3, false); 4837 update_word_crc32(crc, tmp3, tmp2, table0, table1, table2, table3, true); 4838 br(Assembler::GE, L_by16_loop); 4839 adds(len, len, 16-4); 4840 br(Assembler::GE, L_by4_loop); 4841 adds(len, len, 4); 4842 br(Assembler::GT, L_by1_loop); 4843 BIND(L_exit); 4844 mvnw(crc, crc); 4845 } 4846 4847 void MacroAssembler::kernel_crc32c_using_crypto_pmull(Register crc, Register buf, 4848 Register len, Register tmp0, Register tmp1, Register tmp2, Register tmp3) { 4849 Label CRC_by4_loop, CRC_by1_loop, CRC_less128, CRC_by128_pre, CRC_by32_loop, CRC_less32, L_exit; 4850 assert_different_registers(crc, buf, len, tmp0, tmp1, tmp2); 4851 4852 subs(tmp0, len, 384); 4853 br(Assembler::GE, CRC_by128_pre); 4854 BIND(CRC_less128); 4855 subs(len, len, 32); 4856 br(Assembler::GE, CRC_by32_loop); 4857 BIND(CRC_less32); 4858 adds(len, len, 32 - 4); 4859 br(Assembler::GE, CRC_by4_loop); 4860 adds(len, len, 4); 4861 br(Assembler::GT, CRC_by1_loop); 4862 b(L_exit); 4863 4864 BIND(CRC_by32_loop); 4865 ldp(tmp0, tmp1, Address(buf)); 4866 crc32cx(crc, crc, tmp0); 4867 ldr(tmp2, Address(buf, 16)); 4868 crc32cx(crc, crc, tmp1); 4869 ldr(tmp3, Address(buf, 24)); 4870 crc32cx(crc, crc, tmp2); 4871 add(buf, buf, 32); 4872 subs(len, len, 32); 4873 crc32cx(crc, crc, tmp3); 4874 br(Assembler::GE, CRC_by32_loop); 4875 cmn(len, (u1)32); 4876 br(Assembler::NE, CRC_less32); 4877 b(L_exit); 4878 4879 BIND(CRC_by4_loop); 4880 ldrw(tmp0, Address(post(buf, 4))); 4881 subs(len, len, 4); 4882 crc32cw(crc, crc, tmp0); 4883 br(Assembler::GE, CRC_by4_loop); 4884 adds(len, len, 4); 4885 br(Assembler::LE, L_exit); 4886 BIND(CRC_by1_loop); 4887 ldrb(tmp0, Address(post(buf, 1))); 4888 subs(len, len, 1); 4889 crc32cb(crc, crc, tmp0); 4890 br(Assembler::GT, CRC_by1_loop); 4891 b(L_exit); 4892 4893 BIND(CRC_by128_pre); 4894 kernel_crc32_common_fold_using_crypto_pmull(crc, buf, len, tmp0, tmp1, tmp2, 4895 4*256*sizeof(juint) + 8*sizeof(juint) + 0x50); 4896 mov(crc, 0); 4897 crc32cx(crc, crc, tmp0); 4898 crc32cx(crc, crc, tmp1); 4899 4900 cbnz(len, CRC_less128); 4901 4902 BIND(L_exit); 4903 } 4904 4905 void MacroAssembler::kernel_crc32c_using_crc32c(Register crc, Register buf, 4906 Register len, Register tmp0, Register tmp1, Register tmp2, 4907 Register tmp3) { 4908 Label CRC_by64_loop, CRC_by4_loop, CRC_by1_loop, CRC_less64, CRC_by64_pre, CRC_by32_loop, CRC_less32, L_exit; 4909 assert_different_registers(crc, buf, len, tmp0, tmp1, tmp2, tmp3); 4910 4911 subs(len, len, 128); 4912 br(Assembler::GE, CRC_by64_pre); 4913 BIND(CRC_less64); 4914 adds(len, len, 128-32); 4915 br(Assembler::GE, CRC_by32_loop); 4916 BIND(CRC_less32); 4917 adds(len, len, 32-4); 4918 br(Assembler::GE, CRC_by4_loop); 4919 adds(len, len, 4); 4920 br(Assembler::GT, CRC_by1_loop); 4921 b(L_exit); 4922 4923 BIND(CRC_by32_loop); 4924 ldp(tmp0, tmp1, Address(post(buf, 16))); 4925 subs(len, len, 32); 4926 crc32cx(crc, crc, tmp0); 4927 ldr(tmp2, Address(post(buf, 8))); 4928 crc32cx(crc, crc, tmp1); 4929 ldr(tmp3, Address(post(buf, 8))); 4930 crc32cx(crc, crc, tmp2); 4931 crc32cx(crc, crc, tmp3); 4932 br(Assembler::GE, CRC_by32_loop); 4933 cmn(len, (u1)32); 4934 br(Assembler::NE, CRC_less32); 4935 b(L_exit); 4936 4937 BIND(CRC_by4_loop); 4938 ldrw(tmp0, Address(post(buf, 4))); 4939 subs(len, len, 4); 4940 crc32cw(crc, crc, tmp0); 4941 br(Assembler::GE, CRC_by4_loop); 4942 adds(len, len, 4); 4943 br(Assembler::LE, L_exit); 4944 BIND(CRC_by1_loop); 4945 ldrb(tmp0, Address(post(buf, 1))); 4946 subs(len, len, 1); 4947 crc32cb(crc, crc, tmp0); 4948 br(Assembler::GT, CRC_by1_loop); 4949 b(L_exit); 4950 4951 BIND(CRC_by64_pre); 4952 sub(buf, buf, 8); 4953 ldp(tmp0, tmp1, Address(buf, 8)); 4954 crc32cx(crc, crc, tmp0); 4955 ldr(tmp2, Address(buf, 24)); 4956 crc32cx(crc, crc, tmp1); 4957 ldr(tmp3, Address(buf, 32)); 4958 crc32cx(crc, crc, tmp2); 4959 ldr(tmp0, Address(buf, 40)); 4960 crc32cx(crc, crc, tmp3); 4961 ldr(tmp1, Address(buf, 48)); 4962 crc32cx(crc, crc, tmp0); 4963 ldr(tmp2, Address(buf, 56)); 4964 crc32cx(crc, crc, tmp1); 4965 ldr(tmp3, Address(pre(buf, 64))); 4966 4967 b(CRC_by64_loop); 4968 4969 align(CodeEntryAlignment); 4970 BIND(CRC_by64_loop); 4971 subs(len, len, 64); 4972 crc32cx(crc, crc, tmp2); 4973 ldr(tmp0, Address(buf, 8)); 4974 crc32cx(crc, crc, tmp3); 4975 ldr(tmp1, Address(buf, 16)); 4976 crc32cx(crc, crc, tmp0); 4977 ldr(tmp2, Address(buf, 24)); 4978 crc32cx(crc, crc, tmp1); 4979 ldr(tmp3, Address(buf, 32)); 4980 crc32cx(crc, crc, tmp2); 4981 ldr(tmp0, Address(buf, 40)); 4982 crc32cx(crc, crc, tmp3); 4983 ldr(tmp1, Address(buf, 48)); 4984 crc32cx(crc, crc, tmp0); 4985 ldr(tmp2, Address(buf, 56)); 4986 crc32cx(crc, crc, tmp1); 4987 ldr(tmp3, Address(pre(buf, 64))); 4988 br(Assembler::GE, CRC_by64_loop); 4989 4990 // post-loop 4991 crc32cx(crc, crc, tmp2); 4992 crc32cx(crc, crc, tmp3); 4993 4994 sub(len, len, 64); 4995 add(buf, buf, 8); 4996 cmn(len, (u1)128); 4997 br(Assembler::NE, CRC_less64); 4998 BIND(L_exit); 4999 } 5000 5001 /** 5002 * @param crc register containing existing CRC (32-bit) 5003 * @param buf register pointing to input byte buffer (byte*) 5004 * @param len register containing number of bytes 5005 * @param table register that will contain address of CRC table 5006 * @param tmp scratch register 5007 */ 5008 void MacroAssembler::kernel_crc32c(Register crc, Register buf, Register len, 5009 Register table0, Register table1, Register table2, Register table3, 5010 Register tmp, Register tmp2, Register tmp3) { 5011 if (UseCryptoPmullForCRC32) { 5012 kernel_crc32c_using_crypto_pmull(crc, buf, len, table0, table1, table2, table3); 5013 } else { 5014 kernel_crc32c_using_crc32c(crc, buf, len, table0, table1, table2, table3); 5015 } 5016 } 5017 5018 void MacroAssembler::kernel_crc32_common_fold_using_crypto_pmull(Register crc, Register buf, 5019 Register len, Register tmp0, Register tmp1, Register tmp2, size_t table_offset) { 5020 Label CRC_by128_loop; 5021 assert_different_registers(crc, buf, len, tmp0, tmp1, tmp2); 5022 5023 sub(len, len, 256); 5024 Register table = tmp0; 5025 { 5026 uint64_t offset; 5027 adrp(table, ExternalAddress(StubRoutines::crc_table_addr()), offset); 5028 add(table, table, offset); 5029 } 5030 add(table, table, table_offset); 5031 5032 // Registers v0..v7 are used as data registers. 5033 // Registers v16..v31 are used as tmp registers. 5034 sub(buf, buf, 0x10); 5035 ldrq(v0, Address(buf, 0x10)); 5036 ldrq(v1, Address(buf, 0x20)); 5037 ldrq(v2, Address(buf, 0x30)); 5038 ldrq(v3, Address(buf, 0x40)); 5039 ldrq(v4, Address(buf, 0x50)); 5040 ldrq(v5, Address(buf, 0x60)); 5041 ldrq(v6, Address(buf, 0x70)); 5042 ldrq(v7, Address(pre(buf, 0x80))); 5043 5044 movi(v31, T4S, 0); 5045 mov(v31, S, 0, crc); 5046 eor(v0, T16B, v0, v31); 5047 5048 // Register v16 contains constants from the crc table. 5049 ldrq(v16, Address(table)); 5050 b(CRC_by128_loop); 5051 5052 align(OptoLoopAlignment); 5053 BIND(CRC_by128_loop); 5054 pmull (v17, T1Q, v0, v16, T1D); 5055 pmull2(v18, T1Q, v0, v16, T2D); 5056 ldrq(v0, Address(buf, 0x10)); 5057 eor3(v0, T16B, v17, v18, v0); 5058 5059 pmull (v19, T1Q, v1, v16, T1D); 5060 pmull2(v20, T1Q, v1, v16, T2D); 5061 ldrq(v1, Address(buf, 0x20)); 5062 eor3(v1, T16B, v19, v20, v1); 5063 5064 pmull (v21, T1Q, v2, v16, T1D); 5065 pmull2(v22, T1Q, v2, v16, T2D); 5066 ldrq(v2, Address(buf, 0x30)); 5067 eor3(v2, T16B, v21, v22, v2); 5068 5069 pmull (v23, T1Q, v3, v16, T1D); 5070 pmull2(v24, T1Q, v3, v16, T2D); 5071 ldrq(v3, Address(buf, 0x40)); 5072 eor3(v3, T16B, v23, v24, v3); 5073 5074 pmull (v25, T1Q, v4, v16, T1D); 5075 pmull2(v26, T1Q, v4, v16, T2D); 5076 ldrq(v4, Address(buf, 0x50)); 5077 eor3(v4, T16B, v25, v26, v4); 5078 5079 pmull (v27, T1Q, v5, v16, T1D); 5080 pmull2(v28, T1Q, v5, v16, T2D); 5081 ldrq(v5, Address(buf, 0x60)); 5082 eor3(v5, T16B, v27, v28, v5); 5083 5084 pmull (v29, T1Q, v6, v16, T1D); 5085 pmull2(v30, T1Q, v6, v16, T2D); 5086 ldrq(v6, Address(buf, 0x70)); 5087 eor3(v6, T16B, v29, v30, v6); 5088 5089 // Reuse registers v23, v24. 5090 // Using them won't block the first instruction of the next iteration. 5091 pmull (v23, T1Q, v7, v16, T1D); 5092 pmull2(v24, T1Q, v7, v16, T2D); 5093 ldrq(v7, Address(pre(buf, 0x80))); 5094 eor3(v7, T16B, v23, v24, v7); 5095 5096 subs(len, len, 0x80); 5097 br(Assembler::GE, CRC_by128_loop); 5098 5099 // fold into 512 bits 5100 // Use v31 for constants because v16 can be still in use. 5101 ldrq(v31, Address(table, 0x10)); 5102 5103 pmull (v17, T1Q, v0, v31, T1D); 5104 pmull2(v18, T1Q, v0, v31, T2D); 5105 eor3(v0, T16B, v17, v18, v4); 5106 5107 pmull (v19, T1Q, v1, v31, T1D); 5108 pmull2(v20, T1Q, v1, v31, T2D); 5109 eor3(v1, T16B, v19, v20, v5); 5110 5111 pmull (v21, T1Q, v2, v31, T1D); 5112 pmull2(v22, T1Q, v2, v31, T2D); 5113 eor3(v2, T16B, v21, v22, v6); 5114 5115 pmull (v23, T1Q, v3, v31, T1D); 5116 pmull2(v24, T1Q, v3, v31, T2D); 5117 eor3(v3, T16B, v23, v24, v7); 5118 5119 // fold into 128 bits 5120 // Use v17 for constants because v31 can be still in use. 5121 ldrq(v17, Address(table, 0x20)); 5122 pmull (v25, T1Q, v0, v17, T1D); 5123 pmull2(v26, T1Q, v0, v17, T2D); 5124 eor3(v3, T16B, v3, v25, v26); 5125 5126 // Use v18 for constants because v17 can be still in use. 5127 ldrq(v18, Address(table, 0x30)); 5128 pmull (v27, T1Q, v1, v18, T1D); 5129 pmull2(v28, T1Q, v1, v18, T2D); 5130 eor3(v3, T16B, v3, v27, v28); 5131 5132 // Use v19 for constants because v18 can be still in use. 5133 ldrq(v19, Address(table, 0x40)); 5134 pmull (v29, T1Q, v2, v19, T1D); 5135 pmull2(v30, T1Q, v2, v19, T2D); 5136 eor3(v0, T16B, v3, v29, v30); 5137 5138 add(len, len, 0x80); 5139 add(buf, buf, 0x10); 5140 5141 mov(tmp0, v0, D, 0); 5142 mov(tmp1, v0, D, 1); 5143 } 5144 5145 void MacroAssembler::addptr(const Address &dst, int32_t src) { 5146 Address adr; 5147 switch(dst.getMode()) { 5148 case Address::base_plus_offset: 5149 // This is the expected mode, although we allow all the other 5150 // forms below. 5151 adr = form_address(rscratch2, dst.base(), dst.offset(), LogBytesPerWord); 5152 break; 5153 default: 5154 lea(rscratch2, dst); 5155 adr = Address(rscratch2); 5156 break; 5157 } 5158 ldr(rscratch1, adr); 5159 add(rscratch1, rscratch1, src); 5160 str(rscratch1, adr); 5161 } 5162 5163 void MacroAssembler::cmpptr(Register src1, Address src2) { 5164 uint64_t offset; 5165 adrp(rscratch1, src2, offset); 5166 ldr(rscratch1, Address(rscratch1, offset)); 5167 cmp(src1, rscratch1); 5168 } 5169 5170 void MacroAssembler::cmpoop(Register obj1, Register obj2) { 5171 cmp(obj1, obj2); 5172 } 5173 5174 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) { 5175 load_method_holder(rresult, rmethod); 5176 ldr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset())); 5177 } 5178 5179 void MacroAssembler::load_method_holder(Register holder, Register method) { 5180 ldr(holder, Address(method, Method::const_offset())); // ConstMethod* 5181 ldr(holder, Address(holder, ConstMethod::constants_offset())); // ConstantPool* 5182 ldr(holder, Address(holder, ConstantPool::pool_holder_offset())); // InstanceKlass* 5183 } 5184 5185 void MacroAssembler::load_metadata(Register dst, Register src) { 5186 if (UseCompactObjectHeaders) { 5187 load_narrow_klass_compact(dst, src); 5188 } else if (UseCompressedClassPointers) { 5189 ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes())); 5190 } else { 5191 ldr(dst, Address(src, oopDesc::klass_offset_in_bytes())); 5192 } 5193 } 5194 5195 // Loads the obj's Klass* into dst. 5196 // Preserves all registers (incl src, rscratch1 and rscratch2). 5197 // Input: 5198 // src - the oop we want to load the klass from. 5199 // dst - output narrow klass. 5200 void MacroAssembler::load_narrow_klass_compact(Register dst, Register src) { 5201 assert(UseCompactObjectHeaders, "expects UseCompactObjectHeaders"); 5202 ldr(dst, Address(src, oopDesc::mark_offset_in_bytes())); 5203 lsr(dst, dst, markWord::klass_shift); 5204 } 5205 5206 void MacroAssembler::load_klass(Register dst, Register src) { 5207 if (UseCompactObjectHeaders) { 5208 load_narrow_klass_compact(dst, src); 5209 decode_klass_not_null(dst); 5210 } else if (UseCompressedClassPointers) { 5211 ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes())); 5212 decode_klass_not_null(dst); 5213 } else { 5214 ldr(dst, Address(src, oopDesc::klass_offset_in_bytes())); 5215 } 5216 } 5217 5218 void MacroAssembler::restore_cpu_control_state_after_jni(Register tmp1, Register tmp2) { 5219 if (RestoreMXCSROnJNICalls) { 5220 Label OK; 5221 get_fpcr(tmp1); 5222 mov(tmp2, tmp1); 5223 // Set FPCR to the state we need. We do want Round to Nearest. We 5224 // don't want non-IEEE rounding modes or floating-point traps. 5225 bfi(tmp1, zr, 22, 4); // Clear DN, FZ, and Rmode 5226 bfi(tmp1, zr, 8, 5); // Clear exception-control bits (8-12) 5227 bfi(tmp1, zr, 0, 2); // Clear AH:FIZ 5228 eor(tmp2, tmp1, tmp2); 5229 cbz(tmp2, OK); // Only reset FPCR if it's wrong 5230 set_fpcr(tmp1); 5231 bind(OK); 5232 } 5233 } 5234 5235 // ((OopHandle)result).resolve(); 5236 void MacroAssembler::resolve_oop_handle(Register result, Register tmp1, Register tmp2) { 5237 // OopHandle::resolve is an indirection. 5238 access_load_at(T_OBJECT, IN_NATIVE, result, Address(result, 0), tmp1, tmp2); 5239 } 5240 5241 // ((WeakHandle)result).resolve(); 5242 void MacroAssembler::resolve_weak_handle(Register result, Register tmp1, Register tmp2) { 5243 assert_different_registers(result, tmp1, tmp2); 5244 Label resolved; 5245 5246 // A null weak handle resolves to null. 5247 cbz(result, resolved); 5248 5249 // Only 64 bit platforms support GCs that require a tmp register 5250 // WeakHandle::resolve is an indirection like jweak. 5251 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, 5252 result, Address(result), tmp1, tmp2); 5253 bind(resolved); 5254 } 5255 5256 void MacroAssembler::load_mirror(Register dst, Register method, Register tmp1, Register tmp2) { 5257 const int mirror_offset = in_bytes(Klass::java_mirror_offset()); 5258 ldr(dst, Address(rmethod, Method::const_offset())); 5259 ldr(dst, Address(dst, ConstMethod::constants_offset())); 5260 ldr(dst, Address(dst, ConstantPool::pool_holder_offset())); 5261 ldr(dst, Address(dst, mirror_offset)); 5262 resolve_oop_handle(dst, tmp1, tmp2); 5263 } 5264 5265 void MacroAssembler::cmp_klass(Register obj, Register klass, Register tmp) { 5266 assert_different_registers(obj, klass, tmp); 5267 if (UseCompressedClassPointers) { 5268 if (UseCompactObjectHeaders) { 5269 load_narrow_klass_compact(tmp, obj); 5270 } else { 5271 ldrw(tmp, Address(obj, oopDesc::klass_offset_in_bytes())); 5272 } 5273 if (CompressedKlassPointers::base() == nullptr) { 5274 cmp(klass, tmp, LSL, CompressedKlassPointers::shift()); 5275 return; 5276 } else if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0 5277 && CompressedKlassPointers::shift() == 0) { 5278 // Only the bottom 32 bits matter 5279 cmpw(klass, tmp); 5280 return; 5281 } 5282 decode_klass_not_null(tmp); 5283 } else { 5284 ldr(tmp, Address(obj, oopDesc::klass_offset_in_bytes())); 5285 } 5286 cmp(klass, tmp); 5287 } 5288 5289 void MacroAssembler::cmp_klasses_from_objects(Register obj1, Register obj2, Register tmp1, Register tmp2) { 5290 if (UseCompactObjectHeaders) { 5291 load_narrow_klass_compact(tmp1, obj1); 5292 load_narrow_klass_compact(tmp2, obj2); 5293 cmpw(tmp1, tmp2); 5294 } else if (UseCompressedClassPointers) { 5295 ldrw(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes())); 5296 ldrw(tmp2, Address(obj2, oopDesc::klass_offset_in_bytes())); 5297 cmpw(tmp1, tmp2); 5298 } else { 5299 ldr(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes())); 5300 ldr(tmp2, Address(obj2, oopDesc::klass_offset_in_bytes())); 5301 cmp(tmp1, tmp2); 5302 } 5303 } 5304 5305 void MacroAssembler::load_prototype_header(Register dst, Register src) { 5306 load_klass(dst, src); 5307 ldr(dst, Address(dst, Klass::prototype_header_offset())); 5308 } 5309 5310 void MacroAssembler::store_klass(Register dst, Register src) { 5311 // FIXME: Should this be a store release? concurrent gcs assumes 5312 // klass length is valid if klass field is not null. 5313 assert(!UseCompactObjectHeaders, "not with compact headers"); 5314 if (UseCompressedClassPointers) { 5315 encode_klass_not_null(src); 5316 strw(src, Address(dst, oopDesc::klass_offset_in_bytes())); 5317 } else { 5318 str(src, Address(dst, oopDesc::klass_offset_in_bytes())); 5319 } 5320 } 5321 5322 void MacroAssembler::store_klass_gap(Register dst, Register src) { 5323 assert(!UseCompactObjectHeaders, "not with compact headers"); 5324 if (UseCompressedClassPointers) { 5325 // Store to klass gap in destination 5326 strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes())); 5327 } 5328 } 5329 5330 // Algorithm must match CompressedOops::encode. 5331 void MacroAssembler::encode_heap_oop(Register d, Register s) { 5332 #ifdef ASSERT 5333 verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?"); 5334 #endif 5335 verify_oop_msg(s, "broken oop in encode_heap_oop"); 5336 if (CompressedOops::base() == nullptr) { 5337 if (CompressedOops::shift() != 0) { 5338 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5339 lsr(d, s, LogMinObjAlignmentInBytes); 5340 } else { 5341 mov(d, s); 5342 } 5343 } else { 5344 subs(d, s, rheapbase); 5345 csel(d, d, zr, Assembler::HS); 5346 lsr(d, d, LogMinObjAlignmentInBytes); 5347 5348 /* Old algorithm: is this any worse? 5349 Label nonnull; 5350 cbnz(r, nonnull); 5351 sub(r, r, rheapbase); 5352 bind(nonnull); 5353 lsr(r, r, LogMinObjAlignmentInBytes); 5354 */ 5355 } 5356 } 5357 5358 void MacroAssembler::encode_heap_oop_not_null(Register r) { 5359 #ifdef ASSERT 5360 verify_heapbase("MacroAssembler::encode_heap_oop_not_null: heap base corrupted?"); 5361 if (CheckCompressedOops) { 5362 Label ok; 5363 cbnz(r, ok); 5364 stop("null oop passed to encode_heap_oop_not_null"); 5365 bind(ok); 5366 } 5367 #endif 5368 verify_oop_msg(r, "broken oop in encode_heap_oop_not_null"); 5369 if (CompressedOops::base() != nullptr) { 5370 sub(r, r, rheapbase); 5371 } 5372 if (CompressedOops::shift() != 0) { 5373 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5374 lsr(r, r, LogMinObjAlignmentInBytes); 5375 } 5376 } 5377 5378 void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) { 5379 #ifdef ASSERT 5380 verify_heapbase("MacroAssembler::encode_heap_oop_not_null2: heap base corrupted?"); 5381 if (CheckCompressedOops) { 5382 Label ok; 5383 cbnz(src, ok); 5384 stop("null oop passed to encode_heap_oop_not_null2"); 5385 bind(ok); 5386 } 5387 #endif 5388 verify_oop_msg(src, "broken oop in encode_heap_oop_not_null2"); 5389 5390 Register data = src; 5391 if (CompressedOops::base() != nullptr) { 5392 sub(dst, src, rheapbase); 5393 data = dst; 5394 } 5395 if (CompressedOops::shift() != 0) { 5396 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5397 lsr(dst, data, LogMinObjAlignmentInBytes); 5398 data = dst; 5399 } 5400 if (data == src) 5401 mov(dst, src); 5402 } 5403 5404 void MacroAssembler::decode_heap_oop(Register d, Register s) { 5405 #ifdef ASSERT 5406 verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?"); 5407 #endif 5408 if (CompressedOops::base() == nullptr) { 5409 if (CompressedOops::shift() != 0) { 5410 lsl(d, s, CompressedOops::shift()); 5411 } else if (d != s) { 5412 mov(d, s); 5413 } 5414 } else { 5415 Label done; 5416 if (d != s) 5417 mov(d, s); 5418 cbz(s, done); 5419 add(d, rheapbase, s, Assembler::LSL, LogMinObjAlignmentInBytes); 5420 bind(done); 5421 } 5422 verify_oop_msg(d, "broken oop in decode_heap_oop"); 5423 } 5424 5425 void MacroAssembler::decode_heap_oop_not_null(Register r) { 5426 assert (UseCompressedOops, "should only be used for compressed headers"); 5427 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5428 // Cannot assert, unverified entry point counts instructions (see .ad file) 5429 // vtableStubs also counts instructions in pd_code_size_limit. 5430 // Also do not verify_oop as this is called by verify_oop. 5431 if (CompressedOops::shift() != 0) { 5432 assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5433 if (CompressedOops::base() != nullptr) { 5434 add(r, rheapbase, r, Assembler::LSL, LogMinObjAlignmentInBytes); 5435 } else { 5436 add(r, zr, r, Assembler::LSL, LogMinObjAlignmentInBytes); 5437 } 5438 } else { 5439 assert (CompressedOops::base() == nullptr, "sanity"); 5440 } 5441 } 5442 5443 void MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) { 5444 assert (UseCompressedOops, "should only be used for compressed headers"); 5445 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5446 // Cannot assert, unverified entry point counts instructions (see .ad file) 5447 // vtableStubs also counts instructions in pd_code_size_limit. 5448 // Also do not verify_oop as this is called by verify_oop. 5449 if (CompressedOops::shift() != 0) { 5450 assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5451 if (CompressedOops::base() != nullptr) { 5452 add(dst, rheapbase, src, Assembler::LSL, LogMinObjAlignmentInBytes); 5453 } else { 5454 add(dst, zr, src, Assembler::LSL, LogMinObjAlignmentInBytes); 5455 } 5456 } else { 5457 assert (CompressedOops::base() == nullptr, "sanity"); 5458 if (dst != src) { 5459 mov(dst, src); 5460 } 5461 } 5462 } 5463 5464 MacroAssembler::KlassDecodeMode MacroAssembler::_klass_decode_mode(KlassDecodeNone); 5465 5466 MacroAssembler::KlassDecodeMode MacroAssembler::klass_decode_mode() { 5467 assert(Metaspace::initialized(), "metaspace not initialized yet"); 5468 assert(_klass_decode_mode != KlassDecodeNone, "should be initialized"); 5469 return _klass_decode_mode; 5470 } 5471 5472 MacroAssembler::KlassDecodeMode MacroAssembler::klass_decode_mode(address base, int shift, const size_t range) { 5473 assert(UseCompressedClassPointers, "not using compressed class pointers"); 5474 5475 // KlassDecodeMode shouldn't be set already. 5476 assert(_klass_decode_mode == KlassDecodeNone, "set once"); 5477 5478 if (base == nullptr) { 5479 return KlassDecodeZero; 5480 } 5481 5482 if (operand_valid_for_logical_immediate( 5483 /*is32*/false, (uint64_t)base)) { 5484 const uint64_t range_mask = right_n_bits(log2i_ceil(range)); 5485 if (((uint64_t)base & range_mask) == 0) { 5486 return KlassDecodeXor; 5487 } 5488 } 5489 5490 const uint64_t shifted_base = 5491 (uint64_t)base >> shift; 5492 if ((shifted_base & 0xffff0000ffffffff) == 0) { 5493 return KlassDecodeMovk; 5494 } 5495 5496 // No valid encoding. 5497 return KlassDecodeNone; 5498 } 5499 5500 // Check if one of the above decoding modes will work for given base, shift and range. 5501 bool MacroAssembler::check_klass_decode_mode(address base, int shift, const size_t range) { 5502 return klass_decode_mode(base, shift, range) != KlassDecodeNone; 5503 } 5504 5505 bool MacroAssembler::set_klass_decode_mode(address base, int shift, const size_t range) { 5506 _klass_decode_mode = klass_decode_mode(base, shift, range); 5507 return _klass_decode_mode != KlassDecodeNone; 5508 } 5509 5510 void MacroAssembler::encode_klass_not_null(Register dst, Register src) { 5511 switch (klass_decode_mode()) { 5512 case KlassDecodeZero: 5513 if (CompressedKlassPointers::shift() != 0) { 5514 lsr(dst, src, CompressedKlassPointers::shift()); 5515 } else { 5516 if (dst != src) mov(dst, src); 5517 } 5518 break; 5519 5520 case KlassDecodeXor: 5521 if (CompressedKlassPointers::shift() != 0) { 5522 eor(dst, src, (uint64_t)CompressedKlassPointers::base()); 5523 lsr(dst, dst, CompressedKlassPointers::shift()); 5524 } else { 5525 eor(dst, src, (uint64_t)CompressedKlassPointers::base()); 5526 } 5527 break; 5528 5529 case KlassDecodeMovk: 5530 if (CompressedKlassPointers::shift() != 0) { 5531 ubfx(dst, src, CompressedKlassPointers::shift(), 32); 5532 } else { 5533 movw(dst, src); 5534 } 5535 break; 5536 5537 case KlassDecodeNone: 5538 ShouldNotReachHere(); 5539 break; 5540 } 5541 } 5542 5543 void MacroAssembler::encode_klass_not_null(Register r) { 5544 encode_klass_not_null(r, r); 5545 } 5546 5547 void MacroAssembler::decode_klass_not_null(Register dst, Register src) { 5548 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 5549 5550 switch (klass_decode_mode()) { 5551 case KlassDecodeZero: 5552 if (CompressedKlassPointers::shift() != 0) { 5553 lsl(dst, src, CompressedKlassPointers::shift()); 5554 } else { 5555 if (dst != src) mov(dst, src); 5556 } 5557 break; 5558 5559 case KlassDecodeXor: 5560 if (CompressedKlassPointers::shift() != 0) { 5561 lsl(dst, src, CompressedKlassPointers::shift()); 5562 eor(dst, dst, (uint64_t)CompressedKlassPointers::base()); 5563 } else { 5564 eor(dst, src, (uint64_t)CompressedKlassPointers::base()); 5565 } 5566 break; 5567 5568 case KlassDecodeMovk: { 5569 const uint64_t shifted_base = 5570 (uint64_t)CompressedKlassPointers::base() >> CompressedKlassPointers::shift(); 5571 5572 if (dst != src) movw(dst, src); 5573 movk(dst, shifted_base >> 32, 32); 5574 5575 if (CompressedKlassPointers::shift() != 0) { 5576 lsl(dst, dst, CompressedKlassPointers::shift()); 5577 } 5578 5579 break; 5580 } 5581 5582 case KlassDecodeNone: 5583 ShouldNotReachHere(); 5584 break; 5585 } 5586 } 5587 5588 void MacroAssembler::decode_klass_not_null(Register r) { 5589 decode_klass_not_null(r, r); 5590 } 5591 5592 void MacroAssembler::set_narrow_oop(Register dst, jobject obj) { 5593 #ifdef ASSERT 5594 { 5595 ThreadInVMfromUnknown tiv; 5596 assert (UseCompressedOops, "should only be used for compressed oops"); 5597 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5598 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5599 assert(Universe::heap()->is_in(JNIHandles::resolve(obj)), "should be real oop"); 5600 } 5601 #endif 5602 int oop_index = oop_recorder()->find_index(obj); 5603 InstructionMark im(this); 5604 RelocationHolder rspec = oop_Relocation::spec(oop_index); 5605 code_section()->relocate(inst_mark(), rspec); 5606 movz(dst, 0xDEAD, 16); 5607 movk(dst, 0xBEEF); 5608 } 5609 5610 void MacroAssembler::set_narrow_klass(Register dst, Klass* k) { 5611 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 5612 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5613 int index = oop_recorder()->find_index(k); 5614 assert(! Universe::heap()->is_in(k), "should not be an oop"); 5615 5616 InstructionMark im(this); 5617 RelocationHolder rspec = metadata_Relocation::spec(index); 5618 code_section()->relocate(inst_mark(), rspec); 5619 narrowKlass nk = CompressedKlassPointers::encode(k); 5620 movz(dst, (nk >> 16), 16); 5621 movk(dst, nk & 0xffff); 5622 } 5623 5624 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, 5625 Register dst, Address src, 5626 Register tmp1, Register tmp2) { 5627 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); 5628 decorators = AccessInternal::decorator_fixup(decorators, type); 5629 bool as_raw = (decorators & AS_RAW) != 0; 5630 if (as_raw) { 5631 bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, tmp2); 5632 } else { 5633 bs->load_at(this, decorators, type, dst, src, tmp1, tmp2); 5634 } 5635 } 5636 5637 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, 5638 Address dst, Register val, 5639 Register tmp1, Register tmp2, Register tmp3) { 5640 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); 5641 decorators = AccessInternal::decorator_fixup(decorators, type); 5642 bool as_raw = (decorators & AS_RAW) != 0; 5643 if (as_raw) { 5644 bs->BarrierSetAssembler::store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3); 5645 } else { 5646 bs->store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3); 5647 } 5648 } 5649 5650 void MacroAssembler::flat_field_copy(DecoratorSet decorators, Register src, Register dst, 5651 Register inline_layout_info) { 5652 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); 5653 bs->flat_field_copy(this, decorators, src, dst, inline_layout_info); 5654 } 5655 5656 void MacroAssembler::payload_offset(Register inline_klass, Register offset) { 5657 ldr(offset, Address(inline_klass, InstanceKlass::adr_inlineklass_fixed_block_offset())); 5658 ldrw(offset, Address(offset, InlineKlass::payload_offset_offset())); 5659 } 5660 5661 void MacroAssembler::payload_address(Register oop, Register data, Register inline_klass) { 5662 // ((address) (void*) o) + vk->payload_offset(); 5663 Register offset = (data == oop) ? rscratch1 : data; 5664 payload_offset(inline_klass, offset); 5665 if (data == oop) { 5666 add(data, data, offset); 5667 } else { 5668 lea(data, Address(oop, offset)); 5669 } 5670 } 5671 5672 void MacroAssembler::data_for_value_array_index(Register array, Register array_klass, 5673 Register index, Register data) { 5674 assert_different_registers(array, array_klass, index); 5675 assert_different_registers(rscratch1, array, index); 5676 5677 // array->base() + (index << Klass::layout_helper_log2_element_size(lh)); 5678 ldrw(rscratch1, Address(array_klass, Klass::layout_helper_offset())); 5679 5680 // Klass::layout_helper_log2_element_size(lh) 5681 // (lh >> _lh_log2_element_size_shift) & _lh_log2_element_size_mask; 5682 lsr(rscratch1, rscratch1, Klass::_lh_log2_element_size_shift); 5683 andr(rscratch1, rscratch1, Klass::_lh_log2_element_size_mask); 5684 lslv(index, index, rscratch1); 5685 5686 add(data, array, index); 5687 add(data, data, arrayOopDesc::base_offset_in_bytes(T_FLAT_ELEMENT)); 5688 } 5689 5690 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1, 5691 Register tmp2, DecoratorSet decorators) { 5692 access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, tmp2); 5693 } 5694 5695 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1, 5696 Register tmp2, DecoratorSet decorators) { 5697 access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, tmp2); 5698 } 5699 5700 void MacroAssembler::store_heap_oop(Address dst, Register val, Register tmp1, 5701 Register tmp2, Register tmp3, DecoratorSet decorators) { 5702 access_store_at(T_OBJECT, IN_HEAP | decorators, dst, val, tmp1, tmp2, tmp3); 5703 } 5704 5705 // Used for storing nulls. 5706 void MacroAssembler::store_heap_oop_null(Address dst) { 5707 access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg, noreg); 5708 } 5709 5710 Address MacroAssembler::allocate_metadata_address(Metadata* obj) { 5711 assert(oop_recorder() != nullptr, "this assembler needs a Recorder"); 5712 int index = oop_recorder()->allocate_metadata_index(obj); 5713 RelocationHolder rspec = metadata_Relocation::spec(index); 5714 return Address((address)obj, rspec); 5715 } 5716 5717 // Move an oop into a register. 5718 void MacroAssembler::movoop(Register dst, jobject obj) { 5719 int oop_index; 5720 if (obj == nullptr) { 5721 oop_index = oop_recorder()->allocate_oop_index(obj); 5722 } else { 5723 #ifdef ASSERT 5724 { 5725 ThreadInVMfromUnknown tiv; 5726 assert(Universe::heap()->is_in(JNIHandles::resolve(obj)), "should be real oop"); 5727 } 5728 #endif 5729 oop_index = oop_recorder()->find_index(obj); 5730 } 5731 RelocationHolder rspec = oop_Relocation::spec(oop_index); 5732 5733 if (BarrierSet::barrier_set()->barrier_set_assembler()->supports_instruction_patching()) { 5734 mov(dst, Address((address)obj, rspec)); 5735 } else { 5736 address dummy = address(uintptr_t(pc()) & -wordSize); // A nearby aligned address 5737 ldr_constant(dst, Address(dummy, rspec)); 5738 } 5739 5740 } 5741 5742 // Move a metadata address into a register. 5743 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) { 5744 int oop_index; 5745 if (obj == nullptr) { 5746 oop_index = oop_recorder()->allocate_metadata_index(obj); 5747 } else { 5748 oop_index = oop_recorder()->find_index(obj); 5749 } 5750 RelocationHolder rspec = metadata_Relocation::spec(oop_index); 5751 mov(dst, Address((address)obj, rspec)); 5752 } 5753 5754 Address MacroAssembler::constant_oop_address(jobject obj) { 5755 #ifdef ASSERT 5756 { 5757 ThreadInVMfromUnknown tiv; 5758 assert(oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5759 assert(Universe::heap()->is_in(JNIHandles::resolve(obj)), "not an oop"); 5760 } 5761 #endif 5762 int oop_index = oop_recorder()->find_index(obj); 5763 return Address((address)obj, oop_Relocation::spec(oop_index)); 5764 } 5765 5766 // Object / value buffer allocation... 5767 void MacroAssembler::allocate_instance(Register klass, Register new_obj, 5768 Register t1, Register t2, 5769 bool clear_fields, Label& alloc_failed) 5770 { 5771 Label done, initialize_header, initialize_object, slow_case, slow_case_no_pop; 5772 Register layout_size = t1; 5773 assert(new_obj == r0, "needs to be r0"); 5774 assert_different_registers(klass, new_obj, t1, t2); 5775 5776 // get instance_size in InstanceKlass (scaled to a count of bytes) 5777 ldrw(layout_size, Address(klass, Klass::layout_helper_offset())); 5778 // test to see if it is malformed in some way 5779 tst(layout_size, Klass::_lh_instance_slow_path_bit); 5780 br(Assembler::NE, slow_case_no_pop); 5781 5782 // Allocate the instance: 5783 // If TLAB is enabled: 5784 // Try to allocate in the TLAB. 5785 // If fails, go to the slow path. 5786 // Initialize the allocation. 5787 // Exit. 5788 // 5789 // Go to slow path. 5790 5791 if (UseTLAB) { 5792 push(klass); 5793 tlab_allocate(new_obj, layout_size, 0, klass, t2, slow_case); 5794 if (ZeroTLAB || (!clear_fields)) { 5795 // the fields have been already cleared 5796 b(initialize_header); 5797 } else { 5798 // initialize both the header and fields 5799 b(initialize_object); 5800 } 5801 5802 if (clear_fields) { 5803 // The object is initialized before the header. If the object size is 5804 // zero, go directly to the header initialization. 5805 bind(initialize_object); 5806 int header_size = oopDesc::header_size() * HeapWordSize; 5807 assert(is_aligned(header_size, BytesPerLong), "oop header size must be 8-byte-aligned"); 5808 subs(layout_size, layout_size, header_size); 5809 br(Assembler::EQ, initialize_header); 5810 5811 // Initialize topmost object field, divide size by 8, check if odd and 5812 // test if zero. 5813 5814 #ifdef ASSERT 5815 // make sure instance_size was multiple of 8 5816 Label L; 5817 tst(layout_size, 7); 5818 br(Assembler::EQ, L); 5819 stop("object size is not multiple of 8 - adjust this code"); 5820 bind(L); 5821 // must be > 0, no extra check needed here 5822 #endif 5823 5824 lsr(layout_size, layout_size, LogBytesPerLong); 5825 5826 // initialize remaining object fields: instance_size was a multiple of 8 5827 { 5828 Label loop; 5829 Register base = t2; 5830 5831 bind(loop); 5832 add(rscratch1, new_obj, layout_size, Assembler::LSL, LogBytesPerLong); 5833 str(zr, Address(rscratch1, header_size - 1*oopSize)); 5834 subs(layout_size, layout_size, 1); 5835 br(Assembler::NE, loop); 5836 } 5837 } // clear_fields 5838 5839 // initialize object header only. 5840 bind(initialize_header); 5841 pop(klass); 5842 Register mark_word = t2; 5843 if (UseCompactObjectHeaders || EnableValhalla) { 5844 ldr(mark_word, Address(klass, Klass::prototype_header_offset())); 5845 str(mark_word, Address(new_obj, oopDesc::mark_offset_in_bytes())); 5846 } else { 5847 mov(mark_word, (intptr_t)markWord::prototype().value()); 5848 str(mark_word, Address(new_obj, oopDesc::mark_offset_in_bytes())); 5849 } 5850 if (!UseCompactObjectHeaders) { 5851 store_klass_gap(new_obj, zr); // zero klass gap for compressed oops 5852 mov(t2, klass); // preserve klass 5853 store_klass(new_obj, t2); // src klass reg is potentially compressed 5854 } 5855 // TODO: Valhalla removed SharedRuntime::dtrace_object_alloc from here ? 5856 5857 b(done); 5858 } 5859 5860 if (UseTLAB) { 5861 bind(slow_case); 5862 pop(klass); 5863 } 5864 bind(slow_case_no_pop); 5865 b(alloc_failed); 5866 5867 bind(done); 5868 } 5869 5870 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes. 5871 void MacroAssembler::tlab_allocate(Register obj, 5872 Register var_size_in_bytes, 5873 int con_size_in_bytes, 5874 Register t1, 5875 Register t2, 5876 Label& slow_case) { 5877 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); 5878 bs->tlab_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case); 5879 } 5880 5881 void MacroAssembler::inc_held_monitor_count(Register tmp) { 5882 Address dst(rthread, JavaThread::held_monitor_count_offset()); 5883 #ifdef ASSERT 5884 ldr(tmp, dst); 5885 increment(tmp); 5886 str(tmp, dst); 5887 Label ok; 5888 tbz(tmp, 63, ok); 5889 STOP("assert(held monitor count underflow)"); 5890 should_not_reach_here(); 5891 bind(ok); 5892 #else 5893 increment(dst); 5894 #endif 5895 } 5896 5897 void MacroAssembler::dec_held_monitor_count(Register tmp) { 5898 Address dst(rthread, JavaThread::held_monitor_count_offset()); 5899 #ifdef ASSERT 5900 ldr(tmp, dst); 5901 decrement(tmp); 5902 str(tmp, dst); 5903 Label ok; 5904 tbz(tmp, 63, ok); 5905 STOP("assert(held monitor count underflow)"); 5906 should_not_reach_here(); 5907 bind(ok); 5908 #else 5909 decrement(dst); 5910 #endif 5911 } 5912 5913 void MacroAssembler::verify_tlab() { 5914 #ifdef ASSERT 5915 if (UseTLAB && VerifyOops) { 5916 Label next, ok; 5917 5918 stp(rscratch2, rscratch1, Address(pre(sp, -16))); 5919 5920 ldr(rscratch2, Address(rthread, in_bytes(JavaThread::tlab_top_offset()))); 5921 ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_start_offset()))); 5922 cmp(rscratch2, rscratch1); 5923 br(Assembler::HS, next); 5924 STOP("assert(top >= start)"); 5925 should_not_reach_here(); 5926 5927 bind(next); 5928 ldr(rscratch2, Address(rthread, in_bytes(JavaThread::tlab_end_offset()))); 5929 ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_top_offset()))); 5930 cmp(rscratch2, rscratch1); 5931 br(Assembler::HS, ok); 5932 STOP("assert(top <= end)"); 5933 should_not_reach_here(); 5934 5935 bind(ok); 5936 ldp(rscratch2, rscratch1, Address(post(sp, 16))); 5937 } 5938 #endif 5939 } 5940 5941 void MacroAssembler::get_inline_type_field_klass(Register holder_klass, Register index, Register inline_klass) { 5942 inline_layout_info(holder_klass, index, inline_klass); 5943 ldr(inline_klass, Address(inline_klass, InlineLayoutInfo::klass_offset())); 5944 } 5945 5946 void MacroAssembler::inline_layout_info(Register holder_klass, Register index, Register layout_info) { 5947 assert_different_registers(holder_klass, index, layout_info); 5948 InlineLayoutInfo array[2]; 5949 int size = (char*)&array[1] - (char*)&array[0]; // computing size of array elements 5950 if (is_power_of_2(size)) { 5951 lsl(index, index, log2i_exact(size)); // Scale index by power of 2 5952 } else { 5953 mov(layout_info, size); 5954 mul(index, index, layout_info); // Scale the index to be the entry index * array_element_size 5955 } 5956 ldr(layout_info, Address(holder_klass, InstanceKlass::inline_layout_info_array_offset())); 5957 add(layout_info, layout_info, Array<InlineLayoutInfo>::base_offset_in_bytes()); 5958 lea(layout_info, Address(layout_info, index)); 5959 } 5960 5961 // Writes to stack successive pages until offset reached to check for 5962 // stack overflow + shadow pages. This clobbers tmp. 5963 void MacroAssembler::bang_stack_size(Register size, Register tmp) { 5964 assert_different_registers(tmp, size, rscratch1); 5965 mov(tmp, sp); 5966 // Bang stack for total size given plus shadow page size. 5967 // Bang one page at a time because large size can bang beyond yellow and 5968 // red zones. 5969 Label loop; 5970 mov(rscratch1, (int)os::vm_page_size()); 5971 bind(loop); 5972 lea(tmp, Address(tmp, -(int)os::vm_page_size())); 5973 subsw(size, size, rscratch1); 5974 str(size, Address(tmp)); 5975 br(Assembler::GT, loop); 5976 5977 // Bang down shadow pages too. 5978 // At this point, (tmp-0) is the last address touched, so don't 5979 // touch it again. (It was touched as (tmp-pagesize) but then tmp 5980 // was post-decremented.) Skip this address by starting at i=1, and 5981 // touch a few more pages below. N.B. It is important to touch all 5982 // the way down to and including i=StackShadowPages. 5983 for (int i = 0; i < (int)(StackOverflow::stack_shadow_zone_size() / (int)os::vm_page_size()) - 1; i++) { 5984 // this could be any sized move but this is can be a debugging crumb 5985 // so the bigger the better. 5986 lea(tmp, Address(tmp, -(int)os::vm_page_size())); 5987 str(size, Address(tmp)); 5988 } 5989 } 5990 5991 // Move the address of the polling page into dest. 5992 void MacroAssembler::get_polling_page(Register dest, relocInfo::relocType rtype) { 5993 ldr(dest, Address(rthread, JavaThread::polling_page_offset())); 5994 } 5995 5996 // Read the polling page. The address of the polling page must 5997 // already be in r. 5998 address MacroAssembler::read_polling_page(Register r, relocInfo::relocType rtype) { 5999 address mark; 6000 { 6001 InstructionMark im(this); 6002 code_section()->relocate(inst_mark(), rtype); 6003 ldrw(zr, Address(r, 0)); 6004 mark = inst_mark(); 6005 } 6006 verify_cross_modify_fence_not_required(); 6007 return mark; 6008 } 6009 6010 void MacroAssembler::adrp(Register reg1, const Address &dest, uint64_t &byte_offset) { 6011 relocInfo::relocType rtype = dest.rspec().reloc()->type(); 6012 uint64_t low_page = (uint64_t)CodeCache::low_bound() >> 12; 6013 uint64_t high_page = (uint64_t)(CodeCache::high_bound()-1) >> 12; 6014 uint64_t dest_page = (uint64_t)dest.target() >> 12; 6015 int64_t offset_low = dest_page - low_page; 6016 int64_t offset_high = dest_page - high_page; 6017 6018 assert(is_valid_AArch64_address(dest.target()), "bad address"); 6019 assert(dest.getMode() == Address::literal, "ADRP must be applied to a literal address"); 6020 6021 InstructionMark im(this); 6022 code_section()->relocate(inst_mark(), dest.rspec()); 6023 // 8143067: Ensure that the adrp can reach the dest from anywhere within 6024 // the code cache so that if it is relocated we know it will still reach 6025 if (offset_high >= -(1<<20) && offset_low < (1<<20)) { 6026 _adrp(reg1, dest.target()); 6027 } else { 6028 uint64_t target = (uint64_t)dest.target(); 6029 uint64_t adrp_target 6030 = (target & 0xffffffffULL) | ((uint64_t)pc() & 0xffff00000000ULL); 6031 6032 _adrp(reg1, (address)adrp_target); 6033 movk(reg1, target >> 32, 32); 6034 } 6035 byte_offset = (uint64_t)dest.target() & 0xfff; 6036 } 6037 6038 void MacroAssembler::load_byte_map_base(Register reg) { 6039 CardTable::CardValue* byte_map_base = 6040 ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base(); 6041 6042 // Strictly speaking the byte_map_base isn't an address at all, and it might 6043 // even be negative. It is thus materialised as a constant. 6044 mov(reg, (uint64_t)byte_map_base); 6045 } 6046 6047 void MacroAssembler::build_frame(int framesize) { 6048 assert(framesize >= 2 * wordSize, "framesize must include space for FP/LR"); 6049 assert(framesize % (2*wordSize) == 0, "must preserve 2*wordSize alignment"); 6050 protect_return_address(); 6051 if (framesize < ((1 << 9) + 2 * wordSize)) { 6052 sub(sp, sp, framesize); 6053 stp(rfp, lr, Address(sp, framesize - 2 * wordSize)); 6054 if (PreserveFramePointer) add(rfp, sp, framesize - 2 * wordSize); 6055 } else { 6056 stp(rfp, lr, Address(pre(sp, -2 * wordSize))); 6057 if (PreserveFramePointer) mov(rfp, sp); 6058 if (framesize < ((1 << 12) + 2 * wordSize)) 6059 sub(sp, sp, framesize - 2 * wordSize); 6060 else { 6061 mov(rscratch1, framesize - 2 * wordSize); 6062 sub(sp, sp, rscratch1); 6063 } 6064 } 6065 verify_cross_modify_fence_not_required(); 6066 } 6067 6068 void MacroAssembler::remove_frame(int framesize) { 6069 assert(framesize >= 2 * wordSize, "framesize must include space for FP/LR"); 6070 assert(framesize % (2*wordSize) == 0, "must preserve 2*wordSize alignment"); 6071 if (framesize < ((1 << 9) + 2 * wordSize)) { 6072 ldp(rfp, lr, Address(sp, framesize - 2 * wordSize)); 6073 add(sp, sp, framesize); 6074 } else { 6075 if (framesize < ((1 << 12) + 2 * wordSize)) 6076 add(sp, sp, framesize - 2 * wordSize); 6077 else { 6078 mov(rscratch1, framesize - 2 * wordSize); 6079 add(sp, sp, rscratch1); 6080 } 6081 ldp(rfp, lr, Address(post(sp, 2 * wordSize))); 6082 } 6083 authenticate_return_address(); 6084 } 6085 6086 void MacroAssembler::remove_frame(int initial_framesize, bool needs_stack_repair) { 6087 if (needs_stack_repair) { 6088 // Remove the extension of the caller's frame used for inline type unpacking 6089 // 6090 // Right now the stack looks like this: 6091 // 6092 // | Arguments from caller | 6093 // |---------------------------| <-- caller's SP 6094 // | Saved LR #1 | 6095 // | Saved FP #1 | 6096 // |---------------------------| 6097 // | Extension space for | 6098 // | inline arg (un)packing | 6099 // |---------------------------| <-- start of this method's frame 6100 // | Saved LR #2 | 6101 // | Saved FP #2 | 6102 // |---------------------------| <-- FP 6103 // | sp_inc | 6104 // | method locals | 6105 // |---------------------------| <-- SP 6106 // 6107 // There are two copies of FP and LR on the stack. They will be identical 6108 // unless the caller has been deoptimized, in which case LR #1 will be patched 6109 // to point at the deopt blob, and LR #2 will still point into the old method. 6110 // 6111 // The sp_inc stack slot holds the total size of the frame including the 6112 // extension space minus two words for the saved FP and LR. 6113 6114 int sp_inc_offset = initial_framesize - 3 * wordSize; // Immediately below saved LR and FP 6115 6116 ldr(rscratch1, Address(sp, sp_inc_offset)); 6117 add(sp, sp, rscratch1); 6118 ldp(rfp, lr, Address(post(sp, 2 * wordSize))); 6119 } else { 6120 remove_frame(initial_framesize); 6121 } 6122 } 6123 6124 void MacroAssembler::save_stack_increment(int sp_inc, int frame_size) { 6125 int real_frame_size = frame_size + sp_inc; 6126 assert(sp_inc == 0 || sp_inc > 2*wordSize, "invalid sp_inc value"); 6127 assert(real_frame_size >= 2*wordSize, "frame size must include FP/LR space"); 6128 assert((real_frame_size & (StackAlignmentInBytes-1)) == 0, "frame size not aligned"); 6129 6130 int sp_inc_offset = frame_size - 3 * wordSize; // Immediately below saved LR and FP 6131 6132 // Subtract two words for the saved FP and LR as these will be popped 6133 // separately. See remove_frame above. 6134 mov(rscratch1, real_frame_size - 2*wordSize); 6135 str(rscratch1, Address(sp, sp_inc_offset)); 6136 } 6137 6138 // This method counts leading positive bytes (highest bit not set) in provided byte array 6139 address MacroAssembler::count_positives(Register ary1, Register len, Register result) { 6140 // Simple and most common case of aligned small array which is not at the 6141 // end of memory page is placed here. All other cases are in stub. 6142 Label LOOP, END, STUB, STUB_LONG, SET_RESULT, DONE; 6143 const uint64_t UPPER_BIT_MASK=0x8080808080808080; 6144 assert_different_registers(ary1, len, result); 6145 6146 mov(result, len); 6147 cmpw(len, 0); 6148 br(LE, DONE); 6149 cmpw(len, 4 * wordSize); 6150 br(GE, STUB_LONG); // size > 32 then go to stub 6151 6152 int shift = 64 - exact_log2(os::vm_page_size()); 6153 lsl(rscratch1, ary1, shift); 6154 mov(rscratch2, (size_t)(4 * wordSize) << shift); 6155 adds(rscratch2, rscratch1, rscratch2); // At end of page? 6156 br(CS, STUB); // at the end of page then go to stub 6157 subs(len, len, wordSize); 6158 br(LT, END); 6159 6160 BIND(LOOP); 6161 ldr(rscratch1, Address(post(ary1, wordSize))); 6162 tst(rscratch1, UPPER_BIT_MASK); 6163 br(NE, SET_RESULT); 6164 subs(len, len, wordSize); 6165 br(GE, LOOP); 6166 cmpw(len, -wordSize); 6167 br(EQ, DONE); 6168 6169 BIND(END); 6170 ldr(rscratch1, Address(ary1)); 6171 sub(rscratch2, zr, len, LSL, 3); // LSL 3 is to get bits from bytes 6172 lslv(rscratch1, rscratch1, rscratch2); 6173 tst(rscratch1, UPPER_BIT_MASK); 6174 br(NE, SET_RESULT); 6175 b(DONE); 6176 6177 BIND(STUB); 6178 RuntimeAddress count_pos = RuntimeAddress(StubRoutines::aarch64::count_positives()); 6179 assert(count_pos.target() != nullptr, "count_positives stub has not been generated"); 6180 address tpc1 = trampoline_call(count_pos); 6181 if (tpc1 == nullptr) { 6182 DEBUG_ONLY(reset_labels(STUB_LONG, SET_RESULT, DONE)); 6183 postcond(pc() == badAddress); 6184 return nullptr; 6185 } 6186 b(DONE); 6187 6188 BIND(STUB_LONG); 6189 RuntimeAddress count_pos_long = RuntimeAddress(StubRoutines::aarch64::count_positives_long()); 6190 assert(count_pos_long.target() != nullptr, "count_positives_long stub has not been generated"); 6191 address tpc2 = trampoline_call(count_pos_long); 6192 if (tpc2 == nullptr) { 6193 DEBUG_ONLY(reset_labels(SET_RESULT, DONE)); 6194 postcond(pc() == badAddress); 6195 return nullptr; 6196 } 6197 b(DONE); 6198 6199 BIND(SET_RESULT); 6200 6201 add(len, len, wordSize); 6202 sub(result, result, len); 6203 6204 BIND(DONE); 6205 postcond(pc() != badAddress); 6206 return pc(); 6207 } 6208 6209 // Clobbers: rscratch1, rscratch2, rflags 6210 // May also clobber v0-v7 when (!UseSimpleArrayEquals && UseSIMDForArrayEquals) 6211 address MacroAssembler::arrays_equals(Register a1, Register a2, Register tmp3, 6212 Register tmp4, Register tmp5, Register result, 6213 Register cnt1, int elem_size) { 6214 Label DONE, SAME; 6215 Register tmp1 = rscratch1; 6216 Register tmp2 = rscratch2; 6217 int elem_per_word = wordSize/elem_size; 6218 int log_elem_size = exact_log2(elem_size); 6219 int klass_offset = arrayOopDesc::klass_offset_in_bytes(); 6220 int length_offset = arrayOopDesc::length_offset_in_bytes(); 6221 int base_offset 6222 = arrayOopDesc::base_offset_in_bytes(elem_size == 2 ? T_CHAR : T_BYTE); 6223 // When the length offset is not aligned to 8 bytes, 6224 // then we align it down. This is valid because the new 6225 // offset will always be the klass which is the same 6226 // for type arrays. 6227 int start_offset = align_down(length_offset, BytesPerWord); 6228 int extra_length = base_offset - start_offset; 6229 assert(start_offset == length_offset || start_offset == klass_offset, 6230 "start offset must be 8-byte-aligned or be the klass offset"); 6231 assert(base_offset != start_offset, "must include the length field"); 6232 extra_length = extra_length / elem_size; // We count in elements, not bytes. 6233 int stubBytesThreshold = 3 * 64 + (UseSIMDForArrayEquals ? 0 : 16); 6234 6235 assert(elem_size == 1 || elem_size == 2, "must be char or byte"); 6236 assert_different_registers(a1, a2, result, cnt1, rscratch1, rscratch2); 6237 6238 #ifndef PRODUCT 6239 { 6240 const char kind = (elem_size == 2) ? 'U' : 'L'; 6241 char comment[64]; 6242 snprintf(comment, sizeof comment, "array_equals%c{", kind); 6243 BLOCK_COMMENT(comment); 6244 } 6245 #endif 6246 6247 // if (a1 == a2) 6248 // return true; 6249 cmpoop(a1, a2); // May have read barriers for a1 and a2. 6250 br(EQ, SAME); 6251 6252 if (UseSimpleArrayEquals) { 6253 Label NEXT_WORD, SHORT, TAIL03, TAIL01, A_MIGHT_BE_NULL, A_IS_NOT_NULL; 6254 // if (a1 == nullptr || a2 == nullptr) 6255 // return false; 6256 // a1 & a2 == 0 means (some-pointer is null) or 6257 // (very-rare-or-even-probably-impossible-pointer-values) 6258 // so, we can save one branch in most cases 6259 tst(a1, a2); 6260 mov(result, false); 6261 br(EQ, A_MIGHT_BE_NULL); 6262 // if (a1.length != a2.length) 6263 // return false; 6264 bind(A_IS_NOT_NULL); 6265 ldrw(cnt1, Address(a1, length_offset)); 6266 // Increase loop counter by diff between base- and actual start-offset. 6267 addw(cnt1, cnt1, extra_length); 6268 lea(a1, Address(a1, start_offset)); 6269 lea(a2, Address(a2, start_offset)); 6270 // Check for short strings, i.e. smaller than wordSize. 6271 subs(cnt1, cnt1, elem_per_word); 6272 br(Assembler::LT, SHORT); 6273 // Main 8 byte comparison loop. 6274 bind(NEXT_WORD); { 6275 ldr(tmp1, Address(post(a1, wordSize))); 6276 ldr(tmp2, Address(post(a2, wordSize))); 6277 subs(cnt1, cnt1, elem_per_word); 6278 eor(tmp5, tmp1, tmp2); 6279 cbnz(tmp5, DONE); 6280 } br(GT, NEXT_WORD); 6281 // Last longword. In the case where length == 4 we compare the 6282 // same longword twice, but that's still faster than another 6283 // conditional branch. 6284 // cnt1 could be 0, -1, -2, -3, -4 for chars; -4 only happens when 6285 // length == 4. 6286 if (log_elem_size > 0) 6287 lsl(cnt1, cnt1, log_elem_size); 6288 ldr(tmp3, Address(a1, cnt1)); 6289 ldr(tmp4, Address(a2, cnt1)); 6290 eor(tmp5, tmp3, tmp4); 6291 cbnz(tmp5, DONE); 6292 b(SAME); 6293 bind(A_MIGHT_BE_NULL); 6294 // in case both a1 and a2 are not-null, proceed with loads 6295 cbz(a1, DONE); 6296 cbz(a2, DONE); 6297 b(A_IS_NOT_NULL); 6298 bind(SHORT); 6299 6300 tbz(cnt1, 2 - log_elem_size, TAIL03); // 0-7 bytes left. 6301 { 6302 ldrw(tmp1, Address(post(a1, 4))); 6303 ldrw(tmp2, Address(post(a2, 4))); 6304 eorw(tmp5, tmp1, tmp2); 6305 cbnzw(tmp5, DONE); 6306 } 6307 bind(TAIL03); 6308 tbz(cnt1, 1 - log_elem_size, TAIL01); // 0-3 bytes left. 6309 { 6310 ldrh(tmp3, Address(post(a1, 2))); 6311 ldrh(tmp4, Address(post(a2, 2))); 6312 eorw(tmp5, tmp3, tmp4); 6313 cbnzw(tmp5, DONE); 6314 } 6315 bind(TAIL01); 6316 if (elem_size == 1) { // Only needed when comparing byte arrays. 6317 tbz(cnt1, 0, SAME); // 0-1 bytes left. 6318 { 6319 ldrb(tmp1, a1); 6320 ldrb(tmp2, a2); 6321 eorw(tmp5, tmp1, tmp2); 6322 cbnzw(tmp5, DONE); 6323 } 6324 } 6325 } else { 6326 Label NEXT_DWORD, SHORT, TAIL, TAIL2, STUB, 6327 CSET_EQ, LAST_CHECK; 6328 mov(result, false); 6329 cbz(a1, DONE); 6330 ldrw(cnt1, Address(a1, length_offset)); 6331 cbz(a2, DONE); 6332 // Increase loop counter by diff between base- and actual start-offset. 6333 addw(cnt1, cnt1, extra_length); 6334 6335 // on most CPUs a2 is still "locked"(surprisingly) in ldrw and it's 6336 // faster to perform another branch before comparing a1 and a2 6337 cmp(cnt1, (u1)elem_per_word); 6338 br(LE, SHORT); // short or same 6339 ldr(tmp3, Address(pre(a1, start_offset))); 6340 subs(zr, cnt1, stubBytesThreshold); 6341 br(GE, STUB); 6342 ldr(tmp4, Address(pre(a2, start_offset))); 6343 sub(tmp5, zr, cnt1, LSL, 3 + log_elem_size); 6344 6345 // Main 16 byte comparison loop with 2 exits 6346 bind(NEXT_DWORD); { 6347 ldr(tmp1, Address(pre(a1, wordSize))); 6348 ldr(tmp2, Address(pre(a2, wordSize))); 6349 subs(cnt1, cnt1, 2 * elem_per_word); 6350 br(LE, TAIL); 6351 eor(tmp4, tmp3, tmp4); 6352 cbnz(tmp4, DONE); 6353 ldr(tmp3, Address(pre(a1, wordSize))); 6354 ldr(tmp4, Address(pre(a2, wordSize))); 6355 cmp(cnt1, (u1)elem_per_word); 6356 br(LE, TAIL2); 6357 cmp(tmp1, tmp2); 6358 } br(EQ, NEXT_DWORD); 6359 b(DONE); 6360 6361 bind(TAIL); 6362 eor(tmp4, tmp3, tmp4); 6363 eor(tmp2, tmp1, tmp2); 6364 lslv(tmp2, tmp2, tmp5); 6365 orr(tmp5, tmp4, tmp2); 6366 cmp(tmp5, zr); 6367 b(CSET_EQ); 6368 6369 bind(TAIL2); 6370 eor(tmp2, tmp1, tmp2); 6371 cbnz(tmp2, DONE); 6372 b(LAST_CHECK); 6373 6374 bind(STUB); 6375 ldr(tmp4, Address(pre(a2, start_offset))); 6376 if (elem_size == 2) { // convert to byte counter 6377 lsl(cnt1, cnt1, 1); 6378 } 6379 eor(tmp5, tmp3, tmp4); 6380 cbnz(tmp5, DONE); 6381 RuntimeAddress stub = RuntimeAddress(StubRoutines::aarch64::large_array_equals()); 6382 assert(stub.target() != nullptr, "array_equals_long stub has not been generated"); 6383 address tpc = trampoline_call(stub); 6384 if (tpc == nullptr) { 6385 DEBUG_ONLY(reset_labels(SHORT, LAST_CHECK, CSET_EQ, SAME, DONE)); 6386 postcond(pc() == badAddress); 6387 return nullptr; 6388 } 6389 b(DONE); 6390 6391 // (a1 != null && a2 == null) || (a1 != null && a2 != null && a1 == a2) 6392 // so, if a2 == null => return false(0), else return true, so we can return a2 6393 mov(result, a2); 6394 b(DONE); 6395 bind(SHORT); 6396 sub(tmp5, zr, cnt1, LSL, 3 + log_elem_size); 6397 ldr(tmp3, Address(a1, start_offset)); 6398 ldr(tmp4, Address(a2, start_offset)); 6399 bind(LAST_CHECK); 6400 eor(tmp4, tmp3, tmp4); 6401 lslv(tmp5, tmp4, tmp5); 6402 cmp(tmp5, zr); 6403 bind(CSET_EQ); 6404 cset(result, EQ); 6405 b(DONE); 6406 } 6407 6408 bind(SAME); 6409 mov(result, true); 6410 // That's it. 6411 bind(DONE); 6412 6413 BLOCK_COMMENT("} array_equals"); 6414 postcond(pc() != badAddress); 6415 return pc(); 6416 } 6417 6418 // Compare Strings 6419 6420 // For Strings we're passed the address of the first characters in a1 6421 // and a2 and the length in cnt1. 6422 // There are two implementations. For arrays >= 8 bytes, all 6423 // comparisons (including the final one, which may overlap) are 6424 // performed 8 bytes at a time. For strings < 8 bytes, we compare a 6425 // halfword, then a short, and then a byte. 6426 6427 void MacroAssembler::string_equals(Register a1, Register a2, 6428 Register result, Register cnt1) 6429 { 6430 Label SAME, DONE, SHORT, NEXT_WORD; 6431 Register tmp1 = rscratch1; 6432 Register tmp2 = rscratch2; 6433 Register cnt2 = tmp2; // cnt2 only used in array length compare 6434 6435 assert_different_registers(a1, a2, result, cnt1, rscratch1, rscratch2); 6436 6437 #ifndef PRODUCT 6438 { 6439 char comment[64]; 6440 snprintf(comment, sizeof comment, "{string_equalsL"); 6441 BLOCK_COMMENT(comment); 6442 } 6443 #endif 6444 6445 mov(result, false); 6446 6447 // Check for short strings, i.e. smaller than wordSize. 6448 subs(cnt1, cnt1, wordSize); 6449 br(Assembler::LT, SHORT); 6450 // Main 8 byte comparison loop. 6451 bind(NEXT_WORD); { 6452 ldr(tmp1, Address(post(a1, wordSize))); 6453 ldr(tmp2, Address(post(a2, wordSize))); 6454 subs(cnt1, cnt1, wordSize); 6455 eor(tmp1, tmp1, tmp2); 6456 cbnz(tmp1, DONE); 6457 } br(GT, NEXT_WORD); 6458 // Last longword. In the case where length == 4 we compare the 6459 // same longword twice, but that's still faster than another 6460 // conditional branch. 6461 // cnt1 could be 0, -1, -2, -3, -4 for chars; -4 only happens when 6462 // length == 4. 6463 ldr(tmp1, Address(a1, cnt1)); 6464 ldr(tmp2, Address(a2, cnt1)); 6465 eor(tmp2, tmp1, tmp2); 6466 cbnz(tmp2, DONE); 6467 b(SAME); 6468 6469 bind(SHORT); 6470 Label TAIL03, TAIL01; 6471 6472 tbz(cnt1, 2, TAIL03); // 0-7 bytes left. 6473 { 6474 ldrw(tmp1, Address(post(a1, 4))); 6475 ldrw(tmp2, Address(post(a2, 4))); 6476 eorw(tmp1, tmp1, tmp2); 6477 cbnzw(tmp1, DONE); 6478 } 6479 bind(TAIL03); 6480 tbz(cnt1, 1, TAIL01); // 0-3 bytes left. 6481 { 6482 ldrh(tmp1, Address(post(a1, 2))); 6483 ldrh(tmp2, Address(post(a2, 2))); 6484 eorw(tmp1, tmp1, tmp2); 6485 cbnzw(tmp1, DONE); 6486 } 6487 bind(TAIL01); 6488 tbz(cnt1, 0, SAME); // 0-1 bytes left. 6489 { 6490 ldrb(tmp1, a1); 6491 ldrb(tmp2, a2); 6492 eorw(tmp1, tmp1, tmp2); 6493 cbnzw(tmp1, DONE); 6494 } 6495 // Arrays are equal. 6496 bind(SAME); 6497 mov(result, true); 6498 6499 // That's it. 6500 bind(DONE); 6501 BLOCK_COMMENT("} string_equals"); 6502 } 6503 6504 6505 // The size of the blocks erased by the zero_blocks stub. We must 6506 // handle anything smaller than this ourselves in zero_words(). 6507 const int MacroAssembler::zero_words_block_size = 8; 6508 6509 // zero_words() is used by C2 ClearArray patterns and by 6510 // C1_MacroAssembler. It is as small as possible, handling small word 6511 // counts locally and delegating anything larger to the zero_blocks 6512 // stub. It is expanded many times in compiled code, so it is 6513 // important to keep it short. 6514 6515 // ptr: Address of a buffer to be zeroed. 6516 // cnt: Count in HeapWords. 6517 // 6518 // ptr, cnt, rscratch1, and rscratch2 are clobbered. 6519 address MacroAssembler::zero_words(Register ptr, Register cnt) 6520 { 6521 assert(is_power_of_2(zero_words_block_size), "adjust this"); 6522 6523 BLOCK_COMMENT("zero_words {"); 6524 assert(ptr == r10 && cnt == r11, "mismatch in register usage"); 6525 RuntimeAddress zero_blocks = RuntimeAddress(StubRoutines::aarch64::zero_blocks()); 6526 assert(zero_blocks.target() != nullptr, "zero_blocks stub has not been generated"); 6527 6528 subs(rscratch1, cnt, zero_words_block_size); 6529 Label around; 6530 br(LO, around); 6531 { 6532 RuntimeAddress zero_blocks = RuntimeAddress(StubRoutines::aarch64::zero_blocks()); 6533 assert(zero_blocks.target() != nullptr, "zero_blocks stub has not been generated"); 6534 // Make sure this is a C2 compilation. C1 allocates space only for 6535 // trampoline stubs generated by Call LIR ops, and in any case it 6536 // makes sense for a C1 compilation task to proceed as quickly as 6537 // possible. 6538 CompileTask* task; 6539 if (StubRoutines::aarch64::complete() 6540 && Thread::current()->is_Compiler_thread() 6541 && (task = ciEnv::current()->task()) 6542 && is_c2_compile(task->comp_level())) { 6543 address tpc = trampoline_call(zero_blocks); 6544 if (tpc == nullptr) { 6545 DEBUG_ONLY(reset_labels(around)); 6546 return nullptr; 6547 } 6548 } else { 6549 far_call(zero_blocks); 6550 } 6551 } 6552 bind(around); 6553 6554 // We have a few words left to do. zero_blocks has adjusted r10 and r11 6555 // for us. 6556 for (int i = zero_words_block_size >> 1; i > 1; i >>= 1) { 6557 Label l; 6558 tbz(cnt, exact_log2(i), l); 6559 for (int j = 0; j < i; j += 2) { 6560 stp(zr, zr, post(ptr, 2 * BytesPerWord)); 6561 } 6562 bind(l); 6563 } 6564 { 6565 Label l; 6566 tbz(cnt, 0, l); 6567 str(zr, Address(ptr)); 6568 bind(l); 6569 } 6570 6571 BLOCK_COMMENT("} zero_words"); 6572 return pc(); 6573 } 6574 6575 // base: Address of a buffer to be zeroed, 8 bytes aligned. 6576 // cnt: Immediate count in HeapWords. 6577 // 6578 // r10, r11, rscratch1, and rscratch2 are clobbered. 6579 address MacroAssembler::zero_words(Register base, uint64_t cnt) 6580 { 6581 assert(wordSize <= BlockZeroingLowLimit, 6582 "increase BlockZeroingLowLimit"); 6583 address result = nullptr; 6584 if (cnt <= (uint64_t)BlockZeroingLowLimit / BytesPerWord) { 6585 #ifndef PRODUCT 6586 { 6587 char buf[64]; 6588 snprintf(buf, sizeof buf, "zero_words (count = %" PRIu64 ") {", cnt); 6589 BLOCK_COMMENT(buf); 6590 } 6591 #endif 6592 if (cnt >= 16) { 6593 uint64_t loops = cnt/16; 6594 if (loops > 1) { 6595 mov(rscratch2, loops - 1); 6596 } 6597 { 6598 Label loop; 6599 bind(loop); 6600 for (int i = 0; i < 16; i += 2) { 6601 stp(zr, zr, Address(base, i * BytesPerWord)); 6602 } 6603 add(base, base, 16 * BytesPerWord); 6604 if (loops > 1) { 6605 subs(rscratch2, rscratch2, 1); 6606 br(GE, loop); 6607 } 6608 } 6609 } 6610 cnt %= 16; 6611 int i = cnt & 1; // store any odd word to start 6612 if (i) str(zr, Address(base)); 6613 for (; i < (int)cnt; i += 2) { 6614 stp(zr, zr, Address(base, i * wordSize)); 6615 } 6616 BLOCK_COMMENT("} zero_words"); 6617 result = pc(); 6618 } else { 6619 mov(r10, base); mov(r11, cnt); 6620 result = zero_words(r10, r11); 6621 } 6622 return result; 6623 } 6624 6625 // Zero blocks of memory by using DC ZVA. 6626 // 6627 // Aligns the base address first sufficiently for DC ZVA, then uses 6628 // DC ZVA repeatedly for every full block. cnt is the size to be 6629 // zeroed in HeapWords. Returns the count of words left to be zeroed 6630 // in cnt. 6631 // 6632 // NOTE: This is intended to be used in the zero_blocks() stub. If 6633 // you want to use it elsewhere, note that cnt must be >= 2*zva_length. 6634 void MacroAssembler::zero_dcache_blocks(Register base, Register cnt) { 6635 Register tmp = rscratch1; 6636 Register tmp2 = rscratch2; 6637 int zva_length = VM_Version::zva_length(); 6638 Label initial_table_end, loop_zva; 6639 Label fini; 6640 6641 // Base must be 16 byte aligned. If not just return and let caller handle it 6642 tst(base, 0x0f); 6643 br(Assembler::NE, fini); 6644 // Align base with ZVA length. 6645 neg(tmp, base); 6646 andr(tmp, tmp, zva_length - 1); 6647 6648 // tmp: the number of bytes to be filled to align the base with ZVA length. 6649 add(base, base, tmp); 6650 sub(cnt, cnt, tmp, Assembler::ASR, 3); 6651 adr(tmp2, initial_table_end); 6652 sub(tmp2, tmp2, tmp, Assembler::LSR, 2); 6653 br(tmp2); 6654 6655 for (int i = -zva_length + 16; i < 0; i += 16) 6656 stp(zr, zr, Address(base, i)); 6657 bind(initial_table_end); 6658 6659 sub(cnt, cnt, zva_length >> 3); 6660 bind(loop_zva); 6661 dc(Assembler::ZVA, base); 6662 subs(cnt, cnt, zva_length >> 3); 6663 add(base, base, zva_length); 6664 br(Assembler::GE, loop_zva); 6665 add(cnt, cnt, zva_length >> 3); // count not zeroed by DC ZVA 6666 bind(fini); 6667 } 6668 6669 // base: Address of a buffer to be filled, 8 bytes aligned. 6670 // cnt: Count in 8-byte unit. 6671 // value: Value to be filled with. 6672 // base will point to the end of the buffer after filling. 6673 void MacroAssembler::fill_words(Register base, Register cnt, Register value) 6674 { 6675 // Algorithm: 6676 // 6677 // if (cnt == 0) { 6678 // return; 6679 // } 6680 // if ((p & 8) != 0) { 6681 // *p++ = v; 6682 // } 6683 // 6684 // scratch1 = cnt & 14; 6685 // cnt -= scratch1; 6686 // p += scratch1; 6687 // switch (scratch1 / 2) { 6688 // do { 6689 // cnt -= 16; 6690 // p[-16] = v; 6691 // p[-15] = v; 6692 // case 7: 6693 // p[-14] = v; 6694 // p[-13] = v; 6695 // case 6: 6696 // p[-12] = v; 6697 // p[-11] = v; 6698 // // ... 6699 // case 1: 6700 // p[-2] = v; 6701 // p[-1] = v; 6702 // case 0: 6703 // p += 16; 6704 // } while (cnt); 6705 // } 6706 // if ((cnt & 1) == 1) { 6707 // *p++ = v; 6708 // } 6709 6710 assert_different_registers(base, cnt, value, rscratch1, rscratch2); 6711 6712 Label fini, skip, entry, loop; 6713 const int unroll = 8; // Number of stp instructions we'll unroll 6714 6715 cbz(cnt, fini); 6716 tbz(base, 3, skip); 6717 str(value, Address(post(base, 8))); 6718 sub(cnt, cnt, 1); 6719 bind(skip); 6720 6721 andr(rscratch1, cnt, (unroll-1) * 2); 6722 sub(cnt, cnt, rscratch1); 6723 add(base, base, rscratch1, Assembler::LSL, 3); 6724 adr(rscratch2, entry); 6725 sub(rscratch2, rscratch2, rscratch1, Assembler::LSL, 1); 6726 br(rscratch2); 6727 6728 bind(loop); 6729 add(base, base, unroll * 16); 6730 for (int i = -unroll; i < 0; i++) 6731 stp(value, value, Address(base, i * 16)); 6732 bind(entry); 6733 subs(cnt, cnt, unroll * 2); 6734 br(Assembler::GE, loop); 6735 6736 tbz(cnt, 0, fini); 6737 str(value, Address(post(base, 8))); 6738 bind(fini); 6739 } 6740 6741 // Intrinsic for 6742 // 6743 // - sun/nio/cs/ISO_8859_1$Encoder.implEncodeISOArray 6744 // return the number of characters copied. 6745 // - java/lang/StringUTF16.compress 6746 // return index of non-latin1 character if copy fails, otherwise 'len'. 6747 // 6748 // This version always returns the number of characters copied, and does not 6749 // clobber the 'len' register. A successful copy will complete with the post- 6750 // condition: 'res' == 'len', while an unsuccessful copy will exit with the 6751 // post-condition: 0 <= 'res' < 'len'. 6752 // 6753 // NOTE: Attempts to use 'ld2' (and 'umaxv' in the ISO part) has proven to 6754 // degrade performance (on Ampere Altra - Neoverse N1), to an extent 6755 // beyond the acceptable, even though the footprint would be smaller. 6756 // Using 'umaxv' in the ASCII-case comes with a small penalty but does 6757 // avoid additional bloat. 6758 // 6759 // Clobbers: src, dst, res, rscratch1, rscratch2, rflags 6760 void MacroAssembler::encode_iso_array(Register src, Register dst, 6761 Register len, Register res, bool ascii, 6762 FloatRegister vtmp0, FloatRegister vtmp1, 6763 FloatRegister vtmp2, FloatRegister vtmp3, 6764 FloatRegister vtmp4, FloatRegister vtmp5) 6765 { 6766 Register cnt = res; 6767 Register max = rscratch1; 6768 Register chk = rscratch2; 6769 6770 prfm(Address(src), PLDL1STRM); 6771 movw(cnt, len); 6772 6773 #define ASCII(insn) do { if (ascii) { insn; } } while (0) 6774 6775 Label LOOP_32, DONE_32, FAIL_32; 6776 6777 BIND(LOOP_32); 6778 { 6779 cmpw(cnt, 32); 6780 br(LT, DONE_32); 6781 ld1(vtmp0, vtmp1, vtmp2, vtmp3, T8H, Address(post(src, 64))); 6782 // Extract lower bytes. 6783 FloatRegister vlo0 = vtmp4; 6784 FloatRegister vlo1 = vtmp5; 6785 uzp1(vlo0, T16B, vtmp0, vtmp1); 6786 uzp1(vlo1, T16B, vtmp2, vtmp3); 6787 // Merge bits... 6788 orr(vtmp0, T16B, vtmp0, vtmp1); 6789 orr(vtmp2, T16B, vtmp2, vtmp3); 6790 // Extract merged upper bytes. 6791 FloatRegister vhix = vtmp0; 6792 uzp2(vhix, T16B, vtmp0, vtmp2); 6793 // ISO-check on hi-parts (all zero). 6794 // ASCII-check on lo-parts (no sign). 6795 FloatRegister vlox = vtmp1; // Merge lower bytes. 6796 ASCII(orr(vlox, T16B, vlo0, vlo1)); 6797 umov(chk, vhix, D, 1); ASCII(cm(LT, vlox, T16B, vlox)); 6798 fmovd(max, vhix); ASCII(umaxv(vlox, T16B, vlox)); 6799 orr(chk, chk, max); ASCII(umov(max, vlox, B, 0)); 6800 ASCII(orr(chk, chk, max)); 6801 cbnz(chk, FAIL_32); 6802 subw(cnt, cnt, 32); 6803 st1(vlo0, vlo1, T16B, Address(post(dst, 32))); 6804 b(LOOP_32); 6805 } 6806 BIND(FAIL_32); 6807 sub(src, src, 64); 6808 BIND(DONE_32); 6809 6810 Label LOOP_8, SKIP_8; 6811 6812 BIND(LOOP_8); 6813 { 6814 cmpw(cnt, 8); 6815 br(LT, SKIP_8); 6816 FloatRegister vhi = vtmp0; 6817 FloatRegister vlo = vtmp1; 6818 ld1(vtmp3, T8H, src); 6819 uzp1(vlo, T16B, vtmp3, vtmp3); 6820 uzp2(vhi, T16B, vtmp3, vtmp3); 6821 // ISO-check on hi-parts (all zero). 6822 // ASCII-check on lo-parts (no sign). 6823 ASCII(cm(LT, vtmp2, T16B, vlo)); 6824 fmovd(chk, vhi); ASCII(umaxv(vtmp2, T16B, vtmp2)); 6825 ASCII(umov(max, vtmp2, B, 0)); 6826 ASCII(orr(chk, chk, max)); 6827 cbnz(chk, SKIP_8); 6828 6829 strd(vlo, Address(post(dst, 8))); 6830 subw(cnt, cnt, 8); 6831 add(src, src, 16); 6832 b(LOOP_8); 6833 } 6834 BIND(SKIP_8); 6835 6836 #undef ASCII 6837 6838 Label LOOP, DONE; 6839 6840 cbz(cnt, DONE); 6841 BIND(LOOP); 6842 { 6843 Register chr = rscratch1; 6844 ldrh(chr, Address(post(src, 2))); 6845 tst(chr, ascii ? 0xff80 : 0xff00); 6846 br(NE, DONE); 6847 strb(chr, Address(post(dst, 1))); 6848 subs(cnt, cnt, 1); 6849 br(GT, LOOP); 6850 } 6851 BIND(DONE); 6852 // Return index where we stopped. 6853 subw(res, len, cnt); 6854 } 6855 6856 // Inflate byte[] array to char[]. 6857 // Clobbers: src, dst, len, rflags, rscratch1, v0-v6 6858 address MacroAssembler::byte_array_inflate(Register src, Register dst, Register len, 6859 FloatRegister vtmp1, FloatRegister vtmp2, 6860 FloatRegister vtmp3, Register tmp4) { 6861 Label big, done, after_init, to_stub; 6862 6863 assert_different_registers(src, dst, len, tmp4, rscratch1); 6864 6865 fmovd(vtmp1, 0.0); 6866 lsrw(tmp4, len, 3); 6867 bind(after_init); 6868 cbnzw(tmp4, big); 6869 // Short string: less than 8 bytes. 6870 { 6871 Label loop, tiny; 6872 6873 cmpw(len, 4); 6874 br(LT, tiny); 6875 // Use SIMD to do 4 bytes. 6876 ldrs(vtmp2, post(src, 4)); 6877 zip1(vtmp3, T8B, vtmp2, vtmp1); 6878 subw(len, len, 4); 6879 strd(vtmp3, post(dst, 8)); 6880 6881 cbzw(len, done); 6882 6883 // Do the remaining bytes by steam. 6884 bind(loop); 6885 ldrb(tmp4, post(src, 1)); 6886 strh(tmp4, post(dst, 2)); 6887 subw(len, len, 1); 6888 6889 bind(tiny); 6890 cbnz(len, loop); 6891 6892 b(done); 6893 } 6894 6895 if (SoftwarePrefetchHintDistance >= 0) { 6896 bind(to_stub); 6897 RuntimeAddress stub = RuntimeAddress(StubRoutines::aarch64::large_byte_array_inflate()); 6898 assert(stub.target() != nullptr, "large_byte_array_inflate stub has not been generated"); 6899 address tpc = trampoline_call(stub); 6900 if (tpc == nullptr) { 6901 DEBUG_ONLY(reset_labels(big, done)); 6902 postcond(pc() == badAddress); 6903 return nullptr; 6904 } 6905 b(after_init); 6906 } 6907 6908 // Unpack the bytes 8 at a time. 6909 bind(big); 6910 { 6911 Label loop, around, loop_last, loop_start; 6912 6913 if (SoftwarePrefetchHintDistance >= 0) { 6914 const int large_loop_threshold = (64 + 16)/8; 6915 ldrd(vtmp2, post(src, 8)); 6916 andw(len, len, 7); 6917 cmp(tmp4, (u1)large_loop_threshold); 6918 br(GE, to_stub); 6919 b(loop_start); 6920 6921 bind(loop); 6922 ldrd(vtmp2, post(src, 8)); 6923 bind(loop_start); 6924 subs(tmp4, tmp4, 1); 6925 br(EQ, loop_last); 6926 zip1(vtmp2, T16B, vtmp2, vtmp1); 6927 ldrd(vtmp3, post(src, 8)); 6928 st1(vtmp2, T8H, post(dst, 16)); 6929 subs(tmp4, tmp4, 1); 6930 zip1(vtmp3, T16B, vtmp3, vtmp1); 6931 st1(vtmp3, T8H, post(dst, 16)); 6932 br(NE, loop); 6933 b(around); 6934 bind(loop_last); 6935 zip1(vtmp2, T16B, vtmp2, vtmp1); 6936 st1(vtmp2, T8H, post(dst, 16)); 6937 bind(around); 6938 cbz(len, done); 6939 } else { 6940 andw(len, len, 7); 6941 bind(loop); 6942 ldrd(vtmp2, post(src, 8)); 6943 sub(tmp4, tmp4, 1); 6944 zip1(vtmp3, T16B, vtmp2, vtmp1); 6945 st1(vtmp3, T8H, post(dst, 16)); 6946 cbnz(tmp4, loop); 6947 } 6948 } 6949 6950 // Do the tail of up to 8 bytes. 6951 add(src, src, len); 6952 ldrd(vtmp3, Address(src, -8)); 6953 add(dst, dst, len, ext::uxtw, 1); 6954 zip1(vtmp3, T16B, vtmp3, vtmp1); 6955 strq(vtmp3, Address(dst, -16)); 6956 6957 bind(done); 6958 postcond(pc() != badAddress); 6959 return pc(); 6960 } 6961 6962 // Compress char[] array to byte[]. 6963 // Intrinsic for java.lang.StringUTF16.compress(char[] src, int srcOff, byte[] dst, int dstOff, int len) 6964 // Return the array length if every element in array can be encoded, 6965 // otherwise, the index of first non-latin1 (> 0xff) character. 6966 void MacroAssembler::char_array_compress(Register src, Register dst, Register len, 6967 Register res, 6968 FloatRegister tmp0, FloatRegister tmp1, 6969 FloatRegister tmp2, FloatRegister tmp3, 6970 FloatRegister tmp4, FloatRegister tmp5) { 6971 encode_iso_array(src, dst, len, res, false, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5); 6972 } 6973 6974 // java.math.round(double a) 6975 // Returns the closest long to the argument, with ties rounding to 6976 // positive infinity. This requires some fiddling for corner 6977 // cases. We take care to avoid double rounding in e.g. (jlong)(a + 0.5). 6978 void MacroAssembler::java_round_double(Register dst, FloatRegister src, 6979 FloatRegister ftmp) { 6980 Label DONE; 6981 BLOCK_COMMENT("java_round_double: { "); 6982 fmovd(rscratch1, src); 6983 // Use RoundToNearestTiesAway unless src small and -ve. 6984 fcvtasd(dst, src); 6985 // Test if src >= 0 || abs(src) >= 0x1.0p52 6986 eor(rscratch1, rscratch1, UCONST64(1) << 63); // flip sign bit 6987 mov(rscratch2, julong_cast(0x1.0p52)); 6988 cmp(rscratch1, rscratch2); 6989 br(HS, DONE); { 6990 // src < 0 && abs(src) < 0x1.0p52 6991 // src may have a fractional part, so add 0.5 6992 fmovd(ftmp, 0.5); 6993 faddd(ftmp, src, ftmp); 6994 // Convert double to jlong, use RoundTowardsNegative 6995 fcvtmsd(dst, ftmp); 6996 } 6997 bind(DONE); 6998 BLOCK_COMMENT("} java_round_double"); 6999 } 7000 7001 void MacroAssembler::java_round_float(Register dst, FloatRegister src, 7002 FloatRegister ftmp) { 7003 Label DONE; 7004 BLOCK_COMMENT("java_round_float: { "); 7005 fmovs(rscratch1, src); 7006 // Use RoundToNearestTiesAway unless src small and -ve. 7007 fcvtassw(dst, src); 7008 // Test if src >= 0 || abs(src) >= 0x1.0p23 7009 eor(rscratch1, rscratch1, 0x80000000); // flip sign bit 7010 mov(rscratch2, jint_cast(0x1.0p23f)); 7011 cmp(rscratch1, rscratch2); 7012 br(HS, DONE); { 7013 // src < 0 && |src| < 0x1.0p23 7014 // src may have a fractional part, so add 0.5 7015 fmovs(ftmp, 0.5f); 7016 fadds(ftmp, src, ftmp); 7017 // Convert float to jint, use RoundTowardsNegative 7018 fcvtmssw(dst, ftmp); 7019 } 7020 bind(DONE); 7021 BLOCK_COMMENT("} java_round_float"); 7022 } 7023 7024 // get_thread() can be called anywhere inside generated code so we 7025 // need to save whatever non-callee save context might get clobbered 7026 // by the call to JavaThread::aarch64_get_thread_helper() or, indeed, 7027 // the call setup code. 7028 // 7029 // On Linux, aarch64_get_thread_helper() clobbers only r0, r1, and flags. 7030 // On other systems, the helper is a usual C function. 7031 // 7032 void MacroAssembler::get_thread(Register dst) { 7033 RegSet saved_regs = 7034 LINUX_ONLY(RegSet::range(r0, r1) + lr - dst) 7035 NOT_LINUX (RegSet::range(r0, r17) + lr - dst); 7036 7037 protect_return_address(); 7038 push(saved_regs, sp); 7039 7040 mov(lr, CAST_FROM_FN_PTR(address, JavaThread::aarch64_get_thread_helper)); 7041 blr(lr); 7042 if (dst != c_rarg0) { 7043 mov(dst, c_rarg0); 7044 } 7045 7046 pop(saved_regs, sp); 7047 authenticate_return_address(); 7048 } 7049 7050 #ifdef COMPILER2 7051 // C2 compiled method's prolog code 7052 // Moved here from aarch64.ad to support Valhalla code belows 7053 void MacroAssembler::verified_entry(Compile* C, int sp_inc) { 7054 if (C->clinit_barrier_on_entry()) { 7055 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started"); 7056 7057 Label L_skip_barrier; 7058 7059 mov_metadata(rscratch2, C->method()->holder()->constant_encoding()); 7060 clinit_barrier(rscratch2, rscratch1, &L_skip_barrier); 7061 far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); 7062 bind(L_skip_barrier); 7063 } 7064 7065 if (C->max_vector_size() > 0) { 7066 reinitialize_ptrue(); 7067 } 7068 7069 int bangsize = C->output()->bang_size_in_bytes(); 7070 if (C->output()->need_stack_bang(bangsize)) 7071 generate_stack_overflow_check(bangsize); 7072 7073 // n.b. frame size includes space for return pc and rfp 7074 const long framesize = C->output()->frame_size_in_bytes(); 7075 build_frame(framesize); 7076 7077 if (C->needs_stack_repair()) { 7078 save_stack_increment(sp_inc, framesize); 7079 } 7080 7081 if (VerifyStackAtCalls) { 7082 Unimplemented(); 7083 } 7084 } 7085 #endif // COMPILER2 7086 7087 int MacroAssembler::store_inline_type_fields_to_buf(ciInlineKlass* vk, bool from_interpreter) { 7088 assert(InlineTypeReturnedAsFields, "Inline types should never be returned as fields"); 7089 // An inline type might be returned. If fields are in registers we 7090 // need to allocate an inline type instance and initialize it with 7091 // the value of the fields. 7092 Label skip; 7093 // We only need a new buffered inline type if a new one is not returned 7094 tbz(r0, 0, skip); 7095 int call_offset = -1; 7096 7097 // Be careful not to clobber r1-7 which hold returned fields 7098 // Also do not use callee-saved registers as these may be live in the interpreter 7099 Register tmp1 = r13, tmp2 = r14, klass = r15, r0_preserved = r12; 7100 7101 // The following code is similar to allocate_instance but has some slight differences, 7102 // e.g. object size is always not zero, sometimes it's constant; storing klass ptr after 7103 // allocating is not necessary if vk != nullptr, etc. allocate_instance is not aware of these. 7104 Label slow_case; 7105 // 1. Try to allocate a new buffered inline instance either from TLAB or eden space 7106 mov(r0_preserved, r0); // save r0 for slow_case since *_allocate may corrupt it when allocation failed 7107 7108 if (vk != nullptr) { 7109 // Called from C1, where the return type is statically known. 7110 movptr(klass, (intptr_t)vk->get_InlineKlass()); 7111 jint lh = vk->layout_helper(); 7112 assert(lh != Klass::_lh_neutral_value, "inline class in return type must have been resolved"); 7113 if (UseTLAB && !Klass::layout_helper_needs_slow_path(lh)) { 7114 tlab_allocate(r0, noreg, lh, tmp1, tmp2, slow_case); 7115 } else { 7116 b(slow_case); 7117 } 7118 } else { 7119 // Call from interpreter. R0 contains ((the InlineKlass* of the return type) | 0x01) 7120 andr(klass, r0, -2); 7121 if (UseTLAB) { 7122 ldrw(tmp2, Address(klass, Klass::layout_helper_offset())); 7123 tst(tmp2, Klass::_lh_instance_slow_path_bit); 7124 br(Assembler::NE, slow_case); 7125 tlab_allocate(r0, tmp2, 0, tmp1, tmp2, slow_case); 7126 } else { 7127 b(slow_case); 7128 } 7129 } 7130 if (UseTLAB) { 7131 // 2. Initialize buffered inline instance header 7132 Register buffer_obj = r0; 7133 if (UseCompactObjectHeaders) { 7134 ldr(rscratch1, Address(klass, Klass::prototype_header_offset())); 7135 str(rscratch1, Address(buffer_obj, oopDesc::mark_offset_in_bytes())); 7136 } else { 7137 mov(rscratch1, (intptr_t)markWord::inline_type_prototype().value()); 7138 str(rscratch1, Address(buffer_obj, oopDesc::mark_offset_in_bytes())); 7139 store_klass_gap(buffer_obj, zr); 7140 if (vk == nullptr) { 7141 // store_klass corrupts klass, so save it for later use (interpreter case only). 7142 mov(tmp1, klass); 7143 } 7144 store_klass(buffer_obj, klass); 7145 } 7146 // 3. Initialize its fields with an inline class specific handler 7147 if (vk != nullptr) { 7148 far_call(RuntimeAddress(vk->pack_handler())); // no need for call info as this will not safepoint. 7149 } else { 7150 // tmp1 holds klass preserved above 7151 ldr(tmp1, Address(tmp1, InstanceKlass::adr_inlineklass_fixed_block_offset())); 7152 ldr(tmp1, Address(tmp1, InlineKlass::pack_handler_offset())); 7153 blr(tmp1); 7154 } 7155 7156 membar(Assembler::StoreStore); 7157 b(skip); 7158 } else { 7159 // Must have already branched to slow_case above. 7160 DEBUG_ONLY(should_not_reach_here()); 7161 } 7162 bind(slow_case); 7163 // We failed to allocate a new inline type, fall back to a runtime 7164 // call. Some oop field may be live in some registers but we can't 7165 // tell. That runtime call will take care of preserving them 7166 // across a GC if there's one. 7167 mov(r0, r0_preserved); 7168 7169 if (from_interpreter) { 7170 super_call_VM_leaf(StubRoutines::store_inline_type_fields_to_buf()); 7171 } else { 7172 far_call(RuntimeAddress(StubRoutines::store_inline_type_fields_to_buf())); 7173 call_offset = offset(); 7174 } 7175 membar(Assembler::StoreStore); 7176 7177 bind(skip); 7178 return call_offset; 7179 } 7180 7181 // Move a value between registers/stack slots and update the reg_state 7182 bool MacroAssembler::move_helper(VMReg from, VMReg to, BasicType bt, RegState reg_state[]) { 7183 assert(from->is_valid() && to->is_valid(), "source and destination must be valid"); 7184 if (reg_state[to->value()] == reg_written) { 7185 return true; // Already written 7186 } 7187 7188 if (from != to && bt != T_VOID) { 7189 if (reg_state[to->value()] == reg_readonly) { 7190 return false; // Not yet writable 7191 } 7192 if (from->is_reg()) { 7193 if (to->is_reg()) { 7194 if (from->is_Register() && to->is_Register()) { 7195 mov(to->as_Register(), from->as_Register()); 7196 } else if (from->is_FloatRegister() && to->is_FloatRegister()) { 7197 fmovd(to->as_FloatRegister(), from->as_FloatRegister()); 7198 } else { 7199 ShouldNotReachHere(); 7200 } 7201 } else { 7202 int st_off = to->reg2stack() * VMRegImpl::stack_slot_size; 7203 Address to_addr = Address(sp, st_off); 7204 if (from->is_FloatRegister()) { 7205 if (bt == T_DOUBLE) { 7206 strd(from->as_FloatRegister(), to_addr); 7207 } else { 7208 assert(bt == T_FLOAT, "must be float"); 7209 strs(from->as_FloatRegister(), to_addr); 7210 } 7211 } else { 7212 str(from->as_Register(), to_addr); 7213 } 7214 } 7215 } else { 7216 Address from_addr = Address(sp, from->reg2stack() * VMRegImpl::stack_slot_size); 7217 if (to->is_reg()) { 7218 if (to->is_FloatRegister()) { 7219 if (bt == T_DOUBLE) { 7220 ldrd(to->as_FloatRegister(), from_addr); 7221 } else { 7222 assert(bt == T_FLOAT, "must be float"); 7223 ldrs(to->as_FloatRegister(), from_addr); 7224 } 7225 } else { 7226 ldr(to->as_Register(), from_addr); 7227 } 7228 } else { 7229 int st_off = to->reg2stack() * VMRegImpl::stack_slot_size; 7230 ldr(rscratch1, from_addr); 7231 str(rscratch1, Address(sp, st_off)); 7232 } 7233 } 7234 } 7235 7236 // Update register states 7237 reg_state[from->value()] = reg_writable; 7238 reg_state[to->value()] = reg_written; 7239 return true; 7240 } 7241 7242 // Calculate the extra stack space required for packing or unpacking inline 7243 // args and adjust the stack pointer 7244 int MacroAssembler::extend_stack_for_inline_args(int args_on_stack) { 7245 int sp_inc = args_on_stack * VMRegImpl::stack_slot_size; 7246 sp_inc = align_up(sp_inc, StackAlignmentInBytes); 7247 assert(sp_inc > 0, "sanity"); 7248 7249 // Save a copy of the FP and LR here for deoptimization patching and frame walking 7250 stp(rfp, lr, Address(pre(sp, -2 * wordSize))); 7251 7252 // Adjust the stack pointer. This will be repaired on return by MacroAssembler::remove_frame 7253 if (sp_inc < (1 << 9)) { 7254 sub(sp, sp, sp_inc); // Fits in an immediate 7255 } else { 7256 mov(rscratch1, sp_inc); 7257 sub(sp, sp, rscratch1); 7258 } 7259 7260 return sp_inc + 2 * wordSize; // Account for the FP/LR space 7261 } 7262 7263 // Read all fields from an inline type oop and store the values in registers/stack slots 7264 bool MacroAssembler::unpack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index, 7265 VMReg from, int& from_index, VMRegPair* to, int to_count, int& to_index, 7266 RegState reg_state[]) { 7267 assert(sig->at(sig_index)._bt == T_VOID, "should be at end delimiter"); 7268 assert(from->is_valid(), "source must be valid"); 7269 bool progress = false; 7270 #ifdef ASSERT 7271 const int start_offset = offset(); 7272 #endif 7273 7274 Label L_null, L_notNull; 7275 // Don't use r14 as tmp because it's used for spilling (see MacroAssembler::spill_reg_for) 7276 Register tmp1 = r10; 7277 Register tmp2 = r11; 7278 Register fromReg = noreg; 7279 ScalarizedInlineArgsStream stream(sig, sig_index, to, to_count, to_index, -1); 7280 bool done = true; 7281 bool mark_done = true; 7282 VMReg toReg; 7283 BasicType bt; 7284 // Check if argument requires a null check 7285 bool null_check = false; 7286 VMReg nullCheckReg; 7287 while (stream.next(nullCheckReg, bt)) { 7288 if (sig->at(stream.sig_index())._offset == -1) { 7289 null_check = true; 7290 break; 7291 } 7292 } 7293 stream.reset(sig_index, to_index); 7294 while (stream.next(toReg, bt)) { 7295 assert(toReg->is_valid(), "destination must be valid"); 7296 int idx = (int)toReg->value(); 7297 if (reg_state[idx] == reg_readonly) { 7298 if (idx != from->value()) { 7299 mark_done = false; 7300 } 7301 done = false; 7302 continue; 7303 } else if (reg_state[idx] == reg_written) { 7304 continue; 7305 } 7306 assert(reg_state[idx] == reg_writable, "must be writable"); 7307 reg_state[idx] = reg_written; 7308 progress = true; 7309 7310 if (fromReg == noreg) { 7311 if (from->is_reg()) { 7312 fromReg = from->as_Register(); 7313 } else { 7314 int st_off = from->reg2stack() * VMRegImpl::stack_slot_size; 7315 ldr(tmp1, Address(sp, st_off)); 7316 fromReg = tmp1; 7317 } 7318 if (null_check) { 7319 // Nullable inline type argument, emit null check 7320 cbz(fromReg, L_null); 7321 } 7322 } 7323 int off = sig->at(stream.sig_index())._offset; 7324 if (off == -1) { 7325 assert(null_check, "Missing null check at"); 7326 if (toReg->is_stack()) { 7327 int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size; 7328 mov(tmp2, 1); 7329 str(tmp2, Address(sp, st_off)); 7330 } else { 7331 mov(toReg->as_Register(), 1); 7332 } 7333 continue; 7334 } 7335 assert(off > 0, "offset in object should be positive"); 7336 Address fromAddr = Address(fromReg, off); 7337 if (!toReg->is_FloatRegister()) { 7338 Register dst = toReg->is_stack() ? tmp2 : toReg->as_Register(); 7339 if (is_reference_type(bt)) { 7340 load_heap_oop(dst, fromAddr, rscratch1, rscratch2); 7341 } else { 7342 bool is_signed = (bt != T_CHAR) && (bt != T_BOOLEAN); 7343 load_sized_value(dst, fromAddr, type2aelembytes(bt), is_signed); 7344 } 7345 if (toReg->is_stack()) { 7346 int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size; 7347 str(dst, Address(sp, st_off)); 7348 } 7349 } else if (bt == T_DOUBLE) { 7350 ldrd(toReg->as_FloatRegister(), fromAddr); 7351 } else { 7352 assert(bt == T_FLOAT, "must be float"); 7353 ldrs(toReg->as_FloatRegister(), fromAddr); 7354 } 7355 } 7356 if (progress && null_check) { 7357 if (done) { 7358 b(L_notNull); 7359 bind(L_null); 7360 // Set IsInit field to zero to signal that the argument is null. 7361 // Also set all oop fields to zero to make the GC happy. 7362 stream.reset(sig_index, to_index); 7363 while (stream.next(toReg, bt)) { 7364 if (sig->at(stream.sig_index())._offset == -1 || 7365 bt == T_OBJECT || bt == T_ARRAY) { 7366 if (toReg->is_stack()) { 7367 int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size; 7368 str(zr, Address(sp, st_off)); 7369 } else { 7370 mov(toReg->as_Register(), zr); 7371 } 7372 } 7373 } 7374 bind(L_notNull); 7375 } else { 7376 bind(L_null); 7377 } 7378 } 7379 7380 sig_index = stream.sig_index(); 7381 to_index = stream.regs_index(); 7382 7383 if (mark_done && reg_state[from->value()] != reg_written) { 7384 // This is okay because no one else will write to that slot 7385 reg_state[from->value()] = reg_writable; 7386 } 7387 from_index--; 7388 assert(progress || (start_offset == offset()), "should not emit code"); 7389 return done; 7390 } 7391 7392 // Pack fields back into an inline type oop 7393 bool MacroAssembler::pack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index, int vtarg_index, 7394 VMRegPair* from, int from_count, int& from_index, VMReg to, 7395 RegState reg_state[], Register val_array) { 7396 assert(sig->at(sig_index)._bt == T_METADATA, "should be at delimiter"); 7397 assert(to->is_valid(), "destination must be valid"); 7398 7399 if (reg_state[to->value()] == reg_written) { 7400 skip_unpacked_fields(sig, sig_index, from, from_count, from_index); 7401 return true; // Already written 7402 } 7403 7404 // The GC barrier expanded by store_heap_oop below may call into the 7405 // runtime so use callee-saved registers for any values that need to be 7406 // preserved. The GC barrier assembler should take care of saving the 7407 // Java argument registers. 7408 // TODO 8284443 Isn't it an issue if below code uses r14 as tmp when it contains a spilled value? 7409 // Be careful with r14 because it's used for spilling (see MacroAssembler::spill_reg_for). 7410 Register val_obj_tmp = r21; 7411 Register from_reg_tmp = r22; 7412 Register tmp1 = r14; 7413 Register tmp2 = r13; 7414 Register tmp3 = r12; 7415 Register val_obj = to->is_stack() ? val_obj_tmp : to->as_Register(); 7416 7417 assert_different_registers(val_obj_tmp, from_reg_tmp, tmp1, tmp2, tmp3, val_array); 7418 7419 if (reg_state[to->value()] == reg_readonly) { 7420 if (!is_reg_in_unpacked_fields(sig, sig_index, to, from, from_count, from_index)) { 7421 skip_unpacked_fields(sig, sig_index, from, from_count, from_index); 7422 return false; // Not yet writable 7423 } 7424 val_obj = val_obj_tmp; 7425 } 7426 7427 int index = arrayOopDesc::base_offset_in_bytes(T_OBJECT) + vtarg_index * type2aelembytes(T_OBJECT); 7428 load_heap_oop(val_obj, Address(val_array, index), tmp1, tmp2); 7429 7430 ScalarizedInlineArgsStream stream(sig, sig_index, from, from_count, from_index); 7431 VMReg fromReg; 7432 BasicType bt; 7433 Label L_null; 7434 while (stream.next(fromReg, bt)) { 7435 assert(fromReg->is_valid(), "source must be valid"); 7436 reg_state[fromReg->value()] = reg_writable; 7437 7438 int off = sig->at(stream.sig_index())._offset; 7439 if (off == -1) { 7440 // Nullable inline type argument, emit null check 7441 Label L_notNull; 7442 if (fromReg->is_stack()) { 7443 int ld_off = fromReg->reg2stack() * VMRegImpl::stack_slot_size; 7444 ldrb(tmp2, Address(sp, ld_off)); 7445 cbnz(tmp2, L_notNull); 7446 } else { 7447 cbnz(fromReg->as_Register(), L_notNull); 7448 } 7449 mov(val_obj, 0); 7450 b(L_null); 7451 bind(L_notNull); 7452 continue; 7453 } 7454 7455 assert(off > 0, "offset in object should be positive"); 7456 size_t size_in_bytes = is_java_primitive(bt) ? type2aelembytes(bt) : wordSize; 7457 7458 // Pack the scalarized field into the value object. 7459 Address dst(val_obj, off); 7460 7461 if (!fromReg->is_FloatRegister()) { 7462 Register src; 7463 if (fromReg->is_stack()) { 7464 src = from_reg_tmp; 7465 int ld_off = fromReg->reg2stack() * VMRegImpl::stack_slot_size; 7466 load_sized_value(src, Address(sp, ld_off), size_in_bytes, /* is_signed */ false); 7467 } else { 7468 src = fromReg->as_Register(); 7469 } 7470 assert_different_registers(dst.base(), src, tmp1, tmp2, tmp3, val_array); 7471 if (is_reference_type(bt)) { 7472 store_heap_oop(dst, src, tmp1, tmp2, tmp3, IN_HEAP | ACCESS_WRITE | IS_DEST_UNINITIALIZED); 7473 } else { 7474 store_sized_value(dst, src, size_in_bytes); 7475 } 7476 } else if (bt == T_DOUBLE) { 7477 strd(fromReg->as_FloatRegister(), dst); 7478 } else { 7479 assert(bt == T_FLOAT, "must be float"); 7480 strs(fromReg->as_FloatRegister(), dst); 7481 } 7482 } 7483 bind(L_null); 7484 sig_index = stream.sig_index(); 7485 from_index = stream.regs_index(); 7486 7487 assert(reg_state[to->value()] == reg_writable, "must have already been read"); 7488 bool success = move_helper(val_obj->as_VMReg(), to, T_OBJECT, reg_state); 7489 assert(success, "to register must be writeable"); 7490 7491 return true; 7492 } 7493 7494 VMReg MacroAssembler::spill_reg_for(VMReg reg) { 7495 return (reg->is_FloatRegister()) ? v8->as_VMReg() : r14->as_VMReg(); 7496 } 7497 7498 void MacroAssembler::cache_wb(Address line) { 7499 assert(line.getMode() == Address::base_plus_offset, "mode should be base_plus_offset"); 7500 assert(line.index() == noreg, "index should be noreg"); 7501 assert(line.offset() == 0, "offset should be 0"); 7502 // would like to assert this 7503 // assert(line._ext.shift == 0, "shift should be zero"); 7504 if (VM_Version::supports_dcpop()) { 7505 // writeback using clear virtual address to point of persistence 7506 dc(Assembler::CVAP, line.base()); 7507 } else { 7508 // no need to generate anything as Unsafe.writebackMemory should 7509 // never invoke this stub 7510 } 7511 } 7512 7513 void MacroAssembler::cache_wbsync(bool is_pre) { 7514 // we only need a barrier post sync 7515 if (!is_pre) { 7516 membar(Assembler::AnyAny); 7517 } 7518 } 7519 7520 void MacroAssembler::verify_sve_vector_length(Register tmp) { 7521 if (!UseSVE || VM_Version::get_max_supported_sve_vector_length() == FloatRegister::sve_vl_min) { 7522 return; 7523 } 7524 // Make sure that native code does not change SVE vector length. 7525 Label verify_ok; 7526 movw(tmp, zr); 7527 sve_inc(tmp, B); 7528 subsw(zr, tmp, VM_Version::get_initial_sve_vector_length()); 7529 br(EQ, verify_ok); 7530 stop("Error: SVE vector length has changed since jvm startup"); 7531 bind(verify_ok); 7532 } 7533 7534 void MacroAssembler::verify_ptrue() { 7535 Label verify_ok; 7536 if (!UseSVE) { 7537 return; 7538 } 7539 sve_cntp(rscratch1, B, ptrue, ptrue); // get true elements count. 7540 sve_dec(rscratch1, B); 7541 cbz(rscratch1, verify_ok); 7542 stop("Error: the preserved predicate register (p7) elements are not all true"); 7543 bind(verify_ok); 7544 } 7545 7546 void MacroAssembler::safepoint_isb() { 7547 isb(); 7548 #ifndef PRODUCT 7549 if (VerifyCrossModifyFence) { 7550 // Clear the thread state. 7551 strb(zr, Address(rthread, in_bytes(JavaThread::requires_cross_modify_fence_offset()))); 7552 } 7553 #endif 7554 } 7555 7556 #ifndef PRODUCT 7557 void MacroAssembler::verify_cross_modify_fence_not_required() { 7558 if (VerifyCrossModifyFence) { 7559 // Check if thread needs a cross modify fence. 7560 ldrb(rscratch1, Address(rthread, in_bytes(JavaThread::requires_cross_modify_fence_offset()))); 7561 Label fence_not_required; 7562 cbz(rscratch1, fence_not_required); 7563 // If it does then fail. 7564 lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, JavaThread::verify_cross_modify_fence_failure))); 7565 mov(c_rarg0, rthread); 7566 blr(rscratch1); 7567 bind(fence_not_required); 7568 } 7569 } 7570 #endif 7571 7572 void MacroAssembler::spin_wait() { 7573 for (int i = 0; i < VM_Version::spin_wait_desc().inst_count(); ++i) { 7574 switch (VM_Version::spin_wait_desc().inst()) { 7575 case SpinWait::NOP: 7576 nop(); 7577 break; 7578 case SpinWait::ISB: 7579 isb(); 7580 break; 7581 case SpinWait::YIELD: 7582 yield(); 7583 break; 7584 default: 7585 ShouldNotReachHere(); 7586 } 7587 } 7588 } 7589 7590 // Stack frame creation/removal 7591 7592 void MacroAssembler::enter(bool strip_ret_addr) { 7593 if (strip_ret_addr) { 7594 // Addresses can only be signed once. If there are multiple nested frames being created 7595 // in the same function, then the return address needs stripping first. 7596 strip_return_address(); 7597 } 7598 protect_return_address(); 7599 stp(rfp, lr, Address(pre(sp, -2 * wordSize))); 7600 mov(rfp, sp); 7601 } 7602 7603 void MacroAssembler::leave() { 7604 mov(sp, rfp); 7605 ldp(rfp, lr, Address(post(sp, 2 * wordSize))); 7606 authenticate_return_address(); 7607 } 7608 7609 // ROP Protection 7610 // Use the AArch64 PAC feature to add ROP protection for generated code. Use whenever creating/ 7611 // destroying stack frames or whenever directly loading/storing the LR to memory. 7612 // If ROP protection is not set then these functions are no-ops. 7613 // For more details on PAC see pauth_aarch64.hpp. 7614 7615 // Sign the LR. Use during construction of a stack frame, before storing the LR to memory. 7616 // Uses value zero as the modifier. 7617 // 7618 void MacroAssembler::protect_return_address() { 7619 if (VM_Version::use_rop_protection()) { 7620 check_return_address(); 7621 paciaz(); 7622 } 7623 } 7624 7625 // Sign the return value in the given register. Use before updating the LR in the existing stack 7626 // frame for the current function. 7627 // Uses value zero as the modifier. 7628 // 7629 void MacroAssembler::protect_return_address(Register return_reg) { 7630 if (VM_Version::use_rop_protection()) { 7631 check_return_address(return_reg); 7632 paciza(return_reg); 7633 } 7634 } 7635 7636 // Authenticate the LR. Use before function return, after restoring FP and loading LR from memory. 7637 // Uses value zero as the modifier. 7638 // 7639 void MacroAssembler::authenticate_return_address() { 7640 if (VM_Version::use_rop_protection()) { 7641 autiaz(); 7642 check_return_address(); 7643 } 7644 } 7645 7646 // Authenticate the return value in the given register. Use before updating the LR in the existing 7647 // stack frame for the current function. 7648 // Uses value zero as the modifier. 7649 // 7650 void MacroAssembler::authenticate_return_address(Register return_reg) { 7651 if (VM_Version::use_rop_protection()) { 7652 autiza(return_reg); 7653 check_return_address(return_reg); 7654 } 7655 } 7656 7657 // Strip any PAC data from LR without performing any authentication. Use with caution - only if 7658 // there is no guaranteed way of authenticating the LR. 7659 // 7660 void MacroAssembler::strip_return_address() { 7661 if (VM_Version::use_rop_protection()) { 7662 xpaclri(); 7663 } 7664 } 7665 7666 #ifndef PRODUCT 7667 // PAC failures can be difficult to debug. After an authentication failure, a segfault will only 7668 // occur when the pointer is used - ie when the program returns to the invalid LR. At this point 7669 // it is difficult to debug back to the callee function. 7670 // This function simply loads from the address in the given register. 7671 // Use directly after authentication to catch authentication failures. 7672 // Also use before signing to check that the pointer is valid and hasn't already been signed. 7673 // 7674 void MacroAssembler::check_return_address(Register return_reg) { 7675 if (VM_Version::use_rop_protection()) { 7676 ldr(zr, Address(return_reg)); 7677 } 7678 } 7679 #endif 7680 7681 // The java_calling_convention describes stack locations as ideal slots on 7682 // a frame with no abi restrictions. Since we must observe abi restrictions 7683 // (like the placement of the register window) the slots must be biased by 7684 // the following value. 7685 static int reg2offset_in(VMReg r) { 7686 // Account for saved rfp and lr 7687 // This should really be in_preserve_stack_slots 7688 return (r->reg2stack() + 4) * VMRegImpl::stack_slot_size; 7689 } 7690 7691 static int reg2offset_out(VMReg r) { 7692 return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size; 7693 } 7694 7695 // On 64bit we will store integer like items to the stack as 7696 // 64bits items (AArch64 ABI) even though java would only store 7697 // 32bits for a parameter. On 32bit it will simply be 32bits 7698 // So this routine will do 32->32 on 32bit and 32->64 on 64bit 7699 void MacroAssembler::move32_64(VMRegPair src, VMRegPair dst, Register tmp) { 7700 if (src.first()->is_stack()) { 7701 if (dst.first()->is_stack()) { 7702 // stack to stack 7703 ldr(tmp, Address(rfp, reg2offset_in(src.first()))); 7704 str(tmp, Address(sp, reg2offset_out(dst.first()))); 7705 } else { 7706 // stack to reg 7707 ldrsw(dst.first()->as_Register(), Address(rfp, reg2offset_in(src.first()))); 7708 } 7709 } else if (dst.first()->is_stack()) { 7710 // reg to stack 7711 str(src.first()->as_Register(), Address(sp, reg2offset_out(dst.first()))); 7712 } else { 7713 if (dst.first() != src.first()) { 7714 sxtw(dst.first()->as_Register(), src.first()->as_Register()); 7715 } 7716 } 7717 } 7718 7719 // An oop arg. Must pass a handle not the oop itself 7720 void MacroAssembler::object_move( 7721 OopMap* map, 7722 int oop_handle_offset, 7723 int framesize_in_slots, 7724 VMRegPair src, 7725 VMRegPair dst, 7726 bool is_receiver, 7727 int* receiver_offset) { 7728 7729 // must pass a handle. First figure out the location we use as a handle 7730 7731 Register rHandle = dst.first()->is_stack() ? rscratch2 : dst.first()->as_Register(); 7732 7733 // See if oop is null if it is we need no handle 7734 7735 if (src.first()->is_stack()) { 7736 7737 // Oop is already on the stack as an argument 7738 int offset_in_older_frame = src.first()->reg2stack() + SharedRuntime::out_preserve_stack_slots(); 7739 map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + framesize_in_slots)); 7740 if (is_receiver) { 7741 *receiver_offset = (offset_in_older_frame + framesize_in_slots) * VMRegImpl::stack_slot_size; 7742 } 7743 7744 ldr(rscratch1, Address(rfp, reg2offset_in(src.first()))); 7745 lea(rHandle, Address(rfp, reg2offset_in(src.first()))); 7746 // conditionally move a null 7747 cmp(rscratch1, zr); 7748 csel(rHandle, zr, rHandle, Assembler::EQ); 7749 } else { 7750 7751 // Oop is in an a register we must store it to the space we reserve 7752 // on the stack for oop_handles and pass a handle if oop is non-null 7753 7754 const Register rOop = src.first()->as_Register(); 7755 int oop_slot; 7756 if (rOop == j_rarg0) 7757 oop_slot = 0; 7758 else if (rOop == j_rarg1) 7759 oop_slot = 1; 7760 else if (rOop == j_rarg2) 7761 oop_slot = 2; 7762 else if (rOop == j_rarg3) 7763 oop_slot = 3; 7764 else if (rOop == j_rarg4) 7765 oop_slot = 4; 7766 else if (rOop == j_rarg5) 7767 oop_slot = 5; 7768 else if (rOop == j_rarg6) 7769 oop_slot = 6; 7770 else { 7771 assert(rOop == j_rarg7, "wrong register"); 7772 oop_slot = 7; 7773 } 7774 7775 oop_slot = oop_slot * VMRegImpl::slots_per_word + oop_handle_offset; 7776 int offset = oop_slot*VMRegImpl::stack_slot_size; 7777 7778 map->set_oop(VMRegImpl::stack2reg(oop_slot)); 7779 // Store oop in handle area, may be null 7780 str(rOop, Address(sp, offset)); 7781 if (is_receiver) { 7782 *receiver_offset = offset; 7783 } 7784 7785 cmp(rOop, zr); 7786 lea(rHandle, Address(sp, offset)); 7787 // conditionally move a null 7788 csel(rHandle, zr, rHandle, Assembler::EQ); 7789 } 7790 7791 // If arg is on the stack then place it otherwise it is already in correct reg. 7792 if (dst.first()->is_stack()) { 7793 str(rHandle, Address(sp, reg2offset_out(dst.first()))); 7794 } 7795 } 7796 7797 // A float arg may have to do float reg int reg conversion 7798 void MacroAssembler::float_move(VMRegPair src, VMRegPair dst, Register tmp) { 7799 if (src.first()->is_stack()) { 7800 if (dst.first()->is_stack()) { 7801 ldrw(tmp, Address(rfp, reg2offset_in(src.first()))); 7802 strw(tmp, Address(sp, reg2offset_out(dst.first()))); 7803 } else { 7804 ldrs(dst.first()->as_FloatRegister(), Address(rfp, reg2offset_in(src.first()))); 7805 } 7806 } else if (src.first() != dst.first()) { 7807 if (src.is_single_phys_reg() && dst.is_single_phys_reg()) 7808 fmovs(dst.first()->as_FloatRegister(), src.first()->as_FloatRegister()); 7809 else 7810 strs(src.first()->as_FloatRegister(), Address(sp, reg2offset_out(dst.first()))); 7811 } 7812 } 7813 7814 // A long move 7815 void MacroAssembler::long_move(VMRegPair src, VMRegPair dst, Register tmp) { 7816 if (src.first()->is_stack()) { 7817 if (dst.first()->is_stack()) { 7818 // stack to stack 7819 ldr(tmp, Address(rfp, reg2offset_in(src.first()))); 7820 str(tmp, Address(sp, reg2offset_out(dst.first()))); 7821 } else { 7822 // stack to reg 7823 ldr(dst.first()->as_Register(), Address(rfp, reg2offset_in(src.first()))); 7824 } 7825 } else if (dst.first()->is_stack()) { 7826 // reg to stack 7827 // Do we really have to sign extend??? 7828 // __ movslq(src.first()->as_Register(), src.first()->as_Register()); 7829 str(src.first()->as_Register(), Address(sp, reg2offset_out(dst.first()))); 7830 } else { 7831 if (dst.first() != src.first()) { 7832 mov(dst.first()->as_Register(), src.first()->as_Register()); 7833 } 7834 } 7835 } 7836 7837 7838 // A double move 7839 void MacroAssembler::double_move(VMRegPair src, VMRegPair dst, Register tmp) { 7840 if (src.first()->is_stack()) { 7841 if (dst.first()->is_stack()) { 7842 ldr(tmp, Address(rfp, reg2offset_in(src.first()))); 7843 str(tmp, Address(sp, reg2offset_out(dst.first()))); 7844 } else { 7845 ldrd(dst.first()->as_FloatRegister(), Address(rfp, reg2offset_in(src.first()))); 7846 } 7847 } else if (src.first() != dst.first()) { 7848 if (src.is_single_phys_reg() && dst.is_single_phys_reg()) 7849 fmovd(dst.first()->as_FloatRegister(), src.first()->as_FloatRegister()); 7850 else 7851 strd(src.first()->as_FloatRegister(), Address(sp, reg2offset_out(dst.first()))); 7852 } 7853 } 7854 7855 // Implements lightweight-locking. 7856 // 7857 // - obj: the object to be locked 7858 // - t1, t2, t3: temporary registers, will be destroyed 7859 // - slow: branched to if locking fails, absolute offset may larger than 32KB (imm14 encoding). 7860 void MacroAssembler::lightweight_lock(Register basic_lock, Register obj, Register t1, Register t2, Register t3, Label& slow) { 7861 assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking"); 7862 assert_different_registers(basic_lock, obj, t1, t2, t3, rscratch1); 7863 7864 Label push; 7865 const Register top = t1; 7866 const Register mark = t2; 7867 const Register t = t3; 7868 7869 // Preload the markWord. It is important that this is the first 7870 // instruction emitted as it is part of C1's null check semantics. 7871 ldr(mark, Address(obj, oopDesc::mark_offset_in_bytes())); 7872 7873 if (UseObjectMonitorTable) { 7874 // Clear cache in case fast locking succeeds. 7875 str(zr, Address(basic_lock, BasicObjectLock::lock_offset() + in_ByteSize((BasicLock::object_monitor_cache_offset_in_bytes())))); 7876 } 7877 7878 // Check if the lock-stack is full. 7879 ldrw(top, Address(rthread, JavaThread::lock_stack_top_offset())); 7880 cmpw(top, (unsigned)LockStack::end_offset()); 7881 br(Assembler::GE, slow); 7882 7883 // Check for recursion. 7884 subw(t, top, oopSize); 7885 ldr(t, Address(rthread, t)); 7886 cmp(obj, t); 7887 br(Assembler::EQ, push); 7888 7889 // Check header for monitor (0b10). 7890 tst(mark, markWord::monitor_value); 7891 br(Assembler::NE, slow); 7892 7893 // Try to lock. Transition lock bits 0b01 => 0b00 7894 assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea"); 7895 orr(mark, mark, markWord::unlocked_value); 7896 if (EnableValhalla) { 7897 // Mask inline_type bit such that we go to the slow path if object is an inline type 7898 andr(mark, mark, ~((int) markWord::inline_type_bit_in_place)); 7899 } 7900 eor(t, mark, markWord::unlocked_value); 7901 cmpxchg(/*addr*/ obj, /*expected*/ mark, /*new*/ t, Assembler::xword, 7902 /*acquire*/ true, /*release*/ false, /*weak*/ false, noreg); 7903 br(Assembler::NE, slow); 7904 7905 bind(push); 7906 // After successful lock, push object on lock-stack. 7907 str(obj, Address(rthread, top)); 7908 addw(top, top, oopSize); 7909 strw(top, Address(rthread, JavaThread::lock_stack_top_offset())); 7910 } 7911 7912 // Implements lightweight-unlocking. 7913 // 7914 // - obj: the object to be unlocked 7915 // - t1, t2, t3: temporary registers 7916 // - slow: branched to if unlocking fails, absolute offset may larger than 32KB (imm14 encoding). 7917 void MacroAssembler::lightweight_unlock(Register obj, Register t1, Register t2, Register t3, Label& slow) { 7918 assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking"); 7919 // cmpxchg clobbers rscratch1. 7920 assert_different_registers(obj, t1, t2, t3, rscratch1); 7921 7922 #ifdef ASSERT 7923 { 7924 // Check for lock-stack underflow. 7925 Label stack_ok; 7926 ldrw(t1, Address(rthread, JavaThread::lock_stack_top_offset())); 7927 cmpw(t1, (unsigned)LockStack::start_offset()); 7928 br(Assembler::GE, stack_ok); 7929 STOP("Lock-stack underflow"); 7930 bind(stack_ok); 7931 } 7932 #endif 7933 7934 Label unlocked, push_and_slow; 7935 const Register top = t1; 7936 const Register mark = t2; 7937 const Register t = t3; 7938 7939 // Check if obj is top of lock-stack. 7940 ldrw(top, Address(rthread, JavaThread::lock_stack_top_offset())); 7941 subw(top, top, oopSize); 7942 ldr(t, Address(rthread, top)); 7943 cmp(obj, t); 7944 br(Assembler::NE, slow); 7945 7946 // Pop lock-stack. 7947 DEBUG_ONLY(str(zr, Address(rthread, top));) 7948 strw(top, Address(rthread, JavaThread::lock_stack_top_offset())); 7949 7950 // Check if recursive. 7951 subw(t, top, oopSize); 7952 ldr(t, Address(rthread, t)); 7953 cmp(obj, t); 7954 br(Assembler::EQ, unlocked); 7955 7956 // Not recursive. Check header for monitor (0b10). 7957 ldr(mark, Address(obj, oopDesc::mark_offset_in_bytes())); 7958 tbnz(mark, log2i_exact(markWord::monitor_value), push_and_slow); 7959 7960 #ifdef ASSERT 7961 // Check header not unlocked (0b01). 7962 Label not_unlocked; 7963 tbz(mark, log2i_exact(markWord::unlocked_value), not_unlocked); 7964 stop("lightweight_unlock already unlocked"); 7965 bind(not_unlocked); 7966 #endif 7967 7968 // Try to unlock. Transition lock bits 0b00 => 0b01 7969 assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea"); 7970 orr(t, mark, markWord::unlocked_value); 7971 cmpxchg(obj, mark, t, Assembler::xword, 7972 /*acquire*/ false, /*release*/ true, /*weak*/ false, noreg); 7973 br(Assembler::EQ, unlocked); 7974 7975 bind(push_and_slow); 7976 // Restore lock-stack and handle the unlock in runtime. 7977 DEBUG_ONLY(str(obj, Address(rthread, top));) 7978 addw(top, top, oopSize); 7979 strw(top, Address(rthread, JavaThread::lock_stack_top_offset())); 7980 b(slow); 7981 7982 bind(unlocked); 7983 }