1 /* 2 * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved. 3 * Copyright (c) 2014, 2024, Red Hat Inc. All rights reserved. 4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 * 6 * This code is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 only, as 8 * published by the Free Software Foundation. 9 * 10 * This code is distributed in the hope that it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * version 2 for more details (a copy is included in the LICENSE file that 14 * accompanied this code). 15 * 16 * You should have received a copy of the GNU General Public License version 17 * 2 along with this work; if not, write to the Free Software Foundation, 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 * or visit www.oracle.com if you need additional information or have any 22 * questions. 23 * 24 */ 25 26 #include "asm/assembler.hpp" 27 #include "asm/assembler.inline.hpp" 28 #include "ci/ciEnv.hpp" 29 #include "ci/ciInlineKlass.hpp" 30 #include "code/compiledIC.hpp" 31 #include "compiler/compileTask.hpp" 32 #include "compiler/disassembler.hpp" 33 #include "compiler/oopMap.hpp" 34 #include "gc/shared/barrierSet.hpp" 35 #include "gc/shared/barrierSetAssembler.hpp" 36 #include "gc/shared/cardTableBarrierSet.hpp" 37 #include "gc/shared/cardTable.hpp" 38 #include "gc/shared/collectedHeap.hpp" 39 #include "gc/shared/tlab_globals.hpp" 40 #include "interpreter/bytecodeHistogram.hpp" 41 #include "interpreter/interpreter.hpp" 42 #include "interpreter/interpreterRuntime.hpp" 43 #include "jvm.h" 44 #include "memory/resourceArea.hpp" 45 #include "memory/universe.hpp" 46 #include "nativeInst_aarch64.hpp" 47 #include "oops/accessDecorators.hpp" 48 #include "oops/compressedKlass.inline.hpp" 49 #include "oops/compressedOops.inline.hpp" 50 #include "oops/klass.inline.hpp" 51 #include "oops/resolvedFieldEntry.hpp" 52 #include "runtime/continuation.hpp" 53 #include "runtime/globals.hpp" 54 #include "runtime/icache.hpp" 55 #include "runtime/interfaceSupport.inline.hpp" 56 #include "runtime/javaThread.hpp" 57 #include "runtime/jniHandles.inline.hpp" 58 #include "runtime/sharedRuntime.hpp" 59 #include "runtime/signature_cc.hpp" 60 #include "runtime/stubRoutines.hpp" 61 #include "utilities/globalDefinitions.hpp" 62 #include "utilities/powerOfTwo.hpp" 63 #include "vmreg_aarch64.inline.hpp" 64 #ifdef COMPILER1 65 #include "c1/c1_LIRAssembler.hpp" 66 #endif 67 #ifdef COMPILER2 68 #include "oops/oop.hpp" 69 #include "opto/compile.hpp" 70 #include "opto/node.hpp" 71 #include "opto/output.hpp" 72 #endif 73 74 #include <sys/types.h> 75 76 #ifdef PRODUCT 77 #define BLOCK_COMMENT(str) /* nothing */ 78 #else 79 #define BLOCK_COMMENT(str) block_comment(str) 80 #endif 81 #define STOP(str) stop(str); 82 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":") 83 84 #ifdef ASSERT 85 extern "C" void disnm(intptr_t p); 86 #endif 87 // Target-dependent relocation processing 88 // 89 // Instruction sequences whose target may need to be retrieved or 90 // patched are distinguished by their leading instruction, sorting 91 // them into three main instruction groups and related subgroups. 92 // 93 // 1) Branch, Exception and System (insn count = 1) 94 // 1a) Unconditional branch (immediate): 95 // b/bl imm19 96 // 1b) Compare & branch (immediate): 97 // cbz/cbnz Rt imm19 98 // 1c) Test & branch (immediate): 99 // tbz/tbnz Rt imm14 100 // 1d) Conditional branch (immediate): 101 // b.cond imm19 102 // 103 // 2) Loads and Stores (insn count = 1) 104 // 2a) Load register literal: 105 // ldr Rt imm19 106 // 107 // 3) Data Processing Immediate (insn count = 2 or 3) 108 // 3a) PC-rel. addressing 109 // adr/adrp Rx imm21; ldr/str Ry Rx #imm12 110 // adr/adrp Rx imm21; add Ry Rx #imm12 111 // adr/adrp Rx imm21; movk Rx #imm16<<32; ldr/str Ry, [Rx, #offset_in_page] 112 // adr/adrp Rx imm21 113 // adr/adrp Rx imm21; movk Rx #imm16<<32 114 // adr/adrp Rx imm21; movk Rx #imm16<<32; add Ry, Rx, #offset_in_page 115 // The latter form can only happen when the target is an 116 // ExternalAddress, and (by definition) ExternalAddresses don't 117 // move. Because of that property, there is never any need to 118 // patch the last of the three instructions. However, 119 // MacroAssembler::target_addr_for_insn takes all three 120 // instructions into account and returns the correct address. 121 // 3b) Move wide (immediate) 122 // movz Rx #imm16; movk Rx #imm16 << 16; movk Rx #imm16 << 32; 123 // 124 // A switch on a subset of the instruction's bits provides an 125 // efficient dispatch to these subcases. 126 // 127 // insn[28:26] -> main group ('x' == don't care) 128 // 00x -> UNALLOCATED 129 // 100 -> Data Processing Immediate 130 // 101 -> Branch, Exception and System 131 // x1x -> Loads and Stores 132 // 133 // insn[30:25] -> subgroup ('_' == group, 'x' == don't care). 134 // n.b. in some cases extra bits need to be checked to verify the 135 // instruction is as expected 136 // 137 // 1) ... xx101x Branch, Exception and System 138 // 1a) 00___x Unconditional branch (immediate) 139 // 1b) 01___0 Compare & branch (immediate) 140 // 1c) 01___1 Test & branch (immediate) 141 // 1d) 10___0 Conditional branch (immediate) 142 // other Should not happen 143 // 144 // 2) ... xxx1x0 Loads and Stores 145 // 2a) xx1__00 Load/Store register (insn[28] == 1 && insn[24] == 0) 146 // 2aa) x01__00 Load register literal (i.e. requires insn[29] == 0) 147 // strictly should be 64 bit non-FP/SIMD i.e. 148 // 0101_000 (i.e. requires insn[31:24] == 01011000) 149 // 150 // 3) ... xx100x Data Processing Immediate 151 // 3a) xx___00 PC-rel. addressing (n.b. requires insn[24] == 0) 152 // 3b) xx___101 Move wide (immediate) (n.b. requires insn[24:23] == 01) 153 // strictly should be 64 bit movz #imm16<<0 154 // 110___10100 (i.e. requires insn[31:21] == 11010010100) 155 // 156 157 static uint32_t insn_at(address insn_addr, int n) { 158 return ((uint32_t*)insn_addr)[n]; 159 } 160 161 template<typename T> 162 class RelocActions : public AllStatic { 163 164 public: 165 166 static int ALWAYSINLINE run(address insn_addr, address &target) { 167 int instructions = 1; 168 uint32_t insn = insn_at(insn_addr, 0); 169 170 uint32_t dispatch = Instruction_aarch64::extract(insn, 30, 25); 171 switch(dispatch) { 172 case 0b001010: 173 case 0b001011: { 174 instructions = T::unconditionalBranch(insn_addr, target); 175 break; 176 } 177 case 0b101010: // Conditional branch (immediate) 178 case 0b011010: { // Compare & branch (immediate) 179 instructions = T::conditionalBranch(insn_addr, target); 180 break; 181 } 182 case 0b011011: { 183 instructions = T::testAndBranch(insn_addr, target); 184 break; 185 } 186 case 0b001100: 187 case 0b001110: 188 case 0b011100: 189 case 0b011110: 190 case 0b101100: 191 case 0b101110: 192 case 0b111100: 193 case 0b111110: { 194 // load/store 195 if ((Instruction_aarch64::extract(insn, 29, 24) & 0b111011) == 0b011000) { 196 // Load register (literal) 197 instructions = T::loadStore(insn_addr, target); 198 break; 199 } else { 200 // nothing to do 201 assert(target == nullptr, "did not expect to relocate target for polling page load"); 202 } 203 break; 204 } 205 case 0b001000: 206 case 0b011000: 207 case 0b101000: 208 case 0b111000: { 209 // adr/adrp 210 assert(Instruction_aarch64::extract(insn, 28, 24) == 0b10000, "must be"); 211 int shift = Instruction_aarch64::extract(insn, 31, 31); 212 if (shift) { 213 uint32_t insn2 = insn_at(insn_addr, 1); 214 if (Instruction_aarch64::extract(insn2, 29, 24) == 0b111001 && 215 Instruction_aarch64::extract(insn, 4, 0) == 216 Instruction_aarch64::extract(insn2, 9, 5)) { 217 instructions = T::adrp(insn_addr, target, T::adrpMem); 218 } else if (Instruction_aarch64::extract(insn2, 31, 22) == 0b1001000100 && 219 Instruction_aarch64::extract(insn, 4, 0) == 220 Instruction_aarch64::extract(insn2, 4, 0)) { 221 instructions = T::adrp(insn_addr, target, T::adrpAdd); 222 } else if (Instruction_aarch64::extract(insn2, 31, 21) == 0b11110010110 && 223 Instruction_aarch64::extract(insn, 4, 0) == 224 Instruction_aarch64::extract(insn2, 4, 0)) { 225 instructions = T::adrp(insn_addr, target, T::adrpMovk); 226 } else { 227 ShouldNotReachHere(); 228 } 229 } else { 230 instructions = T::adr(insn_addr, target); 231 } 232 break; 233 } 234 case 0b001001: 235 case 0b011001: 236 case 0b101001: 237 case 0b111001: { 238 instructions = T::immediate(insn_addr, target); 239 break; 240 } 241 default: { 242 ShouldNotReachHere(); 243 } 244 } 245 246 T::verify(insn_addr, target); 247 return instructions * NativeInstruction::instruction_size; 248 } 249 }; 250 251 class Patcher : public AllStatic { 252 public: 253 static int unconditionalBranch(address insn_addr, address &target) { 254 intptr_t offset = (target - insn_addr) >> 2; 255 Instruction_aarch64::spatch(insn_addr, 25, 0, offset); 256 return 1; 257 } 258 static int conditionalBranch(address insn_addr, address &target) { 259 intptr_t offset = (target - insn_addr) >> 2; 260 Instruction_aarch64::spatch(insn_addr, 23, 5, offset); 261 return 1; 262 } 263 static int testAndBranch(address insn_addr, address &target) { 264 intptr_t offset = (target - insn_addr) >> 2; 265 Instruction_aarch64::spatch(insn_addr, 18, 5, offset); 266 return 1; 267 } 268 static int loadStore(address insn_addr, address &target) { 269 intptr_t offset = (target - insn_addr) >> 2; 270 Instruction_aarch64::spatch(insn_addr, 23, 5, offset); 271 return 1; 272 } 273 static int adr(address insn_addr, address &target) { 274 #ifdef ASSERT 275 assert(Instruction_aarch64::extract(insn_at(insn_addr, 0), 28, 24) == 0b10000, "must be"); 276 #endif 277 // PC-rel. addressing 278 ptrdiff_t offset = target - insn_addr; 279 int offset_lo = offset & 3; 280 offset >>= 2; 281 Instruction_aarch64::spatch(insn_addr, 23, 5, offset); 282 Instruction_aarch64::patch(insn_addr, 30, 29, offset_lo); 283 return 1; 284 } 285 template<typename U> 286 static int adrp(address insn_addr, address &target, U inner) { 287 int instructions = 1; 288 #ifdef ASSERT 289 assert(Instruction_aarch64::extract(insn_at(insn_addr, 0), 28, 24) == 0b10000, "must be"); 290 #endif 291 ptrdiff_t offset = target - insn_addr; 292 instructions = 2; 293 precond(inner != nullptr); 294 // Give the inner reloc a chance to modify the target. 295 address adjusted_target = target; 296 instructions = inner(insn_addr, adjusted_target); 297 uintptr_t pc_page = (uintptr_t)insn_addr >> 12; 298 uintptr_t adr_page = (uintptr_t)adjusted_target >> 12; 299 offset = adr_page - pc_page; 300 int offset_lo = offset & 3; 301 offset >>= 2; 302 Instruction_aarch64::spatch(insn_addr, 23, 5, offset); 303 Instruction_aarch64::patch(insn_addr, 30, 29, offset_lo); 304 return instructions; 305 } 306 static int adrpMem(address insn_addr, address &target) { 307 uintptr_t dest = (uintptr_t)target; 308 int offset_lo = dest & 0xfff; 309 uint32_t insn2 = insn_at(insn_addr, 1); 310 uint32_t size = Instruction_aarch64::extract(insn2, 31, 30); 311 Instruction_aarch64::patch(insn_addr + sizeof (uint32_t), 21, 10, offset_lo >> size); 312 guarantee(((dest >> size) << size) == dest, "misaligned target"); 313 return 2; 314 } 315 static int adrpAdd(address insn_addr, address &target) { 316 uintptr_t dest = (uintptr_t)target; 317 int offset_lo = dest & 0xfff; 318 Instruction_aarch64::patch(insn_addr + sizeof (uint32_t), 21, 10, offset_lo); 319 return 2; 320 } 321 static int adrpMovk(address insn_addr, address &target) { 322 uintptr_t dest = uintptr_t(target); 323 Instruction_aarch64::patch(insn_addr + sizeof (uint32_t), 20, 5, (uintptr_t)target >> 32); 324 dest = (dest & 0xffffffffULL) | (uintptr_t(insn_addr) & 0xffff00000000ULL); 325 target = address(dest); 326 return 2; 327 } 328 static int immediate(address insn_addr, address &target) { 329 assert(Instruction_aarch64::extract(insn_at(insn_addr, 0), 31, 21) == 0b11010010100, "must be"); 330 uint64_t dest = (uint64_t)target; 331 // Move wide constant 332 assert(nativeInstruction_at(insn_addr+4)->is_movk(), "wrong insns in patch"); 333 assert(nativeInstruction_at(insn_addr+8)->is_movk(), "wrong insns in patch"); 334 Instruction_aarch64::patch(insn_addr, 20, 5, dest & 0xffff); 335 Instruction_aarch64::patch(insn_addr+4, 20, 5, (dest >>= 16) & 0xffff); 336 Instruction_aarch64::patch(insn_addr+8, 20, 5, (dest >>= 16) & 0xffff); 337 return 3; 338 } 339 static void verify(address insn_addr, address &target) { 340 #ifdef ASSERT 341 address address_is = MacroAssembler::target_addr_for_insn(insn_addr); 342 if (!(address_is == target)) { 343 tty->print_cr("%p at %p should be %p", address_is, insn_addr, target); 344 disnm((intptr_t)insn_addr); 345 assert(address_is == target, "should be"); 346 } 347 #endif 348 } 349 }; 350 351 // If insn1 and insn2 use the same register to form an address, either 352 // by an offsetted LDR or a simple ADD, return the offset. If the 353 // second instruction is an LDR, the offset may be scaled. 354 static bool offset_for(uint32_t insn1, uint32_t insn2, ptrdiff_t &byte_offset) { 355 if (Instruction_aarch64::extract(insn2, 29, 24) == 0b111001 && 356 Instruction_aarch64::extract(insn1, 4, 0) == 357 Instruction_aarch64::extract(insn2, 9, 5)) { 358 // Load/store register (unsigned immediate) 359 byte_offset = Instruction_aarch64::extract(insn2, 21, 10); 360 uint32_t size = Instruction_aarch64::extract(insn2, 31, 30); 361 byte_offset <<= size; 362 return true; 363 } else if (Instruction_aarch64::extract(insn2, 31, 22) == 0b1001000100 && 364 Instruction_aarch64::extract(insn1, 4, 0) == 365 Instruction_aarch64::extract(insn2, 4, 0)) { 366 // add (immediate) 367 byte_offset = Instruction_aarch64::extract(insn2, 21, 10); 368 return true; 369 } 370 return false; 371 } 372 373 class AArch64Decoder : public AllStatic { 374 public: 375 376 static int loadStore(address insn_addr, address &target) { 377 intptr_t offset = Instruction_aarch64::sextract(insn_at(insn_addr, 0), 23, 5); 378 target = insn_addr + (offset << 2); 379 return 1; 380 } 381 static int unconditionalBranch(address insn_addr, address &target) { 382 intptr_t offset = Instruction_aarch64::sextract(insn_at(insn_addr, 0), 25, 0); 383 target = insn_addr + (offset << 2); 384 return 1; 385 } 386 static int conditionalBranch(address insn_addr, address &target) { 387 intptr_t offset = Instruction_aarch64::sextract(insn_at(insn_addr, 0), 23, 5); 388 target = address(((uint64_t)insn_addr + (offset << 2))); 389 return 1; 390 } 391 static int testAndBranch(address insn_addr, address &target) { 392 intptr_t offset = Instruction_aarch64::sextract(insn_at(insn_addr, 0), 18, 5); 393 target = address(((uint64_t)insn_addr + (offset << 2))); 394 return 1; 395 } 396 static int adr(address insn_addr, address &target) { 397 // PC-rel. addressing 398 uint32_t insn = insn_at(insn_addr, 0); 399 intptr_t offset = Instruction_aarch64::extract(insn, 30, 29); 400 offset |= Instruction_aarch64::sextract(insn, 23, 5) << 2; 401 target = address((uint64_t)insn_addr + offset); 402 return 1; 403 } 404 template<typename U> 405 static int adrp(address insn_addr, address &target, U inner) { 406 uint32_t insn = insn_at(insn_addr, 0); 407 assert(Instruction_aarch64::extract(insn, 28, 24) == 0b10000, "must be"); 408 intptr_t offset = Instruction_aarch64::extract(insn, 30, 29); 409 offset |= Instruction_aarch64::sextract(insn, 23, 5) << 2; 410 int shift = 12; 411 offset <<= shift; 412 uint64_t target_page = ((uint64_t)insn_addr) + offset; 413 target_page &= ((uint64_t)-1) << shift; 414 uint32_t insn2 = insn_at(insn_addr, 1); 415 target = address(target_page); 416 precond(inner != nullptr); 417 inner(insn_addr, target); 418 return 2; 419 } 420 static int adrpMem(address insn_addr, address &target) { 421 uint32_t insn2 = insn_at(insn_addr, 1); 422 // Load/store register (unsigned immediate) 423 ptrdiff_t byte_offset = Instruction_aarch64::extract(insn2, 21, 10); 424 uint32_t size = Instruction_aarch64::extract(insn2, 31, 30); 425 byte_offset <<= size; 426 target += byte_offset; 427 return 2; 428 } 429 static int adrpAdd(address insn_addr, address &target) { 430 uint32_t insn2 = insn_at(insn_addr, 1); 431 // add (immediate) 432 ptrdiff_t byte_offset = Instruction_aarch64::extract(insn2, 21, 10); 433 target += byte_offset; 434 return 2; 435 } 436 static int adrpMovk(address insn_addr, address &target) { 437 uint32_t insn2 = insn_at(insn_addr, 1); 438 uint64_t dest = uint64_t(target); 439 dest = (dest & 0xffff0000ffffffff) | 440 ((uint64_t)Instruction_aarch64::extract(insn2, 20, 5) << 32); 441 target = address(dest); 442 443 // We know the destination 4k page. Maybe we have a third 444 // instruction. 445 uint32_t insn = insn_at(insn_addr, 0); 446 uint32_t insn3 = insn_at(insn_addr, 2); 447 ptrdiff_t byte_offset; 448 if (offset_for(insn, insn3, byte_offset)) { 449 target += byte_offset; 450 return 3; 451 } else { 452 return 2; 453 } 454 } 455 static int immediate(address insn_addr, address &target) { 456 uint32_t *insns = (uint32_t *)insn_addr; 457 assert(Instruction_aarch64::extract(insns[0], 31, 21) == 0b11010010100, "must be"); 458 // Move wide constant: movz, movk, movk. See movptr(). 459 assert(nativeInstruction_at(insns+1)->is_movk(), "wrong insns in patch"); 460 assert(nativeInstruction_at(insns+2)->is_movk(), "wrong insns in patch"); 461 target = address(uint64_t(Instruction_aarch64::extract(insns[0], 20, 5)) 462 + (uint64_t(Instruction_aarch64::extract(insns[1], 20, 5)) << 16) 463 + (uint64_t(Instruction_aarch64::extract(insns[2], 20, 5)) << 32)); 464 assert(nativeInstruction_at(insn_addr+4)->is_movk(), "wrong insns in patch"); 465 assert(nativeInstruction_at(insn_addr+8)->is_movk(), "wrong insns in patch"); 466 return 3; 467 } 468 static void verify(address insn_addr, address &target) { 469 } 470 }; 471 472 address MacroAssembler::target_addr_for_insn(address insn_addr) { 473 address target; 474 RelocActions<AArch64Decoder>::run(insn_addr, target); 475 return target; 476 } 477 478 // Patch any kind of instruction; there may be several instructions. 479 // Return the total length (in bytes) of the instructions. 480 int MacroAssembler::pd_patch_instruction_size(address insn_addr, address target) { 481 return RelocActions<Patcher>::run(insn_addr, target); 482 } 483 484 int MacroAssembler::patch_oop(address insn_addr, address o) { 485 int instructions; 486 unsigned insn = *(unsigned*)insn_addr; 487 assert(nativeInstruction_at(insn_addr+4)->is_movk(), "wrong insns in patch"); 488 489 // OOPs are either narrow (32 bits) or wide (48 bits). We encode 490 // narrow OOPs by setting the upper 16 bits in the first 491 // instruction. 492 if (Instruction_aarch64::extract(insn, 31, 21) == 0b11010010101) { 493 // Move narrow OOP 494 uint32_t n = CompressedOops::narrow_oop_value(cast_to_oop(o)); 495 Instruction_aarch64::patch(insn_addr, 20, 5, n >> 16); 496 Instruction_aarch64::patch(insn_addr+4, 20, 5, n & 0xffff); 497 instructions = 2; 498 } else { 499 // Move wide OOP 500 assert(nativeInstruction_at(insn_addr+8)->is_movk(), "wrong insns in patch"); 501 uintptr_t dest = (uintptr_t)o; 502 Instruction_aarch64::patch(insn_addr, 20, 5, dest & 0xffff); 503 Instruction_aarch64::patch(insn_addr+4, 20, 5, (dest >>= 16) & 0xffff); 504 Instruction_aarch64::patch(insn_addr+8, 20, 5, (dest >>= 16) & 0xffff); 505 instructions = 3; 506 } 507 return instructions * NativeInstruction::instruction_size; 508 } 509 510 int MacroAssembler::patch_narrow_klass(address insn_addr, narrowKlass n) { 511 // Metadata pointers are either narrow (32 bits) or wide (48 bits). 512 // We encode narrow ones by setting the upper 16 bits in the first 513 // instruction. 514 NativeInstruction *insn = nativeInstruction_at(insn_addr); 515 assert(Instruction_aarch64::extract(insn->encoding(), 31, 21) == 0b11010010101 && 516 nativeInstruction_at(insn_addr+4)->is_movk(), "wrong insns in patch"); 517 518 Instruction_aarch64::patch(insn_addr, 20, 5, n >> 16); 519 Instruction_aarch64::patch(insn_addr+4, 20, 5, n & 0xffff); 520 return 2 * NativeInstruction::instruction_size; 521 } 522 523 address MacroAssembler::target_addr_for_insn_or_null(address insn_addr) { 524 if (NativeInstruction::is_ldrw_to_zr(insn_addr)) { 525 return nullptr; 526 } 527 return MacroAssembler::target_addr_for_insn(insn_addr); 528 } 529 530 void MacroAssembler::safepoint_poll(Label& slow_path, bool at_return, bool in_nmethod, Register tmp) { 531 ldr(tmp, Address(rthread, JavaThread::polling_word_offset())); 532 if (at_return) { 533 // Note that when in_nmethod is set, the stack pointer is incremented before the poll. Therefore, 534 // we may safely use the sp instead to perform the stack watermark check. 535 cmp(in_nmethod ? sp : rfp, tmp); 536 br(Assembler::HI, slow_path); 537 } else { 538 tbnz(tmp, log2i_exact(SafepointMechanism::poll_bit()), slow_path); 539 } 540 } 541 542 void MacroAssembler::rt_call(address dest, Register tmp) { 543 CodeBlob *cb = CodeCache::find_blob(dest); 544 if (cb) { 545 far_call(RuntimeAddress(dest)); 546 } else { 547 lea(tmp, RuntimeAddress(dest)); 548 blr(tmp); 549 } 550 } 551 552 void MacroAssembler::push_cont_fastpath(Register java_thread) { 553 if (!Continuations::enabled()) return; 554 Label done; 555 ldr(rscratch1, Address(java_thread, JavaThread::cont_fastpath_offset())); 556 cmp(sp, rscratch1); 557 br(Assembler::LS, done); 558 mov(rscratch1, sp); // we can't use sp as the source in str 559 str(rscratch1, Address(java_thread, JavaThread::cont_fastpath_offset())); 560 bind(done); 561 } 562 563 void MacroAssembler::pop_cont_fastpath(Register java_thread) { 564 if (!Continuations::enabled()) return; 565 Label done; 566 ldr(rscratch1, Address(java_thread, JavaThread::cont_fastpath_offset())); 567 cmp(sp, rscratch1); 568 br(Assembler::LO, done); 569 str(zr, Address(java_thread, JavaThread::cont_fastpath_offset())); 570 bind(done); 571 } 572 573 void MacroAssembler::reset_last_Java_frame(bool clear_fp) { 574 // we must set sp to zero to clear frame 575 str(zr, Address(rthread, JavaThread::last_Java_sp_offset())); 576 577 // must clear fp, so that compiled frames are not confused; it is 578 // possible that we need it only for debugging 579 if (clear_fp) { 580 str(zr, Address(rthread, JavaThread::last_Java_fp_offset())); 581 } 582 583 // Always clear the pc because it could have been set by make_walkable() 584 str(zr, Address(rthread, JavaThread::last_Java_pc_offset())); 585 } 586 587 // Calls to C land 588 // 589 // When entering C land, the rfp, & resp of the last Java frame have to be recorded 590 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp 591 // has to be reset to 0. This is required to allow proper stack traversal. 592 void MacroAssembler::set_last_Java_frame(Register last_java_sp, 593 Register last_java_fp, 594 Register last_java_pc, 595 Register scratch) { 596 597 if (last_java_pc->is_valid()) { 598 str(last_java_pc, Address(rthread, 599 JavaThread::frame_anchor_offset() 600 + JavaFrameAnchor::last_Java_pc_offset())); 601 } 602 603 // determine last_java_sp register 604 if (last_java_sp == sp) { 605 mov(scratch, sp); 606 last_java_sp = scratch; 607 } else if (!last_java_sp->is_valid()) { 608 last_java_sp = esp; 609 } 610 611 // last_java_fp is optional 612 if (last_java_fp->is_valid()) { 613 str(last_java_fp, Address(rthread, JavaThread::last_Java_fp_offset())); 614 } 615 616 // We must set sp last. 617 str(last_java_sp, Address(rthread, JavaThread::last_Java_sp_offset())); 618 } 619 620 void MacroAssembler::set_last_Java_frame(Register last_java_sp, 621 Register last_java_fp, 622 address last_java_pc, 623 Register scratch) { 624 assert(last_java_pc != nullptr, "must provide a valid PC"); 625 626 adr(scratch, last_java_pc); 627 str(scratch, Address(rthread, 628 JavaThread::frame_anchor_offset() 629 + JavaFrameAnchor::last_Java_pc_offset())); 630 631 set_last_Java_frame(last_java_sp, last_java_fp, noreg, scratch); 632 } 633 634 void MacroAssembler::set_last_Java_frame(Register last_java_sp, 635 Register last_java_fp, 636 Label &L, 637 Register scratch) { 638 if (L.is_bound()) { 639 set_last_Java_frame(last_java_sp, last_java_fp, target(L), scratch); 640 } else { 641 InstructionMark im(this); 642 L.add_patch_at(code(), locator()); 643 set_last_Java_frame(last_java_sp, last_java_fp, pc() /* Patched later */, scratch); 644 } 645 } 646 647 static inline bool target_needs_far_branch(address addr) { 648 if (AOTCodeCache::is_on_for_dump()) { 649 return true; 650 } 651 // codecache size <= 128M 652 if (!MacroAssembler::far_branches()) { 653 return false; 654 } 655 // codecache size > 240M 656 if (MacroAssembler::codestub_branch_needs_far_jump()) { 657 return true; 658 } 659 // codecache size: 128M..240M 660 return !CodeCache::is_non_nmethod(addr); 661 } 662 663 void MacroAssembler::far_call(Address entry, Register tmp) { 664 assert(ReservedCodeCacheSize < 4*G, "branch out of range"); 665 assert(CodeCache::find_blob(entry.target()) != nullptr, 666 "destination of far call not found in code cache"); 667 assert(entry.rspec().type() == relocInfo::external_word_type 668 || entry.rspec().type() == relocInfo::runtime_call_type 669 || entry.rspec().type() == relocInfo::none, "wrong entry relocInfo type"); 670 if (target_needs_far_branch(entry.target())) { 671 uint64_t offset; 672 // We can use ADRP here because we know that the total size of 673 // the code cache cannot exceed 2Gb (ADRP limit is 4GB). 674 adrp(tmp, entry, offset); 675 add(tmp, tmp, offset); 676 blr(tmp); 677 } else { 678 bl(entry); 679 } 680 } 681 682 int MacroAssembler::far_jump(Address entry, Register tmp) { 683 assert(ReservedCodeCacheSize < 4*G, "branch out of range"); 684 assert(CodeCache::find_blob(entry.target()) != nullptr, 685 "destination of far call not found in code cache"); 686 assert(entry.rspec().type() == relocInfo::external_word_type 687 || entry.rspec().type() == relocInfo::runtime_call_type 688 || entry.rspec().type() == relocInfo::none, "wrong entry relocInfo type"); 689 address start = pc(); 690 if (target_needs_far_branch(entry.target())) { 691 uint64_t offset; 692 // We can use ADRP here because we know that the total size of 693 // the code cache cannot exceed 2Gb (ADRP limit is 4GB). 694 adrp(tmp, entry, offset); 695 add(tmp, tmp, offset); 696 br(tmp); 697 } else { 698 b(entry); 699 } 700 return pc() - start; 701 } 702 703 void MacroAssembler::reserved_stack_check() { 704 // testing if reserved zone needs to be enabled 705 Label no_reserved_zone_enabling; 706 707 ldr(rscratch1, Address(rthread, JavaThread::reserved_stack_activation_offset())); 708 cmp(sp, rscratch1); 709 br(Assembler::LO, no_reserved_zone_enabling); 710 711 enter(); // LR and FP are live. 712 lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone))); 713 mov(c_rarg0, rthread); 714 blr(rscratch1); 715 leave(); 716 717 // We have already removed our own frame. 718 // throw_delayed_StackOverflowError will think that it's been 719 // called by our caller. 720 lea(rscratch1, RuntimeAddress(SharedRuntime::throw_delayed_StackOverflowError_entry())); 721 br(rscratch1); 722 should_not_reach_here(); 723 724 bind(no_reserved_zone_enabling); 725 } 726 727 static void pass_arg0(MacroAssembler* masm, Register arg) { 728 if (c_rarg0 != arg ) { 729 masm->mov(c_rarg0, arg); 730 } 731 } 732 733 static void pass_arg1(MacroAssembler* masm, Register arg) { 734 if (c_rarg1 != arg ) { 735 masm->mov(c_rarg1, arg); 736 } 737 } 738 739 static void pass_arg2(MacroAssembler* masm, Register arg) { 740 if (c_rarg2 != arg ) { 741 masm->mov(c_rarg2, arg); 742 } 743 } 744 745 static void pass_arg3(MacroAssembler* masm, Register arg) { 746 if (c_rarg3 != arg ) { 747 masm->mov(c_rarg3, arg); 748 } 749 } 750 751 static bool is_preemptable(address entry_point) { 752 return entry_point == CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter); 753 } 754 755 void MacroAssembler::call_VM_base(Register oop_result, 756 Register java_thread, 757 Register last_java_sp, 758 address entry_point, 759 int number_of_arguments, 760 bool check_exceptions) { 761 // determine java_thread register 762 if (!java_thread->is_valid()) { 763 java_thread = rthread; 764 } 765 766 // determine last_java_sp register 767 if (!last_java_sp->is_valid()) { 768 last_java_sp = esp; 769 } 770 771 // debugging support 772 assert(number_of_arguments >= 0 , "cannot have negative number of arguments"); 773 assert(java_thread == rthread, "unexpected register"); 774 #ifdef ASSERT 775 // TraceBytecodes does not use r12 but saves it over the call, so don't verify 776 // if ((UseCompressedOops || UseCompressedClassPointers) && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?"); 777 #endif // ASSERT 778 779 assert(java_thread != oop_result , "cannot use the same register for java_thread & oop_result"); 780 assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp"); 781 782 // push java thread (becomes first argument of C function) 783 784 mov(c_rarg0, java_thread); 785 786 // set last Java frame before call 787 assert(last_java_sp != rfp, "can't use rfp"); 788 789 Label l; 790 if (is_preemptable(entry_point)) { 791 // skip setting last_pc since we already set it to desired value. 792 set_last_Java_frame(last_java_sp, rfp, noreg, rscratch1); 793 } else { 794 set_last_Java_frame(last_java_sp, rfp, l, rscratch1); 795 } 796 797 // do the call, remove parameters 798 MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments, &l); 799 800 // lr could be poisoned with PAC signature during throw_pending_exception 801 // if it was tail-call optimized by compiler, since lr is not callee-saved 802 // reload it with proper value 803 adr(lr, l); 804 805 // reset last Java frame 806 // Only interpreter should have to clear fp 807 reset_last_Java_frame(true); 808 809 // C++ interp handles this in the interpreter 810 check_and_handle_popframe(java_thread); 811 check_and_handle_earlyret(java_thread); 812 813 if (check_exceptions) { 814 // check for pending exceptions (java_thread is set upon return) 815 ldr(rscratch1, Address(java_thread, in_bytes(Thread::pending_exception_offset()))); 816 Label ok; 817 cbz(rscratch1, ok); 818 lea(rscratch1, RuntimeAddress(StubRoutines::forward_exception_entry())); 819 br(rscratch1); 820 bind(ok); 821 } 822 823 // get oop result if there is one and reset the value in the thread 824 if (oop_result->is_valid()) { 825 get_vm_result_oop(oop_result, java_thread); 826 } 827 } 828 829 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) { 830 call_VM_base(oop_result, noreg, noreg, entry_point, number_of_arguments, check_exceptions); 831 } 832 833 // Check the entry target is always reachable from any branch. 834 static bool is_always_within_branch_range(Address entry) { 835 if (AOTCodeCache::is_on_for_dump()) { 836 return false; 837 } 838 const address target = entry.target(); 839 840 if (!CodeCache::contains(target)) { 841 // We always use trampolines for callees outside CodeCache. 842 assert(entry.rspec().type() == relocInfo::runtime_call_type, "non-runtime call of an external target"); 843 return false; 844 } 845 846 if (!MacroAssembler::far_branches()) { 847 return true; 848 } 849 850 if (entry.rspec().type() == relocInfo::runtime_call_type) { 851 // Runtime calls are calls of a non-compiled method (stubs, adapters). 852 // Non-compiled methods stay forever in CodeCache. 853 // We check whether the longest possible branch is within the branch range. 854 assert(CodeCache::find_blob(target) != nullptr && 855 !CodeCache::find_blob(target)->is_nmethod(), 856 "runtime call of compiled method"); 857 const address right_longest_branch_start = CodeCache::high_bound() - NativeInstruction::instruction_size; 858 const address left_longest_branch_start = CodeCache::low_bound(); 859 const bool is_reachable = Assembler::reachable_from_branch_at(left_longest_branch_start, target) && 860 Assembler::reachable_from_branch_at(right_longest_branch_start, target); 861 return is_reachable; 862 } 863 864 return false; 865 } 866 867 // Maybe emit a call via a trampoline. If the code cache is small 868 // trampolines won't be emitted. 869 address MacroAssembler::trampoline_call(Address entry) { 870 assert(entry.rspec().type() == relocInfo::runtime_call_type 871 || entry.rspec().type() == relocInfo::opt_virtual_call_type 872 || entry.rspec().type() == relocInfo::static_call_type 873 || entry.rspec().type() == relocInfo::virtual_call_type, "wrong reloc type"); 874 875 address target = entry.target(); 876 877 if (!is_always_within_branch_range(entry)) { 878 if (!in_scratch_emit_size()) { 879 // We don't want to emit a trampoline if C2 is generating dummy 880 // code during its branch shortening phase. 881 if (entry.rspec().type() == relocInfo::runtime_call_type) { 882 assert(CodeBuffer::supports_shared_stubs(), "must support shared stubs"); 883 code()->share_trampoline_for(entry.target(), offset()); 884 } else { 885 address stub = emit_trampoline_stub(offset(), target); 886 if (stub == nullptr) { 887 postcond(pc() == badAddress); 888 return nullptr; // CodeCache is full 889 } 890 } 891 } 892 target = pc(); 893 } 894 895 address call_pc = pc(); 896 relocate(entry.rspec()); 897 bl(target); 898 899 postcond(pc() != badAddress); 900 return call_pc; 901 } 902 903 // Emit a trampoline stub for a call to a target which is too far away. 904 // 905 // code sequences: 906 // 907 // call-site: 908 // branch-and-link to <destination> or <trampoline stub> 909 // 910 // Related trampoline stub for this call site in the stub section: 911 // load the call target from the constant pool 912 // branch (LR still points to the call site above) 913 914 address MacroAssembler::emit_trampoline_stub(int insts_call_instruction_offset, 915 address dest) { 916 // Max stub size: alignment nop, TrampolineStub. 917 address stub = start_a_stub(max_trampoline_stub_size()); 918 if (stub == nullptr) { 919 return nullptr; // CodeBuffer::expand failed 920 } 921 922 // Create a trampoline stub relocation which relates this trampoline stub 923 // with the call instruction at insts_call_instruction_offset in the 924 // instructions code-section. 925 align(wordSize); 926 relocate(trampoline_stub_Relocation::spec(code()->insts()->start() 927 + insts_call_instruction_offset)); 928 const int stub_start_offset = offset(); 929 930 // Now, create the trampoline stub's code: 931 // - load the call 932 // - call 933 Label target; 934 ldr(rscratch1, target); 935 br(rscratch1); 936 bind(target); 937 assert(offset() - stub_start_offset == NativeCallTrampolineStub::data_offset, 938 "should be"); 939 emit_int64((int64_t)dest); 940 941 const address stub_start_addr = addr_at(stub_start_offset); 942 943 assert(is_NativeCallTrampolineStub_at(stub_start_addr), "doesn't look like a trampoline"); 944 945 end_a_stub(); 946 return stub_start_addr; 947 } 948 949 int MacroAssembler::max_trampoline_stub_size() { 950 // Max stub size: alignment nop, TrampolineStub. 951 return NativeInstruction::instruction_size + NativeCallTrampolineStub::instruction_size; 952 } 953 954 void MacroAssembler::emit_static_call_stub() { 955 // CompiledDirectCall::set_to_interpreted knows the 956 // exact layout of this stub. 957 958 isb(); 959 mov_metadata(rmethod, nullptr); 960 961 // Jump to the entry point of the c2i stub. 962 if (codestub_branch_needs_far_jump()) { 963 movptr(rscratch1, 0); 964 br(rscratch1); 965 } else { 966 b(pc()); 967 } 968 } 969 970 int MacroAssembler::static_call_stub_size() { 971 if (!codestub_branch_needs_far_jump()) { 972 // isb; movk; movz; movz; b 973 return 5 * NativeInstruction::instruction_size; 974 } 975 // isb; movk; movz; movz; movk; movz; movz; br 976 return 8 * NativeInstruction::instruction_size; 977 } 978 979 void MacroAssembler::c2bool(Register x) { 980 // implements x == 0 ? 0 : 1 981 // note: must only look at least-significant byte of x 982 // since C-style booleans are stored in one byte 983 // only! (was bug) 984 tst(x, 0xff); 985 cset(x, Assembler::NE); 986 } 987 988 address MacroAssembler::ic_call(address entry, jint method_index) { 989 RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index); 990 movptr(rscratch2, (intptr_t)Universe::non_oop_word()); 991 return trampoline_call(Address(entry, rh)); 992 } 993 994 int MacroAssembler::ic_check_size() { 995 int extra_instructions = UseCompactObjectHeaders ? 1 : 0; 996 if (target_needs_far_branch(CAST_FROM_FN_PTR(address, SharedRuntime::get_ic_miss_stub()))) { 997 return NativeInstruction::instruction_size * (7 + extra_instructions); 998 } else { 999 return NativeInstruction::instruction_size * (5 + extra_instructions); 1000 } 1001 } 1002 1003 int MacroAssembler::ic_check(int end_alignment) { 1004 Register receiver = j_rarg0; 1005 Register data = rscratch2; 1006 Register tmp1 = rscratch1; 1007 Register tmp2 = r10; 1008 1009 // The UEP of a code blob ensures that the VEP is padded. However, the padding of the UEP is placed 1010 // before the inline cache check, so we don't have to execute any nop instructions when dispatching 1011 // through the UEP, yet we can ensure that the VEP is aligned appropriately. That's why we align 1012 // before the inline cache check here, and not after 1013 align(end_alignment, offset() + ic_check_size()); 1014 1015 int uep_offset = offset(); 1016 1017 if (UseCompactObjectHeaders) { 1018 load_narrow_klass_compact(tmp1, receiver); 1019 ldrw(tmp2, Address(data, CompiledICData::speculated_klass_offset())); 1020 cmpw(tmp1, tmp2); 1021 } else if (UseCompressedClassPointers) { 1022 ldrw(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes())); 1023 ldrw(tmp2, Address(data, CompiledICData::speculated_klass_offset())); 1024 cmpw(tmp1, tmp2); 1025 } else { 1026 ldr(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes())); 1027 ldr(tmp2, Address(data, CompiledICData::speculated_klass_offset())); 1028 cmp(tmp1, tmp2); 1029 } 1030 1031 Label dont; 1032 br(Assembler::EQ, dont); 1033 far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub())); 1034 bind(dont); 1035 assert((offset() % end_alignment) == 0, "Misaligned verified entry point"); 1036 1037 return uep_offset; 1038 } 1039 1040 // Implementation of call_VM versions 1041 1042 void MacroAssembler::call_VM(Register oop_result, 1043 address entry_point, 1044 bool check_exceptions) { 1045 call_VM_helper(oop_result, entry_point, 0, check_exceptions); 1046 } 1047 1048 void MacroAssembler::call_VM(Register oop_result, 1049 address entry_point, 1050 Register arg_1, 1051 bool check_exceptions) { 1052 pass_arg1(this, arg_1); 1053 call_VM_helper(oop_result, entry_point, 1, check_exceptions); 1054 } 1055 1056 void MacroAssembler::call_VM(Register oop_result, 1057 address entry_point, 1058 Register arg_1, 1059 Register arg_2, 1060 bool check_exceptions) { 1061 assert_different_registers(arg_1, c_rarg2); 1062 pass_arg2(this, arg_2); 1063 pass_arg1(this, arg_1); 1064 call_VM_helper(oop_result, entry_point, 2, check_exceptions); 1065 } 1066 1067 void MacroAssembler::call_VM(Register oop_result, 1068 address entry_point, 1069 Register arg_1, 1070 Register arg_2, 1071 Register arg_3, 1072 bool check_exceptions) { 1073 assert_different_registers(arg_1, c_rarg2, c_rarg3); 1074 assert_different_registers(arg_2, c_rarg3); 1075 pass_arg3(this, arg_3); 1076 1077 pass_arg2(this, arg_2); 1078 1079 pass_arg1(this, arg_1); 1080 call_VM_helper(oop_result, entry_point, 3, check_exceptions); 1081 } 1082 1083 void MacroAssembler::call_VM(Register oop_result, 1084 Register last_java_sp, 1085 address entry_point, 1086 int number_of_arguments, 1087 bool check_exceptions) { 1088 call_VM_base(oop_result, rthread, last_java_sp, entry_point, number_of_arguments, check_exceptions); 1089 } 1090 1091 void MacroAssembler::call_VM(Register oop_result, 1092 Register last_java_sp, 1093 address entry_point, 1094 Register arg_1, 1095 bool check_exceptions) { 1096 pass_arg1(this, arg_1); 1097 call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions); 1098 } 1099 1100 void MacroAssembler::call_VM(Register oop_result, 1101 Register last_java_sp, 1102 address entry_point, 1103 Register arg_1, 1104 Register arg_2, 1105 bool check_exceptions) { 1106 1107 assert_different_registers(arg_1, c_rarg2); 1108 pass_arg2(this, arg_2); 1109 pass_arg1(this, arg_1); 1110 call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions); 1111 } 1112 1113 void MacroAssembler::call_VM(Register oop_result, 1114 Register last_java_sp, 1115 address entry_point, 1116 Register arg_1, 1117 Register arg_2, 1118 Register arg_3, 1119 bool check_exceptions) { 1120 assert_different_registers(arg_1, c_rarg2, c_rarg3); 1121 assert_different_registers(arg_2, c_rarg3); 1122 pass_arg3(this, arg_3); 1123 pass_arg2(this, arg_2); 1124 pass_arg1(this, arg_1); 1125 call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions); 1126 } 1127 1128 1129 void MacroAssembler::get_vm_result_oop(Register oop_result, Register java_thread) { 1130 ldr(oop_result, Address(java_thread, JavaThread::vm_result_oop_offset())); 1131 str(zr, Address(java_thread, JavaThread::vm_result_oop_offset())); 1132 verify_oop_msg(oop_result, "broken oop in call_VM_base"); 1133 } 1134 1135 void MacroAssembler::get_vm_result_metadata(Register metadata_result, Register java_thread) { 1136 ldr(metadata_result, Address(java_thread, JavaThread::vm_result_metadata_offset())); 1137 str(zr, Address(java_thread, JavaThread::vm_result_metadata_offset())); 1138 } 1139 1140 void MacroAssembler::align(int modulus) { 1141 align(modulus, offset()); 1142 } 1143 1144 // Ensure that the code at target bytes offset from the current offset() is aligned 1145 // according to modulus. 1146 void MacroAssembler::align(int modulus, int target) { 1147 int delta = target - offset(); 1148 while ((offset() + delta) % modulus != 0) nop(); 1149 } 1150 1151 void MacroAssembler::post_call_nop() { 1152 if (!Continuations::enabled()) { 1153 return; 1154 } 1155 InstructionMark im(this); 1156 relocate(post_call_nop_Relocation::spec()); 1157 InlineSkippedInstructionsCounter skipCounter(this); 1158 nop(); 1159 movk(zr, 0); 1160 movk(zr, 0); 1161 } 1162 1163 // these are no-ops overridden by InterpreterMacroAssembler 1164 1165 void MacroAssembler::check_and_handle_earlyret(Register java_thread) { } 1166 1167 void MacroAssembler::check_and_handle_popframe(Register java_thread) { } 1168 1169 // Look up the method for a megamorphic invokeinterface call. 1170 // The target method is determined by <intf_klass, itable_index>. 1171 // The receiver klass is in recv_klass. 1172 // On success, the result will be in method_result, and execution falls through. 1173 // On failure, execution transfers to the given label. 1174 void MacroAssembler::lookup_interface_method(Register recv_klass, 1175 Register intf_klass, 1176 RegisterOrConstant itable_index, 1177 Register method_result, 1178 Register scan_temp, 1179 Label& L_no_such_interface, 1180 bool return_method) { 1181 assert_different_registers(recv_klass, intf_klass, scan_temp); 1182 assert_different_registers(method_result, intf_klass, scan_temp); 1183 assert(recv_klass != method_result || !return_method, 1184 "recv_klass can be destroyed when method isn't needed"); 1185 assert(itable_index.is_constant() || itable_index.as_register() == method_result, 1186 "caller must use same register for non-constant itable index as for method"); 1187 1188 // Compute start of first itableOffsetEntry (which is at the end of the vtable) 1189 int vtable_base = in_bytes(Klass::vtable_start_offset()); 1190 int itentry_off = in_bytes(itableMethodEntry::method_offset()); 1191 int scan_step = itableOffsetEntry::size() * wordSize; 1192 int vte_size = vtableEntry::size_in_bytes(); 1193 assert(vte_size == wordSize, "else adjust times_vte_scale"); 1194 1195 ldrw(scan_temp, Address(recv_klass, Klass::vtable_length_offset())); 1196 1197 // Could store the aligned, prescaled offset in the klass. 1198 // lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base)); 1199 lea(scan_temp, Address(recv_klass, scan_temp, Address::lsl(3))); 1200 add(scan_temp, scan_temp, vtable_base); 1201 1202 if (return_method) { 1203 // Adjust recv_klass by scaled itable_index, so we can free itable_index. 1204 assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below"); 1205 // lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off)); 1206 lea(recv_klass, Address(recv_klass, itable_index, Address::lsl(3))); 1207 if (itentry_off) 1208 add(recv_klass, recv_klass, itentry_off); 1209 } 1210 1211 // for (scan = klass->itable(); scan->interface() != nullptr; scan += scan_step) { 1212 // if (scan->interface() == intf) { 1213 // result = (klass + scan->offset() + itable_index); 1214 // } 1215 // } 1216 Label search, found_method; 1217 1218 ldr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset())); 1219 cmp(intf_klass, method_result); 1220 br(Assembler::EQ, found_method); 1221 bind(search); 1222 // Check that the previous entry is non-null. A null entry means that 1223 // the receiver class doesn't implement the interface, and wasn't the 1224 // same as when the caller was compiled. 1225 cbz(method_result, L_no_such_interface); 1226 if (itableOffsetEntry::interface_offset() != 0) { 1227 add(scan_temp, scan_temp, scan_step); 1228 ldr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset())); 1229 } else { 1230 ldr(method_result, Address(pre(scan_temp, scan_step))); 1231 } 1232 cmp(intf_klass, method_result); 1233 br(Assembler::NE, search); 1234 1235 bind(found_method); 1236 1237 // Got a hit. 1238 if (return_method) { 1239 ldrw(scan_temp, Address(scan_temp, itableOffsetEntry::offset_offset())); 1240 ldr(method_result, Address(recv_klass, scan_temp, Address::uxtw(0))); 1241 } 1242 } 1243 1244 // Look up the method for a megamorphic invokeinterface call in a single pass over itable: 1245 // - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICData 1246 // - find a holder_klass (class that implements the method) vtable offset and get the method from vtable by index 1247 // The target method is determined by <holder_klass, itable_index>. 1248 // The receiver klass is in recv_klass. 1249 // On success, the result will be in method_result, and execution falls through. 1250 // On failure, execution transfers to the given label. 1251 void MacroAssembler::lookup_interface_method_stub(Register recv_klass, 1252 Register holder_klass, 1253 Register resolved_klass, 1254 Register method_result, 1255 Register temp_itbl_klass, 1256 Register scan_temp, 1257 int itable_index, 1258 Label& L_no_such_interface) { 1259 // 'method_result' is only used as output register at the very end of this method. 1260 // Until then we can reuse it as 'holder_offset'. 1261 Register holder_offset = method_result; 1262 assert_different_registers(resolved_klass, recv_klass, holder_klass, temp_itbl_klass, scan_temp, holder_offset); 1263 1264 int vtable_start_offset = in_bytes(Klass::vtable_start_offset()); 1265 int itable_offset_entry_size = itableOffsetEntry::size() * wordSize; 1266 int ioffset = in_bytes(itableOffsetEntry::interface_offset()); 1267 int ooffset = in_bytes(itableOffsetEntry::offset_offset()); 1268 1269 Label L_loop_search_resolved_entry, L_resolved_found, L_holder_found; 1270 1271 ldrw(scan_temp, Address(recv_klass, Klass::vtable_length_offset())); 1272 add(recv_klass, recv_klass, vtable_start_offset + ioffset); 1273 // itableOffsetEntry[] itable = recv_klass + Klass::vtable_start_offset() + sizeof(vtableEntry) * recv_klass->_vtable_len; 1274 // temp_itbl_klass = itable[0]._interface; 1275 int vtblEntrySize = vtableEntry::size_in_bytes(); 1276 assert(vtblEntrySize == wordSize, "ldr lsl shift amount must be 3"); 1277 ldr(temp_itbl_klass, Address(recv_klass, scan_temp, Address::lsl(exact_log2(vtblEntrySize)))); 1278 mov(holder_offset, zr); 1279 // scan_temp = &(itable[0]._interface) 1280 lea(scan_temp, Address(recv_klass, scan_temp, Address::lsl(exact_log2(vtblEntrySize)))); 1281 1282 // Initial checks: 1283 // - if (holder_klass != resolved_klass), go to "scan for resolved" 1284 // - if (itable[0] == holder_klass), shortcut to "holder found" 1285 // - if (itable[0] == 0), no such interface 1286 cmp(resolved_klass, holder_klass); 1287 br(Assembler::NE, L_loop_search_resolved_entry); 1288 cmp(holder_klass, temp_itbl_klass); 1289 br(Assembler::EQ, L_holder_found); 1290 cbz(temp_itbl_klass, L_no_such_interface); 1291 1292 // Loop: Look for holder_klass record in itable 1293 // do { 1294 // temp_itbl_klass = *(scan_temp += itable_offset_entry_size); 1295 // if (temp_itbl_klass == holder_klass) { 1296 // goto L_holder_found; // Found! 1297 // } 1298 // } while (temp_itbl_klass != 0); 1299 // goto L_no_such_interface // Not found. 1300 Label L_search_holder; 1301 bind(L_search_holder); 1302 ldr(temp_itbl_klass, Address(pre(scan_temp, itable_offset_entry_size))); 1303 cmp(holder_klass, temp_itbl_klass); 1304 br(Assembler::EQ, L_holder_found); 1305 cbnz(temp_itbl_klass, L_search_holder); 1306 1307 b(L_no_such_interface); 1308 1309 // Loop: Look for resolved_class record in itable 1310 // while (true) { 1311 // temp_itbl_klass = *(scan_temp += itable_offset_entry_size); 1312 // if (temp_itbl_klass == 0) { 1313 // goto L_no_such_interface; 1314 // } 1315 // if (temp_itbl_klass == resolved_klass) { 1316 // goto L_resolved_found; // Found! 1317 // } 1318 // if (temp_itbl_klass == holder_klass) { 1319 // holder_offset = scan_temp; 1320 // } 1321 // } 1322 // 1323 Label L_loop_search_resolved; 1324 bind(L_loop_search_resolved); 1325 ldr(temp_itbl_klass, Address(pre(scan_temp, itable_offset_entry_size))); 1326 bind(L_loop_search_resolved_entry); 1327 cbz(temp_itbl_klass, L_no_such_interface); 1328 cmp(resolved_klass, temp_itbl_klass); 1329 br(Assembler::EQ, L_resolved_found); 1330 cmp(holder_klass, temp_itbl_klass); 1331 br(Assembler::NE, L_loop_search_resolved); 1332 mov(holder_offset, scan_temp); 1333 b(L_loop_search_resolved); 1334 1335 // See if we already have a holder klass. If not, go and scan for it. 1336 bind(L_resolved_found); 1337 cbz(holder_offset, L_search_holder); 1338 mov(scan_temp, holder_offset); 1339 1340 // Finally, scan_temp contains holder_klass vtable offset 1341 bind(L_holder_found); 1342 ldrw(method_result, Address(scan_temp, ooffset - ioffset)); 1343 add(recv_klass, recv_klass, itable_index * wordSize + in_bytes(itableMethodEntry::method_offset()) 1344 - vtable_start_offset - ioffset); // substract offsets to restore the original value of recv_klass 1345 ldr(method_result, Address(recv_klass, method_result, Address::uxtw(0))); 1346 } 1347 1348 // virtual method calling 1349 void MacroAssembler::lookup_virtual_method(Register recv_klass, 1350 RegisterOrConstant vtable_index, 1351 Register method_result) { 1352 assert(vtableEntry::size() * wordSize == 8, 1353 "adjust the scaling in the code below"); 1354 int64_t vtable_offset_in_bytes = in_bytes(Klass::vtable_start_offset() + vtableEntry::method_offset()); 1355 1356 if (vtable_index.is_register()) { 1357 lea(method_result, Address(recv_klass, 1358 vtable_index.as_register(), 1359 Address::lsl(LogBytesPerWord))); 1360 ldr(method_result, Address(method_result, vtable_offset_in_bytes)); 1361 } else { 1362 vtable_offset_in_bytes += vtable_index.as_constant() * wordSize; 1363 ldr(method_result, 1364 form_address(rscratch1, recv_klass, vtable_offset_in_bytes, 0)); 1365 } 1366 } 1367 1368 void MacroAssembler::check_klass_subtype(Register sub_klass, 1369 Register super_klass, 1370 Register temp_reg, 1371 Label& L_success) { 1372 Label L_failure; 1373 check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg, &L_success, &L_failure, nullptr); 1374 check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, noreg, &L_success, nullptr); 1375 bind(L_failure); 1376 } 1377 1378 1379 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass, 1380 Register super_klass, 1381 Register temp_reg, 1382 Label* L_success, 1383 Label* L_failure, 1384 Label* L_slow_path, 1385 Register super_check_offset) { 1386 assert_different_registers(sub_klass, super_klass, temp_reg, super_check_offset); 1387 bool must_load_sco = ! super_check_offset->is_valid(); 1388 if (must_load_sco) { 1389 assert(temp_reg != noreg, "supply either a temp or a register offset"); 1390 } 1391 1392 Label L_fallthrough; 1393 int label_nulls = 0; 1394 if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; } 1395 if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; } 1396 if (L_slow_path == nullptr) { L_slow_path = &L_fallthrough; label_nulls++; } 1397 assert(label_nulls <= 1, "at most one null in the batch"); 1398 1399 int sco_offset = in_bytes(Klass::super_check_offset_offset()); 1400 Address super_check_offset_addr(super_klass, sco_offset); 1401 1402 // Hacked jmp, which may only be used just before L_fallthrough. 1403 #define final_jmp(label) \ 1404 if (&(label) == &L_fallthrough) { /*do nothing*/ } \ 1405 else b(label) /*omit semi*/ 1406 1407 // If the pointers are equal, we are done (e.g., String[] elements). 1408 // This self-check enables sharing of secondary supertype arrays among 1409 // non-primary types such as array-of-interface. Otherwise, each such 1410 // type would need its own customized SSA. 1411 // We move this check to the front of the fast path because many 1412 // type checks are in fact trivially successful in this manner, 1413 // so we get a nicely predicted branch right at the start of the check. 1414 // TODO 8370341 For a direct pointer comparison, we need the refined array klass pointer 1415 cmp(sub_klass, super_klass); 1416 br(Assembler::EQ, *L_success); 1417 1418 // Check the supertype display: 1419 if (must_load_sco) { 1420 ldrw(temp_reg, super_check_offset_addr); 1421 super_check_offset = temp_reg; 1422 } 1423 1424 Address super_check_addr(sub_klass, super_check_offset); 1425 ldr(rscratch1, super_check_addr); 1426 cmp(super_klass, rscratch1); // load displayed supertype 1427 br(Assembler::EQ, *L_success); 1428 1429 // This check has worked decisively for primary supers. 1430 // Secondary supers are sought in the super_cache ('super_cache_addr'). 1431 // (Secondary supers are interfaces and very deeply nested subtypes.) 1432 // This works in the same check above because of a tricky aliasing 1433 // between the super_cache and the primary super display elements. 1434 // (The 'super_check_addr' can address either, as the case requires.) 1435 // Note that the cache is updated below if it does not help us find 1436 // what we need immediately. 1437 // So if it was a primary super, we can just fail immediately. 1438 // Otherwise, it's the slow path for us (no success at this point). 1439 1440 sub(rscratch1, super_check_offset, in_bytes(Klass::secondary_super_cache_offset())); 1441 if (L_failure == &L_fallthrough) { 1442 cbz(rscratch1, *L_slow_path); 1443 } else { 1444 cbnz(rscratch1, *L_failure); 1445 final_jmp(*L_slow_path); 1446 } 1447 1448 bind(L_fallthrough); 1449 1450 #undef final_jmp 1451 } 1452 1453 // These two are taken from x86, but they look generally useful 1454 1455 // scans count pointer sized words at [addr] for occurrence of value, 1456 // generic 1457 void MacroAssembler::repne_scan(Register addr, Register value, Register count, 1458 Register scratch) { 1459 Label Lloop, Lexit; 1460 cbz(count, Lexit); 1461 bind(Lloop); 1462 ldr(scratch, post(addr, wordSize)); 1463 cmp(value, scratch); 1464 br(EQ, Lexit); 1465 sub(count, count, 1); 1466 cbnz(count, Lloop); 1467 bind(Lexit); 1468 } 1469 1470 // scans count 4 byte words at [addr] for occurrence of value, 1471 // generic 1472 void MacroAssembler::repne_scanw(Register addr, Register value, Register count, 1473 Register scratch) { 1474 Label Lloop, Lexit; 1475 cbz(count, Lexit); 1476 bind(Lloop); 1477 ldrw(scratch, post(addr, wordSize)); 1478 cmpw(value, scratch); 1479 br(EQ, Lexit); 1480 sub(count, count, 1); 1481 cbnz(count, Lloop); 1482 bind(Lexit); 1483 } 1484 1485 void MacroAssembler::check_klass_subtype_slow_path_linear(Register sub_klass, 1486 Register super_klass, 1487 Register temp_reg, 1488 Register temp2_reg, 1489 Label* L_success, 1490 Label* L_failure, 1491 bool set_cond_codes) { 1492 // NB! Callers may assume that, when temp2_reg is a valid register, 1493 // this code sets it to a nonzero value. 1494 1495 assert_different_registers(sub_klass, super_klass, temp_reg); 1496 if (temp2_reg != noreg) 1497 assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg, rscratch1); 1498 #define IS_A_TEMP(reg) ((reg) == temp_reg || (reg) == temp2_reg) 1499 1500 Label L_fallthrough; 1501 int label_nulls = 0; 1502 if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; } 1503 if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; } 1504 assert(label_nulls <= 1, "at most one null in the batch"); 1505 1506 // a couple of useful fields in sub_klass: 1507 int ss_offset = in_bytes(Klass::secondary_supers_offset()); 1508 int sc_offset = in_bytes(Klass::secondary_super_cache_offset()); 1509 Address secondary_supers_addr(sub_klass, ss_offset); 1510 Address super_cache_addr( sub_klass, sc_offset); 1511 1512 BLOCK_COMMENT("check_klass_subtype_slow_path"); 1513 1514 // Do a linear scan of the secondary super-klass chain. 1515 // This code is rarely used, so simplicity is a virtue here. 1516 // The repne_scan instruction uses fixed registers, which we must spill. 1517 // Don't worry too much about pre-existing connections with the input regs. 1518 1519 assert(sub_klass != r0, "killed reg"); // killed by mov(r0, super) 1520 assert(sub_klass != r2, "killed reg"); // killed by lea(r2, &pst_counter) 1521 1522 RegSet pushed_registers; 1523 if (!IS_A_TEMP(r2)) pushed_registers += r2; 1524 if (!IS_A_TEMP(r5)) pushed_registers += r5; 1525 1526 if (super_klass != r0) { 1527 if (!IS_A_TEMP(r0)) pushed_registers += r0; 1528 } 1529 1530 push(pushed_registers, sp); 1531 1532 // Get super_klass value into r0 (even if it was in r5 or r2). 1533 if (super_klass != r0) { 1534 mov(r0, super_klass); 1535 } 1536 1537 #ifndef PRODUCT 1538 incrementw(ExternalAddress((address)&SharedRuntime::_partial_subtype_ctr)); 1539 #endif //PRODUCT 1540 1541 // We will consult the secondary-super array. 1542 ldr(r5, secondary_supers_addr); 1543 // Load the array length. 1544 ldrw(r2, Address(r5, Array<Klass*>::length_offset_in_bytes())); 1545 // Skip to start of data. 1546 add(r5, r5, Array<Klass*>::base_offset_in_bytes()); 1547 1548 cmp(sp, zr); // Clear Z flag; SP is never zero 1549 // Scan R2 words at [R5] for an occurrence of R0. 1550 // Set NZ/Z based on last compare. 1551 repne_scan(r5, r0, r2, rscratch1); 1552 1553 // Unspill the temp. registers: 1554 pop(pushed_registers, sp); 1555 1556 br(Assembler::NE, *L_failure); 1557 1558 // Success. Cache the super we found and proceed in triumph. 1559 1560 if (UseSecondarySupersCache) { 1561 str(super_klass, super_cache_addr); 1562 } 1563 1564 if (L_success != &L_fallthrough) { 1565 b(*L_success); 1566 } 1567 1568 #undef IS_A_TEMP 1569 1570 bind(L_fallthrough); 1571 } 1572 1573 // If Register r is invalid, remove a new register from 1574 // available_regs, and add new register to regs_to_push. 1575 Register MacroAssembler::allocate_if_noreg(Register r, 1576 RegSetIterator<Register> &available_regs, 1577 RegSet ®s_to_push) { 1578 if (!r->is_valid()) { 1579 r = *available_regs++; 1580 regs_to_push += r; 1581 } 1582 return r; 1583 } 1584 1585 // check_klass_subtype_slow_path_table() looks for super_klass in the 1586 // hash table belonging to super_klass, branching to L_success or 1587 // L_failure as appropriate. This is essentially a shim which 1588 // allocates registers as necessary then calls 1589 // lookup_secondary_supers_table() to do the work. Any of the temp 1590 // regs may be noreg, in which case this logic will chooses some 1591 // registers push and pop them from the stack. 1592 void MacroAssembler::check_klass_subtype_slow_path_table(Register sub_klass, 1593 Register super_klass, 1594 Register temp_reg, 1595 Register temp2_reg, 1596 Register temp3_reg, 1597 Register result_reg, 1598 FloatRegister vtemp, 1599 Label* L_success, 1600 Label* L_failure, 1601 bool set_cond_codes) { 1602 RegSet temps = RegSet::of(temp_reg, temp2_reg, temp3_reg); 1603 1604 assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg, rscratch1); 1605 1606 Label L_fallthrough; 1607 int label_nulls = 0; 1608 if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; } 1609 if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; } 1610 assert(label_nulls <= 1, "at most one null in the batch"); 1611 1612 BLOCK_COMMENT("check_klass_subtype_slow_path"); 1613 1614 RegSetIterator<Register> available_regs 1615 = (RegSet::range(r0, r15) - temps - sub_klass - super_klass).begin(); 1616 1617 RegSet pushed_regs; 1618 1619 temp_reg = allocate_if_noreg(temp_reg, available_regs, pushed_regs); 1620 temp2_reg = allocate_if_noreg(temp2_reg, available_regs, pushed_regs); 1621 temp3_reg = allocate_if_noreg(temp3_reg, available_regs, pushed_regs); 1622 result_reg = allocate_if_noreg(result_reg, available_regs, pushed_regs); 1623 1624 push(pushed_regs, sp); 1625 1626 lookup_secondary_supers_table_var(sub_klass, 1627 super_klass, 1628 temp_reg, temp2_reg, temp3_reg, vtemp, result_reg, 1629 nullptr); 1630 cmp(result_reg, zr); 1631 1632 // Unspill the temp. registers: 1633 pop(pushed_regs, sp); 1634 1635 // NB! Callers may assume that, when set_cond_codes is true, this 1636 // code sets temp2_reg to a nonzero value. 1637 if (set_cond_codes) { 1638 mov(temp2_reg, 1); 1639 } 1640 1641 br(Assembler::NE, *L_failure); 1642 1643 if (L_success != &L_fallthrough) { 1644 b(*L_success); 1645 } 1646 1647 bind(L_fallthrough); 1648 } 1649 1650 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass, 1651 Register super_klass, 1652 Register temp_reg, 1653 Register temp2_reg, 1654 Label* L_success, 1655 Label* L_failure, 1656 bool set_cond_codes) { 1657 if (UseSecondarySupersTable) { 1658 check_klass_subtype_slow_path_table 1659 (sub_klass, super_klass, temp_reg, temp2_reg, /*temp3*/noreg, /*result*/noreg, 1660 /*vtemp*/fnoreg, 1661 L_success, L_failure, set_cond_codes); 1662 } else { 1663 check_klass_subtype_slow_path_linear 1664 (sub_klass, super_klass, temp_reg, temp2_reg, L_success, L_failure, set_cond_codes); 1665 } 1666 } 1667 1668 1669 // Ensure that the inline code and the stub are using the same registers. 1670 #define LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS \ 1671 do { \ 1672 assert(r_super_klass == r0 && \ 1673 r_array_base == r1 && \ 1674 r_array_length == r2 && \ 1675 (r_array_index == r3 || r_array_index == noreg) && \ 1676 (r_sub_klass == r4 || r_sub_klass == noreg) && \ 1677 (r_bitmap == rscratch2 || r_bitmap == noreg) && \ 1678 (result == r5 || result == noreg), "registers must match aarch64.ad"); \ 1679 } while(0) 1680 1681 bool MacroAssembler::lookup_secondary_supers_table_const(Register r_sub_klass, 1682 Register r_super_klass, 1683 Register temp1, 1684 Register temp2, 1685 Register temp3, 1686 FloatRegister vtemp, 1687 Register result, 1688 u1 super_klass_slot, 1689 bool stub_is_near) { 1690 assert_different_registers(r_sub_klass, temp1, temp2, temp3, result, rscratch1, rscratch2); 1691 1692 Label L_fallthrough; 1693 1694 BLOCK_COMMENT("lookup_secondary_supers_table {"); 1695 1696 const Register 1697 r_array_base = temp1, // r1 1698 r_array_length = temp2, // r2 1699 r_array_index = temp3, // r3 1700 r_bitmap = rscratch2; 1701 1702 LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS; 1703 1704 u1 bit = super_klass_slot; 1705 1706 // Make sure that result is nonzero if the TBZ below misses. 1707 mov(result, 1); 1708 1709 // We're going to need the bitmap in a vector reg and in a core reg, 1710 // so load both now. 1711 ldr(r_bitmap, Address(r_sub_klass, Klass::secondary_supers_bitmap_offset())); 1712 if (bit != 0) { 1713 ldrd(vtemp, Address(r_sub_klass, Klass::secondary_supers_bitmap_offset())); 1714 } 1715 // First check the bitmap to see if super_klass might be present. If 1716 // the bit is zero, we are certain that super_klass is not one of 1717 // the secondary supers. 1718 tbz(r_bitmap, bit, L_fallthrough); 1719 1720 // Get the first array index that can contain super_klass into r_array_index. 1721 if (bit != 0) { 1722 shld(vtemp, vtemp, Klass::SECONDARY_SUPERS_TABLE_MASK - bit); 1723 cnt(vtemp, T8B, vtemp); 1724 addv(vtemp, T8B, vtemp); 1725 fmovd(r_array_index, vtemp); 1726 } else { 1727 mov(r_array_index, (u1)1); 1728 } 1729 // NB! r_array_index is off by 1. It is compensated by keeping r_array_base off by 1 word. 1730 1731 // We will consult the secondary-super array. 1732 ldr(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset()))); 1733 1734 // The value i in r_array_index is >= 1, so even though r_array_base 1735 // points to the length, we don't need to adjust it to point to the 1736 // data. 1737 assert(Array<Klass*>::base_offset_in_bytes() == wordSize, "Adjust this code"); 1738 assert(Array<Klass*>::length_offset_in_bytes() == 0, "Adjust this code"); 1739 1740 ldr(result, Address(r_array_base, r_array_index, Address::lsl(LogBytesPerWord))); 1741 eor(result, result, r_super_klass); 1742 cbz(result, L_fallthrough); // Found a match 1743 1744 // Is there another entry to check? Consult the bitmap. 1745 tbz(r_bitmap, (bit + 1) & Klass::SECONDARY_SUPERS_TABLE_MASK, L_fallthrough); 1746 1747 // Linear probe. 1748 if (bit != 0) { 1749 ror(r_bitmap, r_bitmap, bit); 1750 } 1751 1752 // The slot we just inspected is at secondary_supers[r_array_index - 1]. 1753 // The next slot to be inspected, by the stub we're about to call, 1754 // is secondary_supers[r_array_index]. Bits 0 and 1 in the bitmap 1755 // have been checked. 1756 Address stub = RuntimeAddress(StubRoutines::lookup_secondary_supers_table_slow_path_stub()); 1757 if (stub_is_near) { 1758 bl(stub); 1759 } else { 1760 address call = trampoline_call(stub); 1761 if (call == nullptr) { 1762 return false; // trampoline allocation failed 1763 } 1764 } 1765 1766 BLOCK_COMMENT("} lookup_secondary_supers_table"); 1767 1768 bind(L_fallthrough); 1769 1770 if (VerifySecondarySupers) { 1771 verify_secondary_supers_table(r_sub_klass, r_super_klass, // r4, r0 1772 temp1, temp2, result); // r1, r2, r5 1773 } 1774 return true; 1775 } 1776 1777 // At runtime, return 0 in result if r_super_klass is a superclass of 1778 // r_sub_klass, otherwise return nonzero. Use this version of 1779 // lookup_secondary_supers_table() if you don't know ahead of time 1780 // which superclass will be searched for. Used by interpreter and 1781 // runtime stubs. It is larger and has somewhat greater latency than 1782 // the version above, which takes a constant super_klass_slot. 1783 void MacroAssembler::lookup_secondary_supers_table_var(Register r_sub_klass, 1784 Register r_super_klass, 1785 Register temp1, 1786 Register temp2, 1787 Register temp3, 1788 FloatRegister vtemp, 1789 Register result, 1790 Label *L_success) { 1791 assert_different_registers(r_sub_klass, temp1, temp2, temp3, result, rscratch1, rscratch2); 1792 1793 Label L_fallthrough; 1794 1795 BLOCK_COMMENT("lookup_secondary_supers_table {"); 1796 1797 const Register 1798 r_array_index = temp3, 1799 slot = rscratch1, 1800 r_bitmap = rscratch2; 1801 1802 ldrb(slot, Address(r_super_klass, Klass::hash_slot_offset())); 1803 1804 // Make sure that result is nonzero if the test below misses. 1805 mov(result, 1); 1806 1807 ldr(r_bitmap, Address(r_sub_klass, Klass::secondary_supers_bitmap_offset())); 1808 1809 // First check the bitmap to see if super_klass might be present. If 1810 // the bit is zero, we are certain that super_klass is not one of 1811 // the secondary supers. 1812 1813 // This next instruction is equivalent to: 1814 // mov(tmp_reg, (u1)(Klass::SECONDARY_SUPERS_TABLE_SIZE - 1)); 1815 // sub(temp2, tmp_reg, slot); 1816 eor(temp2, slot, (u1)(Klass::SECONDARY_SUPERS_TABLE_SIZE - 1)); 1817 lslv(temp2, r_bitmap, temp2); 1818 tbz(temp2, Klass::SECONDARY_SUPERS_TABLE_SIZE - 1, L_fallthrough); 1819 1820 bool must_save_v0 = (vtemp == fnoreg); 1821 if (must_save_v0) { 1822 // temp1 and result are free, so use them to preserve vtemp 1823 vtemp = v0; 1824 mov(temp1, vtemp, D, 0); 1825 mov(result, vtemp, D, 1); 1826 } 1827 1828 // Get the first array index that can contain super_klass into r_array_index. 1829 mov(vtemp, D, 0, temp2); 1830 cnt(vtemp, T8B, vtemp); 1831 addv(vtemp, T8B, vtemp); 1832 mov(r_array_index, vtemp, D, 0); 1833 1834 if (must_save_v0) { 1835 mov(vtemp, D, 0, temp1 ); 1836 mov(vtemp, D, 1, result); 1837 } 1838 1839 // NB! r_array_index is off by 1. It is compensated by keeping r_array_base off by 1 word. 1840 1841 const Register 1842 r_array_base = temp1, 1843 r_array_length = temp2; 1844 1845 // The value i in r_array_index is >= 1, so even though r_array_base 1846 // points to the length, we don't need to adjust it to point to the 1847 // data. 1848 assert(Array<Klass*>::base_offset_in_bytes() == wordSize, "Adjust this code"); 1849 assert(Array<Klass*>::length_offset_in_bytes() == 0, "Adjust this code"); 1850 1851 // We will consult the secondary-super array. 1852 ldr(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset()))); 1853 1854 ldr(result, Address(r_array_base, r_array_index, Address::lsl(LogBytesPerWord))); 1855 eor(result, result, r_super_klass); 1856 cbz(result, L_success ? *L_success : L_fallthrough); // Found a match 1857 1858 // Is there another entry to check? Consult the bitmap. 1859 rorv(r_bitmap, r_bitmap, slot); 1860 // rol(r_bitmap, r_bitmap, 1); 1861 tbz(r_bitmap, 1, L_fallthrough); 1862 1863 // The slot we just inspected is at secondary_supers[r_array_index - 1]. 1864 // The next slot to be inspected, by the logic we're about to call, 1865 // is secondary_supers[r_array_index]. Bits 0 and 1 in the bitmap 1866 // have been checked. 1867 lookup_secondary_supers_table_slow_path(r_super_klass, r_array_base, r_array_index, 1868 r_bitmap, r_array_length, result, /*is_stub*/false); 1869 1870 BLOCK_COMMENT("} lookup_secondary_supers_table"); 1871 1872 bind(L_fallthrough); 1873 1874 if (VerifySecondarySupers) { 1875 verify_secondary_supers_table(r_sub_klass, r_super_klass, // r4, r0 1876 temp1, temp2, result); // r1, r2, r5 1877 } 1878 1879 if (L_success) { 1880 cbz(result, *L_success); 1881 } 1882 } 1883 1884 // Called by code generated by check_klass_subtype_slow_path 1885 // above. This is called when there is a collision in the hashed 1886 // lookup in the secondary supers array. 1887 void MacroAssembler::lookup_secondary_supers_table_slow_path(Register r_super_klass, 1888 Register r_array_base, 1889 Register r_array_index, 1890 Register r_bitmap, 1891 Register temp1, 1892 Register result, 1893 bool is_stub) { 1894 assert_different_registers(r_super_klass, r_array_base, r_array_index, r_bitmap, temp1, result, rscratch1); 1895 1896 const Register 1897 r_array_length = temp1, 1898 r_sub_klass = noreg; // unused 1899 1900 if (is_stub) { 1901 LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS; 1902 } 1903 1904 Label L_fallthrough, L_huge; 1905 1906 // Load the array length. 1907 ldrw(r_array_length, Address(r_array_base, Array<Klass*>::length_offset_in_bytes())); 1908 // And adjust the array base to point to the data. 1909 // NB! Effectively increments current slot index by 1. 1910 assert(Array<Klass*>::base_offset_in_bytes() == wordSize, ""); 1911 add(r_array_base, r_array_base, Array<Klass*>::base_offset_in_bytes()); 1912 1913 // The bitmap is full to bursting. 1914 // Implicit invariant: BITMAP_FULL implies (length > 0) 1915 assert(Klass::SECONDARY_SUPERS_BITMAP_FULL == ~uintx(0), ""); 1916 cmpw(r_array_length, (u1)(Klass::SECONDARY_SUPERS_TABLE_SIZE - 2)); 1917 br(GT, L_huge); 1918 1919 // NB! Our caller has checked bits 0 and 1 in the bitmap. The 1920 // current slot (at secondary_supers[r_array_index]) has not yet 1921 // been inspected, and r_array_index may be out of bounds if we 1922 // wrapped around the end of the array. 1923 1924 { // This is conventional linear probing, but instead of terminating 1925 // when a null entry is found in the table, we maintain a bitmap 1926 // in which a 0 indicates missing entries. 1927 // As long as the bitmap is not completely full, 1928 // array_length == popcount(bitmap). The array_length check above 1929 // guarantees there are 0s in the bitmap, so the loop eventually 1930 // terminates. 1931 Label L_loop; 1932 bind(L_loop); 1933 1934 // Check for wraparound. 1935 cmp(r_array_index, r_array_length); 1936 csel(r_array_index, zr, r_array_index, GE); 1937 1938 ldr(rscratch1, Address(r_array_base, r_array_index, Address::lsl(LogBytesPerWord))); 1939 eor(result, rscratch1, r_super_klass); 1940 cbz(result, L_fallthrough); 1941 1942 tbz(r_bitmap, 2, L_fallthrough); // look-ahead check (Bit 2); result is non-zero 1943 1944 ror(r_bitmap, r_bitmap, 1); 1945 add(r_array_index, r_array_index, 1); 1946 b(L_loop); 1947 } 1948 1949 { // Degenerate case: more than 64 secondary supers. 1950 // FIXME: We could do something smarter here, maybe a vectorized 1951 // comparison or a binary search, but is that worth any added 1952 // complexity? 1953 bind(L_huge); 1954 cmp(sp, zr); // Clear Z flag; SP is never zero 1955 repne_scan(r_array_base, r_super_klass, r_array_length, rscratch1); 1956 cset(result, NE); // result == 0 iff we got a match. 1957 } 1958 1959 bind(L_fallthrough); 1960 } 1961 1962 // Make sure that the hashed lookup and a linear scan agree. 1963 void MacroAssembler::verify_secondary_supers_table(Register r_sub_klass, 1964 Register r_super_klass, 1965 Register temp1, 1966 Register temp2, 1967 Register result) { 1968 assert_different_registers(r_sub_klass, r_super_klass, temp1, temp2, result, rscratch1); 1969 1970 const Register 1971 r_array_base = temp1, 1972 r_array_length = temp2, 1973 r_array_index = noreg, // unused 1974 r_bitmap = noreg; // unused 1975 1976 BLOCK_COMMENT("verify_secondary_supers_table {"); 1977 1978 // We will consult the secondary-super array. 1979 ldr(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset()))); 1980 1981 // Load the array length. 1982 ldrw(r_array_length, Address(r_array_base, Array<Klass*>::length_offset_in_bytes())); 1983 // And adjust the array base to point to the data. 1984 add(r_array_base, r_array_base, Array<Klass*>::base_offset_in_bytes()); 1985 1986 cmp(sp, zr); // Clear Z flag; SP is never zero 1987 // Scan R2 words at [R5] for an occurrence of R0. 1988 // Set NZ/Z based on last compare. 1989 repne_scan(/*addr*/r_array_base, /*value*/r_super_klass, /*count*/r_array_length, rscratch2); 1990 // rscratch1 == 0 iff we got a match. 1991 cset(rscratch1, NE); 1992 1993 Label passed; 1994 cmp(result, zr); 1995 cset(result, NE); // normalize result to 0/1 for comparison 1996 1997 cmp(rscratch1, result); 1998 br(EQ, passed); 1999 { 2000 mov(r0, r_super_klass); // r0 <- r0 2001 mov(r1, r_sub_klass); // r1 <- r4 2002 mov(r2, /*expected*/rscratch1); // r2 <- r8 2003 mov(r3, result); // r3 <- r5 2004 mov(r4, (address)("mismatch")); // r4 <- const 2005 rt_call(CAST_FROM_FN_PTR(address, Klass::on_secondary_supers_verification_failure), rscratch2); 2006 should_not_reach_here(); 2007 } 2008 bind(passed); 2009 2010 BLOCK_COMMENT("} verify_secondary_supers_table"); 2011 } 2012 2013 void MacroAssembler::clinit_barrier(Register klass, Register scratch, Label* L_fast_path, Label* L_slow_path) { 2014 assert(L_fast_path != nullptr || L_slow_path != nullptr, "at least one is required"); 2015 assert_different_registers(klass, rthread, scratch); 2016 2017 Label L_fallthrough, L_tmp; 2018 if (L_fast_path == nullptr) { 2019 L_fast_path = &L_fallthrough; 2020 } else if (L_slow_path == nullptr) { 2021 L_slow_path = &L_fallthrough; 2022 } 2023 // Fast path check: class is fully initialized 2024 lea(scratch, Address(klass, InstanceKlass::init_state_offset())); 2025 ldarb(scratch, scratch); 2026 cmp(scratch, InstanceKlass::fully_initialized); 2027 br(Assembler::EQ, *L_fast_path); 2028 2029 // Fast path check: current thread is initializer thread 2030 ldr(scratch, Address(klass, InstanceKlass::init_thread_offset())); 2031 cmp(rthread, scratch); 2032 2033 if (L_slow_path == &L_fallthrough) { 2034 br(Assembler::EQ, *L_fast_path); 2035 bind(*L_slow_path); 2036 } else if (L_fast_path == &L_fallthrough) { 2037 br(Assembler::NE, *L_slow_path); 2038 bind(*L_fast_path); 2039 } else { 2040 Unimplemented(); 2041 } 2042 } 2043 2044 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) { 2045 if (!VerifyOops || VerifyAdapterSharing) { 2046 // Below address of the code string confuses VerifyAdapterSharing 2047 // because it may differ between otherwise equivalent adapters. 2048 return; 2049 } 2050 2051 // Pass register number to verify_oop_subroutine 2052 const char* b = nullptr; 2053 { 2054 ResourceMark rm; 2055 stringStream ss; 2056 ss.print("verify_oop: %s: %s (%s:%d)", reg->name(), s, file, line); 2057 b = code_string(ss.as_string()); 2058 } 2059 BLOCK_COMMENT("verify_oop {"); 2060 2061 strip_return_address(); // This might happen within a stack frame. 2062 protect_return_address(); 2063 stp(r0, rscratch1, Address(pre(sp, -2 * wordSize))); 2064 stp(rscratch2, lr, Address(pre(sp, -2 * wordSize))); 2065 2066 mov(r0, reg); 2067 movptr(rscratch1, (uintptr_t)(address)b); 2068 2069 // call indirectly to solve generation ordering problem 2070 lea(rscratch2, RuntimeAddress(StubRoutines::verify_oop_subroutine_entry_address())); 2071 ldr(rscratch2, Address(rscratch2)); 2072 blr(rscratch2); 2073 2074 ldp(rscratch2, lr, Address(post(sp, 2 * wordSize))); 2075 ldp(r0, rscratch1, Address(post(sp, 2 * wordSize))); 2076 authenticate_return_address(); 2077 2078 BLOCK_COMMENT("} verify_oop"); 2079 } 2080 2081 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) { 2082 if (!VerifyOops || VerifyAdapterSharing) { 2083 // Below address of the code string confuses VerifyAdapterSharing 2084 // because it may differ between otherwise equivalent adapters. 2085 return; 2086 } 2087 2088 const char* b = nullptr; 2089 { 2090 ResourceMark rm; 2091 stringStream ss; 2092 ss.print("verify_oop_addr: %s (%s:%d)", s, file, line); 2093 b = code_string(ss.as_string()); 2094 } 2095 BLOCK_COMMENT("verify_oop_addr {"); 2096 2097 strip_return_address(); // This might happen within a stack frame. 2098 protect_return_address(); 2099 stp(r0, rscratch1, Address(pre(sp, -2 * wordSize))); 2100 stp(rscratch2, lr, Address(pre(sp, -2 * wordSize))); 2101 2102 // addr may contain sp so we will have to adjust it based on the 2103 // pushes that we just did. 2104 if (addr.uses(sp)) { 2105 lea(r0, addr); 2106 ldr(r0, Address(r0, 4 * wordSize)); 2107 } else { 2108 ldr(r0, addr); 2109 } 2110 movptr(rscratch1, (uintptr_t)(address)b); 2111 2112 // call indirectly to solve generation ordering problem 2113 lea(rscratch2, RuntimeAddress(StubRoutines::verify_oop_subroutine_entry_address())); 2114 ldr(rscratch2, Address(rscratch2)); 2115 blr(rscratch2); 2116 2117 ldp(rscratch2, lr, Address(post(sp, 2 * wordSize))); 2118 ldp(r0, rscratch1, Address(post(sp, 2 * wordSize))); 2119 authenticate_return_address(); 2120 2121 BLOCK_COMMENT("} verify_oop_addr"); 2122 } 2123 2124 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot, 2125 int extra_slot_offset) { 2126 // cf. TemplateTable::prepare_invoke(), if (load_receiver). 2127 int stackElementSize = Interpreter::stackElementSize; 2128 int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0); 2129 #ifdef ASSERT 2130 int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1); 2131 assert(offset1 - offset == stackElementSize, "correct arithmetic"); 2132 #endif 2133 if (arg_slot.is_constant()) { 2134 return Address(esp, arg_slot.as_constant() * stackElementSize 2135 + offset); 2136 } else { 2137 add(rscratch1, esp, arg_slot.as_register(), 2138 ext::uxtx, exact_log2(stackElementSize)); 2139 return Address(rscratch1, offset); 2140 } 2141 } 2142 2143 void MacroAssembler::call_VM_leaf_base(address entry_point, 2144 int number_of_arguments, 2145 Label *retaddr) { 2146 Label E, L; 2147 2148 stp(rscratch1, rmethod, Address(pre(sp, -2 * wordSize))); 2149 2150 mov(rscratch1, RuntimeAddress(entry_point)); 2151 blr(rscratch1); 2152 if (retaddr) 2153 bind(*retaddr); 2154 2155 ldp(rscratch1, rmethod, Address(post(sp, 2 * wordSize))); 2156 } 2157 2158 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) { 2159 call_VM_leaf_base(entry_point, number_of_arguments); 2160 } 2161 2162 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) { 2163 pass_arg0(this, arg_0); 2164 call_VM_leaf_base(entry_point, 1); 2165 } 2166 2167 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) { 2168 assert_different_registers(arg_1, c_rarg0); 2169 pass_arg0(this, arg_0); 2170 pass_arg1(this, arg_1); 2171 call_VM_leaf_base(entry_point, 2); 2172 } 2173 2174 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, 2175 Register arg_1, Register arg_2) { 2176 assert_different_registers(arg_1, c_rarg0); 2177 assert_different_registers(arg_2, c_rarg0, c_rarg1); 2178 pass_arg0(this, arg_0); 2179 pass_arg1(this, arg_1); 2180 pass_arg2(this, arg_2); 2181 call_VM_leaf_base(entry_point, 3); 2182 } 2183 2184 void MacroAssembler::super_call_VM_leaf(address entry_point) { 2185 MacroAssembler::call_VM_leaf_base(entry_point, 1); 2186 } 2187 2188 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) { 2189 pass_arg0(this, arg_0); 2190 MacroAssembler::call_VM_leaf_base(entry_point, 1); 2191 } 2192 2193 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) { 2194 2195 assert_different_registers(arg_0, c_rarg1); 2196 pass_arg1(this, arg_1); 2197 pass_arg0(this, arg_0); 2198 MacroAssembler::call_VM_leaf_base(entry_point, 2); 2199 } 2200 2201 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) { 2202 assert_different_registers(arg_0, c_rarg1, c_rarg2); 2203 assert_different_registers(arg_1, c_rarg2); 2204 pass_arg2(this, arg_2); 2205 pass_arg1(this, arg_1); 2206 pass_arg0(this, arg_0); 2207 MacroAssembler::call_VM_leaf_base(entry_point, 3); 2208 } 2209 2210 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) { 2211 assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3); 2212 assert_different_registers(arg_1, c_rarg2, c_rarg3); 2213 assert_different_registers(arg_2, c_rarg3); 2214 pass_arg3(this, arg_3); 2215 pass_arg2(this, arg_2); 2216 pass_arg1(this, arg_1); 2217 pass_arg0(this, arg_0); 2218 MacroAssembler::call_VM_leaf_base(entry_point, 4); 2219 } 2220 2221 void MacroAssembler::null_check(Register reg, int offset) { 2222 if (needs_explicit_null_check(offset)) { 2223 // provoke OS null exception if reg is null by 2224 // accessing M[reg] w/o changing any registers 2225 // NOTE: this is plenty to provoke a segv 2226 ldr(zr, Address(reg)); 2227 } else { 2228 // nothing to do, (later) access of M[reg + offset] 2229 // will provoke OS null exception if reg is null 2230 } 2231 } 2232 2233 void MacroAssembler::test_markword_is_inline_type(Register markword, Label& is_inline_type) { 2234 assert_different_registers(markword, rscratch2); 2235 mov(rscratch2, markWord::inline_type_mask_in_place); 2236 andr(markword, markword, rscratch2); 2237 mov(rscratch2, markWord::inline_type_pattern); 2238 cmp(markword, rscratch2); 2239 br(Assembler::EQ, is_inline_type); 2240 } 2241 2242 void MacroAssembler::test_oop_is_not_inline_type(Register object, Register tmp, Label& not_inline_type, bool can_be_null) { 2243 assert_different_registers(tmp, rscratch1); 2244 if (can_be_null) { 2245 cbz(object, not_inline_type); 2246 } 2247 const int is_inline_type_mask = markWord::inline_type_pattern; 2248 ldr(tmp, Address(object, oopDesc::mark_offset_in_bytes())); 2249 mov(rscratch1, is_inline_type_mask); 2250 andr(tmp, tmp, rscratch1); 2251 cmp(tmp, rscratch1); 2252 br(Assembler::NE, not_inline_type); 2253 } 2254 2255 void MacroAssembler::test_field_is_null_free_inline_type(Register flags, Register temp_reg, Label& is_null_free_inline_type) { 2256 assert(temp_reg == noreg, "not needed"); // keep signature uniform with x86 2257 tbnz(flags, ResolvedFieldEntry::is_null_free_inline_type_shift, is_null_free_inline_type); 2258 } 2259 2260 void MacroAssembler::test_field_is_not_null_free_inline_type(Register flags, Register temp_reg, Label& not_null_free_inline_type) { 2261 assert(temp_reg == noreg, "not needed"); // keep signature uniform with x86 2262 tbz(flags, ResolvedFieldEntry::is_null_free_inline_type_shift, not_null_free_inline_type); 2263 } 2264 2265 void MacroAssembler::test_field_is_flat(Register flags, Register temp_reg, Label& is_flat) { 2266 assert(temp_reg == noreg, "not needed"); // keep signature uniform with x86 2267 tbnz(flags, ResolvedFieldEntry::is_flat_shift, is_flat); 2268 } 2269 2270 void MacroAssembler::test_field_has_null_marker(Register flags, Register temp_reg, Label& has_null_marker) { 2271 assert(temp_reg == noreg, "not needed"); // keep signature uniform with x86 2272 tbnz(flags, ResolvedFieldEntry::has_null_marker_shift, has_null_marker); 2273 } 2274 2275 void MacroAssembler::test_oop_prototype_bit(Register oop, Register temp_reg, int32_t test_bit, bool jmp_set, Label& jmp_label) { 2276 Label test_mark_word; 2277 // load mark word 2278 ldr(temp_reg, Address(oop, oopDesc::mark_offset_in_bytes())); 2279 // check displaced 2280 tst(temp_reg, markWord::unlocked_value); 2281 br(Assembler::NE, test_mark_word); 2282 // slow path use klass prototype 2283 load_prototype_header(temp_reg, oop); 2284 2285 bind(test_mark_word); 2286 andr(temp_reg, temp_reg, test_bit); 2287 if (jmp_set) { 2288 cbnz(temp_reg, jmp_label); 2289 } else { 2290 cbz(temp_reg, jmp_label); 2291 } 2292 } 2293 2294 void MacroAssembler::test_flat_array_oop(Register oop, Register temp_reg, Label& is_flat_array) { 2295 test_oop_prototype_bit(oop, temp_reg, markWord::flat_array_bit_in_place, true, is_flat_array); 2296 } 2297 2298 void MacroAssembler::test_non_flat_array_oop(Register oop, Register temp_reg, 2299 Label&is_non_flat_array) { 2300 test_oop_prototype_bit(oop, temp_reg, markWord::flat_array_bit_in_place, false, is_non_flat_array); 2301 } 2302 2303 void MacroAssembler::test_null_free_array_oop(Register oop, Register temp_reg, Label& is_null_free_array) { 2304 test_oop_prototype_bit(oop, temp_reg, markWord::null_free_array_bit_in_place, true, is_null_free_array); 2305 } 2306 2307 void MacroAssembler::test_non_null_free_array_oop(Register oop, Register temp_reg, Label&is_non_null_free_array) { 2308 test_oop_prototype_bit(oop, temp_reg, markWord::null_free_array_bit_in_place, false, is_non_null_free_array); 2309 } 2310 2311 void MacroAssembler::test_flat_array_layout(Register lh, Label& is_flat_array) { 2312 tst(lh, Klass::_lh_array_tag_flat_value_bit_inplace); 2313 br(Assembler::NE, is_flat_array); 2314 } 2315 2316 void MacroAssembler::test_non_flat_array_layout(Register lh, Label& is_non_flat_array) { 2317 tst(lh, Klass::_lh_array_tag_flat_value_bit_inplace); 2318 br(Assembler::EQ, is_non_flat_array); 2319 } 2320 2321 // MacroAssembler protected routines needed to implement 2322 // public methods 2323 2324 void MacroAssembler::mov(Register r, Address dest) { 2325 code_section()->relocate(pc(), dest.rspec()); 2326 uint64_t imm64 = (uint64_t)dest.target(); 2327 movptr(r, imm64); 2328 } 2329 2330 // Move a constant pointer into r. In AArch64 mode the virtual 2331 // address space is 48 bits in size, so we only need three 2332 // instructions to create a patchable instruction sequence that can 2333 // reach anywhere. 2334 void MacroAssembler::movptr(Register r, uintptr_t imm64) { 2335 #ifndef PRODUCT 2336 { 2337 char buffer[64]; 2338 os::snprintf_checked(buffer, sizeof(buffer), "0x%" PRIX64, (uint64_t)imm64); 2339 block_comment(buffer); 2340 } 2341 #endif 2342 assert(imm64 < (1ull << 48), "48-bit overflow in address constant"); 2343 movz(r, imm64 & 0xffff); 2344 imm64 >>= 16; 2345 movk(r, imm64 & 0xffff, 16); 2346 imm64 >>= 16; 2347 movk(r, imm64 & 0xffff, 32); 2348 } 2349 2350 // Macro to mov replicated immediate to vector register. 2351 // imm64: only the lower 8/16/32 bits are considered for B/H/S type. That is, 2352 // the upper 56/48/32 bits must be zeros for B/H/S type. 2353 // Vd will get the following values for different arrangements in T 2354 // imm64 == hex 000000gh T8B: Vd = ghghghghghghghgh 2355 // imm64 == hex 000000gh T16B: Vd = ghghghghghghghghghghghghghghghgh 2356 // imm64 == hex 0000efgh T4H: Vd = efghefghefghefgh 2357 // imm64 == hex 0000efgh T8H: Vd = efghefghefghefghefghefghefghefgh 2358 // imm64 == hex abcdefgh T2S: Vd = abcdefghabcdefgh 2359 // imm64 == hex abcdefgh T4S: Vd = abcdefghabcdefghabcdefghabcdefgh 2360 // imm64 == hex abcdefgh T1D: Vd = 00000000abcdefgh 2361 // imm64 == hex abcdefgh T2D: Vd = 00000000abcdefgh00000000abcdefgh 2362 // Clobbers rscratch1 2363 void MacroAssembler::mov(FloatRegister Vd, SIMD_Arrangement T, uint64_t imm64) { 2364 assert(T != T1Q, "unsupported"); 2365 if (T == T1D || T == T2D) { 2366 int imm = operand_valid_for_movi_immediate(imm64, T); 2367 if (-1 != imm) { 2368 movi(Vd, T, imm); 2369 } else { 2370 mov(rscratch1, imm64); 2371 dup(Vd, T, rscratch1); 2372 } 2373 return; 2374 } 2375 2376 #ifdef ASSERT 2377 if (T == T8B || T == T16B) assert((imm64 & ~0xff) == 0, "extraneous bits (T8B/T16B)"); 2378 if (T == T4H || T == T8H) assert((imm64 & ~0xffff) == 0, "extraneous bits (T4H/T8H)"); 2379 if (T == T2S || T == T4S) assert((imm64 & ~0xffffffff) == 0, "extraneous bits (T2S/T4S)"); 2380 #endif 2381 int shift = operand_valid_for_movi_immediate(imm64, T); 2382 uint32_t imm32 = imm64 & 0xffffffffULL; 2383 if (shift >= 0) { 2384 movi(Vd, T, (imm32 >> shift) & 0xff, shift); 2385 } else { 2386 movw(rscratch1, imm32); 2387 dup(Vd, T, rscratch1); 2388 } 2389 } 2390 2391 void MacroAssembler::mov_immediate64(Register dst, uint64_t imm64) 2392 { 2393 #ifndef PRODUCT 2394 { 2395 char buffer[64]; 2396 os::snprintf_checked(buffer, sizeof(buffer), "0x%" PRIX64, imm64); 2397 block_comment(buffer); 2398 } 2399 #endif 2400 if (operand_valid_for_logical_immediate(false, imm64)) { 2401 orr(dst, zr, imm64); 2402 } else { 2403 // we can use a combination of MOVZ or MOVN with 2404 // MOVK to build up the constant 2405 uint64_t imm_h[4]; 2406 int zero_count = 0; 2407 int neg_count = 0; 2408 int i; 2409 for (i = 0; i < 4; i++) { 2410 imm_h[i] = ((imm64 >> (i * 16)) & 0xffffL); 2411 if (imm_h[i] == 0) { 2412 zero_count++; 2413 } else if (imm_h[i] == 0xffffL) { 2414 neg_count++; 2415 } 2416 } 2417 if (zero_count == 4) { 2418 // one MOVZ will do 2419 movz(dst, 0); 2420 } else if (neg_count == 4) { 2421 // one MOVN will do 2422 movn(dst, 0); 2423 } else if (zero_count == 3) { 2424 for (i = 0; i < 4; i++) { 2425 if (imm_h[i] != 0L) { 2426 movz(dst, (uint32_t)imm_h[i], (i << 4)); 2427 break; 2428 } 2429 } 2430 } else if (neg_count == 3) { 2431 // one MOVN will do 2432 for (int i = 0; i < 4; i++) { 2433 if (imm_h[i] != 0xffffL) { 2434 movn(dst, (uint32_t)imm_h[i] ^ 0xffffL, (i << 4)); 2435 break; 2436 } 2437 } 2438 } else if (zero_count == 2) { 2439 // one MOVZ and one MOVK will do 2440 for (i = 0; i < 3; i++) { 2441 if (imm_h[i] != 0L) { 2442 movz(dst, (uint32_t)imm_h[i], (i << 4)); 2443 i++; 2444 break; 2445 } 2446 } 2447 for (;i < 4; i++) { 2448 if (imm_h[i] != 0L) { 2449 movk(dst, (uint32_t)imm_h[i], (i << 4)); 2450 } 2451 } 2452 } else if (neg_count == 2) { 2453 // one MOVN and one MOVK will do 2454 for (i = 0; i < 4; i++) { 2455 if (imm_h[i] != 0xffffL) { 2456 movn(dst, (uint32_t)imm_h[i] ^ 0xffffL, (i << 4)); 2457 i++; 2458 break; 2459 } 2460 } 2461 for (;i < 4; i++) { 2462 if (imm_h[i] != 0xffffL) { 2463 movk(dst, (uint32_t)imm_h[i], (i << 4)); 2464 } 2465 } 2466 } else if (zero_count == 1) { 2467 // one MOVZ and two MOVKs will do 2468 for (i = 0; i < 4; i++) { 2469 if (imm_h[i] != 0L) { 2470 movz(dst, (uint32_t)imm_h[i], (i << 4)); 2471 i++; 2472 break; 2473 } 2474 } 2475 for (;i < 4; i++) { 2476 if (imm_h[i] != 0x0L) { 2477 movk(dst, (uint32_t)imm_h[i], (i << 4)); 2478 } 2479 } 2480 } else if (neg_count == 1) { 2481 // one MOVN and two MOVKs will do 2482 for (i = 0; i < 4; i++) { 2483 if (imm_h[i] != 0xffffL) { 2484 movn(dst, (uint32_t)imm_h[i] ^ 0xffffL, (i << 4)); 2485 i++; 2486 break; 2487 } 2488 } 2489 for (;i < 4; i++) { 2490 if (imm_h[i] != 0xffffL) { 2491 movk(dst, (uint32_t)imm_h[i], (i << 4)); 2492 } 2493 } 2494 } else { 2495 // use a MOVZ and 3 MOVKs (makes it easier to debug) 2496 movz(dst, (uint32_t)imm_h[0], 0); 2497 for (i = 1; i < 4; i++) { 2498 movk(dst, (uint32_t)imm_h[i], (i << 4)); 2499 } 2500 } 2501 } 2502 } 2503 2504 void MacroAssembler::mov_immediate32(Register dst, uint32_t imm32) 2505 { 2506 #ifndef PRODUCT 2507 { 2508 char buffer[64]; 2509 os::snprintf_checked(buffer, sizeof(buffer), "0x%" PRIX32, imm32); 2510 block_comment(buffer); 2511 } 2512 #endif 2513 if (operand_valid_for_logical_immediate(true, imm32)) { 2514 orrw(dst, zr, imm32); 2515 } else { 2516 // we can use MOVZ, MOVN or two calls to MOVK to build up the 2517 // constant 2518 uint32_t imm_h[2]; 2519 imm_h[0] = imm32 & 0xffff; 2520 imm_h[1] = ((imm32 >> 16) & 0xffff); 2521 if (imm_h[0] == 0) { 2522 movzw(dst, imm_h[1], 16); 2523 } else if (imm_h[0] == 0xffff) { 2524 movnw(dst, imm_h[1] ^ 0xffff, 16); 2525 } else if (imm_h[1] == 0) { 2526 movzw(dst, imm_h[0], 0); 2527 } else if (imm_h[1] == 0xffff) { 2528 movnw(dst, imm_h[0] ^ 0xffff, 0); 2529 } else { 2530 // use a MOVZ and MOVK (makes it easier to debug) 2531 movzw(dst, imm_h[0], 0); 2532 movkw(dst, imm_h[1], 16); 2533 } 2534 } 2535 } 2536 2537 // Form an address from base + offset in Rd. Rd may or may 2538 // not actually be used: you must use the Address that is returned. 2539 // It is up to you to ensure that the shift provided matches the size 2540 // of your data. 2541 Address MacroAssembler::form_address(Register Rd, Register base, int64_t byte_offset, int shift) { 2542 if (Address::offset_ok_for_immed(byte_offset, shift)) 2543 // It fits; no need for any heroics 2544 return Address(base, byte_offset); 2545 2546 // Don't do anything clever with negative or misaligned offsets 2547 unsigned mask = (1 << shift) - 1; 2548 if (byte_offset < 0 || byte_offset & mask) { 2549 mov(Rd, byte_offset); 2550 add(Rd, base, Rd); 2551 return Address(Rd); 2552 } 2553 2554 // See if we can do this with two 12-bit offsets 2555 { 2556 uint64_t word_offset = byte_offset >> shift; 2557 uint64_t masked_offset = word_offset & 0xfff000; 2558 if (Address::offset_ok_for_immed(word_offset - masked_offset, 0) 2559 && Assembler::operand_valid_for_add_sub_immediate(masked_offset << shift)) { 2560 add(Rd, base, masked_offset << shift); 2561 word_offset -= masked_offset; 2562 return Address(Rd, word_offset << shift); 2563 } 2564 } 2565 2566 // Do it the hard way 2567 mov(Rd, byte_offset); 2568 add(Rd, base, Rd); 2569 return Address(Rd); 2570 } 2571 2572 int MacroAssembler::corrected_idivl(Register result, Register ra, Register rb, 2573 bool want_remainder, Register scratch) 2574 { 2575 // Full implementation of Java idiv and irem. The function 2576 // returns the (pc) offset of the div instruction - may be needed 2577 // for implicit exceptions. 2578 // 2579 // constraint : ra/rb =/= scratch 2580 // normal case 2581 // 2582 // input : ra: dividend 2583 // rb: divisor 2584 // 2585 // result: either 2586 // quotient (= ra idiv rb) 2587 // remainder (= ra irem rb) 2588 2589 assert(ra != scratch && rb != scratch, "reg cannot be scratch"); 2590 2591 int idivl_offset = offset(); 2592 if (! want_remainder) { 2593 sdivw(result, ra, rb); 2594 } else { 2595 sdivw(scratch, ra, rb); 2596 Assembler::msubw(result, scratch, rb, ra); 2597 } 2598 2599 return idivl_offset; 2600 } 2601 2602 int MacroAssembler::corrected_idivq(Register result, Register ra, Register rb, 2603 bool want_remainder, Register scratch) 2604 { 2605 // Full implementation of Java ldiv and lrem. The function 2606 // returns the (pc) offset of the div instruction - may be needed 2607 // for implicit exceptions. 2608 // 2609 // constraint : ra/rb =/= scratch 2610 // normal case 2611 // 2612 // input : ra: dividend 2613 // rb: divisor 2614 // 2615 // result: either 2616 // quotient (= ra idiv rb) 2617 // remainder (= ra irem rb) 2618 2619 assert(ra != scratch && rb != scratch, "reg cannot be scratch"); 2620 2621 int idivq_offset = offset(); 2622 if (! want_remainder) { 2623 sdiv(result, ra, rb); 2624 } else { 2625 sdiv(scratch, ra, rb); 2626 Assembler::msub(result, scratch, rb, ra); 2627 } 2628 2629 return idivq_offset; 2630 } 2631 2632 void MacroAssembler::membar(Membar_mask_bits order_constraint) { 2633 address prev = pc() - NativeMembar::instruction_size; 2634 address last = code()->last_insn(); 2635 if (last != nullptr && nativeInstruction_at(last)->is_Membar() && prev == last) { 2636 NativeMembar *bar = NativeMembar_at(prev); 2637 if (AlwaysMergeDMB) { 2638 bar->set_kind(bar->get_kind() | order_constraint); 2639 BLOCK_COMMENT("merged membar(always)"); 2640 return; 2641 } 2642 // Don't promote DMB ST|DMB LD to DMB (a full barrier) because 2643 // doing so would introduce a StoreLoad which the caller did not 2644 // intend 2645 if (bar->get_kind() == order_constraint 2646 || bar->get_kind() == AnyAny 2647 || order_constraint == AnyAny) { 2648 // We are merging two memory barrier instructions. On AArch64 we 2649 // can do this simply by ORing them together. 2650 bar->set_kind(bar->get_kind() | order_constraint); 2651 BLOCK_COMMENT("merged membar"); 2652 return; 2653 } else { 2654 // A special case like "DMB ST;DMB LD;DMB ST", the last DMB can be skipped 2655 // We need check the last 2 instructions 2656 address prev2 = prev - NativeMembar::instruction_size; 2657 if (last != code()->last_label() && nativeInstruction_at(prev2)->is_Membar()) { 2658 NativeMembar *bar2 = NativeMembar_at(prev2); 2659 assert(bar2->get_kind() == order_constraint, "it should be merged before"); 2660 BLOCK_COMMENT("merged membar(elided)"); 2661 return; 2662 } 2663 } 2664 } 2665 code()->set_last_insn(pc()); 2666 dmb(Assembler::barrier(order_constraint)); 2667 } 2668 2669 bool MacroAssembler::try_merge_ldst(Register rt, const Address &adr, size_t size_in_bytes, bool is_store) { 2670 if (ldst_can_merge(rt, adr, size_in_bytes, is_store)) { 2671 merge_ldst(rt, adr, size_in_bytes, is_store); 2672 code()->clear_last_insn(); 2673 return true; 2674 } else { 2675 assert(size_in_bytes == 8 || size_in_bytes == 4, "only 8 bytes or 4 bytes load/store is supported."); 2676 const uint64_t mask = size_in_bytes - 1; 2677 if (adr.getMode() == Address::base_plus_offset && 2678 (adr.offset() & mask) == 0) { // only supports base_plus_offset. 2679 code()->set_last_insn(pc()); 2680 } 2681 return false; 2682 } 2683 } 2684 2685 void MacroAssembler::ldr(Register Rx, const Address &adr) { 2686 // We always try to merge two adjacent loads into one ldp. 2687 if (!try_merge_ldst(Rx, adr, 8, false)) { 2688 Assembler::ldr(Rx, adr); 2689 } 2690 } 2691 2692 void MacroAssembler::ldrw(Register Rw, const Address &adr) { 2693 // We always try to merge two adjacent loads into one ldp. 2694 if (!try_merge_ldst(Rw, adr, 4, false)) { 2695 Assembler::ldrw(Rw, adr); 2696 } 2697 } 2698 2699 void MacroAssembler::str(Register Rx, const Address &adr) { 2700 // We always try to merge two adjacent stores into one stp. 2701 if (!try_merge_ldst(Rx, adr, 8, true)) { 2702 Assembler::str(Rx, adr); 2703 } 2704 } 2705 2706 void MacroAssembler::strw(Register Rw, const Address &adr) { 2707 // We always try to merge two adjacent stores into one stp. 2708 if (!try_merge_ldst(Rw, adr, 4, true)) { 2709 Assembler::strw(Rw, adr); 2710 } 2711 } 2712 2713 // MacroAssembler routines found actually to be needed 2714 2715 void MacroAssembler::push(Register src) 2716 { 2717 str(src, Address(pre(esp, -1 * wordSize))); 2718 } 2719 2720 void MacroAssembler::pop(Register dst) 2721 { 2722 ldr(dst, Address(post(esp, 1 * wordSize))); 2723 } 2724 2725 // Note: load_unsigned_short used to be called load_unsigned_word. 2726 int MacroAssembler::load_unsigned_short(Register dst, Address src) { 2727 int off = offset(); 2728 ldrh(dst, src); 2729 return off; 2730 } 2731 2732 int MacroAssembler::load_unsigned_byte(Register dst, Address src) { 2733 int off = offset(); 2734 ldrb(dst, src); 2735 return off; 2736 } 2737 2738 int MacroAssembler::load_signed_short(Register dst, Address src) { 2739 int off = offset(); 2740 ldrsh(dst, src); 2741 return off; 2742 } 2743 2744 int MacroAssembler::load_signed_byte(Register dst, Address src) { 2745 int off = offset(); 2746 ldrsb(dst, src); 2747 return off; 2748 } 2749 2750 int MacroAssembler::load_signed_short32(Register dst, Address src) { 2751 int off = offset(); 2752 ldrshw(dst, src); 2753 return off; 2754 } 2755 2756 int MacroAssembler::load_signed_byte32(Register dst, Address src) { 2757 int off = offset(); 2758 ldrsbw(dst, src); 2759 return off; 2760 } 2761 2762 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed) { 2763 switch (size_in_bytes) { 2764 case 8: ldr(dst, src); break; 2765 case 4: ldrw(dst, src); break; 2766 case 2: is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break; 2767 case 1: is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break; 2768 default: ShouldNotReachHere(); 2769 } 2770 } 2771 2772 void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes) { 2773 switch (size_in_bytes) { 2774 case 8: str(src, dst); break; 2775 case 4: strw(src, dst); break; 2776 case 2: strh(src, dst); break; 2777 case 1: strb(src, dst); break; 2778 default: ShouldNotReachHere(); 2779 } 2780 } 2781 2782 void MacroAssembler::decrementw(Register reg, int value) 2783 { 2784 if (value < 0) { incrementw(reg, -value); return; } 2785 if (value == 0) { return; } 2786 if (value < (1 << 12)) { subw(reg, reg, value); return; } 2787 /* else */ { 2788 guarantee(reg != rscratch2, "invalid dst for register decrement"); 2789 movw(rscratch2, (unsigned)value); 2790 subw(reg, reg, rscratch2); 2791 } 2792 } 2793 2794 void MacroAssembler::decrement(Register reg, int value) 2795 { 2796 if (value < 0) { increment(reg, -value); return; } 2797 if (value == 0) { return; } 2798 if (value < (1 << 12)) { sub(reg, reg, value); return; } 2799 /* else */ { 2800 assert(reg != rscratch2, "invalid dst for register decrement"); 2801 mov(rscratch2, (uint64_t)value); 2802 sub(reg, reg, rscratch2); 2803 } 2804 } 2805 2806 void MacroAssembler::decrementw(Address dst, int value) 2807 { 2808 assert(!dst.uses(rscratch1), "invalid dst for address decrement"); 2809 if (dst.getMode() == Address::literal) { 2810 assert(abs(value) < (1 << 12), "invalid value and address mode combination"); 2811 lea(rscratch2, dst); 2812 dst = Address(rscratch2); 2813 } 2814 ldrw(rscratch1, dst); 2815 decrementw(rscratch1, value); 2816 strw(rscratch1, dst); 2817 } 2818 2819 void MacroAssembler::decrement(Address dst, int value) 2820 { 2821 assert(!dst.uses(rscratch1), "invalid address for decrement"); 2822 if (dst.getMode() == Address::literal) { 2823 assert(abs(value) < (1 << 12), "invalid value and address mode combination"); 2824 lea(rscratch2, dst); 2825 dst = Address(rscratch2); 2826 } 2827 ldr(rscratch1, dst); 2828 decrement(rscratch1, value); 2829 str(rscratch1, dst); 2830 } 2831 2832 void MacroAssembler::incrementw(Register reg, int value) 2833 { 2834 if (value < 0) { decrementw(reg, -value); return; } 2835 if (value == 0) { return; } 2836 if (value < (1 << 12)) { addw(reg, reg, value); return; } 2837 /* else */ { 2838 assert(reg != rscratch2, "invalid dst for register increment"); 2839 movw(rscratch2, (unsigned)value); 2840 addw(reg, reg, rscratch2); 2841 } 2842 } 2843 2844 void MacroAssembler::increment(Register reg, int value) 2845 { 2846 if (value < 0) { decrement(reg, -value); return; } 2847 if (value == 0) { return; } 2848 if (value < (1 << 12)) { add(reg, reg, value); return; } 2849 /* else */ { 2850 assert(reg != rscratch2, "invalid dst for register increment"); 2851 movw(rscratch2, (unsigned)value); 2852 add(reg, reg, rscratch2); 2853 } 2854 } 2855 2856 void MacroAssembler::incrementw(Address dst, int value) 2857 { 2858 assert(!dst.uses(rscratch1), "invalid dst for address increment"); 2859 if (dst.getMode() == Address::literal) { 2860 assert(abs(value) < (1 << 12), "invalid value and address mode combination"); 2861 lea(rscratch2, dst); 2862 dst = Address(rscratch2); 2863 } 2864 ldrw(rscratch1, dst); 2865 incrementw(rscratch1, value); 2866 strw(rscratch1, dst); 2867 } 2868 2869 void MacroAssembler::increment(Address dst, int value) 2870 { 2871 assert(!dst.uses(rscratch1), "invalid dst for address increment"); 2872 if (dst.getMode() == Address::literal) { 2873 assert(abs(value) < (1 << 12), "invalid value and address mode combination"); 2874 lea(rscratch2, dst); 2875 dst = Address(rscratch2); 2876 } 2877 ldr(rscratch1, dst); 2878 increment(rscratch1, value); 2879 str(rscratch1, dst); 2880 } 2881 2882 // Push lots of registers in the bit set supplied. Don't push sp. 2883 // Return the number of words pushed 2884 int MacroAssembler::push(unsigned int bitset, Register stack) { 2885 int words_pushed = 0; 2886 2887 // Scan bitset to accumulate register pairs 2888 unsigned char regs[32]; 2889 int count = 0; 2890 for (int reg = 0; reg <= 30; reg++) { 2891 if (1 & bitset) 2892 regs[count++] = reg; 2893 bitset >>= 1; 2894 } 2895 regs[count++] = zr->raw_encoding(); 2896 count &= ~1; // Only push an even number of regs 2897 2898 if (count) { 2899 stp(as_Register(regs[0]), as_Register(regs[1]), 2900 Address(pre(stack, -count * wordSize))); 2901 words_pushed += 2; 2902 } 2903 for (int i = 2; i < count; i += 2) { 2904 stp(as_Register(regs[i]), as_Register(regs[i+1]), 2905 Address(stack, i * wordSize)); 2906 words_pushed += 2; 2907 } 2908 2909 assert(words_pushed == count, "oops, pushed != count"); 2910 2911 return count; 2912 } 2913 2914 int MacroAssembler::pop(unsigned int bitset, Register stack) { 2915 int words_pushed = 0; 2916 2917 // Scan bitset to accumulate register pairs 2918 unsigned char regs[32]; 2919 int count = 0; 2920 for (int reg = 0; reg <= 30; reg++) { 2921 if (1 & bitset) 2922 regs[count++] = reg; 2923 bitset >>= 1; 2924 } 2925 regs[count++] = zr->raw_encoding(); 2926 count &= ~1; 2927 2928 for (int i = 2; i < count; i += 2) { 2929 ldp(as_Register(regs[i]), as_Register(regs[i+1]), 2930 Address(stack, i * wordSize)); 2931 words_pushed += 2; 2932 } 2933 if (count) { 2934 ldp(as_Register(regs[0]), as_Register(regs[1]), 2935 Address(post(stack, count * wordSize))); 2936 words_pushed += 2; 2937 } 2938 2939 assert(words_pushed == count, "oops, pushed != count"); 2940 2941 return count; 2942 } 2943 2944 // Push lots of registers in the bit set supplied. Don't push sp. 2945 // Return the number of dwords pushed 2946 int MacroAssembler::push_fp(unsigned int bitset, Register stack, FpPushPopMode mode) { 2947 int words_pushed = 0; 2948 bool use_sve = false; 2949 int sve_vector_size_in_bytes = 0; 2950 2951 #ifdef COMPILER2 2952 use_sve = Matcher::supports_scalable_vector(); 2953 sve_vector_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE); 2954 #endif 2955 2956 // Scan bitset to accumulate register pairs 2957 unsigned char regs[32]; 2958 int count = 0; 2959 for (int reg = 0; reg <= 31; reg++) { 2960 if (1 & bitset) 2961 regs[count++] = reg; 2962 bitset >>= 1; 2963 } 2964 2965 if (count == 0) { 2966 return 0; 2967 } 2968 2969 if (mode == PushPopFull) { 2970 if (use_sve && sve_vector_size_in_bytes > 16) { 2971 mode = PushPopSVE; 2972 } else { 2973 mode = PushPopNeon; 2974 } 2975 } 2976 2977 #ifndef PRODUCT 2978 { 2979 char buffer[48]; 2980 if (mode == PushPopSVE) { 2981 os::snprintf_checked(buffer, sizeof(buffer), "push_fp: %d SVE registers", count); 2982 } else if (mode == PushPopNeon) { 2983 os::snprintf_checked(buffer, sizeof(buffer), "push_fp: %d Neon registers", count); 2984 } else { 2985 os::snprintf_checked(buffer, sizeof(buffer), "push_fp: %d fp registers", count); 2986 } 2987 block_comment(buffer); 2988 } 2989 #endif 2990 2991 if (mode == PushPopSVE) { 2992 sub(stack, stack, sve_vector_size_in_bytes * count); 2993 for (int i = 0; i < count; i++) { 2994 sve_str(as_FloatRegister(regs[i]), Address(stack, i)); 2995 } 2996 return count * sve_vector_size_in_bytes / 8; 2997 } 2998 2999 if (mode == PushPopNeon) { 3000 if (count == 1) { 3001 strq(as_FloatRegister(regs[0]), Address(pre(stack, -wordSize * 2))); 3002 return 2; 3003 } 3004 3005 bool odd = (count & 1) == 1; 3006 int push_slots = count + (odd ? 1 : 0); 3007 3008 // Always pushing full 128 bit registers. 3009 stpq(as_FloatRegister(regs[0]), as_FloatRegister(regs[1]), Address(pre(stack, -push_slots * wordSize * 2))); 3010 words_pushed += 2; 3011 3012 for (int i = 2; i + 1 < count; i += 2) { 3013 stpq(as_FloatRegister(regs[i]), as_FloatRegister(regs[i+1]), Address(stack, i * wordSize * 2)); 3014 words_pushed += 2; 3015 } 3016 3017 if (odd) { 3018 strq(as_FloatRegister(regs[count - 1]), Address(stack, (count - 1) * wordSize * 2)); 3019 words_pushed++; 3020 } 3021 3022 assert(words_pushed == count, "oops, pushed(%d) != count(%d)", words_pushed, count); 3023 return count * 2; 3024 } 3025 3026 if (mode == PushPopFp) { 3027 bool odd = (count & 1) == 1; 3028 int push_slots = count + (odd ? 1 : 0); 3029 3030 if (count == 1) { 3031 // Stack pointer must be 16 bytes aligned 3032 strd(as_FloatRegister(regs[0]), Address(pre(stack, -push_slots * wordSize))); 3033 return 1; 3034 } 3035 3036 stpd(as_FloatRegister(regs[0]), as_FloatRegister(regs[1]), Address(pre(stack, -push_slots * wordSize))); 3037 words_pushed += 2; 3038 3039 for (int i = 2; i + 1 < count; i += 2) { 3040 stpd(as_FloatRegister(regs[i]), as_FloatRegister(regs[i+1]), Address(stack, i * wordSize)); 3041 words_pushed += 2; 3042 } 3043 3044 if (odd) { 3045 // Stack pointer must be 16 bytes aligned 3046 strd(as_FloatRegister(regs[count - 1]), Address(stack, (count - 1) * wordSize)); 3047 words_pushed++; 3048 } 3049 3050 assert(words_pushed == count, "oops, pushed != count"); 3051 3052 return count; 3053 } 3054 3055 return 0; 3056 } 3057 3058 // Return the number of dwords popped 3059 int MacroAssembler::pop_fp(unsigned int bitset, Register stack, FpPushPopMode mode) { 3060 int words_pushed = 0; 3061 bool use_sve = false; 3062 int sve_vector_size_in_bytes = 0; 3063 3064 #ifdef COMPILER2 3065 use_sve = Matcher::supports_scalable_vector(); 3066 sve_vector_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE); 3067 #endif 3068 // Scan bitset to accumulate register pairs 3069 unsigned char regs[32]; 3070 int count = 0; 3071 for (int reg = 0; reg <= 31; reg++) { 3072 if (1 & bitset) 3073 regs[count++] = reg; 3074 bitset >>= 1; 3075 } 3076 3077 if (count == 0) { 3078 return 0; 3079 } 3080 3081 if (mode == PushPopFull) { 3082 if (use_sve && sve_vector_size_in_bytes > 16) { 3083 mode = PushPopSVE; 3084 } else { 3085 mode = PushPopNeon; 3086 } 3087 } 3088 3089 #ifndef PRODUCT 3090 { 3091 char buffer[48]; 3092 if (mode == PushPopSVE) { 3093 os::snprintf_checked(buffer, sizeof(buffer), "pop_fp: %d SVE registers", count); 3094 } else if (mode == PushPopNeon) { 3095 os::snprintf_checked(buffer, sizeof(buffer), "pop_fp: %d Neon registers", count); 3096 } else { 3097 os::snprintf_checked(buffer, sizeof(buffer), "pop_fp: %d fp registers", count); 3098 } 3099 block_comment(buffer); 3100 } 3101 #endif 3102 3103 if (mode == PushPopSVE) { 3104 for (int i = count - 1; i >= 0; i--) { 3105 sve_ldr(as_FloatRegister(regs[i]), Address(stack, i)); 3106 } 3107 add(stack, stack, sve_vector_size_in_bytes * count); 3108 return count * sve_vector_size_in_bytes / 8; 3109 } 3110 3111 if (mode == PushPopNeon) { 3112 if (count == 1) { 3113 ldrq(as_FloatRegister(regs[0]), Address(post(stack, wordSize * 2))); 3114 return 2; 3115 } 3116 3117 bool odd = (count & 1) == 1; 3118 int push_slots = count + (odd ? 1 : 0); 3119 3120 if (odd) { 3121 ldrq(as_FloatRegister(regs[count - 1]), Address(stack, (count - 1) * wordSize * 2)); 3122 words_pushed++; 3123 } 3124 3125 for (int i = 2; i + 1 < count; i += 2) { 3126 ldpq(as_FloatRegister(regs[i]), as_FloatRegister(regs[i+1]), Address(stack, i * wordSize * 2)); 3127 words_pushed += 2; 3128 } 3129 3130 ldpq(as_FloatRegister(regs[0]), as_FloatRegister(regs[1]), Address(post(stack, push_slots * wordSize * 2))); 3131 words_pushed += 2; 3132 3133 assert(words_pushed == count, "oops, pushed(%d) != count(%d)", words_pushed, count); 3134 3135 return count * 2; 3136 } 3137 3138 if (mode == PushPopFp) { 3139 bool odd = (count & 1) == 1; 3140 int push_slots = count + (odd ? 1 : 0); 3141 3142 if (count == 1) { 3143 ldrd(as_FloatRegister(regs[0]), Address(post(stack, push_slots * wordSize))); 3144 return 1; 3145 } 3146 3147 if (odd) { 3148 ldrd(as_FloatRegister(regs[count - 1]), Address(stack, (count - 1) * wordSize)); 3149 words_pushed++; 3150 } 3151 3152 for (int i = 2; i + 1 < count; i += 2) { 3153 ldpd(as_FloatRegister(regs[i]), as_FloatRegister(regs[i+1]), Address(stack, i * wordSize)); 3154 words_pushed += 2; 3155 } 3156 3157 ldpd(as_FloatRegister(regs[0]), as_FloatRegister(regs[1]), Address(post(stack, push_slots * wordSize))); 3158 words_pushed += 2; 3159 3160 assert(words_pushed == count, "oops, pushed != count"); 3161 3162 return count; 3163 } 3164 3165 return 0; 3166 } 3167 3168 // Return the number of dwords pushed 3169 int MacroAssembler::push_p(unsigned int bitset, Register stack) { 3170 bool use_sve = false; 3171 int sve_predicate_size_in_slots = 0; 3172 3173 #ifdef COMPILER2 3174 use_sve = Matcher::supports_scalable_vector(); 3175 if (use_sve) { 3176 sve_predicate_size_in_slots = Matcher::scalable_predicate_reg_slots(); 3177 } 3178 #endif 3179 3180 if (!use_sve) { 3181 return 0; 3182 } 3183 3184 unsigned char regs[PRegister::number_of_registers]; 3185 int count = 0; 3186 for (int reg = 0; reg < PRegister::number_of_registers; reg++) { 3187 if (1 & bitset) 3188 regs[count++] = reg; 3189 bitset >>= 1; 3190 } 3191 3192 if (count == 0) { 3193 return 0; 3194 } 3195 3196 int total_push_bytes = align_up(sve_predicate_size_in_slots * 3197 VMRegImpl::stack_slot_size * count, 16); 3198 sub(stack, stack, total_push_bytes); 3199 for (int i = 0; i < count; i++) { 3200 sve_str(as_PRegister(regs[i]), Address(stack, i)); 3201 } 3202 return total_push_bytes / 8; 3203 } 3204 3205 // Return the number of dwords popped 3206 int MacroAssembler::pop_p(unsigned int bitset, Register stack) { 3207 bool use_sve = false; 3208 int sve_predicate_size_in_slots = 0; 3209 3210 #ifdef COMPILER2 3211 use_sve = Matcher::supports_scalable_vector(); 3212 if (use_sve) { 3213 sve_predicate_size_in_slots = Matcher::scalable_predicate_reg_slots(); 3214 } 3215 #endif 3216 3217 if (!use_sve) { 3218 return 0; 3219 } 3220 3221 unsigned char regs[PRegister::number_of_registers]; 3222 int count = 0; 3223 for (int reg = 0; reg < PRegister::number_of_registers; reg++) { 3224 if (1 & bitset) 3225 regs[count++] = reg; 3226 bitset >>= 1; 3227 } 3228 3229 if (count == 0) { 3230 return 0; 3231 } 3232 3233 int total_pop_bytes = align_up(sve_predicate_size_in_slots * 3234 VMRegImpl::stack_slot_size * count, 16); 3235 for (int i = count - 1; i >= 0; i--) { 3236 sve_ldr(as_PRegister(regs[i]), Address(stack, i)); 3237 } 3238 add(stack, stack, total_pop_bytes); 3239 return total_pop_bytes / 8; 3240 } 3241 3242 #ifdef ASSERT 3243 void MacroAssembler::verify_heapbase(const char* msg) { 3244 #if 0 3245 assert (UseCompressedOops || UseCompressedClassPointers, "should be compressed"); 3246 assert (Universe::heap() != nullptr, "java heap should be initialized"); 3247 if (!UseCompressedOops || Universe::ptr_base() == nullptr) { 3248 // rheapbase is allocated as general register 3249 return; 3250 } 3251 if (CheckCompressedOops) { 3252 Label ok; 3253 push(1 << rscratch1->encoding(), sp); // cmpptr trashes rscratch1 3254 cmpptr(rheapbase, ExternalAddress(CompressedOops::base_addr())); 3255 br(Assembler::EQ, ok); 3256 stop(msg); 3257 bind(ok); 3258 pop(1 << rscratch1->encoding(), sp); 3259 } 3260 #endif 3261 } 3262 #endif 3263 3264 void MacroAssembler::resolve_jobject(Register value, Register tmp1, Register tmp2) { 3265 assert_different_registers(value, tmp1, tmp2); 3266 Label done, tagged, weak_tagged; 3267 3268 cbz(value, done); // Use null as-is. 3269 tst(value, JNIHandles::tag_mask); // Test for tag. 3270 br(Assembler::NE, tagged); 3271 3272 // Resolve local handle 3273 access_load_at(T_OBJECT, IN_NATIVE | AS_RAW, value, Address(value, 0), tmp1, tmp2); 3274 verify_oop(value); 3275 b(done); 3276 3277 bind(tagged); 3278 STATIC_ASSERT(JNIHandles::TypeTag::weak_global == 0b1); 3279 tbnz(value, 0, weak_tagged); // Test for weak tag. 3280 3281 // Resolve global handle 3282 access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp1, tmp2); 3283 verify_oop(value); 3284 b(done); 3285 3286 bind(weak_tagged); 3287 // Resolve jweak. 3288 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, 3289 value, Address(value, -JNIHandles::TypeTag::weak_global), tmp1, tmp2); 3290 verify_oop(value); 3291 3292 bind(done); 3293 } 3294 3295 void MacroAssembler::resolve_global_jobject(Register value, Register tmp1, Register tmp2) { 3296 assert_different_registers(value, tmp1, tmp2); 3297 Label done; 3298 3299 cbz(value, done); // Use null as-is. 3300 3301 #ifdef ASSERT 3302 { 3303 STATIC_ASSERT(JNIHandles::TypeTag::global == 0b10); 3304 Label valid_global_tag; 3305 tbnz(value, 1, valid_global_tag); // Test for global tag 3306 stop("non global jobject using resolve_global_jobject"); 3307 bind(valid_global_tag); 3308 } 3309 #endif 3310 3311 // Resolve global handle 3312 access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp1, tmp2); 3313 verify_oop(value); 3314 3315 bind(done); 3316 } 3317 3318 void MacroAssembler::stop(const char* msg) { 3319 // Skip AOT caching C strings in scratch buffer. 3320 const char* str = (code_section()->scratch_emit()) ? msg : AOTCodeCache::add_C_string(msg); 3321 BLOCK_COMMENT(str); 3322 // load msg into r0 so we can access it from the signal handler 3323 // ExternalAddress enables saving and restoring via the code cache 3324 lea(c_rarg0, ExternalAddress((address) str)); 3325 dcps1(0xdeae); 3326 } 3327 3328 void MacroAssembler::unimplemented(const char* what) { 3329 const char* buf = nullptr; 3330 { 3331 ResourceMark rm; 3332 stringStream ss; 3333 ss.print("unimplemented: %s", what); 3334 buf = code_string(ss.as_string()); 3335 } 3336 stop(buf); 3337 } 3338 3339 void MacroAssembler::_assert_asm(Assembler::Condition cc, const char* msg) { 3340 #ifdef ASSERT 3341 Label OK; 3342 br(cc, OK); 3343 stop(msg); 3344 bind(OK); 3345 #endif 3346 } 3347 3348 // If a constant does not fit in an immediate field, generate some 3349 // number of MOV instructions and then perform the operation. 3350 void MacroAssembler::wrap_add_sub_imm_insn(Register Rd, Register Rn, uint64_t imm, 3351 add_sub_imm_insn insn1, 3352 add_sub_reg_insn insn2, 3353 bool is32) { 3354 assert(Rd != zr, "Rd = zr and not setting flags?"); 3355 bool fits = operand_valid_for_add_sub_immediate(is32 ? (int32_t)imm : imm); 3356 if (fits) { 3357 (this->*insn1)(Rd, Rn, imm); 3358 } else { 3359 if (g_uabs(imm) < (1 << 24)) { 3360 (this->*insn1)(Rd, Rn, imm & -(1 << 12)); 3361 (this->*insn1)(Rd, Rd, imm & ((1 << 12)-1)); 3362 } else { 3363 assert_different_registers(Rd, Rn); 3364 mov(Rd, imm); 3365 (this->*insn2)(Rd, Rn, Rd, LSL, 0); 3366 } 3367 } 3368 } 3369 3370 // Separate vsn which sets the flags. Optimisations are more restricted 3371 // because we must set the flags correctly. 3372 void MacroAssembler::wrap_adds_subs_imm_insn(Register Rd, Register Rn, uint64_t imm, 3373 add_sub_imm_insn insn1, 3374 add_sub_reg_insn insn2, 3375 bool is32) { 3376 bool fits = operand_valid_for_add_sub_immediate(is32 ? (int32_t)imm : imm); 3377 if (fits) { 3378 (this->*insn1)(Rd, Rn, imm); 3379 } else { 3380 assert_different_registers(Rd, Rn); 3381 assert(Rd != zr, "overflow in immediate operand"); 3382 mov(Rd, imm); 3383 (this->*insn2)(Rd, Rn, Rd, LSL, 0); 3384 } 3385 } 3386 3387 3388 void MacroAssembler::add(Register Rd, Register Rn, RegisterOrConstant increment) { 3389 if (increment.is_register()) { 3390 add(Rd, Rn, increment.as_register()); 3391 } else { 3392 add(Rd, Rn, increment.as_constant()); 3393 } 3394 } 3395 3396 void MacroAssembler::addw(Register Rd, Register Rn, RegisterOrConstant increment) { 3397 if (increment.is_register()) { 3398 addw(Rd, Rn, increment.as_register()); 3399 } else { 3400 addw(Rd, Rn, increment.as_constant()); 3401 } 3402 } 3403 3404 void MacroAssembler::sub(Register Rd, Register Rn, RegisterOrConstant decrement) { 3405 if (decrement.is_register()) { 3406 sub(Rd, Rn, decrement.as_register()); 3407 } else { 3408 sub(Rd, Rn, decrement.as_constant()); 3409 } 3410 } 3411 3412 void MacroAssembler::subw(Register Rd, Register Rn, RegisterOrConstant decrement) { 3413 if (decrement.is_register()) { 3414 subw(Rd, Rn, decrement.as_register()); 3415 } else { 3416 subw(Rd, Rn, decrement.as_constant()); 3417 } 3418 } 3419 3420 void MacroAssembler::reinit_heapbase() 3421 { 3422 if (UseCompressedOops) { 3423 if (Universe::is_fully_initialized()) { 3424 mov(rheapbase, CompressedOops::base()); 3425 } else { 3426 lea(rheapbase, ExternalAddress(CompressedOops::base_addr())); 3427 ldr(rheapbase, Address(rheapbase)); 3428 } 3429 } 3430 } 3431 3432 // this simulates the behaviour of the x86 cmpxchg instruction using a 3433 // load linked/store conditional pair. we use the acquire/release 3434 // versions of these instructions so that we flush pending writes as 3435 // per Java semantics. 3436 3437 // n.b the x86 version assumes the old value to be compared against is 3438 // in rax and updates rax with the value located in memory if the 3439 // cmpxchg fails. we supply a register for the old value explicitly 3440 3441 // the aarch64 load linked/store conditional instructions do not 3442 // accept an offset. so, unlike x86, we must provide a plain register 3443 // to identify the memory word to be compared/exchanged rather than a 3444 // register+offset Address. 3445 3446 void MacroAssembler::cmpxchgptr(Register oldv, Register newv, Register addr, Register tmp, 3447 Label &succeed, Label *fail) { 3448 // oldv holds comparison value 3449 // newv holds value to write in exchange 3450 // addr identifies memory word to compare against/update 3451 if (UseLSE) { 3452 mov(tmp, oldv); 3453 casal(Assembler::xword, oldv, newv, addr); 3454 cmp(tmp, oldv); 3455 br(Assembler::EQ, succeed); 3456 membar(AnyAny); 3457 } else { 3458 Label retry_load, nope; 3459 prfm(Address(addr), PSTL1STRM); 3460 bind(retry_load); 3461 // flush and load exclusive from the memory location 3462 // and fail if it is not what we expect 3463 ldaxr(tmp, addr); 3464 cmp(tmp, oldv); 3465 br(Assembler::NE, nope); 3466 // if we store+flush with no intervening write tmp will be zero 3467 stlxr(tmp, newv, addr); 3468 cbzw(tmp, succeed); 3469 // retry so we only ever return after a load fails to compare 3470 // ensures we don't return a stale value after a failed write. 3471 b(retry_load); 3472 // if the memory word differs we return it in oldv and signal a fail 3473 bind(nope); 3474 membar(AnyAny); 3475 mov(oldv, tmp); 3476 } 3477 if (fail) 3478 b(*fail); 3479 } 3480 3481 void MacroAssembler::cmpxchg_obj_header(Register oldv, Register newv, Register obj, Register tmp, 3482 Label &succeed, Label *fail) { 3483 assert(oopDesc::mark_offset_in_bytes() == 0, "assumption"); 3484 cmpxchgptr(oldv, newv, obj, tmp, succeed, fail); 3485 } 3486 3487 void MacroAssembler::cmpxchgw(Register oldv, Register newv, Register addr, Register tmp, 3488 Label &succeed, Label *fail) { 3489 // oldv holds comparison value 3490 // newv holds value to write in exchange 3491 // addr identifies memory word to compare against/update 3492 // tmp returns 0/1 for success/failure 3493 if (UseLSE) { 3494 mov(tmp, oldv); 3495 casal(Assembler::word, oldv, newv, addr); 3496 cmp(tmp, oldv); 3497 br(Assembler::EQ, succeed); 3498 membar(AnyAny); 3499 } else { 3500 Label retry_load, nope; 3501 prfm(Address(addr), PSTL1STRM); 3502 bind(retry_load); 3503 // flush and load exclusive from the memory location 3504 // and fail if it is not what we expect 3505 ldaxrw(tmp, addr); 3506 cmp(tmp, oldv); 3507 br(Assembler::NE, nope); 3508 // if we store+flush with no intervening write tmp will be zero 3509 stlxrw(tmp, newv, addr); 3510 cbzw(tmp, succeed); 3511 // retry so we only ever return after a load fails to compare 3512 // ensures we don't return a stale value after a failed write. 3513 b(retry_load); 3514 // if the memory word differs we return it in oldv and signal a fail 3515 bind(nope); 3516 membar(AnyAny); 3517 mov(oldv, tmp); 3518 } 3519 if (fail) 3520 b(*fail); 3521 } 3522 3523 // A generic CAS; success or failure is in the EQ flag. A weak CAS 3524 // doesn't retry and may fail spuriously. If the oldval is wanted, 3525 // Pass a register for the result, otherwise pass noreg. 3526 3527 // Clobbers rscratch1 3528 void MacroAssembler::cmpxchg(Register addr, Register expected, 3529 Register new_val, 3530 enum operand_size size, 3531 bool acquire, bool release, 3532 bool weak, 3533 Register result) { 3534 if (result == noreg) result = rscratch1; 3535 BLOCK_COMMENT("cmpxchg {"); 3536 if (UseLSE) { 3537 mov(result, expected); 3538 lse_cas(result, new_val, addr, size, acquire, release, /*not_pair*/ true); 3539 compare_eq(result, expected, size); 3540 #ifdef ASSERT 3541 // Poison rscratch1 which is written on !UseLSE branch 3542 mov(rscratch1, 0x1f1f1f1f1f1f1f1f); 3543 #endif 3544 } else { 3545 Label retry_load, done; 3546 prfm(Address(addr), PSTL1STRM); 3547 bind(retry_load); 3548 load_exclusive(result, addr, size, acquire); 3549 compare_eq(result, expected, size); 3550 br(Assembler::NE, done); 3551 store_exclusive(rscratch1, new_val, addr, size, release); 3552 if (weak) { 3553 cmpw(rscratch1, 0u); // If the store fails, return NE to our caller. 3554 } else { 3555 cbnzw(rscratch1, retry_load); 3556 } 3557 bind(done); 3558 } 3559 BLOCK_COMMENT("} cmpxchg"); 3560 } 3561 3562 // A generic comparison. Only compares for equality, clobbers rscratch1. 3563 void MacroAssembler::compare_eq(Register rm, Register rn, enum operand_size size) { 3564 if (size == xword) { 3565 cmp(rm, rn); 3566 } else if (size == word) { 3567 cmpw(rm, rn); 3568 } else if (size == halfword) { 3569 eorw(rscratch1, rm, rn); 3570 ands(zr, rscratch1, 0xffff); 3571 } else if (size == byte) { 3572 eorw(rscratch1, rm, rn); 3573 ands(zr, rscratch1, 0xff); 3574 } else { 3575 ShouldNotReachHere(); 3576 } 3577 } 3578 3579 3580 static bool different(Register a, RegisterOrConstant b, Register c) { 3581 if (b.is_constant()) 3582 return a != c; 3583 else 3584 return a != b.as_register() && a != c && b.as_register() != c; 3585 } 3586 3587 #define ATOMIC_OP(NAME, LDXR, OP, IOP, AOP, STXR, sz) \ 3588 void MacroAssembler::atomic_##NAME(Register prev, RegisterOrConstant incr, Register addr) { \ 3589 if (UseLSE) { \ 3590 prev = prev->is_valid() ? prev : zr; \ 3591 if (incr.is_register()) { \ 3592 AOP(sz, incr.as_register(), prev, addr); \ 3593 } else { \ 3594 mov(rscratch2, incr.as_constant()); \ 3595 AOP(sz, rscratch2, prev, addr); \ 3596 } \ 3597 return; \ 3598 } \ 3599 Register result = rscratch2; \ 3600 if (prev->is_valid()) \ 3601 result = different(prev, incr, addr) ? prev : rscratch2; \ 3602 \ 3603 Label retry_load; \ 3604 prfm(Address(addr), PSTL1STRM); \ 3605 bind(retry_load); \ 3606 LDXR(result, addr); \ 3607 OP(rscratch1, result, incr); \ 3608 STXR(rscratch2, rscratch1, addr); \ 3609 cbnzw(rscratch2, retry_load); \ 3610 if (prev->is_valid() && prev != result) { \ 3611 IOP(prev, rscratch1, incr); \ 3612 } \ 3613 } 3614 3615 ATOMIC_OP(add, ldxr, add, sub, ldadd, stxr, Assembler::xword) 3616 ATOMIC_OP(addw, ldxrw, addw, subw, ldadd, stxrw, Assembler::word) 3617 ATOMIC_OP(addal, ldaxr, add, sub, ldaddal, stlxr, Assembler::xword) 3618 ATOMIC_OP(addalw, ldaxrw, addw, subw, ldaddal, stlxrw, Assembler::word) 3619 3620 #undef ATOMIC_OP 3621 3622 #define ATOMIC_XCHG(OP, AOP, LDXR, STXR, sz) \ 3623 void MacroAssembler::atomic_##OP(Register prev, Register newv, Register addr) { \ 3624 if (UseLSE) { \ 3625 prev = prev->is_valid() ? prev : zr; \ 3626 AOP(sz, newv, prev, addr); \ 3627 return; \ 3628 } \ 3629 Register result = rscratch2; \ 3630 if (prev->is_valid()) \ 3631 result = different(prev, newv, addr) ? prev : rscratch2; \ 3632 \ 3633 Label retry_load; \ 3634 prfm(Address(addr), PSTL1STRM); \ 3635 bind(retry_load); \ 3636 LDXR(result, addr); \ 3637 STXR(rscratch1, newv, addr); \ 3638 cbnzw(rscratch1, retry_load); \ 3639 if (prev->is_valid() && prev != result) \ 3640 mov(prev, result); \ 3641 } 3642 3643 ATOMIC_XCHG(xchg, swp, ldxr, stxr, Assembler::xword) 3644 ATOMIC_XCHG(xchgw, swp, ldxrw, stxrw, Assembler::word) 3645 ATOMIC_XCHG(xchgl, swpl, ldxr, stlxr, Assembler::xword) 3646 ATOMIC_XCHG(xchglw, swpl, ldxrw, stlxrw, Assembler::word) 3647 ATOMIC_XCHG(xchgal, swpal, ldaxr, stlxr, Assembler::xword) 3648 ATOMIC_XCHG(xchgalw, swpal, ldaxrw, stlxrw, Assembler::word) 3649 3650 #undef ATOMIC_XCHG 3651 3652 #ifndef PRODUCT 3653 extern "C" void findpc(intptr_t x); 3654 #endif 3655 3656 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) 3657 { 3658 // In order to get locks to work, we need to fake a in_VM state 3659 if (ShowMessageBoxOnError ) { 3660 JavaThread* thread = JavaThread::current(); 3661 JavaThreadState saved_state = thread->thread_state(); 3662 thread->set_thread_state(_thread_in_vm); 3663 #ifndef PRODUCT 3664 if (CountBytecodes || TraceBytecodes || StopInterpreterAt) { 3665 ttyLocker ttyl; 3666 BytecodeCounter::print(); 3667 } 3668 #endif 3669 if (os::message_box(msg, "Execution stopped, print registers?")) { 3670 ttyLocker ttyl; 3671 tty->print_cr(" pc = 0x%016" PRIx64, pc); 3672 #ifndef PRODUCT 3673 tty->cr(); 3674 findpc(pc); 3675 tty->cr(); 3676 #endif 3677 tty->print_cr(" r0 = 0x%016" PRIx64, regs[0]); 3678 tty->print_cr(" r1 = 0x%016" PRIx64, regs[1]); 3679 tty->print_cr(" r2 = 0x%016" PRIx64, regs[2]); 3680 tty->print_cr(" r3 = 0x%016" PRIx64, regs[3]); 3681 tty->print_cr(" r4 = 0x%016" PRIx64, regs[4]); 3682 tty->print_cr(" r5 = 0x%016" PRIx64, regs[5]); 3683 tty->print_cr(" r6 = 0x%016" PRIx64, regs[6]); 3684 tty->print_cr(" r7 = 0x%016" PRIx64, regs[7]); 3685 tty->print_cr(" r8 = 0x%016" PRIx64, regs[8]); 3686 tty->print_cr(" r9 = 0x%016" PRIx64, regs[9]); 3687 tty->print_cr("r10 = 0x%016" PRIx64, regs[10]); 3688 tty->print_cr("r11 = 0x%016" PRIx64, regs[11]); 3689 tty->print_cr("r12 = 0x%016" PRIx64, regs[12]); 3690 tty->print_cr("r13 = 0x%016" PRIx64, regs[13]); 3691 tty->print_cr("r14 = 0x%016" PRIx64, regs[14]); 3692 tty->print_cr("r15 = 0x%016" PRIx64, regs[15]); 3693 tty->print_cr("r16 = 0x%016" PRIx64, regs[16]); 3694 tty->print_cr("r17 = 0x%016" PRIx64, regs[17]); 3695 tty->print_cr("r18 = 0x%016" PRIx64, regs[18]); 3696 tty->print_cr("r19 = 0x%016" PRIx64, regs[19]); 3697 tty->print_cr("r20 = 0x%016" PRIx64, regs[20]); 3698 tty->print_cr("r21 = 0x%016" PRIx64, regs[21]); 3699 tty->print_cr("r22 = 0x%016" PRIx64, regs[22]); 3700 tty->print_cr("r23 = 0x%016" PRIx64, regs[23]); 3701 tty->print_cr("r24 = 0x%016" PRIx64, regs[24]); 3702 tty->print_cr("r25 = 0x%016" PRIx64, regs[25]); 3703 tty->print_cr("r26 = 0x%016" PRIx64, regs[26]); 3704 tty->print_cr("r27 = 0x%016" PRIx64, regs[27]); 3705 tty->print_cr("r28 = 0x%016" PRIx64, regs[28]); 3706 tty->print_cr("r30 = 0x%016" PRIx64, regs[30]); 3707 tty->print_cr("r31 = 0x%016" PRIx64, regs[31]); 3708 BREAKPOINT; 3709 } 3710 } 3711 fatal("DEBUG MESSAGE: %s", msg); 3712 } 3713 3714 RegSet MacroAssembler::call_clobbered_gp_registers() { 3715 RegSet regs = RegSet::range(r0, r17) - RegSet::of(rscratch1, rscratch2); 3716 #ifndef R18_RESERVED 3717 regs += r18_tls; 3718 #endif 3719 return regs; 3720 } 3721 3722 void MacroAssembler::push_call_clobbered_registers_except(RegSet exclude) { 3723 int step = 4 * wordSize; 3724 push(call_clobbered_gp_registers() - exclude, sp); 3725 sub(sp, sp, step); 3726 mov(rscratch1, -step); 3727 // Push v0-v7, v16-v31. 3728 for (int i = 31; i>= 4; i -= 4) { 3729 if (i <= v7->encoding() || i >= v16->encoding()) 3730 st1(as_FloatRegister(i-3), as_FloatRegister(i-2), as_FloatRegister(i-1), 3731 as_FloatRegister(i), T1D, Address(post(sp, rscratch1))); 3732 } 3733 st1(as_FloatRegister(0), as_FloatRegister(1), as_FloatRegister(2), 3734 as_FloatRegister(3), T1D, Address(sp)); 3735 } 3736 3737 void MacroAssembler::pop_call_clobbered_registers_except(RegSet exclude) { 3738 for (int i = 0; i < 32; i += 4) { 3739 if (i <= v7->encoding() || i >= v16->encoding()) 3740 ld1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2), 3741 as_FloatRegister(i+3), T1D, Address(post(sp, 4 * wordSize))); 3742 } 3743 3744 reinitialize_ptrue(); 3745 3746 pop(call_clobbered_gp_registers() - exclude, sp); 3747 } 3748 3749 void MacroAssembler::push_CPU_state(bool save_vectors, bool use_sve, 3750 int sve_vector_size_in_bytes, int total_predicate_in_bytes) { 3751 push(RegSet::range(r0, r29), sp); // integer registers except lr & sp 3752 if (save_vectors && use_sve && sve_vector_size_in_bytes > 16) { 3753 sub(sp, sp, sve_vector_size_in_bytes * FloatRegister::number_of_registers); 3754 for (int i = 0; i < FloatRegister::number_of_registers; i++) { 3755 sve_str(as_FloatRegister(i), Address(sp, i)); 3756 } 3757 } else { 3758 int step = (save_vectors ? 8 : 4) * wordSize; 3759 mov(rscratch1, -step); 3760 sub(sp, sp, step); 3761 for (int i = 28; i >= 4; i -= 4) { 3762 st1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2), 3763 as_FloatRegister(i+3), save_vectors ? T2D : T1D, Address(post(sp, rscratch1))); 3764 } 3765 st1(v0, v1, v2, v3, save_vectors ? T2D : T1D, sp); 3766 } 3767 if (save_vectors && use_sve && total_predicate_in_bytes > 0) { 3768 sub(sp, sp, total_predicate_in_bytes); 3769 for (int i = 0; i < PRegister::number_of_registers; i++) { 3770 sve_str(as_PRegister(i), Address(sp, i)); 3771 } 3772 } 3773 } 3774 3775 void MacroAssembler::pop_CPU_state(bool restore_vectors, bool use_sve, 3776 int sve_vector_size_in_bytes, int total_predicate_in_bytes) { 3777 if (restore_vectors && use_sve && total_predicate_in_bytes > 0) { 3778 for (int i = PRegister::number_of_registers - 1; i >= 0; i--) { 3779 sve_ldr(as_PRegister(i), Address(sp, i)); 3780 } 3781 add(sp, sp, total_predicate_in_bytes); 3782 } 3783 if (restore_vectors && use_sve && sve_vector_size_in_bytes > 16) { 3784 for (int i = FloatRegister::number_of_registers - 1; i >= 0; i--) { 3785 sve_ldr(as_FloatRegister(i), Address(sp, i)); 3786 } 3787 add(sp, sp, sve_vector_size_in_bytes * FloatRegister::number_of_registers); 3788 } else { 3789 int step = (restore_vectors ? 8 : 4) * wordSize; 3790 for (int i = 0; i <= 28; i += 4) 3791 ld1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2), 3792 as_FloatRegister(i+3), restore_vectors ? T2D : T1D, Address(post(sp, step))); 3793 } 3794 3795 // We may use predicate registers and rely on ptrue with SVE, 3796 // regardless of wide vector (> 8 bytes) used or not. 3797 if (use_sve) { 3798 reinitialize_ptrue(); 3799 } 3800 3801 // integer registers except lr & sp 3802 pop(RegSet::range(r0, r17), sp); 3803 #ifdef R18_RESERVED 3804 ldp(zr, r19, Address(post(sp, 2 * wordSize))); 3805 pop(RegSet::range(r20, r29), sp); 3806 #else 3807 pop(RegSet::range(r18_tls, r29), sp); 3808 #endif 3809 } 3810 3811 /** 3812 * Helpers for multiply_to_len(). 3813 */ 3814 void MacroAssembler::add2_with_carry(Register final_dest_hi, Register dest_hi, Register dest_lo, 3815 Register src1, Register src2) { 3816 adds(dest_lo, dest_lo, src1); 3817 adc(dest_hi, dest_hi, zr); 3818 adds(dest_lo, dest_lo, src2); 3819 adc(final_dest_hi, dest_hi, zr); 3820 } 3821 3822 // Generate an address from (r + r1 extend offset). "size" is the 3823 // size of the operand. The result may be in rscratch2. 3824 Address MacroAssembler::offsetted_address(Register r, Register r1, 3825 Address::extend ext, int offset, int size) { 3826 if (offset || (ext.shift() % size != 0)) { 3827 lea(rscratch2, Address(r, r1, ext)); 3828 return Address(rscratch2, offset); 3829 } else { 3830 return Address(r, r1, ext); 3831 } 3832 } 3833 3834 Address MacroAssembler::spill_address(int size, int offset, Register tmp) 3835 { 3836 assert(offset >= 0, "spill to negative address?"); 3837 // Offset reachable ? 3838 // Not aligned - 9 bits signed offset 3839 // Aligned - 12 bits unsigned offset shifted 3840 Register base = sp; 3841 if ((offset & (size-1)) && offset >= (1<<8)) { 3842 add(tmp, base, offset & ((1<<12)-1)); 3843 base = tmp; 3844 offset &= -1u<<12; 3845 } 3846 3847 if (offset >= (1<<12) * size) { 3848 add(tmp, base, offset & (((1<<12)-1)<<12)); 3849 base = tmp; 3850 offset &= ~(((1<<12)-1)<<12); 3851 } 3852 3853 return Address(base, offset); 3854 } 3855 3856 Address MacroAssembler::sve_spill_address(int sve_reg_size_in_bytes, int offset, Register tmp) { 3857 assert(offset >= 0, "spill to negative address?"); 3858 3859 Register base = sp; 3860 3861 // An immediate offset in the range 0 to 255 which is multiplied 3862 // by the current vector or predicate register size in bytes. 3863 if (offset % sve_reg_size_in_bytes == 0 && offset < ((1<<8)*sve_reg_size_in_bytes)) { 3864 return Address(base, offset / sve_reg_size_in_bytes); 3865 } 3866 3867 add(tmp, base, offset); 3868 return Address(tmp); 3869 } 3870 3871 // Checks whether offset is aligned. 3872 // Returns true if it is, else false. 3873 bool MacroAssembler::merge_alignment_check(Register base, 3874 size_t size, 3875 int64_t cur_offset, 3876 int64_t prev_offset) const { 3877 if (AvoidUnalignedAccesses) { 3878 if (base == sp) { 3879 // Checks whether low offset if aligned to pair of registers. 3880 int64_t pair_mask = size * 2 - 1; 3881 int64_t offset = prev_offset > cur_offset ? cur_offset : prev_offset; 3882 return (offset & pair_mask) == 0; 3883 } else { // If base is not sp, we can't guarantee the access is aligned. 3884 return false; 3885 } 3886 } else { 3887 int64_t mask = size - 1; 3888 // Load/store pair instruction only supports element size aligned offset. 3889 return (cur_offset & mask) == 0 && (prev_offset & mask) == 0; 3890 } 3891 } 3892 3893 // Checks whether current and previous loads/stores can be merged. 3894 // Returns true if it can be merged, else false. 3895 bool MacroAssembler::ldst_can_merge(Register rt, 3896 const Address &adr, 3897 size_t cur_size_in_bytes, 3898 bool is_store) const { 3899 address prev = pc() - NativeInstruction::instruction_size; 3900 address last = code()->last_insn(); 3901 3902 if (last == nullptr || !nativeInstruction_at(last)->is_Imm_LdSt()) { 3903 return false; 3904 } 3905 3906 if (adr.getMode() != Address::base_plus_offset || prev != last) { 3907 return false; 3908 } 3909 3910 NativeLdSt* prev_ldst = NativeLdSt_at(prev); 3911 size_t prev_size_in_bytes = prev_ldst->size_in_bytes(); 3912 3913 assert(prev_size_in_bytes == 4 || prev_size_in_bytes == 8, "only supports 64/32bit merging."); 3914 assert(cur_size_in_bytes == 4 || cur_size_in_bytes == 8, "only supports 64/32bit merging."); 3915 3916 if (cur_size_in_bytes != prev_size_in_bytes || is_store != prev_ldst->is_store()) { 3917 return false; 3918 } 3919 3920 int64_t max_offset = 63 * prev_size_in_bytes; 3921 int64_t min_offset = -64 * prev_size_in_bytes; 3922 3923 assert(prev_ldst->is_not_pre_post_index(), "pre-index or post-index is not supported to be merged."); 3924 3925 // Only same base can be merged. 3926 if (adr.base() != prev_ldst->base()) { 3927 return false; 3928 } 3929 3930 int64_t cur_offset = adr.offset(); 3931 int64_t prev_offset = prev_ldst->offset(); 3932 size_t diff = abs(cur_offset - prev_offset); 3933 if (diff != prev_size_in_bytes) { 3934 return false; 3935 } 3936 3937 // Following cases can not be merged: 3938 // ldr x2, [x2, #8] 3939 // ldr x3, [x2, #16] 3940 // or: 3941 // ldr x2, [x3, #8] 3942 // ldr x2, [x3, #16] 3943 // If t1 and t2 is the same in "ldp t1, t2, [xn, #imm]", we'll get SIGILL. 3944 if (!is_store && (adr.base() == prev_ldst->target() || rt == prev_ldst->target())) { 3945 return false; 3946 } 3947 3948 int64_t low_offset = prev_offset > cur_offset ? cur_offset : prev_offset; 3949 // Offset range must be in ldp/stp instruction's range. 3950 if (low_offset > max_offset || low_offset < min_offset) { 3951 return false; 3952 } 3953 3954 if (merge_alignment_check(adr.base(), prev_size_in_bytes, cur_offset, prev_offset)) { 3955 return true; 3956 } 3957 3958 return false; 3959 } 3960 3961 // Merge current load/store with previous load/store into ldp/stp. 3962 void MacroAssembler::merge_ldst(Register rt, 3963 const Address &adr, 3964 size_t cur_size_in_bytes, 3965 bool is_store) { 3966 3967 assert(ldst_can_merge(rt, adr, cur_size_in_bytes, is_store) == true, "cur and prev must be able to be merged."); 3968 3969 Register rt_low, rt_high; 3970 address prev = pc() - NativeInstruction::instruction_size; 3971 NativeLdSt* prev_ldst = NativeLdSt_at(prev); 3972 3973 int64_t offset; 3974 3975 if (adr.offset() < prev_ldst->offset()) { 3976 offset = adr.offset(); 3977 rt_low = rt; 3978 rt_high = prev_ldst->target(); 3979 } else { 3980 offset = prev_ldst->offset(); 3981 rt_low = prev_ldst->target(); 3982 rt_high = rt; 3983 } 3984 3985 Address adr_p = Address(prev_ldst->base(), offset); 3986 // Overwrite previous generated binary. 3987 code_section()->set_end(prev); 3988 3989 const size_t sz = prev_ldst->size_in_bytes(); 3990 assert(sz == 8 || sz == 4, "only supports 64/32bit merging."); 3991 if (!is_store) { 3992 BLOCK_COMMENT("merged ldr pair"); 3993 if (sz == 8) { 3994 ldp(rt_low, rt_high, adr_p); 3995 } else { 3996 ldpw(rt_low, rt_high, adr_p); 3997 } 3998 } else { 3999 BLOCK_COMMENT("merged str pair"); 4000 if (sz == 8) { 4001 stp(rt_low, rt_high, adr_p); 4002 } else { 4003 stpw(rt_low, rt_high, adr_p); 4004 } 4005 } 4006 } 4007 4008 /** 4009 * Multiply 64 bit by 64 bit first loop. 4010 */ 4011 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart, 4012 Register y, Register y_idx, Register z, 4013 Register carry, Register product, 4014 Register idx, Register kdx) { 4015 // 4016 // jlong carry, x[], y[], z[]; 4017 // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) { 4018 // huge_128 product = y[idx] * x[xstart] + carry; 4019 // z[kdx] = (jlong)product; 4020 // carry = (jlong)(product >>> 64); 4021 // } 4022 // z[xstart] = carry; 4023 // 4024 4025 Label L_first_loop, L_first_loop_exit; 4026 Label L_one_x, L_one_y, L_multiply; 4027 4028 subsw(xstart, xstart, 1); 4029 br(Assembler::MI, L_one_x); 4030 4031 lea(rscratch1, Address(x, xstart, Address::lsl(LogBytesPerInt))); 4032 ldr(x_xstart, Address(rscratch1)); 4033 ror(x_xstart, x_xstart, 32); // convert big-endian to little-endian 4034 4035 bind(L_first_loop); 4036 subsw(idx, idx, 1); 4037 br(Assembler::MI, L_first_loop_exit); 4038 subsw(idx, idx, 1); 4039 br(Assembler::MI, L_one_y); 4040 lea(rscratch1, Address(y, idx, Address::uxtw(LogBytesPerInt))); 4041 ldr(y_idx, Address(rscratch1)); 4042 ror(y_idx, y_idx, 32); // convert big-endian to little-endian 4043 bind(L_multiply); 4044 4045 // AArch64 has a multiply-accumulate instruction that we can't use 4046 // here because it has no way to process carries, so we have to use 4047 // separate add and adc instructions. Bah. 4048 umulh(rscratch1, x_xstart, y_idx); // x_xstart * y_idx -> rscratch1:product 4049 mul(product, x_xstart, y_idx); 4050 adds(product, product, carry); 4051 adc(carry, rscratch1, zr); // x_xstart * y_idx + carry -> carry:product 4052 4053 subw(kdx, kdx, 2); 4054 ror(product, product, 32); // back to big-endian 4055 str(product, offsetted_address(z, kdx, Address::uxtw(LogBytesPerInt), 0, BytesPerLong)); 4056 4057 b(L_first_loop); 4058 4059 bind(L_one_y); 4060 ldrw(y_idx, Address(y, 0)); 4061 b(L_multiply); 4062 4063 bind(L_one_x); 4064 ldrw(x_xstart, Address(x, 0)); 4065 b(L_first_loop); 4066 4067 bind(L_first_loop_exit); 4068 } 4069 4070 /** 4071 * Multiply 128 bit by 128. Unrolled inner loop. 4072 * 4073 */ 4074 void MacroAssembler::multiply_128_x_128_loop(Register y, Register z, 4075 Register carry, Register carry2, 4076 Register idx, Register jdx, 4077 Register yz_idx1, Register yz_idx2, 4078 Register tmp, Register tmp3, Register tmp4, 4079 Register tmp6, Register product_hi) { 4080 4081 // jlong carry, x[], y[], z[]; 4082 // int kdx = ystart+1; 4083 // for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop 4084 // huge_128 tmp3 = (y[idx+1] * product_hi) + z[kdx+idx+1] + carry; 4085 // jlong carry2 = (jlong)(tmp3 >>> 64); 4086 // huge_128 tmp4 = (y[idx] * product_hi) + z[kdx+idx] + carry2; 4087 // carry = (jlong)(tmp4 >>> 64); 4088 // z[kdx+idx+1] = (jlong)tmp3; 4089 // z[kdx+idx] = (jlong)tmp4; 4090 // } 4091 // idx += 2; 4092 // if (idx > 0) { 4093 // yz_idx1 = (y[idx] * product_hi) + z[kdx+idx] + carry; 4094 // z[kdx+idx] = (jlong)yz_idx1; 4095 // carry = (jlong)(yz_idx1 >>> 64); 4096 // } 4097 // 4098 4099 Label L_third_loop, L_third_loop_exit, L_post_third_loop_done; 4100 4101 lsrw(jdx, idx, 2); 4102 4103 bind(L_third_loop); 4104 4105 subsw(jdx, jdx, 1); 4106 br(Assembler::MI, L_third_loop_exit); 4107 subw(idx, idx, 4); 4108 4109 lea(rscratch1, Address(y, idx, Address::uxtw(LogBytesPerInt))); 4110 4111 ldp(yz_idx2, yz_idx1, Address(rscratch1, 0)); 4112 4113 lea(tmp6, Address(z, idx, Address::uxtw(LogBytesPerInt))); 4114 4115 ror(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian 4116 ror(yz_idx2, yz_idx2, 32); 4117 4118 ldp(rscratch2, rscratch1, Address(tmp6, 0)); 4119 4120 mul(tmp3, product_hi, yz_idx1); // yz_idx1 * product_hi -> tmp4:tmp3 4121 umulh(tmp4, product_hi, yz_idx1); 4122 4123 ror(rscratch1, rscratch1, 32); // convert big-endian to little-endian 4124 ror(rscratch2, rscratch2, 32); 4125 4126 mul(tmp, product_hi, yz_idx2); // yz_idx2 * product_hi -> carry2:tmp 4127 umulh(carry2, product_hi, yz_idx2); 4128 4129 // propagate sum of both multiplications into carry:tmp4:tmp3 4130 adds(tmp3, tmp3, carry); 4131 adc(tmp4, tmp4, zr); 4132 adds(tmp3, tmp3, rscratch1); 4133 adcs(tmp4, tmp4, tmp); 4134 adc(carry, carry2, zr); 4135 adds(tmp4, tmp4, rscratch2); 4136 adc(carry, carry, zr); 4137 4138 ror(tmp3, tmp3, 32); // convert little-endian to big-endian 4139 ror(tmp4, tmp4, 32); 4140 stp(tmp4, tmp3, Address(tmp6, 0)); 4141 4142 b(L_third_loop); 4143 bind (L_third_loop_exit); 4144 4145 andw (idx, idx, 0x3); 4146 cbz(idx, L_post_third_loop_done); 4147 4148 Label L_check_1; 4149 subsw(idx, idx, 2); 4150 br(Assembler::MI, L_check_1); 4151 4152 lea(rscratch1, Address(y, idx, Address::uxtw(LogBytesPerInt))); 4153 ldr(yz_idx1, Address(rscratch1, 0)); 4154 ror(yz_idx1, yz_idx1, 32); 4155 mul(tmp3, product_hi, yz_idx1); // yz_idx1 * product_hi -> tmp4:tmp3 4156 umulh(tmp4, product_hi, yz_idx1); 4157 lea(rscratch1, Address(z, idx, Address::uxtw(LogBytesPerInt))); 4158 ldr(yz_idx2, Address(rscratch1, 0)); 4159 ror(yz_idx2, yz_idx2, 32); 4160 4161 add2_with_carry(carry, tmp4, tmp3, carry, yz_idx2); 4162 4163 ror(tmp3, tmp3, 32); 4164 str(tmp3, Address(rscratch1, 0)); 4165 4166 bind (L_check_1); 4167 4168 andw (idx, idx, 0x1); 4169 subsw(idx, idx, 1); 4170 br(Assembler::MI, L_post_third_loop_done); 4171 ldrw(tmp4, Address(y, idx, Address::uxtw(LogBytesPerInt))); 4172 mul(tmp3, tmp4, product_hi); // tmp4 * product_hi -> carry2:tmp3 4173 umulh(carry2, tmp4, product_hi); 4174 ldrw(tmp4, Address(z, idx, Address::uxtw(LogBytesPerInt))); 4175 4176 add2_with_carry(carry2, tmp3, tmp4, carry); 4177 4178 strw(tmp3, Address(z, idx, Address::uxtw(LogBytesPerInt))); 4179 extr(carry, carry2, tmp3, 32); 4180 4181 bind(L_post_third_loop_done); 4182 } 4183 4184 /** 4185 * Code for BigInteger::multiplyToLen() intrinsic. 4186 * 4187 * r0: x 4188 * r1: xlen 4189 * r2: y 4190 * r3: ylen 4191 * r4: z 4192 * r5: tmp0 4193 * r10: tmp1 4194 * r11: tmp2 4195 * r12: tmp3 4196 * r13: tmp4 4197 * r14: tmp5 4198 * r15: tmp6 4199 * r16: tmp7 4200 * 4201 */ 4202 void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen, 4203 Register z, Register tmp0, 4204 Register tmp1, Register tmp2, Register tmp3, Register tmp4, 4205 Register tmp5, Register tmp6, Register product_hi) { 4206 4207 assert_different_registers(x, xlen, y, ylen, z, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, product_hi); 4208 4209 const Register idx = tmp1; 4210 const Register kdx = tmp2; 4211 const Register xstart = tmp3; 4212 4213 const Register y_idx = tmp4; 4214 const Register carry = tmp5; 4215 const Register product = xlen; 4216 const Register x_xstart = tmp0; 4217 4218 // First Loop. 4219 // 4220 // final static long LONG_MASK = 0xffffffffL; 4221 // int xstart = xlen - 1; 4222 // int ystart = ylen - 1; 4223 // long carry = 0; 4224 // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) { 4225 // long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry; 4226 // z[kdx] = (int)product; 4227 // carry = product >>> 32; 4228 // } 4229 // z[xstart] = (int)carry; 4230 // 4231 4232 movw(idx, ylen); // idx = ylen; 4233 addw(kdx, xlen, ylen); // kdx = xlen+ylen; 4234 mov(carry, zr); // carry = 0; 4235 4236 Label L_done; 4237 4238 movw(xstart, xlen); 4239 subsw(xstart, xstart, 1); 4240 br(Assembler::MI, L_done); 4241 4242 multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx); 4243 4244 Label L_second_loop; 4245 cbzw(kdx, L_second_loop); 4246 4247 Label L_carry; 4248 subw(kdx, kdx, 1); 4249 cbzw(kdx, L_carry); 4250 4251 strw(carry, Address(z, kdx, Address::uxtw(LogBytesPerInt))); 4252 lsr(carry, carry, 32); 4253 subw(kdx, kdx, 1); 4254 4255 bind(L_carry); 4256 strw(carry, Address(z, kdx, Address::uxtw(LogBytesPerInt))); 4257 4258 // Second and third (nested) loops. 4259 // 4260 // for (int i = xstart-1; i >= 0; i--) { // Second loop 4261 // carry = 0; 4262 // for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop 4263 // long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) + 4264 // (z[k] & LONG_MASK) + carry; 4265 // z[k] = (int)product; 4266 // carry = product >>> 32; 4267 // } 4268 // z[i] = (int)carry; 4269 // } 4270 // 4271 // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = product_hi 4272 4273 const Register jdx = tmp1; 4274 4275 bind(L_second_loop); 4276 mov(carry, zr); // carry = 0; 4277 movw(jdx, ylen); // j = ystart+1 4278 4279 subsw(xstart, xstart, 1); // i = xstart-1; 4280 br(Assembler::MI, L_done); 4281 4282 str(z, Address(pre(sp, -4 * wordSize))); 4283 4284 Label L_last_x; 4285 lea(z, offsetted_address(z, xstart, Address::uxtw(LogBytesPerInt), 4, BytesPerInt)); // z = z + k - j 4286 subsw(xstart, xstart, 1); // i = xstart-1; 4287 br(Assembler::MI, L_last_x); 4288 4289 lea(rscratch1, Address(x, xstart, Address::uxtw(LogBytesPerInt))); 4290 ldr(product_hi, Address(rscratch1)); 4291 ror(product_hi, product_hi, 32); // convert big-endian to little-endian 4292 4293 Label L_third_loop_prologue; 4294 bind(L_third_loop_prologue); 4295 4296 str(ylen, Address(sp, wordSize)); 4297 stp(x, xstart, Address(sp, 2 * wordSize)); 4298 multiply_128_x_128_loop(y, z, carry, x, jdx, ylen, product, 4299 tmp2, x_xstart, tmp3, tmp4, tmp6, product_hi); 4300 ldp(z, ylen, Address(post(sp, 2 * wordSize))); 4301 ldp(x, xlen, Address(post(sp, 2 * wordSize))); // copy old xstart -> xlen 4302 4303 addw(tmp3, xlen, 1); 4304 strw(carry, Address(z, tmp3, Address::uxtw(LogBytesPerInt))); 4305 subsw(tmp3, tmp3, 1); 4306 br(Assembler::MI, L_done); 4307 4308 lsr(carry, carry, 32); 4309 strw(carry, Address(z, tmp3, Address::uxtw(LogBytesPerInt))); 4310 b(L_second_loop); 4311 4312 // Next infrequent code is moved outside loops. 4313 bind(L_last_x); 4314 ldrw(product_hi, Address(x, 0)); 4315 b(L_third_loop_prologue); 4316 4317 bind(L_done); 4318 } 4319 4320 // Code for BigInteger::mulAdd intrinsic 4321 // out = r0 4322 // in = r1 4323 // offset = r2 (already out.length-offset) 4324 // len = r3 4325 // k = r4 4326 // 4327 // pseudo code from java implementation: 4328 // carry = 0; 4329 // offset = out.length-offset - 1; 4330 // for (int j=len-1; j >= 0; j--) { 4331 // product = (in[j] & LONG_MASK) * kLong + (out[offset] & LONG_MASK) + carry; 4332 // out[offset--] = (int)product; 4333 // carry = product >>> 32; 4334 // } 4335 // return (int)carry; 4336 void MacroAssembler::mul_add(Register out, Register in, Register offset, 4337 Register len, Register k) { 4338 Label LOOP, END; 4339 // pre-loop 4340 cmp(len, zr); // cmp, not cbz/cbnz: to use condition twice => less branches 4341 csel(out, zr, out, Assembler::EQ); 4342 br(Assembler::EQ, END); 4343 add(in, in, len, LSL, 2); // in[j+1] address 4344 add(offset, out, offset, LSL, 2); // out[offset + 1] address 4345 mov(out, zr); // used to keep carry now 4346 BIND(LOOP); 4347 ldrw(rscratch1, Address(pre(in, -4))); 4348 madd(rscratch1, rscratch1, k, out); 4349 ldrw(rscratch2, Address(pre(offset, -4))); 4350 add(rscratch1, rscratch1, rscratch2); 4351 strw(rscratch1, Address(offset)); 4352 lsr(out, rscratch1, 32); 4353 subs(len, len, 1); 4354 br(Assembler::NE, LOOP); 4355 BIND(END); 4356 } 4357 4358 /** 4359 * Emits code to update CRC-32 with a byte value according to constants in table 4360 * 4361 * @param [in,out]crc Register containing the crc. 4362 * @param [in]val Register containing the byte to fold into the CRC. 4363 * @param [in]table Register containing the table of crc constants. 4364 * 4365 * uint32_t crc; 4366 * val = crc_table[(val ^ crc) & 0xFF]; 4367 * crc = val ^ (crc >> 8); 4368 * 4369 */ 4370 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) { 4371 eor(val, val, crc); 4372 andr(val, val, 0xff); 4373 ldrw(val, Address(table, val, Address::lsl(2))); 4374 eor(crc, val, crc, Assembler::LSR, 8); 4375 } 4376 4377 /** 4378 * Emits code to update CRC-32 with a 32-bit value according to tables 0 to 3 4379 * 4380 * @param [in,out]crc Register containing the crc. 4381 * @param [in]v Register containing the 32-bit to fold into the CRC. 4382 * @param [in]table0 Register containing table 0 of crc constants. 4383 * @param [in]table1 Register containing table 1 of crc constants. 4384 * @param [in]table2 Register containing table 2 of crc constants. 4385 * @param [in]table3 Register containing table 3 of crc constants. 4386 * 4387 * uint32_t crc; 4388 * v = crc ^ v 4389 * crc = table3[v&0xff]^table2[(v>>8)&0xff]^table1[(v>>16)&0xff]^table0[v>>24] 4390 * 4391 */ 4392 void MacroAssembler::update_word_crc32(Register crc, Register v, Register tmp, 4393 Register table0, Register table1, Register table2, Register table3, 4394 bool upper) { 4395 eor(v, crc, v, upper ? LSR:LSL, upper ? 32:0); 4396 uxtb(tmp, v); 4397 ldrw(crc, Address(table3, tmp, Address::lsl(2))); 4398 ubfx(tmp, v, 8, 8); 4399 ldrw(tmp, Address(table2, tmp, Address::lsl(2))); 4400 eor(crc, crc, tmp); 4401 ubfx(tmp, v, 16, 8); 4402 ldrw(tmp, Address(table1, tmp, Address::lsl(2))); 4403 eor(crc, crc, tmp); 4404 ubfx(tmp, v, 24, 8); 4405 ldrw(tmp, Address(table0, tmp, Address::lsl(2))); 4406 eor(crc, crc, tmp); 4407 } 4408 4409 void MacroAssembler::kernel_crc32_using_crypto_pmull(Register crc, Register buf, 4410 Register len, Register tmp0, Register tmp1, Register tmp2, Register tmp3) { 4411 Label CRC_by4_loop, CRC_by1_loop, CRC_less128, CRC_by128_pre, CRC_by32_loop, CRC_less32, L_exit; 4412 assert_different_registers(crc, buf, len, tmp0, tmp1, tmp2); 4413 4414 subs(tmp0, len, 384); 4415 mvnw(crc, crc); 4416 br(Assembler::GE, CRC_by128_pre); 4417 BIND(CRC_less128); 4418 subs(len, len, 32); 4419 br(Assembler::GE, CRC_by32_loop); 4420 BIND(CRC_less32); 4421 adds(len, len, 32 - 4); 4422 br(Assembler::GE, CRC_by4_loop); 4423 adds(len, len, 4); 4424 br(Assembler::GT, CRC_by1_loop); 4425 b(L_exit); 4426 4427 BIND(CRC_by32_loop); 4428 ldp(tmp0, tmp1, Address(buf)); 4429 crc32x(crc, crc, tmp0); 4430 ldp(tmp2, tmp3, Address(buf, 16)); 4431 crc32x(crc, crc, tmp1); 4432 add(buf, buf, 32); 4433 crc32x(crc, crc, tmp2); 4434 subs(len, len, 32); 4435 crc32x(crc, crc, tmp3); 4436 br(Assembler::GE, CRC_by32_loop); 4437 cmn(len, (u1)32); 4438 br(Assembler::NE, CRC_less32); 4439 b(L_exit); 4440 4441 BIND(CRC_by4_loop); 4442 ldrw(tmp0, Address(post(buf, 4))); 4443 subs(len, len, 4); 4444 crc32w(crc, crc, tmp0); 4445 br(Assembler::GE, CRC_by4_loop); 4446 adds(len, len, 4); 4447 br(Assembler::LE, L_exit); 4448 BIND(CRC_by1_loop); 4449 ldrb(tmp0, Address(post(buf, 1))); 4450 subs(len, len, 1); 4451 crc32b(crc, crc, tmp0); 4452 br(Assembler::GT, CRC_by1_loop); 4453 b(L_exit); 4454 4455 BIND(CRC_by128_pre); 4456 kernel_crc32_common_fold_using_crypto_pmull(crc, buf, len, tmp0, tmp1, tmp2, 4457 4*256*sizeof(juint) + 8*sizeof(juint)); 4458 mov(crc, 0); 4459 crc32x(crc, crc, tmp0); 4460 crc32x(crc, crc, tmp1); 4461 4462 cbnz(len, CRC_less128); 4463 4464 BIND(L_exit); 4465 mvnw(crc, crc); 4466 } 4467 4468 void MacroAssembler::kernel_crc32_using_crc32(Register crc, Register buf, 4469 Register len, Register tmp0, Register tmp1, Register tmp2, 4470 Register tmp3) { 4471 Label CRC_by64_loop, CRC_by4_loop, CRC_by1_loop, CRC_less64, CRC_by64_pre, CRC_by32_loop, CRC_less32, L_exit; 4472 assert_different_registers(crc, buf, len, tmp0, tmp1, tmp2, tmp3); 4473 4474 mvnw(crc, crc); 4475 4476 subs(len, len, 128); 4477 br(Assembler::GE, CRC_by64_pre); 4478 BIND(CRC_less64); 4479 adds(len, len, 128-32); 4480 br(Assembler::GE, CRC_by32_loop); 4481 BIND(CRC_less32); 4482 adds(len, len, 32-4); 4483 br(Assembler::GE, CRC_by4_loop); 4484 adds(len, len, 4); 4485 br(Assembler::GT, CRC_by1_loop); 4486 b(L_exit); 4487 4488 BIND(CRC_by32_loop); 4489 ldp(tmp0, tmp1, Address(post(buf, 16))); 4490 subs(len, len, 32); 4491 crc32x(crc, crc, tmp0); 4492 ldr(tmp2, Address(post(buf, 8))); 4493 crc32x(crc, crc, tmp1); 4494 ldr(tmp3, Address(post(buf, 8))); 4495 crc32x(crc, crc, tmp2); 4496 crc32x(crc, crc, tmp3); 4497 br(Assembler::GE, CRC_by32_loop); 4498 cmn(len, (u1)32); 4499 br(Assembler::NE, CRC_less32); 4500 b(L_exit); 4501 4502 BIND(CRC_by4_loop); 4503 ldrw(tmp0, Address(post(buf, 4))); 4504 subs(len, len, 4); 4505 crc32w(crc, crc, tmp0); 4506 br(Assembler::GE, CRC_by4_loop); 4507 adds(len, len, 4); 4508 br(Assembler::LE, L_exit); 4509 BIND(CRC_by1_loop); 4510 ldrb(tmp0, Address(post(buf, 1))); 4511 subs(len, len, 1); 4512 crc32b(crc, crc, tmp0); 4513 br(Assembler::GT, CRC_by1_loop); 4514 b(L_exit); 4515 4516 BIND(CRC_by64_pre); 4517 sub(buf, buf, 8); 4518 ldp(tmp0, tmp1, Address(buf, 8)); 4519 crc32x(crc, crc, tmp0); 4520 ldr(tmp2, Address(buf, 24)); 4521 crc32x(crc, crc, tmp1); 4522 ldr(tmp3, Address(buf, 32)); 4523 crc32x(crc, crc, tmp2); 4524 ldr(tmp0, Address(buf, 40)); 4525 crc32x(crc, crc, tmp3); 4526 ldr(tmp1, Address(buf, 48)); 4527 crc32x(crc, crc, tmp0); 4528 ldr(tmp2, Address(buf, 56)); 4529 crc32x(crc, crc, tmp1); 4530 ldr(tmp3, Address(pre(buf, 64))); 4531 4532 b(CRC_by64_loop); 4533 4534 align(CodeEntryAlignment); 4535 BIND(CRC_by64_loop); 4536 subs(len, len, 64); 4537 crc32x(crc, crc, tmp2); 4538 ldr(tmp0, Address(buf, 8)); 4539 crc32x(crc, crc, tmp3); 4540 ldr(tmp1, Address(buf, 16)); 4541 crc32x(crc, crc, tmp0); 4542 ldr(tmp2, Address(buf, 24)); 4543 crc32x(crc, crc, tmp1); 4544 ldr(tmp3, Address(buf, 32)); 4545 crc32x(crc, crc, tmp2); 4546 ldr(tmp0, Address(buf, 40)); 4547 crc32x(crc, crc, tmp3); 4548 ldr(tmp1, Address(buf, 48)); 4549 crc32x(crc, crc, tmp0); 4550 ldr(tmp2, Address(buf, 56)); 4551 crc32x(crc, crc, tmp1); 4552 ldr(tmp3, Address(pre(buf, 64))); 4553 br(Assembler::GE, CRC_by64_loop); 4554 4555 // post-loop 4556 crc32x(crc, crc, tmp2); 4557 crc32x(crc, crc, tmp3); 4558 4559 sub(len, len, 64); 4560 add(buf, buf, 8); 4561 cmn(len, (u1)128); 4562 br(Assembler::NE, CRC_less64); 4563 BIND(L_exit); 4564 mvnw(crc, crc); 4565 } 4566 4567 /** 4568 * @param crc register containing existing CRC (32-bit) 4569 * @param buf register pointing to input byte buffer (byte*) 4570 * @param len register containing number of bytes 4571 * @param table register that will contain address of CRC table 4572 * @param tmp scratch register 4573 */ 4574 void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len, 4575 Register table0, Register table1, Register table2, Register table3, 4576 Register tmp, Register tmp2, Register tmp3) { 4577 Label L_by16, L_by16_loop, L_by4, L_by4_loop, L_by1, L_by1_loop, L_exit; 4578 4579 if (UseCryptoPmullForCRC32) { 4580 kernel_crc32_using_crypto_pmull(crc, buf, len, table0, table1, table2, table3); 4581 return; 4582 } 4583 4584 if (UseCRC32) { 4585 kernel_crc32_using_crc32(crc, buf, len, table0, table1, table2, table3); 4586 return; 4587 } 4588 4589 mvnw(crc, crc); 4590 4591 { 4592 uint64_t offset; 4593 adrp(table0, ExternalAddress(StubRoutines::crc_table_addr()), offset); 4594 add(table0, table0, offset); 4595 } 4596 add(table1, table0, 1*256*sizeof(juint)); 4597 add(table2, table0, 2*256*sizeof(juint)); 4598 add(table3, table0, 3*256*sizeof(juint)); 4599 4600 { // Neon code start 4601 cmp(len, (u1)64); 4602 br(Assembler::LT, L_by16); 4603 eor(v16, T16B, v16, v16); 4604 4605 Label L_fold; 4606 4607 add(tmp, table0, 4*256*sizeof(juint)); // Point at the Neon constants 4608 4609 ld1(v0, v1, T2D, post(buf, 32)); 4610 ld1r(v4, T2D, post(tmp, 8)); 4611 ld1r(v5, T2D, post(tmp, 8)); 4612 ld1r(v6, T2D, post(tmp, 8)); 4613 ld1r(v7, T2D, post(tmp, 8)); 4614 mov(v16, S, 0, crc); 4615 4616 eor(v0, T16B, v0, v16); 4617 sub(len, len, 64); 4618 4619 BIND(L_fold); 4620 pmull(v22, T8H, v0, v5, T8B); 4621 pmull(v20, T8H, v0, v7, T8B); 4622 pmull(v23, T8H, v0, v4, T8B); 4623 pmull(v21, T8H, v0, v6, T8B); 4624 4625 pmull2(v18, T8H, v0, v5, T16B); 4626 pmull2(v16, T8H, v0, v7, T16B); 4627 pmull2(v19, T8H, v0, v4, T16B); 4628 pmull2(v17, T8H, v0, v6, T16B); 4629 4630 uzp1(v24, T8H, v20, v22); 4631 uzp2(v25, T8H, v20, v22); 4632 eor(v20, T16B, v24, v25); 4633 4634 uzp1(v26, T8H, v16, v18); 4635 uzp2(v27, T8H, v16, v18); 4636 eor(v16, T16B, v26, v27); 4637 4638 ushll2(v22, T4S, v20, T8H, 8); 4639 ushll(v20, T4S, v20, T4H, 8); 4640 4641 ushll2(v18, T4S, v16, T8H, 8); 4642 ushll(v16, T4S, v16, T4H, 8); 4643 4644 eor(v22, T16B, v23, v22); 4645 eor(v18, T16B, v19, v18); 4646 eor(v20, T16B, v21, v20); 4647 eor(v16, T16B, v17, v16); 4648 4649 uzp1(v17, T2D, v16, v20); 4650 uzp2(v21, T2D, v16, v20); 4651 eor(v17, T16B, v17, v21); 4652 4653 ushll2(v20, T2D, v17, T4S, 16); 4654 ushll(v16, T2D, v17, T2S, 16); 4655 4656 eor(v20, T16B, v20, v22); 4657 eor(v16, T16B, v16, v18); 4658 4659 uzp1(v17, T2D, v20, v16); 4660 uzp2(v21, T2D, v20, v16); 4661 eor(v28, T16B, v17, v21); 4662 4663 pmull(v22, T8H, v1, v5, T8B); 4664 pmull(v20, T8H, v1, v7, T8B); 4665 pmull(v23, T8H, v1, v4, T8B); 4666 pmull(v21, T8H, v1, v6, T8B); 4667 4668 pmull2(v18, T8H, v1, v5, T16B); 4669 pmull2(v16, T8H, v1, v7, T16B); 4670 pmull2(v19, T8H, v1, v4, T16B); 4671 pmull2(v17, T8H, v1, v6, T16B); 4672 4673 ld1(v0, v1, T2D, post(buf, 32)); 4674 4675 uzp1(v24, T8H, v20, v22); 4676 uzp2(v25, T8H, v20, v22); 4677 eor(v20, T16B, v24, v25); 4678 4679 uzp1(v26, T8H, v16, v18); 4680 uzp2(v27, T8H, v16, v18); 4681 eor(v16, T16B, v26, v27); 4682 4683 ushll2(v22, T4S, v20, T8H, 8); 4684 ushll(v20, T4S, v20, T4H, 8); 4685 4686 ushll2(v18, T4S, v16, T8H, 8); 4687 ushll(v16, T4S, v16, T4H, 8); 4688 4689 eor(v22, T16B, v23, v22); 4690 eor(v18, T16B, v19, v18); 4691 eor(v20, T16B, v21, v20); 4692 eor(v16, T16B, v17, v16); 4693 4694 uzp1(v17, T2D, v16, v20); 4695 uzp2(v21, T2D, v16, v20); 4696 eor(v16, T16B, v17, v21); 4697 4698 ushll2(v20, T2D, v16, T4S, 16); 4699 ushll(v16, T2D, v16, T2S, 16); 4700 4701 eor(v20, T16B, v22, v20); 4702 eor(v16, T16B, v16, v18); 4703 4704 uzp1(v17, T2D, v20, v16); 4705 uzp2(v21, T2D, v20, v16); 4706 eor(v20, T16B, v17, v21); 4707 4708 shl(v16, T2D, v28, 1); 4709 shl(v17, T2D, v20, 1); 4710 4711 eor(v0, T16B, v0, v16); 4712 eor(v1, T16B, v1, v17); 4713 4714 subs(len, len, 32); 4715 br(Assembler::GE, L_fold); 4716 4717 mov(crc, 0); 4718 mov(tmp, v0, D, 0); 4719 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, false); 4720 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, true); 4721 mov(tmp, v0, D, 1); 4722 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, false); 4723 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, true); 4724 mov(tmp, v1, D, 0); 4725 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, false); 4726 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, true); 4727 mov(tmp, v1, D, 1); 4728 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, false); 4729 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, true); 4730 4731 add(len, len, 32); 4732 } // Neon code end 4733 4734 BIND(L_by16); 4735 subs(len, len, 16); 4736 br(Assembler::GE, L_by16_loop); 4737 adds(len, len, 16-4); 4738 br(Assembler::GE, L_by4_loop); 4739 adds(len, len, 4); 4740 br(Assembler::GT, L_by1_loop); 4741 b(L_exit); 4742 4743 BIND(L_by4_loop); 4744 ldrw(tmp, Address(post(buf, 4))); 4745 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3); 4746 subs(len, len, 4); 4747 br(Assembler::GE, L_by4_loop); 4748 adds(len, len, 4); 4749 br(Assembler::LE, L_exit); 4750 BIND(L_by1_loop); 4751 subs(len, len, 1); 4752 ldrb(tmp, Address(post(buf, 1))); 4753 update_byte_crc32(crc, tmp, table0); 4754 br(Assembler::GT, L_by1_loop); 4755 b(L_exit); 4756 4757 align(CodeEntryAlignment); 4758 BIND(L_by16_loop); 4759 subs(len, len, 16); 4760 ldp(tmp, tmp3, Address(post(buf, 16))); 4761 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, false); 4762 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, true); 4763 update_word_crc32(crc, tmp3, tmp2, table0, table1, table2, table3, false); 4764 update_word_crc32(crc, tmp3, tmp2, table0, table1, table2, table3, true); 4765 br(Assembler::GE, L_by16_loop); 4766 adds(len, len, 16-4); 4767 br(Assembler::GE, L_by4_loop); 4768 adds(len, len, 4); 4769 br(Assembler::GT, L_by1_loop); 4770 BIND(L_exit); 4771 mvnw(crc, crc); 4772 } 4773 4774 void MacroAssembler::kernel_crc32c_using_crypto_pmull(Register crc, Register buf, 4775 Register len, Register tmp0, Register tmp1, Register tmp2, Register tmp3) { 4776 Label CRC_by4_loop, CRC_by1_loop, CRC_less128, CRC_by128_pre, CRC_by32_loop, CRC_less32, L_exit; 4777 assert_different_registers(crc, buf, len, tmp0, tmp1, tmp2); 4778 4779 subs(tmp0, len, 384); 4780 br(Assembler::GE, CRC_by128_pre); 4781 BIND(CRC_less128); 4782 subs(len, len, 32); 4783 br(Assembler::GE, CRC_by32_loop); 4784 BIND(CRC_less32); 4785 adds(len, len, 32 - 4); 4786 br(Assembler::GE, CRC_by4_loop); 4787 adds(len, len, 4); 4788 br(Assembler::GT, CRC_by1_loop); 4789 b(L_exit); 4790 4791 BIND(CRC_by32_loop); 4792 ldp(tmp0, tmp1, Address(buf)); 4793 crc32cx(crc, crc, tmp0); 4794 ldr(tmp2, Address(buf, 16)); 4795 crc32cx(crc, crc, tmp1); 4796 ldr(tmp3, Address(buf, 24)); 4797 crc32cx(crc, crc, tmp2); 4798 add(buf, buf, 32); 4799 subs(len, len, 32); 4800 crc32cx(crc, crc, tmp3); 4801 br(Assembler::GE, CRC_by32_loop); 4802 cmn(len, (u1)32); 4803 br(Assembler::NE, CRC_less32); 4804 b(L_exit); 4805 4806 BIND(CRC_by4_loop); 4807 ldrw(tmp0, Address(post(buf, 4))); 4808 subs(len, len, 4); 4809 crc32cw(crc, crc, tmp0); 4810 br(Assembler::GE, CRC_by4_loop); 4811 adds(len, len, 4); 4812 br(Assembler::LE, L_exit); 4813 BIND(CRC_by1_loop); 4814 ldrb(tmp0, Address(post(buf, 1))); 4815 subs(len, len, 1); 4816 crc32cb(crc, crc, tmp0); 4817 br(Assembler::GT, CRC_by1_loop); 4818 b(L_exit); 4819 4820 BIND(CRC_by128_pre); 4821 kernel_crc32_common_fold_using_crypto_pmull(crc, buf, len, tmp0, tmp1, tmp2, 4822 4*256*sizeof(juint) + 8*sizeof(juint) + 0x50); 4823 mov(crc, 0); 4824 crc32cx(crc, crc, tmp0); 4825 crc32cx(crc, crc, tmp1); 4826 4827 cbnz(len, CRC_less128); 4828 4829 BIND(L_exit); 4830 } 4831 4832 void MacroAssembler::kernel_crc32c_using_crc32c(Register crc, Register buf, 4833 Register len, Register tmp0, Register tmp1, Register tmp2, 4834 Register tmp3) { 4835 Label CRC_by64_loop, CRC_by4_loop, CRC_by1_loop, CRC_less64, CRC_by64_pre, CRC_by32_loop, CRC_less32, L_exit; 4836 assert_different_registers(crc, buf, len, tmp0, tmp1, tmp2, tmp3); 4837 4838 subs(len, len, 128); 4839 br(Assembler::GE, CRC_by64_pre); 4840 BIND(CRC_less64); 4841 adds(len, len, 128-32); 4842 br(Assembler::GE, CRC_by32_loop); 4843 BIND(CRC_less32); 4844 adds(len, len, 32-4); 4845 br(Assembler::GE, CRC_by4_loop); 4846 adds(len, len, 4); 4847 br(Assembler::GT, CRC_by1_loop); 4848 b(L_exit); 4849 4850 BIND(CRC_by32_loop); 4851 ldp(tmp0, tmp1, Address(post(buf, 16))); 4852 subs(len, len, 32); 4853 crc32cx(crc, crc, tmp0); 4854 ldr(tmp2, Address(post(buf, 8))); 4855 crc32cx(crc, crc, tmp1); 4856 ldr(tmp3, Address(post(buf, 8))); 4857 crc32cx(crc, crc, tmp2); 4858 crc32cx(crc, crc, tmp3); 4859 br(Assembler::GE, CRC_by32_loop); 4860 cmn(len, (u1)32); 4861 br(Assembler::NE, CRC_less32); 4862 b(L_exit); 4863 4864 BIND(CRC_by4_loop); 4865 ldrw(tmp0, Address(post(buf, 4))); 4866 subs(len, len, 4); 4867 crc32cw(crc, crc, tmp0); 4868 br(Assembler::GE, CRC_by4_loop); 4869 adds(len, len, 4); 4870 br(Assembler::LE, L_exit); 4871 BIND(CRC_by1_loop); 4872 ldrb(tmp0, Address(post(buf, 1))); 4873 subs(len, len, 1); 4874 crc32cb(crc, crc, tmp0); 4875 br(Assembler::GT, CRC_by1_loop); 4876 b(L_exit); 4877 4878 BIND(CRC_by64_pre); 4879 sub(buf, buf, 8); 4880 ldp(tmp0, tmp1, Address(buf, 8)); 4881 crc32cx(crc, crc, tmp0); 4882 ldr(tmp2, Address(buf, 24)); 4883 crc32cx(crc, crc, tmp1); 4884 ldr(tmp3, Address(buf, 32)); 4885 crc32cx(crc, crc, tmp2); 4886 ldr(tmp0, Address(buf, 40)); 4887 crc32cx(crc, crc, tmp3); 4888 ldr(tmp1, Address(buf, 48)); 4889 crc32cx(crc, crc, tmp0); 4890 ldr(tmp2, Address(buf, 56)); 4891 crc32cx(crc, crc, tmp1); 4892 ldr(tmp3, Address(pre(buf, 64))); 4893 4894 b(CRC_by64_loop); 4895 4896 align(CodeEntryAlignment); 4897 BIND(CRC_by64_loop); 4898 subs(len, len, 64); 4899 crc32cx(crc, crc, tmp2); 4900 ldr(tmp0, Address(buf, 8)); 4901 crc32cx(crc, crc, tmp3); 4902 ldr(tmp1, Address(buf, 16)); 4903 crc32cx(crc, crc, tmp0); 4904 ldr(tmp2, Address(buf, 24)); 4905 crc32cx(crc, crc, tmp1); 4906 ldr(tmp3, Address(buf, 32)); 4907 crc32cx(crc, crc, tmp2); 4908 ldr(tmp0, Address(buf, 40)); 4909 crc32cx(crc, crc, tmp3); 4910 ldr(tmp1, Address(buf, 48)); 4911 crc32cx(crc, crc, tmp0); 4912 ldr(tmp2, Address(buf, 56)); 4913 crc32cx(crc, crc, tmp1); 4914 ldr(tmp3, Address(pre(buf, 64))); 4915 br(Assembler::GE, CRC_by64_loop); 4916 4917 // post-loop 4918 crc32cx(crc, crc, tmp2); 4919 crc32cx(crc, crc, tmp3); 4920 4921 sub(len, len, 64); 4922 add(buf, buf, 8); 4923 cmn(len, (u1)128); 4924 br(Assembler::NE, CRC_less64); 4925 BIND(L_exit); 4926 } 4927 4928 /** 4929 * @param crc register containing existing CRC (32-bit) 4930 * @param buf register pointing to input byte buffer (byte*) 4931 * @param len register containing number of bytes 4932 * @param table register that will contain address of CRC table 4933 * @param tmp scratch register 4934 */ 4935 void MacroAssembler::kernel_crc32c(Register crc, Register buf, Register len, 4936 Register table0, Register table1, Register table2, Register table3, 4937 Register tmp, Register tmp2, Register tmp3) { 4938 if (UseCryptoPmullForCRC32) { 4939 kernel_crc32c_using_crypto_pmull(crc, buf, len, table0, table1, table2, table3); 4940 } else { 4941 kernel_crc32c_using_crc32c(crc, buf, len, table0, table1, table2, table3); 4942 } 4943 } 4944 4945 void MacroAssembler::kernel_crc32_common_fold_using_crypto_pmull(Register crc, Register buf, 4946 Register len, Register tmp0, Register tmp1, Register tmp2, size_t table_offset) { 4947 Label CRC_by128_loop; 4948 assert_different_registers(crc, buf, len, tmp0, tmp1, tmp2); 4949 4950 sub(len, len, 256); 4951 Register table = tmp0; 4952 { 4953 uint64_t offset; 4954 adrp(table, ExternalAddress(StubRoutines::crc_table_addr()), offset); 4955 add(table, table, offset); 4956 } 4957 add(table, table, table_offset); 4958 4959 // Registers v0..v7 are used as data registers. 4960 // Registers v16..v31 are used as tmp registers. 4961 sub(buf, buf, 0x10); 4962 ldrq(v0, Address(buf, 0x10)); 4963 ldrq(v1, Address(buf, 0x20)); 4964 ldrq(v2, Address(buf, 0x30)); 4965 ldrq(v3, Address(buf, 0x40)); 4966 ldrq(v4, Address(buf, 0x50)); 4967 ldrq(v5, Address(buf, 0x60)); 4968 ldrq(v6, Address(buf, 0x70)); 4969 ldrq(v7, Address(pre(buf, 0x80))); 4970 4971 movi(v31, T4S, 0); 4972 mov(v31, S, 0, crc); 4973 eor(v0, T16B, v0, v31); 4974 4975 // Register v16 contains constants from the crc table. 4976 ldrq(v16, Address(table)); 4977 b(CRC_by128_loop); 4978 4979 align(OptoLoopAlignment); 4980 BIND(CRC_by128_loop); 4981 pmull (v17, T1Q, v0, v16, T1D); 4982 pmull2(v18, T1Q, v0, v16, T2D); 4983 ldrq(v0, Address(buf, 0x10)); 4984 eor3(v0, T16B, v17, v18, v0); 4985 4986 pmull (v19, T1Q, v1, v16, T1D); 4987 pmull2(v20, T1Q, v1, v16, T2D); 4988 ldrq(v1, Address(buf, 0x20)); 4989 eor3(v1, T16B, v19, v20, v1); 4990 4991 pmull (v21, T1Q, v2, v16, T1D); 4992 pmull2(v22, T1Q, v2, v16, T2D); 4993 ldrq(v2, Address(buf, 0x30)); 4994 eor3(v2, T16B, v21, v22, v2); 4995 4996 pmull (v23, T1Q, v3, v16, T1D); 4997 pmull2(v24, T1Q, v3, v16, T2D); 4998 ldrq(v3, Address(buf, 0x40)); 4999 eor3(v3, T16B, v23, v24, v3); 5000 5001 pmull (v25, T1Q, v4, v16, T1D); 5002 pmull2(v26, T1Q, v4, v16, T2D); 5003 ldrq(v4, Address(buf, 0x50)); 5004 eor3(v4, T16B, v25, v26, v4); 5005 5006 pmull (v27, T1Q, v5, v16, T1D); 5007 pmull2(v28, T1Q, v5, v16, T2D); 5008 ldrq(v5, Address(buf, 0x60)); 5009 eor3(v5, T16B, v27, v28, v5); 5010 5011 pmull (v29, T1Q, v6, v16, T1D); 5012 pmull2(v30, T1Q, v6, v16, T2D); 5013 ldrq(v6, Address(buf, 0x70)); 5014 eor3(v6, T16B, v29, v30, v6); 5015 5016 // Reuse registers v23, v24. 5017 // Using them won't block the first instruction of the next iteration. 5018 pmull (v23, T1Q, v7, v16, T1D); 5019 pmull2(v24, T1Q, v7, v16, T2D); 5020 ldrq(v7, Address(pre(buf, 0x80))); 5021 eor3(v7, T16B, v23, v24, v7); 5022 5023 subs(len, len, 0x80); 5024 br(Assembler::GE, CRC_by128_loop); 5025 5026 // fold into 512 bits 5027 // Use v31 for constants because v16 can be still in use. 5028 ldrq(v31, Address(table, 0x10)); 5029 5030 pmull (v17, T1Q, v0, v31, T1D); 5031 pmull2(v18, T1Q, v0, v31, T2D); 5032 eor3(v0, T16B, v17, v18, v4); 5033 5034 pmull (v19, T1Q, v1, v31, T1D); 5035 pmull2(v20, T1Q, v1, v31, T2D); 5036 eor3(v1, T16B, v19, v20, v5); 5037 5038 pmull (v21, T1Q, v2, v31, T1D); 5039 pmull2(v22, T1Q, v2, v31, T2D); 5040 eor3(v2, T16B, v21, v22, v6); 5041 5042 pmull (v23, T1Q, v3, v31, T1D); 5043 pmull2(v24, T1Q, v3, v31, T2D); 5044 eor3(v3, T16B, v23, v24, v7); 5045 5046 // fold into 128 bits 5047 // Use v17 for constants because v31 can be still in use. 5048 ldrq(v17, Address(table, 0x20)); 5049 pmull (v25, T1Q, v0, v17, T1D); 5050 pmull2(v26, T1Q, v0, v17, T2D); 5051 eor3(v3, T16B, v3, v25, v26); 5052 5053 // Use v18 for constants because v17 can be still in use. 5054 ldrq(v18, Address(table, 0x30)); 5055 pmull (v27, T1Q, v1, v18, T1D); 5056 pmull2(v28, T1Q, v1, v18, T2D); 5057 eor3(v3, T16B, v3, v27, v28); 5058 5059 // Use v19 for constants because v18 can be still in use. 5060 ldrq(v19, Address(table, 0x40)); 5061 pmull (v29, T1Q, v2, v19, T1D); 5062 pmull2(v30, T1Q, v2, v19, T2D); 5063 eor3(v0, T16B, v3, v29, v30); 5064 5065 add(len, len, 0x80); 5066 add(buf, buf, 0x10); 5067 5068 mov(tmp0, v0, D, 0); 5069 mov(tmp1, v0, D, 1); 5070 } 5071 5072 void MacroAssembler::addptr(const Address &dst, int32_t src) { 5073 Address adr; 5074 switch(dst.getMode()) { 5075 case Address::base_plus_offset: 5076 // This is the expected mode, although we allow all the other 5077 // forms below. 5078 adr = form_address(rscratch2, dst.base(), dst.offset(), LogBytesPerWord); 5079 break; 5080 default: 5081 lea(rscratch2, dst); 5082 adr = Address(rscratch2); 5083 break; 5084 } 5085 ldr(rscratch1, adr); 5086 add(rscratch1, rscratch1, src); 5087 str(rscratch1, adr); 5088 } 5089 5090 void MacroAssembler::cmpptr(Register src1, Address src2) { 5091 uint64_t offset; 5092 adrp(rscratch1, src2, offset); 5093 ldr(rscratch1, Address(rscratch1, offset)); 5094 cmp(src1, rscratch1); 5095 } 5096 5097 void MacroAssembler::cmpoop(Register obj1, Register obj2) { 5098 cmp(obj1, obj2); 5099 } 5100 5101 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) { 5102 load_method_holder(rresult, rmethod); 5103 ldr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset())); 5104 } 5105 5106 void MacroAssembler::load_method_holder(Register holder, Register method) { 5107 ldr(holder, Address(method, Method::const_offset())); // ConstMethod* 5108 ldr(holder, Address(holder, ConstMethod::constants_offset())); // ConstantPool* 5109 ldr(holder, Address(holder, ConstantPool::pool_holder_offset())); // InstanceKlass* 5110 } 5111 5112 void MacroAssembler::load_metadata(Register dst, Register src) { 5113 if (UseCompactObjectHeaders) { 5114 load_narrow_klass_compact(dst, src); 5115 } else if (UseCompressedClassPointers) { 5116 ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes())); 5117 } else { 5118 ldr(dst, Address(src, oopDesc::klass_offset_in_bytes())); 5119 } 5120 } 5121 5122 // Loads the obj's Klass* into dst. 5123 // Preserves all registers (incl src, rscratch1 and rscratch2). 5124 // Input: 5125 // src - the oop we want to load the klass from. 5126 // dst - output narrow klass. 5127 void MacroAssembler::load_narrow_klass_compact(Register dst, Register src) { 5128 assert(UseCompactObjectHeaders, "expects UseCompactObjectHeaders"); 5129 ldr(dst, Address(src, oopDesc::mark_offset_in_bytes())); 5130 lsr(dst, dst, markWord::klass_shift); 5131 } 5132 5133 void MacroAssembler::load_klass(Register dst, Register src) { 5134 if (UseCompactObjectHeaders) { 5135 load_narrow_klass_compact(dst, src); 5136 decode_klass_not_null(dst); 5137 } else if (UseCompressedClassPointers) { 5138 ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes())); 5139 decode_klass_not_null(dst); 5140 } else { 5141 ldr(dst, Address(src, oopDesc::klass_offset_in_bytes())); 5142 } 5143 } 5144 5145 void MacroAssembler::restore_cpu_control_state_after_jni(Register tmp1, Register tmp2) { 5146 if (RestoreMXCSROnJNICalls) { 5147 Label OK; 5148 get_fpcr(tmp1); 5149 mov(tmp2, tmp1); 5150 // Set FPCR to the state we need. We do want Round to Nearest. We 5151 // don't want non-IEEE rounding modes or floating-point traps. 5152 bfi(tmp1, zr, 22, 4); // Clear DN, FZ, and Rmode 5153 bfi(tmp1, zr, 8, 5); // Clear exception-control bits (8-12) 5154 bfi(tmp1, zr, 0, 2); // Clear AH:FIZ 5155 eor(tmp2, tmp1, tmp2); 5156 cbz(tmp2, OK); // Only reset FPCR if it's wrong 5157 set_fpcr(tmp1); 5158 bind(OK); 5159 } 5160 } 5161 5162 // ((OopHandle)result).resolve(); 5163 void MacroAssembler::resolve_oop_handle(Register result, Register tmp1, Register tmp2) { 5164 // OopHandle::resolve is an indirection. 5165 access_load_at(T_OBJECT, IN_NATIVE, result, Address(result, 0), tmp1, tmp2); 5166 } 5167 5168 // ((WeakHandle)result).resolve(); 5169 void MacroAssembler::resolve_weak_handle(Register result, Register tmp1, Register tmp2) { 5170 assert_different_registers(result, tmp1, tmp2); 5171 Label resolved; 5172 5173 // A null weak handle resolves to null. 5174 cbz(result, resolved); 5175 5176 // Only 64 bit platforms support GCs that require a tmp register 5177 // WeakHandle::resolve is an indirection like jweak. 5178 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, 5179 result, Address(result), tmp1, tmp2); 5180 bind(resolved); 5181 } 5182 5183 void MacroAssembler::load_mirror(Register dst, Register method, Register tmp1, Register tmp2) { 5184 const int mirror_offset = in_bytes(Klass::java_mirror_offset()); 5185 ldr(dst, Address(rmethod, Method::const_offset())); 5186 ldr(dst, Address(dst, ConstMethod::constants_offset())); 5187 ldr(dst, Address(dst, ConstantPool::pool_holder_offset())); 5188 ldr(dst, Address(dst, mirror_offset)); 5189 resolve_oop_handle(dst, tmp1, tmp2); 5190 } 5191 5192 void MacroAssembler::cmp_klass(Register obj, Register klass, Register tmp) { 5193 assert_different_registers(obj, klass, tmp); 5194 if (UseCompressedClassPointers) { 5195 if (UseCompactObjectHeaders) { 5196 load_narrow_klass_compact(tmp, obj); 5197 } else { 5198 ldrw(tmp, Address(obj, oopDesc::klass_offset_in_bytes())); 5199 } 5200 if (CompressedKlassPointers::base() == nullptr) { 5201 cmp(klass, tmp, LSL, CompressedKlassPointers::shift()); 5202 return; 5203 } else if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0 5204 && CompressedKlassPointers::shift() == 0) { 5205 // Only the bottom 32 bits matter 5206 cmpw(klass, tmp); 5207 return; 5208 } 5209 decode_klass_not_null(tmp); 5210 } else { 5211 ldr(tmp, Address(obj, oopDesc::klass_offset_in_bytes())); 5212 } 5213 cmp(klass, tmp); 5214 } 5215 5216 void MacroAssembler::cmp_klasses_from_objects(Register obj1, Register obj2, Register tmp1, Register tmp2) { 5217 if (UseCompactObjectHeaders) { 5218 load_narrow_klass_compact(tmp1, obj1); 5219 load_narrow_klass_compact(tmp2, obj2); 5220 cmpw(tmp1, tmp2); 5221 } else if (UseCompressedClassPointers) { 5222 ldrw(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes())); 5223 ldrw(tmp2, Address(obj2, oopDesc::klass_offset_in_bytes())); 5224 cmpw(tmp1, tmp2); 5225 } else { 5226 ldr(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes())); 5227 ldr(tmp2, Address(obj2, oopDesc::klass_offset_in_bytes())); 5228 cmp(tmp1, tmp2); 5229 } 5230 } 5231 5232 void MacroAssembler::load_prototype_header(Register dst, Register src) { 5233 load_klass(dst, src); 5234 ldr(dst, Address(dst, Klass::prototype_header_offset())); 5235 } 5236 5237 void MacroAssembler::store_klass(Register dst, Register src) { 5238 // FIXME: Should this be a store release? concurrent gcs assumes 5239 // klass length is valid if klass field is not null. 5240 assert(!UseCompactObjectHeaders, "not with compact headers"); 5241 if (UseCompressedClassPointers) { 5242 encode_klass_not_null(src); 5243 strw(src, Address(dst, oopDesc::klass_offset_in_bytes())); 5244 } else { 5245 str(src, Address(dst, oopDesc::klass_offset_in_bytes())); 5246 } 5247 } 5248 5249 void MacroAssembler::store_klass_gap(Register dst, Register src) { 5250 assert(!UseCompactObjectHeaders, "not with compact headers"); 5251 if (UseCompressedClassPointers) { 5252 // Store to klass gap in destination 5253 strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes())); 5254 } 5255 } 5256 5257 // Algorithm must match CompressedOops::encode. 5258 void MacroAssembler::encode_heap_oop(Register d, Register s) { 5259 #ifdef ASSERT 5260 verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?"); 5261 #endif 5262 verify_oop_msg(s, "broken oop in encode_heap_oop"); 5263 if (CompressedOops::base() == nullptr) { 5264 if (CompressedOops::shift() != 0) { 5265 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5266 lsr(d, s, LogMinObjAlignmentInBytes); 5267 } else { 5268 mov(d, s); 5269 } 5270 } else { 5271 subs(d, s, rheapbase); 5272 csel(d, d, zr, Assembler::HS); 5273 lsr(d, d, LogMinObjAlignmentInBytes); 5274 5275 /* Old algorithm: is this any worse? 5276 Label nonnull; 5277 cbnz(r, nonnull); 5278 sub(r, r, rheapbase); 5279 bind(nonnull); 5280 lsr(r, r, LogMinObjAlignmentInBytes); 5281 */ 5282 } 5283 } 5284 5285 void MacroAssembler::encode_heap_oop_not_null(Register r) { 5286 #ifdef ASSERT 5287 verify_heapbase("MacroAssembler::encode_heap_oop_not_null: heap base corrupted?"); 5288 if (CheckCompressedOops) { 5289 Label ok; 5290 cbnz(r, ok); 5291 stop("null oop passed to encode_heap_oop_not_null"); 5292 bind(ok); 5293 } 5294 #endif 5295 verify_oop_msg(r, "broken oop in encode_heap_oop_not_null"); 5296 if (CompressedOops::base() != nullptr) { 5297 sub(r, r, rheapbase); 5298 } 5299 if (CompressedOops::shift() != 0) { 5300 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5301 lsr(r, r, LogMinObjAlignmentInBytes); 5302 } 5303 } 5304 5305 void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) { 5306 #ifdef ASSERT 5307 verify_heapbase("MacroAssembler::encode_heap_oop_not_null2: heap base corrupted?"); 5308 if (CheckCompressedOops) { 5309 Label ok; 5310 cbnz(src, ok); 5311 stop("null oop passed to encode_heap_oop_not_null2"); 5312 bind(ok); 5313 } 5314 #endif 5315 verify_oop_msg(src, "broken oop in encode_heap_oop_not_null2"); 5316 5317 Register data = src; 5318 if (CompressedOops::base() != nullptr) { 5319 sub(dst, src, rheapbase); 5320 data = dst; 5321 } 5322 if (CompressedOops::shift() != 0) { 5323 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5324 lsr(dst, data, LogMinObjAlignmentInBytes); 5325 data = dst; 5326 } 5327 if (data == src) 5328 mov(dst, src); 5329 } 5330 5331 void MacroAssembler::decode_heap_oop(Register d, Register s) { 5332 #ifdef ASSERT 5333 verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?"); 5334 #endif 5335 if (CompressedOops::base() == nullptr) { 5336 if (CompressedOops::shift() != 0) { 5337 lsl(d, s, CompressedOops::shift()); 5338 } else if (d != s) { 5339 mov(d, s); 5340 } 5341 } else { 5342 Label done; 5343 if (d != s) 5344 mov(d, s); 5345 cbz(s, done); 5346 add(d, rheapbase, s, Assembler::LSL, LogMinObjAlignmentInBytes); 5347 bind(done); 5348 } 5349 verify_oop_msg(d, "broken oop in decode_heap_oop"); 5350 } 5351 5352 void MacroAssembler::decode_heap_oop_not_null(Register r) { 5353 assert (UseCompressedOops, "should only be used for compressed headers"); 5354 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5355 // Cannot assert, unverified entry point counts instructions (see .ad file) 5356 // vtableStubs also counts instructions in pd_code_size_limit. 5357 // Also do not verify_oop as this is called by verify_oop. 5358 if (CompressedOops::shift() != 0) { 5359 assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5360 if (CompressedOops::base() != nullptr) { 5361 add(r, rheapbase, r, Assembler::LSL, LogMinObjAlignmentInBytes); 5362 } else { 5363 add(r, zr, r, Assembler::LSL, LogMinObjAlignmentInBytes); 5364 } 5365 } else { 5366 assert (CompressedOops::base() == nullptr, "sanity"); 5367 } 5368 } 5369 5370 void MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) { 5371 assert (UseCompressedOops, "should only be used for compressed headers"); 5372 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5373 // Cannot assert, unverified entry point counts instructions (see .ad file) 5374 // vtableStubs also counts instructions in pd_code_size_limit. 5375 // Also do not verify_oop as this is called by verify_oop. 5376 if (CompressedOops::shift() != 0) { 5377 assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5378 if (CompressedOops::base() != nullptr) { 5379 add(dst, rheapbase, src, Assembler::LSL, LogMinObjAlignmentInBytes); 5380 } else { 5381 add(dst, zr, src, Assembler::LSL, LogMinObjAlignmentInBytes); 5382 } 5383 } else { 5384 assert (CompressedOops::base() == nullptr, "sanity"); 5385 if (dst != src) { 5386 mov(dst, src); 5387 } 5388 } 5389 } 5390 5391 MacroAssembler::KlassDecodeMode MacroAssembler::_klass_decode_mode(KlassDecodeNone); 5392 5393 MacroAssembler::KlassDecodeMode MacroAssembler::klass_decode_mode() { 5394 assert(Metaspace::initialized(), "metaspace not initialized yet"); 5395 assert(_klass_decode_mode != KlassDecodeNone, "should be initialized"); 5396 return _klass_decode_mode; 5397 } 5398 5399 MacroAssembler::KlassDecodeMode MacroAssembler::klass_decode_mode(address base, int shift, const size_t range) { 5400 assert(UseCompressedClassPointers, "not using compressed class pointers"); 5401 5402 // KlassDecodeMode shouldn't be set already. 5403 assert(_klass_decode_mode == KlassDecodeNone, "set once"); 5404 5405 if (base == nullptr) { 5406 return KlassDecodeZero; 5407 } 5408 5409 if (operand_valid_for_logical_immediate( 5410 /*is32*/false, (uint64_t)base)) { 5411 const uint64_t range_mask = right_n_bits(log2i_ceil(range)); 5412 if (((uint64_t)base & range_mask) == 0) { 5413 return KlassDecodeXor; 5414 } 5415 } 5416 5417 const uint64_t shifted_base = 5418 (uint64_t)base >> shift; 5419 if ((shifted_base & 0xffff0000ffffffff) == 0) { 5420 return KlassDecodeMovk; 5421 } 5422 5423 // No valid encoding. 5424 return KlassDecodeNone; 5425 } 5426 5427 // Check if one of the above decoding modes will work for given base, shift and range. 5428 bool MacroAssembler::check_klass_decode_mode(address base, int shift, const size_t range) { 5429 return klass_decode_mode(base, shift, range) != KlassDecodeNone; 5430 } 5431 5432 bool MacroAssembler::set_klass_decode_mode(address base, int shift, const size_t range) { 5433 _klass_decode_mode = klass_decode_mode(base, shift, range); 5434 return _klass_decode_mode != KlassDecodeNone; 5435 } 5436 5437 static Register pick_different_tmp(Register dst, Register src) { 5438 auto tmps = RegSet::of(r0, r1, r2) - RegSet::of(src, dst); 5439 return *tmps.begin(); 5440 } 5441 5442 void MacroAssembler::encode_klass_not_null_for_aot(Register dst, Register src) { 5443 // we have to load the klass base from the AOT constants area but 5444 // not the shift because it is not allowed to change 5445 int shift = CompressedKlassPointers::shift(); 5446 assert(shift >= 0 && shift <= CompressedKlassPointers::max_shift(), "unexpected compressed klass shift!"); 5447 if (dst != src) { 5448 // we can load the base into dst, subtract it formthe src and shift down 5449 lea(dst, ExternalAddress(CompressedKlassPointers::base_addr())); 5450 ldr(dst, dst); 5451 sub(dst, src, dst); 5452 lsr(dst, dst, shift); 5453 } else { 5454 // we need an extra register in order to load the coop base 5455 Register tmp = pick_different_tmp(dst, src); 5456 RegSet regs = RegSet::of(tmp); 5457 push(regs, sp); 5458 lea(tmp, ExternalAddress(CompressedKlassPointers::base_addr())); 5459 ldr(tmp, tmp); 5460 sub(dst, src, tmp); 5461 lsr(dst, dst, shift); 5462 pop(regs, sp); 5463 } 5464 } 5465 5466 void MacroAssembler::encode_klass_not_null(Register dst, Register src) { 5467 if (AOTCodeCache::is_on_for_dump()) { 5468 encode_klass_not_null_for_aot(dst, src); 5469 return; 5470 } 5471 5472 switch (klass_decode_mode()) { 5473 case KlassDecodeZero: 5474 if (CompressedKlassPointers::shift() != 0) { 5475 lsr(dst, src, CompressedKlassPointers::shift()); 5476 } else { 5477 if (dst != src) mov(dst, src); 5478 } 5479 break; 5480 5481 case KlassDecodeXor: 5482 if (CompressedKlassPointers::shift() != 0) { 5483 eor(dst, src, (uint64_t)CompressedKlassPointers::base()); 5484 lsr(dst, dst, CompressedKlassPointers::shift()); 5485 } else { 5486 eor(dst, src, (uint64_t)CompressedKlassPointers::base()); 5487 } 5488 break; 5489 5490 case KlassDecodeMovk: 5491 if (CompressedKlassPointers::shift() != 0) { 5492 ubfx(dst, src, CompressedKlassPointers::shift(), 32); 5493 } else { 5494 movw(dst, src); 5495 } 5496 break; 5497 5498 case KlassDecodeNone: 5499 ShouldNotReachHere(); 5500 break; 5501 } 5502 } 5503 5504 void MacroAssembler::encode_klass_not_null(Register r) { 5505 encode_klass_not_null(r, r); 5506 } 5507 5508 void MacroAssembler::decode_klass_not_null_for_aot(Register dst, Register src) { 5509 // we have to load the klass base from the AOT constants area but 5510 // not the shift because it is not allowed to change 5511 int shift = CompressedKlassPointers::shift(); 5512 assert(shift >= 0 && shift <= CompressedKlassPointers::max_shift(), "unexpected compressed klass shift!"); 5513 if (dst != src) { 5514 // we can load the base into dst then add the offset with a suitable shift 5515 lea(dst, ExternalAddress(CompressedKlassPointers::base_addr())); 5516 ldr(dst, dst); 5517 add(dst, dst, src, LSL, shift); 5518 } else { 5519 // we need an extra register in order to load the coop base 5520 Register tmp = pick_different_tmp(dst, src); 5521 RegSet regs = RegSet::of(tmp); 5522 push(regs, sp); 5523 lea(tmp, ExternalAddress(CompressedKlassPointers::base_addr())); 5524 ldr(tmp, tmp); 5525 add(dst, tmp, src, LSL, shift); 5526 pop(regs, sp); 5527 } 5528 } 5529 5530 void MacroAssembler::decode_klass_not_null(Register dst, Register src) { 5531 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 5532 5533 if (AOTCodeCache::is_on_for_dump()) { 5534 decode_klass_not_null_for_aot(dst, src); 5535 return; 5536 } 5537 5538 switch (klass_decode_mode()) { 5539 case KlassDecodeZero: 5540 if (CompressedKlassPointers::shift() != 0) { 5541 lsl(dst, src, CompressedKlassPointers::shift()); 5542 } else { 5543 if (dst != src) mov(dst, src); 5544 } 5545 break; 5546 5547 case KlassDecodeXor: 5548 if (CompressedKlassPointers::shift() != 0) { 5549 lsl(dst, src, CompressedKlassPointers::shift()); 5550 eor(dst, dst, (uint64_t)CompressedKlassPointers::base()); 5551 } else { 5552 eor(dst, src, (uint64_t)CompressedKlassPointers::base()); 5553 } 5554 break; 5555 5556 case KlassDecodeMovk: { 5557 const uint64_t shifted_base = 5558 (uint64_t)CompressedKlassPointers::base() >> CompressedKlassPointers::shift(); 5559 5560 if (dst != src) movw(dst, src); 5561 movk(dst, shifted_base >> 32, 32); 5562 5563 if (CompressedKlassPointers::shift() != 0) { 5564 lsl(dst, dst, CompressedKlassPointers::shift()); 5565 } 5566 5567 break; 5568 } 5569 5570 case KlassDecodeNone: 5571 ShouldNotReachHere(); 5572 break; 5573 } 5574 } 5575 5576 void MacroAssembler::decode_klass_not_null(Register r) { 5577 decode_klass_not_null(r, r); 5578 } 5579 5580 void MacroAssembler::set_narrow_oop(Register dst, jobject obj) { 5581 #ifdef ASSERT 5582 { 5583 ThreadInVMfromUnknown tiv; 5584 assert (UseCompressedOops, "should only be used for compressed oops"); 5585 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5586 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5587 assert(Universe::heap()->is_in(JNIHandles::resolve(obj)), "should be real oop"); 5588 } 5589 #endif 5590 int oop_index = oop_recorder()->find_index(obj); 5591 InstructionMark im(this); 5592 RelocationHolder rspec = oop_Relocation::spec(oop_index); 5593 code_section()->relocate(inst_mark(), rspec); 5594 movz(dst, 0xDEAD, 16); 5595 movk(dst, 0xBEEF); 5596 } 5597 5598 void MacroAssembler::set_narrow_klass(Register dst, Klass* k) { 5599 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 5600 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5601 int index = oop_recorder()->find_index(k); 5602 assert(! Universe::heap()->is_in(k), "should not be an oop"); 5603 5604 InstructionMark im(this); 5605 RelocationHolder rspec = metadata_Relocation::spec(index); 5606 code_section()->relocate(inst_mark(), rspec); 5607 narrowKlass nk = CompressedKlassPointers::encode(k); 5608 movz(dst, (nk >> 16), 16); 5609 movk(dst, nk & 0xffff); 5610 } 5611 5612 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, 5613 Register dst, Address src, 5614 Register tmp1, Register tmp2) { 5615 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); 5616 decorators = AccessInternal::decorator_fixup(decorators, type); 5617 bool as_raw = (decorators & AS_RAW) != 0; 5618 if (as_raw) { 5619 bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, tmp2); 5620 } else { 5621 bs->load_at(this, decorators, type, dst, src, tmp1, tmp2); 5622 } 5623 } 5624 5625 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, 5626 Address dst, Register val, 5627 Register tmp1, Register tmp2, Register tmp3) { 5628 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); 5629 decorators = AccessInternal::decorator_fixup(decorators, type); 5630 bool as_raw = (decorators & AS_RAW) != 0; 5631 if (as_raw) { 5632 bs->BarrierSetAssembler::store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3); 5633 } else { 5634 bs->store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3); 5635 } 5636 } 5637 5638 void MacroAssembler::flat_field_copy(DecoratorSet decorators, Register src, Register dst, 5639 Register inline_layout_info) { 5640 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); 5641 bs->flat_field_copy(this, decorators, src, dst, inline_layout_info); 5642 } 5643 5644 void MacroAssembler::payload_offset(Register inline_klass, Register offset) { 5645 ldr(offset, Address(inline_klass, InstanceKlass::adr_inlineklass_fixed_block_offset())); 5646 ldrw(offset, Address(offset, InlineKlass::payload_offset_offset())); 5647 } 5648 5649 void MacroAssembler::payload_address(Register oop, Register data, Register inline_klass) { 5650 // ((address) (void*) o) + vk->payload_offset(); 5651 Register offset = (data == oop) ? rscratch1 : data; 5652 payload_offset(inline_klass, offset); 5653 if (data == oop) { 5654 add(data, data, offset); 5655 } else { 5656 lea(data, Address(oop, offset)); 5657 } 5658 } 5659 5660 void MacroAssembler::data_for_value_array_index(Register array, Register array_klass, 5661 Register index, Register data) { 5662 assert_different_registers(array, array_klass, index); 5663 assert_different_registers(rscratch1, array, index); 5664 5665 // array->base() + (index << Klass::layout_helper_log2_element_size(lh)); 5666 ldrw(rscratch1, Address(array_klass, Klass::layout_helper_offset())); 5667 5668 // Klass::layout_helper_log2_element_size(lh) 5669 // (lh >> _lh_log2_element_size_shift) & _lh_log2_element_size_mask; 5670 lsr(rscratch1, rscratch1, Klass::_lh_log2_element_size_shift); 5671 andr(rscratch1, rscratch1, Klass::_lh_log2_element_size_mask); 5672 lslv(index, index, rscratch1); 5673 5674 add(data, array, index); 5675 add(data, data, arrayOopDesc::base_offset_in_bytes(T_FLAT_ELEMENT)); 5676 } 5677 5678 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1, 5679 Register tmp2, DecoratorSet decorators) { 5680 access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, tmp2); 5681 } 5682 5683 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1, 5684 Register tmp2, DecoratorSet decorators) { 5685 access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, tmp2); 5686 } 5687 5688 void MacroAssembler::store_heap_oop(Address dst, Register val, Register tmp1, 5689 Register tmp2, Register tmp3, DecoratorSet decorators) { 5690 access_store_at(T_OBJECT, IN_HEAP | decorators, dst, val, tmp1, tmp2, tmp3); 5691 } 5692 5693 // Used for storing nulls. 5694 void MacroAssembler::store_heap_oop_null(Address dst) { 5695 access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg, noreg); 5696 } 5697 5698 Address MacroAssembler::allocate_metadata_address(Metadata* obj) { 5699 assert(oop_recorder() != nullptr, "this assembler needs a Recorder"); 5700 int index = oop_recorder()->allocate_metadata_index(obj); 5701 RelocationHolder rspec = metadata_Relocation::spec(index); 5702 return Address((address)obj, rspec); 5703 } 5704 5705 // Move an oop into a register. 5706 void MacroAssembler::movoop(Register dst, jobject obj) { 5707 int oop_index; 5708 if (obj == nullptr) { 5709 oop_index = oop_recorder()->allocate_oop_index(obj); 5710 } else { 5711 #ifdef ASSERT 5712 { 5713 ThreadInVMfromUnknown tiv; 5714 assert(Universe::heap()->is_in(JNIHandles::resolve(obj)), "should be real oop"); 5715 } 5716 #endif 5717 oop_index = oop_recorder()->find_index(obj); 5718 } 5719 RelocationHolder rspec = oop_Relocation::spec(oop_index); 5720 5721 if (BarrierSet::barrier_set()->barrier_set_assembler()->supports_instruction_patching()) { 5722 mov(dst, Address((address)obj, rspec)); 5723 } else { 5724 address dummy = address(uintptr_t(pc()) & -wordSize); // A nearby aligned address 5725 ldr(dst, Address(dummy, rspec)); 5726 } 5727 } 5728 5729 // Move a metadata address into a register. 5730 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) { 5731 int oop_index; 5732 if (obj == nullptr) { 5733 oop_index = oop_recorder()->allocate_metadata_index(obj); 5734 } else { 5735 oop_index = oop_recorder()->find_index(obj); 5736 } 5737 RelocationHolder rspec = metadata_Relocation::spec(oop_index); 5738 mov(dst, Address((address)obj, rspec)); 5739 } 5740 5741 Address MacroAssembler::constant_oop_address(jobject obj) { 5742 #ifdef ASSERT 5743 { 5744 ThreadInVMfromUnknown tiv; 5745 assert(oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5746 assert(Universe::heap()->is_in(JNIHandles::resolve(obj)), "not an oop"); 5747 } 5748 #endif 5749 int oop_index = oop_recorder()->find_index(obj); 5750 return Address((address)obj, oop_Relocation::spec(oop_index)); 5751 } 5752 5753 // Object / value buffer allocation... 5754 void MacroAssembler::allocate_instance(Register klass, Register new_obj, 5755 Register t1, Register t2, 5756 bool clear_fields, Label& alloc_failed) 5757 { 5758 Label done, initialize_header, initialize_object, slow_case, slow_case_no_pop; 5759 Register layout_size = t1; 5760 assert(new_obj == r0, "needs to be r0"); 5761 assert_different_registers(klass, new_obj, t1, t2); 5762 5763 // get instance_size in InstanceKlass (scaled to a count of bytes) 5764 ldrw(layout_size, Address(klass, Klass::layout_helper_offset())); 5765 // test to see if it is malformed in some way 5766 tst(layout_size, Klass::_lh_instance_slow_path_bit); 5767 br(Assembler::NE, slow_case_no_pop); 5768 5769 // Allocate the instance: 5770 // If TLAB is enabled: 5771 // Try to allocate in the TLAB. 5772 // If fails, go to the slow path. 5773 // Initialize the allocation. 5774 // Exit. 5775 // 5776 // Go to slow path. 5777 5778 if (UseTLAB) { 5779 push(klass); 5780 tlab_allocate(new_obj, layout_size, 0, klass, t2, slow_case); 5781 if (ZeroTLAB || (!clear_fields)) { 5782 // the fields have been already cleared 5783 b(initialize_header); 5784 } else { 5785 // initialize both the header and fields 5786 b(initialize_object); 5787 } 5788 5789 if (clear_fields) { 5790 // The object is initialized before the header. If the object size is 5791 // zero, go directly to the header initialization. 5792 bind(initialize_object); 5793 int header_size = oopDesc::header_size() * HeapWordSize; 5794 assert(is_aligned(header_size, BytesPerLong), "oop header size must be 8-byte-aligned"); 5795 subs(layout_size, layout_size, header_size); 5796 br(Assembler::EQ, initialize_header); 5797 5798 // Initialize topmost object field, divide size by 8, check if odd and 5799 // test if zero. 5800 5801 #ifdef ASSERT 5802 // make sure instance_size was multiple of 8 5803 Label L; 5804 tst(layout_size, 7); 5805 br(Assembler::EQ, L); 5806 stop("object size is not multiple of 8 - adjust this code"); 5807 bind(L); 5808 // must be > 0, no extra check needed here 5809 #endif 5810 5811 lsr(layout_size, layout_size, LogBytesPerLong); 5812 5813 // initialize remaining object fields: instance_size was a multiple of 8 5814 { 5815 Label loop; 5816 Register base = t2; 5817 5818 bind(loop); 5819 add(rscratch1, new_obj, layout_size, Assembler::LSL, LogBytesPerLong); 5820 str(zr, Address(rscratch1, header_size - 1*oopSize)); 5821 subs(layout_size, layout_size, 1); 5822 br(Assembler::NE, loop); 5823 } 5824 } // clear_fields 5825 5826 // initialize object header only. 5827 bind(initialize_header); 5828 pop(klass); 5829 Register mark_word = t2; 5830 if (UseCompactObjectHeaders || EnableValhalla) { 5831 ldr(mark_word, Address(klass, Klass::prototype_header_offset())); 5832 str(mark_word, Address(new_obj, oopDesc::mark_offset_in_bytes())); 5833 } else { 5834 mov(mark_word, (intptr_t)markWord::prototype().value()); 5835 str(mark_word, Address(new_obj, oopDesc::mark_offset_in_bytes())); 5836 } 5837 if (!UseCompactObjectHeaders) { 5838 store_klass_gap(new_obj, zr); // zero klass gap for compressed oops 5839 mov(t2, klass); // preserve klass 5840 store_klass(new_obj, t2); // src klass reg is potentially compressed 5841 } 5842 b(done); 5843 } 5844 5845 if (UseTLAB) { 5846 bind(slow_case); 5847 pop(klass); 5848 } 5849 bind(slow_case_no_pop); 5850 b(alloc_failed); 5851 5852 bind(done); 5853 } 5854 5855 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes. 5856 void MacroAssembler::tlab_allocate(Register obj, 5857 Register var_size_in_bytes, 5858 int con_size_in_bytes, 5859 Register t1, 5860 Register t2, 5861 Label& slow_case) { 5862 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); 5863 bs->tlab_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case); 5864 } 5865 5866 void MacroAssembler::verify_tlab() { 5867 #ifdef ASSERT 5868 if (UseTLAB && VerifyOops) { 5869 Label next, ok; 5870 5871 stp(rscratch2, rscratch1, Address(pre(sp, -16))); 5872 5873 ldr(rscratch2, Address(rthread, in_bytes(JavaThread::tlab_top_offset()))); 5874 ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_start_offset()))); 5875 cmp(rscratch2, rscratch1); 5876 br(Assembler::HS, next); 5877 STOP("assert(top >= start)"); 5878 should_not_reach_here(); 5879 5880 bind(next); 5881 ldr(rscratch2, Address(rthread, in_bytes(JavaThread::tlab_end_offset()))); 5882 ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_top_offset()))); 5883 cmp(rscratch2, rscratch1); 5884 br(Assembler::HS, ok); 5885 STOP("assert(top <= end)"); 5886 should_not_reach_here(); 5887 5888 bind(ok); 5889 ldp(rscratch2, rscratch1, Address(post(sp, 16))); 5890 } 5891 #endif 5892 } 5893 5894 void MacroAssembler::get_inline_type_field_klass(Register holder_klass, Register index, Register inline_klass) { 5895 inline_layout_info(holder_klass, index, inline_klass); 5896 ldr(inline_klass, Address(inline_klass, InlineLayoutInfo::klass_offset())); 5897 } 5898 5899 void MacroAssembler::inline_layout_info(Register holder_klass, Register index, Register layout_info) { 5900 assert_different_registers(holder_klass, index, layout_info); 5901 InlineLayoutInfo array[2]; 5902 int size = (char*)&array[1] - (char*)&array[0]; // computing size of array elements 5903 if (is_power_of_2(size)) { 5904 lsl(index, index, log2i_exact(size)); // Scale index by power of 2 5905 } else { 5906 mov(layout_info, size); 5907 mul(index, index, layout_info); // Scale the index to be the entry index * array_element_size 5908 } 5909 ldr(layout_info, Address(holder_klass, InstanceKlass::inline_layout_info_array_offset())); 5910 add(layout_info, layout_info, Array<InlineLayoutInfo>::base_offset_in_bytes()); 5911 lea(layout_info, Address(layout_info, index)); 5912 } 5913 5914 // Writes to stack successive pages until offset reached to check for 5915 // stack overflow + shadow pages. This clobbers tmp. 5916 void MacroAssembler::bang_stack_size(Register size, Register tmp) { 5917 assert_different_registers(tmp, size, rscratch1); 5918 mov(tmp, sp); 5919 // Bang stack for total size given plus shadow page size. 5920 // Bang one page at a time because large size can bang beyond yellow and 5921 // red zones. 5922 Label loop; 5923 mov(rscratch1, (int)os::vm_page_size()); 5924 bind(loop); 5925 lea(tmp, Address(tmp, -(int)os::vm_page_size())); 5926 subsw(size, size, rscratch1); 5927 str(size, Address(tmp)); 5928 br(Assembler::GT, loop); 5929 5930 // Bang down shadow pages too. 5931 // At this point, (tmp-0) is the last address touched, so don't 5932 // touch it again. (It was touched as (tmp-pagesize) but then tmp 5933 // was post-decremented.) Skip this address by starting at i=1, and 5934 // touch a few more pages below. N.B. It is important to touch all 5935 // the way down to and including i=StackShadowPages. 5936 for (int i = 0; i < (int)(StackOverflow::stack_shadow_zone_size() / (int)os::vm_page_size()) - 1; i++) { 5937 // this could be any sized move but this is can be a debugging crumb 5938 // so the bigger the better. 5939 lea(tmp, Address(tmp, -(int)os::vm_page_size())); 5940 str(size, Address(tmp)); 5941 } 5942 } 5943 5944 // Move the address of the polling page into dest. 5945 void MacroAssembler::get_polling_page(Register dest, relocInfo::relocType rtype) { 5946 ldr(dest, Address(rthread, JavaThread::polling_page_offset())); 5947 } 5948 5949 // Read the polling page. The address of the polling page must 5950 // already be in r. 5951 address MacroAssembler::read_polling_page(Register r, relocInfo::relocType rtype) { 5952 address mark; 5953 { 5954 InstructionMark im(this); 5955 code_section()->relocate(inst_mark(), rtype); 5956 ldrw(zr, Address(r, 0)); 5957 mark = inst_mark(); 5958 } 5959 verify_cross_modify_fence_not_required(); 5960 return mark; 5961 } 5962 5963 void MacroAssembler::adrp(Register reg1, const Address &dest, uint64_t &byte_offset) { 5964 relocInfo::relocType rtype = dest.rspec().reloc()->type(); 5965 uint64_t low_page = (uint64_t)CodeCache::low_bound() >> 12; 5966 uint64_t high_page = (uint64_t)(CodeCache::high_bound()-1) >> 12; 5967 uint64_t dest_page = (uint64_t)dest.target() >> 12; 5968 int64_t offset_low = dest_page - low_page; 5969 int64_t offset_high = dest_page - high_page; 5970 5971 assert(is_valid_AArch64_address(dest.target()), "bad address"); 5972 assert(dest.getMode() == Address::literal, "ADRP must be applied to a literal address"); 5973 5974 InstructionMark im(this); 5975 code_section()->relocate(inst_mark(), dest.rspec()); 5976 // 8143067: Ensure that the adrp can reach the dest from anywhere within 5977 // the code cache so that if it is relocated we know it will still reach 5978 if (offset_high >= -(1<<20) && offset_low < (1<<20)) { 5979 _adrp(reg1, dest.target()); 5980 } else { 5981 uint64_t target = (uint64_t)dest.target(); 5982 uint64_t adrp_target 5983 = (target & 0xffffffffULL) | ((uint64_t)pc() & 0xffff00000000ULL); 5984 5985 _adrp(reg1, (address)adrp_target); 5986 movk(reg1, target >> 32, 32); 5987 } 5988 byte_offset = (uint64_t)dest.target() & 0xfff; 5989 } 5990 5991 void MacroAssembler::load_byte_map_base(Register reg) { 5992 CardTable::CardValue* byte_map_base = 5993 ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base(); 5994 5995 // Strictly speaking the byte_map_base isn't an address at all, and it might 5996 // even be negative. It is thus materialised as a constant. 5997 mov(reg, (uint64_t)byte_map_base); 5998 } 5999 6000 void MacroAssembler::build_frame(int framesize) { 6001 assert(framesize >= 2 * wordSize, "framesize must include space for FP/LR"); 6002 assert(framesize % (2*wordSize) == 0, "must preserve 2*wordSize alignment"); 6003 protect_return_address(); 6004 if (framesize < ((1 << 9) + 2 * wordSize)) { 6005 sub(sp, sp, framesize); 6006 stp(rfp, lr, Address(sp, framesize - 2 * wordSize)); 6007 if (PreserveFramePointer) add(rfp, sp, framesize - 2 * wordSize); 6008 } else { 6009 stp(rfp, lr, Address(pre(sp, -2 * wordSize))); 6010 if (PreserveFramePointer) mov(rfp, sp); 6011 if (framesize < ((1 << 12) + 2 * wordSize)) 6012 sub(sp, sp, framesize - 2 * wordSize); 6013 else { 6014 mov(rscratch1, framesize - 2 * wordSize); 6015 sub(sp, sp, rscratch1); 6016 } 6017 } 6018 verify_cross_modify_fence_not_required(); 6019 } 6020 6021 void MacroAssembler::remove_frame(int framesize) { 6022 assert(framesize >= 2 * wordSize, "framesize must include space for FP/LR"); 6023 assert(framesize % (2*wordSize) == 0, "must preserve 2*wordSize alignment"); 6024 if (framesize < ((1 << 9) + 2 * wordSize)) { 6025 ldp(rfp, lr, Address(sp, framesize - 2 * wordSize)); 6026 add(sp, sp, framesize); 6027 } else { 6028 if (framesize < ((1 << 12) + 2 * wordSize)) 6029 add(sp, sp, framesize - 2 * wordSize); 6030 else { 6031 mov(rscratch1, framesize - 2 * wordSize); 6032 add(sp, sp, rscratch1); 6033 } 6034 ldp(rfp, lr, Address(post(sp, 2 * wordSize))); 6035 } 6036 authenticate_return_address(); 6037 } 6038 6039 void MacroAssembler::remove_frame(int initial_framesize, bool needs_stack_repair) { 6040 if (needs_stack_repair) { 6041 // Remove the extension of the caller's frame used for inline type unpacking 6042 // 6043 // Right now the stack looks like this: 6044 // 6045 // | Arguments from caller | 6046 // |---------------------------| <-- caller's SP 6047 // | Saved LR #1 | 6048 // | Saved FP #1 | 6049 // |---------------------------| 6050 // | Extension space for | 6051 // | inline arg (un)packing | 6052 // |---------------------------| <-- start of this method's frame 6053 // | Saved LR #2 | 6054 // | Saved FP #2 | 6055 // |---------------------------| <-- FP 6056 // | sp_inc | 6057 // | method locals | 6058 // |---------------------------| <-- SP 6059 // 6060 // There are two copies of FP and LR on the stack. They will be identical at 6061 // first, but that can change. 6062 // If the caller has been deoptimized, LR #1 will be patched to point at the 6063 // deopt blob, and LR #2 will still point into the old method. 6064 // If the saved FP (x29) was not used as the frame pointer, but to store an 6065 // oop, the GC will be aware only of FP #2 as the spilled location of x29 and 6066 // will fix only this one. 6067 // 6068 // When restoring, one must then load FP #2 into x29, and LR #1 into x30, 6069 // while keeping in mind that from the scalarized entry point, there will be 6070 // only one copy of each. 6071 // 6072 // The sp_inc stack slot holds the total size of the frame including the 6073 // extension space minus two words for the saved FP and LR. That is how to 6074 // find LR #1. FP #2 is always located just after sp_inc. 6075 6076 int sp_inc_offset = initial_framesize - 3 * wordSize; // Immediately below saved LR and FP 6077 6078 ldr(rscratch1, Address(sp, sp_inc_offset)); 6079 ldr(rfp, Address(sp, sp_inc_offset + wordSize)); 6080 add(sp, sp, rscratch1); 6081 ldr(lr, Address(sp, wordSize)); 6082 add(sp, sp, 2 * wordSize); 6083 } else { 6084 remove_frame(initial_framesize); 6085 } 6086 } 6087 6088 void MacroAssembler::save_stack_increment(int sp_inc, int frame_size) { 6089 int real_frame_size = frame_size + sp_inc; 6090 assert(sp_inc == 0 || sp_inc > 2*wordSize, "invalid sp_inc value"); 6091 assert(real_frame_size >= 2*wordSize, "frame size must include FP/LR space"); 6092 assert((real_frame_size & (StackAlignmentInBytes-1)) == 0, "frame size not aligned"); 6093 6094 int sp_inc_offset = frame_size - 3 * wordSize; // Immediately below saved LR and FP 6095 6096 // Subtract two words for the saved FP and LR as these will be popped 6097 // separately. See remove_frame above. 6098 mov(rscratch1, real_frame_size - 2*wordSize); 6099 str(rscratch1, Address(sp, sp_inc_offset)); 6100 } 6101 6102 // This method counts leading positive bytes (highest bit not set) in provided byte array 6103 address MacroAssembler::count_positives(Register ary1, Register len, Register result) { 6104 // Simple and most common case of aligned small array which is not at the 6105 // end of memory page is placed here. All other cases are in stub. 6106 Label LOOP, END, STUB, STUB_LONG, SET_RESULT, DONE; 6107 const uint64_t UPPER_BIT_MASK=0x8080808080808080; 6108 assert_different_registers(ary1, len, result); 6109 6110 mov(result, len); 6111 cmpw(len, 0); 6112 br(LE, DONE); 6113 cmpw(len, 4 * wordSize); 6114 br(GE, STUB_LONG); // size > 32 then go to stub 6115 6116 int shift = 64 - exact_log2(os::vm_page_size()); 6117 lsl(rscratch1, ary1, shift); 6118 mov(rscratch2, (size_t)(4 * wordSize) << shift); 6119 adds(rscratch2, rscratch1, rscratch2); // At end of page? 6120 br(CS, STUB); // at the end of page then go to stub 6121 subs(len, len, wordSize); 6122 br(LT, END); 6123 6124 BIND(LOOP); 6125 ldr(rscratch1, Address(post(ary1, wordSize))); 6126 tst(rscratch1, UPPER_BIT_MASK); 6127 br(NE, SET_RESULT); 6128 subs(len, len, wordSize); 6129 br(GE, LOOP); 6130 cmpw(len, -wordSize); 6131 br(EQ, DONE); 6132 6133 BIND(END); 6134 ldr(rscratch1, Address(ary1)); 6135 sub(rscratch2, zr, len, LSL, 3); // LSL 3 is to get bits from bytes 6136 lslv(rscratch1, rscratch1, rscratch2); 6137 tst(rscratch1, UPPER_BIT_MASK); 6138 br(NE, SET_RESULT); 6139 b(DONE); 6140 6141 BIND(STUB); 6142 RuntimeAddress count_pos = RuntimeAddress(StubRoutines::aarch64::count_positives()); 6143 assert(count_pos.target() != nullptr, "count_positives stub has not been generated"); 6144 address tpc1 = trampoline_call(count_pos); 6145 if (tpc1 == nullptr) { 6146 DEBUG_ONLY(reset_labels(STUB_LONG, SET_RESULT, DONE)); 6147 postcond(pc() == badAddress); 6148 return nullptr; 6149 } 6150 b(DONE); 6151 6152 BIND(STUB_LONG); 6153 RuntimeAddress count_pos_long = RuntimeAddress(StubRoutines::aarch64::count_positives_long()); 6154 assert(count_pos_long.target() != nullptr, "count_positives_long stub has not been generated"); 6155 address tpc2 = trampoline_call(count_pos_long); 6156 if (tpc2 == nullptr) { 6157 DEBUG_ONLY(reset_labels(SET_RESULT, DONE)); 6158 postcond(pc() == badAddress); 6159 return nullptr; 6160 } 6161 b(DONE); 6162 6163 BIND(SET_RESULT); 6164 6165 add(len, len, wordSize); 6166 sub(result, result, len); 6167 6168 BIND(DONE); 6169 postcond(pc() != badAddress); 6170 return pc(); 6171 } 6172 6173 // Clobbers: rscratch1, rscratch2, rflags 6174 // May also clobber v0-v7 when (!UseSimpleArrayEquals && UseSIMDForArrayEquals) 6175 address MacroAssembler::arrays_equals(Register a1, Register a2, Register tmp3, 6176 Register tmp4, Register tmp5, Register result, 6177 Register cnt1, int elem_size) { 6178 Label DONE, SAME; 6179 Register tmp1 = rscratch1; 6180 Register tmp2 = rscratch2; 6181 int elem_per_word = wordSize/elem_size; 6182 int log_elem_size = exact_log2(elem_size); 6183 int klass_offset = arrayOopDesc::klass_offset_in_bytes(); 6184 int length_offset = arrayOopDesc::length_offset_in_bytes(); 6185 int base_offset 6186 = arrayOopDesc::base_offset_in_bytes(elem_size == 2 ? T_CHAR : T_BYTE); 6187 // When the length offset is not aligned to 8 bytes, 6188 // then we align it down. This is valid because the new 6189 // offset will always be the klass which is the same 6190 // for type arrays. 6191 int start_offset = align_down(length_offset, BytesPerWord); 6192 int extra_length = base_offset - start_offset; 6193 assert(start_offset == length_offset || start_offset == klass_offset, 6194 "start offset must be 8-byte-aligned or be the klass offset"); 6195 assert(base_offset != start_offset, "must include the length field"); 6196 extra_length = extra_length / elem_size; // We count in elements, not bytes. 6197 int stubBytesThreshold = 3 * 64 + (UseSIMDForArrayEquals ? 0 : 16); 6198 6199 assert(elem_size == 1 || elem_size == 2, "must be char or byte"); 6200 assert_different_registers(a1, a2, result, cnt1, rscratch1, rscratch2); 6201 6202 #ifndef PRODUCT 6203 { 6204 const char kind = (elem_size == 2) ? 'U' : 'L'; 6205 char comment[64]; 6206 os::snprintf_checked(comment, sizeof comment, "array_equals%c{", kind); 6207 BLOCK_COMMENT(comment); 6208 } 6209 #endif 6210 6211 // if (a1 == a2) 6212 // return true; 6213 cmpoop(a1, a2); // May have read barriers for a1 and a2. 6214 br(EQ, SAME); 6215 6216 if (UseSimpleArrayEquals) { 6217 Label NEXT_WORD, SHORT, TAIL03, TAIL01, A_MIGHT_BE_NULL, A_IS_NOT_NULL; 6218 // if (a1 == nullptr || a2 == nullptr) 6219 // return false; 6220 // a1 & a2 == 0 means (some-pointer is null) or 6221 // (very-rare-or-even-probably-impossible-pointer-values) 6222 // so, we can save one branch in most cases 6223 tst(a1, a2); 6224 mov(result, false); 6225 br(EQ, A_MIGHT_BE_NULL); 6226 // if (a1.length != a2.length) 6227 // return false; 6228 bind(A_IS_NOT_NULL); 6229 ldrw(cnt1, Address(a1, length_offset)); 6230 // Increase loop counter by diff between base- and actual start-offset. 6231 addw(cnt1, cnt1, extra_length); 6232 lea(a1, Address(a1, start_offset)); 6233 lea(a2, Address(a2, start_offset)); 6234 // Check for short strings, i.e. smaller than wordSize. 6235 subs(cnt1, cnt1, elem_per_word); 6236 br(Assembler::LT, SHORT); 6237 // Main 8 byte comparison loop. 6238 bind(NEXT_WORD); { 6239 ldr(tmp1, Address(post(a1, wordSize))); 6240 ldr(tmp2, Address(post(a2, wordSize))); 6241 subs(cnt1, cnt1, elem_per_word); 6242 eor(tmp5, tmp1, tmp2); 6243 cbnz(tmp5, DONE); 6244 } br(GT, NEXT_WORD); 6245 // Last longword. In the case where length == 4 we compare the 6246 // same longword twice, but that's still faster than another 6247 // conditional branch. 6248 // cnt1 could be 0, -1, -2, -3, -4 for chars; -4 only happens when 6249 // length == 4. 6250 if (log_elem_size > 0) 6251 lsl(cnt1, cnt1, log_elem_size); 6252 ldr(tmp3, Address(a1, cnt1)); 6253 ldr(tmp4, Address(a2, cnt1)); 6254 eor(tmp5, tmp3, tmp4); 6255 cbnz(tmp5, DONE); 6256 b(SAME); 6257 bind(A_MIGHT_BE_NULL); 6258 // in case both a1 and a2 are not-null, proceed with loads 6259 cbz(a1, DONE); 6260 cbz(a2, DONE); 6261 b(A_IS_NOT_NULL); 6262 bind(SHORT); 6263 6264 tbz(cnt1, 2 - log_elem_size, TAIL03); // 0-7 bytes left. 6265 { 6266 ldrw(tmp1, Address(post(a1, 4))); 6267 ldrw(tmp2, Address(post(a2, 4))); 6268 eorw(tmp5, tmp1, tmp2); 6269 cbnzw(tmp5, DONE); 6270 } 6271 bind(TAIL03); 6272 tbz(cnt1, 1 - log_elem_size, TAIL01); // 0-3 bytes left. 6273 { 6274 ldrh(tmp3, Address(post(a1, 2))); 6275 ldrh(tmp4, Address(post(a2, 2))); 6276 eorw(tmp5, tmp3, tmp4); 6277 cbnzw(tmp5, DONE); 6278 } 6279 bind(TAIL01); 6280 if (elem_size == 1) { // Only needed when comparing byte arrays. 6281 tbz(cnt1, 0, SAME); // 0-1 bytes left. 6282 { 6283 ldrb(tmp1, a1); 6284 ldrb(tmp2, a2); 6285 eorw(tmp5, tmp1, tmp2); 6286 cbnzw(tmp5, DONE); 6287 } 6288 } 6289 } else { 6290 Label NEXT_DWORD, SHORT, TAIL, TAIL2, STUB, 6291 CSET_EQ, LAST_CHECK; 6292 mov(result, false); 6293 cbz(a1, DONE); 6294 ldrw(cnt1, Address(a1, length_offset)); 6295 cbz(a2, DONE); 6296 // Increase loop counter by diff between base- and actual start-offset. 6297 addw(cnt1, cnt1, extra_length); 6298 6299 // on most CPUs a2 is still "locked"(surprisingly) in ldrw and it's 6300 // faster to perform another branch before comparing a1 and a2 6301 cmp(cnt1, (u1)elem_per_word); 6302 br(LE, SHORT); // short or same 6303 ldr(tmp3, Address(pre(a1, start_offset))); 6304 subs(zr, cnt1, stubBytesThreshold); 6305 br(GE, STUB); 6306 ldr(tmp4, Address(pre(a2, start_offset))); 6307 sub(tmp5, zr, cnt1, LSL, 3 + log_elem_size); 6308 6309 // Main 16 byte comparison loop with 2 exits 6310 bind(NEXT_DWORD); { 6311 ldr(tmp1, Address(pre(a1, wordSize))); 6312 ldr(tmp2, Address(pre(a2, wordSize))); 6313 subs(cnt1, cnt1, 2 * elem_per_word); 6314 br(LE, TAIL); 6315 eor(tmp4, tmp3, tmp4); 6316 cbnz(tmp4, DONE); 6317 ldr(tmp3, Address(pre(a1, wordSize))); 6318 ldr(tmp4, Address(pre(a2, wordSize))); 6319 cmp(cnt1, (u1)elem_per_word); 6320 br(LE, TAIL2); 6321 cmp(tmp1, tmp2); 6322 } br(EQ, NEXT_DWORD); 6323 b(DONE); 6324 6325 bind(TAIL); 6326 eor(tmp4, tmp3, tmp4); 6327 eor(tmp2, tmp1, tmp2); 6328 lslv(tmp2, tmp2, tmp5); 6329 orr(tmp5, tmp4, tmp2); 6330 cmp(tmp5, zr); 6331 b(CSET_EQ); 6332 6333 bind(TAIL2); 6334 eor(tmp2, tmp1, tmp2); 6335 cbnz(tmp2, DONE); 6336 b(LAST_CHECK); 6337 6338 bind(STUB); 6339 ldr(tmp4, Address(pre(a2, start_offset))); 6340 if (elem_size == 2) { // convert to byte counter 6341 lsl(cnt1, cnt1, 1); 6342 } 6343 eor(tmp5, tmp3, tmp4); 6344 cbnz(tmp5, DONE); 6345 RuntimeAddress stub = RuntimeAddress(StubRoutines::aarch64::large_array_equals()); 6346 assert(stub.target() != nullptr, "array_equals_long stub has not been generated"); 6347 address tpc = trampoline_call(stub); 6348 if (tpc == nullptr) { 6349 DEBUG_ONLY(reset_labels(SHORT, LAST_CHECK, CSET_EQ, SAME, DONE)); 6350 postcond(pc() == badAddress); 6351 return nullptr; 6352 } 6353 b(DONE); 6354 6355 // (a1 != null && a2 == null) || (a1 != null && a2 != null && a1 == a2) 6356 // so, if a2 == null => return false(0), else return true, so we can return a2 6357 mov(result, a2); 6358 b(DONE); 6359 bind(SHORT); 6360 sub(tmp5, zr, cnt1, LSL, 3 + log_elem_size); 6361 ldr(tmp3, Address(a1, start_offset)); 6362 ldr(tmp4, Address(a2, start_offset)); 6363 bind(LAST_CHECK); 6364 eor(tmp4, tmp3, tmp4); 6365 lslv(tmp5, tmp4, tmp5); 6366 cmp(tmp5, zr); 6367 bind(CSET_EQ); 6368 cset(result, EQ); 6369 b(DONE); 6370 } 6371 6372 bind(SAME); 6373 mov(result, true); 6374 // That's it. 6375 bind(DONE); 6376 6377 BLOCK_COMMENT("} array_equals"); 6378 postcond(pc() != badAddress); 6379 return pc(); 6380 } 6381 6382 // Compare Strings 6383 6384 // For Strings we're passed the address of the first characters in a1 6385 // and a2 and the length in cnt1. 6386 // There are two implementations. For arrays >= 8 bytes, all 6387 // comparisons (including the final one, which may overlap) are 6388 // performed 8 bytes at a time. For strings < 8 bytes, we compare a 6389 // halfword, then a short, and then a byte. 6390 6391 void MacroAssembler::string_equals(Register a1, Register a2, 6392 Register result, Register cnt1) 6393 { 6394 Label SAME, DONE, SHORT, NEXT_WORD; 6395 Register tmp1 = rscratch1; 6396 Register tmp2 = rscratch2; 6397 Register cnt2 = tmp2; // cnt2 only used in array length compare 6398 6399 assert_different_registers(a1, a2, result, cnt1, rscratch1, rscratch2); 6400 6401 #ifndef PRODUCT 6402 { 6403 char comment[64]; 6404 os::snprintf_checked(comment, sizeof comment, "{string_equalsL"); 6405 BLOCK_COMMENT(comment); 6406 } 6407 #endif 6408 6409 mov(result, false); 6410 6411 // Check for short strings, i.e. smaller than wordSize. 6412 subs(cnt1, cnt1, wordSize); 6413 br(Assembler::LT, SHORT); 6414 // Main 8 byte comparison loop. 6415 bind(NEXT_WORD); { 6416 ldr(tmp1, Address(post(a1, wordSize))); 6417 ldr(tmp2, Address(post(a2, wordSize))); 6418 subs(cnt1, cnt1, wordSize); 6419 eor(tmp1, tmp1, tmp2); 6420 cbnz(tmp1, DONE); 6421 } br(GT, NEXT_WORD); 6422 // Last longword. In the case where length == 4 we compare the 6423 // same longword twice, but that's still faster than another 6424 // conditional branch. 6425 // cnt1 could be 0, -1, -2, -3, -4 for chars; -4 only happens when 6426 // length == 4. 6427 ldr(tmp1, Address(a1, cnt1)); 6428 ldr(tmp2, Address(a2, cnt1)); 6429 eor(tmp2, tmp1, tmp2); 6430 cbnz(tmp2, DONE); 6431 b(SAME); 6432 6433 bind(SHORT); 6434 Label TAIL03, TAIL01; 6435 6436 tbz(cnt1, 2, TAIL03); // 0-7 bytes left. 6437 { 6438 ldrw(tmp1, Address(post(a1, 4))); 6439 ldrw(tmp2, Address(post(a2, 4))); 6440 eorw(tmp1, tmp1, tmp2); 6441 cbnzw(tmp1, DONE); 6442 } 6443 bind(TAIL03); 6444 tbz(cnt1, 1, TAIL01); // 0-3 bytes left. 6445 { 6446 ldrh(tmp1, Address(post(a1, 2))); 6447 ldrh(tmp2, Address(post(a2, 2))); 6448 eorw(tmp1, tmp1, tmp2); 6449 cbnzw(tmp1, DONE); 6450 } 6451 bind(TAIL01); 6452 tbz(cnt1, 0, SAME); // 0-1 bytes left. 6453 { 6454 ldrb(tmp1, a1); 6455 ldrb(tmp2, a2); 6456 eorw(tmp1, tmp1, tmp2); 6457 cbnzw(tmp1, DONE); 6458 } 6459 // Arrays are equal. 6460 bind(SAME); 6461 mov(result, true); 6462 6463 // That's it. 6464 bind(DONE); 6465 BLOCK_COMMENT("} string_equals"); 6466 } 6467 6468 6469 // The size of the blocks erased by the zero_blocks stub. We must 6470 // handle anything smaller than this ourselves in zero_words(). 6471 const int MacroAssembler::zero_words_block_size = 8; 6472 6473 // zero_words() is used by C2 ClearArray patterns and by 6474 // C1_MacroAssembler. It is as small as possible, handling small word 6475 // counts locally and delegating anything larger to the zero_blocks 6476 // stub. It is expanded many times in compiled code, so it is 6477 // important to keep it short. 6478 6479 // ptr: Address of a buffer to be zeroed. 6480 // cnt: Count in HeapWords. 6481 // 6482 // ptr, cnt, rscratch1, and rscratch2 are clobbered. 6483 address MacroAssembler::zero_words(Register ptr, Register cnt) 6484 { 6485 assert(is_power_of_2(zero_words_block_size), "adjust this"); 6486 6487 BLOCK_COMMENT("zero_words {"); 6488 assert(ptr == r10 && cnt == r11, "mismatch in register usage"); 6489 RuntimeAddress zero_blocks = RuntimeAddress(StubRoutines::aarch64::zero_blocks()); 6490 assert(zero_blocks.target() != nullptr, "zero_blocks stub has not been generated"); 6491 6492 subs(rscratch1, cnt, zero_words_block_size); 6493 Label around; 6494 br(LO, around); 6495 { 6496 RuntimeAddress zero_blocks = RuntimeAddress(StubRoutines::aarch64::zero_blocks()); 6497 assert(zero_blocks.target() != nullptr, "zero_blocks stub has not been generated"); 6498 // Make sure this is a C2 compilation. C1 allocates space only for 6499 // trampoline stubs generated by Call LIR ops, and in any case it 6500 // makes sense for a C1 compilation task to proceed as quickly as 6501 // possible. 6502 CompileTask* task; 6503 if (StubRoutines::aarch64::complete() 6504 && Thread::current()->is_Compiler_thread() 6505 && (task = ciEnv::current()->task()) 6506 && is_c2_compile(task->comp_level())) { 6507 address tpc = trampoline_call(zero_blocks); 6508 if (tpc == nullptr) { 6509 DEBUG_ONLY(reset_labels(around)); 6510 return nullptr; 6511 } 6512 } else { 6513 far_call(zero_blocks); 6514 } 6515 } 6516 bind(around); 6517 6518 // We have a few words left to do. zero_blocks has adjusted r10 and r11 6519 // for us. 6520 for (int i = zero_words_block_size >> 1; i > 1; i >>= 1) { 6521 Label l; 6522 tbz(cnt, exact_log2(i), l); 6523 for (int j = 0; j < i; j += 2) { 6524 stp(zr, zr, post(ptr, 2 * BytesPerWord)); 6525 } 6526 bind(l); 6527 } 6528 { 6529 Label l; 6530 tbz(cnt, 0, l); 6531 str(zr, Address(ptr)); 6532 bind(l); 6533 } 6534 6535 BLOCK_COMMENT("} zero_words"); 6536 return pc(); 6537 } 6538 6539 // base: Address of a buffer to be zeroed, 8 bytes aligned. 6540 // cnt: Immediate count in HeapWords. 6541 // 6542 // r10, r11, rscratch1, and rscratch2 are clobbered. 6543 address MacroAssembler::zero_words(Register base, uint64_t cnt) 6544 { 6545 assert(wordSize <= BlockZeroingLowLimit, 6546 "increase BlockZeroingLowLimit"); 6547 address result = nullptr; 6548 if (cnt <= (uint64_t)BlockZeroingLowLimit / BytesPerWord) { 6549 #ifndef PRODUCT 6550 { 6551 char buf[64]; 6552 os::snprintf_checked(buf, sizeof buf, "zero_words (count = %" PRIu64 ") {", cnt); 6553 BLOCK_COMMENT(buf); 6554 } 6555 #endif 6556 if (cnt >= 16) { 6557 uint64_t loops = cnt/16; 6558 if (loops > 1) { 6559 mov(rscratch2, loops - 1); 6560 } 6561 { 6562 Label loop; 6563 bind(loop); 6564 for (int i = 0; i < 16; i += 2) { 6565 stp(zr, zr, Address(base, i * BytesPerWord)); 6566 } 6567 add(base, base, 16 * BytesPerWord); 6568 if (loops > 1) { 6569 subs(rscratch2, rscratch2, 1); 6570 br(GE, loop); 6571 } 6572 } 6573 } 6574 cnt %= 16; 6575 int i = cnt & 1; // store any odd word to start 6576 if (i) str(zr, Address(base)); 6577 for (; i < (int)cnt; i += 2) { 6578 stp(zr, zr, Address(base, i * wordSize)); 6579 } 6580 BLOCK_COMMENT("} zero_words"); 6581 result = pc(); 6582 } else { 6583 mov(r10, base); mov(r11, cnt); 6584 result = zero_words(r10, r11); 6585 } 6586 return result; 6587 } 6588 6589 // Zero blocks of memory by using DC ZVA. 6590 // 6591 // Aligns the base address first sufficiently for DC ZVA, then uses 6592 // DC ZVA repeatedly for every full block. cnt is the size to be 6593 // zeroed in HeapWords. Returns the count of words left to be zeroed 6594 // in cnt. 6595 // 6596 // NOTE: This is intended to be used in the zero_blocks() stub. If 6597 // you want to use it elsewhere, note that cnt must be >= 2*zva_length. 6598 void MacroAssembler::zero_dcache_blocks(Register base, Register cnt) { 6599 Register tmp = rscratch1; 6600 Register tmp2 = rscratch2; 6601 int zva_length = VM_Version::zva_length(); 6602 Label initial_table_end, loop_zva; 6603 Label fini; 6604 6605 // Base must be 16 byte aligned. If not just return and let caller handle it 6606 tst(base, 0x0f); 6607 br(Assembler::NE, fini); 6608 // Align base with ZVA length. 6609 neg(tmp, base); 6610 andr(tmp, tmp, zva_length - 1); 6611 6612 // tmp: the number of bytes to be filled to align the base with ZVA length. 6613 add(base, base, tmp); 6614 sub(cnt, cnt, tmp, Assembler::ASR, 3); 6615 adr(tmp2, initial_table_end); 6616 sub(tmp2, tmp2, tmp, Assembler::LSR, 2); 6617 br(tmp2); 6618 6619 for (int i = -zva_length + 16; i < 0; i += 16) 6620 stp(zr, zr, Address(base, i)); 6621 bind(initial_table_end); 6622 6623 sub(cnt, cnt, zva_length >> 3); 6624 bind(loop_zva); 6625 dc(Assembler::ZVA, base); 6626 subs(cnt, cnt, zva_length >> 3); 6627 add(base, base, zva_length); 6628 br(Assembler::GE, loop_zva); 6629 add(cnt, cnt, zva_length >> 3); // count not zeroed by DC ZVA 6630 bind(fini); 6631 } 6632 6633 // base: Address of a buffer to be filled, 8 bytes aligned. 6634 // cnt: Count in 8-byte unit. 6635 // value: Value to be filled with. 6636 // base will point to the end of the buffer after filling. 6637 void MacroAssembler::fill_words(Register base, Register cnt, Register value) 6638 { 6639 // Algorithm: 6640 // 6641 // if (cnt == 0) { 6642 // return; 6643 // } 6644 // if ((p & 8) != 0) { 6645 // *p++ = v; 6646 // } 6647 // 6648 // scratch1 = cnt & 14; 6649 // cnt -= scratch1; 6650 // p += scratch1; 6651 // switch (scratch1 / 2) { 6652 // do { 6653 // cnt -= 16; 6654 // p[-16] = v; 6655 // p[-15] = v; 6656 // case 7: 6657 // p[-14] = v; 6658 // p[-13] = v; 6659 // case 6: 6660 // p[-12] = v; 6661 // p[-11] = v; 6662 // // ... 6663 // case 1: 6664 // p[-2] = v; 6665 // p[-1] = v; 6666 // case 0: 6667 // p += 16; 6668 // } while (cnt); 6669 // } 6670 // if ((cnt & 1) == 1) { 6671 // *p++ = v; 6672 // } 6673 6674 assert_different_registers(base, cnt, value, rscratch1, rscratch2); 6675 6676 Label fini, skip, entry, loop; 6677 const int unroll = 8; // Number of stp instructions we'll unroll 6678 6679 cbz(cnt, fini); 6680 tbz(base, 3, skip); 6681 str(value, Address(post(base, 8))); 6682 sub(cnt, cnt, 1); 6683 bind(skip); 6684 6685 andr(rscratch1, cnt, (unroll-1) * 2); 6686 sub(cnt, cnt, rscratch1); 6687 add(base, base, rscratch1, Assembler::LSL, 3); 6688 adr(rscratch2, entry); 6689 sub(rscratch2, rscratch2, rscratch1, Assembler::LSL, 1); 6690 br(rscratch2); 6691 6692 bind(loop); 6693 add(base, base, unroll * 16); 6694 for (int i = -unroll; i < 0; i++) 6695 stp(value, value, Address(base, i * 16)); 6696 bind(entry); 6697 subs(cnt, cnt, unroll * 2); 6698 br(Assembler::GE, loop); 6699 6700 tbz(cnt, 0, fini); 6701 str(value, Address(post(base, 8))); 6702 bind(fini); 6703 } 6704 6705 // Intrinsic for 6706 // 6707 // - sun.nio.cs.ISO_8859_1.Encoder#encodeISOArray0(byte[] sa, int sp, byte[] da, int dp, int len) 6708 // Encodes char[] to byte[] in ISO-8859-1 6709 // 6710 // - java.lang.StringCoding#encodeISOArray0(byte[] sa, int sp, byte[] da, int dp, int len) 6711 // Encodes byte[] (containing UTF-16) to byte[] in ISO-8859-1 6712 // 6713 // - java.lang.StringCoding#encodeAsciiArray0(char[] sa, int sp, byte[] da, int dp, int len) 6714 // Encodes char[] to byte[] in ASCII 6715 // 6716 // This version always returns the number of characters copied, and does not 6717 // clobber the 'len' register. A successful copy will complete with the post- 6718 // condition: 'res' == 'len', while an unsuccessful copy will exit with the 6719 // post-condition: 0 <= 'res' < 'len'. 6720 // 6721 // NOTE: Attempts to use 'ld2' (and 'umaxv' in the ISO part) has proven to 6722 // degrade performance (on Ampere Altra - Neoverse N1), to an extent 6723 // beyond the acceptable, even though the footprint would be smaller. 6724 // Using 'umaxv' in the ASCII-case comes with a small penalty but does 6725 // avoid additional bloat. 6726 // 6727 // Clobbers: src, dst, res, rscratch1, rscratch2, rflags 6728 void MacroAssembler::encode_iso_array(Register src, Register dst, 6729 Register len, Register res, bool ascii, 6730 FloatRegister vtmp0, FloatRegister vtmp1, 6731 FloatRegister vtmp2, FloatRegister vtmp3, 6732 FloatRegister vtmp4, FloatRegister vtmp5) 6733 { 6734 Register cnt = res; 6735 Register max = rscratch1; 6736 Register chk = rscratch2; 6737 6738 prfm(Address(src), PLDL1STRM); 6739 movw(cnt, len); 6740 6741 #define ASCII(insn) do { if (ascii) { insn; } } while (0) 6742 6743 Label LOOP_32, DONE_32, FAIL_32; 6744 6745 BIND(LOOP_32); 6746 { 6747 cmpw(cnt, 32); 6748 br(LT, DONE_32); 6749 ld1(vtmp0, vtmp1, vtmp2, vtmp3, T8H, Address(post(src, 64))); 6750 // Extract lower bytes. 6751 FloatRegister vlo0 = vtmp4; 6752 FloatRegister vlo1 = vtmp5; 6753 uzp1(vlo0, T16B, vtmp0, vtmp1); 6754 uzp1(vlo1, T16B, vtmp2, vtmp3); 6755 // Merge bits... 6756 orr(vtmp0, T16B, vtmp0, vtmp1); 6757 orr(vtmp2, T16B, vtmp2, vtmp3); 6758 // Extract merged upper bytes. 6759 FloatRegister vhix = vtmp0; 6760 uzp2(vhix, T16B, vtmp0, vtmp2); 6761 // ISO-check on hi-parts (all zero). 6762 // ASCII-check on lo-parts (no sign). 6763 FloatRegister vlox = vtmp1; // Merge lower bytes. 6764 ASCII(orr(vlox, T16B, vlo0, vlo1)); 6765 umov(chk, vhix, D, 1); ASCII(cm(LT, vlox, T16B, vlox)); 6766 fmovd(max, vhix); ASCII(umaxv(vlox, T16B, vlox)); 6767 orr(chk, chk, max); ASCII(umov(max, vlox, B, 0)); 6768 ASCII(orr(chk, chk, max)); 6769 cbnz(chk, FAIL_32); 6770 subw(cnt, cnt, 32); 6771 st1(vlo0, vlo1, T16B, Address(post(dst, 32))); 6772 b(LOOP_32); 6773 } 6774 BIND(FAIL_32); 6775 sub(src, src, 64); 6776 BIND(DONE_32); 6777 6778 Label LOOP_8, SKIP_8; 6779 6780 BIND(LOOP_8); 6781 { 6782 cmpw(cnt, 8); 6783 br(LT, SKIP_8); 6784 FloatRegister vhi = vtmp0; 6785 FloatRegister vlo = vtmp1; 6786 ld1(vtmp3, T8H, src); 6787 uzp1(vlo, T16B, vtmp3, vtmp3); 6788 uzp2(vhi, T16B, vtmp3, vtmp3); 6789 // ISO-check on hi-parts (all zero). 6790 // ASCII-check on lo-parts (no sign). 6791 ASCII(cm(LT, vtmp2, T16B, vlo)); 6792 fmovd(chk, vhi); ASCII(umaxv(vtmp2, T16B, vtmp2)); 6793 ASCII(umov(max, vtmp2, B, 0)); 6794 ASCII(orr(chk, chk, max)); 6795 cbnz(chk, SKIP_8); 6796 6797 strd(vlo, Address(post(dst, 8))); 6798 subw(cnt, cnt, 8); 6799 add(src, src, 16); 6800 b(LOOP_8); 6801 } 6802 BIND(SKIP_8); 6803 6804 #undef ASCII 6805 6806 Label LOOP, DONE; 6807 6808 cbz(cnt, DONE); 6809 BIND(LOOP); 6810 { 6811 Register chr = rscratch1; 6812 ldrh(chr, Address(post(src, 2))); 6813 tst(chr, ascii ? 0xff80 : 0xff00); 6814 br(NE, DONE); 6815 strb(chr, Address(post(dst, 1))); 6816 subs(cnt, cnt, 1); 6817 br(GT, LOOP); 6818 } 6819 BIND(DONE); 6820 // Return index where we stopped. 6821 subw(res, len, cnt); 6822 } 6823 6824 // Inflate byte[] array to char[]. 6825 // Clobbers: src, dst, len, rflags, rscratch1, v0-v6 6826 address MacroAssembler::byte_array_inflate(Register src, Register dst, Register len, 6827 FloatRegister vtmp1, FloatRegister vtmp2, 6828 FloatRegister vtmp3, Register tmp4) { 6829 Label big, done, after_init, to_stub; 6830 6831 assert_different_registers(src, dst, len, tmp4, rscratch1); 6832 6833 fmovd(vtmp1, 0.0); 6834 lsrw(tmp4, len, 3); 6835 bind(after_init); 6836 cbnzw(tmp4, big); 6837 // Short string: less than 8 bytes. 6838 { 6839 Label loop, tiny; 6840 6841 cmpw(len, 4); 6842 br(LT, tiny); 6843 // Use SIMD to do 4 bytes. 6844 ldrs(vtmp2, post(src, 4)); 6845 zip1(vtmp3, T8B, vtmp2, vtmp1); 6846 subw(len, len, 4); 6847 strd(vtmp3, post(dst, 8)); 6848 6849 cbzw(len, done); 6850 6851 // Do the remaining bytes by steam. 6852 bind(loop); 6853 ldrb(tmp4, post(src, 1)); 6854 strh(tmp4, post(dst, 2)); 6855 subw(len, len, 1); 6856 6857 bind(tiny); 6858 cbnz(len, loop); 6859 6860 b(done); 6861 } 6862 6863 if (SoftwarePrefetchHintDistance >= 0) { 6864 bind(to_stub); 6865 RuntimeAddress stub = RuntimeAddress(StubRoutines::aarch64::large_byte_array_inflate()); 6866 assert(stub.target() != nullptr, "large_byte_array_inflate stub has not been generated"); 6867 address tpc = trampoline_call(stub); 6868 if (tpc == nullptr) { 6869 DEBUG_ONLY(reset_labels(big, done)); 6870 postcond(pc() == badAddress); 6871 return nullptr; 6872 } 6873 b(after_init); 6874 } 6875 6876 // Unpack the bytes 8 at a time. 6877 bind(big); 6878 { 6879 Label loop, around, loop_last, loop_start; 6880 6881 if (SoftwarePrefetchHintDistance >= 0) { 6882 const int large_loop_threshold = (64 + 16)/8; 6883 ldrd(vtmp2, post(src, 8)); 6884 andw(len, len, 7); 6885 cmp(tmp4, (u1)large_loop_threshold); 6886 br(GE, to_stub); 6887 b(loop_start); 6888 6889 bind(loop); 6890 ldrd(vtmp2, post(src, 8)); 6891 bind(loop_start); 6892 subs(tmp4, tmp4, 1); 6893 br(EQ, loop_last); 6894 zip1(vtmp2, T16B, vtmp2, vtmp1); 6895 ldrd(vtmp3, post(src, 8)); 6896 st1(vtmp2, T8H, post(dst, 16)); 6897 subs(tmp4, tmp4, 1); 6898 zip1(vtmp3, T16B, vtmp3, vtmp1); 6899 st1(vtmp3, T8H, post(dst, 16)); 6900 br(NE, loop); 6901 b(around); 6902 bind(loop_last); 6903 zip1(vtmp2, T16B, vtmp2, vtmp1); 6904 st1(vtmp2, T8H, post(dst, 16)); 6905 bind(around); 6906 cbz(len, done); 6907 } else { 6908 andw(len, len, 7); 6909 bind(loop); 6910 ldrd(vtmp2, post(src, 8)); 6911 sub(tmp4, tmp4, 1); 6912 zip1(vtmp3, T16B, vtmp2, vtmp1); 6913 st1(vtmp3, T8H, post(dst, 16)); 6914 cbnz(tmp4, loop); 6915 } 6916 } 6917 6918 // Do the tail of up to 8 bytes. 6919 add(src, src, len); 6920 ldrd(vtmp3, Address(src, -8)); 6921 add(dst, dst, len, ext::uxtw, 1); 6922 zip1(vtmp3, T16B, vtmp3, vtmp1); 6923 strq(vtmp3, Address(dst, -16)); 6924 6925 bind(done); 6926 postcond(pc() != badAddress); 6927 return pc(); 6928 } 6929 6930 // Compress char[] array to byte[]. 6931 // Intrinsic for java.lang.StringUTF16.compress(char[] src, int srcOff, byte[] dst, int dstOff, int len) 6932 // Return the array length if every element in array can be encoded, 6933 // otherwise, the index of first non-latin1 (> 0xff) character. 6934 void MacroAssembler::char_array_compress(Register src, Register dst, Register len, 6935 Register res, 6936 FloatRegister tmp0, FloatRegister tmp1, 6937 FloatRegister tmp2, FloatRegister tmp3, 6938 FloatRegister tmp4, FloatRegister tmp5) { 6939 encode_iso_array(src, dst, len, res, false, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5); 6940 } 6941 6942 // java.math.round(double a) 6943 // Returns the closest long to the argument, with ties rounding to 6944 // positive infinity. This requires some fiddling for corner 6945 // cases. We take care to avoid double rounding in e.g. (jlong)(a + 0.5). 6946 void MacroAssembler::java_round_double(Register dst, FloatRegister src, 6947 FloatRegister ftmp) { 6948 Label DONE; 6949 BLOCK_COMMENT("java_round_double: { "); 6950 fmovd(rscratch1, src); 6951 // Use RoundToNearestTiesAway unless src small and -ve. 6952 fcvtasd(dst, src); 6953 // Test if src >= 0 || abs(src) >= 0x1.0p52 6954 eor(rscratch1, rscratch1, UCONST64(1) << 63); // flip sign bit 6955 mov(rscratch2, julong_cast(0x1.0p52)); 6956 cmp(rscratch1, rscratch2); 6957 br(HS, DONE); { 6958 // src < 0 && abs(src) < 0x1.0p52 6959 // src may have a fractional part, so add 0.5 6960 fmovd(ftmp, 0.5); 6961 faddd(ftmp, src, ftmp); 6962 // Convert double to jlong, use RoundTowardsNegative 6963 fcvtmsd(dst, ftmp); 6964 } 6965 bind(DONE); 6966 BLOCK_COMMENT("} java_round_double"); 6967 } 6968 6969 void MacroAssembler::java_round_float(Register dst, FloatRegister src, 6970 FloatRegister ftmp) { 6971 Label DONE; 6972 BLOCK_COMMENT("java_round_float: { "); 6973 fmovs(rscratch1, src); 6974 // Use RoundToNearestTiesAway unless src small and -ve. 6975 fcvtassw(dst, src); 6976 // Test if src >= 0 || abs(src) >= 0x1.0p23 6977 eor(rscratch1, rscratch1, 0x80000000); // flip sign bit 6978 mov(rscratch2, jint_cast(0x1.0p23f)); 6979 cmp(rscratch1, rscratch2); 6980 br(HS, DONE); { 6981 // src < 0 && |src| < 0x1.0p23 6982 // src may have a fractional part, so add 0.5 6983 fmovs(ftmp, 0.5f); 6984 fadds(ftmp, src, ftmp); 6985 // Convert float to jint, use RoundTowardsNegative 6986 fcvtmssw(dst, ftmp); 6987 } 6988 bind(DONE); 6989 BLOCK_COMMENT("} java_round_float"); 6990 } 6991 6992 // get_thread() can be called anywhere inside generated code so we 6993 // need to save whatever non-callee save context might get clobbered 6994 // by the call to JavaThread::aarch64_get_thread_helper() or, indeed, 6995 // the call setup code. 6996 // 6997 // On Linux, aarch64_get_thread_helper() clobbers only r0, r1, and flags. 6998 // On other systems, the helper is a usual C function. 6999 // 7000 void MacroAssembler::get_thread(Register dst) { 7001 RegSet saved_regs = 7002 LINUX_ONLY(RegSet::range(r0, r1) + lr - dst) 7003 NOT_LINUX (RegSet::range(r0, r17) + lr - dst); 7004 7005 protect_return_address(); 7006 push(saved_regs, sp); 7007 7008 mov(lr, ExternalAddress(CAST_FROM_FN_PTR(address, JavaThread::aarch64_get_thread_helper))); 7009 blr(lr); 7010 if (dst != c_rarg0) { 7011 mov(dst, c_rarg0); 7012 } 7013 7014 pop(saved_regs, sp); 7015 authenticate_return_address(); 7016 } 7017 7018 #ifdef COMPILER2 7019 // C2 compiled method's prolog code 7020 // Moved here from aarch64.ad to support Valhalla code belows 7021 void MacroAssembler::verified_entry(Compile* C, int sp_inc) { 7022 if (C->clinit_barrier_on_entry()) { 7023 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started"); 7024 7025 Label L_skip_barrier; 7026 7027 mov_metadata(rscratch2, C->method()->holder()->constant_encoding()); 7028 clinit_barrier(rscratch2, rscratch1, &L_skip_barrier); 7029 far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); 7030 bind(L_skip_barrier); 7031 } 7032 7033 if (C->max_vector_size() > 0) { 7034 reinitialize_ptrue(); 7035 } 7036 7037 int bangsize = C->output()->bang_size_in_bytes(); 7038 if (C->output()->need_stack_bang(bangsize)) 7039 generate_stack_overflow_check(bangsize); 7040 7041 // n.b. frame size includes space for return pc and rfp 7042 const long framesize = C->output()->frame_size_in_bytes(); 7043 build_frame(framesize); 7044 7045 if (C->needs_stack_repair()) { 7046 save_stack_increment(sp_inc, framesize); 7047 } 7048 7049 if (VerifyStackAtCalls) { 7050 Unimplemented(); 7051 } 7052 } 7053 #endif // COMPILER2 7054 7055 int MacroAssembler::store_inline_type_fields_to_buf(ciInlineKlass* vk, bool from_interpreter) { 7056 assert(InlineTypeReturnedAsFields, "Inline types should never be returned as fields"); 7057 // An inline type might be returned. If fields are in registers we 7058 // need to allocate an inline type instance and initialize it with 7059 // the value of the fields. 7060 Label skip; 7061 // We only need a new buffered inline type if a new one is not returned 7062 tbz(r0, 0, skip); 7063 int call_offset = -1; 7064 7065 // Be careful not to clobber r1-7 which hold returned fields 7066 // Also do not use callee-saved registers as these may be live in the interpreter 7067 Register tmp1 = r13, tmp2 = r14, klass = r15, r0_preserved = r12; 7068 7069 // The following code is similar to allocate_instance but has some slight differences, 7070 // e.g. object size is always not zero, sometimes it's constant; storing klass ptr after 7071 // allocating is not necessary if vk != nullptr, etc. allocate_instance is not aware of these. 7072 Label slow_case; 7073 // 1. Try to allocate a new buffered inline instance either from TLAB or eden space 7074 mov(r0_preserved, r0); // save r0 for slow_case since *_allocate may corrupt it when allocation failed 7075 7076 if (vk != nullptr) { 7077 // Called from C1, where the return type is statically known. 7078 movptr(klass, (intptr_t)vk->get_InlineKlass()); 7079 jint lh = vk->layout_helper(); 7080 assert(lh != Klass::_lh_neutral_value, "inline class in return type must have been resolved"); 7081 if (UseTLAB && !Klass::layout_helper_needs_slow_path(lh)) { 7082 tlab_allocate(r0, noreg, lh, tmp1, tmp2, slow_case); 7083 } else { 7084 b(slow_case); 7085 } 7086 } else { 7087 // Call from interpreter. R0 contains ((the InlineKlass* of the return type) | 0x01) 7088 andr(klass, r0, -2); 7089 if (UseTLAB) { 7090 ldrw(tmp2, Address(klass, Klass::layout_helper_offset())); 7091 tst(tmp2, Klass::_lh_instance_slow_path_bit); 7092 br(Assembler::NE, slow_case); 7093 tlab_allocate(r0, tmp2, 0, tmp1, tmp2, slow_case); 7094 } else { 7095 b(slow_case); 7096 } 7097 } 7098 if (UseTLAB) { 7099 // 2. Initialize buffered inline instance header 7100 Register buffer_obj = r0; 7101 if (UseCompactObjectHeaders) { 7102 ldr(rscratch1, Address(klass, Klass::prototype_header_offset())); 7103 str(rscratch1, Address(buffer_obj, oopDesc::mark_offset_in_bytes())); 7104 } else { 7105 mov(rscratch1, (intptr_t)markWord::inline_type_prototype().value()); 7106 str(rscratch1, Address(buffer_obj, oopDesc::mark_offset_in_bytes())); 7107 store_klass_gap(buffer_obj, zr); 7108 if (vk == nullptr) { 7109 // store_klass corrupts klass, so save it for later use (interpreter case only). 7110 mov(tmp1, klass); 7111 } 7112 store_klass(buffer_obj, klass); 7113 klass = tmp1; 7114 } 7115 // 3. Initialize its fields with an inline class specific handler 7116 if (vk != nullptr) { 7117 far_call(RuntimeAddress(vk->pack_handler())); // no need for call info as this will not safepoint. 7118 } else { 7119 ldr(tmp1, Address(klass, InstanceKlass::adr_inlineklass_fixed_block_offset())); 7120 ldr(tmp1, Address(tmp1, InlineKlass::pack_handler_offset())); 7121 blr(tmp1); 7122 } 7123 7124 membar(Assembler::StoreStore); 7125 b(skip); 7126 } else { 7127 // Must have already branched to slow_case above. 7128 DEBUG_ONLY(should_not_reach_here()); 7129 } 7130 bind(slow_case); 7131 // We failed to allocate a new inline type, fall back to a runtime 7132 // call. Some oop field may be live in some registers but we can't 7133 // tell. That runtime call will take care of preserving them 7134 // across a GC if there's one. 7135 mov(r0, r0_preserved); 7136 7137 if (from_interpreter) { 7138 super_call_VM_leaf(StubRoutines::store_inline_type_fields_to_buf()); 7139 } else { 7140 far_call(RuntimeAddress(StubRoutines::store_inline_type_fields_to_buf())); 7141 call_offset = offset(); 7142 } 7143 membar(Assembler::StoreStore); 7144 7145 bind(skip); 7146 return call_offset; 7147 } 7148 7149 // Move a value between registers/stack slots and update the reg_state 7150 bool MacroAssembler::move_helper(VMReg from, VMReg to, BasicType bt, RegState reg_state[]) { 7151 assert(from->is_valid() && to->is_valid(), "source and destination must be valid"); 7152 if (reg_state[to->value()] == reg_written) { 7153 return true; // Already written 7154 } 7155 7156 if (from != to && bt != T_VOID) { 7157 if (reg_state[to->value()] == reg_readonly) { 7158 return false; // Not yet writable 7159 } 7160 if (from->is_reg()) { 7161 if (to->is_reg()) { 7162 if (from->is_Register() && to->is_Register()) { 7163 mov(to->as_Register(), from->as_Register()); 7164 } else if (from->is_FloatRegister() && to->is_FloatRegister()) { 7165 fmovd(to->as_FloatRegister(), from->as_FloatRegister()); 7166 } else { 7167 ShouldNotReachHere(); 7168 } 7169 } else { 7170 int st_off = to->reg2stack() * VMRegImpl::stack_slot_size; 7171 Address to_addr = Address(sp, st_off); 7172 if (from->is_FloatRegister()) { 7173 if (bt == T_DOUBLE) { 7174 strd(from->as_FloatRegister(), to_addr); 7175 } else { 7176 assert(bt == T_FLOAT, "must be float"); 7177 strs(from->as_FloatRegister(), to_addr); 7178 } 7179 } else { 7180 str(from->as_Register(), to_addr); 7181 } 7182 } 7183 } else { 7184 Address from_addr = Address(sp, from->reg2stack() * VMRegImpl::stack_slot_size); 7185 if (to->is_reg()) { 7186 if (to->is_FloatRegister()) { 7187 if (bt == T_DOUBLE) { 7188 ldrd(to->as_FloatRegister(), from_addr); 7189 } else { 7190 assert(bt == T_FLOAT, "must be float"); 7191 ldrs(to->as_FloatRegister(), from_addr); 7192 } 7193 } else { 7194 ldr(to->as_Register(), from_addr); 7195 } 7196 } else { 7197 int st_off = to->reg2stack() * VMRegImpl::stack_slot_size; 7198 ldr(rscratch1, from_addr); 7199 str(rscratch1, Address(sp, st_off)); 7200 } 7201 } 7202 } 7203 7204 // Update register states 7205 reg_state[from->value()] = reg_writable; 7206 reg_state[to->value()] = reg_written; 7207 return true; 7208 } 7209 7210 // Calculate the extra stack space required for packing or unpacking inline 7211 // args and adjust the stack pointer 7212 int MacroAssembler::extend_stack_for_inline_args(int args_on_stack) { 7213 int sp_inc = args_on_stack * VMRegImpl::stack_slot_size; 7214 sp_inc = align_up(sp_inc, StackAlignmentInBytes); 7215 assert(sp_inc > 0, "sanity"); 7216 7217 // Save a copy of the FP and LR here for deoptimization patching and frame walking 7218 stp(rfp, lr, Address(pre(sp, -2 * wordSize))); 7219 7220 // Adjust the stack pointer. This will be repaired on return by MacroAssembler::remove_frame 7221 if (sp_inc < (1 << 9)) { 7222 sub(sp, sp, sp_inc); // Fits in an immediate 7223 } else { 7224 mov(rscratch1, sp_inc); 7225 sub(sp, sp, rscratch1); 7226 } 7227 7228 return sp_inc + 2 * wordSize; // Account for the FP/LR space 7229 } 7230 7231 // Read all fields from an inline type oop and store the values in registers/stack slots 7232 bool MacroAssembler::unpack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index, 7233 VMReg from, int& from_index, VMRegPair* to, int to_count, int& to_index, 7234 RegState reg_state[]) { 7235 assert(sig->at(sig_index)._bt == T_VOID, "should be at end delimiter"); 7236 assert(from->is_valid(), "source must be valid"); 7237 bool progress = false; 7238 #ifdef ASSERT 7239 const int start_offset = offset(); 7240 #endif 7241 7242 Label L_null, L_notNull; 7243 // Don't use r14 as tmp because it's used for spilling (see MacroAssembler::spill_reg_for) 7244 // TODO 8366717 We need to make sure that r14 (and potentially other long-life regs) are kept live in slowpath runtime calls in GC barriers 7245 Register tmp1 = r10; 7246 Register tmp2 = r11; 7247 Register fromReg = noreg; 7248 ScalarizedInlineArgsStream stream(sig, sig_index, to, to_count, to_index, -1); 7249 bool done = true; 7250 bool mark_done = true; 7251 VMReg toReg; 7252 BasicType bt; 7253 // Check if argument requires a null check 7254 bool null_check = false; 7255 VMReg nullCheckReg; 7256 while (stream.next(nullCheckReg, bt)) { 7257 if (sig->at(stream.sig_index())._offset == -1) { 7258 null_check = true; 7259 break; 7260 } 7261 } 7262 stream.reset(sig_index, to_index); 7263 while (stream.next(toReg, bt)) { 7264 assert(toReg->is_valid(), "destination must be valid"); 7265 int idx = (int)toReg->value(); 7266 if (reg_state[idx] == reg_readonly) { 7267 if (idx != from->value()) { 7268 mark_done = false; 7269 } 7270 done = false; 7271 continue; 7272 } else if (reg_state[idx] == reg_written) { 7273 continue; 7274 } 7275 assert(reg_state[idx] == reg_writable, "must be writable"); 7276 reg_state[idx] = reg_written; 7277 progress = true; 7278 7279 if (fromReg == noreg) { 7280 if (from->is_reg()) { 7281 fromReg = from->as_Register(); 7282 } else { 7283 int st_off = from->reg2stack() * VMRegImpl::stack_slot_size; 7284 ldr(tmp1, Address(sp, st_off)); 7285 fromReg = tmp1; 7286 } 7287 if (null_check) { 7288 // Nullable inline type argument, emit null check 7289 cbz(fromReg, L_null); 7290 } 7291 } 7292 int off = sig->at(stream.sig_index())._offset; 7293 if (off == -1) { 7294 assert(null_check, "Missing null check at"); 7295 if (toReg->is_stack()) { 7296 int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size; 7297 mov(tmp2, 1); 7298 str(tmp2, Address(sp, st_off)); 7299 } else { 7300 mov(toReg->as_Register(), 1); 7301 } 7302 continue; 7303 } 7304 assert(off > 0, "offset in object should be positive"); 7305 Address fromAddr = Address(fromReg, off); 7306 if (!toReg->is_FloatRegister()) { 7307 Register dst = toReg->is_stack() ? tmp2 : toReg->as_Register(); 7308 if (is_reference_type(bt)) { 7309 load_heap_oop(dst, fromAddr, rscratch1, rscratch2); 7310 } else { 7311 bool is_signed = (bt != T_CHAR) && (bt != T_BOOLEAN); 7312 load_sized_value(dst, fromAddr, type2aelembytes(bt), is_signed); 7313 } 7314 if (toReg->is_stack()) { 7315 int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size; 7316 str(dst, Address(sp, st_off)); 7317 } 7318 } else if (bt == T_DOUBLE) { 7319 ldrd(toReg->as_FloatRegister(), fromAddr); 7320 } else { 7321 assert(bt == T_FLOAT, "must be float"); 7322 ldrs(toReg->as_FloatRegister(), fromAddr); 7323 } 7324 } 7325 if (progress && null_check) { 7326 if (done) { 7327 b(L_notNull); 7328 bind(L_null); 7329 // Set null marker to zero to signal that the argument is null. 7330 // Also set all oop fields to zero to make the GC happy. 7331 stream.reset(sig_index, to_index); 7332 while (stream.next(toReg, bt)) { 7333 if (sig->at(stream.sig_index())._offset == -1 || 7334 bt == T_OBJECT || bt == T_ARRAY) { 7335 if (toReg->is_stack()) { 7336 int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size; 7337 str(zr, Address(sp, st_off)); 7338 } else { 7339 mov(toReg->as_Register(), zr); 7340 } 7341 } 7342 } 7343 bind(L_notNull); 7344 } else { 7345 bind(L_null); 7346 } 7347 } 7348 7349 // TODO 8366717 This is probably okay but looks fishy because stream is reset in the "Set null marker to zero" case just above. Same on x64. 7350 sig_index = stream.sig_index(); 7351 to_index = stream.regs_index(); 7352 7353 if (mark_done && reg_state[from->value()] != reg_written) { 7354 // This is okay because no one else will write to that slot 7355 reg_state[from->value()] = reg_writable; 7356 } 7357 from_index--; 7358 assert(progress || (start_offset == offset()), "should not emit code"); 7359 return done; 7360 } 7361 7362 // Pack fields back into an inline type oop 7363 bool MacroAssembler::pack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index, int vtarg_index, 7364 VMRegPair* from, int from_count, int& from_index, VMReg to, 7365 RegState reg_state[], Register val_array) { 7366 assert(sig->at(sig_index)._bt == T_METADATA, "should be at delimiter"); 7367 assert(to->is_valid(), "destination must be valid"); 7368 7369 if (reg_state[to->value()] == reg_written) { 7370 skip_unpacked_fields(sig, sig_index, from, from_count, from_index); 7371 return true; // Already written 7372 } 7373 7374 // The GC barrier expanded by store_heap_oop below may call into the 7375 // runtime so use callee-saved registers for any values that need to be 7376 // preserved. The GC barrier assembler should take care of saving the 7377 // Java argument registers. 7378 // TODO 8284443 Isn't it an issue if below code uses r14 as tmp when it contains a spilled value? 7379 // Be careful with r14 because it's used for spilling (see MacroAssembler::spill_reg_for). 7380 Register val_obj_tmp = r21; 7381 Register from_reg_tmp = r22; 7382 Register tmp1 = r14; 7383 Register tmp2 = r13; 7384 Register tmp3 = r12; 7385 Register val_obj = to->is_stack() ? val_obj_tmp : to->as_Register(); 7386 7387 assert_different_registers(val_obj_tmp, from_reg_tmp, tmp1, tmp2, tmp3, val_array); 7388 7389 if (reg_state[to->value()] == reg_readonly) { 7390 if (!is_reg_in_unpacked_fields(sig, sig_index, to, from, from_count, from_index)) { 7391 skip_unpacked_fields(sig, sig_index, from, from_count, from_index); 7392 return false; // Not yet writable 7393 } 7394 val_obj = val_obj_tmp; 7395 } 7396 7397 int index = arrayOopDesc::base_offset_in_bytes(T_OBJECT) + vtarg_index * type2aelembytes(T_OBJECT); 7398 load_heap_oop(val_obj, Address(val_array, index), tmp1, tmp2); 7399 7400 ScalarizedInlineArgsStream stream(sig, sig_index, from, from_count, from_index); 7401 VMReg fromReg; 7402 BasicType bt; 7403 Label L_null; 7404 while (stream.next(fromReg, bt)) { 7405 assert(fromReg->is_valid(), "source must be valid"); 7406 reg_state[fromReg->value()] = reg_writable; 7407 7408 int off = sig->at(stream.sig_index())._offset; 7409 if (off == -1) { 7410 // Nullable inline type argument, emit null check 7411 Label L_notNull; 7412 if (fromReg->is_stack()) { 7413 int ld_off = fromReg->reg2stack() * VMRegImpl::stack_slot_size; 7414 ldrb(tmp2, Address(sp, ld_off)); 7415 cbnz(tmp2, L_notNull); 7416 } else { 7417 cbnz(fromReg->as_Register(), L_notNull); 7418 } 7419 mov(val_obj, 0); 7420 b(L_null); 7421 bind(L_notNull); 7422 continue; 7423 } 7424 7425 assert(off > 0, "offset in object should be positive"); 7426 size_t size_in_bytes = is_java_primitive(bt) ? type2aelembytes(bt) : wordSize; 7427 7428 // Pack the scalarized field into the value object. 7429 Address dst(val_obj, off); 7430 if (!fromReg->is_FloatRegister()) { 7431 Register src; 7432 if (fromReg->is_stack()) { 7433 src = from_reg_tmp; 7434 int ld_off = fromReg->reg2stack() * VMRegImpl::stack_slot_size; 7435 load_sized_value(src, Address(sp, ld_off), size_in_bytes, /* is_signed */ false); 7436 } else { 7437 src = fromReg->as_Register(); 7438 } 7439 assert_different_registers(dst.base(), src, tmp1, tmp2, tmp3, val_array); 7440 if (is_reference_type(bt)) { 7441 store_heap_oop(dst, src, tmp1, tmp2, tmp3, IN_HEAP | ACCESS_WRITE | IS_DEST_UNINITIALIZED); 7442 } else { 7443 store_sized_value(dst, src, size_in_bytes); 7444 } 7445 } else if (bt == T_DOUBLE) { 7446 strd(fromReg->as_FloatRegister(), dst); 7447 } else { 7448 assert(bt == T_FLOAT, "must be float"); 7449 strs(fromReg->as_FloatRegister(), dst); 7450 } 7451 } 7452 bind(L_null); 7453 sig_index = stream.sig_index(); 7454 from_index = stream.regs_index(); 7455 7456 assert(reg_state[to->value()] == reg_writable, "must have already been read"); 7457 bool success = move_helper(val_obj->as_VMReg(), to, T_OBJECT, reg_state); 7458 assert(success, "to register must be writeable"); 7459 return true; 7460 } 7461 7462 VMReg MacroAssembler::spill_reg_for(VMReg reg) { 7463 return (reg->is_FloatRegister()) ? v8->as_VMReg() : r14->as_VMReg(); 7464 } 7465 7466 void MacroAssembler::cache_wb(Address line) { 7467 assert(line.getMode() == Address::base_plus_offset, "mode should be base_plus_offset"); 7468 assert(line.index() == noreg, "index should be noreg"); 7469 assert(line.offset() == 0, "offset should be 0"); 7470 // would like to assert this 7471 // assert(line._ext.shift == 0, "shift should be zero"); 7472 if (VM_Version::supports_dcpop()) { 7473 // writeback using clear virtual address to point of persistence 7474 dc(Assembler::CVAP, line.base()); 7475 } else { 7476 // no need to generate anything as Unsafe.writebackMemory should 7477 // never invoke this stub 7478 } 7479 } 7480 7481 void MacroAssembler::cache_wbsync(bool is_pre) { 7482 // we only need a barrier post sync 7483 if (!is_pre) { 7484 membar(Assembler::AnyAny); 7485 } 7486 } 7487 7488 void MacroAssembler::verify_sve_vector_length(Register tmp) { 7489 if (!UseSVE || VM_Version::get_max_supported_sve_vector_length() == FloatRegister::sve_vl_min) { 7490 return; 7491 } 7492 // Make sure that native code does not change SVE vector length. 7493 Label verify_ok; 7494 movw(tmp, zr); 7495 sve_inc(tmp, B); 7496 subsw(zr, tmp, VM_Version::get_initial_sve_vector_length()); 7497 br(EQ, verify_ok); 7498 stop("Error: SVE vector length has changed since jvm startup"); 7499 bind(verify_ok); 7500 } 7501 7502 void MacroAssembler::verify_ptrue() { 7503 Label verify_ok; 7504 if (!UseSVE) { 7505 return; 7506 } 7507 sve_cntp(rscratch1, B, ptrue, ptrue); // get true elements count. 7508 sve_dec(rscratch1, B); 7509 cbz(rscratch1, verify_ok); 7510 stop("Error: the preserved predicate register (p7) elements are not all true"); 7511 bind(verify_ok); 7512 } 7513 7514 void MacroAssembler::safepoint_isb() { 7515 isb(); 7516 #ifndef PRODUCT 7517 if (VerifyCrossModifyFence) { 7518 // Clear the thread state. 7519 strb(zr, Address(rthread, in_bytes(JavaThread::requires_cross_modify_fence_offset()))); 7520 } 7521 #endif 7522 } 7523 7524 #ifndef PRODUCT 7525 void MacroAssembler::verify_cross_modify_fence_not_required() { 7526 if (VerifyCrossModifyFence) { 7527 // Check if thread needs a cross modify fence. 7528 ldrb(rscratch1, Address(rthread, in_bytes(JavaThread::requires_cross_modify_fence_offset()))); 7529 Label fence_not_required; 7530 cbz(rscratch1, fence_not_required); 7531 // If it does then fail. 7532 lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, JavaThread::verify_cross_modify_fence_failure))); 7533 mov(c_rarg0, rthread); 7534 blr(rscratch1); 7535 bind(fence_not_required); 7536 } 7537 } 7538 #endif 7539 7540 void MacroAssembler::spin_wait() { 7541 block_comment("spin_wait {"); 7542 for (int i = 0; i < VM_Version::spin_wait_desc().inst_count(); ++i) { 7543 switch (VM_Version::spin_wait_desc().inst()) { 7544 case SpinWait::NOP: 7545 nop(); 7546 break; 7547 case SpinWait::ISB: 7548 isb(); 7549 break; 7550 case SpinWait::YIELD: 7551 yield(); 7552 break; 7553 case SpinWait::SB: 7554 assert(VM_Version::supports_sb(), "current CPU does not support SB instruction"); 7555 sb(); 7556 break; 7557 default: 7558 ShouldNotReachHere(); 7559 } 7560 } 7561 block_comment("}"); 7562 } 7563 7564 // Stack frame creation/removal 7565 7566 void MacroAssembler::enter(bool strip_ret_addr) { 7567 if (strip_ret_addr) { 7568 // Addresses can only be signed once. If there are multiple nested frames being created 7569 // in the same function, then the return address needs stripping first. 7570 strip_return_address(); 7571 } 7572 protect_return_address(); 7573 stp(rfp, lr, Address(pre(sp, -2 * wordSize))); 7574 mov(rfp, sp); 7575 } 7576 7577 void MacroAssembler::leave() { 7578 mov(sp, rfp); 7579 ldp(rfp, lr, Address(post(sp, 2 * wordSize))); 7580 authenticate_return_address(); 7581 } 7582 7583 // ROP Protection 7584 // Use the AArch64 PAC feature to add ROP protection for generated code. Use whenever creating/ 7585 // destroying stack frames or whenever directly loading/storing the LR to memory. 7586 // If ROP protection is not set then these functions are no-ops. 7587 // For more details on PAC see pauth_aarch64.hpp. 7588 7589 // Sign the LR. Use during construction of a stack frame, before storing the LR to memory. 7590 // Uses value zero as the modifier. 7591 // 7592 void MacroAssembler::protect_return_address() { 7593 if (VM_Version::use_rop_protection()) { 7594 check_return_address(); 7595 paciaz(); 7596 } 7597 } 7598 7599 // Sign the return value in the given register. Use before updating the LR in the existing stack 7600 // frame for the current function. 7601 // Uses value zero as the modifier. 7602 // 7603 void MacroAssembler::protect_return_address(Register return_reg) { 7604 if (VM_Version::use_rop_protection()) { 7605 check_return_address(return_reg); 7606 paciza(return_reg); 7607 } 7608 } 7609 7610 // Authenticate the LR. Use before function return, after restoring FP and loading LR from memory. 7611 // Uses value zero as the modifier. 7612 // 7613 void MacroAssembler::authenticate_return_address() { 7614 if (VM_Version::use_rop_protection()) { 7615 autiaz(); 7616 check_return_address(); 7617 } 7618 } 7619 7620 // Authenticate the return value in the given register. Use before updating the LR in the existing 7621 // stack frame for the current function. 7622 // Uses value zero as the modifier. 7623 // 7624 void MacroAssembler::authenticate_return_address(Register return_reg) { 7625 if (VM_Version::use_rop_protection()) { 7626 autiza(return_reg); 7627 check_return_address(return_reg); 7628 } 7629 } 7630 7631 // Strip any PAC data from LR without performing any authentication. Use with caution - only if 7632 // there is no guaranteed way of authenticating the LR. 7633 // 7634 void MacroAssembler::strip_return_address() { 7635 if (VM_Version::use_rop_protection()) { 7636 xpaclri(); 7637 } 7638 } 7639 7640 #ifndef PRODUCT 7641 // PAC failures can be difficult to debug. After an authentication failure, a segfault will only 7642 // occur when the pointer is used - ie when the program returns to the invalid LR. At this point 7643 // it is difficult to debug back to the callee function. 7644 // This function simply loads from the address in the given register. 7645 // Use directly after authentication to catch authentication failures. 7646 // Also use before signing to check that the pointer is valid and hasn't already been signed. 7647 // 7648 void MacroAssembler::check_return_address(Register return_reg) { 7649 if (VM_Version::use_rop_protection()) { 7650 ldr(zr, Address(return_reg)); 7651 } 7652 } 7653 #endif 7654 7655 // The java_calling_convention describes stack locations as ideal slots on 7656 // a frame with no abi restrictions. Since we must observe abi restrictions 7657 // (like the placement of the register window) the slots must be biased by 7658 // the following value. 7659 static int reg2offset_in(VMReg r) { 7660 // Account for saved rfp and lr 7661 // This should really be in_preserve_stack_slots 7662 return (r->reg2stack() + 4) * VMRegImpl::stack_slot_size; 7663 } 7664 7665 static int reg2offset_out(VMReg r) { 7666 return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size; 7667 } 7668 7669 // On 64bit we will store integer like items to the stack as 7670 // 64bits items (AArch64 ABI) even though java would only store 7671 // 32bits for a parameter. On 32bit it will simply be 32bits 7672 // So this routine will do 32->32 on 32bit and 32->64 on 64bit 7673 void MacroAssembler::move32_64(VMRegPair src, VMRegPair dst, Register tmp) { 7674 if (src.first()->is_stack()) { 7675 if (dst.first()->is_stack()) { 7676 // stack to stack 7677 ldr(tmp, Address(rfp, reg2offset_in(src.first()))); 7678 str(tmp, Address(sp, reg2offset_out(dst.first()))); 7679 } else { 7680 // stack to reg 7681 ldrsw(dst.first()->as_Register(), Address(rfp, reg2offset_in(src.first()))); 7682 } 7683 } else if (dst.first()->is_stack()) { 7684 // reg to stack 7685 str(src.first()->as_Register(), Address(sp, reg2offset_out(dst.first()))); 7686 } else { 7687 if (dst.first() != src.first()) { 7688 sxtw(dst.first()->as_Register(), src.first()->as_Register()); 7689 } 7690 } 7691 } 7692 7693 // An oop arg. Must pass a handle not the oop itself 7694 void MacroAssembler::object_move( 7695 OopMap* map, 7696 int oop_handle_offset, 7697 int framesize_in_slots, 7698 VMRegPair src, 7699 VMRegPair dst, 7700 bool is_receiver, 7701 int* receiver_offset) { 7702 7703 // must pass a handle. First figure out the location we use as a handle 7704 7705 Register rHandle = dst.first()->is_stack() ? rscratch2 : dst.first()->as_Register(); 7706 7707 // See if oop is null if it is we need no handle 7708 7709 if (src.first()->is_stack()) { 7710 7711 // Oop is already on the stack as an argument 7712 int offset_in_older_frame = src.first()->reg2stack() + SharedRuntime::out_preserve_stack_slots(); 7713 map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + framesize_in_slots)); 7714 if (is_receiver) { 7715 *receiver_offset = (offset_in_older_frame + framesize_in_slots) * VMRegImpl::stack_slot_size; 7716 } 7717 7718 ldr(rscratch1, Address(rfp, reg2offset_in(src.first()))); 7719 lea(rHandle, Address(rfp, reg2offset_in(src.first()))); 7720 // conditionally move a null 7721 cmp(rscratch1, zr); 7722 csel(rHandle, zr, rHandle, Assembler::EQ); 7723 } else { 7724 7725 // Oop is in an a register we must store it to the space we reserve 7726 // on the stack for oop_handles and pass a handle if oop is non-null 7727 7728 const Register rOop = src.first()->as_Register(); 7729 int oop_slot; 7730 if (rOop == j_rarg0) 7731 oop_slot = 0; 7732 else if (rOop == j_rarg1) 7733 oop_slot = 1; 7734 else if (rOop == j_rarg2) 7735 oop_slot = 2; 7736 else if (rOop == j_rarg3) 7737 oop_slot = 3; 7738 else if (rOop == j_rarg4) 7739 oop_slot = 4; 7740 else if (rOop == j_rarg5) 7741 oop_slot = 5; 7742 else if (rOop == j_rarg6) 7743 oop_slot = 6; 7744 else { 7745 assert(rOop == j_rarg7, "wrong register"); 7746 oop_slot = 7; 7747 } 7748 7749 oop_slot = oop_slot * VMRegImpl::slots_per_word + oop_handle_offset; 7750 int offset = oop_slot*VMRegImpl::stack_slot_size; 7751 7752 map->set_oop(VMRegImpl::stack2reg(oop_slot)); 7753 // Store oop in handle area, may be null 7754 str(rOop, Address(sp, offset)); 7755 if (is_receiver) { 7756 *receiver_offset = offset; 7757 } 7758 7759 cmp(rOop, zr); 7760 lea(rHandle, Address(sp, offset)); 7761 // conditionally move a null 7762 csel(rHandle, zr, rHandle, Assembler::EQ); 7763 } 7764 7765 // If arg is on the stack then place it otherwise it is already in correct reg. 7766 if (dst.first()->is_stack()) { 7767 str(rHandle, Address(sp, reg2offset_out(dst.first()))); 7768 } 7769 } 7770 7771 // A float arg may have to do float reg int reg conversion 7772 void MacroAssembler::float_move(VMRegPair src, VMRegPair dst, Register tmp) { 7773 if (src.first()->is_stack()) { 7774 if (dst.first()->is_stack()) { 7775 ldrw(tmp, Address(rfp, reg2offset_in(src.first()))); 7776 strw(tmp, Address(sp, reg2offset_out(dst.first()))); 7777 } else { 7778 ldrs(dst.first()->as_FloatRegister(), Address(rfp, reg2offset_in(src.first()))); 7779 } 7780 } else if (src.first() != dst.first()) { 7781 if (src.is_single_phys_reg() && dst.is_single_phys_reg()) 7782 fmovs(dst.first()->as_FloatRegister(), src.first()->as_FloatRegister()); 7783 else 7784 strs(src.first()->as_FloatRegister(), Address(sp, reg2offset_out(dst.first()))); 7785 } 7786 } 7787 7788 // A long move 7789 void MacroAssembler::long_move(VMRegPair src, VMRegPair dst, Register tmp) { 7790 if (src.first()->is_stack()) { 7791 if (dst.first()->is_stack()) { 7792 // stack to stack 7793 ldr(tmp, Address(rfp, reg2offset_in(src.first()))); 7794 str(tmp, Address(sp, reg2offset_out(dst.first()))); 7795 } else { 7796 // stack to reg 7797 ldr(dst.first()->as_Register(), Address(rfp, reg2offset_in(src.first()))); 7798 } 7799 } else if (dst.first()->is_stack()) { 7800 // reg to stack 7801 // Do we really have to sign extend??? 7802 // __ movslq(src.first()->as_Register(), src.first()->as_Register()); 7803 str(src.first()->as_Register(), Address(sp, reg2offset_out(dst.first()))); 7804 } else { 7805 if (dst.first() != src.first()) { 7806 mov(dst.first()->as_Register(), src.first()->as_Register()); 7807 } 7808 } 7809 } 7810 7811 7812 // A double move 7813 void MacroAssembler::double_move(VMRegPair src, VMRegPair dst, Register tmp) { 7814 if (src.first()->is_stack()) { 7815 if (dst.first()->is_stack()) { 7816 ldr(tmp, Address(rfp, reg2offset_in(src.first()))); 7817 str(tmp, Address(sp, reg2offset_out(dst.first()))); 7818 } else { 7819 ldrd(dst.first()->as_FloatRegister(), Address(rfp, reg2offset_in(src.first()))); 7820 } 7821 } else if (src.first() != dst.first()) { 7822 if (src.is_single_phys_reg() && dst.is_single_phys_reg()) 7823 fmovd(dst.first()->as_FloatRegister(), src.first()->as_FloatRegister()); 7824 else 7825 strd(src.first()->as_FloatRegister(), Address(sp, reg2offset_out(dst.first()))); 7826 } 7827 } 7828 7829 // Implements lightweight-locking. 7830 // 7831 // - obj: the object to be locked 7832 // - t1, t2, t3: temporary registers, will be destroyed 7833 // - slow: branched to if locking fails, absolute offset may larger than 32KB (imm14 encoding). 7834 void MacroAssembler::lightweight_lock(Register basic_lock, Register obj, Register t1, Register t2, Register t3, Label& slow) { 7835 assert_different_registers(basic_lock, obj, t1, t2, t3, rscratch1); 7836 7837 Label push; 7838 const Register top = t1; 7839 const Register mark = t2; 7840 const Register t = t3; 7841 7842 // Preload the markWord. It is important that this is the first 7843 // instruction emitted as it is part of C1's null check semantics. 7844 ldr(mark, Address(obj, oopDesc::mark_offset_in_bytes())); 7845 7846 if (UseObjectMonitorTable) { 7847 // Clear cache in case fast locking succeeds or we need to take the slow-path. 7848 str(zr, Address(basic_lock, BasicObjectLock::lock_offset() + in_ByteSize((BasicLock::object_monitor_cache_offset_in_bytes())))); 7849 } 7850 7851 if (DiagnoseSyncOnValueBasedClasses != 0) { 7852 load_klass(t1, obj); 7853 ldrb(t1, Address(t1, Klass::misc_flags_offset())); 7854 tst(t1, KlassFlags::_misc_is_value_based_class); 7855 br(Assembler::NE, slow); 7856 } 7857 7858 // Check if the lock-stack is full. 7859 ldrw(top, Address(rthread, JavaThread::lock_stack_top_offset())); 7860 cmpw(top, (unsigned)LockStack::end_offset()); 7861 br(Assembler::GE, slow); 7862 7863 // Check for recursion. 7864 subw(t, top, oopSize); 7865 ldr(t, Address(rthread, t)); 7866 cmp(obj, t); 7867 br(Assembler::EQ, push); 7868 7869 // Check header for monitor (0b10). 7870 tst(mark, markWord::monitor_value); 7871 br(Assembler::NE, slow); 7872 7873 // Try to lock. Transition lock bits 0b01 => 0b00 7874 assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea"); 7875 orr(mark, mark, markWord::unlocked_value); 7876 // Mask inline_type bit such that we go to the slow path if object is an inline type 7877 andr(mark, mark, ~((int) markWord::inline_type_bit_in_place)); 7878 7879 eor(t, mark, markWord::unlocked_value); 7880 cmpxchg(/*addr*/ obj, /*expected*/ mark, /*new*/ t, Assembler::xword, 7881 /*acquire*/ true, /*release*/ false, /*weak*/ false, noreg); 7882 br(Assembler::NE, slow); 7883 7884 bind(push); 7885 // After successful lock, push object on lock-stack. 7886 str(obj, Address(rthread, top)); 7887 addw(top, top, oopSize); 7888 strw(top, Address(rthread, JavaThread::lock_stack_top_offset())); 7889 } 7890 7891 // Implements lightweight-unlocking. 7892 // 7893 // - obj: the object to be unlocked 7894 // - t1, t2, t3: temporary registers 7895 // - slow: branched to if unlocking fails, absolute offset may larger than 32KB (imm14 encoding). 7896 void MacroAssembler::lightweight_unlock(Register obj, Register t1, Register t2, Register t3, Label& slow) { 7897 // cmpxchg clobbers rscratch1. 7898 assert_different_registers(obj, t1, t2, t3, rscratch1); 7899 7900 #ifdef ASSERT 7901 { 7902 // Check for lock-stack underflow. 7903 Label stack_ok; 7904 ldrw(t1, Address(rthread, JavaThread::lock_stack_top_offset())); 7905 cmpw(t1, (unsigned)LockStack::start_offset()); 7906 br(Assembler::GE, stack_ok); 7907 STOP("Lock-stack underflow"); 7908 bind(stack_ok); 7909 } 7910 #endif 7911 7912 Label unlocked, push_and_slow; 7913 const Register top = t1; 7914 const Register mark = t2; 7915 const Register t = t3; 7916 7917 // Check if obj is top of lock-stack. 7918 ldrw(top, Address(rthread, JavaThread::lock_stack_top_offset())); 7919 subw(top, top, oopSize); 7920 ldr(t, Address(rthread, top)); 7921 cmp(obj, t); 7922 br(Assembler::NE, slow); 7923 7924 // Pop lock-stack. 7925 DEBUG_ONLY(str(zr, Address(rthread, top));) 7926 strw(top, Address(rthread, JavaThread::lock_stack_top_offset())); 7927 7928 // Check if recursive. 7929 subw(t, top, oopSize); 7930 ldr(t, Address(rthread, t)); 7931 cmp(obj, t); 7932 br(Assembler::EQ, unlocked); 7933 7934 // Not recursive. Check header for monitor (0b10). 7935 ldr(mark, Address(obj, oopDesc::mark_offset_in_bytes())); 7936 tbnz(mark, log2i_exact(markWord::monitor_value), push_and_slow); 7937 7938 #ifdef ASSERT 7939 // Check header not unlocked (0b01). 7940 Label not_unlocked; 7941 tbz(mark, log2i_exact(markWord::unlocked_value), not_unlocked); 7942 stop("lightweight_unlock already unlocked"); 7943 bind(not_unlocked); 7944 #endif 7945 7946 // Try to unlock. Transition lock bits 0b00 => 0b01 7947 assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea"); 7948 orr(t, mark, markWord::unlocked_value); 7949 cmpxchg(obj, mark, t, Assembler::xword, 7950 /*acquire*/ false, /*release*/ true, /*weak*/ false, noreg); 7951 br(Assembler::EQ, unlocked); 7952 7953 bind(push_and_slow); 7954 // Restore lock-stack and handle the unlock in runtime. 7955 DEBUG_ONLY(str(obj, Address(rthread, top));) 7956 addw(top, top, oopSize); 7957 strw(top, Address(rthread, JavaThread::lock_stack_top_offset())); 7958 b(slow); 7959 7960 bind(unlocked); 7961 } --- EOF ---