1 /* 2 * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "asm/assembler.hpp" 26 #include "asm/assembler.inline.hpp" 27 #include "code/compiledIC.hpp" 28 #include "compiler/compiler_globals.hpp" 29 #include "compiler/disassembler.hpp" 30 #include "crc32c.h" 31 #include "gc/shared/barrierSet.hpp" 32 #include "gc/shared/barrierSetAssembler.hpp" 33 #include "gc/shared/collectedHeap.inline.hpp" 34 #include "gc/shared/tlab_globals.hpp" 35 #include "interpreter/bytecodeHistogram.hpp" 36 #include "interpreter/interpreter.hpp" 37 #include "interpreter/interpreterRuntime.hpp" 38 #include "jvm.h" 39 #include "memory/resourceArea.hpp" 40 #include "memory/universe.hpp" 41 #include "oops/accessDecorators.hpp" 42 #include "oops/compressedKlass.inline.hpp" 43 #include "oops/compressedOops.inline.hpp" 44 #include "oops/klass.inline.hpp" 45 #include "prims/methodHandles.hpp" 46 #include "runtime/continuation.hpp" 47 #include "runtime/interfaceSupport.inline.hpp" 48 #include "runtime/javaThread.hpp" 49 #include "runtime/jniHandles.hpp" 50 #include "runtime/objectMonitor.hpp" 51 #include "runtime/os.hpp" 52 #include "runtime/safepoint.hpp" 53 #include "runtime/safepointMechanism.hpp" 54 #include "runtime/sharedRuntime.hpp" 55 #include "runtime/stubRoutines.hpp" 56 #include "utilities/checkedCast.hpp" 57 #include "utilities/macros.hpp" 58 59 #ifdef PRODUCT 60 #define BLOCK_COMMENT(str) /* nothing */ 61 #define STOP(error) stop(error) 62 #else 63 #define BLOCK_COMMENT(str) block_comment(str) 64 #define STOP(error) block_comment(error); stop(error) 65 #endif 66 67 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":") 68 69 #ifdef ASSERT 70 bool AbstractAssembler::pd_check_instruction_mark() { return true; } 71 #endif 72 73 static const Assembler::Condition reverse[] = { 74 Assembler::noOverflow /* overflow = 0x0 */ , 75 Assembler::overflow /* noOverflow = 0x1 */ , 76 Assembler::aboveEqual /* carrySet = 0x2, below = 0x2 */ , 77 Assembler::below /* aboveEqual = 0x3, carryClear = 0x3 */ , 78 Assembler::notZero /* zero = 0x4, equal = 0x4 */ , 79 Assembler::zero /* notZero = 0x5, notEqual = 0x5 */ , 80 Assembler::above /* belowEqual = 0x6 */ , 81 Assembler::belowEqual /* above = 0x7 */ , 82 Assembler::positive /* negative = 0x8 */ , 83 Assembler::negative /* positive = 0x9 */ , 84 Assembler::noParity /* parity = 0xa */ , 85 Assembler::parity /* noParity = 0xb */ , 86 Assembler::greaterEqual /* less = 0xc */ , 87 Assembler::less /* greaterEqual = 0xd */ , 88 Assembler::greater /* lessEqual = 0xe */ , 89 Assembler::lessEqual /* greater = 0xf, */ 90 91 }; 92 93 94 // Implementation of MacroAssembler 95 96 Address MacroAssembler::as_Address(AddressLiteral adr) { 97 // amd64 always does this as a pc-rel 98 // we can be absolute or disp based on the instruction type 99 // jmp/call are displacements others are absolute 100 assert(!adr.is_lval(), "must be rval"); 101 assert(reachable(adr), "must be"); 102 return Address(checked_cast<int32_t>(adr.target() - pc()), adr.target(), adr.reloc()); 103 104 } 105 106 Address MacroAssembler::as_Address(ArrayAddress adr, Register rscratch) { 107 AddressLiteral base = adr.base(); 108 lea(rscratch, base); 109 Address index = adr.index(); 110 assert(index._disp == 0, "must not have disp"); // maybe it can? 111 Address array(rscratch, index._index, index._scale, index._disp); 112 return array; 113 } 114 115 void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) { 116 Label L, E; 117 118 #ifdef _WIN64 119 // Windows always allocates space for it's register args 120 assert(num_args <= 4, "only register arguments supported"); 121 subq(rsp, frame::arg_reg_save_area_bytes); 122 #endif 123 124 // Align stack if necessary 125 testl(rsp, 15); 126 jcc(Assembler::zero, L); 127 128 subq(rsp, 8); 129 call(RuntimeAddress(entry_point)); 130 addq(rsp, 8); 131 jmp(E); 132 133 bind(L); 134 call(RuntimeAddress(entry_point)); 135 136 bind(E); 137 138 #ifdef _WIN64 139 // restore stack pointer 140 addq(rsp, frame::arg_reg_save_area_bytes); 141 #endif 142 } 143 144 void MacroAssembler::cmp64(Register src1, AddressLiteral src2, Register rscratch) { 145 assert(!src2.is_lval(), "should use cmpptr"); 146 assert(rscratch != noreg || always_reachable(src2), "missing"); 147 148 if (reachable(src2)) { 149 cmpq(src1, as_Address(src2)); 150 } else { 151 lea(rscratch, src2); 152 Assembler::cmpq(src1, Address(rscratch, 0)); 153 } 154 } 155 156 int MacroAssembler::corrected_idivq(Register reg) { 157 // Full implementation of Java ldiv and lrem; checks for special 158 // case as described in JVM spec., p.243 & p.271. The function 159 // returns the (pc) offset of the idivl instruction - may be needed 160 // for implicit exceptions. 161 // 162 // normal case special case 163 // 164 // input : rax: dividend min_long 165 // reg: divisor (may not be eax/edx) -1 166 // 167 // output: rax: quotient (= rax idiv reg) min_long 168 // rdx: remainder (= rax irem reg) 0 169 assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register"); 170 static const int64_t min_long = 0x8000000000000000; 171 Label normal_case, special_case; 172 173 // check for special case 174 cmp64(rax, ExternalAddress((address) &min_long), rdx /*rscratch*/); 175 jcc(Assembler::notEqual, normal_case); 176 xorl(rdx, rdx); // prepare rdx for possible special case (where 177 // remainder = 0) 178 cmpq(reg, -1); 179 jcc(Assembler::equal, special_case); 180 181 // handle normal case 182 bind(normal_case); 183 cdqq(); 184 int idivq_offset = offset(); 185 idivq(reg); 186 187 // normal and special case exit 188 bind(special_case); 189 190 return idivq_offset; 191 } 192 193 void MacroAssembler::decrementq(Register reg, int value) { 194 if (value == min_jint) { subq(reg, value); return; } 195 if (value < 0) { incrementq(reg, -value); return; } 196 if (value == 0) { ; return; } 197 if (value == 1 && UseIncDec) { decq(reg) ; return; } 198 /* else */ { subq(reg, value) ; return; } 199 } 200 201 void MacroAssembler::decrementq(Address dst, int value) { 202 if (value == min_jint) { subq(dst, value); return; } 203 if (value < 0) { incrementq(dst, -value); return; } 204 if (value == 0) { ; return; } 205 if (value == 1 && UseIncDec) { decq(dst) ; return; } 206 /* else */ { subq(dst, value) ; return; } 207 } 208 209 void MacroAssembler::incrementq(AddressLiteral dst, Register rscratch) { 210 assert(rscratch != noreg || always_reachable(dst), "missing"); 211 212 if (reachable(dst)) { 213 incrementq(as_Address(dst)); 214 } else { 215 lea(rscratch, dst); 216 incrementq(Address(rscratch, 0)); 217 } 218 } 219 220 void MacroAssembler::incrementq(Register reg, int value) { 221 if (value == min_jint) { addq(reg, value); return; } 222 if (value < 0) { decrementq(reg, -value); return; } 223 if (value == 0) { ; return; } 224 if (value == 1 && UseIncDec) { incq(reg) ; return; } 225 /* else */ { addq(reg, value) ; return; } 226 } 227 228 void MacroAssembler::incrementq(Address dst, int value) { 229 if (value == min_jint) { addq(dst, value); return; } 230 if (value < 0) { decrementq(dst, -value); return; } 231 if (value == 0) { ; return; } 232 if (value == 1 && UseIncDec) { incq(dst) ; return; } 233 /* else */ { addq(dst, value) ; return; } 234 } 235 236 // 32bit can do a case table jump in one instruction but we no longer allow the base 237 // to be installed in the Address class 238 void MacroAssembler::jump(ArrayAddress entry, Register rscratch) { 239 lea(rscratch, entry.base()); 240 Address dispatch = entry.index(); 241 assert(dispatch._base == noreg, "must be"); 242 dispatch._base = rscratch; 243 jmp(dispatch); 244 } 245 246 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) { 247 ShouldNotReachHere(); // 64bit doesn't use two regs 248 cmpq(x_lo, y_lo); 249 } 250 251 void MacroAssembler::lea(Register dst, AddressLiteral src) { 252 mov_literal64(dst, (intptr_t)src.target(), src.rspec()); 253 } 254 255 void MacroAssembler::lea(Address dst, AddressLiteral adr, Register rscratch) { 256 lea(rscratch, adr); 257 movptr(dst, rscratch); 258 } 259 260 void MacroAssembler::leave() { 261 // %%% is this really better? Why not on 32bit too? 262 emit_int8((unsigned char)0xC9); // LEAVE 263 } 264 265 void MacroAssembler::lneg(Register hi, Register lo) { 266 ShouldNotReachHere(); // 64bit doesn't use two regs 267 negq(lo); 268 } 269 270 void MacroAssembler::movoop(Register dst, jobject obj) { 271 mov_literal64(dst, (intptr_t)obj, oop_Relocation::spec_for_immediate()); 272 } 273 274 void MacroAssembler::movoop(Address dst, jobject obj, Register rscratch) { 275 mov_literal64(rscratch, (intptr_t)obj, oop_Relocation::spec_for_immediate()); 276 movq(dst, rscratch); 277 } 278 279 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) { 280 mov_literal64(dst, (intptr_t)obj, metadata_Relocation::spec_for_immediate()); 281 } 282 283 void MacroAssembler::mov_metadata(Address dst, Metadata* obj, Register rscratch) { 284 mov_literal64(rscratch, (intptr_t)obj, metadata_Relocation::spec_for_immediate()); 285 movq(dst, rscratch); 286 } 287 288 void MacroAssembler::movptr(Register dst, AddressLiteral src) { 289 if (src.is_lval()) { 290 mov_literal64(dst, (intptr_t)src.target(), src.rspec()); 291 } else { 292 if (reachable(src)) { 293 movq(dst, as_Address(src)); 294 } else { 295 lea(dst, src); 296 movq(dst, Address(dst, 0)); 297 } 298 } 299 } 300 301 void MacroAssembler::movptr(ArrayAddress dst, Register src, Register rscratch) { 302 movq(as_Address(dst, rscratch), src); 303 } 304 305 void MacroAssembler::movptr(Register dst, ArrayAddress src) { 306 movq(dst, as_Address(src, dst /*rscratch*/)); 307 } 308 309 // src should NEVER be a real pointer. Use AddressLiteral for true pointers 310 void MacroAssembler::movptr(Address dst, intptr_t src, Register rscratch) { 311 if (is_simm32(src)) { 312 movptr(dst, checked_cast<int32_t>(src)); 313 } else { 314 mov64(rscratch, src); 315 movq(dst, rscratch); 316 } 317 } 318 319 void MacroAssembler::pushoop(jobject obj, Register rscratch) { 320 movoop(rscratch, obj); 321 push(rscratch); 322 } 323 324 void MacroAssembler::pushklass(Metadata* obj, Register rscratch) { 325 mov_metadata(rscratch, obj); 326 push(rscratch); 327 } 328 329 void MacroAssembler::pushptr(AddressLiteral src, Register rscratch) { 330 lea(rscratch, src); 331 if (src.is_lval()) { 332 push(rscratch); 333 } else { 334 pushq(Address(rscratch, 0)); 335 } 336 } 337 338 static void pass_arg0(MacroAssembler* masm, Register arg) { 339 if (c_rarg0 != arg ) { 340 masm->mov(c_rarg0, arg); 341 } 342 } 343 344 static void pass_arg1(MacroAssembler* masm, Register arg) { 345 if (c_rarg1 != arg ) { 346 masm->mov(c_rarg1, arg); 347 } 348 } 349 350 static void pass_arg2(MacroAssembler* masm, Register arg) { 351 if (c_rarg2 != arg ) { 352 masm->mov(c_rarg2, arg); 353 } 354 } 355 356 static void pass_arg3(MacroAssembler* masm, Register arg) { 357 if (c_rarg3 != arg ) { 358 masm->mov(c_rarg3, arg); 359 } 360 } 361 362 void MacroAssembler::stop(const char* msg) { 363 if (ShowMessageBoxOnError) { 364 address rip = pc(); 365 pusha(); // get regs on stack 366 lea(c_rarg1, InternalAddress(rip)); 367 movq(c_rarg2, rsp); // pass pointer to regs array 368 } 369 lea(c_rarg0, ExternalAddress((address) msg)); 370 andq(rsp, -16); // align stack as required by ABI 371 call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64))); 372 hlt(); 373 } 374 375 void MacroAssembler::warn(const char* msg) { 376 push(rbp); 377 movq(rbp, rsp); 378 andq(rsp, -16); // align stack as required by push_CPU_state and call 379 push_CPU_state(); // keeps alignment at 16 bytes 380 381 #ifdef _WIN64 382 // Windows always allocates space for its register args 383 subq(rsp, frame::arg_reg_save_area_bytes); 384 #endif 385 lea(c_rarg0, ExternalAddress((address) msg)); 386 call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning))); 387 388 #ifdef _WIN64 389 // restore stack pointer 390 addq(rsp, frame::arg_reg_save_area_bytes); 391 #endif 392 pop_CPU_state(); 393 mov(rsp, rbp); 394 pop(rbp); 395 } 396 397 void MacroAssembler::print_state() { 398 address rip = pc(); 399 pusha(); // get regs on stack 400 push(rbp); 401 movq(rbp, rsp); 402 andq(rsp, -16); // align stack as required by push_CPU_state and call 403 push_CPU_state(); // keeps alignment at 16 bytes 404 405 lea(c_rarg0, InternalAddress(rip)); 406 lea(c_rarg1, Address(rbp, wordSize)); // pass pointer to regs array 407 call_VM_leaf(CAST_FROM_FN_PTR(address, MacroAssembler::print_state64), c_rarg0, c_rarg1); 408 409 pop_CPU_state(); 410 mov(rsp, rbp); 411 pop(rbp); 412 popa(); 413 } 414 415 #ifndef PRODUCT 416 extern "C" void findpc(intptr_t x); 417 #endif 418 419 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) { 420 // In order to get locks to work, we need to fake a in_VM state 421 if (ShowMessageBoxOnError) { 422 JavaThread* thread = JavaThread::current(); 423 JavaThreadState saved_state = thread->thread_state(); 424 thread->set_thread_state(_thread_in_vm); 425 #ifndef PRODUCT 426 if (CountBytecodes || TraceBytecodes || StopInterpreterAt) { 427 ttyLocker ttyl; 428 BytecodeCounter::print(); 429 } 430 #endif 431 // To see where a verify_oop failed, get $ebx+40/X for this frame. 432 // XXX correct this offset for amd64 433 // This is the value of eip which points to where verify_oop will return. 434 if (os::message_box(msg, "Execution stopped, print registers?")) { 435 print_state64(pc, regs); 436 BREAKPOINT; 437 } 438 } 439 fatal("DEBUG MESSAGE: %s", msg); 440 } 441 442 void MacroAssembler::print_state64(int64_t pc, int64_t regs[]) { 443 ttyLocker ttyl; 444 DebuggingContext debugging{}; 445 tty->print_cr("rip = 0x%016lx", (intptr_t)pc); 446 #ifndef PRODUCT 447 tty->cr(); 448 findpc(pc); 449 tty->cr(); 450 #endif 451 #define PRINT_REG(rax, value) \ 452 { tty->print("%s = ", #rax); os::print_location(tty, value); } 453 PRINT_REG(rax, regs[15]); 454 PRINT_REG(rbx, regs[12]); 455 PRINT_REG(rcx, regs[14]); 456 PRINT_REG(rdx, regs[13]); 457 PRINT_REG(rdi, regs[8]); 458 PRINT_REG(rsi, regs[9]); 459 PRINT_REG(rbp, regs[10]); 460 // rsp is actually not stored by pusha(), compute the old rsp from regs (rsp after pusha): regs + 16 = old rsp 461 PRINT_REG(rsp, (intptr_t)(®s[16])); 462 PRINT_REG(r8 , regs[7]); 463 PRINT_REG(r9 , regs[6]); 464 PRINT_REG(r10, regs[5]); 465 PRINT_REG(r11, regs[4]); 466 PRINT_REG(r12, regs[3]); 467 PRINT_REG(r13, regs[2]); 468 PRINT_REG(r14, regs[1]); 469 PRINT_REG(r15, regs[0]); 470 #undef PRINT_REG 471 // Print some words near the top of the stack. 472 int64_t* rsp = ®s[16]; 473 int64_t* dump_sp = rsp; 474 for (int col1 = 0; col1 < 8; col1++) { 475 tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp); 476 os::print_location(tty, *dump_sp++); 477 } 478 for (int row = 0; row < 25; row++) { 479 tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp); 480 for (int col = 0; col < 4; col++) { 481 tty->print(" 0x%016lx", (intptr_t)*dump_sp++); 482 } 483 tty->cr(); 484 } 485 // Print some instructions around pc: 486 Disassembler::decode((address)pc-64, (address)pc); 487 tty->print_cr("--------"); 488 Disassembler::decode((address)pc, (address)pc+32); 489 } 490 491 // The java_calling_convention describes stack locations as ideal slots on 492 // a frame with no abi restrictions. Since we must observe abi restrictions 493 // (like the placement of the register window) the slots must be biased by 494 // the following value. 495 static int reg2offset_in(VMReg r) { 496 // Account for saved rbp and return address 497 // This should really be in_preserve_stack_slots 498 return (r->reg2stack() + 4) * VMRegImpl::stack_slot_size; 499 } 500 501 static int reg2offset_out(VMReg r) { 502 return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size; 503 } 504 505 // A long move 506 void MacroAssembler::long_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) { 507 508 // The calling conventions assures us that each VMregpair is either 509 // all really one physical register or adjacent stack slots. 510 511 if (src.is_single_phys_reg() ) { 512 if (dst.is_single_phys_reg()) { 513 if (dst.first() != src.first()) { 514 mov(dst.first()->as_Register(), src.first()->as_Register()); 515 } 516 } else { 517 assert(dst.is_single_reg(), "not a stack pair: (%s, %s), (%s, %s)", 518 src.first()->name(), src.second()->name(), dst.first()->name(), dst.second()->name()); 519 movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_Register()); 520 } 521 } else if (dst.is_single_phys_reg()) { 522 assert(src.is_single_reg(), "not a stack pair"); 523 movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 524 } else { 525 assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs"); 526 movq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 527 movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp); 528 } 529 } 530 531 // A double move 532 void MacroAssembler::double_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) { 533 534 // The calling conventions assures us that each VMregpair is either 535 // all really one physical register or adjacent stack slots. 536 537 if (src.is_single_phys_reg() ) { 538 if (dst.is_single_phys_reg()) { 539 // In theory these overlap but the ordering is such that this is likely a nop 540 if ( src.first() != dst.first()) { 541 movdbl(dst.first()->as_XMMRegister(), src.first()->as_XMMRegister()); 542 } 543 } else { 544 assert(dst.is_single_reg(), "not a stack pair"); 545 movdbl(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_XMMRegister()); 546 } 547 } else if (dst.is_single_phys_reg()) { 548 assert(src.is_single_reg(), "not a stack pair"); 549 movdbl(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 550 } else { 551 assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs"); 552 movq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 553 movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp); 554 } 555 } 556 557 558 // A float arg may have to do float reg int reg conversion 559 void MacroAssembler::float_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) { 560 assert(!src.second()->is_valid() && !dst.second()->is_valid(), "bad float_move"); 561 562 // The calling conventions assures us that each VMregpair is either 563 // all really one physical register or adjacent stack slots. 564 565 if (src.first()->is_stack()) { 566 if (dst.first()->is_stack()) { 567 movl(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 568 movptr(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp); 569 } else { 570 // stack to reg 571 assert(dst.first()->is_XMMRegister(), "only expect xmm registers as parameters"); 572 movflt(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 573 } 574 } else if (dst.first()->is_stack()) { 575 // reg to stack 576 assert(src.first()->is_XMMRegister(), "only expect xmm registers as parameters"); 577 movflt(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_XMMRegister()); 578 } else { 579 // reg to reg 580 // In theory these overlap but the ordering is such that this is likely a nop 581 if ( src.first() != dst.first()) { 582 movdbl(dst.first()->as_XMMRegister(), src.first()->as_XMMRegister()); 583 } 584 } 585 } 586 587 // On 64 bit we will store integer like items to the stack as 588 // 64 bits items (x86_32/64 abi) even though java would only store 589 // 32bits for a parameter. On 32bit it will simply be 32 bits 590 // So this routine will do 32->32 on 32bit and 32->64 on 64bit 591 void MacroAssembler::move32_64(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) { 592 if (src.first()->is_stack()) { 593 if (dst.first()->is_stack()) { 594 // stack to stack 595 movslq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 596 movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp); 597 } else { 598 // stack to reg 599 movslq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 600 } 601 } else if (dst.first()->is_stack()) { 602 // reg to stack 603 // Do we really have to sign extend??? 604 // __ movslq(src.first()->as_Register(), src.first()->as_Register()); 605 movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_Register()); 606 } else { 607 // Do we really have to sign extend??? 608 // __ movslq(dst.first()->as_Register(), src.first()->as_Register()); 609 if (dst.first() != src.first()) { 610 movq(dst.first()->as_Register(), src.first()->as_Register()); 611 } 612 } 613 } 614 615 void MacroAssembler::move_ptr(VMRegPair src, VMRegPair dst) { 616 if (src.first()->is_stack()) { 617 if (dst.first()->is_stack()) { 618 // stack to stack 619 movq(rax, Address(rbp, reg2offset_in(src.first()))); 620 movq(Address(rsp, reg2offset_out(dst.first())), rax); 621 } else { 622 // stack to reg 623 movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()))); 624 } 625 } else if (dst.first()->is_stack()) { 626 // reg to stack 627 movq(Address(rsp, reg2offset_out(dst.first())), src.first()->as_Register()); 628 } else { 629 if (dst.first() != src.first()) { 630 movq(dst.first()->as_Register(), src.first()->as_Register()); 631 } 632 } 633 } 634 635 // An oop arg. Must pass a handle not the oop itself 636 void MacroAssembler::object_move(OopMap* map, 637 int oop_handle_offset, 638 int framesize_in_slots, 639 VMRegPair src, 640 VMRegPair dst, 641 bool is_receiver, 642 int* receiver_offset) { 643 644 // must pass a handle. First figure out the location we use as a handle 645 646 Register rHandle = dst.first()->is_stack() ? rax : dst.first()->as_Register(); 647 648 // See if oop is null if it is we need no handle 649 650 if (src.first()->is_stack()) { 651 652 // Oop is already on the stack as an argument 653 int offset_in_older_frame = src.first()->reg2stack() + SharedRuntime::out_preserve_stack_slots(); 654 map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + framesize_in_slots)); 655 if (is_receiver) { 656 *receiver_offset = (offset_in_older_frame + framesize_in_slots) * VMRegImpl::stack_slot_size; 657 } 658 659 cmpptr(Address(rbp, reg2offset_in(src.first())), NULL_WORD); 660 lea(rHandle, Address(rbp, reg2offset_in(src.first()))); 661 // conditionally move a null 662 cmovptr(Assembler::equal, rHandle, Address(rbp, reg2offset_in(src.first()))); 663 } else { 664 665 // Oop is in a register we must store it to the space we reserve 666 // on the stack for oop_handles and pass a handle if oop is non-null 667 668 const Register rOop = src.first()->as_Register(); 669 int oop_slot; 670 if (rOop == j_rarg0) 671 oop_slot = 0; 672 else if (rOop == j_rarg1) 673 oop_slot = 1; 674 else if (rOop == j_rarg2) 675 oop_slot = 2; 676 else if (rOop == j_rarg3) 677 oop_slot = 3; 678 else if (rOop == j_rarg4) 679 oop_slot = 4; 680 else { 681 assert(rOop == j_rarg5, "wrong register"); 682 oop_slot = 5; 683 } 684 685 oop_slot = oop_slot * VMRegImpl::slots_per_word + oop_handle_offset; 686 int offset = oop_slot*VMRegImpl::stack_slot_size; 687 688 map->set_oop(VMRegImpl::stack2reg(oop_slot)); 689 // Store oop in handle area, may be null 690 movptr(Address(rsp, offset), rOop); 691 if (is_receiver) { 692 *receiver_offset = offset; 693 } 694 695 cmpptr(rOop, NULL_WORD); 696 lea(rHandle, Address(rsp, offset)); 697 // conditionally move a null from the handle area where it was just stored 698 cmovptr(Assembler::equal, rHandle, Address(rsp, offset)); 699 } 700 701 // If arg is on the stack then place it otherwise it is already in correct reg. 702 if (dst.first()->is_stack()) { 703 movptr(Address(rsp, reg2offset_out(dst.first())), rHandle); 704 } 705 } 706 707 void MacroAssembler::addptr(Register dst, int32_t imm32) { 708 addq(dst, imm32); 709 } 710 711 void MacroAssembler::addptr(Register dst, Register src) { 712 addq(dst, src); 713 } 714 715 void MacroAssembler::addptr(Address dst, Register src) { 716 addq(dst, src); 717 } 718 719 void MacroAssembler::addsd(XMMRegister dst, AddressLiteral src, Register rscratch) { 720 assert(rscratch != noreg || always_reachable(src), "missing"); 721 722 if (reachable(src)) { 723 Assembler::addsd(dst, as_Address(src)); 724 } else { 725 lea(rscratch, src); 726 Assembler::addsd(dst, Address(rscratch, 0)); 727 } 728 } 729 730 void MacroAssembler::addss(XMMRegister dst, AddressLiteral src, Register rscratch) { 731 assert(rscratch != noreg || always_reachable(src), "missing"); 732 733 if (reachable(src)) { 734 addss(dst, as_Address(src)); 735 } else { 736 lea(rscratch, src); 737 addss(dst, Address(rscratch, 0)); 738 } 739 } 740 741 void MacroAssembler::addpd(XMMRegister dst, AddressLiteral src, Register rscratch) { 742 assert(rscratch != noreg || always_reachable(src), "missing"); 743 744 if (reachable(src)) { 745 Assembler::addpd(dst, as_Address(src)); 746 } else { 747 lea(rscratch, src); 748 Assembler::addpd(dst, Address(rscratch, 0)); 749 } 750 } 751 752 // See 8273459. Function for ensuring 64-byte alignment, intended for stubs only. 753 // Stub code is generated once and never copied. 754 // NMethods can't use this because they get copied and we can't force alignment > 32 bytes. 755 void MacroAssembler::align64() { 756 align(64, (uint)(uintptr_t)pc()); 757 } 758 759 void MacroAssembler::align32() { 760 align(32, (uint)(uintptr_t)pc()); 761 } 762 763 void MacroAssembler::align(uint modulus) { 764 // 8273459: Ensure alignment is possible with current segment alignment 765 assert(modulus <= (uintx)CodeEntryAlignment, "Alignment must be <= CodeEntryAlignment"); 766 align(modulus, offset()); 767 } 768 769 void MacroAssembler::align(uint modulus, uint target) { 770 if (target % modulus != 0) { 771 nop(modulus - (target % modulus)); 772 } 773 } 774 775 void MacroAssembler::push_f(XMMRegister r) { 776 subptr(rsp, wordSize); 777 movflt(Address(rsp, 0), r); 778 } 779 780 void MacroAssembler::pop_f(XMMRegister r) { 781 movflt(r, Address(rsp, 0)); 782 addptr(rsp, wordSize); 783 } 784 785 void MacroAssembler::push_d(XMMRegister r) { 786 subptr(rsp, 2 * wordSize); 787 movdbl(Address(rsp, 0), r); 788 } 789 790 void MacroAssembler::pop_d(XMMRegister r) { 791 movdbl(r, Address(rsp, 0)); 792 addptr(rsp, 2 * Interpreter::stackElementSize); 793 } 794 795 void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src, Register rscratch) { 796 // Used in sign-masking with aligned address. 797 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 798 assert(rscratch != noreg || always_reachable(src), "missing"); 799 800 if (UseAVX > 2 && 801 (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) && 802 (dst->encoding() >= 16)) { 803 vpand(dst, dst, src, AVX_512bit, rscratch); 804 } else if (reachable(src)) { 805 Assembler::andpd(dst, as_Address(src)); 806 } else { 807 lea(rscratch, src); 808 Assembler::andpd(dst, Address(rscratch, 0)); 809 } 810 } 811 812 void MacroAssembler::andps(XMMRegister dst, AddressLiteral src, Register rscratch) { 813 // Used in sign-masking with aligned address. 814 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 815 assert(rscratch != noreg || always_reachable(src), "missing"); 816 817 if (reachable(src)) { 818 Assembler::andps(dst, as_Address(src)); 819 } else { 820 lea(rscratch, src); 821 Assembler::andps(dst, Address(rscratch, 0)); 822 } 823 } 824 825 void MacroAssembler::andptr(Register dst, int32_t imm32) { 826 andq(dst, imm32); 827 } 828 829 void MacroAssembler::andq(Register dst, AddressLiteral src, Register rscratch) { 830 assert(rscratch != noreg || always_reachable(src), "missing"); 831 832 if (reachable(src)) { 833 andq(dst, as_Address(src)); 834 } else { 835 lea(rscratch, src); 836 andq(dst, Address(rscratch, 0)); 837 } 838 } 839 840 void MacroAssembler::atomic_incl(Address counter_addr) { 841 lock(); 842 incrementl(counter_addr); 843 } 844 845 void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register rscratch) { 846 assert(rscratch != noreg || always_reachable(counter_addr), "missing"); 847 848 if (reachable(counter_addr)) { 849 atomic_incl(as_Address(counter_addr)); 850 } else { 851 lea(rscratch, counter_addr); 852 atomic_incl(Address(rscratch, 0)); 853 } 854 } 855 856 void MacroAssembler::atomic_incq(Address counter_addr) { 857 lock(); 858 incrementq(counter_addr); 859 } 860 861 void MacroAssembler::atomic_incq(AddressLiteral counter_addr, Register rscratch) { 862 assert(rscratch != noreg || always_reachable(counter_addr), "missing"); 863 864 if (reachable(counter_addr)) { 865 atomic_incq(as_Address(counter_addr)); 866 } else { 867 lea(rscratch, counter_addr); 868 atomic_incq(Address(rscratch, 0)); 869 } 870 } 871 872 // Writes to stack successive pages until offset reached to check for 873 // stack overflow + shadow pages. This clobbers tmp. 874 void MacroAssembler::bang_stack_size(Register size, Register tmp) { 875 movptr(tmp, rsp); 876 // Bang stack for total size given plus shadow page size. 877 // Bang one page at a time because large size can bang beyond yellow and 878 // red zones. 879 Label loop; 880 bind(loop); 881 movl(Address(tmp, (-(int)os::vm_page_size())), size ); 882 subptr(tmp, (int)os::vm_page_size()); 883 subl(size, (int)os::vm_page_size()); 884 jcc(Assembler::greater, loop); 885 886 // Bang down shadow pages too. 887 // At this point, (tmp-0) is the last address touched, so don't 888 // touch it again. (It was touched as (tmp-pagesize) but then tmp 889 // was post-decremented.) Skip this address by starting at i=1, and 890 // touch a few more pages below. N.B. It is important to touch all 891 // the way down including all pages in the shadow zone. 892 for (int i = 1; i < ((int)StackOverflow::stack_shadow_zone_size() / (int)os::vm_page_size()); i++) { 893 // this could be any sized move but this is can be a debugging crumb 894 // so the bigger the better. 895 movptr(Address(tmp, (-i*(int)os::vm_page_size())), size ); 896 } 897 } 898 899 void MacroAssembler::reserved_stack_check() { 900 // testing if reserved zone needs to be enabled 901 Label no_reserved_zone_enabling; 902 903 cmpptr(rsp, Address(r15_thread, JavaThread::reserved_stack_activation_offset())); 904 jcc(Assembler::below, no_reserved_zone_enabling); 905 906 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), r15_thread); 907 jump(RuntimeAddress(SharedRuntime::throw_delayed_StackOverflowError_entry())); 908 should_not_reach_here(); 909 910 bind(no_reserved_zone_enabling); 911 } 912 913 void MacroAssembler::c2bool(Register x) { 914 // implements x == 0 ? 0 : 1 915 // note: must only look at least-significant byte of x 916 // since C-style booleans are stored in one byte 917 // only! (was bug) 918 andl(x, 0xFF); 919 setb(Assembler::notZero, x); 920 } 921 922 // Wouldn't need if AddressLiteral version had new name 923 void MacroAssembler::call(Label& L, relocInfo::relocType rtype) { 924 Assembler::call(L, rtype); 925 } 926 927 void MacroAssembler::call(Register entry) { 928 Assembler::call(entry); 929 } 930 931 void MacroAssembler::call(AddressLiteral entry, Register rscratch) { 932 assert(rscratch != noreg || always_reachable(entry), "missing"); 933 934 if (reachable(entry)) { 935 Assembler::call_literal(entry.target(), entry.rspec()); 936 } else { 937 lea(rscratch, entry); 938 Assembler::call(rscratch); 939 } 940 } 941 942 void MacroAssembler::ic_call(address entry, jint method_index) { 943 RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index); 944 // Needs full 64-bit immediate for later patching. 945 mov64(rax, (int64_t)Universe::non_oop_word()); 946 call(AddressLiteral(entry, rh)); 947 } 948 949 int MacroAssembler::ic_check_size() { 950 return UseCompactObjectHeaders ? 17 : 14; 951 } 952 953 int MacroAssembler::ic_check(int end_alignment) { 954 Register receiver = j_rarg0; 955 Register data = rax; 956 Register temp = rscratch1; 957 958 // The UEP of a code blob ensures that the VEP is padded. However, the padding of the UEP is placed 959 // before the inline cache check, so we don't have to execute any nop instructions when dispatching 960 // through the UEP, yet we can ensure that the VEP is aligned appropriately. That's why we align 961 // before the inline cache check here, and not after 962 align(end_alignment, offset() + ic_check_size()); 963 964 int uep_offset = offset(); 965 966 if (UseCompactObjectHeaders) { 967 load_narrow_klass_compact(temp, receiver); 968 cmpl(temp, Address(data, CompiledICData::speculated_klass_offset())); 969 } else if (UseCompressedClassPointers) { 970 movl(temp, Address(receiver, oopDesc::klass_offset_in_bytes())); 971 cmpl(temp, Address(data, CompiledICData::speculated_klass_offset())); 972 } else { 973 movptr(temp, Address(receiver, oopDesc::klass_offset_in_bytes())); 974 cmpptr(temp, Address(data, CompiledICData::speculated_klass_offset())); 975 } 976 977 // if inline cache check fails, then jump to runtime routine 978 jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub())); 979 assert((offset() % end_alignment) == 0, "Misaligned verified entry point (%d, %d, %d)", uep_offset, offset(), end_alignment); 980 981 return uep_offset; 982 } 983 984 void MacroAssembler::emit_static_call_stub() { 985 // Static stub relocation also tags the Method* in the code-stream. 986 mov_metadata(rbx, (Metadata*) nullptr); // Method is zapped till fixup time. 987 // This is recognized as unresolved by relocs/nativeinst/ic code. 988 jump(RuntimeAddress(pc())); 989 } 990 991 // Implementation of call_VM versions 992 993 void MacroAssembler::call_VM(Register oop_result, 994 address entry_point, 995 bool check_exceptions) { 996 Label C, E; 997 call(C, relocInfo::none); 998 jmp(E); 999 1000 bind(C); 1001 call_VM_helper(oop_result, entry_point, 0, check_exceptions); 1002 ret(0); 1003 1004 bind(E); 1005 } 1006 1007 void MacroAssembler::call_VM(Register oop_result, 1008 address entry_point, 1009 Register arg_1, 1010 bool check_exceptions) { 1011 Label C, E; 1012 call(C, relocInfo::none); 1013 jmp(E); 1014 1015 bind(C); 1016 pass_arg1(this, arg_1); 1017 call_VM_helper(oop_result, entry_point, 1, check_exceptions); 1018 ret(0); 1019 1020 bind(E); 1021 } 1022 1023 void MacroAssembler::call_VM(Register oop_result, 1024 address entry_point, 1025 Register arg_1, 1026 Register arg_2, 1027 bool check_exceptions) { 1028 Label C, E; 1029 call(C, relocInfo::none); 1030 jmp(E); 1031 1032 bind(C); 1033 1034 assert_different_registers(arg_1, c_rarg2); 1035 1036 pass_arg2(this, arg_2); 1037 pass_arg1(this, arg_1); 1038 call_VM_helper(oop_result, entry_point, 2, check_exceptions); 1039 ret(0); 1040 1041 bind(E); 1042 } 1043 1044 void MacroAssembler::call_VM(Register oop_result, 1045 address entry_point, 1046 Register arg_1, 1047 Register arg_2, 1048 Register arg_3, 1049 bool check_exceptions) { 1050 Label C, E; 1051 call(C, relocInfo::none); 1052 jmp(E); 1053 1054 bind(C); 1055 1056 assert_different_registers(arg_1, c_rarg2, c_rarg3); 1057 assert_different_registers(arg_2, c_rarg3); 1058 pass_arg3(this, arg_3); 1059 pass_arg2(this, arg_2); 1060 pass_arg1(this, arg_1); 1061 call_VM_helper(oop_result, entry_point, 3, check_exceptions); 1062 ret(0); 1063 1064 bind(E); 1065 } 1066 1067 void MacroAssembler::call_VM(Register oop_result, 1068 Register last_java_sp, 1069 address entry_point, 1070 int number_of_arguments, 1071 bool check_exceptions) { 1072 call_VM_base(oop_result, last_java_sp, entry_point, number_of_arguments, check_exceptions); 1073 } 1074 1075 void MacroAssembler::call_VM(Register oop_result, 1076 Register last_java_sp, 1077 address entry_point, 1078 Register arg_1, 1079 bool check_exceptions) { 1080 pass_arg1(this, arg_1); 1081 call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions); 1082 } 1083 1084 void MacroAssembler::call_VM(Register oop_result, 1085 Register last_java_sp, 1086 address entry_point, 1087 Register arg_1, 1088 Register arg_2, 1089 bool check_exceptions) { 1090 1091 assert_different_registers(arg_1, c_rarg2); 1092 pass_arg2(this, arg_2); 1093 pass_arg1(this, arg_1); 1094 call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions); 1095 } 1096 1097 void MacroAssembler::call_VM(Register oop_result, 1098 Register last_java_sp, 1099 address entry_point, 1100 Register arg_1, 1101 Register arg_2, 1102 Register arg_3, 1103 bool check_exceptions) { 1104 assert_different_registers(arg_1, c_rarg2, c_rarg3); 1105 assert_different_registers(arg_2, c_rarg3); 1106 pass_arg3(this, arg_3); 1107 pass_arg2(this, arg_2); 1108 pass_arg1(this, arg_1); 1109 call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions); 1110 } 1111 1112 void MacroAssembler::super_call_VM(Register oop_result, 1113 Register last_java_sp, 1114 address entry_point, 1115 int number_of_arguments, 1116 bool check_exceptions) { 1117 MacroAssembler::call_VM_base(oop_result, last_java_sp, entry_point, number_of_arguments, check_exceptions); 1118 } 1119 1120 void MacroAssembler::super_call_VM(Register oop_result, 1121 Register last_java_sp, 1122 address entry_point, 1123 Register arg_1, 1124 bool check_exceptions) { 1125 pass_arg1(this, arg_1); 1126 super_call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions); 1127 } 1128 1129 void MacroAssembler::super_call_VM(Register oop_result, 1130 Register last_java_sp, 1131 address entry_point, 1132 Register arg_1, 1133 Register arg_2, 1134 bool check_exceptions) { 1135 1136 assert_different_registers(arg_1, c_rarg2); 1137 pass_arg2(this, arg_2); 1138 pass_arg1(this, arg_1); 1139 super_call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions); 1140 } 1141 1142 void MacroAssembler::super_call_VM(Register oop_result, 1143 Register last_java_sp, 1144 address entry_point, 1145 Register arg_1, 1146 Register arg_2, 1147 Register arg_3, 1148 bool check_exceptions) { 1149 assert_different_registers(arg_1, c_rarg2, c_rarg3); 1150 assert_different_registers(arg_2, c_rarg3); 1151 pass_arg3(this, arg_3); 1152 pass_arg2(this, arg_2); 1153 pass_arg1(this, arg_1); 1154 super_call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions); 1155 } 1156 1157 void MacroAssembler::call_VM_base(Register oop_result, 1158 Register last_java_sp, 1159 address entry_point, 1160 int number_of_arguments, 1161 bool check_exceptions) { 1162 Register java_thread = r15_thread; 1163 1164 // determine last_java_sp register 1165 if (!last_java_sp->is_valid()) { 1166 last_java_sp = rsp; 1167 } 1168 // debugging support 1169 assert(number_of_arguments >= 0 , "cannot have negative number of arguments"); 1170 #ifdef ASSERT 1171 // TraceBytecodes does not use r12 but saves it over the call, so don't verify 1172 // r12 is the heapbase. 1173 if (UseCompressedOops && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?"); 1174 #endif // ASSERT 1175 1176 assert(java_thread != oop_result , "cannot use the same register for java_thread & oop_result"); 1177 assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp"); 1178 1179 // push java thread (becomes first argument of C function) 1180 1181 mov(c_rarg0, r15_thread); 1182 1183 // set last Java frame before call 1184 assert(last_java_sp != rbp, "can't use ebp/rbp"); 1185 1186 // Only interpreter should have to set fp 1187 set_last_Java_frame(last_java_sp, rbp, nullptr, rscratch1); 1188 1189 // do the call, remove parameters 1190 MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments); 1191 1192 #ifdef ASSERT 1193 // Check that thread register is not clobbered. 1194 guarantee(java_thread != rax, "change this code"); 1195 push(rax); 1196 { Label L; 1197 get_thread_slow(rax); 1198 cmpptr(java_thread, rax); 1199 jcc(Assembler::equal, L); 1200 STOP("MacroAssembler::call_VM_base: java_thread not callee saved?"); 1201 bind(L); 1202 } 1203 pop(rax); 1204 #endif 1205 1206 // reset last Java frame 1207 // Only interpreter should have to clear fp 1208 reset_last_Java_frame(true); 1209 1210 // C++ interp handles this in the interpreter 1211 check_and_handle_popframe(); 1212 check_and_handle_earlyret(); 1213 1214 if (check_exceptions) { 1215 // check for pending exceptions (java_thread is set upon return) 1216 cmpptr(Address(r15_thread, Thread::pending_exception_offset()), NULL_WORD); 1217 // This used to conditionally jump to forward_exception however it is 1218 // possible if we relocate that the branch will not reach. So we must jump 1219 // around so we can always reach 1220 1221 Label ok; 1222 jcc(Assembler::equal, ok); 1223 jump(RuntimeAddress(StubRoutines::forward_exception_entry())); 1224 bind(ok); 1225 } 1226 1227 // get oop result if there is one and reset the value in the thread 1228 if (oop_result->is_valid()) { 1229 get_vm_result_oop(oop_result); 1230 } 1231 } 1232 1233 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) { 1234 // Calculate the value for last_Java_sp somewhat subtle. 1235 // call_VM does an intermediate call which places a return address on 1236 // the stack just under the stack pointer as the user finished with it. 1237 // This allows use to retrieve last_Java_pc from last_Java_sp[-1]. 1238 1239 // We've pushed one address, correct last_Java_sp 1240 lea(rax, Address(rsp, wordSize)); 1241 1242 call_VM_base(oop_result, rax, entry_point, number_of_arguments, check_exceptions); 1243 } 1244 1245 // Use this method when MacroAssembler version of call_VM_leaf_base() should be called from Interpreter. 1246 void MacroAssembler::call_VM_leaf0(address entry_point) { 1247 MacroAssembler::call_VM_leaf_base(entry_point, 0); 1248 } 1249 1250 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) { 1251 call_VM_leaf_base(entry_point, number_of_arguments); 1252 } 1253 1254 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) { 1255 pass_arg0(this, arg_0); 1256 call_VM_leaf(entry_point, 1); 1257 } 1258 1259 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) { 1260 1261 assert_different_registers(arg_0, c_rarg1); 1262 pass_arg1(this, arg_1); 1263 pass_arg0(this, arg_0); 1264 call_VM_leaf(entry_point, 2); 1265 } 1266 1267 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) { 1268 assert_different_registers(arg_0, c_rarg1, c_rarg2); 1269 assert_different_registers(arg_1, c_rarg2); 1270 pass_arg2(this, arg_2); 1271 pass_arg1(this, arg_1); 1272 pass_arg0(this, arg_0); 1273 call_VM_leaf(entry_point, 3); 1274 } 1275 1276 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) { 1277 assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3); 1278 assert_different_registers(arg_1, c_rarg2, c_rarg3); 1279 assert_different_registers(arg_2, c_rarg3); 1280 pass_arg3(this, arg_3); 1281 pass_arg2(this, arg_2); 1282 pass_arg1(this, arg_1); 1283 pass_arg0(this, arg_0); 1284 call_VM_leaf(entry_point, 3); 1285 } 1286 1287 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) { 1288 pass_arg0(this, arg_0); 1289 MacroAssembler::call_VM_leaf_base(entry_point, 1); 1290 } 1291 1292 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) { 1293 assert_different_registers(arg_0, c_rarg1); 1294 pass_arg1(this, arg_1); 1295 pass_arg0(this, arg_0); 1296 MacroAssembler::call_VM_leaf_base(entry_point, 2); 1297 } 1298 1299 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) { 1300 assert_different_registers(arg_0, c_rarg1, c_rarg2); 1301 assert_different_registers(arg_1, c_rarg2); 1302 pass_arg2(this, arg_2); 1303 pass_arg1(this, arg_1); 1304 pass_arg0(this, arg_0); 1305 MacroAssembler::call_VM_leaf_base(entry_point, 3); 1306 } 1307 1308 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) { 1309 assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3); 1310 assert_different_registers(arg_1, c_rarg2, c_rarg3); 1311 assert_different_registers(arg_2, c_rarg3); 1312 pass_arg3(this, arg_3); 1313 pass_arg2(this, arg_2); 1314 pass_arg1(this, arg_1); 1315 pass_arg0(this, arg_0); 1316 MacroAssembler::call_VM_leaf_base(entry_point, 4); 1317 } 1318 1319 void MacroAssembler::get_vm_result_oop(Register oop_result) { 1320 movptr(oop_result, Address(r15_thread, JavaThread::vm_result_oop_offset())); 1321 movptr(Address(r15_thread, JavaThread::vm_result_oop_offset()), NULL_WORD); 1322 verify_oop_msg(oop_result, "broken oop in call_VM_base"); 1323 } 1324 1325 void MacroAssembler::get_vm_result_metadata(Register metadata_result) { 1326 movptr(metadata_result, Address(r15_thread, JavaThread::vm_result_metadata_offset())); 1327 movptr(Address(r15_thread, JavaThread::vm_result_metadata_offset()), NULL_WORD); 1328 } 1329 1330 void MacroAssembler::check_and_handle_earlyret() { 1331 } 1332 1333 void MacroAssembler::check_and_handle_popframe() { 1334 } 1335 1336 void MacroAssembler::cmp32(AddressLiteral src1, int32_t imm, Register rscratch) { 1337 assert(rscratch != noreg || always_reachable(src1), "missing"); 1338 1339 if (reachable(src1)) { 1340 cmpl(as_Address(src1), imm); 1341 } else { 1342 lea(rscratch, src1); 1343 cmpl(Address(rscratch, 0), imm); 1344 } 1345 } 1346 1347 void MacroAssembler::cmp32(Register src1, AddressLiteral src2, Register rscratch) { 1348 assert(!src2.is_lval(), "use cmpptr"); 1349 assert(rscratch != noreg || always_reachable(src2), "missing"); 1350 1351 if (reachable(src2)) { 1352 cmpl(src1, as_Address(src2)); 1353 } else { 1354 lea(rscratch, src2); 1355 cmpl(src1, Address(rscratch, 0)); 1356 } 1357 } 1358 1359 void MacroAssembler::cmp32(Register src1, int32_t imm) { 1360 Assembler::cmpl(src1, imm); 1361 } 1362 1363 void MacroAssembler::cmp32(Register src1, Address src2) { 1364 Assembler::cmpl(src1, src2); 1365 } 1366 1367 void MacroAssembler::cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) { 1368 ucomisd(opr1, opr2); 1369 1370 Label L; 1371 if (unordered_is_less) { 1372 movl(dst, -1); 1373 jcc(Assembler::parity, L); 1374 jcc(Assembler::below , L); 1375 movl(dst, 0); 1376 jcc(Assembler::equal , L); 1377 increment(dst); 1378 } else { // unordered is greater 1379 movl(dst, 1); 1380 jcc(Assembler::parity, L); 1381 jcc(Assembler::above , L); 1382 movl(dst, 0); 1383 jcc(Assembler::equal , L); 1384 decrementl(dst); 1385 } 1386 bind(L); 1387 } 1388 1389 void MacroAssembler::cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) { 1390 ucomiss(opr1, opr2); 1391 1392 Label L; 1393 if (unordered_is_less) { 1394 movl(dst, -1); 1395 jcc(Assembler::parity, L); 1396 jcc(Assembler::below , L); 1397 movl(dst, 0); 1398 jcc(Assembler::equal , L); 1399 increment(dst); 1400 } else { // unordered is greater 1401 movl(dst, 1); 1402 jcc(Assembler::parity, L); 1403 jcc(Assembler::above , L); 1404 movl(dst, 0); 1405 jcc(Assembler::equal , L); 1406 decrementl(dst); 1407 } 1408 bind(L); 1409 } 1410 1411 1412 void MacroAssembler::cmp8(AddressLiteral src1, int imm, Register rscratch) { 1413 assert(rscratch != noreg || always_reachable(src1), "missing"); 1414 1415 if (reachable(src1)) { 1416 cmpb(as_Address(src1), imm); 1417 } else { 1418 lea(rscratch, src1); 1419 cmpb(Address(rscratch, 0), imm); 1420 } 1421 } 1422 1423 void MacroAssembler::cmpptr(Register src1, AddressLiteral src2, Register rscratch) { 1424 assert(rscratch != noreg || always_reachable(src2), "missing"); 1425 1426 if (src2.is_lval()) { 1427 movptr(rscratch, src2); 1428 Assembler::cmpq(src1, rscratch); 1429 } else if (reachable(src2)) { 1430 cmpq(src1, as_Address(src2)); 1431 } else { 1432 lea(rscratch, src2); 1433 Assembler::cmpq(src1, Address(rscratch, 0)); 1434 } 1435 } 1436 1437 void MacroAssembler::cmpptr(Address src1, AddressLiteral src2, Register rscratch) { 1438 assert(src2.is_lval(), "not a mem-mem compare"); 1439 // moves src2's literal address 1440 movptr(rscratch, src2); 1441 Assembler::cmpq(src1, rscratch); 1442 } 1443 1444 void MacroAssembler::cmpoop(Register src1, Register src2) { 1445 cmpptr(src1, src2); 1446 } 1447 1448 void MacroAssembler::cmpoop(Register src1, Address src2) { 1449 cmpptr(src1, src2); 1450 } 1451 1452 void MacroAssembler::cmpoop(Register src1, jobject src2, Register rscratch) { 1453 movoop(rscratch, src2); 1454 cmpptr(src1, rscratch); 1455 } 1456 1457 void MacroAssembler::locked_cmpxchgptr(Register reg, AddressLiteral adr, Register rscratch) { 1458 assert(rscratch != noreg || always_reachable(adr), "missing"); 1459 1460 if (reachable(adr)) { 1461 lock(); 1462 cmpxchgptr(reg, as_Address(adr)); 1463 } else { 1464 lea(rscratch, adr); 1465 lock(); 1466 cmpxchgptr(reg, Address(rscratch, 0)); 1467 } 1468 } 1469 1470 void MacroAssembler::cmpxchgptr(Register reg, Address adr) { 1471 cmpxchgq(reg, adr); 1472 } 1473 1474 void MacroAssembler::comisd(XMMRegister dst, AddressLiteral src, Register rscratch) { 1475 assert(rscratch != noreg || always_reachable(src), "missing"); 1476 1477 if (reachable(src)) { 1478 Assembler::comisd(dst, as_Address(src)); 1479 } else { 1480 lea(rscratch, src); 1481 Assembler::comisd(dst, Address(rscratch, 0)); 1482 } 1483 } 1484 1485 void MacroAssembler::comiss(XMMRegister dst, AddressLiteral src, Register rscratch) { 1486 assert(rscratch != noreg || always_reachable(src), "missing"); 1487 1488 if (reachable(src)) { 1489 Assembler::comiss(dst, as_Address(src)); 1490 } else { 1491 lea(rscratch, src); 1492 Assembler::comiss(dst, Address(rscratch, 0)); 1493 } 1494 } 1495 1496 1497 void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr, Register rscratch) { 1498 assert(rscratch != noreg || always_reachable(counter_addr), "missing"); 1499 1500 Condition negated_cond = negate_condition(cond); 1501 Label L; 1502 jcc(negated_cond, L); 1503 pushf(); // Preserve flags 1504 atomic_incl(counter_addr, rscratch); 1505 popf(); 1506 bind(L); 1507 } 1508 1509 int MacroAssembler::corrected_idivl(Register reg) { 1510 // Full implementation of Java idiv and irem; checks for 1511 // special case as described in JVM spec., p.243 & p.271. 1512 // The function returns the (pc) offset of the idivl 1513 // instruction - may be needed for implicit exceptions. 1514 // 1515 // normal case special case 1516 // 1517 // input : rax,: dividend min_int 1518 // reg: divisor (may not be rax,/rdx) -1 1519 // 1520 // output: rax,: quotient (= rax, idiv reg) min_int 1521 // rdx: remainder (= rax, irem reg) 0 1522 assert(reg != rax && reg != rdx, "reg cannot be rax, or rdx register"); 1523 const int min_int = 0x80000000; 1524 Label normal_case, special_case; 1525 1526 // check for special case 1527 cmpl(rax, min_int); 1528 jcc(Assembler::notEqual, normal_case); 1529 xorl(rdx, rdx); // prepare rdx for possible special case (where remainder = 0) 1530 cmpl(reg, -1); 1531 jcc(Assembler::equal, special_case); 1532 1533 // handle normal case 1534 bind(normal_case); 1535 cdql(); 1536 int idivl_offset = offset(); 1537 idivl(reg); 1538 1539 // normal and special case exit 1540 bind(special_case); 1541 1542 return idivl_offset; 1543 } 1544 1545 1546 1547 void MacroAssembler::decrementl(Register reg, int value) { 1548 if (value == min_jint) {subl(reg, value) ; return; } 1549 if (value < 0) { incrementl(reg, -value); return; } 1550 if (value == 0) { ; return; } 1551 if (value == 1 && UseIncDec) { decl(reg) ; return; } 1552 /* else */ { subl(reg, value) ; return; } 1553 } 1554 1555 void MacroAssembler::decrementl(Address dst, int value) { 1556 if (value == min_jint) {subl(dst, value) ; return; } 1557 if (value < 0) { incrementl(dst, -value); return; } 1558 if (value == 0) { ; return; } 1559 if (value == 1 && UseIncDec) { decl(dst) ; return; } 1560 /* else */ { subl(dst, value) ; return; } 1561 } 1562 1563 void MacroAssembler::division_with_shift (Register reg, int shift_value) { 1564 assert(shift_value > 0, "illegal shift value"); 1565 Label _is_positive; 1566 testl (reg, reg); 1567 jcc (Assembler::positive, _is_positive); 1568 int offset = (1 << shift_value) - 1 ; 1569 1570 if (offset == 1) { 1571 incrementl(reg); 1572 } else { 1573 addl(reg, offset); 1574 } 1575 1576 bind (_is_positive); 1577 sarl(reg, shift_value); 1578 } 1579 1580 void MacroAssembler::divsd(XMMRegister dst, AddressLiteral src, Register rscratch) { 1581 assert(rscratch != noreg || always_reachable(src), "missing"); 1582 1583 if (reachable(src)) { 1584 Assembler::divsd(dst, as_Address(src)); 1585 } else { 1586 lea(rscratch, src); 1587 Assembler::divsd(dst, Address(rscratch, 0)); 1588 } 1589 } 1590 1591 void MacroAssembler::divss(XMMRegister dst, AddressLiteral src, Register rscratch) { 1592 assert(rscratch != noreg || always_reachable(src), "missing"); 1593 1594 if (reachable(src)) { 1595 Assembler::divss(dst, as_Address(src)); 1596 } else { 1597 lea(rscratch, src); 1598 Assembler::divss(dst, Address(rscratch, 0)); 1599 } 1600 } 1601 1602 void MacroAssembler::enter() { 1603 push(rbp); 1604 mov(rbp, rsp); 1605 } 1606 1607 void MacroAssembler::post_call_nop() { 1608 if (!Continuations::enabled()) { 1609 return; 1610 } 1611 InstructionMark im(this); 1612 relocate(post_call_nop_Relocation::spec()); 1613 InlineSkippedInstructionsCounter skipCounter(this); 1614 emit_int8((uint8_t)0x0f); 1615 emit_int8((uint8_t)0x1f); 1616 emit_int8((uint8_t)0x84); 1617 emit_int8((uint8_t)0x00); 1618 emit_int32(0x00); 1619 } 1620 1621 // A 5 byte nop that is safe for patching (see patch_verified_entry) 1622 void MacroAssembler::fat_nop() { 1623 if (UseAddressNop) { 1624 addr_nop_5(); 1625 } else { 1626 emit_int8((uint8_t)0x26); // es: 1627 emit_int8((uint8_t)0x2e); // cs: 1628 emit_int8((uint8_t)0x64); // fs: 1629 emit_int8((uint8_t)0x65); // gs: 1630 emit_int8((uint8_t)0x90); 1631 } 1632 } 1633 1634 void MacroAssembler::mulpd(XMMRegister dst, AddressLiteral src, Register rscratch) { 1635 assert(rscratch != noreg || always_reachable(src), "missing"); 1636 if (reachable(src)) { 1637 Assembler::mulpd(dst, as_Address(src)); 1638 } else { 1639 lea(rscratch, src); 1640 Assembler::mulpd(dst, Address(rscratch, 0)); 1641 } 1642 } 1643 1644 // dst = c = a * b + c 1645 void MacroAssembler::fmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) { 1646 Assembler::vfmadd231sd(c, a, b); 1647 if (dst != c) { 1648 movdbl(dst, c); 1649 } 1650 } 1651 1652 // dst = c = a * b + c 1653 void MacroAssembler::fmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) { 1654 Assembler::vfmadd231ss(c, a, b); 1655 if (dst != c) { 1656 movflt(dst, c); 1657 } 1658 } 1659 1660 // dst = c = a * b + c 1661 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) { 1662 Assembler::vfmadd231pd(c, a, b, vector_len); 1663 if (dst != c) { 1664 vmovdqu(dst, c); 1665 } 1666 } 1667 1668 // dst = c = a * b + c 1669 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) { 1670 Assembler::vfmadd231ps(c, a, b, vector_len); 1671 if (dst != c) { 1672 vmovdqu(dst, c); 1673 } 1674 } 1675 1676 // dst = c = a * b + c 1677 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) { 1678 Assembler::vfmadd231pd(c, a, b, vector_len); 1679 if (dst != c) { 1680 vmovdqu(dst, c); 1681 } 1682 } 1683 1684 // dst = c = a * b + c 1685 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) { 1686 Assembler::vfmadd231ps(c, a, b, vector_len); 1687 if (dst != c) { 1688 vmovdqu(dst, c); 1689 } 1690 } 1691 1692 void MacroAssembler::incrementl(AddressLiteral dst, Register rscratch) { 1693 assert(rscratch != noreg || always_reachable(dst), "missing"); 1694 1695 if (reachable(dst)) { 1696 incrementl(as_Address(dst)); 1697 } else { 1698 lea(rscratch, dst); 1699 incrementl(Address(rscratch, 0)); 1700 } 1701 } 1702 1703 void MacroAssembler::incrementl(ArrayAddress dst, Register rscratch) { 1704 incrementl(as_Address(dst, rscratch)); 1705 } 1706 1707 void MacroAssembler::incrementl(Register reg, int value) { 1708 if (value == min_jint) {addl(reg, value) ; return; } 1709 if (value < 0) { decrementl(reg, -value); return; } 1710 if (value == 0) { ; return; } 1711 if (value == 1 && UseIncDec) { incl(reg) ; return; } 1712 /* else */ { addl(reg, value) ; return; } 1713 } 1714 1715 void MacroAssembler::incrementl(Address dst, int value) { 1716 if (value == min_jint) {addl(dst, value) ; return; } 1717 if (value < 0) { decrementl(dst, -value); return; } 1718 if (value == 0) { ; return; } 1719 if (value == 1 && UseIncDec) { incl(dst) ; return; } 1720 /* else */ { addl(dst, value) ; return; } 1721 } 1722 1723 void MacroAssembler::jump(AddressLiteral dst, Register rscratch) { 1724 assert(rscratch != noreg || always_reachable(dst), "missing"); 1725 assert(!dst.rspec().reloc()->is_data(), "should not use ExternalAddress for jump"); 1726 if (reachable(dst)) { 1727 jmp_literal(dst.target(), dst.rspec()); 1728 } else { 1729 lea(rscratch, dst); 1730 jmp(rscratch); 1731 } 1732 } 1733 1734 void MacroAssembler::jump_cc(Condition cc, AddressLiteral dst, Register rscratch) { 1735 assert(rscratch != noreg || always_reachable(dst), "missing"); 1736 assert(!dst.rspec().reloc()->is_data(), "should not use ExternalAddress for jump_cc"); 1737 if (reachable(dst)) { 1738 InstructionMark im(this); 1739 relocate(dst.reloc()); 1740 const int short_size = 2; 1741 const int long_size = 6; 1742 int offs = (intptr_t)dst.target() - ((intptr_t)pc()); 1743 if (dst.reloc() == relocInfo::none && is8bit(offs - short_size)) { 1744 // 0111 tttn #8-bit disp 1745 emit_int8(0x70 | cc); 1746 emit_int8((offs - short_size) & 0xFF); 1747 } else { 1748 // 0000 1111 1000 tttn #32-bit disp 1749 emit_int8(0x0F); 1750 emit_int8((unsigned char)(0x80 | cc)); 1751 emit_int32(offs - long_size); 1752 } 1753 } else { 1754 #ifdef ASSERT 1755 warning("reversing conditional branch"); 1756 #endif /* ASSERT */ 1757 Label skip; 1758 jccb(reverse[cc], skip); 1759 lea(rscratch, dst); 1760 Assembler::jmp(rscratch); 1761 bind(skip); 1762 } 1763 } 1764 1765 void MacroAssembler::cmp32_mxcsr_std(Address mxcsr_save, Register tmp, Register rscratch) { 1766 ExternalAddress mxcsr_std(StubRoutines::x86::addr_mxcsr_std()); 1767 assert(rscratch != noreg || always_reachable(mxcsr_std), "missing"); 1768 1769 stmxcsr(mxcsr_save); 1770 movl(tmp, mxcsr_save); 1771 if (EnableX86ECoreOpts) { 1772 // The mxcsr_std has status bits set for performance on ECore 1773 orl(tmp, 0x003f); 1774 } else { 1775 // Mask out status bits (only check control and mask bits) 1776 andl(tmp, 0xFFC0); 1777 } 1778 cmp32(tmp, mxcsr_std, rscratch); 1779 } 1780 1781 void MacroAssembler::ldmxcsr(AddressLiteral src, Register rscratch) { 1782 assert(rscratch != noreg || always_reachable(src), "missing"); 1783 1784 if (reachable(src)) { 1785 Assembler::ldmxcsr(as_Address(src)); 1786 } else { 1787 lea(rscratch, src); 1788 Assembler::ldmxcsr(Address(rscratch, 0)); 1789 } 1790 } 1791 1792 int MacroAssembler::load_signed_byte(Register dst, Address src) { 1793 int off = offset(); 1794 movsbl(dst, src); // movsxb 1795 return off; 1796 } 1797 1798 // Note: load_signed_short used to be called load_signed_word. 1799 // Although the 'w' in x86 opcodes refers to the term "word" in the assembler 1800 // manual, which means 16 bits, that usage is found nowhere in HotSpot code. 1801 // The term "word" in HotSpot means a 32- or 64-bit machine word. 1802 int MacroAssembler::load_signed_short(Register dst, Address src) { 1803 // This is dubious to me since it seems safe to do a signed 16 => 64 bit 1804 // version but this is what 64bit has always done. This seems to imply 1805 // that users are only using 32bits worth. 1806 int off = offset(); 1807 movswl(dst, src); // movsxw 1808 return off; 1809 } 1810 1811 int MacroAssembler::load_unsigned_byte(Register dst, Address src) { 1812 // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16, 1813 // and "3.9 Partial Register Penalties", p. 22). 1814 int off = offset(); 1815 movzbl(dst, src); // movzxb 1816 return off; 1817 } 1818 1819 // Note: load_unsigned_short used to be called load_unsigned_word. 1820 int MacroAssembler::load_unsigned_short(Register dst, Address src) { 1821 // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16, 1822 // and "3.9 Partial Register Penalties", p. 22). 1823 int off = offset(); 1824 movzwl(dst, src); // movzxw 1825 return off; 1826 } 1827 1828 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2) { 1829 switch (size_in_bytes) { 1830 case 8: movq(dst, src); break; 1831 case 4: movl(dst, src); break; 1832 case 2: is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break; 1833 case 1: is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break; 1834 default: ShouldNotReachHere(); 1835 } 1836 } 1837 1838 void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2) { 1839 switch (size_in_bytes) { 1840 case 8: movq(dst, src); break; 1841 case 4: movl(dst, src); break; 1842 case 2: movw(dst, src); break; 1843 case 1: movb(dst, src); break; 1844 default: ShouldNotReachHere(); 1845 } 1846 } 1847 1848 void MacroAssembler::mov32(AddressLiteral dst, Register src, Register rscratch) { 1849 assert(rscratch != noreg || always_reachable(dst), "missing"); 1850 1851 if (reachable(dst)) { 1852 movl(as_Address(dst), src); 1853 } else { 1854 lea(rscratch, dst); 1855 movl(Address(rscratch, 0), src); 1856 } 1857 } 1858 1859 void MacroAssembler::mov32(Register dst, AddressLiteral src) { 1860 if (reachable(src)) { 1861 movl(dst, as_Address(src)); 1862 } else { 1863 lea(dst, src); 1864 movl(dst, Address(dst, 0)); 1865 } 1866 } 1867 1868 // C++ bool manipulation 1869 1870 void MacroAssembler::movbool(Register dst, Address src) { 1871 if(sizeof(bool) == 1) 1872 movb(dst, src); 1873 else if(sizeof(bool) == 2) 1874 movw(dst, src); 1875 else if(sizeof(bool) == 4) 1876 movl(dst, src); 1877 else 1878 // unsupported 1879 ShouldNotReachHere(); 1880 } 1881 1882 void MacroAssembler::movbool(Address dst, bool boolconst) { 1883 if(sizeof(bool) == 1) 1884 movb(dst, (int) boolconst); 1885 else if(sizeof(bool) == 2) 1886 movw(dst, (int) boolconst); 1887 else if(sizeof(bool) == 4) 1888 movl(dst, (int) boolconst); 1889 else 1890 // unsupported 1891 ShouldNotReachHere(); 1892 } 1893 1894 void MacroAssembler::movbool(Address dst, Register src) { 1895 if(sizeof(bool) == 1) 1896 movb(dst, src); 1897 else if(sizeof(bool) == 2) 1898 movw(dst, src); 1899 else if(sizeof(bool) == 4) 1900 movl(dst, src); 1901 else 1902 // unsupported 1903 ShouldNotReachHere(); 1904 } 1905 1906 void MacroAssembler::movdl(XMMRegister dst, AddressLiteral src, Register rscratch) { 1907 assert(rscratch != noreg || always_reachable(src), "missing"); 1908 1909 if (reachable(src)) { 1910 movdl(dst, as_Address(src)); 1911 } else { 1912 lea(rscratch, src); 1913 movdl(dst, Address(rscratch, 0)); 1914 } 1915 } 1916 1917 void MacroAssembler::movq(XMMRegister dst, AddressLiteral src, Register rscratch) { 1918 assert(rscratch != noreg || always_reachable(src), "missing"); 1919 1920 if (reachable(src)) { 1921 movq(dst, as_Address(src)); 1922 } else { 1923 lea(rscratch, src); 1924 movq(dst, Address(rscratch, 0)); 1925 } 1926 } 1927 1928 void MacroAssembler::movdbl(XMMRegister dst, AddressLiteral src, Register rscratch) { 1929 assert(rscratch != noreg || always_reachable(src), "missing"); 1930 1931 if (reachable(src)) { 1932 if (UseXmmLoadAndClearUpper) { 1933 movsd (dst, as_Address(src)); 1934 } else { 1935 movlpd(dst, as_Address(src)); 1936 } 1937 } else { 1938 lea(rscratch, src); 1939 if (UseXmmLoadAndClearUpper) { 1940 movsd (dst, Address(rscratch, 0)); 1941 } else { 1942 movlpd(dst, Address(rscratch, 0)); 1943 } 1944 } 1945 } 1946 1947 void MacroAssembler::movflt(XMMRegister dst, AddressLiteral src, Register rscratch) { 1948 assert(rscratch != noreg || always_reachable(src), "missing"); 1949 1950 if (reachable(src)) { 1951 movss(dst, as_Address(src)); 1952 } else { 1953 lea(rscratch, src); 1954 movss(dst, Address(rscratch, 0)); 1955 } 1956 } 1957 1958 void MacroAssembler::movptr(Register dst, Register src) { 1959 movq(dst, src); 1960 } 1961 1962 void MacroAssembler::movptr(Register dst, Address src) { 1963 movq(dst, src); 1964 } 1965 1966 // src should NEVER be a real pointer. Use AddressLiteral for true pointers 1967 void MacroAssembler::movptr(Register dst, intptr_t src) { 1968 if (is_uimm32(src)) { 1969 movl(dst, checked_cast<uint32_t>(src)); 1970 } else if (is_simm32(src)) { 1971 movq(dst, checked_cast<int32_t>(src)); 1972 } else { 1973 mov64(dst, src); 1974 } 1975 } 1976 1977 void MacroAssembler::movptr(Address dst, Register src) { 1978 movq(dst, src); 1979 } 1980 1981 void MacroAssembler::movptr(Address dst, int32_t src) { 1982 movslq(dst, src); 1983 } 1984 1985 void MacroAssembler::movdqu(Address dst, XMMRegister src) { 1986 assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 1987 Assembler::movdqu(dst, src); 1988 } 1989 1990 void MacroAssembler::movdqu(XMMRegister dst, Address src) { 1991 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 1992 Assembler::movdqu(dst, src); 1993 } 1994 1995 void MacroAssembler::movdqu(XMMRegister dst, XMMRegister src) { 1996 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 1997 Assembler::movdqu(dst, src); 1998 } 1999 2000 void MacroAssembler::movdqu(XMMRegister dst, AddressLiteral src, Register rscratch) { 2001 assert(rscratch != noreg || always_reachable(src), "missing"); 2002 2003 if (reachable(src)) { 2004 movdqu(dst, as_Address(src)); 2005 } else { 2006 lea(rscratch, src); 2007 movdqu(dst, Address(rscratch, 0)); 2008 } 2009 } 2010 2011 void MacroAssembler::vmovdqu(Address dst, XMMRegister src) { 2012 assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 2013 Assembler::vmovdqu(dst, src); 2014 } 2015 2016 void MacroAssembler::vmovdqu(XMMRegister dst, Address src) { 2017 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 2018 Assembler::vmovdqu(dst, src); 2019 } 2020 2021 void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src) { 2022 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 2023 Assembler::vmovdqu(dst, src); 2024 } 2025 2026 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, Register rscratch) { 2027 assert(rscratch != noreg || always_reachable(src), "missing"); 2028 2029 if (reachable(src)) { 2030 vmovdqu(dst, as_Address(src)); 2031 } 2032 else { 2033 lea(rscratch, src); 2034 vmovdqu(dst, Address(rscratch, 0)); 2035 } 2036 } 2037 2038 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 2039 assert(rscratch != noreg || always_reachable(src), "missing"); 2040 2041 if (vector_len == AVX_512bit) { 2042 evmovdquq(dst, src, AVX_512bit, rscratch); 2043 } else if (vector_len == AVX_256bit) { 2044 vmovdqu(dst, src, rscratch); 2045 } else { 2046 movdqu(dst, src, rscratch); 2047 } 2048 } 2049 2050 void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src, int vector_len) { 2051 if (vector_len == AVX_512bit) { 2052 evmovdquq(dst, src, AVX_512bit); 2053 } else if (vector_len == AVX_256bit) { 2054 vmovdqu(dst, src); 2055 } else { 2056 movdqu(dst, src); 2057 } 2058 } 2059 2060 void MacroAssembler::vmovdqu(Address dst, XMMRegister src, int vector_len) { 2061 if (vector_len == AVX_512bit) { 2062 evmovdquq(dst, src, AVX_512bit); 2063 } else if (vector_len == AVX_256bit) { 2064 vmovdqu(dst, src); 2065 } else { 2066 movdqu(dst, src); 2067 } 2068 } 2069 2070 void MacroAssembler::vmovdqu(XMMRegister dst, Address src, int vector_len) { 2071 if (vector_len == AVX_512bit) { 2072 evmovdquq(dst, src, AVX_512bit); 2073 } else if (vector_len == AVX_256bit) { 2074 vmovdqu(dst, src); 2075 } else { 2076 movdqu(dst, src); 2077 } 2078 } 2079 2080 void MacroAssembler::vmovdqa(XMMRegister dst, AddressLiteral src, Register rscratch) { 2081 assert(rscratch != noreg || always_reachable(src), "missing"); 2082 2083 if (reachable(src)) { 2084 vmovdqa(dst, as_Address(src)); 2085 } 2086 else { 2087 lea(rscratch, src); 2088 vmovdqa(dst, Address(rscratch, 0)); 2089 } 2090 } 2091 2092 void MacroAssembler::vmovdqa(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 2093 assert(rscratch != noreg || always_reachable(src), "missing"); 2094 2095 if (vector_len == AVX_512bit) { 2096 evmovdqaq(dst, src, AVX_512bit, rscratch); 2097 } else if (vector_len == AVX_256bit) { 2098 vmovdqa(dst, src, rscratch); 2099 } else { 2100 movdqa(dst, src, rscratch); 2101 } 2102 } 2103 2104 void MacroAssembler::kmov(KRegister dst, Address src) { 2105 if (VM_Version::supports_avx512bw()) { 2106 kmovql(dst, src); 2107 } else { 2108 assert(VM_Version::supports_evex(), ""); 2109 kmovwl(dst, src); 2110 } 2111 } 2112 2113 void MacroAssembler::kmov(Address dst, KRegister src) { 2114 if (VM_Version::supports_avx512bw()) { 2115 kmovql(dst, src); 2116 } else { 2117 assert(VM_Version::supports_evex(), ""); 2118 kmovwl(dst, src); 2119 } 2120 } 2121 2122 void MacroAssembler::kmov(KRegister dst, KRegister src) { 2123 if (VM_Version::supports_avx512bw()) { 2124 kmovql(dst, src); 2125 } else { 2126 assert(VM_Version::supports_evex(), ""); 2127 kmovwl(dst, src); 2128 } 2129 } 2130 2131 void MacroAssembler::kmov(Register dst, KRegister src) { 2132 if (VM_Version::supports_avx512bw()) { 2133 kmovql(dst, src); 2134 } else { 2135 assert(VM_Version::supports_evex(), ""); 2136 kmovwl(dst, src); 2137 } 2138 } 2139 2140 void MacroAssembler::kmov(KRegister dst, Register src) { 2141 if (VM_Version::supports_avx512bw()) { 2142 kmovql(dst, src); 2143 } else { 2144 assert(VM_Version::supports_evex(), ""); 2145 kmovwl(dst, src); 2146 } 2147 } 2148 2149 void MacroAssembler::kmovql(KRegister dst, AddressLiteral src, Register rscratch) { 2150 assert(rscratch != noreg || always_reachable(src), "missing"); 2151 2152 if (reachable(src)) { 2153 kmovql(dst, as_Address(src)); 2154 } else { 2155 lea(rscratch, src); 2156 kmovql(dst, Address(rscratch, 0)); 2157 } 2158 } 2159 2160 void MacroAssembler::kmovwl(KRegister dst, AddressLiteral src, Register rscratch) { 2161 assert(rscratch != noreg || always_reachable(src), "missing"); 2162 2163 if (reachable(src)) { 2164 kmovwl(dst, as_Address(src)); 2165 } else { 2166 lea(rscratch, src); 2167 kmovwl(dst, Address(rscratch, 0)); 2168 } 2169 } 2170 2171 void MacroAssembler::evmovdqub(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, 2172 int vector_len, Register rscratch) { 2173 assert(rscratch != noreg || always_reachable(src), "missing"); 2174 2175 if (reachable(src)) { 2176 Assembler::evmovdqub(dst, mask, as_Address(src), merge, vector_len); 2177 } else { 2178 lea(rscratch, src); 2179 Assembler::evmovdqub(dst, mask, Address(rscratch, 0), merge, vector_len); 2180 } 2181 } 2182 2183 void MacroAssembler::evmovdquw(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, 2184 int vector_len, Register rscratch) { 2185 assert(rscratch != noreg || always_reachable(src), "missing"); 2186 2187 if (reachable(src)) { 2188 Assembler::evmovdquw(dst, mask, as_Address(src), merge, vector_len); 2189 } else { 2190 lea(rscratch, src); 2191 Assembler::evmovdquw(dst, mask, Address(rscratch, 0), merge, vector_len); 2192 } 2193 } 2194 2195 void MacroAssembler::evmovdqul(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) { 2196 assert(rscratch != noreg || always_reachable(src), "missing"); 2197 2198 if (reachable(src)) { 2199 Assembler::evmovdqul(dst, mask, as_Address(src), merge, vector_len); 2200 } else { 2201 lea(rscratch, src); 2202 Assembler::evmovdqul(dst, mask, Address(rscratch, 0), merge, vector_len); 2203 } 2204 } 2205 2206 void MacroAssembler::evmovdquq(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) { 2207 assert(rscratch != noreg || always_reachable(src), "missing"); 2208 2209 if (reachable(src)) { 2210 Assembler::evmovdquq(dst, mask, as_Address(src), merge, vector_len); 2211 } else { 2212 lea(rscratch, src); 2213 Assembler::evmovdquq(dst, mask, Address(rscratch, 0), merge, vector_len); 2214 } 2215 } 2216 2217 void MacroAssembler::evmovdquq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 2218 assert(rscratch != noreg || always_reachable(src), "missing"); 2219 2220 if (reachable(src)) { 2221 Assembler::evmovdquq(dst, as_Address(src), vector_len); 2222 } else { 2223 lea(rscratch, src); 2224 Assembler::evmovdquq(dst, Address(rscratch, 0), vector_len); 2225 } 2226 } 2227 2228 void MacroAssembler::evmovdqaq(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) { 2229 assert(rscratch != noreg || always_reachable(src), "missing"); 2230 2231 if (reachable(src)) { 2232 Assembler::evmovdqaq(dst, mask, as_Address(src), merge, vector_len); 2233 } else { 2234 lea(rscratch, src); 2235 Assembler::evmovdqaq(dst, mask, Address(rscratch, 0), merge, vector_len); 2236 } 2237 } 2238 2239 void MacroAssembler::evmovdqaq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 2240 assert(rscratch != noreg || always_reachable(src), "missing"); 2241 2242 if (reachable(src)) { 2243 Assembler::evmovdqaq(dst, as_Address(src), vector_len); 2244 } else { 2245 lea(rscratch, src); 2246 Assembler::evmovdqaq(dst, Address(rscratch, 0), vector_len); 2247 } 2248 } 2249 2250 2251 void MacroAssembler::movdqa(XMMRegister dst, AddressLiteral src, Register rscratch) { 2252 assert(rscratch != noreg || always_reachable(src), "missing"); 2253 2254 if (reachable(src)) { 2255 Assembler::movdqa(dst, as_Address(src)); 2256 } else { 2257 lea(rscratch, src); 2258 Assembler::movdqa(dst, Address(rscratch, 0)); 2259 } 2260 } 2261 2262 void MacroAssembler::movsd(XMMRegister dst, AddressLiteral src, Register rscratch) { 2263 assert(rscratch != noreg || always_reachable(src), "missing"); 2264 2265 if (reachable(src)) { 2266 Assembler::movsd(dst, as_Address(src)); 2267 } else { 2268 lea(rscratch, src); 2269 Assembler::movsd(dst, Address(rscratch, 0)); 2270 } 2271 } 2272 2273 void MacroAssembler::movss(XMMRegister dst, AddressLiteral src, Register rscratch) { 2274 assert(rscratch != noreg || always_reachable(src), "missing"); 2275 2276 if (reachable(src)) { 2277 Assembler::movss(dst, as_Address(src)); 2278 } else { 2279 lea(rscratch, src); 2280 Assembler::movss(dst, Address(rscratch, 0)); 2281 } 2282 } 2283 2284 void MacroAssembler::movddup(XMMRegister dst, AddressLiteral src, Register rscratch) { 2285 assert(rscratch != noreg || always_reachable(src), "missing"); 2286 2287 if (reachable(src)) { 2288 Assembler::movddup(dst, as_Address(src)); 2289 } else { 2290 lea(rscratch, src); 2291 Assembler::movddup(dst, Address(rscratch, 0)); 2292 } 2293 } 2294 2295 void MacroAssembler::vmovddup(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 2296 assert(rscratch != noreg || always_reachable(src), "missing"); 2297 2298 if (reachable(src)) { 2299 Assembler::vmovddup(dst, as_Address(src), vector_len); 2300 } else { 2301 lea(rscratch, src); 2302 Assembler::vmovddup(dst, Address(rscratch, 0), vector_len); 2303 } 2304 } 2305 2306 void MacroAssembler::mulsd(XMMRegister dst, AddressLiteral src, Register rscratch) { 2307 assert(rscratch != noreg || always_reachable(src), "missing"); 2308 2309 if (reachable(src)) { 2310 Assembler::mulsd(dst, as_Address(src)); 2311 } else { 2312 lea(rscratch, src); 2313 Assembler::mulsd(dst, Address(rscratch, 0)); 2314 } 2315 } 2316 2317 void MacroAssembler::mulss(XMMRegister dst, AddressLiteral src, Register rscratch) { 2318 assert(rscratch != noreg || always_reachable(src), "missing"); 2319 2320 if (reachable(src)) { 2321 Assembler::mulss(dst, as_Address(src)); 2322 } else { 2323 lea(rscratch, src); 2324 Assembler::mulss(dst, Address(rscratch, 0)); 2325 } 2326 } 2327 2328 void MacroAssembler::null_check(Register reg, int offset) { 2329 if (needs_explicit_null_check(offset)) { 2330 // provoke OS null exception if reg is null by 2331 // accessing M[reg] w/o changing any (non-CC) registers 2332 // NOTE: cmpl is plenty here to provoke a segv 2333 cmpptr(rax, Address(reg, 0)); 2334 // Note: should probably use testl(rax, Address(reg, 0)); 2335 // may be shorter code (however, this version of 2336 // testl needs to be implemented first) 2337 } else { 2338 // nothing to do, (later) access of M[reg + offset] 2339 // will provoke OS null exception if reg is null 2340 } 2341 } 2342 2343 void MacroAssembler::os_breakpoint() { 2344 // instead of directly emitting a breakpoint, call os:breakpoint for better debugability 2345 // (e.g., MSVC can't call ps() otherwise) 2346 call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint))); 2347 } 2348 2349 void MacroAssembler::unimplemented(const char* what) { 2350 const char* buf = nullptr; 2351 { 2352 ResourceMark rm; 2353 stringStream ss; 2354 ss.print("unimplemented: %s", what); 2355 buf = code_string(ss.as_string()); 2356 } 2357 stop(buf); 2358 } 2359 2360 #define XSTATE_BV 0x200 2361 2362 void MacroAssembler::pop_CPU_state() { 2363 pop_FPU_state(); 2364 pop_IU_state(); 2365 } 2366 2367 void MacroAssembler::pop_FPU_state() { 2368 fxrstor(Address(rsp, 0)); 2369 addptr(rsp, FPUStateSizeInWords * wordSize); 2370 } 2371 2372 void MacroAssembler::pop_IU_state() { 2373 popa(); 2374 addq(rsp, 8); 2375 popf(); 2376 } 2377 2378 // Save Integer and Float state 2379 // Warning: Stack must be 16 byte aligned (64bit) 2380 void MacroAssembler::push_CPU_state() { 2381 push_IU_state(); 2382 push_FPU_state(); 2383 } 2384 2385 void MacroAssembler::push_FPU_state() { 2386 subptr(rsp, FPUStateSizeInWords * wordSize); 2387 fxsave(Address(rsp, 0)); 2388 } 2389 2390 void MacroAssembler::push_IU_state() { 2391 // Push flags first because pusha kills them 2392 pushf(); 2393 // Make sure rsp stays 16-byte aligned 2394 subq(rsp, 8); 2395 pusha(); 2396 } 2397 2398 void MacroAssembler::push_cont_fastpath() { 2399 if (!Continuations::enabled()) return; 2400 2401 Label L_done; 2402 cmpptr(rsp, Address(r15_thread, JavaThread::cont_fastpath_offset())); 2403 jccb(Assembler::belowEqual, L_done); 2404 movptr(Address(r15_thread, JavaThread::cont_fastpath_offset()), rsp); 2405 bind(L_done); 2406 } 2407 2408 void MacroAssembler::pop_cont_fastpath() { 2409 if (!Continuations::enabled()) return; 2410 2411 Label L_done; 2412 cmpptr(rsp, Address(r15_thread, JavaThread::cont_fastpath_offset())); 2413 jccb(Assembler::below, L_done); 2414 movptr(Address(r15_thread, JavaThread::cont_fastpath_offset()), 0); 2415 bind(L_done); 2416 } 2417 2418 void MacroAssembler::inc_held_monitor_count() { 2419 incrementq(Address(r15_thread, JavaThread::held_monitor_count_offset())); 2420 } 2421 2422 void MacroAssembler::dec_held_monitor_count() { 2423 decrementq(Address(r15_thread, JavaThread::held_monitor_count_offset())); 2424 } 2425 2426 #ifdef ASSERT 2427 void MacroAssembler::stop_if_in_cont(Register cont, const char* name) { 2428 Label no_cont; 2429 movptr(cont, Address(r15_thread, JavaThread::cont_entry_offset())); 2430 testl(cont, cont); 2431 jcc(Assembler::zero, no_cont); 2432 stop(name); 2433 bind(no_cont); 2434 } 2435 #endif 2436 2437 void MacroAssembler::reset_last_Java_frame(bool clear_fp) { // determine java_thread register 2438 // we must set sp to zero to clear frame 2439 movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), NULL_WORD); 2440 // must clear fp, so that compiled frames are not confused; it is 2441 // possible that we need it only for debugging 2442 if (clear_fp) { 2443 movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()), NULL_WORD); 2444 } 2445 // Always clear the pc because it could have been set by make_walkable() 2446 movptr(Address(r15_thread, JavaThread::last_Java_pc_offset()), NULL_WORD); 2447 vzeroupper(); 2448 } 2449 2450 void MacroAssembler::round_to(Register reg, int modulus) { 2451 addptr(reg, modulus - 1); 2452 andptr(reg, -modulus); 2453 } 2454 2455 void MacroAssembler::safepoint_poll(Label& slow_path, bool at_return, bool in_nmethod) { 2456 if (at_return) { 2457 // Note that when in_nmethod is set, the stack pointer is incremented before the poll. Therefore, 2458 // we may safely use rsp instead to perform the stack watermark check. 2459 cmpptr(in_nmethod ? rsp : rbp, Address(r15_thread, JavaThread::polling_word_offset())); 2460 jcc(Assembler::above, slow_path); 2461 return; 2462 } 2463 testb(Address(r15_thread, JavaThread::polling_word_offset()), SafepointMechanism::poll_bit()); 2464 jcc(Assembler::notZero, slow_path); // handshake bit set implies poll 2465 } 2466 2467 // Calls to C land 2468 // 2469 // When entering C land, the rbp, & rsp of the last Java frame have to be recorded 2470 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp 2471 // has to be reset to 0. This is required to allow proper stack traversal. 2472 void MacroAssembler::set_last_Java_frame(Register last_java_sp, 2473 Register last_java_fp, 2474 address last_java_pc, 2475 Register rscratch) { 2476 vzeroupper(); 2477 // determine last_java_sp register 2478 if (!last_java_sp->is_valid()) { 2479 last_java_sp = rsp; 2480 } 2481 // last_java_fp is optional 2482 if (last_java_fp->is_valid()) { 2483 movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()), last_java_fp); 2484 } 2485 // last_java_pc is optional 2486 if (last_java_pc != nullptr) { 2487 Address java_pc(r15_thread, 2488 JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset()); 2489 lea(java_pc, InternalAddress(last_java_pc), rscratch); 2490 } 2491 movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), last_java_sp); 2492 } 2493 2494 void MacroAssembler::set_last_Java_frame(Register last_java_sp, 2495 Register last_java_fp, 2496 Label &L, 2497 Register scratch) { 2498 lea(scratch, L); 2499 movptr(Address(r15_thread, JavaThread::last_Java_pc_offset()), scratch); 2500 set_last_Java_frame(last_java_sp, last_java_fp, nullptr, scratch); 2501 } 2502 2503 void MacroAssembler::shlptr(Register dst, int imm8) { 2504 shlq(dst, imm8); 2505 } 2506 2507 void MacroAssembler::shrptr(Register dst, int imm8) { 2508 shrq(dst, imm8); 2509 } 2510 2511 void MacroAssembler::sign_extend_byte(Register reg) { 2512 movsbl(reg, reg); // movsxb 2513 } 2514 2515 void MacroAssembler::sign_extend_short(Register reg) { 2516 movswl(reg, reg); // movsxw 2517 } 2518 2519 void MacroAssembler::testl(Address dst, int32_t imm32) { 2520 if (imm32 >= 0 && is8bit(imm32)) { 2521 testb(dst, imm32); 2522 } else { 2523 Assembler::testl(dst, imm32); 2524 } 2525 } 2526 2527 void MacroAssembler::testl(Register dst, int32_t imm32) { 2528 if (imm32 >= 0 && is8bit(imm32) && dst->has_byte_register()) { 2529 testb(dst, imm32); 2530 } else { 2531 Assembler::testl(dst, imm32); 2532 } 2533 } 2534 2535 void MacroAssembler::testl(Register dst, AddressLiteral src) { 2536 assert(always_reachable(src), "Address should be reachable"); 2537 testl(dst, as_Address(src)); 2538 } 2539 2540 void MacroAssembler::testq(Address dst, int32_t imm32) { 2541 if (imm32 >= 0) { 2542 testl(dst, imm32); 2543 } else { 2544 Assembler::testq(dst, imm32); 2545 } 2546 } 2547 2548 void MacroAssembler::testq(Register dst, int32_t imm32) { 2549 if (imm32 >= 0) { 2550 testl(dst, imm32); 2551 } else { 2552 Assembler::testq(dst, imm32); 2553 } 2554 } 2555 2556 void MacroAssembler::pcmpeqb(XMMRegister dst, XMMRegister src) { 2557 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 2558 Assembler::pcmpeqb(dst, src); 2559 } 2560 2561 void MacroAssembler::pcmpeqw(XMMRegister dst, XMMRegister src) { 2562 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 2563 Assembler::pcmpeqw(dst, src); 2564 } 2565 2566 void MacroAssembler::pcmpestri(XMMRegister dst, Address src, int imm8) { 2567 assert((dst->encoding() < 16),"XMM register should be 0-15"); 2568 Assembler::pcmpestri(dst, src, imm8); 2569 } 2570 2571 void MacroAssembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) { 2572 assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15"); 2573 Assembler::pcmpestri(dst, src, imm8); 2574 } 2575 2576 void MacroAssembler::pmovzxbw(XMMRegister dst, XMMRegister src) { 2577 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 2578 Assembler::pmovzxbw(dst, src); 2579 } 2580 2581 void MacroAssembler::pmovzxbw(XMMRegister dst, Address src) { 2582 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 2583 Assembler::pmovzxbw(dst, src); 2584 } 2585 2586 void MacroAssembler::pmovmskb(Register dst, XMMRegister src) { 2587 assert((src->encoding() < 16),"XMM register should be 0-15"); 2588 Assembler::pmovmskb(dst, src); 2589 } 2590 2591 void MacroAssembler::ptest(XMMRegister dst, XMMRegister src) { 2592 assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15"); 2593 Assembler::ptest(dst, src); 2594 } 2595 2596 void MacroAssembler::sqrtss(XMMRegister dst, AddressLiteral src, Register rscratch) { 2597 assert(rscratch != noreg || always_reachable(src), "missing"); 2598 2599 if (reachable(src)) { 2600 Assembler::sqrtss(dst, as_Address(src)); 2601 } else { 2602 lea(rscratch, src); 2603 Assembler::sqrtss(dst, Address(rscratch, 0)); 2604 } 2605 } 2606 2607 void MacroAssembler::subsd(XMMRegister dst, AddressLiteral src, Register rscratch) { 2608 assert(rscratch != noreg || always_reachable(src), "missing"); 2609 2610 if (reachable(src)) { 2611 Assembler::subsd(dst, as_Address(src)); 2612 } else { 2613 lea(rscratch, src); 2614 Assembler::subsd(dst, Address(rscratch, 0)); 2615 } 2616 } 2617 2618 void MacroAssembler::roundsd(XMMRegister dst, AddressLiteral src, int32_t rmode, Register rscratch) { 2619 assert(rscratch != noreg || always_reachable(src), "missing"); 2620 2621 if (reachable(src)) { 2622 Assembler::roundsd(dst, as_Address(src), rmode); 2623 } else { 2624 lea(rscratch, src); 2625 Assembler::roundsd(dst, Address(rscratch, 0), rmode); 2626 } 2627 } 2628 2629 void MacroAssembler::subss(XMMRegister dst, AddressLiteral src, Register rscratch) { 2630 assert(rscratch != noreg || always_reachable(src), "missing"); 2631 2632 if (reachable(src)) { 2633 Assembler::subss(dst, as_Address(src)); 2634 } else { 2635 lea(rscratch, src); 2636 Assembler::subss(dst, Address(rscratch, 0)); 2637 } 2638 } 2639 2640 void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src, Register rscratch) { 2641 assert(rscratch != noreg || always_reachable(src), "missing"); 2642 2643 if (reachable(src)) { 2644 Assembler::ucomisd(dst, as_Address(src)); 2645 } else { 2646 lea(rscratch, src); 2647 Assembler::ucomisd(dst, Address(rscratch, 0)); 2648 } 2649 } 2650 2651 void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src, Register rscratch) { 2652 assert(rscratch != noreg || always_reachable(src), "missing"); 2653 2654 if (reachable(src)) { 2655 Assembler::ucomiss(dst, as_Address(src)); 2656 } else { 2657 lea(rscratch, src); 2658 Assembler::ucomiss(dst, Address(rscratch, 0)); 2659 } 2660 } 2661 2662 void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src, Register rscratch) { 2663 assert(rscratch != noreg || always_reachable(src), "missing"); 2664 2665 // Used in sign-bit flipping with aligned address. 2666 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 2667 2668 if (UseAVX > 2 && 2669 (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) && 2670 (dst->encoding() >= 16)) { 2671 vpxor(dst, dst, src, Assembler::AVX_512bit, rscratch); 2672 } else if (reachable(src)) { 2673 Assembler::xorpd(dst, as_Address(src)); 2674 } else { 2675 lea(rscratch, src); 2676 Assembler::xorpd(dst, Address(rscratch, 0)); 2677 } 2678 } 2679 2680 void MacroAssembler::xorpd(XMMRegister dst, XMMRegister src) { 2681 if (UseAVX > 2 && 2682 (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) && 2683 ((dst->encoding() >= 16) || (src->encoding() >= 16))) { 2684 Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit); 2685 } else { 2686 Assembler::xorpd(dst, src); 2687 } 2688 } 2689 2690 void MacroAssembler::xorps(XMMRegister dst, XMMRegister src) { 2691 if (UseAVX > 2 && 2692 (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) && 2693 ((dst->encoding() >= 16) || (src->encoding() >= 16))) { 2694 Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit); 2695 } else { 2696 Assembler::xorps(dst, src); 2697 } 2698 } 2699 2700 void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src, Register rscratch) { 2701 assert(rscratch != noreg || always_reachable(src), "missing"); 2702 2703 // Used in sign-bit flipping with aligned address. 2704 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 2705 2706 if (UseAVX > 2 && 2707 (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) && 2708 (dst->encoding() >= 16)) { 2709 vpxor(dst, dst, src, Assembler::AVX_512bit, rscratch); 2710 } else if (reachable(src)) { 2711 Assembler::xorps(dst, as_Address(src)); 2712 } else { 2713 lea(rscratch, src); 2714 Assembler::xorps(dst, Address(rscratch, 0)); 2715 } 2716 } 2717 2718 void MacroAssembler::pshufb(XMMRegister dst, AddressLiteral src, Register rscratch) { 2719 assert(rscratch != noreg || always_reachable(src), "missing"); 2720 2721 // Used in sign-bit flipping with aligned address. 2722 bool aligned_adr = (((intptr_t)src.target() & 15) == 0); 2723 assert((UseAVX > 0) || aligned_adr, "SSE mode requires address alignment 16 bytes"); 2724 if (reachable(src)) { 2725 Assembler::pshufb(dst, as_Address(src)); 2726 } else { 2727 lea(rscratch, src); 2728 Assembler::pshufb(dst, Address(rscratch, 0)); 2729 } 2730 } 2731 2732 // AVX 3-operands instructions 2733 2734 void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 2735 assert(rscratch != noreg || always_reachable(src), "missing"); 2736 2737 if (reachable(src)) { 2738 vaddsd(dst, nds, as_Address(src)); 2739 } else { 2740 lea(rscratch, src); 2741 vaddsd(dst, nds, Address(rscratch, 0)); 2742 } 2743 } 2744 2745 void MacroAssembler::vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 2746 assert(rscratch != noreg || always_reachable(src), "missing"); 2747 2748 if (reachable(src)) { 2749 vaddss(dst, nds, as_Address(src)); 2750 } else { 2751 lea(rscratch, src); 2752 vaddss(dst, nds, Address(rscratch, 0)); 2753 } 2754 } 2755 2756 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 2757 assert(UseAVX > 0, "requires some form of AVX"); 2758 assert(rscratch != noreg || always_reachable(src), "missing"); 2759 2760 if (reachable(src)) { 2761 Assembler::vpaddb(dst, nds, as_Address(src), vector_len); 2762 } else { 2763 lea(rscratch, src); 2764 Assembler::vpaddb(dst, nds, Address(rscratch, 0), vector_len); 2765 } 2766 } 2767 2768 void MacroAssembler::vpaddd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 2769 assert(UseAVX > 0, "requires some form of AVX"); 2770 assert(rscratch != noreg || always_reachable(src), "missing"); 2771 2772 if (reachable(src)) { 2773 Assembler::vpaddd(dst, nds, as_Address(src), vector_len); 2774 } else { 2775 lea(rscratch, src); 2776 Assembler::vpaddd(dst, nds, Address(rscratch, 0), vector_len); 2777 } 2778 } 2779 2780 void MacroAssembler::vabsss(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch) { 2781 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); 2782 assert(rscratch != noreg || always_reachable(negate_field), "missing"); 2783 2784 vandps(dst, nds, negate_field, vector_len, rscratch); 2785 } 2786 2787 void MacroAssembler::vabssd(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch) { 2788 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); 2789 assert(rscratch != noreg || always_reachable(negate_field), "missing"); 2790 2791 vandpd(dst, nds, negate_field, vector_len, rscratch); 2792 } 2793 2794 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 2795 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 2796 Assembler::vpaddb(dst, nds, src, vector_len); 2797 } 2798 2799 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 2800 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 2801 Assembler::vpaddb(dst, nds, src, vector_len); 2802 } 2803 2804 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 2805 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 2806 Assembler::vpaddw(dst, nds, src, vector_len); 2807 } 2808 2809 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 2810 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 2811 Assembler::vpaddw(dst, nds, src, vector_len); 2812 } 2813 2814 void MacroAssembler::vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 2815 assert(rscratch != noreg || always_reachable(src), "missing"); 2816 2817 if (reachable(src)) { 2818 Assembler::vpand(dst, nds, as_Address(src), vector_len); 2819 } else { 2820 lea(rscratch, src); 2821 Assembler::vpand(dst, nds, Address(rscratch, 0), vector_len); 2822 } 2823 } 2824 2825 void MacroAssembler::vpbroadcastd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 2826 assert(rscratch != noreg || always_reachable(src), "missing"); 2827 2828 if (reachable(src)) { 2829 Assembler::vpbroadcastd(dst, as_Address(src), vector_len); 2830 } else { 2831 lea(rscratch, src); 2832 Assembler::vpbroadcastd(dst, Address(rscratch, 0), vector_len); 2833 } 2834 } 2835 2836 void MacroAssembler::vbroadcasti128(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 2837 assert(rscratch != noreg || always_reachable(src), "missing"); 2838 2839 if (reachable(src)) { 2840 Assembler::vbroadcasti128(dst, as_Address(src), vector_len); 2841 } else { 2842 lea(rscratch, src); 2843 Assembler::vbroadcasti128(dst, Address(rscratch, 0), vector_len); 2844 } 2845 } 2846 2847 void MacroAssembler::vpbroadcastq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 2848 assert(rscratch != noreg || always_reachable(src), "missing"); 2849 2850 if (reachable(src)) { 2851 Assembler::vpbroadcastq(dst, as_Address(src), vector_len); 2852 } else { 2853 lea(rscratch, src); 2854 Assembler::vpbroadcastq(dst, Address(rscratch, 0), vector_len); 2855 } 2856 } 2857 2858 void MacroAssembler::vbroadcastsd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 2859 assert(rscratch != noreg || always_reachable(src), "missing"); 2860 2861 if (reachable(src)) { 2862 Assembler::vbroadcastsd(dst, as_Address(src), vector_len); 2863 } else { 2864 lea(rscratch, src); 2865 Assembler::vbroadcastsd(dst, Address(rscratch, 0), vector_len); 2866 } 2867 } 2868 2869 void MacroAssembler::vbroadcastss(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 2870 assert(rscratch != noreg || always_reachable(src), "missing"); 2871 2872 if (reachable(src)) { 2873 Assembler::vbroadcastss(dst, as_Address(src), vector_len); 2874 } else { 2875 lea(rscratch, src); 2876 Assembler::vbroadcastss(dst, Address(rscratch, 0), vector_len); 2877 } 2878 } 2879 2880 // Vector float blend 2881 // vblendvps(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len, bool compute_mask = true, XMMRegister scratch = xnoreg) 2882 void MacroAssembler::vblendvps(XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister mask, int vector_len, bool compute_mask, XMMRegister scratch) { 2883 // WARN: Allow dst == (src1|src2), mask == scratch 2884 bool blend_emulation = EnableX86ECoreOpts && UseAVX > 1; 2885 bool scratch_available = scratch != xnoreg && scratch != src1 && scratch != src2 && scratch != dst; 2886 bool dst_available = dst != mask && (dst != src1 || dst != src2); 2887 if (blend_emulation && scratch_available && dst_available) { 2888 if (compute_mask) { 2889 vpsrad(scratch, mask, 32, vector_len); 2890 mask = scratch; 2891 } 2892 if (dst == src1) { 2893 vpandn(dst, mask, src1, vector_len); // if mask == 0, src1 2894 vpand (scratch, mask, src2, vector_len); // if mask == 1, src2 2895 } else { 2896 vpand (dst, mask, src2, vector_len); // if mask == 1, src2 2897 vpandn(scratch, mask, src1, vector_len); // if mask == 0, src1 2898 } 2899 vpor(dst, dst, scratch, vector_len); 2900 } else { 2901 Assembler::vblendvps(dst, src1, src2, mask, vector_len); 2902 } 2903 } 2904 2905 // vblendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len, bool compute_mask = true, XMMRegister scratch = xnoreg) 2906 void MacroAssembler::vblendvpd(XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister mask, int vector_len, bool compute_mask, XMMRegister scratch) { 2907 // WARN: Allow dst == (src1|src2), mask == scratch 2908 bool blend_emulation = EnableX86ECoreOpts && UseAVX > 1; 2909 bool scratch_available = scratch != xnoreg && scratch != src1 && scratch != src2 && scratch != dst && (!compute_mask || scratch != mask); 2910 bool dst_available = dst != mask && (dst != src1 || dst != src2); 2911 if (blend_emulation && scratch_available && dst_available) { 2912 if (compute_mask) { 2913 vpxor(scratch, scratch, scratch, vector_len); 2914 vpcmpgtq(scratch, scratch, mask, vector_len); 2915 mask = scratch; 2916 } 2917 if (dst == src1) { 2918 vpandn(dst, mask, src1, vector_len); // if mask == 0, src 2919 vpand (scratch, mask, src2, vector_len); // if mask == 1, src2 2920 } else { 2921 vpand (dst, mask, src2, vector_len); // if mask == 1, src2 2922 vpandn(scratch, mask, src1, vector_len); // if mask == 0, src 2923 } 2924 vpor(dst, dst, scratch, vector_len); 2925 } else { 2926 Assembler::vblendvpd(dst, src1, src2, mask, vector_len); 2927 } 2928 } 2929 2930 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 2931 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 2932 Assembler::vpcmpeqb(dst, nds, src, vector_len); 2933 } 2934 2935 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) { 2936 assert(((dst->encoding() < 16 && src1->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 2937 Assembler::vpcmpeqb(dst, src1, src2, vector_len); 2938 } 2939 2940 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 2941 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 2942 Assembler::vpcmpeqw(dst, nds, src, vector_len); 2943 } 2944 2945 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 2946 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 2947 Assembler::vpcmpeqw(dst, nds, src, vector_len); 2948 } 2949 2950 void MacroAssembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 2951 assert(rscratch != noreg || always_reachable(src), "missing"); 2952 2953 if (reachable(src)) { 2954 Assembler::evpcmpeqd(kdst, mask, nds, as_Address(src), vector_len); 2955 } else { 2956 lea(rscratch, src); 2957 Assembler::evpcmpeqd(kdst, mask, nds, Address(rscratch, 0), vector_len); 2958 } 2959 } 2960 2961 void MacroAssembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, 2962 int comparison, bool is_signed, int vector_len, Register rscratch) { 2963 assert(rscratch != noreg || always_reachable(src), "missing"); 2964 2965 if (reachable(src)) { 2966 Assembler::evpcmpd(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len); 2967 } else { 2968 lea(rscratch, src); 2969 Assembler::evpcmpd(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len); 2970 } 2971 } 2972 2973 void MacroAssembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, 2974 int comparison, bool is_signed, int vector_len, Register rscratch) { 2975 assert(rscratch != noreg || always_reachable(src), "missing"); 2976 2977 if (reachable(src)) { 2978 Assembler::evpcmpq(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len); 2979 } else { 2980 lea(rscratch, src); 2981 Assembler::evpcmpq(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len); 2982 } 2983 } 2984 2985 void MacroAssembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, 2986 int comparison, bool is_signed, int vector_len, Register rscratch) { 2987 assert(rscratch != noreg || always_reachable(src), "missing"); 2988 2989 if (reachable(src)) { 2990 Assembler::evpcmpb(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len); 2991 } else { 2992 lea(rscratch, src); 2993 Assembler::evpcmpb(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len); 2994 } 2995 } 2996 2997 void MacroAssembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, 2998 int comparison, bool is_signed, int vector_len, Register rscratch) { 2999 assert(rscratch != noreg || always_reachable(src), "missing"); 3000 3001 if (reachable(src)) { 3002 Assembler::evpcmpw(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len); 3003 } else { 3004 lea(rscratch, src); 3005 Assembler::evpcmpw(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len); 3006 } 3007 } 3008 3009 void MacroAssembler::vpcmpCC(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, Width width, int vector_len) { 3010 if (width == Assembler::Q) { 3011 Assembler::vpcmpCCq(dst, nds, src, cond_encoding, vector_len); 3012 } else { 3013 Assembler::vpcmpCCbwd(dst, nds, src, cond_encoding, vector_len); 3014 } 3015 } 3016 3017 void MacroAssembler::vpcmpCCW(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister xtmp, ComparisonPredicate cond, Width width, int vector_len) { 3018 int eq_cond_enc = 0x29; 3019 int gt_cond_enc = 0x37; 3020 if (width != Assembler::Q) { 3021 eq_cond_enc = 0x74 + width; 3022 gt_cond_enc = 0x64 + width; 3023 } 3024 switch (cond) { 3025 case eq: 3026 vpcmpCC(dst, nds, src, eq_cond_enc, width, vector_len); 3027 break; 3028 case neq: 3029 vpcmpCC(dst, nds, src, eq_cond_enc, width, vector_len); 3030 vallones(xtmp, vector_len); 3031 vpxor(dst, xtmp, dst, vector_len); 3032 break; 3033 case le: 3034 vpcmpCC(dst, nds, src, gt_cond_enc, width, vector_len); 3035 vallones(xtmp, vector_len); 3036 vpxor(dst, xtmp, dst, vector_len); 3037 break; 3038 case nlt: 3039 vpcmpCC(dst, src, nds, gt_cond_enc, width, vector_len); 3040 vallones(xtmp, vector_len); 3041 vpxor(dst, xtmp, dst, vector_len); 3042 break; 3043 case lt: 3044 vpcmpCC(dst, src, nds, gt_cond_enc, width, vector_len); 3045 break; 3046 case nle: 3047 vpcmpCC(dst, nds, src, gt_cond_enc, width, vector_len); 3048 break; 3049 default: 3050 assert(false, "Should not reach here"); 3051 } 3052 } 3053 3054 void MacroAssembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) { 3055 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3056 Assembler::vpmovzxbw(dst, src, vector_len); 3057 } 3058 3059 void MacroAssembler::vpmovmskb(Register dst, XMMRegister src, int vector_len) { 3060 assert((src->encoding() < 16),"XMM register should be 0-15"); 3061 Assembler::vpmovmskb(dst, src, vector_len); 3062 } 3063 3064 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3065 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3066 Assembler::vpmullw(dst, nds, src, vector_len); 3067 } 3068 3069 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3070 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3071 Assembler::vpmullw(dst, nds, src, vector_len); 3072 } 3073 3074 void MacroAssembler::vpmulld(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3075 assert((UseAVX > 0), "AVX support is needed"); 3076 assert(rscratch != noreg || always_reachable(src), "missing"); 3077 3078 if (reachable(src)) { 3079 Assembler::vpmulld(dst, nds, as_Address(src), vector_len); 3080 } else { 3081 lea(rscratch, src); 3082 Assembler::vpmulld(dst, nds, Address(rscratch, 0), vector_len); 3083 } 3084 } 3085 3086 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3087 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3088 Assembler::vpsubb(dst, nds, src, vector_len); 3089 } 3090 3091 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3092 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3093 Assembler::vpsubb(dst, nds, src, vector_len); 3094 } 3095 3096 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3097 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3098 Assembler::vpsubw(dst, nds, src, vector_len); 3099 } 3100 3101 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3102 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3103 Assembler::vpsubw(dst, nds, src, vector_len); 3104 } 3105 3106 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3107 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3108 Assembler::vpsraw(dst, nds, shift, vector_len); 3109 } 3110 3111 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3112 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3113 Assembler::vpsraw(dst, nds, shift, vector_len); 3114 } 3115 3116 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3117 assert(UseAVX > 2,""); 3118 if (!VM_Version::supports_avx512vl() && vector_len < 2) { 3119 vector_len = 2; 3120 } 3121 Assembler::evpsraq(dst, nds, shift, vector_len); 3122 } 3123 3124 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3125 assert(UseAVX > 2,""); 3126 if (!VM_Version::supports_avx512vl() && vector_len < 2) { 3127 vector_len = 2; 3128 } 3129 Assembler::evpsraq(dst, nds, shift, vector_len); 3130 } 3131 3132 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3133 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3134 Assembler::vpsrlw(dst, nds, shift, vector_len); 3135 } 3136 3137 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3138 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3139 Assembler::vpsrlw(dst, nds, shift, vector_len); 3140 } 3141 3142 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3143 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3144 Assembler::vpsllw(dst, nds, shift, vector_len); 3145 } 3146 3147 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3148 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3149 Assembler::vpsllw(dst, nds, shift, vector_len); 3150 } 3151 3152 void MacroAssembler::vptest(XMMRegister dst, XMMRegister src) { 3153 assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15"); 3154 Assembler::vptest(dst, src); 3155 } 3156 3157 void MacroAssembler::punpcklbw(XMMRegister dst, XMMRegister src) { 3158 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3159 Assembler::punpcklbw(dst, src); 3160 } 3161 3162 void MacroAssembler::pshufd(XMMRegister dst, Address src, int mode) { 3163 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 3164 Assembler::pshufd(dst, src, mode); 3165 } 3166 3167 void MacroAssembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) { 3168 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3169 Assembler::pshuflw(dst, src, mode); 3170 } 3171 3172 void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3173 assert(rscratch != noreg || always_reachable(src), "missing"); 3174 3175 if (reachable(src)) { 3176 vandpd(dst, nds, as_Address(src), vector_len); 3177 } else { 3178 lea(rscratch, src); 3179 vandpd(dst, nds, Address(rscratch, 0), vector_len); 3180 } 3181 } 3182 3183 void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3184 assert(rscratch != noreg || always_reachable(src), "missing"); 3185 3186 if (reachable(src)) { 3187 vandps(dst, nds, as_Address(src), vector_len); 3188 } else { 3189 lea(rscratch, src); 3190 vandps(dst, nds, Address(rscratch, 0), vector_len); 3191 } 3192 } 3193 3194 void MacroAssembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src, 3195 bool merge, int vector_len, Register rscratch) { 3196 assert(rscratch != noreg || always_reachable(src), "missing"); 3197 3198 if (reachable(src)) { 3199 Assembler::evpord(dst, mask, nds, as_Address(src), merge, vector_len); 3200 } else { 3201 lea(rscratch, src); 3202 Assembler::evpord(dst, mask, nds, Address(rscratch, 0), merge, vector_len); 3203 } 3204 } 3205 3206 void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3207 assert(rscratch != noreg || always_reachable(src), "missing"); 3208 3209 if (reachable(src)) { 3210 vdivsd(dst, nds, as_Address(src)); 3211 } else { 3212 lea(rscratch, src); 3213 vdivsd(dst, nds, Address(rscratch, 0)); 3214 } 3215 } 3216 3217 void MacroAssembler::vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3218 assert(rscratch != noreg || always_reachable(src), "missing"); 3219 3220 if (reachable(src)) { 3221 vdivss(dst, nds, as_Address(src)); 3222 } else { 3223 lea(rscratch, src); 3224 vdivss(dst, nds, Address(rscratch, 0)); 3225 } 3226 } 3227 3228 void MacroAssembler::vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3229 assert(rscratch != noreg || always_reachable(src), "missing"); 3230 3231 if (reachable(src)) { 3232 vmulsd(dst, nds, as_Address(src)); 3233 } else { 3234 lea(rscratch, src); 3235 vmulsd(dst, nds, Address(rscratch, 0)); 3236 } 3237 } 3238 3239 void MacroAssembler::vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3240 assert(rscratch != noreg || always_reachable(src), "missing"); 3241 3242 if (reachable(src)) { 3243 vmulss(dst, nds, as_Address(src)); 3244 } else { 3245 lea(rscratch, src); 3246 vmulss(dst, nds, Address(rscratch, 0)); 3247 } 3248 } 3249 3250 void MacroAssembler::vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3251 assert(rscratch != noreg || always_reachable(src), "missing"); 3252 3253 if (reachable(src)) { 3254 vsubsd(dst, nds, as_Address(src)); 3255 } else { 3256 lea(rscratch, src); 3257 vsubsd(dst, nds, Address(rscratch, 0)); 3258 } 3259 } 3260 3261 void MacroAssembler::vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3262 assert(rscratch != noreg || always_reachable(src), "missing"); 3263 3264 if (reachable(src)) { 3265 vsubss(dst, nds, as_Address(src)); 3266 } else { 3267 lea(rscratch, src); 3268 vsubss(dst, nds, Address(rscratch, 0)); 3269 } 3270 } 3271 3272 void MacroAssembler::vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3273 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); 3274 assert(rscratch != noreg || always_reachable(src), "missing"); 3275 3276 vxorps(dst, nds, src, Assembler::AVX_128bit, rscratch); 3277 } 3278 3279 void MacroAssembler::vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3280 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); 3281 assert(rscratch != noreg || always_reachable(src), "missing"); 3282 3283 vxorpd(dst, nds, src, Assembler::AVX_128bit, rscratch); 3284 } 3285 3286 void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3287 assert(rscratch != noreg || always_reachable(src), "missing"); 3288 3289 if (reachable(src)) { 3290 vxorpd(dst, nds, as_Address(src), vector_len); 3291 } else { 3292 lea(rscratch, src); 3293 vxorpd(dst, nds, Address(rscratch, 0), vector_len); 3294 } 3295 } 3296 3297 void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3298 assert(rscratch != noreg || always_reachable(src), "missing"); 3299 3300 if (reachable(src)) { 3301 vxorps(dst, nds, as_Address(src), vector_len); 3302 } else { 3303 lea(rscratch, src); 3304 vxorps(dst, nds, Address(rscratch, 0), vector_len); 3305 } 3306 } 3307 3308 void MacroAssembler::vpxor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3309 assert(rscratch != noreg || always_reachable(src), "missing"); 3310 3311 if (UseAVX > 1 || (vector_len < 1)) { 3312 if (reachable(src)) { 3313 Assembler::vpxor(dst, nds, as_Address(src), vector_len); 3314 } else { 3315 lea(rscratch, src); 3316 Assembler::vpxor(dst, nds, Address(rscratch, 0), vector_len); 3317 } 3318 } else { 3319 MacroAssembler::vxorpd(dst, nds, src, vector_len, rscratch); 3320 } 3321 } 3322 3323 void MacroAssembler::vpermd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3324 assert(rscratch != noreg || always_reachable(src), "missing"); 3325 3326 if (reachable(src)) { 3327 Assembler::vpermd(dst, nds, as_Address(src), vector_len); 3328 } else { 3329 lea(rscratch, src); 3330 Assembler::vpermd(dst, nds, Address(rscratch, 0), vector_len); 3331 } 3332 } 3333 3334 void MacroAssembler::clear_jobject_tag(Register possibly_non_local) { 3335 const int32_t inverted_mask = ~static_cast<int32_t>(JNIHandles::tag_mask); 3336 STATIC_ASSERT(inverted_mask == -4); // otherwise check this code 3337 // The inverted mask is sign-extended 3338 andptr(possibly_non_local, inverted_mask); 3339 } 3340 3341 void MacroAssembler::resolve_jobject(Register value, 3342 Register tmp) { 3343 Register thread = r15_thread; 3344 assert_different_registers(value, thread, tmp); 3345 Label done, tagged, weak_tagged; 3346 testptr(value, value); 3347 jcc(Assembler::zero, done); // Use null as-is. 3348 testptr(value, JNIHandles::tag_mask); // Test for tag. 3349 jcc(Assembler::notZero, tagged); 3350 3351 // Resolve local handle 3352 access_load_at(T_OBJECT, IN_NATIVE | AS_RAW, value, Address(value, 0), tmp); 3353 verify_oop(value); 3354 jmp(done); 3355 3356 bind(tagged); 3357 testptr(value, JNIHandles::TypeTag::weak_global); // Test for weak tag. 3358 jcc(Assembler::notZero, weak_tagged); 3359 3360 // Resolve global handle 3361 access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp); 3362 verify_oop(value); 3363 jmp(done); 3364 3365 bind(weak_tagged); 3366 // Resolve jweak. 3367 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, 3368 value, Address(value, -JNIHandles::TypeTag::weak_global), tmp); 3369 verify_oop(value); 3370 3371 bind(done); 3372 } 3373 3374 void MacroAssembler::resolve_global_jobject(Register value, 3375 Register tmp) { 3376 Register thread = r15_thread; 3377 assert_different_registers(value, thread, tmp); 3378 Label done; 3379 3380 testptr(value, value); 3381 jcc(Assembler::zero, done); // Use null as-is. 3382 3383 #ifdef ASSERT 3384 { 3385 Label valid_global_tag; 3386 testptr(value, JNIHandles::TypeTag::global); // Test for global tag. 3387 jcc(Assembler::notZero, valid_global_tag); 3388 stop("non global jobject using resolve_global_jobject"); 3389 bind(valid_global_tag); 3390 } 3391 #endif 3392 3393 // Resolve global handle 3394 access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp); 3395 verify_oop(value); 3396 3397 bind(done); 3398 } 3399 3400 void MacroAssembler::subptr(Register dst, int32_t imm32) { 3401 subq(dst, imm32); 3402 } 3403 3404 // Force generation of a 4 byte immediate value even if it fits into 8bit 3405 void MacroAssembler::subptr_imm32(Register dst, int32_t imm32) { 3406 subq_imm32(dst, imm32); 3407 } 3408 3409 void MacroAssembler::subptr(Register dst, Register src) { 3410 subq(dst, src); 3411 } 3412 3413 // C++ bool manipulation 3414 void MacroAssembler::testbool(Register dst) { 3415 if(sizeof(bool) == 1) 3416 testb(dst, 0xff); 3417 else if(sizeof(bool) == 2) { 3418 // testw implementation needed for two byte bools 3419 ShouldNotReachHere(); 3420 } else if(sizeof(bool) == 4) 3421 testl(dst, dst); 3422 else 3423 // unsupported 3424 ShouldNotReachHere(); 3425 } 3426 3427 void MacroAssembler::testptr(Register dst, Register src) { 3428 testq(dst, src); 3429 } 3430 3431 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes. 3432 void MacroAssembler::tlab_allocate(Register obj, 3433 Register var_size_in_bytes, 3434 int con_size_in_bytes, 3435 Register t1, 3436 Register t2, 3437 Label& slow_case) { 3438 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); 3439 bs->tlab_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case); 3440 } 3441 3442 RegSet MacroAssembler::call_clobbered_gp_registers() { 3443 RegSet regs; 3444 regs += RegSet::of(rax, rcx, rdx); 3445 #ifndef _WINDOWS 3446 regs += RegSet::of(rsi, rdi); 3447 #endif 3448 regs += RegSet::range(r8, r11); 3449 if (UseAPX) { 3450 regs += RegSet::range(r16, as_Register(Register::number_of_registers - 1)); 3451 } 3452 return regs; 3453 } 3454 3455 XMMRegSet MacroAssembler::call_clobbered_xmm_registers() { 3456 int num_xmm_registers = XMMRegister::available_xmm_registers(); 3457 #if defined(_WINDOWS) 3458 XMMRegSet result = XMMRegSet::range(xmm0, xmm5); 3459 if (num_xmm_registers > 16) { 3460 result += XMMRegSet::range(xmm16, as_XMMRegister(num_xmm_registers - 1)); 3461 } 3462 return result; 3463 #else 3464 return XMMRegSet::range(xmm0, as_XMMRegister(num_xmm_registers - 1)); 3465 #endif 3466 } 3467 3468 // C1 only ever uses the first double/float of the XMM register. 3469 static int xmm_save_size() { return sizeof(double); } 3470 3471 static void save_xmm_register(MacroAssembler* masm, int offset, XMMRegister reg) { 3472 masm->movdbl(Address(rsp, offset), reg); 3473 } 3474 3475 static void restore_xmm_register(MacroAssembler* masm, int offset, XMMRegister reg) { 3476 masm->movdbl(reg, Address(rsp, offset)); 3477 } 3478 3479 static int register_section_sizes(RegSet gp_registers, XMMRegSet xmm_registers, 3480 bool save_fpu, int& gp_area_size, int& xmm_area_size) { 3481 3482 gp_area_size = align_up(gp_registers.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size, 3483 StackAlignmentInBytes); 3484 xmm_area_size = save_fpu ? xmm_registers.size() * xmm_save_size() : 0; 3485 3486 return gp_area_size + xmm_area_size; 3487 } 3488 3489 void MacroAssembler::push_call_clobbered_registers_except(RegSet exclude, bool save_fpu) { 3490 block_comment("push_call_clobbered_registers start"); 3491 // Regular registers 3492 RegSet gp_registers_to_push = call_clobbered_gp_registers() - exclude; 3493 3494 int gp_area_size; 3495 int xmm_area_size; 3496 int total_save_size = register_section_sizes(gp_registers_to_push, call_clobbered_xmm_registers(), save_fpu, 3497 gp_area_size, xmm_area_size); 3498 subptr(rsp, total_save_size); 3499 3500 push_set(gp_registers_to_push, 0); 3501 3502 if (save_fpu) { 3503 push_set(call_clobbered_xmm_registers(), gp_area_size); 3504 } 3505 3506 block_comment("push_call_clobbered_registers end"); 3507 } 3508 3509 void MacroAssembler::pop_call_clobbered_registers_except(RegSet exclude, bool restore_fpu) { 3510 block_comment("pop_call_clobbered_registers start"); 3511 3512 RegSet gp_registers_to_pop = call_clobbered_gp_registers() - exclude; 3513 3514 int gp_area_size; 3515 int xmm_area_size; 3516 int total_save_size = register_section_sizes(gp_registers_to_pop, call_clobbered_xmm_registers(), restore_fpu, 3517 gp_area_size, xmm_area_size); 3518 3519 if (restore_fpu) { 3520 pop_set(call_clobbered_xmm_registers(), gp_area_size); 3521 } 3522 3523 pop_set(gp_registers_to_pop, 0); 3524 3525 addptr(rsp, total_save_size); 3526 3527 vzeroupper(); 3528 3529 block_comment("pop_call_clobbered_registers end"); 3530 } 3531 3532 void MacroAssembler::push_set(XMMRegSet set, int offset) { 3533 assert(is_aligned(set.size() * xmm_save_size(), StackAlignmentInBytes), "must be"); 3534 int spill_offset = offset; 3535 3536 for (RegSetIterator<XMMRegister> it = set.begin(); *it != xnoreg; ++it) { 3537 save_xmm_register(this, spill_offset, *it); 3538 spill_offset += xmm_save_size(); 3539 } 3540 } 3541 3542 void MacroAssembler::pop_set(XMMRegSet set, int offset) { 3543 int restore_size = set.size() * xmm_save_size(); 3544 assert(is_aligned(restore_size, StackAlignmentInBytes), "must be"); 3545 3546 int restore_offset = offset + restore_size - xmm_save_size(); 3547 3548 for (ReverseRegSetIterator<XMMRegister> it = set.rbegin(); *it != xnoreg; ++it) { 3549 restore_xmm_register(this, restore_offset, *it); 3550 restore_offset -= xmm_save_size(); 3551 } 3552 } 3553 3554 void MacroAssembler::push_set(RegSet set, int offset) { 3555 int spill_offset; 3556 if (offset == -1) { 3557 int register_push_size = set.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size; 3558 int aligned_size = align_up(register_push_size, StackAlignmentInBytes); 3559 subptr(rsp, aligned_size); 3560 spill_offset = 0; 3561 } else { 3562 spill_offset = offset; 3563 } 3564 3565 for (RegSetIterator<Register> it = set.begin(); *it != noreg; ++it) { 3566 movptr(Address(rsp, spill_offset), *it); 3567 spill_offset += Register::max_slots_per_register * VMRegImpl::stack_slot_size; 3568 } 3569 } 3570 3571 void MacroAssembler::pop_set(RegSet set, int offset) { 3572 3573 int gp_reg_size = Register::max_slots_per_register * VMRegImpl::stack_slot_size; 3574 int restore_size = set.size() * gp_reg_size; 3575 int aligned_size = align_up(restore_size, StackAlignmentInBytes); 3576 3577 int restore_offset; 3578 if (offset == -1) { 3579 restore_offset = restore_size - gp_reg_size; 3580 } else { 3581 restore_offset = offset + restore_size - gp_reg_size; 3582 } 3583 for (ReverseRegSetIterator<Register> it = set.rbegin(); *it != noreg; ++it) { 3584 movptr(*it, Address(rsp, restore_offset)); 3585 restore_offset -= gp_reg_size; 3586 } 3587 3588 if (offset == -1) { 3589 addptr(rsp, aligned_size); 3590 } 3591 } 3592 3593 // Preserves the contents of address, destroys the contents length_in_bytes and temp. 3594 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) { 3595 assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different"); 3596 assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord"); 3597 Label done; 3598 3599 testptr(length_in_bytes, length_in_bytes); 3600 jcc(Assembler::zero, done); 3601 3602 // initialize topmost word, divide index by 2, check if odd and test if zero 3603 // note: for the remaining code to work, index must be a multiple of BytesPerWord 3604 #ifdef ASSERT 3605 { 3606 Label L; 3607 testptr(length_in_bytes, BytesPerWord - 1); 3608 jcc(Assembler::zero, L); 3609 stop("length must be a multiple of BytesPerWord"); 3610 bind(L); 3611 } 3612 #endif 3613 Register index = length_in_bytes; 3614 xorptr(temp, temp); // use _zero reg to clear memory (shorter code) 3615 if (UseIncDec) { 3616 shrptr(index, 3); // divide by 8/16 and set carry flag if bit 2 was set 3617 } else { 3618 shrptr(index, 2); // use 2 instructions to avoid partial flag stall 3619 shrptr(index, 1); 3620 } 3621 3622 // initialize remaining object fields: index is a multiple of 2 now 3623 { 3624 Label loop; 3625 bind(loop); 3626 movptr(Address(address, index, Address::times_8, offset_in_bytes - 1*BytesPerWord), temp); 3627 decrement(index); 3628 jcc(Assembler::notZero, loop); 3629 } 3630 3631 bind(done); 3632 } 3633 3634 // Look up the method for a megamorphic invokeinterface call. 3635 // The target method is determined by <intf_klass, itable_index>. 3636 // The receiver klass is in recv_klass. 3637 // On success, the result will be in method_result, and execution falls through. 3638 // On failure, execution transfers to the given label. 3639 void MacroAssembler::lookup_interface_method(Register recv_klass, 3640 Register intf_klass, 3641 RegisterOrConstant itable_index, 3642 Register method_result, 3643 Register scan_temp, 3644 Label& L_no_such_interface, 3645 bool return_method) { 3646 assert_different_registers(recv_klass, intf_klass, scan_temp); 3647 assert_different_registers(method_result, intf_klass, scan_temp); 3648 assert(recv_klass != method_result || !return_method, 3649 "recv_klass can be destroyed when method isn't needed"); 3650 3651 assert(itable_index.is_constant() || itable_index.as_register() == method_result, 3652 "caller must use same register for non-constant itable index as for method"); 3653 3654 // Compute start of first itableOffsetEntry (which is at the end of the vtable) 3655 int vtable_base = in_bytes(Klass::vtable_start_offset()); 3656 int itentry_off = in_bytes(itableMethodEntry::method_offset()); 3657 int scan_step = itableOffsetEntry::size() * wordSize; 3658 int vte_size = vtableEntry::size_in_bytes(); 3659 Address::ScaleFactor times_vte_scale = Address::times_ptr; 3660 assert(vte_size == wordSize, "else adjust times_vte_scale"); 3661 3662 movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset())); 3663 3664 // Could store the aligned, prescaled offset in the klass. 3665 lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base)); 3666 3667 if (return_method) { 3668 // Adjust recv_klass by scaled itable_index, so we can free itable_index. 3669 assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below"); 3670 lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off)); 3671 } 3672 3673 // for (scan = klass->itable(); scan->interface() != nullptr; scan += scan_step) { 3674 // if (scan->interface() == intf) { 3675 // result = (klass + scan->offset() + itable_index); 3676 // } 3677 // } 3678 Label search, found_method; 3679 3680 for (int peel = 1; peel >= 0; peel--) { 3681 movptr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset())); 3682 cmpptr(intf_klass, method_result); 3683 3684 if (peel) { 3685 jccb(Assembler::equal, found_method); 3686 } else { 3687 jccb(Assembler::notEqual, search); 3688 // (invert the test to fall through to found_method...) 3689 } 3690 3691 if (!peel) break; 3692 3693 bind(search); 3694 3695 // Check that the previous entry is non-null. A null entry means that 3696 // the receiver class doesn't implement the interface, and wasn't the 3697 // same as when the caller was compiled. 3698 testptr(method_result, method_result); 3699 jcc(Assembler::zero, L_no_such_interface); 3700 addptr(scan_temp, scan_step); 3701 } 3702 3703 bind(found_method); 3704 3705 if (return_method) { 3706 // Got a hit. 3707 movl(scan_temp, Address(scan_temp, itableOffsetEntry::offset_offset())); 3708 movptr(method_result, Address(recv_klass, scan_temp, Address::times_1)); 3709 } 3710 } 3711 3712 // Look up the method for a megamorphic invokeinterface call in a single pass over itable: 3713 // - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICData 3714 // - find a holder_klass (class that implements the method) vtable offset and get the method from vtable by index 3715 // The target method is determined by <holder_klass, itable_index>. 3716 // The receiver klass is in recv_klass. 3717 // On success, the result will be in method_result, and execution falls through. 3718 // On failure, execution transfers to the given label. 3719 void MacroAssembler::lookup_interface_method_stub(Register recv_klass, 3720 Register holder_klass, 3721 Register resolved_klass, 3722 Register method_result, 3723 Register scan_temp, 3724 Register temp_reg2, 3725 Register receiver, 3726 int itable_index, 3727 Label& L_no_such_interface) { 3728 assert_different_registers(recv_klass, method_result, holder_klass, resolved_klass, scan_temp, temp_reg2, receiver); 3729 Register temp_itbl_klass = method_result; 3730 Register temp_reg = (temp_reg2 == noreg ? recv_klass : temp_reg2); // reuse recv_klass register on 32-bit x86 impl 3731 3732 int vtable_base = in_bytes(Klass::vtable_start_offset()); 3733 int itentry_off = in_bytes(itableMethodEntry::method_offset()); 3734 int scan_step = itableOffsetEntry::size() * wordSize; 3735 int vte_size = vtableEntry::size_in_bytes(); 3736 int ioffset = in_bytes(itableOffsetEntry::interface_offset()); 3737 int ooffset = in_bytes(itableOffsetEntry::offset_offset()); 3738 Address::ScaleFactor times_vte_scale = Address::times_ptr; 3739 assert(vte_size == wordSize, "adjust times_vte_scale"); 3740 3741 Label L_loop_scan_resolved_entry, L_resolved_found, L_holder_found; 3742 3743 // temp_itbl_klass = recv_klass.itable[0] 3744 // scan_temp = &recv_klass.itable[0] + step 3745 movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset())); 3746 movptr(temp_itbl_klass, Address(recv_klass, scan_temp, times_vte_scale, vtable_base + ioffset)); 3747 lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base + ioffset + scan_step)); 3748 xorptr(temp_reg, temp_reg); 3749 3750 // Initial checks: 3751 // - if (holder_klass != resolved_klass), go to "scan for resolved" 3752 // - if (itable[0] == 0), no such interface 3753 // - if (itable[0] == holder_klass), shortcut to "holder found" 3754 cmpptr(holder_klass, resolved_klass); 3755 jccb(Assembler::notEqual, L_loop_scan_resolved_entry); 3756 testptr(temp_itbl_klass, temp_itbl_klass); 3757 jccb(Assembler::zero, L_no_such_interface); 3758 cmpptr(holder_klass, temp_itbl_klass); 3759 jccb(Assembler::equal, L_holder_found); 3760 3761 // Loop: Look for holder_klass record in itable 3762 // do { 3763 // tmp = itable[index]; 3764 // index += step; 3765 // if (tmp == holder_klass) { 3766 // goto L_holder_found; // Found! 3767 // } 3768 // } while (tmp != 0); 3769 // goto L_no_such_interface // Not found. 3770 Label L_scan_holder; 3771 bind(L_scan_holder); 3772 movptr(temp_itbl_klass, Address(scan_temp, 0)); 3773 addptr(scan_temp, scan_step); 3774 cmpptr(holder_klass, temp_itbl_klass); 3775 jccb(Assembler::equal, L_holder_found); 3776 testptr(temp_itbl_klass, temp_itbl_klass); 3777 jccb(Assembler::notZero, L_scan_holder); 3778 3779 jmpb(L_no_such_interface); 3780 3781 // Loop: Look for resolved_class record in itable 3782 // do { 3783 // tmp = itable[index]; 3784 // index += step; 3785 // if (tmp == holder_klass) { 3786 // // Also check if we have met a holder klass 3787 // holder_tmp = itable[index-step-ioffset]; 3788 // } 3789 // if (tmp == resolved_klass) { 3790 // goto L_resolved_found; // Found! 3791 // } 3792 // } while (tmp != 0); 3793 // goto L_no_such_interface // Not found. 3794 // 3795 Label L_loop_scan_resolved; 3796 bind(L_loop_scan_resolved); 3797 movptr(temp_itbl_klass, Address(scan_temp, 0)); 3798 addptr(scan_temp, scan_step); 3799 bind(L_loop_scan_resolved_entry); 3800 cmpptr(holder_klass, temp_itbl_klass); 3801 cmovl(Assembler::equal, temp_reg, Address(scan_temp, ooffset - ioffset - scan_step)); 3802 cmpptr(resolved_klass, temp_itbl_klass); 3803 jccb(Assembler::equal, L_resolved_found); 3804 testptr(temp_itbl_klass, temp_itbl_klass); 3805 jccb(Assembler::notZero, L_loop_scan_resolved); 3806 3807 jmpb(L_no_such_interface); 3808 3809 Label L_ready; 3810 3811 // See if we already have a holder klass. If not, go and scan for it. 3812 bind(L_resolved_found); 3813 testptr(temp_reg, temp_reg); 3814 jccb(Assembler::zero, L_scan_holder); 3815 jmpb(L_ready); 3816 3817 bind(L_holder_found); 3818 movl(temp_reg, Address(scan_temp, ooffset - ioffset - scan_step)); 3819 3820 // Finally, temp_reg contains holder_klass vtable offset 3821 bind(L_ready); 3822 assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below"); 3823 if (temp_reg2 == noreg) { // recv_klass register is clobbered for 32-bit x86 impl 3824 load_klass(scan_temp, receiver, noreg); 3825 movptr(method_result, Address(scan_temp, temp_reg, Address::times_1, itable_index * wordSize + itentry_off)); 3826 } else { 3827 movptr(method_result, Address(recv_klass, temp_reg, Address::times_1, itable_index * wordSize + itentry_off)); 3828 } 3829 } 3830 3831 3832 // virtual method calling 3833 void MacroAssembler::lookup_virtual_method(Register recv_klass, 3834 RegisterOrConstant vtable_index, 3835 Register method_result) { 3836 const ByteSize base = Klass::vtable_start_offset(); 3837 assert(vtableEntry::size() * wordSize == wordSize, "else adjust the scaling in the code below"); 3838 Address vtable_entry_addr(recv_klass, 3839 vtable_index, Address::times_ptr, 3840 base + vtableEntry::method_offset()); 3841 movptr(method_result, vtable_entry_addr); 3842 } 3843 3844 3845 void MacroAssembler::check_klass_subtype(Register sub_klass, 3846 Register super_klass, 3847 Register temp_reg, 3848 Label& L_success) { 3849 Label L_failure; 3850 check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg, &L_success, &L_failure, nullptr); 3851 check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, noreg, &L_success, nullptr); 3852 bind(L_failure); 3853 } 3854 3855 3856 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass, 3857 Register super_klass, 3858 Register temp_reg, 3859 Label* L_success, 3860 Label* L_failure, 3861 Label* L_slow_path, 3862 RegisterOrConstant super_check_offset) { 3863 assert_different_registers(sub_klass, super_klass, temp_reg); 3864 bool must_load_sco = (super_check_offset.constant_or_zero() == -1); 3865 if (super_check_offset.is_register()) { 3866 assert_different_registers(sub_klass, super_klass, 3867 super_check_offset.as_register()); 3868 } else if (must_load_sco) { 3869 assert(temp_reg != noreg, "supply either a temp or a register offset"); 3870 } 3871 3872 Label L_fallthrough; 3873 int label_nulls = 0; 3874 if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; } 3875 if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; } 3876 if (L_slow_path == nullptr) { L_slow_path = &L_fallthrough; label_nulls++; } 3877 assert(label_nulls <= 1, "at most one null in the batch"); 3878 3879 int sc_offset = in_bytes(Klass::secondary_super_cache_offset()); 3880 int sco_offset = in_bytes(Klass::super_check_offset_offset()); 3881 Address super_check_offset_addr(super_klass, sco_offset); 3882 3883 // Hacked jcc, which "knows" that L_fallthrough, at least, is in 3884 // range of a jccb. If this routine grows larger, reconsider at 3885 // least some of these. 3886 #define local_jcc(assembler_cond, label) \ 3887 if (&(label) == &L_fallthrough) jccb(assembler_cond, label); \ 3888 else jcc( assembler_cond, label) /*omit semi*/ 3889 3890 // Hacked jmp, which may only be used just before L_fallthrough. 3891 #define final_jmp(label) \ 3892 if (&(label) == &L_fallthrough) { /*do nothing*/ } \ 3893 else jmp(label) /*omit semi*/ 3894 3895 // If the pointers are equal, we are done (e.g., String[] elements). 3896 // This self-check enables sharing of secondary supertype arrays among 3897 // non-primary types such as array-of-interface. Otherwise, each such 3898 // type would need its own customized SSA. 3899 // We move this check to the front of the fast path because many 3900 // type checks are in fact trivially successful in this manner, 3901 // so we get a nicely predicted branch right at the start of the check. 3902 cmpptr(sub_klass, super_klass); 3903 local_jcc(Assembler::equal, *L_success); 3904 3905 // Check the supertype display: 3906 if (must_load_sco) { 3907 // Positive movl does right thing on LP64. 3908 movl(temp_reg, super_check_offset_addr); 3909 super_check_offset = RegisterOrConstant(temp_reg); 3910 } 3911 Address super_check_addr(sub_klass, super_check_offset, Address::times_1, 0); 3912 cmpptr(super_klass, super_check_addr); // load displayed supertype 3913 3914 // This check has worked decisively for primary supers. 3915 // Secondary supers are sought in the super_cache ('super_cache_addr'). 3916 // (Secondary supers are interfaces and very deeply nested subtypes.) 3917 // This works in the same check above because of a tricky aliasing 3918 // between the super_cache and the primary super display elements. 3919 // (The 'super_check_addr' can address either, as the case requires.) 3920 // Note that the cache is updated below if it does not help us find 3921 // what we need immediately. 3922 // So if it was a primary super, we can just fail immediately. 3923 // Otherwise, it's the slow path for us (no success at this point). 3924 3925 if (super_check_offset.is_register()) { 3926 local_jcc(Assembler::equal, *L_success); 3927 cmpl(super_check_offset.as_register(), sc_offset); 3928 if (L_failure == &L_fallthrough) { 3929 local_jcc(Assembler::equal, *L_slow_path); 3930 } else { 3931 local_jcc(Assembler::notEqual, *L_failure); 3932 final_jmp(*L_slow_path); 3933 } 3934 } else if (super_check_offset.as_constant() == sc_offset) { 3935 // Need a slow path; fast failure is impossible. 3936 if (L_slow_path == &L_fallthrough) { 3937 local_jcc(Assembler::equal, *L_success); 3938 } else { 3939 local_jcc(Assembler::notEqual, *L_slow_path); 3940 final_jmp(*L_success); 3941 } 3942 } else { 3943 // No slow path; it's a fast decision. 3944 if (L_failure == &L_fallthrough) { 3945 local_jcc(Assembler::equal, *L_success); 3946 } else { 3947 local_jcc(Assembler::notEqual, *L_failure); 3948 final_jmp(*L_success); 3949 } 3950 } 3951 3952 bind(L_fallthrough); 3953 3954 #undef local_jcc 3955 #undef final_jmp 3956 } 3957 3958 3959 void MacroAssembler::check_klass_subtype_slow_path_linear(Register sub_klass, 3960 Register super_klass, 3961 Register temp_reg, 3962 Register temp2_reg, 3963 Label* L_success, 3964 Label* L_failure, 3965 bool set_cond_codes) { 3966 assert_different_registers(sub_klass, super_klass, temp_reg); 3967 if (temp2_reg != noreg) 3968 assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg); 3969 #define IS_A_TEMP(reg) ((reg) == temp_reg || (reg) == temp2_reg) 3970 3971 Label L_fallthrough; 3972 int label_nulls = 0; 3973 if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; } 3974 if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; } 3975 assert(label_nulls <= 1, "at most one null in the batch"); 3976 3977 // a couple of useful fields in sub_klass: 3978 int ss_offset = in_bytes(Klass::secondary_supers_offset()); 3979 int sc_offset = in_bytes(Klass::secondary_super_cache_offset()); 3980 Address secondary_supers_addr(sub_klass, ss_offset); 3981 Address super_cache_addr( sub_klass, sc_offset); 3982 3983 // Do a linear scan of the secondary super-klass chain. 3984 // This code is rarely used, so simplicity is a virtue here. 3985 // The repne_scan instruction uses fixed registers, which we must spill. 3986 // Don't worry too much about pre-existing connections with the input regs. 3987 3988 assert(sub_klass != rax, "killed reg"); // killed by mov(rax, super) 3989 assert(sub_klass != rcx, "killed reg"); // killed by lea(rcx, &pst_counter) 3990 3991 // Get super_klass value into rax (even if it was in rdi or rcx). 3992 bool pushed_rax = false, pushed_rcx = false, pushed_rdi = false; 3993 if (super_klass != rax) { 3994 if (!IS_A_TEMP(rax)) { push(rax); pushed_rax = true; } 3995 mov(rax, super_klass); 3996 } 3997 if (!IS_A_TEMP(rcx)) { push(rcx); pushed_rcx = true; } 3998 if (!IS_A_TEMP(rdi)) { push(rdi); pushed_rdi = true; } 3999 4000 #ifndef PRODUCT 4001 uint* pst_counter = &SharedRuntime::_partial_subtype_ctr; 4002 ExternalAddress pst_counter_addr((address) pst_counter); 4003 lea(rcx, pst_counter_addr); 4004 incrementl(Address(rcx, 0)); 4005 #endif //PRODUCT 4006 4007 // We will consult the secondary-super array. 4008 movptr(rdi, secondary_supers_addr); 4009 // Load the array length. (Positive movl does right thing on LP64.) 4010 movl(rcx, Address(rdi, Array<Klass*>::length_offset_in_bytes())); 4011 // Skip to start of data. 4012 addptr(rdi, Array<Klass*>::base_offset_in_bytes()); 4013 4014 // Scan RCX words at [RDI] for an occurrence of RAX. 4015 // Set NZ/Z based on last compare. 4016 // Z flag value will not be set by 'repne' if RCX == 0 since 'repne' does 4017 // not change flags (only scas instruction which is repeated sets flags). 4018 // Set Z = 0 (not equal) before 'repne' to indicate that class was not found. 4019 4020 testptr(rax,rax); // Set Z = 0 4021 repne_scan(); 4022 4023 // Unspill the temp. registers: 4024 if (pushed_rdi) pop(rdi); 4025 if (pushed_rcx) pop(rcx); 4026 if (pushed_rax) pop(rax); 4027 4028 if (set_cond_codes) { 4029 // Special hack for the AD files: rdi is guaranteed non-zero. 4030 assert(!pushed_rdi, "rdi must be left non-null"); 4031 // Also, the condition codes are properly set Z/NZ on succeed/failure. 4032 } 4033 4034 if (L_failure == &L_fallthrough) 4035 jccb(Assembler::notEqual, *L_failure); 4036 else jcc(Assembler::notEqual, *L_failure); 4037 4038 // Success. Cache the super we found and proceed in triumph. 4039 movptr(super_cache_addr, super_klass); 4040 4041 if (L_success != &L_fallthrough) { 4042 jmp(*L_success); 4043 } 4044 4045 #undef IS_A_TEMP 4046 4047 bind(L_fallthrough); 4048 } 4049 4050 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass, 4051 Register super_klass, 4052 Register temp_reg, 4053 Register temp2_reg, 4054 Label* L_success, 4055 Label* L_failure, 4056 bool set_cond_codes) { 4057 assert(set_cond_codes == false, "must be false on 64-bit x86"); 4058 check_klass_subtype_slow_path 4059 (sub_klass, super_klass, temp_reg, temp2_reg, noreg, noreg, 4060 L_success, L_failure); 4061 } 4062 4063 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass, 4064 Register super_klass, 4065 Register temp_reg, 4066 Register temp2_reg, 4067 Register temp3_reg, 4068 Register temp4_reg, 4069 Label* L_success, 4070 Label* L_failure) { 4071 if (UseSecondarySupersTable) { 4072 check_klass_subtype_slow_path_table 4073 (sub_klass, super_klass, temp_reg, temp2_reg, temp3_reg, temp4_reg, 4074 L_success, L_failure); 4075 } else { 4076 check_klass_subtype_slow_path_linear 4077 (sub_klass, super_klass, temp_reg, temp2_reg, L_success, L_failure, /*set_cond_codes*/false); 4078 } 4079 } 4080 4081 Register MacroAssembler::allocate_if_noreg(Register r, 4082 RegSetIterator<Register> &available_regs, 4083 RegSet ®s_to_push) { 4084 if (!r->is_valid()) { 4085 r = *available_regs++; 4086 regs_to_push += r; 4087 } 4088 return r; 4089 } 4090 4091 void MacroAssembler::check_klass_subtype_slow_path_table(Register sub_klass, 4092 Register super_klass, 4093 Register temp_reg, 4094 Register temp2_reg, 4095 Register temp3_reg, 4096 Register result_reg, 4097 Label* L_success, 4098 Label* L_failure) { 4099 // NB! Callers may assume that, when temp2_reg is a valid register, 4100 // this code sets it to a nonzero value. 4101 bool temp2_reg_was_valid = temp2_reg->is_valid(); 4102 4103 RegSet temps = RegSet::of(temp_reg, temp2_reg, temp3_reg); 4104 4105 Label L_fallthrough; 4106 int label_nulls = 0; 4107 if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; } 4108 if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; } 4109 assert(label_nulls <= 1, "at most one null in the batch"); 4110 4111 BLOCK_COMMENT("check_klass_subtype_slow_path_table"); 4112 4113 RegSetIterator<Register> available_regs 4114 = (RegSet::of(rax, rcx, rdx, r8) + r9 + r10 + r11 + r12 - temps - sub_klass - super_klass).begin(); 4115 4116 RegSet pushed_regs; 4117 4118 temp_reg = allocate_if_noreg(temp_reg, available_regs, pushed_regs); 4119 temp2_reg = allocate_if_noreg(temp2_reg, available_regs, pushed_regs); 4120 temp3_reg = allocate_if_noreg(temp3_reg, available_regs, pushed_regs); 4121 result_reg = allocate_if_noreg(result_reg, available_regs, pushed_regs); 4122 Register temp4_reg = allocate_if_noreg(noreg, available_regs, pushed_regs); 4123 4124 assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg, temp3_reg, result_reg); 4125 4126 { 4127 4128 int register_push_size = pushed_regs.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size; 4129 int aligned_size = align_up(register_push_size, StackAlignmentInBytes); 4130 subptr(rsp, aligned_size); 4131 push_set(pushed_regs, 0); 4132 4133 lookup_secondary_supers_table_var(sub_klass, 4134 super_klass, 4135 temp_reg, temp2_reg, temp3_reg, temp4_reg, result_reg); 4136 cmpq(result_reg, 0); 4137 4138 // Unspill the temp. registers: 4139 pop_set(pushed_regs, 0); 4140 // Increment SP but do not clobber flags. 4141 lea(rsp, Address(rsp, aligned_size)); 4142 } 4143 4144 if (temp2_reg_was_valid) { 4145 movq(temp2_reg, 1); 4146 } 4147 4148 jcc(Assembler::notEqual, *L_failure); 4149 4150 if (L_success != &L_fallthrough) { 4151 jmp(*L_success); 4152 } 4153 4154 bind(L_fallthrough); 4155 } 4156 4157 // population_count variant for running without the POPCNT 4158 // instruction, which was introduced with SSE4.2 in 2008. 4159 void MacroAssembler::population_count(Register dst, Register src, 4160 Register scratch1, Register scratch2) { 4161 assert_different_registers(src, scratch1, scratch2); 4162 if (UsePopCountInstruction) { 4163 Assembler::popcntq(dst, src); 4164 } else { 4165 assert_different_registers(src, scratch1, scratch2); 4166 assert_different_registers(dst, scratch1, scratch2); 4167 Label loop, done; 4168 4169 mov(scratch1, src); 4170 // dst = 0; 4171 // while(scratch1 != 0) { 4172 // dst++; 4173 // scratch1 &= (scratch1 - 1); 4174 // } 4175 xorl(dst, dst); 4176 testq(scratch1, scratch1); 4177 jccb(Assembler::equal, done); 4178 { 4179 bind(loop); 4180 incq(dst); 4181 movq(scratch2, scratch1); 4182 decq(scratch2); 4183 andq(scratch1, scratch2); 4184 jccb(Assembler::notEqual, loop); 4185 } 4186 bind(done); 4187 } 4188 #ifdef ASSERT 4189 mov64(scratch1, 0xCafeBabeDeadBeef); 4190 movq(scratch2, scratch1); 4191 #endif 4192 } 4193 4194 // Ensure that the inline code and the stub are using the same registers. 4195 #define LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS \ 4196 do { \ 4197 assert(r_super_klass == rax, "mismatch"); \ 4198 assert(r_array_base == rbx, "mismatch"); \ 4199 assert(r_array_length == rcx, "mismatch"); \ 4200 assert(r_array_index == rdx, "mismatch"); \ 4201 assert(r_sub_klass == rsi || r_sub_klass == noreg, "mismatch"); \ 4202 assert(r_bitmap == r11 || r_bitmap == noreg, "mismatch"); \ 4203 assert(result == rdi || result == noreg, "mismatch"); \ 4204 } while(0) 4205 4206 // Versions of salq and rorq that don't need count to be in rcx 4207 4208 void MacroAssembler::salq(Register dest, Register count) { 4209 if (count == rcx) { 4210 Assembler::salq(dest); 4211 } else { 4212 assert_different_registers(rcx, dest); 4213 xchgq(rcx, count); 4214 Assembler::salq(dest); 4215 xchgq(rcx, count); 4216 } 4217 } 4218 4219 void MacroAssembler::rorq(Register dest, Register count) { 4220 if (count == rcx) { 4221 Assembler::rorq(dest); 4222 } else { 4223 assert_different_registers(rcx, dest); 4224 xchgq(rcx, count); 4225 Assembler::rorq(dest); 4226 xchgq(rcx, count); 4227 } 4228 } 4229 4230 // Return true: we succeeded in generating this code 4231 // 4232 // At runtime, return 0 in result if r_super_klass is a superclass of 4233 // r_sub_klass, otherwise return nonzero. Use this if you know the 4234 // super_klass_slot of the class you're looking for. This is always 4235 // the case for instanceof and checkcast. 4236 void MacroAssembler::lookup_secondary_supers_table_const(Register r_sub_klass, 4237 Register r_super_klass, 4238 Register temp1, 4239 Register temp2, 4240 Register temp3, 4241 Register temp4, 4242 Register result, 4243 u1 super_klass_slot) { 4244 assert_different_registers(r_sub_klass, r_super_klass, temp1, temp2, temp3, temp4, result); 4245 4246 Label L_fallthrough, L_success, L_failure; 4247 4248 BLOCK_COMMENT("lookup_secondary_supers_table {"); 4249 4250 const Register 4251 r_array_index = temp1, 4252 r_array_length = temp2, 4253 r_array_base = temp3, 4254 r_bitmap = temp4; 4255 4256 LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS; 4257 4258 xorq(result, result); // = 0 4259 4260 movq(r_bitmap, Address(r_sub_klass, Klass::secondary_supers_bitmap_offset())); 4261 movq(r_array_index, r_bitmap); 4262 4263 // First check the bitmap to see if super_klass might be present. If 4264 // the bit is zero, we are certain that super_klass is not one of 4265 // the secondary supers. 4266 u1 bit = super_klass_slot; 4267 { 4268 // NB: If the count in a x86 shift instruction is 0, the flags are 4269 // not affected, so we do a testq instead. 4270 int shift_count = Klass::SECONDARY_SUPERS_TABLE_MASK - bit; 4271 if (shift_count != 0) { 4272 salq(r_array_index, shift_count); 4273 } else { 4274 testq(r_array_index, r_array_index); 4275 } 4276 } 4277 // We test the MSB of r_array_index, i.e. its sign bit 4278 jcc(Assembler::positive, L_failure); 4279 4280 // Get the first array index that can contain super_klass into r_array_index. 4281 if (bit != 0) { 4282 population_count(r_array_index, r_array_index, temp2, temp3); 4283 } else { 4284 movl(r_array_index, 1); 4285 } 4286 // NB! r_array_index is off by 1. It is compensated by keeping r_array_base off by 1 word. 4287 4288 // We will consult the secondary-super array. 4289 movptr(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset()))); 4290 4291 // We're asserting that the first word in an Array<Klass*> is the 4292 // length, and the second word is the first word of the data. If 4293 // that ever changes, r_array_base will have to be adjusted here. 4294 assert(Array<Klass*>::base_offset_in_bytes() == wordSize, "Adjust this code"); 4295 assert(Array<Klass*>::length_offset_in_bytes() == 0, "Adjust this code"); 4296 4297 cmpq(r_super_klass, Address(r_array_base, r_array_index, Address::times_8)); 4298 jccb(Assembler::equal, L_success); 4299 4300 // Is there another entry to check? Consult the bitmap. 4301 btq(r_bitmap, (bit + 1) & Klass::SECONDARY_SUPERS_TABLE_MASK); 4302 jccb(Assembler::carryClear, L_failure); 4303 4304 // Linear probe. Rotate the bitmap so that the next bit to test is 4305 // in Bit 1. 4306 if (bit != 0) { 4307 rorq(r_bitmap, bit); 4308 } 4309 4310 // Calls into the stub generated by lookup_secondary_supers_table_slow_path. 4311 // Arguments: r_super_klass, r_array_base, r_array_index, r_bitmap. 4312 // Kills: r_array_length. 4313 // Returns: result. 4314 call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_slow_path_stub())); 4315 // Result (0/1) is in rdi 4316 jmpb(L_fallthrough); 4317 4318 bind(L_failure); 4319 incq(result); // 0 => 1 4320 4321 bind(L_success); 4322 // result = 0; 4323 4324 bind(L_fallthrough); 4325 BLOCK_COMMENT("} lookup_secondary_supers_table"); 4326 4327 if (VerifySecondarySupers) { 4328 verify_secondary_supers_table(r_sub_klass, r_super_klass, result, 4329 temp1, temp2, temp3); 4330 } 4331 } 4332 4333 // At runtime, return 0 in result if r_super_klass is a superclass of 4334 // r_sub_klass, otherwise return nonzero. Use this version of 4335 // lookup_secondary_supers_table() if you don't know ahead of time 4336 // which superclass will be searched for. Used by interpreter and 4337 // runtime stubs. It is larger and has somewhat greater latency than 4338 // the version above, which takes a constant super_klass_slot. 4339 void MacroAssembler::lookup_secondary_supers_table_var(Register r_sub_klass, 4340 Register r_super_klass, 4341 Register temp1, 4342 Register temp2, 4343 Register temp3, 4344 Register temp4, 4345 Register result) { 4346 assert_different_registers(r_sub_klass, r_super_klass, temp1, temp2, temp3, temp4, result); 4347 assert_different_registers(r_sub_klass, r_super_klass, rcx); 4348 RegSet temps = RegSet::of(temp1, temp2, temp3, temp4); 4349 4350 Label L_fallthrough, L_success, L_failure; 4351 4352 BLOCK_COMMENT("lookup_secondary_supers_table {"); 4353 4354 RegSetIterator<Register> available_regs = (temps - rcx).begin(); 4355 4356 // FIXME. Once we are sure that all paths reaching this point really 4357 // do pass rcx as one of our temps we can get rid of the following 4358 // workaround. 4359 assert(temps.contains(rcx), "fix this code"); 4360 4361 // We prefer to have our shift count in rcx. If rcx is one of our 4362 // temps, use it for slot. If not, pick any of our temps. 4363 Register slot; 4364 if (!temps.contains(rcx)) { 4365 slot = *available_regs++; 4366 } else { 4367 slot = rcx; 4368 } 4369 4370 const Register r_array_index = *available_regs++; 4371 const Register r_bitmap = *available_regs++; 4372 4373 // The logic above guarantees this property, but we state it here. 4374 assert_different_registers(r_array_index, r_bitmap, rcx); 4375 4376 movq(r_bitmap, Address(r_sub_klass, Klass::secondary_supers_bitmap_offset())); 4377 movq(r_array_index, r_bitmap); 4378 4379 // First check the bitmap to see if super_klass might be present. If 4380 // the bit is zero, we are certain that super_klass is not one of 4381 // the secondary supers. 4382 movb(slot, Address(r_super_klass, Klass::hash_slot_offset())); 4383 xorl(slot, (u1)(Klass::SECONDARY_SUPERS_TABLE_SIZE - 1)); // slot ^ 63 === 63 - slot (mod 64) 4384 salq(r_array_index, slot); 4385 4386 testq(r_array_index, r_array_index); 4387 // We test the MSB of r_array_index, i.e. its sign bit 4388 jcc(Assembler::positive, L_failure); 4389 4390 const Register r_array_base = *available_regs++; 4391 4392 // Get the first array index that can contain super_klass into r_array_index. 4393 // Note: Clobbers r_array_base and slot. 4394 population_count(r_array_index, r_array_index, /*temp2*/r_array_base, /*temp3*/slot); 4395 4396 // NB! r_array_index is off by 1. It is compensated by keeping r_array_base off by 1 word. 4397 4398 // We will consult the secondary-super array. 4399 movptr(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset()))); 4400 4401 // We're asserting that the first word in an Array<Klass*> is the 4402 // length, and the second word is the first word of the data. If 4403 // that ever changes, r_array_base will have to be adjusted here. 4404 assert(Array<Klass*>::base_offset_in_bytes() == wordSize, "Adjust this code"); 4405 assert(Array<Klass*>::length_offset_in_bytes() == 0, "Adjust this code"); 4406 4407 cmpq(r_super_klass, Address(r_array_base, r_array_index, Address::times_8)); 4408 jccb(Assembler::equal, L_success); 4409 4410 // Restore slot to its true value 4411 movb(slot, Address(r_super_klass, Klass::hash_slot_offset())); 4412 4413 // Linear probe. Rotate the bitmap so that the next bit to test is 4414 // in Bit 1. 4415 rorq(r_bitmap, slot); 4416 4417 // Is there another entry to check? Consult the bitmap. 4418 btq(r_bitmap, 1); 4419 jccb(Assembler::carryClear, L_failure); 4420 4421 // Calls into the stub generated by lookup_secondary_supers_table_slow_path. 4422 // Arguments: r_super_klass, r_array_base, r_array_index, r_bitmap. 4423 // Kills: r_array_length. 4424 // Returns: result. 4425 lookup_secondary_supers_table_slow_path(r_super_klass, 4426 r_array_base, 4427 r_array_index, 4428 r_bitmap, 4429 /*temp1*/result, 4430 /*temp2*/slot, 4431 &L_success, 4432 nullptr); 4433 4434 bind(L_failure); 4435 movq(result, 1); 4436 jmpb(L_fallthrough); 4437 4438 bind(L_success); 4439 xorq(result, result); // = 0 4440 4441 bind(L_fallthrough); 4442 BLOCK_COMMENT("} lookup_secondary_supers_table"); 4443 4444 if (VerifySecondarySupers) { 4445 verify_secondary_supers_table(r_sub_klass, r_super_klass, result, 4446 temp1, temp2, temp3); 4447 } 4448 } 4449 4450 void MacroAssembler::repne_scanq(Register addr, Register value, Register count, Register limit, 4451 Label* L_success, Label* L_failure) { 4452 Label L_loop, L_fallthrough; 4453 { 4454 int label_nulls = 0; 4455 if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; } 4456 if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; } 4457 assert(label_nulls <= 1, "at most one null in the batch"); 4458 } 4459 bind(L_loop); 4460 cmpq(value, Address(addr, count, Address::times_8)); 4461 jcc(Assembler::equal, *L_success); 4462 addl(count, 1); 4463 cmpl(count, limit); 4464 jcc(Assembler::less, L_loop); 4465 4466 if (&L_fallthrough != L_failure) { 4467 jmp(*L_failure); 4468 } 4469 bind(L_fallthrough); 4470 } 4471 4472 // Called by code generated by check_klass_subtype_slow_path 4473 // above. This is called when there is a collision in the hashed 4474 // lookup in the secondary supers array. 4475 void MacroAssembler::lookup_secondary_supers_table_slow_path(Register r_super_klass, 4476 Register r_array_base, 4477 Register r_array_index, 4478 Register r_bitmap, 4479 Register temp1, 4480 Register temp2, 4481 Label* L_success, 4482 Label* L_failure) { 4483 assert_different_registers(r_super_klass, r_array_base, r_array_index, r_bitmap, temp1, temp2); 4484 4485 const Register 4486 r_array_length = temp1, 4487 r_sub_klass = noreg, 4488 result = noreg; 4489 4490 Label L_fallthrough; 4491 int label_nulls = 0; 4492 if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; } 4493 if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; } 4494 assert(label_nulls <= 1, "at most one null in the batch"); 4495 4496 // Load the array length. 4497 movl(r_array_length, Address(r_array_base, Array<Klass*>::length_offset_in_bytes())); 4498 // And adjust the array base to point to the data. 4499 // NB! Effectively increments current slot index by 1. 4500 assert(Array<Klass*>::base_offset_in_bytes() == wordSize, ""); 4501 addptr(r_array_base, Array<Klass*>::base_offset_in_bytes()); 4502 4503 // Linear probe 4504 Label L_huge; 4505 4506 // The bitmap is full to bursting. 4507 // Implicit invariant: BITMAP_FULL implies (length > 0) 4508 cmpl(r_array_length, (int32_t)Klass::SECONDARY_SUPERS_TABLE_SIZE - 2); 4509 jcc(Assembler::greater, L_huge); 4510 4511 // NB! Our caller has checked bits 0 and 1 in the bitmap. The 4512 // current slot (at secondary_supers[r_array_index]) has not yet 4513 // been inspected, and r_array_index may be out of bounds if we 4514 // wrapped around the end of the array. 4515 4516 { // This is conventional linear probing, but instead of terminating 4517 // when a null entry is found in the table, we maintain a bitmap 4518 // in which a 0 indicates missing entries. 4519 // The check above guarantees there are 0s in the bitmap, so the loop 4520 // eventually terminates. 4521 4522 xorl(temp2, temp2); // = 0; 4523 4524 Label L_again; 4525 bind(L_again); 4526 4527 // Check for array wraparound. 4528 cmpl(r_array_index, r_array_length); 4529 cmovl(Assembler::greaterEqual, r_array_index, temp2); 4530 4531 cmpq(r_super_klass, Address(r_array_base, r_array_index, Address::times_8)); 4532 jcc(Assembler::equal, *L_success); 4533 4534 // If the next bit in bitmap is zero, we're done. 4535 btq(r_bitmap, 2); // look-ahead check (Bit 2); Bits 0 and 1 are tested by now 4536 jcc(Assembler::carryClear, *L_failure); 4537 4538 rorq(r_bitmap, 1); // Bits 1/2 => 0/1 4539 addl(r_array_index, 1); 4540 4541 jmp(L_again); 4542 } 4543 4544 { // Degenerate case: more than 64 secondary supers. 4545 // FIXME: We could do something smarter here, maybe a vectorized 4546 // comparison or a binary search, but is that worth any added 4547 // complexity? 4548 bind(L_huge); 4549 xorl(r_array_index, r_array_index); // = 0 4550 repne_scanq(r_array_base, r_super_klass, r_array_index, r_array_length, 4551 L_success, 4552 (&L_fallthrough != L_failure ? L_failure : nullptr)); 4553 4554 bind(L_fallthrough); 4555 } 4556 } 4557 4558 struct VerifyHelperArguments { 4559 Klass* _super; 4560 Klass* _sub; 4561 intptr_t _linear_result; 4562 intptr_t _table_result; 4563 }; 4564 4565 static void verify_secondary_supers_table_helper(const char* msg, VerifyHelperArguments* args) { 4566 Klass::on_secondary_supers_verification_failure(args->_super, 4567 args->_sub, 4568 args->_linear_result, 4569 args->_table_result, 4570 msg); 4571 } 4572 4573 // Make sure that the hashed lookup and a linear scan agree. 4574 void MacroAssembler::verify_secondary_supers_table(Register r_sub_klass, 4575 Register r_super_klass, 4576 Register result, 4577 Register temp1, 4578 Register temp2, 4579 Register temp3) { 4580 const Register 4581 r_array_index = temp1, 4582 r_array_length = temp2, 4583 r_array_base = temp3, 4584 r_bitmap = noreg; 4585 4586 BLOCK_COMMENT("verify_secondary_supers_table {"); 4587 4588 Label L_success, L_failure, L_check, L_done; 4589 4590 movptr(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset()))); 4591 movl(r_array_length, Address(r_array_base, Array<Klass*>::length_offset_in_bytes())); 4592 // And adjust the array base to point to the data. 4593 addptr(r_array_base, Array<Klass*>::base_offset_in_bytes()); 4594 4595 testl(r_array_length, r_array_length); // array_length == 0? 4596 jcc(Assembler::zero, L_failure); 4597 4598 movl(r_array_index, 0); 4599 repne_scanq(r_array_base, r_super_klass, r_array_index, r_array_length, &L_success); 4600 // fall through to L_failure 4601 4602 const Register linear_result = r_array_index; // reuse temp1 4603 4604 bind(L_failure); // not present 4605 movl(linear_result, 1); 4606 jmp(L_check); 4607 4608 bind(L_success); // present 4609 movl(linear_result, 0); 4610 4611 bind(L_check); 4612 cmpl(linear_result, result); 4613 jcc(Assembler::equal, L_done); 4614 4615 { // To avoid calling convention issues, build a record on the stack 4616 // and pass the pointer to that instead. 4617 push(result); 4618 push(linear_result); 4619 push(r_sub_klass); 4620 push(r_super_klass); 4621 movptr(c_rarg1, rsp); 4622 movptr(c_rarg0, (uintptr_t) "mismatch"); 4623 call(RuntimeAddress(CAST_FROM_FN_PTR(address, verify_secondary_supers_table_helper))); 4624 should_not_reach_here(); 4625 } 4626 bind(L_done); 4627 4628 BLOCK_COMMENT("} verify_secondary_supers_table"); 4629 } 4630 4631 #undef LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS 4632 4633 void MacroAssembler::clinit_barrier(Register klass, Label* L_fast_path, Label* L_slow_path) { 4634 assert(L_fast_path != nullptr || L_slow_path != nullptr, "at least one is required"); 4635 4636 Label L_fallthrough; 4637 if (L_fast_path == nullptr) { 4638 L_fast_path = &L_fallthrough; 4639 } else if (L_slow_path == nullptr) { 4640 L_slow_path = &L_fallthrough; 4641 } 4642 4643 // Fast path check: class is fully initialized. 4644 // init_state needs acquire, but x86 is TSO, and so we are already good. 4645 cmpb(Address(klass, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized); 4646 jcc(Assembler::equal, *L_fast_path); 4647 4648 // Fast path check: current thread is initializer thread 4649 cmpptr(r15_thread, Address(klass, InstanceKlass::init_thread_offset())); 4650 if (L_slow_path == &L_fallthrough) { 4651 jcc(Assembler::equal, *L_fast_path); 4652 bind(*L_slow_path); 4653 } else if (L_fast_path == &L_fallthrough) { 4654 jcc(Assembler::notEqual, *L_slow_path); 4655 bind(*L_fast_path); 4656 } else { 4657 Unimplemented(); 4658 } 4659 } 4660 4661 void MacroAssembler::cmov32(Condition cc, Register dst, Address src) { 4662 if (VM_Version::supports_cmov()) { 4663 cmovl(cc, dst, src); 4664 } else { 4665 Label L; 4666 jccb(negate_condition(cc), L); 4667 movl(dst, src); 4668 bind(L); 4669 } 4670 } 4671 4672 void MacroAssembler::cmov32(Condition cc, Register dst, Register src) { 4673 if (VM_Version::supports_cmov()) { 4674 cmovl(cc, dst, src); 4675 } else { 4676 Label L; 4677 jccb(negate_condition(cc), L); 4678 movl(dst, src); 4679 bind(L); 4680 } 4681 } 4682 4683 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) { 4684 if (!VerifyOops) return; 4685 4686 BLOCK_COMMENT("verify_oop {"); 4687 push(rscratch1); 4688 push(rax); // save rax 4689 push(reg); // pass register argument 4690 4691 // Pass register number to verify_oop_subroutine 4692 const char* b = nullptr; 4693 { 4694 ResourceMark rm; 4695 stringStream ss; 4696 ss.print("verify_oop: %s: %s (%s:%d)", reg->name(), s, file, line); 4697 b = code_string(ss.as_string()); 4698 } 4699 AddressLiteral buffer((address) b, external_word_Relocation::spec_for_immediate()); 4700 pushptr(buffer.addr(), rscratch1); 4701 4702 // call indirectly to solve generation ordering problem 4703 movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address())); 4704 call(rax); 4705 // Caller pops the arguments (oop, message) and restores rax, r10 4706 BLOCK_COMMENT("} verify_oop"); 4707 } 4708 4709 void MacroAssembler::vallones(XMMRegister dst, int vector_len) { 4710 if (UseAVX > 2 && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl())) { 4711 // Only pcmpeq has dependency breaking treatment (i.e the execution can begin without 4712 // waiting for the previous result on dst), not vpcmpeqd, so just use vpternlog 4713 vpternlogd(dst, 0xFF, dst, dst, vector_len); 4714 } else if (VM_Version::supports_avx()) { 4715 vpcmpeqd(dst, dst, dst, vector_len); 4716 } else { 4717 pcmpeqd(dst, dst); 4718 } 4719 } 4720 4721 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot, 4722 int extra_slot_offset) { 4723 // cf. TemplateTable::prepare_invoke(), if (load_receiver). 4724 int stackElementSize = Interpreter::stackElementSize; 4725 int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0); 4726 #ifdef ASSERT 4727 int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1); 4728 assert(offset1 - offset == stackElementSize, "correct arithmetic"); 4729 #endif 4730 Register scale_reg = noreg; 4731 Address::ScaleFactor scale_factor = Address::no_scale; 4732 if (arg_slot.is_constant()) { 4733 offset += arg_slot.as_constant() * stackElementSize; 4734 } else { 4735 scale_reg = arg_slot.as_register(); 4736 scale_factor = Address::times(stackElementSize); 4737 } 4738 offset += wordSize; // return PC is on stack 4739 return Address(rsp, scale_reg, scale_factor, offset); 4740 } 4741 4742 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) { 4743 if (!VerifyOops) return; 4744 4745 push(rscratch1); 4746 push(rax); // save rax, 4747 // addr may contain rsp so we will have to adjust it based on the push 4748 // we just did (and on 64 bit we do two pushes) 4749 // NOTE: 64bit seemed to have had a bug in that it did movq(addr, rax); which 4750 // stores rax into addr which is backwards of what was intended. 4751 if (addr.uses(rsp)) { 4752 lea(rax, addr); 4753 pushptr(Address(rax, 2 * BytesPerWord)); 4754 } else { 4755 pushptr(addr); 4756 } 4757 4758 // Pass register number to verify_oop_subroutine 4759 const char* b = nullptr; 4760 { 4761 ResourceMark rm; 4762 stringStream ss; 4763 ss.print("verify_oop_addr: %s (%s:%d)", s, file, line); 4764 b = code_string(ss.as_string()); 4765 } 4766 AddressLiteral buffer((address) b, external_word_Relocation::spec_for_immediate()); 4767 pushptr(buffer.addr(), rscratch1); 4768 4769 // call indirectly to solve generation ordering problem 4770 movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address())); 4771 call(rax); 4772 // Caller pops the arguments (addr, message) and restores rax, r10. 4773 } 4774 4775 void MacroAssembler::verify_tlab() { 4776 #ifdef ASSERT 4777 if (UseTLAB && VerifyOops) { 4778 Label next, ok; 4779 Register t1 = rsi; 4780 4781 push(t1); 4782 4783 movptr(t1, Address(r15_thread, in_bytes(JavaThread::tlab_top_offset()))); 4784 cmpptr(t1, Address(r15_thread, in_bytes(JavaThread::tlab_start_offset()))); 4785 jcc(Assembler::aboveEqual, next); 4786 STOP("assert(top >= start)"); 4787 should_not_reach_here(); 4788 4789 bind(next); 4790 movptr(t1, Address(r15_thread, in_bytes(JavaThread::tlab_end_offset()))); 4791 cmpptr(t1, Address(r15_thread, in_bytes(JavaThread::tlab_top_offset()))); 4792 jcc(Assembler::aboveEqual, ok); 4793 STOP("assert(top <= end)"); 4794 should_not_reach_here(); 4795 4796 bind(ok); 4797 pop(t1); 4798 } 4799 #endif 4800 } 4801 4802 class ControlWord { 4803 public: 4804 int32_t _value; 4805 4806 int rounding_control() const { return (_value >> 10) & 3 ; } 4807 int precision_control() const { return (_value >> 8) & 3 ; } 4808 bool precision() const { return ((_value >> 5) & 1) != 0; } 4809 bool underflow() const { return ((_value >> 4) & 1) != 0; } 4810 bool overflow() const { return ((_value >> 3) & 1) != 0; } 4811 bool zero_divide() const { return ((_value >> 2) & 1) != 0; } 4812 bool denormalized() const { return ((_value >> 1) & 1) != 0; } 4813 bool invalid() const { return ((_value >> 0) & 1) != 0; } 4814 4815 void print() const { 4816 // rounding control 4817 const char* rc; 4818 switch (rounding_control()) { 4819 case 0: rc = "round near"; break; 4820 case 1: rc = "round down"; break; 4821 case 2: rc = "round up "; break; 4822 case 3: rc = "chop "; break; 4823 default: 4824 rc = nullptr; // silence compiler warnings 4825 fatal("Unknown rounding control: %d", rounding_control()); 4826 }; 4827 // precision control 4828 const char* pc; 4829 switch (precision_control()) { 4830 case 0: pc = "24 bits "; break; 4831 case 1: pc = "reserved"; break; 4832 case 2: pc = "53 bits "; break; 4833 case 3: pc = "64 bits "; break; 4834 default: 4835 pc = nullptr; // silence compiler warnings 4836 fatal("Unknown precision control: %d", precision_control()); 4837 }; 4838 // flags 4839 char f[9]; 4840 f[0] = ' '; 4841 f[1] = ' '; 4842 f[2] = (precision ()) ? 'P' : 'p'; 4843 f[3] = (underflow ()) ? 'U' : 'u'; 4844 f[4] = (overflow ()) ? 'O' : 'o'; 4845 f[5] = (zero_divide ()) ? 'Z' : 'z'; 4846 f[6] = (denormalized()) ? 'D' : 'd'; 4847 f[7] = (invalid ()) ? 'I' : 'i'; 4848 f[8] = '\x0'; 4849 // output 4850 printf("%04x masks = %s, %s, %s", _value & 0xFFFF, f, rc, pc); 4851 } 4852 4853 }; 4854 4855 class StatusWord { 4856 public: 4857 int32_t _value; 4858 4859 bool busy() const { return ((_value >> 15) & 1) != 0; } 4860 bool C3() const { return ((_value >> 14) & 1) != 0; } 4861 bool C2() const { return ((_value >> 10) & 1) != 0; } 4862 bool C1() const { return ((_value >> 9) & 1) != 0; } 4863 bool C0() const { return ((_value >> 8) & 1) != 0; } 4864 int top() const { return (_value >> 11) & 7 ; } 4865 bool error_status() const { return ((_value >> 7) & 1) != 0; } 4866 bool stack_fault() const { return ((_value >> 6) & 1) != 0; } 4867 bool precision() const { return ((_value >> 5) & 1) != 0; } 4868 bool underflow() const { return ((_value >> 4) & 1) != 0; } 4869 bool overflow() const { return ((_value >> 3) & 1) != 0; } 4870 bool zero_divide() const { return ((_value >> 2) & 1) != 0; } 4871 bool denormalized() const { return ((_value >> 1) & 1) != 0; } 4872 bool invalid() const { return ((_value >> 0) & 1) != 0; } 4873 4874 void print() const { 4875 // condition codes 4876 char c[5]; 4877 c[0] = (C3()) ? '3' : '-'; 4878 c[1] = (C2()) ? '2' : '-'; 4879 c[2] = (C1()) ? '1' : '-'; 4880 c[3] = (C0()) ? '0' : '-'; 4881 c[4] = '\x0'; 4882 // flags 4883 char f[9]; 4884 f[0] = (error_status()) ? 'E' : '-'; 4885 f[1] = (stack_fault ()) ? 'S' : '-'; 4886 f[2] = (precision ()) ? 'P' : '-'; 4887 f[3] = (underflow ()) ? 'U' : '-'; 4888 f[4] = (overflow ()) ? 'O' : '-'; 4889 f[5] = (zero_divide ()) ? 'Z' : '-'; 4890 f[6] = (denormalized()) ? 'D' : '-'; 4891 f[7] = (invalid ()) ? 'I' : '-'; 4892 f[8] = '\x0'; 4893 // output 4894 printf("%04x flags = %s, cc = %s, top = %d", _value & 0xFFFF, f, c, top()); 4895 } 4896 4897 }; 4898 4899 class TagWord { 4900 public: 4901 int32_t _value; 4902 4903 int tag_at(int i) const { return (_value >> (i*2)) & 3; } 4904 4905 void print() const { 4906 printf("%04x", _value & 0xFFFF); 4907 } 4908 4909 }; 4910 4911 class FPU_Register { 4912 public: 4913 int32_t _m0; 4914 int32_t _m1; 4915 int16_t _ex; 4916 4917 bool is_indefinite() const { 4918 return _ex == -1 && _m1 == (int32_t)0xC0000000 && _m0 == 0; 4919 } 4920 4921 void print() const { 4922 char sign = (_ex < 0) ? '-' : '+'; 4923 const char* kind = (_ex == 0x7FFF || _ex == (int16_t)-1) ? "NaN" : " "; 4924 printf("%c%04hx.%08x%08x %s", sign, _ex, _m1, _m0, kind); 4925 }; 4926 4927 }; 4928 4929 class FPU_State { 4930 public: 4931 enum { 4932 register_size = 10, 4933 number_of_registers = 8, 4934 register_mask = 7 4935 }; 4936 4937 ControlWord _control_word; 4938 StatusWord _status_word; 4939 TagWord _tag_word; 4940 int32_t _error_offset; 4941 int32_t _error_selector; 4942 int32_t _data_offset; 4943 int32_t _data_selector; 4944 int8_t _register[register_size * number_of_registers]; 4945 4946 int tag_for_st(int i) const { return _tag_word.tag_at((_status_word.top() + i) & register_mask); } 4947 FPU_Register* st(int i) const { return (FPU_Register*)&_register[register_size * i]; } 4948 4949 const char* tag_as_string(int tag) const { 4950 switch (tag) { 4951 case 0: return "valid"; 4952 case 1: return "zero"; 4953 case 2: return "special"; 4954 case 3: return "empty"; 4955 } 4956 ShouldNotReachHere(); 4957 return nullptr; 4958 } 4959 4960 void print() const { 4961 // print computation registers 4962 { int t = _status_word.top(); 4963 for (int i = 0; i < number_of_registers; i++) { 4964 int j = (i - t) & register_mask; 4965 printf("%c r%d = ST%d = ", (j == 0 ? '*' : ' '), i, j); 4966 st(j)->print(); 4967 printf(" %s\n", tag_as_string(_tag_word.tag_at(i))); 4968 } 4969 } 4970 printf("\n"); 4971 // print control registers 4972 printf("ctrl = "); _control_word.print(); printf("\n"); 4973 printf("stat = "); _status_word .print(); printf("\n"); 4974 printf("tags = "); _tag_word .print(); printf("\n"); 4975 } 4976 4977 }; 4978 4979 class Flag_Register { 4980 public: 4981 int32_t _value; 4982 4983 bool overflow() const { return ((_value >> 11) & 1) != 0; } 4984 bool direction() const { return ((_value >> 10) & 1) != 0; } 4985 bool sign() const { return ((_value >> 7) & 1) != 0; } 4986 bool zero() const { return ((_value >> 6) & 1) != 0; } 4987 bool auxiliary_carry() const { return ((_value >> 4) & 1) != 0; } 4988 bool parity() const { return ((_value >> 2) & 1) != 0; } 4989 bool carry() const { return ((_value >> 0) & 1) != 0; } 4990 4991 void print() const { 4992 // flags 4993 char f[8]; 4994 f[0] = (overflow ()) ? 'O' : '-'; 4995 f[1] = (direction ()) ? 'D' : '-'; 4996 f[2] = (sign ()) ? 'S' : '-'; 4997 f[3] = (zero ()) ? 'Z' : '-'; 4998 f[4] = (auxiliary_carry()) ? 'A' : '-'; 4999 f[5] = (parity ()) ? 'P' : '-'; 5000 f[6] = (carry ()) ? 'C' : '-'; 5001 f[7] = '\x0'; 5002 // output 5003 printf("%08x flags = %s", _value, f); 5004 } 5005 5006 }; 5007 5008 class IU_Register { 5009 public: 5010 int32_t _value; 5011 5012 void print() const { 5013 printf("%08x %11d", _value, _value); 5014 } 5015 5016 }; 5017 5018 class IU_State { 5019 public: 5020 Flag_Register _eflags; 5021 IU_Register _rdi; 5022 IU_Register _rsi; 5023 IU_Register _rbp; 5024 IU_Register _rsp; 5025 IU_Register _rbx; 5026 IU_Register _rdx; 5027 IU_Register _rcx; 5028 IU_Register _rax; 5029 5030 void print() const { 5031 // computation registers 5032 printf("rax, = "); _rax.print(); printf("\n"); 5033 printf("rbx, = "); _rbx.print(); printf("\n"); 5034 printf("rcx = "); _rcx.print(); printf("\n"); 5035 printf("rdx = "); _rdx.print(); printf("\n"); 5036 printf("rdi = "); _rdi.print(); printf("\n"); 5037 printf("rsi = "); _rsi.print(); printf("\n"); 5038 printf("rbp, = "); _rbp.print(); printf("\n"); 5039 printf("rsp = "); _rsp.print(); printf("\n"); 5040 printf("\n"); 5041 // control registers 5042 printf("flgs = "); _eflags.print(); printf("\n"); 5043 } 5044 }; 5045 5046 5047 class CPU_State { 5048 public: 5049 FPU_State _fpu_state; 5050 IU_State _iu_state; 5051 5052 void print() const { 5053 printf("--------------------------------------------------\n"); 5054 _iu_state .print(); 5055 printf("\n"); 5056 _fpu_state.print(); 5057 printf("--------------------------------------------------\n"); 5058 } 5059 5060 }; 5061 5062 5063 static void _print_CPU_state(CPU_State* state) { 5064 state->print(); 5065 }; 5066 5067 5068 void MacroAssembler::print_CPU_state() { 5069 push_CPU_state(); 5070 push(rsp); // pass CPU state 5071 call(RuntimeAddress(CAST_FROM_FN_PTR(address, _print_CPU_state))); 5072 addptr(rsp, wordSize); // discard argument 5073 pop_CPU_state(); 5074 } 5075 5076 void MacroAssembler::restore_cpu_control_state_after_jni(Register rscratch) { 5077 // Either restore the MXCSR register after returning from the JNI Call 5078 // or verify that it wasn't changed (with -Xcheck:jni flag). 5079 if (VM_Version::supports_sse()) { 5080 if (RestoreMXCSROnJNICalls) { 5081 ldmxcsr(ExternalAddress(StubRoutines::x86::addr_mxcsr_std()), rscratch); 5082 } else if (CheckJNICalls) { 5083 call(RuntimeAddress(StubRoutines::x86::verify_mxcsr_entry())); 5084 } 5085 } 5086 // Clear upper bits of YMM registers to avoid SSE <-> AVX transition penalty. 5087 vzeroupper(); 5088 } 5089 5090 // ((OopHandle)result).resolve(); 5091 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) { 5092 assert_different_registers(result, tmp); 5093 5094 // Only 64 bit platforms support GCs that require a tmp register 5095 // Only IN_HEAP loads require a thread_tmp register 5096 // OopHandle::resolve is an indirection like jobject. 5097 access_load_at(T_OBJECT, IN_NATIVE, 5098 result, Address(result, 0), tmp); 5099 } 5100 5101 // ((WeakHandle)result).resolve(); 5102 void MacroAssembler::resolve_weak_handle(Register rresult, Register rtmp) { 5103 assert_different_registers(rresult, rtmp); 5104 Label resolved; 5105 5106 // A null weak handle resolves to null. 5107 cmpptr(rresult, 0); 5108 jcc(Assembler::equal, resolved); 5109 5110 // Only 64 bit platforms support GCs that require a tmp register 5111 // Only IN_HEAP loads require a thread_tmp register 5112 // WeakHandle::resolve is an indirection like jweak. 5113 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, 5114 rresult, Address(rresult, 0), rtmp); 5115 bind(resolved); 5116 } 5117 5118 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) { 5119 // get mirror 5120 const int mirror_offset = in_bytes(Klass::java_mirror_offset()); 5121 load_method_holder(mirror, method); 5122 movptr(mirror, Address(mirror, mirror_offset)); 5123 resolve_oop_handle(mirror, tmp); 5124 } 5125 5126 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) { 5127 load_method_holder(rresult, rmethod); 5128 movptr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset())); 5129 } 5130 5131 void MacroAssembler::load_method_holder(Register holder, Register method) { 5132 movptr(holder, Address(method, Method::const_offset())); // ConstMethod* 5133 movptr(holder, Address(holder, ConstMethod::constants_offset())); // ConstantPool* 5134 movptr(holder, Address(holder, ConstantPool::pool_holder_offset())); // InstanceKlass* 5135 } 5136 5137 void MacroAssembler::load_narrow_klass_compact(Register dst, Register src) { 5138 assert(UseCompactObjectHeaders, "expect compact object headers"); 5139 movl(dst, Address(src, oopDesc::mark_offset_in_bytes())); 5140 shrl(dst, markWord::klass_shift); 5141 } 5142 5143 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) { 5144 assert_different_registers(src, tmp); 5145 assert_different_registers(dst, tmp); 5146 5147 if (UseCompactObjectHeaders) { 5148 load_narrow_klass_compact(dst, src); 5149 decode_klass_not_null(dst, tmp); 5150 } else if (UseCompressedClassPointers) { 5151 movl(dst, Address(src, oopDesc::klass_offset_in_bytes())); 5152 decode_klass_not_null(dst, tmp); 5153 } else { 5154 movptr(dst, Address(src, oopDesc::klass_offset_in_bytes())); 5155 } 5156 } 5157 5158 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) { 5159 assert(!UseCompactObjectHeaders, "not with compact headers"); 5160 assert_different_registers(src, tmp); 5161 assert_different_registers(dst, tmp); 5162 if (UseCompressedClassPointers) { 5163 encode_klass_not_null(src, tmp); 5164 movl(Address(dst, oopDesc::klass_offset_in_bytes()), src); 5165 } else { 5166 movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src); 5167 } 5168 } 5169 5170 void MacroAssembler::cmp_klass(Register klass, Register obj, Register tmp) { 5171 if (UseCompactObjectHeaders) { 5172 assert(tmp != noreg, "need tmp"); 5173 assert_different_registers(klass, obj, tmp); 5174 load_narrow_klass_compact(tmp, obj); 5175 cmpl(klass, tmp); 5176 } else if (UseCompressedClassPointers) { 5177 cmpl(klass, Address(obj, oopDesc::klass_offset_in_bytes())); 5178 } else { 5179 cmpptr(klass, Address(obj, oopDesc::klass_offset_in_bytes())); 5180 } 5181 } 5182 5183 void MacroAssembler::cmp_klasses_from_objects(Register obj1, Register obj2, Register tmp1, Register tmp2) { 5184 if (UseCompactObjectHeaders) { 5185 assert(tmp2 != noreg, "need tmp2"); 5186 assert_different_registers(obj1, obj2, tmp1, tmp2); 5187 load_narrow_klass_compact(tmp1, obj1); 5188 load_narrow_klass_compact(tmp2, obj2); 5189 cmpl(tmp1, tmp2); 5190 } else if (UseCompressedClassPointers) { 5191 movl(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes())); 5192 cmpl(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes())); 5193 } else { 5194 movptr(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes())); 5195 cmpptr(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes())); 5196 } 5197 } 5198 5199 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src, 5200 Register tmp1) { 5201 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); 5202 decorators = AccessInternal::decorator_fixup(decorators, type); 5203 bool as_raw = (decorators & AS_RAW) != 0; 5204 if (as_raw) { 5205 bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1); 5206 } else { 5207 bs->load_at(this, decorators, type, dst, src, tmp1); 5208 } 5209 } 5210 5211 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register val, 5212 Register tmp1, Register tmp2, Register tmp3) { 5213 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); 5214 decorators = AccessInternal::decorator_fixup(decorators, type); 5215 bool as_raw = (decorators & AS_RAW) != 0; 5216 if (as_raw) { 5217 bs->BarrierSetAssembler::store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3); 5218 } else { 5219 bs->store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3); 5220 } 5221 } 5222 5223 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1, DecoratorSet decorators) { 5224 access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1); 5225 } 5226 5227 // Doesn't do verification, generates fixed size code 5228 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1, DecoratorSet decorators) { 5229 access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1); 5230 } 5231 5232 void MacroAssembler::store_heap_oop(Address dst, Register val, Register tmp1, 5233 Register tmp2, Register tmp3, DecoratorSet decorators) { 5234 access_store_at(T_OBJECT, IN_HEAP | decorators, dst, val, tmp1, tmp2, tmp3); 5235 } 5236 5237 // Used for storing nulls. 5238 void MacroAssembler::store_heap_oop_null(Address dst) { 5239 access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg, noreg); 5240 } 5241 5242 void MacroAssembler::store_klass_gap(Register dst, Register src) { 5243 assert(!UseCompactObjectHeaders, "Don't use with compact headers"); 5244 if (UseCompressedClassPointers) { 5245 // Store to klass gap in destination 5246 movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src); 5247 } 5248 } 5249 5250 #ifdef ASSERT 5251 void MacroAssembler::verify_heapbase(const char* msg) { 5252 assert (UseCompressedOops, "should be compressed"); 5253 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5254 if (CheckCompressedOops) { 5255 Label ok; 5256 ExternalAddress src2(CompressedOops::base_addr()); 5257 const bool is_src2_reachable = reachable(src2); 5258 if (!is_src2_reachable) { 5259 push(rscratch1); // cmpptr trashes rscratch1 5260 } 5261 cmpptr(r12_heapbase, src2, rscratch1); 5262 jcc(Assembler::equal, ok); 5263 STOP(msg); 5264 bind(ok); 5265 if (!is_src2_reachable) { 5266 pop(rscratch1); 5267 } 5268 } 5269 } 5270 #endif 5271 5272 // Algorithm must match oop.inline.hpp encode_heap_oop. 5273 void MacroAssembler::encode_heap_oop(Register r) { 5274 #ifdef ASSERT 5275 verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?"); 5276 #endif 5277 verify_oop_msg(r, "broken oop in encode_heap_oop"); 5278 if (CompressedOops::base() == nullptr) { 5279 if (CompressedOops::shift() != 0) { 5280 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5281 shrq(r, LogMinObjAlignmentInBytes); 5282 } 5283 return; 5284 } 5285 testq(r, r); 5286 cmovq(Assembler::equal, r, r12_heapbase); 5287 subq(r, r12_heapbase); 5288 shrq(r, LogMinObjAlignmentInBytes); 5289 } 5290 5291 void MacroAssembler::encode_heap_oop_not_null(Register r) { 5292 #ifdef ASSERT 5293 verify_heapbase("MacroAssembler::encode_heap_oop_not_null: heap base corrupted?"); 5294 if (CheckCompressedOops) { 5295 Label ok; 5296 testq(r, r); 5297 jcc(Assembler::notEqual, ok); 5298 STOP("null oop passed to encode_heap_oop_not_null"); 5299 bind(ok); 5300 } 5301 #endif 5302 verify_oop_msg(r, "broken oop in encode_heap_oop_not_null"); 5303 if (CompressedOops::base() != nullptr) { 5304 subq(r, r12_heapbase); 5305 } 5306 if (CompressedOops::shift() != 0) { 5307 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5308 shrq(r, LogMinObjAlignmentInBytes); 5309 } 5310 } 5311 5312 void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) { 5313 #ifdef ASSERT 5314 verify_heapbase("MacroAssembler::encode_heap_oop_not_null2: heap base corrupted?"); 5315 if (CheckCompressedOops) { 5316 Label ok; 5317 testq(src, src); 5318 jcc(Assembler::notEqual, ok); 5319 STOP("null oop passed to encode_heap_oop_not_null2"); 5320 bind(ok); 5321 } 5322 #endif 5323 verify_oop_msg(src, "broken oop in encode_heap_oop_not_null2"); 5324 if (dst != src) { 5325 movq(dst, src); 5326 } 5327 if (CompressedOops::base() != nullptr) { 5328 subq(dst, r12_heapbase); 5329 } 5330 if (CompressedOops::shift() != 0) { 5331 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5332 shrq(dst, LogMinObjAlignmentInBytes); 5333 } 5334 } 5335 5336 void MacroAssembler::decode_heap_oop(Register r) { 5337 #ifdef ASSERT 5338 verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?"); 5339 #endif 5340 if (CompressedOops::base() == nullptr) { 5341 if (CompressedOops::shift() != 0) { 5342 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5343 shlq(r, LogMinObjAlignmentInBytes); 5344 } 5345 } else { 5346 Label done; 5347 shlq(r, LogMinObjAlignmentInBytes); 5348 jccb(Assembler::equal, done); 5349 addq(r, r12_heapbase); 5350 bind(done); 5351 } 5352 verify_oop_msg(r, "broken oop in decode_heap_oop"); 5353 } 5354 5355 void MacroAssembler::decode_heap_oop_not_null(Register r) { 5356 // Note: it will change flags 5357 assert (UseCompressedOops, "should only be used for compressed headers"); 5358 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5359 // Cannot assert, unverified entry point counts instructions (see .ad file) 5360 // vtableStubs also counts instructions in pd_code_size_limit. 5361 // Also do not verify_oop as this is called by verify_oop. 5362 if (CompressedOops::shift() != 0) { 5363 assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5364 shlq(r, LogMinObjAlignmentInBytes); 5365 if (CompressedOops::base() != nullptr) { 5366 addq(r, r12_heapbase); 5367 } 5368 } else { 5369 assert (CompressedOops::base() == nullptr, "sanity"); 5370 } 5371 } 5372 5373 void MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) { 5374 // Note: it will change flags 5375 assert (UseCompressedOops, "should only be used for compressed headers"); 5376 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5377 // Cannot assert, unverified entry point counts instructions (see .ad file) 5378 // vtableStubs also counts instructions in pd_code_size_limit. 5379 // Also do not verify_oop as this is called by verify_oop. 5380 if (CompressedOops::shift() != 0) { 5381 assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 5382 if (LogMinObjAlignmentInBytes == Address::times_8) { 5383 leaq(dst, Address(r12_heapbase, src, Address::times_8, 0)); 5384 } else { 5385 if (dst != src) { 5386 movq(dst, src); 5387 } 5388 shlq(dst, LogMinObjAlignmentInBytes); 5389 if (CompressedOops::base() != nullptr) { 5390 addq(dst, r12_heapbase); 5391 } 5392 } 5393 } else { 5394 assert (CompressedOops::base() == nullptr, "sanity"); 5395 if (dst != src) { 5396 movq(dst, src); 5397 } 5398 } 5399 } 5400 5401 void MacroAssembler::encode_klass_not_null(Register r, Register tmp) { 5402 assert_different_registers(r, tmp); 5403 if (CompressedKlassPointers::base() != nullptr) { 5404 mov64(tmp, (int64_t)CompressedKlassPointers::base()); 5405 subq(r, tmp); 5406 } 5407 if (CompressedKlassPointers::shift() != 0) { 5408 shrq(r, CompressedKlassPointers::shift()); 5409 } 5410 } 5411 5412 void MacroAssembler::encode_and_move_klass_not_null(Register dst, Register src) { 5413 assert_different_registers(src, dst); 5414 if (CompressedKlassPointers::base() != nullptr) { 5415 mov64(dst, -(int64_t)CompressedKlassPointers::base()); 5416 addq(dst, src); 5417 } else { 5418 movptr(dst, src); 5419 } 5420 if (CompressedKlassPointers::shift() != 0) { 5421 shrq(dst, CompressedKlassPointers::shift()); 5422 } 5423 } 5424 5425 void MacroAssembler::decode_klass_not_null(Register r, Register tmp) { 5426 assert_different_registers(r, tmp); 5427 // Note: it will change flags 5428 assert(UseCompressedClassPointers, "should only be used for compressed headers"); 5429 // Cannot assert, unverified entry point counts instructions (see .ad file) 5430 // vtableStubs also counts instructions in pd_code_size_limit. 5431 // Also do not verify_oop as this is called by verify_oop. 5432 if (CompressedKlassPointers::shift() != 0) { 5433 shlq(r, CompressedKlassPointers::shift()); 5434 } 5435 if (CompressedKlassPointers::base() != nullptr) { 5436 mov64(tmp, (int64_t)CompressedKlassPointers::base()); 5437 addq(r, tmp); 5438 } 5439 } 5440 5441 void MacroAssembler::decode_and_move_klass_not_null(Register dst, Register src) { 5442 assert_different_registers(src, dst); 5443 // Note: it will change flags 5444 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 5445 // Cannot assert, unverified entry point counts instructions (see .ad file) 5446 // vtableStubs also counts instructions in pd_code_size_limit. 5447 // Also do not verify_oop as this is called by verify_oop. 5448 5449 if (CompressedKlassPointers::base() == nullptr && 5450 CompressedKlassPointers::shift() == 0) { 5451 // The best case scenario is that there is no base or shift. Then it is already 5452 // a pointer that needs nothing but a register rename. 5453 movl(dst, src); 5454 } else { 5455 if (CompressedKlassPointers::shift() <= Address::times_8) { 5456 if (CompressedKlassPointers::base() != nullptr) { 5457 mov64(dst, (int64_t)CompressedKlassPointers::base()); 5458 } else { 5459 xorq(dst, dst); 5460 } 5461 if (CompressedKlassPointers::shift() != 0) { 5462 assert(CompressedKlassPointers::shift() == Address::times_8, "klass not aligned on 64bits?"); 5463 leaq(dst, Address(dst, src, Address::times_8, 0)); 5464 } else { 5465 addq(dst, src); 5466 } 5467 } else { 5468 if (CompressedKlassPointers::base() != nullptr) { 5469 const uint64_t base_right_shifted = 5470 (uint64_t)CompressedKlassPointers::base() >> CompressedKlassPointers::shift(); 5471 mov64(dst, base_right_shifted); 5472 } else { 5473 xorq(dst, dst); 5474 } 5475 addq(dst, src); 5476 shlq(dst, CompressedKlassPointers::shift()); 5477 } 5478 } 5479 } 5480 5481 void MacroAssembler::set_narrow_oop(Register dst, jobject obj) { 5482 assert (UseCompressedOops, "should only be used for compressed headers"); 5483 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5484 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5485 int oop_index = oop_recorder()->find_index(obj); 5486 RelocationHolder rspec = oop_Relocation::spec(oop_index); 5487 mov_narrow_oop(dst, oop_index, rspec); 5488 } 5489 5490 void MacroAssembler::set_narrow_oop(Address dst, jobject obj) { 5491 assert (UseCompressedOops, "should only be used for compressed headers"); 5492 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5493 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5494 int oop_index = oop_recorder()->find_index(obj); 5495 RelocationHolder rspec = oop_Relocation::spec(oop_index); 5496 mov_narrow_oop(dst, oop_index, rspec); 5497 } 5498 5499 void MacroAssembler::set_narrow_klass(Register dst, Klass* k) { 5500 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 5501 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5502 int klass_index = oop_recorder()->find_index(k); 5503 RelocationHolder rspec = metadata_Relocation::spec(klass_index); 5504 mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec); 5505 } 5506 5507 void MacroAssembler::set_narrow_klass(Address dst, Klass* k) { 5508 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 5509 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5510 int klass_index = oop_recorder()->find_index(k); 5511 RelocationHolder rspec = metadata_Relocation::spec(klass_index); 5512 mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec); 5513 } 5514 5515 void MacroAssembler::cmp_narrow_oop(Register dst, jobject obj) { 5516 assert (UseCompressedOops, "should only be used for compressed headers"); 5517 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5518 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5519 int oop_index = oop_recorder()->find_index(obj); 5520 RelocationHolder rspec = oop_Relocation::spec(oop_index); 5521 Assembler::cmp_narrow_oop(dst, oop_index, rspec); 5522 } 5523 5524 void MacroAssembler::cmp_narrow_oop(Address dst, jobject obj) { 5525 assert (UseCompressedOops, "should only be used for compressed headers"); 5526 assert (Universe::heap() != nullptr, "java heap should be initialized"); 5527 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5528 int oop_index = oop_recorder()->find_index(obj); 5529 RelocationHolder rspec = oop_Relocation::spec(oop_index); 5530 Assembler::cmp_narrow_oop(dst, oop_index, rspec); 5531 } 5532 5533 void MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) { 5534 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 5535 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5536 int klass_index = oop_recorder()->find_index(k); 5537 RelocationHolder rspec = metadata_Relocation::spec(klass_index); 5538 Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec); 5539 } 5540 5541 void MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) { 5542 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 5543 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 5544 int klass_index = oop_recorder()->find_index(k); 5545 RelocationHolder rspec = metadata_Relocation::spec(klass_index); 5546 Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec); 5547 } 5548 5549 void MacroAssembler::reinit_heapbase() { 5550 if (UseCompressedOops) { 5551 if (Universe::heap() != nullptr) { 5552 if (CompressedOops::base() == nullptr) { 5553 MacroAssembler::xorptr(r12_heapbase, r12_heapbase); 5554 } else { 5555 mov64(r12_heapbase, (int64_t)CompressedOops::base()); 5556 } 5557 } else { 5558 movptr(r12_heapbase, ExternalAddress(CompressedOops::base_addr())); 5559 } 5560 } 5561 } 5562 5563 #if COMPILER2_OR_JVMCI 5564 5565 // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM/ZMM registers 5566 void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register rtmp, XMMRegister xtmp, KRegister mask) { 5567 // cnt - number of qwords (8-byte words). 5568 // base - start address, qword aligned. 5569 Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end; 5570 bool use64byteVector = (MaxVectorSize == 64) && (VM_Version::avx3_threshold() == 0); 5571 if (use64byteVector) { 5572 vpxor(xtmp, xtmp, xtmp, AVX_512bit); 5573 } else if (MaxVectorSize >= 32) { 5574 vpxor(xtmp, xtmp, xtmp, AVX_256bit); 5575 } else { 5576 pxor(xtmp, xtmp); 5577 } 5578 jmp(L_zero_64_bytes); 5579 5580 BIND(L_loop); 5581 if (MaxVectorSize >= 32) { 5582 fill64(base, 0, xtmp, use64byteVector); 5583 } else { 5584 movdqu(Address(base, 0), xtmp); 5585 movdqu(Address(base, 16), xtmp); 5586 movdqu(Address(base, 32), xtmp); 5587 movdqu(Address(base, 48), xtmp); 5588 } 5589 addptr(base, 64); 5590 5591 BIND(L_zero_64_bytes); 5592 subptr(cnt, 8); 5593 jccb(Assembler::greaterEqual, L_loop); 5594 5595 // Copy trailing 64 bytes 5596 if (use64byteVector) { 5597 addptr(cnt, 8); 5598 jccb(Assembler::equal, L_end); 5599 fill64_masked(3, base, 0, xtmp, mask, cnt, rtmp, true); 5600 jmp(L_end); 5601 } else { 5602 addptr(cnt, 4); 5603 jccb(Assembler::less, L_tail); 5604 if (MaxVectorSize >= 32) { 5605 vmovdqu(Address(base, 0), xtmp); 5606 } else { 5607 movdqu(Address(base, 0), xtmp); 5608 movdqu(Address(base, 16), xtmp); 5609 } 5610 } 5611 addptr(base, 32); 5612 subptr(cnt, 4); 5613 5614 BIND(L_tail); 5615 addptr(cnt, 4); 5616 jccb(Assembler::lessEqual, L_end); 5617 if (UseAVX > 2 && MaxVectorSize >= 32 && VM_Version::supports_avx512vl()) { 5618 fill32_masked(3, base, 0, xtmp, mask, cnt, rtmp); 5619 } else { 5620 decrement(cnt); 5621 5622 BIND(L_sloop); 5623 movq(Address(base, 0), xtmp); 5624 addptr(base, 8); 5625 decrement(cnt); 5626 jccb(Assembler::greaterEqual, L_sloop); 5627 } 5628 BIND(L_end); 5629 } 5630 5631 // Clearing constant sized memory using YMM/ZMM registers. 5632 void MacroAssembler::clear_mem(Register base, int cnt, Register rtmp, XMMRegister xtmp, KRegister mask) { 5633 assert(UseAVX > 2 && VM_Version::supports_avx512vl(), ""); 5634 bool use64byteVector = (MaxVectorSize > 32) && (VM_Version::avx3_threshold() == 0); 5635 5636 int vector64_count = (cnt & (~0x7)) >> 3; 5637 cnt = cnt & 0x7; 5638 const int fill64_per_loop = 4; 5639 const int max_unrolled_fill64 = 8; 5640 5641 // 64 byte initialization loop. 5642 vpxor(xtmp, xtmp, xtmp, use64byteVector ? AVX_512bit : AVX_256bit); 5643 int start64 = 0; 5644 if (vector64_count > max_unrolled_fill64) { 5645 Label LOOP; 5646 Register index = rtmp; 5647 5648 start64 = vector64_count - (vector64_count % fill64_per_loop); 5649 5650 movl(index, 0); 5651 BIND(LOOP); 5652 for (int i = 0; i < fill64_per_loop; i++) { 5653 fill64(Address(base, index, Address::times_1, i * 64), xtmp, use64byteVector); 5654 } 5655 addl(index, fill64_per_loop * 64); 5656 cmpl(index, start64 * 64); 5657 jccb(Assembler::less, LOOP); 5658 } 5659 for (int i = start64; i < vector64_count; i++) { 5660 fill64(base, i * 64, xtmp, use64byteVector); 5661 } 5662 5663 // Clear remaining 64 byte tail. 5664 int disp = vector64_count * 64; 5665 if (cnt) { 5666 switch (cnt) { 5667 case 1: 5668 movq(Address(base, disp), xtmp); 5669 break; 5670 case 2: 5671 evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_128bit); 5672 break; 5673 case 3: 5674 movl(rtmp, 0x7); 5675 kmovwl(mask, rtmp); 5676 evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_256bit); 5677 break; 5678 case 4: 5679 evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit); 5680 break; 5681 case 5: 5682 if (use64byteVector) { 5683 movl(rtmp, 0x1F); 5684 kmovwl(mask, rtmp); 5685 evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit); 5686 } else { 5687 evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit); 5688 movq(Address(base, disp + 32), xtmp); 5689 } 5690 break; 5691 case 6: 5692 if (use64byteVector) { 5693 movl(rtmp, 0x3F); 5694 kmovwl(mask, rtmp); 5695 evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit); 5696 } else { 5697 evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit); 5698 evmovdqu(T_LONG, k0, Address(base, disp + 32), xtmp, false, Assembler::AVX_128bit); 5699 } 5700 break; 5701 case 7: 5702 if (use64byteVector) { 5703 movl(rtmp, 0x7F); 5704 kmovwl(mask, rtmp); 5705 evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit); 5706 } else { 5707 evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit); 5708 movl(rtmp, 0x7); 5709 kmovwl(mask, rtmp); 5710 evmovdqu(T_LONG, mask, Address(base, disp + 32), xtmp, true, Assembler::AVX_256bit); 5711 } 5712 break; 5713 default: 5714 fatal("Unexpected length : %d\n",cnt); 5715 break; 5716 } 5717 } 5718 } 5719 5720 void MacroAssembler::clear_mem(Register base, Register cnt, Register tmp, XMMRegister xtmp, 5721 bool is_large, KRegister mask) { 5722 // cnt - number of qwords (8-byte words). 5723 // base - start address, qword aligned. 5724 // is_large - if optimizers know cnt is larger than InitArrayShortSize 5725 assert(base==rdi, "base register must be edi for rep stos"); 5726 assert(tmp==rax, "tmp register must be eax for rep stos"); 5727 assert(cnt==rcx, "cnt register must be ecx for rep stos"); 5728 assert(InitArrayShortSize % BytesPerLong == 0, 5729 "InitArrayShortSize should be the multiple of BytesPerLong"); 5730 5731 Label DONE; 5732 if (!is_large || !UseXMMForObjInit) { 5733 xorptr(tmp, tmp); 5734 } 5735 5736 if (!is_large) { 5737 Label LOOP, LONG; 5738 cmpptr(cnt, InitArrayShortSize/BytesPerLong); 5739 jccb(Assembler::greater, LONG); 5740 5741 decrement(cnt); 5742 jccb(Assembler::negative, DONE); // Zero length 5743 5744 // Use individual pointer-sized stores for small counts: 5745 BIND(LOOP); 5746 movptr(Address(base, cnt, Address::times_ptr), tmp); 5747 decrement(cnt); 5748 jccb(Assembler::greaterEqual, LOOP); 5749 jmpb(DONE); 5750 5751 BIND(LONG); 5752 } 5753 5754 // Use longer rep-prefixed ops for non-small counts: 5755 if (UseFastStosb) { 5756 shlptr(cnt, 3); // convert to number of bytes 5757 rep_stosb(); 5758 } else if (UseXMMForObjInit) { 5759 xmm_clear_mem(base, cnt, tmp, xtmp, mask); 5760 } else { 5761 rep_stos(); 5762 } 5763 5764 BIND(DONE); 5765 } 5766 5767 #endif //COMPILER2_OR_JVMCI 5768 5769 5770 void MacroAssembler::generate_fill(BasicType t, bool aligned, 5771 Register to, Register value, Register count, 5772 Register rtmp, XMMRegister xtmp) { 5773 ShortBranchVerifier sbv(this); 5774 assert_different_registers(to, value, count, rtmp); 5775 Label L_exit; 5776 Label L_fill_2_bytes, L_fill_4_bytes; 5777 5778 #if defined(COMPILER2) 5779 if(MaxVectorSize >=32 && 5780 VM_Version::supports_avx512vlbw() && 5781 VM_Version::supports_bmi2()) { 5782 generate_fill_avx3(t, to, value, count, rtmp, xtmp); 5783 return; 5784 } 5785 #endif 5786 5787 int shift = -1; 5788 switch (t) { 5789 case T_BYTE: 5790 shift = 2; 5791 break; 5792 case T_SHORT: 5793 shift = 1; 5794 break; 5795 case T_INT: 5796 shift = 0; 5797 break; 5798 default: ShouldNotReachHere(); 5799 } 5800 5801 if (t == T_BYTE) { 5802 andl(value, 0xff); 5803 movl(rtmp, value); 5804 shll(rtmp, 8); 5805 orl(value, rtmp); 5806 } 5807 if (t == T_SHORT) { 5808 andl(value, 0xffff); 5809 } 5810 if (t == T_BYTE || t == T_SHORT) { 5811 movl(rtmp, value); 5812 shll(rtmp, 16); 5813 orl(value, rtmp); 5814 } 5815 5816 cmpptr(count, 2<<shift); // Short arrays (< 8 bytes) fill by element 5817 jcc(Assembler::below, L_fill_4_bytes); // use unsigned cmp 5818 if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) { 5819 Label L_skip_align2; 5820 // align source address at 4 bytes address boundary 5821 if (t == T_BYTE) { 5822 Label L_skip_align1; 5823 // One byte misalignment happens only for byte arrays 5824 testptr(to, 1); 5825 jccb(Assembler::zero, L_skip_align1); 5826 movb(Address(to, 0), value); 5827 increment(to); 5828 decrement(count); 5829 BIND(L_skip_align1); 5830 } 5831 // Two bytes misalignment happens only for byte and short (char) arrays 5832 testptr(to, 2); 5833 jccb(Assembler::zero, L_skip_align2); 5834 movw(Address(to, 0), value); 5835 addptr(to, 2); 5836 subptr(count, 1<<(shift-1)); 5837 BIND(L_skip_align2); 5838 } 5839 { 5840 Label L_fill_32_bytes; 5841 if (!UseUnalignedLoadStores) { 5842 // align to 8 bytes, we know we are 4 byte aligned to start 5843 testptr(to, 4); 5844 jccb(Assembler::zero, L_fill_32_bytes); 5845 movl(Address(to, 0), value); 5846 addptr(to, 4); 5847 subptr(count, 1<<shift); 5848 } 5849 BIND(L_fill_32_bytes); 5850 { 5851 Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes; 5852 movdl(xtmp, value); 5853 if (UseAVX >= 2 && UseUnalignedLoadStores) { 5854 Label L_check_fill_32_bytes; 5855 if (UseAVX > 2) { 5856 // Fill 64-byte chunks 5857 Label L_fill_64_bytes_loop_avx3, L_check_fill_64_bytes_avx2; 5858 5859 // If number of bytes to fill < VM_Version::avx3_threshold(), perform fill using AVX2 5860 cmpptr(count, VM_Version::avx3_threshold()); 5861 jccb(Assembler::below, L_check_fill_64_bytes_avx2); 5862 5863 vpbroadcastd(xtmp, xtmp, Assembler::AVX_512bit); 5864 5865 subptr(count, 16 << shift); 5866 jccb(Assembler::less, L_check_fill_32_bytes); 5867 align(16); 5868 5869 BIND(L_fill_64_bytes_loop_avx3); 5870 evmovdqul(Address(to, 0), xtmp, Assembler::AVX_512bit); 5871 addptr(to, 64); 5872 subptr(count, 16 << shift); 5873 jcc(Assembler::greaterEqual, L_fill_64_bytes_loop_avx3); 5874 jmpb(L_check_fill_32_bytes); 5875 5876 BIND(L_check_fill_64_bytes_avx2); 5877 } 5878 // Fill 64-byte chunks 5879 Label L_fill_64_bytes_loop; 5880 vpbroadcastd(xtmp, xtmp, Assembler::AVX_256bit); 5881 5882 subptr(count, 16 << shift); 5883 jcc(Assembler::less, L_check_fill_32_bytes); 5884 align(16); 5885 5886 BIND(L_fill_64_bytes_loop); 5887 vmovdqu(Address(to, 0), xtmp); 5888 vmovdqu(Address(to, 32), xtmp); 5889 addptr(to, 64); 5890 subptr(count, 16 << shift); 5891 jcc(Assembler::greaterEqual, L_fill_64_bytes_loop); 5892 5893 BIND(L_check_fill_32_bytes); 5894 addptr(count, 8 << shift); 5895 jccb(Assembler::less, L_check_fill_8_bytes); 5896 vmovdqu(Address(to, 0), xtmp); 5897 addptr(to, 32); 5898 subptr(count, 8 << shift); 5899 5900 BIND(L_check_fill_8_bytes); 5901 // clean upper bits of YMM registers 5902 movdl(xtmp, value); 5903 pshufd(xtmp, xtmp, 0); 5904 } else { 5905 // Fill 32-byte chunks 5906 pshufd(xtmp, xtmp, 0); 5907 5908 subptr(count, 8 << shift); 5909 jcc(Assembler::less, L_check_fill_8_bytes); 5910 align(16); 5911 5912 BIND(L_fill_32_bytes_loop); 5913 5914 if (UseUnalignedLoadStores) { 5915 movdqu(Address(to, 0), xtmp); 5916 movdqu(Address(to, 16), xtmp); 5917 } else { 5918 movq(Address(to, 0), xtmp); 5919 movq(Address(to, 8), xtmp); 5920 movq(Address(to, 16), xtmp); 5921 movq(Address(to, 24), xtmp); 5922 } 5923 5924 addptr(to, 32); 5925 subptr(count, 8 << shift); 5926 jcc(Assembler::greaterEqual, L_fill_32_bytes_loop); 5927 5928 BIND(L_check_fill_8_bytes); 5929 } 5930 addptr(count, 8 << shift); 5931 jccb(Assembler::zero, L_exit); 5932 jmpb(L_fill_8_bytes); 5933 5934 // 5935 // length is too short, just fill qwords 5936 // 5937 BIND(L_fill_8_bytes_loop); 5938 movq(Address(to, 0), xtmp); 5939 addptr(to, 8); 5940 BIND(L_fill_8_bytes); 5941 subptr(count, 1 << (shift + 1)); 5942 jcc(Assembler::greaterEqual, L_fill_8_bytes_loop); 5943 } 5944 } 5945 // fill trailing 4 bytes 5946 BIND(L_fill_4_bytes); 5947 testl(count, 1<<shift); 5948 jccb(Assembler::zero, L_fill_2_bytes); 5949 movl(Address(to, 0), value); 5950 if (t == T_BYTE || t == T_SHORT) { 5951 Label L_fill_byte; 5952 addptr(to, 4); 5953 BIND(L_fill_2_bytes); 5954 // fill trailing 2 bytes 5955 testl(count, 1<<(shift-1)); 5956 jccb(Assembler::zero, L_fill_byte); 5957 movw(Address(to, 0), value); 5958 if (t == T_BYTE) { 5959 addptr(to, 2); 5960 BIND(L_fill_byte); 5961 // fill trailing byte 5962 testl(count, 1); 5963 jccb(Assembler::zero, L_exit); 5964 movb(Address(to, 0), value); 5965 } else { 5966 BIND(L_fill_byte); 5967 } 5968 } else { 5969 BIND(L_fill_2_bytes); 5970 } 5971 BIND(L_exit); 5972 } 5973 5974 void MacroAssembler::evpbroadcast(BasicType type, XMMRegister dst, Register src, int vector_len) { 5975 switch(type) { 5976 case T_BYTE: 5977 case T_BOOLEAN: 5978 evpbroadcastb(dst, src, vector_len); 5979 break; 5980 case T_SHORT: 5981 case T_CHAR: 5982 evpbroadcastw(dst, src, vector_len); 5983 break; 5984 case T_INT: 5985 case T_FLOAT: 5986 evpbroadcastd(dst, src, vector_len); 5987 break; 5988 case T_LONG: 5989 case T_DOUBLE: 5990 evpbroadcastq(dst, src, vector_len); 5991 break; 5992 default: 5993 fatal("Unhandled type : %s", type2name(type)); 5994 break; 5995 } 5996 } 5997 5998 // encode char[] to byte[] in ISO_8859_1 or ASCII 5999 //@IntrinsicCandidate 6000 //private static int implEncodeISOArray(byte[] sa, int sp, 6001 //byte[] da, int dp, int len) { 6002 // int i = 0; 6003 // for (; i < len; i++) { 6004 // char c = StringUTF16.getChar(sa, sp++); 6005 // if (c > '\u00FF') 6006 // break; 6007 // da[dp++] = (byte)c; 6008 // } 6009 // return i; 6010 //} 6011 // 6012 //@IntrinsicCandidate 6013 //private static int implEncodeAsciiArray(char[] sa, int sp, 6014 // byte[] da, int dp, int len) { 6015 // int i = 0; 6016 // for (; i < len; i++) { 6017 // char c = sa[sp++]; 6018 // if (c >= '\u0080') 6019 // break; 6020 // da[dp++] = (byte)c; 6021 // } 6022 // return i; 6023 //} 6024 void MacroAssembler::encode_iso_array(Register src, Register dst, Register len, 6025 XMMRegister tmp1Reg, XMMRegister tmp2Reg, 6026 XMMRegister tmp3Reg, XMMRegister tmp4Reg, 6027 Register tmp5, Register result, bool ascii) { 6028 6029 // rsi: src 6030 // rdi: dst 6031 // rdx: len 6032 // rcx: tmp5 6033 // rax: result 6034 ShortBranchVerifier sbv(this); 6035 assert_different_registers(src, dst, len, tmp5, result); 6036 Label L_done, L_copy_1_char, L_copy_1_char_exit; 6037 6038 int mask = ascii ? 0xff80ff80 : 0xff00ff00; 6039 int short_mask = ascii ? 0xff80 : 0xff00; 6040 6041 // set result 6042 xorl(result, result); 6043 // check for zero length 6044 testl(len, len); 6045 jcc(Assembler::zero, L_done); 6046 6047 movl(result, len); 6048 6049 // Setup pointers 6050 lea(src, Address(src, len, Address::times_2)); // char[] 6051 lea(dst, Address(dst, len, Address::times_1)); // byte[] 6052 negptr(len); 6053 6054 if (UseSSE42Intrinsics || UseAVX >= 2) { 6055 Label L_copy_8_chars, L_copy_8_chars_exit; 6056 Label L_chars_16_check, L_copy_16_chars, L_copy_16_chars_exit; 6057 6058 if (UseAVX >= 2) { 6059 Label L_chars_32_check, L_copy_32_chars, L_copy_32_chars_exit; 6060 movl(tmp5, mask); // create mask to test for Unicode or non-ASCII chars in vector 6061 movdl(tmp1Reg, tmp5); 6062 vpbroadcastd(tmp1Reg, tmp1Reg, Assembler::AVX_256bit); 6063 jmp(L_chars_32_check); 6064 6065 bind(L_copy_32_chars); 6066 vmovdqu(tmp3Reg, Address(src, len, Address::times_2, -64)); 6067 vmovdqu(tmp4Reg, Address(src, len, Address::times_2, -32)); 6068 vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1); 6069 vptest(tmp2Reg, tmp1Reg); // check for Unicode or non-ASCII chars in vector 6070 jccb(Assembler::notZero, L_copy_32_chars_exit); 6071 vpackuswb(tmp3Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1); 6072 vpermq(tmp4Reg, tmp3Reg, 0xD8, /* vector_len */ 1); 6073 vmovdqu(Address(dst, len, Address::times_1, -32), tmp4Reg); 6074 6075 bind(L_chars_32_check); 6076 addptr(len, 32); 6077 jcc(Assembler::lessEqual, L_copy_32_chars); 6078 6079 bind(L_copy_32_chars_exit); 6080 subptr(len, 16); 6081 jccb(Assembler::greater, L_copy_16_chars_exit); 6082 6083 } else if (UseSSE42Intrinsics) { 6084 movl(tmp5, mask); // create mask to test for Unicode or non-ASCII chars in vector 6085 movdl(tmp1Reg, tmp5); 6086 pshufd(tmp1Reg, tmp1Reg, 0); 6087 jmpb(L_chars_16_check); 6088 } 6089 6090 bind(L_copy_16_chars); 6091 if (UseAVX >= 2) { 6092 vmovdqu(tmp2Reg, Address(src, len, Address::times_2, -32)); 6093 vptest(tmp2Reg, tmp1Reg); 6094 jcc(Assembler::notZero, L_copy_16_chars_exit); 6095 vpackuswb(tmp2Reg, tmp2Reg, tmp1Reg, /* vector_len */ 1); 6096 vpermq(tmp3Reg, tmp2Reg, 0xD8, /* vector_len */ 1); 6097 } else { 6098 if (UseAVX > 0) { 6099 movdqu(tmp3Reg, Address(src, len, Address::times_2, -32)); 6100 movdqu(tmp4Reg, Address(src, len, Address::times_2, -16)); 6101 vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 0); 6102 } else { 6103 movdqu(tmp3Reg, Address(src, len, Address::times_2, -32)); 6104 por(tmp2Reg, tmp3Reg); 6105 movdqu(tmp4Reg, Address(src, len, Address::times_2, -16)); 6106 por(tmp2Reg, tmp4Reg); 6107 } 6108 ptest(tmp2Reg, tmp1Reg); // check for Unicode or non-ASCII chars in vector 6109 jccb(Assembler::notZero, L_copy_16_chars_exit); 6110 packuswb(tmp3Reg, tmp4Reg); 6111 } 6112 movdqu(Address(dst, len, Address::times_1, -16), tmp3Reg); 6113 6114 bind(L_chars_16_check); 6115 addptr(len, 16); 6116 jcc(Assembler::lessEqual, L_copy_16_chars); 6117 6118 bind(L_copy_16_chars_exit); 6119 if (UseAVX >= 2) { 6120 // clean upper bits of YMM registers 6121 vpxor(tmp2Reg, tmp2Reg); 6122 vpxor(tmp3Reg, tmp3Reg); 6123 vpxor(tmp4Reg, tmp4Reg); 6124 movdl(tmp1Reg, tmp5); 6125 pshufd(tmp1Reg, tmp1Reg, 0); 6126 } 6127 subptr(len, 8); 6128 jccb(Assembler::greater, L_copy_8_chars_exit); 6129 6130 bind(L_copy_8_chars); 6131 movdqu(tmp3Reg, Address(src, len, Address::times_2, -16)); 6132 ptest(tmp3Reg, tmp1Reg); 6133 jccb(Assembler::notZero, L_copy_8_chars_exit); 6134 packuswb(tmp3Reg, tmp1Reg); 6135 movq(Address(dst, len, Address::times_1, -8), tmp3Reg); 6136 addptr(len, 8); 6137 jccb(Assembler::lessEqual, L_copy_8_chars); 6138 6139 bind(L_copy_8_chars_exit); 6140 subptr(len, 8); 6141 jccb(Assembler::zero, L_done); 6142 } 6143 6144 bind(L_copy_1_char); 6145 load_unsigned_short(tmp5, Address(src, len, Address::times_2, 0)); 6146 testl(tmp5, short_mask); // check if Unicode or non-ASCII char 6147 jccb(Assembler::notZero, L_copy_1_char_exit); 6148 movb(Address(dst, len, Address::times_1, 0), tmp5); 6149 addptr(len, 1); 6150 jccb(Assembler::less, L_copy_1_char); 6151 6152 bind(L_copy_1_char_exit); 6153 addptr(result, len); // len is negative count of not processed elements 6154 6155 bind(L_done); 6156 } 6157 6158 /** 6159 * Helper for multiply_to_len(). 6160 */ 6161 void MacroAssembler::add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2) { 6162 addq(dest_lo, src1); 6163 adcq(dest_hi, 0); 6164 addq(dest_lo, src2); 6165 adcq(dest_hi, 0); 6166 } 6167 6168 /** 6169 * Multiply 64 bit by 64 bit first loop. 6170 */ 6171 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart, 6172 Register y, Register y_idx, Register z, 6173 Register carry, Register product, 6174 Register idx, Register kdx) { 6175 // 6176 // jlong carry, x[], y[], z[]; 6177 // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) { 6178 // huge_128 product = y[idx] * x[xstart] + carry; 6179 // z[kdx] = (jlong)product; 6180 // carry = (jlong)(product >>> 64); 6181 // } 6182 // z[xstart] = carry; 6183 // 6184 6185 Label L_first_loop, L_first_loop_exit; 6186 Label L_one_x, L_one_y, L_multiply; 6187 6188 decrementl(xstart); 6189 jcc(Assembler::negative, L_one_x); 6190 6191 movq(x_xstart, Address(x, xstart, Address::times_4, 0)); 6192 rorq(x_xstart, 32); // convert big-endian to little-endian 6193 6194 bind(L_first_loop); 6195 decrementl(idx); 6196 jcc(Assembler::negative, L_first_loop_exit); 6197 decrementl(idx); 6198 jcc(Assembler::negative, L_one_y); 6199 movq(y_idx, Address(y, idx, Address::times_4, 0)); 6200 rorq(y_idx, 32); // convert big-endian to little-endian 6201 bind(L_multiply); 6202 movq(product, x_xstart); 6203 mulq(y_idx); // product(rax) * y_idx -> rdx:rax 6204 addq(product, carry); 6205 adcq(rdx, 0); 6206 subl(kdx, 2); 6207 movl(Address(z, kdx, Address::times_4, 4), product); 6208 shrq(product, 32); 6209 movl(Address(z, kdx, Address::times_4, 0), product); 6210 movq(carry, rdx); 6211 jmp(L_first_loop); 6212 6213 bind(L_one_y); 6214 movl(y_idx, Address(y, 0)); 6215 jmp(L_multiply); 6216 6217 bind(L_one_x); 6218 movl(x_xstart, Address(x, 0)); 6219 jmp(L_first_loop); 6220 6221 bind(L_first_loop_exit); 6222 } 6223 6224 /** 6225 * Multiply 64 bit by 64 bit and add 128 bit. 6226 */ 6227 void MacroAssembler::multiply_add_128_x_128(Register x_xstart, Register y, Register z, 6228 Register yz_idx, Register idx, 6229 Register carry, Register product, int offset) { 6230 // huge_128 product = (y[idx] * x_xstart) + z[kdx] + carry; 6231 // z[kdx] = (jlong)product; 6232 6233 movq(yz_idx, Address(y, idx, Address::times_4, offset)); 6234 rorq(yz_idx, 32); // convert big-endian to little-endian 6235 movq(product, x_xstart); 6236 mulq(yz_idx); // product(rax) * yz_idx -> rdx:product(rax) 6237 movq(yz_idx, Address(z, idx, Address::times_4, offset)); 6238 rorq(yz_idx, 32); // convert big-endian to little-endian 6239 6240 add2_with_carry(rdx, product, carry, yz_idx); 6241 6242 movl(Address(z, idx, Address::times_4, offset+4), product); 6243 shrq(product, 32); 6244 movl(Address(z, idx, Address::times_4, offset), product); 6245 6246 } 6247 6248 /** 6249 * Multiply 128 bit by 128 bit. Unrolled inner loop. 6250 */ 6251 void MacroAssembler::multiply_128_x_128_loop(Register x_xstart, Register y, Register z, 6252 Register yz_idx, Register idx, Register jdx, 6253 Register carry, Register product, 6254 Register carry2) { 6255 // jlong carry, x[], y[], z[]; 6256 // int kdx = ystart+1; 6257 // for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop 6258 // huge_128 product = (y[idx+1] * x_xstart) + z[kdx+idx+1] + carry; 6259 // z[kdx+idx+1] = (jlong)product; 6260 // jlong carry2 = (jlong)(product >>> 64); 6261 // product = (y[idx] * x_xstart) + z[kdx+idx] + carry2; 6262 // z[kdx+idx] = (jlong)product; 6263 // carry = (jlong)(product >>> 64); 6264 // } 6265 // idx += 2; 6266 // if (idx > 0) { 6267 // product = (y[idx] * x_xstart) + z[kdx+idx] + carry; 6268 // z[kdx+idx] = (jlong)product; 6269 // carry = (jlong)(product >>> 64); 6270 // } 6271 // 6272 6273 Label L_third_loop, L_third_loop_exit, L_post_third_loop_done; 6274 6275 movl(jdx, idx); 6276 andl(jdx, 0xFFFFFFFC); 6277 shrl(jdx, 2); 6278 6279 bind(L_third_loop); 6280 subl(jdx, 1); 6281 jcc(Assembler::negative, L_third_loop_exit); 6282 subl(idx, 4); 6283 6284 multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 8); 6285 movq(carry2, rdx); 6286 6287 multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry2, product, 0); 6288 movq(carry, rdx); 6289 jmp(L_third_loop); 6290 6291 bind (L_third_loop_exit); 6292 6293 andl (idx, 0x3); 6294 jcc(Assembler::zero, L_post_third_loop_done); 6295 6296 Label L_check_1; 6297 subl(idx, 2); 6298 jcc(Assembler::negative, L_check_1); 6299 6300 multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 0); 6301 movq(carry, rdx); 6302 6303 bind (L_check_1); 6304 addl (idx, 0x2); 6305 andl (idx, 0x1); 6306 subl(idx, 1); 6307 jcc(Assembler::negative, L_post_third_loop_done); 6308 6309 movl(yz_idx, Address(y, idx, Address::times_4, 0)); 6310 movq(product, x_xstart); 6311 mulq(yz_idx); // product(rax) * yz_idx -> rdx:product(rax) 6312 movl(yz_idx, Address(z, idx, Address::times_4, 0)); 6313 6314 add2_with_carry(rdx, product, yz_idx, carry); 6315 6316 movl(Address(z, idx, Address::times_4, 0), product); 6317 shrq(product, 32); 6318 6319 shlq(rdx, 32); 6320 orq(product, rdx); 6321 movq(carry, product); 6322 6323 bind(L_post_third_loop_done); 6324 } 6325 6326 /** 6327 * Multiply 128 bit by 128 bit using BMI2. Unrolled inner loop. 6328 * 6329 */ 6330 void MacroAssembler::multiply_128_x_128_bmi2_loop(Register y, Register z, 6331 Register carry, Register carry2, 6332 Register idx, Register jdx, 6333 Register yz_idx1, Register yz_idx2, 6334 Register tmp, Register tmp3, Register tmp4) { 6335 assert(UseBMI2Instructions, "should be used only when BMI2 is available"); 6336 6337 // jlong carry, x[], y[], z[]; 6338 // int kdx = ystart+1; 6339 // for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop 6340 // huge_128 tmp3 = (y[idx+1] * rdx) + z[kdx+idx+1] + carry; 6341 // jlong carry2 = (jlong)(tmp3 >>> 64); 6342 // huge_128 tmp4 = (y[idx] * rdx) + z[kdx+idx] + carry2; 6343 // carry = (jlong)(tmp4 >>> 64); 6344 // z[kdx+idx+1] = (jlong)tmp3; 6345 // z[kdx+idx] = (jlong)tmp4; 6346 // } 6347 // idx += 2; 6348 // if (idx > 0) { 6349 // yz_idx1 = (y[idx] * rdx) + z[kdx+idx] + carry; 6350 // z[kdx+idx] = (jlong)yz_idx1; 6351 // carry = (jlong)(yz_idx1 >>> 64); 6352 // } 6353 // 6354 6355 Label L_third_loop, L_third_loop_exit, L_post_third_loop_done; 6356 6357 movl(jdx, idx); 6358 andl(jdx, 0xFFFFFFFC); 6359 shrl(jdx, 2); 6360 6361 bind(L_third_loop); 6362 subl(jdx, 1); 6363 jcc(Assembler::negative, L_third_loop_exit); 6364 subl(idx, 4); 6365 6366 movq(yz_idx1, Address(y, idx, Address::times_4, 8)); 6367 rorxq(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian 6368 movq(yz_idx2, Address(y, idx, Address::times_4, 0)); 6369 rorxq(yz_idx2, yz_idx2, 32); 6370 6371 mulxq(tmp4, tmp3, yz_idx1); // yz_idx1 * rdx -> tmp4:tmp3 6372 mulxq(carry2, tmp, yz_idx2); // yz_idx2 * rdx -> carry2:tmp 6373 6374 movq(yz_idx1, Address(z, idx, Address::times_4, 8)); 6375 rorxq(yz_idx1, yz_idx1, 32); 6376 movq(yz_idx2, Address(z, idx, Address::times_4, 0)); 6377 rorxq(yz_idx2, yz_idx2, 32); 6378 6379 if (VM_Version::supports_adx()) { 6380 adcxq(tmp3, carry); 6381 adoxq(tmp3, yz_idx1); 6382 6383 adcxq(tmp4, tmp); 6384 adoxq(tmp4, yz_idx2); 6385 6386 movl(carry, 0); // does not affect flags 6387 adcxq(carry2, carry); 6388 adoxq(carry2, carry); 6389 } else { 6390 add2_with_carry(tmp4, tmp3, carry, yz_idx1); 6391 add2_with_carry(carry2, tmp4, tmp, yz_idx2); 6392 } 6393 movq(carry, carry2); 6394 6395 movl(Address(z, idx, Address::times_4, 12), tmp3); 6396 shrq(tmp3, 32); 6397 movl(Address(z, idx, Address::times_4, 8), tmp3); 6398 6399 movl(Address(z, idx, Address::times_4, 4), tmp4); 6400 shrq(tmp4, 32); 6401 movl(Address(z, idx, Address::times_4, 0), tmp4); 6402 6403 jmp(L_third_loop); 6404 6405 bind (L_third_loop_exit); 6406 6407 andl (idx, 0x3); 6408 jcc(Assembler::zero, L_post_third_loop_done); 6409 6410 Label L_check_1; 6411 subl(idx, 2); 6412 jcc(Assembler::negative, L_check_1); 6413 6414 movq(yz_idx1, Address(y, idx, Address::times_4, 0)); 6415 rorxq(yz_idx1, yz_idx1, 32); 6416 mulxq(tmp4, tmp3, yz_idx1); // yz_idx1 * rdx -> tmp4:tmp3 6417 movq(yz_idx2, Address(z, idx, Address::times_4, 0)); 6418 rorxq(yz_idx2, yz_idx2, 32); 6419 6420 add2_with_carry(tmp4, tmp3, carry, yz_idx2); 6421 6422 movl(Address(z, idx, Address::times_4, 4), tmp3); 6423 shrq(tmp3, 32); 6424 movl(Address(z, idx, Address::times_4, 0), tmp3); 6425 movq(carry, tmp4); 6426 6427 bind (L_check_1); 6428 addl (idx, 0x2); 6429 andl (idx, 0x1); 6430 subl(idx, 1); 6431 jcc(Assembler::negative, L_post_third_loop_done); 6432 movl(tmp4, Address(y, idx, Address::times_4, 0)); 6433 mulxq(carry2, tmp3, tmp4); // tmp4 * rdx -> carry2:tmp3 6434 movl(tmp4, Address(z, idx, Address::times_4, 0)); 6435 6436 add2_with_carry(carry2, tmp3, tmp4, carry); 6437 6438 movl(Address(z, idx, Address::times_4, 0), tmp3); 6439 shrq(tmp3, 32); 6440 6441 shlq(carry2, 32); 6442 orq(tmp3, carry2); 6443 movq(carry, tmp3); 6444 6445 bind(L_post_third_loop_done); 6446 } 6447 6448 /** 6449 * Code for BigInteger::multiplyToLen() intrinsic. 6450 * 6451 * rdi: x 6452 * rax: xlen 6453 * rsi: y 6454 * rcx: ylen 6455 * r8: z 6456 * r11: tmp0 6457 * r12: tmp1 6458 * r13: tmp2 6459 * r14: tmp3 6460 * r15: tmp4 6461 * rbx: tmp5 6462 * 6463 */ 6464 void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen, Register z, Register tmp0, 6465 Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5) { 6466 ShortBranchVerifier sbv(this); 6467 assert_different_registers(x, xlen, y, ylen, z, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, rdx); 6468 6469 push(tmp0); 6470 push(tmp1); 6471 push(tmp2); 6472 push(tmp3); 6473 push(tmp4); 6474 push(tmp5); 6475 6476 push(xlen); 6477 6478 const Register idx = tmp1; 6479 const Register kdx = tmp2; 6480 const Register xstart = tmp3; 6481 6482 const Register y_idx = tmp4; 6483 const Register carry = tmp5; 6484 const Register product = xlen; 6485 const Register x_xstart = tmp0; 6486 6487 // First Loop. 6488 // 6489 // final static long LONG_MASK = 0xffffffffL; 6490 // int xstart = xlen - 1; 6491 // int ystart = ylen - 1; 6492 // long carry = 0; 6493 // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) { 6494 // long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry; 6495 // z[kdx] = (int)product; 6496 // carry = product >>> 32; 6497 // } 6498 // z[xstart] = (int)carry; 6499 // 6500 6501 movl(idx, ylen); // idx = ylen; 6502 lea(kdx, Address(xlen, ylen)); // kdx = xlen+ylen; 6503 xorq(carry, carry); // carry = 0; 6504 6505 Label L_done; 6506 6507 movl(xstart, xlen); 6508 decrementl(xstart); 6509 jcc(Assembler::negative, L_done); 6510 6511 multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx); 6512 6513 Label L_second_loop; 6514 testl(kdx, kdx); 6515 jcc(Assembler::zero, L_second_loop); 6516 6517 Label L_carry; 6518 subl(kdx, 1); 6519 jcc(Assembler::zero, L_carry); 6520 6521 movl(Address(z, kdx, Address::times_4, 0), carry); 6522 shrq(carry, 32); 6523 subl(kdx, 1); 6524 6525 bind(L_carry); 6526 movl(Address(z, kdx, Address::times_4, 0), carry); 6527 6528 // Second and third (nested) loops. 6529 // 6530 // for (int i = xstart-1; i >= 0; i--) { // Second loop 6531 // carry = 0; 6532 // for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop 6533 // long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) + 6534 // (z[k] & LONG_MASK) + carry; 6535 // z[k] = (int)product; 6536 // carry = product >>> 32; 6537 // } 6538 // z[i] = (int)carry; 6539 // } 6540 // 6541 // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = rdx 6542 6543 const Register jdx = tmp1; 6544 6545 bind(L_second_loop); 6546 xorl(carry, carry); // carry = 0; 6547 movl(jdx, ylen); // j = ystart+1 6548 6549 subl(xstart, 1); // i = xstart-1; 6550 jcc(Assembler::negative, L_done); 6551 6552 push (z); 6553 6554 Label L_last_x; 6555 lea(z, Address(z, xstart, Address::times_4, 4)); // z = z + k - j 6556 subl(xstart, 1); // i = xstart-1; 6557 jcc(Assembler::negative, L_last_x); 6558 6559 if (UseBMI2Instructions) { 6560 movq(rdx, Address(x, xstart, Address::times_4, 0)); 6561 rorxq(rdx, rdx, 32); // convert big-endian to little-endian 6562 } else { 6563 movq(x_xstart, Address(x, xstart, Address::times_4, 0)); 6564 rorq(x_xstart, 32); // convert big-endian to little-endian 6565 } 6566 6567 Label L_third_loop_prologue; 6568 bind(L_third_loop_prologue); 6569 6570 push (x); 6571 push (xstart); 6572 push (ylen); 6573 6574 6575 if (UseBMI2Instructions) { 6576 multiply_128_x_128_bmi2_loop(y, z, carry, x, jdx, ylen, product, tmp2, x_xstart, tmp3, tmp4); 6577 } else { // !UseBMI2Instructions 6578 multiply_128_x_128_loop(x_xstart, y, z, y_idx, jdx, ylen, carry, product, x); 6579 } 6580 6581 pop(ylen); 6582 pop(xlen); 6583 pop(x); 6584 pop(z); 6585 6586 movl(tmp3, xlen); 6587 addl(tmp3, 1); 6588 movl(Address(z, tmp3, Address::times_4, 0), carry); 6589 subl(tmp3, 1); 6590 jccb(Assembler::negative, L_done); 6591 6592 shrq(carry, 32); 6593 movl(Address(z, tmp3, Address::times_4, 0), carry); 6594 jmp(L_second_loop); 6595 6596 // Next infrequent code is moved outside loops. 6597 bind(L_last_x); 6598 if (UseBMI2Instructions) { 6599 movl(rdx, Address(x, 0)); 6600 } else { 6601 movl(x_xstart, Address(x, 0)); 6602 } 6603 jmp(L_third_loop_prologue); 6604 6605 bind(L_done); 6606 6607 pop(xlen); 6608 6609 pop(tmp5); 6610 pop(tmp4); 6611 pop(tmp3); 6612 pop(tmp2); 6613 pop(tmp1); 6614 pop(tmp0); 6615 } 6616 6617 void MacroAssembler::vectorized_mismatch(Register obja, Register objb, Register length, Register log2_array_indxscale, 6618 Register result, Register tmp1, Register tmp2, XMMRegister rymm0, XMMRegister rymm1, XMMRegister rymm2){ 6619 assert(UseSSE42Intrinsics, "SSE4.2 must be enabled."); 6620 Label VECTOR16_LOOP, VECTOR8_LOOP, VECTOR4_LOOP; 6621 Label VECTOR8_TAIL, VECTOR4_TAIL; 6622 Label VECTOR32_NOT_EQUAL, VECTOR16_NOT_EQUAL, VECTOR8_NOT_EQUAL, VECTOR4_NOT_EQUAL; 6623 Label SAME_TILL_END, DONE; 6624 Label BYTES_LOOP, BYTES_TAIL, BYTES_NOT_EQUAL; 6625 6626 //scale is in rcx in both Win64 and Unix 6627 ShortBranchVerifier sbv(this); 6628 6629 shlq(length); 6630 xorq(result, result); 6631 6632 if ((AVX3Threshold == 0) && (UseAVX > 2) && 6633 VM_Version::supports_avx512vlbw()) { 6634 Label VECTOR64_LOOP, VECTOR64_NOT_EQUAL, VECTOR32_TAIL; 6635 6636 cmpq(length, 64); 6637 jcc(Assembler::less, VECTOR32_TAIL); 6638 6639 movq(tmp1, length); 6640 andq(tmp1, 0x3F); // tail count 6641 andq(length, ~(0x3F)); //vector count 6642 6643 bind(VECTOR64_LOOP); 6644 // AVX512 code to compare 64 byte vectors. 6645 evmovdqub(rymm0, Address(obja, result), Assembler::AVX_512bit); 6646 evpcmpeqb(k7, rymm0, Address(objb, result), Assembler::AVX_512bit); 6647 kortestql(k7, k7); 6648 jcc(Assembler::aboveEqual, VECTOR64_NOT_EQUAL); // mismatch 6649 addq(result, 64); 6650 subq(length, 64); 6651 jccb(Assembler::notZero, VECTOR64_LOOP); 6652 6653 //bind(VECTOR64_TAIL); 6654 testq(tmp1, tmp1); 6655 jcc(Assembler::zero, SAME_TILL_END); 6656 6657 //bind(VECTOR64_TAIL); 6658 // AVX512 code to compare up to 63 byte vectors. 6659 mov64(tmp2, 0xFFFFFFFFFFFFFFFF); 6660 shlxq(tmp2, tmp2, tmp1); 6661 notq(tmp2); 6662 kmovql(k3, tmp2); 6663 6664 evmovdqub(rymm0, k3, Address(obja, result), false, Assembler::AVX_512bit); 6665 evpcmpeqb(k7, k3, rymm0, Address(objb, result), Assembler::AVX_512bit); 6666 6667 ktestql(k7, k3); 6668 jcc(Assembler::below, SAME_TILL_END); // not mismatch 6669 6670 bind(VECTOR64_NOT_EQUAL); 6671 kmovql(tmp1, k7); 6672 notq(tmp1); 6673 tzcntq(tmp1, tmp1); 6674 addq(result, tmp1); 6675 shrq(result); 6676 jmp(DONE); 6677 bind(VECTOR32_TAIL); 6678 } 6679 6680 cmpq(length, 8); 6681 jcc(Assembler::equal, VECTOR8_LOOP); 6682 jcc(Assembler::less, VECTOR4_TAIL); 6683 6684 if (UseAVX >= 2) { 6685 Label VECTOR16_TAIL, VECTOR32_LOOP; 6686 6687 cmpq(length, 16); 6688 jcc(Assembler::equal, VECTOR16_LOOP); 6689 jcc(Assembler::less, VECTOR8_LOOP); 6690 6691 cmpq(length, 32); 6692 jccb(Assembler::less, VECTOR16_TAIL); 6693 6694 subq(length, 32); 6695 bind(VECTOR32_LOOP); 6696 vmovdqu(rymm0, Address(obja, result)); 6697 vmovdqu(rymm1, Address(objb, result)); 6698 vpxor(rymm2, rymm0, rymm1, Assembler::AVX_256bit); 6699 vptest(rymm2, rymm2); 6700 jcc(Assembler::notZero, VECTOR32_NOT_EQUAL);//mismatch found 6701 addq(result, 32); 6702 subq(length, 32); 6703 jcc(Assembler::greaterEqual, VECTOR32_LOOP); 6704 addq(length, 32); 6705 jcc(Assembler::equal, SAME_TILL_END); 6706 //falling through if less than 32 bytes left //close the branch here. 6707 6708 bind(VECTOR16_TAIL); 6709 cmpq(length, 16); 6710 jccb(Assembler::less, VECTOR8_TAIL); 6711 bind(VECTOR16_LOOP); 6712 movdqu(rymm0, Address(obja, result)); 6713 movdqu(rymm1, Address(objb, result)); 6714 vpxor(rymm2, rymm0, rymm1, Assembler::AVX_128bit); 6715 ptest(rymm2, rymm2); 6716 jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found 6717 addq(result, 16); 6718 subq(length, 16); 6719 jcc(Assembler::equal, SAME_TILL_END); 6720 //falling through if less than 16 bytes left 6721 } else {//regular intrinsics 6722 6723 cmpq(length, 16); 6724 jccb(Assembler::less, VECTOR8_TAIL); 6725 6726 subq(length, 16); 6727 bind(VECTOR16_LOOP); 6728 movdqu(rymm0, Address(obja, result)); 6729 movdqu(rymm1, Address(objb, result)); 6730 pxor(rymm0, rymm1); 6731 ptest(rymm0, rymm0); 6732 jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found 6733 addq(result, 16); 6734 subq(length, 16); 6735 jccb(Assembler::greaterEqual, VECTOR16_LOOP); 6736 addq(length, 16); 6737 jcc(Assembler::equal, SAME_TILL_END); 6738 //falling through if less than 16 bytes left 6739 } 6740 6741 bind(VECTOR8_TAIL); 6742 cmpq(length, 8); 6743 jccb(Assembler::less, VECTOR4_TAIL); 6744 bind(VECTOR8_LOOP); 6745 movq(tmp1, Address(obja, result)); 6746 movq(tmp2, Address(objb, result)); 6747 xorq(tmp1, tmp2); 6748 testq(tmp1, tmp1); 6749 jcc(Assembler::notZero, VECTOR8_NOT_EQUAL);//mismatch found 6750 addq(result, 8); 6751 subq(length, 8); 6752 jcc(Assembler::equal, SAME_TILL_END); 6753 //falling through if less than 8 bytes left 6754 6755 bind(VECTOR4_TAIL); 6756 cmpq(length, 4); 6757 jccb(Assembler::less, BYTES_TAIL); 6758 bind(VECTOR4_LOOP); 6759 movl(tmp1, Address(obja, result)); 6760 xorl(tmp1, Address(objb, result)); 6761 testl(tmp1, tmp1); 6762 jcc(Assembler::notZero, VECTOR4_NOT_EQUAL);//mismatch found 6763 addq(result, 4); 6764 subq(length, 4); 6765 jcc(Assembler::equal, SAME_TILL_END); 6766 //falling through if less than 4 bytes left 6767 6768 bind(BYTES_TAIL); 6769 bind(BYTES_LOOP); 6770 load_unsigned_byte(tmp1, Address(obja, result)); 6771 load_unsigned_byte(tmp2, Address(objb, result)); 6772 xorl(tmp1, tmp2); 6773 testl(tmp1, tmp1); 6774 jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found 6775 decq(length); 6776 jcc(Assembler::zero, SAME_TILL_END); 6777 incq(result); 6778 load_unsigned_byte(tmp1, Address(obja, result)); 6779 load_unsigned_byte(tmp2, Address(objb, result)); 6780 xorl(tmp1, tmp2); 6781 testl(tmp1, tmp1); 6782 jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found 6783 decq(length); 6784 jcc(Assembler::zero, SAME_TILL_END); 6785 incq(result); 6786 load_unsigned_byte(tmp1, Address(obja, result)); 6787 load_unsigned_byte(tmp2, Address(objb, result)); 6788 xorl(tmp1, tmp2); 6789 testl(tmp1, tmp1); 6790 jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found 6791 jmp(SAME_TILL_END); 6792 6793 if (UseAVX >= 2) { 6794 bind(VECTOR32_NOT_EQUAL); 6795 vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_256bit); 6796 vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_256bit); 6797 vpxor(rymm0, rymm0, rymm2, Assembler::AVX_256bit); 6798 vpmovmskb(tmp1, rymm0); 6799 bsfq(tmp1, tmp1); 6800 addq(result, tmp1); 6801 shrq(result); 6802 jmp(DONE); 6803 } 6804 6805 bind(VECTOR16_NOT_EQUAL); 6806 if (UseAVX >= 2) { 6807 vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_128bit); 6808 vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_128bit); 6809 pxor(rymm0, rymm2); 6810 } else { 6811 pcmpeqb(rymm2, rymm2); 6812 pxor(rymm0, rymm1); 6813 pcmpeqb(rymm0, rymm1); 6814 pxor(rymm0, rymm2); 6815 } 6816 pmovmskb(tmp1, rymm0); 6817 bsfq(tmp1, tmp1); 6818 addq(result, tmp1); 6819 shrq(result); 6820 jmpb(DONE); 6821 6822 bind(VECTOR8_NOT_EQUAL); 6823 bind(VECTOR4_NOT_EQUAL); 6824 bsfq(tmp1, tmp1); 6825 shrq(tmp1, 3); 6826 addq(result, tmp1); 6827 bind(BYTES_NOT_EQUAL); 6828 shrq(result); 6829 jmpb(DONE); 6830 6831 bind(SAME_TILL_END); 6832 mov64(result, -1); 6833 6834 bind(DONE); 6835 } 6836 6837 //Helper functions for square_to_len() 6838 6839 /** 6840 * Store the squares of x[], right shifted one bit (divided by 2) into z[] 6841 * Preserves x and z and modifies rest of the registers. 6842 */ 6843 void MacroAssembler::square_rshift(Register x, Register xlen, Register z, Register tmp1, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) { 6844 // Perform square and right shift by 1 6845 // Handle odd xlen case first, then for even xlen do the following 6846 // jlong carry = 0; 6847 // for (int j=0, i=0; j < xlen; j+=2, i+=4) { 6848 // huge_128 product = x[j:j+1] * x[j:j+1]; 6849 // z[i:i+1] = (carry << 63) | (jlong)(product >>> 65); 6850 // z[i+2:i+3] = (jlong)(product >>> 1); 6851 // carry = (jlong)product; 6852 // } 6853 6854 xorq(tmp5, tmp5); // carry 6855 xorq(rdxReg, rdxReg); 6856 xorl(tmp1, tmp1); // index for x 6857 xorl(tmp4, tmp4); // index for z 6858 6859 Label L_first_loop, L_first_loop_exit; 6860 6861 testl(xlen, 1); 6862 jccb(Assembler::zero, L_first_loop); //jump if xlen is even 6863 6864 // Square and right shift by 1 the odd element using 32 bit multiply 6865 movl(raxReg, Address(x, tmp1, Address::times_4, 0)); 6866 imulq(raxReg, raxReg); 6867 shrq(raxReg, 1); 6868 adcq(tmp5, 0); 6869 movq(Address(z, tmp4, Address::times_4, 0), raxReg); 6870 incrementl(tmp1); 6871 addl(tmp4, 2); 6872 6873 // Square and right shift by 1 the rest using 64 bit multiply 6874 bind(L_first_loop); 6875 cmpptr(tmp1, xlen); 6876 jccb(Assembler::equal, L_first_loop_exit); 6877 6878 // Square 6879 movq(raxReg, Address(x, tmp1, Address::times_4, 0)); 6880 rorq(raxReg, 32); // convert big-endian to little-endian 6881 mulq(raxReg); // 64-bit multiply rax * rax -> rdx:rax 6882 6883 // Right shift by 1 and save carry 6884 shrq(tmp5, 1); // rdx:rax:tmp5 = (tmp5:rdx:rax) >>> 1 6885 rcrq(rdxReg, 1); 6886 rcrq(raxReg, 1); 6887 adcq(tmp5, 0); 6888 6889 // Store result in z 6890 movq(Address(z, tmp4, Address::times_4, 0), rdxReg); 6891 movq(Address(z, tmp4, Address::times_4, 8), raxReg); 6892 6893 // Update indices for x and z 6894 addl(tmp1, 2); 6895 addl(tmp4, 4); 6896 jmp(L_first_loop); 6897 6898 bind(L_first_loop_exit); 6899 } 6900 6901 6902 /** 6903 * Perform the following multiply add operation using BMI2 instructions 6904 * carry:sum = sum + op1*op2 + carry 6905 * op2 should be in rdx 6906 * op2 is preserved, all other registers are modified 6907 */ 6908 void MacroAssembler::multiply_add_64_bmi2(Register sum, Register op1, Register op2, Register carry, Register tmp2) { 6909 // assert op2 is rdx 6910 mulxq(tmp2, op1, op1); // op1 * op2 -> tmp2:op1 6911 addq(sum, carry); 6912 adcq(tmp2, 0); 6913 addq(sum, op1); 6914 adcq(tmp2, 0); 6915 movq(carry, tmp2); 6916 } 6917 6918 /** 6919 * Perform the following multiply add operation: 6920 * carry:sum = sum + op1*op2 + carry 6921 * Preserves op1, op2 and modifies rest of registers 6922 */ 6923 void MacroAssembler::multiply_add_64(Register sum, Register op1, Register op2, Register carry, Register rdxReg, Register raxReg) { 6924 // rdx:rax = op1 * op2 6925 movq(raxReg, op2); 6926 mulq(op1); 6927 6928 // rdx:rax = sum + carry + rdx:rax 6929 addq(sum, carry); 6930 adcq(rdxReg, 0); 6931 addq(sum, raxReg); 6932 adcq(rdxReg, 0); 6933 6934 // carry:sum = rdx:sum 6935 movq(carry, rdxReg); 6936 } 6937 6938 /** 6939 * Add 64 bit long carry into z[] with carry propagation. 6940 * Preserves z and carry register values and modifies rest of registers. 6941 * 6942 */ 6943 void MacroAssembler::add_one_64(Register z, Register zlen, Register carry, Register tmp1) { 6944 Label L_fourth_loop, L_fourth_loop_exit; 6945 6946 movl(tmp1, 1); 6947 subl(zlen, 2); 6948 addq(Address(z, zlen, Address::times_4, 0), carry); 6949 6950 bind(L_fourth_loop); 6951 jccb(Assembler::carryClear, L_fourth_loop_exit); 6952 subl(zlen, 2); 6953 jccb(Assembler::negative, L_fourth_loop_exit); 6954 addq(Address(z, zlen, Address::times_4, 0), tmp1); 6955 jmp(L_fourth_loop); 6956 bind(L_fourth_loop_exit); 6957 } 6958 6959 /** 6960 * Shift z[] left by 1 bit. 6961 * Preserves x, len, z and zlen registers and modifies rest of the registers. 6962 * 6963 */ 6964 void MacroAssembler::lshift_by_1(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4) { 6965 6966 Label L_fifth_loop, L_fifth_loop_exit; 6967 6968 // Fifth loop 6969 // Perform primitiveLeftShift(z, zlen, 1) 6970 6971 const Register prev_carry = tmp1; 6972 const Register new_carry = tmp4; 6973 const Register value = tmp2; 6974 const Register zidx = tmp3; 6975 6976 // int zidx, carry; 6977 // long value; 6978 // carry = 0; 6979 // for (zidx = zlen-2; zidx >=0; zidx -= 2) { 6980 // (carry:value) = (z[i] << 1) | carry ; 6981 // z[i] = value; 6982 // } 6983 6984 movl(zidx, zlen); 6985 xorl(prev_carry, prev_carry); // clear carry flag and prev_carry register 6986 6987 bind(L_fifth_loop); 6988 decl(zidx); // Use decl to preserve carry flag 6989 decl(zidx); 6990 jccb(Assembler::negative, L_fifth_loop_exit); 6991 6992 if (UseBMI2Instructions) { 6993 movq(value, Address(z, zidx, Address::times_4, 0)); 6994 rclq(value, 1); 6995 rorxq(value, value, 32); 6996 movq(Address(z, zidx, Address::times_4, 0), value); // Store back in big endian form 6997 } 6998 else { 6999 // clear new_carry 7000 xorl(new_carry, new_carry); 7001 7002 // Shift z[i] by 1, or in previous carry and save new carry 7003 movq(value, Address(z, zidx, Address::times_4, 0)); 7004 shlq(value, 1); 7005 adcl(new_carry, 0); 7006 7007 orq(value, prev_carry); 7008 rorq(value, 0x20); 7009 movq(Address(z, zidx, Address::times_4, 0), value); // Store back in big endian form 7010 7011 // Set previous carry = new carry 7012 movl(prev_carry, new_carry); 7013 } 7014 jmp(L_fifth_loop); 7015 7016 bind(L_fifth_loop_exit); 7017 } 7018 7019 7020 /** 7021 * Code for BigInteger::squareToLen() intrinsic 7022 * 7023 * rdi: x 7024 * rsi: len 7025 * r8: z 7026 * rcx: zlen 7027 * r12: tmp1 7028 * r13: tmp2 7029 * r14: tmp3 7030 * r15: tmp4 7031 * rbx: tmp5 7032 * 7033 */ 7034 void MacroAssembler::square_to_len(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) { 7035 7036 Label L_second_loop, L_second_loop_exit, L_third_loop, L_third_loop_exit, L_last_x, L_multiply; 7037 push(tmp1); 7038 push(tmp2); 7039 push(tmp3); 7040 push(tmp4); 7041 push(tmp5); 7042 7043 // First loop 7044 // Store the squares, right shifted one bit (i.e., divided by 2). 7045 square_rshift(x, len, z, tmp1, tmp3, tmp4, tmp5, rdxReg, raxReg); 7046 7047 // Add in off-diagonal sums. 7048 // 7049 // Second, third (nested) and fourth loops. 7050 // zlen +=2; 7051 // for (int xidx=len-2,zidx=zlen-4; xidx > 0; xidx-=2,zidx-=4) { 7052 // carry = 0; 7053 // long op2 = x[xidx:xidx+1]; 7054 // for (int j=xidx-2,k=zidx; j >= 0; j-=2) { 7055 // k -= 2; 7056 // long op1 = x[j:j+1]; 7057 // long sum = z[k:k+1]; 7058 // carry:sum = multiply_add_64(sum, op1, op2, carry, tmp_regs); 7059 // z[k:k+1] = sum; 7060 // } 7061 // add_one_64(z, k, carry, tmp_regs); 7062 // } 7063 7064 const Register carry = tmp5; 7065 const Register sum = tmp3; 7066 const Register op1 = tmp4; 7067 Register op2 = tmp2; 7068 7069 push(zlen); 7070 push(len); 7071 addl(zlen,2); 7072 bind(L_second_loop); 7073 xorq(carry, carry); 7074 subl(zlen, 4); 7075 subl(len, 2); 7076 push(zlen); 7077 push(len); 7078 cmpl(len, 0); 7079 jccb(Assembler::lessEqual, L_second_loop_exit); 7080 7081 // Multiply an array by one 64 bit long. 7082 if (UseBMI2Instructions) { 7083 op2 = rdxReg; 7084 movq(op2, Address(x, len, Address::times_4, 0)); 7085 rorxq(op2, op2, 32); 7086 } 7087 else { 7088 movq(op2, Address(x, len, Address::times_4, 0)); 7089 rorq(op2, 32); 7090 } 7091 7092 bind(L_third_loop); 7093 decrementl(len); 7094 jccb(Assembler::negative, L_third_loop_exit); 7095 decrementl(len); 7096 jccb(Assembler::negative, L_last_x); 7097 7098 movq(op1, Address(x, len, Address::times_4, 0)); 7099 rorq(op1, 32); 7100 7101 bind(L_multiply); 7102 subl(zlen, 2); 7103 movq(sum, Address(z, zlen, Address::times_4, 0)); 7104 7105 // Multiply 64 bit by 64 bit and add 64 bits lower half and upper 64 bits as carry. 7106 if (UseBMI2Instructions) { 7107 multiply_add_64_bmi2(sum, op1, op2, carry, tmp2); 7108 } 7109 else { 7110 multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg); 7111 } 7112 7113 movq(Address(z, zlen, Address::times_4, 0), sum); 7114 7115 jmp(L_third_loop); 7116 bind(L_third_loop_exit); 7117 7118 // Fourth loop 7119 // Add 64 bit long carry into z with carry propagation. 7120 // Uses offsetted zlen. 7121 add_one_64(z, zlen, carry, tmp1); 7122 7123 pop(len); 7124 pop(zlen); 7125 jmp(L_second_loop); 7126 7127 // Next infrequent code is moved outside loops. 7128 bind(L_last_x); 7129 movl(op1, Address(x, 0)); 7130 jmp(L_multiply); 7131 7132 bind(L_second_loop_exit); 7133 pop(len); 7134 pop(zlen); 7135 pop(len); 7136 pop(zlen); 7137 7138 // Fifth loop 7139 // Shift z left 1 bit. 7140 lshift_by_1(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4); 7141 7142 // z[zlen-1] |= x[len-1] & 1; 7143 movl(tmp3, Address(x, len, Address::times_4, -4)); 7144 andl(tmp3, 1); 7145 orl(Address(z, zlen, Address::times_4, -4), tmp3); 7146 7147 pop(tmp5); 7148 pop(tmp4); 7149 pop(tmp3); 7150 pop(tmp2); 7151 pop(tmp1); 7152 } 7153 7154 /** 7155 * Helper function for mul_add() 7156 * Multiply the in[] by int k and add to out[] starting at offset offs using 7157 * 128 bit by 32 bit multiply and return the carry in tmp5. 7158 * Only quad int aligned length of in[] is operated on in this function. 7159 * k is in rdxReg for BMI2Instructions, for others it is in tmp2. 7160 * This function preserves out, in and k registers. 7161 * len and offset point to the appropriate index in "in" & "out" correspondingly 7162 * tmp5 has the carry. 7163 * other registers are temporary and are modified. 7164 * 7165 */ 7166 void MacroAssembler::mul_add_128_x_32_loop(Register out, Register in, 7167 Register offset, Register len, Register tmp1, Register tmp2, Register tmp3, 7168 Register tmp4, Register tmp5, Register rdxReg, Register raxReg) { 7169 7170 Label L_first_loop, L_first_loop_exit; 7171 7172 movl(tmp1, len); 7173 shrl(tmp1, 2); 7174 7175 bind(L_first_loop); 7176 subl(tmp1, 1); 7177 jccb(Assembler::negative, L_first_loop_exit); 7178 7179 subl(len, 4); 7180 subl(offset, 4); 7181 7182 Register op2 = tmp2; 7183 const Register sum = tmp3; 7184 const Register op1 = tmp4; 7185 const Register carry = tmp5; 7186 7187 if (UseBMI2Instructions) { 7188 op2 = rdxReg; 7189 } 7190 7191 movq(op1, Address(in, len, Address::times_4, 8)); 7192 rorq(op1, 32); 7193 movq(sum, Address(out, offset, Address::times_4, 8)); 7194 rorq(sum, 32); 7195 if (UseBMI2Instructions) { 7196 multiply_add_64_bmi2(sum, op1, op2, carry, raxReg); 7197 } 7198 else { 7199 multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg); 7200 } 7201 // Store back in big endian from little endian 7202 rorq(sum, 0x20); 7203 movq(Address(out, offset, Address::times_4, 8), sum); 7204 7205 movq(op1, Address(in, len, Address::times_4, 0)); 7206 rorq(op1, 32); 7207 movq(sum, Address(out, offset, Address::times_4, 0)); 7208 rorq(sum, 32); 7209 if (UseBMI2Instructions) { 7210 multiply_add_64_bmi2(sum, op1, op2, carry, raxReg); 7211 } 7212 else { 7213 multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg); 7214 } 7215 // Store back in big endian from little endian 7216 rorq(sum, 0x20); 7217 movq(Address(out, offset, Address::times_4, 0), sum); 7218 7219 jmp(L_first_loop); 7220 bind(L_first_loop_exit); 7221 } 7222 7223 /** 7224 * Code for BigInteger::mulAdd() intrinsic 7225 * 7226 * rdi: out 7227 * rsi: in 7228 * r11: offs (out.length - offset) 7229 * rcx: len 7230 * r8: k 7231 * r12: tmp1 7232 * r13: tmp2 7233 * r14: tmp3 7234 * r15: tmp4 7235 * rbx: tmp5 7236 * Multiply the in[] by word k and add to out[], return the carry in rax 7237 */ 7238 void MacroAssembler::mul_add(Register out, Register in, Register offs, 7239 Register len, Register k, Register tmp1, Register tmp2, Register tmp3, 7240 Register tmp4, Register tmp5, Register rdxReg, Register raxReg) { 7241 7242 Label L_carry, L_last_in, L_done; 7243 7244 // carry = 0; 7245 // for (int j=len-1; j >= 0; j--) { 7246 // long product = (in[j] & LONG_MASK) * kLong + 7247 // (out[offs] & LONG_MASK) + carry; 7248 // out[offs--] = (int)product; 7249 // carry = product >>> 32; 7250 // } 7251 // 7252 push(tmp1); 7253 push(tmp2); 7254 push(tmp3); 7255 push(tmp4); 7256 push(tmp5); 7257 7258 Register op2 = tmp2; 7259 const Register sum = tmp3; 7260 const Register op1 = tmp4; 7261 const Register carry = tmp5; 7262 7263 if (UseBMI2Instructions) { 7264 op2 = rdxReg; 7265 movl(op2, k); 7266 } 7267 else { 7268 movl(op2, k); 7269 } 7270 7271 xorq(carry, carry); 7272 7273 //First loop 7274 7275 //Multiply in[] by k in a 4 way unrolled loop using 128 bit by 32 bit multiply 7276 //The carry is in tmp5 7277 mul_add_128_x_32_loop(out, in, offs, len, tmp1, tmp2, tmp3, tmp4, tmp5, rdxReg, raxReg); 7278 7279 //Multiply the trailing in[] entry using 64 bit by 32 bit, if any 7280 decrementl(len); 7281 jccb(Assembler::negative, L_carry); 7282 decrementl(len); 7283 jccb(Assembler::negative, L_last_in); 7284 7285 movq(op1, Address(in, len, Address::times_4, 0)); 7286 rorq(op1, 32); 7287 7288 subl(offs, 2); 7289 movq(sum, Address(out, offs, Address::times_4, 0)); 7290 rorq(sum, 32); 7291 7292 if (UseBMI2Instructions) { 7293 multiply_add_64_bmi2(sum, op1, op2, carry, raxReg); 7294 } 7295 else { 7296 multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg); 7297 } 7298 7299 // Store back in big endian from little endian 7300 rorq(sum, 0x20); 7301 movq(Address(out, offs, Address::times_4, 0), sum); 7302 7303 testl(len, len); 7304 jccb(Assembler::zero, L_carry); 7305 7306 //Multiply the last in[] entry, if any 7307 bind(L_last_in); 7308 movl(op1, Address(in, 0)); 7309 movl(sum, Address(out, offs, Address::times_4, -4)); 7310 7311 movl(raxReg, k); 7312 mull(op1); //tmp4 * eax -> edx:eax 7313 addl(sum, carry); 7314 adcl(rdxReg, 0); 7315 addl(sum, raxReg); 7316 adcl(rdxReg, 0); 7317 movl(carry, rdxReg); 7318 7319 movl(Address(out, offs, Address::times_4, -4), sum); 7320 7321 bind(L_carry); 7322 //return tmp5/carry as carry in rax 7323 movl(rax, carry); 7324 7325 bind(L_done); 7326 pop(tmp5); 7327 pop(tmp4); 7328 pop(tmp3); 7329 pop(tmp2); 7330 pop(tmp1); 7331 } 7332 7333 /** 7334 * Emits code to update CRC-32 with a byte value according to constants in table 7335 * 7336 * @param [in,out]crc Register containing the crc. 7337 * @param [in]val Register containing the byte to fold into the CRC. 7338 * @param [in]table Register containing the table of crc constants. 7339 * 7340 * uint32_t crc; 7341 * val = crc_table[(val ^ crc) & 0xFF]; 7342 * crc = val ^ (crc >> 8); 7343 * 7344 */ 7345 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) { 7346 xorl(val, crc); 7347 andl(val, 0xFF); 7348 shrl(crc, 8); // unsigned shift 7349 xorl(crc, Address(table, val, Address::times_4, 0)); 7350 } 7351 7352 /** 7353 * Fold 128-bit data chunk 7354 */ 7355 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset) { 7356 if (UseAVX > 0) { 7357 vpclmulhdq(xtmp, xK, xcrc); // [123:64] 7358 vpclmulldq(xcrc, xK, xcrc); // [63:0] 7359 vpxor(xcrc, xcrc, Address(buf, offset), 0 /* vector_len */); 7360 pxor(xcrc, xtmp); 7361 } else { 7362 movdqa(xtmp, xcrc); 7363 pclmulhdq(xtmp, xK); // [123:64] 7364 pclmulldq(xcrc, xK); // [63:0] 7365 pxor(xcrc, xtmp); 7366 movdqu(xtmp, Address(buf, offset)); 7367 pxor(xcrc, xtmp); 7368 } 7369 } 7370 7371 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, XMMRegister xbuf) { 7372 if (UseAVX > 0) { 7373 vpclmulhdq(xtmp, xK, xcrc); 7374 vpclmulldq(xcrc, xK, xcrc); 7375 pxor(xcrc, xbuf); 7376 pxor(xcrc, xtmp); 7377 } else { 7378 movdqa(xtmp, xcrc); 7379 pclmulhdq(xtmp, xK); 7380 pclmulldq(xcrc, xK); 7381 pxor(xcrc, xbuf); 7382 pxor(xcrc, xtmp); 7383 } 7384 } 7385 7386 /** 7387 * 8-bit folds to compute 32-bit CRC 7388 * 7389 * uint64_t xcrc; 7390 * timesXtoThe32[xcrc & 0xFF] ^ (xcrc >> 8); 7391 */ 7392 void MacroAssembler::fold_8bit_crc32(XMMRegister xcrc, Register table, XMMRegister xtmp, Register tmp) { 7393 movdl(tmp, xcrc); 7394 andl(tmp, 0xFF); 7395 movdl(xtmp, Address(table, tmp, Address::times_4, 0)); 7396 psrldq(xcrc, 1); // unsigned shift one byte 7397 pxor(xcrc, xtmp); 7398 } 7399 7400 /** 7401 * uint32_t crc; 7402 * timesXtoThe32[crc & 0xFF] ^ (crc >> 8); 7403 */ 7404 void MacroAssembler::fold_8bit_crc32(Register crc, Register table, Register tmp) { 7405 movl(tmp, crc); 7406 andl(tmp, 0xFF); 7407 shrl(crc, 8); 7408 xorl(crc, Address(table, tmp, Address::times_4, 0)); 7409 } 7410 7411 /** 7412 * @param crc register containing existing CRC (32-bit) 7413 * @param buf register pointing to input byte buffer (byte*) 7414 * @param len register containing number of bytes 7415 * @param table register that will contain address of CRC table 7416 * @param tmp scratch register 7417 */ 7418 void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len, Register table, Register tmp) { 7419 assert_different_registers(crc, buf, len, table, tmp, rax); 7420 7421 Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned; 7422 Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop; 7423 7424 // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge 7425 // context for the registers used, where all instructions below are using 128-bit mode 7426 // On EVEX without VL and BW, these instructions will all be AVX. 7427 lea(table, ExternalAddress(StubRoutines::crc_table_addr())); 7428 notl(crc); // ~crc 7429 cmpl(len, 16); 7430 jcc(Assembler::less, L_tail); 7431 7432 // Align buffer to 16 bytes 7433 movl(tmp, buf); 7434 andl(tmp, 0xF); 7435 jccb(Assembler::zero, L_aligned); 7436 subl(tmp, 16); 7437 addl(len, tmp); 7438 7439 align(4); 7440 BIND(L_align_loop); 7441 movsbl(rax, Address(buf, 0)); // load byte with sign extension 7442 update_byte_crc32(crc, rax, table); 7443 increment(buf); 7444 incrementl(tmp); 7445 jccb(Assembler::less, L_align_loop); 7446 7447 BIND(L_aligned); 7448 movl(tmp, len); // save 7449 shrl(len, 4); 7450 jcc(Assembler::zero, L_tail_restore); 7451 7452 // Fold crc into first bytes of vector 7453 movdqa(xmm1, Address(buf, 0)); 7454 movdl(rax, xmm1); 7455 xorl(crc, rax); 7456 if (VM_Version::supports_sse4_1()) { 7457 pinsrd(xmm1, crc, 0); 7458 } else { 7459 pinsrw(xmm1, crc, 0); 7460 shrl(crc, 16); 7461 pinsrw(xmm1, crc, 1); 7462 } 7463 addptr(buf, 16); 7464 subl(len, 4); // len > 0 7465 jcc(Assembler::less, L_fold_tail); 7466 7467 movdqa(xmm2, Address(buf, 0)); 7468 movdqa(xmm3, Address(buf, 16)); 7469 movdqa(xmm4, Address(buf, 32)); 7470 addptr(buf, 48); 7471 subl(len, 3); 7472 jcc(Assembler::lessEqual, L_fold_512b); 7473 7474 // Fold total 512 bits of polynomial on each iteration, 7475 // 128 bits per each of 4 parallel streams. 7476 movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 32), rscratch1); 7477 7478 align32(); 7479 BIND(L_fold_512b_loop); 7480 fold_128bit_crc32(xmm1, xmm0, xmm5, buf, 0); 7481 fold_128bit_crc32(xmm2, xmm0, xmm5, buf, 16); 7482 fold_128bit_crc32(xmm3, xmm0, xmm5, buf, 32); 7483 fold_128bit_crc32(xmm4, xmm0, xmm5, buf, 48); 7484 addptr(buf, 64); 7485 subl(len, 4); 7486 jcc(Assembler::greater, L_fold_512b_loop); 7487 7488 // Fold 512 bits to 128 bits. 7489 BIND(L_fold_512b); 7490 movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16), rscratch1); 7491 fold_128bit_crc32(xmm1, xmm0, xmm5, xmm2); 7492 fold_128bit_crc32(xmm1, xmm0, xmm5, xmm3); 7493 fold_128bit_crc32(xmm1, xmm0, xmm5, xmm4); 7494 7495 // Fold the rest of 128 bits data chunks 7496 BIND(L_fold_tail); 7497 addl(len, 3); 7498 jccb(Assembler::lessEqual, L_fold_128b); 7499 movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16), rscratch1); 7500 7501 BIND(L_fold_tail_loop); 7502 fold_128bit_crc32(xmm1, xmm0, xmm5, buf, 0); 7503 addptr(buf, 16); 7504 decrementl(len); 7505 jccb(Assembler::greater, L_fold_tail_loop); 7506 7507 // Fold 128 bits in xmm1 down into 32 bits in crc register. 7508 BIND(L_fold_128b); 7509 movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr()), rscratch1); 7510 if (UseAVX > 0) { 7511 vpclmulqdq(xmm2, xmm0, xmm1, 0x1); 7512 vpand(xmm3, xmm0, xmm2, 0 /* vector_len */); 7513 vpclmulqdq(xmm0, xmm0, xmm3, 0x1); 7514 } else { 7515 movdqa(xmm2, xmm0); 7516 pclmulqdq(xmm2, xmm1, 0x1); 7517 movdqa(xmm3, xmm0); 7518 pand(xmm3, xmm2); 7519 pclmulqdq(xmm0, xmm3, 0x1); 7520 } 7521 psrldq(xmm1, 8); 7522 psrldq(xmm2, 4); 7523 pxor(xmm0, xmm1); 7524 pxor(xmm0, xmm2); 7525 7526 // 8 8-bit folds to compute 32-bit CRC. 7527 for (int j = 0; j < 4; j++) { 7528 fold_8bit_crc32(xmm0, table, xmm1, rax); 7529 } 7530 movdl(crc, xmm0); // mov 32 bits to general register 7531 for (int j = 0; j < 4; j++) { 7532 fold_8bit_crc32(crc, table, rax); 7533 } 7534 7535 BIND(L_tail_restore); 7536 movl(len, tmp); // restore 7537 BIND(L_tail); 7538 andl(len, 0xf); 7539 jccb(Assembler::zero, L_exit); 7540 7541 // Fold the rest of bytes 7542 align(4); 7543 BIND(L_tail_loop); 7544 movsbl(rax, Address(buf, 0)); // load byte with sign extension 7545 update_byte_crc32(crc, rax, table); 7546 increment(buf); 7547 decrementl(len); 7548 jccb(Assembler::greater, L_tail_loop); 7549 7550 BIND(L_exit); 7551 notl(crc); // ~c 7552 } 7553 7554 // Helper function for AVX 512 CRC32 7555 // Fold 512-bit data chunks 7556 void MacroAssembler::fold512bit_crc32_avx512(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, 7557 Register pos, int offset) { 7558 evmovdquq(xmm3, Address(buf, pos, Address::times_1, offset), Assembler::AVX_512bit); 7559 evpclmulqdq(xtmp, xcrc, xK, 0x10, Assembler::AVX_512bit); // [123:64] 7560 evpclmulqdq(xmm2, xcrc, xK, 0x01, Assembler::AVX_512bit); // [63:0] 7561 evpxorq(xcrc, xtmp, xmm2, Assembler::AVX_512bit /* vector_len */); 7562 evpxorq(xcrc, xcrc, xmm3, Assembler::AVX_512bit /* vector_len */); 7563 } 7564 7565 // Helper function for AVX 512 CRC32 7566 // Compute CRC32 for < 256B buffers 7567 void MacroAssembler::kernel_crc32_avx512_256B(Register crc, Register buf, Register len, Register table, Register pos, 7568 Register tmp1, Register tmp2, Label& L_barrett, Label& L_16B_reduction_loop, 7569 Label& L_get_last_two_xmms, Label& L_128_done, Label& L_cleanup) { 7570 7571 Label L_less_than_32, L_exact_16_left, L_less_than_16_left; 7572 Label L_less_than_8_left, L_less_than_4_left, L_less_than_2_left, L_zero_left; 7573 Label L_only_less_than_4, L_only_less_than_3, L_only_less_than_2; 7574 7575 // check if there is enough buffer to be able to fold 16B at a time 7576 cmpl(len, 32); 7577 jcc(Assembler::less, L_less_than_32); 7578 7579 // if there is, load the constants 7580 movdqu(xmm10, Address(table, 1 * 16)); //rk1 and rk2 in xmm10 7581 movdl(xmm0, crc); // get the initial crc value 7582 movdqu(xmm7, Address(buf, pos, Address::times_1, 0 * 16)); //load the plaintext 7583 pxor(xmm7, xmm0); 7584 7585 // update the buffer pointer 7586 addl(pos, 16); 7587 //update the counter.subtract 32 instead of 16 to save one instruction from the loop 7588 subl(len, 32); 7589 jmp(L_16B_reduction_loop); 7590 7591 bind(L_less_than_32); 7592 //mov initial crc to the return value. this is necessary for zero - length buffers. 7593 movl(rax, crc); 7594 testl(len, len); 7595 jcc(Assembler::equal, L_cleanup); 7596 7597 movdl(xmm0, crc); //get the initial crc value 7598 7599 cmpl(len, 16); 7600 jcc(Assembler::equal, L_exact_16_left); 7601 jcc(Assembler::less, L_less_than_16_left); 7602 7603 movdqu(xmm7, Address(buf, pos, Address::times_1, 0 * 16)); //load the plaintext 7604 pxor(xmm7, xmm0); //xor the initial crc value 7605 addl(pos, 16); 7606 subl(len, 16); 7607 movdqu(xmm10, Address(table, 1 * 16)); // rk1 and rk2 in xmm10 7608 jmp(L_get_last_two_xmms); 7609 7610 bind(L_less_than_16_left); 7611 //use stack space to load data less than 16 bytes, zero - out the 16B in memory first. 7612 pxor(xmm1, xmm1); 7613 movptr(tmp1, rsp); 7614 movdqu(Address(tmp1, 0 * 16), xmm1); 7615 7616 cmpl(len, 4); 7617 jcc(Assembler::less, L_only_less_than_4); 7618 7619 //backup the counter value 7620 movl(tmp2, len); 7621 cmpl(len, 8); 7622 jcc(Assembler::less, L_less_than_8_left); 7623 7624 //load 8 Bytes 7625 movq(rax, Address(buf, pos, Address::times_1, 0 * 16)); 7626 movq(Address(tmp1, 0 * 16), rax); 7627 addptr(tmp1, 8); 7628 subl(len, 8); 7629 addl(pos, 8); 7630 7631 bind(L_less_than_8_left); 7632 cmpl(len, 4); 7633 jcc(Assembler::less, L_less_than_4_left); 7634 7635 //load 4 Bytes 7636 movl(rax, Address(buf, pos, Address::times_1, 0)); 7637 movl(Address(tmp1, 0 * 16), rax); 7638 addptr(tmp1, 4); 7639 subl(len, 4); 7640 addl(pos, 4); 7641 7642 bind(L_less_than_4_left); 7643 cmpl(len, 2); 7644 jcc(Assembler::less, L_less_than_2_left); 7645 7646 // load 2 Bytes 7647 movw(rax, Address(buf, pos, Address::times_1, 0)); 7648 movl(Address(tmp1, 0 * 16), rax); 7649 addptr(tmp1, 2); 7650 subl(len, 2); 7651 addl(pos, 2); 7652 7653 bind(L_less_than_2_left); 7654 cmpl(len, 1); 7655 jcc(Assembler::less, L_zero_left); 7656 7657 // load 1 Byte 7658 movb(rax, Address(buf, pos, Address::times_1, 0)); 7659 movb(Address(tmp1, 0 * 16), rax); 7660 7661 bind(L_zero_left); 7662 movdqu(xmm7, Address(rsp, 0)); 7663 pxor(xmm7, xmm0); //xor the initial crc value 7664 7665 lea(rax, ExternalAddress(StubRoutines::x86::shuf_table_crc32_avx512_addr())); 7666 movdqu(xmm0, Address(rax, tmp2)); 7667 pshufb(xmm7, xmm0); 7668 jmp(L_128_done); 7669 7670 bind(L_exact_16_left); 7671 movdqu(xmm7, Address(buf, pos, Address::times_1, 0)); 7672 pxor(xmm7, xmm0); //xor the initial crc value 7673 jmp(L_128_done); 7674 7675 bind(L_only_less_than_4); 7676 cmpl(len, 3); 7677 jcc(Assembler::less, L_only_less_than_3); 7678 7679 // load 3 Bytes 7680 movb(rax, Address(buf, pos, Address::times_1, 0)); 7681 movb(Address(tmp1, 0), rax); 7682 7683 movb(rax, Address(buf, pos, Address::times_1, 1)); 7684 movb(Address(tmp1, 1), rax); 7685 7686 movb(rax, Address(buf, pos, Address::times_1, 2)); 7687 movb(Address(tmp1, 2), rax); 7688 7689 movdqu(xmm7, Address(rsp, 0)); 7690 pxor(xmm7, xmm0); //xor the initial crc value 7691 7692 pslldq(xmm7, 0x5); 7693 jmp(L_barrett); 7694 bind(L_only_less_than_3); 7695 cmpl(len, 2); 7696 jcc(Assembler::less, L_only_less_than_2); 7697 7698 // load 2 Bytes 7699 movb(rax, Address(buf, pos, Address::times_1, 0)); 7700 movb(Address(tmp1, 0), rax); 7701 7702 movb(rax, Address(buf, pos, Address::times_1, 1)); 7703 movb(Address(tmp1, 1), rax); 7704 7705 movdqu(xmm7, Address(rsp, 0)); 7706 pxor(xmm7, xmm0); //xor the initial crc value 7707 7708 pslldq(xmm7, 0x6); 7709 jmp(L_barrett); 7710 7711 bind(L_only_less_than_2); 7712 //load 1 Byte 7713 movb(rax, Address(buf, pos, Address::times_1, 0)); 7714 movb(Address(tmp1, 0), rax); 7715 7716 movdqu(xmm7, Address(rsp, 0)); 7717 pxor(xmm7, xmm0); //xor the initial crc value 7718 7719 pslldq(xmm7, 0x7); 7720 } 7721 7722 /** 7723 * Compute CRC32 using AVX512 instructions 7724 * param crc register containing existing CRC (32-bit) 7725 * param buf register pointing to input byte buffer (byte*) 7726 * param len register containing number of bytes 7727 * param table address of crc or crc32c table 7728 * param tmp1 scratch register 7729 * param tmp2 scratch register 7730 * return rax result register 7731 * 7732 * This routine is identical for crc32c with the exception of the precomputed constant 7733 * table which will be passed as the table argument. The calculation steps are 7734 * the same for both variants. 7735 */ 7736 void MacroAssembler::kernel_crc32_avx512(Register crc, Register buf, Register len, Register table, Register tmp1, Register tmp2) { 7737 assert_different_registers(crc, buf, len, table, tmp1, tmp2, rax, r12); 7738 7739 Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned; 7740 Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop; 7741 Label L_less_than_256, L_fold_128_B_loop, L_fold_256_B_loop; 7742 Label L_fold_128_B_register, L_final_reduction_for_128, L_16B_reduction_loop; 7743 Label L_128_done, L_get_last_two_xmms, L_barrett, L_cleanup; 7744 7745 const Register pos = r12; 7746 push(r12); 7747 subptr(rsp, 16 * 2 + 8); 7748 7749 // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge 7750 // context for the registers used, where all instructions below are using 128-bit mode 7751 // On EVEX without VL and BW, these instructions will all be AVX. 7752 movl(pos, 0); 7753 7754 // check if smaller than 256B 7755 cmpl(len, 256); 7756 jcc(Assembler::less, L_less_than_256); 7757 7758 // load the initial crc value 7759 movdl(xmm10, crc); 7760 7761 // receive the initial 64B data, xor the initial crc value 7762 evmovdquq(xmm0, Address(buf, pos, Address::times_1, 0 * 64), Assembler::AVX_512bit); 7763 evmovdquq(xmm4, Address(buf, pos, Address::times_1, 1 * 64), Assembler::AVX_512bit); 7764 evpxorq(xmm0, xmm0, xmm10, Assembler::AVX_512bit); 7765 evbroadcasti32x4(xmm10, Address(table, 2 * 16), Assembler::AVX_512bit); //zmm10 has rk3 and rk4 7766 7767 subl(len, 256); 7768 cmpl(len, 256); 7769 jcc(Assembler::less, L_fold_128_B_loop); 7770 7771 evmovdquq(xmm7, Address(buf, pos, Address::times_1, 2 * 64), Assembler::AVX_512bit); 7772 evmovdquq(xmm8, Address(buf, pos, Address::times_1, 3 * 64), Assembler::AVX_512bit); 7773 evbroadcasti32x4(xmm16, Address(table, 0 * 16), Assembler::AVX_512bit); //zmm16 has rk-1 and rk-2 7774 subl(len, 256); 7775 7776 bind(L_fold_256_B_loop); 7777 addl(pos, 256); 7778 fold512bit_crc32_avx512(xmm0, xmm16, xmm1, buf, pos, 0 * 64); 7779 fold512bit_crc32_avx512(xmm4, xmm16, xmm1, buf, pos, 1 * 64); 7780 fold512bit_crc32_avx512(xmm7, xmm16, xmm1, buf, pos, 2 * 64); 7781 fold512bit_crc32_avx512(xmm8, xmm16, xmm1, buf, pos, 3 * 64); 7782 7783 subl(len, 256); 7784 jcc(Assembler::greaterEqual, L_fold_256_B_loop); 7785 7786 // Fold 256 into 128 7787 addl(pos, 256); 7788 evpclmulqdq(xmm1, xmm0, xmm10, 0x01, Assembler::AVX_512bit); 7789 evpclmulqdq(xmm2, xmm0, xmm10, 0x10, Assembler::AVX_512bit); 7790 vpternlogq(xmm7, 0x96, xmm1, xmm2, Assembler::AVX_512bit); // xor ABC 7791 7792 evpclmulqdq(xmm5, xmm4, xmm10, 0x01, Assembler::AVX_512bit); 7793 evpclmulqdq(xmm6, xmm4, xmm10, 0x10, Assembler::AVX_512bit); 7794 vpternlogq(xmm8, 0x96, xmm5, xmm6, Assembler::AVX_512bit); // xor ABC 7795 7796 evmovdquq(xmm0, xmm7, Assembler::AVX_512bit); 7797 evmovdquq(xmm4, xmm8, Assembler::AVX_512bit); 7798 7799 addl(len, 128); 7800 jmp(L_fold_128_B_register); 7801 7802 // at this section of the code, there is 128 * x + y(0 <= y<128) bytes of buffer.The fold_128_B_loop 7803 // loop will fold 128B at a time until we have 128 + y Bytes of buffer 7804 7805 // fold 128B at a time.This section of the code folds 8 xmm registers in parallel 7806 bind(L_fold_128_B_loop); 7807 addl(pos, 128); 7808 fold512bit_crc32_avx512(xmm0, xmm10, xmm1, buf, pos, 0 * 64); 7809 fold512bit_crc32_avx512(xmm4, xmm10, xmm1, buf, pos, 1 * 64); 7810 7811 subl(len, 128); 7812 jcc(Assembler::greaterEqual, L_fold_128_B_loop); 7813 7814 addl(pos, 128); 7815 7816 // at this point, the buffer pointer is pointing at the last y Bytes of the buffer, where 0 <= y < 128 7817 // the 128B of folded data is in 8 of the xmm registers : xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 7818 bind(L_fold_128_B_register); 7819 evmovdquq(xmm16, Address(table, 5 * 16), Assembler::AVX_512bit); // multiply by rk9-rk16 7820 evmovdquq(xmm11, Address(table, 9 * 16), Assembler::AVX_512bit); // multiply by rk17-rk20, rk1,rk2, 0,0 7821 evpclmulqdq(xmm1, xmm0, xmm16, 0x01, Assembler::AVX_512bit); 7822 evpclmulqdq(xmm2, xmm0, xmm16, 0x10, Assembler::AVX_512bit); 7823 // save last that has no multiplicand 7824 vextracti64x2(xmm7, xmm4, 3); 7825 7826 evpclmulqdq(xmm5, xmm4, xmm11, 0x01, Assembler::AVX_512bit); 7827 evpclmulqdq(xmm6, xmm4, xmm11, 0x10, Assembler::AVX_512bit); 7828 // Needed later in reduction loop 7829 movdqu(xmm10, Address(table, 1 * 16)); 7830 vpternlogq(xmm1, 0x96, xmm2, xmm5, Assembler::AVX_512bit); // xor ABC 7831 vpternlogq(xmm1, 0x96, xmm6, xmm7, Assembler::AVX_512bit); // xor ABC 7832 7833 // Swap 1,0,3,2 - 01 00 11 10 7834 evshufi64x2(xmm8, xmm1, xmm1, 0x4e, Assembler::AVX_512bit); 7835 evpxorq(xmm8, xmm8, xmm1, Assembler::AVX_256bit); 7836 vextracti128(xmm5, xmm8, 1); 7837 evpxorq(xmm7, xmm5, xmm8, Assembler::AVX_128bit); 7838 7839 // instead of 128, we add 128 - 16 to the loop counter to save 1 instruction from the loop 7840 // instead of a cmp instruction, we use the negative flag with the jl instruction 7841 addl(len, 128 - 16); 7842 jcc(Assembler::less, L_final_reduction_for_128); 7843 7844 bind(L_16B_reduction_loop); 7845 vpclmulqdq(xmm8, xmm7, xmm10, 0x01); 7846 vpclmulqdq(xmm7, xmm7, xmm10, 0x10); 7847 vpxor(xmm7, xmm7, xmm8, Assembler::AVX_128bit); 7848 movdqu(xmm0, Address(buf, pos, Address::times_1, 0 * 16)); 7849 vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit); 7850 addl(pos, 16); 7851 subl(len, 16); 7852 jcc(Assembler::greaterEqual, L_16B_reduction_loop); 7853 7854 bind(L_final_reduction_for_128); 7855 addl(len, 16); 7856 jcc(Assembler::equal, L_128_done); 7857 7858 bind(L_get_last_two_xmms); 7859 movdqu(xmm2, xmm7); 7860 addl(pos, len); 7861 movdqu(xmm1, Address(buf, pos, Address::times_1, -16)); 7862 subl(pos, len); 7863 7864 // get rid of the extra data that was loaded before 7865 // load the shift constant 7866 lea(rax, ExternalAddress(StubRoutines::x86::shuf_table_crc32_avx512_addr())); 7867 movdqu(xmm0, Address(rax, len)); 7868 addl(rax, len); 7869 7870 vpshufb(xmm7, xmm7, xmm0, Assembler::AVX_128bit); 7871 //Change mask to 512 7872 vpxor(xmm0, xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr() + 2 * 16), Assembler::AVX_128bit, tmp2); 7873 vpshufb(xmm2, xmm2, xmm0, Assembler::AVX_128bit); 7874 7875 blendvpb(xmm2, xmm2, xmm1, xmm0, Assembler::AVX_128bit); 7876 vpclmulqdq(xmm8, xmm7, xmm10, 0x01); 7877 vpclmulqdq(xmm7, xmm7, xmm10, 0x10); 7878 vpxor(xmm7, xmm7, xmm8, Assembler::AVX_128bit); 7879 vpxor(xmm7, xmm7, xmm2, Assembler::AVX_128bit); 7880 7881 bind(L_128_done); 7882 // compute crc of a 128-bit value 7883 movdqu(xmm10, Address(table, 3 * 16)); 7884 movdqu(xmm0, xmm7); 7885 7886 // 64b fold 7887 vpclmulqdq(xmm7, xmm7, xmm10, 0x0); 7888 vpsrldq(xmm0, xmm0, 0x8, Assembler::AVX_128bit); 7889 vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit); 7890 7891 // 32b fold 7892 movdqu(xmm0, xmm7); 7893 vpslldq(xmm7, xmm7, 0x4, Assembler::AVX_128bit); 7894 vpclmulqdq(xmm7, xmm7, xmm10, 0x10); 7895 vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit); 7896 jmp(L_barrett); 7897 7898 bind(L_less_than_256); 7899 kernel_crc32_avx512_256B(crc, buf, len, table, pos, tmp1, tmp2, L_barrett, L_16B_reduction_loop, L_get_last_two_xmms, L_128_done, L_cleanup); 7900 7901 //barrett reduction 7902 bind(L_barrett); 7903 vpand(xmm7, xmm7, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr() + 1 * 16), Assembler::AVX_128bit, tmp2); 7904 movdqu(xmm1, xmm7); 7905 movdqu(xmm2, xmm7); 7906 movdqu(xmm10, Address(table, 4 * 16)); 7907 7908 pclmulqdq(xmm7, xmm10, 0x0); 7909 pxor(xmm7, xmm2); 7910 vpand(xmm7, xmm7, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr()), Assembler::AVX_128bit, tmp2); 7911 movdqu(xmm2, xmm7); 7912 pclmulqdq(xmm7, xmm10, 0x10); 7913 pxor(xmm7, xmm2); 7914 pxor(xmm7, xmm1); 7915 pextrd(crc, xmm7, 2); 7916 7917 bind(L_cleanup); 7918 addptr(rsp, 16 * 2 + 8); 7919 pop(r12); 7920 } 7921 7922 // S. Gueron / Information Processing Letters 112 (2012) 184 7923 // Algorithm 4: Computing carry-less multiplication using a precomputed lookup table. 7924 // Input: A 32 bit value B = [byte3, byte2, byte1, byte0]. 7925 // Output: the 64-bit carry-less product of B * CONST 7926 void MacroAssembler::crc32c_ipl_alg4(Register in, uint32_t n, 7927 Register tmp1, Register tmp2, Register tmp3) { 7928 lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr())); 7929 if (n > 0) { 7930 addq(tmp3, n * 256 * 8); 7931 } 7932 // Q1 = TABLEExt[n][B & 0xFF]; 7933 movl(tmp1, in); 7934 andl(tmp1, 0x000000FF); 7935 shll(tmp1, 3); 7936 addq(tmp1, tmp3); 7937 movq(tmp1, Address(tmp1, 0)); 7938 7939 // Q2 = TABLEExt[n][B >> 8 & 0xFF]; 7940 movl(tmp2, in); 7941 shrl(tmp2, 8); 7942 andl(tmp2, 0x000000FF); 7943 shll(tmp2, 3); 7944 addq(tmp2, tmp3); 7945 movq(tmp2, Address(tmp2, 0)); 7946 7947 shlq(tmp2, 8); 7948 xorq(tmp1, tmp2); 7949 7950 // Q3 = TABLEExt[n][B >> 16 & 0xFF]; 7951 movl(tmp2, in); 7952 shrl(tmp2, 16); 7953 andl(tmp2, 0x000000FF); 7954 shll(tmp2, 3); 7955 addq(tmp2, tmp3); 7956 movq(tmp2, Address(tmp2, 0)); 7957 7958 shlq(tmp2, 16); 7959 xorq(tmp1, tmp2); 7960 7961 // Q4 = TABLEExt[n][B >> 24 & 0xFF]; 7962 shrl(in, 24); 7963 andl(in, 0x000000FF); 7964 shll(in, 3); 7965 addq(in, tmp3); 7966 movq(in, Address(in, 0)); 7967 7968 shlq(in, 24); 7969 xorq(in, tmp1); 7970 // return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24; 7971 } 7972 7973 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1, 7974 Register in_out, 7975 uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported, 7976 XMMRegister w_xtmp2, 7977 Register tmp1, 7978 Register n_tmp2, Register n_tmp3) { 7979 if (is_pclmulqdq_supported) { 7980 movdl(w_xtmp1, in_out); // modified blindly 7981 7982 movl(tmp1, const_or_pre_comp_const_index); 7983 movdl(w_xtmp2, tmp1); 7984 pclmulqdq(w_xtmp1, w_xtmp2, 0); 7985 7986 movdq(in_out, w_xtmp1); 7987 } else { 7988 crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3); 7989 } 7990 } 7991 7992 // Recombination Alternative 2: No bit-reflections 7993 // T1 = (CRC_A * U1) << 1 7994 // T2 = (CRC_B * U2) << 1 7995 // C1 = T1 >> 32 7996 // C2 = T2 >> 32 7997 // T1 = T1 & 0xFFFFFFFF 7998 // T2 = T2 & 0xFFFFFFFF 7999 // T1 = CRC32(0, T1) 8000 // T2 = CRC32(0, T2) 8001 // C1 = C1 ^ T1 8002 // C2 = C2 ^ T2 8003 // CRC = C1 ^ C2 ^ CRC_C 8004 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2, 8005 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, 8006 Register tmp1, Register tmp2, 8007 Register n_tmp3) { 8008 crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3); 8009 crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3); 8010 shlq(in_out, 1); 8011 movl(tmp1, in_out); 8012 shrq(in_out, 32); 8013 xorl(tmp2, tmp2); 8014 crc32(tmp2, tmp1, 4); 8015 xorl(in_out, tmp2); // we don't care about upper 32 bit contents here 8016 shlq(in1, 1); 8017 movl(tmp1, in1); 8018 shrq(in1, 32); 8019 xorl(tmp2, tmp2); 8020 crc32(tmp2, tmp1, 4); 8021 xorl(in1, tmp2); 8022 xorl(in_out, in1); 8023 xorl(in_out, in2); 8024 } 8025 8026 // Set N to predefined value 8027 // Subtract from a length of a buffer 8028 // execute in a loop: 8029 // CRC_A = 0xFFFFFFFF, CRC_B = 0, CRC_C = 0 8030 // for i = 1 to N do 8031 // CRC_A = CRC32(CRC_A, A[i]) 8032 // CRC_B = CRC32(CRC_B, B[i]) 8033 // CRC_C = CRC32(CRC_C, C[i]) 8034 // end for 8035 // Recombine 8036 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, 8037 Register in_out1, Register in_out2, Register in_out3, 8038 Register tmp1, Register tmp2, Register tmp3, 8039 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, 8040 Register tmp4, Register tmp5, 8041 Register n_tmp6) { 8042 Label L_processPartitions; 8043 Label L_processPartition; 8044 Label L_exit; 8045 8046 bind(L_processPartitions); 8047 cmpl(in_out1, 3 * size); 8048 jcc(Assembler::less, L_exit); 8049 xorl(tmp1, tmp1); 8050 xorl(tmp2, tmp2); 8051 movq(tmp3, in_out2); 8052 addq(tmp3, size); 8053 8054 bind(L_processPartition); 8055 crc32(in_out3, Address(in_out2, 0), 8); 8056 crc32(tmp1, Address(in_out2, size), 8); 8057 crc32(tmp2, Address(in_out2, size * 2), 8); 8058 addq(in_out2, 8); 8059 cmpq(in_out2, tmp3); 8060 jcc(Assembler::less, L_processPartition); 8061 crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2, 8062 w_xtmp1, w_xtmp2, w_xtmp3, 8063 tmp4, tmp5, 8064 n_tmp6); 8065 addq(in_out2, 2 * size); 8066 subl(in_out1, 3 * size); 8067 jmp(L_processPartitions); 8068 8069 bind(L_exit); 8070 } 8071 8072 // Algorithm 2: Pipelined usage of the CRC32 instruction. 8073 // Input: A buffer I of L bytes. 8074 // Output: the CRC32C value of the buffer. 8075 // Notations: 8076 // Write L = 24N + r, with N = floor (L/24). 8077 // r = L mod 24 (0 <= r < 24). 8078 // Consider I as the concatenation of A|B|C|R, where A, B, C, each, 8079 // N quadwords, and R consists of r bytes. 8080 // A[j] = I [8j+7:8j], j= 0, 1, ..., N-1 8081 // B[j] = I [N + 8j+7:N + 8j], j= 0, 1, ..., N-1 8082 // C[j] = I [2N + 8j+7:2N + 8j], j= 0, 1, ..., N-1 8083 // if r > 0 R[j] = I [3N +j], j= 0, 1, ...,r-1 8084 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2, 8085 Register tmp1, Register tmp2, Register tmp3, 8086 Register tmp4, Register tmp5, Register tmp6, 8087 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, 8088 bool is_pclmulqdq_supported) { 8089 uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS]; 8090 Label L_wordByWord; 8091 Label L_byteByByteProlog; 8092 Label L_byteByByte; 8093 Label L_exit; 8094 8095 if (is_pclmulqdq_supported ) { 8096 const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::crc32c_table_addr(); 8097 const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 1); 8098 8099 const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 2); 8100 const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 3); 8101 8102 const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 4); 8103 const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 5); 8104 assert((CRC32C_NUM_PRECOMPUTED_CONSTANTS - 1 ) == 5, "Checking whether you declared all of the constants based on the number of \"chunks\""); 8105 } else { 8106 const_or_pre_comp_const_index[0] = 1; 8107 const_or_pre_comp_const_index[1] = 0; 8108 8109 const_or_pre_comp_const_index[2] = 3; 8110 const_or_pre_comp_const_index[3] = 2; 8111 8112 const_or_pre_comp_const_index[4] = 5; 8113 const_or_pre_comp_const_index[5] = 4; 8114 } 8115 crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported, 8116 in2, in1, in_out, 8117 tmp1, tmp2, tmp3, 8118 w_xtmp1, w_xtmp2, w_xtmp3, 8119 tmp4, tmp5, 8120 tmp6); 8121 crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported, 8122 in2, in1, in_out, 8123 tmp1, tmp2, tmp3, 8124 w_xtmp1, w_xtmp2, w_xtmp3, 8125 tmp4, tmp5, 8126 tmp6); 8127 crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported, 8128 in2, in1, in_out, 8129 tmp1, tmp2, tmp3, 8130 w_xtmp1, w_xtmp2, w_xtmp3, 8131 tmp4, tmp5, 8132 tmp6); 8133 movl(tmp1, in2); 8134 andl(tmp1, 0x00000007); 8135 negl(tmp1); 8136 addl(tmp1, in2); 8137 addq(tmp1, in1); 8138 8139 cmpq(in1, tmp1); 8140 jccb(Assembler::greaterEqual, L_byteByByteProlog); 8141 align(16); 8142 BIND(L_wordByWord); 8143 crc32(in_out, Address(in1, 0), 8); 8144 addq(in1, 8); 8145 cmpq(in1, tmp1); 8146 jcc(Assembler::less, L_wordByWord); 8147 8148 BIND(L_byteByByteProlog); 8149 andl(in2, 0x00000007); 8150 movl(tmp2, 1); 8151 8152 cmpl(tmp2, in2); 8153 jccb(Assembler::greater, L_exit); 8154 BIND(L_byteByByte); 8155 crc32(in_out, Address(in1, 0), 1); 8156 incq(in1); 8157 incl(tmp2); 8158 cmpl(tmp2, in2); 8159 jcc(Assembler::lessEqual, L_byteByByte); 8160 8161 BIND(L_exit); 8162 } 8163 #undef BIND 8164 #undef BLOCK_COMMENT 8165 8166 // Compress char[] array to byte[]. 8167 // Intrinsic for java.lang.StringUTF16.compress(char[] src, int srcOff, byte[] dst, int dstOff, int len) 8168 // Return the array length if every element in array can be encoded, 8169 // otherwise, the index of first non-latin1 (> 0xff) character. 8170 // @IntrinsicCandidate 8171 // public static int compress(char[] src, int srcOff, byte[] dst, int dstOff, int len) { 8172 // for (int i = 0; i < len; i++) { 8173 // char c = src[srcOff]; 8174 // if (c > 0xff) { 8175 // return i; // return index of non-latin1 char 8176 // } 8177 // dst[dstOff] = (byte)c; 8178 // srcOff++; 8179 // dstOff++; 8180 // } 8181 // return len; 8182 // } 8183 void MacroAssembler::char_array_compress(Register src, Register dst, Register len, 8184 XMMRegister tmp1Reg, XMMRegister tmp2Reg, 8185 XMMRegister tmp3Reg, XMMRegister tmp4Reg, 8186 Register tmp5, Register result, KRegister mask1, KRegister mask2) { 8187 Label copy_chars_loop, done, reset_sp, copy_tail; 8188 8189 // rsi: src 8190 // rdi: dst 8191 // rdx: len 8192 // rcx: tmp5 8193 // rax: result 8194 8195 // rsi holds start addr of source char[] to be compressed 8196 // rdi holds start addr of destination byte[] 8197 // rdx holds length 8198 8199 assert(len != result, ""); 8200 8201 // save length for return 8202 movl(result, len); 8203 8204 if ((AVX3Threshold == 0) && (UseAVX > 2) && // AVX512 8205 VM_Version::supports_avx512vlbw() && 8206 VM_Version::supports_bmi2()) { 8207 8208 Label copy_32_loop, copy_loop_tail, below_threshold, reset_for_copy_tail; 8209 8210 // alignment 8211 Label post_alignment; 8212 8213 // if length of the string is less than 32, handle it the old fashioned way 8214 testl(len, -32); 8215 jcc(Assembler::zero, below_threshold); 8216 8217 // First check whether a character is compressible ( <= 0xFF). 8218 // Create mask to test for Unicode chars inside zmm vector 8219 movl(tmp5, 0x00FF); 8220 evpbroadcastw(tmp2Reg, tmp5, Assembler::AVX_512bit); 8221 8222 testl(len, -64); 8223 jccb(Assembler::zero, post_alignment); 8224 8225 movl(tmp5, dst); 8226 andl(tmp5, (32 - 1)); 8227 negl(tmp5); 8228 andl(tmp5, (32 - 1)); 8229 8230 // bail out when there is nothing to be done 8231 testl(tmp5, 0xFFFFFFFF); 8232 jccb(Assembler::zero, post_alignment); 8233 8234 // ~(~0 << len), where len is the # of remaining elements to process 8235 movl(len, 0xFFFFFFFF); 8236 shlxl(len, len, tmp5); 8237 notl(len); 8238 kmovdl(mask2, len); 8239 movl(len, result); 8240 8241 evmovdquw(tmp1Reg, mask2, Address(src, 0), /*merge*/ false, Assembler::AVX_512bit); 8242 evpcmpw(mask1, mask2, tmp1Reg, tmp2Reg, Assembler::le, /*signed*/ false, Assembler::AVX_512bit); 8243 ktestd(mask1, mask2); 8244 jcc(Assembler::carryClear, copy_tail); 8245 8246 evpmovwb(Address(dst, 0), mask2, tmp1Reg, Assembler::AVX_512bit); 8247 8248 addptr(src, tmp5); 8249 addptr(src, tmp5); 8250 addptr(dst, tmp5); 8251 subl(len, tmp5); 8252 8253 bind(post_alignment); 8254 // end of alignment 8255 8256 movl(tmp5, len); 8257 andl(tmp5, (32 - 1)); // tail count (in chars) 8258 andl(len, ~(32 - 1)); // vector count (in chars) 8259 jccb(Assembler::zero, copy_loop_tail); 8260 8261 lea(src, Address(src, len, Address::times_2)); 8262 lea(dst, Address(dst, len, Address::times_1)); 8263 negptr(len); 8264 8265 bind(copy_32_loop); 8266 evmovdquw(tmp1Reg, Address(src, len, Address::times_2), Assembler::AVX_512bit); 8267 evpcmpuw(mask1, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit); 8268 kortestdl(mask1, mask1); 8269 jccb(Assembler::carryClear, reset_for_copy_tail); 8270 8271 // All elements in current processed chunk are valid candidates for 8272 // compression. Write a truncated byte elements to the memory. 8273 evpmovwb(Address(dst, len, Address::times_1), tmp1Reg, Assembler::AVX_512bit); 8274 addptr(len, 32); 8275 jccb(Assembler::notZero, copy_32_loop); 8276 8277 bind(copy_loop_tail); 8278 // bail out when there is nothing to be done 8279 testl(tmp5, 0xFFFFFFFF); 8280 jcc(Assembler::zero, done); 8281 8282 movl(len, tmp5); 8283 8284 // ~(~0 << len), where len is the # of remaining elements to process 8285 movl(tmp5, 0xFFFFFFFF); 8286 shlxl(tmp5, tmp5, len); 8287 notl(tmp5); 8288 8289 kmovdl(mask2, tmp5); 8290 8291 evmovdquw(tmp1Reg, mask2, Address(src, 0), /*merge*/ false, Assembler::AVX_512bit); 8292 evpcmpw(mask1, mask2, tmp1Reg, tmp2Reg, Assembler::le, /*signed*/ false, Assembler::AVX_512bit); 8293 ktestd(mask1, mask2); 8294 jcc(Assembler::carryClear, copy_tail); 8295 8296 evpmovwb(Address(dst, 0), mask2, tmp1Reg, Assembler::AVX_512bit); 8297 jmp(done); 8298 8299 bind(reset_for_copy_tail); 8300 lea(src, Address(src, tmp5, Address::times_2)); 8301 lea(dst, Address(dst, tmp5, Address::times_1)); 8302 subptr(len, tmp5); 8303 jmp(copy_chars_loop); 8304 8305 bind(below_threshold); 8306 } 8307 8308 if (UseSSE42Intrinsics) { 8309 Label copy_32_loop, copy_16, copy_tail_sse, reset_for_copy_tail; 8310 8311 // vectored compression 8312 testl(len, 0xfffffff8); 8313 jcc(Assembler::zero, copy_tail); 8314 8315 movl(tmp5, 0xff00ff00); // create mask to test for Unicode chars in vectors 8316 movdl(tmp1Reg, tmp5); 8317 pshufd(tmp1Reg, tmp1Reg, 0); // store Unicode mask in tmp1Reg 8318 8319 andl(len, 0xfffffff0); 8320 jccb(Assembler::zero, copy_16); 8321 8322 // compress 16 chars per iter 8323 pxor(tmp4Reg, tmp4Reg); 8324 8325 lea(src, Address(src, len, Address::times_2)); 8326 lea(dst, Address(dst, len, Address::times_1)); 8327 negptr(len); 8328 8329 bind(copy_32_loop); 8330 movdqu(tmp2Reg, Address(src, len, Address::times_2)); // load 1st 8 characters 8331 por(tmp4Reg, tmp2Reg); 8332 movdqu(tmp3Reg, Address(src, len, Address::times_2, 16)); // load next 8 characters 8333 por(tmp4Reg, tmp3Reg); 8334 ptest(tmp4Reg, tmp1Reg); // check for Unicode chars in next vector 8335 jccb(Assembler::notZero, reset_for_copy_tail); 8336 packuswb(tmp2Reg, tmp3Reg); // only ASCII chars; compress each to 1 byte 8337 movdqu(Address(dst, len, Address::times_1), tmp2Reg); 8338 addptr(len, 16); 8339 jccb(Assembler::notZero, copy_32_loop); 8340 8341 // compress next vector of 8 chars (if any) 8342 bind(copy_16); 8343 // len = 0 8344 testl(result, 0x00000008); // check if there's a block of 8 chars to compress 8345 jccb(Assembler::zero, copy_tail_sse); 8346 8347 pxor(tmp3Reg, tmp3Reg); 8348 8349 movdqu(tmp2Reg, Address(src, 0)); 8350 ptest(tmp2Reg, tmp1Reg); // check for Unicode chars in vector 8351 jccb(Assembler::notZero, reset_for_copy_tail); 8352 packuswb(tmp2Reg, tmp3Reg); // only LATIN1 chars; compress each to 1 byte 8353 movq(Address(dst, 0), tmp2Reg); 8354 addptr(src, 16); 8355 addptr(dst, 8); 8356 jmpb(copy_tail_sse); 8357 8358 bind(reset_for_copy_tail); 8359 movl(tmp5, result); 8360 andl(tmp5, 0x0000000f); 8361 lea(src, Address(src, tmp5, Address::times_2)); 8362 lea(dst, Address(dst, tmp5, Address::times_1)); 8363 subptr(len, tmp5); 8364 jmpb(copy_chars_loop); 8365 8366 bind(copy_tail_sse); 8367 movl(len, result); 8368 andl(len, 0x00000007); // tail count (in chars) 8369 } 8370 // compress 1 char per iter 8371 bind(copy_tail); 8372 testl(len, len); 8373 jccb(Assembler::zero, done); 8374 lea(src, Address(src, len, Address::times_2)); 8375 lea(dst, Address(dst, len, Address::times_1)); 8376 negptr(len); 8377 8378 bind(copy_chars_loop); 8379 load_unsigned_short(tmp5, Address(src, len, Address::times_2)); 8380 testl(tmp5, 0xff00); // check if Unicode char 8381 jccb(Assembler::notZero, reset_sp); 8382 movb(Address(dst, len, Address::times_1), tmp5); // ASCII char; compress to 1 byte 8383 increment(len); 8384 jccb(Assembler::notZero, copy_chars_loop); 8385 8386 // add len then return (len will be zero if compress succeeded, otherwise negative) 8387 bind(reset_sp); 8388 addl(result, len); 8389 8390 bind(done); 8391 } 8392 8393 // Inflate byte[] array to char[]. 8394 // ..\jdk\src\java.base\share\classes\java\lang\StringLatin1.java 8395 // @IntrinsicCandidate 8396 // private static void inflate(byte[] src, int srcOff, char[] dst, int dstOff, int len) { 8397 // for (int i = 0; i < len; i++) { 8398 // dst[dstOff++] = (char)(src[srcOff++] & 0xff); 8399 // } 8400 // } 8401 void MacroAssembler::byte_array_inflate(Register src, Register dst, Register len, 8402 XMMRegister tmp1, Register tmp2, KRegister mask) { 8403 Label copy_chars_loop, done, below_threshold, avx3_threshold; 8404 // rsi: src 8405 // rdi: dst 8406 // rdx: len 8407 // rcx: tmp2 8408 8409 // rsi holds start addr of source byte[] to be inflated 8410 // rdi holds start addr of destination char[] 8411 // rdx holds length 8412 assert_different_registers(src, dst, len, tmp2); 8413 movl(tmp2, len); 8414 if ((UseAVX > 2) && // AVX512 8415 VM_Version::supports_avx512vlbw() && 8416 VM_Version::supports_bmi2()) { 8417 8418 Label copy_32_loop, copy_tail; 8419 Register tmp3_aliased = len; 8420 8421 // if length of the string is less than 16, handle it in an old fashioned way 8422 testl(len, -16); 8423 jcc(Assembler::zero, below_threshold); 8424 8425 testl(len, -1 * AVX3Threshold); 8426 jcc(Assembler::zero, avx3_threshold); 8427 8428 // In order to use only one arithmetic operation for the main loop we use 8429 // this pre-calculation 8430 andl(tmp2, (32 - 1)); // tail count (in chars), 32 element wide loop 8431 andl(len, -32); // vector count 8432 jccb(Assembler::zero, copy_tail); 8433 8434 lea(src, Address(src, len, Address::times_1)); 8435 lea(dst, Address(dst, len, Address::times_2)); 8436 negptr(len); 8437 8438 8439 // inflate 32 chars per iter 8440 bind(copy_32_loop); 8441 vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_512bit); 8442 evmovdquw(Address(dst, len, Address::times_2), tmp1, Assembler::AVX_512bit); 8443 addptr(len, 32); 8444 jcc(Assembler::notZero, copy_32_loop); 8445 8446 bind(copy_tail); 8447 // bail out when there is nothing to be done 8448 testl(tmp2, -1); // we don't destroy the contents of tmp2 here 8449 jcc(Assembler::zero, done); 8450 8451 // ~(~0 << length), where length is the # of remaining elements to process 8452 movl(tmp3_aliased, -1); 8453 shlxl(tmp3_aliased, tmp3_aliased, tmp2); 8454 notl(tmp3_aliased); 8455 kmovdl(mask, tmp3_aliased); 8456 evpmovzxbw(tmp1, mask, Address(src, 0), Assembler::AVX_512bit); 8457 evmovdquw(Address(dst, 0), mask, tmp1, /*merge*/ true, Assembler::AVX_512bit); 8458 8459 jmp(done); 8460 bind(avx3_threshold); 8461 } 8462 if (UseSSE42Intrinsics) { 8463 Label copy_16_loop, copy_8_loop, copy_bytes, copy_new_tail, copy_tail; 8464 8465 if (UseAVX > 1) { 8466 andl(tmp2, (16 - 1)); 8467 andl(len, -16); 8468 jccb(Assembler::zero, copy_new_tail); 8469 } else { 8470 andl(tmp2, 0x00000007); // tail count (in chars) 8471 andl(len, 0xfffffff8); // vector count (in chars) 8472 jccb(Assembler::zero, copy_tail); 8473 } 8474 8475 // vectored inflation 8476 lea(src, Address(src, len, Address::times_1)); 8477 lea(dst, Address(dst, len, Address::times_2)); 8478 negptr(len); 8479 8480 if (UseAVX > 1) { 8481 bind(copy_16_loop); 8482 vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_256bit); 8483 vmovdqu(Address(dst, len, Address::times_2), tmp1); 8484 addptr(len, 16); 8485 jcc(Assembler::notZero, copy_16_loop); 8486 8487 bind(below_threshold); 8488 bind(copy_new_tail); 8489 movl(len, tmp2); 8490 andl(tmp2, 0x00000007); 8491 andl(len, 0xFFFFFFF8); 8492 jccb(Assembler::zero, copy_tail); 8493 8494 pmovzxbw(tmp1, Address(src, 0)); 8495 movdqu(Address(dst, 0), tmp1); 8496 addptr(src, 8); 8497 addptr(dst, 2 * 8); 8498 8499 jmp(copy_tail, true); 8500 } 8501 8502 // inflate 8 chars per iter 8503 bind(copy_8_loop); 8504 pmovzxbw(tmp1, Address(src, len, Address::times_1)); // unpack to 8 words 8505 movdqu(Address(dst, len, Address::times_2), tmp1); 8506 addptr(len, 8); 8507 jcc(Assembler::notZero, copy_8_loop); 8508 8509 bind(copy_tail); 8510 movl(len, tmp2); 8511 8512 cmpl(len, 4); 8513 jccb(Assembler::less, copy_bytes); 8514 8515 movdl(tmp1, Address(src, 0)); // load 4 byte chars 8516 pmovzxbw(tmp1, tmp1); 8517 movq(Address(dst, 0), tmp1); 8518 subptr(len, 4); 8519 addptr(src, 4); 8520 addptr(dst, 8); 8521 8522 bind(copy_bytes); 8523 } else { 8524 bind(below_threshold); 8525 } 8526 8527 testl(len, len); 8528 jccb(Assembler::zero, done); 8529 lea(src, Address(src, len, Address::times_1)); 8530 lea(dst, Address(dst, len, Address::times_2)); 8531 negptr(len); 8532 8533 // inflate 1 char per iter 8534 bind(copy_chars_loop); 8535 load_unsigned_byte(tmp2, Address(src, len, Address::times_1)); // load byte char 8536 movw(Address(dst, len, Address::times_2), tmp2); // inflate byte char to word 8537 increment(len); 8538 jcc(Assembler::notZero, copy_chars_loop); 8539 8540 bind(done); 8541 } 8542 8543 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, XMMRegister src, bool merge, int vector_len) { 8544 switch(type) { 8545 case T_BYTE: 8546 case T_BOOLEAN: 8547 evmovdqub(dst, kmask, src, merge, vector_len); 8548 break; 8549 case T_CHAR: 8550 case T_SHORT: 8551 evmovdquw(dst, kmask, src, merge, vector_len); 8552 break; 8553 case T_INT: 8554 case T_FLOAT: 8555 evmovdqul(dst, kmask, src, merge, vector_len); 8556 break; 8557 case T_LONG: 8558 case T_DOUBLE: 8559 evmovdquq(dst, kmask, src, merge, vector_len); 8560 break; 8561 default: 8562 fatal("Unexpected type argument %s", type2name(type)); 8563 break; 8564 } 8565 } 8566 8567 8568 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, Address src, bool merge, int vector_len) { 8569 switch(type) { 8570 case T_BYTE: 8571 case T_BOOLEAN: 8572 evmovdqub(dst, kmask, src, merge, vector_len); 8573 break; 8574 case T_CHAR: 8575 case T_SHORT: 8576 evmovdquw(dst, kmask, src, merge, vector_len); 8577 break; 8578 case T_INT: 8579 case T_FLOAT: 8580 evmovdqul(dst, kmask, src, merge, vector_len); 8581 break; 8582 case T_LONG: 8583 case T_DOUBLE: 8584 evmovdquq(dst, kmask, src, merge, vector_len); 8585 break; 8586 default: 8587 fatal("Unexpected type argument %s", type2name(type)); 8588 break; 8589 } 8590 } 8591 8592 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, Address dst, XMMRegister src, bool merge, int vector_len) { 8593 switch(type) { 8594 case T_BYTE: 8595 case T_BOOLEAN: 8596 evmovdqub(dst, kmask, src, merge, vector_len); 8597 break; 8598 case T_CHAR: 8599 case T_SHORT: 8600 evmovdquw(dst, kmask, src, merge, vector_len); 8601 break; 8602 case T_INT: 8603 case T_FLOAT: 8604 evmovdqul(dst, kmask, src, merge, vector_len); 8605 break; 8606 case T_LONG: 8607 case T_DOUBLE: 8608 evmovdquq(dst, kmask, src, merge, vector_len); 8609 break; 8610 default: 8611 fatal("Unexpected type argument %s", type2name(type)); 8612 break; 8613 } 8614 } 8615 8616 void MacroAssembler::knot(uint masklen, KRegister dst, KRegister src, KRegister ktmp, Register rtmp) { 8617 switch(masklen) { 8618 case 2: 8619 knotbl(dst, src); 8620 movl(rtmp, 3); 8621 kmovbl(ktmp, rtmp); 8622 kandbl(dst, ktmp, dst); 8623 break; 8624 case 4: 8625 knotbl(dst, src); 8626 movl(rtmp, 15); 8627 kmovbl(ktmp, rtmp); 8628 kandbl(dst, ktmp, dst); 8629 break; 8630 case 8: 8631 knotbl(dst, src); 8632 break; 8633 case 16: 8634 knotwl(dst, src); 8635 break; 8636 case 32: 8637 knotdl(dst, src); 8638 break; 8639 case 64: 8640 knotql(dst, src); 8641 break; 8642 default: 8643 fatal("Unexpected vector length %d", masklen); 8644 break; 8645 } 8646 } 8647 8648 void MacroAssembler::kand(BasicType type, KRegister dst, KRegister src1, KRegister src2) { 8649 switch(type) { 8650 case T_BOOLEAN: 8651 case T_BYTE: 8652 kandbl(dst, src1, src2); 8653 break; 8654 case T_CHAR: 8655 case T_SHORT: 8656 kandwl(dst, src1, src2); 8657 break; 8658 case T_INT: 8659 case T_FLOAT: 8660 kanddl(dst, src1, src2); 8661 break; 8662 case T_LONG: 8663 case T_DOUBLE: 8664 kandql(dst, src1, src2); 8665 break; 8666 default: 8667 fatal("Unexpected type argument %s", type2name(type)); 8668 break; 8669 } 8670 } 8671 8672 void MacroAssembler::kor(BasicType type, KRegister dst, KRegister src1, KRegister src2) { 8673 switch(type) { 8674 case T_BOOLEAN: 8675 case T_BYTE: 8676 korbl(dst, src1, src2); 8677 break; 8678 case T_CHAR: 8679 case T_SHORT: 8680 korwl(dst, src1, src2); 8681 break; 8682 case T_INT: 8683 case T_FLOAT: 8684 kordl(dst, src1, src2); 8685 break; 8686 case T_LONG: 8687 case T_DOUBLE: 8688 korql(dst, src1, src2); 8689 break; 8690 default: 8691 fatal("Unexpected type argument %s", type2name(type)); 8692 break; 8693 } 8694 } 8695 8696 void MacroAssembler::kxor(BasicType type, KRegister dst, KRegister src1, KRegister src2) { 8697 switch(type) { 8698 case T_BOOLEAN: 8699 case T_BYTE: 8700 kxorbl(dst, src1, src2); 8701 break; 8702 case T_CHAR: 8703 case T_SHORT: 8704 kxorwl(dst, src1, src2); 8705 break; 8706 case T_INT: 8707 case T_FLOAT: 8708 kxordl(dst, src1, src2); 8709 break; 8710 case T_LONG: 8711 case T_DOUBLE: 8712 kxorql(dst, src1, src2); 8713 break; 8714 default: 8715 fatal("Unexpected type argument %s", type2name(type)); 8716 break; 8717 } 8718 } 8719 8720 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 8721 switch(type) { 8722 case T_BOOLEAN: 8723 case T_BYTE: 8724 evpermb(dst, mask, nds, src, merge, vector_len); break; 8725 case T_CHAR: 8726 case T_SHORT: 8727 evpermw(dst, mask, nds, src, merge, vector_len); break; 8728 case T_INT: 8729 case T_FLOAT: 8730 evpermd(dst, mask, nds, src, merge, vector_len); break; 8731 case T_LONG: 8732 case T_DOUBLE: 8733 evpermq(dst, mask, nds, src, merge, vector_len); break; 8734 default: 8735 fatal("Unexpected type argument %s", type2name(type)); break; 8736 } 8737 } 8738 8739 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 8740 switch(type) { 8741 case T_BOOLEAN: 8742 case T_BYTE: 8743 evpermb(dst, mask, nds, src, merge, vector_len); break; 8744 case T_CHAR: 8745 case T_SHORT: 8746 evpermw(dst, mask, nds, src, merge, vector_len); break; 8747 case T_INT: 8748 case T_FLOAT: 8749 evpermd(dst, mask, nds, src, merge, vector_len); break; 8750 case T_LONG: 8751 case T_DOUBLE: 8752 evpermq(dst, mask, nds, src, merge, vector_len); break; 8753 default: 8754 fatal("Unexpected type argument %s", type2name(type)); break; 8755 } 8756 } 8757 8758 void MacroAssembler::evpminu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 8759 switch(type) { 8760 case T_BYTE: 8761 evpminub(dst, mask, nds, src, merge, vector_len); break; 8762 case T_SHORT: 8763 evpminuw(dst, mask, nds, src, merge, vector_len); break; 8764 case T_INT: 8765 evpminud(dst, mask, nds, src, merge, vector_len); break; 8766 case T_LONG: 8767 evpminuq(dst, mask, nds, src, merge, vector_len); break; 8768 default: 8769 fatal("Unexpected type argument %s", type2name(type)); break; 8770 } 8771 } 8772 8773 void MacroAssembler::evpmaxu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 8774 switch(type) { 8775 case T_BYTE: 8776 evpmaxub(dst, mask, nds, src, merge, vector_len); break; 8777 case T_SHORT: 8778 evpmaxuw(dst, mask, nds, src, merge, vector_len); break; 8779 case T_INT: 8780 evpmaxud(dst, mask, nds, src, merge, vector_len); break; 8781 case T_LONG: 8782 evpmaxuq(dst, mask, nds, src, merge, vector_len); break; 8783 default: 8784 fatal("Unexpected type argument %s", type2name(type)); break; 8785 } 8786 } 8787 8788 void MacroAssembler::evpminu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 8789 switch(type) { 8790 case T_BYTE: 8791 evpminub(dst, mask, nds, src, merge, vector_len); break; 8792 case T_SHORT: 8793 evpminuw(dst, mask, nds, src, merge, vector_len); break; 8794 case T_INT: 8795 evpminud(dst, mask, nds, src, merge, vector_len); break; 8796 case T_LONG: 8797 evpminuq(dst, mask, nds, src, merge, vector_len); break; 8798 default: 8799 fatal("Unexpected type argument %s", type2name(type)); break; 8800 } 8801 } 8802 8803 void MacroAssembler::evpmaxu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 8804 switch(type) { 8805 case T_BYTE: 8806 evpmaxub(dst, mask, nds, src, merge, vector_len); break; 8807 case T_SHORT: 8808 evpmaxuw(dst, mask, nds, src, merge, vector_len); break; 8809 case T_INT: 8810 evpmaxud(dst, mask, nds, src, merge, vector_len); break; 8811 case T_LONG: 8812 evpmaxuq(dst, mask, nds, src, merge, vector_len); break; 8813 default: 8814 fatal("Unexpected type argument %s", type2name(type)); break; 8815 } 8816 } 8817 8818 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 8819 switch(type) { 8820 case T_BYTE: 8821 evpminsb(dst, mask, nds, src, merge, vector_len); break; 8822 case T_SHORT: 8823 evpminsw(dst, mask, nds, src, merge, vector_len); break; 8824 case T_INT: 8825 evpminsd(dst, mask, nds, src, merge, vector_len); break; 8826 case T_LONG: 8827 evpminsq(dst, mask, nds, src, merge, vector_len); break; 8828 default: 8829 fatal("Unexpected type argument %s", type2name(type)); break; 8830 } 8831 } 8832 8833 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 8834 switch(type) { 8835 case T_BYTE: 8836 evpmaxsb(dst, mask, nds, src, merge, vector_len); break; 8837 case T_SHORT: 8838 evpmaxsw(dst, mask, nds, src, merge, vector_len); break; 8839 case T_INT: 8840 evpmaxsd(dst, mask, nds, src, merge, vector_len); break; 8841 case T_LONG: 8842 evpmaxsq(dst, mask, nds, src, merge, vector_len); break; 8843 default: 8844 fatal("Unexpected type argument %s", type2name(type)); break; 8845 } 8846 } 8847 8848 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 8849 switch(type) { 8850 case T_BYTE: 8851 evpminsb(dst, mask, nds, src, merge, vector_len); break; 8852 case T_SHORT: 8853 evpminsw(dst, mask, nds, src, merge, vector_len); break; 8854 case T_INT: 8855 evpminsd(dst, mask, nds, src, merge, vector_len); break; 8856 case T_LONG: 8857 evpminsq(dst, mask, nds, src, merge, vector_len); break; 8858 default: 8859 fatal("Unexpected type argument %s", type2name(type)); break; 8860 } 8861 } 8862 8863 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 8864 switch(type) { 8865 case T_BYTE: 8866 evpmaxsb(dst, mask, nds, src, merge, vector_len); break; 8867 case T_SHORT: 8868 evpmaxsw(dst, mask, nds, src, merge, vector_len); break; 8869 case T_INT: 8870 evpmaxsd(dst, mask, nds, src, merge, vector_len); break; 8871 case T_LONG: 8872 evpmaxsq(dst, mask, nds, src, merge, vector_len); break; 8873 default: 8874 fatal("Unexpected type argument %s", type2name(type)); break; 8875 } 8876 } 8877 8878 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 8879 switch(type) { 8880 case T_INT: 8881 evpxord(dst, mask, nds, src, merge, vector_len); break; 8882 case T_LONG: 8883 evpxorq(dst, mask, nds, src, merge, vector_len); break; 8884 default: 8885 fatal("Unexpected type argument %s", type2name(type)); break; 8886 } 8887 } 8888 8889 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 8890 switch(type) { 8891 case T_INT: 8892 evpxord(dst, mask, nds, src, merge, vector_len); break; 8893 case T_LONG: 8894 evpxorq(dst, mask, nds, src, merge, vector_len); break; 8895 default: 8896 fatal("Unexpected type argument %s", type2name(type)); break; 8897 } 8898 } 8899 8900 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 8901 switch(type) { 8902 case T_INT: 8903 Assembler::evpord(dst, mask, nds, src, merge, vector_len); break; 8904 case T_LONG: 8905 evporq(dst, mask, nds, src, merge, vector_len); break; 8906 default: 8907 fatal("Unexpected type argument %s", type2name(type)); break; 8908 } 8909 } 8910 8911 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 8912 switch(type) { 8913 case T_INT: 8914 Assembler::evpord(dst, mask, nds, src, merge, vector_len); break; 8915 case T_LONG: 8916 evporq(dst, mask, nds, src, merge, vector_len); break; 8917 default: 8918 fatal("Unexpected type argument %s", type2name(type)); break; 8919 } 8920 } 8921 8922 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 8923 switch(type) { 8924 case T_INT: 8925 evpandd(dst, mask, nds, src, merge, vector_len); break; 8926 case T_LONG: 8927 evpandq(dst, mask, nds, src, merge, vector_len); break; 8928 default: 8929 fatal("Unexpected type argument %s", type2name(type)); break; 8930 } 8931 } 8932 8933 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 8934 switch(type) { 8935 case T_INT: 8936 evpandd(dst, mask, nds, src, merge, vector_len); break; 8937 case T_LONG: 8938 evpandq(dst, mask, nds, src, merge, vector_len); break; 8939 default: 8940 fatal("Unexpected type argument %s", type2name(type)); break; 8941 } 8942 } 8943 8944 void MacroAssembler::kortest(uint masklen, KRegister src1, KRegister src2) { 8945 switch(masklen) { 8946 case 8: 8947 kortestbl(src1, src2); 8948 break; 8949 case 16: 8950 kortestwl(src1, src2); 8951 break; 8952 case 32: 8953 kortestdl(src1, src2); 8954 break; 8955 case 64: 8956 kortestql(src1, src2); 8957 break; 8958 default: 8959 fatal("Unexpected mask length %d", masklen); 8960 break; 8961 } 8962 } 8963 8964 8965 void MacroAssembler::ktest(uint masklen, KRegister src1, KRegister src2) { 8966 switch(masklen) { 8967 case 8: 8968 ktestbl(src1, src2); 8969 break; 8970 case 16: 8971 ktestwl(src1, src2); 8972 break; 8973 case 32: 8974 ktestdl(src1, src2); 8975 break; 8976 case 64: 8977 ktestql(src1, src2); 8978 break; 8979 default: 8980 fatal("Unexpected mask length %d", masklen); 8981 break; 8982 } 8983 } 8984 8985 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) { 8986 switch(type) { 8987 case T_INT: 8988 evprold(dst, mask, src, shift, merge, vlen_enc); break; 8989 case T_LONG: 8990 evprolq(dst, mask, src, shift, merge, vlen_enc); break; 8991 default: 8992 fatal("Unexpected type argument %s", type2name(type)); break; 8993 break; 8994 } 8995 } 8996 8997 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) { 8998 switch(type) { 8999 case T_INT: 9000 evprord(dst, mask, src, shift, merge, vlen_enc); break; 9001 case T_LONG: 9002 evprorq(dst, mask, src, shift, merge, vlen_enc); break; 9003 default: 9004 fatal("Unexpected type argument %s", type2name(type)); break; 9005 } 9006 } 9007 9008 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) { 9009 switch(type) { 9010 case T_INT: 9011 evprolvd(dst, mask, src1, src2, merge, vlen_enc); break; 9012 case T_LONG: 9013 evprolvq(dst, mask, src1, src2, merge, vlen_enc); break; 9014 default: 9015 fatal("Unexpected type argument %s", type2name(type)); break; 9016 } 9017 } 9018 9019 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) { 9020 switch(type) { 9021 case T_INT: 9022 evprorvd(dst, mask, src1, src2, merge, vlen_enc); break; 9023 case T_LONG: 9024 evprorvq(dst, mask, src1, src2, merge, vlen_enc); break; 9025 default: 9026 fatal("Unexpected type argument %s", type2name(type)); break; 9027 } 9028 } 9029 9030 void MacroAssembler::evpandq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 9031 assert(rscratch != noreg || always_reachable(src), "missing"); 9032 9033 if (reachable(src)) { 9034 evpandq(dst, nds, as_Address(src), vector_len); 9035 } else { 9036 lea(rscratch, src); 9037 evpandq(dst, nds, Address(rscratch, 0), vector_len); 9038 } 9039 } 9040 9041 void MacroAssembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src, bool merge, int vector_len, Register rscratch) { 9042 assert(rscratch != noreg || always_reachable(src), "missing"); 9043 9044 if (reachable(src)) { 9045 Assembler::evpaddq(dst, mask, nds, as_Address(src), merge, vector_len); 9046 } else { 9047 lea(rscratch, src); 9048 Assembler::evpaddq(dst, mask, nds, Address(rscratch, 0), merge, vector_len); 9049 } 9050 } 9051 9052 void MacroAssembler::evporq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 9053 assert(rscratch != noreg || always_reachable(src), "missing"); 9054 9055 if (reachable(src)) { 9056 evporq(dst, nds, as_Address(src), vector_len); 9057 } else { 9058 lea(rscratch, src); 9059 evporq(dst, nds, Address(rscratch, 0), vector_len); 9060 } 9061 } 9062 9063 void MacroAssembler::vpshufb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 9064 assert(rscratch != noreg || always_reachable(src), "missing"); 9065 9066 if (reachable(src)) { 9067 vpshufb(dst, nds, as_Address(src), vector_len); 9068 } else { 9069 lea(rscratch, src); 9070 vpshufb(dst, nds, Address(rscratch, 0), vector_len); 9071 } 9072 } 9073 9074 void MacroAssembler::vpor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 9075 assert(rscratch != noreg || always_reachable(src), "missing"); 9076 9077 if (reachable(src)) { 9078 Assembler::vpor(dst, nds, as_Address(src), vector_len); 9079 } else { 9080 lea(rscratch, src); 9081 Assembler::vpor(dst, nds, Address(rscratch, 0), vector_len); 9082 } 9083 } 9084 9085 void MacroAssembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, AddressLiteral src3, int vector_len, Register rscratch) { 9086 assert(rscratch != noreg || always_reachable(src3), "missing"); 9087 9088 if (reachable(src3)) { 9089 vpternlogq(dst, imm8, src2, as_Address(src3), vector_len); 9090 } else { 9091 lea(rscratch, src3); 9092 vpternlogq(dst, imm8, src2, Address(rscratch, 0), vector_len); 9093 } 9094 } 9095 9096 #if COMPILER2_OR_JVMCI 9097 9098 void MacroAssembler::fill_masked(BasicType bt, Address dst, XMMRegister xmm, KRegister mask, 9099 Register length, Register temp, int vec_enc) { 9100 // Computing mask for predicated vector store. 9101 movptr(temp, -1); 9102 bzhiq(temp, temp, length); 9103 kmov(mask, temp); 9104 evmovdqu(bt, mask, dst, xmm, true, vec_enc); 9105 } 9106 9107 // Set memory operation for length "less than" 64 bytes. 9108 void MacroAssembler::fill64_masked(uint shift, Register dst, int disp, 9109 XMMRegister xmm, KRegister mask, Register length, 9110 Register temp, bool use64byteVector) { 9111 assert(MaxVectorSize >= 32, "vector length should be >= 32"); 9112 const BasicType type[] = { T_BYTE, T_SHORT, T_INT, T_LONG}; 9113 if (!use64byteVector) { 9114 fill32(dst, disp, xmm); 9115 subptr(length, 32 >> shift); 9116 fill32_masked(shift, dst, disp + 32, xmm, mask, length, temp); 9117 } else { 9118 assert(MaxVectorSize == 64, "vector length != 64"); 9119 fill_masked(type[shift], Address(dst, disp), xmm, mask, length, temp, Assembler::AVX_512bit); 9120 } 9121 } 9122 9123 9124 void MacroAssembler::fill32_masked(uint shift, Register dst, int disp, 9125 XMMRegister xmm, KRegister mask, Register length, 9126 Register temp) { 9127 assert(MaxVectorSize >= 32, "vector length should be >= 32"); 9128 const BasicType type[] = { T_BYTE, T_SHORT, T_INT, T_LONG}; 9129 fill_masked(type[shift], Address(dst, disp), xmm, mask, length, temp, Assembler::AVX_256bit); 9130 } 9131 9132 9133 void MacroAssembler::fill32(Address dst, XMMRegister xmm) { 9134 assert(MaxVectorSize >= 32, "vector length should be >= 32"); 9135 vmovdqu(dst, xmm); 9136 } 9137 9138 void MacroAssembler::fill32(Register dst, int disp, XMMRegister xmm) { 9139 fill32(Address(dst, disp), xmm); 9140 } 9141 9142 void MacroAssembler::fill64(Address dst, XMMRegister xmm, bool use64byteVector) { 9143 assert(MaxVectorSize >= 32, "vector length should be >= 32"); 9144 if (!use64byteVector) { 9145 fill32(dst, xmm); 9146 fill32(dst.plus_disp(32), xmm); 9147 } else { 9148 evmovdquq(dst, xmm, Assembler::AVX_512bit); 9149 } 9150 } 9151 9152 void MacroAssembler::fill64(Register dst, int disp, XMMRegister xmm, bool use64byteVector) { 9153 fill64(Address(dst, disp), xmm, use64byteVector); 9154 } 9155 9156 void MacroAssembler::generate_fill_avx3(BasicType type, Register to, Register value, 9157 Register count, Register rtmp, XMMRegister xtmp) { 9158 Label L_exit; 9159 Label L_fill_start; 9160 Label L_fill_64_bytes; 9161 Label L_fill_96_bytes; 9162 Label L_fill_128_bytes; 9163 Label L_fill_128_bytes_loop; 9164 Label L_fill_128_loop_header; 9165 Label L_fill_128_bytes_loop_header; 9166 Label L_fill_128_bytes_loop_pre_header; 9167 Label L_fill_zmm_sequence; 9168 9169 int shift = -1; 9170 int avx3threshold = VM_Version::avx3_threshold(); 9171 switch(type) { 9172 case T_BYTE: shift = 0; 9173 break; 9174 case T_SHORT: shift = 1; 9175 break; 9176 case T_INT: shift = 2; 9177 break; 9178 /* Uncomment when LONG fill stubs are supported. 9179 case T_LONG: shift = 3; 9180 break; 9181 */ 9182 default: 9183 fatal("Unhandled type: %s\n", type2name(type)); 9184 } 9185 9186 if ((avx3threshold != 0) || (MaxVectorSize == 32)) { 9187 9188 if (MaxVectorSize == 64) { 9189 cmpq(count, avx3threshold >> shift); 9190 jcc(Assembler::greater, L_fill_zmm_sequence); 9191 } 9192 9193 evpbroadcast(type, xtmp, value, Assembler::AVX_256bit); 9194 9195 bind(L_fill_start); 9196 9197 cmpq(count, 32 >> shift); 9198 jccb(Assembler::greater, L_fill_64_bytes); 9199 fill32_masked(shift, to, 0, xtmp, k2, count, rtmp); 9200 jmp(L_exit); 9201 9202 bind(L_fill_64_bytes); 9203 cmpq(count, 64 >> shift); 9204 jccb(Assembler::greater, L_fill_96_bytes); 9205 fill64_masked(shift, to, 0, xtmp, k2, count, rtmp); 9206 jmp(L_exit); 9207 9208 bind(L_fill_96_bytes); 9209 cmpq(count, 96 >> shift); 9210 jccb(Assembler::greater, L_fill_128_bytes); 9211 fill64(to, 0, xtmp); 9212 subq(count, 64 >> shift); 9213 fill32_masked(shift, to, 64, xtmp, k2, count, rtmp); 9214 jmp(L_exit); 9215 9216 bind(L_fill_128_bytes); 9217 cmpq(count, 128 >> shift); 9218 jccb(Assembler::greater, L_fill_128_bytes_loop_pre_header); 9219 fill64(to, 0, xtmp); 9220 fill32(to, 64, xtmp); 9221 subq(count, 96 >> shift); 9222 fill32_masked(shift, to, 96, xtmp, k2, count, rtmp); 9223 jmp(L_exit); 9224 9225 bind(L_fill_128_bytes_loop_pre_header); 9226 { 9227 mov(rtmp, to); 9228 andq(rtmp, 31); 9229 jccb(Assembler::zero, L_fill_128_bytes_loop_header); 9230 negq(rtmp); 9231 addq(rtmp, 32); 9232 mov64(r8, -1L); 9233 bzhiq(r8, r8, rtmp); 9234 kmovql(k2, r8); 9235 evmovdqu(T_BYTE, k2, Address(to, 0), xtmp, true, Assembler::AVX_256bit); 9236 addq(to, rtmp); 9237 shrq(rtmp, shift); 9238 subq(count, rtmp); 9239 } 9240 9241 cmpq(count, 128 >> shift); 9242 jcc(Assembler::less, L_fill_start); 9243 9244 bind(L_fill_128_bytes_loop_header); 9245 subq(count, 128 >> shift); 9246 9247 align32(); 9248 bind(L_fill_128_bytes_loop); 9249 fill64(to, 0, xtmp); 9250 fill64(to, 64, xtmp); 9251 addq(to, 128); 9252 subq(count, 128 >> shift); 9253 jccb(Assembler::greaterEqual, L_fill_128_bytes_loop); 9254 9255 addq(count, 128 >> shift); 9256 jcc(Assembler::zero, L_exit); 9257 jmp(L_fill_start); 9258 } 9259 9260 if (MaxVectorSize == 64) { 9261 // Sequence using 64 byte ZMM register. 9262 Label L_fill_128_bytes_zmm; 9263 Label L_fill_192_bytes_zmm; 9264 Label L_fill_192_bytes_loop_zmm; 9265 Label L_fill_192_bytes_loop_header_zmm; 9266 Label L_fill_192_bytes_loop_pre_header_zmm; 9267 Label L_fill_start_zmm_sequence; 9268 9269 bind(L_fill_zmm_sequence); 9270 evpbroadcast(type, xtmp, value, Assembler::AVX_512bit); 9271 9272 bind(L_fill_start_zmm_sequence); 9273 cmpq(count, 64 >> shift); 9274 jccb(Assembler::greater, L_fill_128_bytes_zmm); 9275 fill64_masked(shift, to, 0, xtmp, k2, count, rtmp, true); 9276 jmp(L_exit); 9277 9278 bind(L_fill_128_bytes_zmm); 9279 cmpq(count, 128 >> shift); 9280 jccb(Assembler::greater, L_fill_192_bytes_zmm); 9281 fill64(to, 0, xtmp, true); 9282 subq(count, 64 >> shift); 9283 fill64_masked(shift, to, 64, xtmp, k2, count, rtmp, true); 9284 jmp(L_exit); 9285 9286 bind(L_fill_192_bytes_zmm); 9287 cmpq(count, 192 >> shift); 9288 jccb(Assembler::greater, L_fill_192_bytes_loop_pre_header_zmm); 9289 fill64(to, 0, xtmp, true); 9290 fill64(to, 64, xtmp, true); 9291 subq(count, 128 >> shift); 9292 fill64_masked(shift, to, 128, xtmp, k2, count, rtmp, true); 9293 jmp(L_exit); 9294 9295 bind(L_fill_192_bytes_loop_pre_header_zmm); 9296 { 9297 movq(rtmp, to); 9298 andq(rtmp, 63); 9299 jccb(Assembler::zero, L_fill_192_bytes_loop_header_zmm); 9300 negq(rtmp); 9301 addq(rtmp, 64); 9302 mov64(r8, -1L); 9303 bzhiq(r8, r8, rtmp); 9304 kmovql(k2, r8); 9305 evmovdqu(T_BYTE, k2, Address(to, 0), xtmp, true, Assembler::AVX_512bit); 9306 addq(to, rtmp); 9307 shrq(rtmp, shift); 9308 subq(count, rtmp); 9309 } 9310 9311 cmpq(count, 192 >> shift); 9312 jcc(Assembler::less, L_fill_start_zmm_sequence); 9313 9314 bind(L_fill_192_bytes_loop_header_zmm); 9315 subq(count, 192 >> shift); 9316 9317 align32(); 9318 bind(L_fill_192_bytes_loop_zmm); 9319 fill64(to, 0, xtmp, true); 9320 fill64(to, 64, xtmp, true); 9321 fill64(to, 128, xtmp, true); 9322 addq(to, 192); 9323 subq(count, 192 >> shift); 9324 jccb(Assembler::greaterEqual, L_fill_192_bytes_loop_zmm); 9325 9326 addq(count, 192 >> shift); 9327 jcc(Assembler::zero, L_exit); 9328 jmp(L_fill_start_zmm_sequence); 9329 } 9330 bind(L_exit); 9331 } 9332 #endif //COMPILER2_OR_JVMCI 9333 9334 9335 void MacroAssembler::convert_f2i(Register dst, XMMRegister src) { 9336 Label done; 9337 cvttss2sil(dst, src); 9338 // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub 9339 cmpl(dst, 0x80000000); // float_sign_flip 9340 jccb(Assembler::notEqual, done); 9341 subptr(rsp, 8); 9342 movflt(Address(rsp, 0), src); 9343 call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup()))); 9344 pop(dst); 9345 bind(done); 9346 } 9347 9348 void MacroAssembler::convert_d2i(Register dst, XMMRegister src) { 9349 Label done; 9350 cvttsd2sil(dst, src); 9351 // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub 9352 cmpl(dst, 0x80000000); // float_sign_flip 9353 jccb(Assembler::notEqual, done); 9354 subptr(rsp, 8); 9355 movdbl(Address(rsp, 0), src); 9356 call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup()))); 9357 pop(dst); 9358 bind(done); 9359 } 9360 9361 void MacroAssembler::convert_f2l(Register dst, XMMRegister src) { 9362 Label done; 9363 cvttss2siq(dst, src); 9364 cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip())); 9365 jccb(Assembler::notEqual, done); 9366 subptr(rsp, 8); 9367 movflt(Address(rsp, 0), src); 9368 call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup()))); 9369 pop(dst); 9370 bind(done); 9371 } 9372 9373 void MacroAssembler::round_float(Register dst, XMMRegister src, Register rtmp, Register rcx) { 9374 // Following code is line by line assembly translation rounding algorithm. 9375 // Please refer to java.lang.Math.round(float) algorithm for details. 9376 const int32_t FloatConsts_EXP_BIT_MASK = 0x7F800000; 9377 const int32_t FloatConsts_SIGNIFICAND_WIDTH = 24; 9378 const int32_t FloatConsts_EXP_BIAS = 127; 9379 const int32_t FloatConsts_SIGNIF_BIT_MASK = 0x007FFFFF; 9380 const int32_t MINUS_32 = 0xFFFFFFE0; 9381 Label L_special_case, L_block1, L_exit; 9382 movl(rtmp, FloatConsts_EXP_BIT_MASK); 9383 movdl(dst, src); 9384 andl(dst, rtmp); 9385 sarl(dst, FloatConsts_SIGNIFICAND_WIDTH - 1); 9386 movl(rtmp, FloatConsts_SIGNIFICAND_WIDTH - 2 + FloatConsts_EXP_BIAS); 9387 subl(rtmp, dst); 9388 movl(rcx, rtmp); 9389 movl(dst, MINUS_32); 9390 testl(rtmp, dst); 9391 jccb(Assembler::notEqual, L_special_case); 9392 movdl(dst, src); 9393 andl(dst, FloatConsts_SIGNIF_BIT_MASK); 9394 orl(dst, FloatConsts_SIGNIF_BIT_MASK + 1); 9395 movdl(rtmp, src); 9396 testl(rtmp, rtmp); 9397 jccb(Assembler::greaterEqual, L_block1); 9398 negl(dst); 9399 bind(L_block1); 9400 sarl(dst); 9401 addl(dst, 0x1); 9402 sarl(dst, 0x1); 9403 jmp(L_exit); 9404 bind(L_special_case); 9405 convert_f2i(dst, src); 9406 bind(L_exit); 9407 } 9408 9409 void MacroAssembler::round_double(Register dst, XMMRegister src, Register rtmp, Register rcx) { 9410 // Following code is line by line assembly translation rounding algorithm. 9411 // Please refer to java.lang.Math.round(double) algorithm for details. 9412 const int64_t DoubleConsts_EXP_BIT_MASK = 0x7FF0000000000000L; 9413 const int64_t DoubleConsts_SIGNIFICAND_WIDTH = 53; 9414 const int64_t DoubleConsts_EXP_BIAS = 1023; 9415 const int64_t DoubleConsts_SIGNIF_BIT_MASK = 0x000FFFFFFFFFFFFFL; 9416 const int64_t MINUS_64 = 0xFFFFFFFFFFFFFFC0L; 9417 Label L_special_case, L_block1, L_exit; 9418 mov64(rtmp, DoubleConsts_EXP_BIT_MASK); 9419 movq(dst, src); 9420 andq(dst, rtmp); 9421 sarq(dst, DoubleConsts_SIGNIFICAND_WIDTH - 1); 9422 mov64(rtmp, DoubleConsts_SIGNIFICAND_WIDTH - 2 + DoubleConsts_EXP_BIAS); 9423 subq(rtmp, dst); 9424 movq(rcx, rtmp); 9425 mov64(dst, MINUS_64); 9426 testq(rtmp, dst); 9427 jccb(Assembler::notEqual, L_special_case); 9428 movq(dst, src); 9429 mov64(rtmp, DoubleConsts_SIGNIF_BIT_MASK); 9430 andq(dst, rtmp); 9431 mov64(rtmp, DoubleConsts_SIGNIF_BIT_MASK + 1); 9432 orq(dst, rtmp); 9433 movq(rtmp, src); 9434 testq(rtmp, rtmp); 9435 jccb(Assembler::greaterEqual, L_block1); 9436 negq(dst); 9437 bind(L_block1); 9438 sarq(dst); 9439 addq(dst, 0x1); 9440 sarq(dst, 0x1); 9441 jmp(L_exit); 9442 bind(L_special_case); 9443 convert_d2l(dst, src); 9444 bind(L_exit); 9445 } 9446 9447 void MacroAssembler::convert_d2l(Register dst, XMMRegister src) { 9448 Label done; 9449 cvttsd2siq(dst, src); 9450 cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip())); 9451 jccb(Assembler::notEqual, done); 9452 subptr(rsp, 8); 9453 movdbl(Address(rsp, 0), src); 9454 call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup()))); 9455 pop(dst); 9456 bind(done); 9457 } 9458 9459 void MacroAssembler::cache_wb(Address line) 9460 { 9461 // 64 bit cpus always support clflush 9462 assert(VM_Version::supports_clflush(), "clflush should be available"); 9463 bool optimized = VM_Version::supports_clflushopt(); 9464 bool no_evict = VM_Version::supports_clwb(); 9465 9466 // prefer clwb (writeback without evict) otherwise 9467 // prefer clflushopt (potentially parallel writeback with evict) 9468 // otherwise fallback on clflush (serial writeback with evict) 9469 9470 if (optimized) { 9471 if (no_evict) { 9472 clwb(line); 9473 } else { 9474 clflushopt(line); 9475 } 9476 } else { 9477 // no need for fence when using CLFLUSH 9478 clflush(line); 9479 } 9480 } 9481 9482 void MacroAssembler::cache_wbsync(bool is_pre) 9483 { 9484 assert(VM_Version::supports_clflush(), "clflush should be available"); 9485 bool optimized = VM_Version::supports_clflushopt(); 9486 bool no_evict = VM_Version::supports_clwb(); 9487 9488 // pick the correct implementation 9489 9490 if (!is_pre && (optimized || no_evict)) { 9491 // need an sfence for post flush when using clflushopt or clwb 9492 // otherwise no no need for any synchroniaztion 9493 9494 sfence(); 9495 } 9496 } 9497 9498 Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) { 9499 switch (cond) { 9500 // Note some conditions are synonyms for others 9501 case Assembler::zero: return Assembler::notZero; 9502 case Assembler::notZero: return Assembler::zero; 9503 case Assembler::less: return Assembler::greaterEqual; 9504 case Assembler::lessEqual: return Assembler::greater; 9505 case Assembler::greater: return Assembler::lessEqual; 9506 case Assembler::greaterEqual: return Assembler::less; 9507 case Assembler::below: return Assembler::aboveEqual; 9508 case Assembler::belowEqual: return Assembler::above; 9509 case Assembler::above: return Assembler::belowEqual; 9510 case Assembler::aboveEqual: return Assembler::below; 9511 case Assembler::overflow: return Assembler::noOverflow; 9512 case Assembler::noOverflow: return Assembler::overflow; 9513 case Assembler::negative: return Assembler::positive; 9514 case Assembler::positive: return Assembler::negative; 9515 case Assembler::parity: return Assembler::noParity; 9516 case Assembler::noParity: return Assembler::parity; 9517 } 9518 ShouldNotReachHere(); return Assembler::overflow; 9519 } 9520 9521 // This is simply a call to Thread::current() 9522 void MacroAssembler::get_thread_slow(Register thread) { 9523 if (thread != rax) { 9524 push(rax); 9525 } 9526 push(rdi); 9527 push(rsi); 9528 push(rdx); 9529 push(rcx); 9530 push(r8); 9531 push(r9); 9532 push(r10); 9533 push(r11); 9534 9535 MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0); 9536 9537 pop(r11); 9538 pop(r10); 9539 pop(r9); 9540 pop(r8); 9541 pop(rcx); 9542 pop(rdx); 9543 pop(rsi); 9544 pop(rdi); 9545 if (thread != rax) { 9546 mov(thread, rax); 9547 pop(rax); 9548 } 9549 } 9550 9551 void MacroAssembler::check_stack_alignment(Register sp, const char* msg, unsigned bias, Register tmp) { 9552 Label L_stack_ok; 9553 if (bias == 0) { 9554 testptr(sp, 2 * wordSize - 1); 9555 } else { 9556 // lea(tmp, Address(rsp, bias); 9557 mov(tmp, sp); 9558 addptr(tmp, bias); 9559 testptr(tmp, 2 * wordSize - 1); 9560 } 9561 jcc(Assembler::equal, L_stack_ok); 9562 block_comment(msg); 9563 stop(msg); 9564 bind(L_stack_ok); 9565 } 9566 9567 // Implements lightweight-locking. 9568 // 9569 // obj: the object to be locked 9570 // reg_rax: rax 9571 // thread: the thread which attempts to lock obj 9572 // tmp: a temporary register 9573 void MacroAssembler::lightweight_lock(Register basic_lock, Register obj, Register reg_rax, Register tmp, Label& slow) { 9574 Register thread = r15_thread; 9575 9576 assert(reg_rax == rax, ""); 9577 assert_different_registers(basic_lock, obj, reg_rax, thread, tmp); 9578 9579 Label push; 9580 const Register top = tmp; 9581 9582 // Preload the markWord. It is important that this is the first 9583 // instruction emitted as it is part of C1's null check semantics. 9584 movptr(reg_rax, Address(obj, oopDesc::mark_offset_in_bytes())); 9585 9586 if (UseObjectMonitorTable) { 9587 // Clear cache in case fast locking succeeds or we need to take the slow-path. 9588 movptr(Address(basic_lock, BasicObjectLock::lock_offset() + in_ByteSize((BasicLock::object_monitor_cache_offset_in_bytes()))), 0); 9589 } 9590 9591 if (DiagnoseSyncOnValueBasedClasses != 0) { 9592 load_klass(tmp, obj, rscratch1); 9593 testb(Address(tmp, Klass::misc_flags_offset()), KlassFlags::_misc_is_value_based_class); 9594 jcc(Assembler::notZero, slow); 9595 } 9596 9597 // Load top. 9598 movl(top, Address(thread, JavaThread::lock_stack_top_offset())); 9599 9600 // Check if the lock-stack is full. 9601 cmpl(top, LockStack::end_offset()); 9602 jcc(Assembler::greaterEqual, slow); 9603 9604 // Check for recursion. 9605 cmpptr(obj, Address(thread, top, Address::times_1, -oopSize)); 9606 jcc(Assembler::equal, push); 9607 9608 // Check header for monitor (0b10). 9609 testptr(reg_rax, markWord::monitor_value); 9610 jcc(Assembler::notZero, slow); 9611 9612 // Try to lock. Transition lock bits 0b01 => 0b00 9613 movptr(tmp, reg_rax); 9614 andptr(tmp, ~(int32_t)markWord::unlocked_value); 9615 orptr(reg_rax, markWord::unlocked_value); 9616 lock(); cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes())); 9617 jcc(Assembler::notEqual, slow); 9618 9619 // Restore top, CAS clobbers register. 9620 movl(top, Address(thread, JavaThread::lock_stack_top_offset())); 9621 9622 bind(push); 9623 // After successful lock, push object on lock-stack. 9624 movptr(Address(thread, top), obj); 9625 incrementl(top, oopSize); 9626 movl(Address(thread, JavaThread::lock_stack_top_offset()), top); 9627 } 9628 9629 // Implements lightweight-unlocking. 9630 // 9631 // obj: the object to be unlocked 9632 // reg_rax: rax 9633 // thread: the thread 9634 // tmp: a temporary register 9635 void MacroAssembler::lightweight_unlock(Register obj, Register reg_rax, Register tmp, Label& slow) { 9636 Register thread = r15_thread; 9637 9638 assert(reg_rax == rax, ""); 9639 assert_different_registers(obj, reg_rax, thread, tmp); 9640 9641 Label unlocked, push_and_slow; 9642 const Register top = tmp; 9643 9644 // Check if obj is top of lock-stack. 9645 movl(top, Address(thread, JavaThread::lock_stack_top_offset())); 9646 cmpptr(obj, Address(thread, top, Address::times_1, -oopSize)); 9647 jcc(Assembler::notEqual, slow); 9648 9649 // Pop lock-stack. 9650 DEBUG_ONLY(movptr(Address(thread, top, Address::times_1, -oopSize), 0);) 9651 subl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize); 9652 9653 // Check if recursive. 9654 cmpptr(obj, Address(thread, top, Address::times_1, -2 * oopSize)); 9655 jcc(Assembler::equal, unlocked); 9656 9657 // Not recursive. Check header for monitor (0b10). 9658 movptr(reg_rax, Address(obj, oopDesc::mark_offset_in_bytes())); 9659 testptr(reg_rax, markWord::monitor_value); 9660 jcc(Assembler::notZero, push_and_slow); 9661 9662 #ifdef ASSERT 9663 // Check header not unlocked (0b01). 9664 Label not_unlocked; 9665 testptr(reg_rax, markWord::unlocked_value); 9666 jcc(Assembler::zero, not_unlocked); 9667 stop("lightweight_unlock already unlocked"); 9668 bind(not_unlocked); 9669 #endif 9670 9671 // Try to unlock. Transition lock bits 0b00 => 0b01 9672 movptr(tmp, reg_rax); 9673 orptr(tmp, markWord::unlocked_value); 9674 lock(); cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes())); 9675 jcc(Assembler::equal, unlocked); 9676 9677 bind(push_and_slow); 9678 // Restore lock-stack and handle the unlock in runtime. 9679 #ifdef ASSERT 9680 movl(top, Address(thread, JavaThread::lock_stack_top_offset())); 9681 movptr(Address(thread, top), obj); 9682 #endif 9683 addl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize); 9684 jmp(slow); 9685 9686 bind(unlocked); 9687 } 9688 9689 // Saves legacy GPRs state on stack. 9690 void MacroAssembler::save_legacy_gprs() { 9691 subq(rsp, 16 * wordSize); 9692 movq(Address(rsp, 15 * wordSize), rax); 9693 movq(Address(rsp, 14 * wordSize), rcx); 9694 movq(Address(rsp, 13 * wordSize), rdx); 9695 movq(Address(rsp, 12 * wordSize), rbx); 9696 movq(Address(rsp, 10 * wordSize), rbp); 9697 movq(Address(rsp, 9 * wordSize), rsi); 9698 movq(Address(rsp, 8 * wordSize), rdi); 9699 movq(Address(rsp, 7 * wordSize), r8); 9700 movq(Address(rsp, 6 * wordSize), r9); 9701 movq(Address(rsp, 5 * wordSize), r10); 9702 movq(Address(rsp, 4 * wordSize), r11); 9703 movq(Address(rsp, 3 * wordSize), r12); 9704 movq(Address(rsp, 2 * wordSize), r13); 9705 movq(Address(rsp, wordSize), r14); 9706 movq(Address(rsp, 0), r15); 9707 } 9708 9709 // Resotres back legacy GPRs state from stack. 9710 void MacroAssembler::restore_legacy_gprs() { 9711 movq(r15, Address(rsp, 0)); 9712 movq(r14, Address(rsp, wordSize)); 9713 movq(r13, Address(rsp, 2 * wordSize)); 9714 movq(r12, Address(rsp, 3 * wordSize)); 9715 movq(r11, Address(rsp, 4 * wordSize)); 9716 movq(r10, Address(rsp, 5 * wordSize)); 9717 movq(r9, Address(rsp, 6 * wordSize)); 9718 movq(r8, Address(rsp, 7 * wordSize)); 9719 movq(rdi, Address(rsp, 8 * wordSize)); 9720 movq(rsi, Address(rsp, 9 * wordSize)); 9721 movq(rbp, Address(rsp, 10 * wordSize)); 9722 movq(rbx, Address(rsp, 12 * wordSize)); 9723 movq(rdx, Address(rsp, 13 * wordSize)); 9724 movq(rcx, Address(rsp, 14 * wordSize)); 9725 movq(rax, Address(rsp, 15 * wordSize)); 9726 addq(rsp, 16 * wordSize); 9727 } 9728 9729 void MacroAssembler::setcc(Assembler::Condition comparison, Register dst) { 9730 if (VM_Version::supports_apx_f()) { 9731 esetzucc(comparison, dst); 9732 } else { 9733 setb(comparison, dst); 9734 movzbl(dst, dst); 9735 } 9736 }