1 /* 2 * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "asm/assembler.hpp" 26 #include "asm/assembler.inline.hpp" 27 #include "code/compiledIC.hpp" 28 #include "compiler/compiler_globals.hpp" 29 #include "compiler/disassembler.hpp" 30 #include "crc32c.h" 31 #include "gc/shared/barrierSet.hpp" 32 #include "gc/shared/barrierSetAssembler.hpp" 33 #include "gc/shared/collectedHeap.inline.hpp" 34 #include "gc/shared/tlab_globals.hpp" 35 #include "interpreter/bytecodeHistogram.hpp" 36 #include "interpreter/interpreter.hpp" 37 #include "interpreter/interpreterRuntime.hpp" 38 #include "jvm.h" 39 #include "memory/resourceArea.hpp" 40 #include "memory/universe.hpp" 41 #include "oops/accessDecorators.hpp" 42 #include "oops/compressedKlass.inline.hpp" 43 #include "oops/compressedOops.inline.hpp" 44 #include "oops/klass.inline.hpp" 45 #include "prims/methodHandles.hpp" 46 #include "runtime/continuation.hpp" 47 #include "runtime/interfaceSupport.inline.hpp" 48 #include "runtime/javaThread.hpp" 49 #include "runtime/jniHandles.hpp" 50 #include "runtime/objectMonitor.hpp" 51 #include "runtime/os.hpp" 52 #include "runtime/safepoint.hpp" 53 #include "runtime/safepointMechanism.hpp" 54 #include "runtime/sharedRuntime.hpp" 55 #include "runtime/stubRoutines.hpp" 56 #include "utilities/checkedCast.hpp" 57 #include "utilities/macros.hpp" 58 59 #ifdef PRODUCT 60 #define BLOCK_COMMENT(str) /* nothing */ 61 #define STOP(error) stop(error) 62 #else 63 #define BLOCK_COMMENT(str) block_comment(str) 64 #define STOP(error) block_comment(error); stop(error) 65 #endif 66 67 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":") 68 69 #ifdef ASSERT 70 bool AbstractAssembler::pd_check_instruction_mark() { return true; } 71 #endif 72 73 static const Assembler::Condition reverse[] = { 74 Assembler::noOverflow /* overflow = 0x0 */ , 75 Assembler::overflow /* noOverflow = 0x1 */ , 76 Assembler::aboveEqual /* carrySet = 0x2, below = 0x2 */ , 77 Assembler::below /* aboveEqual = 0x3, carryClear = 0x3 */ , 78 Assembler::notZero /* zero = 0x4, equal = 0x4 */ , 79 Assembler::zero /* notZero = 0x5, notEqual = 0x5 */ , 80 Assembler::above /* belowEqual = 0x6 */ , 81 Assembler::belowEqual /* above = 0x7 */ , 82 Assembler::positive /* negative = 0x8 */ , 83 Assembler::negative /* positive = 0x9 */ , 84 Assembler::noParity /* parity = 0xa */ , 85 Assembler::parity /* noParity = 0xb */ , 86 Assembler::greaterEqual /* less = 0xc */ , 87 Assembler::less /* greaterEqual = 0xd */ , 88 Assembler::greater /* lessEqual = 0xe */ , 89 Assembler::lessEqual /* greater = 0xf, */ 90 91 }; 92 93 94 // Implementation of MacroAssembler 95 96 // First all the versions that have distinct versions depending on 32/64 bit 97 // Unless the difference is trivial (1 line or so). 98 99 #ifndef _LP64 100 101 // 32bit versions 102 103 Address MacroAssembler::as_Address(AddressLiteral adr) { 104 return Address(adr.target(), adr.rspec()); 105 } 106 107 Address MacroAssembler::as_Address(ArrayAddress adr, Register rscratch) { 108 assert(rscratch == noreg, ""); 109 return Address::make_array(adr); 110 } 111 112 void MacroAssembler::call_VM_leaf_base(address entry_point, 113 int number_of_arguments) { 114 call(RuntimeAddress(entry_point)); 115 increment(rsp, number_of_arguments * wordSize); 116 } 117 118 void MacroAssembler::cmpklass(Address src1, Metadata* obj) { 119 cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate()); 120 } 121 122 123 void MacroAssembler::cmpklass(Register src1, Metadata* obj) { 124 cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate()); 125 } 126 127 void MacroAssembler::cmpoop(Address src1, jobject obj) { 128 cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate()); 129 } 130 131 void MacroAssembler::cmpoop(Register src1, jobject obj, Register rscratch) { 132 assert(rscratch == noreg, "redundant"); 133 cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate()); 134 } 135 136 void MacroAssembler::extend_sign(Register hi, Register lo) { 137 // According to Intel Doc. AP-526, "Integer Divide", p.18. 138 if (VM_Version::is_P6() && hi == rdx && lo == rax) { 139 cdql(); 140 } else { 141 movl(hi, lo); 142 sarl(hi, 31); 143 } 144 } 145 146 void MacroAssembler::jC2(Register tmp, Label& L) { 147 // set parity bit if FPU flag C2 is set (via rax) 148 save_rax(tmp); 149 fwait(); fnstsw_ax(); 150 sahf(); 151 restore_rax(tmp); 152 // branch 153 jcc(Assembler::parity, L); 154 } 155 156 void MacroAssembler::jnC2(Register tmp, Label& L) { 157 // set parity bit if FPU flag C2 is set (via rax) 158 save_rax(tmp); 159 fwait(); fnstsw_ax(); 160 sahf(); 161 restore_rax(tmp); 162 // branch 163 jcc(Assembler::noParity, L); 164 } 165 166 // 32bit can do a case table jump in one instruction but we no longer allow the base 167 // to be installed in the Address class 168 void MacroAssembler::jump(ArrayAddress entry, Register rscratch) { 169 assert(rscratch == noreg, "not needed"); 170 jmp(as_Address(entry, noreg)); 171 } 172 173 // Note: y_lo will be destroyed 174 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) { 175 // Long compare for Java (semantics as described in JVM spec.) 176 Label high, low, done; 177 178 cmpl(x_hi, y_hi); 179 jcc(Assembler::less, low); 180 jcc(Assembler::greater, high); 181 // x_hi is the return register 182 xorl(x_hi, x_hi); 183 cmpl(x_lo, y_lo); 184 jcc(Assembler::below, low); 185 jcc(Assembler::equal, done); 186 187 bind(high); 188 xorl(x_hi, x_hi); 189 increment(x_hi); 190 jmp(done); 191 192 bind(low); 193 xorl(x_hi, x_hi); 194 decrementl(x_hi); 195 196 bind(done); 197 } 198 199 void MacroAssembler::lea(Register dst, AddressLiteral src) { 200 mov_literal32(dst, (int32_t)src.target(), src.rspec()); 201 } 202 203 void MacroAssembler::lea(Address dst, AddressLiteral adr, Register rscratch) { 204 assert(rscratch == noreg, "not needed"); 205 206 // leal(dst, as_Address(adr)); 207 // see note in movl as to why we must use a move 208 mov_literal32(dst, (int32_t)adr.target(), adr.rspec()); 209 } 210 211 void MacroAssembler::leave() { 212 mov(rsp, rbp); 213 pop(rbp); 214 } 215 216 void MacroAssembler::lmul(int x_rsp_offset, int y_rsp_offset) { 217 // Multiplication of two Java long values stored on the stack 218 // as illustrated below. Result is in rdx:rax. 219 // 220 // rsp ---> [ ?? ] \ \ 221 // .... | y_rsp_offset | 222 // [ y_lo ] / (in bytes) | x_rsp_offset 223 // [ y_hi ] | (in bytes) 224 // .... | 225 // [ x_lo ] / 226 // [ x_hi ] 227 // .... 228 // 229 // Basic idea: lo(result) = lo(x_lo * y_lo) 230 // hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi) 231 Address x_hi(rsp, x_rsp_offset + wordSize); Address x_lo(rsp, x_rsp_offset); 232 Address y_hi(rsp, y_rsp_offset + wordSize); Address y_lo(rsp, y_rsp_offset); 233 Label quick; 234 // load x_hi, y_hi and check if quick 235 // multiplication is possible 236 movl(rbx, x_hi); 237 movl(rcx, y_hi); 238 movl(rax, rbx); 239 orl(rbx, rcx); // rbx, = 0 <=> x_hi = 0 and y_hi = 0 240 jcc(Assembler::zero, quick); // if rbx, = 0 do quick multiply 241 // do full multiplication 242 // 1st step 243 mull(y_lo); // x_hi * y_lo 244 movl(rbx, rax); // save lo(x_hi * y_lo) in rbx, 245 // 2nd step 246 movl(rax, x_lo); 247 mull(rcx); // x_lo * y_hi 248 addl(rbx, rax); // add lo(x_lo * y_hi) to rbx, 249 // 3rd step 250 bind(quick); // note: rbx, = 0 if quick multiply! 251 movl(rax, x_lo); 252 mull(y_lo); // x_lo * y_lo 253 addl(rdx, rbx); // correct hi(x_lo * y_lo) 254 } 255 256 void MacroAssembler::lneg(Register hi, Register lo) { 257 negl(lo); 258 adcl(hi, 0); 259 negl(hi); 260 } 261 262 void MacroAssembler::lshl(Register hi, Register lo) { 263 // Java shift left long support (semantics as described in JVM spec., p.305) 264 // (basic idea for shift counts s >= n: x << s == (x << n) << (s - n)) 265 // shift value is in rcx ! 266 assert(hi != rcx, "must not use rcx"); 267 assert(lo != rcx, "must not use rcx"); 268 const Register s = rcx; // shift count 269 const int n = BitsPerWord; 270 Label L; 271 andl(s, 0x3f); // s := s & 0x3f (s < 0x40) 272 cmpl(s, n); // if (s < n) 273 jcc(Assembler::less, L); // else (s >= n) 274 movl(hi, lo); // x := x << n 275 xorl(lo, lo); 276 // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n! 277 bind(L); // s (mod n) < n 278 shldl(hi, lo); // x := x << s 279 shll(lo); 280 } 281 282 283 void MacroAssembler::lshr(Register hi, Register lo, bool sign_extension) { 284 // Java shift right long support (semantics as described in JVM spec., p.306 & p.310) 285 // (basic idea for shift counts s >= n: x >> s == (x >> n) >> (s - n)) 286 assert(hi != rcx, "must not use rcx"); 287 assert(lo != rcx, "must not use rcx"); 288 const Register s = rcx; // shift count 289 const int n = BitsPerWord; 290 Label L; 291 andl(s, 0x3f); // s := s & 0x3f (s < 0x40) 292 cmpl(s, n); // if (s < n) 293 jcc(Assembler::less, L); // else (s >= n) 294 movl(lo, hi); // x := x >> n 295 if (sign_extension) sarl(hi, 31); 296 else xorl(hi, hi); 297 // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n! 298 bind(L); // s (mod n) < n 299 shrdl(lo, hi); // x := x >> s 300 if (sign_extension) sarl(hi); 301 else shrl(hi); 302 } 303 304 void MacroAssembler::movoop(Register dst, jobject obj) { 305 mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate()); 306 } 307 308 void MacroAssembler::movoop(Address dst, jobject obj, Register rscratch) { 309 assert(rscratch == noreg, "redundant"); 310 mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate()); 311 } 312 313 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) { 314 mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate()); 315 } 316 317 void MacroAssembler::mov_metadata(Address dst, Metadata* obj, Register rscratch) { 318 assert(rscratch == noreg, "redundant"); 319 mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate()); 320 } 321 322 void MacroAssembler::movptr(Register dst, AddressLiteral src) { 323 if (src.is_lval()) { 324 mov_literal32(dst, (intptr_t)src.target(), src.rspec()); 325 } else { 326 movl(dst, as_Address(src)); 327 } 328 } 329 330 void MacroAssembler::movptr(ArrayAddress dst, Register src, Register rscratch) { 331 assert(rscratch == noreg, "redundant"); 332 movl(as_Address(dst, noreg), src); 333 } 334 335 void MacroAssembler::movptr(Register dst, ArrayAddress src) { 336 movl(dst, as_Address(src, noreg)); 337 } 338 339 void MacroAssembler::movptr(Address dst, intptr_t src, Register rscratch) { 340 assert(rscratch == noreg, "redundant"); 341 movl(dst, src); 342 } 343 344 void MacroAssembler::pushoop(jobject obj, Register rscratch) { 345 assert(rscratch == noreg, "redundant"); 346 push_literal32((int32_t)obj, oop_Relocation::spec_for_immediate()); 347 } 348 349 void MacroAssembler::pushklass(Metadata* obj, Register rscratch) { 350 assert(rscratch == noreg, "redundant"); 351 push_literal32((int32_t)obj, metadata_Relocation::spec_for_immediate()); 352 } 353 354 void MacroAssembler::pushptr(AddressLiteral src, Register rscratch) { 355 assert(rscratch == noreg, "redundant"); 356 if (src.is_lval()) { 357 push_literal32((int32_t)src.target(), src.rspec()); 358 } else { 359 pushl(as_Address(src)); 360 } 361 } 362 363 static void pass_arg0(MacroAssembler* masm, Register arg) { 364 masm->push(arg); 365 } 366 367 static void pass_arg1(MacroAssembler* masm, Register arg) { 368 masm->push(arg); 369 } 370 371 static void pass_arg2(MacroAssembler* masm, Register arg) { 372 masm->push(arg); 373 } 374 375 static void pass_arg3(MacroAssembler* masm, Register arg) { 376 masm->push(arg); 377 } 378 379 #ifndef PRODUCT 380 extern "C" void findpc(intptr_t x); 381 #endif 382 383 void MacroAssembler::debug32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip, char* msg) { 384 // In order to get locks to work, we need to fake a in_VM state 385 JavaThread* thread = JavaThread::current(); 386 JavaThreadState saved_state = thread->thread_state(); 387 thread->set_thread_state(_thread_in_vm); 388 if (ShowMessageBoxOnError) { 389 JavaThread* thread = JavaThread::current(); 390 JavaThreadState saved_state = thread->thread_state(); 391 thread->set_thread_state(_thread_in_vm); 392 if (CountBytecodes || TraceBytecodes || StopInterpreterAt) { 393 ttyLocker ttyl; 394 BytecodeCounter::print(); 395 } 396 // To see where a verify_oop failed, get $ebx+40/X for this frame. 397 // This is the value of eip which points to where verify_oop will return. 398 if (os::message_box(msg, "Execution stopped, print registers?")) { 399 print_state32(rdi, rsi, rbp, rsp, rbx, rdx, rcx, rax, eip); 400 BREAKPOINT; 401 } 402 } 403 fatal("DEBUG MESSAGE: %s", msg); 404 } 405 406 void MacroAssembler::print_state32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip) { 407 ttyLocker ttyl; 408 DebuggingContext debugging{}; 409 tty->print_cr("eip = 0x%08x", eip); 410 #ifndef PRODUCT 411 if ((WizardMode || Verbose) && PrintMiscellaneous) { 412 tty->cr(); 413 findpc(eip); 414 tty->cr(); 415 } 416 #endif 417 #define PRINT_REG(rax) \ 418 { tty->print("%s = ", #rax); os::print_location(tty, rax); } 419 PRINT_REG(rax); 420 PRINT_REG(rbx); 421 PRINT_REG(rcx); 422 PRINT_REG(rdx); 423 PRINT_REG(rdi); 424 PRINT_REG(rsi); 425 PRINT_REG(rbp); 426 PRINT_REG(rsp); 427 #undef PRINT_REG 428 // Print some words near top of staack. 429 int* dump_sp = (int*) rsp; 430 for (int col1 = 0; col1 < 8; col1++) { 431 tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp); 432 os::print_location(tty, *dump_sp++); 433 } 434 for (int row = 0; row < 16; row++) { 435 tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp); 436 for (int col = 0; col < 8; col++) { 437 tty->print(" 0x%08x", *dump_sp++); 438 } 439 tty->cr(); 440 } 441 // Print some instructions around pc: 442 Disassembler::decode((address)eip-64, (address)eip); 443 tty->print_cr("--------"); 444 Disassembler::decode((address)eip, (address)eip+32); 445 } 446 447 void MacroAssembler::stop(const char* msg) { 448 // push address of message 449 ExternalAddress message((address)msg); 450 pushptr(message.addr(), noreg); 451 { Label L; call(L, relocInfo::none); bind(L); } // push eip 452 pusha(); // push registers 453 call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32))); 454 hlt(); 455 } 456 457 void MacroAssembler::warn(const char* msg) { 458 push_CPU_state(); 459 460 // push address of message 461 ExternalAddress message((address)msg); 462 pushptr(message.addr(), noreg); 463 464 call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning))); 465 addl(rsp, wordSize); // discard argument 466 pop_CPU_state(); 467 } 468 469 void MacroAssembler::print_state() { 470 { Label L; call(L, relocInfo::none); bind(L); } // push eip 471 pusha(); // push registers 472 473 push_CPU_state(); 474 call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::print_state32))); 475 pop_CPU_state(); 476 477 popa(); 478 addl(rsp, wordSize); 479 } 480 481 #else // _LP64 482 483 // 64 bit versions 484 485 Address MacroAssembler::as_Address(AddressLiteral adr) { 486 // amd64 always does this as a pc-rel 487 // we can be absolute or disp based on the instruction type 488 // jmp/call are displacements others are absolute 489 assert(!adr.is_lval(), "must be rval"); 490 assert(reachable(adr), "must be"); 491 return Address(checked_cast<int32_t>(adr.target() - pc()), adr.target(), adr.reloc()); 492 493 } 494 495 Address MacroAssembler::as_Address(ArrayAddress adr, Register rscratch) { 496 AddressLiteral base = adr.base(); 497 lea(rscratch, base); 498 Address index = adr.index(); 499 assert(index._disp == 0, "must not have disp"); // maybe it can? 500 Address array(rscratch, index._index, index._scale, index._disp); 501 return array; 502 } 503 504 void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) { 505 Label L, E; 506 507 #ifdef _WIN64 508 // Windows always allocates space for it's register args 509 assert(num_args <= 4, "only register arguments supported"); 510 subq(rsp, frame::arg_reg_save_area_bytes); 511 #endif 512 513 // Align stack if necessary 514 testl(rsp, 15); 515 jcc(Assembler::zero, L); 516 517 subq(rsp, 8); 518 call(RuntimeAddress(entry_point)); 519 addq(rsp, 8); 520 jmp(E); 521 522 bind(L); 523 call(RuntimeAddress(entry_point)); 524 525 bind(E); 526 527 #ifdef _WIN64 528 // restore stack pointer 529 addq(rsp, frame::arg_reg_save_area_bytes); 530 #endif 531 } 532 533 void MacroAssembler::cmp64(Register src1, AddressLiteral src2, Register rscratch) { 534 assert(!src2.is_lval(), "should use cmpptr"); 535 assert(rscratch != noreg || always_reachable(src2), "missing"); 536 537 if (reachable(src2)) { 538 cmpq(src1, as_Address(src2)); 539 } else { 540 lea(rscratch, src2); 541 Assembler::cmpq(src1, Address(rscratch, 0)); 542 } 543 } 544 545 int MacroAssembler::corrected_idivq(Register reg) { 546 // Full implementation of Java ldiv and lrem; checks for special 547 // case as described in JVM spec., p.243 & p.271. The function 548 // returns the (pc) offset of the idivl instruction - may be needed 549 // for implicit exceptions. 550 // 551 // normal case special case 552 // 553 // input : rax: dividend min_long 554 // reg: divisor (may not be eax/edx) -1 555 // 556 // output: rax: quotient (= rax idiv reg) min_long 557 // rdx: remainder (= rax irem reg) 0 558 assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register"); 559 static const int64_t min_long = 0x8000000000000000; 560 Label normal_case, special_case; 561 562 // check for special case 563 cmp64(rax, ExternalAddress((address) &min_long), rdx /*rscratch*/); 564 jcc(Assembler::notEqual, normal_case); 565 xorl(rdx, rdx); // prepare rdx for possible special case (where 566 // remainder = 0) 567 cmpq(reg, -1); 568 jcc(Assembler::equal, special_case); 569 570 // handle normal case 571 bind(normal_case); 572 cdqq(); 573 int idivq_offset = offset(); 574 idivq(reg); 575 576 // normal and special case exit 577 bind(special_case); 578 579 return idivq_offset; 580 } 581 582 void MacroAssembler::decrementq(Register reg, int value) { 583 if (value == min_jint) { subq(reg, value); return; } 584 if (value < 0) { incrementq(reg, -value); return; } 585 if (value == 0) { ; return; } 586 if (value == 1 && UseIncDec) { decq(reg) ; return; } 587 /* else */ { subq(reg, value) ; return; } 588 } 589 590 void MacroAssembler::decrementq(Address dst, int value) { 591 if (value == min_jint) { subq(dst, value); return; } 592 if (value < 0) { incrementq(dst, -value); return; } 593 if (value == 0) { ; return; } 594 if (value == 1 && UseIncDec) { decq(dst) ; return; } 595 /* else */ { subq(dst, value) ; return; } 596 } 597 598 void MacroAssembler::incrementq(AddressLiteral dst, Register rscratch) { 599 assert(rscratch != noreg || always_reachable(dst), "missing"); 600 601 if (reachable(dst)) { 602 incrementq(as_Address(dst)); 603 } else { 604 lea(rscratch, dst); 605 incrementq(Address(rscratch, 0)); 606 } 607 } 608 609 void MacroAssembler::incrementq(Register reg, int value) { 610 if (value == min_jint) { addq(reg, value); return; } 611 if (value < 0) { decrementq(reg, -value); return; } 612 if (value == 0) { ; return; } 613 if (value == 1 && UseIncDec) { incq(reg) ; return; } 614 /* else */ { addq(reg, value) ; return; } 615 } 616 617 void MacroAssembler::incrementq(Address dst, int value) { 618 if (value == min_jint) { addq(dst, value); return; } 619 if (value < 0) { decrementq(dst, -value); return; } 620 if (value == 0) { ; return; } 621 if (value == 1 && UseIncDec) { incq(dst) ; return; } 622 /* else */ { addq(dst, value) ; return; } 623 } 624 625 // 32bit can do a case table jump in one instruction but we no longer allow the base 626 // to be installed in the Address class 627 void MacroAssembler::jump(ArrayAddress entry, Register rscratch) { 628 lea(rscratch, entry.base()); 629 Address dispatch = entry.index(); 630 assert(dispatch._base == noreg, "must be"); 631 dispatch._base = rscratch; 632 jmp(dispatch); 633 } 634 635 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) { 636 ShouldNotReachHere(); // 64bit doesn't use two regs 637 cmpq(x_lo, y_lo); 638 } 639 640 void MacroAssembler::lea(Register dst, AddressLiteral src) { 641 mov_literal64(dst, (intptr_t)src.target(), src.rspec()); 642 } 643 644 void MacroAssembler::lea(Address dst, AddressLiteral adr, Register rscratch) { 645 lea(rscratch, adr); 646 movptr(dst, rscratch); 647 } 648 649 void MacroAssembler::leave() { 650 // %%% is this really better? Why not on 32bit too? 651 emit_int8((unsigned char)0xC9); // LEAVE 652 } 653 654 void MacroAssembler::lneg(Register hi, Register lo) { 655 ShouldNotReachHere(); // 64bit doesn't use two regs 656 negq(lo); 657 } 658 659 void MacroAssembler::movoop(Register dst, jobject obj) { 660 mov_literal64(dst, (intptr_t)obj, oop_Relocation::spec_for_immediate()); 661 } 662 663 void MacroAssembler::movoop(Address dst, jobject obj, Register rscratch) { 664 mov_literal64(rscratch, (intptr_t)obj, oop_Relocation::spec_for_immediate()); 665 movq(dst, rscratch); 666 } 667 668 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) { 669 mov_literal64(dst, (intptr_t)obj, metadata_Relocation::spec_for_immediate()); 670 } 671 672 void MacroAssembler::mov_metadata(Address dst, Metadata* obj, Register rscratch) { 673 mov_literal64(rscratch, (intptr_t)obj, metadata_Relocation::spec_for_immediate()); 674 movq(dst, rscratch); 675 } 676 677 void MacroAssembler::movptr(Register dst, AddressLiteral src) { 678 if (src.is_lval()) { 679 mov_literal64(dst, (intptr_t)src.target(), src.rspec()); 680 } else { 681 if (reachable(src)) { 682 movq(dst, as_Address(src)); 683 } else { 684 lea(dst, src); 685 movq(dst, Address(dst, 0)); 686 } 687 } 688 } 689 690 void MacroAssembler::movptr(ArrayAddress dst, Register src, Register rscratch) { 691 movq(as_Address(dst, rscratch), src); 692 } 693 694 void MacroAssembler::movptr(Register dst, ArrayAddress src) { 695 movq(dst, as_Address(src, dst /*rscratch*/)); 696 } 697 698 // src should NEVER be a real pointer. Use AddressLiteral for true pointers 699 void MacroAssembler::movptr(Address dst, intptr_t src, Register rscratch) { 700 if (is_simm32(src)) { 701 movptr(dst, checked_cast<int32_t>(src)); 702 } else { 703 mov64(rscratch, src); 704 movq(dst, rscratch); 705 } 706 } 707 708 void MacroAssembler::pushoop(jobject obj, Register rscratch) { 709 movoop(rscratch, obj); 710 push(rscratch); 711 } 712 713 void MacroAssembler::pushklass(Metadata* obj, Register rscratch) { 714 mov_metadata(rscratch, obj); 715 push(rscratch); 716 } 717 718 void MacroAssembler::pushptr(AddressLiteral src, Register rscratch) { 719 lea(rscratch, src); 720 if (src.is_lval()) { 721 push(rscratch); 722 } else { 723 pushq(Address(rscratch, 0)); 724 } 725 } 726 727 void MacroAssembler::reset_last_Java_frame(bool clear_fp) { 728 reset_last_Java_frame(r15_thread, clear_fp); 729 } 730 731 void MacroAssembler::set_last_Java_frame(Register last_java_sp, 732 Register last_java_fp, 733 address last_java_pc, 734 Register rscratch) { 735 set_last_Java_frame(r15_thread, last_java_sp, last_java_fp, last_java_pc, rscratch); 736 } 737 738 static void pass_arg0(MacroAssembler* masm, Register arg) { 739 if (c_rarg0 != arg ) { 740 masm->mov(c_rarg0, arg); 741 } 742 } 743 744 static void pass_arg1(MacroAssembler* masm, Register arg) { 745 if (c_rarg1 != arg ) { 746 masm->mov(c_rarg1, arg); 747 } 748 } 749 750 static void pass_arg2(MacroAssembler* masm, Register arg) { 751 if (c_rarg2 != arg ) { 752 masm->mov(c_rarg2, arg); 753 } 754 } 755 756 static void pass_arg3(MacroAssembler* masm, Register arg) { 757 if (c_rarg3 != arg ) { 758 masm->mov(c_rarg3, arg); 759 } 760 } 761 762 void MacroAssembler::stop(const char* msg) { 763 if (ShowMessageBoxOnError) { 764 address rip = pc(); 765 pusha(); // get regs on stack 766 lea(c_rarg1, InternalAddress(rip)); 767 movq(c_rarg2, rsp); // pass pointer to regs array 768 } 769 lea(c_rarg0, ExternalAddress((address) msg)); 770 andq(rsp, -16); // align stack as required by ABI 771 call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64))); 772 hlt(); 773 } 774 775 void MacroAssembler::warn(const char* msg) { 776 push(rbp); 777 movq(rbp, rsp); 778 andq(rsp, -16); // align stack as required by push_CPU_state and call 779 push_CPU_state(); // keeps alignment at 16 bytes 780 781 lea(c_rarg0, ExternalAddress((address) msg)); 782 call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning))); 783 784 pop_CPU_state(); 785 mov(rsp, rbp); 786 pop(rbp); 787 } 788 789 void MacroAssembler::print_state() { 790 address rip = pc(); 791 pusha(); // get regs on stack 792 push(rbp); 793 movq(rbp, rsp); 794 andq(rsp, -16); // align stack as required by push_CPU_state and call 795 push_CPU_state(); // keeps alignment at 16 bytes 796 797 lea(c_rarg0, InternalAddress(rip)); 798 lea(c_rarg1, Address(rbp, wordSize)); // pass pointer to regs array 799 call_VM_leaf(CAST_FROM_FN_PTR(address, MacroAssembler::print_state64), c_rarg0, c_rarg1); 800 801 pop_CPU_state(); 802 mov(rsp, rbp); 803 pop(rbp); 804 popa(); 805 } 806 807 #ifndef PRODUCT 808 extern "C" void findpc(intptr_t x); 809 #endif 810 811 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) { 812 // In order to get locks to work, we need to fake a in_VM state 813 if (ShowMessageBoxOnError) { 814 JavaThread* thread = JavaThread::current(); 815 JavaThreadState saved_state = thread->thread_state(); 816 thread->set_thread_state(_thread_in_vm); 817 #ifndef PRODUCT 818 if (CountBytecodes || TraceBytecodes || StopInterpreterAt) { 819 ttyLocker ttyl; 820 BytecodeCounter::print(); 821 } 822 #endif 823 // To see where a verify_oop failed, get $ebx+40/X for this frame. 824 // XXX correct this offset for amd64 825 // This is the value of eip which points to where verify_oop will return. 826 if (os::message_box(msg, "Execution stopped, print registers?")) { 827 print_state64(pc, regs); 828 BREAKPOINT; 829 } 830 } 831 fatal("DEBUG MESSAGE: %s", msg); 832 } 833 834 void MacroAssembler::print_state64(int64_t pc, int64_t regs[]) { 835 ttyLocker ttyl; 836 DebuggingContext debugging{}; 837 tty->print_cr("rip = 0x%016lx", (intptr_t)pc); 838 #ifndef PRODUCT 839 tty->cr(); 840 findpc(pc); 841 tty->cr(); 842 #endif 843 #define PRINT_REG(rax, value) \ 844 { tty->print("%s = ", #rax); os::print_location(tty, value); } 845 PRINT_REG(rax, regs[15]); 846 PRINT_REG(rbx, regs[12]); 847 PRINT_REG(rcx, regs[14]); 848 PRINT_REG(rdx, regs[13]); 849 PRINT_REG(rdi, regs[8]); 850 PRINT_REG(rsi, regs[9]); 851 PRINT_REG(rbp, regs[10]); 852 // rsp is actually not stored by pusha(), compute the old rsp from regs (rsp after pusha): regs + 16 = old rsp 853 PRINT_REG(rsp, (intptr_t)(®s[16])); 854 PRINT_REG(r8 , regs[7]); 855 PRINT_REG(r9 , regs[6]); 856 PRINT_REG(r10, regs[5]); 857 PRINT_REG(r11, regs[4]); 858 PRINT_REG(r12, regs[3]); 859 PRINT_REG(r13, regs[2]); 860 PRINT_REG(r14, regs[1]); 861 PRINT_REG(r15, regs[0]); 862 #undef PRINT_REG 863 // Print some words near the top of the stack. 864 int64_t* rsp = ®s[16]; 865 int64_t* dump_sp = rsp; 866 for (int col1 = 0; col1 < 8; col1++) { 867 tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp); 868 os::print_location(tty, *dump_sp++); 869 } 870 for (int row = 0; row < 25; row++) { 871 tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp); 872 for (int col = 0; col < 4; col++) { 873 tty->print(" 0x%016lx", (intptr_t)*dump_sp++); 874 } 875 tty->cr(); 876 } 877 // Print some instructions around pc: 878 Disassembler::decode((address)pc-64, (address)pc); 879 tty->print_cr("--------"); 880 Disassembler::decode((address)pc, (address)pc+32); 881 } 882 883 // The java_calling_convention describes stack locations as ideal slots on 884 // a frame with no abi restrictions. Since we must observe abi restrictions 885 // (like the placement of the register window) the slots must be biased by 886 // the following value. 887 static int reg2offset_in(VMReg r) { 888 // Account for saved rbp and return address 889 // This should really be in_preserve_stack_slots 890 return (r->reg2stack() + 4) * VMRegImpl::stack_slot_size; 891 } 892 893 static int reg2offset_out(VMReg r) { 894 return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size; 895 } 896 897 // A long move 898 void MacroAssembler::long_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) { 899 900 // The calling conventions assures us that each VMregpair is either 901 // all really one physical register or adjacent stack slots. 902 903 if (src.is_single_phys_reg() ) { 904 if (dst.is_single_phys_reg()) { 905 if (dst.first() != src.first()) { 906 mov(dst.first()->as_Register(), src.first()->as_Register()); 907 } 908 } else { 909 assert(dst.is_single_reg(), "not a stack pair: (%s, %s), (%s, %s)", 910 src.first()->name(), src.second()->name(), dst.first()->name(), dst.second()->name()); 911 movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_Register()); 912 } 913 } else if (dst.is_single_phys_reg()) { 914 assert(src.is_single_reg(), "not a stack pair"); 915 movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 916 } else { 917 assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs"); 918 movq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 919 movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp); 920 } 921 } 922 923 // A double move 924 void MacroAssembler::double_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) { 925 926 // The calling conventions assures us that each VMregpair is either 927 // all really one physical register or adjacent stack slots. 928 929 if (src.is_single_phys_reg() ) { 930 if (dst.is_single_phys_reg()) { 931 // In theory these overlap but the ordering is such that this is likely a nop 932 if ( src.first() != dst.first()) { 933 movdbl(dst.first()->as_XMMRegister(), src.first()->as_XMMRegister()); 934 } 935 } else { 936 assert(dst.is_single_reg(), "not a stack pair"); 937 movdbl(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_XMMRegister()); 938 } 939 } else if (dst.is_single_phys_reg()) { 940 assert(src.is_single_reg(), "not a stack pair"); 941 movdbl(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 942 } else { 943 assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs"); 944 movq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 945 movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp); 946 } 947 } 948 949 950 // A float arg may have to do float reg int reg conversion 951 void MacroAssembler::float_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) { 952 assert(!src.second()->is_valid() && !dst.second()->is_valid(), "bad float_move"); 953 954 // The calling conventions assures us that each VMregpair is either 955 // all really one physical register or adjacent stack slots. 956 957 if (src.first()->is_stack()) { 958 if (dst.first()->is_stack()) { 959 movl(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 960 movptr(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp); 961 } else { 962 // stack to reg 963 assert(dst.first()->is_XMMRegister(), "only expect xmm registers as parameters"); 964 movflt(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 965 } 966 } else if (dst.first()->is_stack()) { 967 // reg to stack 968 assert(src.first()->is_XMMRegister(), "only expect xmm registers as parameters"); 969 movflt(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_XMMRegister()); 970 } else { 971 // reg to reg 972 // In theory these overlap but the ordering is such that this is likely a nop 973 if ( src.first() != dst.first()) { 974 movdbl(dst.first()->as_XMMRegister(), src.first()->as_XMMRegister()); 975 } 976 } 977 } 978 979 // On 64 bit we will store integer like items to the stack as 980 // 64 bits items (x86_32/64 abi) even though java would only store 981 // 32bits for a parameter. On 32bit it will simply be 32 bits 982 // So this routine will do 32->32 on 32bit and 32->64 on 64bit 983 void MacroAssembler::move32_64(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) { 984 if (src.first()->is_stack()) { 985 if (dst.first()->is_stack()) { 986 // stack to stack 987 movslq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 988 movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp); 989 } else { 990 // stack to reg 991 movslq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias)); 992 } 993 } else if (dst.first()->is_stack()) { 994 // reg to stack 995 // Do we really have to sign extend??? 996 // __ movslq(src.first()->as_Register(), src.first()->as_Register()); 997 movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_Register()); 998 } else { 999 // Do we really have to sign extend??? 1000 // __ movslq(dst.first()->as_Register(), src.first()->as_Register()); 1001 if (dst.first() != src.first()) { 1002 movq(dst.first()->as_Register(), src.first()->as_Register()); 1003 } 1004 } 1005 } 1006 1007 void MacroAssembler::move_ptr(VMRegPair src, VMRegPair dst) { 1008 if (src.first()->is_stack()) { 1009 if (dst.first()->is_stack()) { 1010 // stack to stack 1011 movq(rax, Address(rbp, reg2offset_in(src.first()))); 1012 movq(Address(rsp, reg2offset_out(dst.first())), rax); 1013 } else { 1014 // stack to reg 1015 movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()))); 1016 } 1017 } else if (dst.first()->is_stack()) { 1018 // reg to stack 1019 movq(Address(rsp, reg2offset_out(dst.first())), src.first()->as_Register()); 1020 } else { 1021 if (dst.first() != src.first()) { 1022 movq(dst.first()->as_Register(), src.first()->as_Register()); 1023 } 1024 } 1025 } 1026 1027 // An oop arg. Must pass a handle not the oop itself 1028 void MacroAssembler::object_move(OopMap* map, 1029 int oop_handle_offset, 1030 int framesize_in_slots, 1031 VMRegPair src, 1032 VMRegPair dst, 1033 bool is_receiver, 1034 int* receiver_offset) { 1035 1036 // must pass a handle. First figure out the location we use as a handle 1037 1038 Register rHandle = dst.first()->is_stack() ? rax : dst.first()->as_Register(); 1039 1040 // See if oop is null if it is we need no handle 1041 1042 if (src.first()->is_stack()) { 1043 1044 // Oop is already on the stack as an argument 1045 int offset_in_older_frame = src.first()->reg2stack() + SharedRuntime::out_preserve_stack_slots(); 1046 map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + framesize_in_slots)); 1047 if (is_receiver) { 1048 *receiver_offset = (offset_in_older_frame + framesize_in_slots) * VMRegImpl::stack_slot_size; 1049 } 1050 1051 cmpptr(Address(rbp, reg2offset_in(src.first())), NULL_WORD); 1052 lea(rHandle, Address(rbp, reg2offset_in(src.first()))); 1053 // conditionally move a null 1054 cmovptr(Assembler::equal, rHandle, Address(rbp, reg2offset_in(src.first()))); 1055 } else { 1056 1057 // Oop is in a register we must store it to the space we reserve 1058 // on the stack for oop_handles and pass a handle if oop is non-null 1059 1060 const Register rOop = src.first()->as_Register(); 1061 int oop_slot; 1062 if (rOop == j_rarg0) 1063 oop_slot = 0; 1064 else if (rOop == j_rarg1) 1065 oop_slot = 1; 1066 else if (rOop == j_rarg2) 1067 oop_slot = 2; 1068 else if (rOop == j_rarg3) 1069 oop_slot = 3; 1070 else if (rOop == j_rarg4) 1071 oop_slot = 4; 1072 else { 1073 assert(rOop == j_rarg5, "wrong register"); 1074 oop_slot = 5; 1075 } 1076 1077 oop_slot = oop_slot * VMRegImpl::slots_per_word + oop_handle_offset; 1078 int offset = oop_slot*VMRegImpl::stack_slot_size; 1079 1080 map->set_oop(VMRegImpl::stack2reg(oop_slot)); 1081 // Store oop in handle area, may be null 1082 movptr(Address(rsp, offset), rOop); 1083 if (is_receiver) { 1084 *receiver_offset = offset; 1085 } 1086 1087 cmpptr(rOop, NULL_WORD); 1088 lea(rHandle, Address(rsp, offset)); 1089 // conditionally move a null from the handle area where it was just stored 1090 cmovptr(Assembler::equal, rHandle, Address(rsp, offset)); 1091 } 1092 1093 // If arg is on the stack then place it otherwise it is already in correct reg. 1094 if (dst.first()->is_stack()) { 1095 movptr(Address(rsp, reg2offset_out(dst.first())), rHandle); 1096 } 1097 } 1098 1099 #endif // _LP64 1100 1101 // Now versions that are common to 32/64 bit 1102 1103 void MacroAssembler::addptr(Register dst, int32_t imm32) { 1104 LP64_ONLY(addq(dst, imm32)) NOT_LP64(addl(dst, imm32)); 1105 } 1106 1107 void MacroAssembler::addptr(Register dst, Register src) { 1108 LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src)); 1109 } 1110 1111 void MacroAssembler::addptr(Address dst, Register src) { 1112 LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src)); 1113 } 1114 1115 void MacroAssembler::addsd(XMMRegister dst, AddressLiteral src, Register rscratch) { 1116 assert(rscratch != noreg || always_reachable(src), "missing"); 1117 1118 if (reachable(src)) { 1119 Assembler::addsd(dst, as_Address(src)); 1120 } else { 1121 lea(rscratch, src); 1122 Assembler::addsd(dst, Address(rscratch, 0)); 1123 } 1124 } 1125 1126 void MacroAssembler::addss(XMMRegister dst, AddressLiteral src, Register rscratch) { 1127 assert(rscratch != noreg || always_reachable(src), "missing"); 1128 1129 if (reachable(src)) { 1130 addss(dst, as_Address(src)); 1131 } else { 1132 lea(rscratch, src); 1133 addss(dst, Address(rscratch, 0)); 1134 } 1135 } 1136 1137 void MacroAssembler::addpd(XMMRegister dst, AddressLiteral src, Register rscratch) { 1138 assert(rscratch != noreg || always_reachable(src), "missing"); 1139 1140 if (reachable(src)) { 1141 Assembler::addpd(dst, as_Address(src)); 1142 } else { 1143 lea(rscratch, src); 1144 Assembler::addpd(dst, Address(rscratch, 0)); 1145 } 1146 } 1147 1148 // See 8273459. Function for ensuring 64-byte alignment, intended for stubs only. 1149 // Stub code is generated once and never copied. 1150 // NMethods can't use this because they get copied and we can't force alignment > 32 bytes. 1151 void MacroAssembler::align64() { 1152 align(64, (uint)(uintptr_t)pc()); 1153 } 1154 1155 void MacroAssembler::align32() { 1156 align(32, (uint)(uintptr_t)pc()); 1157 } 1158 1159 void MacroAssembler::align(uint modulus) { 1160 // 8273459: Ensure alignment is possible with current segment alignment 1161 assert(modulus <= (uintx)CodeEntryAlignment, "Alignment must be <= CodeEntryAlignment"); 1162 align(modulus, offset()); 1163 } 1164 1165 void MacroAssembler::align(uint modulus, uint target) { 1166 if (target % modulus != 0) { 1167 nop(modulus - (target % modulus)); 1168 } 1169 } 1170 1171 void MacroAssembler::push_f(XMMRegister r) { 1172 subptr(rsp, wordSize); 1173 movflt(Address(rsp, 0), r); 1174 } 1175 1176 void MacroAssembler::pop_f(XMMRegister r) { 1177 movflt(r, Address(rsp, 0)); 1178 addptr(rsp, wordSize); 1179 } 1180 1181 void MacroAssembler::push_d(XMMRegister r) { 1182 subptr(rsp, 2 * wordSize); 1183 movdbl(Address(rsp, 0), r); 1184 } 1185 1186 void MacroAssembler::pop_d(XMMRegister r) { 1187 movdbl(r, Address(rsp, 0)); 1188 addptr(rsp, 2 * Interpreter::stackElementSize); 1189 } 1190 1191 void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src, Register rscratch) { 1192 // Used in sign-masking with aligned address. 1193 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 1194 assert(rscratch != noreg || always_reachable(src), "missing"); 1195 1196 if (UseAVX > 2 && 1197 (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) && 1198 (dst->encoding() >= 16)) { 1199 vpand(dst, dst, src, AVX_512bit, rscratch); 1200 } else if (reachable(src)) { 1201 Assembler::andpd(dst, as_Address(src)); 1202 } else { 1203 lea(rscratch, src); 1204 Assembler::andpd(dst, Address(rscratch, 0)); 1205 } 1206 } 1207 1208 void MacroAssembler::andps(XMMRegister dst, AddressLiteral src, Register rscratch) { 1209 // Used in sign-masking with aligned address. 1210 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 1211 assert(rscratch != noreg || always_reachable(src), "missing"); 1212 1213 if (reachable(src)) { 1214 Assembler::andps(dst, as_Address(src)); 1215 } else { 1216 lea(rscratch, src); 1217 Assembler::andps(dst, Address(rscratch, 0)); 1218 } 1219 } 1220 1221 void MacroAssembler::andptr(Register dst, int32_t imm32) { 1222 LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32)); 1223 } 1224 1225 #ifdef _LP64 1226 void MacroAssembler::andq(Register dst, AddressLiteral src, Register rscratch) { 1227 assert(rscratch != noreg || always_reachable(src), "missing"); 1228 1229 if (reachable(src)) { 1230 andq(dst, as_Address(src)); 1231 } else { 1232 lea(rscratch, src); 1233 andq(dst, Address(rscratch, 0)); 1234 } 1235 } 1236 #endif 1237 1238 void MacroAssembler::atomic_incl(Address counter_addr) { 1239 lock(); 1240 incrementl(counter_addr); 1241 } 1242 1243 void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register rscratch) { 1244 assert(rscratch != noreg || always_reachable(counter_addr), "missing"); 1245 1246 if (reachable(counter_addr)) { 1247 atomic_incl(as_Address(counter_addr)); 1248 } else { 1249 lea(rscratch, counter_addr); 1250 atomic_incl(Address(rscratch, 0)); 1251 } 1252 } 1253 1254 #ifdef _LP64 1255 void MacroAssembler::atomic_incq(Address counter_addr) { 1256 lock(); 1257 incrementq(counter_addr); 1258 } 1259 1260 void MacroAssembler::atomic_incq(AddressLiteral counter_addr, Register rscratch) { 1261 assert(rscratch != noreg || always_reachable(counter_addr), "missing"); 1262 1263 if (reachable(counter_addr)) { 1264 atomic_incq(as_Address(counter_addr)); 1265 } else { 1266 lea(rscratch, counter_addr); 1267 atomic_incq(Address(rscratch, 0)); 1268 } 1269 } 1270 #endif 1271 1272 // Writes to stack successive pages until offset reached to check for 1273 // stack overflow + shadow pages. This clobbers tmp. 1274 void MacroAssembler::bang_stack_size(Register size, Register tmp) { 1275 movptr(tmp, rsp); 1276 // Bang stack for total size given plus shadow page size. 1277 // Bang one page at a time because large size can bang beyond yellow and 1278 // red zones. 1279 Label loop; 1280 bind(loop); 1281 movl(Address(tmp, (-(int)os::vm_page_size())), size ); 1282 subptr(tmp, (int)os::vm_page_size()); 1283 subl(size, (int)os::vm_page_size()); 1284 jcc(Assembler::greater, loop); 1285 1286 // Bang down shadow pages too. 1287 // At this point, (tmp-0) is the last address touched, so don't 1288 // touch it again. (It was touched as (tmp-pagesize) but then tmp 1289 // was post-decremented.) Skip this address by starting at i=1, and 1290 // touch a few more pages below. N.B. It is important to touch all 1291 // the way down including all pages in the shadow zone. 1292 for (int i = 1; i < ((int)StackOverflow::stack_shadow_zone_size() / (int)os::vm_page_size()); i++) { 1293 // this could be any sized move but this is can be a debugging crumb 1294 // so the bigger the better. 1295 movptr(Address(tmp, (-i*(int)os::vm_page_size())), size ); 1296 } 1297 } 1298 1299 void MacroAssembler::reserved_stack_check() { 1300 // testing if reserved zone needs to be enabled 1301 Label no_reserved_zone_enabling; 1302 Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread); 1303 NOT_LP64(get_thread(rsi);) 1304 1305 cmpptr(rsp, Address(thread, JavaThread::reserved_stack_activation_offset())); 1306 jcc(Assembler::below, no_reserved_zone_enabling); 1307 1308 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), thread); 1309 jump(RuntimeAddress(SharedRuntime::throw_delayed_StackOverflowError_entry())); 1310 should_not_reach_here(); 1311 1312 bind(no_reserved_zone_enabling); 1313 } 1314 1315 void MacroAssembler::c2bool(Register x) { 1316 // implements x == 0 ? 0 : 1 1317 // note: must only look at least-significant byte of x 1318 // since C-style booleans are stored in one byte 1319 // only! (was bug) 1320 andl(x, 0xFF); 1321 setb(Assembler::notZero, x); 1322 } 1323 1324 // Wouldn't need if AddressLiteral version had new name 1325 void MacroAssembler::call(Label& L, relocInfo::relocType rtype) { 1326 Assembler::call(L, rtype); 1327 } 1328 1329 void MacroAssembler::call(Register entry) { 1330 Assembler::call(entry); 1331 } 1332 1333 void MacroAssembler::call(AddressLiteral entry, Register rscratch) { 1334 assert(rscratch != noreg || always_reachable(entry), "missing"); 1335 1336 if (reachable(entry)) { 1337 Assembler::call_literal(entry.target(), entry.rspec()); 1338 } else { 1339 lea(rscratch, entry); 1340 Assembler::call(rscratch); 1341 } 1342 } 1343 1344 void MacroAssembler::ic_call(address entry, jint method_index) { 1345 RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index); 1346 #ifdef _LP64 1347 // Needs full 64-bit immediate for later patching. 1348 mov64(rax, (int64_t)Universe::non_oop_word()); 1349 #else 1350 movptr(rax, (intptr_t)Universe::non_oop_word()); 1351 #endif 1352 call(AddressLiteral(entry, rh)); 1353 } 1354 1355 int MacroAssembler::ic_check_size() { 1356 return 1357 LP64_ONLY(UseCompactObjectHeaders ? 17 : 14) NOT_LP64(12); 1358 } 1359 1360 int MacroAssembler::ic_check(int end_alignment) { 1361 Register receiver = LP64_ONLY(j_rarg0) NOT_LP64(rcx); 1362 Register data = rax; 1363 Register temp = LP64_ONLY(rscratch1) NOT_LP64(rbx); 1364 1365 // The UEP of a code blob ensures that the VEP is padded. However, the padding of the UEP is placed 1366 // before the inline cache check, so we don't have to execute any nop instructions when dispatching 1367 // through the UEP, yet we can ensure that the VEP is aligned appropriately. That's why we align 1368 // before the inline cache check here, and not after 1369 align(end_alignment, offset() + ic_check_size()); 1370 1371 int uep_offset = offset(); 1372 1373 #ifdef _LP64 1374 if (UseCompactObjectHeaders) { 1375 load_narrow_klass_compact(temp, receiver); 1376 cmpl(temp, Address(data, CompiledICData::speculated_klass_offset())); 1377 } else 1378 #endif 1379 if (UseCompressedClassPointers) { 1380 movl(temp, Address(receiver, oopDesc::klass_offset_in_bytes())); 1381 cmpl(temp, Address(data, CompiledICData::speculated_klass_offset())); 1382 } else { 1383 movptr(temp, Address(receiver, oopDesc::klass_offset_in_bytes())); 1384 cmpptr(temp, Address(data, CompiledICData::speculated_klass_offset())); 1385 } 1386 1387 // if inline cache check fails, then jump to runtime routine 1388 jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub())); 1389 assert((offset() % end_alignment) == 0, "Misaligned verified entry point (%d, %d, %d)", uep_offset, offset(), end_alignment); 1390 1391 return uep_offset; 1392 } 1393 1394 void MacroAssembler::emit_static_call_stub() { 1395 // Static stub relocation also tags the Method* in the code-stream. 1396 mov_metadata(rbx, (Metadata*) nullptr); // Method is zapped till fixup time. 1397 // This is recognized as unresolved by relocs/nativeinst/ic code. 1398 jump(RuntimeAddress(pc())); 1399 } 1400 1401 // Implementation of call_VM versions 1402 1403 void MacroAssembler::call_VM(Register oop_result, 1404 address entry_point, 1405 bool check_exceptions) { 1406 Label C, E; 1407 call(C, relocInfo::none); 1408 jmp(E); 1409 1410 bind(C); 1411 call_VM_helper(oop_result, entry_point, 0, check_exceptions); 1412 ret(0); 1413 1414 bind(E); 1415 } 1416 1417 void MacroAssembler::call_VM(Register oop_result, 1418 address entry_point, 1419 Register arg_1, 1420 bool check_exceptions) { 1421 Label C, E; 1422 call(C, relocInfo::none); 1423 jmp(E); 1424 1425 bind(C); 1426 pass_arg1(this, arg_1); 1427 call_VM_helper(oop_result, entry_point, 1, check_exceptions); 1428 ret(0); 1429 1430 bind(E); 1431 } 1432 1433 void MacroAssembler::call_VM(Register oop_result, 1434 address entry_point, 1435 Register arg_1, 1436 Register arg_2, 1437 bool check_exceptions) { 1438 Label C, E; 1439 call(C, relocInfo::none); 1440 jmp(E); 1441 1442 bind(C); 1443 1444 LP64_ONLY(assert_different_registers(arg_1, c_rarg2)); 1445 1446 pass_arg2(this, arg_2); 1447 pass_arg1(this, arg_1); 1448 call_VM_helper(oop_result, entry_point, 2, check_exceptions); 1449 ret(0); 1450 1451 bind(E); 1452 } 1453 1454 void MacroAssembler::call_VM(Register oop_result, 1455 address entry_point, 1456 Register arg_1, 1457 Register arg_2, 1458 Register arg_3, 1459 bool check_exceptions) { 1460 Label C, E; 1461 call(C, relocInfo::none); 1462 jmp(E); 1463 1464 bind(C); 1465 1466 LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3)); 1467 LP64_ONLY(assert_different_registers(arg_2, c_rarg3)); 1468 pass_arg3(this, arg_3); 1469 pass_arg2(this, arg_2); 1470 pass_arg1(this, arg_1); 1471 call_VM_helper(oop_result, entry_point, 3, check_exceptions); 1472 ret(0); 1473 1474 bind(E); 1475 } 1476 1477 void MacroAssembler::call_VM(Register oop_result, 1478 Register last_java_sp, 1479 address entry_point, 1480 int number_of_arguments, 1481 bool check_exceptions) { 1482 Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg); 1483 call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions); 1484 } 1485 1486 void MacroAssembler::call_VM(Register oop_result, 1487 Register last_java_sp, 1488 address entry_point, 1489 Register arg_1, 1490 bool check_exceptions) { 1491 pass_arg1(this, arg_1); 1492 call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions); 1493 } 1494 1495 void MacroAssembler::call_VM(Register oop_result, 1496 Register last_java_sp, 1497 address entry_point, 1498 Register arg_1, 1499 Register arg_2, 1500 bool check_exceptions) { 1501 1502 LP64_ONLY(assert_different_registers(arg_1, c_rarg2)); 1503 pass_arg2(this, arg_2); 1504 pass_arg1(this, arg_1); 1505 call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions); 1506 } 1507 1508 void MacroAssembler::call_VM(Register oop_result, 1509 Register last_java_sp, 1510 address entry_point, 1511 Register arg_1, 1512 Register arg_2, 1513 Register arg_3, 1514 bool check_exceptions) { 1515 LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3)); 1516 LP64_ONLY(assert_different_registers(arg_2, c_rarg3)); 1517 pass_arg3(this, arg_3); 1518 pass_arg2(this, arg_2); 1519 pass_arg1(this, arg_1); 1520 call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions); 1521 } 1522 1523 void MacroAssembler::super_call_VM(Register oop_result, 1524 Register last_java_sp, 1525 address entry_point, 1526 int number_of_arguments, 1527 bool check_exceptions) { 1528 Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg); 1529 MacroAssembler::call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions); 1530 } 1531 1532 void MacroAssembler::super_call_VM(Register oop_result, 1533 Register last_java_sp, 1534 address entry_point, 1535 Register arg_1, 1536 bool check_exceptions) { 1537 pass_arg1(this, arg_1); 1538 super_call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions); 1539 } 1540 1541 void MacroAssembler::super_call_VM(Register oop_result, 1542 Register last_java_sp, 1543 address entry_point, 1544 Register arg_1, 1545 Register arg_2, 1546 bool check_exceptions) { 1547 1548 LP64_ONLY(assert_different_registers(arg_1, c_rarg2)); 1549 pass_arg2(this, arg_2); 1550 pass_arg1(this, arg_1); 1551 super_call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions); 1552 } 1553 1554 void MacroAssembler::super_call_VM(Register oop_result, 1555 Register last_java_sp, 1556 address entry_point, 1557 Register arg_1, 1558 Register arg_2, 1559 Register arg_3, 1560 bool check_exceptions) { 1561 LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3)); 1562 LP64_ONLY(assert_different_registers(arg_2, c_rarg3)); 1563 pass_arg3(this, arg_3); 1564 pass_arg2(this, arg_2); 1565 pass_arg1(this, arg_1); 1566 super_call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions); 1567 } 1568 1569 void MacroAssembler::call_VM_base(Register oop_result, 1570 Register java_thread, 1571 Register last_java_sp, 1572 address entry_point, 1573 int number_of_arguments, 1574 bool check_exceptions) { 1575 // determine java_thread register 1576 if (!java_thread->is_valid()) { 1577 #ifdef _LP64 1578 java_thread = r15_thread; 1579 #else 1580 java_thread = rdi; 1581 get_thread(java_thread); 1582 #endif // LP64 1583 } 1584 // determine last_java_sp register 1585 if (!last_java_sp->is_valid()) { 1586 last_java_sp = rsp; 1587 } 1588 // debugging support 1589 assert(number_of_arguments >= 0 , "cannot have negative number of arguments"); 1590 LP64_ONLY(assert(java_thread == r15_thread, "unexpected register")); 1591 #ifdef ASSERT 1592 // TraceBytecodes does not use r12 but saves it over the call, so don't verify 1593 // r12 is the heapbase. 1594 LP64_ONLY(if (UseCompressedOops && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");) 1595 #endif // ASSERT 1596 1597 assert(java_thread != oop_result , "cannot use the same register for java_thread & oop_result"); 1598 assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp"); 1599 1600 // push java thread (becomes first argument of C function) 1601 1602 NOT_LP64(push(java_thread); number_of_arguments++); 1603 LP64_ONLY(mov(c_rarg0, r15_thread)); 1604 1605 // set last Java frame before call 1606 assert(last_java_sp != rbp, "can't use ebp/rbp"); 1607 1608 // Only interpreter should have to set fp 1609 set_last_Java_frame(java_thread, last_java_sp, rbp, nullptr, rscratch1); 1610 1611 // do the call, remove parameters 1612 MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments); 1613 1614 // restore the thread (cannot use the pushed argument since arguments 1615 // may be overwritten by C code generated by an optimizing compiler); 1616 // however can use the register value directly if it is callee saved. 1617 if (LP64_ONLY(true ||) java_thread == rdi || java_thread == rsi) { 1618 // rdi & rsi (also r15) are callee saved -> nothing to do 1619 #ifdef ASSERT 1620 guarantee(java_thread != rax, "change this code"); 1621 push(rax); 1622 { Label L; 1623 get_thread(rax); 1624 cmpptr(java_thread, rax); 1625 jcc(Assembler::equal, L); 1626 STOP("MacroAssembler::call_VM_base: rdi not callee saved?"); 1627 bind(L); 1628 } 1629 pop(rax); 1630 #endif 1631 } else { 1632 get_thread(java_thread); 1633 } 1634 // reset last Java frame 1635 // Only interpreter should have to clear fp 1636 reset_last_Java_frame(java_thread, true); 1637 1638 // C++ interp handles this in the interpreter 1639 check_and_handle_popframe(java_thread); 1640 check_and_handle_earlyret(java_thread); 1641 1642 if (check_exceptions) { 1643 // check for pending exceptions (java_thread is set upon return) 1644 cmpptr(Address(java_thread, Thread::pending_exception_offset()), NULL_WORD); 1645 #ifndef _LP64 1646 jump_cc(Assembler::notEqual, 1647 RuntimeAddress(StubRoutines::forward_exception_entry())); 1648 #else 1649 // This used to conditionally jump to forward_exception however it is 1650 // possible if we relocate that the branch will not reach. So we must jump 1651 // around so we can always reach 1652 1653 Label ok; 1654 jcc(Assembler::equal, ok); 1655 jump(RuntimeAddress(StubRoutines::forward_exception_entry())); 1656 bind(ok); 1657 #endif // LP64 1658 } 1659 1660 // get oop result if there is one and reset the value in the thread 1661 if (oop_result->is_valid()) { 1662 get_vm_result(oop_result, java_thread); 1663 } 1664 } 1665 1666 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) { 1667 1668 // Calculate the value for last_Java_sp 1669 // somewhat subtle. call_VM does an intermediate call 1670 // which places a return address on the stack just under the 1671 // stack pointer as the user finished with it. This allows 1672 // use to retrieve last_Java_pc from last_Java_sp[-1]. 1673 // On 32bit we then have to push additional args on the stack to accomplish 1674 // the actual requested call. On 64bit call_VM only can use register args 1675 // so the only extra space is the return address that call_VM created. 1676 // This hopefully explains the calculations here. 1677 1678 #ifdef _LP64 1679 // We've pushed one address, correct last_Java_sp 1680 lea(rax, Address(rsp, wordSize)); 1681 #else 1682 lea(rax, Address(rsp, (1 + number_of_arguments) * wordSize)); 1683 #endif // LP64 1684 1685 call_VM_base(oop_result, noreg, rax, entry_point, number_of_arguments, check_exceptions); 1686 1687 } 1688 1689 // Use this method when MacroAssembler version of call_VM_leaf_base() should be called from Interpreter. 1690 void MacroAssembler::call_VM_leaf0(address entry_point) { 1691 MacroAssembler::call_VM_leaf_base(entry_point, 0); 1692 } 1693 1694 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) { 1695 call_VM_leaf_base(entry_point, number_of_arguments); 1696 } 1697 1698 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) { 1699 pass_arg0(this, arg_0); 1700 call_VM_leaf(entry_point, 1); 1701 } 1702 1703 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) { 1704 1705 LP64_ONLY(assert_different_registers(arg_0, c_rarg1)); 1706 pass_arg1(this, arg_1); 1707 pass_arg0(this, arg_0); 1708 call_VM_leaf(entry_point, 2); 1709 } 1710 1711 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) { 1712 LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2)); 1713 LP64_ONLY(assert_different_registers(arg_1, c_rarg2)); 1714 pass_arg2(this, arg_2); 1715 pass_arg1(this, arg_1); 1716 pass_arg0(this, arg_0); 1717 call_VM_leaf(entry_point, 3); 1718 } 1719 1720 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) { 1721 LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3)); 1722 LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3)); 1723 LP64_ONLY(assert_different_registers(arg_2, c_rarg3)); 1724 pass_arg3(this, arg_3); 1725 pass_arg2(this, arg_2); 1726 pass_arg1(this, arg_1); 1727 pass_arg0(this, arg_0); 1728 call_VM_leaf(entry_point, 3); 1729 } 1730 1731 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) { 1732 pass_arg0(this, arg_0); 1733 MacroAssembler::call_VM_leaf_base(entry_point, 1); 1734 } 1735 1736 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) { 1737 LP64_ONLY(assert_different_registers(arg_0, c_rarg1)); 1738 pass_arg1(this, arg_1); 1739 pass_arg0(this, arg_0); 1740 MacroAssembler::call_VM_leaf_base(entry_point, 2); 1741 } 1742 1743 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) { 1744 LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2)); 1745 LP64_ONLY(assert_different_registers(arg_1, c_rarg2)); 1746 pass_arg2(this, arg_2); 1747 pass_arg1(this, arg_1); 1748 pass_arg0(this, arg_0); 1749 MacroAssembler::call_VM_leaf_base(entry_point, 3); 1750 } 1751 1752 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) { 1753 LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3)); 1754 LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3)); 1755 LP64_ONLY(assert_different_registers(arg_2, c_rarg3)); 1756 pass_arg3(this, arg_3); 1757 pass_arg2(this, arg_2); 1758 pass_arg1(this, arg_1); 1759 pass_arg0(this, arg_0); 1760 MacroAssembler::call_VM_leaf_base(entry_point, 4); 1761 } 1762 1763 void MacroAssembler::get_vm_result(Register oop_result, Register java_thread) { 1764 movptr(oop_result, Address(java_thread, JavaThread::vm_result_offset())); 1765 movptr(Address(java_thread, JavaThread::vm_result_offset()), NULL_WORD); 1766 verify_oop_msg(oop_result, "broken oop in call_VM_base"); 1767 } 1768 1769 void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) { 1770 movptr(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset())); 1771 movptr(Address(java_thread, JavaThread::vm_result_2_offset()), NULL_WORD); 1772 } 1773 1774 void MacroAssembler::check_and_handle_earlyret(Register java_thread) { 1775 } 1776 1777 void MacroAssembler::check_and_handle_popframe(Register java_thread) { 1778 } 1779 1780 void MacroAssembler::cmp32(AddressLiteral src1, int32_t imm, Register rscratch) { 1781 assert(rscratch != noreg || always_reachable(src1), "missing"); 1782 1783 if (reachable(src1)) { 1784 cmpl(as_Address(src1), imm); 1785 } else { 1786 lea(rscratch, src1); 1787 cmpl(Address(rscratch, 0), imm); 1788 } 1789 } 1790 1791 void MacroAssembler::cmp32(Register src1, AddressLiteral src2, Register rscratch) { 1792 assert(!src2.is_lval(), "use cmpptr"); 1793 assert(rscratch != noreg || always_reachable(src2), "missing"); 1794 1795 if (reachable(src2)) { 1796 cmpl(src1, as_Address(src2)); 1797 } else { 1798 lea(rscratch, src2); 1799 cmpl(src1, Address(rscratch, 0)); 1800 } 1801 } 1802 1803 void MacroAssembler::cmp32(Register src1, int32_t imm) { 1804 Assembler::cmpl(src1, imm); 1805 } 1806 1807 void MacroAssembler::cmp32(Register src1, Address src2) { 1808 Assembler::cmpl(src1, src2); 1809 } 1810 1811 void MacroAssembler::cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) { 1812 ucomisd(opr1, opr2); 1813 1814 Label L; 1815 if (unordered_is_less) { 1816 movl(dst, -1); 1817 jcc(Assembler::parity, L); 1818 jcc(Assembler::below , L); 1819 movl(dst, 0); 1820 jcc(Assembler::equal , L); 1821 increment(dst); 1822 } else { // unordered is greater 1823 movl(dst, 1); 1824 jcc(Assembler::parity, L); 1825 jcc(Assembler::above , L); 1826 movl(dst, 0); 1827 jcc(Assembler::equal , L); 1828 decrementl(dst); 1829 } 1830 bind(L); 1831 } 1832 1833 void MacroAssembler::cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) { 1834 ucomiss(opr1, opr2); 1835 1836 Label L; 1837 if (unordered_is_less) { 1838 movl(dst, -1); 1839 jcc(Assembler::parity, L); 1840 jcc(Assembler::below , L); 1841 movl(dst, 0); 1842 jcc(Assembler::equal , L); 1843 increment(dst); 1844 } else { // unordered is greater 1845 movl(dst, 1); 1846 jcc(Assembler::parity, L); 1847 jcc(Assembler::above , L); 1848 movl(dst, 0); 1849 jcc(Assembler::equal , L); 1850 decrementl(dst); 1851 } 1852 bind(L); 1853 } 1854 1855 1856 void MacroAssembler::cmp8(AddressLiteral src1, int imm, Register rscratch) { 1857 assert(rscratch != noreg || always_reachable(src1), "missing"); 1858 1859 if (reachable(src1)) { 1860 cmpb(as_Address(src1), imm); 1861 } else { 1862 lea(rscratch, src1); 1863 cmpb(Address(rscratch, 0), imm); 1864 } 1865 } 1866 1867 void MacroAssembler::cmpptr(Register src1, AddressLiteral src2, Register rscratch) { 1868 #ifdef _LP64 1869 assert(rscratch != noreg || always_reachable(src2), "missing"); 1870 1871 if (src2.is_lval()) { 1872 movptr(rscratch, src2); 1873 Assembler::cmpq(src1, rscratch); 1874 } else if (reachable(src2)) { 1875 cmpq(src1, as_Address(src2)); 1876 } else { 1877 lea(rscratch, src2); 1878 Assembler::cmpq(src1, Address(rscratch, 0)); 1879 } 1880 #else 1881 assert(rscratch == noreg, "not needed"); 1882 if (src2.is_lval()) { 1883 cmp_literal32(src1, (int32_t)src2.target(), src2.rspec()); 1884 } else { 1885 cmpl(src1, as_Address(src2)); 1886 } 1887 #endif // _LP64 1888 } 1889 1890 void MacroAssembler::cmpptr(Address src1, AddressLiteral src2, Register rscratch) { 1891 assert(src2.is_lval(), "not a mem-mem compare"); 1892 #ifdef _LP64 1893 // moves src2's literal address 1894 movptr(rscratch, src2); 1895 Assembler::cmpq(src1, rscratch); 1896 #else 1897 assert(rscratch == noreg, "not needed"); 1898 cmp_literal32(src1, (int32_t)src2.target(), src2.rspec()); 1899 #endif // _LP64 1900 } 1901 1902 void MacroAssembler::cmpoop(Register src1, Register src2) { 1903 cmpptr(src1, src2); 1904 } 1905 1906 void MacroAssembler::cmpoop(Register src1, Address src2) { 1907 cmpptr(src1, src2); 1908 } 1909 1910 #ifdef _LP64 1911 void MacroAssembler::cmpoop(Register src1, jobject src2, Register rscratch) { 1912 movoop(rscratch, src2); 1913 cmpptr(src1, rscratch); 1914 } 1915 #endif 1916 1917 void MacroAssembler::locked_cmpxchgptr(Register reg, AddressLiteral adr, Register rscratch) { 1918 assert(rscratch != noreg || always_reachable(adr), "missing"); 1919 1920 if (reachable(adr)) { 1921 lock(); 1922 cmpxchgptr(reg, as_Address(adr)); 1923 } else { 1924 lea(rscratch, adr); 1925 lock(); 1926 cmpxchgptr(reg, Address(rscratch, 0)); 1927 } 1928 } 1929 1930 void MacroAssembler::cmpxchgptr(Register reg, Address adr) { 1931 LP64_ONLY(cmpxchgq(reg, adr)) NOT_LP64(cmpxchgl(reg, adr)); 1932 } 1933 1934 void MacroAssembler::comisd(XMMRegister dst, AddressLiteral src, Register rscratch) { 1935 assert(rscratch != noreg || always_reachable(src), "missing"); 1936 1937 if (reachable(src)) { 1938 Assembler::comisd(dst, as_Address(src)); 1939 } else { 1940 lea(rscratch, src); 1941 Assembler::comisd(dst, Address(rscratch, 0)); 1942 } 1943 } 1944 1945 void MacroAssembler::comiss(XMMRegister dst, AddressLiteral src, Register rscratch) { 1946 assert(rscratch != noreg || always_reachable(src), "missing"); 1947 1948 if (reachable(src)) { 1949 Assembler::comiss(dst, as_Address(src)); 1950 } else { 1951 lea(rscratch, src); 1952 Assembler::comiss(dst, Address(rscratch, 0)); 1953 } 1954 } 1955 1956 1957 void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr, Register rscratch) { 1958 assert(rscratch != noreg || always_reachable(counter_addr), "missing"); 1959 1960 Condition negated_cond = negate_condition(cond); 1961 Label L; 1962 jcc(negated_cond, L); 1963 pushf(); // Preserve flags 1964 atomic_incl(counter_addr, rscratch); 1965 popf(); 1966 bind(L); 1967 } 1968 1969 int MacroAssembler::corrected_idivl(Register reg) { 1970 // Full implementation of Java idiv and irem; checks for 1971 // special case as described in JVM spec., p.243 & p.271. 1972 // The function returns the (pc) offset of the idivl 1973 // instruction - may be needed for implicit exceptions. 1974 // 1975 // normal case special case 1976 // 1977 // input : rax,: dividend min_int 1978 // reg: divisor (may not be rax,/rdx) -1 1979 // 1980 // output: rax,: quotient (= rax, idiv reg) min_int 1981 // rdx: remainder (= rax, irem reg) 0 1982 assert(reg != rax && reg != rdx, "reg cannot be rax, or rdx register"); 1983 const int min_int = 0x80000000; 1984 Label normal_case, special_case; 1985 1986 // check for special case 1987 cmpl(rax, min_int); 1988 jcc(Assembler::notEqual, normal_case); 1989 xorl(rdx, rdx); // prepare rdx for possible special case (where remainder = 0) 1990 cmpl(reg, -1); 1991 jcc(Assembler::equal, special_case); 1992 1993 // handle normal case 1994 bind(normal_case); 1995 cdql(); 1996 int idivl_offset = offset(); 1997 idivl(reg); 1998 1999 // normal and special case exit 2000 bind(special_case); 2001 2002 return idivl_offset; 2003 } 2004 2005 2006 2007 void MacroAssembler::decrementl(Register reg, int value) { 2008 if (value == min_jint) {subl(reg, value) ; return; } 2009 if (value < 0) { incrementl(reg, -value); return; } 2010 if (value == 0) { ; return; } 2011 if (value == 1 && UseIncDec) { decl(reg) ; return; } 2012 /* else */ { subl(reg, value) ; return; } 2013 } 2014 2015 void MacroAssembler::decrementl(Address dst, int value) { 2016 if (value == min_jint) {subl(dst, value) ; return; } 2017 if (value < 0) { incrementl(dst, -value); return; } 2018 if (value == 0) { ; return; } 2019 if (value == 1 && UseIncDec) { decl(dst) ; return; } 2020 /* else */ { subl(dst, value) ; return; } 2021 } 2022 2023 void MacroAssembler::division_with_shift (Register reg, int shift_value) { 2024 assert(shift_value > 0, "illegal shift value"); 2025 Label _is_positive; 2026 testl (reg, reg); 2027 jcc (Assembler::positive, _is_positive); 2028 int offset = (1 << shift_value) - 1 ; 2029 2030 if (offset == 1) { 2031 incrementl(reg); 2032 } else { 2033 addl(reg, offset); 2034 } 2035 2036 bind (_is_positive); 2037 sarl(reg, shift_value); 2038 } 2039 2040 void MacroAssembler::divsd(XMMRegister dst, AddressLiteral src, Register rscratch) { 2041 assert(rscratch != noreg || always_reachable(src), "missing"); 2042 2043 if (reachable(src)) { 2044 Assembler::divsd(dst, as_Address(src)); 2045 } else { 2046 lea(rscratch, src); 2047 Assembler::divsd(dst, Address(rscratch, 0)); 2048 } 2049 } 2050 2051 void MacroAssembler::divss(XMMRegister dst, AddressLiteral src, Register rscratch) { 2052 assert(rscratch != noreg || always_reachable(src), "missing"); 2053 2054 if (reachable(src)) { 2055 Assembler::divss(dst, as_Address(src)); 2056 } else { 2057 lea(rscratch, src); 2058 Assembler::divss(dst, Address(rscratch, 0)); 2059 } 2060 } 2061 2062 void MacroAssembler::enter() { 2063 push(rbp); 2064 mov(rbp, rsp); 2065 } 2066 2067 void MacroAssembler::post_call_nop() { 2068 if (!Continuations::enabled()) { 2069 return; 2070 } 2071 InstructionMark im(this); 2072 relocate(post_call_nop_Relocation::spec()); 2073 InlineSkippedInstructionsCounter skipCounter(this); 2074 emit_int8((uint8_t)0x0f); 2075 emit_int8((uint8_t)0x1f); 2076 emit_int8((uint8_t)0x84); 2077 emit_int8((uint8_t)0x00); 2078 emit_int32(0x00); 2079 } 2080 2081 // A 5 byte nop that is safe for patching (see patch_verified_entry) 2082 void MacroAssembler::fat_nop() { 2083 if (UseAddressNop) { 2084 addr_nop_5(); 2085 } else { 2086 emit_int8((uint8_t)0x26); // es: 2087 emit_int8((uint8_t)0x2e); // cs: 2088 emit_int8((uint8_t)0x64); // fs: 2089 emit_int8((uint8_t)0x65); // gs: 2090 emit_int8((uint8_t)0x90); 2091 } 2092 } 2093 2094 #ifndef _LP64 2095 void MacroAssembler::fcmp(Register tmp) { 2096 fcmp(tmp, 1, true, true); 2097 } 2098 2099 void MacroAssembler::fcmp(Register tmp, int index, bool pop_left, bool pop_right) { 2100 assert(!pop_right || pop_left, "usage error"); 2101 if (VM_Version::supports_cmov()) { 2102 assert(tmp == noreg, "unneeded temp"); 2103 if (pop_left) { 2104 fucomip(index); 2105 } else { 2106 fucomi(index); 2107 } 2108 if (pop_right) { 2109 fpop(); 2110 } 2111 } else { 2112 assert(tmp != noreg, "need temp"); 2113 if (pop_left) { 2114 if (pop_right) { 2115 fcompp(); 2116 } else { 2117 fcomp(index); 2118 } 2119 } else { 2120 fcom(index); 2121 } 2122 // convert FPU condition into eflags condition via rax, 2123 save_rax(tmp); 2124 fwait(); fnstsw_ax(); 2125 sahf(); 2126 restore_rax(tmp); 2127 } 2128 // condition codes set as follows: 2129 // 2130 // CF (corresponds to C0) if x < y 2131 // PF (corresponds to C2) if unordered 2132 // ZF (corresponds to C3) if x = y 2133 } 2134 2135 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less) { 2136 fcmp2int(dst, unordered_is_less, 1, true, true); 2137 } 2138 2139 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less, int index, bool pop_left, bool pop_right) { 2140 fcmp(VM_Version::supports_cmov() ? noreg : dst, index, pop_left, pop_right); 2141 Label L; 2142 if (unordered_is_less) { 2143 movl(dst, -1); 2144 jcc(Assembler::parity, L); 2145 jcc(Assembler::below , L); 2146 movl(dst, 0); 2147 jcc(Assembler::equal , L); 2148 increment(dst); 2149 } else { // unordered is greater 2150 movl(dst, 1); 2151 jcc(Assembler::parity, L); 2152 jcc(Assembler::above , L); 2153 movl(dst, 0); 2154 jcc(Assembler::equal , L); 2155 decrementl(dst); 2156 } 2157 bind(L); 2158 } 2159 2160 void MacroAssembler::fld_d(AddressLiteral src) { 2161 fld_d(as_Address(src)); 2162 } 2163 2164 void MacroAssembler::fld_s(AddressLiteral src) { 2165 fld_s(as_Address(src)); 2166 } 2167 2168 void MacroAssembler::fldcw(AddressLiteral src) { 2169 fldcw(as_Address(src)); 2170 } 2171 2172 void MacroAssembler::fpop() { 2173 ffree(); 2174 fincstp(); 2175 } 2176 2177 void MacroAssembler::fremr(Register tmp) { 2178 save_rax(tmp); 2179 { Label L; 2180 bind(L); 2181 fprem(); 2182 fwait(); fnstsw_ax(); 2183 sahf(); 2184 jcc(Assembler::parity, L); 2185 } 2186 restore_rax(tmp); 2187 // Result is in ST0. 2188 // Note: fxch & fpop to get rid of ST1 2189 // (otherwise FPU stack could overflow eventually) 2190 fxch(1); 2191 fpop(); 2192 } 2193 2194 void MacroAssembler::empty_FPU_stack() { 2195 if (VM_Version::supports_mmx()) { 2196 emms(); 2197 } else { 2198 for (int i = 8; i-- > 0; ) ffree(i); 2199 } 2200 } 2201 #endif // !LP64 2202 2203 void MacroAssembler::mulpd(XMMRegister dst, AddressLiteral src, Register rscratch) { 2204 assert(rscratch != noreg || always_reachable(src), "missing"); 2205 if (reachable(src)) { 2206 Assembler::mulpd(dst, as_Address(src)); 2207 } else { 2208 lea(rscratch, src); 2209 Assembler::mulpd(dst, Address(rscratch, 0)); 2210 } 2211 } 2212 2213 void MacroAssembler::load_float(Address src) { 2214 #ifdef _LP64 2215 movflt(xmm0, src); 2216 #else 2217 if (UseSSE >= 1) { 2218 movflt(xmm0, src); 2219 } else { 2220 fld_s(src); 2221 } 2222 #endif // LP64 2223 } 2224 2225 void MacroAssembler::store_float(Address dst) { 2226 #ifdef _LP64 2227 movflt(dst, xmm0); 2228 #else 2229 if (UseSSE >= 1) { 2230 movflt(dst, xmm0); 2231 } else { 2232 fstp_s(dst); 2233 } 2234 #endif // LP64 2235 } 2236 2237 void MacroAssembler::load_double(Address src) { 2238 #ifdef _LP64 2239 movdbl(xmm0, src); 2240 #else 2241 if (UseSSE >= 2) { 2242 movdbl(xmm0, src); 2243 } else { 2244 fld_d(src); 2245 } 2246 #endif // LP64 2247 } 2248 2249 void MacroAssembler::store_double(Address dst) { 2250 #ifdef _LP64 2251 movdbl(dst, xmm0); 2252 #else 2253 if (UseSSE >= 2) { 2254 movdbl(dst, xmm0); 2255 } else { 2256 fstp_d(dst); 2257 } 2258 #endif // LP64 2259 } 2260 2261 // dst = c = a * b + c 2262 void MacroAssembler::fmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) { 2263 Assembler::vfmadd231sd(c, a, b); 2264 if (dst != c) { 2265 movdbl(dst, c); 2266 } 2267 } 2268 2269 // dst = c = a * b + c 2270 void MacroAssembler::fmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) { 2271 Assembler::vfmadd231ss(c, a, b); 2272 if (dst != c) { 2273 movflt(dst, c); 2274 } 2275 } 2276 2277 // dst = c = a * b + c 2278 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) { 2279 Assembler::vfmadd231pd(c, a, b, vector_len); 2280 if (dst != c) { 2281 vmovdqu(dst, c); 2282 } 2283 } 2284 2285 // dst = c = a * b + c 2286 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) { 2287 Assembler::vfmadd231ps(c, a, b, vector_len); 2288 if (dst != c) { 2289 vmovdqu(dst, c); 2290 } 2291 } 2292 2293 // dst = c = a * b + c 2294 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) { 2295 Assembler::vfmadd231pd(c, a, b, vector_len); 2296 if (dst != c) { 2297 vmovdqu(dst, c); 2298 } 2299 } 2300 2301 // dst = c = a * b + c 2302 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) { 2303 Assembler::vfmadd231ps(c, a, b, vector_len); 2304 if (dst != c) { 2305 vmovdqu(dst, c); 2306 } 2307 } 2308 2309 void MacroAssembler::incrementl(AddressLiteral dst, Register rscratch) { 2310 assert(rscratch != noreg || always_reachable(dst), "missing"); 2311 2312 if (reachable(dst)) { 2313 incrementl(as_Address(dst)); 2314 } else { 2315 lea(rscratch, dst); 2316 incrementl(Address(rscratch, 0)); 2317 } 2318 } 2319 2320 void MacroAssembler::incrementl(ArrayAddress dst, Register rscratch) { 2321 incrementl(as_Address(dst, rscratch)); 2322 } 2323 2324 void MacroAssembler::incrementl(Register reg, int value) { 2325 if (value == min_jint) {addl(reg, value) ; return; } 2326 if (value < 0) { decrementl(reg, -value); return; } 2327 if (value == 0) { ; return; } 2328 if (value == 1 && UseIncDec) { incl(reg) ; return; } 2329 /* else */ { addl(reg, value) ; return; } 2330 } 2331 2332 void MacroAssembler::incrementl(Address dst, int value) { 2333 if (value == min_jint) {addl(dst, value) ; return; } 2334 if (value < 0) { decrementl(dst, -value); return; } 2335 if (value == 0) { ; return; } 2336 if (value == 1 && UseIncDec) { incl(dst) ; return; } 2337 /* else */ { addl(dst, value) ; return; } 2338 } 2339 2340 void MacroAssembler::jump(AddressLiteral dst, Register rscratch) { 2341 assert(rscratch != noreg || always_reachable(dst), "missing"); 2342 assert(!dst.rspec().reloc()->is_data(), "should not use ExternalAddress for jump"); 2343 if (reachable(dst)) { 2344 jmp_literal(dst.target(), dst.rspec()); 2345 } else { 2346 lea(rscratch, dst); 2347 jmp(rscratch); 2348 } 2349 } 2350 2351 void MacroAssembler::jump_cc(Condition cc, AddressLiteral dst, Register rscratch) { 2352 assert(rscratch != noreg || always_reachable(dst), "missing"); 2353 assert(!dst.rspec().reloc()->is_data(), "should not use ExternalAddress for jump_cc"); 2354 if (reachable(dst)) { 2355 InstructionMark im(this); 2356 relocate(dst.reloc()); 2357 const int short_size = 2; 2358 const int long_size = 6; 2359 int offs = (intptr_t)dst.target() - ((intptr_t)pc()); 2360 if (dst.reloc() == relocInfo::none && is8bit(offs - short_size)) { 2361 // 0111 tttn #8-bit disp 2362 emit_int8(0x70 | cc); 2363 emit_int8((offs - short_size) & 0xFF); 2364 } else { 2365 // 0000 1111 1000 tttn #32-bit disp 2366 emit_int8(0x0F); 2367 emit_int8((unsigned char)(0x80 | cc)); 2368 emit_int32(offs - long_size); 2369 } 2370 } else { 2371 #ifdef ASSERT 2372 warning("reversing conditional branch"); 2373 #endif /* ASSERT */ 2374 Label skip; 2375 jccb(reverse[cc], skip); 2376 lea(rscratch, dst); 2377 Assembler::jmp(rscratch); 2378 bind(skip); 2379 } 2380 } 2381 2382 void MacroAssembler::ldmxcsr(AddressLiteral src, Register rscratch) { 2383 assert(rscratch != noreg || always_reachable(src), "missing"); 2384 2385 if (reachable(src)) { 2386 Assembler::ldmxcsr(as_Address(src)); 2387 } else { 2388 lea(rscratch, src); 2389 Assembler::ldmxcsr(Address(rscratch, 0)); 2390 } 2391 } 2392 2393 int MacroAssembler::load_signed_byte(Register dst, Address src) { 2394 int off; 2395 if (LP64_ONLY(true ||) VM_Version::is_P6()) { 2396 off = offset(); 2397 movsbl(dst, src); // movsxb 2398 } else { 2399 off = load_unsigned_byte(dst, src); 2400 shll(dst, 24); 2401 sarl(dst, 24); 2402 } 2403 return off; 2404 } 2405 2406 // Note: load_signed_short used to be called load_signed_word. 2407 // Although the 'w' in x86 opcodes refers to the term "word" in the assembler 2408 // manual, which means 16 bits, that usage is found nowhere in HotSpot code. 2409 // The term "word" in HotSpot means a 32- or 64-bit machine word. 2410 int MacroAssembler::load_signed_short(Register dst, Address src) { 2411 int off; 2412 if (LP64_ONLY(true ||) VM_Version::is_P6()) { 2413 // This is dubious to me since it seems safe to do a signed 16 => 64 bit 2414 // version but this is what 64bit has always done. This seems to imply 2415 // that users are only using 32bits worth. 2416 off = offset(); 2417 movswl(dst, src); // movsxw 2418 } else { 2419 off = load_unsigned_short(dst, src); 2420 shll(dst, 16); 2421 sarl(dst, 16); 2422 } 2423 return off; 2424 } 2425 2426 int MacroAssembler::load_unsigned_byte(Register dst, Address src) { 2427 // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16, 2428 // and "3.9 Partial Register Penalties", p. 22). 2429 int off; 2430 if (LP64_ONLY(true || ) VM_Version::is_P6() || src.uses(dst)) { 2431 off = offset(); 2432 movzbl(dst, src); // movzxb 2433 } else { 2434 xorl(dst, dst); 2435 off = offset(); 2436 movb(dst, src); 2437 } 2438 return off; 2439 } 2440 2441 // Note: load_unsigned_short used to be called load_unsigned_word. 2442 int MacroAssembler::load_unsigned_short(Register dst, Address src) { 2443 // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16, 2444 // and "3.9 Partial Register Penalties", p. 22). 2445 int off; 2446 if (LP64_ONLY(true ||) VM_Version::is_P6() || src.uses(dst)) { 2447 off = offset(); 2448 movzwl(dst, src); // movzxw 2449 } else { 2450 xorl(dst, dst); 2451 off = offset(); 2452 movw(dst, src); 2453 } 2454 return off; 2455 } 2456 2457 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2) { 2458 switch (size_in_bytes) { 2459 #ifndef _LP64 2460 case 8: 2461 assert(dst2 != noreg, "second dest register required"); 2462 movl(dst, src); 2463 movl(dst2, src.plus_disp(BytesPerInt)); 2464 break; 2465 #else 2466 case 8: movq(dst, src); break; 2467 #endif 2468 case 4: movl(dst, src); break; 2469 case 2: is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break; 2470 case 1: is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break; 2471 default: ShouldNotReachHere(); 2472 } 2473 } 2474 2475 void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2) { 2476 switch (size_in_bytes) { 2477 #ifndef _LP64 2478 case 8: 2479 assert(src2 != noreg, "second source register required"); 2480 movl(dst, src); 2481 movl(dst.plus_disp(BytesPerInt), src2); 2482 break; 2483 #else 2484 case 8: movq(dst, src); break; 2485 #endif 2486 case 4: movl(dst, src); break; 2487 case 2: movw(dst, src); break; 2488 case 1: movb(dst, src); break; 2489 default: ShouldNotReachHere(); 2490 } 2491 } 2492 2493 void MacroAssembler::mov32(AddressLiteral dst, Register src, Register rscratch) { 2494 assert(rscratch != noreg || always_reachable(dst), "missing"); 2495 2496 if (reachable(dst)) { 2497 movl(as_Address(dst), src); 2498 } else { 2499 lea(rscratch, dst); 2500 movl(Address(rscratch, 0), src); 2501 } 2502 } 2503 2504 void MacroAssembler::mov32(Register dst, AddressLiteral src) { 2505 if (reachable(src)) { 2506 movl(dst, as_Address(src)); 2507 } else { 2508 lea(dst, src); 2509 movl(dst, Address(dst, 0)); 2510 } 2511 } 2512 2513 // C++ bool manipulation 2514 2515 void MacroAssembler::movbool(Register dst, Address src) { 2516 if(sizeof(bool) == 1) 2517 movb(dst, src); 2518 else if(sizeof(bool) == 2) 2519 movw(dst, src); 2520 else if(sizeof(bool) == 4) 2521 movl(dst, src); 2522 else 2523 // unsupported 2524 ShouldNotReachHere(); 2525 } 2526 2527 void MacroAssembler::movbool(Address dst, bool boolconst) { 2528 if(sizeof(bool) == 1) 2529 movb(dst, (int) boolconst); 2530 else if(sizeof(bool) == 2) 2531 movw(dst, (int) boolconst); 2532 else if(sizeof(bool) == 4) 2533 movl(dst, (int) boolconst); 2534 else 2535 // unsupported 2536 ShouldNotReachHere(); 2537 } 2538 2539 void MacroAssembler::movbool(Address dst, Register src) { 2540 if(sizeof(bool) == 1) 2541 movb(dst, src); 2542 else if(sizeof(bool) == 2) 2543 movw(dst, src); 2544 else if(sizeof(bool) == 4) 2545 movl(dst, src); 2546 else 2547 // unsupported 2548 ShouldNotReachHere(); 2549 } 2550 2551 void MacroAssembler::movdl(XMMRegister dst, AddressLiteral src, Register rscratch) { 2552 assert(rscratch != noreg || always_reachable(src), "missing"); 2553 2554 if (reachable(src)) { 2555 movdl(dst, as_Address(src)); 2556 } else { 2557 lea(rscratch, src); 2558 movdl(dst, Address(rscratch, 0)); 2559 } 2560 } 2561 2562 void MacroAssembler::movq(XMMRegister dst, AddressLiteral src, Register rscratch) { 2563 assert(rscratch != noreg || always_reachable(src), "missing"); 2564 2565 if (reachable(src)) { 2566 movq(dst, as_Address(src)); 2567 } else { 2568 lea(rscratch, src); 2569 movq(dst, Address(rscratch, 0)); 2570 } 2571 } 2572 2573 void MacroAssembler::movdbl(XMMRegister dst, AddressLiteral src, Register rscratch) { 2574 assert(rscratch != noreg || always_reachable(src), "missing"); 2575 2576 if (reachable(src)) { 2577 if (UseXmmLoadAndClearUpper) { 2578 movsd (dst, as_Address(src)); 2579 } else { 2580 movlpd(dst, as_Address(src)); 2581 } 2582 } else { 2583 lea(rscratch, src); 2584 if (UseXmmLoadAndClearUpper) { 2585 movsd (dst, Address(rscratch, 0)); 2586 } else { 2587 movlpd(dst, Address(rscratch, 0)); 2588 } 2589 } 2590 } 2591 2592 void MacroAssembler::movflt(XMMRegister dst, AddressLiteral src, Register rscratch) { 2593 assert(rscratch != noreg || always_reachable(src), "missing"); 2594 2595 if (reachable(src)) { 2596 movss(dst, as_Address(src)); 2597 } else { 2598 lea(rscratch, src); 2599 movss(dst, Address(rscratch, 0)); 2600 } 2601 } 2602 2603 void MacroAssembler::movptr(Register dst, Register src) { 2604 LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src)); 2605 } 2606 2607 void MacroAssembler::movptr(Register dst, Address src) { 2608 LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src)); 2609 } 2610 2611 // src should NEVER be a real pointer. Use AddressLiteral for true pointers 2612 void MacroAssembler::movptr(Register dst, intptr_t src) { 2613 #ifdef _LP64 2614 if (is_uimm32(src)) { 2615 movl(dst, checked_cast<uint32_t>(src)); 2616 } else if (is_simm32(src)) { 2617 movq(dst, checked_cast<int32_t>(src)); 2618 } else { 2619 mov64(dst, src); 2620 } 2621 #else 2622 movl(dst, src); 2623 #endif 2624 } 2625 2626 void MacroAssembler::movptr(Address dst, Register src) { 2627 LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src)); 2628 } 2629 2630 void MacroAssembler::movptr(Address dst, int32_t src) { 2631 LP64_ONLY(movslq(dst, src)) NOT_LP64(movl(dst, src)); 2632 } 2633 2634 void MacroAssembler::movdqu(Address dst, XMMRegister src) { 2635 assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 2636 Assembler::movdqu(dst, src); 2637 } 2638 2639 void MacroAssembler::movdqu(XMMRegister dst, Address src) { 2640 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 2641 Assembler::movdqu(dst, src); 2642 } 2643 2644 void MacroAssembler::movdqu(XMMRegister dst, XMMRegister src) { 2645 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 2646 Assembler::movdqu(dst, src); 2647 } 2648 2649 void MacroAssembler::movdqu(XMMRegister dst, AddressLiteral src, Register rscratch) { 2650 assert(rscratch != noreg || always_reachable(src), "missing"); 2651 2652 if (reachable(src)) { 2653 movdqu(dst, as_Address(src)); 2654 } else { 2655 lea(rscratch, src); 2656 movdqu(dst, Address(rscratch, 0)); 2657 } 2658 } 2659 2660 void MacroAssembler::vmovdqu(Address dst, XMMRegister src) { 2661 assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 2662 Assembler::vmovdqu(dst, src); 2663 } 2664 2665 void MacroAssembler::vmovdqu(XMMRegister dst, Address src) { 2666 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 2667 Assembler::vmovdqu(dst, src); 2668 } 2669 2670 void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src) { 2671 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 2672 Assembler::vmovdqu(dst, src); 2673 } 2674 2675 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, Register rscratch) { 2676 assert(rscratch != noreg || always_reachable(src), "missing"); 2677 2678 if (reachable(src)) { 2679 vmovdqu(dst, as_Address(src)); 2680 } 2681 else { 2682 lea(rscratch, src); 2683 vmovdqu(dst, Address(rscratch, 0)); 2684 } 2685 } 2686 2687 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 2688 assert(rscratch != noreg || always_reachable(src), "missing"); 2689 2690 if (vector_len == AVX_512bit) { 2691 evmovdquq(dst, src, AVX_512bit, rscratch); 2692 } else if (vector_len == AVX_256bit) { 2693 vmovdqu(dst, src, rscratch); 2694 } else { 2695 movdqu(dst, src, rscratch); 2696 } 2697 } 2698 2699 void MacroAssembler::kmov(KRegister dst, Address src) { 2700 if (VM_Version::supports_avx512bw()) { 2701 kmovql(dst, src); 2702 } else { 2703 assert(VM_Version::supports_evex(), ""); 2704 kmovwl(dst, src); 2705 } 2706 } 2707 2708 void MacroAssembler::kmov(Address dst, KRegister src) { 2709 if (VM_Version::supports_avx512bw()) { 2710 kmovql(dst, src); 2711 } else { 2712 assert(VM_Version::supports_evex(), ""); 2713 kmovwl(dst, src); 2714 } 2715 } 2716 2717 void MacroAssembler::kmov(KRegister dst, KRegister src) { 2718 if (VM_Version::supports_avx512bw()) { 2719 kmovql(dst, src); 2720 } else { 2721 assert(VM_Version::supports_evex(), ""); 2722 kmovwl(dst, src); 2723 } 2724 } 2725 2726 void MacroAssembler::kmov(Register dst, KRegister src) { 2727 if (VM_Version::supports_avx512bw()) { 2728 kmovql(dst, src); 2729 } else { 2730 assert(VM_Version::supports_evex(), ""); 2731 kmovwl(dst, src); 2732 } 2733 } 2734 2735 void MacroAssembler::kmov(KRegister dst, Register src) { 2736 if (VM_Version::supports_avx512bw()) { 2737 kmovql(dst, src); 2738 } else { 2739 assert(VM_Version::supports_evex(), ""); 2740 kmovwl(dst, src); 2741 } 2742 } 2743 2744 void MacroAssembler::kmovql(KRegister dst, AddressLiteral src, Register rscratch) { 2745 assert(rscratch != noreg || always_reachable(src), "missing"); 2746 2747 if (reachable(src)) { 2748 kmovql(dst, as_Address(src)); 2749 } else { 2750 lea(rscratch, src); 2751 kmovql(dst, Address(rscratch, 0)); 2752 } 2753 } 2754 2755 void MacroAssembler::kmovwl(KRegister dst, AddressLiteral src, Register rscratch) { 2756 assert(rscratch != noreg || always_reachable(src), "missing"); 2757 2758 if (reachable(src)) { 2759 kmovwl(dst, as_Address(src)); 2760 } else { 2761 lea(rscratch, src); 2762 kmovwl(dst, Address(rscratch, 0)); 2763 } 2764 } 2765 2766 void MacroAssembler::evmovdqub(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, 2767 int vector_len, Register rscratch) { 2768 assert(rscratch != noreg || always_reachable(src), "missing"); 2769 2770 if (reachable(src)) { 2771 Assembler::evmovdqub(dst, mask, as_Address(src), merge, vector_len); 2772 } else { 2773 lea(rscratch, src); 2774 Assembler::evmovdqub(dst, mask, Address(rscratch, 0), merge, vector_len); 2775 } 2776 } 2777 2778 void MacroAssembler::evmovdquw(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, 2779 int vector_len, Register rscratch) { 2780 assert(rscratch != noreg || always_reachable(src), "missing"); 2781 2782 if (reachable(src)) { 2783 Assembler::evmovdquw(dst, mask, as_Address(src), merge, vector_len); 2784 } else { 2785 lea(rscratch, src); 2786 Assembler::evmovdquw(dst, mask, Address(rscratch, 0), merge, vector_len); 2787 } 2788 } 2789 2790 void MacroAssembler::evmovdqul(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) { 2791 assert(rscratch != noreg || always_reachable(src), "missing"); 2792 2793 if (reachable(src)) { 2794 Assembler::evmovdqul(dst, mask, as_Address(src), merge, vector_len); 2795 } else { 2796 lea(rscratch, src); 2797 Assembler::evmovdqul(dst, mask, Address(rscratch, 0), merge, vector_len); 2798 } 2799 } 2800 2801 void MacroAssembler::evmovdquq(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) { 2802 assert(rscratch != noreg || always_reachable(src), "missing"); 2803 2804 if (reachable(src)) { 2805 Assembler::evmovdquq(dst, mask, as_Address(src), merge, vector_len); 2806 } else { 2807 lea(rscratch, src); 2808 Assembler::evmovdquq(dst, mask, Address(rscratch, 0), merge, vector_len); 2809 } 2810 } 2811 2812 void MacroAssembler::evmovdquq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 2813 assert(rscratch != noreg || always_reachable(src), "missing"); 2814 2815 if (reachable(src)) { 2816 Assembler::evmovdquq(dst, as_Address(src), vector_len); 2817 } else { 2818 lea(rscratch, src); 2819 Assembler::evmovdquq(dst, Address(rscratch, 0), vector_len); 2820 } 2821 } 2822 2823 void MacroAssembler::movdqa(XMMRegister dst, AddressLiteral src, Register rscratch) { 2824 assert(rscratch != noreg || always_reachable(src), "missing"); 2825 2826 if (reachable(src)) { 2827 Assembler::movdqa(dst, as_Address(src)); 2828 } else { 2829 lea(rscratch, src); 2830 Assembler::movdqa(dst, Address(rscratch, 0)); 2831 } 2832 } 2833 2834 void MacroAssembler::movsd(XMMRegister dst, AddressLiteral src, Register rscratch) { 2835 assert(rscratch != noreg || always_reachable(src), "missing"); 2836 2837 if (reachable(src)) { 2838 Assembler::movsd(dst, as_Address(src)); 2839 } else { 2840 lea(rscratch, src); 2841 Assembler::movsd(dst, Address(rscratch, 0)); 2842 } 2843 } 2844 2845 void MacroAssembler::movss(XMMRegister dst, AddressLiteral src, Register rscratch) { 2846 assert(rscratch != noreg || always_reachable(src), "missing"); 2847 2848 if (reachable(src)) { 2849 Assembler::movss(dst, as_Address(src)); 2850 } else { 2851 lea(rscratch, src); 2852 Assembler::movss(dst, Address(rscratch, 0)); 2853 } 2854 } 2855 2856 void MacroAssembler::movddup(XMMRegister dst, AddressLiteral src, Register rscratch) { 2857 assert(rscratch != noreg || always_reachable(src), "missing"); 2858 2859 if (reachable(src)) { 2860 Assembler::movddup(dst, as_Address(src)); 2861 } else { 2862 lea(rscratch, src); 2863 Assembler::movddup(dst, Address(rscratch, 0)); 2864 } 2865 } 2866 2867 void MacroAssembler::vmovddup(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 2868 assert(rscratch != noreg || always_reachable(src), "missing"); 2869 2870 if (reachable(src)) { 2871 Assembler::vmovddup(dst, as_Address(src), vector_len); 2872 } else { 2873 lea(rscratch, src); 2874 Assembler::vmovddup(dst, Address(rscratch, 0), vector_len); 2875 } 2876 } 2877 2878 void MacroAssembler::mulsd(XMMRegister dst, AddressLiteral src, Register rscratch) { 2879 assert(rscratch != noreg || always_reachable(src), "missing"); 2880 2881 if (reachable(src)) { 2882 Assembler::mulsd(dst, as_Address(src)); 2883 } else { 2884 lea(rscratch, src); 2885 Assembler::mulsd(dst, Address(rscratch, 0)); 2886 } 2887 } 2888 2889 void MacroAssembler::mulss(XMMRegister dst, AddressLiteral src, Register rscratch) { 2890 assert(rscratch != noreg || always_reachable(src), "missing"); 2891 2892 if (reachable(src)) { 2893 Assembler::mulss(dst, as_Address(src)); 2894 } else { 2895 lea(rscratch, src); 2896 Assembler::mulss(dst, Address(rscratch, 0)); 2897 } 2898 } 2899 2900 void MacroAssembler::null_check(Register reg, int offset) { 2901 if (needs_explicit_null_check(offset)) { 2902 // provoke OS null exception if reg is null by 2903 // accessing M[reg] w/o changing any (non-CC) registers 2904 // NOTE: cmpl is plenty here to provoke a segv 2905 cmpptr(rax, Address(reg, 0)); 2906 // Note: should probably use testl(rax, Address(reg, 0)); 2907 // may be shorter code (however, this version of 2908 // testl needs to be implemented first) 2909 } else { 2910 // nothing to do, (later) access of M[reg + offset] 2911 // will provoke OS null exception if reg is null 2912 } 2913 } 2914 2915 void MacroAssembler::os_breakpoint() { 2916 // instead of directly emitting a breakpoint, call os:breakpoint for better debugability 2917 // (e.g., MSVC can't call ps() otherwise) 2918 call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint))); 2919 } 2920 2921 void MacroAssembler::unimplemented(const char* what) { 2922 const char* buf = nullptr; 2923 { 2924 ResourceMark rm; 2925 stringStream ss; 2926 ss.print("unimplemented: %s", what); 2927 buf = code_string(ss.as_string()); 2928 } 2929 stop(buf); 2930 } 2931 2932 #ifdef _LP64 2933 #define XSTATE_BV 0x200 2934 #endif 2935 2936 void MacroAssembler::pop_CPU_state() { 2937 pop_FPU_state(); 2938 pop_IU_state(); 2939 } 2940 2941 void MacroAssembler::pop_FPU_state() { 2942 #ifndef _LP64 2943 frstor(Address(rsp, 0)); 2944 #else 2945 fxrstor(Address(rsp, 0)); 2946 #endif 2947 addptr(rsp, FPUStateSizeInWords * wordSize); 2948 } 2949 2950 void MacroAssembler::pop_IU_state() { 2951 popa(); 2952 LP64_ONLY(addq(rsp, 8)); 2953 popf(); 2954 } 2955 2956 // Save Integer and Float state 2957 // Warning: Stack must be 16 byte aligned (64bit) 2958 void MacroAssembler::push_CPU_state() { 2959 push_IU_state(); 2960 push_FPU_state(); 2961 } 2962 2963 void MacroAssembler::push_FPU_state() { 2964 subptr(rsp, FPUStateSizeInWords * wordSize); 2965 #ifndef _LP64 2966 fnsave(Address(rsp, 0)); 2967 fwait(); 2968 #else 2969 fxsave(Address(rsp, 0)); 2970 #endif // LP64 2971 } 2972 2973 void MacroAssembler::push_IU_state() { 2974 // Push flags first because pusha kills them 2975 pushf(); 2976 // Make sure rsp stays 16-byte aligned 2977 LP64_ONLY(subq(rsp, 8)); 2978 pusha(); 2979 } 2980 2981 void MacroAssembler::push_cont_fastpath() { 2982 if (!Continuations::enabled()) return; 2983 2984 #ifndef _LP64 2985 Register rthread = rax; 2986 Register rrealsp = rbx; 2987 push(rthread); 2988 push(rrealsp); 2989 2990 get_thread(rthread); 2991 2992 // The code below wants the original RSP. 2993 // Move it back after the pushes above. 2994 movptr(rrealsp, rsp); 2995 addptr(rrealsp, 2*wordSize); 2996 #else 2997 Register rthread = r15_thread; 2998 Register rrealsp = rsp; 2999 #endif 3000 3001 Label done; 3002 cmpptr(rrealsp, Address(rthread, JavaThread::cont_fastpath_offset())); 3003 jccb(Assembler::belowEqual, done); 3004 movptr(Address(rthread, JavaThread::cont_fastpath_offset()), rrealsp); 3005 bind(done); 3006 3007 #ifndef _LP64 3008 pop(rrealsp); 3009 pop(rthread); 3010 #endif 3011 } 3012 3013 void MacroAssembler::pop_cont_fastpath() { 3014 if (!Continuations::enabled()) return; 3015 3016 #ifndef _LP64 3017 Register rthread = rax; 3018 Register rrealsp = rbx; 3019 push(rthread); 3020 push(rrealsp); 3021 3022 get_thread(rthread); 3023 3024 // The code below wants the original RSP. 3025 // Move it back after the pushes above. 3026 movptr(rrealsp, rsp); 3027 addptr(rrealsp, 2*wordSize); 3028 #else 3029 Register rthread = r15_thread; 3030 Register rrealsp = rsp; 3031 #endif 3032 3033 Label done; 3034 cmpptr(rrealsp, Address(rthread, JavaThread::cont_fastpath_offset())); 3035 jccb(Assembler::below, done); 3036 movptr(Address(rthread, JavaThread::cont_fastpath_offset()), 0); 3037 bind(done); 3038 3039 #ifndef _LP64 3040 pop(rrealsp); 3041 pop(rthread); 3042 #endif 3043 } 3044 3045 void MacroAssembler::inc_held_monitor_count() { 3046 #ifdef _LP64 3047 incrementq(Address(r15_thread, JavaThread::held_monitor_count_offset())); 3048 #endif 3049 } 3050 3051 void MacroAssembler::dec_held_monitor_count() { 3052 #ifdef _LP64 3053 decrementq(Address(r15_thread, JavaThread::held_monitor_count_offset())); 3054 #endif 3055 } 3056 3057 #ifdef ASSERT 3058 void MacroAssembler::stop_if_in_cont(Register cont, const char* name) { 3059 #ifdef _LP64 3060 Label no_cont; 3061 movptr(cont, Address(r15_thread, JavaThread::cont_entry_offset())); 3062 testl(cont, cont); 3063 jcc(Assembler::zero, no_cont); 3064 stop(name); 3065 bind(no_cont); 3066 #else 3067 Unimplemented(); 3068 #endif 3069 } 3070 #endif 3071 3072 void MacroAssembler::reset_last_Java_frame(Register java_thread, bool clear_fp) { // determine java_thread register 3073 if (!java_thread->is_valid()) { 3074 java_thread = rdi; 3075 get_thread(java_thread); 3076 } 3077 // we must set sp to zero to clear frame 3078 movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), NULL_WORD); 3079 // must clear fp, so that compiled frames are not confused; it is 3080 // possible that we need it only for debugging 3081 if (clear_fp) { 3082 movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), NULL_WORD); 3083 } 3084 // Always clear the pc because it could have been set by make_walkable() 3085 movptr(Address(java_thread, JavaThread::last_Java_pc_offset()), NULL_WORD); 3086 vzeroupper(); 3087 } 3088 3089 void MacroAssembler::restore_rax(Register tmp) { 3090 if (tmp == noreg) pop(rax); 3091 else if (tmp != rax) mov(rax, tmp); 3092 } 3093 3094 void MacroAssembler::round_to(Register reg, int modulus) { 3095 addptr(reg, modulus - 1); 3096 andptr(reg, -modulus); 3097 } 3098 3099 void MacroAssembler::save_rax(Register tmp) { 3100 if (tmp == noreg) push(rax); 3101 else if (tmp != rax) mov(tmp, rax); 3102 } 3103 3104 void MacroAssembler::safepoint_poll(Label& slow_path, Register thread_reg, bool at_return, bool in_nmethod) { 3105 if (at_return) { 3106 // Note that when in_nmethod is set, the stack pointer is incremented before the poll. Therefore, 3107 // we may safely use rsp instead to perform the stack watermark check. 3108 cmpptr(in_nmethod ? rsp : rbp, Address(thread_reg, JavaThread::polling_word_offset())); 3109 jcc(Assembler::above, slow_path); 3110 return; 3111 } 3112 testb(Address(thread_reg, JavaThread::polling_word_offset()), SafepointMechanism::poll_bit()); 3113 jcc(Assembler::notZero, slow_path); // handshake bit set implies poll 3114 } 3115 3116 // Calls to C land 3117 // 3118 // When entering C land, the rbp, & rsp of the last Java frame have to be recorded 3119 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp 3120 // has to be reset to 0. This is required to allow proper stack traversal. 3121 void MacroAssembler::set_last_Java_frame(Register java_thread, 3122 Register last_java_sp, 3123 Register last_java_fp, 3124 address last_java_pc, 3125 Register rscratch) { 3126 vzeroupper(); 3127 // determine java_thread register 3128 if (!java_thread->is_valid()) { 3129 java_thread = rdi; 3130 get_thread(java_thread); 3131 } 3132 // determine last_java_sp register 3133 if (!last_java_sp->is_valid()) { 3134 last_java_sp = rsp; 3135 } 3136 // last_java_fp is optional 3137 if (last_java_fp->is_valid()) { 3138 movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), last_java_fp); 3139 } 3140 // last_java_pc is optional 3141 if (last_java_pc != nullptr) { 3142 Address java_pc(java_thread, 3143 JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset()); 3144 lea(java_pc, InternalAddress(last_java_pc), rscratch); 3145 } 3146 movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), last_java_sp); 3147 } 3148 3149 #ifdef _LP64 3150 void MacroAssembler::set_last_Java_frame(Register last_java_sp, 3151 Register last_java_fp, 3152 Label &L, 3153 Register scratch) { 3154 lea(scratch, L); 3155 movptr(Address(r15_thread, JavaThread::last_Java_pc_offset()), scratch); 3156 set_last_Java_frame(r15_thread, last_java_sp, last_java_fp, nullptr, scratch); 3157 } 3158 #endif 3159 3160 void MacroAssembler::shlptr(Register dst, int imm8) { 3161 LP64_ONLY(shlq(dst, imm8)) NOT_LP64(shll(dst, imm8)); 3162 } 3163 3164 void MacroAssembler::shrptr(Register dst, int imm8) { 3165 LP64_ONLY(shrq(dst, imm8)) NOT_LP64(shrl(dst, imm8)); 3166 } 3167 3168 void MacroAssembler::sign_extend_byte(Register reg) { 3169 if (LP64_ONLY(true ||) (VM_Version::is_P6() && reg->has_byte_register())) { 3170 movsbl(reg, reg); // movsxb 3171 } else { 3172 shll(reg, 24); 3173 sarl(reg, 24); 3174 } 3175 } 3176 3177 void MacroAssembler::sign_extend_short(Register reg) { 3178 if (LP64_ONLY(true ||) VM_Version::is_P6()) { 3179 movswl(reg, reg); // movsxw 3180 } else { 3181 shll(reg, 16); 3182 sarl(reg, 16); 3183 } 3184 } 3185 3186 void MacroAssembler::testl(Address dst, int32_t imm32) { 3187 if (imm32 >= 0 && is8bit(imm32)) { 3188 testb(dst, imm32); 3189 } else { 3190 Assembler::testl(dst, imm32); 3191 } 3192 } 3193 3194 void MacroAssembler::testl(Register dst, int32_t imm32) { 3195 if (imm32 >= 0 && is8bit(imm32) && dst->has_byte_register()) { 3196 testb(dst, imm32); 3197 } else { 3198 Assembler::testl(dst, imm32); 3199 } 3200 } 3201 3202 void MacroAssembler::testl(Register dst, AddressLiteral src) { 3203 assert(always_reachable(src), "Address should be reachable"); 3204 testl(dst, as_Address(src)); 3205 } 3206 3207 #ifdef _LP64 3208 3209 void MacroAssembler::testq(Address dst, int32_t imm32) { 3210 if (imm32 >= 0) { 3211 testl(dst, imm32); 3212 } else { 3213 Assembler::testq(dst, imm32); 3214 } 3215 } 3216 3217 void MacroAssembler::testq(Register dst, int32_t imm32) { 3218 if (imm32 >= 0) { 3219 testl(dst, imm32); 3220 } else { 3221 Assembler::testq(dst, imm32); 3222 } 3223 } 3224 3225 #endif 3226 3227 void MacroAssembler::pcmpeqb(XMMRegister dst, XMMRegister src) { 3228 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3229 Assembler::pcmpeqb(dst, src); 3230 } 3231 3232 void MacroAssembler::pcmpeqw(XMMRegister dst, XMMRegister src) { 3233 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3234 Assembler::pcmpeqw(dst, src); 3235 } 3236 3237 void MacroAssembler::pcmpestri(XMMRegister dst, Address src, int imm8) { 3238 assert((dst->encoding() < 16),"XMM register should be 0-15"); 3239 Assembler::pcmpestri(dst, src, imm8); 3240 } 3241 3242 void MacroAssembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) { 3243 assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15"); 3244 Assembler::pcmpestri(dst, src, imm8); 3245 } 3246 3247 void MacroAssembler::pmovzxbw(XMMRegister dst, XMMRegister src) { 3248 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3249 Assembler::pmovzxbw(dst, src); 3250 } 3251 3252 void MacroAssembler::pmovzxbw(XMMRegister dst, Address src) { 3253 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3254 Assembler::pmovzxbw(dst, src); 3255 } 3256 3257 void MacroAssembler::pmovmskb(Register dst, XMMRegister src) { 3258 assert((src->encoding() < 16),"XMM register should be 0-15"); 3259 Assembler::pmovmskb(dst, src); 3260 } 3261 3262 void MacroAssembler::ptest(XMMRegister dst, XMMRegister src) { 3263 assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15"); 3264 Assembler::ptest(dst, src); 3265 } 3266 3267 void MacroAssembler::sqrtss(XMMRegister dst, AddressLiteral src, Register rscratch) { 3268 assert(rscratch != noreg || always_reachable(src), "missing"); 3269 3270 if (reachable(src)) { 3271 Assembler::sqrtss(dst, as_Address(src)); 3272 } else { 3273 lea(rscratch, src); 3274 Assembler::sqrtss(dst, Address(rscratch, 0)); 3275 } 3276 } 3277 3278 void MacroAssembler::subsd(XMMRegister dst, AddressLiteral src, Register rscratch) { 3279 assert(rscratch != noreg || always_reachable(src), "missing"); 3280 3281 if (reachable(src)) { 3282 Assembler::subsd(dst, as_Address(src)); 3283 } else { 3284 lea(rscratch, src); 3285 Assembler::subsd(dst, Address(rscratch, 0)); 3286 } 3287 } 3288 3289 void MacroAssembler::roundsd(XMMRegister dst, AddressLiteral src, int32_t rmode, Register rscratch) { 3290 assert(rscratch != noreg || always_reachable(src), "missing"); 3291 3292 if (reachable(src)) { 3293 Assembler::roundsd(dst, as_Address(src), rmode); 3294 } else { 3295 lea(rscratch, src); 3296 Assembler::roundsd(dst, Address(rscratch, 0), rmode); 3297 } 3298 } 3299 3300 void MacroAssembler::subss(XMMRegister dst, AddressLiteral src, Register rscratch) { 3301 assert(rscratch != noreg || always_reachable(src), "missing"); 3302 3303 if (reachable(src)) { 3304 Assembler::subss(dst, as_Address(src)); 3305 } else { 3306 lea(rscratch, src); 3307 Assembler::subss(dst, Address(rscratch, 0)); 3308 } 3309 } 3310 3311 void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src, Register rscratch) { 3312 assert(rscratch != noreg || always_reachable(src), "missing"); 3313 3314 if (reachable(src)) { 3315 Assembler::ucomisd(dst, as_Address(src)); 3316 } else { 3317 lea(rscratch, src); 3318 Assembler::ucomisd(dst, Address(rscratch, 0)); 3319 } 3320 } 3321 3322 void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src, Register rscratch) { 3323 assert(rscratch != noreg || always_reachable(src), "missing"); 3324 3325 if (reachable(src)) { 3326 Assembler::ucomiss(dst, as_Address(src)); 3327 } else { 3328 lea(rscratch, src); 3329 Assembler::ucomiss(dst, Address(rscratch, 0)); 3330 } 3331 } 3332 3333 void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src, Register rscratch) { 3334 assert(rscratch != noreg || always_reachable(src), "missing"); 3335 3336 // Used in sign-bit flipping with aligned address. 3337 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 3338 3339 if (UseAVX > 2 && 3340 (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) && 3341 (dst->encoding() >= 16)) { 3342 vpxor(dst, dst, src, Assembler::AVX_512bit, rscratch); 3343 } else if (reachable(src)) { 3344 Assembler::xorpd(dst, as_Address(src)); 3345 } else { 3346 lea(rscratch, src); 3347 Assembler::xorpd(dst, Address(rscratch, 0)); 3348 } 3349 } 3350 3351 void MacroAssembler::xorpd(XMMRegister dst, XMMRegister src) { 3352 if (UseAVX > 2 && 3353 (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) && 3354 ((dst->encoding() >= 16) || (src->encoding() >= 16))) { 3355 Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit); 3356 } else { 3357 Assembler::xorpd(dst, src); 3358 } 3359 } 3360 3361 void MacroAssembler::xorps(XMMRegister dst, XMMRegister src) { 3362 if (UseAVX > 2 && 3363 (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) && 3364 ((dst->encoding() >= 16) || (src->encoding() >= 16))) { 3365 Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit); 3366 } else { 3367 Assembler::xorps(dst, src); 3368 } 3369 } 3370 3371 void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src, Register rscratch) { 3372 assert(rscratch != noreg || always_reachable(src), "missing"); 3373 3374 // Used in sign-bit flipping with aligned address. 3375 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 3376 3377 if (UseAVX > 2 && 3378 (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) && 3379 (dst->encoding() >= 16)) { 3380 vpxor(dst, dst, src, Assembler::AVX_512bit, rscratch); 3381 } else if (reachable(src)) { 3382 Assembler::xorps(dst, as_Address(src)); 3383 } else { 3384 lea(rscratch, src); 3385 Assembler::xorps(dst, Address(rscratch, 0)); 3386 } 3387 } 3388 3389 void MacroAssembler::pshufb(XMMRegister dst, AddressLiteral src, Register rscratch) { 3390 assert(rscratch != noreg || always_reachable(src), "missing"); 3391 3392 // Used in sign-bit flipping with aligned address. 3393 bool aligned_adr = (((intptr_t)src.target() & 15) == 0); 3394 assert((UseAVX > 0) || aligned_adr, "SSE mode requires address alignment 16 bytes"); 3395 if (reachable(src)) { 3396 Assembler::pshufb(dst, as_Address(src)); 3397 } else { 3398 lea(rscratch, src); 3399 Assembler::pshufb(dst, Address(rscratch, 0)); 3400 } 3401 } 3402 3403 // AVX 3-operands instructions 3404 3405 void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3406 assert(rscratch != noreg || always_reachable(src), "missing"); 3407 3408 if (reachable(src)) { 3409 vaddsd(dst, nds, as_Address(src)); 3410 } else { 3411 lea(rscratch, src); 3412 vaddsd(dst, nds, Address(rscratch, 0)); 3413 } 3414 } 3415 3416 void MacroAssembler::vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3417 assert(rscratch != noreg || always_reachable(src), "missing"); 3418 3419 if (reachable(src)) { 3420 vaddss(dst, nds, as_Address(src)); 3421 } else { 3422 lea(rscratch, src); 3423 vaddss(dst, nds, Address(rscratch, 0)); 3424 } 3425 } 3426 3427 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3428 assert(UseAVX > 0, "requires some form of AVX"); 3429 assert(rscratch != noreg || always_reachable(src), "missing"); 3430 3431 if (reachable(src)) { 3432 Assembler::vpaddb(dst, nds, as_Address(src), vector_len); 3433 } else { 3434 lea(rscratch, src); 3435 Assembler::vpaddb(dst, nds, Address(rscratch, 0), vector_len); 3436 } 3437 } 3438 3439 void MacroAssembler::vpaddd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3440 assert(UseAVX > 0, "requires some form of AVX"); 3441 assert(rscratch != noreg || always_reachable(src), "missing"); 3442 3443 if (reachable(src)) { 3444 Assembler::vpaddd(dst, nds, as_Address(src), vector_len); 3445 } else { 3446 lea(rscratch, src); 3447 Assembler::vpaddd(dst, nds, Address(rscratch, 0), vector_len); 3448 } 3449 } 3450 3451 void MacroAssembler::vabsss(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch) { 3452 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); 3453 assert(rscratch != noreg || always_reachable(negate_field), "missing"); 3454 3455 vandps(dst, nds, negate_field, vector_len, rscratch); 3456 } 3457 3458 void MacroAssembler::vabssd(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch) { 3459 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); 3460 assert(rscratch != noreg || always_reachable(negate_field), "missing"); 3461 3462 vandpd(dst, nds, negate_field, vector_len, rscratch); 3463 } 3464 3465 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3466 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3467 Assembler::vpaddb(dst, nds, src, vector_len); 3468 } 3469 3470 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3471 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3472 Assembler::vpaddb(dst, nds, src, vector_len); 3473 } 3474 3475 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3476 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3477 Assembler::vpaddw(dst, nds, src, vector_len); 3478 } 3479 3480 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3481 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3482 Assembler::vpaddw(dst, nds, src, vector_len); 3483 } 3484 3485 void MacroAssembler::vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3486 assert(rscratch != noreg || always_reachable(src), "missing"); 3487 3488 if (reachable(src)) { 3489 Assembler::vpand(dst, nds, as_Address(src), vector_len); 3490 } else { 3491 lea(rscratch, src); 3492 Assembler::vpand(dst, nds, Address(rscratch, 0), vector_len); 3493 } 3494 } 3495 3496 void MacroAssembler::vpbroadcastd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 3497 assert(rscratch != noreg || always_reachable(src), "missing"); 3498 3499 if (reachable(src)) { 3500 Assembler::vpbroadcastd(dst, as_Address(src), vector_len); 3501 } else { 3502 lea(rscratch, src); 3503 Assembler::vpbroadcastd(dst, Address(rscratch, 0), vector_len); 3504 } 3505 } 3506 3507 void MacroAssembler::vbroadcasti128(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 3508 assert(rscratch != noreg || always_reachable(src), "missing"); 3509 3510 if (reachable(src)) { 3511 Assembler::vbroadcasti128(dst, as_Address(src), vector_len); 3512 } else { 3513 lea(rscratch, src); 3514 Assembler::vbroadcasti128(dst, Address(rscratch, 0), vector_len); 3515 } 3516 } 3517 3518 void MacroAssembler::vpbroadcastq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 3519 assert(rscratch != noreg || always_reachable(src), "missing"); 3520 3521 if (reachable(src)) { 3522 Assembler::vpbroadcastq(dst, as_Address(src), vector_len); 3523 } else { 3524 lea(rscratch, src); 3525 Assembler::vpbroadcastq(dst, Address(rscratch, 0), vector_len); 3526 } 3527 } 3528 3529 void MacroAssembler::vbroadcastsd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 3530 assert(rscratch != noreg || always_reachable(src), "missing"); 3531 3532 if (reachable(src)) { 3533 Assembler::vbroadcastsd(dst, as_Address(src), vector_len); 3534 } else { 3535 lea(rscratch, src); 3536 Assembler::vbroadcastsd(dst, Address(rscratch, 0), vector_len); 3537 } 3538 } 3539 3540 void MacroAssembler::vbroadcastss(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 3541 assert(rscratch != noreg || always_reachable(src), "missing"); 3542 3543 if (reachable(src)) { 3544 Assembler::vbroadcastss(dst, as_Address(src), vector_len); 3545 } else { 3546 lea(rscratch, src); 3547 Assembler::vbroadcastss(dst, Address(rscratch, 0), vector_len); 3548 } 3549 } 3550 3551 // Vector float blend 3552 // vblendvps(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len, bool compute_mask = true, XMMRegister scratch = xnoreg) 3553 void MacroAssembler::vblendvps(XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister mask, int vector_len, bool compute_mask, XMMRegister scratch) { 3554 // WARN: Allow dst == (src1|src2), mask == scratch 3555 bool blend_emulation = EnableX86ECoreOpts && UseAVX > 1; 3556 bool scratch_available = scratch != xnoreg && scratch != src1 && scratch != src2 && scratch != dst; 3557 bool dst_available = dst != mask && (dst != src1 || dst != src2); 3558 if (blend_emulation && scratch_available && dst_available) { 3559 if (compute_mask) { 3560 vpsrad(scratch, mask, 32, vector_len); 3561 mask = scratch; 3562 } 3563 if (dst == src1) { 3564 vpandn(dst, mask, src1, vector_len); // if mask == 0, src1 3565 vpand (scratch, mask, src2, vector_len); // if mask == 1, src2 3566 } else { 3567 vpand (dst, mask, src2, vector_len); // if mask == 1, src2 3568 vpandn(scratch, mask, src1, vector_len); // if mask == 0, src1 3569 } 3570 vpor(dst, dst, scratch, vector_len); 3571 } else { 3572 Assembler::vblendvps(dst, src1, src2, mask, vector_len); 3573 } 3574 } 3575 3576 // vblendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len, bool compute_mask = true, XMMRegister scratch = xnoreg) 3577 void MacroAssembler::vblendvpd(XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister mask, int vector_len, bool compute_mask, XMMRegister scratch) { 3578 // WARN: Allow dst == (src1|src2), mask == scratch 3579 bool blend_emulation = EnableX86ECoreOpts && UseAVX > 1; 3580 bool scratch_available = scratch != xnoreg && scratch != src1 && scratch != src2 && scratch != dst && (!compute_mask || scratch != mask); 3581 bool dst_available = dst != mask && (dst != src1 || dst != src2); 3582 if (blend_emulation && scratch_available && dst_available) { 3583 if (compute_mask) { 3584 vpxor(scratch, scratch, scratch, vector_len); 3585 vpcmpgtq(scratch, scratch, mask, vector_len); 3586 mask = scratch; 3587 } 3588 if (dst == src1) { 3589 vpandn(dst, mask, src1, vector_len); // if mask == 0, src 3590 vpand (scratch, mask, src2, vector_len); // if mask == 1, src2 3591 } else { 3592 vpand (dst, mask, src2, vector_len); // if mask == 1, src2 3593 vpandn(scratch, mask, src1, vector_len); // if mask == 0, src 3594 } 3595 vpor(dst, dst, scratch, vector_len); 3596 } else { 3597 Assembler::vblendvpd(dst, src1, src2, mask, vector_len); 3598 } 3599 } 3600 3601 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3602 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3603 Assembler::vpcmpeqb(dst, nds, src, vector_len); 3604 } 3605 3606 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) { 3607 assert(((dst->encoding() < 16 && src1->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3608 Assembler::vpcmpeqb(dst, src1, src2, vector_len); 3609 } 3610 3611 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3612 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3613 Assembler::vpcmpeqw(dst, nds, src, vector_len); 3614 } 3615 3616 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3617 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3618 Assembler::vpcmpeqw(dst, nds, src, vector_len); 3619 } 3620 3621 void MacroAssembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3622 assert(rscratch != noreg || always_reachable(src), "missing"); 3623 3624 if (reachable(src)) { 3625 Assembler::evpcmpeqd(kdst, mask, nds, as_Address(src), vector_len); 3626 } else { 3627 lea(rscratch, src); 3628 Assembler::evpcmpeqd(kdst, mask, nds, Address(rscratch, 0), vector_len); 3629 } 3630 } 3631 3632 void MacroAssembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, 3633 int comparison, bool is_signed, int vector_len, Register rscratch) { 3634 assert(rscratch != noreg || always_reachable(src), "missing"); 3635 3636 if (reachable(src)) { 3637 Assembler::evpcmpd(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len); 3638 } else { 3639 lea(rscratch, src); 3640 Assembler::evpcmpd(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len); 3641 } 3642 } 3643 3644 void MacroAssembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, 3645 int comparison, bool is_signed, int vector_len, Register rscratch) { 3646 assert(rscratch != noreg || always_reachable(src), "missing"); 3647 3648 if (reachable(src)) { 3649 Assembler::evpcmpq(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len); 3650 } else { 3651 lea(rscratch, src); 3652 Assembler::evpcmpq(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len); 3653 } 3654 } 3655 3656 void MacroAssembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, 3657 int comparison, bool is_signed, int vector_len, Register rscratch) { 3658 assert(rscratch != noreg || always_reachable(src), "missing"); 3659 3660 if (reachable(src)) { 3661 Assembler::evpcmpb(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len); 3662 } else { 3663 lea(rscratch, src); 3664 Assembler::evpcmpb(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len); 3665 } 3666 } 3667 3668 void MacroAssembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, 3669 int comparison, bool is_signed, int vector_len, Register rscratch) { 3670 assert(rscratch != noreg || always_reachable(src), "missing"); 3671 3672 if (reachable(src)) { 3673 Assembler::evpcmpw(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len); 3674 } else { 3675 lea(rscratch, src); 3676 Assembler::evpcmpw(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len); 3677 } 3678 } 3679 3680 void MacroAssembler::vpcmpCC(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, Width width, int vector_len) { 3681 if (width == Assembler::Q) { 3682 Assembler::vpcmpCCq(dst, nds, src, cond_encoding, vector_len); 3683 } else { 3684 Assembler::vpcmpCCbwd(dst, nds, src, cond_encoding, vector_len); 3685 } 3686 } 3687 3688 void MacroAssembler::vpcmpCCW(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister xtmp, ComparisonPredicate cond, Width width, int vector_len) { 3689 int eq_cond_enc = 0x29; 3690 int gt_cond_enc = 0x37; 3691 if (width != Assembler::Q) { 3692 eq_cond_enc = 0x74 + width; 3693 gt_cond_enc = 0x64 + width; 3694 } 3695 switch (cond) { 3696 case eq: 3697 vpcmpCC(dst, nds, src, eq_cond_enc, width, vector_len); 3698 break; 3699 case neq: 3700 vpcmpCC(dst, nds, src, eq_cond_enc, width, vector_len); 3701 vallones(xtmp, vector_len); 3702 vpxor(dst, xtmp, dst, vector_len); 3703 break; 3704 case le: 3705 vpcmpCC(dst, nds, src, gt_cond_enc, width, vector_len); 3706 vallones(xtmp, vector_len); 3707 vpxor(dst, xtmp, dst, vector_len); 3708 break; 3709 case nlt: 3710 vpcmpCC(dst, src, nds, gt_cond_enc, width, vector_len); 3711 vallones(xtmp, vector_len); 3712 vpxor(dst, xtmp, dst, vector_len); 3713 break; 3714 case lt: 3715 vpcmpCC(dst, src, nds, gt_cond_enc, width, vector_len); 3716 break; 3717 case nle: 3718 vpcmpCC(dst, nds, src, gt_cond_enc, width, vector_len); 3719 break; 3720 default: 3721 assert(false, "Should not reach here"); 3722 } 3723 } 3724 3725 void MacroAssembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) { 3726 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3727 Assembler::vpmovzxbw(dst, src, vector_len); 3728 } 3729 3730 void MacroAssembler::vpmovmskb(Register dst, XMMRegister src, int vector_len) { 3731 assert((src->encoding() < 16),"XMM register should be 0-15"); 3732 Assembler::vpmovmskb(dst, src, vector_len); 3733 } 3734 3735 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3736 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3737 Assembler::vpmullw(dst, nds, src, vector_len); 3738 } 3739 3740 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3741 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3742 Assembler::vpmullw(dst, nds, src, vector_len); 3743 } 3744 3745 void MacroAssembler::vpmulld(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3746 assert((UseAVX > 0), "AVX support is needed"); 3747 assert(rscratch != noreg || always_reachable(src), "missing"); 3748 3749 if (reachable(src)) { 3750 Assembler::vpmulld(dst, nds, as_Address(src), vector_len); 3751 } else { 3752 lea(rscratch, src); 3753 Assembler::vpmulld(dst, nds, Address(rscratch, 0), vector_len); 3754 } 3755 } 3756 3757 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3758 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3759 Assembler::vpsubb(dst, nds, src, vector_len); 3760 } 3761 3762 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3763 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3764 Assembler::vpsubb(dst, nds, src, vector_len); 3765 } 3766 3767 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3768 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3769 Assembler::vpsubw(dst, nds, src, vector_len); 3770 } 3771 3772 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3773 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3774 Assembler::vpsubw(dst, nds, src, vector_len); 3775 } 3776 3777 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3778 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3779 Assembler::vpsraw(dst, nds, shift, vector_len); 3780 } 3781 3782 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3783 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3784 Assembler::vpsraw(dst, nds, shift, vector_len); 3785 } 3786 3787 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3788 assert(UseAVX > 2,""); 3789 if (!VM_Version::supports_avx512vl() && vector_len < 2) { 3790 vector_len = 2; 3791 } 3792 Assembler::evpsraq(dst, nds, shift, vector_len); 3793 } 3794 3795 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3796 assert(UseAVX > 2,""); 3797 if (!VM_Version::supports_avx512vl() && vector_len < 2) { 3798 vector_len = 2; 3799 } 3800 Assembler::evpsraq(dst, nds, shift, vector_len); 3801 } 3802 3803 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3804 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3805 Assembler::vpsrlw(dst, nds, shift, vector_len); 3806 } 3807 3808 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3809 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3810 Assembler::vpsrlw(dst, nds, shift, vector_len); 3811 } 3812 3813 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3814 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3815 Assembler::vpsllw(dst, nds, shift, vector_len); 3816 } 3817 3818 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3819 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3820 Assembler::vpsllw(dst, nds, shift, vector_len); 3821 } 3822 3823 void MacroAssembler::vptest(XMMRegister dst, XMMRegister src) { 3824 assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15"); 3825 Assembler::vptest(dst, src); 3826 } 3827 3828 void MacroAssembler::punpcklbw(XMMRegister dst, XMMRegister src) { 3829 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3830 Assembler::punpcklbw(dst, src); 3831 } 3832 3833 void MacroAssembler::pshufd(XMMRegister dst, Address src, int mode) { 3834 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 3835 Assembler::pshufd(dst, src, mode); 3836 } 3837 3838 void MacroAssembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) { 3839 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3840 Assembler::pshuflw(dst, src, mode); 3841 } 3842 3843 void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3844 assert(rscratch != noreg || always_reachable(src), "missing"); 3845 3846 if (reachable(src)) { 3847 vandpd(dst, nds, as_Address(src), vector_len); 3848 } else { 3849 lea(rscratch, src); 3850 vandpd(dst, nds, Address(rscratch, 0), vector_len); 3851 } 3852 } 3853 3854 void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3855 assert(rscratch != noreg || always_reachable(src), "missing"); 3856 3857 if (reachable(src)) { 3858 vandps(dst, nds, as_Address(src), vector_len); 3859 } else { 3860 lea(rscratch, src); 3861 vandps(dst, nds, Address(rscratch, 0), vector_len); 3862 } 3863 } 3864 3865 void MacroAssembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src, 3866 bool merge, int vector_len, Register rscratch) { 3867 assert(rscratch != noreg || always_reachable(src), "missing"); 3868 3869 if (reachable(src)) { 3870 Assembler::evpord(dst, mask, nds, as_Address(src), merge, vector_len); 3871 } else { 3872 lea(rscratch, src); 3873 Assembler::evpord(dst, mask, nds, Address(rscratch, 0), merge, vector_len); 3874 } 3875 } 3876 3877 void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3878 assert(rscratch != noreg || always_reachable(src), "missing"); 3879 3880 if (reachable(src)) { 3881 vdivsd(dst, nds, as_Address(src)); 3882 } else { 3883 lea(rscratch, src); 3884 vdivsd(dst, nds, Address(rscratch, 0)); 3885 } 3886 } 3887 3888 void MacroAssembler::vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3889 assert(rscratch != noreg || always_reachable(src), "missing"); 3890 3891 if (reachable(src)) { 3892 vdivss(dst, nds, as_Address(src)); 3893 } else { 3894 lea(rscratch, src); 3895 vdivss(dst, nds, Address(rscratch, 0)); 3896 } 3897 } 3898 3899 void MacroAssembler::vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3900 assert(rscratch != noreg || always_reachable(src), "missing"); 3901 3902 if (reachable(src)) { 3903 vmulsd(dst, nds, as_Address(src)); 3904 } else { 3905 lea(rscratch, src); 3906 vmulsd(dst, nds, Address(rscratch, 0)); 3907 } 3908 } 3909 3910 void MacroAssembler::vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3911 assert(rscratch != noreg || always_reachable(src), "missing"); 3912 3913 if (reachable(src)) { 3914 vmulss(dst, nds, as_Address(src)); 3915 } else { 3916 lea(rscratch, src); 3917 vmulss(dst, nds, Address(rscratch, 0)); 3918 } 3919 } 3920 3921 void MacroAssembler::vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3922 assert(rscratch != noreg || always_reachable(src), "missing"); 3923 3924 if (reachable(src)) { 3925 vsubsd(dst, nds, as_Address(src)); 3926 } else { 3927 lea(rscratch, src); 3928 vsubsd(dst, nds, Address(rscratch, 0)); 3929 } 3930 } 3931 3932 void MacroAssembler::vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3933 assert(rscratch != noreg || always_reachable(src), "missing"); 3934 3935 if (reachable(src)) { 3936 vsubss(dst, nds, as_Address(src)); 3937 } else { 3938 lea(rscratch, src); 3939 vsubss(dst, nds, Address(rscratch, 0)); 3940 } 3941 } 3942 3943 void MacroAssembler::vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3944 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); 3945 assert(rscratch != noreg || always_reachable(src), "missing"); 3946 3947 vxorps(dst, nds, src, Assembler::AVX_128bit, rscratch); 3948 } 3949 3950 void MacroAssembler::vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) { 3951 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); 3952 assert(rscratch != noreg || always_reachable(src), "missing"); 3953 3954 vxorpd(dst, nds, src, Assembler::AVX_128bit, rscratch); 3955 } 3956 3957 void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3958 assert(rscratch != noreg || always_reachable(src), "missing"); 3959 3960 if (reachable(src)) { 3961 vxorpd(dst, nds, as_Address(src), vector_len); 3962 } else { 3963 lea(rscratch, src); 3964 vxorpd(dst, nds, Address(rscratch, 0), vector_len); 3965 } 3966 } 3967 3968 void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3969 assert(rscratch != noreg || always_reachable(src), "missing"); 3970 3971 if (reachable(src)) { 3972 vxorps(dst, nds, as_Address(src), vector_len); 3973 } else { 3974 lea(rscratch, src); 3975 vxorps(dst, nds, Address(rscratch, 0), vector_len); 3976 } 3977 } 3978 3979 void MacroAssembler::vpxor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3980 assert(rscratch != noreg || always_reachable(src), "missing"); 3981 3982 if (UseAVX > 1 || (vector_len < 1)) { 3983 if (reachable(src)) { 3984 Assembler::vpxor(dst, nds, as_Address(src), vector_len); 3985 } else { 3986 lea(rscratch, src); 3987 Assembler::vpxor(dst, nds, Address(rscratch, 0), vector_len); 3988 } 3989 } else { 3990 MacroAssembler::vxorpd(dst, nds, src, vector_len, rscratch); 3991 } 3992 } 3993 3994 void MacroAssembler::vpermd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 3995 assert(rscratch != noreg || always_reachable(src), "missing"); 3996 3997 if (reachable(src)) { 3998 Assembler::vpermd(dst, nds, as_Address(src), vector_len); 3999 } else { 4000 lea(rscratch, src); 4001 Assembler::vpermd(dst, nds, Address(rscratch, 0), vector_len); 4002 } 4003 } 4004 4005 void MacroAssembler::clear_jobject_tag(Register possibly_non_local) { 4006 const int32_t inverted_mask = ~static_cast<int32_t>(JNIHandles::tag_mask); 4007 STATIC_ASSERT(inverted_mask == -4); // otherwise check this code 4008 // The inverted mask is sign-extended 4009 andptr(possibly_non_local, inverted_mask); 4010 } 4011 4012 void MacroAssembler::resolve_jobject(Register value, 4013 Register thread, 4014 Register tmp) { 4015 assert_different_registers(value, thread, tmp); 4016 Label done, tagged, weak_tagged; 4017 testptr(value, value); 4018 jcc(Assembler::zero, done); // Use null as-is. 4019 testptr(value, JNIHandles::tag_mask); // Test for tag. 4020 jcc(Assembler::notZero, tagged); 4021 4022 // Resolve local handle 4023 access_load_at(T_OBJECT, IN_NATIVE | AS_RAW, value, Address(value, 0), tmp, thread); 4024 verify_oop(value); 4025 jmp(done); 4026 4027 bind(tagged); 4028 testptr(value, JNIHandles::TypeTag::weak_global); // Test for weak tag. 4029 jcc(Assembler::notZero, weak_tagged); 4030 4031 // Resolve global handle 4032 access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp, thread); 4033 verify_oop(value); 4034 jmp(done); 4035 4036 bind(weak_tagged); 4037 // Resolve jweak. 4038 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, 4039 value, Address(value, -JNIHandles::TypeTag::weak_global), tmp, thread); 4040 verify_oop(value); 4041 4042 bind(done); 4043 } 4044 4045 void MacroAssembler::resolve_global_jobject(Register value, 4046 Register thread, 4047 Register tmp) { 4048 assert_different_registers(value, thread, tmp); 4049 Label done; 4050 4051 testptr(value, value); 4052 jcc(Assembler::zero, done); // Use null as-is. 4053 4054 #ifdef ASSERT 4055 { 4056 Label valid_global_tag; 4057 testptr(value, JNIHandles::TypeTag::global); // Test for global tag. 4058 jcc(Assembler::notZero, valid_global_tag); 4059 stop("non global jobject using resolve_global_jobject"); 4060 bind(valid_global_tag); 4061 } 4062 #endif 4063 4064 // Resolve global handle 4065 access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp, thread); 4066 verify_oop(value); 4067 4068 bind(done); 4069 } 4070 4071 void MacroAssembler::subptr(Register dst, int32_t imm32) { 4072 LP64_ONLY(subq(dst, imm32)) NOT_LP64(subl(dst, imm32)); 4073 } 4074 4075 // Force generation of a 4 byte immediate value even if it fits into 8bit 4076 void MacroAssembler::subptr_imm32(Register dst, int32_t imm32) { 4077 LP64_ONLY(subq_imm32(dst, imm32)) NOT_LP64(subl_imm32(dst, imm32)); 4078 } 4079 4080 void MacroAssembler::subptr(Register dst, Register src) { 4081 LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src)); 4082 } 4083 4084 // C++ bool manipulation 4085 void MacroAssembler::testbool(Register dst) { 4086 if(sizeof(bool) == 1) 4087 testb(dst, 0xff); 4088 else if(sizeof(bool) == 2) { 4089 // testw implementation needed for two byte bools 4090 ShouldNotReachHere(); 4091 } else if(sizeof(bool) == 4) 4092 testl(dst, dst); 4093 else 4094 // unsupported 4095 ShouldNotReachHere(); 4096 } 4097 4098 void MacroAssembler::testptr(Register dst, Register src) { 4099 LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src)); 4100 } 4101 4102 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes. 4103 void MacroAssembler::tlab_allocate(Register thread, Register obj, 4104 Register var_size_in_bytes, 4105 int con_size_in_bytes, 4106 Register t1, 4107 Register t2, 4108 Label& slow_case) { 4109 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); 4110 bs->tlab_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case); 4111 } 4112 4113 RegSet MacroAssembler::call_clobbered_gp_registers() { 4114 RegSet regs; 4115 #ifdef _LP64 4116 regs += RegSet::of(rax, rcx, rdx); 4117 #ifndef _WINDOWS 4118 regs += RegSet::of(rsi, rdi); 4119 #endif 4120 regs += RegSet::range(r8, r11); 4121 #else 4122 regs += RegSet::of(rax, rcx, rdx); 4123 #endif 4124 #ifdef _LP64 4125 if (UseAPX) { 4126 regs += RegSet::range(r16, as_Register(Register::number_of_registers - 1)); 4127 } 4128 #endif 4129 return regs; 4130 } 4131 4132 XMMRegSet MacroAssembler::call_clobbered_xmm_registers() { 4133 int num_xmm_registers = XMMRegister::available_xmm_registers(); 4134 #if defined(_WINDOWS) 4135 XMMRegSet result = XMMRegSet::range(xmm0, xmm5); 4136 if (num_xmm_registers > 16) { 4137 result += XMMRegSet::range(xmm16, as_XMMRegister(num_xmm_registers - 1)); 4138 } 4139 return result; 4140 #else 4141 return XMMRegSet::range(xmm0, as_XMMRegister(num_xmm_registers - 1)); 4142 #endif 4143 } 4144 4145 static int FPUSaveAreaSize = align_up(108, StackAlignmentInBytes); // 108 bytes needed for FPU state by fsave/frstor 4146 4147 #ifndef _LP64 4148 static bool use_x87_registers() { return UseSSE < 2; } 4149 #endif 4150 static bool use_xmm_registers() { return UseSSE >= 1; } 4151 4152 // C1 only ever uses the first double/float of the XMM register. 4153 static int xmm_save_size() { return UseSSE >= 2 ? sizeof(double) : sizeof(float); } 4154 4155 static void save_xmm_register(MacroAssembler* masm, int offset, XMMRegister reg) { 4156 if (UseSSE == 1) { 4157 masm->movflt(Address(rsp, offset), reg); 4158 } else { 4159 masm->movdbl(Address(rsp, offset), reg); 4160 } 4161 } 4162 4163 static void restore_xmm_register(MacroAssembler* masm, int offset, XMMRegister reg) { 4164 if (UseSSE == 1) { 4165 masm->movflt(reg, Address(rsp, offset)); 4166 } else { 4167 masm->movdbl(reg, Address(rsp, offset)); 4168 } 4169 } 4170 4171 static int register_section_sizes(RegSet gp_registers, XMMRegSet xmm_registers, 4172 bool save_fpu, int& gp_area_size, 4173 int& fp_area_size, int& xmm_area_size) { 4174 4175 gp_area_size = align_up(gp_registers.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size, 4176 StackAlignmentInBytes); 4177 #ifdef _LP64 4178 fp_area_size = 0; 4179 #else 4180 fp_area_size = (save_fpu && use_x87_registers()) ? FPUSaveAreaSize : 0; 4181 #endif 4182 xmm_area_size = (save_fpu && use_xmm_registers()) ? xmm_registers.size() * xmm_save_size() : 0; 4183 4184 return gp_area_size + fp_area_size + xmm_area_size; 4185 } 4186 4187 void MacroAssembler::push_call_clobbered_registers_except(RegSet exclude, bool save_fpu) { 4188 block_comment("push_call_clobbered_registers start"); 4189 // Regular registers 4190 RegSet gp_registers_to_push = call_clobbered_gp_registers() - exclude; 4191 4192 int gp_area_size; 4193 int fp_area_size; 4194 int xmm_area_size; 4195 int total_save_size = register_section_sizes(gp_registers_to_push, call_clobbered_xmm_registers(), save_fpu, 4196 gp_area_size, fp_area_size, xmm_area_size); 4197 subptr(rsp, total_save_size); 4198 4199 push_set(gp_registers_to_push, 0); 4200 4201 #ifndef _LP64 4202 if (save_fpu && use_x87_registers()) { 4203 fnsave(Address(rsp, gp_area_size)); 4204 fwait(); 4205 } 4206 #endif 4207 if (save_fpu && use_xmm_registers()) { 4208 push_set(call_clobbered_xmm_registers(), gp_area_size + fp_area_size); 4209 } 4210 4211 block_comment("push_call_clobbered_registers end"); 4212 } 4213 4214 void MacroAssembler::pop_call_clobbered_registers_except(RegSet exclude, bool restore_fpu) { 4215 block_comment("pop_call_clobbered_registers start"); 4216 4217 RegSet gp_registers_to_pop = call_clobbered_gp_registers() - exclude; 4218 4219 int gp_area_size; 4220 int fp_area_size; 4221 int xmm_area_size; 4222 int total_save_size = register_section_sizes(gp_registers_to_pop, call_clobbered_xmm_registers(), restore_fpu, 4223 gp_area_size, fp_area_size, xmm_area_size); 4224 4225 if (restore_fpu && use_xmm_registers()) { 4226 pop_set(call_clobbered_xmm_registers(), gp_area_size + fp_area_size); 4227 } 4228 #ifndef _LP64 4229 if (restore_fpu && use_x87_registers()) { 4230 frstor(Address(rsp, gp_area_size)); 4231 } 4232 #endif 4233 4234 pop_set(gp_registers_to_pop, 0); 4235 4236 addptr(rsp, total_save_size); 4237 4238 vzeroupper(); 4239 4240 block_comment("pop_call_clobbered_registers end"); 4241 } 4242 4243 void MacroAssembler::push_set(XMMRegSet set, int offset) { 4244 assert(is_aligned(set.size() * xmm_save_size(), StackAlignmentInBytes), "must be"); 4245 int spill_offset = offset; 4246 4247 for (RegSetIterator<XMMRegister> it = set.begin(); *it != xnoreg; ++it) { 4248 save_xmm_register(this, spill_offset, *it); 4249 spill_offset += xmm_save_size(); 4250 } 4251 } 4252 4253 void MacroAssembler::pop_set(XMMRegSet set, int offset) { 4254 int restore_size = set.size() * xmm_save_size(); 4255 assert(is_aligned(restore_size, StackAlignmentInBytes), "must be"); 4256 4257 int restore_offset = offset + restore_size - xmm_save_size(); 4258 4259 for (ReverseRegSetIterator<XMMRegister> it = set.rbegin(); *it != xnoreg; ++it) { 4260 restore_xmm_register(this, restore_offset, *it); 4261 restore_offset -= xmm_save_size(); 4262 } 4263 } 4264 4265 void MacroAssembler::push_set(RegSet set, int offset) { 4266 int spill_offset; 4267 if (offset == -1) { 4268 int register_push_size = set.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size; 4269 int aligned_size = align_up(register_push_size, StackAlignmentInBytes); 4270 subptr(rsp, aligned_size); 4271 spill_offset = 0; 4272 } else { 4273 spill_offset = offset; 4274 } 4275 4276 for (RegSetIterator<Register> it = set.begin(); *it != noreg; ++it) { 4277 movptr(Address(rsp, spill_offset), *it); 4278 spill_offset += Register::max_slots_per_register * VMRegImpl::stack_slot_size; 4279 } 4280 } 4281 4282 void MacroAssembler::pop_set(RegSet set, int offset) { 4283 4284 int gp_reg_size = Register::max_slots_per_register * VMRegImpl::stack_slot_size; 4285 int restore_size = set.size() * gp_reg_size; 4286 int aligned_size = align_up(restore_size, StackAlignmentInBytes); 4287 4288 int restore_offset; 4289 if (offset == -1) { 4290 restore_offset = restore_size - gp_reg_size; 4291 } else { 4292 restore_offset = offset + restore_size - gp_reg_size; 4293 } 4294 for (ReverseRegSetIterator<Register> it = set.rbegin(); *it != noreg; ++it) { 4295 movptr(*it, Address(rsp, restore_offset)); 4296 restore_offset -= gp_reg_size; 4297 } 4298 4299 if (offset == -1) { 4300 addptr(rsp, aligned_size); 4301 } 4302 } 4303 4304 // Preserves the contents of address, destroys the contents length_in_bytes and temp. 4305 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) { 4306 assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different"); 4307 assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord"); 4308 Label done; 4309 4310 testptr(length_in_bytes, length_in_bytes); 4311 jcc(Assembler::zero, done); 4312 4313 // initialize topmost word, divide index by 2, check if odd and test if zero 4314 // note: for the remaining code to work, index must be a multiple of BytesPerWord 4315 #ifdef ASSERT 4316 { 4317 Label L; 4318 testptr(length_in_bytes, BytesPerWord - 1); 4319 jcc(Assembler::zero, L); 4320 stop("length must be a multiple of BytesPerWord"); 4321 bind(L); 4322 } 4323 #endif 4324 Register index = length_in_bytes; 4325 xorptr(temp, temp); // use _zero reg to clear memory (shorter code) 4326 if (UseIncDec) { 4327 shrptr(index, 3); // divide by 8/16 and set carry flag if bit 2 was set 4328 } else { 4329 shrptr(index, 2); // use 2 instructions to avoid partial flag stall 4330 shrptr(index, 1); 4331 } 4332 #ifndef _LP64 4333 // index could have not been a multiple of 8 (i.e., bit 2 was set) 4334 { 4335 Label even; 4336 // note: if index was a multiple of 8, then it cannot 4337 // be 0 now otherwise it must have been 0 before 4338 // => if it is even, we don't need to check for 0 again 4339 jcc(Assembler::carryClear, even); 4340 // clear topmost word (no jump would be needed if conditional assignment worked here) 4341 movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp); 4342 // index could be 0 now, must check again 4343 jcc(Assembler::zero, done); 4344 bind(even); 4345 } 4346 #endif // !_LP64 4347 // initialize remaining object fields: index is a multiple of 2 now 4348 { 4349 Label loop; 4350 bind(loop); 4351 movptr(Address(address, index, Address::times_8, offset_in_bytes - 1*BytesPerWord), temp); 4352 NOT_LP64(movptr(Address(address, index, Address::times_8, offset_in_bytes - 2*BytesPerWord), temp);) 4353 decrement(index); 4354 jcc(Assembler::notZero, loop); 4355 } 4356 4357 bind(done); 4358 } 4359 4360 // Look up the method for a megamorphic invokeinterface call. 4361 // The target method is determined by <intf_klass, itable_index>. 4362 // The receiver klass is in recv_klass. 4363 // On success, the result will be in method_result, and execution falls through. 4364 // On failure, execution transfers to the given label. 4365 void MacroAssembler::lookup_interface_method(Register recv_klass, 4366 Register intf_klass, 4367 RegisterOrConstant itable_index, 4368 Register method_result, 4369 Register scan_temp, 4370 Label& L_no_such_interface, 4371 bool return_method) { 4372 assert_different_registers(recv_klass, intf_klass, scan_temp); 4373 assert_different_registers(method_result, intf_klass, scan_temp); 4374 assert(recv_klass != method_result || !return_method, 4375 "recv_klass can be destroyed when method isn't needed"); 4376 4377 assert(itable_index.is_constant() || itable_index.as_register() == method_result, 4378 "caller must use same register for non-constant itable index as for method"); 4379 4380 // Compute start of first itableOffsetEntry (which is at the end of the vtable) 4381 int vtable_base = in_bytes(Klass::vtable_start_offset()); 4382 int itentry_off = in_bytes(itableMethodEntry::method_offset()); 4383 int scan_step = itableOffsetEntry::size() * wordSize; 4384 int vte_size = vtableEntry::size_in_bytes(); 4385 Address::ScaleFactor times_vte_scale = Address::times_ptr; 4386 assert(vte_size == wordSize, "else adjust times_vte_scale"); 4387 4388 movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset())); 4389 4390 // Could store the aligned, prescaled offset in the klass. 4391 lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base)); 4392 4393 if (return_method) { 4394 // Adjust recv_klass by scaled itable_index, so we can free itable_index. 4395 assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below"); 4396 lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off)); 4397 } 4398 4399 // for (scan = klass->itable(); scan->interface() != nullptr; scan += scan_step) { 4400 // if (scan->interface() == intf) { 4401 // result = (klass + scan->offset() + itable_index); 4402 // } 4403 // } 4404 Label search, found_method; 4405 4406 for (int peel = 1; peel >= 0; peel--) { 4407 movptr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset())); 4408 cmpptr(intf_klass, method_result); 4409 4410 if (peel) { 4411 jccb(Assembler::equal, found_method); 4412 } else { 4413 jccb(Assembler::notEqual, search); 4414 // (invert the test to fall through to found_method...) 4415 } 4416 4417 if (!peel) break; 4418 4419 bind(search); 4420 4421 // Check that the previous entry is non-null. A null entry means that 4422 // the receiver class doesn't implement the interface, and wasn't the 4423 // same as when the caller was compiled. 4424 testptr(method_result, method_result); 4425 jcc(Assembler::zero, L_no_such_interface); 4426 addptr(scan_temp, scan_step); 4427 } 4428 4429 bind(found_method); 4430 4431 if (return_method) { 4432 // Got a hit. 4433 movl(scan_temp, Address(scan_temp, itableOffsetEntry::offset_offset())); 4434 movptr(method_result, Address(recv_klass, scan_temp, Address::times_1)); 4435 } 4436 } 4437 4438 // Look up the method for a megamorphic invokeinterface call in a single pass over itable: 4439 // - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICData 4440 // - find a holder_klass (class that implements the method) vtable offset and get the method from vtable by index 4441 // The target method is determined by <holder_klass, itable_index>. 4442 // The receiver klass is in recv_klass. 4443 // On success, the result will be in method_result, and execution falls through. 4444 // On failure, execution transfers to the given label. 4445 void MacroAssembler::lookup_interface_method_stub(Register recv_klass, 4446 Register holder_klass, 4447 Register resolved_klass, 4448 Register method_result, 4449 Register scan_temp, 4450 Register temp_reg2, 4451 Register receiver, 4452 int itable_index, 4453 Label& L_no_such_interface) { 4454 assert_different_registers(recv_klass, method_result, holder_klass, resolved_klass, scan_temp, temp_reg2, receiver); 4455 Register temp_itbl_klass = method_result; 4456 Register temp_reg = (temp_reg2 == noreg ? recv_klass : temp_reg2); // reuse recv_klass register on 32-bit x86 impl 4457 4458 int vtable_base = in_bytes(Klass::vtable_start_offset()); 4459 int itentry_off = in_bytes(itableMethodEntry::method_offset()); 4460 int scan_step = itableOffsetEntry::size() * wordSize; 4461 int vte_size = vtableEntry::size_in_bytes(); 4462 int ioffset = in_bytes(itableOffsetEntry::interface_offset()); 4463 int ooffset = in_bytes(itableOffsetEntry::offset_offset()); 4464 Address::ScaleFactor times_vte_scale = Address::times_ptr; 4465 assert(vte_size == wordSize, "adjust times_vte_scale"); 4466 4467 Label L_loop_scan_resolved_entry, L_resolved_found, L_holder_found; 4468 4469 // temp_itbl_klass = recv_klass.itable[0] 4470 // scan_temp = &recv_klass.itable[0] + step 4471 movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset())); 4472 movptr(temp_itbl_klass, Address(recv_klass, scan_temp, times_vte_scale, vtable_base + ioffset)); 4473 lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base + ioffset + scan_step)); 4474 xorptr(temp_reg, temp_reg); 4475 4476 // Initial checks: 4477 // - if (holder_klass != resolved_klass), go to "scan for resolved" 4478 // - if (itable[0] == 0), no such interface 4479 // - if (itable[0] == holder_klass), shortcut to "holder found" 4480 cmpptr(holder_klass, resolved_klass); 4481 jccb(Assembler::notEqual, L_loop_scan_resolved_entry); 4482 testptr(temp_itbl_klass, temp_itbl_klass); 4483 jccb(Assembler::zero, L_no_such_interface); 4484 cmpptr(holder_klass, temp_itbl_klass); 4485 jccb(Assembler::equal, L_holder_found); 4486 4487 // Loop: Look for holder_klass record in itable 4488 // do { 4489 // tmp = itable[index]; 4490 // index += step; 4491 // if (tmp == holder_klass) { 4492 // goto L_holder_found; // Found! 4493 // } 4494 // } while (tmp != 0); 4495 // goto L_no_such_interface // Not found. 4496 Label L_scan_holder; 4497 bind(L_scan_holder); 4498 movptr(temp_itbl_klass, Address(scan_temp, 0)); 4499 addptr(scan_temp, scan_step); 4500 cmpptr(holder_klass, temp_itbl_klass); 4501 jccb(Assembler::equal, L_holder_found); 4502 testptr(temp_itbl_klass, temp_itbl_klass); 4503 jccb(Assembler::notZero, L_scan_holder); 4504 4505 jmpb(L_no_such_interface); 4506 4507 // Loop: Look for resolved_class record in itable 4508 // do { 4509 // tmp = itable[index]; 4510 // index += step; 4511 // if (tmp == holder_klass) { 4512 // // Also check if we have met a holder klass 4513 // holder_tmp = itable[index-step-ioffset]; 4514 // } 4515 // if (tmp == resolved_klass) { 4516 // goto L_resolved_found; // Found! 4517 // } 4518 // } while (tmp != 0); 4519 // goto L_no_such_interface // Not found. 4520 // 4521 Label L_loop_scan_resolved; 4522 bind(L_loop_scan_resolved); 4523 movptr(temp_itbl_klass, Address(scan_temp, 0)); 4524 addptr(scan_temp, scan_step); 4525 bind(L_loop_scan_resolved_entry); 4526 cmpptr(holder_klass, temp_itbl_klass); 4527 cmovl(Assembler::equal, temp_reg, Address(scan_temp, ooffset - ioffset - scan_step)); 4528 cmpptr(resolved_klass, temp_itbl_klass); 4529 jccb(Assembler::equal, L_resolved_found); 4530 testptr(temp_itbl_klass, temp_itbl_klass); 4531 jccb(Assembler::notZero, L_loop_scan_resolved); 4532 4533 jmpb(L_no_such_interface); 4534 4535 Label L_ready; 4536 4537 // See if we already have a holder klass. If not, go and scan for it. 4538 bind(L_resolved_found); 4539 testptr(temp_reg, temp_reg); 4540 jccb(Assembler::zero, L_scan_holder); 4541 jmpb(L_ready); 4542 4543 bind(L_holder_found); 4544 movl(temp_reg, Address(scan_temp, ooffset - ioffset - scan_step)); 4545 4546 // Finally, temp_reg contains holder_klass vtable offset 4547 bind(L_ready); 4548 assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below"); 4549 if (temp_reg2 == noreg) { // recv_klass register is clobbered for 32-bit x86 impl 4550 load_klass(scan_temp, receiver, noreg); 4551 movptr(method_result, Address(scan_temp, temp_reg, Address::times_1, itable_index * wordSize + itentry_off)); 4552 } else { 4553 movptr(method_result, Address(recv_klass, temp_reg, Address::times_1, itable_index * wordSize + itentry_off)); 4554 } 4555 } 4556 4557 4558 // virtual method calling 4559 void MacroAssembler::lookup_virtual_method(Register recv_klass, 4560 RegisterOrConstant vtable_index, 4561 Register method_result) { 4562 const ByteSize base = Klass::vtable_start_offset(); 4563 assert(vtableEntry::size() * wordSize == wordSize, "else adjust the scaling in the code below"); 4564 Address vtable_entry_addr(recv_klass, 4565 vtable_index, Address::times_ptr, 4566 base + vtableEntry::method_offset()); 4567 movptr(method_result, vtable_entry_addr); 4568 } 4569 4570 4571 void MacroAssembler::check_klass_subtype(Register sub_klass, 4572 Register super_klass, 4573 Register temp_reg, 4574 Label& L_success) { 4575 Label L_failure; 4576 check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg, &L_success, &L_failure, nullptr); 4577 check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, noreg, &L_success, nullptr); 4578 bind(L_failure); 4579 } 4580 4581 4582 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass, 4583 Register super_klass, 4584 Register temp_reg, 4585 Label* L_success, 4586 Label* L_failure, 4587 Label* L_slow_path, 4588 RegisterOrConstant super_check_offset) { 4589 assert_different_registers(sub_klass, super_klass, temp_reg); 4590 bool must_load_sco = (super_check_offset.constant_or_zero() == -1); 4591 if (super_check_offset.is_register()) { 4592 assert_different_registers(sub_klass, super_klass, 4593 super_check_offset.as_register()); 4594 } else if (must_load_sco) { 4595 assert(temp_reg != noreg, "supply either a temp or a register offset"); 4596 } 4597 4598 Label L_fallthrough; 4599 int label_nulls = 0; 4600 if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; } 4601 if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; } 4602 if (L_slow_path == nullptr) { L_slow_path = &L_fallthrough; label_nulls++; } 4603 assert(label_nulls <= 1, "at most one null in the batch"); 4604 4605 int sc_offset = in_bytes(Klass::secondary_super_cache_offset()); 4606 int sco_offset = in_bytes(Klass::super_check_offset_offset()); 4607 Address super_check_offset_addr(super_klass, sco_offset); 4608 4609 // Hacked jcc, which "knows" that L_fallthrough, at least, is in 4610 // range of a jccb. If this routine grows larger, reconsider at 4611 // least some of these. 4612 #define local_jcc(assembler_cond, label) \ 4613 if (&(label) == &L_fallthrough) jccb(assembler_cond, label); \ 4614 else jcc( assembler_cond, label) /*omit semi*/ 4615 4616 // Hacked jmp, which may only be used just before L_fallthrough. 4617 #define final_jmp(label) \ 4618 if (&(label) == &L_fallthrough) { /*do nothing*/ } \ 4619 else jmp(label) /*omit semi*/ 4620 4621 // If the pointers are equal, we are done (e.g., String[] elements). 4622 // This self-check enables sharing of secondary supertype arrays among 4623 // non-primary types such as array-of-interface. Otherwise, each such 4624 // type would need its own customized SSA. 4625 // We move this check to the front of the fast path because many 4626 // type checks are in fact trivially successful in this manner, 4627 // so we get a nicely predicted branch right at the start of the check. 4628 cmpptr(sub_klass, super_klass); 4629 local_jcc(Assembler::equal, *L_success); 4630 4631 // Check the supertype display: 4632 if (must_load_sco) { 4633 // Positive movl does right thing on LP64. 4634 movl(temp_reg, super_check_offset_addr); 4635 super_check_offset = RegisterOrConstant(temp_reg); 4636 } 4637 Address super_check_addr(sub_klass, super_check_offset, Address::times_1, 0); 4638 cmpptr(super_klass, super_check_addr); // load displayed supertype 4639 4640 // This check has worked decisively for primary supers. 4641 // Secondary supers are sought in the super_cache ('super_cache_addr'). 4642 // (Secondary supers are interfaces and very deeply nested subtypes.) 4643 // This works in the same check above because of a tricky aliasing 4644 // between the super_cache and the primary super display elements. 4645 // (The 'super_check_addr' can address either, as the case requires.) 4646 // Note that the cache is updated below if it does not help us find 4647 // what we need immediately. 4648 // So if it was a primary super, we can just fail immediately. 4649 // Otherwise, it's the slow path for us (no success at this point). 4650 4651 if (super_check_offset.is_register()) { 4652 local_jcc(Assembler::equal, *L_success); 4653 cmpl(super_check_offset.as_register(), sc_offset); 4654 if (L_failure == &L_fallthrough) { 4655 local_jcc(Assembler::equal, *L_slow_path); 4656 } else { 4657 local_jcc(Assembler::notEqual, *L_failure); 4658 final_jmp(*L_slow_path); 4659 } 4660 } else if (super_check_offset.as_constant() == sc_offset) { 4661 // Need a slow path; fast failure is impossible. 4662 if (L_slow_path == &L_fallthrough) { 4663 local_jcc(Assembler::equal, *L_success); 4664 } else { 4665 local_jcc(Assembler::notEqual, *L_slow_path); 4666 final_jmp(*L_success); 4667 } 4668 } else { 4669 // No slow path; it's a fast decision. 4670 if (L_failure == &L_fallthrough) { 4671 local_jcc(Assembler::equal, *L_success); 4672 } else { 4673 local_jcc(Assembler::notEqual, *L_failure); 4674 final_jmp(*L_success); 4675 } 4676 } 4677 4678 bind(L_fallthrough); 4679 4680 #undef local_jcc 4681 #undef final_jmp 4682 } 4683 4684 4685 void MacroAssembler::check_klass_subtype_slow_path_linear(Register sub_klass, 4686 Register super_klass, 4687 Register temp_reg, 4688 Register temp2_reg, 4689 Label* L_success, 4690 Label* L_failure, 4691 bool set_cond_codes) { 4692 assert_different_registers(sub_klass, super_klass, temp_reg); 4693 if (temp2_reg != noreg) 4694 assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg); 4695 #define IS_A_TEMP(reg) ((reg) == temp_reg || (reg) == temp2_reg) 4696 4697 Label L_fallthrough; 4698 int label_nulls = 0; 4699 if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; } 4700 if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; } 4701 assert(label_nulls <= 1, "at most one null in the batch"); 4702 4703 // a couple of useful fields in sub_klass: 4704 int ss_offset = in_bytes(Klass::secondary_supers_offset()); 4705 int sc_offset = in_bytes(Klass::secondary_super_cache_offset()); 4706 Address secondary_supers_addr(sub_klass, ss_offset); 4707 Address super_cache_addr( sub_klass, sc_offset); 4708 4709 // Do a linear scan of the secondary super-klass chain. 4710 // This code is rarely used, so simplicity is a virtue here. 4711 // The repne_scan instruction uses fixed registers, which we must spill. 4712 // Don't worry too much about pre-existing connections with the input regs. 4713 4714 assert(sub_klass != rax, "killed reg"); // killed by mov(rax, super) 4715 assert(sub_klass != rcx, "killed reg"); // killed by lea(rcx, &pst_counter) 4716 4717 // Get super_klass value into rax (even if it was in rdi or rcx). 4718 bool pushed_rax = false, pushed_rcx = false, pushed_rdi = false; 4719 if (super_klass != rax) { 4720 if (!IS_A_TEMP(rax)) { push(rax); pushed_rax = true; } 4721 mov(rax, super_klass); 4722 } 4723 if (!IS_A_TEMP(rcx)) { push(rcx); pushed_rcx = true; } 4724 if (!IS_A_TEMP(rdi)) { push(rdi); pushed_rdi = true; } 4725 4726 #ifndef PRODUCT 4727 uint* pst_counter = &SharedRuntime::_partial_subtype_ctr; 4728 ExternalAddress pst_counter_addr((address) pst_counter); 4729 NOT_LP64( incrementl(pst_counter_addr) ); 4730 LP64_ONLY( lea(rcx, pst_counter_addr) ); 4731 LP64_ONLY( incrementl(Address(rcx, 0)) ); 4732 #endif //PRODUCT 4733 4734 // We will consult the secondary-super array. 4735 movptr(rdi, secondary_supers_addr); 4736 // Load the array length. (Positive movl does right thing on LP64.) 4737 movl(rcx, Address(rdi, Array<Klass*>::length_offset_in_bytes())); 4738 // Skip to start of data. 4739 addptr(rdi, Array<Klass*>::base_offset_in_bytes()); 4740 4741 // Scan RCX words at [RDI] for an occurrence of RAX. 4742 // Set NZ/Z based on last compare. 4743 // Z flag value will not be set by 'repne' if RCX == 0 since 'repne' does 4744 // not change flags (only scas instruction which is repeated sets flags). 4745 // Set Z = 0 (not equal) before 'repne' to indicate that class was not found. 4746 4747 testptr(rax,rax); // Set Z = 0 4748 repne_scan(); 4749 4750 // Unspill the temp. registers: 4751 if (pushed_rdi) pop(rdi); 4752 if (pushed_rcx) pop(rcx); 4753 if (pushed_rax) pop(rax); 4754 4755 if (set_cond_codes) { 4756 // Special hack for the AD files: rdi is guaranteed non-zero. 4757 assert(!pushed_rdi, "rdi must be left non-null"); 4758 // Also, the condition codes are properly set Z/NZ on succeed/failure. 4759 } 4760 4761 if (L_failure == &L_fallthrough) 4762 jccb(Assembler::notEqual, *L_failure); 4763 else jcc(Assembler::notEqual, *L_failure); 4764 4765 // Success. Cache the super we found and proceed in triumph. 4766 movptr(super_cache_addr, super_klass); 4767 4768 if (L_success != &L_fallthrough) { 4769 jmp(*L_success); 4770 } 4771 4772 #undef IS_A_TEMP 4773 4774 bind(L_fallthrough); 4775 } 4776 4777 #ifndef _LP64 4778 4779 // 32-bit x86 only: always use the linear search. 4780 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass, 4781 Register super_klass, 4782 Register temp_reg, 4783 Register temp2_reg, 4784 Label* L_success, 4785 Label* L_failure, 4786 bool set_cond_codes) { 4787 check_klass_subtype_slow_path_linear 4788 (sub_klass, super_klass, temp_reg, temp2_reg, L_success, L_failure, set_cond_codes); 4789 } 4790 4791 #else // _LP64 4792 4793 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass, 4794 Register super_klass, 4795 Register temp_reg, 4796 Register temp2_reg, 4797 Label* L_success, 4798 Label* L_failure, 4799 bool set_cond_codes) { 4800 assert(set_cond_codes == false, "must be false on 64-bit x86"); 4801 check_klass_subtype_slow_path 4802 (sub_klass, super_klass, temp_reg, temp2_reg, noreg, noreg, 4803 L_success, L_failure); 4804 } 4805 4806 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass, 4807 Register super_klass, 4808 Register temp_reg, 4809 Register temp2_reg, 4810 Register temp3_reg, 4811 Register temp4_reg, 4812 Label* L_success, 4813 Label* L_failure) { 4814 if (UseSecondarySupersTable) { 4815 check_klass_subtype_slow_path_table 4816 (sub_klass, super_klass, temp_reg, temp2_reg, temp3_reg, temp4_reg, 4817 L_success, L_failure); 4818 } else { 4819 check_klass_subtype_slow_path_linear 4820 (sub_klass, super_klass, temp_reg, temp2_reg, L_success, L_failure, /*set_cond_codes*/false); 4821 } 4822 } 4823 4824 Register MacroAssembler::allocate_if_noreg(Register r, 4825 RegSetIterator<Register> &available_regs, 4826 RegSet ®s_to_push) { 4827 if (!r->is_valid()) { 4828 r = *available_regs++; 4829 regs_to_push += r; 4830 } 4831 return r; 4832 } 4833 4834 void MacroAssembler::check_klass_subtype_slow_path_table(Register sub_klass, 4835 Register super_klass, 4836 Register temp_reg, 4837 Register temp2_reg, 4838 Register temp3_reg, 4839 Register result_reg, 4840 Label* L_success, 4841 Label* L_failure) { 4842 // NB! Callers may assume that, when temp2_reg is a valid register, 4843 // this code sets it to a nonzero value. 4844 bool temp2_reg_was_valid = temp2_reg->is_valid(); 4845 4846 RegSet temps = RegSet::of(temp_reg, temp2_reg, temp3_reg); 4847 4848 Label L_fallthrough; 4849 int label_nulls = 0; 4850 if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; } 4851 if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; } 4852 assert(label_nulls <= 1, "at most one null in the batch"); 4853 4854 BLOCK_COMMENT("check_klass_subtype_slow_path_table"); 4855 4856 RegSetIterator<Register> available_regs 4857 = (RegSet::of(rax, rcx, rdx, r8) + r9 + r10 + r11 + r12 - temps - sub_klass - super_klass).begin(); 4858 4859 RegSet pushed_regs; 4860 4861 temp_reg = allocate_if_noreg(temp_reg, available_regs, pushed_regs); 4862 temp2_reg = allocate_if_noreg(temp2_reg, available_regs, pushed_regs); 4863 temp3_reg = allocate_if_noreg(temp3_reg, available_regs, pushed_regs); 4864 result_reg = allocate_if_noreg(result_reg, available_regs, pushed_regs); 4865 Register temp4_reg = allocate_if_noreg(noreg, available_regs, pushed_regs); 4866 4867 assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg, temp3_reg, result_reg); 4868 4869 { 4870 4871 int register_push_size = pushed_regs.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size; 4872 int aligned_size = align_up(register_push_size, StackAlignmentInBytes); 4873 subptr(rsp, aligned_size); 4874 push_set(pushed_regs, 0); 4875 4876 lookup_secondary_supers_table_var(sub_klass, 4877 super_klass, 4878 temp_reg, temp2_reg, temp3_reg, temp4_reg, result_reg); 4879 cmpq(result_reg, 0); 4880 4881 // Unspill the temp. registers: 4882 pop_set(pushed_regs, 0); 4883 // Increment SP but do not clobber flags. 4884 lea(rsp, Address(rsp, aligned_size)); 4885 } 4886 4887 if (temp2_reg_was_valid) { 4888 movq(temp2_reg, 1); 4889 } 4890 4891 jcc(Assembler::notEqual, *L_failure); 4892 4893 if (L_success != &L_fallthrough) { 4894 jmp(*L_success); 4895 } 4896 4897 bind(L_fallthrough); 4898 } 4899 4900 // population_count variant for running without the POPCNT 4901 // instruction, which was introduced with SSE4.2 in 2008. 4902 void MacroAssembler::population_count(Register dst, Register src, 4903 Register scratch1, Register scratch2) { 4904 assert_different_registers(src, scratch1, scratch2); 4905 if (UsePopCountInstruction) { 4906 Assembler::popcntq(dst, src); 4907 } else { 4908 assert_different_registers(src, scratch1, scratch2); 4909 assert_different_registers(dst, scratch1, scratch2); 4910 Label loop, done; 4911 4912 mov(scratch1, src); 4913 // dst = 0; 4914 // while(scratch1 != 0) { 4915 // dst++; 4916 // scratch1 &= (scratch1 - 1); 4917 // } 4918 xorl(dst, dst); 4919 testq(scratch1, scratch1); 4920 jccb(Assembler::equal, done); 4921 { 4922 bind(loop); 4923 incq(dst); 4924 movq(scratch2, scratch1); 4925 decq(scratch2); 4926 andq(scratch1, scratch2); 4927 jccb(Assembler::notEqual, loop); 4928 } 4929 bind(done); 4930 } 4931 #ifdef ASSERT 4932 mov64(scratch1, 0xCafeBabeDeadBeef); 4933 movq(scratch2, scratch1); 4934 #endif 4935 } 4936 4937 // Ensure that the inline code and the stub are using the same registers. 4938 #define LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS \ 4939 do { \ 4940 assert(r_super_klass == rax, "mismatch"); \ 4941 assert(r_array_base == rbx, "mismatch"); \ 4942 assert(r_array_length == rcx, "mismatch"); \ 4943 assert(r_array_index == rdx, "mismatch"); \ 4944 assert(r_sub_klass == rsi || r_sub_klass == noreg, "mismatch"); \ 4945 assert(r_bitmap == r11 || r_bitmap == noreg, "mismatch"); \ 4946 assert(result == rdi || result == noreg, "mismatch"); \ 4947 } while(0) 4948 4949 // Versions of salq and rorq that don't need count to be in rcx 4950 4951 void MacroAssembler::salq(Register dest, Register count) { 4952 if (count == rcx) { 4953 Assembler::salq(dest); 4954 } else { 4955 assert_different_registers(rcx, dest); 4956 xchgq(rcx, count); 4957 Assembler::salq(dest); 4958 xchgq(rcx, count); 4959 } 4960 } 4961 4962 void MacroAssembler::rorq(Register dest, Register count) { 4963 if (count == rcx) { 4964 Assembler::rorq(dest); 4965 } else { 4966 assert_different_registers(rcx, dest); 4967 xchgq(rcx, count); 4968 Assembler::rorq(dest); 4969 xchgq(rcx, count); 4970 } 4971 } 4972 4973 // Return true: we succeeded in generating this code 4974 // 4975 // At runtime, return 0 in result if r_super_klass is a superclass of 4976 // r_sub_klass, otherwise return nonzero. Use this if you know the 4977 // super_klass_slot of the class you're looking for. This is always 4978 // the case for instanceof and checkcast. 4979 void MacroAssembler::lookup_secondary_supers_table_const(Register r_sub_klass, 4980 Register r_super_klass, 4981 Register temp1, 4982 Register temp2, 4983 Register temp3, 4984 Register temp4, 4985 Register result, 4986 u1 super_klass_slot) { 4987 assert_different_registers(r_sub_klass, r_super_klass, temp1, temp2, temp3, temp4, result); 4988 4989 Label L_fallthrough, L_success, L_failure; 4990 4991 BLOCK_COMMENT("lookup_secondary_supers_table {"); 4992 4993 const Register 4994 r_array_index = temp1, 4995 r_array_length = temp2, 4996 r_array_base = temp3, 4997 r_bitmap = temp4; 4998 4999 LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS; 5000 5001 xorq(result, result); // = 0 5002 5003 movq(r_bitmap, Address(r_sub_klass, Klass::secondary_supers_bitmap_offset())); 5004 movq(r_array_index, r_bitmap); 5005 5006 // First check the bitmap to see if super_klass might be present. If 5007 // the bit is zero, we are certain that super_klass is not one of 5008 // the secondary supers. 5009 u1 bit = super_klass_slot; 5010 { 5011 // NB: If the count in a x86 shift instruction is 0, the flags are 5012 // not affected, so we do a testq instead. 5013 int shift_count = Klass::SECONDARY_SUPERS_TABLE_MASK - bit; 5014 if (shift_count != 0) { 5015 salq(r_array_index, shift_count); 5016 } else { 5017 testq(r_array_index, r_array_index); 5018 } 5019 } 5020 // We test the MSB of r_array_index, i.e. its sign bit 5021 jcc(Assembler::positive, L_failure); 5022 5023 // Get the first array index that can contain super_klass into r_array_index. 5024 if (bit != 0) { 5025 population_count(r_array_index, r_array_index, temp2, temp3); 5026 } else { 5027 movl(r_array_index, 1); 5028 } 5029 // NB! r_array_index is off by 1. It is compensated by keeping r_array_base off by 1 word. 5030 5031 // We will consult the secondary-super array. 5032 movptr(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset()))); 5033 5034 // We're asserting that the first word in an Array<Klass*> is the 5035 // length, and the second word is the first word of the data. If 5036 // that ever changes, r_array_base will have to be adjusted here. 5037 assert(Array<Klass*>::base_offset_in_bytes() == wordSize, "Adjust this code"); 5038 assert(Array<Klass*>::length_offset_in_bytes() == 0, "Adjust this code"); 5039 5040 cmpq(r_super_klass, Address(r_array_base, r_array_index, Address::times_8)); 5041 jccb(Assembler::equal, L_success); 5042 5043 // Is there another entry to check? Consult the bitmap. 5044 btq(r_bitmap, (bit + 1) & Klass::SECONDARY_SUPERS_TABLE_MASK); 5045 jccb(Assembler::carryClear, L_failure); 5046 5047 // Linear probe. Rotate the bitmap so that the next bit to test is 5048 // in Bit 1. 5049 if (bit != 0) { 5050 rorq(r_bitmap, bit); 5051 } 5052 5053 // Calls into the stub generated by lookup_secondary_supers_table_slow_path. 5054 // Arguments: r_super_klass, r_array_base, r_array_index, r_bitmap. 5055 // Kills: r_array_length. 5056 // Returns: result. 5057 call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_slow_path_stub())); 5058 // Result (0/1) is in rdi 5059 jmpb(L_fallthrough); 5060 5061 bind(L_failure); 5062 incq(result); // 0 => 1 5063 5064 bind(L_success); 5065 // result = 0; 5066 5067 bind(L_fallthrough); 5068 BLOCK_COMMENT("} lookup_secondary_supers_table"); 5069 5070 if (VerifySecondarySupers) { 5071 verify_secondary_supers_table(r_sub_klass, r_super_klass, result, 5072 temp1, temp2, temp3); 5073 } 5074 } 5075 5076 // At runtime, return 0 in result if r_super_klass is a superclass of 5077 // r_sub_klass, otherwise return nonzero. Use this version of 5078 // lookup_secondary_supers_table() if you don't know ahead of time 5079 // which superclass will be searched for. Used by interpreter and 5080 // runtime stubs. It is larger and has somewhat greater latency than 5081 // the version above, which takes a constant super_klass_slot. 5082 void MacroAssembler::lookup_secondary_supers_table_var(Register r_sub_klass, 5083 Register r_super_klass, 5084 Register temp1, 5085 Register temp2, 5086 Register temp3, 5087 Register temp4, 5088 Register result) { 5089 assert_different_registers(r_sub_klass, r_super_klass, temp1, temp2, temp3, temp4, result); 5090 assert_different_registers(r_sub_klass, r_super_klass, rcx); 5091 RegSet temps = RegSet::of(temp1, temp2, temp3, temp4); 5092 5093 Label L_fallthrough, L_success, L_failure; 5094 5095 BLOCK_COMMENT("lookup_secondary_supers_table {"); 5096 5097 RegSetIterator<Register> available_regs = (temps - rcx).begin(); 5098 5099 // FIXME. Once we are sure that all paths reaching this point really 5100 // do pass rcx as one of our temps we can get rid of the following 5101 // workaround. 5102 assert(temps.contains(rcx), "fix this code"); 5103 5104 // We prefer to have our shift count in rcx. If rcx is one of our 5105 // temps, use it for slot. If not, pick any of our temps. 5106 Register slot; 5107 if (!temps.contains(rcx)) { 5108 slot = *available_regs++; 5109 } else { 5110 slot = rcx; 5111 } 5112 5113 const Register r_array_index = *available_regs++; 5114 const Register r_bitmap = *available_regs++; 5115 5116 // The logic above guarantees this property, but we state it here. 5117 assert_different_registers(r_array_index, r_bitmap, rcx); 5118 5119 movq(r_bitmap, Address(r_sub_klass, Klass::secondary_supers_bitmap_offset())); 5120 movq(r_array_index, r_bitmap); 5121 5122 // First check the bitmap to see if super_klass might be present. If 5123 // the bit is zero, we are certain that super_klass is not one of 5124 // the secondary supers. 5125 movb(slot, Address(r_super_klass, Klass::hash_slot_offset())); 5126 xorl(slot, (u1)(Klass::SECONDARY_SUPERS_TABLE_SIZE - 1)); // slot ^ 63 === 63 - slot (mod 64) 5127 salq(r_array_index, slot); 5128 5129 testq(r_array_index, r_array_index); 5130 // We test the MSB of r_array_index, i.e. its sign bit 5131 jcc(Assembler::positive, L_failure); 5132 5133 const Register r_array_base = *available_regs++; 5134 5135 // Get the first array index that can contain super_klass into r_array_index. 5136 // Note: Clobbers r_array_base and slot. 5137 population_count(r_array_index, r_array_index, /*temp2*/r_array_base, /*temp3*/slot); 5138 5139 // NB! r_array_index is off by 1. It is compensated by keeping r_array_base off by 1 word. 5140 5141 // We will consult the secondary-super array. 5142 movptr(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset()))); 5143 5144 // We're asserting that the first word in an Array<Klass*> is the 5145 // length, and the second word is the first word of the data. If 5146 // that ever changes, r_array_base will have to be adjusted here. 5147 assert(Array<Klass*>::base_offset_in_bytes() == wordSize, "Adjust this code"); 5148 assert(Array<Klass*>::length_offset_in_bytes() == 0, "Adjust this code"); 5149 5150 cmpq(r_super_klass, Address(r_array_base, r_array_index, Address::times_8)); 5151 jccb(Assembler::equal, L_success); 5152 5153 // Restore slot to its true value 5154 movb(slot, Address(r_super_klass, Klass::hash_slot_offset())); 5155 5156 // Linear probe. Rotate the bitmap so that the next bit to test is 5157 // in Bit 1. 5158 rorq(r_bitmap, slot); 5159 5160 // Is there another entry to check? Consult the bitmap. 5161 btq(r_bitmap, 1); 5162 jccb(Assembler::carryClear, L_failure); 5163 5164 // Calls into the stub generated by lookup_secondary_supers_table_slow_path. 5165 // Arguments: r_super_klass, r_array_base, r_array_index, r_bitmap. 5166 // Kills: r_array_length. 5167 // Returns: result. 5168 lookup_secondary_supers_table_slow_path(r_super_klass, 5169 r_array_base, 5170 r_array_index, 5171 r_bitmap, 5172 /*temp1*/result, 5173 /*temp2*/slot, 5174 &L_success, 5175 nullptr); 5176 5177 bind(L_failure); 5178 movq(result, 1); 5179 jmpb(L_fallthrough); 5180 5181 bind(L_success); 5182 xorq(result, result); // = 0 5183 5184 bind(L_fallthrough); 5185 BLOCK_COMMENT("} lookup_secondary_supers_table"); 5186 5187 if (VerifySecondarySupers) { 5188 verify_secondary_supers_table(r_sub_klass, r_super_klass, result, 5189 temp1, temp2, temp3); 5190 } 5191 } 5192 5193 void MacroAssembler::repne_scanq(Register addr, Register value, Register count, Register limit, 5194 Label* L_success, Label* L_failure) { 5195 Label L_loop, L_fallthrough; 5196 { 5197 int label_nulls = 0; 5198 if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; } 5199 if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; } 5200 assert(label_nulls <= 1, "at most one null in the batch"); 5201 } 5202 bind(L_loop); 5203 cmpq(value, Address(addr, count, Address::times_8)); 5204 jcc(Assembler::equal, *L_success); 5205 addl(count, 1); 5206 cmpl(count, limit); 5207 jcc(Assembler::less, L_loop); 5208 5209 if (&L_fallthrough != L_failure) { 5210 jmp(*L_failure); 5211 } 5212 bind(L_fallthrough); 5213 } 5214 5215 // Called by code generated by check_klass_subtype_slow_path 5216 // above. This is called when there is a collision in the hashed 5217 // lookup in the secondary supers array. 5218 void MacroAssembler::lookup_secondary_supers_table_slow_path(Register r_super_klass, 5219 Register r_array_base, 5220 Register r_array_index, 5221 Register r_bitmap, 5222 Register temp1, 5223 Register temp2, 5224 Label* L_success, 5225 Label* L_failure) { 5226 assert_different_registers(r_super_klass, r_array_base, r_array_index, r_bitmap, temp1, temp2); 5227 5228 const Register 5229 r_array_length = temp1, 5230 r_sub_klass = noreg, 5231 result = noreg; 5232 5233 Label L_fallthrough; 5234 int label_nulls = 0; 5235 if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; } 5236 if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; } 5237 assert(label_nulls <= 1, "at most one null in the batch"); 5238 5239 // Load the array length. 5240 movl(r_array_length, Address(r_array_base, Array<Klass*>::length_offset_in_bytes())); 5241 // And adjust the array base to point to the data. 5242 // NB! Effectively increments current slot index by 1. 5243 assert(Array<Klass*>::base_offset_in_bytes() == wordSize, ""); 5244 addptr(r_array_base, Array<Klass*>::base_offset_in_bytes()); 5245 5246 // Linear probe 5247 Label L_huge; 5248 5249 // The bitmap is full to bursting. 5250 // Implicit invariant: BITMAP_FULL implies (length > 0) 5251 cmpl(r_array_length, (int32_t)Klass::SECONDARY_SUPERS_TABLE_SIZE - 2); 5252 jcc(Assembler::greater, L_huge); 5253 5254 // NB! Our caller has checked bits 0 and 1 in the bitmap. The 5255 // current slot (at secondary_supers[r_array_index]) has not yet 5256 // been inspected, and r_array_index may be out of bounds if we 5257 // wrapped around the end of the array. 5258 5259 { // This is conventional linear probing, but instead of terminating 5260 // when a null entry is found in the table, we maintain a bitmap 5261 // in which a 0 indicates missing entries. 5262 // The check above guarantees there are 0s in the bitmap, so the loop 5263 // eventually terminates. 5264 5265 xorl(temp2, temp2); // = 0; 5266 5267 Label L_again; 5268 bind(L_again); 5269 5270 // Check for array wraparound. 5271 cmpl(r_array_index, r_array_length); 5272 cmovl(Assembler::greaterEqual, r_array_index, temp2); 5273 5274 cmpq(r_super_klass, Address(r_array_base, r_array_index, Address::times_8)); 5275 jcc(Assembler::equal, *L_success); 5276 5277 // If the next bit in bitmap is zero, we're done. 5278 btq(r_bitmap, 2); // look-ahead check (Bit 2); Bits 0 and 1 are tested by now 5279 jcc(Assembler::carryClear, *L_failure); 5280 5281 rorq(r_bitmap, 1); // Bits 1/2 => 0/1 5282 addl(r_array_index, 1); 5283 5284 jmp(L_again); 5285 } 5286 5287 { // Degenerate case: more than 64 secondary supers. 5288 // FIXME: We could do something smarter here, maybe a vectorized 5289 // comparison or a binary search, but is that worth any added 5290 // complexity? 5291 bind(L_huge); 5292 xorl(r_array_index, r_array_index); // = 0 5293 repne_scanq(r_array_base, r_super_klass, r_array_index, r_array_length, 5294 L_success, 5295 (&L_fallthrough != L_failure ? L_failure : nullptr)); 5296 5297 bind(L_fallthrough); 5298 } 5299 } 5300 5301 struct VerifyHelperArguments { 5302 Klass* _super; 5303 Klass* _sub; 5304 intptr_t _linear_result; 5305 intptr_t _table_result; 5306 }; 5307 5308 static void verify_secondary_supers_table_helper(const char* msg, VerifyHelperArguments* args) { 5309 Klass::on_secondary_supers_verification_failure(args->_super, 5310 args->_sub, 5311 args->_linear_result, 5312 args->_table_result, 5313 msg); 5314 } 5315 5316 // Make sure that the hashed lookup and a linear scan agree. 5317 void MacroAssembler::verify_secondary_supers_table(Register r_sub_klass, 5318 Register r_super_klass, 5319 Register result, 5320 Register temp1, 5321 Register temp2, 5322 Register temp3) { 5323 const Register 5324 r_array_index = temp1, 5325 r_array_length = temp2, 5326 r_array_base = temp3, 5327 r_bitmap = noreg; 5328 5329 BLOCK_COMMENT("verify_secondary_supers_table {"); 5330 5331 Label L_success, L_failure, L_check, L_done; 5332 5333 movptr(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset()))); 5334 movl(r_array_length, Address(r_array_base, Array<Klass*>::length_offset_in_bytes())); 5335 // And adjust the array base to point to the data. 5336 addptr(r_array_base, Array<Klass*>::base_offset_in_bytes()); 5337 5338 testl(r_array_length, r_array_length); // array_length == 0? 5339 jcc(Assembler::zero, L_failure); 5340 5341 movl(r_array_index, 0); 5342 repne_scanq(r_array_base, r_super_klass, r_array_index, r_array_length, &L_success); 5343 // fall through to L_failure 5344 5345 const Register linear_result = r_array_index; // reuse temp1 5346 5347 bind(L_failure); // not present 5348 movl(linear_result, 1); 5349 jmp(L_check); 5350 5351 bind(L_success); // present 5352 movl(linear_result, 0); 5353 5354 bind(L_check); 5355 cmpl(linear_result, result); 5356 jcc(Assembler::equal, L_done); 5357 5358 { // To avoid calling convention issues, build a record on the stack 5359 // and pass the pointer to that instead. 5360 push(result); 5361 push(linear_result); 5362 push(r_sub_klass); 5363 push(r_super_klass); 5364 movptr(c_rarg1, rsp); 5365 movptr(c_rarg0, (uintptr_t) "mismatch"); 5366 call(RuntimeAddress(CAST_FROM_FN_PTR(address, verify_secondary_supers_table_helper))); 5367 should_not_reach_here(); 5368 } 5369 bind(L_done); 5370 5371 BLOCK_COMMENT("} verify_secondary_supers_table"); 5372 } 5373 5374 #undef LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS 5375 5376 #endif // LP64 5377 5378 void MacroAssembler::clinit_barrier(Register klass, Register thread, Label* L_fast_path, Label* L_slow_path) { 5379 assert(L_fast_path != nullptr || L_slow_path != nullptr, "at least one is required"); 5380 5381 Label L_fallthrough; 5382 if (L_fast_path == nullptr) { 5383 L_fast_path = &L_fallthrough; 5384 } else if (L_slow_path == nullptr) { 5385 L_slow_path = &L_fallthrough; 5386 } 5387 5388 // Fast path check: class is fully initialized. 5389 // init_state needs acquire, but x86 is TSO, and so we are already good. 5390 cmpb(Address(klass, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized); 5391 jcc(Assembler::equal, *L_fast_path); 5392 5393 // Fast path check: current thread is initializer thread 5394 cmpptr(thread, Address(klass, InstanceKlass::init_thread_offset())); 5395 if (L_slow_path == &L_fallthrough) { 5396 jcc(Assembler::equal, *L_fast_path); 5397 bind(*L_slow_path); 5398 } else if (L_fast_path == &L_fallthrough) { 5399 jcc(Assembler::notEqual, *L_slow_path); 5400 bind(*L_fast_path); 5401 } else { 5402 Unimplemented(); 5403 } 5404 } 5405 5406 void MacroAssembler::cmov32(Condition cc, Register dst, Address src) { 5407 if (VM_Version::supports_cmov()) { 5408 cmovl(cc, dst, src); 5409 } else { 5410 Label L; 5411 jccb(negate_condition(cc), L); 5412 movl(dst, src); 5413 bind(L); 5414 } 5415 } 5416 5417 void MacroAssembler::cmov32(Condition cc, Register dst, Register src) { 5418 if (VM_Version::supports_cmov()) { 5419 cmovl(cc, dst, src); 5420 } else { 5421 Label L; 5422 jccb(negate_condition(cc), L); 5423 movl(dst, src); 5424 bind(L); 5425 } 5426 } 5427 5428 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) { 5429 if (!VerifyOops) return; 5430 5431 BLOCK_COMMENT("verify_oop {"); 5432 #ifdef _LP64 5433 push(rscratch1); 5434 #endif 5435 push(rax); // save rax 5436 push(reg); // pass register argument 5437 5438 // Pass register number to verify_oop_subroutine 5439 const char* b = nullptr; 5440 { 5441 ResourceMark rm; 5442 stringStream ss; 5443 ss.print("verify_oop: %s: %s (%s:%d)", reg->name(), s, file, line); 5444 b = code_string(ss.as_string()); 5445 } 5446 AddressLiteral buffer((address) b, external_word_Relocation::spec_for_immediate()); 5447 pushptr(buffer.addr(), rscratch1); 5448 5449 // call indirectly to solve generation ordering problem 5450 movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address())); 5451 call(rax); 5452 // Caller pops the arguments (oop, message) and restores rax, r10 5453 BLOCK_COMMENT("} verify_oop"); 5454 } 5455 5456 void MacroAssembler::vallones(XMMRegister dst, int vector_len) { 5457 if (UseAVX > 2 && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl())) { 5458 // Only pcmpeq has dependency breaking treatment (i.e the execution can begin without 5459 // waiting for the previous result on dst), not vpcmpeqd, so just use vpternlog 5460 vpternlogd(dst, 0xFF, dst, dst, vector_len); 5461 } else if (VM_Version::supports_avx()) { 5462 vpcmpeqd(dst, dst, dst, vector_len); 5463 } else { 5464 pcmpeqd(dst, dst); 5465 } 5466 } 5467 5468 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot, 5469 int extra_slot_offset) { 5470 // cf. TemplateTable::prepare_invoke(), if (load_receiver). 5471 int stackElementSize = Interpreter::stackElementSize; 5472 int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0); 5473 #ifdef ASSERT 5474 int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1); 5475 assert(offset1 - offset == stackElementSize, "correct arithmetic"); 5476 #endif 5477 Register scale_reg = noreg; 5478 Address::ScaleFactor scale_factor = Address::no_scale; 5479 if (arg_slot.is_constant()) { 5480 offset += arg_slot.as_constant() * stackElementSize; 5481 } else { 5482 scale_reg = arg_slot.as_register(); 5483 scale_factor = Address::times(stackElementSize); 5484 } 5485 offset += wordSize; // return PC is on stack 5486 return Address(rsp, scale_reg, scale_factor, offset); 5487 } 5488 5489 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) { 5490 if (!VerifyOops) return; 5491 5492 #ifdef _LP64 5493 push(rscratch1); 5494 #endif 5495 push(rax); // save rax, 5496 // addr may contain rsp so we will have to adjust it based on the push 5497 // we just did (and on 64 bit we do two pushes) 5498 // NOTE: 64bit seemed to have had a bug in that it did movq(addr, rax); which 5499 // stores rax into addr which is backwards of what was intended. 5500 if (addr.uses(rsp)) { 5501 lea(rax, addr); 5502 pushptr(Address(rax, LP64_ONLY(2 *) BytesPerWord)); 5503 } else { 5504 pushptr(addr); 5505 } 5506 5507 // Pass register number to verify_oop_subroutine 5508 const char* b = nullptr; 5509 { 5510 ResourceMark rm; 5511 stringStream ss; 5512 ss.print("verify_oop_addr: %s (%s:%d)", s, file, line); 5513 b = code_string(ss.as_string()); 5514 } 5515 AddressLiteral buffer((address) b, external_word_Relocation::spec_for_immediate()); 5516 pushptr(buffer.addr(), rscratch1); 5517 5518 // call indirectly to solve generation ordering problem 5519 movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address())); 5520 call(rax); 5521 // Caller pops the arguments (addr, message) and restores rax, r10. 5522 } 5523 5524 void MacroAssembler::verify_tlab() { 5525 #ifdef ASSERT 5526 if (UseTLAB && VerifyOops) { 5527 Label next, ok; 5528 Register t1 = rsi; 5529 Register thread_reg = NOT_LP64(rbx) LP64_ONLY(r15_thread); 5530 5531 push(t1); 5532 NOT_LP64(push(thread_reg)); 5533 NOT_LP64(get_thread(thread_reg)); 5534 5535 movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset()))); 5536 cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset()))); 5537 jcc(Assembler::aboveEqual, next); 5538 STOP("assert(top >= start)"); 5539 should_not_reach_here(); 5540 5541 bind(next); 5542 movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset()))); 5543 cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset()))); 5544 jcc(Assembler::aboveEqual, ok); 5545 STOP("assert(top <= end)"); 5546 should_not_reach_here(); 5547 5548 bind(ok); 5549 NOT_LP64(pop(thread_reg)); 5550 pop(t1); 5551 } 5552 #endif 5553 } 5554 5555 class ControlWord { 5556 public: 5557 int32_t _value; 5558 5559 int rounding_control() const { return (_value >> 10) & 3 ; } 5560 int precision_control() const { return (_value >> 8) & 3 ; } 5561 bool precision() const { return ((_value >> 5) & 1) != 0; } 5562 bool underflow() const { return ((_value >> 4) & 1) != 0; } 5563 bool overflow() const { return ((_value >> 3) & 1) != 0; } 5564 bool zero_divide() const { return ((_value >> 2) & 1) != 0; } 5565 bool denormalized() const { return ((_value >> 1) & 1) != 0; } 5566 bool invalid() const { return ((_value >> 0) & 1) != 0; } 5567 5568 void print() const { 5569 // rounding control 5570 const char* rc; 5571 switch (rounding_control()) { 5572 case 0: rc = "round near"; break; 5573 case 1: rc = "round down"; break; 5574 case 2: rc = "round up "; break; 5575 case 3: rc = "chop "; break; 5576 default: 5577 rc = nullptr; // silence compiler warnings 5578 fatal("Unknown rounding control: %d", rounding_control()); 5579 }; 5580 // precision control 5581 const char* pc; 5582 switch (precision_control()) { 5583 case 0: pc = "24 bits "; break; 5584 case 1: pc = "reserved"; break; 5585 case 2: pc = "53 bits "; break; 5586 case 3: pc = "64 bits "; break; 5587 default: 5588 pc = nullptr; // silence compiler warnings 5589 fatal("Unknown precision control: %d", precision_control()); 5590 }; 5591 // flags 5592 char f[9]; 5593 f[0] = ' '; 5594 f[1] = ' '; 5595 f[2] = (precision ()) ? 'P' : 'p'; 5596 f[3] = (underflow ()) ? 'U' : 'u'; 5597 f[4] = (overflow ()) ? 'O' : 'o'; 5598 f[5] = (zero_divide ()) ? 'Z' : 'z'; 5599 f[6] = (denormalized()) ? 'D' : 'd'; 5600 f[7] = (invalid ()) ? 'I' : 'i'; 5601 f[8] = '\x0'; 5602 // output 5603 printf("%04x masks = %s, %s, %s", _value & 0xFFFF, f, rc, pc); 5604 } 5605 5606 }; 5607 5608 class StatusWord { 5609 public: 5610 int32_t _value; 5611 5612 bool busy() const { return ((_value >> 15) & 1) != 0; } 5613 bool C3() const { return ((_value >> 14) & 1) != 0; } 5614 bool C2() const { return ((_value >> 10) & 1) != 0; } 5615 bool C1() const { return ((_value >> 9) & 1) != 0; } 5616 bool C0() const { return ((_value >> 8) & 1) != 0; } 5617 int top() const { return (_value >> 11) & 7 ; } 5618 bool error_status() const { return ((_value >> 7) & 1) != 0; } 5619 bool stack_fault() const { return ((_value >> 6) & 1) != 0; } 5620 bool precision() const { return ((_value >> 5) & 1) != 0; } 5621 bool underflow() const { return ((_value >> 4) & 1) != 0; } 5622 bool overflow() const { return ((_value >> 3) & 1) != 0; } 5623 bool zero_divide() const { return ((_value >> 2) & 1) != 0; } 5624 bool denormalized() const { return ((_value >> 1) & 1) != 0; } 5625 bool invalid() const { return ((_value >> 0) & 1) != 0; } 5626 5627 void print() const { 5628 // condition codes 5629 char c[5]; 5630 c[0] = (C3()) ? '3' : '-'; 5631 c[1] = (C2()) ? '2' : '-'; 5632 c[2] = (C1()) ? '1' : '-'; 5633 c[3] = (C0()) ? '0' : '-'; 5634 c[4] = '\x0'; 5635 // flags 5636 char f[9]; 5637 f[0] = (error_status()) ? 'E' : '-'; 5638 f[1] = (stack_fault ()) ? 'S' : '-'; 5639 f[2] = (precision ()) ? 'P' : '-'; 5640 f[3] = (underflow ()) ? 'U' : '-'; 5641 f[4] = (overflow ()) ? 'O' : '-'; 5642 f[5] = (zero_divide ()) ? 'Z' : '-'; 5643 f[6] = (denormalized()) ? 'D' : '-'; 5644 f[7] = (invalid ()) ? 'I' : '-'; 5645 f[8] = '\x0'; 5646 // output 5647 printf("%04x flags = %s, cc = %s, top = %d", _value & 0xFFFF, f, c, top()); 5648 } 5649 5650 }; 5651 5652 class TagWord { 5653 public: 5654 int32_t _value; 5655 5656 int tag_at(int i) const { return (_value >> (i*2)) & 3; } 5657 5658 void print() const { 5659 printf("%04x", _value & 0xFFFF); 5660 } 5661 5662 }; 5663 5664 class FPU_Register { 5665 public: 5666 int32_t _m0; 5667 int32_t _m1; 5668 int16_t _ex; 5669 5670 bool is_indefinite() const { 5671 return _ex == -1 && _m1 == (int32_t)0xC0000000 && _m0 == 0; 5672 } 5673 5674 void print() const { 5675 char sign = (_ex < 0) ? '-' : '+'; 5676 const char* kind = (_ex == 0x7FFF || _ex == (int16_t)-1) ? "NaN" : " "; 5677 printf("%c%04hx.%08x%08x %s", sign, _ex, _m1, _m0, kind); 5678 }; 5679 5680 }; 5681 5682 class FPU_State { 5683 public: 5684 enum { 5685 register_size = 10, 5686 number_of_registers = 8, 5687 register_mask = 7 5688 }; 5689 5690 ControlWord _control_word; 5691 StatusWord _status_word; 5692 TagWord _tag_word; 5693 int32_t _error_offset; 5694 int32_t _error_selector; 5695 int32_t _data_offset; 5696 int32_t _data_selector; 5697 int8_t _register[register_size * number_of_registers]; 5698 5699 int tag_for_st(int i) const { return _tag_word.tag_at((_status_word.top() + i) & register_mask); } 5700 FPU_Register* st(int i) const { return (FPU_Register*)&_register[register_size * i]; } 5701 5702 const char* tag_as_string(int tag) const { 5703 switch (tag) { 5704 case 0: return "valid"; 5705 case 1: return "zero"; 5706 case 2: return "special"; 5707 case 3: return "empty"; 5708 } 5709 ShouldNotReachHere(); 5710 return nullptr; 5711 } 5712 5713 void print() const { 5714 // print computation registers 5715 { int t = _status_word.top(); 5716 for (int i = 0; i < number_of_registers; i++) { 5717 int j = (i - t) & register_mask; 5718 printf("%c r%d = ST%d = ", (j == 0 ? '*' : ' '), i, j); 5719 st(j)->print(); 5720 printf(" %s\n", tag_as_string(_tag_word.tag_at(i))); 5721 } 5722 } 5723 printf("\n"); 5724 // print control registers 5725 printf("ctrl = "); _control_word.print(); printf("\n"); 5726 printf("stat = "); _status_word .print(); printf("\n"); 5727 printf("tags = "); _tag_word .print(); printf("\n"); 5728 } 5729 5730 }; 5731 5732 class Flag_Register { 5733 public: 5734 int32_t _value; 5735 5736 bool overflow() const { return ((_value >> 11) & 1) != 0; } 5737 bool direction() const { return ((_value >> 10) & 1) != 0; } 5738 bool sign() const { return ((_value >> 7) & 1) != 0; } 5739 bool zero() const { return ((_value >> 6) & 1) != 0; } 5740 bool auxiliary_carry() const { return ((_value >> 4) & 1) != 0; } 5741 bool parity() const { return ((_value >> 2) & 1) != 0; } 5742 bool carry() const { return ((_value >> 0) & 1) != 0; } 5743 5744 void print() const { 5745 // flags 5746 char f[8]; 5747 f[0] = (overflow ()) ? 'O' : '-'; 5748 f[1] = (direction ()) ? 'D' : '-'; 5749 f[2] = (sign ()) ? 'S' : '-'; 5750 f[3] = (zero ()) ? 'Z' : '-'; 5751 f[4] = (auxiliary_carry()) ? 'A' : '-'; 5752 f[5] = (parity ()) ? 'P' : '-'; 5753 f[6] = (carry ()) ? 'C' : '-'; 5754 f[7] = '\x0'; 5755 // output 5756 printf("%08x flags = %s", _value, f); 5757 } 5758 5759 }; 5760 5761 class IU_Register { 5762 public: 5763 int32_t _value; 5764 5765 void print() const { 5766 printf("%08x %11d", _value, _value); 5767 } 5768 5769 }; 5770 5771 class IU_State { 5772 public: 5773 Flag_Register _eflags; 5774 IU_Register _rdi; 5775 IU_Register _rsi; 5776 IU_Register _rbp; 5777 IU_Register _rsp; 5778 IU_Register _rbx; 5779 IU_Register _rdx; 5780 IU_Register _rcx; 5781 IU_Register _rax; 5782 5783 void print() const { 5784 // computation registers 5785 printf("rax, = "); _rax.print(); printf("\n"); 5786 printf("rbx, = "); _rbx.print(); printf("\n"); 5787 printf("rcx = "); _rcx.print(); printf("\n"); 5788 printf("rdx = "); _rdx.print(); printf("\n"); 5789 printf("rdi = "); _rdi.print(); printf("\n"); 5790 printf("rsi = "); _rsi.print(); printf("\n"); 5791 printf("rbp, = "); _rbp.print(); printf("\n"); 5792 printf("rsp = "); _rsp.print(); printf("\n"); 5793 printf("\n"); 5794 // control registers 5795 printf("flgs = "); _eflags.print(); printf("\n"); 5796 } 5797 }; 5798 5799 5800 class CPU_State { 5801 public: 5802 FPU_State _fpu_state; 5803 IU_State _iu_state; 5804 5805 void print() const { 5806 printf("--------------------------------------------------\n"); 5807 _iu_state .print(); 5808 printf("\n"); 5809 _fpu_state.print(); 5810 printf("--------------------------------------------------\n"); 5811 } 5812 5813 }; 5814 5815 5816 static void _print_CPU_state(CPU_State* state) { 5817 state->print(); 5818 }; 5819 5820 5821 void MacroAssembler::print_CPU_state() { 5822 push_CPU_state(); 5823 push(rsp); // pass CPU state 5824 call(RuntimeAddress(CAST_FROM_FN_PTR(address, _print_CPU_state))); 5825 addptr(rsp, wordSize); // discard argument 5826 pop_CPU_state(); 5827 } 5828 5829 5830 #ifndef _LP64 5831 static bool _verify_FPU(int stack_depth, char* s, CPU_State* state) { 5832 static int counter = 0; 5833 FPU_State* fs = &state->_fpu_state; 5834 counter++; 5835 // For leaf calls, only verify that the top few elements remain empty. 5836 // We only need 1 empty at the top for C2 code. 5837 if( stack_depth < 0 ) { 5838 if( fs->tag_for_st(7) != 3 ) { 5839 printf("FPR7 not empty\n"); 5840 state->print(); 5841 assert(false, "error"); 5842 return false; 5843 } 5844 return true; // All other stack states do not matter 5845 } 5846 5847 assert((fs->_control_word._value & 0xffff) == StubRoutines::x86::fpu_cntrl_wrd_std(), 5848 "bad FPU control word"); 5849 5850 // compute stack depth 5851 int i = 0; 5852 while (i < FPU_State::number_of_registers && fs->tag_for_st(i) < 3) i++; 5853 int d = i; 5854 while (i < FPU_State::number_of_registers && fs->tag_for_st(i) == 3) i++; 5855 // verify findings 5856 if (i != FPU_State::number_of_registers) { 5857 // stack not contiguous 5858 printf("%s: stack not contiguous at ST%d\n", s, i); 5859 state->print(); 5860 assert(false, "error"); 5861 return false; 5862 } 5863 // check if computed stack depth corresponds to expected stack depth 5864 if (stack_depth < 0) { 5865 // expected stack depth is -stack_depth or less 5866 if (d > -stack_depth) { 5867 // too many elements on the stack 5868 printf("%s: <= %d stack elements expected but found %d\n", s, -stack_depth, d); 5869 state->print(); 5870 assert(false, "error"); 5871 return false; 5872 } 5873 } else { 5874 // expected stack depth is stack_depth 5875 if (d != stack_depth) { 5876 // wrong stack depth 5877 printf("%s: %d stack elements expected but found %d\n", s, stack_depth, d); 5878 state->print(); 5879 assert(false, "error"); 5880 return false; 5881 } 5882 } 5883 // everything is cool 5884 return true; 5885 } 5886 5887 void MacroAssembler::verify_FPU(int stack_depth, const char* s) { 5888 if (!VerifyFPU) return; 5889 push_CPU_state(); 5890 push(rsp); // pass CPU state 5891 ExternalAddress msg((address) s); 5892 // pass message string s 5893 pushptr(msg.addr(), noreg); 5894 push(stack_depth); // pass stack depth 5895 call(RuntimeAddress(CAST_FROM_FN_PTR(address, _verify_FPU))); 5896 addptr(rsp, 3 * wordSize); // discard arguments 5897 // check for error 5898 { Label L; 5899 testl(rax, rax); 5900 jcc(Assembler::notZero, L); 5901 int3(); // break if error condition 5902 bind(L); 5903 } 5904 pop_CPU_state(); 5905 } 5906 #endif // _LP64 5907 5908 void MacroAssembler::restore_cpu_control_state_after_jni(Register rscratch) { 5909 // Either restore the MXCSR register after returning from the JNI Call 5910 // or verify that it wasn't changed (with -Xcheck:jni flag). 5911 if (VM_Version::supports_sse()) { 5912 if (RestoreMXCSROnJNICalls) { 5913 ldmxcsr(ExternalAddress(StubRoutines::x86::addr_mxcsr_std()), rscratch); 5914 } else if (CheckJNICalls) { 5915 call(RuntimeAddress(StubRoutines::x86::verify_mxcsr_entry())); 5916 } 5917 } 5918 // Clear upper bits of YMM registers to avoid SSE <-> AVX transition penalty. 5919 vzeroupper(); 5920 5921 #ifndef _LP64 5922 // Either restore the x87 floating pointer control word after returning 5923 // from the JNI call or verify that it wasn't changed. 5924 if (CheckJNICalls) { 5925 call(RuntimeAddress(StubRoutines::x86::verify_fpu_cntrl_wrd_entry())); 5926 } 5927 #endif // _LP64 5928 } 5929 5930 // ((OopHandle)result).resolve(); 5931 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) { 5932 assert_different_registers(result, tmp); 5933 5934 // Only 64 bit platforms support GCs that require a tmp register 5935 // Only IN_HEAP loads require a thread_tmp register 5936 // OopHandle::resolve is an indirection like jobject. 5937 access_load_at(T_OBJECT, IN_NATIVE, 5938 result, Address(result, 0), tmp, /*tmp_thread*/noreg); 5939 } 5940 5941 // ((WeakHandle)result).resolve(); 5942 void MacroAssembler::resolve_weak_handle(Register rresult, Register rtmp) { 5943 assert_different_registers(rresult, rtmp); 5944 Label resolved; 5945 5946 // A null weak handle resolves to null. 5947 cmpptr(rresult, 0); 5948 jcc(Assembler::equal, resolved); 5949 5950 // Only 64 bit platforms support GCs that require a tmp register 5951 // Only IN_HEAP loads require a thread_tmp register 5952 // WeakHandle::resolve is an indirection like jweak. 5953 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, 5954 rresult, Address(rresult, 0), rtmp, /*tmp_thread*/noreg); 5955 bind(resolved); 5956 } 5957 5958 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) { 5959 // get mirror 5960 const int mirror_offset = in_bytes(Klass::java_mirror_offset()); 5961 load_method_holder(mirror, method); 5962 movptr(mirror, Address(mirror, mirror_offset)); 5963 resolve_oop_handle(mirror, tmp); 5964 } 5965 5966 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) { 5967 load_method_holder(rresult, rmethod); 5968 movptr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset())); 5969 } 5970 5971 void MacroAssembler::load_method_holder(Register holder, Register method) { 5972 movptr(holder, Address(method, Method::const_offset())); // ConstMethod* 5973 movptr(holder, Address(holder, ConstMethod::constants_offset())); // ConstantPool* 5974 movptr(holder, Address(holder, ConstantPool::pool_holder_offset())); // InstanceKlass* 5975 } 5976 5977 #ifdef _LP64 5978 void MacroAssembler::load_narrow_klass_compact(Register dst, Register src) { 5979 assert(UseCompactObjectHeaders, "expect compact object headers"); 5980 movl(dst, Address(src, oopDesc::mark_offset_in_bytes())); 5981 shrl(dst, markWord::klass_shift); 5982 } 5983 #endif 5984 5985 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) { 5986 assert_different_registers(src, tmp); 5987 assert_different_registers(dst, tmp); 5988 #ifdef _LP64 5989 if (UseCompactObjectHeaders) { 5990 load_narrow_klass_compact(dst, src); 5991 decode_klass_not_null(dst, tmp); 5992 } else if (UseCompressedClassPointers) { 5993 movl(dst, Address(src, oopDesc::klass_offset_in_bytes())); 5994 decode_klass_not_null(dst, tmp); 5995 } else 5996 #endif 5997 { 5998 movptr(dst, Address(src, oopDesc::klass_offset_in_bytes())); 5999 } 6000 } 6001 6002 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) { 6003 assert(!UseCompactObjectHeaders, "not with compact headers"); 6004 assert_different_registers(src, tmp); 6005 assert_different_registers(dst, tmp); 6006 #ifdef _LP64 6007 if (UseCompressedClassPointers) { 6008 encode_klass_not_null(src, tmp); 6009 movl(Address(dst, oopDesc::klass_offset_in_bytes()), src); 6010 } else 6011 #endif 6012 movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src); 6013 } 6014 6015 void MacroAssembler::cmp_klass(Register klass, Register obj, Register tmp) { 6016 #ifdef _LP64 6017 if (UseCompactObjectHeaders) { 6018 assert(tmp != noreg, "need tmp"); 6019 assert_different_registers(klass, obj, tmp); 6020 load_narrow_klass_compact(tmp, obj); 6021 cmpl(klass, tmp); 6022 } else if (UseCompressedClassPointers) { 6023 cmpl(klass, Address(obj, oopDesc::klass_offset_in_bytes())); 6024 } else 6025 #endif 6026 { 6027 cmpptr(klass, Address(obj, oopDesc::klass_offset_in_bytes())); 6028 } 6029 } 6030 6031 void MacroAssembler::cmp_klasses_from_objects(Register obj1, Register obj2, Register tmp1, Register tmp2) { 6032 #ifdef _LP64 6033 if (UseCompactObjectHeaders) { 6034 assert(tmp2 != noreg, "need tmp2"); 6035 assert_different_registers(obj1, obj2, tmp1, tmp2); 6036 load_narrow_klass_compact(tmp1, obj1); 6037 load_narrow_klass_compact(tmp2, obj2); 6038 cmpl(tmp1, tmp2); 6039 } else if (UseCompressedClassPointers) { 6040 movl(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes())); 6041 cmpl(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes())); 6042 } else 6043 #endif 6044 { 6045 movptr(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes())); 6046 cmpptr(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes())); 6047 } 6048 } 6049 6050 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src, 6051 Register tmp1, Register thread_tmp) { 6052 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); 6053 decorators = AccessInternal::decorator_fixup(decorators, type); 6054 bool as_raw = (decorators & AS_RAW) != 0; 6055 if (as_raw) { 6056 bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp); 6057 } else { 6058 bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp); 6059 } 6060 } 6061 6062 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register val, 6063 Register tmp1, Register tmp2, Register tmp3) { 6064 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); 6065 decorators = AccessInternal::decorator_fixup(decorators, type); 6066 bool as_raw = (decorators & AS_RAW) != 0; 6067 if (as_raw) { 6068 bs->BarrierSetAssembler::store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3); 6069 } else { 6070 bs->store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3); 6071 } 6072 } 6073 6074 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1, 6075 Register thread_tmp, DecoratorSet decorators) { 6076 access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp); 6077 } 6078 6079 // Doesn't do verification, generates fixed size code 6080 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1, 6081 Register thread_tmp, DecoratorSet decorators) { 6082 access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, thread_tmp); 6083 } 6084 6085 void MacroAssembler::store_heap_oop(Address dst, Register val, Register tmp1, 6086 Register tmp2, Register tmp3, DecoratorSet decorators) { 6087 access_store_at(T_OBJECT, IN_HEAP | decorators, dst, val, tmp1, tmp2, tmp3); 6088 } 6089 6090 // Used for storing nulls. 6091 void MacroAssembler::store_heap_oop_null(Address dst) { 6092 access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg, noreg); 6093 } 6094 6095 #ifdef _LP64 6096 void MacroAssembler::store_klass_gap(Register dst, Register src) { 6097 assert(!UseCompactObjectHeaders, "Don't use with compact headers"); 6098 if (UseCompressedClassPointers) { 6099 // Store to klass gap in destination 6100 movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src); 6101 } 6102 } 6103 6104 #ifdef ASSERT 6105 void MacroAssembler::verify_heapbase(const char* msg) { 6106 assert (UseCompressedOops, "should be compressed"); 6107 assert (Universe::heap() != nullptr, "java heap should be initialized"); 6108 if (CheckCompressedOops) { 6109 Label ok; 6110 ExternalAddress src2(CompressedOops::base_addr()); 6111 const bool is_src2_reachable = reachable(src2); 6112 if (!is_src2_reachable) { 6113 push(rscratch1); // cmpptr trashes rscratch1 6114 } 6115 cmpptr(r12_heapbase, src2, rscratch1); 6116 jcc(Assembler::equal, ok); 6117 STOP(msg); 6118 bind(ok); 6119 if (!is_src2_reachable) { 6120 pop(rscratch1); 6121 } 6122 } 6123 } 6124 #endif 6125 6126 // Algorithm must match oop.inline.hpp encode_heap_oop. 6127 void MacroAssembler::encode_heap_oop(Register r) { 6128 #ifdef ASSERT 6129 verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?"); 6130 #endif 6131 verify_oop_msg(r, "broken oop in encode_heap_oop"); 6132 if (CompressedOops::base() == nullptr) { 6133 if (CompressedOops::shift() != 0) { 6134 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 6135 shrq(r, LogMinObjAlignmentInBytes); 6136 } 6137 return; 6138 } 6139 testq(r, r); 6140 cmovq(Assembler::equal, r, r12_heapbase); 6141 subq(r, r12_heapbase); 6142 shrq(r, LogMinObjAlignmentInBytes); 6143 } 6144 6145 void MacroAssembler::encode_heap_oop_not_null(Register r) { 6146 #ifdef ASSERT 6147 verify_heapbase("MacroAssembler::encode_heap_oop_not_null: heap base corrupted?"); 6148 if (CheckCompressedOops) { 6149 Label ok; 6150 testq(r, r); 6151 jcc(Assembler::notEqual, ok); 6152 STOP("null oop passed to encode_heap_oop_not_null"); 6153 bind(ok); 6154 } 6155 #endif 6156 verify_oop_msg(r, "broken oop in encode_heap_oop_not_null"); 6157 if (CompressedOops::base() != nullptr) { 6158 subq(r, r12_heapbase); 6159 } 6160 if (CompressedOops::shift() != 0) { 6161 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 6162 shrq(r, LogMinObjAlignmentInBytes); 6163 } 6164 } 6165 6166 void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) { 6167 #ifdef ASSERT 6168 verify_heapbase("MacroAssembler::encode_heap_oop_not_null2: heap base corrupted?"); 6169 if (CheckCompressedOops) { 6170 Label ok; 6171 testq(src, src); 6172 jcc(Assembler::notEqual, ok); 6173 STOP("null oop passed to encode_heap_oop_not_null2"); 6174 bind(ok); 6175 } 6176 #endif 6177 verify_oop_msg(src, "broken oop in encode_heap_oop_not_null2"); 6178 if (dst != src) { 6179 movq(dst, src); 6180 } 6181 if (CompressedOops::base() != nullptr) { 6182 subq(dst, r12_heapbase); 6183 } 6184 if (CompressedOops::shift() != 0) { 6185 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 6186 shrq(dst, LogMinObjAlignmentInBytes); 6187 } 6188 } 6189 6190 void MacroAssembler::decode_heap_oop(Register r) { 6191 #ifdef ASSERT 6192 verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?"); 6193 #endif 6194 if (CompressedOops::base() == nullptr) { 6195 if (CompressedOops::shift() != 0) { 6196 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 6197 shlq(r, LogMinObjAlignmentInBytes); 6198 } 6199 } else { 6200 Label done; 6201 shlq(r, LogMinObjAlignmentInBytes); 6202 jccb(Assembler::equal, done); 6203 addq(r, r12_heapbase); 6204 bind(done); 6205 } 6206 verify_oop_msg(r, "broken oop in decode_heap_oop"); 6207 } 6208 6209 void MacroAssembler::decode_heap_oop_not_null(Register r) { 6210 // Note: it will change flags 6211 assert (UseCompressedOops, "should only be used for compressed headers"); 6212 assert (Universe::heap() != nullptr, "java heap should be initialized"); 6213 // Cannot assert, unverified entry point counts instructions (see .ad file) 6214 // vtableStubs also counts instructions in pd_code_size_limit. 6215 // Also do not verify_oop as this is called by verify_oop. 6216 if (CompressedOops::shift() != 0) { 6217 assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 6218 shlq(r, LogMinObjAlignmentInBytes); 6219 if (CompressedOops::base() != nullptr) { 6220 addq(r, r12_heapbase); 6221 } 6222 } else { 6223 assert (CompressedOops::base() == nullptr, "sanity"); 6224 } 6225 } 6226 6227 void MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) { 6228 // Note: it will change flags 6229 assert (UseCompressedOops, "should only be used for compressed headers"); 6230 assert (Universe::heap() != nullptr, "java heap should be initialized"); 6231 // Cannot assert, unverified entry point counts instructions (see .ad file) 6232 // vtableStubs also counts instructions in pd_code_size_limit. 6233 // Also do not verify_oop as this is called by verify_oop. 6234 if (CompressedOops::shift() != 0) { 6235 assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 6236 if (LogMinObjAlignmentInBytes == Address::times_8) { 6237 leaq(dst, Address(r12_heapbase, src, Address::times_8, 0)); 6238 } else { 6239 if (dst != src) { 6240 movq(dst, src); 6241 } 6242 shlq(dst, LogMinObjAlignmentInBytes); 6243 if (CompressedOops::base() != nullptr) { 6244 addq(dst, r12_heapbase); 6245 } 6246 } 6247 } else { 6248 assert (CompressedOops::base() == nullptr, "sanity"); 6249 if (dst != src) { 6250 movq(dst, src); 6251 } 6252 } 6253 } 6254 6255 void MacroAssembler::encode_klass_not_null(Register r, Register tmp) { 6256 assert_different_registers(r, tmp); 6257 if (CompressedKlassPointers::base() != nullptr) { 6258 mov64(tmp, (int64_t)CompressedKlassPointers::base()); 6259 subq(r, tmp); 6260 } 6261 if (CompressedKlassPointers::shift() != 0) { 6262 shrq(r, CompressedKlassPointers::shift()); 6263 } 6264 } 6265 6266 void MacroAssembler::encode_and_move_klass_not_null(Register dst, Register src) { 6267 assert_different_registers(src, dst); 6268 if (CompressedKlassPointers::base() != nullptr) { 6269 mov64(dst, -(int64_t)CompressedKlassPointers::base()); 6270 addq(dst, src); 6271 } else { 6272 movptr(dst, src); 6273 } 6274 if (CompressedKlassPointers::shift() != 0) { 6275 shrq(dst, CompressedKlassPointers::shift()); 6276 } 6277 } 6278 6279 void MacroAssembler::decode_klass_not_null(Register r, Register tmp) { 6280 assert_different_registers(r, tmp); 6281 // Note: it will change flags 6282 assert(UseCompressedClassPointers, "should only be used for compressed headers"); 6283 // Cannot assert, unverified entry point counts instructions (see .ad file) 6284 // vtableStubs also counts instructions in pd_code_size_limit. 6285 // Also do not verify_oop as this is called by verify_oop. 6286 if (CompressedKlassPointers::shift() != 0) { 6287 shlq(r, CompressedKlassPointers::shift()); 6288 } 6289 if (CompressedKlassPointers::base() != nullptr) { 6290 mov64(tmp, (int64_t)CompressedKlassPointers::base()); 6291 addq(r, tmp); 6292 } 6293 } 6294 6295 void MacroAssembler::decode_and_move_klass_not_null(Register dst, Register src) { 6296 assert_different_registers(src, dst); 6297 // Note: it will change flags 6298 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 6299 // Cannot assert, unverified entry point counts instructions (see .ad file) 6300 // vtableStubs also counts instructions in pd_code_size_limit. 6301 // Also do not verify_oop as this is called by verify_oop. 6302 6303 if (CompressedKlassPointers::base() == nullptr && 6304 CompressedKlassPointers::shift() == 0) { 6305 // The best case scenario is that there is no base or shift. Then it is already 6306 // a pointer that needs nothing but a register rename. 6307 movl(dst, src); 6308 } else { 6309 if (CompressedKlassPointers::shift() <= Address::times_8) { 6310 if (CompressedKlassPointers::base() != nullptr) { 6311 mov64(dst, (int64_t)CompressedKlassPointers::base()); 6312 } else { 6313 xorq(dst, dst); 6314 } 6315 if (CompressedKlassPointers::shift() != 0) { 6316 assert(CompressedKlassPointers::shift() == Address::times_8, "klass not aligned on 64bits?"); 6317 leaq(dst, Address(dst, src, Address::times_8, 0)); 6318 } else { 6319 addq(dst, src); 6320 } 6321 } else { 6322 if (CompressedKlassPointers::base() != nullptr) { 6323 const uint64_t base_right_shifted = 6324 (uint64_t)CompressedKlassPointers::base() >> CompressedKlassPointers::shift(); 6325 mov64(dst, base_right_shifted); 6326 } else { 6327 xorq(dst, dst); 6328 } 6329 addq(dst, src); 6330 shlq(dst, CompressedKlassPointers::shift()); 6331 } 6332 } 6333 } 6334 6335 void MacroAssembler::set_narrow_oop(Register dst, jobject obj) { 6336 assert (UseCompressedOops, "should only be used for compressed headers"); 6337 assert (Universe::heap() != nullptr, "java heap should be initialized"); 6338 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 6339 int oop_index = oop_recorder()->find_index(obj); 6340 RelocationHolder rspec = oop_Relocation::spec(oop_index); 6341 mov_narrow_oop(dst, oop_index, rspec); 6342 } 6343 6344 void MacroAssembler::set_narrow_oop(Address dst, jobject obj) { 6345 assert (UseCompressedOops, "should only be used for compressed headers"); 6346 assert (Universe::heap() != nullptr, "java heap should be initialized"); 6347 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 6348 int oop_index = oop_recorder()->find_index(obj); 6349 RelocationHolder rspec = oop_Relocation::spec(oop_index); 6350 mov_narrow_oop(dst, oop_index, rspec); 6351 } 6352 6353 void MacroAssembler::set_narrow_klass(Register dst, Klass* k) { 6354 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 6355 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 6356 int klass_index = oop_recorder()->find_index(k); 6357 RelocationHolder rspec = metadata_Relocation::spec(klass_index); 6358 mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec); 6359 } 6360 6361 void MacroAssembler::set_narrow_klass(Address dst, Klass* k) { 6362 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 6363 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 6364 int klass_index = oop_recorder()->find_index(k); 6365 RelocationHolder rspec = metadata_Relocation::spec(klass_index); 6366 mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec); 6367 } 6368 6369 void MacroAssembler::cmp_narrow_oop(Register dst, jobject obj) { 6370 assert (UseCompressedOops, "should only be used for compressed headers"); 6371 assert (Universe::heap() != nullptr, "java heap should be initialized"); 6372 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 6373 int oop_index = oop_recorder()->find_index(obj); 6374 RelocationHolder rspec = oop_Relocation::spec(oop_index); 6375 Assembler::cmp_narrow_oop(dst, oop_index, rspec); 6376 } 6377 6378 void MacroAssembler::cmp_narrow_oop(Address dst, jobject obj) { 6379 assert (UseCompressedOops, "should only be used for compressed headers"); 6380 assert (Universe::heap() != nullptr, "java heap should be initialized"); 6381 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 6382 int oop_index = oop_recorder()->find_index(obj); 6383 RelocationHolder rspec = oop_Relocation::spec(oop_index); 6384 Assembler::cmp_narrow_oop(dst, oop_index, rspec); 6385 } 6386 6387 void MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) { 6388 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 6389 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 6390 int klass_index = oop_recorder()->find_index(k); 6391 RelocationHolder rspec = metadata_Relocation::spec(klass_index); 6392 Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec); 6393 } 6394 6395 void MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) { 6396 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 6397 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 6398 int klass_index = oop_recorder()->find_index(k); 6399 RelocationHolder rspec = metadata_Relocation::spec(klass_index); 6400 Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec); 6401 } 6402 6403 void MacroAssembler::reinit_heapbase() { 6404 if (UseCompressedOops) { 6405 if (Universe::heap() != nullptr) { 6406 if (CompressedOops::base() == nullptr) { 6407 MacroAssembler::xorptr(r12_heapbase, r12_heapbase); 6408 } else { 6409 mov64(r12_heapbase, (int64_t)CompressedOops::base()); 6410 } 6411 } else { 6412 movptr(r12_heapbase, ExternalAddress(CompressedOops::base_addr())); 6413 } 6414 } 6415 } 6416 6417 #endif // _LP64 6418 6419 #if COMPILER2_OR_JVMCI 6420 6421 // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM/ZMM registers 6422 void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register rtmp, XMMRegister xtmp, KRegister mask) { 6423 // cnt - number of qwords (8-byte words). 6424 // base - start address, qword aligned. 6425 Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end; 6426 bool use64byteVector = (MaxVectorSize == 64) && (VM_Version::avx3_threshold() == 0); 6427 if (use64byteVector) { 6428 vpxor(xtmp, xtmp, xtmp, AVX_512bit); 6429 } else if (MaxVectorSize >= 32) { 6430 vpxor(xtmp, xtmp, xtmp, AVX_256bit); 6431 } else { 6432 pxor(xtmp, xtmp); 6433 } 6434 jmp(L_zero_64_bytes); 6435 6436 BIND(L_loop); 6437 if (MaxVectorSize >= 32) { 6438 fill64(base, 0, xtmp, use64byteVector); 6439 } else { 6440 movdqu(Address(base, 0), xtmp); 6441 movdqu(Address(base, 16), xtmp); 6442 movdqu(Address(base, 32), xtmp); 6443 movdqu(Address(base, 48), xtmp); 6444 } 6445 addptr(base, 64); 6446 6447 BIND(L_zero_64_bytes); 6448 subptr(cnt, 8); 6449 jccb(Assembler::greaterEqual, L_loop); 6450 6451 // Copy trailing 64 bytes 6452 if (use64byteVector) { 6453 addptr(cnt, 8); 6454 jccb(Assembler::equal, L_end); 6455 fill64_masked(3, base, 0, xtmp, mask, cnt, rtmp, true); 6456 jmp(L_end); 6457 } else { 6458 addptr(cnt, 4); 6459 jccb(Assembler::less, L_tail); 6460 if (MaxVectorSize >= 32) { 6461 vmovdqu(Address(base, 0), xtmp); 6462 } else { 6463 movdqu(Address(base, 0), xtmp); 6464 movdqu(Address(base, 16), xtmp); 6465 } 6466 } 6467 addptr(base, 32); 6468 subptr(cnt, 4); 6469 6470 BIND(L_tail); 6471 addptr(cnt, 4); 6472 jccb(Assembler::lessEqual, L_end); 6473 if (UseAVX > 2 && MaxVectorSize >= 32 && VM_Version::supports_avx512vl()) { 6474 fill32_masked(3, base, 0, xtmp, mask, cnt, rtmp); 6475 } else { 6476 decrement(cnt); 6477 6478 BIND(L_sloop); 6479 movq(Address(base, 0), xtmp); 6480 addptr(base, 8); 6481 decrement(cnt); 6482 jccb(Assembler::greaterEqual, L_sloop); 6483 } 6484 BIND(L_end); 6485 } 6486 6487 // Clearing constant sized memory using YMM/ZMM registers. 6488 void MacroAssembler::clear_mem(Register base, int cnt, Register rtmp, XMMRegister xtmp, KRegister mask) { 6489 assert(UseAVX > 2 && VM_Version::supports_avx512vl(), ""); 6490 bool use64byteVector = (MaxVectorSize > 32) && (VM_Version::avx3_threshold() == 0); 6491 6492 int vector64_count = (cnt & (~0x7)) >> 3; 6493 cnt = cnt & 0x7; 6494 const int fill64_per_loop = 4; 6495 const int max_unrolled_fill64 = 8; 6496 6497 // 64 byte initialization loop. 6498 vpxor(xtmp, xtmp, xtmp, use64byteVector ? AVX_512bit : AVX_256bit); 6499 int start64 = 0; 6500 if (vector64_count > max_unrolled_fill64) { 6501 Label LOOP; 6502 Register index = rtmp; 6503 6504 start64 = vector64_count - (vector64_count % fill64_per_loop); 6505 6506 movl(index, 0); 6507 BIND(LOOP); 6508 for (int i = 0; i < fill64_per_loop; i++) { 6509 fill64(Address(base, index, Address::times_1, i * 64), xtmp, use64byteVector); 6510 } 6511 addl(index, fill64_per_loop * 64); 6512 cmpl(index, start64 * 64); 6513 jccb(Assembler::less, LOOP); 6514 } 6515 for (int i = start64; i < vector64_count; i++) { 6516 fill64(base, i * 64, xtmp, use64byteVector); 6517 } 6518 6519 // Clear remaining 64 byte tail. 6520 int disp = vector64_count * 64; 6521 if (cnt) { 6522 switch (cnt) { 6523 case 1: 6524 movq(Address(base, disp), xtmp); 6525 break; 6526 case 2: 6527 evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_128bit); 6528 break; 6529 case 3: 6530 movl(rtmp, 0x7); 6531 kmovwl(mask, rtmp); 6532 evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_256bit); 6533 break; 6534 case 4: 6535 evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit); 6536 break; 6537 case 5: 6538 if (use64byteVector) { 6539 movl(rtmp, 0x1F); 6540 kmovwl(mask, rtmp); 6541 evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit); 6542 } else { 6543 evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit); 6544 movq(Address(base, disp + 32), xtmp); 6545 } 6546 break; 6547 case 6: 6548 if (use64byteVector) { 6549 movl(rtmp, 0x3F); 6550 kmovwl(mask, rtmp); 6551 evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit); 6552 } else { 6553 evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit); 6554 evmovdqu(T_LONG, k0, Address(base, disp + 32), xtmp, false, Assembler::AVX_128bit); 6555 } 6556 break; 6557 case 7: 6558 if (use64byteVector) { 6559 movl(rtmp, 0x7F); 6560 kmovwl(mask, rtmp); 6561 evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit); 6562 } else { 6563 evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit); 6564 movl(rtmp, 0x7); 6565 kmovwl(mask, rtmp); 6566 evmovdqu(T_LONG, mask, Address(base, disp + 32), xtmp, true, Assembler::AVX_256bit); 6567 } 6568 break; 6569 default: 6570 fatal("Unexpected length : %d\n",cnt); 6571 break; 6572 } 6573 } 6574 } 6575 6576 void MacroAssembler::clear_mem(Register base, Register cnt, Register tmp, XMMRegister xtmp, 6577 bool is_large, KRegister mask) { 6578 // cnt - number of qwords (8-byte words). 6579 // base - start address, qword aligned. 6580 // is_large - if optimizers know cnt is larger than InitArrayShortSize 6581 assert(base==rdi, "base register must be edi for rep stos"); 6582 assert(tmp==rax, "tmp register must be eax for rep stos"); 6583 assert(cnt==rcx, "cnt register must be ecx for rep stos"); 6584 assert(InitArrayShortSize % BytesPerLong == 0, 6585 "InitArrayShortSize should be the multiple of BytesPerLong"); 6586 6587 Label DONE; 6588 if (!is_large || !UseXMMForObjInit) { 6589 xorptr(tmp, tmp); 6590 } 6591 6592 if (!is_large) { 6593 Label LOOP, LONG; 6594 cmpptr(cnt, InitArrayShortSize/BytesPerLong); 6595 jccb(Assembler::greater, LONG); 6596 6597 NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM 6598 6599 decrement(cnt); 6600 jccb(Assembler::negative, DONE); // Zero length 6601 6602 // Use individual pointer-sized stores for small counts: 6603 BIND(LOOP); 6604 movptr(Address(base, cnt, Address::times_ptr), tmp); 6605 decrement(cnt); 6606 jccb(Assembler::greaterEqual, LOOP); 6607 jmpb(DONE); 6608 6609 BIND(LONG); 6610 } 6611 6612 // Use longer rep-prefixed ops for non-small counts: 6613 if (UseFastStosb) { 6614 shlptr(cnt, 3); // convert to number of bytes 6615 rep_stosb(); 6616 } else if (UseXMMForObjInit) { 6617 xmm_clear_mem(base, cnt, tmp, xtmp, mask); 6618 } else { 6619 NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM 6620 rep_stos(); 6621 } 6622 6623 BIND(DONE); 6624 } 6625 6626 #endif //COMPILER2_OR_JVMCI 6627 6628 6629 void MacroAssembler::generate_fill(BasicType t, bool aligned, 6630 Register to, Register value, Register count, 6631 Register rtmp, XMMRegister xtmp) { 6632 ShortBranchVerifier sbv(this); 6633 assert_different_registers(to, value, count, rtmp); 6634 Label L_exit; 6635 Label L_fill_2_bytes, L_fill_4_bytes; 6636 6637 #if defined(COMPILER2) && defined(_LP64) 6638 if(MaxVectorSize >=32 && 6639 VM_Version::supports_avx512vlbw() && 6640 VM_Version::supports_bmi2()) { 6641 generate_fill_avx3(t, to, value, count, rtmp, xtmp); 6642 return; 6643 } 6644 #endif 6645 6646 int shift = -1; 6647 switch (t) { 6648 case T_BYTE: 6649 shift = 2; 6650 break; 6651 case T_SHORT: 6652 shift = 1; 6653 break; 6654 case T_INT: 6655 shift = 0; 6656 break; 6657 default: ShouldNotReachHere(); 6658 } 6659 6660 if (t == T_BYTE) { 6661 andl(value, 0xff); 6662 movl(rtmp, value); 6663 shll(rtmp, 8); 6664 orl(value, rtmp); 6665 } 6666 if (t == T_SHORT) { 6667 andl(value, 0xffff); 6668 } 6669 if (t == T_BYTE || t == T_SHORT) { 6670 movl(rtmp, value); 6671 shll(rtmp, 16); 6672 orl(value, rtmp); 6673 } 6674 6675 cmpptr(count, 2<<shift); // Short arrays (< 8 bytes) fill by element 6676 jcc(Assembler::below, L_fill_4_bytes); // use unsigned cmp 6677 if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) { 6678 Label L_skip_align2; 6679 // align source address at 4 bytes address boundary 6680 if (t == T_BYTE) { 6681 Label L_skip_align1; 6682 // One byte misalignment happens only for byte arrays 6683 testptr(to, 1); 6684 jccb(Assembler::zero, L_skip_align1); 6685 movb(Address(to, 0), value); 6686 increment(to); 6687 decrement(count); 6688 BIND(L_skip_align1); 6689 } 6690 // Two bytes misalignment happens only for byte and short (char) arrays 6691 testptr(to, 2); 6692 jccb(Assembler::zero, L_skip_align2); 6693 movw(Address(to, 0), value); 6694 addptr(to, 2); 6695 subptr(count, 1<<(shift-1)); 6696 BIND(L_skip_align2); 6697 } 6698 if (UseSSE < 2) { 6699 Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes; 6700 // Fill 32-byte chunks 6701 subptr(count, 8 << shift); 6702 jcc(Assembler::less, L_check_fill_8_bytes); 6703 align(16); 6704 6705 BIND(L_fill_32_bytes_loop); 6706 6707 for (int i = 0; i < 32; i += 4) { 6708 movl(Address(to, i), value); 6709 } 6710 6711 addptr(to, 32); 6712 subptr(count, 8 << shift); 6713 jcc(Assembler::greaterEqual, L_fill_32_bytes_loop); 6714 BIND(L_check_fill_8_bytes); 6715 addptr(count, 8 << shift); 6716 jccb(Assembler::zero, L_exit); 6717 jmpb(L_fill_8_bytes); 6718 6719 // 6720 // length is too short, just fill qwords 6721 // 6722 BIND(L_fill_8_bytes_loop); 6723 movl(Address(to, 0), value); 6724 movl(Address(to, 4), value); 6725 addptr(to, 8); 6726 BIND(L_fill_8_bytes); 6727 subptr(count, 1 << (shift + 1)); 6728 jcc(Assembler::greaterEqual, L_fill_8_bytes_loop); 6729 // fall through to fill 4 bytes 6730 } else { 6731 Label L_fill_32_bytes; 6732 if (!UseUnalignedLoadStores) { 6733 // align to 8 bytes, we know we are 4 byte aligned to start 6734 testptr(to, 4); 6735 jccb(Assembler::zero, L_fill_32_bytes); 6736 movl(Address(to, 0), value); 6737 addptr(to, 4); 6738 subptr(count, 1<<shift); 6739 } 6740 BIND(L_fill_32_bytes); 6741 { 6742 assert( UseSSE >= 2, "supported cpu only" ); 6743 Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes; 6744 movdl(xtmp, value); 6745 if (UseAVX >= 2 && UseUnalignedLoadStores) { 6746 Label L_check_fill_32_bytes; 6747 if (UseAVX > 2) { 6748 // Fill 64-byte chunks 6749 Label L_fill_64_bytes_loop_avx3, L_check_fill_64_bytes_avx2; 6750 6751 // If number of bytes to fill < VM_Version::avx3_threshold(), perform fill using AVX2 6752 cmpptr(count, VM_Version::avx3_threshold()); 6753 jccb(Assembler::below, L_check_fill_64_bytes_avx2); 6754 6755 vpbroadcastd(xtmp, xtmp, Assembler::AVX_512bit); 6756 6757 subptr(count, 16 << shift); 6758 jccb(Assembler::less, L_check_fill_32_bytes); 6759 align(16); 6760 6761 BIND(L_fill_64_bytes_loop_avx3); 6762 evmovdqul(Address(to, 0), xtmp, Assembler::AVX_512bit); 6763 addptr(to, 64); 6764 subptr(count, 16 << shift); 6765 jcc(Assembler::greaterEqual, L_fill_64_bytes_loop_avx3); 6766 jmpb(L_check_fill_32_bytes); 6767 6768 BIND(L_check_fill_64_bytes_avx2); 6769 } 6770 // Fill 64-byte chunks 6771 Label L_fill_64_bytes_loop; 6772 vpbroadcastd(xtmp, xtmp, Assembler::AVX_256bit); 6773 6774 subptr(count, 16 << shift); 6775 jcc(Assembler::less, L_check_fill_32_bytes); 6776 align(16); 6777 6778 BIND(L_fill_64_bytes_loop); 6779 vmovdqu(Address(to, 0), xtmp); 6780 vmovdqu(Address(to, 32), xtmp); 6781 addptr(to, 64); 6782 subptr(count, 16 << shift); 6783 jcc(Assembler::greaterEqual, L_fill_64_bytes_loop); 6784 6785 BIND(L_check_fill_32_bytes); 6786 addptr(count, 8 << shift); 6787 jccb(Assembler::less, L_check_fill_8_bytes); 6788 vmovdqu(Address(to, 0), xtmp); 6789 addptr(to, 32); 6790 subptr(count, 8 << shift); 6791 6792 BIND(L_check_fill_8_bytes); 6793 // clean upper bits of YMM registers 6794 movdl(xtmp, value); 6795 pshufd(xtmp, xtmp, 0); 6796 } else { 6797 // Fill 32-byte chunks 6798 pshufd(xtmp, xtmp, 0); 6799 6800 subptr(count, 8 << shift); 6801 jcc(Assembler::less, L_check_fill_8_bytes); 6802 align(16); 6803 6804 BIND(L_fill_32_bytes_loop); 6805 6806 if (UseUnalignedLoadStores) { 6807 movdqu(Address(to, 0), xtmp); 6808 movdqu(Address(to, 16), xtmp); 6809 } else { 6810 movq(Address(to, 0), xtmp); 6811 movq(Address(to, 8), xtmp); 6812 movq(Address(to, 16), xtmp); 6813 movq(Address(to, 24), xtmp); 6814 } 6815 6816 addptr(to, 32); 6817 subptr(count, 8 << shift); 6818 jcc(Assembler::greaterEqual, L_fill_32_bytes_loop); 6819 6820 BIND(L_check_fill_8_bytes); 6821 } 6822 addptr(count, 8 << shift); 6823 jccb(Assembler::zero, L_exit); 6824 jmpb(L_fill_8_bytes); 6825 6826 // 6827 // length is too short, just fill qwords 6828 // 6829 BIND(L_fill_8_bytes_loop); 6830 movq(Address(to, 0), xtmp); 6831 addptr(to, 8); 6832 BIND(L_fill_8_bytes); 6833 subptr(count, 1 << (shift + 1)); 6834 jcc(Assembler::greaterEqual, L_fill_8_bytes_loop); 6835 } 6836 } 6837 // fill trailing 4 bytes 6838 BIND(L_fill_4_bytes); 6839 testl(count, 1<<shift); 6840 jccb(Assembler::zero, L_fill_2_bytes); 6841 movl(Address(to, 0), value); 6842 if (t == T_BYTE || t == T_SHORT) { 6843 Label L_fill_byte; 6844 addptr(to, 4); 6845 BIND(L_fill_2_bytes); 6846 // fill trailing 2 bytes 6847 testl(count, 1<<(shift-1)); 6848 jccb(Assembler::zero, L_fill_byte); 6849 movw(Address(to, 0), value); 6850 if (t == T_BYTE) { 6851 addptr(to, 2); 6852 BIND(L_fill_byte); 6853 // fill trailing byte 6854 testl(count, 1); 6855 jccb(Assembler::zero, L_exit); 6856 movb(Address(to, 0), value); 6857 } else { 6858 BIND(L_fill_byte); 6859 } 6860 } else { 6861 BIND(L_fill_2_bytes); 6862 } 6863 BIND(L_exit); 6864 } 6865 6866 void MacroAssembler::evpbroadcast(BasicType type, XMMRegister dst, Register src, int vector_len) { 6867 switch(type) { 6868 case T_BYTE: 6869 case T_BOOLEAN: 6870 evpbroadcastb(dst, src, vector_len); 6871 break; 6872 case T_SHORT: 6873 case T_CHAR: 6874 evpbroadcastw(dst, src, vector_len); 6875 break; 6876 case T_INT: 6877 case T_FLOAT: 6878 evpbroadcastd(dst, src, vector_len); 6879 break; 6880 case T_LONG: 6881 case T_DOUBLE: 6882 evpbroadcastq(dst, src, vector_len); 6883 break; 6884 default: 6885 fatal("Unhandled type : %s", type2name(type)); 6886 break; 6887 } 6888 } 6889 6890 // encode char[] to byte[] in ISO_8859_1 or ASCII 6891 //@IntrinsicCandidate 6892 //private static int implEncodeISOArray(byte[] sa, int sp, 6893 //byte[] da, int dp, int len) { 6894 // int i = 0; 6895 // for (; i < len; i++) { 6896 // char c = StringUTF16.getChar(sa, sp++); 6897 // if (c > '\u00FF') 6898 // break; 6899 // da[dp++] = (byte)c; 6900 // } 6901 // return i; 6902 //} 6903 // 6904 //@IntrinsicCandidate 6905 //private static int implEncodeAsciiArray(char[] sa, int sp, 6906 // byte[] da, int dp, int len) { 6907 // int i = 0; 6908 // for (; i < len; i++) { 6909 // char c = sa[sp++]; 6910 // if (c >= '\u0080') 6911 // break; 6912 // da[dp++] = (byte)c; 6913 // } 6914 // return i; 6915 //} 6916 void MacroAssembler::encode_iso_array(Register src, Register dst, Register len, 6917 XMMRegister tmp1Reg, XMMRegister tmp2Reg, 6918 XMMRegister tmp3Reg, XMMRegister tmp4Reg, 6919 Register tmp5, Register result, bool ascii) { 6920 6921 // rsi: src 6922 // rdi: dst 6923 // rdx: len 6924 // rcx: tmp5 6925 // rax: result 6926 ShortBranchVerifier sbv(this); 6927 assert_different_registers(src, dst, len, tmp5, result); 6928 Label L_done, L_copy_1_char, L_copy_1_char_exit; 6929 6930 int mask = ascii ? 0xff80ff80 : 0xff00ff00; 6931 int short_mask = ascii ? 0xff80 : 0xff00; 6932 6933 // set result 6934 xorl(result, result); 6935 // check for zero length 6936 testl(len, len); 6937 jcc(Assembler::zero, L_done); 6938 6939 movl(result, len); 6940 6941 // Setup pointers 6942 lea(src, Address(src, len, Address::times_2)); // char[] 6943 lea(dst, Address(dst, len, Address::times_1)); // byte[] 6944 negptr(len); 6945 6946 if (UseSSE42Intrinsics || UseAVX >= 2) { 6947 Label L_copy_8_chars, L_copy_8_chars_exit; 6948 Label L_chars_16_check, L_copy_16_chars, L_copy_16_chars_exit; 6949 6950 if (UseAVX >= 2) { 6951 Label L_chars_32_check, L_copy_32_chars, L_copy_32_chars_exit; 6952 movl(tmp5, mask); // create mask to test for Unicode or non-ASCII chars in vector 6953 movdl(tmp1Reg, tmp5); 6954 vpbroadcastd(tmp1Reg, tmp1Reg, Assembler::AVX_256bit); 6955 jmp(L_chars_32_check); 6956 6957 bind(L_copy_32_chars); 6958 vmovdqu(tmp3Reg, Address(src, len, Address::times_2, -64)); 6959 vmovdqu(tmp4Reg, Address(src, len, Address::times_2, -32)); 6960 vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1); 6961 vptest(tmp2Reg, tmp1Reg); // check for Unicode or non-ASCII chars in vector 6962 jccb(Assembler::notZero, L_copy_32_chars_exit); 6963 vpackuswb(tmp3Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1); 6964 vpermq(tmp4Reg, tmp3Reg, 0xD8, /* vector_len */ 1); 6965 vmovdqu(Address(dst, len, Address::times_1, -32), tmp4Reg); 6966 6967 bind(L_chars_32_check); 6968 addptr(len, 32); 6969 jcc(Assembler::lessEqual, L_copy_32_chars); 6970 6971 bind(L_copy_32_chars_exit); 6972 subptr(len, 16); 6973 jccb(Assembler::greater, L_copy_16_chars_exit); 6974 6975 } else if (UseSSE42Intrinsics) { 6976 movl(tmp5, mask); // create mask to test for Unicode or non-ASCII chars in vector 6977 movdl(tmp1Reg, tmp5); 6978 pshufd(tmp1Reg, tmp1Reg, 0); 6979 jmpb(L_chars_16_check); 6980 } 6981 6982 bind(L_copy_16_chars); 6983 if (UseAVX >= 2) { 6984 vmovdqu(tmp2Reg, Address(src, len, Address::times_2, -32)); 6985 vptest(tmp2Reg, tmp1Reg); 6986 jcc(Assembler::notZero, L_copy_16_chars_exit); 6987 vpackuswb(tmp2Reg, tmp2Reg, tmp1Reg, /* vector_len */ 1); 6988 vpermq(tmp3Reg, tmp2Reg, 0xD8, /* vector_len */ 1); 6989 } else { 6990 if (UseAVX > 0) { 6991 movdqu(tmp3Reg, Address(src, len, Address::times_2, -32)); 6992 movdqu(tmp4Reg, Address(src, len, Address::times_2, -16)); 6993 vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 0); 6994 } else { 6995 movdqu(tmp3Reg, Address(src, len, Address::times_2, -32)); 6996 por(tmp2Reg, tmp3Reg); 6997 movdqu(tmp4Reg, Address(src, len, Address::times_2, -16)); 6998 por(tmp2Reg, tmp4Reg); 6999 } 7000 ptest(tmp2Reg, tmp1Reg); // check for Unicode or non-ASCII chars in vector 7001 jccb(Assembler::notZero, L_copy_16_chars_exit); 7002 packuswb(tmp3Reg, tmp4Reg); 7003 } 7004 movdqu(Address(dst, len, Address::times_1, -16), tmp3Reg); 7005 7006 bind(L_chars_16_check); 7007 addptr(len, 16); 7008 jcc(Assembler::lessEqual, L_copy_16_chars); 7009 7010 bind(L_copy_16_chars_exit); 7011 if (UseAVX >= 2) { 7012 // clean upper bits of YMM registers 7013 vpxor(tmp2Reg, tmp2Reg); 7014 vpxor(tmp3Reg, tmp3Reg); 7015 vpxor(tmp4Reg, tmp4Reg); 7016 movdl(tmp1Reg, tmp5); 7017 pshufd(tmp1Reg, tmp1Reg, 0); 7018 } 7019 subptr(len, 8); 7020 jccb(Assembler::greater, L_copy_8_chars_exit); 7021 7022 bind(L_copy_8_chars); 7023 movdqu(tmp3Reg, Address(src, len, Address::times_2, -16)); 7024 ptest(tmp3Reg, tmp1Reg); 7025 jccb(Assembler::notZero, L_copy_8_chars_exit); 7026 packuswb(tmp3Reg, tmp1Reg); 7027 movq(Address(dst, len, Address::times_1, -8), tmp3Reg); 7028 addptr(len, 8); 7029 jccb(Assembler::lessEqual, L_copy_8_chars); 7030 7031 bind(L_copy_8_chars_exit); 7032 subptr(len, 8); 7033 jccb(Assembler::zero, L_done); 7034 } 7035 7036 bind(L_copy_1_char); 7037 load_unsigned_short(tmp5, Address(src, len, Address::times_2, 0)); 7038 testl(tmp5, short_mask); // check if Unicode or non-ASCII char 7039 jccb(Assembler::notZero, L_copy_1_char_exit); 7040 movb(Address(dst, len, Address::times_1, 0), tmp5); 7041 addptr(len, 1); 7042 jccb(Assembler::less, L_copy_1_char); 7043 7044 bind(L_copy_1_char_exit); 7045 addptr(result, len); // len is negative count of not processed elements 7046 7047 bind(L_done); 7048 } 7049 7050 #ifdef _LP64 7051 /** 7052 * Helper for multiply_to_len(). 7053 */ 7054 void MacroAssembler::add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2) { 7055 addq(dest_lo, src1); 7056 adcq(dest_hi, 0); 7057 addq(dest_lo, src2); 7058 adcq(dest_hi, 0); 7059 } 7060 7061 /** 7062 * Multiply 64 bit by 64 bit first loop. 7063 */ 7064 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart, 7065 Register y, Register y_idx, Register z, 7066 Register carry, Register product, 7067 Register idx, Register kdx) { 7068 // 7069 // jlong carry, x[], y[], z[]; 7070 // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) { 7071 // huge_128 product = y[idx] * x[xstart] + carry; 7072 // z[kdx] = (jlong)product; 7073 // carry = (jlong)(product >>> 64); 7074 // } 7075 // z[xstart] = carry; 7076 // 7077 7078 Label L_first_loop, L_first_loop_exit; 7079 Label L_one_x, L_one_y, L_multiply; 7080 7081 decrementl(xstart); 7082 jcc(Assembler::negative, L_one_x); 7083 7084 movq(x_xstart, Address(x, xstart, Address::times_4, 0)); 7085 rorq(x_xstart, 32); // convert big-endian to little-endian 7086 7087 bind(L_first_loop); 7088 decrementl(idx); 7089 jcc(Assembler::negative, L_first_loop_exit); 7090 decrementl(idx); 7091 jcc(Assembler::negative, L_one_y); 7092 movq(y_idx, Address(y, idx, Address::times_4, 0)); 7093 rorq(y_idx, 32); // convert big-endian to little-endian 7094 bind(L_multiply); 7095 movq(product, x_xstart); 7096 mulq(y_idx); // product(rax) * y_idx -> rdx:rax 7097 addq(product, carry); 7098 adcq(rdx, 0); 7099 subl(kdx, 2); 7100 movl(Address(z, kdx, Address::times_4, 4), product); 7101 shrq(product, 32); 7102 movl(Address(z, kdx, Address::times_4, 0), product); 7103 movq(carry, rdx); 7104 jmp(L_first_loop); 7105 7106 bind(L_one_y); 7107 movl(y_idx, Address(y, 0)); 7108 jmp(L_multiply); 7109 7110 bind(L_one_x); 7111 movl(x_xstart, Address(x, 0)); 7112 jmp(L_first_loop); 7113 7114 bind(L_first_loop_exit); 7115 } 7116 7117 /** 7118 * Multiply 64 bit by 64 bit and add 128 bit. 7119 */ 7120 void MacroAssembler::multiply_add_128_x_128(Register x_xstart, Register y, Register z, 7121 Register yz_idx, Register idx, 7122 Register carry, Register product, int offset) { 7123 // huge_128 product = (y[idx] * x_xstart) + z[kdx] + carry; 7124 // z[kdx] = (jlong)product; 7125 7126 movq(yz_idx, Address(y, idx, Address::times_4, offset)); 7127 rorq(yz_idx, 32); // convert big-endian to little-endian 7128 movq(product, x_xstart); 7129 mulq(yz_idx); // product(rax) * yz_idx -> rdx:product(rax) 7130 movq(yz_idx, Address(z, idx, Address::times_4, offset)); 7131 rorq(yz_idx, 32); // convert big-endian to little-endian 7132 7133 add2_with_carry(rdx, product, carry, yz_idx); 7134 7135 movl(Address(z, idx, Address::times_4, offset+4), product); 7136 shrq(product, 32); 7137 movl(Address(z, idx, Address::times_4, offset), product); 7138 7139 } 7140 7141 /** 7142 * Multiply 128 bit by 128 bit. Unrolled inner loop. 7143 */ 7144 void MacroAssembler::multiply_128_x_128_loop(Register x_xstart, Register y, Register z, 7145 Register yz_idx, Register idx, Register jdx, 7146 Register carry, Register product, 7147 Register carry2) { 7148 // jlong carry, x[], y[], z[]; 7149 // int kdx = ystart+1; 7150 // for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop 7151 // huge_128 product = (y[idx+1] * x_xstart) + z[kdx+idx+1] + carry; 7152 // z[kdx+idx+1] = (jlong)product; 7153 // jlong carry2 = (jlong)(product >>> 64); 7154 // product = (y[idx] * x_xstart) + z[kdx+idx] + carry2; 7155 // z[kdx+idx] = (jlong)product; 7156 // carry = (jlong)(product >>> 64); 7157 // } 7158 // idx += 2; 7159 // if (idx > 0) { 7160 // product = (y[idx] * x_xstart) + z[kdx+idx] + carry; 7161 // z[kdx+idx] = (jlong)product; 7162 // carry = (jlong)(product >>> 64); 7163 // } 7164 // 7165 7166 Label L_third_loop, L_third_loop_exit, L_post_third_loop_done; 7167 7168 movl(jdx, idx); 7169 andl(jdx, 0xFFFFFFFC); 7170 shrl(jdx, 2); 7171 7172 bind(L_third_loop); 7173 subl(jdx, 1); 7174 jcc(Assembler::negative, L_third_loop_exit); 7175 subl(idx, 4); 7176 7177 multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 8); 7178 movq(carry2, rdx); 7179 7180 multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry2, product, 0); 7181 movq(carry, rdx); 7182 jmp(L_third_loop); 7183 7184 bind (L_third_loop_exit); 7185 7186 andl (idx, 0x3); 7187 jcc(Assembler::zero, L_post_third_loop_done); 7188 7189 Label L_check_1; 7190 subl(idx, 2); 7191 jcc(Assembler::negative, L_check_1); 7192 7193 multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 0); 7194 movq(carry, rdx); 7195 7196 bind (L_check_1); 7197 addl (idx, 0x2); 7198 andl (idx, 0x1); 7199 subl(idx, 1); 7200 jcc(Assembler::negative, L_post_third_loop_done); 7201 7202 movl(yz_idx, Address(y, idx, Address::times_4, 0)); 7203 movq(product, x_xstart); 7204 mulq(yz_idx); // product(rax) * yz_idx -> rdx:product(rax) 7205 movl(yz_idx, Address(z, idx, Address::times_4, 0)); 7206 7207 add2_with_carry(rdx, product, yz_idx, carry); 7208 7209 movl(Address(z, idx, Address::times_4, 0), product); 7210 shrq(product, 32); 7211 7212 shlq(rdx, 32); 7213 orq(product, rdx); 7214 movq(carry, product); 7215 7216 bind(L_post_third_loop_done); 7217 } 7218 7219 /** 7220 * Multiply 128 bit by 128 bit using BMI2. Unrolled inner loop. 7221 * 7222 */ 7223 void MacroAssembler::multiply_128_x_128_bmi2_loop(Register y, Register z, 7224 Register carry, Register carry2, 7225 Register idx, Register jdx, 7226 Register yz_idx1, Register yz_idx2, 7227 Register tmp, Register tmp3, Register tmp4) { 7228 assert(UseBMI2Instructions, "should be used only when BMI2 is available"); 7229 7230 // jlong carry, x[], y[], z[]; 7231 // int kdx = ystart+1; 7232 // for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop 7233 // huge_128 tmp3 = (y[idx+1] * rdx) + z[kdx+idx+1] + carry; 7234 // jlong carry2 = (jlong)(tmp3 >>> 64); 7235 // huge_128 tmp4 = (y[idx] * rdx) + z[kdx+idx] + carry2; 7236 // carry = (jlong)(tmp4 >>> 64); 7237 // z[kdx+idx+1] = (jlong)tmp3; 7238 // z[kdx+idx] = (jlong)tmp4; 7239 // } 7240 // idx += 2; 7241 // if (idx > 0) { 7242 // yz_idx1 = (y[idx] * rdx) + z[kdx+idx] + carry; 7243 // z[kdx+idx] = (jlong)yz_idx1; 7244 // carry = (jlong)(yz_idx1 >>> 64); 7245 // } 7246 // 7247 7248 Label L_third_loop, L_third_loop_exit, L_post_third_loop_done; 7249 7250 movl(jdx, idx); 7251 andl(jdx, 0xFFFFFFFC); 7252 shrl(jdx, 2); 7253 7254 bind(L_third_loop); 7255 subl(jdx, 1); 7256 jcc(Assembler::negative, L_third_loop_exit); 7257 subl(idx, 4); 7258 7259 movq(yz_idx1, Address(y, idx, Address::times_4, 8)); 7260 rorxq(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian 7261 movq(yz_idx2, Address(y, idx, Address::times_4, 0)); 7262 rorxq(yz_idx2, yz_idx2, 32); 7263 7264 mulxq(tmp4, tmp3, yz_idx1); // yz_idx1 * rdx -> tmp4:tmp3 7265 mulxq(carry2, tmp, yz_idx2); // yz_idx2 * rdx -> carry2:tmp 7266 7267 movq(yz_idx1, Address(z, idx, Address::times_4, 8)); 7268 rorxq(yz_idx1, yz_idx1, 32); 7269 movq(yz_idx2, Address(z, idx, Address::times_4, 0)); 7270 rorxq(yz_idx2, yz_idx2, 32); 7271 7272 if (VM_Version::supports_adx()) { 7273 adcxq(tmp3, carry); 7274 adoxq(tmp3, yz_idx1); 7275 7276 adcxq(tmp4, tmp); 7277 adoxq(tmp4, yz_idx2); 7278 7279 movl(carry, 0); // does not affect flags 7280 adcxq(carry2, carry); 7281 adoxq(carry2, carry); 7282 } else { 7283 add2_with_carry(tmp4, tmp3, carry, yz_idx1); 7284 add2_with_carry(carry2, tmp4, tmp, yz_idx2); 7285 } 7286 movq(carry, carry2); 7287 7288 movl(Address(z, idx, Address::times_4, 12), tmp3); 7289 shrq(tmp3, 32); 7290 movl(Address(z, idx, Address::times_4, 8), tmp3); 7291 7292 movl(Address(z, idx, Address::times_4, 4), tmp4); 7293 shrq(tmp4, 32); 7294 movl(Address(z, idx, Address::times_4, 0), tmp4); 7295 7296 jmp(L_third_loop); 7297 7298 bind (L_third_loop_exit); 7299 7300 andl (idx, 0x3); 7301 jcc(Assembler::zero, L_post_third_loop_done); 7302 7303 Label L_check_1; 7304 subl(idx, 2); 7305 jcc(Assembler::negative, L_check_1); 7306 7307 movq(yz_idx1, Address(y, idx, Address::times_4, 0)); 7308 rorxq(yz_idx1, yz_idx1, 32); 7309 mulxq(tmp4, tmp3, yz_idx1); // yz_idx1 * rdx -> tmp4:tmp3 7310 movq(yz_idx2, Address(z, idx, Address::times_4, 0)); 7311 rorxq(yz_idx2, yz_idx2, 32); 7312 7313 add2_with_carry(tmp4, tmp3, carry, yz_idx2); 7314 7315 movl(Address(z, idx, Address::times_4, 4), tmp3); 7316 shrq(tmp3, 32); 7317 movl(Address(z, idx, Address::times_4, 0), tmp3); 7318 movq(carry, tmp4); 7319 7320 bind (L_check_1); 7321 addl (idx, 0x2); 7322 andl (idx, 0x1); 7323 subl(idx, 1); 7324 jcc(Assembler::negative, L_post_third_loop_done); 7325 movl(tmp4, Address(y, idx, Address::times_4, 0)); 7326 mulxq(carry2, tmp3, tmp4); // tmp4 * rdx -> carry2:tmp3 7327 movl(tmp4, Address(z, idx, Address::times_4, 0)); 7328 7329 add2_with_carry(carry2, tmp3, tmp4, carry); 7330 7331 movl(Address(z, idx, Address::times_4, 0), tmp3); 7332 shrq(tmp3, 32); 7333 7334 shlq(carry2, 32); 7335 orq(tmp3, carry2); 7336 movq(carry, tmp3); 7337 7338 bind(L_post_third_loop_done); 7339 } 7340 7341 /** 7342 * Code for BigInteger::multiplyToLen() intrinsic. 7343 * 7344 * rdi: x 7345 * rax: xlen 7346 * rsi: y 7347 * rcx: ylen 7348 * r8: z 7349 * r11: tmp0 7350 * r12: tmp1 7351 * r13: tmp2 7352 * r14: tmp3 7353 * r15: tmp4 7354 * rbx: tmp5 7355 * 7356 */ 7357 void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen, Register z, Register tmp0, 7358 Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5) { 7359 ShortBranchVerifier sbv(this); 7360 assert_different_registers(x, xlen, y, ylen, z, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, rdx); 7361 7362 push(tmp0); 7363 push(tmp1); 7364 push(tmp2); 7365 push(tmp3); 7366 push(tmp4); 7367 push(tmp5); 7368 7369 push(xlen); 7370 7371 const Register idx = tmp1; 7372 const Register kdx = tmp2; 7373 const Register xstart = tmp3; 7374 7375 const Register y_idx = tmp4; 7376 const Register carry = tmp5; 7377 const Register product = xlen; 7378 const Register x_xstart = tmp0; 7379 7380 // First Loop. 7381 // 7382 // final static long LONG_MASK = 0xffffffffL; 7383 // int xstart = xlen - 1; 7384 // int ystart = ylen - 1; 7385 // long carry = 0; 7386 // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) { 7387 // long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry; 7388 // z[kdx] = (int)product; 7389 // carry = product >>> 32; 7390 // } 7391 // z[xstart] = (int)carry; 7392 // 7393 7394 movl(idx, ylen); // idx = ylen; 7395 lea(kdx, Address(xlen, ylen)); // kdx = xlen+ylen; 7396 xorq(carry, carry); // carry = 0; 7397 7398 Label L_done; 7399 7400 movl(xstart, xlen); 7401 decrementl(xstart); 7402 jcc(Assembler::negative, L_done); 7403 7404 multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx); 7405 7406 Label L_second_loop; 7407 testl(kdx, kdx); 7408 jcc(Assembler::zero, L_second_loop); 7409 7410 Label L_carry; 7411 subl(kdx, 1); 7412 jcc(Assembler::zero, L_carry); 7413 7414 movl(Address(z, kdx, Address::times_4, 0), carry); 7415 shrq(carry, 32); 7416 subl(kdx, 1); 7417 7418 bind(L_carry); 7419 movl(Address(z, kdx, Address::times_4, 0), carry); 7420 7421 // Second and third (nested) loops. 7422 // 7423 // for (int i = xstart-1; i >= 0; i--) { // Second loop 7424 // carry = 0; 7425 // for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop 7426 // long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) + 7427 // (z[k] & LONG_MASK) + carry; 7428 // z[k] = (int)product; 7429 // carry = product >>> 32; 7430 // } 7431 // z[i] = (int)carry; 7432 // } 7433 // 7434 // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = rdx 7435 7436 const Register jdx = tmp1; 7437 7438 bind(L_second_loop); 7439 xorl(carry, carry); // carry = 0; 7440 movl(jdx, ylen); // j = ystart+1 7441 7442 subl(xstart, 1); // i = xstart-1; 7443 jcc(Assembler::negative, L_done); 7444 7445 push (z); 7446 7447 Label L_last_x; 7448 lea(z, Address(z, xstart, Address::times_4, 4)); // z = z + k - j 7449 subl(xstart, 1); // i = xstart-1; 7450 jcc(Assembler::negative, L_last_x); 7451 7452 if (UseBMI2Instructions) { 7453 movq(rdx, Address(x, xstart, Address::times_4, 0)); 7454 rorxq(rdx, rdx, 32); // convert big-endian to little-endian 7455 } else { 7456 movq(x_xstart, Address(x, xstart, Address::times_4, 0)); 7457 rorq(x_xstart, 32); // convert big-endian to little-endian 7458 } 7459 7460 Label L_third_loop_prologue; 7461 bind(L_third_loop_prologue); 7462 7463 push (x); 7464 push (xstart); 7465 push (ylen); 7466 7467 7468 if (UseBMI2Instructions) { 7469 multiply_128_x_128_bmi2_loop(y, z, carry, x, jdx, ylen, product, tmp2, x_xstart, tmp3, tmp4); 7470 } else { // !UseBMI2Instructions 7471 multiply_128_x_128_loop(x_xstart, y, z, y_idx, jdx, ylen, carry, product, x); 7472 } 7473 7474 pop(ylen); 7475 pop(xlen); 7476 pop(x); 7477 pop(z); 7478 7479 movl(tmp3, xlen); 7480 addl(tmp3, 1); 7481 movl(Address(z, tmp3, Address::times_4, 0), carry); 7482 subl(tmp3, 1); 7483 jccb(Assembler::negative, L_done); 7484 7485 shrq(carry, 32); 7486 movl(Address(z, tmp3, Address::times_4, 0), carry); 7487 jmp(L_second_loop); 7488 7489 // Next infrequent code is moved outside loops. 7490 bind(L_last_x); 7491 if (UseBMI2Instructions) { 7492 movl(rdx, Address(x, 0)); 7493 } else { 7494 movl(x_xstart, Address(x, 0)); 7495 } 7496 jmp(L_third_loop_prologue); 7497 7498 bind(L_done); 7499 7500 pop(xlen); 7501 7502 pop(tmp5); 7503 pop(tmp4); 7504 pop(tmp3); 7505 pop(tmp2); 7506 pop(tmp1); 7507 pop(tmp0); 7508 } 7509 7510 void MacroAssembler::vectorized_mismatch(Register obja, Register objb, Register length, Register log2_array_indxscale, 7511 Register result, Register tmp1, Register tmp2, XMMRegister rymm0, XMMRegister rymm1, XMMRegister rymm2){ 7512 assert(UseSSE42Intrinsics, "SSE4.2 must be enabled."); 7513 Label VECTOR16_LOOP, VECTOR8_LOOP, VECTOR4_LOOP; 7514 Label VECTOR8_TAIL, VECTOR4_TAIL; 7515 Label VECTOR32_NOT_EQUAL, VECTOR16_NOT_EQUAL, VECTOR8_NOT_EQUAL, VECTOR4_NOT_EQUAL; 7516 Label SAME_TILL_END, DONE; 7517 Label BYTES_LOOP, BYTES_TAIL, BYTES_NOT_EQUAL; 7518 7519 //scale is in rcx in both Win64 and Unix 7520 ShortBranchVerifier sbv(this); 7521 7522 shlq(length); 7523 xorq(result, result); 7524 7525 if ((AVX3Threshold == 0) && (UseAVX > 2) && 7526 VM_Version::supports_avx512vlbw()) { 7527 Label VECTOR64_LOOP, VECTOR64_NOT_EQUAL, VECTOR32_TAIL; 7528 7529 cmpq(length, 64); 7530 jcc(Assembler::less, VECTOR32_TAIL); 7531 7532 movq(tmp1, length); 7533 andq(tmp1, 0x3F); // tail count 7534 andq(length, ~(0x3F)); //vector count 7535 7536 bind(VECTOR64_LOOP); 7537 // AVX512 code to compare 64 byte vectors. 7538 evmovdqub(rymm0, Address(obja, result), Assembler::AVX_512bit); 7539 evpcmpeqb(k7, rymm0, Address(objb, result), Assembler::AVX_512bit); 7540 kortestql(k7, k7); 7541 jcc(Assembler::aboveEqual, VECTOR64_NOT_EQUAL); // mismatch 7542 addq(result, 64); 7543 subq(length, 64); 7544 jccb(Assembler::notZero, VECTOR64_LOOP); 7545 7546 //bind(VECTOR64_TAIL); 7547 testq(tmp1, tmp1); 7548 jcc(Assembler::zero, SAME_TILL_END); 7549 7550 //bind(VECTOR64_TAIL); 7551 // AVX512 code to compare up to 63 byte vectors. 7552 mov64(tmp2, 0xFFFFFFFFFFFFFFFF); 7553 shlxq(tmp2, tmp2, tmp1); 7554 notq(tmp2); 7555 kmovql(k3, tmp2); 7556 7557 evmovdqub(rymm0, k3, Address(obja, result), false, Assembler::AVX_512bit); 7558 evpcmpeqb(k7, k3, rymm0, Address(objb, result), Assembler::AVX_512bit); 7559 7560 ktestql(k7, k3); 7561 jcc(Assembler::below, SAME_TILL_END); // not mismatch 7562 7563 bind(VECTOR64_NOT_EQUAL); 7564 kmovql(tmp1, k7); 7565 notq(tmp1); 7566 tzcntq(tmp1, tmp1); 7567 addq(result, tmp1); 7568 shrq(result); 7569 jmp(DONE); 7570 bind(VECTOR32_TAIL); 7571 } 7572 7573 cmpq(length, 8); 7574 jcc(Assembler::equal, VECTOR8_LOOP); 7575 jcc(Assembler::less, VECTOR4_TAIL); 7576 7577 if (UseAVX >= 2) { 7578 Label VECTOR16_TAIL, VECTOR32_LOOP; 7579 7580 cmpq(length, 16); 7581 jcc(Assembler::equal, VECTOR16_LOOP); 7582 jcc(Assembler::less, VECTOR8_LOOP); 7583 7584 cmpq(length, 32); 7585 jccb(Assembler::less, VECTOR16_TAIL); 7586 7587 subq(length, 32); 7588 bind(VECTOR32_LOOP); 7589 vmovdqu(rymm0, Address(obja, result)); 7590 vmovdqu(rymm1, Address(objb, result)); 7591 vpxor(rymm2, rymm0, rymm1, Assembler::AVX_256bit); 7592 vptest(rymm2, rymm2); 7593 jcc(Assembler::notZero, VECTOR32_NOT_EQUAL);//mismatch found 7594 addq(result, 32); 7595 subq(length, 32); 7596 jcc(Assembler::greaterEqual, VECTOR32_LOOP); 7597 addq(length, 32); 7598 jcc(Assembler::equal, SAME_TILL_END); 7599 //falling through if less than 32 bytes left //close the branch here. 7600 7601 bind(VECTOR16_TAIL); 7602 cmpq(length, 16); 7603 jccb(Assembler::less, VECTOR8_TAIL); 7604 bind(VECTOR16_LOOP); 7605 movdqu(rymm0, Address(obja, result)); 7606 movdqu(rymm1, Address(objb, result)); 7607 vpxor(rymm2, rymm0, rymm1, Assembler::AVX_128bit); 7608 ptest(rymm2, rymm2); 7609 jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found 7610 addq(result, 16); 7611 subq(length, 16); 7612 jcc(Assembler::equal, SAME_TILL_END); 7613 //falling through if less than 16 bytes left 7614 } else {//regular intrinsics 7615 7616 cmpq(length, 16); 7617 jccb(Assembler::less, VECTOR8_TAIL); 7618 7619 subq(length, 16); 7620 bind(VECTOR16_LOOP); 7621 movdqu(rymm0, Address(obja, result)); 7622 movdqu(rymm1, Address(objb, result)); 7623 pxor(rymm0, rymm1); 7624 ptest(rymm0, rymm0); 7625 jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found 7626 addq(result, 16); 7627 subq(length, 16); 7628 jccb(Assembler::greaterEqual, VECTOR16_LOOP); 7629 addq(length, 16); 7630 jcc(Assembler::equal, SAME_TILL_END); 7631 //falling through if less than 16 bytes left 7632 } 7633 7634 bind(VECTOR8_TAIL); 7635 cmpq(length, 8); 7636 jccb(Assembler::less, VECTOR4_TAIL); 7637 bind(VECTOR8_LOOP); 7638 movq(tmp1, Address(obja, result)); 7639 movq(tmp2, Address(objb, result)); 7640 xorq(tmp1, tmp2); 7641 testq(tmp1, tmp1); 7642 jcc(Assembler::notZero, VECTOR8_NOT_EQUAL);//mismatch found 7643 addq(result, 8); 7644 subq(length, 8); 7645 jcc(Assembler::equal, SAME_TILL_END); 7646 //falling through if less than 8 bytes left 7647 7648 bind(VECTOR4_TAIL); 7649 cmpq(length, 4); 7650 jccb(Assembler::less, BYTES_TAIL); 7651 bind(VECTOR4_LOOP); 7652 movl(tmp1, Address(obja, result)); 7653 xorl(tmp1, Address(objb, result)); 7654 testl(tmp1, tmp1); 7655 jcc(Assembler::notZero, VECTOR4_NOT_EQUAL);//mismatch found 7656 addq(result, 4); 7657 subq(length, 4); 7658 jcc(Assembler::equal, SAME_TILL_END); 7659 //falling through if less than 4 bytes left 7660 7661 bind(BYTES_TAIL); 7662 bind(BYTES_LOOP); 7663 load_unsigned_byte(tmp1, Address(obja, result)); 7664 load_unsigned_byte(tmp2, Address(objb, result)); 7665 xorl(tmp1, tmp2); 7666 testl(tmp1, tmp1); 7667 jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found 7668 decq(length); 7669 jcc(Assembler::zero, SAME_TILL_END); 7670 incq(result); 7671 load_unsigned_byte(tmp1, Address(obja, result)); 7672 load_unsigned_byte(tmp2, Address(objb, result)); 7673 xorl(tmp1, tmp2); 7674 testl(tmp1, tmp1); 7675 jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found 7676 decq(length); 7677 jcc(Assembler::zero, SAME_TILL_END); 7678 incq(result); 7679 load_unsigned_byte(tmp1, Address(obja, result)); 7680 load_unsigned_byte(tmp2, Address(objb, result)); 7681 xorl(tmp1, tmp2); 7682 testl(tmp1, tmp1); 7683 jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found 7684 jmp(SAME_TILL_END); 7685 7686 if (UseAVX >= 2) { 7687 bind(VECTOR32_NOT_EQUAL); 7688 vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_256bit); 7689 vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_256bit); 7690 vpxor(rymm0, rymm0, rymm2, Assembler::AVX_256bit); 7691 vpmovmskb(tmp1, rymm0); 7692 bsfq(tmp1, tmp1); 7693 addq(result, tmp1); 7694 shrq(result); 7695 jmp(DONE); 7696 } 7697 7698 bind(VECTOR16_NOT_EQUAL); 7699 if (UseAVX >= 2) { 7700 vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_128bit); 7701 vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_128bit); 7702 pxor(rymm0, rymm2); 7703 } else { 7704 pcmpeqb(rymm2, rymm2); 7705 pxor(rymm0, rymm1); 7706 pcmpeqb(rymm0, rymm1); 7707 pxor(rymm0, rymm2); 7708 } 7709 pmovmskb(tmp1, rymm0); 7710 bsfq(tmp1, tmp1); 7711 addq(result, tmp1); 7712 shrq(result); 7713 jmpb(DONE); 7714 7715 bind(VECTOR8_NOT_EQUAL); 7716 bind(VECTOR4_NOT_EQUAL); 7717 bsfq(tmp1, tmp1); 7718 shrq(tmp1, 3); 7719 addq(result, tmp1); 7720 bind(BYTES_NOT_EQUAL); 7721 shrq(result); 7722 jmpb(DONE); 7723 7724 bind(SAME_TILL_END); 7725 mov64(result, -1); 7726 7727 bind(DONE); 7728 } 7729 7730 //Helper functions for square_to_len() 7731 7732 /** 7733 * Store the squares of x[], right shifted one bit (divided by 2) into z[] 7734 * Preserves x and z and modifies rest of the registers. 7735 */ 7736 void MacroAssembler::square_rshift(Register x, Register xlen, Register z, Register tmp1, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) { 7737 // Perform square and right shift by 1 7738 // Handle odd xlen case first, then for even xlen do the following 7739 // jlong carry = 0; 7740 // for (int j=0, i=0; j < xlen; j+=2, i+=4) { 7741 // huge_128 product = x[j:j+1] * x[j:j+1]; 7742 // z[i:i+1] = (carry << 63) | (jlong)(product >>> 65); 7743 // z[i+2:i+3] = (jlong)(product >>> 1); 7744 // carry = (jlong)product; 7745 // } 7746 7747 xorq(tmp5, tmp5); // carry 7748 xorq(rdxReg, rdxReg); 7749 xorl(tmp1, tmp1); // index for x 7750 xorl(tmp4, tmp4); // index for z 7751 7752 Label L_first_loop, L_first_loop_exit; 7753 7754 testl(xlen, 1); 7755 jccb(Assembler::zero, L_first_loop); //jump if xlen is even 7756 7757 // Square and right shift by 1 the odd element using 32 bit multiply 7758 movl(raxReg, Address(x, tmp1, Address::times_4, 0)); 7759 imulq(raxReg, raxReg); 7760 shrq(raxReg, 1); 7761 adcq(tmp5, 0); 7762 movq(Address(z, tmp4, Address::times_4, 0), raxReg); 7763 incrementl(tmp1); 7764 addl(tmp4, 2); 7765 7766 // Square and right shift by 1 the rest using 64 bit multiply 7767 bind(L_first_loop); 7768 cmpptr(tmp1, xlen); 7769 jccb(Assembler::equal, L_first_loop_exit); 7770 7771 // Square 7772 movq(raxReg, Address(x, tmp1, Address::times_4, 0)); 7773 rorq(raxReg, 32); // convert big-endian to little-endian 7774 mulq(raxReg); // 64-bit multiply rax * rax -> rdx:rax 7775 7776 // Right shift by 1 and save carry 7777 shrq(tmp5, 1); // rdx:rax:tmp5 = (tmp5:rdx:rax) >>> 1 7778 rcrq(rdxReg, 1); 7779 rcrq(raxReg, 1); 7780 adcq(tmp5, 0); 7781 7782 // Store result in z 7783 movq(Address(z, tmp4, Address::times_4, 0), rdxReg); 7784 movq(Address(z, tmp4, Address::times_4, 8), raxReg); 7785 7786 // Update indices for x and z 7787 addl(tmp1, 2); 7788 addl(tmp4, 4); 7789 jmp(L_first_loop); 7790 7791 bind(L_first_loop_exit); 7792 } 7793 7794 7795 /** 7796 * Perform the following multiply add operation using BMI2 instructions 7797 * carry:sum = sum + op1*op2 + carry 7798 * op2 should be in rdx 7799 * op2 is preserved, all other registers are modified 7800 */ 7801 void MacroAssembler::multiply_add_64_bmi2(Register sum, Register op1, Register op2, Register carry, Register tmp2) { 7802 // assert op2 is rdx 7803 mulxq(tmp2, op1, op1); // op1 * op2 -> tmp2:op1 7804 addq(sum, carry); 7805 adcq(tmp2, 0); 7806 addq(sum, op1); 7807 adcq(tmp2, 0); 7808 movq(carry, tmp2); 7809 } 7810 7811 /** 7812 * Perform the following multiply add operation: 7813 * carry:sum = sum + op1*op2 + carry 7814 * Preserves op1, op2 and modifies rest of registers 7815 */ 7816 void MacroAssembler::multiply_add_64(Register sum, Register op1, Register op2, Register carry, Register rdxReg, Register raxReg) { 7817 // rdx:rax = op1 * op2 7818 movq(raxReg, op2); 7819 mulq(op1); 7820 7821 // rdx:rax = sum + carry + rdx:rax 7822 addq(sum, carry); 7823 adcq(rdxReg, 0); 7824 addq(sum, raxReg); 7825 adcq(rdxReg, 0); 7826 7827 // carry:sum = rdx:sum 7828 movq(carry, rdxReg); 7829 } 7830 7831 /** 7832 * Add 64 bit long carry into z[] with carry propagation. 7833 * Preserves z and carry register values and modifies rest of registers. 7834 * 7835 */ 7836 void MacroAssembler::add_one_64(Register z, Register zlen, Register carry, Register tmp1) { 7837 Label L_fourth_loop, L_fourth_loop_exit; 7838 7839 movl(tmp1, 1); 7840 subl(zlen, 2); 7841 addq(Address(z, zlen, Address::times_4, 0), carry); 7842 7843 bind(L_fourth_loop); 7844 jccb(Assembler::carryClear, L_fourth_loop_exit); 7845 subl(zlen, 2); 7846 jccb(Assembler::negative, L_fourth_loop_exit); 7847 addq(Address(z, zlen, Address::times_4, 0), tmp1); 7848 jmp(L_fourth_loop); 7849 bind(L_fourth_loop_exit); 7850 } 7851 7852 /** 7853 * Shift z[] left by 1 bit. 7854 * Preserves x, len, z and zlen registers and modifies rest of the registers. 7855 * 7856 */ 7857 void MacroAssembler::lshift_by_1(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4) { 7858 7859 Label L_fifth_loop, L_fifth_loop_exit; 7860 7861 // Fifth loop 7862 // Perform primitiveLeftShift(z, zlen, 1) 7863 7864 const Register prev_carry = tmp1; 7865 const Register new_carry = tmp4; 7866 const Register value = tmp2; 7867 const Register zidx = tmp3; 7868 7869 // int zidx, carry; 7870 // long value; 7871 // carry = 0; 7872 // for (zidx = zlen-2; zidx >=0; zidx -= 2) { 7873 // (carry:value) = (z[i] << 1) | carry ; 7874 // z[i] = value; 7875 // } 7876 7877 movl(zidx, zlen); 7878 xorl(prev_carry, prev_carry); // clear carry flag and prev_carry register 7879 7880 bind(L_fifth_loop); 7881 decl(zidx); // Use decl to preserve carry flag 7882 decl(zidx); 7883 jccb(Assembler::negative, L_fifth_loop_exit); 7884 7885 if (UseBMI2Instructions) { 7886 movq(value, Address(z, zidx, Address::times_4, 0)); 7887 rclq(value, 1); 7888 rorxq(value, value, 32); 7889 movq(Address(z, zidx, Address::times_4, 0), value); // Store back in big endian form 7890 } 7891 else { 7892 // clear new_carry 7893 xorl(new_carry, new_carry); 7894 7895 // Shift z[i] by 1, or in previous carry and save new carry 7896 movq(value, Address(z, zidx, Address::times_4, 0)); 7897 shlq(value, 1); 7898 adcl(new_carry, 0); 7899 7900 orq(value, prev_carry); 7901 rorq(value, 0x20); 7902 movq(Address(z, zidx, Address::times_4, 0), value); // Store back in big endian form 7903 7904 // Set previous carry = new carry 7905 movl(prev_carry, new_carry); 7906 } 7907 jmp(L_fifth_loop); 7908 7909 bind(L_fifth_loop_exit); 7910 } 7911 7912 7913 /** 7914 * Code for BigInteger::squareToLen() intrinsic 7915 * 7916 * rdi: x 7917 * rsi: len 7918 * r8: z 7919 * rcx: zlen 7920 * r12: tmp1 7921 * r13: tmp2 7922 * r14: tmp3 7923 * r15: tmp4 7924 * rbx: tmp5 7925 * 7926 */ 7927 void MacroAssembler::square_to_len(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) { 7928 7929 Label L_second_loop, L_second_loop_exit, L_third_loop, L_third_loop_exit, L_last_x, L_multiply; 7930 push(tmp1); 7931 push(tmp2); 7932 push(tmp3); 7933 push(tmp4); 7934 push(tmp5); 7935 7936 // First loop 7937 // Store the squares, right shifted one bit (i.e., divided by 2). 7938 square_rshift(x, len, z, tmp1, tmp3, tmp4, tmp5, rdxReg, raxReg); 7939 7940 // Add in off-diagonal sums. 7941 // 7942 // Second, third (nested) and fourth loops. 7943 // zlen +=2; 7944 // for (int xidx=len-2,zidx=zlen-4; xidx > 0; xidx-=2,zidx-=4) { 7945 // carry = 0; 7946 // long op2 = x[xidx:xidx+1]; 7947 // for (int j=xidx-2,k=zidx; j >= 0; j-=2) { 7948 // k -= 2; 7949 // long op1 = x[j:j+1]; 7950 // long sum = z[k:k+1]; 7951 // carry:sum = multiply_add_64(sum, op1, op2, carry, tmp_regs); 7952 // z[k:k+1] = sum; 7953 // } 7954 // add_one_64(z, k, carry, tmp_regs); 7955 // } 7956 7957 const Register carry = tmp5; 7958 const Register sum = tmp3; 7959 const Register op1 = tmp4; 7960 Register op2 = tmp2; 7961 7962 push(zlen); 7963 push(len); 7964 addl(zlen,2); 7965 bind(L_second_loop); 7966 xorq(carry, carry); 7967 subl(zlen, 4); 7968 subl(len, 2); 7969 push(zlen); 7970 push(len); 7971 cmpl(len, 0); 7972 jccb(Assembler::lessEqual, L_second_loop_exit); 7973 7974 // Multiply an array by one 64 bit long. 7975 if (UseBMI2Instructions) { 7976 op2 = rdxReg; 7977 movq(op2, Address(x, len, Address::times_4, 0)); 7978 rorxq(op2, op2, 32); 7979 } 7980 else { 7981 movq(op2, Address(x, len, Address::times_4, 0)); 7982 rorq(op2, 32); 7983 } 7984 7985 bind(L_third_loop); 7986 decrementl(len); 7987 jccb(Assembler::negative, L_third_loop_exit); 7988 decrementl(len); 7989 jccb(Assembler::negative, L_last_x); 7990 7991 movq(op1, Address(x, len, Address::times_4, 0)); 7992 rorq(op1, 32); 7993 7994 bind(L_multiply); 7995 subl(zlen, 2); 7996 movq(sum, Address(z, zlen, Address::times_4, 0)); 7997 7998 // Multiply 64 bit by 64 bit and add 64 bits lower half and upper 64 bits as carry. 7999 if (UseBMI2Instructions) { 8000 multiply_add_64_bmi2(sum, op1, op2, carry, tmp2); 8001 } 8002 else { 8003 multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg); 8004 } 8005 8006 movq(Address(z, zlen, Address::times_4, 0), sum); 8007 8008 jmp(L_third_loop); 8009 bind(L_third_loop_exit); 8010 8011 // Fourth loop 8012 // Add 64 bit long carry into z with carry propagation. 8013 // Uses offsetted zlen. 8014 add_one_64(z, zlen, carry, tmp1); 8015 8016 pop(len); 8017 pop(zlen); 8018 jmp(L_second_loop); 8019 8020 // Next infrequent code is moved outside loops. 8021 bind(L_last_x); 8022 movl(op1, Address(x, 0)); 8023 jmp(L_multiply); 8024 8025 bind(L_second_loop_exit); 8026 pop(len); 8027 pop(zlen); 8028 pop(len); 8029 pop(zlen); 8030 8031 // Fifth loop 8032 // Shift z left 1 bit. 8033 lshift_by_1(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4); 8034 8035 // z[zlen-1] |= x[len-1] & 1; 8036 movl(tmp3, Address(x, len, Address::times_4, -4)); 8037 andl(tmp3, 1); 8038 orl(Address(z, zlen, Address::times_4, -4), tmp3); 8039 8040 pop(tmp5); 8041 pop(tmp4); 8042 pop(tmp3); 8043 pop(tmp2); 8044 pop(tmp1); 8045 } 8046 8047 /** 8048 * Helper function for mul_add() 8049 * Multiply the in[] by int k and add to out[] starting at offset offs using 8050 * 128 bit by 32 bit multiply and return the carry in tmp5. 8051 * Only quad int aligned length of in[] is operated on in this function. 8052 * k is in rdxReg for BMI2Instructions, for others it is in tmp2. 8053 * This function preserves out, in and k registers. 8054 * len and offset point to the appropriate index in "in" & "out" correspondingly 8055 * tmp5 has the carry. 8056 * other registers are temporary and are modified. 8057 * 8058 */ 8059 void MacroAssembler::mul_add_128_x_32_loop(Register out, Register in, 8060 Register offset, Register len, Register tmp1, Register tmp2, Register tmp3, 8061 Register tmp4, Register tmp5, Register rdxReg, Register raxReg) { 8062 8063 Label L_first_loop, L_first_loop_exit; 8064 8065 movl(tmp1, len); 8066 shrl(tmp1, 2); 8067 8068 bind(L_first_loop); 8069 subl(tmp1, 1); 8070 jccb(Assembler::negative, L_first_loop_exit); 8071 8072 subl(len, 4); 8073 subl(offset, 4); 8074 8075 Register op2 = tmp2; 8076 const Register sum = tmp3; 8077 const Register op1 = tmp4; 8078 const Register carry = tmp5; 8079 8080 if (UseBMI2Instructions) { 8081 op2 = rdxReg; 8082 } 8083 8084 movq(op1, Address(in, len, Address::times_4, 8)); 8085 rorq(op1, 32); 8086 movq(sum, Address(out, offset, Address::times_4, 8)); 8087 rorq(sum, 32); 8088 if (UseBMI2Instructions) { 8089 multiply_add_64_bmi2(sum, op1, op2, carry, raxReg); 8090 } 8091 else { 8092 multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg); 8093 } 8094 // Store back in big endian from little endian 8095 rorq(sum, 0x20); 8096 movq(Address(out, offset, Address::times_4, 8), sum); 8097 8098 movq(op1, Address(in, len, Address::times_4, 0)); 8099 rorq(op1, 32); 8100 movq(sum, Address(out, offset, Address::times_4, 0)); 8101 rorq(sum, 32); 8102 if (UseBMI2Instructions) { 8103 multiply_add_64_bmi2(sum, op1, op2, carry, raxReg); 8104 } 8105 else { 8106 multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg); 8107 } 8108 // Store back in big endian from little endian 8109 rorq(sum, 0x20); 8110 movq(Address(out, offset, Address::times_4, 0), sum); 8111 8112 jmp(L_first_loop); 8113 bind(L_first_loop_exit); 8114 } 8115 8116 /** 8117 * Code for BigInteger::mulAdd() intrinsic 8118 * 8119 * rdi: out 8120 * rsi: in 8121 * r11: offs (out.length - offset) 8122 * rcx: len 8123 * r8: k 8124 * r12: tmp1 8125 * r13: tmp2 8126 * r14: tmp3 8127 * r15: tmp4 8128 * rbx: tmp5 8129 * Multiply the in[] by word k and add to out[], return the carry in rax 8130 */ 8131 void MacroAssembler::mul_add(Register out, Register in, Register offs, 8132 Register len, Register k, Register tmp1, Register tmp2, Register tmp3, 8133 Register tmp4, Register tmp5, Register rdxReg, Register raxReg) { 8134 8135 Label L_carry, L_last_in, L_done; 8136 8137 // carry = 0; 8138 // for (int j=len-1; j >= 0; j--) { 8139 // long product = (in[j] & LONG_MASK) * kLong + 8140 // (out[offs] & LONG_MASK) + carry; 8141 // out[offs--] = (int)product; 8142 // carry = product >>> 32; 8143 // } 8144 // 8145 push(tmp1); 8146 push(tmp2); 8147 push(tmp3); 8148 push(tmp4); 8149 push(tmp5); 8150 8151 Register op2 = tmp2; 8152 const Register sum = tmp3; 8153 const Register op1 = tmp4; 8154 const Register carry = tmp5; 8155 8156 if (UseBMI2Instructions) { 8157 op2 = rdxReg; 8158 movl(op2, k); 8159 } 8160 else { 8161 movl(op2, k); 8162 } 8163 8164 xorq(carry, carry); 8165 8166 //First loop 8167 8168 //Multiply in[] by k in a 4 way unrolled loop using 128 bit by 32 bit multiply 8169 //The carry is in tmp5 8170 mul_add_128_x_32_loop(out, in, offs, len, tmp1, tmp2, tmp3, tmp4, tmp5, rdxReg, raxReg); 8171 8172 //Multiply the trailing in[] entry using 64 bit by 32 bit, if any 8173 decrementl(len); 8174 jccb(Assembler::negative, L_carry); 8175 decrementl(len); 8176 jccb(Assembler::negative, L_last_in); 8177 8178 movq(op1, Address(in, len, Address::times_4, 0)); 8179 rorq(op1, 32); 8180 8181 subl(offs, 2); 8182 movq(sum, Address(out, offs, Address::times_4, 0)); 8183 rorq(sum, 32); 8184 8185 if (UseBMI2Instructions) { 8186 multiply_add_64_bmi2(sum, op1, op2, carry, raxReg); 8187 } 8188 else { 8189 multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg); 8190 } 8191 8192 // Store back in big endian from little endian 8193 rorq(sum, 0x20); 8194 movq(Address(out, offs, Address::times_4, 0), sum); 8195 8196 testl(len, len); 8197 jccb(Assembler::zero, L_carry); 8198 8199 //Multiply the last in[] entry, if any 8200 bind(L_last_in); 8201 movl(op1, Address(in, 0)); 8202 movl(sum, Address(out, offs, Address::times_4, -4)); 8203 8204 movl(raxReg, k); 8205 mull(op1); //tmp4 * eax -> edx:eax 8206 addl(sum, carry); 8207 adcl(rdxReg, 0); 8208 addl(sum, raxReg); 8209 adcl(rdxReg, 0); 8210 movl(carry, rdxReg); 8211 8212 movl(Address(out, offs, Address::times_4, -4), sum); 8213 8214 bind(L_carry); 8215 //return tmp5/carry as carry in rax 8216 movl(rax, carry); 8217 8218 bind(L_done); 8219 pop(tmp5); 8220 pop(tmp4); 8221 pop(tmp3); 8222 pop(tmp2); 8223 pop(tmp1); 8224 } 8225 #endif 8226 8227 /** 8228 * Emits code to update CRC-32 with a byte value according to constants in table 8229 * 8230 * @param [in,out]crc Register containing the crc. 8231 * @param [in]val Register containing the byte to fold into the CRC. 8232 * @param [in]table Register containing the table of crc constants. 8233 * 8234 * uint32_t crc; 8235 * val = crc_table[(val ^ crc) & 0xFF]; 8236 * crc = val ^ (crc >> 8); 8237 * 8238 */ 8239 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) { 8240 xorl(val, crc); 8241 andl(val, 0xFF); 8242 shrl(crc, 8); // unsigned shift 8243 xorl(crc, Address(table, val, Address::times_4, 0)); 8244 } 8245 8246 /** 8247 * Fold 128-bit data chunk 8248 */ 8249 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset) { 8250 if (UseAVX > 0) { 8251 vpclmulhdq(xtmp, xK, xcrc); // [123:64] 8252 vpclmulldq(xcrc, xK, xcrc); // [63:0] 8253 vpxor(xcrc, xcrc, Address(buf, offset), 0 /* vector_len */); 8254 pxor(xcrc, xtmp); 8255 } else { 8256 movdqa(xtmp, xcrc); 8257 pclmulhdq(xtmp, xK); // [123:64] 8258 pclmulldq(xcrc, xK); // [63:0] 8259 pxor(xcrc, xtmp); 8260 movdqu(xtmp, Address(buf, offset)); 8261 pxor(xcrc, xtmp); 8262 } 8263 } 8264 8265 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, XMMRegister xbuf) { 8266 if (UseAVX > 0) { 8267 vpclmulhdq(xtmp, xK, xcrc); 8268 vpclmulldq(xcrc, xK, xcrc); 8269 pxor(xcrc, xbuf); 8270 pxor(xcrc, xtmp); 8271 } else { 8272 movdqa(xtmp, xcrc); 8273 pclmulhdq(xtmp, xK); 8274 pclmulldq(xcrc, xK); 8275 pxor(xcrc, xbuf); 8276 pxor(xcrc, xtmp); 8277 } 8278 } 8279 8280 /** 8281 * 8-bit folds to compute 32-bit CRC 8282 * 8283 * uint64_t xcrc; 8284 * timesXtoThe32[xcrc & 0xFF] ^ (xcrc >> 8); 8285 */ 8286 void MacroAssembler::fold_8bit_crc32(XMMRegister xcrc, Register table, XMMRegister xtmp, Register tmp) { 8287 movdl(tmp, xcrc); 8288 andl(tmp, 0xFF); 8289 movdl(xtmp, Address(table, tmp, Address::times_4, 0)); 8290 psrldq(xcrc, 1); // unsigned shift one byte 8291 pxor(xcrc, xtmp); 8292 } 8293 8294 /** 8295 * uint32_t crc; 8296 * timesXtoThe32[crc & 0xFF] ^ (crc >> 8); 8297 */ 8298 void MacroAssembler::fold_8bit_crc32(Register crc, Register table, Register tmp) { 8299 movl(tmp, crc); 8300 andl(tmp, 0xFF); 8301 shrl(crc, 8); 8302 xorl(crc, Address(table, tmp, Address::times_4, 0)); 8303 } 8304 8305 /** 8306 * @param crc register containing existing CRC (32-bit) 8307 * @param buf register pointing to input byte buffer (byte*) 8308 * @param len register containing number of bytes 8309 * @param table register that will contain address of CRC table 8310 * @param tmp scratch register 8311 */ 8312 void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len, Register table, Register tmp) { 8313 assert_different_registers(crc, buf, len, table, tmp, rax); 8314 8315 Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned; 8316 Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop; 8317 8318 // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge 8319 // context for the registers used, where all instructions below are using 128-bit mode 8320 // On EVEX without VL and BW, these instructions will all be AVX. 8321 lea(table, ExternalAddress(StubRoutines::crc_table_addr())); 8322 notl(crc); // ~crc 8323 cmpl(len, 16); 8324 jcc(Assembler::less, L_tail); 8325 8326 // Align buffer to 16 bytes 8327 movl(tmp, buf); 8328 andl(tmp, 0xF); 8329 jccb(Assembler::zero, L_aligned); 8330 subl(tmp, 16); 8331 addl(len, tmp); 8332 8333 align(4); 8334 BIND(L_align_loop); 8335 movsbl(rax, Address(buf, 0)); // load byte with sign extension 8336 update_byte_crc32(crc, rax, table); 8337 increment(buf); 8338 incrementl(tmp); 8339 jccb(Assembler::less, L_align_loop); 8340 8341 BIND(L_aligned); 8342 movl(tmp, len); // save 8343 shrl(len, 4); 8344 jcc(Assembler::zero, L_tail_restore); 8345 8346 // Fold crc into first bytes of vector 8347 movdqa(xmm1, Address(buf, 0)); 8348 movdl(rax, xmm1); 8349 xorl(crc, rax); 8350 if (VM_Version::supports_sse4_1()) { 8351 pinsrd(xmm1, crc, 0); 8352 } else { 8353 pinsrw(xmm1, crc, 0); 8354 shrl(crc, 16); 8355 pinsrw(xmm1, crc, 1); 8356 } 8357 addptr(buf, 16); 8358 subl(len, 4); // len > 0 8359 jcc(Assembler::less, L_fold_tail); 8360 8361 movdqa(xmm2, Address(buf, 0)); 8362 movdqa(xmm3, Address(buf, 16)); 8363 movdqa(xmm4, Address(buf, 32)); 8364 addptr(buf, 48); 8365 subl(len, 3); 8366 jcc(Assembler::lessEqual, L_fold_512b); 8367 8368 // Fold total 512 bits of polynomial on each iteration, 8369 // 128 bits per each of 4 parallel streams. 8370 movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 32), rscratch1); 8371 8372 align32(); 8373 BIND(L_fold_512b_loop); 8374 fold_128bit_crc32(xmm1, xmm0, xmm5, buf, 0); 8375 fold_128bit_crc32(xmm2, xmm0, xmm5, buf, 16); 8376 fold_128bit_crc32(xmm3, xmm0, xmm5, buf, 32); 8377 fold_128bit_crc32(xmm4, xmm0, xmm5, buf, 48); 8378 addptr(buf, 64); 8379 subl(len, 4); 8380 jcc(Assembler::greater, L_fold_512b_loop); 8381 8382 // Fold 512 bits to 128 bits. 8383 BIND(L_fold_512b); 8384 movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16), rscratch1); 8385 fold_128bit_crc32(xmm1, xmm0, xmm5, xmm2); 8386 fold_128bit_crc32(xmm1, xmm0, xmm5, xmm3); 8387 fold_128bit_crc32(xmm1, xmm0, xmm5, xmm4); 8388 8389 // Fold the rest of 128 bits data chunks 8390 BIND(L_fold_tail); 8391 addl(len, 3); 8392 jccb(Assembler::lessEqual, L_fold_128b); 8393 movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16), rscratch1); 8394 8395 BIND(L_fold_tail_loop); 8396 fold_128bit_crc32(xmm1, xmm0, xmm5, buf, 0); 8397 addptr(buf, 16); 8398 decrementl(len); 8399 jccb(Assembler::greater, L_fold_tail_loop); 8400 8401 // Fold 128 bits in xmm1 down into 32 bits in crc register. 8402 BIND(L_fold_128b); 8403 movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr()), rscratch1); 8404 if (UseAVX > 0) { 8405 vpclmulqdq(xmm2, xmm0, xmm1, 0x1); 8406 vpand(xmm3, xmm0, xmm2, 0 /* vector_len */); 8407 vpclmulqdq(xmm0, xmm0, xmm3, 0x1); 8408 } else { 8409 movdqa(xmm2, xmm0); 8410 pclmulqdq(xmm2, xmm1, 0x1); 8411 movdqa(xmm3, xmm0); 8412 pand(xmm3, xmm2); 8413 pclmulqdq(xmm0, xmm3, 0x1); 8414 } 8415 psrldq(xmm1, 8); 8416 psrldq(xmm2, 4); 8417 pxor(xmm0, xmm1); 8418 pxor(xmm0, xmm2); 8419 8420 // 8 8-bit folds to compute 32-bit CRC. 8421 for (int j = 0; j < 4; j++) { 8422 fold_8bit_crc32(xmm0, table, xmm1, rax); 8423 } 8424 movdl(crc, xmm0); // mov 32 bits to general register 8425 for (int j = 0; j < 4; j++) { 8426 fold_8bit_crc32(crc, table, rax); 8427 } 8428 8429 BIND(L_tail_restore); 8430 movl(len, tmp); // restore 8431 BIND(L_tail); 8432 andl(len, 0xf); 8433 jccb(Assembler::zero, L_exit); 8434 8435 // Fold the rest of bytes 8436 align(4); 8437 BIND(L_tail_loop); 8438 movsbl(rax, Address(buf, 0)); // load byte with sign extension 8439 update_byte_crc32(crc, rax, table); 8440 increment(buf); 8441 decrementl(len); 8442 jccb(Assembler::greater, L_tail_loop); 8443 8444 BIND(L_exit); 8445 notl(crc); // ~c 8446 } 8447 8448 #ifdef _LP64 8449 // Helper function for AVX 512 CRC32 8450 // Fold 512-bit data chunks 8451 void MacroAssembler::fold512bit_crc32_avx512(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, 8452 Register pos, int offset) { 8453 evmovdquq(xmm3, Address(buf, pos, Address::times_1, offset), Assembler::AVX_512bit); 8454 evpclmulqdq(xtmp, xcrc, xK, 0x10, Assembler::AVX_512bit); // [123:64] 8455 evpclmulqdq(xmm2, xcrc, xK, 0x01, Assembler::AVX_512bit); // [63:0] 8456 evpxorq(xcrc, xtmp, xmm2, Assembler::AVX_512bit /* vector_len */); 8457 evpxorq(xcrc, xcrc, xmm3, Assembler::AVX_512bit /* vector_len */); 8458 } 8459 8460 // Helper function for AVX 512 CRC32 8461 // Compute CRC32 for < 256B buffers 8462 void MacroAssembler::kernel_crc32_avx512_256B(Register crc, Register buf, Register len, Register table, Register pos, 8463 Register tmp1, Register tmp2, Label& L_barrett, Label& L_16B_reduction_loop, 8464 Label& L_get_last_two_xmms, Label& L_128_done, Label& L_cleanup) { 8465 8466 Label L_less_than_32, L_exact_16_left, L_less_than_16_left; 8467 Label L_less_than_8_left, L_less_than_4_left, L_less_than_2_left, L_zero_left; 8468 Label L_only_less_than_4, L_only_less_than_3, L_only_less_than_2; 8469 8470 // check if there is enough buffer to be able to fold 16B at a time 8471 cmpl(len, 32); 8472 jcc(Assembler::less, L_less_than_32); 8473 8474 // if there is, load the constants 8475 movdqu(xmm10, Address(table, 1 * 16)); //rk1 and rk2 in xmm10 8476 movdl(xmm0, crc); // get the initial crc value 8477 movdqu(xmm7, Address(buf, pos, Address::times_1, 0 * 16)); //load the plaintext 8478 pxor(xmm7, xmm0); 8479 8480 // update the buffer pointer 8481 addl(pos, 16); 8482 //update the counter.subtract 32 instead of 16 to save one instruction from the loop 8483 subl(len, 32); 8484 jmp(L_16B_reduction_loop); 8485 8486 bind(L_less_than_32); 8487 //mov initial crc to the return value. this is necessary for zero - length buffers. 8488 movl(rax, crc); 8489 testl(len, len); 8490 jcc(Assembler::equal, L_cleanup); 8491 8492 movdl(xmm0, crc); //get the initial crc value 8493 8494 cmpl(len, 16); 8495 jcc(Assembler::equal, L_exact_16_left); 8496 jcc(Assembler::less, L_less_than_16_left); 8497 8498 movdqu(xmm7, Address(buf, pos, Address::times_1, 0 * 16)); //load the plaintext 8499 pxor(xmm7, xmm0); //xor the initial crc value 8500 addl(pos, 16); 8501 subl(len, 16); 8502 movdqu(xmm10, Address(table, 1 * 16)); // rk1 and rk2 in xmm10 8503 jmp(L_get_last_two_xmms); 8504 8505 bind(L_less_than_16_left); 8506 //use stack space to load data less than 16 bytes, zero - out the 16B in memory first. 8507 pxor(xmm1, xmm1); 8508 movptr(tmp1, rsp); 8509 movdqu(Address(tmp1, 0 * 16), xmm1); 8510 8511 cmpl(len, 4); 8512 jcc(Assembler::less, L_only_less_than_4); 8513 8514 //backup the counter value 8515 movl(tmp2, len); 8516 cmpl(len, 8); 8517 jcc(Assembler::less, L_less_than_8_left); 8518 8519 //load 8 Bytes 8520 movq(rax, Address(buf, pos, Address::times_1, 0 * 16)); 8521 movq(Address(tmp1, 0 * 16), rax); 8522 addptr(tmp1, 8); 8523 subl(len, 8); 8524 addl(pos, 8); 8525 8526 bind(L_less_than_8_left); 8527 cmpl(len, 4); 8528 jcc(Assembler::less, L_less_than_4_left); 8529 8530 //load 4 Bytes 8531 movl(rax, Address(buf, pos, Address::times_1, 0)); 8532 movl(Address(tmp1, 0 * 16), rax); 8533 addptr(tmp1, 4); 8534 subl(len, 4); 8535 addl(pos, 4); 8536 8537 bind(L_less_than_4_left); 8538 cmpl(len, 2); 8539 jcc(Assembler::less, L_less_than_2_left); 8540 8541 // load 2 Bytes 8542 movw(rax, Address(buf, pos, Address::times_1, 0)); 8543 movl(Address(tmp1, 0 * 16), rax); 8544 addptr(tmp1, 2); 8545 subl(len, 2); 8546 addl(pos, 2); 8547 8548 bind(L_less_than_2_left); 8549 cmpl(len, 1); 8550 jcc(Assembler::less, L_zero_left); 8551 8552 // load 1 Byte 8553 movb(rax, Address(buf, pos, Address::times_1, 0)); 8554 movb(Address(tmp1, 0 * 16), rax); 8555 8556 bind(L_zero_left); 8557 movdqu(xmm7, Address(rsp, 0)); 8558 pxor(xmm7, xmm0); //xor the initial crc value 8559 8560 lea(rax, ExternalAddress(StubRoutines::x86::shuf_table_crc32_avx512_addr())); 8561 movdqu(xmm0, Address(rax, tmp2)); 8562 pshufb(xmm7, xmm0); 8563 jmp(L_128_done); 8564 8565 bind(L_exact_16_left); 8566 movdqu(xmm7, Address(buf, pos, Address::times_1, 0)); 8567 pxor(xmm7, xmm0); //xor the initial crc value 8568 jmp(L_128_done); 8569 8570 bind(L_only_less_than_4); 8571 cmpl(len, 3); 8572 jcc(Assembler::less, L_only_less_than_3); 8573 8574 // load 3 Bytes 8575 movb(rax, Address(buf, pos, Address::times_1, 0)); 8576 movb(Address(tmp1, 0), rax); 8577 8578 movb(rax, Address(buf, pos, Address::times_1, 1)); 8579 movb(Address(tmp1, 1), rax); 8580 8581 movb(rax, Address(buf, pos, Address::times_1, 2)); 8582 movb(Address(tmp1, 2), rax); 8583 8584 movdqu(xmm7, Address(rsp, 0)); 8585 pxor(xmm7, xmm0); //xor the initial crc value 8586 8587 pslldq(xmm7, 0x5); 8588 jmp(L_barrett); 8589 bind(L_only_less_than_3); 8590 cmpl(len, 2); 8591 jcc(Assembler::less, L_only_less_than_2); 8592 8593 // load 2 Bytes 8594 movb(rax, Address(buf, pos, Address::times_1, 0)); 8595 movb(Address(tmp1, 0), rax); 8596 8597 movb(rax, Address(buf, pos, Address::times_1, 1)); 8598 movb(Address(tmp1, 1), rax); 8599 8600 movdqu(xmm7, Address(rsp, 0)); 8601 pxor(xmm7, xmm0); //xor the initial crc value 8602 8603 pslldq(xmm7, 0x6); 8604 jmp(L_barrett); 8605 8606 bind(L_only_less_than_2); 8607 //load 1 Byte 8608 movb(rax, Address(buf, pos, Address::times_1, 0)); 8609 movb(Address(tmp1, 0), rax); 8610 8611 movdqu(xmm7, Address(rsp, 0)); 8612 pxor(xmm7, xmm0); //xor the initial crc value 8613 8614 pslldq(xmm7, 0x7); 8615 } 8616 8617 /** 8618 * Compute CRC32 using AVX512 instructions 8619 * param crc register containing existing CRC (32-bit) 8620 * param buf register pointing to input byte buffer (byte*) 8621 * param len register containing number of bytes 8622 * param table address of crc or crc32c table 8623 * param tmp1 scratch register 8624 * param tmp2 scratch register 8625 * return rax result register 8626 * 8627 * This routine is identical for crc32c with the exception of the precomputed constant 8628 * table which will be passed as the table argument. The calculation steps are 8629 * the same for both variants. 8630 */ 8631 void MacroAssembler::kernel_crc32_avx512(Register crc, Register buf, Register len, Register table, Register tmp1, Register tmp2) { 8632 assert_different_registers(crc, buf, len, table, tmp1, tmp2, rax, r12); 8633 8634 Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned; 8635 Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop; 8636 Label L_less_than_256, L_fold_128_B_loop, L_fold_256_B_loop; 8637 Label L_fold_128_B_register, L_final_reduction_for_128, L_16B_reduction_loop; 8638 Label L_128_done, L_get_last_two_xmms, L_barrett, L_cleanup; 8639 8640 const Register pos = r12; 8641 push(r12); 8642 subptr(rsp, 16 * 2 + 8); 8643 8644 // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge 8645 // context for the registers used, where all instructions below are using 128-bit mode 8646 // On EVEX without VL and BW, these instructions will all be AVX. 8647 movl(pos, 0); 8648 8649 // check if smaller than 256B 8650 cmpl(len, 256); 8651 jcc(Assembler::less, L_less_than_256); 8652 8653 // load the initial crc value 8654 movdl(xmm10, crc); 8655 8656 // receive the initial 64B data, xor the initial crc value 8657 evmovdquq(xmm0, Address(buf, pos, Address::times_1, 0 * 64), Assembler::AVX_512bit); 8658 evmovdquq(xmm4, Address(buf, pos, Address::times_1, 1 * 64), Assembler::AVX_512bit); 8659 evpxorq(xmm0, xmm0, xmm10, Assembler::AVX_512bit); 8660 evbroadcasti32x4(xmm10, Address(table, 2 * 16), Assembler::AVX_512bit); //zmm10 has rk3 and rk4 8661 8662 subl(len, 256); 8663 cmpl(len, 256); 8664 jcc(Assembler::less, L_fold_128_B_loop); 8665 8666 evmovdquq(xmm7, Address(buf, pos, Address::times_1, 2 * 64), Assembler::AVX_512bit); 8667 evmovdquq(xmm8, Address(buf, pos, Address::times_1, 3 * 64), Assembler::AVX_512bit); 8668 evbroadcasti32x4(xmm16, Address(table, 0 * 16), Assembler::AVX_512bit); //zmm16 has rk-1 and rk-2 8669 subl(len, 256); 8670 8671 bind(L_fold_256_B_loop); 8672 addl(pos, 256); 8673 fold512bit_crc32_avx512(xmm0, xmm16, xmm1, buf, pos, 0 * 64); 8674 fold512bit_crc32_avx512(xmm4, xmm16, xmm1, buf, pos, 1 * 64); 8675 fold512bit_crc32_avx512(xmm7, xmm16, xmm1, buf, pos, 2 * 64); 8676 fold512bit_crc32_avx512(xmm8, xmm16, xmm1, buf, pos, 3 * 64); 8677 8678 subl(len, 256); 8679 jcc(Assembler::greaterEqual, L_fold_256_B_loop); 8680 8681 // Fold 256 into 128 8682 addl(pos, 256); 8683 evpclmulqdq(xmm1, xmm0, xmm10, 0x01, Assembler::AVX_512bit); 8684 evpclmulqdq(xmm2, xmm0, xmm10, 0x10, Assembler::AVX_512bit); 8685 vpternlogq(xmm7, 0x96, xmm1, xmm2, Assembler::AVX_512bit); // xor ABC 8686 8687 evpclmulqdq(xmm5, xmm4, xmm10, 0x01, Assembler::AVX_512bit); 8688 evpclmulqdq(xmm6, xmm4, xmm10, 0x10, Assembler::AVX_512bit); 8689 vpternlogq(xmm8, 0x96, xmm5, xmm6, Assembler::AVX_512bit); // xor ABC 8690 8691 evmovdquq(xmm0, xmm7, Assembler::AVX_512bit); 8692 evmovdquq(xmm4, xmm8, Assembler::AVX_512bit); 8693 8694 addl(len, 128); 8695 jmp(L_fold_128_B_register); 8696 8697 // at this section of the code, there is 128 * x + y(0 <= y<128) bytes of buffer.The fold_128_B_loop 8698 // loop will fold 128B at a time until we have 128 + y Bytes of buffer 8699 8700 // fold 128B at a time.This section of the code folds 8 xmm registers in parallel 8701 bind(L_fold_128_B_loop); 8702 addl(pos, 128); 8703 fold512bit_crc32_avx512(xmm0, xmm10, xmm1, buf, pos, 0 * 64); 8704 fold512bit_crc32_avx512(xmm4, xmm10, xmm1, buf, pos, 1 * 64); 8705 8706 subl(len, 128); 8707 jcc(Assembler::greaterEqual, L_fold_128_B_loop); 8708 8709 addl(pos, 128); 8710 8711 // at this point, the buffer pointer is pointing at the last y Bytes of the buffer, where 0 <= y < 128 8712 // the 128B of folded data is in 8 of the xmm registers : xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 8713 bind(L_fold_128_B_register); 8714 evmovdquq(xmm16, Address(table, 5 * 16), Assembler::AVX_512bit); // multiply by rk9-rk16 8715 evmovdquq(xmm11, Address(table, 9 * 16), Assembler::AVX_512bit); // multiply by rk17-rk20, rk1,rk2, 0,0 8716 evpclmulqdq(xmm1, xmm0, xmm16, 0x01, Assembler::AVX_512bit); 8717 evpclmulqdq(xmm2, xmm0, xmm16, 0x10, Assembler::AVX_512bit); 8718 // save last that has no multiplicand 8719 vextracti64x2(xmm7, xmm4, 3); 8720 8721 evpclmulqdq(xmm5, xmm4, xmm11, 0x01, Assembler::AVX_512bit); 8722 evpclmulqdq(xmm6, xmm4, xmm11, 0x10, Assembler::AVX_512bit); 8723 // Needed later in reduction loop 8724 movdqu(xmm10, Address(table, 1 * 16)); 8725 vpternlogq(xmm1, 0x96, xmm2, xmm5, Assembler::AVX_512bit); // xor ABC 8726 vpternlogq(xmm1, 0x96, xmm6, xmm7, Assembler::AVX_512bit); // xor ABC 8727 8728 // Swap 1,0,3,2 - 01 00 11 10 8729 evshufi64x2(xmm8, xmm1, xmm1, 0x4e, Assembler::AVX_512bit); 8730 evpxorq(xmm8, xmm8, xmm1, Assembler::AVX_256bit); 8731 vextracti128(xmm5, xmm8, 1); 8732 evpxorq(xmm7, xmm5, xmm8, Assembler::AVX_128bit); 8733 8734 // instead of 128, we add 128 - 16 to the loop counter to save 1 instruction from the loop 8735 // instead of a cmp instruction, we use the negative flag with the jl instruction 8736 addl(len, 128 - 16); 8737 jcc(Assembler::less, L_final_reduction_for_128); 8738 8739 bind(L_16B_reduction_loop); 8740 vpclmulqdq(xmm8, xmm7, xmm10, 0x01); 8741 vpclmulqdq(xmm7, xmm7, xmm10, 0x10); 8742 vpxor(xmm7, xmm7, xmm8, Assembler::AVX_128bit); 8743 movdqu(xmm0, Address(buf, pos, Address::times_1, 0 * 16)); 8744 vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit); 8745 addl(pos, 16); 8746 subl(len, 16); 8747 jcc(Assembler::greaterEqual, L_16B_reduction_loop); 8748 8749 bind(L_final_reduction_for_128); 8750 addl(len, 16); 8751 jcc(Assembler::equal, L_128_done); 8752 8753 bind(L_get_last_two_xmms); 8754 movdqu(xmm2, xmm7); 8755 addl(pos, len); 8756 movdqu(xmm1, Address(buf, pos, Address::times_1, -16)); 8757 subl(pos, len); 8758 8759 // get rid of the extra data that was loaded before 8760 // load the shift constant 8761 lea(rax, ExternalAddress(StubRoutines::x86::shuf_table_crc32_avx512_addr())); 8762 movdqu(xmm0, Address(rax, len)); 8763 addl(rax, len); 8764 8765 vpshufb(xmm7, xmm7, xmm0, Assembler::AVX_128bit); 8766 //Change mask to 512 8767 vpxor(xmm0, xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr() + 2 * 16), Assembler::AVX_128bit, tmp2); 8768 vpshufb(xmm2, xmm2, xmm0, Assembler::AVX_128bit); 8769 8770 blendvpb(xmm2, xmm2, xmm1, xmm0, Assembler::AVX_128bit); 8771 vpclmulqdq(xmm8, xmm7, xmm10, 0x01); 8772 vpclmulqdq(xmm7, xmm7, xmm10, 0x10); 8773 vpxor(xmm7, xmm7, xmm8, Assembler::AVX_128bit); 8774 vpxor(xmm7, xmm7, xmm2, Assembler::AVX_128bit); 8775 8776 bind(L_128_done); 8777 // compute crc of a 128-bit value 8778 movdqu(xmm10, Address(table, 3 * 16)); 8779 movdqu(xmm0, xmm7); 8780 8781 // 64b fold 8782 vpclmulqdq(xmm7, xmm7, xmm10, 0x0); 8783 vpsrldq(xmm0, xmm0, 0x8, Assembler::AVX_128bit); 8784 vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit); 8785 8786 // 32b fold 8787 movdqu(xmm0, xmm7); 8788 vpslldq(xmm7, xmm7, 0x4, Assembler::AVX_128bit); 8789 vpclmulqdq(xmm7, xmm7, xmm10, 0x10); 8790 vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit); 8791 jmp(L_barrett); 8792 8793 bind(L_less_than_256); 8794 kernel_crc32_avx512_256B(crc, buf, len, table, pos, tmp1, tmp2, L_barrett, L_16B_reduction_loop, L_get_last_two_xmms, L_128_done, L_cleanup); 8795 8796 //barrett reduction 8797 bind(L_barrett); 8798 vpand(xmm7, xmm7, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr() + 1 * 16), Assembler::AVX_128bit, tmp2); 8799 movdqu(xmm1, xmm7); 8800 movdqu(xmm2, xmm7); 8801 movdqu(xmm10, Address(table, 4 * 16)); 8802 8803 pclmulqdq(xmm7, xmm10, 0x0); 8804 pxor(xmm7, xmm2); 8805 vpand(xmm7, xmm7, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr()), Assembler::AVX_128bit, tmp2); 8806 movdqu(xmm2, xmm7); 8807 pclmulqdq(xmm7, xmm10, 0x10); 8808 pxor(xmm7, xmm2); 8809 pxor(xmm7, xmm1); 8810 pextrd(crc, xmm7, 2); 8811 8812 bind(L_cleanup); 8813 addptr(rsp, 16 * 2 + 8); 8814 pop(r12); 8815 } 8816 8817 // S. Gueron / Information Processing Letters 112 (2012) 184 8818 // Algorithm 4: Computing carry-less multiplication using a precomputed lookup table. 8819 // Input: A 32 bit value B = [byte3, byte2, byte1, byte0]. 8820 // Output: the 64-bit carry-less product of B * CONST 8821 void MacroAssembler::crc32c_ipl_alg4(Register in, uint32_t n, 8822 Register tmp1, Register tmp2, Register tmp3) { 8823 lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr())); 8824 if (n > 0) { 8825 addq(tmp3, n * 256 * 8); 8826 } 8827 // Q1 = TABLEExt[n][B & 0xFF]; 8828 movl(tmp1, in); 8829 andl(tmp1, 0x000000FF); 8830 shll(tmp1, 3); 8831 addq(tmp1, tmp3); 8832 movq(tmp1, Address(tmp1, 0)); 8833 8834 // Q2 = TABLEExt[n][B >> 8 & 0xFF]; 8835 movl(tmp2, in); 8836 shrl(tmp2, 8); 8837 andl(tmp2, 0x000000FF); 8838 shll(tmp2, 3); 8839 addq(tmp2, tmp3); 8840 movq(tmp2, Address(tmp2, 0)); 8841 8842 shlq(tmp2, 8); 8843 xorq(tmp1, tmp2); 8844 8845 // Q3 = TABLEExt[n][B >> 16 & 0xFF]; 8846 movl(tmp2, in); 8847 shrl(tmp2, 16); 8848 andl(tmp2, 0x000000FF); 8849 shll(tmp2, 3); 8850 addq(tmp2, tmp3); 8851 movq(tmp2, Address(tmp2, 0)); 8852 8853 shlq(tmp2, 16); 8854 xorq(tmp1, tmp2); 8855 8856 // Q4 = TABLEExt[n][B >> 24 & 0xFF]; 8857 shrl(in, 24); 8858 andl(in, 0x000000FF); 8859 shll(in, 3); 8860 addq(in, tmp3); 8861 movq(in, Address(in, 0)); 8862 8863 shlq(in, 24); 8864 xorq(in, tmp1); 8865 // return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24; 8866 } 8867 8868 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1, 8869 Register in_out, 8870 uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported, 8871 XMMRegister w_xtmp2, 8872 Register tmp1, 8873 Register n_tmp2, Register n_tmp3) { 8874 if (is_pclmulqdq_supported) { 8875 movdl(w_xtmp1, in_out); // modified blindly 8876 8877 movl(tmp1, const_or_pre_comp_const_index); 8878 movdl(w_xtmp2, tmp1); 8879 pclmulqdq(w_xtmp1, w_xtmp2, 0); 8880 8881 movdq(in_out, w_xtmp1); 8882 } else { 8883 crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3); 8884 } 8885 } 8886 8887 // Recombination Alternative 2: No bit-reflections 8888 // T1 = (CRC_A * U1) << 1 8889 // T2 = (CRC_B * U2) << 1 8890 // C1 = T1 >> 32 8891 // C2 = T2 >> 32 8892 // T1 = T1 & 0xFFFFFFFF 8893 // T2 = T2 & 0xFFFFFFFF 8894 // T1 = CRC32(0, T1) 8895 // T2 = CRC32(0, T2) 8896 // C1 = C1 ^ T1 8897 // C2 = C2 ^ T2 8898 // CRC = C1 ^ C2 ^ CRC_C 8899 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2, 8900 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, 8901 Register tmp1, Register tmp2, 8902 Register n_tmp3) { 8903 crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3); 8904 crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3); 8905 shlq(in_out, 1); 8906 movl(tmp1, in_out); 8907 shrq(in_out, 32); 8908 xorl(tmp2, tmp2); 8909 crc32(tmp2, tmp1, 4); 8910 xorl(in_out, tmp2); // we don't care about upper 32 bit contents here 8911 shlq(in1, 1); 8912 movl(tmp1, in1); 8913 shrq(in1, 32); 8914 xorl(tmp2, tmp2); 8915 crc32(tmp2, tmp1, 4); 8916 xorl(in1, tmp2); 8917 xorl(in_out, in1); 8918 xorl(in_out, in2); 8919 } 8920 8921 // Set N to predefined value 8922 // Subtract from a length of a buffer 8923 // execute in a loop: 8924 // CRC_A = 0xFFFFFFFF, CRC_B = 0, CRC_C = 0 8925 // for i = 1 to N do 8926 // CRC_A = CRC32(CRC_A, A[i]) 8927 // CRC_B = CRC32(CRC_B, B[i]) 8928 // CRC_C = CRC32(CRC_C, C[i]) 8929 // end for 8930 // Recombine 8931 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, 8932 Register in_out1, Register in_out2, Register in_out3, 8933 Register tmp1, Register tmp2, Register tmp3, 8934 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, 8935 Register tmp4, Register tmp5, 8936 Register n_tmp6) { 8937 Label L_processPartitions; 8938 Label L_processPartition; 8939 Label L_exit; 8940 8941 bind(L_processPartitions); 8942 cmpl(in_out1, 3 * size); 8943 jcc(Assembler::less, L_exit); 8944 xorl(tmp1, tmp1); 8945 xorl(tmp2, tmp2); 8946 movq(tmp3, in_out2); 8947 addq(tmp3, size); 8948 8949 bind(L_processPartition); 8950 crc32(in_out3, Address(in_out2, 0), 8); 8951 crc32(tmp1, Address(in_out2, size), 8); 8952 crc32(tmp2, Address(in_out2, size * 2), 8); 8953 addq(in_out2, 8); 8954 cmpq(in_out2, tmp3); 8955 jcc(Assembler::less, L_processPartition); 8956 crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2, 8957 w_xtmp1, w_xtmp2, w_xtmp3, 8958 tmp4, tmp5, 8959 n_tmp6); 8960 addq(in_out2, 2 * size); 8961 subl(in_out1, 3 * size); 8962 jmp(L_processPartitions); 8963 8964 bind(L_exit); 8965 } 8966 #else 8967 void MacroAssembler::crc32c_ipl_alg4(Register in_out, uint32_t n, 8968 Register tmp1, Register tmp2, Register tmp3, 8969 XMMRegister xtmp1, XMMRegister xtmp2) { 8970 lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr())); 8971 if (n > 0) { 8972 addl(tmp3, n * 256 * 8); 8973 } 8974 // Q1 = TABLEExt[n][B & 0xFF]; 8975 movl(tmp1, in_out); 8976 andl(tmp1, 0x000000FF); 8977 shll(tmp1, 3); 8978 addl(tmp1, tmp3); 8979 movq(xtmp1, Address(tmp1, 0)); 8980 8981 // Q2 = TABLEExt[n][B >> 8 & 0xFF]; 8982 movl(tmp2, in_out); 8983 shrl(tmp2, 8); 8984 andl(tmp2, 0x000000FF); 8985 shll(tmp2, 3); 8986 addl(tmp2, tmp3); 8987 movq(xtmp2, Address(tmp2, 0)); 8988 8989 psllq(xtmp2, 8); 8990 pxor(xtmp1, xtmp2); 8991 8992 // Q3 = TABLEExt[n][B >> 16 & 0xFF]; 8993 movl(tmp2, in_out); 8994 shrl(tmp2, 16); 8995 andl(tmp2, 0x000000FF); 8996 shll(tmp2, 3); 8997 addl(tmp2, tmp3); 8998 movq(xtmp2, Address(tmp2, 0)); 8999 9000 psllq(xtmp2, 16); 9001 pxor(xtmp1, xtmp2); 9002 9003 // Q4 = TABLEExt[n][B >> 24 & 0xFF]; 9004 shrl(in_out, 24); 9005 andl(in_out, 0x000000FF); 9006 shll(in_out, 3); 9007 addl(in_out, tmp3); 9008 movq(xtmp2, Address(in_out, 0)); 9009 9010 psllq(xtmp2, 24); 9011 pxor(xtmp1, xtmp2); // Result in CXMM 9012 // return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24; 9013 } 9014 9015 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1, 9016 Register in_out, 9017 uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported, 9018 XMMRegister w_xtmp2, 9019 Register tmp1, 9020 Register n_tmp2, Register n_tmp3) { 9021 if (is_pclmulqdq_supported) { 9022 movdl(w_xtmp1, in_out); 9023 9024 movl(tmp1, const_or_pre_comp_const_index); 9025 movdl(w_xtmp2, tmp1); 9026 pclmulqdq(w_xtmp1, w_xtmp2, 0); 9027 // Keep result in XMM since GPR is 32 bit in length 9028 } else { 9029 crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3, w_xtmp1, w_xtmp2); 9030 } 9031 } 9032 9033 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2, 9034 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, 9035 Register tmp1, Register tmp2, 9036 Register n_tmp3) { 9037 crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3); 9038 crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3); 9039 9040 psllq(w_xtmp1, 1); 9041 movdl(tmp1, w_xtmp1); 9042 psrlq(w_xtmp1, 32); 9043 movdl(in_out, w_xtmp1); 9044 9045 xorl(tmp2, tmp2); 9046 crc32(tmp2, tmp1, 4); 9047 xorl(in_out, tmp2); 9048 9049 psllq(w_xtmp2, 1); 9050 movdl(tmp1, w_xtmp2); 9051 psrlq(w_xtmp2, 32); 9052 movdl(in1, w_xtmp2); 9053 9054 xorl(tmp2, tmp2); 9055 crc32(tmp2, tmp1, 4); 9056 xorl(in1, tmp2); 9057 xorl(in_out, in1); 9058 xorl(in_out, in2); 9059 } 9060 9061 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, 9062 Register in_out1, Register in_out2, Register in_out3, 9063 Register tmp1, Register tmp2, Register tmp3, 9064 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, 9065 Register tmp4, Register tmp5, 9066 Register n_tmp6) { 9067 Label L_processPartitions; 9068 Label L_processPartition; 9069 Label L_exit; 9070 9071 bind(L_processPartitions); 9072 cmpl(in_out1, 3 * size); 9073 jcc(Assembler::less, L_exit); 9074 xorl(tmp1, tmp1); 9075 xorl(tmp2, tmp2); 9076 movl(tmp3, in_out2); 9077 addl(tmp3, size); 9078 9079 bind(L_processPartition); 9080 crc32(in_out3, Address(in_out2, 0), 4); 9081 crc32(tmp1, Address(in_out2, size), 4); 9082 crc32(tmp2, Address(in_out2, size*2), 4); 9083 crc32(in_out3, Address(in_out2, 0+4), 4); 9084 crc32(tmp1, Address(in_out2, size+4), 4); 9085 crc32(tmp2, Address(in_out2, size*2+4), 4); 9086 addl(in_out2, 8); 9087 cmpl(in_out2, tmp3); 9088 jcc(Assembler::less, L_processPartition); 9089 9090 push(tmp3); 9091 push(in_out1); 9092 push(in_out2); 9093 tmp4 = tmp3; 9094 tmp5 = in_out1; 9095 n_tmp6 = in_out2; 9096 9097 crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2, 9098 w_xtmp1, w_xtmp2, w_xtmp3, 9099 tmp4, tmp5, 9100 n_tmp6); 9101 9102 pop(in_out2); 9103 pop(in_out1); 9104 pop(tmp3); 9105 9106 addl(in_out2, 2 * size); 9107 subl(in_out1, 3 * size); 9108 jmp(L_processPartitions); 9109 9110 bind(L_exit); 9111 } 9112 #endif //LP64 9113 9114 #ifdef _LP64 9115 // Algorithm 2: Pipelined usage of the CRC32 instruction. 9116 // Input: A buffer I of L bytes. 9117 // Output: the CRC32C value of the buffer. 9118 // Notations: 9119 // Write L = 24N + r, with N = floor (L/24). 9120 // r = L mod 24 (0 <= r < 24). 9121 // Consider I as the concatenation of A|B|C|R, where A, B, C, each, 9122 // N quadwords, and R consists of r bytes. 9123 // A[j] = I [8j+7:8j], j= 0, 1, ..., N-1 9124 // B[j] = I [N + 8j+7:N + 8j], j= 0, 1, ..., N-1 9125 // C[j] = I [2N + 8j+7:2N + 8j], j= 0, 1, ..., N-1 9126 // if r > 0 R[j] = I [3N +j], j= 0, 1, ...,r-1 9127 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2, 9128 Register tmp1, Register tmp2, Register tmp3, 9129 Register tmp4, Register tmp5, Register tmp6, 9130 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, 9131 bool is_pclmulqdq_supported) { 9132 uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS]; 9133 Label L_wordByWord; 9134 Label L_byteByByteProlog; 9135 Label L_byteByByte; 9136 Label L_exit; 9137 9138 if (is_pclmulqdq_supported ) { 9139 const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::crc32c_table_addr(); 9140 const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 1); 9141 9142 const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 2); 9143 const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 3); 9144 9145 const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 4); 9146 const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 5); 9147 assert((CRC32C_NUM_PRECOMPUTED_CONSTANTS - 1 ) == 5, "Checking whether you declared all of the constants based on the number of \"chunks\""); 9148 } else { 9149 const_or_pre_comp_const_index[0] = 1; 9150 const_or_pre_comp_const_index[1] = 0; 9151 9152 const_or_pre_comp_const_index[2] = 3; 9153 const_or_pre_comp_const_index[3] = 2; 9154 9155 const_or_pre_comp_const_index[4] = 5; 9156 const_or_pre_comp_const_index[5] = 4; 9157 } 9158 crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported, 9159 in2, in1, in_out, 9160 tmp1, tmp2, tmp3, 9161 w_xtmp1, w_xtmp2, w_xtmp3, 9162 tmp4, tmp5, 9163 tmp6); 9164 crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported, 9165 in2, in1, in_out, 9166 tmp1, tmp2, tmp3, 9167 w_xtmp1, w_xtmp2, w_xtmp3, 9168 tmp4, tmp5, 9169 tmp6); 9170 crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported, 9171 in2, in1, in_out, 9172 tmp1, tmp2, tmp3, 9173 w_xtmp1, w_xtmp2, w_xtmp3, 9174 tmp4, tmp5, 9175 tmp6); 9176 movl(tmp1, in2); 9177 andl(tmp1, 0x00000007); 9178 negl(tmp1); 9179 addl(tmp1, in2); 9180 addq(tmp1, in1); 9181 9182 cmpq(in1, tmp1); 9183 jccb(Assembler::greaterEqual, L_byteByByteProlog); 9184 align(16); 9185 BIND(L_wordByWord); 9186 crc32(in_out, Address(in1, 0), 8); 9187 addq(in1, 8); 9188 cmpq(in1, tmp1); 9189 jcc(Assembler::less, L_wordByWord); 9190 9191 BIND(L_byteByByteProlog); 9192 andl(in2, 0x00000007); 9193 movl(tmp2, 1); 9194 9195 cmpl(tmp2, in2); 9196 jccb(Assembler::greater, L_exit); 9197 BIND(L_byteByByte); 9198 crc32(in_out, Address(in1, 0), 1); 9199 incq(in1); 9200 incl(tmp2); 9201 cmpl(tmp2, in2); 9202 jcc(Assembler::lessEqual, L_byteByByte); 9203 9204 BIND(L_exit); 9205 } 9206 #else 9207 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2, 9208 Register tmp1, Register tmp2, Register tmp3, 9209 Register tmp4, Register tmp5, Register tmp6, 9210 XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3, 9211 bool is_pclmulqdq_supported) { 9212 uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS]; 9213 Label L_wordByWord; 9214 Label L_byteByByteProlog; 9215 Label L_byteByByte; 9216 Label L_exit; 9217 9218 if (is_pclmulqdq_supported) { 9219 const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::crc32c_table_addr(); 9220 const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 1); 9221 9222 const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 2); 9223 const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 3); 9224 9225 const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 4); 9226 const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 5); 9227 } else { 9228 const_or_pre_comp_const_index[0] = 1; 9229 const_or_pre_comp_const_index[1] = 0; 9230 9231 const_or_pre_comp_const_index[2] = 3; 9232 const_or_pre_comp_const_index[3] = 2; 9233 9234 const_or_pre_comp_const_index[4] = 5; 9235 const_or_pre_comp_const_index[5] = 4; 9236 } 9237 crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported, 9238 in2, in1, in_out, 9239 tmp1, tmp2, tmp3, 9240 w_xtmp1, w_xtmp2, w_xtmp3, 9241 tmp4, tmp5, 9242 tmp6); 9243 crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported, 9244 in2, in1, in_out, 9245 tmp1, tmp2, tmp3, 9246 w_xtmp1, w_xtmp2, w_xtmp3, 9247 tmp4, tmp5, 9248 tmp6); 9249 crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported, 9250 in2, in1, in_out, 9251 tmp1, tmp2, tmp3, 9252 w_xtmp1, w_xtmp2, w_xtmp3, 9253 tmp4, tmp5, 9254 tmp6); 9255 movl(tmp1, in2); 9256 andl(tmp1, 0x00000007); 9257 negl(tmp1); 9258 addl(tmp1, in2); 9259 addl(tmp1, in1); 9260 9261 BIND(L_wordByWord); 9262 cmpl(in1, tmp1); 9263 jcc(Assembler::greaterEqual, L_byteByByteProlog); 9264 crc32(in_out, Address(in1,0), 4); 9265 addl(in1, 4); 9266 jmp(L_wordByWord); 9267 9268 BIND(L_byteByByteProlog); 9269 andl(in2, 0x00000007); 9270 movl(tmp2, 1); 9271 9272 BIND(L_byteByByte); 9273 cmpl(tmp2, in2); 9274 jccb(Assembler::greater, L_exit); 9275 movb(tmp1, Address(in1, 0)); 9276 crc32(in_out, tmp1, 1); 9277 incl(in1); 9278 incl(tmp2); 9279 jmp(L_byteByByte); 9280 9281 BIND(L_exit); 9282 } 9283 #endif // LP64 9284 #undef BIND 9285 #undef BLOCK_COMMENT 9286 9287 // Compress char[] array to byte[]. 9288 // Intrinsic for java.lang.StringUTF16.compress(char[] src, int srcOff, byte[] dst, int dstOff, int len) 9289 // Return the array length if every element in array can be encoded, 9290 // otherwise, the index of first non-latin1 (> 0xff) character. 9291 // @IntrinsicCandidate 9292 // public static int compress(char[] src, int srcOff, byte[] dst, int dstOff, int len) { 9293 // for (int i = 0; i < len; i++) { 9294 // char c = src[srcOff]; 9295 // if (c > 0xff) { 9296 // return i; // return index of non-latin1 char 9297 // } 9298 // dst[dstOff] = (byte)c; 9299 // srcOff++; 9300 // dstOff++; 9301 // } 9302 // return len; 9303 // } 9304 void MacroAssembler::char_array_compress(Register src, Register dst, Register len, 9305 XMMRegister tmp1Reg, XMMRegister tmp2Reg, 9306 XMMRegister tmp3Reg, XMMRegister tmp4Reg, 9307 Register tmp5, Register result, KRegister mask1, KRegister mask2) { 9308 Label copy_chars_loop, done, reset_sp, copy_tail; 9309 9310 // rsi: src 9311 // rdi: dst 9312 // rdx: len 9313 // rcx: tmp5 9314 // rax: result 9315 9316 // rsi holds start addr of source char[] to be compressed 9317 // rdi holds start addr of destination byte[] 9318 // rdx holds length 9319 9320 assert(len != result, ""); 9321 9322 // save length for return 9323 movl(result, len); 9324 9325 if ((AVX3Threshold == 0) && (UseAVX > 2) && // AVX512 9326 VM_Version::supports_avx512vlbw() && 9327 VM_Version::supports_bmi2()) { 9328 9329 Label copy_32_loop, copy_loop_tail, below_threshold, reset_for_copy_tail; 9330 9331 // alignment 9332 Label post_alignment; 9333 9334 // if length of the string is less than 32, handle it the old fashioned way 9335 testl(len, -32); 9336 jcc(Assembler::zero, below_threshold); 9337 9338 // First check whether a character is compressible ( <= 0xFF). 9339 // Create mask to test for Unicode chars inside zmm vector 9340 movl(tmp5, 0x00FF); 9341 evpbroadcastw(tmp2Reg, tmp5, Assembler::AVX_512bit); 9342 9343 testl(len, -64); 9344 jccb(Assembler::zero, post_alignment); 9345 9346 movl(tmp5, dst); 9347 andl(tmp5, (32 - 1)); 9348 negl(tmp5); 9349 andl(tmp5, (32 - 1)); 9350 9351 // bail out when there is nothing to be done 9352 testl(tmp5, 0xFFFFFFFF); 9353 jccb(Assembler::zero, post_alignment); 9354 9355 // ~(~0 << len), where len is the # of remaining elements to process 9356 movl(len, 0xFFFFFFFF); 9357 shlxl(len, len, tmp5); 9358 notl(len); 9359 kmovdl(mask2, len); 9360 movl(len, result); 9361 9362 evmovdquw(tmp1Reg, mask2, Address(src, 0), /*merge*/ false, Assembler::AVX_512bit); 9363 evpcmpw(mask1, mask2, tmp1Reg, tmp2Reg, Assembler::le, /*signed*/ false, Assembler::AVX_512bit); 9364 ktestd(mask1, mask2); 9365 jcc(Assembler::carryClear, copy_tail); 9366 9367 evpmovwb(Address(dst, 0), mask2, tmp1Reg, Assembler::AVX_512bit); 9368 9369 addptr(src, tmp5); 9370 addptr(src, tmp5); 9371 addptr(dst, tmp5); 9372 subl(len, tmp5); 9373 9374 bind(post_alignment); 9375 // end of alignment 9376 9377 movl(tmp5, len); 9378 andl(tmp5, (32 - 1)); // tail count (in chars) 9379 andl(len, ~(32 - 1)); // vector count (in chars) 9380 jccb(Assembler::zero, copy_loop_tail); 9381 9382 lea(src, Address(src, len, Address::times_2)); 9383 lea(dst, Address(dst, len, Address::times_1)); 9384 negptr(len); 9385 9386 bind(copy_32_loop); 9387 evmovdquw(tmp1Reg, Address(src, len, Address::times_2), Assembler::AVX_512bit); 9388 evpcmpuw(mask1, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit); 9389 kortestdl(mask1, mask1); 9390 jccb(Assembler::carryClear, reset_for_copy_tail); 9391 9392 // All elements in current processed chunk are valid candidates for 9393 // compression. Write a truncated byte elements to the memory. 9394 evpmovwb(Address(dst, len, Address::times_1), tmp1Reg, Assembler::AVX_512bit); 9395 addptr(len, 32); 9396 jccb(Assembler::notZero, copy_32_loop); 9397 9398 bind(copy_loop_tail); 9399 // bail out when there is nothing to be done 9400 testl(tmp5, 0xFFFFFFFF); 9401 jcc(Assembler::zero, done); 9402 9403 movl(len, tmp5); 9404 9405 // ~(~0 << len), where len is the # of remaining elements to process 9406 movl(tmp5, 0xFFFFFFFF); 9407 shlxl(tmp5, tmp5, len); 9408 notl(tmp5); 9409 9410 kmovdl(mask2, tmp5); 9411 9412 evmovdquw(tmp1Reg, mask2, Address(src, 0), /*merge*/ false, Assembler::AVX_512bit); 9413 evpcmpw(mask1, mask2, tmp1Reg, tmp2Reg, Assembler::le, /*signed*/ false, Assembler::AVX_512bit); 9414 ktestd(mask1, mask2); 9415 jcc(Assembler::carryClear, copy_tail); 9416 9417 evpmovwb(Address(dst, 0), mask2, tmp1Reg, Assembler::AVX_512bit); 9418 jmp(done); 9419 9420 bind(reset_for_copy_tail); 9421 lea(src, Address(src, tmp5, Address::times_2)); 9422 lea(dst, Address(dst, tmp5, Address::times_1)); 9423 subptr(len, tmp5); 9424 jmp(copy_chars_loop); 9425 9426 bind(below_threshold); 9427 } 9428 9429 if (UseSSE42Intrinsics) { 9430 Label copy_32_loop, copy_16, copy_tail_sse, reset_for_copy_tail; 9431 9432 // vectored compression 9433 testl(len, 0xfffffff8); 9434 jcc(Assembler::zero, copy_tail); 9435 9436 movl(tmp5, 0xff00ff00); // create mask to test for Unicode chars in vectors 9437 movdl(tmp1Reg, tmp5); 9438 pshufd(tmp1Reg, tmp1Reg, 0); // store Unicode mask in tmp1Reg 9439 9440 andl(len, 0xfffffff0); 9441 jccb(Assembler::zero, copy_16); 9442 9443 // compress 16 chars per iter 9444 pxor(tmp4Reg, tmp4Reg); 9445 9446 lea(src, Address(src, len, Address::times_2)); 9447 lea(dst, Address(dst, len, Address::times_1)); 9448 negptr(len); 9449 9450 bind(copy_32_loop); 9451 movdqu(tmp2Reg, Address(src, len, Address::times_2)); // load 1st 8 characters 9452 por(tmp4Reg, tmp2Reg); 9453 movdqu(tmp3Reg, Address(src, len, Address::times_2, 16)); // load next 8 characters 9454 por(tmp4Reg, tmp3Reg); 9455 ptest(tmp4Reg, tmp1Reg); // check for Unicode chars in next vector 9456 jccb(Assembler::notZero, reset_for_copy_tail); 9457 packuswb(tmp2Reg, tmp3Reg); // only ASCII chars; compress each to 1 byte 9458 movdqu(Address(dst, len, Address::times_1), tmp2Reg); 9459 addptr(len, 16); 9460 jccb(Assembler::notZero, copy_32_loop); 9461 9462 // compress next vector of 8 chars (if any) 9463 bind(copy_16); 9464 // len = 0 9465 testl(result, 0x00000008); // check if there's a block of 8 chars to compress 9466 jccb(Assembler::zero, copy_tail_sse); 9467 9468 pxor(tmp3Reg, tmp3Reg); 9469 9470 movdqu(tmp2Reg, Address(src, 0)); 9471 ptest(tmp2Reg, tmp1Reg); // check for Unicode chars in vector 9472 jccb(Assembler::notZero, reset_for_copy_tail); 9473 packuswb(tmp2Reg, tmp3Reg); // only LATIN1 chars; compress each to 1 byte 9474 movq(Address(dst, 0), tmp2Reg); 9475 addptr(src, 16); 9476 addptr(dst, 8); 9477 jmpb(copy_tail_sse); 9478 9479 bind(reset_for_copy_tail); 9480 movl(tmp5, result); 9481 andl(tmp5, 0x0000000f); 9482 lea(src, Address(src, tmp5, Address::times_2)); 9483 lea(dst, Address(dst, tmp5, Address::times_1)); 9484 subptr(len, tmp5); 9485 jmpb(copy_chars_loop); 9486 9487 bind(copy_tail_sse); 9488 movl(len, result); 9489 andl(len, 0x00000007); // tail count (in chars) 9490 } 9491 // compress 1 char per iter 9492 bind(copy_tail); 9493 testl(len, len); 9494 jccb(Assembler::zero, done); 9495 lea(src, Address(src, len, Address::times_2)); 9496 lea(dst, Address(dst, len, Address::times_1)); 9497 negptr(len); 9498 9499 bind(copy_chars_loop); 9500 load_unsigned_short(tmp5, Address(src, len, Address::times_2)); 9501 testl(tmp5, 0xff00); // check if Unicode char 9502 jccb(Assembler::notZero, reset_sp); 9503 movb(Address(dst, len, Address::times_1), tmp5); // ASCII char; compress to 1 byte 9504 increment(len); 9505 jccb(Assembler::notZero, copy_chars_loop); 9506 9507 // add len then return (len will be zero if compress succeeded, otherwise negative) 9508 bind(reset_sp); 9509 addl(result, len); 9510 9511 bind(done); 9512 } 9513 9514 // Inflate byte[] array to char[]. 9515 // ..\jdk\src\java.base\share\classes\java\lang\StringLatin1.java 9516 // @IntrinsicCandidate 9517 // private static void inflate(byte[] src, int srcOff, char[] dst, int dstOff, int len) { 9518 // for (int i = 0; i < len; i++) { 9519 // dst[dstOff++] = (char)(src[srcOff++] & 0xff); 9520 // } 9521 // } 9522 void MacroAssembler::byte_array_inflate(Register src, Register dst, Register len, 9523 XMMRegister tmp1, Register tmp2, KRegister mask) { 9524 Label copy_chars_loop, done, below_threshold, avx3_threshold; 9525 // rsi: src 9526 // rdi: dst 9527 // rdx: len 9528 // rcx: tmp2 9529 9530 // rsi holds start addr of source byte[] to be inflated 9531 // rdi holds start addr of destination char[] 9532 // rdx holds length 9533 assert_different_registers(src, dst, len, tmp2); 9534 movl(tmp2, len); 9535 if ((UseAVX > 2) && // AVX512 9536 VM_Version::supports_avx512vlbw() && 9537 VM_Version::supports_bmi2()) { 9538 9539 Label copy_32_loop, copy_tail; 9540 Register tmp3_aliased = len; 9541 9542 // if length of the string is less than 16, handle it in an old fashioned way 9543 testl(len, -16); 9544 jcc(Assembler::zero, below_threshold); 9545 9546 testl(len, -1 * AVX3Threshold); 9547 jcc(Assembler::zero, avx3_threshold); 9548 9549 // In order to use only one arithmetic operation for the main loop we use 9550 // this pre-calculation 9551 andl(tmp2, (32 - 1)); // tail count (in chars), 32 element wide loop 9552 andl(len, -32); // vector count 9553 jccb(Assembler::zero, copy_tail); 9554 9555 lea(src, Address(src, len, Address::times_1)); 9556 lea(dst, Address(dst, len, Address::times_2)); 9557 negptr(len); 9558 9559 9560 // inflate 32 chars per iter 9561 bind(copy_32_loop); 9562 vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_512bit); 9563 evmovdquw(Address(dst, len, Address::times_2), tmp1, Assembler::AVX_512bit); 9564 addptr(len, 32); 9565 jcc(Assembler::notZero, copy_32_loop); 9566 9567 bind(copy_tail); 9568 // bail out when there is nothing to be done 9569 testl(tmp2, -1); // we don't destroy the contents of tmp2 here 9570 jcc(Assembler::zero, done); 9571 9572 // ~(~0 << length), where length is the # of remaining elements to process 9573 movl(tmp3_aliased, -1); 9574 shlxl(tmp3_aliased, tmp3_aliased, tmp2); 9575 notl(tmp3_aliased); 9576 kmovdl(mask, tmp3_aliased); 9577 evpmovzxbw(tmp1, mask, Address(src, 0), Assembler::AVX_512bit); 9578 evmovdquw(Address(dst, 0), mask, tmp1, /*merge*/ true, Assembler::AVX_512bit); 9579 9580 jmp(done); 9581 bind(avx3_threshold); 9582 } 9583 if (UseSSE42Intrinsics) { 9584 Label copy_16_loop, copy_8_loop, copy_bytes, copy_new_tail, copy_tail; 9585 9586 if (UseAVX > 1) { 9587 andl(tmp2, (16 - 1)); 9588 andl(len, -16); 9589 jccb(Assembler::zero, copy_new_tail); 9590 } else { 9591 andl(tmp2, 0x00000007); // tail count (in chars) 9592 andl(len, 0xfffffff8); // vector count (in chars) 9593 jccb(Assembler::zero, copy_tail); 9594 } 9595 9596 // vectored inflation 9597 lea(src, Address(src, len, Address::times_1)); 9598 lea(dst, Address(dst, len, Address::times_2)); 9599 negptr(len); 9600 9601 if (UseAVX > 1) { 9602 bind(copy_16_loop); 9603 vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_256bit); 9604 vmovdqu(Address(dst, len, Address::times_2), tmp1); 9605 addptr(len, 16); 9606 jcc(Assembler::notZero, copy_16_loop); 9607 9608 bind(below_threshold); 9609 bind(copy_new_tail); 9610 movl(len, tmp2); 9611 andl(tmp2, 0x00000007); 9612 andl(len, 0xFFFFFFF8); 9613 jccb(Assembler::zero, copy_tail); 9614 9615 pmovzxbw(tmp1, Address(src, 0)); 9616 movdqu(Address(dst, 0), tmp1); 9617 addptr(src, 8); 9618 addptr(dst, 2 * 8); 9619 9620 jmp(copy_tail, true); 9621 } 9622 9623 // inflate 8 chars per iter 9624 bind(copy_8_loop); 9625 pmovzxbw(tmp1, Address(src, len, Address::times_1)); // unpack to 8 words 9626 movdqu(Address(dst, len, Address::times_2), tmp1); 9627 addptr(len, 8); 9628 jcc(Assembler::notZero, copy_8_loop); 9629 9630 bind(copy_tail); 9631 movl(len, tmp2); 9632 9633 cmpl(len, 4); 9634 jccb(Assembler::less, copy_bytes); 9635 9636 movdl(tmp1, Address(src, 0)); // load 4 byte chars 9637 pmovzxbw(tmp1, tmp1); 9638 movq(Address(dst, 0), tmp1); 9639 subptr(len, 4); 9640 addptr(src, 4); 9641 addptr(dst, 8); 9642 9643 bind(copy_bytes); 9644 } else { 9645 bind(below_threshold); 9646 } 9647 9648 testl(len, len); 9649 jccb(Assembler::zero, done); 9650 lea(src, Address(src, len, Address::times_1)); 9651 lea(dst, Address(dst, len, Address::times_2)); 9652 negptr(len); 9653 9654 // inflate 1 char per iter 9655 bind(copy_chars_loop); 9656 load_unsigned_byte(tmp2, Address(src, len, Address::times_1)); // load byte char 9657 movw(Address(dst, len, Address::times_2), tmp2); // inflate byte char to word 9658 increment(len); 9659 jcc(Assembler::notZero, copy_chars_loop); 9660 9661 bind(done); 9662 } 9663 9664 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, XMMRegister src, bool merge, int vector_len) { 9665 switch(type) { 9666 case T_BYTE: 9667 case T_BOOLEAN: 9668 evmovdqub(dst, kmask, src, merge, vector_len); 9669 break; 9670 case T_CHAR: 9671 case T_SHORT: 9672 evmovdquw(dst, kmask, src, merge, vector_len); 9673 break; 9674 case T_INT: 9675 case T_FLOAT: 9676 evmovdqul(dst, kmask, src, merge, vector_len); 9677 break; 9678 case T_LONG: 9679 case T_DOUBLE: 9680 evmovdquq(dst, kmask, src, merge, vector_len); 9681 break; 9682 default: 9683 fatal("Unexpected type argument %s", type2name(type)); 9684 break; 9685 } 9686 } 9687 9688 9689 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, Address src, bool merge, int vector_len) { 9690 switch(type) { 9691 case T_BYTE: 9692 case T_BOOLEAN: 9693 evmovdqub(dst, kmask, src, merge, vector_len); 9694 break; 9695 case T_CHAR: 9696 case T_SHORT: 9697 evmovdquw(dst, kmask, src, merge, vector_len); 9698 break; 9699 case T_INT: 9700 case T_FLOAT: 9701 evmovdqul(dst, kmask, src, merge, vector_len); 9702 break; 9703 case T_LONG: 9704 case T_DOUBLE: 9705 evmovdquq(dst, kmask, src, merge, vector_len); 9706 break; 9707 default: 9708 fatal("Unexpected type argument %s", type2name(type)); 9709 break; 9710 } 9711 } 9712 9713 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, Address dst, XMMRegister src, bool merge, int vector_len) { 9714 switch(type) { 9715 case T_BYTE: 9716 case T_BOOLEAN: 9717 evmovdqub(dst, kmask, src, merge, vector_len); 9718 break; 9719 case T_CHAR: 9720 case T_SHORT: 9721 evmovdquw(dst, kmask, src, merge, vector_len); 9722 break; 9723 case T_INT: 9724 case T_FLOAT: 9725 evmovdqul(dst, kmask, src, merge, vector_len); 9726 break; 9727 case T_LONG: 9728 case T_DOUBLE: 9729 evmovdquq(dst, kmask, src, merge, vector_len); 9730 break; 9731 default: 9732 fatal("Unexpected type argument %s", type2name(type)); 9733 break; 9734 } 9735 } 9736 9737 void MacroAssembler::knot(uint masklen, KRegister dst, KRegister src, KRegister ktmp, Register rtmp) { 9738 switch(masklen) { 9739 case 2: 9740 knotbl(dst, src); 9741 movl(rtmp, 3); 9742 kmovbl(ktmp, rtmp); 9743 kandbl(dst, ktmp, dst); 9744 break; 9745 case 4: 9746 knotbl(dst, src); 9747 movl(rtmp, 15); 9748 kmovbl(ktmp, rtmp); 9749 kandbl(dst, ktmp, dst); 9750 break; 9751 case 8: 9752 knotbl(dst, src); 9753 break; 9754 case 16: 9755 knotwl(dst, src); 9756 break; 9757 case 32: 9758 knotdl(dst, src); 9759 break; 9760 case 64: 9761 knotql(dst, src); 9762 break; 9763 default: 9764 fatal("Unexpected vector length %d", masklen); 9765 break; 9766 } 9767 } 9768 9769 void MacroAssembler::kand(BasicType type, KRegister dst, KRegister src1, KRegister src2) { 9770 switch(type) { 9771 case T_BOOLEAN: 9772 case T_BYTE: 9773 kandbl(dst, src1, src2); 9774 break; 9775 case T_CHAR: 9776 case T_SHORT: 9777 kandwl(dst, src1, src2); 9778 break; 9779 case T_INT: 9780 case T_FLOAT: 9781 kanddl(dst, src1, src2); 9782 break; 9783 case T_LONG: 9784 case T_DOUBLE: 9785 kandql(dst, src1, src2); 9786 break; 9787 default: 9788 fatal("Unexpected type argument %s", type2name(type)); 9789 break; 9790 } 9791 } 9792 9793 void MacroAssembler::kor(BasicType type, KRegister dst, KRegister src1, KRegister src2) { 9794 switch(type) { 9795 case T_BOOLEAN: 9796 case T_BYTE: 9797 korbl(dst, src1, src2); 9798 break; 9799 case T_CHAR: 9800 case T_SHORT: 9801 korwl(dst, src1, src2); 9802 break; 9803 case T_INT: 9804 case T_FLOAT: 9805 kordl(dst, src1, src2); 9806 break; 9807 case T_LONG: 9808 case T_DOUBLE: 9809 korql(dst, src1, src2); 9810 break; 9811 default: 9812 fatal("Unexpected type argument %s", type2name(type)); 9813 break; 9814 } 9815 } 9816 9817 void MacroAssembler::kxor(BasicType type, KRegister dst, KRegister src1, KRegister src2) { 9818 switch(type) { 9819 case T_BOOLEAN: 9820 case T_BYTE: 9821 kxorbl(dst, src1, src2); 9822 break; 9823 case T_CHAR: 9824 case T_SHORT: 9825 kxorwl(dst, src1, src2); 9826 break; 9827 case T_INT: 9828 case T_FLOAT: 9829 kxordl(dst, src1, src2); 9830 break; 9831 case T_LONG: 9832 case T_DOUBLE: 9833 kxorql(dst, src1, src2); 9834 break; 9835 default: 9836 fatal("Unexpected type argument %s", type2name(type)); 9837 break; 9838 } 9839 } 9840 9841 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9842 switch(type) { 9843 case T_BOOLEAN: 9844 case T_BYTE: 9845 evpermb(dst, mask, nds, src, merge, vector_len); break; 9846 case T_CHAR: 9847 case T_SHORT: 9848 evpermw(dst, mask, nds, src, merge, vector_len); break; 9849 case T_INT: 9850 case T_FLOAT: 9851 evpermd(dst, mask, nds, src, merge, vector_len); break; 9852 case T_LONG: 9853 case T_DOUBLE: 9854 evpermq(dst, mask, nds, src, merge, vector_len); break; 9855 default: 9856 fatal("Unexpected type argument %s", type2name(type)); break; 9857 } 9858 } 9859 9860 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9861 switch(type) { 9862 case T_BOOLEAN: 9863 case T_BYTE: 9864 evpermb(dst, mask, nds, src, merge, vector_len); break; 9865 case T_CHAR: 9866 case T_SHORT: 9867 evpermw(dst, mask, nds, src, merge, vector_len); break; 9868 case T_INT: 9869 case T_FLOAT: 9870 evpermd(dst, mask, nds, src, merge, vector_len); break; 9871 case T_LONG: 9872 case T_DOUBLE: 9873 evpermq(dst, mask, nds, src, merge, vector_len); break; 9874 default: 9875 fatal("Unexpected type argument %s", type2name(type)); break; 9876 } 9877 } 9878 9879 void MacroAssembler::evpminu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9880 switch(type) { 9881 case T_BYTE: 9882 evpminub(dst, mask, nds, src, merge, vector_len); break; 9883 case T_SHORT: 9884 evpminuw(dst, mask, nds, src, merge, vector_len); break; 9885 case T_INT: 9886 evpminud(dst, mask, nds, src, merge, vector_len); break; 9887 case T_LONG: 9888 evpminuq(dst, mask, nds, src, merge, vector_len); break; 9889 default: 9890 fatal("Unexpected type argument %s", type2name(type)); break; 9891 } 9892 } 9893 9894 void MacroAssembler::evpmaxu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9895 switch(type) { 9896 case T_BYTE: 9897 evpmaxub(dst, mask, nds, src, merge, vector_len); break; 9898 case T_SHORT: 9899 evpmaxuw(dst, mask, nds, src, merge, vector_len); break; 9900 case T_INT: 9901 evpmaxud(dst, mask, nds, src, merge, vector_len); break; 9902 case T_LONG: 9903 evpmaxuq(dst, mask, nds, src, merge, vector_len); break; 9904 default: 9905 fatal("Unexpected type argument %s", type2name(type)); break; 9906 } 9907 } 9908 9909 void MacroAssembler::evpminu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9910 switch(type) { 9911 case T_BYTE: 9912 evpminub(dst, mask, nds, src, merge, vector_len); break; 9913 case T_SHORT: 9914 evpminuw(dst, mask, nds, src, merge, vector_len); break; 9915 case T_INT: 9916 evpminud(dst, mask, nds, src, merge, vector_len); break; 9917 case T_LONG: 9918 evpminuq(dst, mask, nds, src, merge, vector_len); break; 9919 default: 9920 fatal("Unexpected type argument %s", type2name(type)); break; 9921 } 9922 } 9923 9924 void MacroAssembler::evpmaxu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9925 switch(type) { 9926 case T_BYTE: 9927 evpmaxub(dst, mask, nds, src, merge, vector_len); break; 9928 case T_SHORT: 9929 evpmaxuw(dst, mask, nds, src, merge, vector_len); break; 9930 case T_INT: 9931 evpmaxud(dst, mask, nds, src, merge, vector_len); break; 9932 case T_LONG: 9933 evpmaxuq(dst, mask, nds, src, merge, vector_len); break; 9934 default: 9935 fatal("Unexpected type argument %s", type2name(type)); break; 9936 } 9937 } 9938 9939 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9940 switch(type) { 9941 case T_BYTE: 9942 evpminsb(dst, mask, nds, src, merge, vector_len); break; 9943 case T_SHORT: 9944 evpminsw(dst, mask, nds, src, merge, vector_len); break; 9945 case T_INT: 9946 evpminsd(dst, mask, nds, src, merge, vector_len); break; 9947 case T_LONG: 9948 evpminsq(dst, mask, nds, src, merge, vector_len); break; 9949 default: 9950 fatal("Unexpected type argument %s", type2name(type)); break; 9951 } 9952 } 9953 9954 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 9955 switch(type) { 9956 case T_BYTE: 9957 evpmaxsb(dst, mask, nds, src, merge, vector_len); break; 9958 case T_SHORT: 9959 evpmaxsw(dst, mask, nds, src, merge, vector_len); break; 9960 case T_INT: 9961 evpmaxsd(dst, mask, nds, src, merge, vector_len); break; 9962 case T_LONG: 9963 evpmaxsq(dst, mask, nds, src, merge, vector_len); break; 9964 default: 9965 fatal("Unexpected type argument %s", type2name(type)); break; 9966 } 9967 } 9968 9969 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9970 switch(type) { 9971 case T_BYTE: 9972 evpminsb(dst, mask, nds, src, merge, vector_len); break; 9973 case T_SHORT: 9974 evpminsw(dst, mask, nds, src, merge, vector_len); break; 9975 case T_INT: 9976 evpminsd(dst, mask, nds, src, merge, vector_len); break; 9977 case T_LONG: 9978 evpminsq(dst, mask, nds, src, merge, vector_len); break; 9979 default: 9980 fatal("Unexpected type argument %s", type2name(type)); break; 9981 } 9982 } 9983 9984 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 9985 switch(type) { 9986 case T_BYTE: 9987 evpmaxsb(dst, mask, nds, src, merge, vector_len); break; 9988 case T_SHORT: 9989 evpmaxsw(dst, mask, nds, src, merge, vector_len); break; 9990 case T_INT: 9991 evpmaxsd(dst, mask, nds, src, merge, vector_len); break; 9992 case T_LONG: 9993 evpmaxsq(dst, mask, nds, src, merge, vector_len); break; 9994 default: 9995 fatal("Unexpected type argument %s", type2name(type)); break; 9996 } 9997 } 9998 9999 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10000 switch(type) { 10001 case T_INT: 10002 evpxord(dst, mask, nds, src, merge, vector_len); break; 10003 case T_LONG: 10004 evpxorq(dst, mask, nds, src, merge, vector_len); break; 10005 default: 10006 fatal("Unexpected type argument %s", type2name(type)); break; 10007 } 10008 } 10009 10010 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 10011 switch(type) { 10012 case T_INT: 10013 evpxord(dst, mask, nds, src, merge, vector_len); break; 10014 case T_LONG: 10015 evpxorq(dst, mask, nds, src, merge, vector_len); break; 10016 default: 10017 fatal("Unexpected type argument %s", type2name(type)); break; 10018 } 10019 } 10020 10021 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10022 switch(type) { 10023 case T_INT: 10024 Assembler::evpord(dst, mask, nds, src, merge, vector_len); break; 10025 case T_LONG: 10026 evporq(dst, mask, nds, src, merge, vector_len); break; 10027 default: 10028 fatal("Unexpected type argument %s", type2name(type)); break; 10029 } 10030 } 10031 10032 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 10033 switch(type) { 10034 case T_INT: 10035 Assembler::evpord(dst, mask, nds, src, merge, vector_len); break; 10036 case T_LONG: 10037 evporq(dst, mask, nds, src, merge, vector_len); break; 10038 default: 10039 fatal("Unexpected type argument %s", type2name(type)); break; 10040 } 10041 } 10042 10043 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) { 10044 switch(type) { 10045 case T_INT: 10046 evpandd(dst, mask, nds, src, merge, vector_len); break; 10047 case T_LONG: 10048 evpandq(dst, mask, nds, src, merge, vector_len); break; 10049 default: 10050 fatal("Unexpected type argument %s", type2name(type)); break; 10051 } 10052 } 10053 10054 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) { 10055 switch(type) { 10056 case T_INT: 10057 evpandd(dst, mask, nds, src, merge, vector_len); break; 10058 case T_LONG: 10059 evpandq(dst, mask, nds, src, merge, vector_len); break; 10060 default: 10061 fatal("Unexpected type argument %s", type2name(type)); break; 10062 } 10063 } 10064 10065 void MacroAssembler::kortest(uint masklen, KRegister src1, KRegister src2) { 10066 switch(masklen) { 10067 case 8: 10068 kortestbl(src1, src2); 10069 break; 10070 case 16: 10071 kortestwl(src1, src2); 10072 break; 10073 case 32: 10074 kortestdl(src1, src2); 10075 break; 10076 case 64: 10077 kortestql(src1, src2); 10078 break; 10079 default: 10080 fatal("Unexpected mask length %d", masklen); 10081 break; 10082 } 10083 } 10084 10085 10086 void MacroAssembler::ktest(uint masklen, KRegister src1, KRegister src2) { 10087 switch(masklen) { 10088 case 8: 10089 ktestbl(src1, src2); 10090 break; 10091 case 16: 10092 ktestwl(src1, src2); 10093 break; 10094 case 32: 10095 ktestdl(src1, src2); 10096 break; 10097 case 64: 10098 ktestql(src1, src2); 10099 break; 10100 default: 10101 fatal("Unexpected mask length %d", masklen); 10102 break; 10103 } 10104 } 10105 10106 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) { 10107 switch(type) { 10108 case T_INT: 10109 evprold(dst, mask, src, shift, merge, vlen_enc); break; 10110 case T_LONG: 10111 evprolq(dst, mask, src, shift, merge, vlen_enc); break; 10112 default: 10113 fatal("Unexpected type argument %s", type2name(type)); break; 10114 break; 10115 } 10116 } 10117 10118 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) { 10119 switch(type) { 10120 case T_INT: 10121 evprord(dst, mask, src, shift, merge, vlen_enc); break; 10122 case T_LONG: 10123 evprorq(dst, mask, src, shift, merge, vlen_enc); break; 10124 default: 10125 fatal("Unexpected type argument %s", type2name(type)); break; 10126 } 10127 } 10128 10129 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) { 10130 switch(type) { 10131 case T_INT: 10132 evprolvd(dst, mask, src1, src2, merge, vlen_enc); break; 10133 case T_LONG: 10134 evprolvq(dst, mask, src1, src2, merge, vlen_enc); break; 10135 default: 10136 fatal("Unexpected type argument %s", type2name(type)); break; 10137 } 10138 } 10139 10140 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) { 10141 switch(type) { 10142 case T_INT: 10143 evprorvd(dst, mask, src1, src2, merge, vlen_enc); break; 10144 case T_LONG: 10145 evprorvq(dst, mask, src1, src2, merge, vlen_enc); break; 10146 default: 10147 fatal("Unexpected type argument %s", type2name(type)); break; 10148 } 10149 } 10150 10151 void MacroAssembler::evpandq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 10152 assert(rscratch != noreg || always_reachable(src), "missing"); 10153 10154 if (reachable(src)) { 10155 evpandq(dst, nds, as_Address(src), vector_len); 10156 } else { 10157 lea(rscratch, src); 10158 evpandq(dst, nds, Address(rscratch, 0), vector_len); 10159 } 10160 } 10161 10162 void MacroAssembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src, bool merge, int vector_len, Register rscratch) { 10163 assert(rscratch != noreg || always_reachable(src), "missing"); 10164 10165 if (reachable(src)) { 10166 Assembler::evpaddq(dst, mask, nds, as_Address(src), merge, vector_len); 10167 } else { 10168 lea(rscratch, src); 10169 Assembler::evpaddq(dst, mask, nds, Address(rscratch, 0), merge, vector_len); 10170 } 10171 } 10172 10173 void MacroAssembler::evporq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 10174 assert(rscratch != noreg || always_reachable(src), "missing"); 10175 10176 if (reachable(src)) { 10177 evporq(dst, nds, as_Address(src), vector_len); 10178 } else { 10179 lea(rscratch, src); 10180 evporq(dst, nds, Address(rscratch, 0), vector_len); 10181 } 10182 } 10183 10184 void MacroAssembler::vpshufb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 10185 assert(rscratch != noreg || always_reachable(src), "missing"); 10186 10187 if (reachable(src)) { 10188 vpshufb(dst, nds, as_Address(src), vector_len); 10189 } else { 10190 lea(rscratch, src); 10191 vpshufb(dst, nds, Address(rscratch, 0), vector_len); 10192 } 10193 } 10194 10195 void MacroAssembler::vpor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) { 10196 assert(rscratch != noreg || always_reachable(src), "missing"); 10197 10198 if (reachable(src)) { 10199 Assembler::vpor(dst, nds, as_Address(src), vector_len); 10200 } else { 10201 lea(rscratch, src); 10202 Assembler::vpor(dst, nds, Address(rscratch, 0), vector_len); 10203 } 10204 } 10205 10206 void MacroAssembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, AddressLiteral src3, int vector_len, Register rscratch) { 10207 assert(rscratch != noreg || always_reachable(src3), "missing"); 10208 10209 if (reachable(src3)) { 10210 vpternlogq(dst, imm8, src2, as_Address(src3), vector_len); 10211 } else { 10212 lea(rscratch, src3); 10213 vpternlogq(dst, imm8, src2, Address(rscratch, 0), vector_len); 10214 } 10215 } 10216 10217 #if COMPILER2_OR_JVMCI 10218 10219 void MacroAssembler::fill_masked(BasicType bt, Address dst, XMMRegister xmm, KRegister mask, 10220 Register length, Register temp, int vec_enc) { 10221 // Computing mask for predicated vector store. 10222 movptr(temp, -1); 10223 bzhiq(temp, temp, length); 10224 kmov(mask, temp); 10225 evmovdqu(bt, mask, dst, xmm, true, vec_enc); 10226 } 10227 10228 // Set memory operation for length "less than" 64 bytes. 10229 void MacroAssembler::fill64_masked(uint shift, Register dst, int disp, 10230 XMMRegister xmm, KRegister mask, Register length, 10231 Register temp, bool use64byteVector) { 10232 assert(MaxVectorSize >= 32, "vector length should be >= 32"); 10233 const BasicType type[] = { T_BYTE, T_SHORT, T_INT, T_LONG}; 10234 if (!use64byteVector) { 10235 fill32(dst, disp, xmm); 10236 subptr(length, 32 >> shift); 10237 fill32_masked(shift, dst, disp + 32, xmm, mask, length, temp); 10238 } else { 10239 assert(MaxVectorSize == 64, "vector length != 64"); 10240 fill_masked(type[shift], Address(dst, disp), xmm, mask, length, temp, Assembler::AVX_512bit); 10241 } 10242 } 10243 10244 10245 void MacroAssembler::fill32_masked(uint shift, Register dst, int disp, 10246 XMMRegister xmm, KRegister mask, Register length, 10247 Register temp) { 10248 assert(MaxVectorSize >= 32, "vector length should be >= 32"); 10249 const BasicType type[] = { T_BYTE, T_SHORT, T_INT, T_LONG}; 10250 fill_masked(type[shift], Address(dst, disp), xmm, mask, length, temp, Assembler::AVX_256bit); 10251 } 10252 10253 10254 void MacroAssembler::fill32(Address dst, XMMRegister xmm) { 10255 assert(MaxVectorSize >= 32, "vector length should be >= 32"); 10256 vmovdqu(dst, xmm); 10257 } 10258 10259 void MacroAssembler::fill32(Register dst, int disp, XMMRegister xmm) { 10260 fill32(Address(dst, disp), xmm); 10261 } 10262 10263 void MacroAssembler::fill64(Address dst, XMMRegister xmm, bool use64byteVector) { 10264 assert(MaxVectorSize >= 32, "vector length should be >= 32"); 10265 if (!use64byteVector) { 10266 fill32(dst, xmm); 10267 fill32(dst.plus_disp(32), xmm); 10268 } else { 10269 evmovdquq(dst, xmm, Assembler::AVX_512bit); 10270 } 10271 } 10272 10273 void MacroAssembler::fill64(Register dst, int disp, XMMRegister xmm, bool use64byteVector) { 10274 fill64(Address(dst, disp), xmm, use64byteVector); 10275 } 10276 10277 #ifdef _LP64 10278 void MacroAssembler::generate_fill_avx3(BasicType type, Register to, Register value, 10279 Register count, Register rtmp, XMMRegister xtmp) { 10280 Label L_exit; 10281 Label L_fill_start; 10282 Label L_fill_64_bytes; 10283 Label L_fill_96_bytes; 10284 Label L_fill_128_bytes; 10285 Label L_fill_128_bytes_loop; 10286 Label L_fill_128_loop_header; 10287 Label L_fill_128_bytes_loop_header; 10288 Label L_fill_128_bytes_loop_pre_header; 10289 Label L_fill_zmm_sequence; 10290 10291 int shift = -1; 10292 int avx3threshold = VM_Version::avx3_threshold(); 10293 switch(type) { 10294 case T_BYTE: shift = 0; 10295 break; 10296 case T_SHORT: shift = 1; 10297 break; 10298 case T_INT: shift = 2; 10299 break; 10300 /* Uncomment when LONG fill stubs are supported. 10301 case T_LONG: shift = 3; 10302 break; 10303 */ 10304 default: 10305 fatal("Unhandled type: %s\n", type2name(type)); 10306 } 10307 10308 if ((avx3threshold != 0) || (MaxVectorSize == 32)) { 10309 10310 if (MaxVectorSize == 64) { 10311 cmpq(count, avx3threshold >> shift); 10312 jcc(Assembler::greater, L_fill_zmm_sequence); 10313 } 10314 10315 evpbroadcast(type, xtmp, value, Assembler::AVX_256bit); 10316 10317 bind(L_fill_start); 10318 10319 cmpq(count, 32 >> shift); 10320 jccb(Assembler::greater, L_fill_64_bytes); 10321 fill32_masked(shift, to, 0, xtmp, k2, count, rtmp); 10322 jmp(L_exit); 10323 10324 bind(L_fill_64_bytes); 10325 cmpq(count, 64 >> shift); 10326 jccb(Assembler::greater, L_fill_96_bytes); 10327 fill64_masked(shift, to, 0, xtmp, k2, count, rtmp); 10328 jmp(L_exit); 10329 10330 bind(L_fill_96_bytes); 10331 cmpq(count, 96 >> shift); 10332 jccb(Assembler::greater, L_fill_128_bytes); 10333 fill64(to, 0, xtmp); 10334 subq(count, 64 >> shift); 10335 fill32_masked(shift, to, 64, xtmp, k2, count, rtmp); 10336 jmp(L_exit); 10337 10338 bind(L_fill_128_bytes); 10339 cmpq(count, 128 >> shift); 10340 jccb(Assembler::greater, L_fill_128_bytes_loop_pre_header); 10341 fill64(to, 0, xtmp); 10342 fill32(to, 64, xtmp); 10343 subq(count, 96 >> shift); 10344 fill32_masked(shift, to, 96, xtmp, k2, count, rtmp); 10345 jmp(L_exit); 10346 10347 bind(L_fill_128_bytes_loop_pre_header); 10348 { 10349 mov(rtmp, to); 10350 andq(rtmp, 31); 10351 jccb(Assembler::zero, L_fill_128_bytes_loop_header); 10352 negq(rtmp); 10353 addq(rtmp, 32); 10354 mov64(r8, -1L); 10355 bzhiq(r8, r8, rtmp); 10356 kmovql(k2, r8); 10357 evmovdqu(T_BYTE, k2, Address(to, 0), xtmp, true, Assembler::AVX_256bit); 10358 addq(to, rtmp); 10359 shrq(rtmp, shift); 10360 subq(count, rtmp); 10361 } 10362 10363 cmpq(count, 128 >> shift); 10364 jcc(Assembler::less, L_fill_start); 10365 10366 bind(L_fill_128_bytes_loop_header); 10367 subq(count, 128 >> shift); 10368 10369 align32(); 10370 bind(L_fill_128_bytes_loop); 10371 fill64(to, 0, xtmp); 10372 fill64(to, 64, xtmp); 10373 addq(to, 128); 10374 subq(count, 128 >> shift); 10375 jccb(Assembler::greaterEqual, L_fill_128_bytes_loop); 10376 10377 addq(count, 128 >> shift); 10378 jcc(Assembler::zero, L_exit); 10379 jmp(L_fill_start); 10380 } 10381 10382 if (MaxVectorSize == 64) { 10383 // Sequence using 64 byte ZMM register. 10384 Label L_fill_128_bytes_zmm; 10385 Label L_fill_192_bytes_zmm; 10386 Label L_fill_192_bytes_loop_zmm; 10387 Label L_fill_192_bytes_loop_header_zmm; 10388 Label L_fill_192_bytes_loop_pre_header_zmm; 10389 Label L_fill_start_zmm_sequence; 10390 10391 bind(L_fill_zmm_sequence); 10392 evpbroadcast(type, xtmp, value, Assembler::AVX_512bit); 10393 10394 bind(L_fill_start_zmm_sequence); 10395 cmpq(count, 64 >> shift); 10396 jccb(Assembler::greater, L_fill_128_bytes_zmm); 10397 fill64_masked(shift, to, 0, xtmp, k2, count, rtmp, true); 10398 jmp(L_exit); 10399 10400 bind(L_fill_128_bytes_zmm); 10401 cmpq(count, 128 >> shift); 10402 jccb(Assembler::greater, L_fill_192_bytes_zmm); 10403 fill64(to, 0, xtmp, true); 10404 subq(count, 64 >> shift); 10405 fill64_masked(shift, to, 64, xtmp, k2, count, rtmp, true); 10406 jmp(L_exit); 10407 10408 bind(L_fill_192_bytes_zmm); 10409 cmpq(count, 192 >> shift); 10410 jccb(Assembler::greater, L_fill_192_bytes_loop_pre_header_zmm); 10411 fill64(to, 0, xtmp, true); 10412 fill64(to, 64, xtmp, true); 10413 subq(count, 128 >> shift); 10414 fill64_masked(shift, to, 128, xtmp, k2, count, rtmp, true); 10415 jmp(L_exit); 10416 10417 bind(L_fill_192_bytes_loop_pre_header_zmm); 10418 { 10419 movq(rtmp, to); 10420 andq(rtmp, 63); 10421 jccb(Assembler::zero, L_fill_192_bytes_loop_header_zmm); 10422 negq(rtmp); 10423 addq(rtmp, 64); 10424 mov64(r8, -1L); 10425 bzhiq(r8, r8, rtmp); 10426 kmovql(k2, r8); 10427 evmovdqu(T_BYTE, k2, Address(to, 0), xtmp, true, Assembler::AVX_512bit); 10428 addq(to, rtmp); 10429 shrq(rtmp, shift); 10430 subq(count, rtmp); 10431 } 10432 10433 cmpq(count, 192 >> shift); 10434 jcc(Assembler::less, L_fill_start_zmm_sequence); 10435 10436 bind(L_fill_192_bytes_loop_header_zmm); 10437 subq(count, 192 >> shift); 10438 10439 align32(); 10440 bind(L_fill_192_bytes_loop_zmm); 10441 fill64(to, 0, xtmp, true); 10442 fill64(to, 64, xtmp, true); 10443 fill64(to, 128, xtmp, true); 10444 addq(to, 192); 10445 subq(count, 192 >> shift); 10446 jccb(Assembler::greaterEqual, L_fill_192_bytes_loop_zmm); 10447 10448 addq(count, 192 >> shift); 10449 jcc(Assembler::zero, L_exit); 10450 jmp(L_fill_start_zmm_sequence); 10451 } 10452 bind(L_exit); 10453 } 10454 #endif 10455 #endif //COMPILER2_OR_JVMCI 10456 10457 10458 #ifdef _LP64 10459 void MacroAssembler::convert_f2i(Register dst, XMMRegister src) { 10460 Label done; 10461 cvttss2sil(dst, src); 10462 // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub 10463 cmpl(dst, 0x80000000); // float_sign_flip 10464 jccb(Assembler::notEqual, done); 10465 subptr(rsp, 8); 10466 movflt(Address(rsp, 0), src); 10467 call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup()))); 10468 pop(dst); 10469 bind(done); 10470 } 10471 10472 void MacroAssembler::convert_d2i(Register dst, XMMRegister src) { 10473 Label done; 10474 cvttsd2sil(dst, src); 10475 // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub 10476 cmpl(dst, 0x80000000); // float_sign_flip 10477 jccb(Assembler::notEqual, done); 10478 subptr(rsp, 8); 10479 movdbl(Address(rsp, 0), src); 10480 call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup()))); 10481 pop(dst); 10482 bind(done); 10483 } 10484 10485 void MacroAssembler::convert_f2l(Register dst, XMMRegister src) { 10486 Label done; 10487 cvttss2siq(dst, src); 10488 cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip())); 10489 jccb(Assembler::notEqual, done); 10490 subptr(rsp, 8); 10491 movflt(Address(rsp, 0), src); 10492 call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup()))); 10493 pop(dst); 10494 bind(done); 10495 } 10496 10497 void MacroAssembler::round_float(Register dst, XMMRegister src, Register rtmp, Register rcx) { 10498 // Following code is line by line assembly translation rounding algorithm. 10499 // Please refer to java.lang.Math.round(float) algorithm for details. 10500 const int32_t FloatConsts_EXP_BIT_MASK = 0x7F800000; 10501 const int32_t FloatConsts_SIGNIFICAND_WIDTH = 24; 10502 const int32_t FloatConsts_EXP_BIAS = 127; 10503 const int32_t FloatConsts_SIGNIF_BIT_MASK = 0x007FFFFF; 10504 const int32_t MINUS_32 = 0xFFFFFFE0; 10505 Label L_special_case, L_block1, L_exit; 10506 movl(rtmp, FloatConsts_EXP_BIT_MASK); 10507 movdl(dst, src); 10508 andl(dst, rtmp); 10509 sarl(dst, FloatConsts_SIGNIFICAND_WIDTH - 1); 10510 movl(rtmp, FloatConsts_SIGNIFICAND_WIDTH - 2 + FloatConsts_EXP_BIAS); 10511 subl(rtmp, dst); 10512 movl(rcx, rtmp); 10513 movl(dst, MINUS_32); 10514 testl(rtmp, dst); 10515 jccb(Assembler::notEqual, L_special_case); 10516 movdl(dst, src); 10517 andl(dst, FloatConsts_SIGNIF_BIT_MASK); 10518 orl(dst, FloatConsts_SIGNIF_BIT_MASK + 1); 10519 movdl(rtmp, src); 10520 testl(rtmp, rtmp); 10521 jccb(Assembler::greaterEqual, L_block1); 10522 negl(dst); 10523 bind(L_block1); 10524 sarl(dst); 10525 addl(dst, 0x1); 10526 sarl(dst, 0x1); 10527 jmp(L_exit); 10528 bind(L_special_case); 10529 convert_f2i(dst, src); 10530 bind(L_exit); 10531 } 10532 10533 void MacroAssembler::round_double(Register dst, XMMRegister src, Register rtmp, Register rcx) { 10534 // Following code is line by line assembly translation rounding algorithm. 10535 // Please refer to java.lang.Math.round(double) algorithm for details. 10536 const int64_t DoubleConsts_EXP_BIT_MASK = 0x7FF0000000000000L; 10537 const int64_t DoubleConsts_SIGNIFICAND_WIDTH = 53; 10538 const int64_t DoubleConsts_EXP_BIAS = 1023; 10539 const int64_t DoubleConsts_SIGNIF_BIT_MASK = 0x000FFFFFFFFFFFFFL; 10540 const int64_t MINUS_64 = 0xFFFFFFFFFFFFFFC0L; 10541 Label L_special_case, L_block1, L_exit; 10542 mov64(rtmp, DoubleConsts_EXP_BIT_MASK); 10543 movq(dst, src); 10544 andq(dst, rtmp); 10545 sarq(dst, DoubleConsts_SIGNIFICAND_WIDTH - 1); 10546 mov64(rtmp, DoubleConsts_SIGNIFICAND_WIDTH - 2 + DoubleConsts_EXP_BIAS); 10547 subq(rtmp, dst); 10548 movq(rcx, rtmp); 10549 mov64(dst, MINUS_64); 10550 testq(rtmp, dst); 10551 jccb(Assembler::notEqual, L_special_case); 10552 movq(dst, src); 10553 mov64(rtmp, DoubleConsts_SIGNIF_BIT_MASK); 10554 andq(dst, rtmp); 10555 mov64(rtmp, DoubleConsts_SIGNIF_BIT_MASK + 1); 10556 orq(dst, rtmp); 10557 movq(rtmp, src); 10558 testq(rtmp, rtmp); 10559 jccb(Assembler::greaterEqual, L_block1); 10560 negq(dst); 10561 bind(L_block1); 10562 sarq(dst); 10563 addq(dst, 0x1); 10564 sarq(dst, 0x1); 10565 jmp(L_exit); 10566 bind(L_special_case); 10567 convert_d2l(dst, src); 10568 bind(L_exit); 10569 } 10570 10571 void MacroAssembler::convert_d2l(Register dst, XMMRegister src) { 10572 Label done; 10573 cvttsd2siq(dst, src); 10574 cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip())); 10575 jccb(Assembler::notEqual, done); 10576 subptr(rsp, 8); 10577 movdbl(Address(rsp, 0), src); 10578 call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup()))); 10579 pop(dst); 10580 bind(done); 10581 } 10582 10583 void MacroAssembler::cache_wb(Address line) 10584 { 10585 // 64 bit cpus always support clflush 10586 assert(VM_Version::supports_clflush(), "clflush should be available"); 10587 bool optimized = VM_Version::supports_clflushopt(); 10588 bool no_evict = VM_Version::supports_clwb(); 10589 10590 // prefer clwb (writeback without evict) otherwise 10591 // prefer clflushopt (potentially parallel writeback with evict) 10592 // otherwise fallback on clflush (serial writeback with evict) 10593 10594 if (optimized) { 10595 if (no_evict) { 10596 clwb(line); 10597 } else { 10598 clflushopt(line); 10599 } 10600 } else { 10601 // no need for fence when using CLFLUSH 10602 clflush(line); 10603 } 10604 } 10605 10606 void MacroAssembler::cache_wbsync(bool is_pre) 10607 { 10608 assert(VM_Version::supports_clflush(), "clflush should be available"); 10609 bool optimized = VM_Version::supports_clflushopt(); 10610 bool no_evict = VM_Version::supports_clwb(); 10611 10612 // pick the correct implementation 10613 10614 if (!is_pre && (optimized || no_evict)) { 10615 // need an sfence for post flush when using clflushopt or clwb 10616 // otherwise no no need for any synchroniaztion 10617 10618 sfence(); 10619 } 10620 } 10621 10622 #endif // _LP64 10623 10624 Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) { 10625 switch (cond) { 10626 // Note some conditions are synonyms for others 10627 case Assembler::zero: return Assembler::notZero; 10628 case Assembler::notZero: return Assembler::zero; 10629 case Assembler::less: return Assembler::greaterEqual; 10630 case Assembler::lessEqual: return Assembler::greater; 10631 case Assembler::greater: return Assembler::lessEqual; 10632 case Assembler::greaterEqual: return Assembler::less; 10633 case Assembler::below: return Assembler::aboveEqual; 10634 case Assembler::belowEqual: return Assembler::above; 10635 case Assembler::above: return Assembler::belowEqual; 10636 case Assembler::aboveEqual: return Assembler::below; 10637 case Assembler::overflow: return Assembler::noOverflow; 10638 case Assembler::noOverflow: return Assembler::overflow; 10639 case Assembler::negative: return Assembler::positive; 10640 case Assembler::positive: return Assembler::negative; 10641 case Assembler::parity: return Assembler::noParity; 10642 case Assembler::noParity: return Assembler::parity; 10643 } 10644 ShouldNotReachHere(); return Assembler::overflow; 10645 } 10646 10647 // This is simply a call to Thread::current() 10648 void MacroAssembler::get_thread(Register thread) { 10649 if (thread != rax) { 10650 push(rax); 10651 } 10652 LP64_ONLY(push(rdi);) 10653 LP64_ONLY(push(rsi);) 10654 push(rdx); 10655 push(rcx); 10656 #ifdef _LP64 10657 push(r8); 10658 push(r9); 10659 push(r10); 10660 push(r11); 10661 #endif 10662 10663 MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0); 10664 10665 #ifdef _LP64 10666 pop(r11); 10667 pop(r10); 10668 pop(r9); 10669 pop(r8); 10670 #endif 10671 pop(rcx); 10672 pop(rdx); 10673 LP64_ONLY(pop(rsi);) 10674 LP64_ONLY(pop(rdi);) 10675 if (thread != rax) { 10676 mov(thread, rax); 10677 pop(rax); 10678 } 10679 } 10680 10681 void MacroAssembler::check_stack_alignment(Register sp, const char* msg, unsigned bias, Register tmp) { 10682 Label L_stack_ok; 10683 if (bias == 0) { 10684 testptr(sp, 2 * wordSize - 1); 10685 } else { 10686 // lea(tmp, Address(rsp, bias); 10687 mov(tmp, sp); 10688 addptr(tmp, bias); 10689 testptr(tmp, 2 * wordSize - 1); 10690 } 10691 jcc(Assembler::equal, L_stack_ok); 10692 block_comment(msg); 10693 stop(msg); 10694 bind(L_stack_ok); 10695 } 10696 10697 // Implements lightweight-locking. 10698 // 10699 // obj: the object to be locked 10700 // reg_rax: rax 10701 // thread: the thread which attempts to lock obj 10702 // tmp: a temporary register 10703 void MacroAssembler::lightweight_lock(Register basic_lock, Register obj, Register reg_rax, Register thread, Register tmp, Label& slow) { 10704 assert(reg_rax == rax, ""); 10705 assert_different_registers(basic_lock, obj, reg_rax, thread, tmp); 10706 10707 Label push; 10708 const Register top = tmp; 10709 10710 // Preload the markWord. It is important that this is the first 10711 // instruction emitted as it is part of C1's null check semantics. 10712 movptr(reg_rax, Address(obj, oopDesc::mark_offset_in_bytes())); 10713 10714 if (UseObjectMonitorTable) { 10715 // Clear cache in case fast locking succeeds. 10716 movptr(Address(basic_lock, BasicObjectLock::lock_offset() + in_ByteSize((BasicLock::object_monitor_cache_offset_in_bytes()))), 0); 10717 } 10718 10719 // Load top. 10720 movl(top, Address(thread, JavaThread::lock_stack_top_offset())); 10721 10722 // Check if the lock-stack is full. 10723 cmpl(top, LockStack::end_offset()); 10724 jcc(Assembler::greaterEqual, slow); 10725 10726 // Check for recursion. 10727 cmpptr(obj, Address(thread, top, Address::times_1, -oopSize)); 10728 jcc(Assembler::equal, push); 10729 10730 // Check header for monitor (0b10). 10731 testptr(reg_rax, markWord::monitor_value); 10732 jcc(Assembler::notZero, slow); 10733 10734 // Try to lock. Transition lock bits 0b01 => 0b00 10735 movptr(tmp, reg_rax); 10736 andptr(tmp, ~(int32_t)markWord::unlocked_value); 10737 orptr(reg_rax, markWord::unlocked_value); 10738 lock(); cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes())); 10739 jcc(Assembler::notEqual, slow); 10740 10741 // Restore top, CAS clobbers register. 10742 movl(top, Address(thread, JavaThread::lock_stack_top_offset())); 10743 10744 bind(push); 10745 // After successful lock, push object on lock-stack. 10746 movptr(Address(thread, top), obj); 10747 incrementl(top, oopSize); 10748 movl(Address(thread, JavaThread::lock_stack_top_offset()), top); 10749 } 10750 10751 // Implements lightweight-unlocking. 10752 // 10753 // obj: the object to be unlocked 10754 // reg_rax: rax 10755 // thread: the thread 10756 // tmp: a temporary register 10757 void MacroAssembler::lightweight_unlock(Register obj, Register reg_rax, Register thread, Register tmp, Label& slow) { 10758 assert(reg_rax == rax, ""); 10759 assert_different_registers(obj, reg_rax, thread, tmp); 10760 10761 Label unlocked, push_and_slow; 10762 const Register top = tmp; 10763 10764 // Check if obj is top of lock-stack. 10765 movl(top, Address(thread, JavaThread::lock_stack_top_offset())); 10766 cmpptr(obj, Address(thread, top, Address::times_1, -oopSize)); 10767 jcc(Assembler::notEqual, slow); 10768 10769 // Pop lock-stack. 10770 DEBUG_ONLY(movptr(Address(thread, top, Address::times_1, -oopSize), 0);) 10771 subl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize); 10772 10773 // Check if recursive. 10774 cmpptr(obj, Address(thread, top, Address::times_1, -2 * oopSize)); 10775 jcc(Assembler::equal, unlocked); 10776 10777 // Not recursive. Check header for monitor (0b10). 10778 movptr(reg_rax, Address(obj, oopDesc::mark_offset_in_bytes())); 10779 testptr(reg_rax, markWord::monitor_value); 10780 jcc(Assembler::notZero, push_and_slow); 10781 10782 #ifdef ASSERT 10783 // Check header not unlocked (0b01). 10784 Label not_unlocked; 10785 testptr(reg_rax, markWord::unlocked_value); 10786 jcc(Assembler::zero, not_unlocked); 10787 stop("lightweight_unlock already unlocked"); 10788 bind(not_unlocked); 10789 #endif 10790 10791 // Try to unlock. Transition lock bits 0b00 => 0b01 10792 movptr(tmp, reg_rax); 10793 orptr(tmp, markWord::unlocked_value); 10794 lock(); cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes())); 10795 jcc(Assembler::equal, unlocked); 10796 10797 bind(push_and_slow); 10798 // Restore lock-stack and handle the unlock in runtime. 10799 #ifdef ASSERT 10800 movl(top, Address(thread, JavaThread::lock_stack_top_offset())); 10801 movptr(Address(thread, top), obj); 10802 #endif 10803 addl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize); 10804 jmp(slow); 10805 10806 bind(unlocked); 10807 } 10808 10809 #ifdef _LP64 10810 // Saves legacy GPRs state on stack. 10811 void MacroAssembler::save_legacy_gprs() { 10812 subq(rsp, 16 * wordSize); 10813 movq(Address(rsp, 15 * wordSize), rax); 10814 movq(Address(rsp, 14 * wordSize), rcx); 10815 movq(Address(rsp, 13 * wordSize), rdx); 10816 movq(Address(rsp, 12 * wordSize), rbx); 10817 movq(Address(rsp, 10 * wordSize), rbp); 10818 movq(Address(rsp, 9 * wordSize), rsi); 10819 movq(Address(rsp, 8 * wordSize), rdi); 10820 movq(Address(rsp, 7 * wordSize), r8); 10821 movq(Address(rsp, 6 * wordSize), r9); 10822 movq(Address(rsp, 5 * wordSize), r10); 10823 movq(Address(rsp, 4 * wordSize), r11); 10824 movq(Address(rsp, 3 * wordSize), r12); 10825 movq(Address(rsp, 2 * wordSize), r13); 10826 movq(Address(rsp, wordSize), r14); 10827 movq(Address(rsp, 0), r15); 10828 } 10829 10830 // Resotres back legacy GPRs state from stack. 10831 void MacroAssembler::restore_legacy_gprs() { 10832 movq(r15, Address(rsp, 0)); 10833 movq(r14, Address(rsp, wordSize)); 10834 movq(r13, Address(rsp, 2 * wordSize)); 10835 movq(r12, Address(rsp, 3 * wordSize)); 10836 movq(r11, Address(rsp, 4 * wordSize)); 10837 movq(r10, Address(rsp, 5 * wordSize)); 10838 movq(r9, Address(rsp, 6 * wordSize)); 10839 movq(r8, Address(rsp, 7 * wordSize)); 10840 movq(rdi, Address(rsp, 8 * wordSize)); 10841 movq(rsi, Address(rsp, 9 * wordSize)); 10842 movq(rbp, Address(rsp, 10 * wordSize)); 10843 movq(rbx, Address(rsp, 12 * wordSize)); 10844 movq(rdx, Address(rsp, 13 * wordSize)); 10845 movq(rcx, Address(rsp, 14 * wordSize)); 10846 movq(rax, Address(rsp, 15 * wordSize)); 10847 addq(rsp, 16 * wordSize); 10848 } 10849 10850 void MacroAssembler::setcc(Assembler::Condition comparison, Register dst) { 10851 if (VM_Version::supports_apx_f()) { 10852 esetzucc(comparison, dst); 10853 } else { 10854 setb(comparison, dst); 10855 movzbl(dst, dst); 10856 } 10857 } 10858 #endif